@tyvm/knowhow 0.0.18 → 0.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/plugins/downloader/plugin.ts +5 -2
- package/src/plugins/url.ts +8 -2
- package/src/processors/TokenCompressor.ts +9 -3
- package/ts_build/src/plugins/downloader/plugin.js +5 -2
- package/ts_build/src/plugins/downloader/plugin.js.map +1 -1
- package/ts_build/src/plugins/url.js +6 -2
- package/ts_build/src/plugins/url.js.map +1 -1
- package/ts_build/src/processors/TokenCompressor.d.ts +2 -1
- package/ts_build/src/processors/TokenCompressor.js +8 -3
- package/ts_build/src/processors/TokenCompressor.js.map +1 -1
package/package.json
CHANGED
|
@@ -8,7 +8,7 @@ export class DownloaderPlugin extends PluginBase {
|
|
|
8
8
|
static readonly meta: PluginMeta = {
|
|
9
9
|
key: "downloader",
|
|
10
10
|
name: "Downloader Plugin",
|
|
11
|
-
requires: []
|
|
11
|
+
requires: [],
|
|
12
12
|
};
|
|
13
13
|
|
|
14
14
|
constructor() {
|
|
@@ -20,7 +20,7 @@ export class DownloaderPlugin extends PluginBase {
|
|
|
20
20
|
extractUrls(userInput: string): string[] {
|
|
21
21
|
const urlRegex = /https:\/\/[^\s]+/gim;
|
|
22
22
|
const matches = userInput.match(urlRegex) || [];
|
|
23
|
-
return matches;
|
|
23
|
+
return Array.from(new Set(matches));
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
async call(userInput: string): Promise<string> {
|
|
@@ -28,6 +28,9 @@ export class DownloaderPlugin extends PluginBase {
|
|
|
28
28
|
if (urls.length === 0) {
|
|
29
29
|
return "DOWNLOADER PLUGIN: No URLs found in the input";
|
|
30
30
|
}
|
|
31
|
+
if (urls.length > 10) {
|
|
32
|
+
return "DOWNLOADER PLUGIN: Too many URLs found in the input. Skipping likely unintentional bulk download.";
|
|
33
|
+
}
|
|
31
34
|
let transcript = "";
|
|
32
35
|
for (const url of urls) {
|
|
33
36
|
if (this.skipExt.includes(url.split(".").pop() || "")) {
|
package/src/plugins/url.ts
CHANGED
|
@@ -9,7 +9,7 @@ export class UrlPlugin extends PluginBase implements Plugin {
|
|
|
9
9
|
static readonly meta: PluginMeta = {
|
|
10
10
|
key: "url",
|
|
11
11
|
name: "URL Plugin",
|
|
12
|
-
requires: []
|
|
12
|
+
requires: [],
|
|
13
13
|
};
|
|
14
14
|
|
|
15
15
|
constructor() {
|
|
@@ -24,7 +24,9 @@ export class UrlPlugin extends PluginBase implements Plugin {
|
|
|
24
24
|
|
|
25
25
|
extractUrls(userPrompt: string): string[] {
|
|
26
26
|
const urlRegex = /(https?:\/\/[^\s]+)/g;
|
|
27
|
-
|
|
27
|
+
const urls = userPrompt.match(urlRegex) || [];
|
|
28
|
+
|
|
29
|
+
return Array.from(new Set(urls));
|
|
28
30
|
}
|
|
29
31
|
|
|
30
32
|
async fetchAndParseUrl(url: string): Promise<MinimalEmbedding | null> {
|
|
@@ -50,6 +52,10 @@ export class UrlPlugin extends PluginBase implements Plugin {
|
|
|
50
52
|
return "URL PLUGIN: No URLs detected.";
|
|
51
53
|
}
|
|
52
54
|
|
|
55
|
+
if (urls.length > 10) {
|
|
56
|
+
return "URL PLUGIN: Too many URLs detected. Skipping like unintentional bulk browse.";
|
|
57
|
+
}
|
|
58
|
+
|
|
53
59
|
const results = await Promise.all(urls.map(this.fetchAndParseUrl));
|
|
54
60
|
const validResults = results.filter(
|
|
55
61
|
(r): r is MinimalEmbedding => r !== null
|
|
@@ -8,12 +8,12 @@ interface TokenCompressorStorage {
|
|
|
8
8
|
|
|
9
9
|
export class TokenCompressor {
|
|
10
10
|
private storage: TokenCompressorStorage = {};
|
|
11
|
-
private maxTokens: number =
|
|
11
|
+
private maxTokens: number = 4000;
|
|
12
12
|
private compressionRatio: number = 0.1;
|
|
13
13
|
private keyPrefix: string = "compressed_";
|
|
14
|
-
private jsonPropertyThreshold: number = 20000;
|
|
15
14
|
private toolName: string = expandTokensDefinition.function.name;
|
|
16
|
-
private characterLimit: number =
|
|
15
|
+
private characterLimit: number = this.maxTokens * 4;
|
|
16
|
+
private jsonPropertyThreshold: number = this.maxTokens;
|
|
17
17
|
|
|
18
18
|
constructor(toolsService?: ToolsService) {
|
|
19
19
|
this.registerTool(toolsService);
|
|
@@ -24,6 +24,12 @@ export class TokenCompressor {
|
|
|
24
24
|
return Math.ceil(text.length / 4);
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
public setMaxTokens(maxTokens: number): void {
|
|
28
|
+
this.maxTokens = maxTokens;
|
|
29
|
+
this.characterLimit = maxTokens * 4;
|
|
30
|
+
this.jsonPropertyThreshold = maxTokens;
|
|
31
|
+
}
|
|
32
|
+
|
|
27
33
|
/**
|
|
28
34
|
* Attempts to parse content as JSON and returns parsed object if successful
|
|
29
35
|
*/
|
|
@@ -12,7 +12,7 @@ class DownloaderPlugin extends PluginBase_1.PluginBase {
|
|
|
12
12
|
static meta = {
|
|
13
13
|
key: "downloader",
|
|
14
14
|
name: "Downloader Plugin",
|
|
15
|
-
requires: []
|
|
15
|
+
requires: [],
|
|
16
16
|
};
|
|
17
17
|
constructor() {
|
|
18
18
|
super(DownloaderPlugin.meta);
|
|
@@ -21,13 +21,16 @@ class DownloaderPlugin extends PluginBase_1.PluginBase {
|
|
|
21
21
|
extractUrls(userInput) {
|
|
22
22
|
const urlRegex = /https:\/\/[^\s]+/gim;
|
|
23
23
|
const matches = userInput.match(urlRegex) || [];
|
|
24
|
-
return matches;
|
|
24
|
+
return Array.from(new Set(matches));
|
|
25
25
|
}
|
|
26
26
|
async call(userInput) {
|
|
27
27
|
const urls = this.extractUrls(userInput);
|
|
28
28
|
if (urls.length === 0) {
|
|
29
29
|
return "DOWNLOADER PLUGIN: No URLs found in the input";
|
|
30
30
|
}
|
|
31
|
+
if (urls.length > 10) {
|
|
32
|
+
return "DOWNLOADER PLUGIN: Too many URLs found in the input. Skipping likely unintentional bulk download.";
|
|
33
|
+
}
|
|
31
34
|
let transcript = "";
|
|
32
35
|
for (const url of urls) {
|
|
33
36
|
if (this.skipExt.includes(url.split(".").pop() || "")) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../../../../src/plugins/downloader/plugin.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AACpB,8CAAuD;AAEvD,iDAA+D;AAC/D,6CAA0C;AAE1C,MAAa,gBAAiB,SAAQ,uBAAU;IAC9C,MAAM,CAAU,IAAI,GAAe;QACjC,GAAG,EAAE,YAAY;QACjB,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,EAAE;KACb,CAAC;IAEF;QACE,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAExC,WAAW,CAAC,SAAiB;QAC3B,MAAM,QAAQ,GAAG,qBAAqB,CAAC;QACvC,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../../../../src/plugins/downloader/plugin.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AACpB,8CAAuD;AAEvD,iDAA+D;AAC/D,6CAA0C;AAE1C,MAAa,gBAAiB,SAAQ,uBAAU;IAC9C,MAAM,CAAU,IAAI,GAAe;QACjC,GAAG,EAAE,YAAY;QACjB,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,EAAE;KACb,CAAC;IAEF;QACE,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAExC,WAAW,CAAC,SAAiB;QAC3B,MAAM,QAAQ,GAAG,qBAAqB,CAAC;QACvC,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,OAAO,+CAA+C,CAAC;SACxD;QACD,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE;YACpB,OAAO,mGAAmG,CAAC;SAC5G;QACD,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE;gBACrD,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC;gBAChD,SAAS;aACV;YACD,IAAI;gBACF,OAAO,CAAC,GAAG,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC;gBAClD,MAAM,WAAW,GAAG,qBAAqB,CAAC;gBAC1C,MAAM,QAAQ,GAAG,MAAM,uBAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;gBAC7D,MAAM,QAAQ,GAAG,GAAG,WAAW,GAAG,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAChE,UAAU,IAAI,MAAM,IAAA,0BAAa,EAAC,QAAQ,CAAC,CAAC;aAC7C;YAAC,OAAO,CAAC,EAAE;gBACV,OAAO,CAAC,GAAG,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;aACxD;SACF;QACD,OAAO,qBAAqB,GAAG,UAAU,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAiB;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,OAAO,EAAE,CAAC;SACX;QAED,MAAM,UAAU,GAAuB,EAAE,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,MAAM,WAAW,GAAG,qBAAqB,CAAC;YAC1C,MAAM,QAAQ,GAAG,MAAM,uBAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,GAAG,WAAW,GAAG,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;YAChE,MAAM,SAAS,GAAG,MAAM,IAAA,yBAAY,EAAC,QAAQ,CAAC,CAAC;YAE/C,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE;gBAC7B,IAAI,KAAK,CAAC,aAAa,EAAE;oBACvB,UAAU,CAAC,IAAI,CAAC;wBACd,EAAE,EAAE,GAAG,GAAG,UAAU,KAAK,EAAE;wBAC3B,IAAI,EAAE,KAAK,CAAC,aAAa;wBACzB,QAAQ,EAAE;4BACR,GAAG;4BACH,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW;4BACpC,SAAS,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG;4BACtC,KAAK,EAAE,YAAE,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC;yBACnD;qBACF,CAAC,CAAC;iBACJ;gBAED,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;oBAC3B,UAAU,CAAC,IAAI,CAAC;wBACd,EAAE,EAAE,GAAG,GAAG,UAAU,KAAK,EAAE;wBAC3B,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW;wBAC7B,QAAQ,EAAE;4BACR,GAAG;4BACH,SAAS,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG;yBACvC;qBACF,CAAC,CAAC;iBACJ;gBAED,KAAK,EAAE,CAAC;aACT;SACF;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;;AA1FH,4CA2FC"}
|
|
@@ -10,7 +10,7 @@ class UrlPlugin extends PluginBase_1.PluginBase {
|
|
|
10
10
|
static meta = {
|
|
11
11
|
key: "url",
|
|
12
12
|
name: "URL Plugin",
|
|
13
|
-
requires: []
|
|
13
|
+
requires: [],
|
|
14
14
|
};
|
|
15
15
|
constructor() {
|
|
16
16
|
super(UrlPlugin.meta);
|
|
@@ -22,7 +22,8 @@ class UrlPlugin extends PluginBase_1.PluginBase {
|
|
|
22
22
|
}
|
|
23
23
|
extractUrls(userPrompt) {
|
|
24
24
|
const urlRegex = /(https?:\/\/[^\s]+)/g;
|
|
25
|
-
|
|
25
|
+
const urls = userPrompt.match(urlRegex) || [];
|
|
26
|
+
return Array.from(new Set(urls));
|
|
26
27
|
}
|
|
27
28
|
async fetchAndParseUrl(url) {
|
|
28
29
|
try {
|
|
@@ -44,6 +45,9 @@ class UrlPlugin extends PluginBase_1.PluginBase {
|
|
|
44
45
|
if (urls.length === 0) {
|
|
45
46
|
return "URL PLUGIN: No URLs detected.";
|
|
46
47
|
}
|
|
48
|
+
if (urls.length > 10) {
|
|
49
|
+
return "URL PLUGIN: Too many URLs detected. Skipping like unintentional bulk browse.";
|
|
50
|
+
}
|
|
47
51
|
const results = await Promise.all(urls.map(this.fetchAndParseUrl));
|
|
48
52
|
const validResults = results.filter((r) => r !== null);
|
|
49
53
|
if (validResults.length === 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url.js","sourceRoot":"","sources":["../../../src/plugins/url.ts"],"names":[],"mappings":";;;;;;AAAA,6CAAsD;AAKtD,8EAAsD;AAEtD,MAAa,SAAU,SAAQ,uBAAU;IACvC,MAAM,CAAU,IAAI,GAAe;QACjC,GAAG,EAAE,KAAK;QACV,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,EAAE;KACb,CAAC;IAEF;QACE,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAkB;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACtE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAyB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACrE,CAAC;IAED,WAAW,CAAC,UAAkB;QAC5B,MAAM,QAAQ,GAAG,sBAAsB,CAAC;QACxC,
|
|
1
|
+
{"version":3,"file":"url.js","sourceRoot":"","sources":["../../../src/plugins/url.ts"],"names":[],"mappings":";;;;;;AAAA,6CAAsD;AAKtD,8EAAsD;AAEtD,MAAa,SAAU,SAAQ,uBAAU;IACvC,MAAM,CAAU,IAAI,GAAe;QACjC,GAAG,EAAE,KAAK;QACV,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,EAAE;KACb,CAAC;IAEF;QACE,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAkB;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACtE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAyB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACrE,CAAC;IAED,WAAW,CAAC,UAAkB;QAC5B,MAAM,QAAQ,GAAG,sBAAsB,CAAC;QACxC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE9C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,GAAW;QAChC,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAA,qBAAW,EAAC,GAAG,CAAC,CAAC;YAEpC,OAAO,CAAC,GAAG,CAAC,oCAAoC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;YAE9D,OAAO;gBACL,EAAE,EAAE,GAAG,GAAG,MAAM;gBAChB,IAAI;gBACJ,QAAQ,EAAE,EAAE,GAAG,EAAE;aAClB,CAAC;SACH;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,iCAAiC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC;SACb;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,UAAkB;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,OAAO,+BAA+B,CAAC;SACxC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE;YACpB,OAAO,8EAA8E,CAAC;SACvF;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACnE,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CACjC,CAAC,CAAC,EAAyB,EAAE,CAAC,CAAC,KAAK,IAAI,CACzC,CAAC;QAEF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;YAC7B,OAAO,gDAAgD,CAAC;SACzD;QAED,MAAM,gBAAgB,GAAG,YAAY;aAClC,GAAG,CACF,CAAC,MAAM,EAAE,EAAE,CACT,QAAQ,MAAM,CAAC,QAAQ,CAAC,GAAG,iBAAiB,MAAM,CAAC,IAAI,CAAC,SAAS,CAC/D,CAAC,EACD,GAAG,CACJ,KAAK,CACT;aACA,IAAI,CAAC,aAAa,CAAC,CAAC;QAEvB,OAAO,sEAAsE,gBAAgB,EAAE,CAAC;IAClG,CAAC;;AAvEH,8BAwEC"}
|
|
@@ -6,11 +6,12 @@ export declare class TokenCompressor {
|
|
|
6
6
|
private maxTokens;
|
|
7
7
|
private compressionRatio;
|
|
8
8
|
private keyPrefix;
|
|
9
|
-
private jsonPropertyThreshold;
|
|
10
9
|
private toolName;
|
|
11
10
|
private characterLimit;
|
|
11
|
+
private jsonPropertyThreshold;
|
|
12
12
|
constructor(toolsService?: ToolsService);
|
|
13
13
|
private estimateTokens;
|
|
14
|
+
setMaxTokens(maxTokens: number): void;
|
|
14
15
|
private tryParseJson;
|
|
15
16
|
compressStringInChunks(content: string, path?: string): string;
|
|
16
17
|
compressContent(content: string, path?: string): string;
|
|
@@ -3,18 +3,23 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.expandTokensDefinition = exports.TokenCompressor = void 0;
|
|
4
4
|
class TokenCompressor {
|
|
5
5
|
storage = {};
|
|
6
|
-
maxTokens =
|
|
6
|
+
maxTokens = 4000;
|
|
7
7
|
compressionRatio = 0.1;
|
|
8
8
|
keyPrefix = "compressed_";
|
|
9
|
-
jsonPropertyThreshold = 20000;
|
|
10
9
|
toolName = exports.expandTokensDefinition.function.name;
|
|
11
|
-
characterLimit =
|
|
10
|
+
characterLimit = this.maxTokens * 4;
|
|
11
|
+
jsonPropertyThreshold = this.maxTokens;
|
|
12
12
|
constructor(toolsService) {
|
|
13
13
|
this.registerTool(toolsService);
|
|
14
14
|
}
|
|
15
15
|
estimateTokens(text) {
|
|
16
16
|
return Math.ceil(text.length / 4);
|
|
17
17
|
}
|
|
18
|
+
setMaxTokens(maxTokens) {
|
|
19
|
+
this.maxTokens = maxTokens;
|
|
20
|
+
this.characterLimit = maxTokens * 4;
|
|
21
|
+
this.jsonPropertyThreshold = maxTokens;
|
|
22
|
+
}
|
|
18
23
|
tryParseJson(content) {
|
|
19
24
|
try {
|
|
20
25
|
return JSON.parse(content);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TokenCompressor.js","sourceRoot":"","sources":["../../../src/processors/TokenCompressor.ts"],"names":[],"mappings":";;;AAQA,MAAa,eAAe;IAClB,OAAO,GAA2B,EAAE,CAAC;IACrC,SAAS,GAAW,
|
|
1
|
+
{"version":3,"file":"TokenCompressor.js","sourceRoot":"","sources":["../../../src/processors/TokenCompressor.ts"],"names":[],"mappings":";;;AAQA,MAAa,eAAe;IAClB,OAAO,GAA2B,EAAE,CAAC;IACrC,SAAS,GAAW,IAAI,CAAC;IACzB,gBAAgB,GAAW,GAAG,CAAC;IAC/B,SAAS,GAAW,aAAa,CAAC;IAClC,QAAQ,GAAW,8BAAsB,CAAC,QAAQ,CAAC,IAAI,CAAC;IACxD,cAAc,GAAW,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IAC5C,qBAAqB,GAAW,IAAI,CAAC,SAAS,CAAC;IAEvD,YAAY,YAA2B;QACrC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;IAClC,CAAC;IAGO,cAAc,CAAC,IAAY;QACjC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC;IAEM,YAAY,CAAC,SAAiB;QACnC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,SAAS,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;IACzC,CAAC;IAKO,YAAY,CAAC,OAAe;QAClC,IAAI;YACF,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;SAC5B;QAAC,MAAM;YACN,OAAO,IAAI,CAAC;SACb;IACH,CAAC;IAKM,sBAAsB,CAAC,OAAe,EAAE,OAAe,EAAE;QAC9D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE;YACzC,OAAO,OAAO,CAAC;SAChB;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,SAAS,GAAG,OAAO,CAAC;QAGxB,OAAO,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE;YAC7C,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC;YAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACtB,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;SAChD;QAGD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACxB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;SAC3B;QAGD,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;YAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/B,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEvB,IAAI,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAG7B,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACzB,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACjC,YAAY,IAAI,wBAAwB,OAAO,GAAG,CAAC;aACpD;YAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC;SAClC;QAGD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QAEjC,OAAO,wBAAwB,WAAW,cAAc,UAAU,kBAAkB,QAAQ,WAAW,IAAI,cAAc,OAAO,CAAC,SAAS,CACxI,CAAC,EACD,GAAG,CACJ,aACC,IAAI,CAAC,QACP,mBAAmB,QAAQ,+EAA+E,CAAC;IAC7G,CAAC;IAKM,eAAe,CAAC,OAAe,EAAE,OAAe,EAAE;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAE5C,IAAI,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE;YAC5B,OAAO,OAAO,CAAC;SAChB;QAGD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,OAAO,EAAE;YAEX,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACjE,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAGjE,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;YAChE,IAAI,gBAAgB,GAAG,MAAM,GAAG,GAAG,EAAE;gBACnC,OAAO,iBAAiB,CAAC;aAC1B;SACF;QAGD,OAAO,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC;IAMM,sBAAsB,CAAC,GAAQ,EAAE,OAAe,EAAE;QACvD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAEtB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC7C,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,CACvD,CAAC;YAIF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;YAC1E,IAAI,aAAa,IAAI,IAAI,CAAC,qBAAqB,GAAG,GAAG,EAAE;gBACrD,OAAO,cAAc,CAAC;aACvB;YAGD,MAAM,UAAU,GAAU,EAAE,CAAC;YAC7B,IAAI,YAAY,GAAU,EAAE,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;gBACnD,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;gBAC/B,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAE3B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;gBACjD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAErD,IAAI,WAAW,GAAG,IAAI,CAAC,qBAAqB,EAAE;oBAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;oBAEhC,MAAM,IAAI,GAAG,kCAAkC,WAAW,YACxD,YAAY,CAAC,MACf,iBAAiB,GAAG,WAAW,IAAI,IAAI,CAAC,MACtC,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAC5B,eAAe,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,aAC1C,IAAI,CAAC,QACP,mBAAmB,GAAG,2BAA2B,CAAC;oBAClD,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAEzB,YAAY,GAAG,EAAE,CAAC;iBACnB;aACF;YAID,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC3B,UAAU,CAAC,OAAO,CAAC,GAAG,YAAY,CAAC,CAAC;aACrC;YACD,OAAO,UAAU,CAAC;SACnB;QAGD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;YAClC,MAAM,MAAM,GAAQ,EAAE,CAAC;YACvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;gBAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC9C,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;aAC3D;YAGD,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;YACnD,IAAI,MAAM,GAAG,IAAI,CAAC,qBAAqB,EAAE;gBACvC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC;gBAEnC,OAAO,6BAA6B,MAAM,kBAAkB,GAAG,WAAW,IAAI,WAAW,MAAM,CAAC,IAAI,CAClG,MAAM,CACP,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,cAAc,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,aACxD,IAAI,CAAC,QACP,mBAAmB,GAAG,6BAA6B,CAAC;aACrD;YACD,OAAO,MAAM,CAAC;SACf;QAGD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;YAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,UAAU,EAAE;gBACd,MAAM,cAAc,GAAG,IAAI,CAAC,sBAAsB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACrE,MAAM,oBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAErE,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBAChD,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;gBAEnE,IAAI,gBAAgB,GAAG,cAAc,GAAG,GAAG,EAAE;oBAC3C,OAAO,oBAAoB,CAAC;iBAC7B;aACF;YAGD,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YACxC,IAAI,MAAM,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE;gBACpC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBAExB,OAAO,+BAA+B,MAAM,kBAAkB,GAAG,WAAW,IAAI,cAAc,GAAG,CAAC,SAAS,CACzG,CAAC,EACD,GAAG,CACJ,aACC,IAAI,CAAC,QACP,mBAAmB,GAAG,6BAA6B,CAAC;aACrD;YACD,OAAO,GAAG,CAAC;SACZ;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,WAAW;QACjB,OAAO,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE;aACnD,QAAQ,CAAC,EAAE,CAAC;aACZ,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IACpB,CAAC;IAEM,gBAAgB,CAAC,OAAgB;QACtC,IAAI,OAAO,CAAC,UAAU,EAAE;YACtB,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,UAAU,EAAE;gBACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE;oBAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;oBACzC,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;oBAEzC,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;wBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;wBAEzB,MAAM,UAAU,GAAG,2BAA2B,MAAM,kBAAkB,GAAG,cAAc,IAAI,CAAC,SAAS,CACnG,CAAC,EACD,GAAG,CACJ,aACC,IAAI,CAAC,QACP,mBAAmB,GAAG,+BAA+B,CAAC;wBAEtD,QAAQ,CAAC,QAAQ,CAAC,SAAS,GAAG,UAAU,CAAC;qBAC1C;iBACF;aACF;SACF;IACH,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,OAAgB;QAM3C,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;YACvC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACzD;aAEI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YACvC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE;gBAClC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE;oBACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC7C;aACF;SACF;QAGD,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAED,eAAe,CACb,QAAoC;QAEpC,OAAO,KAAK,EAAE,gBAA2B,EAAE,gBAA2B,EAAE,EAAE;YACxE,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE;gBACtC,IAAI,QAAQ,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;oBAClC,SAAS;iBACV;gBACD,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;aACrC;QACH,CAAC,CAAC;IACJ,CAAC;IAOD,cAAc,CAAC,GAAW;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;IACnC,CAAC;IAED,YAAY;QACV,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;IACpB,CAAC;IAED,cAAc;QACZ,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;IAED,cAAc;QACZ,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC1C,CAAC;IAED,YAAY,CAAC,YAA2B;QACtC,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;YACxD,YAAY,CAAC,OAAO,CAAC,8BAAsB,CAAC,CAAC;YAC7C,YAAY,CAAC,YAAY,CAAC;gBACxB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAW,EAAE,EAAE;oBAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;oBAEtC,IAAI,CAAC,IAAI,EAAE;wBACT,OAAO,iCAAiC,GAAG,sBAAsB,IAAI,CAAC,cAAc,EAAE,CAAC,IAAI,CACzF,IAAI,CACL,EAAE,CAAC;qBACL;oBACD,OAAO,IAAI,CAAC;gBACd,CAAC;aACF,CAAC,CAAC;SACJ;IACH,CAAC;CACF;AA9UD,0CA8UC;AAEY,QAAA,sBAAsB,GAAS;IAC1C,IAAI,EAAE,UAAU;IAChB,QAAQ,EAAE;QACR,IAAI,EAAE,cAAc;QACpB,WAAW,EACT,mKAAmK;QACrK,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,4CAA4C;iBAC1D;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;SAClB;KACF;CACF,CAAC"}
|