@ramtinj95/opencode-tokenscope 1.6.1 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/tokenscope-lib/analyzer.d.ts +1 -1
- package/dist/tokenscope-lib/analyzer.d.ts.map +1 -1
- package/dist/tokenscope-lib/analyzer.js +14 -12
- package/dist/tokenscope-lib/analyzer.js.map +1 -1
- package/dist/tokenscope-lib/config.d.ts +1 -1
- package/dist/tokenscope-lib/config.d.ts.map +1 -1
- package/dist/tokenscope-lib/config.js +6 -7
- package/dist/tokenscope-lib/config.js.map +1 -1
- package/dist/tokenscope-lib/tokenizer.d.ts +8 -5
- package/dist/tokenscope-lib/tokenizer.d.ts.map +1 -1
- package/dist/tokenscope-lib/tokenizer.js +52 -19
- package/dist/tokenscope-lib/tokenizer.js.map +1 -1
- package/dist/tokenscope-lib/types.d.ts +1 -1
- package/dist/tokenscope-lib/types.d.ts.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -62,7 +62,7 @@ Then restart OpenCode and run `/tokenscope`
|
|
|
62
62
|
|---------------------------|----------|
|
|
63
63
|
| `"@ramtinj95/opencode-tokenscope"` | Uses the version installed at install time. **Never auto-updates.** |
|
|
64
64
|
| `"@ramtinj95/opencode-tokenscope@latest"` | Fetches latest version **every time OpenCode starts**. |
|
|
65
|
-
| `"@ramtinj95/opencode-tokenscope@1.6.
|
|
65
|
+
| `"@ramtinj95/opencode-tokenscope@1.6.2"` | Pins to exact version 1.6.2. Never updates. |
|
|
66
66
|
|
|
67
67
|
To manually update:
|
|
68
68
|
```bash
|
|
@@ -150,7 +150,7 @@ cat token-usage-output.txt
|
|
|
150
150
|
- **API Call Tracking**: See total API calls for main session and subagents
|
|
151
151
|
- **Top Contributors**: Identify the biggest token consumers
|
|
152
152
|
- **Model Normalization**: Handles `provider/model` format automatically
|
|
153
|
-
- **Multi-Tokenizer Support**: Uses official tokenizers (tiktoken for OpenAI,
|
|
153
|
+
- **Multi-Tokenizer Support**: Uses official tokenizers (tiktoken for OpenAI, lightweight Hugging Face tokenizers for others)
|
|
154
154
|
- **Configurable Sections**: Enable/disable analysis features via `tokenscope-config.json`
|
|
155
155
|
|
|
156
156
|
### Skill Analysis
|
|
@@ -9,7 +9,7 @@ export declare class ModelResolver {
|
|
|
9
9
|
resolveModelAndProvider(messages: SessionMessage[]): ModelAndProvider;
|
|
10
10
|
resolveTokenModel(messages: SessionMessage[]): TokenModel;
|
|
11
11
|
private resolveOpenAIModel;
|
|
12
|
-
private
|
|
12
|
+
private resolveHuggingFaceTokenizerModel;
|
|
13
13
|
private mapOpenAI;
|
|
14
14
|
private getProviderID;
|
|
15
15
|
private getModelID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/analyzer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,cAAc,EAEd,UAAU,EACV,aAAa,EACb,mBAAmB,EAMpB,MAAM,SAAS,CAAA;AAGhB,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAK9C,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,UAAU,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,qBAAa,aAAa;IACxB,uBAAuB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,gBAAgB;IA4BrE,iBAAiB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,UAAU;IAezD,OAAO,CAAC,kBAAkB;IAa1B,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/analyzer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,cAAc,EAEd,UAAU,EACV,aAAa,EACb,mBAAmB,EAMpB,MAAM,SAAS,CAAA;AAGhB,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAK9C,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,UAAU,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,qBAAa,aAAa;IACxB,uBAAuB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,gBAAgB;IA4BrE,iBAAiB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,UAAU;IAezD,OAAO,CAAC,kBAAkB;IAa1B,OAAO,CAAC,gCAAgC;IAkCxC,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,YAAY;CAGrB;AAID,qBAAa,gBAAgB;IAC3B,oBAAoB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,mBAAmB,EAAE;IAmCvE,mBAAmB,CAAC,QAAQ,EAAE,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,mBAAmB,EAAE;IAgBlG,kBAAkB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,mBAAmB,EAAE;IA0BrE,qBAAqB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAiBtE,qBAAqB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,MAAM,EAAE;IAI3D,qBAAqB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,mBAAmB,EAAE;IAmBxE,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,oBAAoB;IAoB5B,OAAO,CAAC,UAAU;CAInB;AAID,qBAAa,mBAAmB;IAE5B,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,gBAAgB;gBADhB,gBAAgB,EAAE,gBAAgB,EAClC,gBAAgB,EAAE,gBAAgB;IAGtC,OAAO,CACX,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,cAAc,EAAE,EAC1B,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,CAAC;YAkDX,aAAa;IAsB3B,OAAO,CAAC,yBAAyB;CA+ClC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Analysis classes - ModelResolver, ContentCollector, TokenAnalysisEngine
|
|
2
2
|
import { isToolPart as toolGuard, isReasoningPart as reasoningGuard, isTextPart as textGuard } from "./types";
|
|
3
|
-
import { OPENAI_MODEL_MAP,
|
|
3
|
+
import { OPENAI_MODEL_MAP, HUGGINGFACE_TOKENIZER_MODEL_MAP, PROVIDER_DEFAULTS } from "./config";
|
|
4
4
|
import { summarizeTelemetry } from "./telemetry";
|
|
5
5
|
export class ModelResolver {
|
|
6
6
|
resolveModelAndProvider(messages) {
|
|
@@ -33,9 +33,9 @@ export class ModelResolver {
|
|
|
33
33
|
const openaiModel = this.resolveOpenAIModel(modelID, providerID);
|
|
34
34
|
if (openaiModel)
|
|
35
35
|
return openaiModel;
|
|
36
|
-
const
|
|
37
|
-
if (
|
|
38
|
-
return
|
|
36
|
+
const huggingFaceTokenizerModel = this.resolveHuggingFaceTokenizerModel(modelID, providerID);
|
|
37
|
+
if (huggingFaceTokenizerModel)
|
|
38
|
+
return huggingFaceTokenizerModel;
|
|
39
39
|
}
|
|
40
40
|
return { name: "approx", spec: { kind: "approx" } };
|
|
41
41
|
}
|
|
@@ -49,28 +49,30 @@ export class ModelResolver {
|
|
|
49
49
|
}
|
|
50
50
|
return undefined;
|
|
51
51
|
}
|
|
52
|
-
|
|
53
|
-
if (modelID &&
|
|
54
|
-
return { name: modelID, spec: { kind: "
|
|
52
|
+
resolveHuggingFaceTokenizerModel(modelID, providerID) {
|
|
53
|
+
if (modelID && HUGGINGFACE_TOKENIZER_MODEL_MAP[modelID]) {
|
|
54
|
+
return { name: modelID, spec: { kind: "huggingface", hub: HUGGINGFACE_TOKENIZER_MODEL_MAP[modelID] } };
|
|
55
55
|
}
|
|
56
56
|
if (providerID && PROVIDER_DEFAULTS[providerID]) {
|
|
57
57
|
return { name: modelID ?? providerID, spec: PROVIDER_DEFAULTS[providerID] };
|
|
58
58
|
}
|
|
59
|
-
// Prefix-based fallbacks
|
|
60
59
|
if (modelID?.startsWith("claude")) {
|
|
61
|
-
return { name: modelID, spec: { kind: "
|
|
60
|
+
return { name: modelID, spec: { kind: "huggingface", hub: "Xenova/claude-tokenizer" } };
|
|
62
61
|
}
|
|
63
62
|
if (modelID?.startsWith("llama")) {
|
|
64
63
|
return {
|
|
65
64
|
name: modelID,
|
|
66
|
-
spec: {
|
|
65
|
+
spec: {
|
|
66
|
+
kind: "huggingface",
|
|
67
|
+
hub: HUGGINGFACE_TOKENIZER_MODEL_MAP[modelID] ?? "Xenova/Meta-Llama-3.1-Tokenizer",
|
|
68
|
+
},
|
|
67
69
|
};
|
|
68
70
|
}
|
|
69
71
|
if (modelID?.startsWith("mistral")) {
|
|
70
|
-
return { name: modelID, spec: { kind: "
|
|
72
|
+
return { name: modelID, spec: { kind: "huggingface", hub: "Xenova/mistral-tokenizer-v3" } };
|
|
71
73
|
}
|
|
72
74
|
if (modelID?.startsWith("deepseek")) {
|
|
73
|
-
return { name: modelID, spec: { kind: "
|
|
75
|
+
return { name: modelID, spec: { kind: "huggingface", hub: "deepseek-ai/DeepSeek-V3" } };
|
|
74
76
|
}
|
|
75
77
|
return undefined;
|
|
76
78
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyzer.js","sourceRoot":"","sources":["../../tokenscope-lib/analyzer.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAc1E,OAAO,EAAE,UAAU,IAAI,SAAS,EAAE,eAAe,IAAI,cAAc,EAAE,UAAU,IAAI,SAAS,EAAE,MAAM,SAAS,CAAA;AAC7G,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAA;AAEtF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAUhD,MAAM,OAAO,aAAa;IACxB,uBAAuB,CAAC,QAA0B;QAChD,IAAI,kBAAkB,GAAG,WAAW,CAAA;QACpC,IAAI,eAAe,GAAG,0BAA0B,CAAA;QAEhD,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;YAExC,IAAI,UAAU,EAAE,CAAC;gBACf,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,kBAAkB,CAAA;YAC1E,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACZ,eAAe,GAAG,OAAO,CAAA;YAC3B,CAAC;YACD,IAAI,UAAU,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAK;YACP,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAE9C,OAAO;YACL,KAAK;YACL,UAAU,EAAE,kBAAkB;YAC9B,OAAO,EAAE,eAAe;SACzB,CAAA;IACH,CAAC;IAED,iBAAiB,CAAC,QAA0B;QAC1C,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAA;YAC3D,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAA;YAEjE,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;YAChE,IAAI,WAAW;gBAAE,OAAO,WAAW,CAAA;YAEnC,MAAM,gBAAgB,GAAG,IAAI,CAAC,wBAAwB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;YAC3E,IAAI,gBAAgB;gBAAE,OAAO,gBAAgB,CAAA;QAC/C,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAA;IACrD,CAAC;IAEO,kBAAkB,CAAC,OAAgB,EAAE,UAAmB;QAC9D,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,UAAU,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YACnF,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;YACtC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAA;QAC/E,CAAC;QAED,IAAI,OAAO,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,EAAE,CAAA;QACxF,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,wBAAwB,CAAC,OAAgB,EAAE,UAAmB;QACpE,IAAI,OAAO,IAAI,sBAAsB,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,CAAA;QAChG,CAAC;QAED,IAAI,UAAU,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,UAAU,EAAE,IAAI,EAAE,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAA;QAC7E,CAAC;QAED,yBAAyB;QACzB,IAAI,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAA;QAC1F,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,iCAAiC,EAAE;aAC1G,CAAA;QACH,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,6BAA6B,EAAE,EAAE,CAAA;QAC9F,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAA;QAC1F,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,SAAS,CAAC,OAAgB;QAChC,IAAI,CAAC,OAAO;YAAE,OAAO,aAAa,CAAA;QAClC,OAAO,gBAAgB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAA;IAC7C,CAAC;IAEO,aAAa,CAAC,OAAuB;QAC3C,OAAO,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAA;IAClE,CAAC;IAEO,UAAU,CAAC,OAAuB;QACxC,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAA;IAC5D,CAAC;IAEO,YAAY,CAAC,KAAc;QACjC,OAAO,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAA;IACtD,CAAC;CACF;AAED,qBAAqB;AAErB,MAAM,OAAO,gBAAgB;IAC3B,oBAAoB,CAAC,QAA0B;QAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;QACzC,MAAM,SAAS,GAAG,CAAC,KAAyB,EAAE,EAAE;YAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,OAAO,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBACnC,IAAI,OAAO;wBAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;gBAC5C,CAAC;gBACD,OAAM;YACR,CAAC;YAED,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YACpC,IAAI,OAAO;gBAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAC5C,CAAC,CAAA;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,2EAA2E;YAC3E,qEAAqE;YACrE,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtE,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAChC,CAAC;YAED,kFAAkF;YAClF,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC/C,IAAI,OAAO;oBAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3D,KAAK,EAAE,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC;YACpD,OAAO;SACR,CAAC,CAAC,CAAA;IACL,CAAC;IAED,mBAAmB,CAAC,QAA0B,EAAE,IAA0B;QACxE,MAAM,OAAO,GAA0B,EAAE,CAAA;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAC/C,IAAI,CAAC,OAAO;gBAAE,SAAQ;YAEtB,KAAK,IAAI,CAAC,CAAA;YACV,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;QACvE,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,kBAAkB,CAAC,QAA0B;QAC3C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;QAC7C,MAAM,oBAAoB,GAAG,mCAAmC,CAAA;QAEhE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAE9B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW;oBAAE,SAAQ;gBAE/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;gBACvF,MAAM,MAAM,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAA;gBAClD,IAAI,CAAC,MAAM;oBAAE,SAAQ;gBAErB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAA;gBACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAChD,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAA;YACzE,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACrE,KAAK,EAAE,QAAQ;YACf,OAAO;SACR,CAAC,CAAC,CAAA;IACL,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAA;gBACpC,IAAI,QAAQ,EAAE,CAAC;oBACb,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/D,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACvE,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,MAAM,OAAO,GAA0B,EAAE,CAAA;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAEnC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAA;gBAChD,IAAI,CAAC,IAAI;oBAAE,SAAQ;gBAEnB,KAAK,IAAI,CAAC,CAAA;gBACV,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;YAC9D,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAEO,WAAW,CAAC,KAA2B;QAC7C,OAAO,KAAK;aACT,MAAM,CAAC,SAAS,CAAC;aACjB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;aAC9B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,MAAM,CAAC,CAAA;IACjB,CAAC;IAEO,oBAAoB,CAAC,OAAe,EAAE,KAAa;QACzD,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAA;QAEnC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,mBAAmB,CAAA;QAC3G,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG;YAAE,OAAO,oBAAoB,CAAA;QAC7G,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,kBAAkB,CAAA;QAChF,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,oBAAoB,CAAA;QACpH,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAAE,OAAO,kBAAkB,CAAA;QAChH,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,OAAO,mBAAmB,CAAA;QAChH,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzF,OAAO,uBAAuB,CAAA;QAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,oBAAoB,CAAA;QACnH,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAAE,OAAO,iBAAiB,CAAA;QAC5G,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAAE,OAAO,iBAAiB,CAAA;QAC3F,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACxF,OAAO,uBAAuB,CAAA;QAEhC,OAAO,UAAU,KAAK,EAAE,CAAA;IAC1B,CAAC;IAEO,UAAU,CAAC,KAAa;QAC9B,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;QACxB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAChD,CAAC;CACF;AAED,wBAAwB;AAExB,MAAM,OAAO,mBAAmB;IAEpB;IACA;IAFV,YACU,gBAAkC,EAClC,gBAAkC;QADlC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,qBAAgB,GAAhB,gBAAgB,CAAkB;IACzC,CAAC;IAEJ,KAAK,CAAC,OAAO,CACX,SAAiB,EACjB,QAA0B,EAC1B,UAAsB,EACtB,UAAkB;QAElB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;QAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;QACvF,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QACtE,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAC5E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAE5E,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACpE,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,UAAU,CAAC;YACnE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC;YAC7D,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,CAAC;YACvE,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,CAAC;YAChE,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,CAAC;SACzE,CAAC,CAAA;QAEF,MAAM,QAAQ,GAAkB;YAC9B,SAAS;YACT,KAAK,EAAE,UAAU;YACjB,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE;YACzD,WAAW,EACT,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW;YAC3G,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;YACnB,qBAAqB,EAAE,CAAC;YACxB,YAAY,EAAE,CAAC;YACf,kBAAkB,EAAE,CAAC;YACrB,mBAAmB,EAAE,CAAC;YACtB,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;YACnB,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,CAAC;YACtB,oBAAoB,EAAE,CAAC;YACvB,6BAA6B,EAAE,SAAS;YACxC,WAAW,EAAE,CAAC;YACd,cAAc,EAAE,CAAC;YACjB,cAAc;YACd,cAAc;YACd,QAAQ,EAAE,EAAE;SACb,CAAA;QAED,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAElD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,KAAa,EACb,OAA8B,EAC9B,KAAiB,EACjB,UAAkB;QAElB,MAAM,OAAO,GAAoB,EAAE,CAAA;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;YAC7E,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;YAC/C,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAEzE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAA;IACtE,CAAC;IAEO,yBAAyB,CAAC,QAAuB,EAAE,QAA0B;QACnF,MAAM,SAAS,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QAE9C,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAA;QAC5C,QAAQ,CAAC,YAAY,GAAG,SAAS,CAAC,YAAY,CAAA;QAC9C,QAAQ,CAAC,eAAe,GAAG,SAAS,CAAC,eAAe,CAAA;QACpD,QAAQ,CAAC,eAAe,GAAG,SAAS,CAAC,eAAe,CAAA;QACpD,QAAQ,CAAC,gBAAgB,GAAG,SAAS,CAAC,gBAAgB,CAAA;QACtD,QAAQ,CAAC,qBAAqB,GAAG,SAAS,CAAC,qBAAqB,CAAA;QAChE,QAAQ,CAAC,YAAY,GAAG,SAAS,CAAC,YAAY,CAAA;QAC9C,QAAQ,CAAC,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAA;QAC1D,QAAQ,CAAC,mBAAmB,GAAG,SAAS,CAAC,mBAAmB,CAAA;QAC5D,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAA;QAC5C,QAAQ,CAAC,cAAc,GAAG,SAAS,CAAC,cAAc,CAAA;QAClD,QAAQ,CAAC,eAAe,GAAG,SAAS,CAAC,eAAe,CAAA;QACpD,QAAQ,CAAC,gBAAgB,GAAG,SAAS,CAAC,gBAAgB,CAAA;QACtD,QAAQ,CAAC,mBAAmB,GAAG,SAAS,CAAC,mBAAmB,CAAA;QAC5D,QAAQ,CAAC,mBAAmB,GAAG,SAAS,CAAC,mBAAmB,CAAA;QAC5D,QAAQ,CAAC,oBAAoB,GAAG,SAAS,CAAC,oBAAoB,CAAA;QAC9D,QAAQ,CAAC,6BAA6B,GAAG,SAAS,CAAC,6BAA6B,CAAA;QAEhF,MAAM,mBAAmB,GAAG,SAAS,CAAC,eAAe,GAAG,SAAS,CAAC,mBAAmB,CAAA;QACrF,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW,CAAA;QACtG,MAAM,4BAA4B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,mBAAmB,GAAG,iBAAiB,CAAC,CAAA;QACzF,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAA;QACpE,MAAM,qBAAqB,GACzB,4BAA4B,IAAI,GAAG;YACnC,4BAA4B,IAAI,mBAAmB,GAAG,IAAI;YAC1D,4BAA4B,IAAI,iBAAiB,GAAG,GAAG,CAAA;QAEzD,IAAI,4BAA4B,IAAI,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7D,MAAM,aAAa,GAAG,qBAAqB;gBACzC,CAAC,CAAC,sCAAsC;gBACxC,CAAC,CAAC,yCAAyC,CAAA;YAE7C,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,GAAG,4BAA4B,CAAA;YACrE,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC,CAAA;YACrG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAA;QAC5E,CAAC;QAED,QAAQ,CAAC,WAAW;YAClB,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW;gBACtC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW;gBACpC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW;gBACzC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW;gBACrC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW,CAAA;IAC7C,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"analyzer.js","sourceRoot":"","sources":["../../tokenscope-lib/analyzer.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAc1E,OAAO,EAAE,UAAU,IAAI,SAAS,EAAE,eAAe,IAAI,cAAc,EAAE,UAAU,IAAI,SAAS,EAAE,MAAM,SAAS,CAAA;AAC7G,OAAO,EAAE,gBAAgB,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAA;AAE/F,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAUhD,MAAM,OAAO,aAAa;IACxB,uBAAuB,CAAC,QAA0B;QAChD,IAAI,kBAAkB,GAAG,WAAW,CAAA;QACpC,IAAI,eAAe,GAAG,0BAA0B,CAAA;QAEhD,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;YAExC,IAAI,UAAU,EAAE,CAAC;gBACf,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,kBAAkB,CAAA;YAC1E,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACZ,eAAe,GAAG,OAAO,CAAA;YAC3B,CAAC;YACD,IAAI,UAAU,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAK;YACP,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAE9C,OAAO;YACL,KAAK;YACL,UAAU,EAAE,kBAAkB;YAC9B,OAAO,EAAE,eAAe;SACzB,CAAA;IACH,CAAC;IAED,iBAAiB,CAAC,QAA0B;QAC1C,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAA;YAC3D,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAA;YAEjE,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;YAChE,IAAI,WAAW;gBAAE,OAAO,WAAW,CAAA;YAEnC,MAAM,yBAAyB,GAAG,IAAI,CAAC,gCAAgC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;YAC5F,IAAI,yBAAyB;gBAAE,OAAO,yBAAyB,CAAA;QACjE,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAA;IACrD,CAAC;IAEO,kBAAkB,CAAC,OAAgB,EAAE,UAAmB;QAC9D,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,UAAU,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YACnF,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;YACtC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAA;QAC/E,CAAC;QAED,IAAI,OAAO,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,EAAE,CAAA;QACxF,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,gCAAgC,CAAC,OAAgB,EAAE,UAAmB;QAC5E,IAAI,OAAO,IAAI,+BAA+B,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,+BAA+B,CAAC,OAAO,CAAC,EAAE,EAAE,CAAA;QACxG,CAAC;QAED,IAAI,UAAU,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,UAAU,EAAE,IAAI,EAAE,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAA;QAC7E,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAA;QACzF,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE;oBACJ,IAAI,EAAE,aAAa;oBACnB,GAAG,EAAE,+BAA+B,CAAC,OAAO,CAAC,IAAI,iCAAiC;iBACnF;aACF,CAAA;QACH,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,6BAA6B,EAAE,EAAE,CAAA;QAC7F,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAA;QACzF,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,SAAS,CAAC,OAAgB;QAChC,IAAI,CAAC,OAAO;YAAE,OAAO,aAAa,CAAA;QAClC,OAAO,gBAAgB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAA;IAC7C,CAAC;IAEO,aAAa,CAAC,OAAuB;QAC3C,OAAO,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAA;IAClE,CAAC;IAEO,UAAU,CAAC,OAAuB;QACxC,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAA;IAC5D,CAAC;IAEO,YAAY,CAAC,KAAc;QACjC,OAAO,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAA;IACtD,CAAC;CACF;AAED,qBAAqB;AAErB,MAAM,OAAO,gBAAgB;IAC3B,oBAAoB,CAAC,QAA0B;QAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;QACzC,MAAM,SAAS,GAAG,CAAC,KAAyB,EAAE,EAAE;YAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,OAAO,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBACnC,IAAI,OAAO;wBAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;gBAC5C,CAAC;gBACD,OAAM;YACR,CAAC;YAED,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YACpC,IAAI,OAAO;gBAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAC5C,CAAC,CAAA;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,2EAA2E;YAC3E,qEAAqE;YACrE,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtE,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAChC,CAAC;YAED,kFAAkF;YAClF,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC/C,IAAI,OAAO;oBAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3D,KAAK,EAAE,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC;YACpD,OAAO;SACR,CAAC,CAAC,CAAA;IACL,CAAC;IAED,mBAAmB,CAAC,QAA0B,EAAE,IAA0B;QACxE,MAAM,OAAO,GAA0B,EAAE,CAAA;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAC/C,IAAI,CAAC,OAAO;gBAAE,SAAQ;YAEtB,KAAK,IAAI,CAAC,CAAA;YACV,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;QACvE,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,kBAAkB,CAAC,QAA0B;QAC3C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;QAC7C,MAAM,oBAAoB,GAAG,mCAAmC,CAAA;QAEhE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAE9B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW;oBAAE,SAAQ;gBAE/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;gBACvF,MAAM,MAAM,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAA;gBAClD,IAAI,CAAC,MAAM;oBAAE,SAAQ;gBAErB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAA;gBACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAChD,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAA;YACzE,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACrE,KAAK,EAAE,QAAQ;YACf,OAAO;SACR,CAAC,CAAC,CAAA;IACL,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAA;gBACpC,IAAI,QAAQ,EAAE,CAAC;oBACb,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/D,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACvE,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,MAAM,OAAO,GAA0B,EAAE,CAAA;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAEnC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAA;gBAChD,IAAI,CAAC,IAAI;oBAAE,SAAQ;gBAEnB,KAAK,IAAI,CAAC,CAAA;gBACV,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;YAC9D,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAEO,WAAW,CAAC,KAA2B;QAC7C,OAAO,KAAK;aACT,MAAM,CAAC,SAAS,CAAC;aACjB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;aAC9B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,MAAM,CAAC,CAAA;IACjB,CAAC;IAEO,oBAAoB,CAAC,OAAe,EAAE,KAAa;QACzD,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAA;QAEnC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,mBAAmB,CAAA;QAC3G,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG;YAAE,OAAO,oBAAoB,CAAA;QAC7G,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,kBAAkB,CAAA;QAChF,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,oBAAoB,CAAA;QACpH,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAAE,OAAO,kBAAkB,CAAA;QAChH,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,OAAO,mBAAmB,CAAA;QAChH,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzF,OAAO,uBAAuB,CAAA;QAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,oBAAoB,CAAA;QACnH,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAAE,OAAO,iBAAiB,CAAA;QAC5G,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAAE,OAAO,iBAAiB,CAAA;QAC3F,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACxF,OAAO,uBAAuB,CAAA;QAEhC,OAAO,UAAU,KAAK,EAAE,CAAA;IAC1B,CAAC;IAEO,UAAU,CAAC,KAAa;QAC9B,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;QACxB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAChD,CAAC;CACF;AAED,wBAAwB;AAExB,MAAM,OAAO,mBAAmB;IAEpB;IACA;IAFV,YACU,gBAAkC,EAClC,gBAAkC;QADlC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,qBAAgB,GAAhB,gBAAgB,CAAkB;IACzC,CAAC;IAEJ,KAAK,CAAC,OAAO,CACX,SAAiB,EACjB,QAA0B,EAC1B,UAAsB,EACtB,UAAkB;QAElB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;QAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;QACvF,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QACtE,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAC5E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAE5E,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACpE,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,UAAU,CAAC;YACnE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC;YAC7D,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,CAAC;YACvE,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,CAAC;YAChE,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,CAAC;SACzE,CAAC,CAAA;QAEF,MAAM,QAAQ,GAAkB;YAC9B,SAAS;YACT,KAAK,EAAE,UAAU;YACjB,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE;YACzD,WAAW,EACT,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW;YAC3G,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;YACnB,qBAAqB,EAAE,CAAC;YACxB,YAAY,EAAE,CAAC;YACf,kBAAkB,EAAE,CAAC;YACrB,mBAAmB,EAAE,CAAC;YACtB,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;YACnB,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,CAAC;YACtB,oBAAoB,EAAE,CAAC;YACvB,6BAA6B,EAAE,SAAS;YACxC,WAAW,EAAE,CAAC;YACd,cAAc,EAAE,CAAC;YACjB,cAAc;YACd,cAAc;YACd,QAAQ,EAAE,EAAE;SACb,CAAA;QAED,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAElD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,KAAa,EACb,OAA8B,EAC9B,KAAiB,EACjB,UAAkB;QAElB,MAAM,OAAO,GAAoB,EAAE,CAAA;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;YAC7E,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;YAC/C,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAEzE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAA;IACtE,CAAC;IAEO,yBAAyB,CAAC,QAAuB,EAAE,QAA0B;QACnF,MAAM,SAAS,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QAE9C,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAA;QAC5C,QAAQ,CAAC,YAAY,GAAG,SAAS,CAAC,YAAY,CAAA;QAC9C,QAAQ,CAAC,eAAe,GAAG,SAAS,CAAC,eAAe,CAAA;QACpD,QAAQ,CAAC,eAAe,GAAG,SAAS,CAAC,eAAe,CAAA;QACpD,QAAQ,CAAC,gBAAgB,GAAG,SAAS,CAAC,gBAAgB,CAAA;QACtD,QAAQ,CAAC,qBAAqB,GAAG,SAAS,CAAC,qBAAqB,CAAA;QAChE,QAAQ,CAAC,YAAY,GAAG,SAAS,CAAC,YAAY,CAAA;QAC9C,QAAQ,CAAC,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAA;QAC1D,QAAQ,CAAC,mBAAmB,GAAG,SAAS,CAAC,mBAAmB,CAAA;QAC5D,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAA;QAC5C,QAAQ,CAAC,cAAc,GAAG,SAAS,CAAC,cAAc,CAAA;QAClD,QAAQ,CAAC,eAAe,GAAG,SAAS,CAAC,eAAe,CAAA;QACpD,QAAQ,CAAC,gBAAgB,GAAG,SAAS,CAAC,gBAAgB,CAAA;QACtD,QAAQ,CAAC,mBAAmB,GAAG,SAAS,CAAC,mBAAmB,CAAA;QAC5D,QAAQ,CAAC,mBAAmB,GAAG,SAAS,CAAC,mBAAmB,CAAA;QAC5D,QAAQ,CAAC,oBAAoB,GAAG,SAAS,CAAC,oBAAoB,CAAA;QAC9D,QAAQ,CAAC,6BAA6B,GAAG,SAAS,CAAC,6BAA6B,CAAA;QAEhF,MAAM,mBAAmB,GAAG,SAAS,CAAC,eAAe,GAAG,SAAS,CAAC,mBAAmB,CAAA;QACrF,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW,CAAA;QACtG,MAAM,4BAA4B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,mBAAmB,GAAG,iBAAiB,CAAC,CAAA;QACzF,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAA;QACpE,MAAM,qBAAqB,GACzB,4BAA4B,IAAI,GAAG;YACnC,4BAA4B,IAAI,mBAAmB,GAAG,IAAI;YAC1D,4BAA4B,IAAI,iBAAiB,GAAG,GAAG,CAAA;QAEzD,IAAI,4BAA4B,IAAI,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7D,MAAM,aAAa,GAAG,qBAAqB;gBACzC,CAAC,CAAC,sCAAsC;gBACxC,CAAC,CAAC,yCAAyC,CAAA;YAE7C,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,GAAG,4BAA4B,CAAA;YACrE,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC,CAAA;YACrG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAA;QAC5E,CAAC;QAED,QAAQ,CAAC,WAAW;YAClB,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW;gBACtC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW;gBACpC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW;gBACzC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW;gBACrC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW,CAAA;IAC7C,CAAC;CACF"}
|
|
@@ -4,6 +4,6 @@ export declare function loadModelPricing(): Promise<Record<string, ModelPricing>
|
|
|
4
4
|
export declare const DEFAULT_TOKENSCOPE_CONFIG: TokenscopeConfig;
|
|
5
5
|
export declare function loadTokenscopeConfig(): Promise<TokenscopeConfig>;
|
|
6
6
|
export declare const OPENAI_MODEL_MAP: Record<string, string>;
|
|
7
|
-
export declare const
|
|
7
|
+
export declare const HUGGINGFACE_TOKENIZER_MODEL_MAP: Record<string, string>;
|
|
8
8
|
export declare const PROVIDER_DEFAULTS: Record<string, TokenizerSpec>;
|
|
9
9
|
//# sourceMappingURL=config.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/config.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAE5E,eAAO,MAAM,mBAAmB,IAAI,CAAA;AAQpC,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAY9E;AAID,eAAO,MAAM,yBAAyB,EAAE,gBAMvC,CAAA;AAaD,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAatE;AAGD,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAiBnD,CAAA;AAGD,eAAO,MAAM,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/config.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAE5E,eAAO,MAAM,mBAAmB,IAAI,CAAA;AAQpC,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAY9E;AAID,eAAO,MAAM,yBAAyB,EAAE,gBAMvC,CAAA;AAaD,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAatE;AAGD,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAiBnD,CAAA;AAGD,eAAO,MAAM,+BAA+B,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CA2BlE,CAAA;AAGD,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAK3D,CAAA"}
|
|
@@ -72,8 +72,8 @@ export const OPENAI_MODEL_MAP = {
|
|
|
72
72
|
"text-embedding-3-small": "text-embedding-3-small",
|
|
73
73
|
"text-embedding-ada-002": "text-embedding-ada-002",
|
|
74
74
|
};
|
|
75
|
-
//
|
|
76
|
-
export const
|
|
75
|
+
// Hugging Face tokenizer model mapping
|
|
76
|
+
export const HUGGINGFACE_TOKENIZER_MODEL_MAP = {
|
|
77
77
|
"claude-opus-4": "Xenova/claude-tokenizer",
|
|
78
78
|
"claude-sonnet-4": "Xenova/claude-tokenizer",
|
|
79
79
|
"claude-3.7-sonnet": "Xenova/claude-tokenizer",
|
|
@@ -103,10 +103,9 @@ export const TRANSFORMERS_MODEL_MAP = {
|
|
|
103
103
|
};
|
|
104
104
|
// Provider default tokenizers
|
|
105
105
|
export const PROVIDER_DEFAULTS = {
|
|
106
|
-
anthropic: { kind: "
|
|
107
|
-
meta: { kind: "
|
|
108
|
-
mistral: { kind: "
|
|
109
|
-
deepseek: { kind: "
|
|
110
|
-
google: { kind: "transformers", hub: "google/gemma-2-9b-it" },
|
|
106
|
+
anthropic: { kind: "huggingface", hub: "Xenova/claude-tokenizer" },
|
|
107
|
+
meta: { kind: "huggingface", hub: "Xenova/Meta-Llama-3.1-Tokenizer" },
|
|
108
|
+
mistral: { kind: "huggingface", hub: "Xenova/mistral-tokenizer-v3" },
|
|
109
|
+
deepseek: { kind: "huggingface", hub: "deepseek-ai/DeepSeek-V3" },
|
|
111
110
|
};
|
|
112
111
|
//# sourceMappingURL=config.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../tokenscope-lib/config.ts"],"names":[],"mappings":"AAAA,0DAA0D;AAE1D,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,MAAM,aAAa,CAAA;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AAGnC,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAA;AACpC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;AAClF,MAAM,8BAA8B,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,wBAAwB,CAAC,CAAA;AACxF,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,wBAAwB,CAAC,CAAA;AAEzG,gBAAgB;AAChB,IAAI,aAAa,GAAwC,IAAI,CAAA;AAE7D,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,IAAI,aAAa;QAAE,OAAO,aAAa,CAAA;IAEvC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAA;QACzD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAClD,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAChC,OAAO,aAAc,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,aAAa,GAAG,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,CAAA;QACjF,OAAO,aAAa,CAAA;IACtB,CAAC;AACH,CAAC;AAED,wCAAwC;AAExC,MAAM,CAAC,MAAM,yBAAyB,GAAqB;IACzD,sBAAsB,EAAE,IAAI;IAC5B,0BAA0B,EAAE,IAAI;IAChC,qBAAqB,EAAE,IAAI;IAC3B,sBAAsB,EAAE,IAAI;IAC5B,mBAAmB,EAAE,IAAI;CAC1B,CAAA;AAED,IAAI,uBAAuB,GAA4B,IAAI,CAAA;AAE3D,KAAK,UAAU,wBAAwB,CAAC,UAAkB;IACxD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAClD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAA8B,CAAA;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,IAAI,uBAAuB;QAAE,OAAO,uBAAuB,CAAA;IAE3D,MAAM,UAAU,GAAG,MAAM,wBAAwB,CAAC,2BAA2B,CAAC,CAAA;IAC9E,MAAM,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,wBAAwB,CAAC,8BAA8B,CAAC,CAAA;IAExG,uBAAuB,GAAG;QACxB,GAAG,yBAAyB;QAC5B,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;QACxB,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC;KACtB,CAAA;IAED,OAAO,uBAAuB,CAAA;AAChC,CAAC;AAED,oCAAoC;AACpC,MAAM,CAAC,MAAM,gBAAgB,GAA2B;IACtD,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,QAAQ;IACnB,IAAI,EAAE,QAAQ;IACd,SAAS,EAAE,QAAQ;IACnB,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,QAAQ;IACnB,cAAc,EAAE,QAAQ;IACxB,QAAQ,EAAE,QAAQ;IAClB,aAAa,EAAE,aAAa;IAC5B,aAAa,EAAE,OAAO;IACtB,OAAO,EAAE,OAAO;IAChB,eAAe,EAAE,eAAe;IAChC,wBAAwB,EAAE,wBAAwB;IAClD,wBAAwB,EAAE,wBAAwB;IAClD,wBAAwB,EAAE,wBAAwB;CACnD,CAAA;AAED,
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../tokenscope-lib/config.ts"],"names":[],"mappings":"AAAA,0DAA0D;AAE1D,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,MAAM,aAAa,CAAA;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AAGnC,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAA;AACpC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;AAClF,MAAM,8BAA8B,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,wBAAwB,CAAC,CAAA;AACxF,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,wBAAwB,CAAC,CAAA;AAEzG,gBAAgB;AAChB,IAAI,aAAa,GAAwC,IAAI,CAAA;AAE7D,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,IAAI,aAAa;QAAE,OAAO,aAAa,CAAA;IAEvC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAA;QACzD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAClD,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAChC,OAAO,aAAc,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,aAAa,GAAG,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,CAAA;QACjF,OAAO,aAAa,CAAA;IACtB,CAAC;AACH,CAAC;AAED,wCAAwC;AAExC,MAAM,CAAC,MAAM,yBAAyB,GAAqB;IACzD,sBAAsB,EAAE,IAAI;IAC5B,0BAA0B,EAAE,IAAI;IAChC,qBAAqB,EAAE,IAAI;IAC3B,sBAAsB,EAAE,IAAI;IAC5B,mBAAmB,EAAE,IAAI;CAC1B,CAAA;AAED,IAAI,uBAAuB,GAA4B,IAAI,CAAA;AAE3D,KAAK,UAAU,wBAAwB,CAAC,UAAkB;IACxD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAClD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAA8B,CAAA;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,IAAI,uBAAuB;QAAE,OAAO,uBAAuB,CAAA;IAE3D,MAAM,UAAU,GAAG,MAAM,wBAAwB,CAAC,2BAA2B,CAAC,CAAA;IAC9E,MAAM,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,wBAAwB,CAAC,8BAA8B,CAAC,CAAA;IAExG,uBAAuB,GAAG;QACxB,GAAG,yBAAyB;QAC5B,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;QACxB,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC;KACtB,CAAA;IAED,OAAO,uBAAuB,CAAA;AAChC,CAAC;AAED,oCAAoC;AACpC,MAAM,CAAC,MAAM,gBAAgB,GAA2B;IACtD,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,QAAQ;IACnB,IAAI,EAAE,QAAQ;IACd,SAAS,EAAE,QAAQ;IACnB,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,QAAQ;IACnB,cAAc,EAAE,QAAQ;IACxB,QAAQ,EAAE,QAAQ;IAClB,aAAa,EAAE,aAAa;IAC5B,aAAa,EAAE,OAAO;IACtB,OAAO,EAAE,OAAO;IAChB,eAAe,EAAE,eAAe;IAChC,wBAAwB,EAAE,wBAAwB;IAClD,wBAAwB,EAAE,wBAAwB;IAClD,wBAAwB,EAAE,wBAAwB;CACnD,CAAA;AAED,uCAAuC;AACvC,MAAM,CAAC,MAAM,+BAA+B,GAA2B;IACrE,eAAe,EAAE,yBAAyB;IAC1C,iBAAiB,EAAE,yBAAyB;IAC5C,mBAAmB,EAAE,yBAAyB;IAC9C,mBAAmB,EAAE,yBAAyB;IAC9C,kBAAkB,EAAE,yBAAyB;IAC7C,eAAe,EAAE,yBAAyB;IAC1C,iBAAiB,EAAE,yBAAyB;IAC5C,gBAAgB,EAAE,yBAAyB;IAC3C,YAAY,EAAE,yBAAyB;IACvC,YAAY,EAAE,yBAAyB;IACvC,oBAAoB,EAAE,yBAAyB;IAC/C,SAAS,EAAE,yBAAyB;IACpC,WAAW,EAAE,gCAAgC;IAC7C,WAAW,EAAE,4BAA4B;IACzC,WAAW,EAAE,iCAAiC;IAC9C,SAAS,EAAE,6BAA6B;IACxC,SAAS,EAAE,yBAAyB;IACpC,YAAY,EAAE,6BAA6B;IAC3C,aAAa,EAAE,yBAAyB;IACxC,aAAa,EAAE,yBAAyB;IACxC,aAAa,EAAE,yBAAyB;IACxC,eAAe,EAAE,6BAA6B;IAC9C,eAAe,EAAE,6BAA6B;IAC9C,cAAc,EAAE,wCAAwC;IACxD,gBAAgB,EAAE,wCAAwC;IAC1D,WAAW,EAAE,6BAA6B;CAC3C,CAAA;AAED,8BAA8B;AAC9B,MAAM,CAAC,MAAM,iBAAiB,GAAkC;IAC9D,SAAS,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,yBAAyB,EAAE;IAClE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,iCAAiC,EAAE;IACrE,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,6BAA6B,EAAE;IACpE,QAAQ,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,yBAAyB,EAAE;CAClE,CAAA"}
|
|
@@ -3,18 +3,21 @@ import { WarningCollector } from "./warnings";
|
|
|
3
3
|
export declare class TokenizerManager {
|
|
4
4
|
private warnings?;
|
|
5
5
|
private tiktokenCache;
|
|
6
|
-
private
|
|
6
|
+
private huggingFaceTokenizerCache;
|
|
7
7
|
private tiktokenModule?;
|
|
8
|
-
private
|
|
8
|
+
private huggingFaceTokenizersModule?;
|
|
9
9
|
constructor(warnings?: WarningCollector | undefined);
|
|
10
10
|
countTokens(content: string, model: TokenModel): Promise<number>;
|
|
11
11
|
private approximateTokenCount;
|
|
12
12
|
private countWithTiktoken;
|
|
13
|
-
private
|
|
13
|
+
private countWithHuggingFaceTokenizer;
|
|
14
14
|
private loadTiktokenEncoder;
|
|
15
15
|
private loadTiktokenModule;
|
|
16
|
-
private
|
|
17
|
-
private
|
|
16
|
+
private loadHuggingFaceTokenizer;
|
|
17
|
+
private loadHuggingFaceTokenizersModule;
|
|
18
|
+
private fetchHuggingFaceJson;
|
|
19
|
+
private getHuggingFaceHeaders;
|
|
20
|
+
private buildHuggingFaceFallbackWarning;
|
|
18
21
|
private importRuntimePackage;
|
|
19
22
|
}
|
|
20
23
|
//# sourceMappingURL=tokenizer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/tokenizer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACzC,OAAO,EAAE,gBAAgB,EAAsB,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/tokenizer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACzC,OAAO,EAAE,gBAAgB,EAAsB,MAAM,YAAY,CAAA;AAIjE,qBAAa,gBAAgB;IAMf,OAAO,CAAC,QAAQ,CAAC;IAL7B,OAAO,CAAC,aAAa,CAAyB;IAC9C,OAAO,CAAC,yBAAyB,CAAyB;IAC1D,OAAO,CAAC,cAAc,CAAC,CAAc;IACrC,OAAO,CAAC,2BAA2B,CAAC,CAAc;gBAE9B,QAAQ,CAAC,EAAE,gBAAgB,YAAA;IAEzC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBtE,OAAO,CAAC,qBAAqB;YAIf,iBAAiB;YASjB,6BAA6B;YAc7B,mBAAmB;YAwBnB,kBAAkB;YAOlB,wBAAwB;YA8BxB,+BAA+B;YAO/B,oBAAoB;IAgBlC,OAAO,CAAC,qBAAqB;IAK7B,OAAO,CAAC,+BAA+B;YASzB,oBAAoB;CAUnC"}
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
// TokenizerManager - handles token counting with multiple backends
|
|
2
2
|
import { formatErrorMessage } from "./warnings";
|
|
3
|
+
const APPROXIMATE_ONLY_HUGGINGFACE_HUBS = new Set(["google/gemma-2-9b-it"]);
|
|
3
4
|
export class TokenizerManager {
|
|
4
5
|
warnings;
|
|
5
6
|
tiktokenCache = new Map();
|
|
6
|
-
|
|
7
|
+
huggingFaceTokenizerCache = new Map();
|
|
7
8
|
tiktokenModule;
|
|
8
|
-
|
|
9
|
+
huggingFaceTokenizersModule;
|
|
9
10
|
constructor(warnings) {
|
|
10
11
|
this.warnings = warnings;
|
|
11
12
|
}
|
|
@@ -18,8 +19,8 @@ export class TokenizerManager {
|
|
|
18
19
|
return this.approximateTokenCount(content);
|
|
19
20
|
case "tiktoken":
|
|
20
21
|
return await this.countWithTiktoken(content, model.spec.model);
|
|
21
|
-
case "
|
|
22
|
-
return await this.
|
|
22
|
+
case "huggingface":
|
|
23
|
+
return await this.countWithHuggingFaceTokenizer(content, model.spec.hub);
|
|
23
24
|
}
|
|
24
25
|
}
|
|
25
26
|
catch (error) {
|
|
@@ -39,14 +40,14 @@ export class TokenizerManager {
|
|
|
39
40
|
return this.approximateTokenCount(content);
|
|
40
41
|
}
|
|
41
42
|
}
|
|
42
|
-
async
|
|
43
|
-
const tokenizer = await this.
|
|
43
|
+
async countWithHuggingFaceTokenizer(content, hub) {
|
|
44
|
+
const tokenizer = await this.loadHuggingFaceTokenizer(hub);
|
|
44
45
|
if (!tokenizer || typeof tokenizer.encode !== "function") {
|
|
45
46
|
return this.approximateTokenCount(content);
|
|
46
47
|
}
|
|
47
48
|
try {
|
|
48
49
|
const encoding = await tokenizer.encode(content);
|
|
49
|
-
return Array.isArray(encoding) ? encoding.length :
|
|
50
|
+
return Array.isArray(encoding?.ids) ? encoding.ids.length : this.approximateTokenCount(content);
|
|
50
51
|
}
|
|
51
52
|
catch {
|
|
52
53
|
return this.approximateTokenCount(content);
|
|
@@ -78,27 +79,59 @@ export class TokenizerManager {
|
|
|
78
79
|
}
|
|
79
80
|
return this.tiktokenModule;
|
|
80
81
|
}
|
|
81
|
-
async
|
|
82
|
-
if (this.
|
|
83
|
-
return this.
|
|
82
|
+
async loadHuggingFaceTokenizer(hub) {
|
|
83
|
+
if (this.huggingFaceTokenizerCache.has(hub)) {
|
|
84
|
+
return this.huggingFaceTokenizerCache.get(hub);
|
|
85
|
+
}
|
|
86
|
+
if (APPROXIMATE_ONLY_HUGGINGFACE_HUBS.has(hub)) {
|
|
87
|
+
this.warnings?.add(`TokenScope used approximate token counting for '${hub}' because it only loads public tokenizers directly in analysis mode.`, `huggingface-tokenizer-approx-only:${hub}`);
|
|
88
|
+
this.huggingFaceTokenizerCache.set(hub, null);
|
|
89
|
+
return null;
|
|
84
90
|
}
|
|
85
91
|
try {
|
|
86
|
-
const {
|
|
87
|
-
const
|
|
88
|
-
|
|
92
|
+
const { Tokenizer } = await this.loadHuggingFaceTokenizersModule();
|
|
93
|
+
const [tokenizerJson, tokenizerConfig] = await Promise.all([
|
|
94
|
+
this.fetchHuggingFaceJson(hub, "tokenizer.json"),
|
|
95
|
+
this.fetchHuggingFaceJson(hub, "tokenizer_config.json", true),
|
|
96
|
+
]);
|
|
97
|
+
const tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig ?? {});
|
|
98
|
+
this.huggingFaceTokenizerCache.set(hub, tokenizer);
|
|
89
99
|
return tokenizer;
|
|
90
100
|
}
|
|
91
101
|
catch (error) {
|
|
92
|
-
this.warnings?.add(
|
|
93
|
-
this.
|
|
102
|
+
this.warnings?.add(this.buildHuggingFaceFallbackWarning(hub, error), `huggingface-tokenizer-load:${hub}`);
|
|
103
|
+
this.huggingFaceTokenizerCache.set(hub, null);
|
|
94
104
|
return null;
|
|
95
105
|
}
|
|
96
106
|
}
|
|
97
|
-
async
|
|
98
|
-
if (!this.
|
|
99
|
-
this.
|
|
107
|
+
async loadHuggingFaceTokenizersModule() {
|
|
108
|
+
if (!this.huggingFaceTokenizersModule) {
|
|
109
|
+
this.huggingFaceTokenizersModule = this.importRuntimePackage("@huggingface/tokenizers");
|
|
110
|
+
}
|
|
111
|
+
return this.huggingFaceTokenizersModule;
|
|
112
|
+
}
|
|
113
|
+
async fetchHuggingFaceJson(hub, file, optional = false) {
|
|
114
|
+
const response = await fetch(`https://huggingface.co/${hub}/raw/main/${file}`, {
|
|
115
|
+
headers: this.getHuggingFaceHeaders(),
|
|
116
|
+
});
|
|
117
|
+
if (!response.ok) {
|
|
118
|
+
if (optional && response.status === 404) {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
throw new Error(`Failed to fetch ${file} for '${hub}' (HTTP ${response.status})`);
|
|
122
|
+
}
|
|
123
|
+
return response.json();
|
|
124
|
+
}
|
|
125
|
+
getHuggingFaceHeaders() {
|
|
126
|
+
const token = process.env.HF_TOKEN ?? process.env.HUGGING_FACE_HUB_TOKEN ?? process.env.HUGGINGFACE_TOKEN;
|
|
127
|
+
return token ? { Authorization: `Bearer ${token}` } : undefined;
|
|
128
|
+
}
|
|
129
|
+
buildHuggingFaceFallbackWarning(hub, error) {
|
|
130
|
+
const message = formatErrorMessage(error);
|
|
131
|
+
if (message.includes("Token analyzer dependency '@huggingface/tokenizers' could not be loaded")) {
|
|
132
|
+
return `TokenScope used approximate token counting for '${hub}' because the lightweight tokenizer runtime was unavailable.`;
|
|
100
133
|
}
|
|
101
|
-
return
|
|
134
|
+
return `TokenScope used approximate token counting for '${hub}' because an exact public tokenizer could not be loaded.`;
|
|
102
135
|
}
|
|
103
136
|
async importRuntimePackage(pkg) {
|
|
104
137
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../tokenscope-lib/tokenizer.ts"],"names":[],"mappings":"AAAA,mEAAmE;AAGnE,OAAO,EAAoB,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAEjE,MAAM,OAAO,gBAAgB;IAMP;IALZ,aAAa,GAAG,IAAI,GAAG,EAAe,CAAA;IACtC,
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../tokenscope-lib/tokenizer.ts"],"names":[],"mappings":"AAAA,mEAAmE;AAGnE,OAAO,EAAoB,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAEjE,MAAM,iCAAiC,GAAG,IAAI,GAAG,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAA;AAE3E,MAAM,OAAO,gBAAgB;IAMP;IALZ,aAAa,GAAG,IAAI,GAAG,EAAe,CAAA;IACtC,yBAAyB,GAAG,IAAI,GAAG,EAAe,CAAA;IAClD,cAAc,CAAe;IAC7B,2BAA2B,CAAe;IAElD,YAAoB,QAA2B;QAA3B,aAAQ,GAAR,QAAQ,CAAmB;IAAG,CAAC;IAEnD,KAAK,CAAC,WAAW,CAAC,OAAe,EAAE,KAAiB;QAClD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAA;QAE7B,IAAI,CAAC;YACH,QAAQ,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxB,KAAK,QAAQ;oBACX,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAA;gBAC5C,KAAK,UAAU;oBACb,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAChE,KAAK,aAAa;oBAChB,OAAO,MAAM,IAAI,CAAC,6BAA6B,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAC5E,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,EAAE,GAAG,CAChB,2DAA2D,KAAK,CAAC,IAAI,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,EACtG,eAAe,KAAK,CAAC,IAAI,EAAE,CAC5B,CAAA;YACD,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC;IAEO,qBAAqB,CAAC,OAAe;QAC3C,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACtC,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,OAAe,EAAE,KAAa;QAC5D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrD,IAAI,CAAC;YACH,OAAO,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAA;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,OAAe,EAAE,GAAW;QACtE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAA;QAC1D,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAA;QAC5C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YAChD,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAA;QACjG,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,KAAa;QAC7C,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtC,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAA;QAC9E,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,OAAO,EAAE,WAAW,CAAA;QAE/D,IAAI,OAAO,WAAW,KAAK,UAAU,EAAE,CAAC;YACtC,OAAO,EAAE,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAA;QAC/E,CAAC;QAED,IAAI,OAAO,CAAA;QACX,IAAI,CAAC;YACH,OAAO,GAAG,OAAO,gBAAgB,KAAK,UAAU,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QACjG,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,WAAW,CAAC,aAAa,CAAC,CAAA;QACtC,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QACtC,OAAO,OAAO,CAAA;IAChB,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAA;QAChE,CAAC;QACD,OAAO,IAAI,CAAC,cAAc,CAAA;IAC5B,CAAC;IAEO,KAAK,CAAC,wBAAwB,CAAC,GAAW;QAChD,IAAI,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAChD,CAAC;QAED,IAAI,iCAAiC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,QAAQ,EAAE,GAAG,CAChB,mDAAmD,GAAG,sEAAsE,EAC5H,qCAAqC,GAAG,EAAE,CAC3C,CAAA;YACD,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YAC7C,OAAO,IAAI,CAAA;QACb,CAAC;QAED,IAAI,CAAC;YACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAA;YAClE,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACzD,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,gBAAgB,CAAC;gBAChD,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,EAAE,IAAI,CAAC;aAC9D,CAAC,CAAA;YACF,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,aAAa,EAAE,eAAe,IAAI,EAAE,CAAC,CAAA;YACrE,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;YAClD,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,+BAA+B,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,8BAA8B,GAAG,EAAE,CAAC,CAAA;YACzG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YAC7C,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,+BAA+B;QAC3C,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACtC,IAAI,CAAC,2BAA2B,GAAG,IAAI,CAAC,oBAAoB,CAAC,yBAAyB,CAAC,CAAA;QACzF,CAAC;QACD,OAAO,IAAI,CAAC,2BAA2B,CAAA;IACzC,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,GAAW,EAAE,IAAY,EAAE,QAAQ,GAAG,KAAK;QAC5E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,0BAA0B,GAAG,aAAa,IAAI,EAAE,EAAE;YAC7E,OAAO,EAAE,IAAI,CAAC,qBAAqB,EAAE;SACtC,CAAC,CAAA;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACxC,OAAO,IAAI,CAAA;YACb,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,SAAS,GAAG,WAAW,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;QACnF,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAA;IACxB,CAAC;IAEO,qBAAqB;QAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAA;QACzG,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;IACjE,CAAC;IAEO,+BAA+B,CAAC,GAAW,EAAE,KAAc;QACjE,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAA;QACzC,IAAI,OAAO,CAAC,QAAQ,CAAC,yEAAyE,CAAC,EAAE,CAAC;YAChG,OAAO,mDAAmD,GAAG,8DAA8D,CAAA;QAC7H,CAAC;QAED,OAAO,mDAAmD,GAAG,0DAA0D,CAAA;IACzH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,GAAW;QAC5C,IAAI,CAAC;YACH,OAAO,MAAM,MAAM,CAAC,GAAG,CAAC,CAAA;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,8BAA8B,GAAG,yBAAyB;gBACxD,yDAAyD,kBAAkB,CAAC,KAAK,CAAC,EAAE,CACvF,CAAA;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,kBAAkB,CAAA;IACxB,KAAK,EAAE,kBAAkB,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE;QACN,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,CAAC,EAAE,MAAM,CAAA;KACjB,CAAA;IACD,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;IAC1B,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,CAAA;CACF;AAED,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GACnD;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,SAAS,CAAA;CAAE,GAChD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAA;AAE5C,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAAA;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE;QACL,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,GAAG,CAAC,EAAE,MAAM,CAAA;QACZ,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,aAAa,EAAE,CAAA;IACxB,UAAU,EAAE,aAAa,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,UAAU,CAAA;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,UAAU,EAAE;QACV,MAAM,EAAE,eAAe,CAAA;QACvB,IAAI,EAAE,eAAe,CAAA;QACrB,SAAS,EAAE,eAAe,CAAA;QAC1B,KAAK,EAAE,eAAe,CAAA;QACtB,SAAS,EAAE,eAAe,CAAA;KAC3B,CAAA;IACD,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,qBAAqB,EAAE,MAAM,CAAA;IAC7B,YAAY,EAAE,MAAM,CAAA;IACpB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,mBAAmB,EAAE,MAAM,CAAA;IAC3B,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,mBAAmB,EAAE,MAAM,CAAA;IAC3B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,6BAA6B,CAAC,EAAE,MAAM,CAAA;IACtC,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACnC,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IAEnC,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAA;IACpC,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,aAAa,CAAC,EAAE,aAAa,CAAA;CAC9B;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,aAAa,CAAA;CACpB;AAED,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,kBAAkB,CAAA;IACxB,KAAK,EAAE,kBAAkB,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE;QACN,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,CAAC,EAAE,MAAM,CAAA;KACjB,CAAA;IACD,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;IAC1B,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,CAAA;CACF;AAED,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GACnD;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,SAAS,CAAA;CAAE,GAChD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAA;AAE5C,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAAA;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE;QACL,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,GAAG,CAAC,EAAE,MAAM,CAAA;QACZ,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,aAAa,EAAE,CAAA;IACxB,UAAU,EAAE,aAAa,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,UAAU,CAAA;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,UAAU,EAAE;QACV,MAAM,EAAE,eAAe,CAAA;QACvB,IAAI,EAAE,eAAe,CAAA;QACrB,SAAS,EAAE,eAAe,CAAA;QAC1B,KAAK,EAAE,eAAe,CAAA;QACtB,SAAS,EAAE,eAAe,CAAA;KAC3B,CAAA;IACD,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,qBAAqB,EAAE,MAAM,CAAA;IAC7B,YAAY,EAAE,MAAM,CAAA;IACpB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,mBAAmB,EAAE,MAAM,CAAA;IAC3B,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,mBAAmB,EAAE,MAAM,CAAA;IAC3B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,6BAA6B,CAAC,EAAE,MAAM,CAAA;IACtC,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACnC,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IAEnC,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAA;IACpC,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,aAAa,CAAC,EAAE,aAAa,CAAA;CAC9B;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,aAAa,CAAA;CACpB;AAED,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CAAA;AAEtB,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,cAAc,EAAE,OAAO,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;IACtB,iBAAiB,EAAE,MAAM,CAAA;IACzB,oBAAoB,EAAE,MAAM,CAAA;IAC5B,kBAAkB,EAAE,MAAM,CAAA;IAC1B,mBAAmB,EAAE,MAAM,CAAA;IAC3B,sBAAsB,EAAE,MAAM,CAAA;IAC9B,uBAAuB,EAAE,MAAM,CAAA;IAC/B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,qBAAqB,EAAE,MAAM,CAAA;IAC7B,wBAAwB,EAAE,MAAM,CAAA;IAChC,yBAAyB,EAAE,MAAM,CAAA;IACjC,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,EAAE,MAAM,CAAA;IACrB,qBAAqB,EAAE,MAAM,CAAA;IAC7B,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,eAAe,EAAE,CAAA;IAC5B,gBAAgB,EAAE,MAAM,CAAA;IACxB,iBAAiB,EAAE,MAAM,CAAA;IACzB,oBAAoB,EAAE,MAAM,CAAA;IAC5B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,qBAAqB,EAAE,MAAM,CAAA;IAC7B,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,aAAa,EAAE,MAAM,CAAA;CACtB;AAID,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,OAAO,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB,EAAE,gBAAgB,CAAA;IAClC,eAAe,EAAE,gBAAgB,GAAG;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAA;IACzD,kBAAkB,EAAE,gBAAgB,GAAG;QAAE,UAAU,EAAE,MAAM,EAAE,CAAA;KAAE,CAAA;IAC/D,WAAW,EAAE,gBAAgB,GAAG;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAA;IACrD,kBAAkB,EAAE,gBAAgB,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAA;IAC5D,kBAAkB,EAAE,MAAM,CAAA;CAC3B;AAID,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,eAAe,EAAE,MAAM,CAAA;IACvB,aAAa,EAAE,MAAM,CAAA;IACrB,cAAc,EAAE,OAAO,CAAA;CACxB;AAID,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB,EAAE,MAAM,CAAA;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,eAAe,EAAE,cAAc,EAAE,CAAA;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAA;IACvC,YAAY,EAAE,WAAW,EAAE,CAAA;IAC3B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,4BAA4B,EAAE,MAAM,CAAA;IACpC,iBAAiB,EAAE,MAAM,CAAA;IACzB,4BAA4B,EAAE,MAAM,CAAA;IACpC,0BAA0B,EAAE,MAAM,CAAA;IAClC,yBAAyB,EAAE,MAAM,CAAA;CAClC;AAID,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,gBAAgB,EAAE,MAAM,CAAA;IACxB,gBAAgB,EAAE,MAAM,CAAA;IACxB,YAAY,EAAE,MAAM,CAAA;IACpB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,CAAA;CACrB;AAID,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,mBAAmB,CAAA;IACzB,QAAQ,EAAE,eAAe,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,mBAAmB,CAAA;IACzB,KAAK,EAAE,YAAY,EAAE,CAAA;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,GAAG,WAAW,CAAA;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE;QACN,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,CAAC,EAAE,MAAM,CAAA;KACjB,CAAA;IACD,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC/B,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE;QACN,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAChC,CAAA;CACF;AAID,MAAM,WAAW,gBAAgB;IAC/B,sBAAsB,EAAE,OAAO,CAAA;IAC/B,0BAA0B,EAAE,OAAO,CAAA;IACnC,qBAAqB,EAAE,OAAO,CAAA;IAC9B,sBAAsB,EAAE,OAAO,CAAA;IAC/B,mBAAmB,EAAE,OAAO,CAAA;CAC7B;AAID,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAA;IACpC,eAAe,CAAC,EAAE,eAAe,CAAA;CAClC;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAID,wBAAgB,UAAU,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,SAAS,CAAA;CAAE,CAE7G;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,IAAI;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAErG;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,CAEhH"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package.json",
|
|
3
3
|
"name": "@ramtinj95/opencode-tokenscope",
|
|
4
|
-
"version": "1.6.
|
|
4
|
+
"version": "1.6.2",
|
|
5
5
|
"description": "OpenCode plugin for detailed token usage analysis with breakdowns by category, visual charts, and subagent cost tracking",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "./dist/tokenscope.js",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"@opencode-ai/plugin": ">=0.13.7"
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@huggingface/
|
|
38
|
+
"@huggingface/tokenizers": "^0.1.3",
|
|
39
39
|
"js-tiktoken": "^1.0.15"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|