@trazum/core 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -8
- package/dist/advisories.d.ts.map +1 -1
- package/dist/advisories.js +11 -1
- package/dist/advisories.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +4 -1
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +4 -1
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +10 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/language.d.ts +38 -0
- package/dist/language.d.ts.map +1 -0
- package/dist/language.js +109 -0
- package/dist/language.js.map +1 -0
- package/dist/tokenizer.d.ts +33 -0
- package/dist/tokenizer.d.ts.map +1 -1
- package/dist/tokenizer.js +94 -2
- package/dist/tokenizer.js.map +1 -1
- package/package.json +1 -1
- package/src/advisories.ts +12 -1
- package/src/i18n/en.ts +4 -0
- package/src/i18n/es.ts +4 -0
- package/src/i18n/types.ts +10 -0
- package/src/index.ts +2 -1
- package/src/language.ts +112 -0
- package/src/tokenizer.ts +98 -2
package/README.md
CHANGED
|
@@ -10,11 +10,6 @@ reading them too.
|
|
|
10
10
|
Part of [Trazum](https://github.com/Davmunrey/Trazum). For the terminal, see
|
|
11
11
|
[`@trazum/cli`](https://www.npmjs.com/package/@trazum/cli).
|
|
12
12
|
|
|
13
|
-
> [!NOTE]
|
|
14
|
-
> **Not published yet.** The command below will not work today — there is no tag
|
|
15
|
-
> and `npm view` returns 404. Run it from source meanwhile:
|
|
16
|
-
> [github.com/Davmunrey/Trazum](https://github.com/Davmunrey/Trazum#getting-started).
|
|
17
|
-
|
|
18
13
|
```bash
|
|
19
14
|
npm install @trazum/core
|
|
20
15
|
```
|
|
@@ -138,9 +133,27 @@ const refined = await refineWithLlm(result, openAiCompatible({
|
|
|
138
133
|
|
|
139
134
|
## Token counts
|
|
140
135
|
|
|
141
|
-
The bundled estimator is a dependency-free heuristic,
|
|
142
|
-
|
|
143
|
-
|
|
136
|
+
The bundled estimator is a dependency-free heuristic, and it compares two versions
|
|
137
|
+
of the same prompt well — which is what it is for. Its error band is **±15%**,
|
|
138
|
+
exported as `ESTIMATE_ERROR_BAND_PCT` so you can print the same number the report
|
|
139
|
+
prints rather than hard-coding it.
|
|
140
|
+
|
|
141
|
+
That band is measured, against the official counting endpoint, over 21 samples in
|
|
142
|
+
seven languages and six text types — worst case 11.2%. It was an unchecked design
|
|
143
|
+
target for eight releases and it was false; the 1.9.0 entry in the changelog says
|
|
144
|
+
what was wrong.
|
|
145
|
+
|
|
146
|
+
Two things follow from how it was measured, and both matter if you depend on the
|
|
147
|
+
number:
|
|
148
|
+
|
|
149
|
+
- **The band is a Claude number.** The estimator is tuned against Claude's
|
|
150
|
+
tokenizer. Other families tokenize differently and nothing here bounds the error
|
|
151
|
+
for them.
|
|
152
|
+
- **It detects the language and divides accordingly**, because one divisor
|
|
153
|
+
calibrated on English measured −37.3% on German. `detectTextLanguage` answers
|
|
154
|
+
`null` when it cannot tell, which falls back to the English behaviour.
|
|
155
|
+
|
|
156
|
+
For exact figures, pass a real counter — the counting endpoint is free:
|
|
144
157
|
|
|
145
158
|
```ts
|
|
146
159
|
import { countTokensAnthropic, withExactTokenCounts } from '@trazum/core';
|
package/dist/advisories.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"advisories.d.ts","sourceRoot":"","sources":["../src/advisories.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAG9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAKrD,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAQrF;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAclF;AAsDD,MAAM,WAAW,eAAe;IAC9B,0EAA0E;IAC1E,EAAE,CAAC,EAAE,IAAI,CAAC;IACV,wEAAwE;IACxE,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAC7B,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,YAAY,EACnB,OAAO,GAAE,eAAoB,GAC5B,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"advisories.d.ts","sourceRoot":"","sources":["../src/advisories.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAG9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAKrD,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAQrF;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAclF;AAsDD,MAAM,WAAW,eAAe;IAC9B,0EAA0E;IAC1E,EAAE,CAAC,EAAE,IAAI,CAAC;IACV,wEAAwE;IACxE,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAC7B,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,YAAY,EACnB,OAAO,GAAE,eAAoB,GAC5B,QAAQ,EAAE,CA4UZ"}
|
package/dist/advisories.js
CHANGED
|
@@ -4,7 +4,7 @@ import { COMPLEX_SIGNALS, SIMPLE_SIGNALS } from './phrases.js';
|
|
|
4
4
|
import { BUNDLED_CATALOGUE, effectivePricing, modelFrom, multipliersFor } from './pricing.js';
|
|
5
5
|
import { formatUsd } from './savings.js';
|
|
6
6
|
import { analyzeExamples, findContradictions, findMovableSchema, findRestatedFormat } from './structure.js';
|
|
7
|
-
import { estimateTokens } from './tokenizer.js';
|
|
7
|
+
import { ESTIMATE_ERROR_BAND_PCT, estimateTokens } from './tokenizer.js';
|
|
8
8
|
function countSignals(haystack, signals) {
|
|
9
9
|
let count = 0;
|
|
10
10
|
for (const signal of signals)
|
|
@@ -171,6 +171,16 @@ export function buildAdvisories(optimizedPrompt, tokensAfter, usage, options = {
|
|
|
171
171
|
prefixTokens: cache.stablePrefixTokens,
|
|
172
172
|
totalTokens: tokensAfter,
|
|
173
173
|
mentionLowerMinimum: minTokens > 512,
|
|
174
|
+
/**
|
|
175
|
+
* Only when the number is an estimate and near the line.
|
|
176
|
+
*
|
|
177
|
+
* `count` defaults to `estimateTokens`; a caller who supplied their own
|
|
178
|
+
* counter — `--exact-tokens`, or the official endpoint — gets an
|
|
179
|
+
* authoritative number and no hedge, because hedging a measured figure
|
|
180
|
+
* is its own kind of dishonesty.
|
|
181
|
+
*/
|
|
182
|
+
couldReachMinimum: count === estimateTokens &&
|
|
183
|
+
cache.stablePrefixTokens * (1 + ESTIMATE_ERROR_BAND_PCT / 100) >= minTokens,
|
|
174
184
|
}),
|
|
175
185
|
estimatedMonthlyUsd: null,
|
|
176
186
|
});
|
package/dist/advisories.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"advisories.js","sourceRoot":"","sources":["../src/advisories.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,EAC7D,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,SAAS,YAAY,CAAC,QAAgB,EAAE,OAA0B;IAChE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,MAAM,IAAI,OAAO;QAAE,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,KAAK,EAAE,CAAC;IACrE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,MAAc;IAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,KAAK,GACP,YAAY,CAAC,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAE3F,IAAI,MAAM,GAAG,IAAI;QAAE,KAAK,IAAI,CAAC,CAAC;SACzB,IAAI,MAAM,GAAG,IAAI;QAAE,KAAK,IAAI,CAAC,CAAC;SAC9B,IAAI,MAAM,GAAG,GAAG;QAAE,KAAK,IAAI,CAAC,CAAC;IAElC,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IAEvC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAChC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,GAAyC;IACvD;;;;;;;;;;OAUG;IACH,OAAO,EAAE,MAAM,CAAC,iBAAiB;IACjC,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,IAAI,EAAE,CAAC;IACP,QAAQ,EAAE,CAAC;CACZ,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,SAAS,cAAc,CACrB,IAA0B,EAC1B,EAAQ,EACR,OAAyB,EACzB,QAA4B;IAE5B,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,IAAI;QACf,CAAC,CAAC,QAAQ,KAAK,QAAQ;QACvB,0EAA0E;QAC1E,mCAAmC;QACnC,CAAC,CAAC,aAAa,KAAK,KAAK,CAC5B,CAAC;IACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACnC,gBAAgB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC1F,CAAC;AACJ,CAAC;AAYD,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAC7B,eAAuB,EACvB,WAAmB,EACnB,KAAmB,EACnB,OAAO,GAAoB,EAAE;IAE7B,MAAM,EAAE,EAAE,GAAG,IAAI,IAAI,EAAE,EAAE,KAAK,GAAG,cAAc,EAAE,MAAM,EAAE,OAAO,GAAG,iBAAiB,EAAE,GAAG,OAAO,CAAC;IACjG,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAe,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAClF,8EAA8E;IAC9E,4EAA4E;IAC5E,sEAAsE;IACtE,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjE,MAAM,eAAe,GACnB,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,YAAY,GAAG,KAAK,CAAC,aAAa,GAAG,WAAW,CAAC;IAC/E,MAAM,gBAAgB,GACpB,CAAC,KAAK,CAAC,eAAe,GAAG,SAAS,CAAC,GAAG,aAAa,GAAG,KAAK,CAAC,aAAa,GAAG,WAAW,CAAC;IAE1F,yBAAyB;IACzB,IAAI,WAAW,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC;QACtC,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,kBAAkB;YACtB,QAAQ,EAAE,SAAS;YACnB,GAAG,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC;gBAC9B,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,KAAK,CAAC,WAAW;gBAC5B,aAAa,EAAE,KAAK,CAAC,aAAa;aACnC,CAAC;YACF,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,yBAAyB;IACzB,wEAAwE;IACxE,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,KAAK,GAAG,kBAAkB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACzD,6EAA6E;IAC7E,4EAA4E;IAC5E,+EAA+E;IAC/E,4DAA4D;IAC5D;;;;;;;;;;OAUG;IACH,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,KAAK,MAAM,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,IAAI,CAAC;IAE9G,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,IAAI,YAAY,EAAE,CAAC;QAC5C,MAAM,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QACjF,MAAM,gBAAgB,GAAG,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,YAAY,GAAG,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC;QAE9E,MAAM,SAAS,GAAG,KAAK,CAAC,cAAc,IAAI,CAAC,CAAC;QAE5C,IAAI,KAAK,CAAC,kBAAkB,IAAI,SAAS,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,gBAAgB,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,UAAU,CAAC,IAAI,CAAC;oBACd,EAAE,EAAE,gBAAgB;oBACpB,QAAQ,EAAE,aAAa;oBACvB,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC;wBAC5B,WAAW,EAAE,KAAK,CAAC,gBAAgB;wBACnC,YAAY,EAAE,KAAK,CAAC,kBAAkB;wBACtC,WAAW,EAAE,WAAW;wBACxB,SAAS;wBACT,SAAS,EAAE,KAAK,CAAC,WAAW;wBAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC;wBACrC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC;wBAC1C,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;wBAC9C,QAAQ,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,UAAU,CAAC,KAAK,UAAU;qBACvD,CAAC;oBACF,mBAAmB,EAAE,MAAM;iBAC5B,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC;oBACd,EAAE,EAAE,6BAA6B;oBACjC,QAAQ,EAAE,MAAM;oBAChB,GAAG,CAAC,CAAC,UAAU,CAAC,uBAAuB,EAAE;oBACzC,mBAAmB,EAAE,IAAI;iBAC1B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC;gBACd,EAAE,EAAE,qBAAqB;gBACzB,QAAQ,EAAE,MAAM;gBAChB,GAAG,CAAC,CAAC,UAAU,CAAC,iBAAiB,CAAC;oBAChC,SAAS,EAAE,KAAK,CAAC,WAAW;oBAC5B,SAAS;oBACT,WAAW,EAAE,KAAK,CAAC,gBAAgB;oBACnC,YAAY,EAAE,KAAK,CAAC,kBAAkB;oBACtC,WAAW,EAAE,WAAW;oBACxB,mBAAmB,EAAE,SAAS,GAAG,GAAG;iBACrC,CAAC;gBACF,mBAAmB,EAAE,IAAI;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,yEAAyE;QACzE,kDAAkD;QAClD,IACE,KAAK,CAAC,gBAAgB;YACtB,KAAK,CAAC,iBAAiB,IAAI,GAAG;YAC9B,KAAK,CAAC,iBAAiB,IAAI,WAAW,GAAG,GAAG,EAC5C,CAAC;YACD,MAAM,YAAY,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YACjF,MAAM,MAAM,GAAG,eAAe,GAAG,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC;YACxE,UAAU,CAAC,IAAI,CAAC;gBACd,EAAE,EAAE,sBAAsB;gBAC1B,QAAQ,EAAE,aAAa;gBACvB,GAAG,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC;oBACjC,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;oBAC1C,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;oBACxC,WAAW,EAAE,KAAK,CAAC,gBAAgB;iBACpC,CAAC;gBACF,mBAAmB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;aAChD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,8EAA8E;IAC9E,4EAA4E;IAC5E,+BAA+B;IAC/B,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QAC5E,MAAM,MAAM,GAAG,CAAC,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QACxE,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,WAAW;YACf,QAAQ,EAAE,aAAa;YACvB,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC1B,mBAAmB,EAAE,MAAM;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IAC5B,MAAM,aAAa,GAAG,aAAa,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAClE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,UAAU,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnF,MAAM,SAAS,GAAG,cAAc,CAAC,aAAa,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7E,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACzD,MAAM,gBAAgB,GACpB,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,gBAAgB,CAAC,YAAY;gBACxD,CAAC,KAAK,CAAC,eAAe,GAAG,SAAS,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC;gBACvE,KAAK,CAAC,aAAa;gBACnB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,MAAM,GAAG,eAAe,GAAG,gBAAgB,GAAG,gBAAgB,CAAC;YACrE,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,UAAU,CAAC,IAAI,CAAC;oBACd,EAAE,EAAE,iBAAiB;oBACrB,QAAQ,EAAE,aAAa;oBACvB,GAAG,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC;wBAC7B,SAAS,EAAE,KAAK,CAAC,WAAW;wBAC5B,IAAI,EAAE,aAAa;wBACnB,aAAa,EAAE,SAAS,CAAC,WAAW;wBACpC,UAAU,EAAE,SAAS,CAAC,eAAe,GAAG,gBAAgB,CAAC;wBACzD,YAAY,EAAE,SAAS,CAAC,gBAAgB,CAAC;qBAC1C,CAAC;oBACF,mBAAmB,EAAE,MAAM;iBAC5B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,IAAI,gBAAgB,GAAG,eAAe,GAAG,CAAC,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QACxE,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,kBAAkB;YACtB,QAAQ,EAAE,MAAM;YAChB,GAAG,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC;gBAC9B,SAAS,EAAE,SAAS,CAAC,gBAAgB,CAAC;gBACtC,QAAQ,EAAE,SAAS,CAAC,eAAe,CAAC;aACrC,CAAC;YACF,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,6CAA6C;IAC7C,yEAAyE;IACzE,6EAA6E;IAC7E,MAAM,cAAc,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,KAAK,EAAE,CAAC;QACV,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,4BAA4B;YAChC,QAAQ,EAAE,SAAS;YACnB,GAAG,CAAC,CAAC,UAAU,CAAC,yBAAyB,CAAC;gBACxC,IAAI,EAAE,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC;gBACrC,UAAU,EAAE,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBAChD,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO;gBAC7B,WAAW,EAAE,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACjD,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO;gBAC9B,UAAU,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC;aACtC,CAAC;YACF,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,mDAAmD;IACnD,MAAM,QAAQ,GAAG,eAAe,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACzD,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QAClE,MAAM,MAAM,GACV,WAAW,GAAG,CAAC;YACb,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,GAAG,WAAW,CAAC;YACvE,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC/E,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,oBAAoB;YACxB,QAAQ,EAAE,aAAa;YACvB,GAAG,CAAC,CAAC,UAAU,CAAC,iBAAiB,CAAC;gBAChC,cAAc,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM;gBACzC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM;gBACpC,eAAe,EAAE,QAAQ,CAAC,eAAe;gBACzC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;aAClD,CAAC;YACF,mBAAmB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;SAChD,CAAC,CAAC;IACL,CAAC;IAED,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,iBAAiB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAC1D,IAAI,OAAO,EAAE,CAAC;QACZ;;;;;;;;;;WAUG;QACH,MAAM,MAAM,GACV,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpF,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,uBAAuB;YAC3B,QAAQ,EAAE,aAAa;YACvB,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBACvD,GAAG,EAAE,OAAO,CAAC,GAAG;aACjB,CAAC;YACF,mBAAmB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;SAChD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GACV,WAAW,GAAG,CAAC;YACb,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,cAAc,GAAG,WAAW,CAAC;YACtE,CAAC,CAAC,CAAC,CAAC;QACR,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,wBAAwB;YAC5B,QAAQ,EAAE,aAAa;YACvB,GAAG,CAAC,CAAC,UAAU,CAAC,oBAAoB,CAAC;gBACnC,aAAa,EAAE,QAAQ,CAAC,YAAY,CAAC,MAAM;gBAC3C,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;gBAChC,cAAc,EAAE,QAAQ,CAAC,cAAc;gBACvC,OAAO,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;aACjE,CAAC;YACF,mBAAmB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;SAChD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,YAAY,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChC,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,SAAS;YACnB,GAAG,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC;gBAC3B,SAAS,EAAE,KAAK,CAAC,WAAW;gBAC5B,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,YAAY;gBACpC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,aAAa;gBACtC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK;gBACxB,SAAS,EAAE,KAAK,CAAC,YAAY;gBAC7B,UAAU,EAAE,KAAK,CAAC,aAAa;aAChC,CAAC;YACF,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,6EAA6E;IAC7E,oCAAoC;IACpC,MAAM,cAAc,GAAyC;QAC3D,OAAO,EAAE,CAAC;QACV,WAAW,EAAE,CAAC;QACd,IAAI,EAAE,CAAC;KACR,CAAC;IAEF,OAAO,UAAU,CAAC,IAAI,CACpB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC;QACvD,CAAC,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,CAC9D,CAAC;AACJ,CAAC"}
|
|
1
|
+
{"version":3,"file":"advisories.js","sourceRoot":"","sources":["../src/advisories.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,EAC7D,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGzE,SAAS,YAAY,CAAC,QAAgB,EAAE,OAA0B;IAChE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,MAAM,IAAI,OAAO;QAAE,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,KAAK,EAAE,CAAC;IACrE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,MAAc;IAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,KAAK,GACP,YAAY,CAAC,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAE3F,IAAI,MAAM,GAAG,IAAI;QAAE,KAAK,IAAI,CAAC,CAAC;SACzB,IAAI,MAAM,GAAG,IAAI;QAAE,KAAK,IAAI,CAAC,CAAC;SAC9B,IAAI,MAAM,GAAG,GAAG;QAAE,KAAK,IAAI,CAAC,CAAC;IAElC,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IAEvC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAChC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,GAAyC;IACvD;;;;;;;;;;OAUG;IACH,OAAO,EAAE,MAAM,CAAC,iBAAiB;IACjC,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,IAAI,EAAE,CAAC;IACP,QAAQ,EAAE,CAAC;CACZ,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,SAAS,cAAc,CACrB,IAA0B,EAC1B,EAAQ,EACR,OAAyB,EACzB,QAA4B;IAE5B,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,IAAI;QACf,CAAC,CAAC,QAAQ,KAAK,QAAQ;QACvB,0EAA0E;QAC1E,mCAAmC;QACnC,CAAC,CAAC,aAAa,KAAK,KAAK,CAC5B,CAAC;IACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACnC,gBAAgB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC1F,CAAC;AACJ,CAAC;AAYD,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAC7B,eAAuB,EACvB,WAAmB,EACnB,KAAmB,EACnB,OAAO,GAAoB,EAAE;IAE7B,MAAM,EAAE,EAAE,GAAG,IAAI,IAAI,EAAE,EAAE,KAAK,GAAG,cAAc,EAAE,MAAM,EAAE,OAAO,GAAG,iBAAiB,EAAE,GAAG,OAAO,CAAC;IACjG,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAe,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAClF,8EAA8E;IAC9E,4EAA4E;IAC5E,sEAAsE;IACtE,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjE,MAAM,eAAe,GACnB,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,YAAY,GAAG,KAAK,CAAC,aAAa,GAAG,WAAW,CAAC;IAC/E,MAAM,gBAAgB,GACpB,CAAC,KAAK,CAAC,eAAe,GAAG,SAAS,CAAC,GAAG,aAAa,GAAG,KAAK,CAAC,aAAa,GAAG,WAAW,CAAC;IAE1F,yBAAyB;IACzB,IAAI,WAAW,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC;QACtC,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,kBAAkB;YACtB,QAAQ,EAAE,SAAS;YACnB,GAAG,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC;gBAC9B,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,KAAK,CAAC,WAAW;gBAC5B,aAAa,EAAE,KAAK,CAAC,aAAa;aACnC,CAAC;YACF,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,yBAAyB;IACzB,wEAAwE;IACxE,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,KAAK,GAAG,kBAAkB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACzD,6EAA6E;IAC7E,4EAA4E;IAC5E,+EAA+E;IAC/E,4DAA4D;IAC5D;;;;;;;;;;OAUG;IACH,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,KAAK,MAAM,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,IAAI,CAAC;IAE9G,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,IAAI,YAAY,EAAE,CAAC;QAC5C,MAAM,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QACjF,MAAM,gBAAgB,GAAG,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,YAAY,GAAG,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC;QAE9E,MAAM,SAAS,GAAG,KAAK,CAAC,cAAc,IAAI,CAAC,CAAC;QAE5C,IAAI,KAAK,CAAC,kBAAkB,IAAI,SAAS,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,gBAAgB,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,UAAU,CAAC,IAAI,CAAC;oBACd,EAAE,EAAE,gBAAgB;oBACpB,QAAQ,EAAE,aAAa;oBACvB,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC;wBAC5B,WAAW,EAAE,KAAK,CAAC,gBAAgB;wBACnC,YAAY,EAAE,KAAK,CAAC,kBAAkB;wBACtC,WAAW,EAAE,WAAW;wBACxB,SAAS;wBACT,SAAS,EAAE,KAAK,CAAC,WAAW;wBAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC;wBACrC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC;wBAC1C,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;wBAC9C,QAAQ,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,UAAU,CAAC,KAAK,UAAU;qBACvD,CAAC;oBACF,mBAAmB,EAAE,MAAM;iBAC5B,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC;oBACd,EAAE,EAAE,6BAA6B;oBACjC,QAAQ,EAAE,MAAM;oBAChB,GAAG,CAAC,CAAC,UAAU,CAAC,uBAAuB,EAAE;oBACzC,mBAAmB,EAAE,IAAI;iBAC1B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC;gBACd,EAAE,EAAE,qBAAqB;gBACzB,QAAQ,EAAE,MAAM;gBAChB,GAAG,CAAC,CAAC,UAAU,CAAC,iBAAiB,CAAC;oBAChC,SAAS,EAAE,KAAK,CAAC,WAAW;oBAC5B,SAAS;oBACT,WAAW,EAAE,KAAK,CAAC,gBAAgB;oBACnC,YAAY,EAAE,KAAK,CAAC,kBAAkB;oBACtC,WAAW,EAAE,WAAW;oBACxB,mBAAmB,EAAE,SAAS,GAAG,GAAG;oBACpC;;;;;;;uBAOG;oBACH,iBAAiB,EACf,KAAK,KAAK,cAAc;wBACxB,KAAK,CAAC,kBAAkB,GAAG,CAAC,CAAC,GAAG,uBAAuB,GAAG,GAAG,CAAC,IAAI,SAAS;iBAC9E,CAAC;gBACF,mBAAmB,EAAE,IAAI;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,yEAAyE;QACzE,kDAAkD;QAClD,IACE,KAAK,CAAC,gBAAgB;YACtB,KAAK,CAAC,iBAAiB,IAAI,GAAG;YAC9B,KAAK,CAAC,iBAAiB,IAAI,WAAW,GAAG,GAAG,EAC5C,CAAC;YACD,MAAM,YAAY,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YACjF,MAAM,MAAM,GAAG,eAAe,GAAG,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC;YACxE,UAAU,CAAC,IAAI,CAAC;gBACd,EAAE,EAAE,sBAAsB;gBAC1B,QAAQ,EAAE,aAAa;gBACvB,GAAG,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC;oBACjC,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;oBAC1C,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;oBACxC,WAAW,EAAE,KAAK,CAAC,gBAAgB;iBACpC,CAAC;gBACF,mBAAmB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;aAChD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,8EAA8E;IAC9E,4EAA4E;IAC5E,+BAA+B;IAC/B,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QAC5E,MAAM,MAAM,GAAG,CAAC,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QACxE,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,WAAW;YACf,QAAQ,EAAE,aAAa;YACvB,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC1B,mBAAmB,EAAE,MAAM;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IAC5B,MAAM,aAAa,GAAG,aAAa,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAClE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,UAAU,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnF,MAAM,SAAS,GAAG,cAAc,CAAC,aAAa,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7E,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACzD,MAAM,gBAAgB,GACpB,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,gBAAgB,CAAC,YAAY;gBACxD,CAAC,KAAK,CAAC,eAAe,GAAG,SAAS,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC;gBACvE,KAAK,CAAC,aAAa;gBACnB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,MAAM,GAAG,eAAe,GAAG,gBAAgB,GAAG,gBAAgB,CAAC;YACrE,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,UAAU,CAAC,IAAI,CAAC;oBACd,EAAE,EAAE,iBAAiB;oBACrB,QAAQ,EAAE,aAAa;oBACvB,GAAG,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC;wBAC7B,SAAS,EAAE,KAAK,CAAC,WAAW;wBAC5B,IAAI,EAAE,aAAa;wBACnB,aAAa,EAAE,SAAS,CAAC,WAAW;wBACpC,UAAU,EAAE,SAAS,CAAC,eAAe,GAAG,gBAAgB,CAAC;wBACzD,YAAY,EAAE,SAAS,CAAC,gBAAgB,CAAC;qBAC1C,CAAC;oBACF,mBAAmB,EAAE,MAAM;iBAC5B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,IAAI,gBAAgB,GAAG,eAAe,GAAG,CAAC,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QACxE,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,kBAAkB;YACtB,QAAQ,EAAE,MAAM;YAChB,GAAG,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC;gBAC9B,SAAS,EAAE,SAAS,CAAC,gBAAgB,CAAC;gBACtC,QAAQ,EAAE,SAAS,CAAC,eAAe,CAAC;aACrC,CAAC;YACF,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,6CAA6C;IAC7C,yEAAyE;IACzE,6EAA6E;IAC7E,MAAM,cAAc,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,KAAK,EAAE,CAAC;QACV,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,4BAA4B;YAChC,QAAQ,EAAE,SAAS;YACnB,GAAG,CAAC,CAAC,UAAU,CAAC,yBAAyB,CAAC;gBACxC,IAAI,EAAE,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC;gBACrC,UAAU,EAAE,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBAChD,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO;gBAC7B,WAAW,EAAE,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACjD,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO;gBAC9B,UAAU,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC;aACtC,CAAC;YACF,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,mDAAmD;IACnD,MAAM,QAAQ,GAAG,eAAe,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACzD,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QAClE,MAAM,MAAM,GACV,WAAW,GAAG,CAAC;YACb,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,GAAG,WAAW,CAAC;YACvE,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC/E,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,oBAAoB;YACxB,QAAQ,EAAE,aAAa;YACvB,GAAG,CAAC,CAAC,UAAU,CAAC,iBAAiB,CAAC;gBAChC,cAAc,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM;gBACzC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM;gBACpC,eAAe,EAAE,QAAQ,CAAC,eAAe;gBACzC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;aAClD,CAAC;YACF,mBAAmB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;SAChD,CAAC,CAAC;IACL,CAAC;IAED,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,iBAAiB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAC1D,IAAI,OAAO,EAAE,CAAC;QACZ;;;;;;;;;;WAUG;QACH,MAAM,MAAM,GACV,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpF,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,uBAAuB;YAC3B,QAAQ,EAAE,aAAa;YACvB,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBACvD,GAAG,EAAE,OAAO,CAAC,GAAG;aACjB,CAAC;YACF,mBAAmB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;SAChD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GACV,WAAW,GAAG,CAAC;YACb,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,cAAc,GAAG,WAAW,CAAC;YACtE,CAAC,CAAC,CAAC,CAAC;QACR,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,wBAAwB;YAC5B,QAAQ,EAAE,aAAa;YACvB,GAAG,CAAC,CAAC,UAAU,CAAC,oBAAoB,CAAC;gBACnC,aAAa,EAAE,QAAQ,CAAC,YAAY,CAAC,MAAM;gBAC3C,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;gBAChC,cAAc,EAAE,QAAQ,CAAC,cAAc;gBACvC,OAAO,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;aACjE,CAAC;YACF,mBAAmB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;SAChD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,YAAY,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChC,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,SAAS;YACnB,GAAG,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC;gBAC3B,SAAS,EAAE,KAAK,CAAC,WAAW;gBAC5B,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,YAAY;gBACpC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,aAAa;gBACtC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK;gBACxB,SAAS,EAAE,KAAK,CAAC,YAAY;gBAC7B,UAAU,EAAE,KAAK,CAAC,aAAa;aAChC,CAAC;YACF,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,6EAA6E;IAC7E,oCAAoC;IACpC,MAAM,cAAc,GAAyC;QAC3D,OAAO,EAAE,CAAC;QACV,WAAW,EAAE,CAAC;QACd,IAAI,EAAE,CAAC;KACR,CAAC;IAEF,OAAO,UAAU,CAAC,IAAI,CACpB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC;QACvD,CAAC,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,CAC9D,CAAC;AACJ,CAAC"}
|
package/dist/i18n/en.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"en.d.ts","sourceRoot":"","sources":["../../src/i18n/en.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/C,iFAAiF;AACjF,eAAO,MAAM,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"en.d.ts","sourceRoot":"","sources":["../../src/i18n/en.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/C,iFAAiF;AACjF,eAAO,MAAM,EAAE,EAAE,YA0OhB,CAAC"}
|
package/dist/i18n/en.js
CHANGED
|
@@ -87,7 +87,7 @@ export const en = {
|
|
|
87
87
|
title: 'At that hit rate, caching does not pay off',
|
|
88
88
|
detail: 'A cache write costs 125% of the input price and a read costs 10%. Below roughly a 28% hit rate you pay more than you save. Either raise prefix reuse or leave caching off.',
|
|
89
89
|
}),
|
|
90
|
-
belowCacheMinimum: ({ modelName, minTokens, placeholder, prefixTokens, totalTokens, mentionLowerMinimum, }) => {
|
|
90
|
+
belowCacheMinimum: ({ modelName, minTokens, placeholder, prefixTokens, totalTokens, mentionLowerMinimum, couldReachMinimum, }) => {
|
|
91
91
|
const reason = placeholder
|
|
92
92
|
? `here the first variable placeholder (${placeholder}) shows up at ~${n(prefixTokens)} tokens and only what precedes it can be cached`
|
|
93
93
|
: `this prompt has ~${n(totalTokens)}`;
|
|
@@ -96,6 +96,9 @@ export const en = {
|
|
|
96
96
|
detail: `${modelName} needs at least ${n(minTokens)} prefix tokens to cache; ${reason}. Setting cache_control will not error, it simply will not cache.` +
|
|
97
97
|
(mentionLowerMinimum
|
|
98
98
|
? ' Claude Opus 5 lowers that minimum to 512 tokens, so short prompts that miss here would cache there.'
|
|
99
|
+
: '') +
|
|
100
|
+
(couldReachMinimum
|
|
101
|
+
? ' That prefix count is an estimate and it is close to the line, so the real one may already be above it — check with --exact-tokens before deciding this is not available to you. The counting endpoint is free.'
|
|
99
102
|
: ''),
|
|
100
103
|
};
|
|
101
104
|
},
|
package/dist/i18n/en.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"en.js","sourceRoot":"","sources":["../../src/i18n/en.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;AAEnE,iFAAiF;AACjF,MAAM,CAAC,MAAM,EAAE,GAAiB;IAC9B,MAAM,EAAE,IAAI;IACZ,YAAY,EAAE,OAAO;IAErB,KAAK,EAAE;QACL,kBAAkB,EAAE;YAClB,KAAK,EAAE,qBAAqB;YAC5B,SAAS,EACP,uMAAuM;SAC1M;QACD,uBAAuB,EAAE;YACvB,KAAK,EAAE,2BAA2B;YAClC,SAAS,EACP,gPAAgP;SACnP;QACD,iBAAiB,EAAE;YACjB,KAAK,EAAE,gBAAgB;YACvB,SAAS,EACP,0LAA0L;SAC7L;QACD,iBAAiB,EAAE;YACjB,KAAK,EAAE,gBAAgB;YACvB,SAAS,EACP,mKAAmK;SACtK;QACD,UAAU,EAAE;YACV,KAAK,EAAE,qBAAqB;YAC5B,SAAS,EACP,8JAA8J;SACjK;QACD,MAAM,EAAE;YACN,KAAK,EAAE,4BAA4B;YACnC,SAAS,EACP,+HAA+H;SAClI;QACD,MAAM,EAAE;YACN,KAAK,EAAE,SAAS;YAChB,SAAS,EACP,2HAA2H;SAC9H;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,cAAc;YACrB,SAAS,EACP,kJAAkJ;SACrJ;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,gCAAgC;YACvC,SAAS,EACP,+OAA+O;SAClP;QACD,QAAQ,EAAE;YACR,KAAK,EAAE,kBAAkB;YACzB,SAAS,EACP,uNAAuN;SAC1N;QACD,UAAU,EAAE;YACV,KAAK,EAAE,uBAAuB;YAC9B,SAAS,EACP,sKAAsK;SACzK;QACD,UAAU,EAAE;YACV,KAAK,EAAE,kBAAkB;YACzB,SAAS,EACP,wLAAwL;SAC3L;KACF;IAED,GAAG,EAAE;QACH,aAAa,EAAE,GAAG,EAAE,CAAC,uCAAuC;QAC5D,uBAAuB,EAAE,CAAC,KAAK,EAAE,EAAE,CACjC,qBAAqB,KAAK,oGAAoG;QAChI,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAC5B,8BAA8B,KAAK,OAAO,MAAM,8CAA8C;QAChG,gBAAgB,EAAE,CAAC,WAAW,EAAE,EAAE,CAChC,yBAAyB,WAAW,wGAAwG;KAC/I;IAED,OAAO,EAAE;QACP,WAAW,EAAE,GAAG,EAAE,CAChB,oFAAoF;QACtF,mBAAmB,EAAE,GAAG,EAAE,CACxB,8EAA8E;QAChF,sBAAsB,EAAE,GAAG,EAAE,CAAC,8DAA8D;QAC5F,WAAW,EAAE,GAAG,EAAE,CAAC,wCAAwC;QAC3D,QAAQ,EAAE,GAAG,EAAE,CAAC,mDAAmD;KACpE;IAED,UAAU,EAAE;QACV,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAC1D,KAAK,EAAE,+CAA+C;YACtD,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,CAAC,eAAe,SAAS,YAAY,CAAC,CAAC,aAAa,CAAC,kFAAkF;SACpL,CAAC;QAEF,aAAa,EAAE,CAAC,EACd,WAAW,EACX,YAAY,EACZ,WAAW,EACX,SAAS,EACT,SAAS,EACT,UAAU,EACV,OAAO,EACP,QAAQ,EACR,QAAQ,GACT,EAAE,EAAE;YACH,MAAM,KAAK,GAAG,WAAW;gBACvB,CAAC,CAAC,+DAA+D,WAAW,UAAU,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC,WAAW,CAAC,uBAAuB,SAAS,MAAM,CAAC,CAAC,SAAS,CAAC,2BAA2B;gBACpN,CAAC,CAAC,mGAAmG,SAAS,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC;YACpJ,MAAM,GAAG,GAAG,QAAQ;gBAClB,CAAC,CAAC,6HAA6H;gBAC/H,CAAC,CAAC,GAAG,SAAS,wKAAwK,CAAC;YACzL,OAAO;gBACL,KAAK,EAAE,8CAA8C;gBACrD,MAAM,EAAE,GAAG,KAAK,SAAS,UAAU,kCAAkC,OAAO,0CAA0C,QAAQ,MAAM,GAAG,EAAE;aAC1I,CAAC;QACJ,CAAC;QAED,uBAAuB,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9B,KAAK,EAAE,4CAA4C;YACnD,MAAM,EACJ,4KAA4K;SAC/K,CAAC;QAEF,iBAAiB,EAAE,CAAC,EAClB,SAAS,EACT,SAAS,EACT,WAAW,EACX,YAAY,EACZ,WAAW,EACX,mBAAmB,
|
|
1
|
+
{"version":3,"file":"en.js","sourceRoot":"","sources":["../../src/i18n/en.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;AAEnE,iFAAiF;AACjF,MAAM,CAAC,MAAM,EAAE,GAAiB;IAC9B,MAAM,EAAE,IAAI;IACZ,YAAY,EAAE,OAAO;IAErB,KAAK,EAAE;QACL,kBAAkB,EAAE;YAClB,KAAK,EAAE,qBAAqB;YAC5B,SAAS,EACP,uMAAuM;SAC1M;QACD,uBAAuB,EAAE;YACvB,KAAK,EAAE,2BAA2B;YAClC,SAAS,EACP,gPAAgP;SACnP;QACD,iBAAiB,EAAE;YACjB,KAAK,EAAE,gBAAgB;YACvB,SAAS,EACP,0LAA0L;SAC7L;QACD,iBAAiB,EAAE;YACjB,KAAK,EAAE,gBAAgB;YACvB,SAAS,EACP,mKAAmK;SACtK;QACD,UAAU,EAAE;YACV,KAAK,EAAE,qBAAqB;YAC5B,SAAS,EACP,8JAA8J;SACjK;QACD,MAAM,EAAE;YACN,KAAK,EAAE,4BAA4B;YACnC,SAAS,EACP,+HAA+H;SAClI;QACD,MAAM,EAAE;YACN,KAAK,EAAE,SAAS;YAChB,SAAS,EACP,2HAA2H;SAC9H;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,cAAc;YACrB,SAAS,EACP,kJAAkJ;SACrJ;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,gCAAgC;YACvC,SAAS,EACP,+OAA+O;SAClP;QACD,QAAQ,EAAE;YACR,KAAK,EAAE,kBAAkB;YACzB,SAAS,EACP,uNAAuN;SAC1N;QACD,UAAU,EAAE;YACV,KAAK,EAAE,uBAAuB;YAC9B,SAAS,EACP,sKAAsK;SACzK;QACD,UAAU,EAAE;YACV,KAAK,EAAE,kBAAkB;YACzB,SAAS,EACP,wLAAwL;SAC3L;KACF;IAED,GAAG,EAAE;QACH,aAAa,EAAE,GAAG,EAAE,CAAC,uCAAuC;QAC5D,uBAAuB,EAAE,CAAC,KAAK,EAAE,EAAE,CACjC,qBAAqB,KAAK,oGAAoG;QAChI,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAC5B,8BAA8B,KAAK,OAAO,MAAM,8CAA8C;QAChG,gBAAgB,EAAE,CAAC,WAAW,EAAE,EAAE,CAChC,yBAAyB,WAAW,wGAAwG;KAC/I;IAED,OAAO,EAAE;QACP,WAAW,EAAE,GAAG,EAAE,CAChB,oFAAoF;QACtF,mBAAmB,EAAE,GAAG,EAAE,CACxB,8EAA8E;QAChF,sBAAsB,EAAE,GAAG,EAAE,CAAC,8DAA8D;QAC5F,WAAW,EAAE,GAAG,EAAE,CAAC,wCAAwC;QAC3D,QAAQ,EAAE,GAAG,EAAE,CAAC,mDAAmD;KACpE;IAED,UAAU,EAAE;QACV,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAC1D,KAAK,EAAE,+CAA+C;YACtD,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,CAAC,eAAe,SAAS,YAAY,CAAC,CAAC,aAAa,CAAC,kFAAkF;SACpL,CAAC;QAEF,aAAa,EAAE,CAAC,EACd,WAAW,EACX,YAAY,EACZ,WAAW,EACX,SAAS,EACT,SAAS,EACT,UAAU,EACV,OAAO,EACP,QAAQ,EACR,QAAQ,GACT,EAAE,EAAE;YACH,MAAM,KAAK,GAAG,WAAW;gBACvB,CAAC,CAAC,+DAA+D,WAAW,UAAU,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC,WAAW,CAAC,uBAAuB,SAAS,MAAM,CAAC,CAAC,SAAS,CAAC,2BAA2B;gBACpN,CAAC,CAAC,mGAAmG,SAAS,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC;YACpJ,MAAM,GAAG,GAAG,QAAQ;gBAClB,CAAC,CAAC,6HAA6H;gBAC/H,CAAC,CAAC,GAAG,SAAS,wKAAwK,CAAC;YACzL,OAAO;gBACL,KAAK,EAAE,8CAA8C;gBACrD,MAAM,EAAE,GAAG,KAAK,SAAS,UAAU,kCAAkC,OAAO,0CAA0C,QAAQ,MAAM,GAAG,EAAE;aAC1I,CAAC;QACJ,CAAC;QAED,uBAAuB,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9B,KAAK,EAAE,4CAA4C;YACnD,MAAM,EACJ,4KAA4K;SAC/K,CAAC;QAEF,iBAAiB,EAAE,CAAC,EAClB,SAAS,EACT,SAAS,EACT,WAAW,EACX,YAAY,EACZ,WAAW,EACX,mBAAmB,EACnB,iBAAiB,GAClB,EAAE,EAAE;YACH,MAAM,MAAM,GAAG,WAAW;gBACxB,CAAC,CAAC,wCAAwC,WAAW,kBAAkB,CAAC,CAAC,YAAY,CAAC,iDAAiD;gBACvI,CAAC,CAAC,oBAAoB,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;YACzC,OAAO;gBACL,KAAK,EAAE,6BAA6B;gBACpC,MAAM,EACJ,GAAG,SAAS,mBAAmB,CAAC,CAAC,SAAS,CAAC,4BAA4B,MAAM,mEAAmE;oBAChJ,CAAC,mBAAmB;wBAClB,CAAC,CAAC,sGAAsG;wBACxG,CAAC,CAAC,EAAE,CAAC;oBACP,CAAC,iBAAiB;wBAChB,CAAC,CAAC,iNAAiN;wBACnN,CAAC,CAAC,EAAE,CAAC;aACV,CAAC;QACJ,CAAC;QAED,kBAAkB,EAAE,CAAC,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;YACrE,KAAK,EAAE,6DAA6D;YACpE,MAAM,EAAE,UAAU,CAAC,CAAC,iBAAiB,CAAC,8BAA8B,QAAQ,6DAA6D,WAAW,uPAAuP;SAC5Y,CAAC;QAEF,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YACf,KAAK,EAAE,kDAAkD;YACzD,MAAM,EACJ,+OAA+O;SAClP,CAAC;QAEF,cAAc,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;YACjF,KAAK,EAAE,0BAA0B,SAAS,EAAE;YAC5C,MAAM,EAAE,mDAAmD,IAAI,oBAAoB,aAAa,sBAAsB,UAAU,OAAO,YAAY,iKAAiK;SACrT,CAAC;QAEF,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7C,KAAK,EAAE,4CAA4C;YACnD,MAAM,EAAE,uBAAuB,SAAS,sBAAsB,QAAQ,oNAAoN;SAC3R,CAAC;QAEF,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YACvF,KAAK,EAAE,+CAA+C;YACtD,MAAM,EAAE,GAAG,SAAS,gCAAgC,UAAU,IAAI,WAAW,6BAA6B,KAAK,iCAAiC,SAAS,IAAI,UAAU,iDAAiD;SACzN,CAAC;QAEF,yBAAyB,EAAE,CAAC,EAC1B,IAAI,EACJ,UAAU,EACV,YAAY,EACZ,WAAW,EACX,aAAa,EACb,UAAU,GACX,EAAE,EAAE,CAAC,CAAC;YACL,KAAK,EAAE,mCAAmC,IAAI,EAAE;YAChD,MAAM,EACJ,YAAY,UAAU,MAAM,YAAY,uBAAuB,WAAW,MAAM,aAAa,MAAM;gBACnG,yIAAyI;gBACzI,qEAAqE;gBACrE,CAAC,UAAU,GAAG,CAAC;oBACb,CAAC,CAAC,IAAI,UAAU,UAAU,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,gCAAgC;oBAC7F,CAAC,CAAC,EAAE,CAAC;SACV,CAAC;QAEF,iBAAiB,EAAE,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,eAAe,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC,CAAC;YACzF,KAAK,EAAE,GAAG,cAAc,OAAO,UAAU,iCAAiC;YAC1E,MAAM,EACJ,cAAc,gBAAgB,4FAA4F,CAAC,CAAC,eAAe,CAAC,kCAAkC;gBAC9K,oHAAoH;gBACpH,uGAAuG;SAC1G,CAAC;QAEF,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,KAAK,EAAE,qEAAqE;YAC5E,MAAM,EACJ,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,MAAM,gBAAgB,mBAAmB,GAAG,aAAa,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,yBAAyB;gBAC7J,+QAA+Q;gBAC/Q,oMAAoM;SACvM,CAAC;QAEF,oBAAoB,EAAE,CAAC,EAAE,aAAa,EAAE,UAAU,EAAE,cAAc,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YACjF,KAAK,EAAE,sCAAsC;YAC7C,MAAM,EACJ,uBAAuB,aAAa,WAAW,UAAU,wCAAwC,OAAO,oBAAoB,CAAC,CAAC,cAAc,CAAC,yBAAyB;gBACtK,6FAA6F;gBAC7F,6JAA6J;SAChK,CAAC;KACH;IAED,iBAAiB,EAAE;QACjB,mBAAmB,EAAE,4BAA4B;QACjD,eAAe,EAAE,mBAAmB;QACpC,iBAAiB,EAAE,+BAA+B;QAClD,sBAAsB,EAAE,+BAA+B;KACxD;IAED,mBAAmB,EAAE;QACnB,gBAAgB,EAAE,0BAA0B;QAC5C,iBAAiB,EAAE,2BAA2B;QAC9C,aAAa,EAAE,MAAM;QACrB,iBAAiB,EAAE,UAAU;QAC7B,mBAAmB,EAAE,YAAY;QACjC,cAAc,EAAE,eAAe;QAC/B,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,oBAAoB;QACvC,kBAAkB,EAAE,oBAAoB;KACzC;CACF,CAAC"}
|
package/dist/i18n/es.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"es.d.ts","sourceRoot":"","sources":["../../src/i18n/es.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/C,wEAAwE;AACxE,eAAO,MAAM,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"es.d.ts","sourceRoot":"","sources":["../../src/i18n/es.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/C,wEAAwE;AACxE,eAAO,MAAM,EAAE,EAAE,YA0OhB,CAAC"}
|
package/dist/i18n/es.js
CHANGED
|
@@ -87,7 +87,7 @@ export const es = {
|
|
|
87
87
|
title: 'Con esa tasa de acierto, la caché no compensa',
|
|
88
88
|
detail: 'Escribir en caché cuesta un 125% del precio de entrada y leer un 10%. Por debajo de un ~28% de aciertos pagas más de lo que ahorras. Sube la reutilización del prefijo o deja la caché desactivada.',
|
|
89
89
|
}),
|
|
90
|
-
belowCacheMinimum: ({ modelName, minTokens, placeholder, prefixTokens, totalTokens, mentionLowerMinimum, }) => {
|
|
90
|
+
belowCacheMinimum: ({ modelName, minTokens, placeholder, prefixTokens, totalTokens, mentionLowerMinimum, couldReachMinimum, }) => {
|
|
91
91
|
const reason = placeholder
|
|
92
92
|
? `aquí el primer marcador variable (${placeholder}) aparece a los ~${n(prefixTokens)} tokens y solo lo anterior puede cachearse`
|
|
93
93
|
: `este prompt tiene ~${n(totalTokens)}`;
|
|
@@ -96,6 +96,9 @@ export const es = {
|
|
|
96
96
|
detail: `${modelName} necesita al menos ${n(minTokens)} tokens de prefijo para cachear; ${reason}. Marcar cache_control no dará error, simplemente no cacheará.` +
|
|
97
97
|
(mentionLowerMinimum
|
|
98
98
|
? ' Claude Opus 5 baja ese mínimo a 512 tokens, así que prompts cortos que aquí no cachean, allí sí.'
|
|
99
|
+
: '') +
|
|
100
|
+
(couldReachMinimum
|
|
101
|
+
? ' Ese recuento del prefijo es una estimación y está cerca del límite, así que el real puede estar ya por encima: compruébalo con --exact-tokens antes de dar esto por perdido. El endpoint de conteo es gratis.'
|
|
99
102
|
: ''),
|
|
100
103
|
};
|
|
101
104
|
},
|
package/dist/i18n/es.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"es.js","sourceRoot":"","sources":["../../src/i18n/es.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;AAEnE,wEAAwE;AACxE,MAAM,CAAC,MAAM,EAAE,GAAiB;IAC9B,MAAM,EAAE,IAAI;IACZ,YAAY,EAAE,OAAO;IAErB,KAAK,EAAE;QACL,kBAAkB,EAAE;YAClB,KAAK,EAAE,oBAAoB;YAC3B,SAAS,EACP,mMAAmM;SACtM;QACD,uBAAuB,EAAE;YACvB,KAAK,EAAE,yBAAyB;YAChC,SAAS,EACP,8OAA8O;SACjP;QACD,iBAAiB,EAAE;YACjB,KAAK,EAAE,kBAAkB;YACzB,SAAS,EACP,0MAA0M;SAC7M;QACD,iBAAiB,EAAE;YACjB,KAAK,EAAE,mBAAmB;YAC1B,SAAS,EACP,yKAAyK;SAC5K;QACD,UAAU,EAAE;YACV,KAAK,EAAE,sBAAsB;YAC7B,SAAS,EACP,2JAA2J;SAC9J;QACD,MAAM,EAAE;YACN,KAAK,EAAE,qBAAqB;YAC5B,SAAS,EACP,gIAAgI;SACnI;QACD,MAAM,EAAE;YACN,KAAK,EAAE,oBAAoB;YAC3B,SAAS,EACP,+GAA+G;SAClH;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,kBAAkB;YACzB,SAAS,EACP,yIAAyI;SAC5I;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,oCAAoC;YAC3C,SAAS,EACP,0OAA0O;SAC7O;QACD,QAAQ,EAAE;YACR,KAAK,EAAE,uBAAuB;YAC9B,SAAS,EACP,qPAAqP;SACxP;QACD,UAAU,EAAE;YACV,KAAK,EAAE,yBAAyB;YAChC,SAAS,EACP,kMAAkM;SACrM;QACD,UAAU,EAAE;YACV,KAAK,EAAE,uCAAuC;YAC9C,SAAS,EACP,yMAAyM;SAC5M;KACF;IAED,GAAG,EAAE;QACH,aAAa,EAAE,GAAG,EAAE,CAAC,yCAAyC;QAC9D,uBAAuB,EAAE,CAAC,KAAK,EAAE,EAAE,CACjC,oBAAoB,KAAK,wGAAwG;QACnI,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAC5B,iCAAiC,KAAK,OAAO,MAAM,gDAAgD;QACrG,gBAAgB,EAAE,CAAC,WAAW,EAAE,EAAE,CAChC,iCAAiC,WAAW,6FAA6F;KAC5I;IAED,OAAO,EAAE;QACP,WAAW,EAAE,GAAG,EAAE,CAChB,oFAAoF;QACtF,mBAAmB,EAAE,GAAG,EAAE,CACxB,kFAAkF;QACpF,sBAAsB,EAAE,GAAG,EAAE,CAAC,wDAAwD;QACtF,WAAW,EAAE,GAAG,EAAE,CAAC,sCAAsC;QACzD,QAAQ,EAAE,GAAG,EAAE,CAAC,+CAA+C;KAChE;IAED,UAAU,EAAE;QACV,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAC1D,KAAK,EAAE,6CAA6C;YACpD,MAAM,EAAE,+BAA+B,CAAC,CAAC,MAAM,CAAC,aAAa,SAAS,WAAW,CAAC,CAAC,aAAa,CAAC,mFAAmF;SACrL,CAAC;QAEF,aAAa,EAAE,CAAC,EACd,WAAW,EACX,YAAY,EACZ,WAAW,EACX,SAAS,EACT,SAAS,EACT,UAAU,EACV,OAAO,EACP,QAAQ,EACR,QAAQ,GACT,EAAE,EAAE;YACH,MAAM,KAAK,GAAG,WAAW;gBACvB,CAAC,CAAC,sDAAsD,WAAW,UAAU,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,WAAW,CAAC,uDAAuD,CAAC,CAAC,SAAS,CAAC,OAAO,SAAS,GAAG;gBAC3M,CAAC,CAAC,yGAAyG,CAAC,CAAC,SAAS,CAAC,cAAc,SAAS,GAAG,CAAC;YACpJ,MAAM,GAAG,GAAG,QAAQ;gBAClB,CAAC,CAAC,qIAAqI;gBACvI,CAAC,CAAC,GAAG,SAAS,wLAAwL,CAAC;YACzM,OAAO;gBACL,KAAK,EAAE,6CAA6C;gBACpD,MAAM,EAAE,GAAG,KAAK,gCAAgC,UAAU,oCAAoC,OAAO,6CAA6C,QAAQ,MAAM,GAAG,EAAE;aACtK,CAAC;QACJ,CAAC;QAED,uBAAuB,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9B,KAAK,EAAE,+CAA+C;YACtD,MAAM,EACJ,qMAAqM;SACxM,CAAC;QAEF,iBAAiB,EAAE,CAAC,EAClB,SAAS,EACT,SAAS,EACT,WAAW,EACX,YAAY,EACZ,WAAW,EACX,mBAAmB,
|
|
1
|
+
{"version":3,"file":"es.js","sourceRoot":"","sources":["../../src/i18n/es.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;AAEnE,wEAAwE;AACxE,MAAM,CAAC,MAAM,EAAE,GAAiB;IAC9B,MAAM,EAAE,IAAI;IACZ,YAAY,EAAE,OAAO;IAErB,KAAK,EAAE;QACL,kBAAkB,EAAE;YAClB,KAAK,EAAE,oBAAoB;YAC3B,SAAS,EACP,mMAAmM;SACtM;QACD,uBAAuB,EAAE;YACvB,KAAK,EAAE,yBAAyB;YAChC,SAAS,EACP,8OAA8O;SACjP;QACD,iBAAiB,EAAE;YACjB,KAAK,EAAE,kBAAkB;YACzB,SAAS,EACP,0MAA0M;SAC7M;QACD,iBAAiB,EAAE;YACjB,KAAK,EAAE,mBAAmB;YAC1B,SAAS,EACP,yKAAyK;SAC5K;QACD,UAAU,EAAE;YACV,KAAK,EAAE,sBAAsB;YAC7B,SAAS,EACP,2JAA2J;SAC9J;QACD,MAAM,EAAE;YACN,KAAK,EAAE,qBAAqB;YAC5B,SAAS,EACP,gIAAgI;SACnI;QACD,MAAM,EAAE;YACN,KAAK,EAAE,oBAAoB;YAC3B,SAAS,EACP,+GAA+G;SAClH;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,kBAAkB;YACzB,SAAS,EACP,yIAAyI;SAC5I;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,oCAAoC;YAC3C,SAAS,EACP,0OAA0O;SAC7O;QACD,QAAQ,EAAE;YACR,KAAK,EAAE,uBAAuB;YAC9B,SAAS,EACP,qPAAqP;SACxP;QACD,UAAU,EAAE;YACV,KAAK,EAAE,yBAAyB;YAChC,SAAS,EACP,kMAAkM;SACrM;QACD,UAAU,EAAE;YACV,KAAK,EAAE,uCAAuC;YAC9C,SAAS,EACP,yMAAyM;SAC5M;KACF;IAED,GAAG,EAAE;QACH,aAAa,EAAE,GAAG,EAAE,CAAC,yCAAyC;QAC9D,uBAAuB,EAAE,CAAC,KAAK,EAAE,EAAE,CACjC,oBAAoB,KAAK,wGAAwG;QACnI,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAC5B,iCAAiC,KAAK,OAAO,MAAM,gDAAgD;QACrG,gBAAgB,EAAE,CAAC,WAAW,EAAE,EAAE,CAChC,iCAAiC,WAAW,6FAA6F;KAC5I;IAED,OAAO,EAAE;QACP,WAAW,EAAE,GAAG,EAAE,CAChB,oFAAoF;QACtF,mBAAmB,EAAE,GAAG,EAAE,CACxB,kFAAkF;QACpF,sBAAsB,EAAE,GAAG,EAAE,CAAC,wDAAwD;QACtF,WAAW,EAAE,GAAG,EAAE,CAAC,sCAAsC;QACzD,QAAQ,EAAE,GAAG,EAAE,CAAC,+CAA+C;KAChE;IAED,UAAU,EAAE;QACV,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAC1D,KAAK,EAAE,6CAA6C;YACpD,MAAM,EAAE,+BAA+B,CAAC,CAAC,MAAM,CAAC,aAAa,SAAS,WAAW,CAAC,CAAC,aAAa,CAAC,mFAAmF;SACrL,CAAC;QAEF,aAAa,EAAE,CAAC,EACd,WAAW,EACX,YAAY,EACZ,WAAW,EACX,SAAS,EACT,SAAS,EACT,UAAU,EACV,OAAO,EACP,QAAQ,EACR,QAAQ,GACT,EAAE,EAAE;YACH,MAAM,KAAK,GAAG,WAAW;gBACvB,CAAC,CAAC,sDAAsD,WAAW,UAAU,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,WAAW,CAAC,uDAAuD,CAAC,CAAC,SAAS,CAAC,OAAO,SAAS,GAAG;gBAC3M,CAAC,CAAC,yGAAyG,CAAC,CAAC,SAAS,CAAC,cAAc,SAAS,GAAG,CAAC;YACpJ,MAAM,GAAG,GAAG,QAAQ;gBAClB,CAAC,CAAC,qIAAqI;gBACvI,CAAC,CAAC,GAAG,SAAS,wLAAwL,CAAC;YACzM,OAAO;gBACL,KAAK,EAAE,6CAA6C;gBACpD,MAAM,EAAE,GAAG,KAAK,gCAAgC,UAAU,oCAAoC,OAAO,6CAA6C,QAAQ,MAAM,GAAG,EAAE;aACtK,CAAC;QACJ,CAAC;QAED,uBAAuB,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9B,KAAK,EAAE,+CAA+C;YACtD,MAAM,EACJ,qMAAqM;SACxM,CAAC;QAEF,iBAAiB,EAAE,CAAC,EAClB,SAAS,EACT,SAAS,EACT,WAAW,EACX,YAAY,EACZ,WAAW,EACX,mBAAmB,EACnB,iBAAiB,GAClB,EAAE,EAAE;YACH,MAAM,MAAM,GAAG,WAAW;gBACxB,CAAC,CAAC,qCAAqC,WAAW,oBAAoB,CAAC,CAAC,YAAY,CAAC,4CAA4C;gBACjI,CAAC,CAAC,sBAAsB,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3C,OAAO;gBACL,KAAK,EAAE,iCAAiC;gBACxC,MAAM,EACJ,GAAG,SAAS,sBAAsB,CAAC,CAAC,SAAS,CAAC,oCAAoC,MAAM,gEAAgE;oBACxJ,CAAC,mBAAmB;wBAClB,CAAC,CAAC,mGAAmG;wBACrG,CAAC,CAAC,EAAE,CAAC;oBACP,CAAC,iBAAiB;wBAChB,CAAC,CAAC,gNAAgN;wBAClN,CAAC,CAAC,EAAE,CAAC;aACV,CAAC;QACJ,CAAC;QAED,kBAAkB,EAAE,CAAC,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;YACrE,KAAK,EAAE,4DAA4D;YACnE,MAAM,EAAE,SAAS,CAAC,CAAC,iBAAiB,CAAC,oCAAoC,QAAQ,4DAA4D,WAAW,8OAA8O;SACvY,CAAC;QAEF,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YACf,KAAK,EAAE,iDAAiD;YACxD,MAAM,EACJ,+QAA+Q;SAClR,CAAC;QAEF,cAAc,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;YACjF,KAAK,EAAE,gCAAgC,SAAS,EAAE;YAClD,MAAM,EAAE,gEAAgE,IAAI,UAAU,aAAa,gBAAgB,UAAU,MAAM,YAAY,yJAAyJ;SACzS,CAAC;QAEF,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7C,KAAK,EAAE,6CAA6C;YACpD,MAAM,EAAE,oBAAoB,SAAS,oBAAoB,QAAQ,6NAA6N;SAC/R,CAAC;QAEF,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YACvF,KAAK,EAAE,yCAAyC;YAChD,MAAM,EAAE,GAAG,SAAS,gCAAgC,UAAU,IAAI,WAAW,kCAAkC,KAAK,kCAAkC,SAAS,IAAI,UAAU,6CAA6C;SAC3N,CAAC;QAEF,yBAAyB,EAAE,CAAC,EAC1B,IAAI,EACJ,UAAU,EACV,YAAY,EACZ,WAAW,EACX,aAAa,EACb,UAAU,GACX,EAAE,EAAE,CAAC,CAAC;YACL,KAAK,EAAE,0CAA0C,IAAI,EAAE;YACvD,MAAM,EACJ,YAAY,UAAU,MAAM,YAAY,kBAAkB,WAAW,MAAM,aAAa,MAAM;gBAC9F,mIAAmI;gBACnI,+DAA+D;gBAC/D,CAAC,UAAU,GAAG,CAAC;oBACb,CAAC,CAAC,QAAQ,UAAU,IAAI,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,uCAAuC;oBAC9G,CAAC,CAAC,EAAE,CAAC;SACV,CAAC;QAEF,iBAAiB,EAAE,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,eAAe,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC,CAAC;YACzF,KAAK,EAAE,GAAG,cAAc,OAAO,UAAU,gCAAgC;YACzE,MAAM,EACJ,gBAAgB,gBAAgB,gEAAgE,CAAC,CAAC,eAAe,CAAC,qCAAqC;gBACvJ,sHAAsH;gBACtH,4GAA4G;SAC/G,CAAC;QAEF,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,KAAK,EAAE,iEAAiE;YACxE,MAAM,EACJ,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,MAAM,qBAAqB,qBAAqB,GAAG,YAAY,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,2BAA2B;gBAC3K,wSAAwS;gBACxS,qLAAqL;SACxL,CAAC;QAEF,oBAAoB,EAAE,CAAC,EAAE,aAAa,EAAE,UAAU,EAAE,cAAc,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YACjF,KAAK,EAAE,kDAAkD;YACzD,MAAM,EACJ,qBAAqB,aAAa,WAAW,UAAU,qCAAqC,OAAO,oBAAoB,CAAC,CAAC,cAAc,CAAC,2BAA2B;gBACnK,wGAAwG;gBACxG,8KAA8K;SACjL,CAAC;KACH;IAED,iBAAiB,EAAE;QACjB,mBAAmB,EAAE,2BAA2B;QAChD,eAAe,EAAE,sBAAsB;QACvC,iBAAiB,EAAE,6BAA6B;QAChD,sBAAsB,EAAE,4BAA4B;KACrD;IAED,mBAAmB,EAAE;QACnB,gBAAgB,EAAE,yBAAyB;QAC3C,iBAAiB,EAAE,4BAA4B;QAC/C,aAAa,EAAE,MAAM;QACrB,iBAAiB,EAAE,UAAU;QAC7B,mBAAmB,EAAE,aAAa;QAClC,cAAc,EAAE,eAAe;QAC/B,iBAAiB,EAAE,sBAAsB;QACzC,iBAAiB,EAAE,yBAAyB;QAC5C,kBAAkB,EAAE,yBAAyB;KAC9C;CACF,CAAC"}
|
package/dist/i18n/types.d.ts
CHANGED
|
@@ -63,6 +63,16 @@ export interface BelowCacheMinimumParams {
|
|
|
63
63
|
totalTokens: number;
|
|
64
64
|
/** Whether to point at Claude Opus 5's lower 512-token minimum. */
|
|
65
65
|
mentionLowerMinimum: boolean;
|
|
66
|
+
/**
|
|
67
|
+
* The prefix is *estimated* and close enough to the minimum that the real
|
|
68
|
+
* count could be above it.
|
|
69
|
+
*
|
|
70
|
+
* Without this the advisory asserts "caching will not work here" from a number
|
|
71
|
+
* measured to ±15%, and on a prefix near the threshold that is not an imprecise
|
|
72
|
+
* figure — it is wrong advice, and it costs the reader the largest saving
|
|
73
|
+
* Trazum offers.
|
|
74
|
+
*/
|
|
75
|
+
couldReachMinimum: boolean;
|
|
66
76
|
}
|
|
67
77
|
export interface CachePrefixReorderParams {
|
|
68
78
|
staticTokensAfter: number;
|
package/dist/i18n/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/i18n/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,mEAAmE;AACnE,eAAO,MAAM,OAAO,YAAI,IAAI,EAAE,IAAI,CAAU,CAAC;AAE7C,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9C,eAAO,MAAM,cAAc,EAAE,MAAa,CAAC;AAE3C;;;;;;GAMG;AACH,MAAM,MAAM,MAAM,GACd,kBAAkB,GAClB,uBAAuB,GACvB,iBAAiB,GACjB,iBAAiB,GACjB,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,cAAc,GACd,YAAY,GACZ,UAAU,GACV,YAAY,GACZ,YAAY,CAAC;AAEjB,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,mBAAmB;IAClC,sEAAsE;IACtE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,mBAAmB,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/i18n/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,mEAAmE;AACnE,eAAO,MAAM,OAAO,YAAI,IAAI,EAAE,IAAI,CAAU,CAAC;AAE7C,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9C,eAAO,MAAM,cAAc,EAAE,MAAa,CAAC;AAE3C;;;;;;GAMG;AACH,MAAM,MAAM,MAAM,GACd,kBAAkB,GAClB,uBAAuB,GACvB,iBAAiB,GACjB,iBAAiB,GACjB,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,cAAc,GACd,YAAY,GACZ,UAAU,GACV,YAAY,GACZ,YAAY,CAAC;AAEjB,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,mBAAmB;IAClC,sEAAsE;IACtE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,mBAAmB,EAAE,OAAO,CAAC;IAC7B;;;;;;;;OAQG;IACH,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,wBAAwB;IACvC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,+BAA+B;IAC9C,wEAAwE;IACxE,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,0DAA0D;IAC1D,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAuB;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,6CAA6C;IAC7C,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,mBAAmB;IAClC,8CAA8C;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,0BAA0B;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,aAAa,IAAI,MAAM,CAAC;IACxB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IAC/C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IAClD,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAAC;CAC/C;AAED,0EAA0E;AAC1E,MAAM,WAAW,eAAe;IAC9B,WAAW,IAAI,MAAM,CAAC;IACtB,mBAAmB,IAAI,MAAM,CAAC;IAC9B,sBAAsB,IAAI,MAAM,CAAC;IACjC,WAAW,IAAI,MAAM,CAAC;IACtB,QAAQ,IAAI,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChC,GAAG,EAAE,WAAW,CAAC;IACjB,OAAO,EAAE,eAAe,CAAC;IACzB,UAAU,EAAE;QACV,eAAe,CAAC,CAAC,EAAE,qBAAqB,GAAG,gBAAgB,CAAC;QAC5D,aAAa,CAAC,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;QACxD,uBAAuB,IAAI,gBAAgB,CAAC;QAC5C,iBAAiB,CAAC,CAAC,EAAE,uBAAuB,GAAG,gBAAgB,CAAC;QAChE,kBAAkB,CAAC,CAAC,EAAE,wBAAwB,GAAG,gBAAgB,CAAC;QAClE,QAAQ,IAAI,gBAAgB,CAAC;QAC7B,cAAc,CAAC,CAAC,EAAE,oBAAoB,GAAG,gBAAgB,CAAC;QAC1D,eAAe,CAAC,CAAC,EAAE,qBAAqB,GAAG,gBAAgB,CAAC;QAC5D,YAAY,CAAC,CAAC,EAAE,kBAAkB,GAAG,gBAAgB,CAAC;QACtD,yBAAyB,CAAC,CAAC,EAAE,+BAA+B,GAAG,gBAAgB,CAAC;QAChF,iBAAiB,CAAC,CAAC,EAAE,uBAAuB,GAAG,gBAAgB,CAAC;QAChE,oBAAoB,CAAC,CAAC,EAAE,0BAA0B,GAAG,gBAAgB,CAAC;QACtE,aAAa,CAAC,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;KACzD,CAAC;IACF,0DAA0D;IAC1D,iBAAiB,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC;IACvD,yEAAyE;IACzE,mBAAmB,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;CAC3D;AAED;;;GAGG;AACH,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,eAAe,GACf,iBAAiB,GACjB,sBAAsB,CAAC;AAE3B;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAC5B,gBAAgB,GAChB,iBAAiB,GACjB,aAAa,GACb,iBAAiB,GACjB,mBAAmB,GACnB,cAAc,GACd,iBAAiB,GACjB,iBAAiB,GACjB,kBAAkB,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from './types.js';
|
|
2
|
-
export { estimateTokens, countTokensAnthropic } from './tokenizer.js';
|
|
2
|
+
export { ESTIMATE_ERROR_BAND_PCT, estimateTokens, countTokensAnthropic } from './tokenizer.js';
|
|
3
|
+
export { DETECTABLE_LANGUAGES, detectTextLanguage } from './language.js';
|
|
3
4
|
export { countSentences, profilePrompt } from './profile.js';
|
|
4
5
|
export { PHRASE_LANGUAGES } from './phrases.js';
|
|
5
6
|
export { toPromptfoo } from './promptfoo.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAC1F,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpG,YAAY,EACV,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AACjF,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAG3E,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAMpD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACrF,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,YAAY,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAKhD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,YAAY,EACV,aAAa,EACb,cAAc,EACd,aAAa,EACb,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAKtD,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,YAAY,EACV,eAAe,EACf,YAAY,EACZ,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAItF,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAClB,YAAY,EACV,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGhG,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAGpE,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAG3E,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG/C,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAK3F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACzF,YAAY,EAAE,mBAAmB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAGjF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGxF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,aAAa,GACd,MAAM,cAAc,CAAC;AAKtB,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAKzD,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from './types.js';
|
|
2
|
-
export { estimateTokens, countTokensAnthropic } from './tokenizer.js';
|
|
2
|
+
export { ESTIMATE_ERROR_BAND_PCT, estimateTokens, countTokensAnthropic } from './tokenizer.js';
|
|
3
|
+
export { DETECTABLE_LANGUAGES, detectTextLanguage } from './language.js';
|
|
3
4
|
export { countSentences, profilePrompt } from './profile.js';
|
|
4
5
|
export { PHRASE_LANGUAGES } from './phrases.js';
|
|
5
6
|
export { toPromptfoo } from './promptfoo.js';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAI1C,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQpG,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AAItB,uEAAuE;AACvE,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,8EAA8E;AAC9E,+EAA+E;AAC/E,6EAA6E;AAC7E,UAAU;AACV,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAErF,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AAQvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,+EAA+E;AAC/E,+EAA+E;AAC/E,6DAA6D;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAS/C,gFAAgF;AAChF,gFAAgF;AAChF,iEAAiE;AACjE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAOtE,4EAA4E;AAC5E,6EAA6E;AAC7E,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAGtF,8EAA8E;AAC9E,2EAA2E;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAWlB,uBAAuB;AACvB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AAGzB,sBAAsB;AACtB,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAEpE,kDAAkD;AAClD,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAGlB,6BAA6B;AAC7B,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAGpE,qEAAqE;AACrE,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAG3E,kFAAkF;AAClF,kFAAkF;AAClF,wCAAwC;AACxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAGzF,mEAAmE;AACnE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGjE,gCAAgC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAQ9C,0EAA0E;AAC1E,+EAA+E;AAC/E,8EAA8E;AAC9E,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEzD,8EAA8E;AAC9E,6EAA6E;AAC7E,2DAA2D;AAC3D,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which language a prompt is written in, to the extent it can be told cheaply.
|
|
3
|
+
*
|
|
4
|
+
* **Why the estimator needs this at all.** Measured against the counting
|
|
5
|
+
* endpoint, `estimateTokens` is accurate on English (+1.0%) and badly low on every
|
|
6
|
+
* other Latin language: German -37.3%, Spanish -22.9%, French -15.1%. Characters
|
|
7
|
+
* per token says why — English 3.44, French 2.66, Spanish 2.53, German 2.02 —
|
|
8
|
+
* while the estimator applied one divisor to all of them. Non-English text is
|
|
9
|
+
* thinner in the merge table, so the same number of characters costs more tokens.
|
|
10
|
+
*
|
|
11
|
+
* **Accents are not the signal.** That was the first hypothesis and it was tested
|
|
12
|
+
* and killed: a Spanish sample with zero accented characters comes out at -22.9%,
|
|
13
|
+
* against -22.1% for accented Spanish. Diacritics correlate with non-English text
|
|
14
|
+
* in a corpus and not in a prompt, and weighting them moved the figure by three
|
|
15
|
+
* points. What separates these languages is which words they are made of.
|
|
16
|
+
*
|
|
17
|
+
* So this counts function words — the shortest, commonest, most language-specific
|
|
18
|
+
* tokens there are. `the of and to` against `der die und ist` against
|
|
19
|
+
* `que de la en`.
|
|
20
|
+
*
|
|
21
|
+
* **It answers `null` when unsure, and that is the important part.** A prompt is
|
|
22
|
+
* not an essay: it can be three lines, or English instructions wrapped around a
|
|
23
|
+
* Spanish example, or a JSON schema with no prose at all. Guessing on those would
|
|
24
|
+
* apply a language's divisor to text that is not in that language, which is how a
|
|
25
|
+
* fix for one case becomes a regression for four. `null` means "use the default",
|
|
26
|
+
* and the default is the English-calibrated behaviour this has always had.
|
|
27
|
+
*/
|
|
28
|
+
/** Languages this can name. Exported so callers can key a table off it. */
|
|
29
|
+
export declare const DETECTABLE_LANGUAGES: readonly string[];
|
|
30
|
+
/**
|
|
31
|
+
* The language of `text`, or `null` when no answer is safe.
|
|
32
|
+
*
|
|
33
|
+
* Case-insensitive, whole words only, and it stops reading after a bounded
|
|
34
|
+
* prefix: a prompt can be a megabyte, the answer does not get better after a few
|
|
35
|
+
* thousand words, and this runs inside `estimateTokens` on every call.
|
|
36
|
+
*/
|
|
37
|
+
export declare function detectTextLanguage(text: string): string | null;
|
|
38
|
+
//# sourceMappingURL=language.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"language.d.ts","sourceRoot":"","sources":["../src/language.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AA+BH,2EAA2E;AAC3E,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EAAgC,CAAC;AAgBnF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA8B9D"}
|
package/dist/language.js
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which language a prompt is written in, to the extent it can be told cheaply.
|
|
3
|
+
*
|
|
4
|
+
* **Why the estimator needs this at all.** Measured against the counting
|
|
5
|
+
* endpoint, `estimateTokens` is accurate on English (+1.0%) and badly low on every
|
|
6
|
+
* other Latin language: German -37.3%, Spanish -22.9%, French -15.1%. Characters
|
|
7
|
+
* per token says why — English 3.44, French 2.66, Spanish 2.53, German 2.02 —
|
|
8
|
+
* while the estimator applied one divisor to all of them. Non-English text is
|
|
9
|
+
* thinner in the merge table, so the same number of characters costs more tokens.
|
|
10
|
+
*
|
|
11
|
+
* **Accents are not the signal.** That was the first hypothesis and it was tested
|
|
12
|
+
* and killed: a Spanish sample with zero accented characters comes out at -22.9%,
|
|
13
|
+
* against -22.1% for accented Spanish. Diacritics correlate with non-English text
|
|
14
|
+
* in a corpus and not in a prompt, and weighting them moved the figure by three
|
|
15
|
+
* points. What separates these languages is which words they are made of.
|
|
16
|
+
*
|
|
17
|
+
* So this counts function words — the shortest, commonest, most language-specific
|
|
18
|
+
* tokens there are. `the of and to` against `der die und ist` against
|
|
19
|
+
* `que de la en`.
|
|
20
|
+
*
|
|
21
|
+
* **It answers `null` when unsure, and that is the important part.** A prompt is
|
|
22
|
+
* not an essay: it can be three lines, or English instructions wrapped around a
|
|
23
|
+
* Spanish example, or a JSON schema with no prose at all. Guessing on those would
|
|
24
|
+
* apply a language's divisor to text that is not in that language, which is how a
|
|
25
|
+
* fix for one case becomes a regression for four. `null` means "use the default",
|
|
26
|
+
* and the default is the English-calibrated behaviour this has always had.
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* Function words per language, lower-cased, matched whole.
|
|
30
|
+
*
|
|
31
|
+
* Chosen for frequency and for *not overlapping*, which matters more: `de` is
|
|
32
|
+
* frequent in Spanish, French and Portuguese, so it earns nothing and is a
|
|
33
|
+
* liability. Each list is words that are common in its own language and rare in
|
|
34
|
+
* the others, which is what lets a small count decide.
|
|
35
|
+
*
|
|
36
|
+
* `en` is included even though it is the default, because the margin rule needs to
|
|
37
|
+
* know when English *won*: a Spanish prompt quoting an English sentence should not
|
|
38
|
+
* become Spanish on two hits.
|
|
39
|
+
*/
|
|
40
|
+
const FUNCTION_WORDS = {
|
|
41
|
+
en: ['the', 'and', 'of', 'to', 'is', 'that', 'with', 'for', 'you', 'this', 'are', 'not'],
|
|
42
|
+
es: ['que', 'los', 'las', 'del', 'una', 'para', 'con', 'como', 'pero', 'sus', 'este', 'siempre'],
|
|
43
|
+
fr: ['les', 'des', 'une', 'dans', 'pour', 'avec', 'vous', 'est', 'sur', 'aux', 'cette', 'toujours'],
|
|
44
|
+
de: ['der', 'die', 'das', 'und', 'nicht', 'sie', 'ist', 'mit', 'auf', 'einen', 'einer', 'immer'],
|
|
45
|
+
pt: ['que', 'dos', 'das', 'uma', 'para', 'com', 'como', 'não', 'seu', 'este', 'pelo', 'sempre'],
|
|
46
|
+
/**
|
|
47
|
+
* Rebuilt once, because half of it was Spanish. `per con del una sempre` are as
|
|
48
|
+
* common in Spanish as in Italian, so they earned nothing and cost the margin
|
|
49
|
+
* rule its answer: an Italian code-review prompt scored a tie and came back
|
|
50
|
+
* `null`, fell through to the English divisor, and measured -21.9%. These are
|
|
51
|
+
* words Italian has and Spanish does not.
|
|
52
|
+
*/
|
|
53
|
+
it: ['il', 'della', 'nella', 'nel', 'che', 'quando', 'senza', 'gli', 'delle', 'degli', 'più', 'anche'],
|
|
54
|
+
nl: ['het', 'een', 'van', 'niet', 'zijn', 'met', 'voor', 'dat', 'aan', 'deze', 'wordt', 'altijd'],
|
|
55
|
+
};
|
|
56
|
+
/** Languages this can name. Exported so callers can key a table off it. */
|
|
57
|
+
export const DETECTABLE_LANGUAGES = Object.keys(FUNCTION_WORDS);
|
|
58
|
+
/**
|
|
59
|
+
* How many hits the winner needs, absolutely and relative to the runner-up.
|
|
60
|
+
*
|
|
61
|
+
* Both bars exist for the same reason and neither is enough alone. The absolute
|
|
62
|
+
* one stops a three-line prompt being classified on a single word. The ratio stops
|
|
63
|
+
* a document that is genuinely mixed — English instructions around a Spanish
|
|
64
|
+
* example — from being called whichever language happened to appear once more.
|
|
65
|
+
*
|
|
66
|
+
* Deliberately cautious in the direction of `null`. A wrong language costs
|
|
67
|
+
* accuracy on text that was fine before; `null` costs only the improvement.
|
|
68
|
+
*/
|
|
69
|
+
const MIN_HITS = 4;
|
|
70
|
+
const MIN_RATIO = 1.6;
|
|
71
|
+
/**
|
|
72
|
+
* The language of `text`, or `null` when no answer is safe.
|
|
73
|
+
*
|
|
74
|
+
* Case-insensitive, whole words only, and it stops reading after a bounded
|
|
75
|
+
* prefix: a prompt can be a megabyte, the answer does not get better after a few
|
|
76
|
+
* thousand words, and this runs inside `estimateTokens` on every call.
|
|
77
|
+
*/
|
|
78
|
+
export function detectTextLanguage(text) {
|
|
79
|
+
if (!text)
|
|
80
|
+
return null;
|
|
81
|
+
// Bounded so this stays cheap on a large prompt. Function words are frequent
|
|
82
|
+
// enough that a prefix this size decides anything a whole document would.
|
|
83
|
+
const sample = text.length > 20_000 ? text.slice(0, 20_000) : text;
|
|
84
|
+
const words = sample.toLowerCase().match(/[\p{L}]+/gu);
|
|
85
|
+
if (!words || words.length < MIN_HITS * 2)
|
|
86
|
+
return null;
|
|
87
|
+
const seen = new Set(words);
|
|
88
|
+
const scores = [];
|
|
89
|
+
for (const [language, list] of Object.entries(FUNCTION_WORDS)) {
|
|
90
|
+
let hits = 0;
|
|
91
|
+
// Distinct words rather than occurrences: a prompt that repeats "the" forty
|
|
92
|
+
// times is not more English than one that uses forty different English words,
|
|
93
|
+
// and counting occurrences lets one hammered word decide.
|
|
94
|
+
for (const word of list)
|
|
95
|
+
if (seen.has(word))
|
|
96
|
+
hits++;
|
|
97
|
+
scores.push([language, hits]);
|
|
98
|
+
}
|
|
99
|
+
scores.sort((a, b) => b[1] - a[1]);
|
|
100
|
+
const [best, bestHits] = scores[0];
|
|
101
|
+
const runnerUp = scores[1]?.[1] ?? 0;
|
|
102
|
+
if (bestHits < MIN_HITS)
|
|
103
|
+
return null;
|
|
104
|
+
// A runner-up of zero is a clear win; guard the division rather than special-case.
|
|
105
|
+
if (runnerUp > 0 && bestHits / runnerUp < MIN_RATIO)
|
|
106
|
+
return null;
|
|
107
|
+
return best;
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=language.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"language.js","sourceRoot":"","sources":["../src/language.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,cAAc,GAAgD;IAClE,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC;IACxF,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC;IAChG,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,CAAC;IACnG,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;IAChG,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;IAC/F;;;;;;OAMG;IACH,EAAE,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;IACtG,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC;CAClG,CAAC;AAEF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,oBAAoB,GAAsB,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;AAEnF;;;;;;;;;;GAUG;AACH,MAAM,QAAQ,GAAG,CAAC,CAAC;AACnB,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,6EAA6E;IAC7E,0EAA0E;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEnE,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACvD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC5B,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAC9D,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,4EAA4E;QAC5E,8EAA8E;QAC9E,0DAA0D;QAC1D,KAAK,MAAM,IAAI,IAAI,IAAI;YAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,IAAI,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;IACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAErC,IAAI,QAAQ,GAAG,QAAQ;QAAE,OAAO,IAAI,CAAC;IACrC,mFAAmF;IACnF,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,QAAQ,GAAG,SAAS;QAAE,OAAO,IAAI,CAAC;IAEjE,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/dist/tokenizer.d.ts
CHANGED
|
@@ -20,6 +20,39 @@
|
|
|
20
20
|
* For exact numbers use `countTokensAnthropic` (the official token-counting
|
|
21
21
|
* endpoint, which is free) or pass your own `TokenCounter`.
|
|
22
22
|
*/
|
|
23
|
+
/**
|
|
24
|
+
* The error band this estimator is published under, as a percentage.
|
|
25
|
+
*
|
|
26
|
+
* **Measured, not chosen.** It was `15` for eight releases — a design target
|
|
27
|
+
* nobody had checked — and the first run of `scripts/measure-token-band.mjs`
|
|
28
|
+
* against the official counting endpoint found two of eight samples outside it,
|
|
29
|
+
* both underestimating. Underestimating tokens means under-reporting cost, which
|
|
30
|
+
* is the flattering direction and the worst one for this tool.
|
|
31
|
+
*
|
|
32
|
+
* `15` is the measured worst case rounded up, by the same rule that briefly made
|
|
33
|
+
* it 25: the corpus now tops out at 11.2%, on Japanese. It landing back on the
|
|
34
|
+
* number that was a guess for eight releases is a coincidence and not a
|
|
35
|
+
* restoration — that 15 bounded nothing, and this one bounds eleven measured
|
|
36
|
+
* samples across four languages and six text types.
|
|
37
|
+
*
|
|
38
|
+
* **Read one caveat before trusting it.** Four of those samples are the Latin
|
|
39
|
+
* languages whose divisors in `DIVISOR_BY_LANGUAGE` were calibrated on one or two
|
|
40
|
+
* samples each, so their residuals are in-sample and optimistic by construction.
|
|
41
|
+
* The band is set by the seven samples nothing was fitted to — worst 11.2% — and
|
|
42
|
+
* the honest test of it is the next held-out sample in Spanish, French or German.
|
|
43
|
+
* The corpus grows one sample at a time now, so that test is cheap to run.
|
|
44
|
+
*
|
|
45
|
+
* **What got it there was not accents.** A Spanish sample with zero accented
|
|
46
|
+
* characters measured -22.9% against -22.1% for accented Spanish, which killed
|
|
47
|
+
* diacritics as a signal. Languages differ in how many tokens their words cost —
|
|
48
|
+
* English 3.44 characters per token, German 2.02 — so the estimator detects the
|
|
49
|
+
* language and divides accordingly. See `language.ts`.
|
|
50
|
+
*
|
|
51
|
+
* Exported so every report, README and tool description reads the same number.
|
|
52
|
+
* It was a literal in twenty-four files before this, with the only machine-
|
|
53
|
+
* readable copy in a test.
|
|
54
|
+
*/
|
|
55
|
+
export declare const ESTIMATE_ERROR_BAND_PCT = 15;
|
|
23
56
|
/**
|
|
24
57
|
* Estimates how many tokens `text` occupies.
|
|
25
58
|
*
|
package/dist/tokenizer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;
|
|
1
|
+
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAKH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAwD1C;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA0GnD;AAED,sDAAsD;AACtD,MAAM,MAAM,iBAAiB,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,gFAAgF;IAChF,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,uBAAuB,GAAG,iBAAiB,CA6BxF"}
|
package/dist/tokenizer.js
CHANGED
|
@@ -20,10 +20,85 @@
|
|
|
20
20
|
* For exact numbers use `countTokensAnthropic` (the official token-counting
|
|
21
21
|
* endpoint, which is free) or pass your own `TokenCounter`.
|
|
22
22
|
*/
|
|
23
|
+
import { detectTextLanguage } from './language.js';
|
|
23
24
|
import { SAFE_FETCH_INIT, checkedEndpoint } from './net.js';
|
|
25
|
+
/**
|
|
26
|
+
* The error band this estimator is published under, as a percentage.
|
|
27
|
+
*
|
|
28
|
+
* **Measured, not chosen.** It was `15` for eight releases — a design target
|
|
29
|
+
* nobody had checked — and the first run of `scripts/measure-token-band.mjs`
|
|
30
|
+
* against the official counting endpoint found two of eight samples outside it,
|
|
31
|
+
* both underestimating. Underestimating tokens means under-reporting cost, which
|
|
32
|
+
* is the flattering direction and the worst one for this tool.
|
|
33
|
+
*
|
|
34
|
+
* `15` is the measured worst case rounded up, by the same rule that briefly made
|
|
35
|
+
* it 25: the corpus now tops out at 11.2%, on Japanese. It landing back on the
|
|
36
|
+
* number that was a guess for eight releases is a coincidence and not a
|
|
37
|
+
* restoration — that 15 bounded nothing, and this one bounds eleven measured
|
|
38
|
+
* samples across four languages and six text types.
|
|
39
|
+
*
|
|
40
|
+
* **Read one caveat before trusting it.** Four of those samples are the Latin
|
|
41
|
+
* languages whose divisors in `DIVISOR_BY_LANGUAGE` were calibrated on one or two
|
|
42
|
+
* samples each, so their residuals are in-sample and optimistic by construction.
|
|
43
|
+
* The band is set by the seven samples nothing was fitted to — worst 11.2% — and
|
|
44
|
+
* the honest test of it is the next held-out sample in Spanish, French or German.
|
|
45
|
+
* The corpus grows one sample at a time now, so that test is cheap to run.
|
|
46
|
+
*
|
|
47
|
+
* **What got it there was not accents.** A Spanish sample with zero accented
|
|
48
|
+
* characters measured -22.9% against -22.1% for accented Spanish, which killed
|
|
49
|
+
* diacritics as a signal. Languages differ in how many tokens their words cost —
|
|
50
|
+
* English 3.44 characters per token, German 2.02 — so the estimator detects the
|
|
51
|
+
* language and divides accordingly. See `language.ts`.
|
|
52
|
+
*
|
|
53
|
+
* Exported so every report, README and tool description reads the same number.
|
|
54
|
+
* It was a literal in twenty-four files before this, with the only machine-
|
|
55
|
+
* readable copy in a test.
|
|
56
|
+
*/
|
|
57
|
+
export const ESTIMATE_ERROR_BAND_PCT = 15;
|
|
24
58
|
const CJK = /[-ヿ㐀-䶿一-鿿가-]/;
|
|
25
59
|
const LETTER = /[A-Za-zÀ-ɏͰ-ϿЀ-ӿ]/;
|
|
26
60
|
const DIGIT = /[0-9]/;
|
|
61
|
+
/**
|
|
62
|
+
* Characters per token, per language.
|
|
63
|
+
*
|
|
64
|
+
* Measured, one entry at a time, against `test/fixtures/token-ground-truth.json`.
|
|
65
|
+
* English keeps 4 because it measures +1.0% there; the others are lower because
|
|
66
|
+
* they are thinner in the merge table and cost more tokens for the same text.
|
|
67
|
+
*
|
|
68
|
+
* **Every one of these now has a held-out test.** Each language was calibrated on
|
|
69
|
+
* support prompts and then measured again on a code-review prompt — a different
|
|
70
|
+
* register, different vocabulary, different length — and the divisors held:
|
|
71
|
+
* English +1.0% then +0.4%, German -9.2% then -8.5%, French -1.2% then -5.8%,
|
|
72
|
+
* Spanish -6.2% then -9.7% under the previous values. That is the evidence that
|
|
73
|
+
* these fit a language rather than a template, and it is why the band can rest on
|
|
74
|
+
* them now.
|
|
75
|
+
*
|
|
76
|
+
* `en` stays at a round 4 rather than the 4.05 the search prefers: a hundredth of
|
|
77
|
+
* a divisor is precision the twenty-one samples cannot support, and it changes no
|
|
78
|
+
* estimate.
|
|
79
|
+
*
|
|
80
|
+
* A language absent from this table falls through to `DEFAULT_DIVISOR`, which is
|
|
81
|
+
* the English number and the behaviour this estimator has always had. Adding a
|
|
82
|
+
* language means measuring it, not guessing it.
|
|
83
|
+
*/
|
|
84
|
+
const DIVISOR_BY_LANGUAGE = {
|
|
85
|
+
en: 4,
|
|
86
|
+
es: 2.8,
|
|
87
|
+
fr: 3.05,
|
|
88
|
+
de: 2.25,
|
|
89
|
+
it: 2.8,
|
|
90
|
+
pt: 3.05,
|
|
91
|
+
nl: 2.65,
|
|
92
|
+
};
|
|
93
|
+
/**
|
|
94
|
+
* What a prompt gets when the language could not be told.
|
|
95
|
+
*
|
|
96
|
+
* The English value on purpose. An unknown-language prompt is most often English
|
|
97
|
+
* or code, and lowering this would inflate every estimate that the detector
|
|
98
|
+
* declined to classify — a change that helps nothing measured and hurts the two
|
|
99
|
+
* samples the estimator gets right.
|
|
100
|
+
*/
|
|
101
|
+
const DEFAULT_DIVISOR = 4;
|
|
27
102
|
/** Effective word length: non-ASCII characters split into more tokens. */
|
|
28
103
|
function effectiveLength(word) {
|
|
29
104
|
let len = 0;
|
|
@@ -46,6 +121,10 @@ function effectiveLength(word) {
|
|
|
46
121
|
export function estimateTokens(text) {
|
|
47
122
|
if (!text)
|
|
48
123
|
return 0;
|
|
124
|
+
// Detected once for the whole text, not per word. A prompt is one document; a
|
|
125
|
+
// per-word guess would be noise, and this runs on every call.
|
|
126
|
+
const language = detectTextLanguage(text);
|
|
127
|
+
const divisor = (language === null ? undefined : DIVISOR_BY_LANGUAGE[language]) ?? DEFAULT_DIVISOR;
|
|
49
128
|
let total = 0;
|
|
50
129
|
let i = 0;
|
|
51
130
|
const chars = Array.from(text);
|
|
@@ -79,7 +158,7 @@ export function estimateTokens(text) {
|
|
|
79
158
|
word += chars[i];
|
|
80
159
|
i++;
|
|
81
160
|
}
|
|
82
|
-
total += Math.max(1, Math.ceil(effectiveLength(word) /
|
|
161
|
+
total += Math.max(1, Math.ceil(effectiveLength(word) / divisor));
|
|
83
162
|
continue;
|
|
84
163
|
}
|
|
85
164
|
if (DIGIT.test(ch)) {
|
|
@@ -88,7 +167,20 @@ export function estimateTokens(text) {
|
|
|
88
167
|
n++;
|
|
89
168
|
i++;
|
|
90
169
|
}
|
|
91
|
-
|
|
170
|
+
/**
|
|
171
|
+
* 1.5 digits per token, not 3.
|
|
172
|
+
*
|
|
173
|
+
* The 3 was a guess and it was the single worst constant in this file:
|
|
174
|
+
* measured against the counting endpoint, the numeric-heavy sample came out
|
|
175
|
+
* **30.6% under** — by far the largest error in the corpus. Claude's
|
|
176
|
+
* tokenizer splits long digit runs far more finely than prose, because a
|
|
177
|
+
* merge table cannot cover every number.
|
|
178
|
+
*
|
|
179
|
+
* Corrected in isolation, which is why it can be trusted: changing only
|
|
180
|
+
* this takes that sample from -30.6% to -5.0% and moves nothing else more
|
|
181
|
+
* than four points. See `test/fixtures/token-ground-truth.json`.
|
|
182
|
+
*/
|
|
183
|
+
total += Math.ceil(n / 1.5);
|
|
92
184
|
continue;
|
|
93
185
|
}
|
|
94
186
|
// Outside the BMP (emoji, unusual symbols): usually 2+ tokens.
|
package/dist/tokenizer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE5D,MAAM,GAAG,GAAG,gBAAgB,CAAC;AAC7B,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB,0EAA0E;AAC1E,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,EAAE,IAAI,IAAI;QAAE,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7C,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBAC/C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,EAAE,CAAC;gBAC7E,IAAI,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC;gBAClB,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C,MAAM,GAAG,GAAG,gBAAgB,CAAC;AAC7B,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,mBAAmB,GAAqC;IAC5D,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;CACT,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,0EAA0E;AAC1E,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,EAAE,IAAI,IAAI;QAAE,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,8EAA8E;IAC9E,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,IAAI,eAAe,CAAC;IAEnG,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7C,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBAC/C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,EAAE,CAAC;gBAC7E,IAAI,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC;gBAClB,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACjD,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD;;;;;;;;;;;;eAYG;YACH,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,+DAA+D;QAC/D,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OACE,CAAC,GAAG,KAAK,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACtB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACpB,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;YAChB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,MAAM,EACnC,CAAC;YACD,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,8EAA8E;YAC9E,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAeD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAgC;IACnE,MAAM,EACJ,MAAM,EACN,KAAK,GAAG,eAAe,EACvB,OAAO,GAAG,2BAA2B,EACrC,SAAS,GAAG,KAAK,EACjB,aAAa,GAAG,KAAK,GACtB,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAE7F,OAAO,KAAK,EAAE,IAAY,EAAmB,EAAE;QAC7C,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,QAAQ,2BAA2B,EAAE;YAClE,GAAG,eAAe;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,YAAY;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6B,CAAC;QAC5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@trazum/core",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.9.0",
|
|
4
4
|
"description": "Trazum core: priced advisories for LLM prompts (caching, model tier, batching, schemas), plus deterministic trimming, token counting and pricing.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "David Mu\u00f1oz Rey",
|
package/src/advisories.ts
CHANGED
|
@@ -7,7 +7,7 @@ import type { PricingCatalogue } from './pricing.js';
|
|
|
7
7
|
import { formatUsd } from './savings.js';
|
|
8
8
|
import { analyzeExamples, findContradictions, findMovableSchema,
|
|
9
9
|
findRestatedFormat } from './structure.js';
|
|
10
|
-
import { estimateTokens } from './tokenizer.js';
|
|
10
|
+
import { ESTIMATE_ERROR_BAND_PCT, estimateTokens } from './tokenizer.js';
|
|
11
11
|
import type { Advisory, ModelPricing, TokenCounter, UsageProfile } from './types.js';
|
|
12
12
|
|
|
13
13
|
function countSignals(haystack: string, signals: readonly string[]): number {
|
|
@@ -207,6 +207,17 @@ export function buildAdvisories(
|
|
|
207
207
|
prefixTokens: cache.stablePrefixTokens,
|
|
208
208
|
totalTokens: tokensAfter,
|
|
209
209
|
mentionLowerMinimum: minTokens > 512,
|
|
210
|
+
/**
|
|
211
|
+
* Only when the number is an estimate and near the line.
|
|
212
|
+
*
|
|
213
|
+
* `count` defaults to `estimateTokens`; a caller who supplied their own
|
|
214
|
+
* counter — `--exact-tokens`, or the official endpoint — gets an
|
|
215
|
+
* authoritative number and no hedge, because hedging a measured figure
|
|
216
|
+
* is its own kind of dishonesty.
|
|
217
|
+
*/
|
|
218
|
+
couldReachMinimum:
|
|
219
|
+
count === estimateTokens &&
|
|
220
|
+
cache.stablePrefixTokens * (1 + ESTIMATE_ERROR_BAND_PCT / 100) >= minTokens,
|
|
210
221
|
}),
|
|
211
222
|
estimatedMonthlyUsd: null,
|
|
212
223
|
});
|
package/src/i18n/en.ts
CHANGED
|
@@ -132,6 +132,7 @@ export const en: CoreMessages = {
|
|
|
132
132
|
prefixTokens,
|
|
133
133
|
totalTokens,
|
|
134
134
|
mentionLowerMinimum,
|
|
135
|
+
couldReachMinimum,
|
|
135
136
|
}) => {
|
|
136
137
|
const reason = placeholder
|
|
137
138
|
? `here the first variable placeholder (${placeholder}) shows up at ~${n(prefixTokens)} tokens and only what precedes it can be cached`
|
|
@@ -142,6 +143,9 @@ export const en: CoreMessages = {
|
|
|
142
143
|
`${modelName} needs at least ${n(minTokens)} prefix tokens to cache; ${reason}. Setting cache_control will not error, it simply will not cache.` +
|
|
143
144
|
(mentionLowerMinimum
|
|
144
145
|
? ' Claude Opus 5 lowers that minimum to 512 tokens, so short prompts that miss here would cache there.'
|
|
146
|
+
: '') +
|
|
147
|
+
(couldReachMinimum
|
|
148
|
+
? ' That prefix count is an estimate and it is close to the line, so the real one may already be above it — check with --exact-tokens before deciding this is not available to you. The counting endpoint is free.'
|
|
145
149
|
: ''),
|
|
146
150
|
};
|
|
147
151
|
},
|
package/src/i18n/es.ts
CHANGED
|
@@ -132,6 +132,7 @@ export const es: CoreMessages = {
|
|
|
132
132
|
prefixTokens,
|
|
133
133
|
totalTokens,
|
|
134
134
|
mentionLowerMinimum,
|
|
135
|
+
couldReachMinimum,
|
|
135
136
|
}) => {
|
|
136
137
|
const reason = placeholder
|
|
137
138
|
? `aquí el primer marcador variable (${placeholder}) aparece a los ~${n(prefixTokens)} tokens y solo lo anterior puede cachearse`
|
|
@@ -142,6 +143,9 @@ export const es: CoreMessages = {
|
|
|
142
143
|
`${modelName} necesita al menos ${n(minTokens)} tokens de prefijo para cachear; ${reason}. Marcar cache_control no dará error, simplemente no cacheará.` +
|
|
143
144
|
(mentionLowerMinimum
|
|
144
145
|
? ' Claude Opus 5 baja ese mínimo a 512 tokens, así que prompts cortos que aquí no cachean, allí sí.'
|
|
146
|
+
: '') +
|
|
147
|
+
(couldReachMinimum
|
|
148
|
+
? ' Ese recuento del prefijo es una estimación y está cerca del límite, así que el real puede estar ya por encima: compruébalo con --exact-tokens antes de dar esto por perdido. El endpoint de conteo es gratis.'
|
|
145
149
|
: ''),
|
|
146
150
|
};
|
|
147
151
|
},
|
package/src/i18n/types.ts
CHANGED
|
@@ -88,6 +88,16 @@ export interface BelowCacheMinimumParams {
|
|
|
88
88
|
totalTokens: number;
|
|
89
89
|
/** Whether to point at Claude Opus 5's lower 512-token minimum. */
|
|
90
90
|
mentionLowerMinimum: boolean;
|
|
91
|
+
/**
|
|
92
|
+
* The prefix is *estimated* and close enough to the minimum that the real
|
|
93
|
+
* count could be above it.
|
|
94
|
+
*
|
|
95
|
+
* Without this the advisory asserts "caching will not work here" from a number
|
|
96
|
+
* measured to ±15%, and on a prefix near the threshold that is not an imprecise
|
|
97
|
+
* figure — it is wrong advice, and it costs the reader the largest saving
|
|
98
|
+
* Trazum offers.
|
|
99
|
+
*/
|
|
100
|
+
couldReachMinimum: boolean;
|
|
91
101
|
}
|
|
92
102
|
|
|
93
103
|
export interface CachePrefixReorderParams {
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from './types.js';
|
|
2
|
-
export { estimateTokens, countTokensAnthropic } from './tokenizer.js';
|
|
2
|
+
export { ESTIMATE_ERROR_BAND_PCT, estimateTokens, countTokensAnthropic } from './tokenizer.js';
|
|
3
|
+
export { DETECTABLE_LANGUAGES, detectTextLanguage } from './language.js';
|
|
3
4
|
export { countSentences, profilePrompt } from './profile.js';
|
|
4
5
|
export { PHRASE_LANGUAGES } from './phrases.js';
|
|
5
6
|
export { toPromptfoo } from './promptfoo.js';
|
package/src/language.ts
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which language a prompt is written in, to the extent it can be told cheaply.
|
|
3
|
+
*
|
|
4
|
+
* **Why the estimator needs this at all.** Measured against the counting
|
|
5
|
+
* endpoint, `estimateTokens` is accurate on English (+1.0%) and badly low on every
|
|
6
|
+
* other Latin language: German -37.3%, Spanish -22.9%, French -15.1%. Characters
|
|
7
|
+
* per token says why — English 3.44, French 2.66, Spanish 2.53, German 2.02 —
|
|
8
|
+
* while the estimator applied one divisor to all of them. Non-English text is
|
|
9
|
+
* thinner in the merge table, so the same number of characters costs more tokens.
|
|
10
|
+
*
|
|
11
|
+
* **Accents are not the signal.** That was the first hypothesis and it was tested
|
|
12
|
+
* and killed: a Spanish sample with zero accented characters comes out at -22.9%,
|
|
13
|
+
* against -22.1% for accented Spanish. Diacritics correlate with non-English text
|
|
14
|
+
* in a corpus and not in a prompt, and weighting them moved the figure by three
|
|
15
|
+
* points. What separates these languages is which words they are made of.
|
|
16
|
+
*
|
|
17
|
+
* So this counts function words — the shortest, commonest, most language-specific
|
|
18
|
+
* tokens there are. `the of and to` against `der die und ist` against
|
|
19
|
+
* `que de la en`.
|
|
20
|
+
*
|
|
21
|
+
* **It answers `null` when unsure, and that is the important part.** A prompt is
|
|
22
|
+
* not an essay: it can be three lines, or English instructions wrapped around a
|
|
23
|
+
* Spanish example, or a JSON schema with no prose at all. Guessing on those would
|
|
24
|
+
* apply a language's divisor to text that is not in that language, which is how a
|
|
25
|
+
* fix for one case becomes a regression for four. `null` means "use the default",
|
|
26
|
+
* and the default is the English-calibrated behaviour this has always had.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Function words per language, lower-cased, matched whole.
|
|
31
|
+
*
|
|
32
|
+
* Chosen for frequency and for *not overlapping*, which matters more: `de` is
|
|
33
|
+
* frequent in Spanish, French and Portuguese, so it earns nothing and is a
|
|
34
|
+
* liability. Each list is words that are common in its own language and rare in
|
|
35
|
+
* the others, which is what lets a small count decide.
|
|
36
|
+
*
|
|
37
|
+
* `en` is included even though it is the default, because the margin rule needs to
|
|
38
|
+
* know when English *won*: a Spanish prompt quoting an English sentence should not
|
|
39
|
+
* become Spanish on two hits.
|
|
40
|
+
*/
|
|
41
|
+
const FUNCTION_WORDS: Readonly<Record<string, readonly string[]>> = {
|
|
42
|
+
en: ['the', 'and', 'of', 'to', 'is', 'that', 'with', 'for', 'you', 'this', 'are', 'not'],
|
|
43
|
+
es: ['que', 'los', 'las', 'del', 'una', 'para', 'con', 'como', 'pero', 'sus', 'este', 'siempre'],
|
|
44
|
+
fr: ['les', 'des', 'une', 'dans', 'pour', 'avec', 'vous', 'est', 'sur', 'aux', 'cette', 'toujours'],
|
|
45
|
+
de: ['der', 'die', 'das', 'und', 'nicht', 'sie', 'ist', 'mit', 'auf', 'einen', 'einer', 'immer'],
|
|
46
|
+
pt: ['que', 'dos', 'das', 'uma', 'para', 'com', 'como', 'não', 'seu', 'este', 'pelo', 'sempre'],
|
|
47
|
+
/**
|
|
48
|
+
* Rebuilt once, because half of it was Spanish. `per con del una sempre` are as
|
|
49
|
+
* common in Spanish as in Italian, so they earned nothing and cost the margin
|
|
50
|
+
* rule its answer: an Italian code-review prompt scored a tie and came back
|
|
51
|
+
* `null`, fell through to the English divisor, and measured -21.9%. These are
|
|
52
|
+
* words Italian has and Spanish does not.
|
|
53
|
+
*/
|
|
54
|
+
it: ['il', 'della', 'nella', 'nel', 'che', 'quando', 'senza', 'gli', 'delle', 'degli', 'più', 'anche'],
|
|
55
|
+
nl: ['het', 'een', 'van', 'niet', 'zijn', 'met', 'voor', 'dat', 'aan', 'deze', 'wordt', 'altijd'],
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/** Languages this can name. Exported so callers can key a table off it. */
|
|
59
|
+
export const DETECTABLE_LANGUAGES: readonly string[] = Object.keys(FUNCTION_WORDS);
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* How many hits the winner needs, absolutely and relative to the runner-up.
|
|
63
|
+
*
|
|
64
|
+
* Both bars exist for the same reason and neither is enough alone. The absolute
|
|
65
|
+
* one stops a three-line prompt being classified on a single word. The ratio stops
|
|
66
|
+
* a document that is genuinely mixed — English instructions around a Spanish
|
|
67
|
+
* example — from being called whichever language happened to appear once more.
|
|
68
|
+
*
|
|
69
|
+
* Deliberately cautious in the direction of `null`. A wrong language costs
|
|
70
|
+
* accuracy on text that was fine before; `null` costs only the improvement.
|
|
71
|
+
*/
|
|
72
|
+
const MIN_HITS = 4;
|
|
73
|
+
const MIN_RATIO = 1.6;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The language of `text`, or `null` when no answer is safe.
|
|
77
|
+
*
|
|
78
|
+
* Case-insensitive, whole words only, and it stops reading after a bounded
|
|
79
|
+
* prefix: a prompt can be a megabyte, the answer does not get better after a few
|
|
80
|
+
* thousand words, and this runs inside `estimateTokens` on every call.
|
|
81
|
+
*/
|
|
82
|
+
export function detectTextLanguage(text: string): string | null {
|
|
83
|
+
if (!text) return null;
|
|
84
|
+
|
|
85
|
+
// Bounded so this stays cheap on a large prompt. Function words are frequent
|
|
86
|
+
// enough that a prefix this size decides anything a whole document would.
|
|
87
|
+
const sample = text.length > 20_000 ? text.slice(0, 20_000) : text;
|
|
88
|
+
|
|
89
|
+
const words = sample.toLowerCase().match(/[\p{L}]+/gu);
|
|
90
|
+
if (!words || words.length < MIN_HITS * 2) return null;
|
|
91
|
+
|
|
92
|
+
const seen = new Set(words);
|
|
93
|
+
const scores: Array<[string, number]> = [];
|
|
94
|
+
for (const [language, list] of Object.entries(FUNCTION_WORDS)) {
|
|
95
|
+
let hits = 0;
|
|
96
|
+
// Distinct words rather than occurrences: a prompt that repeats "the" forty
|
|
97
|
+
// times is not more English than one that uses forty different English words,
|
|
98
|
+
// and counting occurrences lets one hammered word decide.
|
|
99
|
+
for (const word of list) if (seen.has(word)) hits++;
|
|
100
|
+
scores.push([language, hits]);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
scores.sort((a, b) => b[1] - a[1]);
|
|
104
|
+
const [best, bestHits] = scores[0]!;
|
|
105
|
+
const runnerUp = scores[1]?.[1] ?? 0;
|
|
106
|
+
|
|
107
|
+
if (bestHits < MIN_HITS) return null;
|
|
108
|
+
// A runner-up of zero is a clear win; guard the division rather than special-case.
|
|
109
|
+
if (runnerUp > 0 && bestHits / runnerUp < MIN_RATIO) return null;
|
|
110
|
+
|
|
111
|
+
return best;
|
|
112
|
+
}
|
package/src/tokenizer.ts
CHANGED
|
@@ -21,12 +21,90 @@
|
|
|
21
21
|
* endpoint, which is free) or pass your own `TokenCounter`.
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
+
import { detectTextLanguage } from './language.js';
|
|
24
25
|
import { SAFE_FETCH_INIT, checkedEndpoint } from './net.js';
|
|
25
26
|
|
|
27
|
+
/**
|
|
28
|
+
* The error band this estimator is published under, as a percentage.
|
|
29
|
+
*
|
|
30
|
+
* **Measured, not chosen.** It was `15` for eight releases — a design target
|
|
31
|
+
* nobody had checked — and the first run of `scripts/measure-token-band.mjs`
|
|
32
|
+
* against the official counting endpoint found two of eight samples outside it,
|
|
33
|
+
* both underestimating. Underestimating tokens means under-reporting cost, which
|
|
34
|
+
* is the flattering direction and the worst one for this tool.
|
|
35
|
+
*
|
|
36
|
+
* `15` is the measured worst case rounded up, by the same rule that briefly made
|
|
37
|
+
* it 25: the corpus now tops out at 11.2%, on Japanese. It landing back on the
|
|
38
|
+
* number that was a guess for eight releases is a coincidence and not a
|
|
39
|
+
* restoration — that 15 bounded nothing, and this one bounds eleven measured
|
|
40
|
+
* samples across four languages and six text types.
|
|
41
|
+
*
|
|
42
|
+
* **Read one caveat before trusting it.** Four of those samples are the Latin
|
|
43
|
+
* languages whose divisors in `DIVISOR_BY_LANGUAGE` were calibrated on one or two
|
|
44
|
+
* samples each, so their residuals are in-sample and optimistic by construction.
|
|
45
|
+
* The band is set by the seven samples nothing was fitted to — worst 11.2% — and
|
|
46
|
+
* the honest test of it is the next held-out sample in Spanish, French or German.
|
|
47
|
+
* The corpus grows one sample at a time now, so that test is cheap to run.
|
|
48
|
+
*
|
|
49
|
+
* **What got it there was not accents.** A Spanish sample with zero accented
|
|
50
|
+
* characters measured -22.9% against -22.1% for accented Spanish, which killed
|
|
51
|
+
* diacritics as a signal. Languages differ in how many tokens their words cost —
|
|
52
|
+
* English 3.44 characters per token, German 2.02 — so the estimator detects the
|
|
53
|
+
* language and divides accordingly. See `language.ts`.
|
|
54
|
+
*
|
|
55
|
+
* Exported so every report, README and tool description reads the same number.
|
|
56
|
+
* It was a literal in twenty-four files before this, with the only machine-
|
|
57
|
+
* readable copy in a test.
|
|
58
|
+
*/
|
|
59
|
+
export const ESTIMATE_ERROR_BAND_PCT = 15;
|
|
60
|
+
|
|
26
61
|
const CJK = /[-ヿ㐀-䶿一-鿿가-]/;
|
|
27
62
|
const LETTER = /[A-Za-zÀ-ɏͰ-ϿЀ-ӿ]/;
|
|
28
63
|
const DIGIT = /[0-9]/;
|
|
29
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Characters per token, per language.
|
|
67
|
+
*
|
|
68
|
+
* Measured, one entry at a time, against `test/fixtures/token-ground-truth.json`.
|
|
69
|
+
* English keeps 4 because it measures +1.0% there; the others are lower because
|
|
70
|
+
* they are thinner in the merge table and cost more tokens for the same text.
|
|
71
|
+
*
|
|
72
|
+
* **Every one of these now has a held-out test.** Each language was calibrated on
|
|
73
|
+
* support prompts and then measured again on a code-review prompt — a different
|
|
74
|
+
* register, different vocabulary, different length — and the divisors held:
|
|
75
|
+
* English +1.0% then +0.4%, German -9.2% then -8.5%, French -1.2% then -5.8%,
|
|
76
|
+
* Spanish -6.2% then -9.7% under the previous values. That is the evidence that
|
|
77
|
+
* these fit a language rather than a template, and it is why the band can rest on
|
|
78
|
+
* them now.
|
|
79
|
+
*
|
|
80
|
+
* `en` stays at a round 4 rather than the 4.05 the search prefers: a hundredth of
|
|
81
|
+
* a divisor is precision the twenty-one samples cannot support, and it changes no
|
|
82
|
+
* estimate.
|
|
83
|
+
*
|
|
84
|
+
* A language absent from this table falls through to `DEFAULT_DIVISOR`, which is
|
|
85
|
+
* the English number and the behaviour this estimator has always had. Adding a
|
|
86
|
+
* language means measuring it, not guessing it.
|
|
87
|
+
*/
|
|
88
|
+
const DIVISOR_BY_LANGUAGE: Readonly<Record<string, number>> = {
|
|
89
|
+
en: 4,
|
|
90
|
+
es: 2.8,
|
|
91
|
+
fr: 3.05,
|
|
92
|
+
de: 2.25,
|
|
93
|
+
it: 2.8,
|
|
94
|
+
pt: 3.05,
|
|
95
|
+
nl: 2.65,
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* What a prompt gets when the language could not be told.
|
|
100
|
+
*
|
|
101
|
+
* The English value on purpose. An unknown-language prompt is most often English
|
|
102
|
+
* or code, and lowering this would inflate every estimate that the detector
|
|
103
|
+
* declined to classify — a change that helps nothing measured and hurts the two
|
|
104
|
+
* samples the estimator gets right.
|
|
105
|
+
*/
|
|
106
|
+
const DEFAULT_DIVISOR = 4;
|
|
107
|
+
|
|
30
108
|
/** Effective word length: non-ASCII characters split into more tokens. */
|
|
31
109
|
function effectiveLength(word: string): number {
|
|
32
110
|
let len = 0;
|
|
@@ -49,6 +127,11 @@ function effectiveLength(word: string): number {
|
|
|
49
127
|
export function estimateTokens(text: string): number {
|
|
50
128
|
if (!text) return 0;
|
|
51
129
|
|
|
130
|
+
// Detected once for the whole text, not per word. A prompt is one document; a
|
|
131
|
+
// per-word guess would be noise, and this runs on every call.
|
|
132
|
+
const language = detectTextLanguage(text);
|
|
133
|
+
const divisor = (language === null ? undefined : DIVISOR_BY_LANGUAGE[language]) ?? DEFAULT_DIVISOR;
|
|
134
|
+
|
|
52
135
|
let total = 0;
|
|
53
136
|
let i = 0;
|
|
54
137
|
const chars = Array.from(text);
|
|
@@ -87,7 +170,7 @@ export function estimateTokens(text: string): number {
|
|
|
87
170
|
word += chars[i]!;
|
|
88
171
|
i++;
|
|
89
172
|
}
|
|
90
|
-
total += Math.max(1, Math.ceil(effectiveLength(word) /
|
|
173
|
+
total += Math.max(1, Math.ceil(effectiveLength(word) / divisor));
|
|
91
174
|
continue;
|
|
92
175
|
}
|
|
93
176
|
|
|
@@ -97,7 +180,20 @@ export function estimateTokens(text: string): number {
|
|
|
97
180
|
n++;
|
|
98
181
|
i++;
|
|
99
182
|
}
|
|
100
|
-
|
|
183
|
+
/**
|
|
184
|
+
* 1.5 digits per token, not 3.
|
|
185
|
+
*
|
|
186
|
+
* The 3 was a guess and it was the single worst constant in this file:
|
|
187
|
+
* measured against the counting endpoint, the numeric-heavy sample came out
|
|
188
|
+
* **30.6% under** — by far the largest error in the corpus. Claude's
|
|
189
|
+
* tokenizer splits long digit runs far more finely than prose, because a
|
|
190
|
+
* merge table cannot cover every number.
|
|
191
|
+
*
|
|
192
|
+
* Corrected in isolation, which is why it can be trusted: changing only
|
|
193
|
+
* this takes that sample from -30.6% to -5.0% and moves nothing else more
|
|
194
|
+
* than four points. See `test/fixtures/token-ground-truth.json`.
|
|
195
|
+
*/
|
|
196
|
+
total += Math.ceil(n / 1.5);
|
|
101
197
|
continue;
|
|
102
198
|
}
|
|
103
199
|
|