@trazum/core 1.83.0 → 1.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/band.d.ts.map +1 -1
- package/dist/band.js +12 -0
- package/dist/band.js.map +1 -1
- package/dist/optimize.d.ts +2 -2
- package/dist/optimize.d.ts.map +1 -1
- package/dist/optimize.js +15 -1
- package/dist/optimize.js.map +1 -1
- package/dist/tokenizer.d.ts +4 -2
- package/dist/tokenizer.d.ts.map +1 -1
- package/dist/tokenizer.js +4 -2
- package/dist/tokenizer.js.map +1 -1
- package/dist/types.d.ts +40 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/band.ts +12 -0
- package/src/optimize.ts +16 -0
- package/src/tokenizer.ts +4 -2
- package/src/types.ts +41 -0
package/dist/band.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"band.d.ts","sourceRoot":"","sources":["../src/band.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH;;;;;;;;GAQG;AACH,eAAO,MAAM,KAAK;IAChB,8EAA8E;aAC9E,OAAO,EAAE,EAAE;IACX,sEAAsE;aACtE,QAAQ,EAAE,EAAE;IACZ;;;;;;;;OAQG;aACH,GAAG,EAAE,CAAC;IACN;;;;;;;OAOG;aACH,KAAK,EAAE,CAAC;CACA,CAAC;AAEX,MAAM,MAAM,UAAU,GAAG,MAAM,OAAO,KAAK,CAAC;AAwC5C,uFAAuF;AACvF,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CA0BlD;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE5C;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAEvE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,0BAA0B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"band.d.ts","sourceRoot":"","sources":["../src/band.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH;;;;;;;;GAQG;AACH,eAAO,MAAM,KAAK;IAChB,8EAA8E;aAC9E,OAAO,EAAE,EAAE;IACX,sEAAsE;aACtE,QAAQ,EAAE,EAAE;IACZ;;;;;;;;OAQG;aACH,GAAG,EAAE,CAAC;IACN;;;;;;;OAOG;aACH,KAAK,EAAE,CAAC;CACA,CAAC;AAEX,MAAM,MAAM,UAAU,GAAG,MAAM,OAAO,KAAK,CAAC;AAwC5C,uFAAuF;AACvF,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CA0BlD;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE5C;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAEvE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,0BAA0B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAiBvE,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAG3E"}
|
package/dist/band.js
CHANGED
|
@@ -194,6 +194,18 @@ export const MEASURED_FOREIGN_ERROR_PCT = {
|
|
|
194
194
|
deepseek: 94.5,
|
|
195
195
|
/** Worst on `german-technical`, where even Latin prose is more than twice out. */
|
|
196
196
|
mistral: 103.1,
|
|
197
|
+
/**
|
|
198
|
+
* Worst on `german-prose`, and the worst of the four.
|
|
199
|
+
*
|
|
200
|
+
* The measurement sat in this repository unclaimed for a release: the
|
|
201
|
+
* fixture was written by a real key against `gpt-5`, and this file went on
|
|
202
|
+
* answering `null`, so every report told a GPT user nobody had looked. The
|
|
203
|
+
* figure is not a surprise -- `o200k_base` packs Latin prose far more densely
|
|
204
|
+
* than the estimator's Claude-calibrated divisors expect -- but a report
|
|
205
|
+
* saying *unmeasured* where the answer is *more than twice out* is the
|
|
206
|
+
* flattering direction, and this whole file exists to refuse that.
|
|
207
|
+
*/
|
|
208
|
+
openai: 112.4,
|
|
197
209
|
};
|
|
198
210
|
/**
|
|
199
211
|
* How far out the estimator has been measured against this provider, or null.
|
package/dist/band.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"band.js","sourceRoot":"","sources":["../src/band.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,8EAA8E;IAC9E,OAAO,EAAE,EAAE;IACX,sEAAsE;IACtE,QAAQ,EAAE,EAAE;IACZ;;;;;;;;OAQG;IACH,GAAG,EAAE,CAAC;IACN;;;;;;;OAOG;IACH,KAAK,EAAE,CAAC;CACA,CAAC;AAIX;;;;;;;;;;;;;;;GAeG;AACH,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B;;;;;;;;;;;;;EAaE;AACF,MAAM,GAAG,GAAG,qEAAqE,CAAC;AAClF,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB,uFAAuF;AACvF,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAEvC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,GAAG,IAAI,CAAC,CAAC;aACtB,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,CAAC,CAAC;aAChC,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC;IAE3C;;;;;;MAME;IACF,IAAI,GAAG,GAAG,CAAC,IAAI,SAAS;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,MAAM,GAAG,CAAC,IAAI,WAAW;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,OAAO,GAAG,CAAC,IAAI,YAAY;QAAE,OAAO,UAAU,CAAC;IACnD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAuB;IACtD,OAAO,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAqC;IAC1E,sFAAsF;IACtF,QAAQ,EAAE,IAAI;IACd,kFAAkF;IAClF,OAAO,EAAE,KAAK;
|
|
1
|
+
{"version":3,"file":"band.js","sourceRoot":"","sources":["../src/band.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,8EAA8E;IAC9E,OAAO,EAAE,EAAE;IACX,sEAAsE;IACtE,QAAQ,EAAE,EAAE;IACZ;;;;;;;;OAQG;IACH,GAAG,EAAE,CAAC;IACN;;;;;;;OAOG;IACH,KAAK,EAAE,CAAC;CACA,CAAC;AAIX;;;;;;;;;;;;;;;GAeG;AACH,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B;;;;;;;;;;;;;EAaE;AACF,MAAM,GAAG,GAAG,qEAAqE,CAAC;AAClF,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB,uFAAuF;AACvF,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAEvC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,GAAG,IAAI,CAAC,CAAC;aACtB,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,CAAC,CAAC;aAChC,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC;IAE3C;;;;;;MAME;IACF,IAAI,GAAG,GAAG,CAAC,IAAI,SAAS;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,MAAM,GAAG,CAAC,IAAI,WAAW;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,OAAO,GAAG,CAAC,IAAI,YAAY;QAAE,OAAO,UAAU,CAAC;IACnD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAuB;IACtD,OAAO,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAqC;IAC1E,sFAAsF;IACtF,QAAQ,EAAE,IAAI;IACd,kFAAkF;IAClF,OAAO,EAAE,KAAK;IACd;;;;;;;;;;OAUG;IACH,MAAM,EAAE,KAAK;CACd,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAuB;IAC1D,IAAI,QAAQ,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACnC,OAAO,0BAA0B,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;AACtD,CAAC"}
|
package/dist/optimize.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { PricingCatalogue } from './pricing.js';
|
|
2
2
|
import { join, segment } from './segment.js';
|
|
3
3
|
import type { AsyncTokenCounter } from './tokenizer.js';
|
|
4
|
-
import type { OptimizationResult, OptimizeOptions, Segment, UsageProfile } from './types.js';
|
|
4
|
+
import type { OptimizationResult, OptimizeOptions, Segment, TokenProvenance, UsageProfile } from './types.js';
|
|
5
5
|
export declare const DEFAULT_USAGE: UsageProfile;
|
|
6
6
|
interface Masked {
|
|
7
7
|
text: string;
|
|
@@ -32,7 +32,7 @@ export declare function optimize(prompt: string, options?: OptimizeOptions): Opt
|
|
|
32
32
|
* attribution stays on the heuristic estimate: asking a remote counter once
|
|
33
33
|
* per rule would multiply the calls without changing any decision.
|
|
34
34
|
*/
|
|
35
|
-
export declare function withExactTokenCounts(result: OptimizationResult, counter: AsyncTokenCounter, pricing?: PricingCatalogue): Promise<OptimizationResult>;
|
|
35
|
+
export declare function withExactTokenCounts(result: OptimizationResult, counter: AsyncTokenCounter, pricing?: PricingCatalogue, countedBy?: TokenProvenance | null): Promise<OptimizationResult>;
|
|
36
36
|
/** Re-exports of internals useful to anyone extending the library. */
|
|
37
37
|
export { join, mask as maskProtected, segment, unmask as unmaskProtected };
|
|
38
38
|
//# sourceMappingURL=optimize.d.ts.map
|
package/dist/optimize.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"optimize.d.ts","sourceRoot":"","sources":["../src/optimize.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EACV,kBAAkB,EAClB,eAAe,EAEf,OAAO,EAEP,YAAY,EACb,MAAM,YAAY,CAAC;AAQpB,eAAO,MAAM,aAAa,EAAE,YAM3B,CAAC;AAEF,UAAU,MAAM;IACd,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;GAIG;AACH,iBAAS,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,CAqBzC;AAED,uCAAuC;AACvC,iBAAS,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CASrD;AAOD;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,kBAAkB,
|
|
1
|
+
{"version":3,"file":"optimize.d.ts","sourceRoot":"","sources":["../src/optimize.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EACV,kBAAkB,EAClB,eAAe,EAEf,OAAO,EAEP,eAAe,EACf,YAAY,EACb,MAAM,YAAY,CAAC;AAQpB,eAAO,MAAM,aAAa,EAAE,YAM3B,CAAC;AAEF,UAAU,MAAM;IACd,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;GAIG;AACH,iBAAS,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,CAqBzC;AAED,uCAAuC;AACvC,iBAAS,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CASrD;AAOD;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,kBAAkB,CAqI1F;AAED;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,iBAAiB,EAC1B,OAAO,GAAE,gBAAoC,EAC7C,SAAS,GAAE,eAAe,GAAG,IAAW,GACvC,OAAO,CAAC,kBAAkB,CAAC,CAiD7B;AAED,sEAAsE;AACtE,OAAO,EAAE,IAAI,EAAE,IAAI,IAAI,aAAa,EAAE,OAAO,EAAE,MAAM,IAAI,eAAe,EAAE,CAAC"}
|
package/dist/optimize.js
CHANGED
|
@@ -173,6 +173,14 @@ export function optimize(prompt, options = {}) {
|
|
|
173
173
|
usage,
|
|
174
174
|
locale,
|
|
175
175
|
tokenSource: options.tokenCounter ? 'external' : 'heuristic',
|
|
176
|
+
/*
|
|
177
|
+
* Null even when a counter was supplied here, and that is not an oversight.
|
|
178
|
+
* `optimize` takes a bare function: it can see that something other than
|
|
179
|
+
* the heuristic ran and cannot see whose it was. Naming one would be an
|
|
180
|
+
* invention, and `withExactTokenCounts` is where a caller that does know
|
|
181
|
+
* says so.
|
|
182
|
+
*/
|
|
183
|
+
countedBy: null,
|
|
176
184
|
pricingSource: {
|
|
177
185
|
lastReviewed: pricing.lastReviewed,
|
|
178
186
|
overriddenModels: pricing.overriddenModels,
|
|
@@ -188,7 +196,7 @@ export function optimize(prompt, options = {}) {
|
|
|
188
196
|
* attribution stays on the heuristic estimate: asking a remote counter once
|
|
189
197
|
* per rule would multiply the calls without changing any decision.
|
|
190
198
|
*/
|
|
191
|
-
export async function withExactTokenCounts(result, counter, pricing = BUNDLED_CATALOGUE) {
|
|
199
|
+
export async function withExactTokenCounts(result, counter, pricing = BUNDLED_CATALOGUE, countedBy = null) {
|
|
192
200
|
// A result priced against an overlay cannot be recomputed against the bundled
|
|
193
201
|
// catalogue: the token counts would come from one source and the money from
|
|
194
202
|
// another, and the report would disagree with itself with nothing to show why.
|
|
@@ -219,6 +227,12 @@ export async function withExactTokenCounts(result, counter, pricing = BUNDLED_CA
|
|
|
219
227
|
savings,
|
|
220
228
|
advisories,
|
|
221
229
|
tokenSource: 'external',
|
|
230
|
+
/*
|
|
231
|
+
* Optional and defaulting to null rather than to a guess. A caller that
|
|
232
|
+
* knows whose counter it passed says so; one that does not gets `external`
|
|
233
|
+
* with no claim attached, which is exactly as much as this function knows.
|
|
234
|
+
*/
|
|
235
|
+
countedBy,
|
|
222
236
|
pricingSource: {
|
|
223
237
|
lastReviewed: pricing.lastReviewed,
|
|
224
238
|
overriddenModels: pricing.overriddenModels,
|
package/dist/optimize.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"optimize.js","sourceRoot":"","sources":["../src/optimize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAEhE,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"optimize.js","sourceRoot":"","sources":["../src/optimize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAEhE,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAWzC,MAAM,SAAS,GAAG,QAAQ,CAAC;AAC3B,MAAM,UAAU,GAAG,QAAQ,CAAC;AAC5B,MAAM,SAAS,GAAG,MAAM,CAAC;AACzB,kDAAkD;AAClD,MAAM,aAAa,GAAG,MAAM,GAAG,SAAS,CAAC;AAEzC,MAAM,CAAC,MAAM,aAAa,GAAiB;IACzC,KAAK,EAAE,aAAa;IACpB,aAAa,EAAE,IAAI;IACnB,eAAe,EAAE,GAAG;IACpB,YAAY,EAAE,GAAG;IACjB,aAAa,EAAE,KAAK;CACrB,CAAC;AAQF;;;;GAIG;AACH,SAAS,IAAI,CAAC,QAAmB;IAC/B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC;YACjB,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;YAClC,wEAAwE;YACxE,4BAA4B;YAC5B,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC;YACjB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,IAAI,SAAS,GAAG,IAAI,GAAG,UAAU,CAAC;IACxC,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACzB,CAAC;AAED,uCAAuC;AACvC,SAAS,MAAM,CAAC,IAAY,EAAE,KAAe;IAC3C,OAAO,IAAI,CAAC,OAAO,CACjB,IAAI,MAAM,CAAC,GAAG,SAAS,aAAa,UAAU,EAAE,EAAE,GAAG,CAAC,EACtD,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;QAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,OAAO,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IACnD,CAAC,CACF,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,SAAS,iBAAiB,CAAC,QAAmB;IAC5C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACzF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAc,EAAE,OAAO,GAAoB,EAAE;IACpE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC;IACtC;;;;;;;;;;MAUE;IACF,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CACb,wBAAwB,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CACtF,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAW,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC;IACxD,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAiB,OAAO,CAAC,YAAY,IAAI,cAAc,CAAC;IAEnE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAEvD,IAAI,OAAO,GAAG,cAAc,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE7C;;;;;;;;;;;;;;;MAeE;IACF,MAAM,kBAAkB,GAAG,CAAC,CAAC;IAC7B,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,kBAAkB,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,WAAW,GAAG,OAAO,CAAC;QAE5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAAE,SAAS;YACpC,IAAI,IAAI,CAAC,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,YAAY;gBAAE,SAAS;YAEpE,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC;YACtC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,IAAI,IAAI,KAAK,CAAC,IAAI,SAAS,KAAK,OAAO;gBAAE,SAAS;YAElD,MAAM,iBAAiB,GAAG,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAEnD,uEAAuE;YACvE,iDAAiD;YACjD,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YACpF,IAAI,aAAa;gBAAE,SAAS;YAE5B,MAAM,KAAK,GAAG,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACvC,yEAAyE;YACzE,oEAAoE;YACpE,MAAM,OAAO,GAAG,cAAc,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;YACnE,OAAO,GAAG,SAAS,CAAC;YACpB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE;oBAClB,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,IAAI;oBACJ,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;oBACxC,OAAO;iBACR,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,qEAAqE;gBACrE,wEAAwE;gBACxE,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC;gBACtB,QAAQ,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC;gBACpD,QAAQ,CAAC,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,IAAI,OAAO,KAAK,WAAW;YAAE,MAAM;IACrC,CAAC;IAED,MAAM,WAAW,GAAiB,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAEvD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IAErC,MAAM,KAAK,GAAiB,EAAE,GAAG,aAAa,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IACnE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,iBAAiB,CAAC;IACrD,MAAM,OAAO,GAAG,cAAc,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;IACtF,MAAM,UAAU,GAAG,eAAe,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAE9F,OAAO;QACL,QAAQ,EAAE,MAAM;QAChB,SAAS;QACT,YAAY;QACZ,WAAW;QACX,WAAW,EAAE,YAAY,GAAG,WAAW;QACvC,YAAY,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACxF,KAAK,EAAE,WAAW;QAClB,UAAU;QACV,OAAO;QACP,KAAK;QACL,MAAM;QACN,WAAW,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW;QAC5D;;;;;;WAMG;QACH,SAAS,EAAE,IAAI;QACf,aAAa,EAAE;YACb,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAA0B,EAC1B,OAA0B,EAC1B,OAAO,GAAqB,iBAAiB,EAC7C,SAAS,GAA2B,IAAI;IAExC,8EAA8E;IAC9E,4EAA4E;IAC5E,+EAA+E;IAC/E,+EAA+E;IAC/E,mDAAmD;IACnD,MAAM,QAAQ,GACZ,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAChD,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9C,IAAI,QAAQ,IAAI,OAAO,KAAK,iBAAiB,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,4EAA4E;YAC1E,6EAA6E;YAC7E,wEAAwE,CAC3E,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;KAC1B,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,cAAc,CAAC,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;IAC7F,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,EAAE;QAC9E,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO;KACR,CAAC,CAAC;IAEH,OAAO;QACL,GAAG,MAAM;QACT,YAAY;QACZ,WAAW;QACX,WAAW,EAAE,YAAY,GAAG,WAAW;QACvC,YAAY,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACxF,OAAO;QACP,UAAU;QACV,WAAW,EAAE,UAAU;QACvB;;;;WAIG;QACH,SAAS;QACT,aAAa,EAAE;YACb,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC;KACF,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,OAAO,EAAE,IAAI,EAAE,IAAI,IAAI,aAAa,EAAE,OAAO,EAAE,MAAM,IAAI,eAAe,EAAE,CAAC"}
|
package/dist/tokenizer.d.ts
CHANGED
|
@@ -61,8 +61,10 @@ export declare const ESTIMATE_ERROR_BAND_PCT = 33;
|
|
|
61
61
|
* estimator **against Claude**. Forty-seven samples, one counting endpoint, one
|
|
62
62
|
* family. Trazum prices seven, and the two that have since been run are far
|
|
63
63
|
* outside anything published here: DeepSeek 94.5% at worst, Mistral 103.1%. On
|
|
64
|
-
* a
|
|
65
|
-
*
|
|
64
|
+
* a Gemini prompt nobody has measured it at all, and `measuredForeignError`
|
|
65
|
+
* answers null rather than the nearest number. On a GPT prompt somebody has:
|
|
66
|
+
* 112.4%, the worst of the four measured, and the figure this file reported as
|
|
67
|
+
* unmeasured for a release while its own fixture held the answer.
|
|
66
68
|
*
|
|
67
69
|
* Exported as a name rather than left implicit in a comment because the band
|
|
68
70
|
* had already leaked out of its domain in three places: an advisory that told a
|
package/dist/tokenizer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAKH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAE1C
|
|
1
|
+
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAKH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAE1C;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,wBAAwB,cAAc,CAAC;AAEpD;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,OAAO,CAExE;AA+HD;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAiJnD;AAED,sDAAsD;AACtD,MAAM,MAAM,iBAAiB,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,gFAAgF;IAChF,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,uBAAuB,GAAG,iBAAiB,CA6BxF"}
|
package/dist/tokenizer.js
CHANGED
|
@@ -63,8 +63,10 @@ export const ESTIMATE_ERROR_BAND_PCT = 33;
|
|
|
63
63
|
* estimator **against Claude**. Forty-seven samples, one counting endpoint, one
|
|
64
64
|
* family. Trazum prices seven, and the two that have since been run are far
|
|
65
65
|
* outside anything published here: DeepSeek 94.5% at worst, Mistral 103.1%. On
|
|
66
|
-
* a
|
|
67
|
-
*
|
|
66
|
+
* a Gemini prompt nobody has measured it at all, and `measuredForeignError`
|
|
67
|
+
* answers null rather than the nearest number. On a GPT prompt somebody has:
|
|
68
|
+
* 112.4%, the worst of the four measured, and the figure this file reported as
|
|
69
|
+
* unmeasured for a release while its own fixture held the answer.
|
|
68
70
|
*
|
|
69
71
|
* Exported as a name rather than left implicit in a comment because the band
|
|
70
72
|
* had already leaked out of its domain in three places: an advisory that told a
|
package/dist/tokenizer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC;AAEpD;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,QAAmC;IAC7D,OAAO,QAAQ,KAAK,wBAAwB,CAAC;AAC/C,CAAC;AAED,MAAM,GAAG,GAAG,gBAAgB,CAAC;AAE7B;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,IAAI,GAAG,OAAO,CAAC;AACrB,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,mBAAmB,GAAqC;IAC5D,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;CACT,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,MAAM,MAAM,GAAG,OAAO,CAAC;AAEvB;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,0EAA0E;AAC1E,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,EAAE,IAAI,IAAI;QAAE,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,8EAA8E;IAC9E,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,IAAI,eAAe,CAAC;IAEnG,IAAI,KAAK,GAAG,CAAC,CAAC;IACd;;;OAGG;IACH,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7C,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB;;;;;;;;eAQG;YACH,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBAC/C,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;oBAC/B,CAAC,CAAC,oBAAoB;oBACtB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;wBACtB,CAAC,CAAC,sBAAsB;wBACxB,CAAC,CAAC,mBAAmB,CAAC;gBAC1B,CAAC,EAAE,CAAC;YACN,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,EAAE,CAAC;gBAC7E,IAAI,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC;gBAClB,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACjD,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD;;;;;;;;;;;;eAYG;YACH,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,+DAA+D;QAC/D,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OACE,CAAC,GAAG,KAAK,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACtB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACpB,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;YAChB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,MAAM,EACnC,CAAC;YACD,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,8EAA8E;YAC9E,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,CAAC;YACN;;;;;;;;;;;;;;;;;;;;;eAqBG;YACH,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,mFAAmF;IACnF,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACtC,CAAC;AAeD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAgC;IACnE,MAAM,EACJ,MAAM,EACN,KAAK,GAAG,eAAe,EACvB,OAAO,GAAG,2BAA2B,EACrC,SAAS,GAAG,KAAK,EACjB,aAAa,GAAG,KAAK,GACtB,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAE7F,OAAO,KAAK,EAAE,IAAY,EAAmB,EAAE;QAC7C,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,QAAQ,2BAA2B,EAAE;YAClE,GAAG,eAAe;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,YAAY;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6B,CAAC;QAC5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -367,6 +367,32 @@ export interface OptimizeOptions {
|
|
|
367
367
|
/** Counts tokens in a text. Synchronous so it can run inside the rule loop. */
|
|
368
368
|
export type TokenCounter = (text: string) => number;
|
|
369
369
|
/** Full result of an optimisation. */
|
|
370
|
+
/** Which counter produced a figure, named rather than left as *external*. */
|
|
371
|
+
export interface TokenProvenance {
|
|
372
|
+
/**
|
|
373
|
+
* A stable identifier for the counter, not a sentence.
|
|
374
|
+
*
|
|
375
|
+
* `anthropic-count-tokens` for the official endpoint,
|
|
376
|
+
* `openai-tiktoken` for the local rank tables. A consumer branches on this;
|
|
377
|
+
* `detail` is what a person reads.
|
|
378
|
+
*/
|
|
379
|
+
counter: string;
|
|
380
|
+
/** The model the count was made for, since an encoding is a property of one. */
|
|
381
|
+
model: string;
|
|
382
|
+
/**
|
|
383
|
+
* What a reader needs to know about this particular count -- the encoding, or
|
|
384
|
+
* the endpoint. Free text, and never the only place a fact lives.
|
|
385
|
+
*/
|
|
386
|
+
detail: string;
|
|
387
|
+
/**
|
|
388
|
+
* Whether the count came from this machine or from a request.
|
|
389
|
+
*
|
|
390
|
+
* The one distinction a privacy-conscious reader actually cares about: a
|
|
391
|
+
* local counter read the prompt in this process, a remote one sent it
|
|
392
|
+
* somewhere. Both are `external` to the heuristic, and only one leaves.
|
|
393
|
+
*/
|
|
394
|
+
where: 'local' | 'remote';
|
|
395
|
+
}
|
|
370
396
|
export interface OptimizationResult {
|
|
371
397
|
original: string;
|
|
372
398
|
optimized: string;
|
|
@@ -383,6 +409,20 @@ export interface OptimizationResult {
|
|
|
383
409
|
locale: Locale;
|
|
384
410
|
/** How tokens were counted, so the caller knows the margin of error. */
|
|
385
411
|
tokenSource: 'heuristic' | 'external';
|
|
412
|
+
/**
|
|
413
|
+
* Which counter produced the figures, when it was not the heuristic.
|
|
414
|
+
*
|
|
415
|
+
* `tokenSource` says *not the estimator*, which is the question a margin of
|
|
416
|
+
* error depends on. It does not say **whose** counter, and once more than one
|
|
417
|
+
* external counter exists that difference matters: a count from Anthropic's
|
|
418
|
+
* endpoint and a count from OpenAI's rank tables are both `external` and are
|
|
419
|
+
* not interchangeable, and a reader repricing one workload against another
|
|
420
|
+
* model needs to know which they are holding.
|
|
421
|
+
*
|
|
422
|
+
* `null` when `tokenSource` is `heuristic`, so the two fields cannot drift
|
|
423
|
+
* into disagreeing about whether anything external was used at all.
|
|
424
|
+
*/
|
|
425
|
+
countedBy: TokenProvenance | null;
|
|
386
426
|
/**
|
|
387
427
|
* Where the prices came from.
|
|
388
428
|
*
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,oCAAoC;AAEpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,gEAAgE;AAChE;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,YAAI,MAAM,EAAE,YAAY,CAAU,CAAC;AAE3D,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,SAAS,CAAC;IACjB,wEAAwE;IACxE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACrD;AAED,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,qFAAqF;AACrF,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,aAAa,GAAG,SAAS,CAAC;AAElE,wCAAwC;AACxC,MAAM,MAAM,UAAU,GAClB,kBAAkB,GAClB,oBAAoB,GACpB,gBAAgB,GAChB,6BAA6B,GAC7B,qBAAqB,GACrB,sBAAsB,GACtB,WAAW,GACX,iBAAiB,GACjB,uBAAuB,GACvB,kBAAkB,GAClB,eAAe,GACf,eAAe,GACf,4BAA4B,GAC5B,oBAAoB,GACpB,wBAAwB,GACxB,uBAAuB,CAAC;AAE5B,sEAAsE;AACtE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,UAAU,CAAC;IACf,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,mDAAmD;AACnD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,eAAe;IAC9B,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,WAAW;AACrB,qEAAqE;AACnE,UAAU;AACZ,+DAA+D;GAC7D,WAAW;AACb,gCAAgC;GAC9B,MAAM;AACR;;;;;;;;;;;;;;GAcG;GACD,SAAS,CAAC;AAEd;;;;;;;GAOG;AACH,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;AAE5E;;;;;;;;;GASG;AACH,MAAM,MAAM,aAAa;AACvB;;;;;;GAMG;AACD;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AAChD;;;;;;;GAOG;GACD;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,YAAY,EAAE,OAAO,CAAA;CAAE,CAAC;AAElE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,WAAW;IAC1B,gFAAgF;IAChF,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,aAAa,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,sDAAsD;IACtD,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,EAAE;QACR,yCAAyC;QACzC,EAAE,EAAE,MAAM,CAAC;QACX,2DAA2D;QAC3D,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,6CAA6C;IAC7C,UAAU,EAAE,UAAU,CAAC;IACvB;;;;OAIG;IACH,IAAI,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IAC3D,gDAAgD;IAChD,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,gEAAgE;QAChE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC;IACtB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,4DAA4D;AAC5D,MAAM,WAAW,YAAY;IAC3B,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,sCAAsC;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,8DAA8D;IAC9D,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,aAAa,CAAA;KAAE,CAAC;IACzD,QAAQ,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,aAAa,CAAA;KAAE,CAAC;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,kEAAkE;AAClE,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,SAAS,GAAG,WAAW,CAAC;IAC9B,gEAAgE;IAChE,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,cAAc,GACtB,aAAa,GACb,aAAa,GACb,KAAK,GACL,aAAa,GACb,SAAS,CAAC;AAEd,4BAA4B;AAC5B,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,iDAAiD;IACjD,KAAK,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9B,oEAAoE;IACpE,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED,+EAA+E;AAC/E,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AAEpD,sCAAsC;AACtC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB,OAAO,EAAE,aAAa,CAAC;IACvB,KAAK,EAAE,YAAY,CAAC;IACpB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,WAAW,EAAE,WAAW,GAAG,UAAU,CAAC;IACtC;;;;;;OAMG;IACH,aAAa,EAAE;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,+DAA+D;QAC/D,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,yDAAyD;QACzD,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,yCAAyC;IACzC,GAAG,CAAC,EAAE,aAAa,CAAC;CACrB;AAED,uCAAuC;AACvC,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,qEAAqE;AACrE,MAAM,WAAW,WAAW;IAC1B,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpE;AAED,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,oCAAoC;AAEpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,gEAAgE;AAChE;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,YAAI,MAAM,EAAE,YAAY,CAAU,CAAC;AAE3D,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,SAAS,CAAC;IACjB,wEAAwE;IACxE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACrD;AAED,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,qFAAqF;AACrF,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,aAAa,GAAG,SAAS,CAAC;AAElE,wCAAwC;AACxC,MAAM,MAAM,UAAU,GAClB,kBAAkB,GAClB,oBAAoB,GACpB,gBAAgB,GAChB,6BAA6B,GAC7B,qBAAqB,GACrB,sBAAsB,GACtB,WAAW,GACX,iBAAiB,GACjB,uBAAuB,GACvB,kBAAkB,GAClB,eAAe,GACf,eAAe,GACf,4BAA4B,GAC5B,oBAAoB,GACpB,wBAAwB,GACxB,uBAAuB,CAAC;AAE5B,sEAAsE;AACtE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,UAAU,CAAC;IACf,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,mDAAmD;AACnD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,eAAe;IAC9B,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,WAAW;AACrB,qEAAqE;AACnE,UAAU;AACZ,+DAA+D;GAC7D,WAAW;AACb,gCAAgC;GAC9B,MAAM;AACR;;;;;;;;;;;;;;GAcG;GACD,SAAS,CAAC;AAEd;;;;;;;GAOG;AACH,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;AAE5E;;;;;;;;;GASG;AACH,MAAM,MAAM,aAAa;AACvB;;;;;;GAMG;AACD;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AAChD;;;;;;;GAOG;GACD;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,YAAY,EAAE,OAAO,CAAA;CAAE,CAAC;AAElE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,WAAW;IAC1B,gFAAgF;IAChF,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,aAAa,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,sDAAsD;IACtD,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,EAAE;QACR,yCAAyC;QACzC,EAAE,EAAE,MAAM,CAAC;QACX,2DAA2D;QAC3D,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,6CAA6C;IAC7C,UAAU,EAAE,UAAU,CAAC;IACvB;;;;OAIG;IACH,IAAI,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IAC3D,gDAAgD;IAChD,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,gEAAgE;QAChE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC;IACtB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,4DAA4D;AAC5D,MAAM,WAAW,YAAY;IAC3B,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,sCAAsC;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,8DAA8D;IAC9D,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,aAAa,CAAA;KAAE,CAAC;IACzD,QAAQ,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,aAAa,CAAA;KAAE,CAAC;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,kEAAkE;AAClE,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,SAAS,GAAG,WAAW,CAAC;IAC9B,gEAAgE;IAChE,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,cAAc,GACtB,aAAa,GACb,aAAa,GACb,KAAK,GACL,aAAa,GACb,SAAS,CAAC;AAEd,4BAA4B;AAC5B,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,iDAAiD;IACjD,KAAK,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9B,oEAAoE;IACpE,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED,+EAA+E;AAC/E,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AAEpD,sCAAsC;AACtC,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,KAAK,EAAE,OAAO,GAAG,QAAQ,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB,OAAO,EAAE,aAAa,CAAC;IACvB,KAAK,EAAE,YAAY,CAAC;IACpB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,WAAW,EAAE,WAAW,GAAG,UAAU,CAAC;IACtC;;;;;;;;;;;;OAYG;IACH,SAAS,EAAE,eAAe,GAAG,IAAI,CAAC;IAClC;;;;;;OAMG;IACH,aAAa,EAAE;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,+DAA+D;QAC/D,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,yDAAyD;QACzD,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,yCAAyC;IACzC,GAAG,CAAC,EAAE,aAAa,CAAC;CACrB;AAED,uCAAuC;AACvC,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,qEAAqE;AACrE,MAAM,WAAW,WAAW;IAC1B,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpE;AAED,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@trazum/core",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.85.0",
|
|
4
4
|
"description": "Trazum core: priced advisories for LLM prompts (caching, model tier, batching, schemas), plus deterministic trimming, token counting and pricing.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "David Mu\u00f1oz Rey",
|
package/src/band.ts
CHANGED
|
@@ -198,6 +198,18 @@ export const MEASURED_FOREIGN_ERROR_PCT: Readonly<Record<string, number>> = {
|
|
|
198
198
|
deepseek: 94.5,
|
|
199
199
|
/** Worst on `german-technical`, where even Latin prose is more than twice out. */
|
|
200
200
|
mistral: 103.1,
|
|
201
|
+
/**
|
|
202
|
+
* Worst on `german-prose`, and the worst of the four.
|
|
203
|
+
*
|
|
204
|
+
* The measurement sat in this repository unclaimed for a release: the
|
|
205
|
+
* fixture was written by a real key against `gpt-5`, and this file went on
|
|
206
|
+
* answering `null`, so every report told a GPT user nobody had looked. The
|
|
207
|
+
* figure is not a surprise -- `o200k_base` packs Latin prose far more densely
|
|
208
|
+
* than the estimator's Claude-calibrated divisors expect -- but a report
|
|
209
|
+
* saying *unmeasured* where the answer is *more than twice out* is the
|
|
210
|
+
* flattering direction, and this whole file exists to refuse that.
|
|
211
|
+
*/
|
|
212
|
+
openai: 112.4,
|
|
201
213
|
};
|
|
202
214
|
|
|
203
215
|
/**
|
package/src/optimize.ts
CHANGED
|
@@ -16,6 +16,7 @@ import type {
|
|
|
16
16
|
RuleResult,
|
|
17
17
|
Segment,
|
|
18
18
|
TokenCounter,
|
|
19
|
+
TokenProvenance,
|
|
19
20
|
UsageProfile,
|
|
20
21
|
} from './types.js';
|
|
21
22
|
|
|
@@ -210,6 +211,14 @@ export function optimize(prompt: string, options: OptimizeOptions = {}): Optimiz
|
|
|
210
211
|
usage,
|
|
211
212
|
locale,
|
|
212
213
|
tokenSource: options.tokenCounter ? 'external' : 'heuristic',
|
|
214
|
+
/*
|
|
215
|
+
* Null even when a counter was supplied here, and that is not an oversight.
|
|
216
|
+
* `optimize` takes a bare function: it can see that something other than
|
|
217
|
+
* the heuristic ran and cannot see whose it was. Naming one would be an
|
|
218
|
+
* invention, and `withExactTokenCounts` is where a caller that does know
|
|
219
|
+
* says so.
|
|
220
|
+
*/
|
|
221
|
+
countedBy: null,
|
|
213
222
|
pricingSource: {
|
|
214
223
|
lastReviewed: pricing.lastReviewed,
|
|
215
224
|
overriddenModels: pricing.overriddenModels,
|
|
@@ -230,6 +239,7 @@ export async function withExactTokenCounts(
|
|
|
230
239
|
result: OptimizationResult,
|
|
231
240
|
counter: AsyncTokenCounter,
|
|
232
241
|
pricing: PricingCatalogue = BUNDLED_CATALOGUE,
|
|
242
|
+
countedBy: TokenProvenance | null = null,
|
|
233
243
|
): Promise<OptimizationResult> {
|
|
234
244
|
// A result priced against an overlay cannot be recomputed against the bundled
|
|
235
245
|
// catalogue: the token counts would come from one source and the money from
|
|
@@ -267,6 +277,12 @@ export async function withExactTokenCounts(
|
|
|
267
277
|
savings,
|
|
268
278
|
advisories,
|
|
269
279
|
tokenSource: 'external',
|
|
280
|
+
/*
|
|
281
|
+
* Optional and defaulting to null rather than to a guess. A caller that
|
|
282
|
+
* knows whose counter it passed says so; one that does not gets `external`
|
|
283
|
+
* with no claim attached, which is exactly as much as this function knows.
|
|
284
|
+
*/
|
|
285
|
+
countedBy,
|
|
270
286
|
pricingSource: {
|
|
271
287
|
lastReviewed: pricing.lastReviewed,
|
|
272
288
|
overriddenModels: pricing.overriddenModels,
|
package/src/tokenizer.ts
CHANGED
|
@@ -66,8 +66,10 @@ export const ESTIMATE_ERROR_BAND_PCT = 33;
|
|
|
66
66
|
* estimator **against Claude**. Forty-seven samples, one counting endpoint, one
|
|
67
67
|
* family. Trazum prices seven, and the two that have since been run are far
|
|
68
68
|
* outside anything published here: DeepSeek 94.5% at worst, Mistral 103.1%. On
|
|
69
|
-
* a
|
|
70
|
-
*
|
|
69
|
+
* a Gemini prompt nobody has measured it at all, and `measuredForeignError`
|
|
70
|
+
* answers null rather than the nearest number. On a GPT prompt somebody has:
|
|
71
|
+
* 112.4%, the worst of the four measured, and the figure this file reported as
|
|
72
|
+
* unmeasured for a release while its own fixture held the answer.
|
|
71
73
|
*
|
|
72
74
|
* Exported as a name rather than left implicit in a comment because the band
|
|
73
75
|
* had already leaked out of its domain in three places: an advisory that told a
|
package/src/types.ts
CHANGED
|
@@ -394,6 +394,33 @@ export interface OptimizeOptions {
|
|
|
394
394
|
export type TokenCounter = (text: string) => number;
|
|
395
395
|
|
|
396
396
|
/** Full result of an optimisation. */
|
|
397
|
+
/** Which counter produced a figure, named rather than left as *external*. */
|
|
398
|
+
export interface TokenProvenance {
|
|
399
|
+
/**
|
|
400
|
+
* A stable identifier for the counter, not a sentence.
|
|
401
|
+
*
|
|
402
|
+
* `anthropic-count-tokens` for the official endpoint,
|
|
403
|
+
* `openai-tiktoken` for the local rank tables. A consumer branches on this;
|
|
404
|
+
* `detail` is what a person reads.
|
|
405
|
+
*/
|
|
406
|
+
counter: string;
|
|
407
|
+
/** The model the count was made for, since an encoding is a property of one. */
|
|
408
|
+
model: string;
|
|
409
|
+
/**
|
|
410
|
+
* What a reader needs to know about this particular count -- the encoding, or
|
|
411
|
+
* the endpoint. Free text, and never the only place a fact lives.
|
|
412
|
+
*/
|
|
413
|
+
detail: string;
|
|
414
|
+
/**
|
|
415
|
+
* Whether the count came from this machine or from a request.
|
|
416
|
+
*
|
|
417
|
+
* The one distinction a privacy-conscious reader actually cares about: a
|
|
418
|
+
* local counter read the prompt in this process, a remote one sent it
|
|
419
|
+
* somewhere. Both are `external` to the heuristic, and only one leaves.
|
|
420
|
+
*/
|
|
421
|
+
where: 'local' | 'remote';
|
|
422
|
+
}
|
|
423
|
+
|
|
397
424
|
export interface OptimizationResult {
|
|
398
425
|
original: string;
|
|
399
426
|
optimized: string;
|
|
@@ -410,6 +437,20 @@ export interface OptimizationResult {
|
|
|
410
437
|
locale: Locale;
|
|
411
438
|
/** How tokens were counted, so the caller knows the margin of error. */
|
|
412
439
|
tokenSource: 'heuristic' | 'external';
|
|
440
|
+
/**
|
|
441
|
+
* Which counter produced the figures, when it was not the heuristic.
|
|
442
|
+
*
|
|
443
|
+
* `tokenSource` says *not the estimator*, which is the question a margin of
|
|
444
|
+
* error depends on. It does not say **whose** counter, and once more than one
|
|
445
|
+
* external counter exists that difference matters: a count from Anthropic's
|
|
446
|
+
* endpoint and a count from OpenAI's rank tables are both `external` and are
|
|
447
|
+
* not interchangeable, and a reader repricing one workload against another
|
|
448
|
+
* model needs to know which they are holding.
|
|
449
|
+
*
|
|
450
|
+
* `null` when `tokenSource` is `heuristic`, so the two fields cannot drift
|
|
451
|
+
* into disagreeing about whether anything external was used at all.
|
|
452
|
+
*/
|
|
453
|
+
countedBy: TokenProvenance | null;
|
|
413
454
|
/**
|
|
414
455
|
* Where the prices came from.
|
|
415
456
|
*
|