@trazum/core 1.81.0 → 1.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -18
- package/dist/advisories.d.ts +33 -1
- package/dist/advisories.d.ts.map +1 -1
- package/dist/advisories.js +103 -16
- package/dist/advisories.js.map +1 -1
- package/dist/band.d.ts +131 -0
- package/dist/band.d.ts.map +1 -0
- package/dist/band.js +210 -0
- package/dist/band.js.map +1 -0
- package/dist/conform.d.ts +1 -1
- package/dist/conform.d.ts.map +1 -1
- package/dist/conform.js +84 -1
- package/dist/conform.js.map +1 -1
- package/dist/helicone.d.ts +92 -0
- package/dist/helicone.d.ts.map +1 -0
- package/dist/helicone.js +194 -0
- package/dist/helicone.js.map +1 -0
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +9 -3
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +9 -3
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +43 -1
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +10 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/litellm.d.ts +98 -0
- package/dist/litellm.d.ts.map +1 -0
- package/dist/litellm.js +207 -0
- package/dist/litellm.js.map +1 -0
- package/dist/measured-profile.d.ts +4 -0
- package/dist/measured-profile.d.ts.map +1 -1
- package/dist/measured-profile.js +22 -1
- package/dist/measured-profile.js.map +1 -1
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +36 -0
- package/dist/schemas.js.map +1 -1
- package/dist/tokenizer.d.ts +45 -49
- package/dist/tokenizer.d.ts.map +1 -1
- package/dist/tokenizer.js +100 -51
- package/dist/tokenizer.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/verdict-bridge.d.ts +113 -0
- package/dist/verdict-bridge.d.ts.map +1 -0
- package/dist/verdict-bridge.js +127 -0
- package/dist/verdict-bridge.js.map +1 -0
- package/package.json +1 -1
- package/src/advisories.ts +132 -15
- package/src/band.ts +213 -0
- package/src/conform.ts +100 -1
- package/src/helicone.ts +227 -0
- package/src/i18n/en.ts +10 -3
- package/src/i18n/es.ts +10 -3
- package/src/i18n/types.ts +45 -1
- package/src/index.ts +17 -1
- package/src/litellm.ts +243 -0
- package/src/measured-profile.ts +26 -2
- package/src/schemas.ts +36 -0
- package/src/tokenizer.ts +101 -51
- package/src/types.ts +1 -0
- package/src/verdict-bridge.ts +200 -0
package/src/advisories.ts
CHANGED
|
@@ -7,7 +7,8 @@ import type { PricingCatalogue } from './pricing.js';
|
|
|
7
7
|
import { formatUsd } from './savings.js';
|
|
8
8
|
import { analyzeExamples, findContradictions, findMovableSchema,
|
|
9
9
|
findRestatedFormat } from './structure.js';
|
|
10
|
-
import {
|
|
10
|
+
import { bandGoverns, estimateTokens } from './tokenizer.js';
|
|
11
|
+
import { bandFor } from './band.js';
|
|
11
12
|
import type { Advisory, ModelPricing, TokenCounter, UsageProfile } from './types.js';
|
|
12
13
|
|
|
13
14
|
function countSignals(haystack: string, signals: readonly string[]): number {
|
|
@@ -16,17 +17,43 @@ function countSignals(haystack: string, signals: readonly string[]): number {
|
|
|
16
17
|
return count;
|
|
17
18
|
}
|
|
18
19
|
|
|
20
|
+
/** What `recommendTierDetailed` saw, not only what it concluded. */
|
|
21
|
+
export interface TierRecommendation {
|
|
22
|
+
tier: ModelPricing['tier'];
|
|
23
|
+
complexSignals: number;
|
|
24
|
+
simpleSignals: number;
|
|
25
|
+
/**
|
|
26
|
+
* True when the prompt asks for depth and for brevity at once, and the
|
|
27
|
+
* heuristic has no business preferring either.
|
|
28
|
+
*
|
|
29
|
+
* The score subtracts one side from the other, so three complex signals
|
|
30
|
+
* against three simple ones cancels to zero and returns `sonnet` — the same
|
|
31
|
+
* answer as a prompt with **no signals at all**. Those are opposite
|
|
32
|
+
* situations reported identically: one is "nothing here suggests a tier", the
|
|
33
|
+
* other is "this prompt contradicts itself about which tier it needs", and
|
|
34
|
+
* only the second is worth telling somebody about.
|
|
35
|
+
*
|
|
36
|
+
* The threshold is a lead of one signal or fewer, and it comes from the
|
|
37
|
+
* weights above rather than from taste: each signal moves the score by 2,
|
|
38
|
+
* prompt size moves it by up to 2, so a one-signal lead is inside what
|
|
39
|
+
* length alone contributes. A lead of two or more is a majority the size
|
|
40
|
+
* term cannot manufacture.
|
|
41
|
+
*/
|
|
42
|
+
conflicted: boolean;
|
|
43
|
+
}
|
|
44
|
+
|
|
19
45
|
/**
|
|
20
|
-
* Estimates the capability tier the prompt needs.
|
|
46
|
+
* Estimates the capability tier the prompt needs, and says what it saw.
|
|
21
47
|
*
|
|
22
48
|
* This is a keyword-and-size heuristic, not a judgement about answer quality.
|
|
23
49
|
* Treat it as a hypothesis to validate with your own evaluations before
|
|
24
50
|
* moving down a tier in production.
|
|
25
51
|
*/
|
|
26
|
-
export function
|
|
52
|
+
export function recommendTierDetailed(prompt: string, tokens: number): TierRecommendation {
|
|
27
53
|
const haystack = prompt.toLowerCase();
|
|
28
|
-
|
|
29
|
-
|
|
54
|
+
const complexSignals = countSignals(haystack, COMPLEX_SIGNALS);
|
|
55
|
+
const simpleSignals = countSignals(haystack, SIMPLE_SIGNALS);
|
|
56
|
+
let score = complexSignals * 2 - simpleSignals * 2;
|
|
30
57
|
|
|
31
58
|
if (tokens > 4000) score += 2;
|
|
32
59
|
else if (tokens > 1500) score += 1;
|
|
@@ -34,9 +61,25 @@ export function recommendTier(prompt: string, tokens: number): ModelPricing['tie
|
|
|
34
61
|
|
|
35
62
|
if (/```|~~~/.test(prompt)) score += 1;
|
|
36
63
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
64
|
+
const tier = score >= 3 ? 'opus' : score >= 0 ? 'sonnet' : 'haiku';
|
|
65
|
+
return {
|
|
66
|
+
tier,
|
|
67
|
+
complexSignals,
|
|
68
|
+
simpleSignals,
|
|
69
|
+
conflicted:
|
|
70
|
+
complexSignals > 0 && simpleSignals > 0 && Math.abs(complexSignals - simpleSignals) <= 1,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The tier alone, unchanged.
|
|
76
|
+
*
|
|
77
|
+
* Kept because it is public API and the 1.x line does not change shapes.
|
|
78
|
+
* Callers that need to know whether the answer is trustworthy reach for
|
|
79
|
+
* `recommendTierDetailed`.
|
|
80
|
+
*/
|
|
81
|
+
export function recommendTier(prompt: string, tokens: number): ModelPricing['tier'] {
|
|
82
|
+
return recommendTierDetailed(prompt, tokens).tier;
|
|
40
83
|
}
|
|
41
84
|
|
|
42
85
|
const TIER_ORDER: Record<ModelPricing['tier'], number> = {
|
|
@@ -142,7 +185,18 @@ export function buildAdvisories(
|
|
|
142
185
|
* outright rather than degrading, so there is no partial result to notice.
|
|
143
186
|
*/
|
|
144
187
|
const estimated = count === estimateTokens;
|
|
145
|
-
|
|
188
|
+
/**
|
|
189
|
+
* The band of **this prompt**, not one number for all text.
|
|
190
|
+
*
|
|
191
|
+
* These three hedges — the context window, the cache minimum both ways — are
|
|
192
|
+
* the places an estimate is compared against a hard threshold and the answer
|
|
193
|
+
* stated as fact, so the width of the hedge decides whether the statement is
|
|
194
|
+
* safe. A single 10% was measured on a corpus of thirteen prose files and one
|
|
195
|
+
* each of code, numeric and punctuation, and the estimator is 25% out on a
|
|
196
|
+
* SQL migration and 33% out on a CSV ledger. Hedging a code prompt by ten
|
|
197
|
+
* points was hedging by a third of what it needed.
|
|
198
|
+
*/
|
|
199
|
+
const band = bandFor(optimizedPrompt) / 100;
|
|
146
200
|
|
|
147
201
|
/**
|
|
148
202
|
* Whether that band describes **this** model's tokenizer.
|
|
@@ -254,15 +308,36 @@ export function buildAdvisories(
|
|
|
254
308
|
*/
|
|
255
309
|
nearMinimum:
|
|
256
310
|
count === estimateTokens &&
|
|
257
|
-
cache.stablePrefixTokens * (1 -
|
|
311
|
+
cache.stablePrefixTokens * (1 - band) < minTokens,
|
|
258
312
|
}),
|
|
259
313
|
estimatedMonthlyUsd: saving,
|
|
260
314
|
});
|
|
261
315
|
} else {
|
|
316
|
+
/**
|
|
317
|
+
* The threshold is derived from this model's own multipliers, not
|
|
318
|
+
* quoted.
|
|
319
|
+
*
|
|
320
|
+
* Break-even is where a cached token costs exactly what an uncached one
|
|
321
|
+
* does: `h*read + (1-h)*write = 1`, so `h = (1 - write) / (read -
|
|
322
|
+
* write)`. At Anthropic's 1.25 and 0.1 that is 21.74%, and the sentence
|
|
323
|
+
* this replaces said 28% for every model in the catalogue.
|
|
324
|
+
*
|
|
325
|
+
* A write multiplier of 1 has no such threshold. Writing then costs
|
|
326
|
+
* exactly what not caching costs, so caching cannot lose money at any
|
|
327
|
+
* hit rate, and `(1 - 1) / (r - 1)` is 0 rather than a small number.
|
|
328
|
+
* Eight of the eighteen models are that shape, and the fixed sentence
|
|
329
|
+
* advised all eight to consider turning caching off.
|
|
330
|
+
*/
|
|
331
|
+
const write = rates.cacheWrite5m;
|
|
262
332
|
advisories.push({
|
|
263
333
|
id: 'prompt-caching-not-worth-it',
|
|
264
334
|
severity: 'info',
|
|
265
|
-
...t.advisories.promptCachingNotWorthIt(
|
|
335
|
+
...t.advisories.promptCachingNotWorthIt({
|
|
336
|
+
readPct: Math.round(rates.cacheRead * 100),
|
|
337
|
+
writePct: Math.round(write * 100),
|
|
338
|
+
breakEvenPct:
|
|
339
|
+
write <= 1 ? null : Math.round(((1 - write) / (rates.cacheRead - write)) * 1000) / 10,
|
|
340
|
+
}),
|
|
266
341
|
estimatedMonthlyUsd: null,
|
|
267
342
|
});
|
|
268
343
|
}
|
|
@@ -287,7 +362,7 @@ export function buildAdvisories(
|
|
|
287
362
|
*/
|
|
288
363
|
couldReachMinimum:
|
|
289
364
|
count === estimateTokens &&
|
|
290
|
-
cache.stablePrefixTokens * (1 +
|
|
365
|
+
cache.stablePrefixTokens * (1 + band) >= minTokens,
|
|
291
366
|
}),
|
|
292
367
|
estimatedMonthlyUsd: null,
|
|
293
368
|
});
|
|
@@ -374,7 +449,34 @@ export function buildAdvisories(
|
|
|
374
449
|
}
|
|
375
450
|
|
|
376
451
|
// --- Recommended model ---
|
|
377
|
-
const
|
|
452
|
+
const recommendation = recommendTierDetailed(optimizedPrompt, tokensAfter);
|
|
453
|
+
const suggestedTier = recommendation.tier;
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* When the prompt argues with itself about which tier it needs, say so and
|
|
457
|
+
* recommend nothing.
|
|
458
|
+
*
|
|
459
|
+
* The score subtracts one side from the other, so a prompt demanding depth
|
|
460
|
+
* and brevity in equal measure cancels to zero and lands on `sonnet` — the
|
|
461
|
+
* identical answer to a prompt with no signals at all. Reporting two opposite
|
|
462
|
+
* situations with one number is the failure; the fix is not a better score,
|
|
463
|
+
* it is declining to pretend there is one.
|
|
464
|
+
*
|
|
465
|
+
* This returns before the downgrade block rather than alongside it, because a
|
|
466
|
+
* saving computed from a tier the heuristic cannot stand behind is a figure
|
|
467
|
+
* with a dollar sign on it and nothing underneath.
|
|
468
|
+
*/
|
|
469
|
+
if (recommendation.conflicted) {
|
|
470
|
+
advisories.push({
|
|
471
|
+
id: 'tier-signals-conflict',
|
|
472
|
+
severity: 'info',
|
|
473
|
+
...t.advisories.tierSignalsConflict({
|
|
474
|
+
complexSignals: recommendation.complexSignals,
|
|
475
|
+
simpleSignals: recommendation.simpleSignals,
|
|
476
|
+
}),
|
|
477
|
+
estimatedMonthlyUsd: null,
|
|
478
|
+
});
|
|
479
|
+
}
|
|
378
480
|
/**
|
|
379
481
|
* A model whose capability nobody recorded is never told it is overpowered.
|
|
380
482
|
*
|
|
@@ -392,15 +494,30 @@ export function buildAdvisories(
|
|
|
392
494
|
* and a condition that forgets it both have to happen before a model of
|
|
393
495
|
* unrecorded capability is told it is overpowered.
|
|
394
496
|
*/
|
|
395
|
-
if (model.tier !== 'unknown' && TIER_ORDER[suggestedTier] < TIER_ORDER[model.tier]) {
|
|
497
|
+
if (!recommendation.conflicted && model.tier !== 'unknown' && TIER_ORDER[suggestedTier] < TIER_ORDER[model.tier]) {
|
|
396
498
|
const candidate = cheapestInTier(suggestedTier, on, pricing, model.provider);
|
|
397
499
|
if (candidate) {
|
|
398
500
|
const candidatePricing = effectivePricing(candidate, on);
|
|
501
|
+
/**
|
|
502
|
+
* The candidate's own batch multiplier, not a constant.
|
|
503
|
+
*
|
|
504
|
+
* This line read `usage.batchEligible ? 0.5 : 1`, while the current
|
|
505
|
+
* model's cost twenty lines above already used `rates.batch ?? 1` for
|
|
506
|
+
* exactly the reason stated there: a discount that is not per model is an
|
|
507
|
+
* invented saving rather than an imprecise one.
|
|
508
|
+
*
|
|
509
|
+
* Three models in the catalogue carry `batch: null` — kimi-k2,
|
|
510
|
+
* deepseek-v3 and grok-4 — which means no batch API at all. Halving their
|
|
511
|
+
* cost offered money that cannot be bought at any price, made the
|
|
512
|
+
* downgrade look twice as good as it is, and could turn a saving that was
|
|
513
|
+
* negative into one this advisory reports.
|
|
514
|
+
*/
|
|
515
|
+
const candidateBatch = usage.batchEligible ? (multipliersFor(candidate).batch ?? 1) : 1;
|
|
399
516
|
const candidateMonthly =
|
|
400
517
|
((tokensAfter / 1_000_000) * candidatePricing.inputPerMTok +
|
|
401
518
|
(usage.avgOutputTokens / 1_000_000) * candidatePricing.outputPerMTok) *
|
|
402
519
|
usage.callsPerMonth *
|
|
403
|
-
|
|
520
|
+
candidateBatch;
|
|
404
521
|
const saving = monthlyInputUsd + monthlyOutputUsd - candidateMonthly;
|
|
405
522
|
if (saving > 0) {
|
|
406
523
|
advisories.push({
|
package/src/band.ts
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The error band Trazum is entitled to print about a given piece of text.
|
|
3
|
+
*
|
|
4
|
+
* ## Why this exists
|
|
5
|
+
*
|
|
6
|
+
* For a long time every report said `±10%`, everywhere, about everything. That
|
|
7
|
+
* number came from a corpus of twenty-one samples that held **thirteen files of
|
|
8
|
+
* Latin prose and exactly one each of code, numeric and punctuation** — and
|
|
9
|
+
* those single files were the set the estimator's constants had been fitted to.
|
|
10
|
+
* So the published band was not a measurement of the estimator. It was a
|
|
11
|
+
* measurement of its own training set.
|
|
12
|
+
*
|
|
13
|
+
* Seven ordinary samples in the thin classes broke it. Measured against
|
|
14
|
+
* Anthropic's counting endpoint, the same estimator that is 4.6% out on prose
|
|
15
|
+
* is **24.9% out on a SQL migration** and **32.5% out on a CSV ledger**. Telling
|
|
16
|
+
* somebody `±10%` about their ledger was telling them a number that is wrong
|
|
17
|
+
* about their prompt specifically.
|
|
18
|
+
*
|
|
19
|
+
* ## Why it does not name a text type
|
|
20
|
+
*
|
|
21
|
+
* The obvious design is a classifier — prose, code, few-shot, punctuation — and
|
|
22
|
+
* a band per class. The corpus says it cannot be built. Measured by character
|
|
23
|
+
* mix, code and punctuation **overlap completely**: `code-sql` is 7.7% symbols
|
|
24
|
+
* and `code-heavy` 15.6%, while `punctuation-markup` is 17.5% and
|
|
25
|
+
* `punctuation-heavy` 29.7%. Two of the three few-shot samples are
|
|
26
|
+
* indistinguishable from prose. A classifier over those would be a guess
|
|
27
|
+
* wearing a measurement's name, and it would hand out the wrong band precisely
|
|
28
|
+
* where the band matters most.
|
|
29
|
+
*
|
|
30
|
+
* So this does not classify. It sorts text into the **buckets the evidence
|
|
31
|
+
* actually separates**, and where two classes overlap it gives the worse of
|
|
32
|
+
* their bands. An overstated uncertainty costs a reader some confidence; an
|
|
33
|
+
* understated one tells them a wrong number and lets them act on it.
|
|
34
|
+
*
|
|
35
|
+
* ## What holds it
|
|
36
|
+
*
|
|
37
|
+
* `token-band.test.js` runs every corpus sample through `bandFor` and requires
|
|
38
|
+
* the sample's own measured error to be inside the band it was given. That is
|
|
39
|
+
* the whole promise, and it cannot be satisfied by tuning the thresholds: a
|
|
40
|
+
* sample sorted into a friendlier bucket gets a smaller band and fails.
|
|
41
|
+
*
|
|
42
|
+
* The figures below are derived from the fixture by the same test, which fails
|
|
43
|
+
* when they drift. They are written here rather than imported because this file
|
|
44
|
+
* ships and the fixture does not.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The buckets, worst measured error first.
|
|
49
|
+
*
|
|
50
|
+
* Every number is `Math.ceil` of the worst sample in that bucket, from
|
|
51
|
+
* `test/fixtures/token-ground-truth.json` — Anthropic's, because that is the
|
|
52
|
+
* tokenizer this estimator was built for and the only one whose fixture governs
|
|
53
|
+
* a published claim. A DeepSeek or Mistral prompt is further out still, and
|
|
54
|
+
* `providerBandCaveat` is what says so.
|
|
55
|
+
*/
|
|
56
|
+
export const BANDS = {
|
|
57
|
+
/** Digit-dominant: ledgers, tables, exports. The worst the estimator does. */
|
|
58
|
+
numeric: 33,
|
|
59
|
+
/** Symbol-rich: code, markup, anything a merge table covers badly. */
|
|
60
|
+
symbolic: 26,
|
|
61
|
+
/**
|
|
62
|
+
* CJK, all three scripts.
|
|
63
|
+
*
|
|
64
|
+
* Was 10 and is 4, and the difference is the second Korean sample rather than
|
|
65
|
+
* anything in this file. Hangul had been charged a placeholder that nothing
|
|
66
|
+
* measured; one sample moved it to within ten points and two moved it to
|
|
67
|
+
* within three. The class's worst went from 10.6% to 3.2% and its standard
|
|
68
|
+
* deviation from 4.0 to 1.0.
|
|
69
|
+
*/
|
|
70
|
+
cjk: 4,
|
|
71
|
+
/**
|
|
72
|
+
* Latin prose and few-shot blocks made of it. What most prompts are.
|
|
73
|
+
*
|
|
74
|
+
* Widened from 5 by `numeric-matrix`, a confusion matrix whose class labels
|
|
75
|
+
* make letters the dominant class: composition put it here, its error came
|
|
76
|
+
* out at 5.6%, and the band follows the measurement rather than the other way
|
|
77
|
+
* round. Eighteen samples, deviation 1.4 points.
|
|
78
|
+
*/
|
|
79
|
+
prose: 6,
|
|
80
|
+
} as const;
|
|
81
|
+
|
|
82
|
+
export type BandBucket = keyof typeof BANDS;
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Where the thresholds come from.
|
|
86
|
+
*
|
|
87
|
+
* Each is the widest gap the corpus offers between the bucket it admits and
|
|
88
|
+
* everything below it, so none of them is a round number somebody liked:
|
|
89
|
+
*
|
|
90
|
+
* - `cjk` at 0.5: the CJK samples are 90.1% to 94.5% CJK and every other sample
|
|
91
|
+
* is 0.0%. Nothing in the corpus lies between.
|
|
92
|
+
* - `numeric` at 0.30: the two numeric samples are 47.7% and 49.7% digits; the
|
|
93
|
+
* next highest anywhere is `few-shot-extraction` at 15.3%.
|
|
94
|
+
* - `symbolic` at 0.07: this one is thin and is stated as thin. `code-sql` is
|
|
95
|
+
* 7.7% symbols and `portuguese-prose` is 6.1%, so 1.6 points of margin decide
|
|
96
|
+
* it. Thin in the safe direction: a prose sample that crosses it is given the
|
|
97
|
+
* symbolic band, which is wider than it needs, and no sample is given a band
|
|
98
|
+
* narrower than its own error.
|
|
99
|
+
*/
|
|
100
|
+
const CJK_SHARE = 0.5;
|
|
101
|
+
const DIGIT_SHARE = 0.3;
|
|
102
|
+
const SYMBOL_SHARE = 0.07;
|
|
103
|
+
|
|
104
|
+
/*
|
|
105
|
+
Written as escapes rather than as the characters themselves, and CodeQL is
|
|
106
|
+
why. The last range was typed as a pair of literal ideographs meaning
|
|
107
|
+
"compatibility ideographs, U+F900 to U+FAFF" — and the opening character was
|
|
108
|
+
U+8C48, the ordinary unified ideograph that shares the glyph, not the
|
|
109
|
+
compatibility one. So the range ran from U+8C48 to U+FAFF and swallowed the
|
|
110
|
+
Yi syllables, the private-use area and both surrogate halves, which meant a
|
|
111
|
+
page of astral emoji was sorted `cjk` and handed the narrowest band in the
|
|
112
|
+
file: ±4% about text nothing has ever measured.
|
|
113
|
+
|
|
114
|
+
A literal that looks right and is not is the failure mode this release exists
|
|
115
|
+
to fix, so the ranges are numbers now and the test below plants one character
|
|
116
|
+
from each block that used to be caught.
|
|
117
|
+
*/
|
|
118
|
+
const CJK = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff]/;
|
|
119
|
+
const LETTER = /[A-Za-zÀ-ɏͰ-ϿЀ-ӿ]/;
|
|
120
|
+
const DIGIT = /[0-9]/;
|
|
121
|
+
|
|
122
|
+
/** The bucket a piece of text falls in, by the composition the corpus separates on. */
|
|
123
|
+
export function bucketFor(text: string): BandBucket {
|
|
124
|
+
const chars = [...text].filter((ch) => !/\s/.test(ch));
|
|
125
|
+
if (chars.length === 0) return 'prose';
|
|
126
|
+
|
|
127
|
+
let cjk = 0;
|
|
128
|
+
let digits = 0;
|
|
129
|
+
let letters = 0;
|
|
130
|
+
for (const ch of chars) {
|
|
131
|
+
if (CJK.test(ch)) cjk += 1;
|
|
132
|
+
else if (DIGIT.test(ch)) digits += 1;
|
|
133
|
+
else if (LETTER.test(ch)) letters += 1;
|
|
134
|
+
}
|
|
135
|
+
const n = chars.length;
|
|
136
|
+
const symbols = n - cjk - digits - letters;
|
|
137
|
+
|
|
138
|
+
/*
|
|
139
|
+
Order is the design. CJK first because it is the one bucket with no overlap
|
|
140
|
+
at all, then digits, then symbols — each test only runs on text the previous
|
|
141
|
+
one declined, so a Korean prompt with quotation marks is CJK and not
|
|
142
|
+
symbolic. Reversing any two would let a wide bucket swallow a narrow one and
|
|
143
|
+
hand out a band nobody measured.
|
|
144
|
+
*/
|
|
145
|
+
if (cjk / n >= CJK_SHARE) return 'cjk';
|
|
146
|
+
if (digits / n >= DIGIT_SHARE) return 'numeric';
|
|
147
|
+
if (symbols / n >= SYMBOL_SHARE) return 'symbolic';
|
|
148
|
+
return 'prose';
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* The band, as a percentage, that may be printed about this text.
|
|
153
|
+
*
|
|
154
|
+
* Never a single number for all text again. A caller that wants one figure for
|
|
155
|
+
* a mixed report should take the widest band across the texts it covers, not an
|
|
156
|
+
* average: averaging two uncertainties produces a third that describes neither.
|
|
157
|
+
*/
|
|
158
|
+
export function bandFor(text: string): number {
|
|
159
|
+
return BANDS[bucketFor(text)];
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Whether the published band applies to a model at all.
|
|
164
|
+
*
|
|
165
|
+
* It does not, and this is the sentence that has been missing. The estimator is
|
|
166
|
+
* calibrated on Claude's tokenizer, and the same 47 samples measured against
|
|
167
|
+
* DeepSeek's own counter came out **94.5% wrong** at worst and against
|
|
168
|
+
* Mistral's **103.1%**. Trazum prices seven providers; the band is one
|
|
169
|
+
* provider's.
|
|
170
|
+
*
|
|
171
|
+
* Returns null when the model is one the band was measured on, and the provider
|
|
172
|
+
* id otherwise, so a caller can say which foreign tokenizer it is estimating
|
|
173
|
+
* against rather than printing a figure that belongs to a different one.
|
|
174
|
+
*/
|
|
175
|
+
export function foreignTokenizer(provider: string | null): string | null {
|
|
176
|
+
return provider === null || provider === 'anthropic' ? null : provider;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* The worst error measured against a provider's own tokenizer, where anybody
|
|
181
|
+
* has run it.
|
|
182
|
+
*
|
|
183
|
+
* Two entries, and the emptiness of the rest is the point. `foreignTokenizer`
|
|
184
|
+
* could always say *this is not Claude*; what it could not say is how far off.
|
|
185
|
+
* A hedge with no figure reads as a formality, and a reader discounts it — so
|
|
186
|
+
* where the corpus has been run against the family's own counter, the report
|
|
187
|
+
* says the number, and where it has not, it says nobody has measured rather
|
|
188
|
+
* than borrowing one of these.
|
|
189
|
+
*
|
|
190
|
+
* Every value is the worst of the 47 samples in
|
|
191
|
+
* `test/fixtures/token-ground-truth.<provider>.json`, and
|
|
192
|
+
* `token-band.test.js` recomputes both from the fixtures and fails on drift.
|
|
193
|
+
* Written here rather than imported because this file ships and the fixtures
|
|
194
|
+
* do not.
|
|
195
|
+
*/
|
|
196
|
+
export const MEASURED_FOREIGN_ERROR_PCT: Readonly<Record<string, number>> = {
|
|
197
|
+
/** Worst on `cjk-chinese`: the estimator's CJK charge is Claude's, not this one's. */
|
|
198
|
+
deepseek: 94.5,
|
|
199
|
+
/** Worst on `german-technical`, where even Latin prose is more than twice out. */
|
|
200
|
+
mistral: 103.1,
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* How far out the estimator has been measured against this provider, or null.
|
|
205
|
+
*
|
|
206
|
+
* Null means unmeasured, never zero: a family nobody has run is an open
|
|
207
|
+
* question, and answering it with a number would be the exact fault this whole
|
|
208
|
+
* file exists to undo.
|
|
209
|
+
*/
|
|
210
|
+
export function measuredForeignError(provider: string | null): number | null {
|
|
211
|
+
if (provider === null) return null;
|
|
212
|
+
return MEASURED_FOREIGN_ERROR_PCT[provider] ?? null;
|
|
213
|
+
}
|
package/src/conform.ts
CHANGED
|
@@ -54,6 +54,11 @@ export const CONTRACT_NAMES = [
|
|
|
54
54
|
'bench',
|
|
55
55
|
'prompt-draft',
|
|
56
56
|
'position',
|
|
57
|
+
'routing-measurement',
|
|
58
|
+
'example-pruning',
|
|
59
|
+
'store-inventory',
|
|
60
|
+
'conformance',
|
|
61
|
+
'watch-cycle',
|
|
57
62
|
] as const;
|
|
58
63
|
|
|
59
64
|
export type ContractName = (typeof CONTRACT_NAMES)[number];
|
|
@@ -394,6 +399,62 @@ const DOCUMENT_RULES: Record<Exclude<ContractName, 'usage-log'>, FieldRule[]> =
|
|
|
394
399
|
rule('cannotSay', 'an array of sentences — what is deliberately not answered', isArray),
|
|
395
400
|
rule('unpricedRecords', 'a count of records the catalogue cannot price', isNumber),
|
|
396
401
|
],
|
|
402
|
+
'routing-measurement': [
|
|
403
|
+
rule('slice', 'the workload object — the calls this measured', isObject),
|
|
404
|
+
rule('evaluation', 'the measurement object — verdict and agreement rates', isObject),
|
|
405
|
+
],
|
|
406
|
+
'store-inventory': [
|
|
407
|
+
rule('providers', 'an array — one entry per provider held', isArray),
|
|
408
|
+
rule('totalRecords', 'records the store resolves to', isNumber),
|
|
409
|
+
rule(
|
|
410
|
+
'span',
|
|
411
|
+
'an object, or null when the store holds nothing',
|
|
412
|
+
(v) => v === null || isObject(v),
|
|
413
|
+
),
|
|
414
|
+
rule('possiblyDouble', 'records the store could not tell apart', isNumber),
|
|
415
|
+
rule('unknownVersion', 'records written by a newer schema', isNumber),
|
|
416
|
+
rule('totalUsd', 'what the held records priced to', isNumber),
|
|
417
|
+
rule('unreadable', 'an array — files the store could not parse', isArray),
|
|
418
|
+
],
|
|
419
|
+
conformance: [
|
|
420
|
+
rule(
|
|
421
|
+
'contract',
|
|
422
|
+
'the contract the input looked like, or null when nothing did',
|
|
423
|
+
(v) => v === null || typeof v === 'string',
|
|
424
|
+
),
|
|
425
|
+
rule(
|
|
426
|
+
'because',
|
|
427
|
+
'why nothing matched, or null when something did — never a bare refusal',
|
|
428
|
+
(v) => v === null || typeof v === 'string',
|
|
429
|
+
),
|
|
430
|
+
rule(
|
|
431
|
+
'records',
|
|
432
|
+
'records examined for a log, or null for a single document',
|
|
433
|
+
(v) => v === null || isNumber(v),
|
|
434
|
+
),
|
|
435
|
+
rule('problems', 'an array — one entry per fault found', isArray),
|
|
436
|
+
rule('unavailable', 'an array — findings this shape cannot support', isArray),
|
|
437
|
+
rule('conforms', 'a boolean — the verdict', (v) => typeof v === 'boolean'),
|
|
438
|
+
],
|
|
439
|
+
'watch-cycle': [
|
|
440
|
+
rule('firedAtMs', 'the instant the cycle ran', isNumber),
|
|
441
|
+
rule('crossings', 'an array — gates over their limit, news this cycle', isArray),
|
|
442
|
+
rule('suppressed', 'an array — still over, already reported', isArray),
|
|
443
|
+
rule('abstentions', 'an array — gates that could not be judged', isArray),
|
|
444
|
+
rule(
|
|
445
|
+
'gap',
|
|
446
|
+
'an object, or null when this cycle watched the whole stretch',
|
|
447
|
+
(v) => v === null || isObject(v),
|
|
448
|
+
),
|
|
449
|
+
],
|
|
450
|
+
'example-pruning': [
|
|
451
|
+
rule('provider', 'the provider the calls went to', (v) => typeof v === 'string'),
|
|
452
|
+
rule('model', 'the model that answered', (v) => typeof v === 'string'),
|
|
453
|
+
rule('selfAgreement', 'the model’s agreement with itself, the yardstick every removal is judged against', isNumber),
|
|
454
|
+
rule('recoverableTokens', 'tokens held by examples whose removal changed nothing measurable', isNumber),
|
|
455
|
+
rule('callsMade', 'calls actually spent', isNumber),
|
|
456
|
+
rule('contributions', 'an array — one entry per example', isArray),
|
|
457
|
+
],
|
|
397
458
|
pulse: [
|
|
398
459
|
rule('beats', 'an array — one per kind', isArray),
|
|
399
460
|
rule('nowMs', 'the instant the ages were taken against', isNumber),
|
|
@@ -554,6 +615,44 @@ function contractOf(doc: Record<string, unknown>): Exclude<ContractName, 'usage-
|
|
|
554
615
|
if (Array.isArray(doc.actions) && typeof doc.arrived === 'number') return 'verification';
|
|
555
616
|
if (Array.isArray(doc.actions)) return 'plan';
|
|
556
617
|
if (typeof doc.verdict === 'string' && 'restsOn' in doc) return 'cost-answer';
|
|
618
|
+
/*
|
|
619
|
+
Two the inference could never reach, found by the guard that demands every
|
|
620
|
+
contract be recognisable from a document of its own shape.
|
|
621
|
+
|
|
622
|
+
The cost answer's branch above tested `restsOn`, which its contract does
|
|
623
|
+
not require: the required set is verdict, call and budget, so a conforming
|
|
624
|
+
answer that omitted the optional field was recognised as nothing at all.
|
|
625
|
+
The prompt draft had no branch of any kind — its schema has been published
|
|
626
|
+
since 1.68 and `trazum conform draft.json` has never known what it was.
|
|
627
|
+
*/
|
|
628
|
+
if (typeof doc.verdict === 'string' && 'call' in doc && 'budget' in doc) return 'cost-answer';
|
|
629
|
+
if (typeof doc.prompt === 'string' && Array.isArray(doc.sections) && Array.isArray(doc.missing)) {
|
|
630
|
+
return 'prompt-draft';
|
|
631
|
+
}
|
|
632
|
+
/*
|
|
633
|
+
The five the JSON sweep contracted, and the reason they are here.
|
|
634
|
+
|
|
635
|
+
All five arrived with rules, a schema, a row in the interchange table and
|
|
636
|
+
a parity test, and none of them arrived with a branch in this function —
|
|
637
|
+
so `trazum conform verdict.json` read a real routing measurement as a
|
|
638
|
+
usage log and answered with advice about `stop_reason`. A contract that
|
|
639
|
+
cannot be recognised is a contract only reachable by somebody who already
|
|
640
|
+
knows its name, which is the opposite of what this page is for. Found by
|
|
641
|
+
running the command against its own output; `contract-coverage.test.js`
|
|
642
|
+
now requires every name to be inferable from a document.
|
|
643
|
+
|
|
644
|
+
Conformance sits before the connected report deliberately: both carry an
|
|
645
|
+
`unavailable` array, and only the conformance verdict carries `conforms`.
|
|
646
|
+
*/
|
|
647
|
+
if (Array.isArray(doc.problems) && Array.isArray(doc.unavailable) && typeof doc.conforms === 'boolean') {
|
|
648
|
+
return 'conformance';
|
|
649
|
+
}
|
|
650
|
+
if (isObject(doc.evaluation) && 'slice' in doc) return 'routing-measurement';
|
|
651
|
+
if (Array.isArray(doc.contributions) && typeof doc.recoverableTokens === 'number') {
|
|
652
|
+
return 'example-pruning';
|
|
653
|
+
}
|
|
654
|
+
if (Array.isArray(doc.providers) && typeof doc.totalRecords === 'number') return 'store-inventory';
|
|
655
|
+
if (Array.isArray(doc.crossings) && Array.isArray(doc.abstentions)) return 'watch-cycle';
|
|
557
656
|
if (typeof doc.provider === 'string' && Array.isArray(doc.unavailable)) return 'connected';
|
|
558
657
|
return null;
|
|
559
658
|
}
|
|
@@ -650,7 +749,7 @@ export function conform(text: string, options: ConformOptions = {}): Conformance
|
|
|
650
749
|
schemaVersion: 1,
|
|
651
750
|
contract: null,
|
|
652
751
|
because:
|
|
653
|
-
'no contract recognised it — a roll-up has contributors and notMerged, a profile has byLabelAndModel, a plan has actions, a verification adds arrived, a history has periods and runs, a connected report has provider and unavailable, a cost answer has verdict and
|
|
752
|
+
'no contract recognised it — a roll-up has contributors and notMerged, a profile has byLabelAndModel, a plan has actions, a verification adds arrived, a history has periods and runs, a connected report has provider and unavailable, a cost answer has verdict with call and budget, a prompt draft has prompt, sections and missing, a routing measurement has slice and evaluation, an example pruning has contributions and recoverableTokens, a store inventory has providers and totalRecords, a conformance verdict has problems and conforms, a watch cycle has crossings and abstentions',
|
|
654
753
|
problems: [],
|
|
655
754
|
unavailable: [],
|
|
656
755
|
records: null,
|