@trazum/core 1.81.0 → 1.83.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -18
- package/dist/advisories.d.ts +33 -1
- package/dist/advisories.d.ts.map +1 -1
- package/dist/advisories.js +130 -20
- package/dist/advisories.js.map +1 -1
- package/dist/band.d.ts +131 -0
- package/dist/band.d.ts.map +1 -0
- package/dist/band.js +210 -0
- package/dist/band.js.map +1 -0
- package/dist/conform.d.ts +1 -1
- package/dist/conform.d.ts.map +1 -1
- package/dist/conform.js +84 -1
- package/dist/conform.js.map +1 -1
- package/dist/helicone.d.ts +92 -0
- package/dist/helicone.d.ts.map +1 -0
- package/dist/helicone.js +194 -0
- package/dist/helicone.js.map +1 -0
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +16 -3
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +17 -3
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +60 -1
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +16 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -3
- package/dist/index.js.map +1 -1
- package/dist/langsmith.d.ts +104 -0
- package/dist/langsmith.d.ts.map +1 -0
- package/dist/langsmith.js +270 -0
- package/dist/langsmith.js.map +1 -0
- package/dist/levers.d.ts +1 -1
- package/dist/levers.js +2 -2
- package/dist/levers.js.map +1 -1
- package/dist/litellm.d.ts +98 -0
- package/dist/litellm.d.ts.map +1 -0
- package/dist/litellm.js +207 -0
- package/dist/litellm.js.map +1 -0
- package/dist/measured-profile.d.ts +4 -0
- package/dist/measured-profile.d.ts.map +1 -1
- package/dist/measured-profile.js +22 -1
- package/dist/measured-profile.js.map +1 -1
- package/dist/pricing.d.ts +98 -3
- package/dist/pricing.d.ts.map +1 -1
- package/dist/pricing.js +393 -35
- package/dist/pricing.js.map +1 -1
- package/dist/receipt.d.ts +253 -0
- package/dist/receipt.d.ts.map +1 -0
- package/dist/receipt.js +111 -0
- package/dist/receipt.js.map +1 -0
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +36 -0
- package/dist/schemas.js.map +1 -1
- package/dist/tokenizer.d.ts +45 -49
- package/dist/tokenizer.d.ts.map +1 -1
- package/dist/tokenizer.js +100 -51
- package/dist/tokenizer.js.map +1 -1
- package/dist/types.d.ts +108 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/verdict-bridge.d.ts +113 -0
- package/dist/verdict-bridge.d.ts.map +1 -0
- package/dist/verdict-bridge.js +127 -0
- package/dist/verdict-bridge.js.map +1 -0
- package/package.json +1 -1
- package/src/advisories.ts +166 -19
- package/src/band.ts +213 -0
- package/src/conform.ts +100 -1
- package/src/helicone.ts +227 -0
- package/src/i18n/en.ts +19 -3
- package/src/i18n/es.ts +20 -3
- package/src/i18n/types.ts +63 -1
- package/src/index.ts +31 -2
- package/src/langsmith.ts +315 -0
- package/src/levers.ts +3 -3
- package/src/litellm.ts +243 -0
- package/src/measured-profile.ts +26 -2
- package/src/pricing.ts +430 -30
- package/src/receipt.ts +351 -0
- package/src/schemas.ts +36 -0
- package/src/tokenizer.ts +101 -51
- package/src/types.ts +104 -0
- package/src/verdict-bridge.ts +200 -0
package/dist/tokenizer.js
CHANGED
|
@@ -2,20 +2,22 @@
|
|
|
2
2
|
* Dependency-free token estimator.
|
|
3
3
|
*
|
|
4
4
|
* This is NOT a real tokenizer: it is a heuristic calibrated per character
|
|
5
|
-
* class
|
|
6
|
-
*
|
|
7
|
-
* two versions of the same prompt — but do NOT bill anyone from it.
|
|
5
|
+
* class, built for comparing two versions of the same prompt — but do NOT bill
|
|
6
|
+
* anyone from it.
|
|
8
7
|
*
|
|
9
|
-
* **
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
8
|
+
* **Its accuracy is not one number, and the years it was written as one were
|
|
9
|
+
* the mistake.** The branches below treat CJK, digits and punctuation quite
|
|
10
|
+
* differently from words, and there was never a reason those should land on the
|
|
11
|
+
* same accuracy. Measured over 47 samples against the official counting
|
|
12
|
+
* endpoint they do not: the worst error is 3.2% on CJK, 5.6% on Latin prose,
|
|
13
|
+
* 25.1% on code and quoting, and 32.5% on a digit-dominant table. `band.ts`
|
|
14
|
+
* holds the bands those measurements earn and `bandFor(text)` is what a caller
|
|
15
|
+
* should print; the constant below is only the widest of them, for callers that
|
|
16
|
+
* have no text in hand.
|
|
16
17
|
*
|
|
17
|
-
* `test/token-band.test.js`
|
|
18
|
-
*
|
|
18
|
+
* `test/token-band.test.js` holds every sample against the band `bandFor` gives
|
|
19
|
+
* it, and a sample edited since it was measured fails rather than passing
|
|
20
|
+
* quietly.
|
|
19
21
|
*
|
|
20
22
|
* For exact numbers use `countTokensAnthropic` (the official token-counting
|
|
21
23
|
* endpoint, which is free) or pass your own `TokenCounter`.
|
|
@@ -23,52 +25,46 @@
|
|
|
23
25
|
import { detectTextLanguage } from './language.js';
|
|
24
26
|
import { SAFE_FETCH_INIT, checkedEndpoint } from './net.js';
|
|
25
27
|
/**
|
|
26
|
-
* The
|
|
28
|
+
* The widest band this estimator is published under, as a percentage.
|
|
27
29
|
*
|
|
28
|
-
* **
|
|
29
|
-
* releases as a design target nobody had checked; the first
|
|
30
|
-
* of eight samples outside it and it went to `25`; fixing
|
|
31
|
-
* calibrating per language brought it back to `15
|
|
32
|
-
*
|
|
30
|
+
* **A history worth keeping, because each value was believed at the time.** It
|
|
31
|
+
* was `15` for eight releases as a design target nobody had checked; the first
|
|
32
|
+
* measurement found two of eight samples outside it and it went to `25`; fixing
|
|
33
|
+
* the digit divisor and calibrating per language brought it back to `15`, and
|
|
34
|
+
* splitting kana from han — Japanese had been +11.2% and Chinese −3.2% under one
|
|
35
|
+
* rule — brought it to `10`. Four values, each an improvement, and all four
|
|
36
|
+
* wrong in the same way.
|
|
33
37
|
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
+
* **The corpus they were measured on was the corpus they were fitted to.**
|
|
39
|
+
* Twenty-one samples held thirteen files of Latin prose and exactly one each of
|
|
40
|
+
* code, numeric and punctuation, and those single files were where the constants
|
|
41
|
+
* came from. The worst error it could show was 6.4%, and 10 looked like a
|
|
42
|
+
* comfortable margin over it. Neither figure was about the estimator.
|
|
38
43
|
*
|
|
39
|
-
* **
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* safe direction for a tool that reports money.
|
|
44
|
+
* **It is no longer one number, and this one is the widest.** A corpus of
|
|
45
|
+
* thirteen prose files and one each of code, numeric and punctuation put the
|
|
46
|
+
* band at 10, and those single files were the set the constants had been fitted
|
|
47
|
+
* to. Measured on forty-seven, the same estimator is 6% out on prose and 33%
|
|
48
|
+
* out on a CSV ledger. `bandFor` in `band.ts` is what a caller that has the
|
|
49
|
+
* text should use.
|
|
46
50
|
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* **Which samples the band rests on matters more than the number.** Eight of the
|
|
54
|
-
* twenty-one had a constant fitted to them — seven Latin divisors and the digit
|
|
55
|
-
* divisor — so their residuals are optimistic by construction. The two worst
|
|
56
|
-
* errors in the corpus, `code-heavy` at 6.4% and `punctuation-heavy` at 5.7%, are
|
|
57
|
-
* fitted to nothing at all, and they are what sets this figure.
|
|
58
|
-
*
|
|
59
|
-
* Exported so every report, README and tool description reads the same number. It
|
|
60
|
-
* was a literal in twenty-four files before this, with the only machine-readable
|
|
61
|
-
* copy in a test, and `token-band.test.js` now fails any file that states a
|
|
62
|
-
* different one.
|
|
51
|
+
* This stays for the callers that do not have it — a figure covering the whole
|
|
52
|
+
* catalogue of text, which can only be the worst of them. Widening it from 10
|
|
53
|
+
* to 33 makes every claim that reads it true where it had been false, at the
|
|
54
|
+
* cost of understating the estimator on prose. That trade is deliberate: a
|
|
55
|
+
* caller with no text cannot know which it is holding, and there is only one
|
|
56
|
+
* safe direction to guess in.
|
|
63
57
|
*/
|
|
64
|
-
export const ESTIMATE_ERROR_BAND_PCT =
|
|
58
|
+
export const ESTIMATE_ERROR_BAND_PCT = 33;
|
|
65
59
|
/**
|
|
66
60
|
* Whose tokenizer that band was measured against.
|
|
67
61
|
*
|
|
68
62
|
* The number above is not a property of the estimator; it is a property of the
|
|
69
|
-
* estimator **against Claude**.
|
|
70
|
-
* family. Trazum prices seven, and
|
|
71
|
-
*
|
|
63
|
+
* estimator **against Claude**. Forty-seven samples, one counting endpoint, one
|
|
64
|
+
* family. Trazum prices seven, and the two that have since been run are far
|
|
65
|
+
* outside anything published here: DeepSeek 94.5% at worst, Mistral 103.1%. On
|
|
66
|
+
* a GPT or a Gemini prompt nobody has measured it at all, and
|
|
67
|
+
* `measuredForeignError` answers null rather than the nearest number.
|
|
72
68
|
*
|
|
73
69
|
* Exported as a name rather than left implicit in a comment because the band
|
|
74
70
|
* had already leaked out of its domain in three places: an advisory that told a
|
|
@@ -164,6 +160,33 @@ const DIVISOR_BY_LANGUAGE = {
|
|
|
164
160
|
*/
|
|
165
161
|
const KANA_TOKENS_PER_CHAR = 0.75;
|
|
166
162
|
const HAN_TOKENS_PER_CHAR = 1.05;
|
|
163
|
+
/**
|
|
164
|
+
* Hangul, measured at last.
|
|
165
|
+
*
|
|
166
|
+
* The comment above used to end *"Hangul keeps the old cost of 1, because
|
|
167
|
+
* nothing here measures Korean. Guessing it from Japanese would be inventing a
|
|
168
|
+
* figure."* That was right, and it stayed right for exactly as long as nobody
|
|
169
|
+
* measured Korean. `cjk-korean.txt` now does: at 1 token per character the
|
|
170
|
+
* estimate came out **20.0% under** — 232 against 290 — and 1.20 takes it to
|
|
171
|
+
* -10.0%.
|
|
172
|
+
*
|
|
173
|
+
* **1.20 was the one-sample answer and it was wrong by ten points.** A second
|
|
174
|
+
* Korean sample in a different register — a deployment review against support
|
|
175
|
+
* copy — put both at -10.6% and -10.0% under it, and solving the constant on
|
|
176
|
+
* the two together lands on 1.35, where both come out at 0.0%.
|
|
177
|
+
*
|
|
178
|
+
* Two independent texts agreeing to within four tenths of a point across the
|
|
179
|
+
* whole search is what makes this a property of Hangul rather than a fit to a
|
|
180
|
+
* file. Their residuals move in lockstep at every candidate value: -10.6/-10.0
|
|
181
|
+
* at 1.20, -3.5/-3.1 at 1.30, 3.2/3.4 at 1.40. One sample cannot show that and
|
|
182
|
+
* two can, which is the entire argument for measuring more of the thin classes
|
|
183
|
+
* rather than tuning harder on the thin ones.
|
|
184
|
+
*
|
|
185
|
+
* Still in-sample: the honest test is a third Korean prompt, and the corpus
|
|
186
|
+
* grows one at a time.
|
|
187
|
+
*/
|
|
188
|
+
const HANGUL_TOKENS_PER_CHAR = 1.35;
|
|
189
|
+
const HANGUL = /[가-]/;
|
|
167
190
|
/**
|
|
168
191
|
* What a prompt gets when the language could not be told.
|
|
169
192
|
*
|
|
@@ -233,7 +256,11 @@ export function estimateTokens(text) {
|
|
|
233
256
|
* where the loop happens to break, not of what the text costs.
|
|
234
257
|
*/
|
|
235
258
|
while (i < chars.length && CJK.test(chars[i])) {
|
|
236
|
-
cjkTokens += KANA.test(chars[i])
|
|
259
|
+
cjkTokens += KANA.test(chars[i])
|
|
260
|
+
? KANA_TOKENS_PER_CHAR
|
|
261
|
+
: HANGUL.test(chars[i])
|
|
262
|
+
? HANGUL_TOKENS_PER_CHAR
|
|
263
|
+
: HAN_TOKENS_PER_CHAR;
|
|
237
264
|
i++;
|
|
238
265
|
}
|
|
239
266
|
continue;
|
|
@@ -295,7 +322,29 @@ export function estimateTokens(text) {
|
|
|
295
322
|
i++;
|
|
296
323
|
}
|
|
297
324
|
else {
|
|
298
|
-
|
|
325
|
+
/**
|
|
326
|
+
* One token per symbol, not one per two.
|
|
327
|
+
*
|
|
328
|
+
* Two was the shipped value and every symbol-dense sample in the corpus
|
|
329
|
+
* undercounted under it, in the same direction, by amounts that grow with
|
|
330
|
+
* density: punctuation-heavy -5.7%, code-heavy -6.4%,
|
|
331
|
+
* punctuation-markup -11.4%, code-sql -25.6%, code-shell -29.8%. Five
|
|
332
|
+
* samples agreeing on a direction is a shape, not a fit — a merge table
|
|
333
|
+
* covers `, ` and `. ` and covers `${`, `"$`, `--` and `‑` far less
|
|
334
|
+
* well, and shell and SQL are made of the second kind.
|
|
335
|
+
*
|
|
336
|
+
* Searched over the whole corpus rather than the class it helps, so a
|
|
337
|
+
* value that fixed code by wrecking prose could not look like a win: at
|
|
338
|
+
* 1.0 the samples outside the published band drop from five to three and
|
|
339
|
+
* the mean error from 6.6% to 5.1%, with no sample moved more than a
|
|
340
|
+
* point in the wrong direction.
|
|
341
|
+
*
|
|
342
|
+
* The search wanted to go below 1.0 and was not allowed to. Below one is
|
|
343
|
+
* more than a token per symbol, which may well be true of SQL — code-sql
|
|
344
|
+
* is still 24.9% under — but it is a claim about a class this corpus has
|
|
345
|
+
* two samples of, and two samples do not earn a constant.
|
|
346
|
+
*/
|
|
347
|
+
total += Math.ceil(n / 1);
|
|
299
348
|
}
|
|
300
349
|
}
|
|
301
350
|
// Rounded once, over the whole document, rather than per run — see the CJK branch.
|
package/dist/tokenizer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC;AAEpD;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,QAAmC;IAC7D,OAAO,QAAQ,KAAK,wBAAwB,CAAC;AAC/C,CAAC;AAED,MAAM,GAAG,GAAG,gBAAgB,CAAC;AAE7B;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,IAAI,GAAG,OAAO,CAAC;AACrB,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,mBAAmB,GAAqC;IAC5D,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;CACT,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,MAAM,MAAM,GAAG,OAAO,CAAC;AAEvB;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,0EAA0E;AAC1E,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,EAAE,IAAI,IAAI;QAAE,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,8EAA8E;IAC9E,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,IAAI,eAAe,CAAC;IAEnG,IAAI,KAAK,GAAG,CAAC,CAAC;IACd;;;OAGG;IACH,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7C,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB;;;;;;;;eAQG;YACH,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBAC/C,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;oBAC/B,CAAC,CAAC,oBAAoB;oBACtB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;wBACtB,CAAC,CAAC,sBAAsB;wBACxB,CAAC,CAAC,mBAAmB,CAAC;gBAC1B,CAAC,EAAE,CAAC;YACN,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,EAAE,CAAC;gBAC7E,IAAI,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC;gBAClB,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACjD,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD;;;;;;;;;;;;eAYG;YACH,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,+DAA+D;QAC/D,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OACE,CAAC,GAAG,KAAK,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACtB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACpB,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;YAChB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,MAAM,EACnC,CAAC;YACD,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,8EAA8E;YAC9E,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,CAAC;YACN;;;;;;;;;;;;;;;;;;;;;eAqBG;YACH,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,mFAAmF;IACnF,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACtC,CAAC;AAeD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAgC;IACnE,MAAM,EACJ,MAAM,EACN,KAAK,GAAG,eAAe,EACvB,OAAO,GAAG,2BAA2B,EACrC,SAAS,GAAG,KAAK,EACjB,aAAa,GAAG,KAAK,GACtB,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAE7F,OAAO,KAAK,EAAE,IAAY,EAAmB,EAAE;QAC7C,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,QAAQ,2BAA2B,EAAE;YAClE,GAAG,eAAe;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,YAAY;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6B,CAAC;QAC5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -48,7 +48,7 @@ export interface RuleResult {
|
|
|
48
48
|
/** Severity of an advisory: the higher it is, the more money is usually at stake. */
|
|
49
49
|
export type AdvisorySeverity = 'info' | 'opportunity' | 'warning';
|
|
50
50
|
/** Every advisory the core can emit. */
|
|
51
|
-
export type AdvisoryId = 'context-overflow' | 'context-near-limit' | 'prompt-caching' | 'prompt-caching-not-worth-it' | 'below-cache-minimum' | 'cache-prefix-reorder' | 'batch-api' | 'model-downgrade' | 'output-dominated' | 'promo-pricing' | 'contradictory-instructions' | 'redundant-examples' | 'restated-output-format' | 'movable-output-schema';
|
|
51
|
+
export type AdvisoryId = 'context-overflow' | 'context-near-limit' | 'prompt-caching' | 'prompt-caching-not-worth-it' | 'below-cache-minimum' | 'cache-prefix-reorder' | 'batch-api' | 'model-downgrade' | 'tier-signals-conflict' | 'output-dominated' | 'promo-pricing' | 'model-retired' | 'contradictory-instructions' | 'redundant-examples' | 'restated-output-format' | 'movable-output-schema';
|
|
52
52
|
/** A recommendation that does NOT modify the prompt, only informs. */
|
|
53
53
|
export interface Advisory {
|
|
54
54
|
id: AdvisoryId;
|
|
@@ -125,6 +125,64 @@ export type CachingMode =
|
|
|
125
125
|
* what it says.
|
|
126
126
|
*/
|
|
127
127
|
export type Capability = 'small' | 'mid' | 'large' | 'frontier' | 'unknown';
|
|
128
|
+
/**
|
|
129
|
+
* What makes a conditional rate apply, as data a machine can evaluate.
|
|
130
|
+
*
|
|
131
|
+
* Deliberately a closed union rather than a predicate function. A catalogue
|
|
132
|
+
* entry is reviewed the way a price is reviewed, and a reviewer can check
|
|
133
|
+
* "peak is 01:00-04:00 and 06:00-10:00 UTC on weekdays" against a provider's
|
|
134
|
+
* page; nobody can review a closure. It also means the condition survives
|
|
135
|
+
* `JSON.stringify`, which matters because the catalogue can be overlaid from a
|
|
136
|
+
* file.
|
|
137
|
+
*/
|
|
138
|
+
export type TierCondition =
|
|
139
|
+
/**
|
|
140
|
+
* The prompt is larger than `tokens` input tokens.
|
|
141
|
+
*
|
|
142
|
+
* Decidable only where the token count is known, which is most of the places
|
|
143
|
+
* that price a specific call and none of the places that merely rank models.
|
|
144
|
+
* `effectivePricing` says which happened rather than guessing.
|
|
145
|
+
*/
|
|
146
|
+
{
|
|
147
|
+
kind: 'input-tokens-above';
|
|
148
|
+
tokens: number;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The call falls inside one of `hours` in UTC, optionally on weekdays only.
|
|
152
|
+
*
|
|
153
|
+
* Decidable everywhere, because every caller already passes the date it is
|
|
154
|
+
* pricing for. `hours` lists the hour each window *starts*, so 01:00-04:00
|
|
155
|
+
* is `[1, 2, 3]` and not `[1, 4]`: an endpoint convention is the kind of
|
|
156
|
+
* thing two readers disagree about silently.
|
|
157
|
+
*/
|
|
158
|
+
| {
|
|
159
|
+
kind: 'utc-hours';
|
|
160
|
+
hours: number[];
|
|
161
|
+
weekdaysOnly: boolean;
|
|
162
|
+
};
|
|
163
|
+
/**
|
|
164
|
+
* A rate that applies when its condition holds.
|
|
165
|
+
*
|
|
166
|
+
* Deliberately carries no multipliers of its own, and the first draft did. The
|
|
167
|
+
* reasoning for adding them was that DeepSeek halves the cache-hit price at
|
|
168
|
+
* off-peak along with everything else, so a cache read priced off the base
|
|
169
|
+
* would be wrong about the money — which is true of the *price* and false of
|
|
170
|
+
* the *ratio*: $0.007 against $0.22 and $0.014 against $0.44 are the same
|
|
171
|
+
* 3.18% to the last digit, on both models. Multiplying a tier's own input rate
|
|
172
|
+
* by the model's multiplier already gives the right answer.
|
|
173
|
+
*
|
|
174
|
+
* It was removed because the guard written to prove it mattered could not: with
|
|
175
|
+
* the tier multiplier equal to the base one, dropping it changed nothing and
|
|
176
|
+
* the test passed either way. An option nothing can distinguish from its
|
|
177
|
+
* absence is a claim nobody checks, which is worse than not having it.
|
|
178
|
+
*/
|
|
179
|
+
export interface PricingTier {
|
|
180
|
+
/** Short, stable, and printed: a reader has to be able to see which applied. */
|
|
181
|
+
id: string;
|
|
182
|
+
when: TierCondition;
|
|
183
|
+
inputPerMTok: number;
|
|
184
|
+
outputPerMTok: number;
|
|
185
|
+
}
|
|
128
186
|
export interface ModelPricing {
|
|
129
187
|
id: string;
|
|
130
188
|
displayName: string;
|
|
@@ -156,6 +214,32 @@ export interface ModelPricing {
|
|
|
156
214
|
* buy is advice that wastes their time.
|
|
157
215
|
*/
|
|
158
216
|
recommendable?: boolean;
|
|
217
|
+
/**
|
|
218
|
+
* Set when the provider refused a real request for this id.
|
|
219
|
+
*
|
|
220
|
+
* A retired model keeps its price and loses its future. Both halves matter:
|
|
221
|
+
* a log full of `deepseek-v3` calls records money that was genuinely spent,
|
|
222
|
+
* so deleting the row would make somebody's own history unpriceable — and a
|
|
223
|
+
* *new* call on that id will not happen at all, so quoting it as an option is
|
|
224
|
+
* a number for a question nobody can ask.
|
|
225
|
+
*
|
|
226
|
+
* `on` is the day `scripts/check-model-availability.mjs` was refused, and
|
|
227
|
+
* `because` is the provider's own sentence, quoted rather than paraphrased.
|
|
228
|
+
* Neither is a judgement, and neither implies anything about what the
|
|
229
|
+
* replacement costs: that is a price, and a price comes off the provider's
|
|
230
|
+
* own page or not at all.
|
|
231
|
+
*
|
|
232
|
+
* A retired model is never recommended, never offered as a cheaper
|
|
233
|
+
* alternative and never chosen as a tier's representative, in the same places
|
|
234
|
+
* `recommendable: false` is honoured — this is a stronger statement than that
|
|
235
|
+
* one and must not be weaker in effect.
|
|
236
|
+
*/
|
|
237
|
+
retired?: {
|
|
238
|
+
/** ISO date the refusal was recorded. */
|
|
239
|
+
on: string;
|
|
240
|
+
/** What the provider answered, in the provider's words. */
|
|
241
|
+
because: string;
|
|
242
|
+
};
|
|
159
243
|
/** Capability, on a vendor-neutral scale. */
|
|
160
244
|
capability: Capability;
|
|
161
245
|
/**
|
|
@@ -171,6 +255,29 @@ export interface ModelPricing {
|
|
|
171
255
|
/** ISO date (inclusive) the promotional price applies until. */
|
|
172
256
|
until: string;
|
|
173
257
|
};
|
|
258
|
+
/**
|
|
259
|
+
* Rates that apply only under a stated condition.
|
|
260
|
+
*
|
|
261
|
+
* The catalogue held one input price and one output price per model until a
|
|
262
|
+
* real credential was pointed at the providers and two of them turned out not
|
|
263
|
+
* to charge that way:
|
|
264
|
+
*
|
|
265
|
+
* - **DeepSeek V4 bills by the clock.** Peak is 01:00-04:00 and 06:00-10:00
|
|
266
|
+
* UTC, Monday to Friday, and off-peak is exactly half. That is 35 hours of
|
|
267
|
+
* 168, so the common case is the cheap one and a single number would have
|
|
268
|
+
* been wrong by 2x in whichever direction it was chosen.
|
|
269
|
+
* - **Gemini 3.1 Pro bills by prompt size**, $2/$12 up to 200k input tokens
|
|
270
|
+
* and $4/$18 above it.
|
|
271
|
+
*
|
|
272
|
+
* Both are *conditions*, so both are written as data rather than as prose in
|
|
273
|
+
* `notes`: a note is something a reader has to apply by hand, and this
|
|
274
|
+
* product exists to stop people doing arithmetic by hand.
|
|
275
|
+
*
|
|
276
|
+
* The base `inputPerMTok` and `outputPerMTok` stay the rate that applies when
|
|
277
|
+
* no condition matches, which keeps every model that has no tiers unchanged
|
|
278
|
+
* and keeps this field optional in effect as well as in the type.
|
|
279
|
+
*/
|
|
280
|
+
tiers?: PricingTier[];
|
|
174
281
|
/**
|
|
175
282
|
* Other ids that bill at exactly this rate.
|
|
176
283
|
*
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,oCAAoC;AAEpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,gEAAgE;AAChE;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,YAAI,MAAM,EAAE,YAAY,CAAU,CAAC;AAE3D,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,SAAS,CAAC;IACjB,wEAAwE;IACxE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACrD;AAED,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,qFAAqF;AACrF,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,aAAa,GAAG,SAAS,CAAC;AAElE,wCAAwC;AACxC,MAAM,MAAM,UAAU,GAClB,kBAAkB,GAClB,oBAAoB,GACpB,gBAAgB,GAChB,6BAA6B,GAC7B,qBAAqB,GACrB,sBAAsB,GACtB,WAAW,GACX,iBAAiB,GACjB,kBAAkB,GAClB,eAAe,GACf,4BAA4B,GAC5B,oBAAoB,GACpB,wBAAwB,GACxB,uBAAuB,CAAC;AAE5B,sEAAsE;AACtE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,UAAU,CAAC;IACf,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,mDAAmD;AACnD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,eAAe;IAC9B,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,WAAW;AACrB,qEAAqE;AACnE,UAAU;AACZ,+DAA+D;GAC7D,WAAW;AACb,gCAAgC;GAC9B,MAAM;AACR;;;;;;;;;;;;;;GAcG;GACD,SAAS,CAAC;AAEd;;;;;;;GAOG;AACH,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;AAE5E,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,sDAAsD;IACtD,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6CAA6C;IAC7C,UAAU,EAAE,UAAU,CAAC;IACvB;;;;OAIG;IACH,IAAI,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IAC3D,gDAAgD;IAChD,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,gEAAgE;QAChE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,4DAA4D;AAC5D,MAAM,WAAW,YAAY;IAC3B,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,sCAAsC;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,8DAA8D;IAC9D,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,aAAa,CAAA;KAAE,CAAC;IACzD,QAAQ,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,aAAa,CAAA;KAAE,CAAC;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,kEAAkE;AAClE,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,SAAS,GAAG,WAAW,CAAC;IAC9B,gEAAgE;IAChE,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,cAAc,GACtB,aAAa,GACb,aAAa,GACb,KAAK,GACL,aAAa,GACb,SAAS,CAAC;AAEd,4BAA4B;AAC5B,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,iDAAiD;IACjD,KAAK,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9B,oEAAoE;IACpE,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED,+EAA+E;AAC/E,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AAEpD,sCAAsC;AACtC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB,OAAO,EAAE,aAAa,CAAC;IACvB,KAAK,EAAE,YAAY,CAAC;IACpB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,WAAW,EAAE,WAAW,GAAG,UAAU,CAAC;IACtC;;;;;;OAMG;IACH,aAAa,EAAE;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,+DAA+D;QAC/D,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,yDAAyD;QACzD,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,yCAAyC;IACzC,GAAG,CAAC,EAAE,aAAa,CAAC;CACrB;AAED,uCAAuC;AACvC,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,qEAAqE;AACrE,MAAM,WAAW,WAAW;IAC1B,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpE;AAED,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,oCAAoC;AAEpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,gEAAgE;AAChE;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,YAAI,MAAM,EAAE,YAAY,CAAU,CAAC;AAE3D,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,SAAS,CAAC;IACjB,wEAAwE;IACxE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACrD;AAED,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,qFAAqF;AACrF,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,aAAa,GAAG,SAAS,CAAC;AAElE,wCAAwC;AACxC,MAAM,MAAM,UAAU,GAClB,kBAAkB,GAClB,oBAAoB,GACpB,gBAAgB,GAChB,6BAA6B,GAC7B,qBAAqB,GACrB,sBAAsB,GACtB,WAAW,GACX,iBAAiB,GACjB,uBAAuB,GACvB,kBAAkB,GAClB,eAAe,GACf,eAAe,GACf,4BAA4B,GAC5B,oBAAoB,GACpB,wBAAwB,GACxB,uBAAuB,CAAC;AAE5B,sEAAsE;AACtE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,UAAU,CAAC;IACf,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,mDAAmD;AACnD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,eAAe;IAC9B,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,WAAW;AACrB,qEAAqE;AACnE,UAAU;AACZ,+DAA+D;GAC7D,WAAW;AACb,gCAAgC;GAC9B,MAAM;AACR;;;;;;;;;;;;;;GAcG;GACD,SAAS,CAAC;AAEd;;;;;;;GAOG;AACH,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;AAE5E;;;;;;;;;GASG;AACH,MAAM,MAAM,aAAa;AACvB;;;;;;GAMG;AACD;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AAChD;;;;;;;GAOG;GACD;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,YAAY,EAAE,OAAO,CAAA;CAAE,CAAC;AAElE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,WAAW;IAC1B,gFAAgF;IAChF,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,aAAa,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,sDAAsD;IACtD,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,EAAE;QACR,yCAAyC;QACzC,EAAE,EAAE,MAAM,CAAC;QACX,2DAA2D;QAC3D,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,6CAA6C;IAC7C,UAAU,EAAE,UAAU,CAAC;IACvB;;;;OAIG;IACH,IAAI,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IAC3D,gDAAgD;IAChD,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,gEAAgE;QAChE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC;IACtB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,4DAA4D;AAC5D,MAAM,WAAW,YAAY;IAC3B,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,sCAAsC;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,8DAA8D;IAC9D,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,aAAa,CAAA;KAAE,CAAC;IACzD,QAAQ,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,aAAa,CAAA;KAAE,CAAC;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,kEAAkE;AAClE,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,SAAS,GAAG,WAAW,CAAC;IAC9B,gEAAgE;IAChE,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,cAAc,GACtB,aAAa,GACb,aAAa,GACb,KAAK,GACL,aAAa,GACb,SAAS,CAAC;AAEd,4BAA4B;AAC5B,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,iDAAiD;IACjD,KAAK,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9B,oEAAoE;IACpE,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED,+EAA+E;AAC/E,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AAEpD,sCAAsC;AACtC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB,OAAO,EAAE,aAAa,CAAC;IACvB,KAAK,EAAE,YAAY,CAAC;IACpB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,WAAW,EAAE,WAAW,GAAG,UAAU,CAAC;IACtC;;;;;;OAMG;IACH,aAAa,EAAE;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,+DAA+D;QAC/D,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,yDAAyD;QACzD,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,yCAAyC;IACzC,GAAG,CAAC,EAAE,aAAa,CAAC;CACrB;AAED,uCAAuC;AACvC,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,qEAAqE;AACrE,MAAM,WAAW,WAAW;IAC1B,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpE;AAED,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a dropped `route --json` verdict, so quality can stand beside cost.
|
|
3
|
+
*
|
|
4
|
+
* The bill is arithmetic: every figure in it comes from records already on
|
|
5
|
+
* disk, and it can say exactly what a cheaper model would have saved. It can
|
|
6
|
+
* say nothing at all about whether that model still does the job. That answer
|
|
7
|
+
* costs provider calls and a credential, which is why `trazum route` exists
|
|
8
|
+
* and why no surface that only reads a log will ever produce it.
|
|
9
|
+
*
|
|
10
|
+
* This is the bridge between the two, and it is deliberately one-directional:
|
|
11
|
+
* the measurement is made where the credential is, written out as the
|
|
12
|
+
* `routing-measurement` contract, and read back here by a pure function that
|
|
13
|
+
* touches no network and holds no key. A reader who has run `route` once can
|
|
14
|
+
* carry the verdict to any surface that prices the same workload.
|
|
15
|
+
*
|
|
16
|
+
* It was not possible to write until the document was worth reading. Before
|
|
17
|
+
* the JSON-text sweep, `route --json` printed the whole `EvalReport` — every
|
|
18
|
+
* case input and three model answers per case — so a bridge would have been a
|
|
19
|
+
* feature for carrying prompt text into a browser. The document carries the
|
|
20
|
+
* measurement and nothing else now, and `conform` is what says so rather than
|
|
21
|
+
* this file's own reading: the contract validates, the bridge only maps.
|
|
22
|
+
*/
|
|
23
|
+
import type { EvalVerdict } from './evaluate.js';
|
|
24
|
+
/** What a dropped routing measurement says, mapped onto what a bill can match. */
|
|
25
|
+
export interface DroppedVerdict {
|
|
26
|
+
/** The workload the measurement was made on, or null when it was unlabelled. */
|
|
27
|
+
label: string | null;
|
|
28
|
+
/**
|
|
29
|
+
* The model those calls go to today, from the bill's own slice.
|
|
30
|
+
*
|
|
31
|
+
* Not the model that answered. `route` builds its baseline provider from
|
|
32
|
+
* the environment, so the model in the measurement is whatever
|
|
33
|
+
* `TRAZUM_LLM_MODEL` names, and it is only the log's model when the reader
|
|
34
|
+
* configured it that way. Pairing on it matched nothing on a real run
|
|
35
|
+
* against an OpenAI-compatible endpoint — `evaluation.model` said
|
|
36
|
+
* `stub-strong` while the log said `claude-opus-5` — and the bill would
|
|
37
|
+
* have shown no verdict at all with no explanation.
|
|
38
|
+
*/
|
|
39
|
+
model: string;
|
|
40
|
+
/**
|
|
41
|
+
* The model that actually answered, when it is not the one the log records.
|
|
42
|
+
*
|
|
43
|
+
* Null when they agree, which is the ordinary case. Not null is worth
|
|
44
|
+
* saying out loud rather than smoothing over: a verdict measured on a
|
|
45
|
+
* stand-in is a weaker claim about this workload than one measured on the
|
|
46
|
+
* model the workload actually uses, and only this field can tell the
|
|
47
|
+
* difference.
|
|
48
|
+
*/
|
|
49
|
+
measuredOn: string | null;
|
|
50
|
+
/** The model measured against it. */
|
|
51
|
+
candidateModel: string;
|
|
52
|
+
verdict: EvalVerdict;
|
|
53
|
+
/** The model's agreement with itself. The yardstick the other rate is read against. */
|
|
54
|
+
selfAgreement: number;
|
|
55
|
+
/** Agreement between the two models, same prompt on both sides. */
|
|
56
|
+
crossAgreement: number;
|
|
57
|
+
/** Cases the measurement covered, so a two-case verdict cannot pose as a hundred. */
|
|
58
|
+
cases: number;
|
|
59
|
+
/** Calls it cost to reach. */
|
|
60
|
+
callsMade: number;
|
|
61
|
+
/**
|
|
62
|
+
* What the measurement's own bill priced the route at, when the document
|
|
63
|
+
* carried it. Null rather than zero when it did not: a saving nobody
|
|
64
|
+
* measured is not a saving of nothing.
|
|
65
|
+
*/
|
|
66
|
+
savingUsd: number | null;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* What the bridge did with a file.
|
|
70
|
+
*
|
|
71
|
+
* `null` is not a failure: it means the file is not a routing measurement at
|
|
72
|
+
* all, and the caller should go on treating it as whatever else it was. A
|
|
73
|
+
* refusal is the other case — it looked like one and could not be read — and
|
|
74
|
+
* it names what is missing, because a refusal with nothing after it is
|
|
75
|
+
* indistinguishable from a bug.
|
|
76
|
+
*/
|
|
77
|
+
export type BridgeReading = {
|
|
78
|
+
kind: 'verdict';
|
|
79
|
+
verdict: DroppedVerdict;
|
|
80
|
+
} | {
|
|
81
|
+
kind: 'refusal';
|
|
82
|
+
because: string;
|
|
83
|
+
};
|
|
84
|
+
/**
|
|
85
|
+
* Reads a dropped `trazum route --json` document.
|
|
86
|
+
*
|
|
87
|
+
* Returns null when the text is something else entirely. Validation is
|
|
88
|
+
* `conform`'s, against the published `routing-measurement` contract, so this
|
|
89
|
+
* function cannot drift from the schema a connector author builds against —
|
|
90
|
+
* and a document that fails carries the contract's own sentences back to the
|
|
91
|
+
* reader rather than a second opinion written here.
|
|
92
|
+
*/
|
|
93
|
+
export declare function readDroppedVerdict(text: string): BridgeReading | null;
|
|
94
|
+
/**
|
|
95
|
+
* Whether a verdict describes the route a given slice of the bill is offering.
|
|
96
|
+
*
|
|
97
|
+
* Both halves must agree or the pairing is a lie: the same workload, the same
|
|
98
|
+
* model it goes to now, and the same candidate. A verdict measured on `chat`
|
|
99
|
+
* shown against `summarise`'s saving would be the exact fault this repository
|
|
100
|
+
* keeps finding in itself — a number describing something other than what was
|
|
101
|
+
* measured — and matching on the model alone would produce it on any log with
|
|
102
|
+
* two workloads on one model.
|
|
103
|
+
*/
|
|
104
|
+
export declare function verdictMatchesSlice(verdict: DroppedVerdict, slice: {
|
|
105
|
+
label: string;
|
|
106
|
+
model: string;
|
|
107
|
+
route: {
|
|
108
|
+
candidate: {
|
|
109
|
+
id: string;
|
|
110
|
+
};
|
|
111
|
+
} | null;
|
|
112
|
+
}): boolean;
|
|
113
|
+
//# sourceMappingURL=verdict-bridge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verdict-bridge.d.ts","sourceRoot":"","sources":["../src/verdict-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAGjD,kFAAkF;AAClF,MAAM,WAAW,cAAc;IAC7B,gFAAgF;IAChF,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB;;;;;;;;;;OAUG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;;;OAQG;IACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,qCAAqC;IACrC,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,uFAAuF;IACvF,aAAa,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,cAAc,EAAE,MAAM,CAAC;IACvB,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAkBzC;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAkErE;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE;QAAE,SAAS,EAAE;YAAE,EAAE,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,GAAG,IAAI,CAAA;CAAE,GACnF,OAAO,CAQT"}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a dropped `route --json` verdict, so quality can stand beside cost.
|
|
3
|
+
*
|
|
4
|
+
* The bill is arithmetic: every figure in it comes from records already on
|
|
5
|
+
* disk, and it can say exactly what a cheaper model would have saved. It can
|
|
6
|
+
* say nothing at all about whether that model still does the job. That answer
|
|
7
|
+
* costs provider calls and a credential, which is why `trazum route` exists
|
|
8
|
+
* and why no surface that only reads a log will ever produce it.
|
|
9
|
+
*
|
|
10
|
+
* This is the bridge between the two, and it is deliberately one-directional:
|
|
11
|
+
* the measurement is made where the credential is, written out as the
|
|
12
|
+
* `routing-measurement` contract, and read back here by a pure function that
|
|
13
|
+
* touches no network and holds no key. A reader who has run `route` once can
|
|
14
|
+
* carry the verdict to any surface that prices the same workload.
|
|
15
|
+
*
|
|
16
|
+
* It was not possible to write until the document was worth reading. Before
|
|
17
|
+
* the JSON-text sweep, `route --json` printed the whole `EvalReport` — every
|
|
18
|
+
* case input and three model answers per case — so a bridge would have been a
|
|
19
|
+
* feature for carrying prompt text into a browser. The document carries the
|
|
20
|
+
* measurement and nothing else now, and `conform` is what says so rather than
|
|
21
|
+
* this file's own reading: the contract validates, the bridge only maps.
|
|
22
|
+
*/
|
|
23
|
+
import { conform } from './conform.js';
|
|
24
|
+
import { UNLABELLED } from './usage.js';
|
|
25
|
+
const isObject = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
26
|
+
/**
|
|
27
|
+
* Whether this file is even claiming to be a routing measurement.
|
|
28
|
+
*
|
|
29
|
+
* Shape only, and deliberately narrow: an `evaluation` object carrying a
|
|
30
|
+
* `verdict`. Anything looser would swallow a usage log or a profile and
|
|
31
|
+
* refuse it in this file's words instead of letting the caller price it.
|
|
32
|
+
*/
|
|
33
|
+
function claimsToBeOne(parsed) {
|
|
34
|
+
if (!isObject(parsed))
|
|
35
|
+
return false;
|
|
36
|
+
const evaluation = parsed.evaluation;
|
|
37
|
+
return isObject(evaluation) && typeof evaluation.verdict === 'string';
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Reads a dropped `trazum route --json` document.
|
|
41
|
+
*
|
|
42
|
+
* Returns null when the text is something else entirely. Validation is
|
|
43
|
+
* `conform`'s, against the published `routing-measurement` contract, so this
|
|
44
|
+
* function cannot drift from the schema a connector author builds against —
|
|
45
|
+
* and a document that fails carries the contract's own sentences back to the
|
|
46
|
+
* reader rather than a second opinion written here.
|
|
47
|
+
*/
|
|
48
|
+
export function readDroppedVerdict(text) {
|
|
49
|
+
let parsed;
|
|
50
|
+
try {
|
|
51
|
+
parsed = JSON.parse(text);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
if (!claimsToBeOne(parsed))
|
|
57
|
+
return null;
|
|
58
|
+
const report = conform(text, { contract: 'routing-measurement' });
|
|
59
|
+
if (!report.conforms) {
|
|
60
|
+
const detail = report.problems.length > 0
|
|
61
|
+
? report.problems.map((problem) => `${problem.at}: ${problem.detail}`).join('; ')
|
|
62
|
+
: (report.because ?? 'it does not match the routing-measurement contract');
|
|
63
|
+
return { kind: 'refusal', because: detail };
|
|
64
|
+
}
|
|
65
|
+
const document = parsed;
|
|
66
|
+
const slice = isObject(document.slice) ? document.slice : {};
|
|
67
|
+
const evaluation = document.evaluation;
|
|
68
|
+
/**
|
|
69
|
+
* Every field below is read defensively even though `conform` has passed,
|
|
70
|
+
* because the contract requires `slice` and `evaluation` to be objects and
|
|
71
|
+
* says nothing about what is inside them. A document from a future version
|
|
72
|
+
* with a thinner slice is a document this reads what it can from, not one
|
|
73
|
+
* it crashes on.
|
|
74
|
+
*/
|
|
75
|
+
const answered = typeof evaluation.model === 'string' ? evaluation.model : null;
|
|
76
|
+
// The slice's model first: that is the fact the bill can be matched against.
|
|
77
|
+
const model = typeof slice.model === 'string' ? slice.model : answered;
|
|
78
|
+
const candidateModel = typeof evaluation.candidateModel === 'string' ? evaluation.candidateModel : null;
|
|
79
|
+
const verdict = typeof evaluation.verdict === 'string' ? evaluation.verdict : null;
|
|
80
|
+
if (model === null || candidateModel === null || verdict === null) {
|
|
81
|
+
return {
|
|
82
|
+
kind: 'refusal',
|
|
83
|
+
because: 'the evaluation names no model, candidate model or verdict, so there is nothing to set beside a bill',
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
const number = (value) => (typeof value === 'number' ? value : 0);
|
|
87
|
+
const route = isObject(slice.route) ? slice.route : null;
|
|
88
|
+
const rawLabel = typeof slice.label === 'string' ? slice.label : null;
|
|
89
|
+
return {
|
|
90
|
+
kind: 'verdict',
|
|
91
|
+
verdict: {
|
|
92
|
+
// `UNLABELLED` is the profile's own sentinel for calls that carry no
|
|
93
|
+
// label. It travels through the document as that sentinel and comes
|
|
94
|
+
// back as null here, so the bridge speaks the bill's language rather
|
|
95
|
+
// than leaking an internal marker into a caption.
|
|
96
|
+
label: rawLabel === null || rawLabel === UNLABELLED ? null : rawLabel,
|
|
97
|
+
model,
|
|
98
|
+
measuredOn: answered !== null && answered !== model ? answered : null,
|
|
99
|
+
candidateModel,
|
|
100
|
+
verdict,
|
|
101
|
+
selfAgreement: number(evaluation.selfAgreement),
|
|
102
|
+
crossAgreement: number(evaluation.crossAgreement),
|
|
103
|
+
cases: Array.isArray(evaluation.cases) ? evaluation.cases.length : 0,
|
|
104
|
+
callsMade: number(evaluation.callsMade),
|
|
105
|
+
savingUsd: route !== null && typeof route.savingUsd === 'number' ? route.savingUsd : null,
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Whether a verdict describes the route a given slice of the bill is offering.
|
|
111
|
+
*
|
|
112
|
+
* Both halves must agree or the pairing is a lie: the same workload, the same
|
|
113
|
+
* model it goes to now, and the same candidate. A verdict measured on `chat`
|
|
114
|
+
* shown against `summarise`'s saving would be the exact fault this repository
|
|
115
|
+
* keeps finding in itself — a number describing something other than what was
|
|
116
|
+
* measured — and matching on the model alone would produce it on any log with
|
|
117
|
+
* two workloads on one model.
|
|
118
|
+
*/
|
|
119
|
+
export function verdictMatchesSlice(verdict, slice) {
|
|
120
|
+
if (slice.route === null)
|
|
121
|
+
return false;
|
|
122
|
+
const sliceLabel = slice.label === UNLABELLED ? null : slice.label;
|
|
123
|
+
return (sliceLabel === verdict.label &&
|
|
124
|
+
slice.model === verdict.model &&
|
|
125
|
+
slice.route.candidate.id === verdict.candidateModel);
|
|
126
|
+
}
|
|
127
|
+
//# sourceMappingURL=verdict-bridge.js.map
|