ai-model-verifier 3.9.5 → 3.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/evidence.d.ts.map +1 -1
- package/dist/engine/evidence.js +18 -4
- package/dist/engine/evidence.js.map +1 -1
- package/dist/models/tokenizers.d.ts +10 -0
- package/dist/models/tokenizers.d.ts.map +1 -1
- package/dist/models/tokenizers.js +17 -0
- package/dist/models/tokenizers.js.map +1 -1
- package/dist/rules/tokenizer-family.d.ts +18 -1
- package/dist/rules/tokenizer-family.d.ts.map +1 -1
- package/dist/rules/tokenizer-family.js +53 -1
- package/dist/rules/tokenizer-family.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evidence.d.ts","sourceRoot":"","sources":["../../src/engine/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAUL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"evidence.d.ts","sourceRoot":"","sources":["../../src/engine/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAUL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,OAAO,EAA2B,KAAK,MAAM,EAAE,MAAM,WAAW,CAAC;AACjE,OAAO,EAAiB,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,6BAA6B,CAAC;AAQrC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,eAAe,CAAC;IACvB,IAAI,EAAE,eAAe,CAAC;IACtB,iEAAiE;IACjE,OAAO,EAAE,eAAe,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;CAClD,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,eAAe,CAAC;IACrB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,IAAI,CAAC;AAET;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,iDAAiD;IACjD,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,2EAA2E;IAC3E,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,SAAS,EAAE,iBAAiB,CAAC;IAC7B,6EAA6E;IAC7E,YAAY,EAAE,oBAAoB,CAAC;IACnC,gFAAgF;IAChF,WAAW,EAAE,eAAe,GAAG,IAAI,CAAC;IACpC,2DAA2D;IAC3D,aAAa,EAAE,qBAAqB,CAAC;IACrC,oEAAoE;IACpE,UAAU,EAAE,eAAe,GAAG,IAAI,CAAC;IACnC,wFAAwF;IACxF,kBAAkB,EAAE,eAAe,GAAG,IAAI,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC;AAE5C,oFAAoF;AACpF,eAAO,MAAM,cAAc,qJAUgB,CAAC;AA4H5C,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAwC;IACpE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAuC;IACrE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwC;IACnE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwC;IAE3E,YAAY,GAAG,EAAE,MAAM,EAUtB;IAED,GAAG,CAAC,CAAC,SAAS,WAAW,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;CAuB7D"}
|
package/dist/engine/evidence.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { COUNT_PROBE_MAX_TOKENS, DIVERSE_PROMPT, ASKED_THINKING_PROMPT, FLOOR_PROMPT, LONG_PROMPT, SHORT_PROMPT, SIGNATURE_PROMPT, SCRIPT_BASE_PROMPT, SCRIPT_VARIANTS, } from "../probes/prompts";
|
|
2
|
+
import { isTransientError } from "../identity/signals";
|
|
2
3
|
import { callWire, chat, wireCtx } from "./context";
|
|
3
4
|
import { collectProbes } from "./probe-runner";
|
|
4
5
|
import { collectSurvey } from "./survey-runner";
|
|
@@ -33,11 +34,24 @@ async function collectFixedText(ctx) {
|
|
|
33
34
|
]);
|
|
34
35
|
return { short, long, diverse };
|
|
35
36
|
}
|
|
37
|
+
const SCRIPT_RETRY_MS = 1500;
|
|
38
|
+
const transient = (r) => r.status === null
|
|
39
|
+
? isTransientError(r.error ?? "")
|
|
40
|
+
: isTransientError(`HTTP ${r.status}`);
|
|
36
41
|
async function collectScriptTokens(ctx) {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
42
|
+
// One more try on a busy answer: a timed out anchor left wusrouter's made up
|
|
43
|
+
// counts unmeasured, and an unmeasured family never fails.
|
|
44
|
+
const ask = async (prompt) => {
|
|
45
|
+
const send = () => chat(ctx, {
|
|
46
|
+
maxTokens: COUNT_PROBE_MAX_TOKENS,
|
|
47
|
+
messages: [{ role: "user", content: prompt }],
|
|
48
|
+
});
|
|
49
|
+
const first = await send();
|
|
50
|
+
if (!transient(first))
|
|
51
|
+
return first;
|
|
52
|
+
await new Promise((resolve) => setTimeout(resolve, SCRIPT_RETRY_MS));
|
|
53
|
+
return send();
|
|
54
|
+
};
|
|
41
55
|
// The anchor first: a relay that caches prefixes then bills every run the same way.
|
|
42
56
|
const base = await ask(SCRIPT_BASE_PROMPT);
|
|
43
57
|
const run = (k) => ask(SCRIPT_BASE_PROMPT + SCRIPT_VARIANTS[k]);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evidence.js","sourceRoot":"","sources":["../../src/engine/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,qBAAqB,EACrB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GAEhB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAe,MAAM,WAAW,CAAC;AACjE,OAAO,EAAE,aAAa,EAAkB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AACpE,OAAO,EACL,wBAAwB,GAEzB,MAAM,6BAA6B,CAAC;AAErC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,mFAAmF;AACnF,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,6FAA6F;AAC7F,MAAM,gBAAgB,GAAG,IAAI,CAAC;AA8C9B,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,QAAQ;IACR,QAAQ;IACR,mBAAmB;IACnB,eAAe;IACf,WAAW;IACX,cAAc;IACd,aAAa;IACb,YAAY;IACZ,oBAAoB;CACqB,CAAC;AAE5C,MAAM,EAAE,GAAG,CAAC,CAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC;AAEvE,KAAK,UAAU,gBAAgB,CAAC,GAAW;IACzC,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,EAAE,CAC7B,IAAI,CAAC,GAAG,EAAE;QACR,SAAS,EAAE,sBAAsB;QACjC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC,CAAC;IACL,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC/C,GAAG,CAAC,YAAY,CAAC;QACjB,GAAG,CAAC,WAAW,CAAC;QAChB,GAAG,CAAC,cAAc,CAAC;KACpB,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,GAAW;IAC5C,MAAM,GAAG,GAAG,
|
|
1
|
+
{"version":3,"file":"evidence.js","sourceRoot":"","sources":["../../src/engine/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,qBAAqB,EACrB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GAEhB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAe,MAAM,WAAW,CAAC;AACjE,OAAO,EAAE,aAAa,EAAkB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AACpE,OAAO,EACL,wBAAwB,GAEzB,MAAM,6BAA6B,CAAC;AAErC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,mFAAmF;AACnF,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,6FAA6F;AAC7F,MAAM,gBAAgB,GAAG,IAAI,CAAC;AA8C9B,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,QAAQ;IACR,QAAQ;IACR,mBAAmB;IACnB,eAAe;IACf,WAAW;IACX,cAAc;IACd,aAAa;IACb,YAAY;IACZ,oBAAoB;CACqB,CAAC;AAE5C,MAAM,EAAE,GAAG,CAAC,CAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC;AAEvE,KAAK,UAAU,gBAAgB,CAAC,GAAW;IACzC,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,EAAE,CAC7B,IAAI,CAAC,GAAG,EAAE;QACR,SAAS,EAAE,sBAAsB;QACjC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC,CAAC;IACL,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC/C,GAAG,CAAC,YAAY,CAAC;QACjB,GAAG,CAAC,WAAW,CAAC;QAChB,GAAG,CAAC,cAAc,CAAC;KACpB,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,MAAM,SAAS,GAAG,CAAC,CAAkB,EAAE,EAAE,CACvC,CAAC,CAAC,MAAM,KAAK,IAAI;IACf,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;IACjC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;AAE3C,KAAK,UAAU,mBAAmB,CAAC,GAAW;IAC5C,6EAA6E;IAC7E,2DAA2D;IAC3D,MAAM,GAAG,GAAG,KAAK,EAAE,MAAc,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,GAAG,EAAE,CAChB,IAAI,CAAC,GAAG,EAAE;YACR,SAAS,EAAE,sBAAsB;YACjC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SAC9C,CAAC,CAAC;QACL,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC;QACrE,OAAO,IAAI,EAAE,CAAC;IAChB,CAAC,CAAC;IACF,oFAAoF;IACpF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,CAAC,CAAgB,EAAE,EAAE,CAC/B,GAAG,CAAC,kBAAkB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACpD,GAAG,CAAC,QAAQ,CAAC;QACb,GAAG,CAAC,KAAK,CAAC;QACV,GAAG,CAAC,QAAQ,CAAC;QACb,GAAG,CAAC,MAAM,CAAC;KACZ,CAAC,CAAC;IACH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;AAC3D,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,GAAW,EACX,KAAoB;IAEpB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;IACvC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI;QACpE,OAAO,IAAI,CAAC;IACd,OAAO,QAAQ,CACb,GAAG,EACH,KAAK,CACH;QACE,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;KACpD,EACD,OAAO,CAAC,GAAG,CAAC,CACb,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,GAAW;IAEX,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;IAChC,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACxE,2EAA2E;IAC3E,wDAAwD;IACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;QAC1B,SAAS,EAAE,oBAAoB;QAC/B,QAAQ;QACR,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,sBAAsB,EAAE;KACzD,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAC9D,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAC3B,2EAA2E;IAC3E,2EAA2E;IAC3E,4EAA4E;IAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;QAC7B,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;KACvE,CAAC,CAAC;IACH,uEAAuE;IACvE,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;AAChF,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAW;IAC1C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IACzC,OAAO,IAAI,CAAC,GAAG,EAAE;QACf,SAAS,EAAE,gBAAgB;QAC3B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;KACpD,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,GAAW;IAEX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe;QAAE,OAAO,IAAI,CAAC;IAC5C,OAAO,IAAI,CAAC,GAAG,EAAE;QACf,SAAS,EAAE,gBAAgB;QAC3B,eAAe,EAAE,MAAM;QACvB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,CAAC;KAC7D,CAAC,CAAC;AACL,CAAC;AAED,MAAM,IAAI,GAAG,CAAI,EAAoB,EAAE,EAAE;IACvC,IAAI,CAAyB,CAAC;IAC9B,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,OAAO,aAAa;IACP,MAAM,CAA6B;IACnC,MAAM,CAAiC;IACvC,iBAAiB,CAAmC;IACpD,SAAS,CAAmC;IAC5C,YAAY,CAAsC;IAClD,WAAW,CAAwC;IACnD,aAAa,CAAuC;IACpD,UAAU,CAAwC;IAClD,kBAAkB,CAAwC;IAE3E,YAAY,GAAW;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,CAAC;IAGD,GAAG,CAAC,GAAgB;QAClB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,QAAQ;gBACX,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,KAAK,QAAQ;gBACX,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,KAAK,mBAAmB;gBACtB,OAAO,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAClC,KAAK,WAAW;gBACd,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,KAAK,cAAc;gBACjB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,KAAK,aAAa;gBAChB,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,KAAK,eAAe;gBAClB,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,YAAY;gBACf,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;YAC3B,KAAK,oBAAoB;gBACvB,OAAO,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACrC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -38,4 +38,14 @@ export declare const knowsMaker: (maker: string, families?: readonly TokenizerFa
|
|
|
38
38
|
* UTIM's "gemini-4-argon" billed 80/32/50/92.
|
|
39
39
|
*/
|
|
40
40
|
export declare function isCharacterEstimate(measured: ScriptVector): boolean;
|
|
41
|
+
/**
|
|
42
|
+
* The fewest tokens any run could bill: a third of the leanest known family on
|
|
43
|
+
* that run, and a run's full prompt count includes the anchor on top of it.
|
|
44
|
+
* wusrouter's "claude-fable-5.1-free" billed 4 for the 300 digit run where the
|
|
45
|
+
* leanest family bills 34, and its "claude-opus-5.5" billed less for that run
|
|
46
|
+
* than for the anchor alone.
|
|
47
|
+
*/
|
|
48
|
+
export declare function scriptFloor(k: ScriptVariant, families?: readonly TokenizerFamily[]): number;
|
|
49
|
+
/** Every run whose full prompt count is below what any tokenizer bills for its text alone. */
|
|
50
|
+
export declare function impossibleRuns(counts: Readonly<Record<ScriptVariant, number | null>>): ScriptVariant[];
|
|
41
51
|
//# sourceMappingURL=tokenizers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../../src/models/tokenizers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;AAEnE,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,OAAO,EAAE,CAAC;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AASF,eAAO,MAAM,kBAAkB,EAAE,SAAS,eAAe,EAmCxD,CAAC;AAEF,uEAAuE;AACvE,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,YAAY,EACtB,QAAQ,GAAE,SAAS,eAAe,EAAuB,GACxD,eAAe,EAAE,CAMnB;AAED,eAAO,MAAM,UAAU,UACd,MAAM,aACH,SAAS,eAAe,EAAE,KACnC,OAAkE,CAAC;AAEtE;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO,CAKnE"}
|
|
1
|
+
{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../../src/models/tokenizers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;AAEnE,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,OAAO,EAAE,CAAC;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AASF,eAAO,MAAM,kBAAkB,EAAE,SAAS,eAAe,EAmCxD,CAAC;AAEF,uEAAuE;AACvE,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,YAAY,EACtB,QAAQ,GAAE,SAAS,eAAe,EAAuB,GACxD,eAAe,EAAE,CAMnB;AAED,eAAO,MAAM,UAAU,UACd,MAAM,aACH,SAAS,eAAe,EAAE,KACnC,OAAkE,CAAC;AAEtE;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO,CAKnE;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CACzB,CAAC,EAAE,aAAa,EAChB,QAAQ,GAAE,SAAS,eAAe,EAAuB,GACxD,MAAM,CAER;AAED,8FAA8F;AAC9F,wBAAgB,cAAc,CAC5B,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,GACrD,aAAa,EAAE,CAKjB"}
|
|
@@ -69,4 +69,21 @@ export function isCharacterEstimate(measured) {
|
|
|
69
69
|
return ratio >= 0.2 && ratio <= 0.3;
|
|
70
70
|
});
|
|
71
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* The fewest tokens any run could bill: a third of the leanest known family on
|
|
74
|
+
* that run, and a run's full prompt count includes the anchor on top of it.
|
|
75
|
+
* wusrouter's "claude-fable-5.1-free" billed 4 for the 300 digit run where the
|
|
76
|
+
* leanest family bills 34, and its "claude-opus-5.5" billed less for that run
|
|
77
|
+
* than for the anchor alone.
|
|
78
|
+
*/
|
|
79
|
+
export function scriptFloor(k, families = TOKENIZER_FAMILIES) {
|
|
80
|
+
return Math.floor(Math.min(...families.map((f) => f.vector[k])) / 3);
|
|
81
|
+
}
|
|
82
|
+
/** Every run whose full prompt count is below what any tokenizer bills for its text alone. */
|
|
83
|
+
export function impossibleRuns(counts) {
|
|
84
|
+
return SCRIPT_VARIANT_KEYS.filter((k) => {
|
|
85
|
+
const n = counts[k];
|
|
86
|
+
return n !== null && n < scriptFloor(k);
|
|
87
|
+
});
|
|
88
|
+
}
|
|
72
89
|
//# sourceMappingURL=tokenizers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizers.js","sourceRoot":"","sources":["../../src/models/tokenizers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EACL,mBAAmB,EACnB,eAAe,GAEhB,MAAM,mBAAmB,CAAC;AAe3B,MAAM,CAAC,GAAG,CACR,MAAc,EACd,GAAW,EACX,MAAc,EACd,IAAY,EACE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAEnD,MAAM,CAAC,MAAM,kBAAkB,GAA+B;IAC5D;QACE,EAAE,EAAE,OAAO;QACX,MAAM,EAAE,CAAC,QAAQ,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;QAC7B,SAAS,EAAE,IAAI;KAChB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,CAAC,QAAQ,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;QAC7B,SAAS,EAAE,IAAI;KAChB;IACD,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACvE,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAClE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC9D;QACE,EAAE,EAAE,UAAU;QACd,MAAM,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC;QAC3C,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;KAC7B;IACD;QACE,EAAE,EAAE,MAAM;QACV,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC;QACxC,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;KAC7B;IACD,4EAA4E;IAC5E,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACpE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACpE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAChE,EAAE,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC3E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;CAC9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,MAAM,UAAU,sBAAsB,CACpC,QAAsB,EACtB,QAAQ,GAA+B,kBAAkB;IAEzD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3B,mBAAmB,CAAC,KAAK,CACvB,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAC/D,CACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,QAAQ,GAA+B,kBAAkB,EAChD,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;AAEtE;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAsB;IACxD,OAAO,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3D,OAAO,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
1
|
+
{"version":3,"file":"tokenizers.js","sourceRoot":"","sources":["../../src/models/tokenizers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EACL,mBAAmB,EACnB,eAAe,GAEhB,MAAM,mBAAmB,CAAC;AAe3B,MAAM,CAAC,GAAG,CACR,MAAc,EACd,GAAW,EACX,MAAc,EACd,IAAY,EACE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAEnD,MAAM,CAAC,MAAM,kBAAkB,GAA+B;IAC5D;QACE,EAAE,EAAE,OAAO;QACX,MAAM,EAAE,CAAC,QAAQ,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;QAC7B,SAAS,EAAE,IAAI;KAChB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,CAAC,QAAQ,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;QAC7B,SAAS,EAAE,IAAI;KAChB;IACD,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACvE,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAClE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC9D;QACE,EAAE,EAAE,UAAU;QACd,MAAM,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC;QAC3C,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;KAC7B;IACD;QACE,EAAE,EAAE,MAAM;QACV,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC;QACxC,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;KAC7B;IACD,4EAA4E;IAC5E,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACpE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACpE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAChE,EAAE,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC3E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;CAC9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,MAAM,UAAU,sBAAsB,CACpC,QAAsB,EACtB,QAAQ,GAA+B,kBAAkB;IAEzD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3B,mBAAmB,CAAC,KAAK,CACvB,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAC/D,CACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,QAAQ,GAA+B,kBAAkB,EAChD,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;AAEtE;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAsB;IACxD,OAAO,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3D,OAAO,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CACzB,CAAgB,EAChB,QAAQ,GAA+B,kBAAkB;IAEzD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAC5B,MAAsD;IAEtD,OAAO,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACtC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -14,7 +14,15 @@
|
|
|
14
14
|
*
|
|
15
15
|
* Also fails when every run bills about one token per four characters: that
|
|
16
16
|
* is a provider making up usage so the vocabulary behind it cannot be seen.
|
|
17
|
+
* Same for two runs billed below what any tokenizer needs for their text
|
|
18
|
+
* alone. One such run is only inconclusive: a7 Claude merchants that pass
|
|
19
|
+
* everything else bill the repeated digits run at 1 to 4 now and then, which
|
|
20
|
+
* reads as a relay cache, not made up counts.
|
|
21
|
+
*
|
|
22
|
+
* Runs that never answered are inconclusive rather than silent: the probes
|
|
23
|
+
* alone call any good persona prompt genuine.
|
|
17
24
|
*/
|
|
25
|
+
import type { ScriptVariant } from "../probes/prompts";
|
|
18
26
|
import { type ScriptVector } from "../models/tokenizers";
|
|
19
27
|
export type TokenizerFamilyState = "own"
|
|
20
28
|
/** Matches another maker's family and none of the claimed maker's. */
|
|
@@ -23,14 +31,23 @@ export type TokenizerFamilyState = "own"
|
|
|
23
31
|
| "estimated"
|
|
24
32
|
/** Billed as characters / 4: the provider makes up usage, hiding the backend. */
|
|
25
33
|
| "char-estimated"
|
|
34
|
+
/** Runs billed below what any tokenizer needs for their text: made up usage or a cache. */
|
|
35
|
+
| "impossible"
|
|
26
36
|
/** Measured but matches no known family. */
|
|
27
|
-
| "unknown" | "unmeasured"
|
|
37
|
+
| "unknown" | "unmeasured"
|
|
38
|
+
/** A run timed out or the endpoint was busy, so nothing could be read. */
|
|
39
|
+
| "unanswered";
|
|
28
40
|
export type TokenizerFamilyResult = {
|
|
29
41
|
state: TokenizerFamilyState;
|
|
30
42
|
vector: ScriptVector | null;
|
|
31
43
|
families: string[];
|
|
32
44
|
makers: string[];
|
|
33
45
|
reason?: string;
|
|
46
|
+
/** Runs billed below every tokenizer, with their full prompt counts. */
|
|
47
|
+
impossible?: {
|
|
48
|
+
run: ScriptVariant;
|
|
49
|
+
prompt: number;
|
|
50
|
+
}[];
|
|
34
51
|
};
|
|
35
52
|
/** Judge one measured vector against the claimed maker. Pure. */
|
|
36
53
|
export declare function judgeTokenizerFamily(maker: string, vector: ScriptVector): TokenizerFamilyResult;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-family.d.ts","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-family.d.ts","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAKL,KAAK,YAAY,EAClB,MAAM,sBAAsB,CAAC;AAG9B,MAAM,MAAM,oBAAoB,GAC5B,KAAK;AACP,sEAAsE;GACpE,SAAS;AACX,mEAAmE;GACjE,WAAW;AACb,iFAAiF;GAC/E,gBAAgB;AAClB,2FAA2F;GACzF,YAAY;AACd,4CAA4C;GAC1C,SAAS,GACT,YAAY;AACd,0EAA0E;GACxE,YAAY,CAAC;AAEjB,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,UAAU,CAAC,EAAE;QAAE,GAAG,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACvD,CAAC;AAEF,iEAAiE;AACjE,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,YAAY,GACnB,qBAAqB,CAsBvB;AA4DD,eAAO,MAAM,mBAAmB,wEA+C9B,CAAC"}
|
|
@@ -14,8 +14,16 @@
|
|
|
14
14
|
*
|
|
15
15
|
* Also fails when every run bills about one token per four characters: that
|
|
16
16
|
* is a provider making up usage so the vocabulary behind it cannot be seen.
|
|
17
|
+
* Same for two runs billed below what any tokenizer needs for their text
|
|
18
|
+
* alone. One such run is only inconclusive: a7 Claude merchants that pass
|
|
19
|
+
* everything else bill the repeated digits run at 1 to 4 now and then, which
|
|
20
|
+
* reads as a relay cache, not made up counts.
|
|
21
|
+
*
|
|
22
|
+
* Runs that never answered are inconclusive rather than silent: the probes
|
|
23
|
+
* alone call any good persona prompt genuine.
|
|
17
24
|
*/
|
|
18
|
-
import {
|
|
25
|
+
import { isTransientError } from "../identity/signals";
|
|
26
|
+
import { impossibleRuns, isCharacterEstimate, knowsMaker, matchTokenizerFamilies, } from "../models/tokenizers";
|
|
19
27
|
import { defineRule } from "./types";
|
|
20
28
|
/** Judge one measured vector against the claimed maker. Pure. */
|
|
21
29
|
export function judgeTokenizerFamily(maker, vector) {
|
|
@@ -48,6 +56,28 @@ async function familyResult(ctx) {
|
|
|
48
56
|
const prompt = (r) => r.status !== null && r.status < 400
|
|
49
57
|
? (ctx.wire.read.meta(r.data).usage?.prompt ?? null)
|
|
50
58
|
: null;
|
|
59
|
+
const counts = {
|
|
60
|
+
digits: prompt(ev.variants.digits),
|
|
61
|
+
cjk: prompt(ev.variants.cjk),
|
|
62
|
+
spaces: prompt(ev.variants.spaces),
|
|
63
|
+
code: prompt(ev.variants.code),
|
|
64
|
+
};
|
|
65
|
+
const impossible = impossibleRuns(counts).flatMap((run) => {
|
|
66
|
+
const prompt = counts[run];
|
|
67
|
+
return prompt === null ? [] : [{ run, prompt }];
|
|
68
|
+
});
|
|
69
|
+
if (impossible.length > 0)
|
|
70
|
+
return { state: "impossible", ...empty, impossible };
|
|
71
|
+
const all = [ev.base, ...Object.values(ev.variants)];
|
|
72
|
+
const busy = all.find((r) => r.status === null
|
|
73
|
+
? isTransientError(r.error ?? "")
|
|
74
|
+
: isTransientError(`HTTP ${r.status}`));
|
|
75
|
+
if (busy)
|
|
76
|
+
return {
|
|
77
|
+
state: "unanswered",
|
|
78
|
+
...empty,
|
|
79
|
+
reason: `${busy.status ?? "network"}: ${busy.error ?? "busy"}`,
|
|
80
|
+
};
|
|
51
81
|
const base = prompt(ev.base);
|
|
52
82
|
if (base === null)
|
|
53
83
|
return {
|
|
@@ -81,6 +111,28 @@ export const tokenizerFamilyRule = defineRule({
|
|
|
81
111
|
applies: (ctx) => ctx.wire.id !== "gemini",
|
|
82
112
|
judge: async (ctx) => {
|
|
83
113
|
const r = await familyResult(ctx);
|
|
114
|
+
if (r.state === "impossible" && r.impossible) {
|
|
115
|
+
const runs = r.impossible
|
|
116
|
+
.map((i) => `${i.prompt} for the ${i.run} run`)
|
|
117
|
+
.join(", ");
|
|
118
|
+
return r.impossible.length > 1
|
|
119
|
+
? {
|
|
120
|
+
severity: "fail",
|
|
121
|
+
reason: `tokenizer-family: billed ${runs}, below what any known tokenizer needs for that text alone: the provider makes up usage and hides the model's real counts`,
|
|
122
|
+
data: r,
|
|
123
|
+
}
|
|
124
|
+
: {
|
|
125
|
+
severity: "inconclusive",
|
|
126
|
+
reason: `tokenizer-family: billed ${runs}, below what any known tokenizer needs for that text alone: made up usage or a cached answer, so the vocabulary could not be read`,
|
|
127
|
+
data: r,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
if (r.state === "unanswered" && knowsMaker(ctx.maker.id))
|
|
131
|
+
return {
|
|
132
|
+
severity: "inconclusive",
|
|
133
|
+
reason: `tokenizer-family: the token count runs did not answer (${r.reason}), so the backend's vocabulary was never measured`,
|
|
134
|
+
data: r,
|
|
135
|
+
};
|
|
84
136
|
if (!r.vector)
|
|
85
137
|
return null;
|
|
86
138
|
const v = r.vector;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-family.js","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-family.js","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,UAAU,EACV,sBAAsB,GAEvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,UAAU,EAAgB,MAAM,SAAS,CAAC;AA4BnD,iEAAiE;AACjE,MAAM,UAAU,oBAAoB,CAClC,KAAa,EACb,MAAoB;IAEpB,MAAM,OAAO,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG;QACX,MAAM;QACN,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;KACvD,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;YACjE,GAAG,IAAI;SACR,CAAC;IACJ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;QACxD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC9E,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QACpB,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,IAAI;YACP,MAAM,EAAE,wBAAwB,KAAK,EAAE;SACxC,CAAC;IACJ,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,GAA4B;IAE5B,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,CAAC,CAAsB,EAAE,EAAE,CACxC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG;QACjC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;QACpD,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAClC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;QAC5B,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAClC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;KAC/B,CAAC;IACF,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QACxD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IACH,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QACvB,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC,CAAC,MAAM,KAAK,IAAI;QACf,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CACzC,CAAC;IACF,IAAI,IAAI;QACN,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,IAAI,CAAC,KAAK,IAAI,MAAM,EAAE;SAC/D,CAAC;IACJ,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,IAAI;QACf,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,UAAU,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,UAAU,EAAE;SAChF,CAAC;IACJ,4EAA4E;IAC5E,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAC1E,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,MAAM,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;QACrE,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,2BAA2B;SACpC,CAAC;IACJ,OAAO,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC;IAC5C,EAAE,EAAE,kBAAkB;IACtB,KAAK,EAAE,UAAU;IACjB,KAAK,EAAE,CAAC,cAAc,CAAC;IACvB,KAAK,EAAE,iBAAiB;IACxB,qGAAqG;IACrG,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,QAAQ;IAC1C,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,CAAC,KAAK,KAAK,YAAY,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,YAAY,CAAC,CAAC,GAAG,MAAM,CAAC;iBAC9C,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,OAAO,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;gBAC5B,CAAC,CAAC;oBACE,QAAQ,EAAE,MAAM;oBAChB,MAAM,EAAE,4BAA4B,IAAI,2HAA2H;oBACnK,IAAI,EAAE,CAAC;iBACR;gBACH,CAAC,CAAC;oBACE,QAAQ,EAAE,cAAc;oBACxB,MAAM,EAAE,4BAA4B,IAAI,mIAAmI;oBAC3K,IAAI,EAAE,CAAC;iBACR,CAAC;QACR,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,KAAK,YAAY,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,OAAO;gBACL,QAAQ,EAAE,cAAc;gBACxB,MAAM,EAAE,0DAA0D,CAAC,CAAC,MAAM,mDAAmD;gBAC7H,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnB,IAAI,CAAC,CAAC,KAAK,KAAK,gBAAgB;YAC9B,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,qKAAqK;gBAChP,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACvC,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;YACnK,IAAI,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;CACtE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-model-verifier",
|
|
3
|
-
"version": "3.9.
|
|
3
|
+
"version": "3.9.6",
|
|
4
4
|
"description": "Verify an AI API endpoint really serves the model it claims: a vendor adapter per wire, a maker table for who trained what, a rule table over shared evidence, and per model facts. Model substitution, Claude thinking signatures, token billing accuracy, envelope and throughput evidence.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|