ai-model-verifier 3.9.7 → 3.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/evidence.d.ts +2 -0
- package/dist/engine/evidence.d.ts.map +1 -1
- package/dist/engine/evidence.js +8 -1
- package/dist/engine/evidence.js.map +1 -1
- package/dist/rules/tokenizer-family.d.ts +8 -0
- package/dist/rules/tokenizer-family.d.ts.map +1 -1
- package/dist/rules/tokenizer-family.js +21 -1
- package/dist/rules/tokenizer-family.js.map +1 -1
- package/package.json +1 -1
|
@@ -13,6 +13,8 @@ export type FixedTextEvidence = {
|
|
|
13
13
|
export type ScriptTokensEvidence = {
|
|
14
14
|
base: TransportResult;
|
|
15
15
|
variants: Record<ScriptVariant, TransportResult>;
|
|
16
|
+
/** The Chinese run twice more, each behind its own nonce so no cache can tell them apart. */
|
|
17
|
+
repeats: [TransportResult, TransportResult];
|
|
16
18
|
};
|
|
17
19
|
export type ThinkingReplyEvidence = {
|
|
18
20
|
res: TransportResult;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evidence.d.ts","sourceRoot":"","sources":["../../src/engine/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAUL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,OAAO,EAA2B,KAAK,MAAM,EAAE,MAAM,WAAW,CAAC;AACjE,OAAO,EAAiB,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,6BAA6B,CAAC;AAQrC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,eAAe,CAAC;IACvB,IAAI,EAAE,eAAe,CAAC;IACtB,iEAAiE;IACjE,OAAO,EAAE,eAAe,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"evidence.d.ts","sourceRoot":"","sources":["../../src/engine/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAUL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,OAAO,EAA2B,KAAK,MAAM,EAAE,MAAM,WAAW,CAAC;AACjE,OAAO,EAAiB,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,6BAA6B,CAAC;AAQrC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,eAAe,CAAC;IACvB,IAAI,EAAE,eAAe,CAAC;IACtB,iEAAiE;IACjE,OAAO,EAAE,eAAe,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IACjD,6FAA6F;IAC7F,OAAO,EAAE,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;CAC7C,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,eAAe,CAAC;IACrB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,IAAI,CAAC;AAET;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,iDAAiD;IACjD,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,2EAA2E;IAC3E,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,SAAS,EAAE,iBAAiB,CAAC;IAC7B,6EAA6E;IAC7E,YAAY,EAAE,oBAAoB,CAAC;IACnC,gFAAgF;IAChF,WAAW,EAAE,eAAe,GAAG,IAAI,CAAC;IACpC,2DAA2D;IAC3D,aAAa,EAAE,qBAAqB,CAAC;IACrC,oEAAoE;IACpE,UAAU,EAAE,eAAe,GAAG,IAAI,CAAC;IACnC,wFAAwF;IACxF,kBAAkB,EAAE,eAAe,GAAG,IAAI,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC;AAE5C,oFAAoF;AACpF,eAAO,MAAM,cAAc,qJAUgB,CAAC;AAsI5C,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAwC;IACpE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAuC;IACrE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwC;IACnE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwC;IAE3E,YAAY,GAAG,EAAE,MAAM,EAUtB;IAED,GAAG,CAAC,CAAC,SAAS,WAAW,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;CAuB7D"}
|
package/dist/engine/evidence.js
CHANGED
|
@@ -63,7 +63,14 @@ async function collectScriptTokens(ctx) {
|
|
|
63
63
|
const cjk = await run("cjk");
|
|
64
64
|
const spaces = await run("spaces");
|
|
65
65
|
const code = await run("code");
|
|
66
|
-
|
|
66
|
+
const repeat = () => ask(`[${ctx.nonce()}] ${SCRIPT_BASE_PROMPT}${SCRIPT_VARIANTS.cjk}`);
|
|
67
|
+
const first = await repeat();
|
|
68
|
+
const second = await repeat();
|
|
69
|
+
return {
|
|
70
|
+
base,
|
|
71
|
+
variants: { digits, cjk, spaces, code },
|
|
72
|
+
repeats: [first, second],
|
|
73
|
+
};
|
|
67
74
|
}
|
|
68
75
|
async function collectCountTokens(ctx, store) {
|
|
69
76
|
const count = ctx.wire.ops.countTokens;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evidence.js","sourceRoot":"","sources":["../../src/engine/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,qBAAqB,EACrB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GAEhB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAe,MAAM,WAAW,CAAC;AACjE,OAAO,EAAE,aAAa,EAAkB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AACpE,OAAO,EACL,wBAAwB,GAEzB,MAAM,6BAA6B,CAAC;AAErC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,mFAAmF;AACnF,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,6FAA6F;AAC7F,MAAM,gBAAgB,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"evidence.js","sourceRoot":"","sources":["../../src/engine/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,qBAAqB,EACrB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GAEhB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAe,MAAM,WAAW,CAAC;AACjE,OAAO,EAAE,aAAa,EAAkB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AACpE,OAAO,EACL,wBAAwB,GAEzB,MAAM,6BAA6B,CAAC;AAErC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,mFAAmF;AACnF,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,6FAA6F;AAC7F,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAgD9B,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,QAAQ;IACR,QAAQ;IACR,mBAAmB;IACnB,eAAe;IACf,WAAW;IACX,cAAc;IACd,aAAa;IACb,YAAY;IACZ,oBAAoB;CACqB,CAAC;AAE5C,MAAM,EAAE,GAAG,CAAC,CAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC;AAEvE,KAAK,UAAU,gBAAgB,CAAC,GAAW;IACzC,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,EAAE,CAC7B,IAAI,CAAC,GAAG,EAAE;QACR,SAAS,EAAE,sBAAsB;QACjC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC,CAAC;IACL,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC/C,GAAG,CAAC,YAAY,CAAC;QACjB,GAAG,CAAC,WAAW,CAAC;QAChB,GAAG,CAAC,cAAc,CAAC;KACpB,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAClC,CAAC;AAED,2EAA2E;AAC3E,2DAA2D;AAC3D,MAAM,eAAe,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;AAE7C,MAAM,SAAS,GAAG,CAAC,CAAkB,EAAE,EAAE,CACvC,CAAC,CAAC,MAAM,KAAK,IAAI;IACf,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;IACjC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;AAE3C,KAAK,UAAU,mBAAmB,CAAC,GAAW;IAC5C,MAAM,GAAG,GAAG,KAAK,EAAE,MAAc,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,GAAG,EAAE,CAChB,IAAI,CAAC,GAAG,EAAE;YACR,SAAS,EAAE,sBAAsB;YACjC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SAC9C,CAAC,CAAC;QACL,IAAI,GAAG,GAAG,MAAM,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;gBAAE,MAAM;YAC3B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YAC1D,GAAG,GAAG,MAAM,IAAI,EAAE,CAAC;QACrB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,oFAAoF;IACpF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,CAAC,CAAgB,EAAE,EAAE,CAC/B,GAAG,CAAC,kBAAkB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,2EAA2E;IAC3E,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,GAAG,EAAE,CAClB,GAAG,CAAC,IAAI,GAAG,CAAC,KAAK,EAAE,KAAK,kBAAkB,GAAG,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,MAAM,MAAM,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,MAAM,MAAM,EAAE,CAAC;IAC9B,OAAO;QACL,IAAI;QACJ,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;QACvC,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;KACzB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,GAAW,EACX,KAAoB;IAEpB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;IACvC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI;QACpE,OAAO,IAAI,CAAC;IACd,OAAO,QAAQ,CACb,GAAG,EACH,KAAK,CACH;QACE,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;KACpD,EACD,OAAO,CAAC,GAAG,CAAC,CACb,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,GAAW;IAEX,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;IAChC,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACxE,2EAA2E;IAC3E,wDAAwD;IACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;QAC1B,SAAS,EAAE,oBAAoB;QAC/B,QAAQ;QACR,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,sBAAsB,EAAE;KACzD,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAC9D,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAC3B,2EAA2E;IAC3E,2EAA2E;IAC3E,4EAA4E;IAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;QAC7B,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;KACvE,CAAC,CAAC;IACH,uEAAuE;IACvE,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;AAChF,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAW;IAC1C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IACzC,OAAO,IAAI,CAAC,GAAG,EAAE;QACf,SAAS,EAAE,gBAAgB;QAC3B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;KACpD,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,GAAW;IAEX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe;QAAE,OAAO,IAAI,CAAC;IAC5C,OAAO,IAAI,CAAC,GAAG,EAAE;QACf,SAAS,EAAE,gBAAgB;QAC3B,eAAe,EAAE,MAAM;QACvB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,CAAC;KAC7D,CAAC,CAAC;AACL,CAAC;AAED,MAAM,IAAI,GAAG,CAAI,EAAoB,EAAE,EAAE;IACvC,IAAI,CAAyB,CAAC;IAC9B,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,OAAO,aAAa;IACP,MAAM,CAA6B;IACnC,MAAM,CAAiC;IACvC,iBAAiB,CAAmC;IACpD,SAAS,CAAmC;IAC5C,YAAY,CAAsC;IAClD,WAAW,CAAwC;IACnD,aAAa,CAAuC;IACpD,UAAU,CAAwC;IAClD,kBAAkB,CAAwC;IAE3E,YAAY,GAAW;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,CAAC;IAGD,GAAG,CAAC,GAAgB;QAClB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,QAAQ;gBACX,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,KAAK,QAAQ;gBACX,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,KAAK,mBAAmB;gBACtB,OAAO,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAClC,KAAK,WAAW;gBACd,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,KAAK,cAAc;gBACjB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,KAAK,aAAa;gBAChB,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,KAAK,eAAe;gBAClB,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,YAAY;gBACf,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;YAC3B,KAAK,oBAAoB;gBACvB,OAAO,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACrC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -19,6 +19,10 @@
|
|
|
19
19
|
* everything else bill the repeated digits run at 1 to 4 now and then, which
|
|
20
20
|
* reads as a relay cache, not made up counts.
|
|
21
21
|
*
|
|
22
|
+
* The Chinese run is sent twice more behind two different nonces: a real
|
|
23
|
+
* tokenizer bills the same text the same way every time, wusrouter billed it
|
|
24
|
+
* 89, 92, 8 and 40, and every passing Claude lane we logged bills one vector.
|
|
25
|
+
*
|
|
22
26
|
* Runs that never answered are inconclusive rather than silent: the probes
|
|
23
27
|
* alone call any good persona prompt genuine.
|
|
24
28
|
*/
|
|
@@ -35,6 +39,8 @@ export type TokenizerFamilyState = "own"
|
|
|
35
39
|
| "impossible"
|
|
36
40
|
/** Measured but matches no known family. */
|
|
37
41
|
| "unknown" | "unmeasured"
|
|
42
|
+
/** The same text behind two nonces billed far apart: the counts are made up. */
|
|
43
|
+
| "unstable"
|
|
38
44
|
/** A run timed out or the endpoint was busy, so nothing could be read. */
|
|
39
45
|
| "unanswered";
|
|
40
46
|
export type TokenizerFamilyResult = {
|
|
@@ -48,6 +54,8 @@ export type TokenizerFamilyResult = {
|
|
|
48
54
|
run: ScriptVariant;
|
|
49
55
|
prompt: number;
|
|
50
56
|
}[];
|
|
57
|
+
/** Prompt counts of the two nonce copies of the Chinese run. */
|
|
58
|
+
repeats?: [number, number];
|
|
51
59
|
};
|
|
52
60
|
/** Judge one measured vector against the claimed maker. Pure. */
|
|
53
61
|
export declare function judgeTokenizerFamily(maker: string, vector: ScriptVector): TokenizerFamilyResult;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-family.d.ts","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-family.d.ts","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAKL,KAAK,YAAY,EAClB,MAAM,sBAAsB,CAAC;AAG9B,MAAM,MAAM,oBAAoB,GAC5B,KAAK;AACP,sEAAsE;GACpE,SAAS;AACX,mEAAmE;GACjE,WAAW;AACb,iFAAiF;GAC/E,gBAAgB;AAClB,2FAA2F;GACzF,YAAY;AACd,4CAA4C;GAC1C,SAAS,GACT,YAAY;AACd,gFAAgF;GAC9E,UAAU;AACZ,0EAA0E;GACxE,YAAY,CAAC;AAEjB,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,UAAU,CAAC,EAAE;QAAE,GAAG,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gEAAgE;IAChE,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC5B,CAAC;AAUF,iEAAiE;AACjE,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,YAAY,GACnB,qBAAqB,CAsBvB;AAgED,eAAO,MAAM,mBAAmB,wEAqD9B,CAAC"}
|
|
@@ -19,12 +19,22 @@
|
|
|
19
19
|
* everything else bill the repeated digits run at 1 to 4 now and then, which
|
|
20
20
|
* reads as a relay cache, not made up counts.
|
|
21
21
|
*
|
|
22
|
+
* The Chinese run is sent twice more behind two different nonces: a real
|
|
23
|
+
* tokenizer bills the same text the same way every time, wusrouter billed it
|
|
24
|
+
* 89, 92, 8 and 40, and every passing Claude lane we logged bills one vector.
|
|
25
|
+
*
|
|
22
26
|
* Runs that never answered are inconclusive rather than silent: the probes
|
|
23
27
|
* alone call any good persona prompt genuine.
|
|
24
28
|
*/
|
|
25
29
|
import { isTransientError } from "../identity/signals";
|
|
26
30
|
import { impossibleRuns, isCharacterEstimate, knowsMaker, matchTokenizerFamilies, } from "../models/tokenizers";
|
|
27
31
|
import { defineRule } from "./types";
|
|
32
|
+
/**
|
|
33
|
+
* More than double and ten tokens apart is no tokenizer. A router may serve the
|
|
34
|
+
* two copies through a provider that counts and one that estimates (Claude 151
|
|
35
|
+
* against o200k's 101 on this text), so a closer pair proves nothing.
|
|
36
|
+
*/
|
|
37
|
+
const REPEAT_UNSTABLE = (a, b) => Math.abs(a - b) > 10 && Math.max(a, b) > 2 * Math.min(a, b);
|
|
28
38
|
/** Judge one measured vector against the claimed maker. Pure. */
|
|
29
39
|
export function judgeTokenizerFamily(maker, vector) {
|
|
30
40
|
const matched = matchTokenizerFamilies(vector);
|
|
@@ -62,13 +72,17 @@ async function familyResult(ctx) {
|
|
|
62
72
|
spaces: prompt(ev.variants.spaces),
|
|
63
73
|
code: prompt(ev.variants.code),
|
|
64
74
|
};
|
|
75
|
+
const r1 = prompt(ev.repeats[0]);
|
|
76
|
+
const r2 = prompt(ev.repeats[1]);
|
|
77
|
+
if (r1 !== null && r2 !== null && REPEAT_UNSTABLE(r1, r2))
|
|
78
|
+
return { state: "unstable", ...empty, repeats: [r1, r2] };
|
|
65
79
|
const impossible = impossibleRuns(counts).flatMap((run) => {
|
|
66
80
|
const prompt = counts[run];
|
|
67
81
|
return prompt === null ? [] : [{ run, prompt }];
|
|
68
82
|
});
|
|
69
83
|
if (impossible.length > 0)
|
|
70
84
|
return { state: "impossible", ...empty, impossible };
|
|
71
|
-
const all = [ev.base, ...Object.values(ev.variants)];
|
|
85
|
+
const all = [ev.base, ...Object.values(ev.variants), ...ev.repeats];
|
|
72
86
|
const busy = all.find((r) => r.status === null
|
|
73
87
|
? isTransientError(r.error ?? "")
|
|
74
88
|
: isTransientError(`HTTP ${r.status}`));
|
|
@@ -111,6 +125,12 @@ export const tokenizerFamilyRule = defineRule({
|
|
|
111
125
|
applies: (ctx) => ctx.wire.id !== "gemini",
|
|
112
126
|
judge: async (ctx) => {
|
|
113
127
|
const r = await familyResult(ctx);
|
|
128
|
+
if (r.state === "unstable" && r.repeats)
|
|
129
|
+
return {
|
|
130
|
+
severity: "fail",
|
|
131
|
+
reason: `tokenizer-family: billed the same Chinese text ${r.repeats[0]} and then ${r.repeats[1]} prompt tokens; a real tokenizer bills identical text identically, so the provider makes up usage and hides the model's real counts`,
|
|
132
|
+
data: r,
|
|
133
|
+
};
|
|
114
134
|
if (r.state === "impossible" && r.impossible) {
|
|
115
135
|
const runs = r.impossible
|
|
116
136
|
.map((i) => `${i.prompt} for the ${i.run} run`)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-family.js","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-family.js","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,UAAU,EACV,sBAAsB,GAEvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,UAAU,EAAgB,MAAM,SAAS,CAAC;AAgCnD;;;;GAIG;AACH,MAAM,eAAe,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAC/C,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE9D,iEAAiE;AACjE,MAAM,UAAU,oBAAoB,CAClC,KAAa,EACb,MAAoB;IAEpB,MAAM,OAAO,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG;QACX,MAAM;QACN,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;KACvD,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;YACjE,GAAG,IAAI;SACR,CAAC;IACJ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;QACxD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC9E,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QACpB,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,IAAI;YACP,MAAM,EAAE,wBAAwB,KAAK,EAAE;SACxC,CAAC;IACJ,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,GAA4B;IAE5B,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,CAAC,CAAsB,EAAE,EAAE,CACxC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG;QACjC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;QACpD,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAClC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;QAC5B,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAClC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;KAC/B,CAAC;IACF,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,eAAe,CAAC,EAAE,EAAE,EAAE,CAAC;QACvD,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;IAC5D,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QACxD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IACH,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QACvB,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC,CAAC,MAAM,KAAK,IAAI;QACf,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CACzC,CAAC;IACF,IAAI,IAAI;QACN,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,IAAI,CAAC,KAAK,IAAI,MAAM,EAAE;SAC/D,CAAC;IACJ,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,IAAI;QACf,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,UAAU,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,UAAU,EAAE;SAChF,CAAC;IACJ,4EAA4E;IAC5E,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAC1E,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,MAAM,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;QACrE,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,2BAA2B;SACpC,CAAC;IACJ,OAAO,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC;IAC5C,EAAE,EAAE,kBAAkB;IACtB,KAAK,EAAE,UAAU;IACjB,KAAK,EAAE,CAAC,cAAc,CAAC;IACvB,KAAK,EAAE,iBAAiB;IACxB,qGAAqG;IACrG,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,QAAQ;IAC1C,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,OAAO;YACrC,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,kDAAkD,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qIAAqI;gBACpO,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,KAAK,KAAK,YAAY,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,YAAY,CAAC,CAAC,GAAG,MAAM,CAAC;iBAC9C,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,OAAO,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;gBAC5B,CAAC,CAAC;oBACE,QAAQ,EAAE,MAAM;oBAChB,MAAM,EAAE,4BAA4B,IAAI,2HAA2H;oBACnK,IAAI,EAAE,CAAC;iBACR;gBACH,CAAC,CAAC;oBACE,QAAQ,EAAE,cAAc;oBACxB,MAAM,EAAE,4BAA4B,IAAI,mIAAmI;oBAC3K,IAAI,EAAE,CAAC;iBACR,CAAC;QACR,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,KAAK,YAAY,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,OAAO;gBACL,QAAQ,EAAE,cAAc;gBACxB,MAAM,EAAE,0DAA0D,CAAC,CAAC,MAAM,mDAAmD;gBAC7H,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnB,IAAI,CAAC,CAAC,KAAK,KAAK,gBAAgB;YAC9B,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,qKAAqK;gBAChP,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACvC,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;YACnK,IAAI,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;CACtE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-model-verifier",
|
|
3
|
-
"version": "3.9.
|
|
3
|
+
"version": "3.9.8",
|
|
4
4
|
"description": "Verify an AI API endpoint really serves the model it claims: a vendor adapter per wire, a maker table for who trained what, a rule table over shared evidence, and per model facts. Model substitution, Claude thinking signatures, token billing accuracy, envelope and throughput evidence.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|