ai-model-verifier 3.9.10 → 3.9.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -18,9 +18,11 @@
|
|
|
18
18
|
* The a7 Claude merchants that did so (3825, 5630, 1169) also refused the
|
|
19
19
|
* Tiananmen question the way Chinese models do, so it was never a cache.
|
|
20
20
|
*
|
|
21
|
-
* The Chinese run is sent twice more behind
|
|
22
|
-
*
|
|
23
|
-
*
|
|
21
|
+
* The Chinese run is sent twice more, each behind its own leading nonce so no
|
|
22
|
+
* cache can serve it. A copy billed below the leanest family's count for that
|
|
23
|
+
* text alone is made up: wusrouter billed it 28 and 29. The two copies are not
|
|
24
|
+
* compared with each other: a router may send them to lanes that inject
|
|
25
|
+
* different wrappers (ours billed 221 and 9830, both honest).
|
|
24
26
|
*
|
|
25
27
|
* Runs that never answered are inconclusive rather than silent: the probes
|
|
26
28
|
* alone call any good persona prompt genuine.
|
|
@@ -38,7 +40,7 @@ export type TokenizerFamilyState = "own"
|
|
|
38
40
|
| "impossible"
|
|
39
41
|
/** Measured but matches no known family. */
|
|
40
42
|
| "unknown" | "unmeasured"
|
|
41
|
-
/**
|
|
43
|
+
/** A nonce copy of the Chinese run billed below every tokenizer: made up usage. */
|
|
42
44
|
| "unstable"
|
|
43
45
|
/** A run timed out or the endpoint was busy, so nothing could be read. */
|
|
44
46
|
| "unanswered";
|
|
@@ -53,8 +55,8 @@ export type TokenizerFamilyResult = {
|
|
|
53
55
|
run: ScriptVariant;
|
|
54
56
|
prompt: number;
|
|
55
57
|
}[];
|
|
56
|
-
/**
|
|
57
|
-
|
|
58
|
+
/** The lowest prompt count of the nonce copies of the Chinese run. */
|
|
59
|
+
repeat?: number;
|
|
58
60
|
};
|
|
59
61
|
/** Judge one measured vector against the claimed maker. Pure. */
|
|
60
62
|
export declare function judgeTokenizerFamily(maker: string, vector: ScriptVector): TokenizerFamilyResult;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-family.d.ts","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-family.d.ts","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,sBAAsB,CAAC;AAG9B,MAAM,MAAM,oBAAoB,GAC5B,KAAK;AACP,sEAAsE;GACpE,SAAS;AACX,mEAAmE;GACjE,WAAW;AACb,iFAAiF;GAC/E,gBAAgB;AAClB,gFAAgF;GAC9E,YAAY;AACd,4CAA4C;GAC1C,SAAS,GACT,YAAY;AACd,mFAAmF;GACjF,UAAU;AACZ,0EAA0E;GACxE,YAAY,CAAC;AAEjB,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,UAAU,CAAC,EAAE;QAAE,GAAG,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAKF,iEAAiE;AACjE,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,YAAY,GACnB,qBAAqB,CAsBvB;AAgED,eAAO,MAAM,mBAAmB,wEA+C9B,CAAC"}
|
|
@@ -18,22 +18,20 @@
|
|
|
18
18
|
* The a7 Claude merchants that did so (3825, 5630, 1169) also refused the
|
|
19
19
|
* Tiananmen question the way Chinese models do, so it was never a cache.
|
|
20
20
|
*
|
|
21
|
-
* The Chinese run is sent twice more behind
|
|
22
|
-
*
|
|
23
|
-
*
|
|
21
|
+
* The Chinese run is sent twice more, each behind its own leading nonce so no
|
|
22
|
+
* cache can serve it. A copy billed below the leanest family's count for that
|
|
23
|
+
* text alone is made up: wusrouter billed it 28 and 29. The two copies are not
|
|
24
|
+
* compared with each other: a router may send them to lanes that inject
|
|
25
|
+
* different wrappers (ours billed 221 and 9830, both honest).
|
|
24
26
|
*
|
|
25
27
|
* Runs that never answered are inconclusive rather than silent: the probes
|
|
26
28
|
* alone call any good persona prompt genuine.
|
|
27
29
|
*/
|
|
28
30
|
import { isTransientError } from "../identity/signals";
|
|
29
|
-
import { impossibleRuns, isCharacterEstimate, knowsMaker, matchTokenizerFamilies, } from "../models/tokenizers";
|
|
31
|
+
import { impossibleRuns, isCharacterEstimate, knowsMaker, matchTokenizerFamilies, TOKENIZER_FAMILIES, } from "../models/tokenizers";
|
|
30
32
|
import { defineRule } from "./types";
|
|
31
|
-
/**
|
|
32
|
-
|
|
33
|
-
* two copies through a provider that counts and one that estimates (Claude 151
|
|
34
|
-
* against o200k's 101 on this text), so a closer pair proves nothing.
|
|
35
|
-
*/
|
|
36
|
-
const REPEAT_UNSTABLE = (a, b) => Math.abs(a - b) > 10 && Math.max(a, b) > 2 * Math.min(a, b);
|
|
33
|
+
/** The leanest known family's count for the Chinese text alone; a whole nonce copy cannot bill less. */
|
|
34
|
+
const REPEAT_FLOOR = Math.min(...TOKENIZER_FAMILIES.map((f) => f.vector.cjk));
|
|
37
35
|
/** Judge one measured vector against the claimed maker. Pure. */
|
|
38
36
|
export function judgeTokenizerFamily(maker, vector) {
|
|
39
37
|
const matched = matchTokenizerFamilies(vector);
|
|
@@ -73,8 +71,9 @@ async function familyResult(ctx) {
|
|
|
73
71
|
};
|
|
74
72
|
const r1 = prompt(ev.repeats[0]);
|
|
75
73
|
const r2 = prompt(ev.repeats[1]);
|
|
76
|
-
|
|
77
|
-
|
|
74
|
+
const low = Math.min(...[r1, r2].filter((n) => n !== null));
|
|
75
|
+
if (low < REPEAT_FLOOR)
|
|
76
|
+
return { state: "unstable", ...empty, repeat: low };
|
|
78
77
|
const impossible = impossibleRuns(counts).flatMap((run) => {
|
|
79
78
|
const prompt = counts[run];
|
|
80
79
|
return prompt === null ? [] : [{ run, prompt }];
|
|
@@ -124,10 +123,10 @@ export const tokenizerFamilyRule = defineRule({
|
|
|
124
123
|
applies: (ctx) => ctx.wire.id !== "gemini",
|
|
125
124
|
judge: async (ctx) => {
|
|
126
125
|
const r = await familyResult(ctx);
|
|
127
|
-
if (r.state === "unstable" && r.
|
|
126
|
+
if (r.state === "unstable" && r.repeat !== undefined)
|
|
128
127
|
return {
|
|
129
128
|
severity: "fail",
|
|
130
|
-
reason: `tokenizer-family: billed
|
|
129
|
+
reason: `tokenizer-family: billed ${r.repeat} prompt tokens for a nonce copy of the Chinese run, below the ${REPEAT_FLOOR} the leanest known tokenizer needs for that text alone: the provider makes up usage and hides the model's real counts`,
|
|
131
130
|
data: r,
|
|
132
131
|
};
|
|
133
132
|
if (r.state === "impossible" && r.impossible) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-family.js","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-family.js","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,UAAU,EACV,sBAAsB,EACtB,kBAAkB,GAEnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,UAAU,EAAgB,MAAM,SAAS,CAAC;AAgCnD,wGAAwG;AACxG,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;AAE9E,iEAAiE;AACjE,MAAM,UAAU,oBAAoB,CAClC,KAAa,EACb,MAAoB;IAEpB,MAAM,OAAO,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG;QACX,MAAM;QACN,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;KACvD,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;YACjE,GAAG,IAAI;SACR,CAAC;IACJ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;QACxD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC9E,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QACpB,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,IAAI;YACP,MAAM,EAAE,wBAAwB,KAAK,EAAE;SACxC,CAAC;IACJ,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,GAA4B;IAE5B,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,CAAC,CAAsB,EAAE,EAAE,CACxC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG;QACjC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;QACpD,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAClC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;QAC5B,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAClC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;KAC/B,CAAC;IACF,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACzE,IAAI,GAAG,GAAG,YAAY;QAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC5E,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QACxD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IACH,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QACvB,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC,CAAC,MAAM,KAAK,IAAI;QACf,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CACzC,CAAC;IACF,IAAI,IAAI;QACN,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,IAAI,CAAC,KAAK,IAAI,MAAM,EAAE;SAC/D,CAAC;IACJ,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,IAAI;QACf,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,UAAU,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,UAAU,EAAE;SAChF,CAAC;IACJ,4EAA4E;IAC5E,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAC1E,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,MAAM,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;QACrE,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,2BAA2B;SACpC,CAAC;IACJ,OAAO,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC;IAC5C,EAAE,EAAE,kBAAkB;IACtB,KAAK,EAAE,UAAU;IACjB,KAAK,EAAE,CAAC,cAAc,CAAC;IACvB,KAAK,EAAE,iBAAiB;IACxB,qGAAqG;IACrG,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,QAAQ;IAC1C,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;YAClD,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,iEAAiE,YAAY,uHAAuH;gBAChP,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,KAAK,KAAK,YAAY,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,YAAY,CAAC,CAAC,GAAG,MAAM,CAAC;iBAC9C,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,4BAA4B,IAAI,2HAA2H;gBACnK,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,KAAK,YAAY,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,OAAO;gBACL,QAAQ,EAAE,cAAc;gBACxB,MAAM,EAAE,0DAA0D,CAAC,CAAC,MAAM,mDAAmD;gBAC7H,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnB,IAAI,CAAC,CAAC,KAAK,KAAK,gBAAgB;YAC9B,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,qKAAqK;gBAChP,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACvC,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;YACnK,IAAI,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;CACtE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-model-verifier",
|
|
3
|
-
"version": "3.9.
|
|
3
|
+
"version": "3.9.11",
|
|
4
4
|
"description": "Verify an AI API endpoint really serves the model it claims: a vendor adapter per wire, a maker table for who trained what, a rule table over shared evidence, and per model facts. Model substitution, Claude thinking signatures, token billing accuracy, envelope and throughput evidence.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|