ai-model-verifier 3.9.1 → 3.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/models/tokenizers.d.ts +7 -0
- package/dist/models/tokenizers.d.ts.map +1 -1
- package/dist/models/tokenizers.js +13 -1
- package/dist/models/tokenizers.js.map +1 -1
- package/dist/rules/thinking-honored.d.ts +6 -0
- package/dist/rules/thinking-honored.d.ts.map +1 -1
- package/dist/rules/thinking-honored.js +25 -3
- package/dist/rules/thinking-honored.js.map +1 -1
- package/dist/rules/tokenizer-family.d.ts +5 -0
- package/dist/rules/tokenizer-family.d.ts.map +1 -1
- package/dist/rules/tokenizer-family.js +17 -3
- package/dist/rules/tokenizer-family.js.map +1 -1
- package/package.json +1 -1
|
@@ -31,4 +31,11 @@ export declare const TOKENIZER_FAMILIES: readonly TokenizerFamily[];
|
|
|
31
31
|
export declare const SCRIPT_TOLERANCE = 1;
|
|
32
32
|
export declare function matchTokenizerFamilies(measured: ScriptVector, families?: readonly TokenizerFamily[]): TokenizerFamily[];
|
|
33
33
|
export declare const knowsMaker: (maker: string, families?: readonly TokenizerFamily[]) => boolean;
|
|
34
|
+
/**
|
|
35
|
+
* Billed as characters divided by about four on every run: no tokenizer does
|
|
36
|
+
* that on the Chinese run (the leanest known bills 0.47 a character), so the
|
|
37
|
+
* provider is computing usage itself and hiding the backend's real counts.
|
|
38
|
+
* UTIM's "gemini-4-argon" billed 80/32/50/92.
|
|
39
|
+
*/
|
|
40
|
+
export declare function isCharacterEstimate(measured: ScriptVector): boolean;
|
|
34
41
|
//# sourceMappingURL=tokenizers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../../src/models/tokenizers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,
|
|
1
|
+
{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../../src/models/tokenizers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;AAEnE,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,OAAO,EAAE,CAAC;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AASF,eAAO,MAAM,kBAAkB,EAAE,SAAS,eAAe,EAmCxD,CAAC;AAEF,uEAAuE;AACvE,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,YAAY,EACtB,QAAQ,GAAE,SAAS,eAAe,EAAuB,GACxD,eAAe,EAAE,CAMnB;AAED,eAAO,MAAM,UAAU,UACd,MAAM,aACH,SAAS,eAAe,EAAE,KACnC,OAAkE,CAAC;AAEtE;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO,CAKnE"}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* all bill 101/61/101/120, so those entries name every maker and only rule out
|
|
14
14
|
* the others.
|
|
15
15
|
*/
|
|
16
|
-
import { SCRIPT_VARIANT_KEYS } from "../probes/prompts";
|
|
16
|
+
import { SCRIPT_VARIANT_KEYS, SCRIPT_VARIANTS, } from "../probes/prompts";
|
|
17
17
|
const v = (digits, cjk, spaces, code) => ({ digits, cjk, spaces, code });
|
|
18
18
|
export const TOKENIZER_FAMILIES = [
|
|
19
19
|
{
|
|
@@ -57,4 +57,16 @@ export function matchTokenizerFamilies(measured, families = TOKENIZER_FAMILIES)
|
|
|
57
57
|
return families.filter((f) => SCRIPT_VARIANT_KEYS.every((k) => Math.abs(f.vector[k] - measured[k]) <= SCRIPT_TOLERANCE));
|
|
58
58
|
}
|
|
59
59
|
export const knowsMaker = (maker, families = TOKENIZER_FAMILIES) => families.some((f) => f.makers.some((m) => m === maker));
|
|
60
|
+
/**
|
|
61
|
+
* Billed as characters divided by about four on every run: no tokenizer does
|
|
62
|
+
* that on the Chinese run (the leanest known bills 0.47 a character), so the
|
|
63
|
+
* provider is computing usage itself and hiding the backend's real counts.
|
|
64
|
+
* UTIM's "gemini-4-argon" billed 80/32/50/92.
|
|
65
|
+
*/
|
|
66
|
+
export function isCharacterEstimate(measured) {
|
|
67
|
+
return SCRIPT_VARIANT_KEYS.every((k) => {
|
|
68
|
+
const ratio = measured[k] / [...SCRIPT_VARIANTS[k]].length;
|
|
69
|
+
return ratio >= 0.2 && ratio <= 0.3;
|
|
70
|
+
});
|
|
71
|
+
}
|
|
60
72
|
//# sourceMappingURL=tokenizers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizers.js","sourceRoot":"","sources":["../../src/models/tokenizers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,
|
|
1
|
+
{"version":3,"file":"tokenizers.js","sourceRoot":"","sources":["../../src/models/tokenizers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EACL,mBAAmB,EACnB,eAAe,GAEhB,MAAM,mBAAmB,CAAC;AAe3B,MAAM,CAAC,GAAG,CACR,MAAc,EACd,GAAW,EACX,MAAc,EACd,IAAY,EACE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAEnD,MAAM,CAAC,MAAM,kBAAkB,GAA+B;IAC5D;QACE,EAAE,EAAE,OAAO;QACX,MAAM,EAAE,CAAC,QAAQ,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;QAC7B,SAAS,EAAE,IAAI;KAChB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,CAAC,QAAQ,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;QAC7B,SAAS,EAAE,IAAI;KAChB;IACD,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACvE,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAClE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC9D;QACE,EAAE,EAAE,UAAU;QACd,MAAM,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC;QAC3C,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;KAC7B;IACD;QACE,EAAE,EAAE,MAAM;QACV,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC;QACxC,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;KAC7B;IACD,4EAA4E;IAC5E,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACpE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACpE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAChE,EAAE,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC3E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;IAC7E,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE;CAC9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,MAAM,UAAU,sBAAsB,CACpC,QAAsB,EACtB,QAAQ,GAA+B,kBAAkB;IAEzD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3B,mBAAmB,CAAC,KAAK,CACvB,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAC/D,CACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,QAAQ,GAA+B,kBAAkB,EAChD,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;AAEtE;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAsB;IACxD,OAAO,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3D,OAAO,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -10,6 +10,10 @@
|
|
|
10
10
|
* Reasoning tokens count when the reply reports them; otherwise the completion
|
|
11
11
|
* count stands in, since a relay that folds thoughts into the answer still bills
|
|
12
12
|
* them there.
|
|
13
|
+
*
|
|
14
|
+
* A reply carrying Google's thought signature came from Gemini itself, so too
|
|
15
|
+
* little thought there means effort capped by the seller (gg's cheap Argon
|
|
16
|
+
* group, 8 tokens at high), not another model: inconclusive, not suspicious.
|
|
13
17
|
*/
|
|
14
18
|
/** A third of the lowest genuine high-effort count, room for shorter thoughts. */
|
|
15
19
|
export declare const MIN_ASKED_THINKING_TOKENS = 100;
|
|
@@ -32,5 +36,7 @@ export declare function judgeThinkingHonored(usage: {
|
|
|
32
36
|
reasoning: number | null;
|
|
33
37
|
completion: number | null;
|
|
34
38
|
} | null, min?: number): ThinkingHonoredResult;
|
|
39
|
+
/** Gemini's own thought signature, on its native wire or passed through an OpenAI-shape relay. */
|
|
40
|
+
export declare function carriesThoughtSignature(data: unknown): boolean;
|
|
35
41
|
export declare const thinkingHonoredRule: import("./types").Rule<"thinking-honored", "askedThinkingReply", "evidence">;
|
|
36
42
|
//# sourceMappingURL=thinking-honored.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"thinking-honored.d.ts","sourceRoot":"","sources":["../../src/rules/thinking-honored.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"thinking-honored.d.ts","sourceRoot":"","sources":["../../src/rules/thinking-honored.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,kFAAkF;AAClF,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C,MAAM,MAAM,oBAAoB,GAC5B,SAAS;AACX,uFAAuF;GACrF,SAAS;AACX,yDAAyD;GACvD,YAAY;AACd,oDAAoD;GAClD,SAAS,CAAC;AAEd,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AASF,mFAAmF;AACnF,wBAAgB,oBAAoB,CAClC,KAAK,EAAE;IAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,EACrE,GAAG,SAA4B,GAC9B,qBAAqB,CA2BvB;AAED,kGAAkG;AAClG,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAI9D;AAgBD,eAAO,MAAM,mBAAmB,8EAwB9B,CAAC"}
|
|
@@ -10,6 +10,10 @@
|
|
|
10
10
|
* Reasoning tokens count when the reply reports them; otherwise the completion
|
|
11
11
|
* count stands in, since a relay that folds thoughts into the answer still bills
|
|
12
12
|
* them there.
|
|
13
|
+
*
|
|
14
|
+
* A reply carrying Google's thought signature came from Gemini itself, so too
|
|
15
|
+
* little thought there means effort capped by the seller (gg's cheap Argon
|
|
16
|
+
* group, 8 tokens at high), not another model: inconclusive, not suspicious.
|
|
13
17
|
*/
|
|
14
18
|
import { defineRule } from "./types";
|
|
15
19
|
/** A third of the lowest genuine high-effort count, room for shorter thoughts. */
|
|
@@ -48,6 +52,13 @@ export function judgeThinkingHonored(usage, min = MIN_ASKED_THINKING_TOKENS) {
|
|
|
48
52
|
};
|
|
49
53
|
return { state: "thought", ...base };
|
|
50
54
|
}
|
|
55
|
+
/** Gemini's own thought signature, on its native wire or passed through an OpenAI-shape relay. */
|
|
56
|
+
export function carriesThoughtSignature(data) {
|
|
57
|
+
if (data === null || data === undefined)
|
|
58
|
+
return false;
|
|
59
|
+
const text = typeof data === "string" ? data : JSON.stringify(data);
|
|
60
|
+
return /"thought_?[sS]ignature"\s*:\s*"[A-Za-z0-9+/=_-]{16,}/.test(text);
|
|
61
|
+
}
|
|
51
62
|
async function honoredResult(ctx) {
|
|
52
63
|
const res = await ctx.evidence("askedThinkingReply");
|
|
53
64
|
if (!res)
|
|
@@ -68,9 +79,20 @@ export const thinkingHonoredRule = defineRule({
|
|
|
68
79
|
applies: (ctx) => ctx.facts.thinksWhenAsked && ctx.wire.read.reasoningUsage !== undefined,
|
|
69
80
|
judge: async (ctx) => {
|
|
70
81
|
const r = await honoredResult(ctx);
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
82
|
+
if (r.state !== "ignored")
|
|
83
|
+
return null;
|
|
84
|
+
const res = await ctx.evidence("askedThinkingReply");
|
|
85
|
+
if (res && carriesThoughtSignature(res.data))
|
|
86
|
+
return {
|
|
87
|
+
severity: "inconclusive",
|
|
88
|
+
reason: `thinking-locked: ${r.reason}, but the reply carries Google's thought signature, so the seller caps reasoning on real Gemini`,
|
|
89
|
+
data: r,
|
|
90
|
+
};
|
|
91
|
+
return {
|
|
92
|
+
severity: "fail",
|
|
93
|
+
reason: `thinking-ignored: ${r.reason}`,
|
|
94
|
+
data: r,
|
|
95
|
+
};
|
|
74
96
|
},
|
|
75
97
|
report: async (ctx) => ({ thinkingHonored: await honoredResult(ctx) }),
|
|
76
98
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"thinking-honored.js","sourceRoot":"","sources":["../../src/rules/thinking-honored.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"thinking-honored.js","sourceRoot":"","sources":["../../src/rules/thinking-honored.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAE,UAAU,EAAgB,MAAM,SAAS,CAAC;AAEnD,kFAAkF;AAClF,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAmB7C,MAAM,OAAO,GAA0B;IACrC,KAAK,EAAE,SAAS;IAChB,cAAc,EAAE,IAAI;IACpB,eAAe,EAAE,IAAI;IACrB,gBAAgB,EAAE,IAAI;CACvB,CAAC;AAEF,mFAAmF;AACnF,MAAM,UAAU,oBAAoB,CAClC,KAAqE,EACrE,GAAG,GAAG,yBAAyB;IAE/B,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;QACnE,OAAO,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC1E,MAAM,QAAQ,GACZ,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC;QAC7C,CAAC,CAAC,KAAK,CAAC,SAAS;QACjB,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;IACvB,MAAM,IAAI,GAAG;QACX,cAAc,EAAE,QAAQ;QACxB,eAAe,EAAE,KAAK,CAAC,SAAS;QAChC,gBAAgB,EAAE,KAAK,CAAC,UAAU;KACnC,CAAC;IACF,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,CAAC;QACrC,OAAO,QAAQ,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC;YAC7C,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE;YACzD,CAAC,CAAC;gBACE,KAAK,EAAE,SAAS;gBAChB,GAAG,IAAI;gBACP,MAAM,EAAE,4DAA4D;aACrE,CAAC;IACR,IAAI,QAAQ,GAAG,GAAG;QAChB,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,IAAI;YACP,MAAM,EAAE,0BAA0B,QAAQ,4BAA4B,GAAG,EAAE;SAC5E,CAAC;IACJ,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,uBAAuB,CAAC,IAAa;IACnD,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACpE,OAAO,sDAAsD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAkC;IAElC,MAAM,GAAG,GAA2B,MAAM,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IAC7E,IAAI,CAAC,GAAG;QAAE,OAAO,OAAO,CAAC;IACzB,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI;QACjC,OAAO;YACL,GAAG,OAAO;YACV,KAAK,EAAE,YAAY;YACnB,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,QAAQ,GAAG,CAAC,MAAM,IAAI,MAAM,EAAE;SACpD,CAAC;IACJ,OAAO,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC;IAC5C,EAAE,EAAE,kBAAkB;IACtB,KAAK,EAAE,UAAU;IACjB,KAAK,EAAE,CAAC,oBAAoB,CAAC;IAC7B,KAAK,EAAE,eAAe;IACtB,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CACf,GAAG,CAAC,KAAK,CAAC,eAAe,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS;IACzE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,CAAC,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QACrD,IAAI,GAAG,IAAI,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC;YAC1C,OAAO;gBACL,QAAQ,EAAE,cAAc;gBACxB,MAAM,EAAE,oBAAoB,CAAC,CAAC,MAAM,iGAAiG;gBACrI,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,qBAAqB,CAAC,CAAC,MAAM,EAAE;YACvC,IAAI,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;CACvE,CAAC,CAAC"}
|
|
@@ -11,6 +11,9 @@
|
|
|
11
11
|
* the claimed maker has known vectors, the measured vector matches none of
|
|
12
12
|
* them, it does match another family, and that family is not one relays
|
|
13
13
|
* estimate usage with (OpenAI's).
|
|
14
|
+
*
|
|
15
|
+
* Also fails when every run bills about one token per four characters: that
|
|
16
|
+
* is a provider making up usage so the vocabulary behind it cannot be seen.
|
|
14
17
|
*/
|
|
15
18
|
import { type ScriptVector } from "../models/tokenizers";
|
|
16
19
|
export type TokenizerFamilyState = "own"
|
|
@@ -18,6 +21,8 @@ export type TokenizerFamilyState = "own"
|
|
|
18
21
|
| "foreign"
|
|
19
22
|
/** Matches only an estimator family: the relay may be counting. */
|
|
20
23
|
| "estimated"
|
|
24
|
+
/** Billed as characters / 4: the provider makes up usage, hiding the backend. */
|
|
25
|
+
| "char-estimated"
|
|
21
26
|
/** Measured but matches no known family. */
|
|
22
27
|
| "unknown" | "unmeasured";
|
|
23
28
|
export type TokenizerFamilyResult = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-family.d.ts","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-family.d.ts","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,sBAAsB,CAAC;AAG9B,MAAM,MAAM,oBAAoB,GAC5B,KAAK;AACP,sEAAsE;GACpE,SAAS;AACX,mEAAmE;GACjE,WAAW;AACb,iFAAiF;GAC/E,gBAAgB;AAClB,4CAA4C;GAC1C,SAAS,GACT,YAAY,CAAC;AAEjB,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,iEAAiE;AACjE,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,YAAY,GACnB,qBAAqB,CAsBvB;AAoCD,eAAO,MAAM,mBAAmB,wEAyB9B,CAAC"}
|
|
@@ -11,8 +11,11 @@
|
|
|
11
11
|
* the claimed maker has known vectors, the measured vector matches none of
|
|
12
12
|
* them, it does match another family, and that family is not one relays
|
|
13
13
|
* estimate usage with (OpenAI's).
|
|
14
|
+
*
|
|
15
|
+
* Also fails when every run bills about one token per four characters: that
|
|
16
|
+
* is a provider making up usage so the vocabulary behind it cannot be seen.
|
|
14
17
|
*/
|
|
15
|
-
import { knowsMaker, matchTokenizerFamilies, } from "../models/tokenizers";
|
|
18
|
+
import { isCharacterEstimate, knowsMaker, matchTokenizerFamilies, } from "../models/tokenizers";
|
|
16
19
|
import { defineRule } from "./types";
|
|
17
20
|
/** Judge one measured vector against the claimed maker. Pure. */
|
|
18
21
|
export function judgeTokenizerFamily(maker, vector) {
|
|
@@ -23,7 +26,10 @@ export function judgeTokenizerFamily(maker, vector) {
|
|
|
23
26
|
makers: [...new Set(matched.flatMap((f) => f.makers))],
|
|
24
27
|
};
|
|
25
28
|
if (matched.length === 0)
|
|
26
|
-
return {
|
|
29
|
+
return {
|
|
30
|
+
state: isCharacterEstimate(vector) ? "char-estimated" : "unknown",
|
|
31
|
+
...base,
|
|
32
|
+
};
|
|
27
33
|
if (matched.some((f) => f.makers.some((m) => m === maker)))
|
|
28
34
|
return { state: "own", ...base };
|
|
29
35
|
if (matched.every((f) => f.estimator))
|
|
@@ -75,9 +81,17 @@ export const tokenizerFamilyRule = defineRule({
|
|
|
75
81
|
applies: (ctx) => ctx.wire.id !== "gemini",
|
|
76
82
|
judge: async (ctx) => {
|
|
77
83
|
const r = await familyResult(ctx);
|
|
78
|
-
if (
|
|
84
|
+
if (!r.vector)
|
|
79
85
|
return null;
|
|
80
86
|
const v = r.vector;
|
|
87
|
+
if (r.state === "char-estimated")
|
|
88
|
+
return {
|
|
89
|
+
severity: "fail",
|
|
90
|
+
reason: `tokenizer-family: billed ${v.digits}/${v.cjk}/${v.spaces}/${v.code}, about one token per four characters on every run including Chinese, which no tokenizer does: the provider computes usage itself and hides the model's real counts`,
|
|
91
|
+
data: r,
|
|
92
|
+
};
|
|
93
|
+
if (r.state !== "foreign")
|
|
94
|
+
return null;
|
|
81
95
|
return {
|
|
82
96
|
severity: "fail",
|
|
83
97
|
reason: `tokenizer-family: billed ${v.digits}/${v.cjk}/${v.spaces}/${v.code}, the ${r.families.join("/")} vocabulary (${r.makers.join(", ")}), not ${ctx.maker.id}`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-family.js","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-family.js","sourceRoot":"","sources":["../../src/rules/tokenizer-family.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EACL,mBAAmB,EACnB,UAAU,EACV,sBAAsB,GAEvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,UAAU,EAAgB,MAAM,SAAS,CAAC;AAsBnD,iEAAiE;AACjE,MAAM,UAAU,oBAAoB,CAClC,KAAa,EACb,MAAoB;IAEpB,MAAM,OAAO,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG;QACX,MAAM;QACN,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;KACvD,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;YACjE,GAAG,IAAI;SACR,CAAC;IACJ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;QACxD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC9E,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QACpB,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,IAAI;YACP,MAAM,EAAE,wBAAwB,KAAK,EAAE;SACxC,CAAC;IACJ,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,GAA4B;IAE5B,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,CAAC,CAAsB,EAAE,EAAE,CACxC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG;QACjC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;QACpD,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,IAAI;QACf,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,UAAU,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,UAAU,EAAE;SAChF,CAAC;IACJ,4EAA4E;IAC5E,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAC1E,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,MAAM,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;QACrE,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,MAAM,EAAE,2BAA2B;SACpC,CAAC;IACJ,OAAO,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC;IAC5C,EAAE,EAAE,kBAAkB;IACtB,KAAK,EAAE,UAAU;IACjB,KAAK,EAAE,CAAC,cAAc,CAAC;IACvB,KAAK,EAAE,iBAAiB;IACxB,qGAAqG;IACrG,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,QAAQ;IAC1C,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnB,IAAI,CAAC,CAAC,KAAK,KAAK,gBAAgB;YAC9B,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,qKAAqK;gBAChP,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACvC,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,4BAA4B,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;YACnK,IAAI,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;CACtE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-model-verifier",
|
|
3
|
-
"version": "3.9.
|
|
3
|
+
"version": "3.9.3",
|
|
4
4
|
"description": "Verify an AI API endpoint really serves the model it claims: a vendor adapter per wire, a maker table for who trained what, a rule table over shared evidence, and per model facts. Model substitution, Claude thinking signatures, token billing accuracy, envelope and throughput evidence.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|