ai-model-verifier 3.9.2 → 3.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -10,6 +10,10 @@
|
|
|
10
10
|
* Reasoning tokens count when the reply reports them; otherwise the completion
|
|
11
11
|
* count stands in, since a relay that folds thoughts into the answer still bills
|
|
12
12
|
* them there.
|
|
13
|
+
*
|
|
14
|
+
* A reply carrying Google's thought signature came from Gemini itself, so too
|
|
15
|
+
* little thought there means effort capped by the seller (gg's cheap Argon
|
|
16
|
+
* group, 8 tokens at high), not another model: inconclusive, not suspicious.
|
|
13
17
|
*/
|
|
14
18
|
/** A third of the lowest genuine high-effort count, room for shorter thoughts. */
|
|
15
19
|
export declare const MIN_ASKED_THINKING_TOKENS = 100;
|
|
@@ -32,5 +36,7 @@ export declare function judgeThinkingHonored(usage: {
|
|
|
32
36
|
reasoning: number | null;
|
|
33
37
|
completion: number | null;
|
|
34
38
|
} | null, min?: number): ThinkingHonoredResult;
|
|
39
|
+
/** Gemini's own thought signature, on its native wire or passed through an OpenAI-shape relay. */
|
|
40
|
+
export declare function carriesThoughtSignature(data: unknown): boolean;
|
|
35
41
|
export declare const thinkingHonoredRule: import("./types").Rule<"thinking-honored", "askedThinkingReply", "evidence">;
|
|
36
42
|
//# sourceMappingURL=thinking-honored.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"thinking-honored.d.ts","sourceRoot":"","sources":["../../src/rules/thinking-honored.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"thinking-honored.d.ts","sourceRoot":"","sources":["../../src/rules/thinking-honored.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,kFAAkF;AAClF,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C,MAAM,MAAM,oBAAoB,GAC5B,SAAS;AACX,uFAAuF;GACrF,SAAS;AACX,yDAAyD;GACvD,YAAY;AACd,oDAAoD;GAClD,SAAS,CAAC;AAEd,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AASF,mFAAmF;AACnF,wBAAgB,oBAAoB,CAClC,KAAK,EAAE;IAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,EACrE,GAAG,SAA4B,GAC9B,qBAAqB,CA2BvB;AAED,kGAAkG;AAClG,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAI9D;AAgBD,eAAO,MAAM,mBAAmB,8EAwB9B,CAAC"}
|
|
@@ -10,6 +10,10 @@
|
|
|
10
10
|
* Reasoning tokens count when the reply reports them; otherwise the completion
|
|
11
11
|
* count stands in, since a relay that folds thoughts into the answer still bills
|
|
12
12
|
* them there.
|
|
13
|
+
*
|
|
14
|
+
* A reply carrying Google's thought signature came from Gemini itself, so too
|
|
15
|
+
* little thought there means effort capped by the seller (gg's cheap Argon
|
|
16
|
+
* group, 8 tokens at high), not another model: inconclusive, not suspicious.
|
|
13
17
|
*/
|
|
14
18
|
import { defineRule } from "./types";
|
|
15
19
|
/** A third of the lowest genuine high-effort count, room for shorter thoughts. */
|
|
@@ -48,6 +52,13 @@ export function judgeThinkingHonored(usage, min = MIN_ASKED_THINKING_TOKENS) {
|
|
|
48
52
|
};
|
|
49
53
|
return { state: "thought", ...base };
|
|
50
54
|
}
|
|
55
|
+
/** Gemini's own thought signature, on its native wire or passed through an OpenAI-shape relay. */
|
|
56
|
+
export function carriesThoughtSignature(data) {
|
|
57
|
+
if (data === null || data === undefined)
|
|
58
|
+
return false;
|
|
59
|
+
const text = typeof data === "string" ? data : JSON.stringify(data);
|
|
60
|
+
return /"thought_?[sS]ignature"\s*:\s*"[A-Za-z0-9+/=_-]{16,}/.test(text);
|
|
61
|
+
}
|
|
51
62
|
async function honoredResult(ctx) {
|
|
52
63
|
const res = await ctx.evidence("askedThinkingReply");
|
|
53
64
|
if (!res)
|
|
@@ -68,9 +79,20 @@ export const thinkingHonoredRule = defineRule({
|
|
|
68
79
|
applies: (ctx) => ctx.facts.thinksWhenAsked && ctx.wire.read.reasoningUsage !== undefined,
|
|
69
80
|
judge: async (ctx) => {
|
|
70
81
|
const r = await honoredResult(ctx);
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
82
|
+
if (r.state !== "ignored")
|
|
83
|
+
return null;
|
|
84
|
+
const res = await ctx.evidence("askedThinkingReply");
|
|
85
|
+
if (res && carriesThoughtSignature(res.data))
|
|
86
|
+
return {
|
|
87
|
+
severity: "inconclusive",
|
|
88
|
+
reason: `thinking-locked: ${r.reason}, but the reply carries Google's thought signature, so the seller caps reasoning on real Gemini`,
|
|
89
|
+
data: r,
|
|
90
|
+
};
|
|
91
|
+
return {
|
|
92
|
+
severity: "fail",
|
|
93
|
+
reason: `thinking-ignored: ${r.reason}`,
|
|
94
|
+
data: r,
|
|
95
|
+
};
|
|
74
96
|
},
|
|
75
97
|
report: async (ctx) => ({ thinkingHonored: await honoredResult(ctx) }),
|
|
76
98
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"thinking-honored.js","sourceRoot":"","sources":["../../src/rules/thinking-honored.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"thinking-honored.js","sourceRoot":"","sources":["../../src/rules/thinking-honored.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAE,UAAU,EAAgB,MAAM,SAAS,CAAC;AAEnD,kFAAkF;AAClF,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAmB7C,MAAM,OAAO,GAA0B;IACrC,KAAK,EAAE,SAAS;IAChB,cAAc,EAAE,IAAI;IACpB,eAAe,EAAE,IAAI;IACrB,gBAAgB,EAAE,IAAI;CACvB,CAAC;AAEF,mFAAmF;AACnF,MAAM,UAAU,oBAAoB,CAClC,KAAqE,EACrE,GAAG,GAAG,yBAAyB;IAE/B,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;QACnE,OAAO,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC1E,MAAM,QAAQ,GACZ,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC;QAC7C,CAAC,CAAC,KAAK,CAAC,SAAS;QACjB,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;IACvB,MAAM,IAAI,GAAG;QACX,cAAc,EAAE,QAAQ;QACxB,eAAe,EAAE,KAAK,CAAC,SAAS;QAChC,gBAAgB,EAAE,KAAK,CAAC,UAAU;KACnC,CAAC;IACF,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,CAAC;QACrC,OAAO,QAAQ,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC;YAC7C,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE;YACzD,CAAC,CAAC;gBACE,KAAK,EAAE,SAAS;gBAChB,GAAG,IAAI;gBACP,MAAM,EAAE,4DAA4D;aACrE,CAAC;IACR,IAAI,QAAQ,GAAG,GAAG;QAChB,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,IAAI;YACP,MAAM,EAAE,0BAA0B,QAAQ,4BAA4B,GAAG,EAAE;SAC5E,CAAC;IACJ,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,uBAAuB,CAAC,IAAa;IACnD,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACpE,OAAO,sDAAsD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAkC;IAElC,MAAM,GAAG,GAA2B,MAAM,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IAC7E,IAAI,CAAC,GAAG;QAAE,OAAO,OAAO,CAAC;IACzB,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI;QACjC,OAAO;YACL,GAAG,OAAO;YACV,KAAK,EAAE,YAAY;YACnB,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,QAAQ,GAAG,CAAC,MAAM,IAAI,MAAM,EAAE;SACpD,CAAC;IACJ,OAAO,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC;IAC5C,EAAE,EAAE,kBAAkB;IACtB,KAAK,EAAE,UAAU;IACjB,KAAK,EAAE,CAAC,oBAAoB,CAAC;IAC7B,KAAK,EAAE,eAAe;IACtB,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CACf,GAAG,CAAC,KAAK,CAAC,eAAe,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS;IACzE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,CAAC,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QACrD,IAAI,GAAG,IAAI,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC;YAC1C,OAAO;gBACL,QAAQ,EAAE,cAAc;gBACxB,MAAM,EAAE,oBAAoB,CAAC,CAAC,MAAM,iGAAiG;gBACrI,IAAI,EAAE,CAAC;aACR,CAAC;QACJ,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,qBAAqB,CAAC,CAAC,MAAM,EAAE;YACvC,IAAI,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;CACvE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-model-verifier",
|
|
3
|
-
"version": "3.9.
|
|
3
|
+
"version": "3.9.3",
|
|
4
4
|
"description": "Verify an AI API endpoint really serves the model it claims: a vendor adapter per wire, a maker table for who trained what, a rule table over shared evidence, and per model facts. Model substitution, Claude thinking signatures, token billing accuracy, envelope and throughput evidence.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|