ai-model-verifier 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -36,12 +36,13 @@ foreign vendor identity, Claude tier substitution (opus billed, sonnet served),
|
|
|
36
36
|
envelope-level model substitution, and (opt-in, `checkThinkingFloor`) a
|
|
37
37
|
model that cannot switch thinking off answering with no reasoning tokens, the
|
|
38
38
|
signature of a flash tier sold as Gemini 2.5 Pro, and (opt-in,
|
|
39
|
-
`checkTokenizerFingerprint`) the input-token
|
|
40
|
-
system prompt can coach
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
39
|
+
`checkTokenizerFingerprint`) the input-token delta for a fixed text, which no
|
|
40
|
+
system prompt can coach. Its job is drift: the delta is deterministic per
|
|
41
|
+
endpoint, so a lane whose delta moves between two measurements changed its
|
|
42
|
+
backend whatever the reply now calls itself. It does not name a tier on its
|
|
43
|
+
own (Opus 4.6, Sonnet 4.6 and Haiku 4.5 share a tokenizer, and relays count
|
|
44
|
+
differently from one another), so the signature table ships empty and is a
|
|
45
|
+
per-endpoint calibration the caller supplies.
|
|
45
46
|
|
|
46
47
|
## License
|
|
47
48
|
|
|
@@ -1,36 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Tokenizer fingerprint: how many input tokens the endpoint bills for a fixed
|
|
3
|
-
* run of text
|
|
3
|
+
* run of text, measured as the difference between a long and a short prompt so
|
|
4
|
+
* anything a relay injects cancels out.
|
|
4
5
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* every lane whose reply named claude-haiku-4-5 billed a delta of exactly 90
|
|
11
|
-
* (18 of 18), while lanes serving opus-4-6 and fable clustered at 84 on the
|
|
12
|
-
* same relay path. Two of the lanes at 90 echoed an opus name in the `model`
|
|
13
|
-
* field and were haiku by every other measure.
|
|
6
|
+
* What it is good for: drift. The delta is deterministic per endpoint (68
|
|
7
|
+
* marketplace lanes measured twice each agreed to the token on every repeat),
|
|
8
|
+
* so a change between two measurements of the same lane means the backend
|
|
9
|
+
* changed, whatever the reply now calls itself. Keep the prior per lane and
|
|
10
|
+
* compare with `fingerprintDrifted`.
|
|
14
11
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
12
|
+
* What it is NOT good for: naming the tier from the number alone. Opus 4.6,
|
|
13
|
+
* Sonnet 4.6 and Haiku 4.5 share one tokenizer, so they bill the same delta on
|
|
14
|
+
* the same path, and relays count differently from each other (77, 90, 134,
|
|
15
|
+
* 160, 211 and 609 were all seen for one text on lanes echoing one model name)
|
|
16
|
+
* or invent usage outright (13 tokens for eighty words, negative deltas on
|
|
17
|
+
* repeat). A signature table therefore ships EMPTY: only a caller who has
|
|
18
|
+
* calibrated a table on a specific endpoint family, per model generation and
|
|
19
|
+
* across several text lengths, should pass one, and even then only
|
|
20
|
+
* between-model differences on the same endpoint are evidence.
|
|
24
21
|
*/
|
|
25
22
|
import type { TransportFn } from "../transport";
|
|
26
23
|
/** Wire format the endpoint speaks; Claude is sold over both. */
|
|
27
24
|
export type CountWire = "anthropic" | "openai";
|
|
28
25
|
export type TierSignatures = Readonly<Record<string, readonly number[]>>;
|
|
29
|
-
/**
|
|
30
|
-
* Deltas measured on lanes whose reply named the tier, 2026-09-12. Only haiku
|
|
31
|
-
* is listed: its 18 samples agreed to the token, while opus and fable spread
|
|
32
|
-
* across relay counting paths and would condemn honest lanes.
|
|
33
|
-
*/
|
|
26
|
+
/** Empty on purpose; see the module note. Pass your own calibration. */
|
|
34
27
|
export declare const DEFAULT_TIER_SIGNATURES: TierSignatures;
|
|
35
28
|
export type TokenizerFingerprintState = "measured" | "unmeasured";
|
|
36
29
|
export type TokenizerFingerprintResult = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-fingerprint.d.ts","sourceRoot":"","sources":["../../src/detectors/tokenizer-fingerprint.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-fingerprint.d.ts","sourceRoot":"","sources":["../../src/detectors/tokenizer-fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOhD,iEAAiE;AACjE,MAAM,MAAM,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE/C,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC;AAEzE,wEAAwE;AACxE,eAAO,MAAM,uBAAuB,EAAE,cAAmB,CAAC;AAE1D,MAAM,MAAM,yBAAyB,GAAG,UAAU,GAAG,YAAY,CAAC;AAElE,MAAM,MAAM,0BAA0B,GAAG;IACvC,KAAK,EAAE,yBAAyB,CAAC;IACjC,6DAA6D;IAC7D,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,yEAAyE;IACzE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,4DAA4D;IAC5D,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAkBF,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,cAAwC,GACnD,MAAM,GAAG,IAAI,CAIf;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,0BAA0B,EAClC,KAAK,EAAE,SAAS,MAAM,EAAE,GACvB,MAAM,GAAG,IAAI,CAMf;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAChC,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAOT;AAED,wBAAsB,2BAA2B,CAAC,IAAI,EAAE;IACtD,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,cAAc,CAAC;CAC7B,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAoEtC"}
|
|
@@ -1,34 +1,27 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Tokenizer fingerprint: how many input tokens the endpoint bills for a fixed
|
|
3
|
-
* run of text
|
|
3
|
+
* run of text, measured as the difference between a long and a short prompt so
|
|
4
|
+
* anything a relay injects cancels out.
|
|
4
5
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* every lane whose reply named claude-haiku-4-5 billed a delta of exactly 90
|
|
11
|
-
* (18 of 18), while lanes serving opus-4-6 and fable clustered at 84 on the
|
|
12
|
-
* same relay path. Two of the lanes at 90 echoed an opus name in the `model`
|
|
13
|
-
* field and were haiku by every other measure.
|
|
6
|
+
* What it is good for: drift. The delta is deterministic per endpoint (68
|
|
7
|
+
* marketplace lanes measured twice each agreed to the token on every repeat),
|
|
8
|
+
* so a change between two measurements of the same lane means the backend
|
|
9
|
+
* changed, whatever the reply now calls itself. Keep the prior per lane and
|
|
10
|
+
* compare with `fingerprintDrifted`.
|
|
14
11
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
12
|
+
* What it is NOT good for: naming the tier from the number alone. Opus 4.6,
|
|
13
|
+
* Sonnet 4.6 and Haiku 4.5 share one tokenizer, so they bill the same delta on
|
|
14
|
+
* the same path, and relays count differently from each other (77, 90, 134,
|
|
15
|
+
* 160, 211 and 609 were all seen for one text on lanes echoing one model name)
|
|
16
|
+
* or invent usage outright (13 tokens for eighty words, negative deltas on
|
|
17
|
+
* repeat). A signature table therefore ships EMPTY: only a caller who has
|
|
18
|
+
* calibrated a table on a specific endpoint family, per model generation and
|
|
19
|
+
* across several text lengths, should pass one, and even then only
|
|
20
|
+
* between-model differences on the same endpoint are evidence.
|
|
24
21
|
*/
|
|
25
22
|
import { COUNT_PROBE_MAX_TOKENS, LONG_PROMPT, SHORT_PROMPT, } from "../internal/fixed-text";
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
-
* is listed: its 18 samples agreed to the token, while opus and fable spread
|
|
29
|
-
* across relay counting paths and would condemn honest lanes.
|
|
30
|
-
*/
|
|
31
|
-
export const DEFAULT_TIER_SIGNATURES = { haiku: [90] };
|
|
23
|
+
/** Empty on purpose; see the module note. Pass your own calibration. */
|
|
24
|
+
export const DEFAULT_TIER_SIGNATURES = {};
|
|
32
25
|
const intOf = (v) => typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : null;
|
|
33
26
|
function readInputTokens(data, wire) {
|
|
34
27
|
const u = data?.usage;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer-fingerprint.js","sourceRoot":"","sources":["../../src/detectors/tokenizer-fingerprint.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokenizer-fingerprint.js","sourceRoot":"","sources":["../../src/detectors/tokenizer-fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,EACL,sBAAsB,EACtB,WAAW,EACX,YAAY,GACb,MAAM,wBAAwB,CAAC;AAOhC,wEAAwE;AACxE,MAAM,CAAC,MAAM,uBAAuB,GAAmB,EAAE,CAAC;AAiB1D,MAAM,KAAK,GAAG,CAAC,CAAU,EAAiB,EAAE,CAC1C,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAEpE,SAAS,eAAe,CAAC,IAAa,EAAE,IAAe;IACrD,MAAM,CAAC,GAAI,IAAmD,EAAE,KAAK,CAAC;IACtE,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7C,OAAO,IAAI,KAAK,WAAW;QACzB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QAC1B,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,eAAe,CAAC,IAAa;IACpC,MAAM,CAAC,GAAI,IAAmC,EAAE,KAAK,CAAC;IACtD,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,UAAU,GAAmB,uBAAuB;IAEpD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;QACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CACvC,cAAsB,EACtB,MAAkC,EAClC,KAAwB;IAExB,IAAI,MAAM,CAAC,KAAK,KAAK,UAAU,IAAI,MAAM,CAAC,WAAW,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC5E,MAAM,GAAG,GAAG,cAAc,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC;AACvE,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,kBAAkB,CAChC,KAAgC,EAChC,OAAmC;IAEnC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,OAAO,CAAC,KAAK,KAAK,UAAU;QAC5B,OAAO,CAAC,KAAK,KAAK,IAAI;QACtB,OAAO,CAAC,KAAK,KAAK,KAAK,CACxB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAQjD;IACC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,GAAG,GACP,IAAI,CAAC,IAAI,KAAK,WAAW;QACvB,CAAC,CAAC,GAAG,IAAI,cAAc;QACvB,CAAC,CAAC,GAAG,IAAI,sBAAsB,CAAC;IACpC,MAAM,OAAO,GACX,IAAI,CAAC,IAAI,KAAK,WAAW;QACvB,CAAC,CAAC;YACE,cAAc,EAAE,kBAAkB;YAClC,WAAW,EAAE,IAAI,CAAC,MAAM;YACxB,mBAAmB,EAAE,YAAY;SAClC;QACH,CAAC,CAAC;YACE,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;SACvC,CAAC;IACR,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,EAAE,CAC7B,IAAI,CAAC,SAAS,CAAC;QACb,IAAI,EAAE,QAAQ;QACd,GAAG;QACH,OAAO;QACP,OAAO,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,sBAAsB;YAClC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SAC9C;QACD,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IAEL,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC;QACjB,GAAG,CAAC,WAAW,CAAC;KACjB,CAAC,CAAC;IACH,MAAM,KAAK,GAAG;QACZ,KAAK,EAAE,IAAI;QACX,gBAAgB,EAAE,IAAI;QACtB,eAAe,EAAE,IAAI;QACrB,WAAW,EAAE,IAAI;QACjB,WAAW,EAAE,IAAI;KAClB,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC;QACjC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG;YACtC,OAAO;gBACL,KAAK,EAAE,YAAY;gBACnB,GAAG,KAAK;gBACR,MAAM,EAAE,gBAAgB,CAAC,CAAC,MAAM,IAAI,SAAS,EAAE;aAChD,CAAC;IAEN,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,MAAM,WAAW,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,KAAK,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;QACjC,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,GAAG,KAAK;YACR,WAAW;YACX,MAAM,EAAE,mBAAmB;SAC5B,CAAC;IACJ,MAAM,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC;IAC3B,OAAO;QACL,KAAK,EAAE,UAAU;QACjB,KAAK;QACL,gBAAgB,EAAE,KAAK;QACvB,eAAe,EAAE,IAAI;QACrB,WAAW;QACX,WAAW,EAAE,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC;KAClD,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-model-verifier",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.1",
|
|
4
4
|
"description": "Verify an AI API endpoint really serves the model it claims: model substitution, Claude thinking signatures, token-billing accuracy, response-envelope and throughput evidence.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|