ai-model-verifier 1.4.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -21
- package/dist/engine/context.d.ts +43 -0
- package/dist/engine/context.d.ts.map +1 -0
- package/dist/engine/context.js +50 -0
- package/dist/engine/context.js.map +1 -0
- package/dist/engine/evidence.d.ts +40 -0
- package/dist/engine/evidence.d.ts.map +1 -0
- package/dist/engine/evidence.js +108 -0
- package/dist/engine/evidence.js.map +1 -0
- package/dist/engine/handshake.d.ts +16 -0
- package/dist/engine/handshake.d.ts.map +1 -0
- package/dist/{handshake.js → engine/handshake.js} +35 -63
- package/dist/engine/handshake.js.map +1 -0
- package/dist/engine/probe-runner.d.ts +14 -0
- package/dist/engine/probe-runner.d.ts.map +1 -0
- package/dist/engine/probe-runner.js +163 -0
- package/dist/engine/probe-runner.js.map +1 -0
- package/dist/engine/verdict.d.ts +15 -0
- package/dist/engine/verdict.d.ts.map +1 -0
- package/dist/engine/verdict.js +19 -0
- package/dist/engine/verdict.js.map +1 -0
- package/dist/highlight.d.ts +2 -2
- package/dist/highlight.d.ts.map +1 -1
- package/dist/highlight.js +10 -9
- package/dist/highlight.js.map +1 -1
- package/dist/{patterns.d.ts → identity/patterns.d.ts} +7 -0
- package/dist/identity/patterns.d.ts.map +1 -0
- package/dist/{patterns.js → identity/patterns.js} +7 -0
- package/dist/identity/patterns.js.map +1 -0
- package/dist/identity/signals.d.ts +15 -0
- package/dist/identity/signals.d.ts.map +1 -0
- package/dist/{signals.js → identity/signals.js} +15 -24
- package/dist/identity/signals.js.map +1 -0
- package/dist/index.d.ts +15 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -5
- package/dist/index.js.map +1 -1
- package/dist/internal/utils.d.ts +3 -1
- package/dist/internal/utils.d.ts.map +1 -1
- package/dist/internal/utils.js +12 -6
- package/dist/internal/utils.js.map +1 -1
- package/dist/models/catalog.d.ts +7 -0
- package/dist/models/catalog.d.ts.map +1 -0
- package/dist/models/catalog.js +23 -0
- package/dist/models/catalog.js.map +1 -0
- package/dist/models/facts.d.ts +56 -0
- package/dist/models/facts.d.ts.map +1 -0
- package/dist/models/facts.js +91 -0
- package/dist/models/facts.js.map +1 -0
- package/dist/models/index.d.ts +6 -0
- package/dist/models/index.d.ts.map +1 -0
- package/dist/models/index.js +5 -0
- package/dist/models/index.js.map +1 -0
- package/dist/{substitution.d.ts → models/substitution.d.ts} +3 -4
- package/dist/models/substitution.d.ts.map +1 -0
- package/dist/{substitution.js → models/substitution.js} +3 -4
- package/dist/models/substitution.js.map +1 -0
- package/dist/models/tiers.d.ts +14 -0
- package/dist/models/tiers.d.ts.map +1 -0
- package/dist/models/tiers.js +35 -0
- package/dist/models/tiers.js.map +1 -0
- package/dist/probes/prompts.d.ts +23 -0
- package/dist/probes/prompts.d.ts.map +1 -0
- package/dist/probes/prompts.js +31 -0
- package/dist/probes/prompts.js.map +1 -0
- package/dist/probes/table.d.ts +37 -0
- package/dist/probes/table.d.ts.map +1 -0
- package/dist/probes/table.js +57 -0
- package/dist/probes/table.js.map +1 -0
- package/dist/registry.d.ts +26 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/registry.js +23 -0
- package/dist/registry.js.map +1 -0
- package/dist/rules/index.d.ts +15 -0
- package/dist/rules/index.d.ts.map +1 -0
- package/dist/rules/index.js +8 -0
- package/dist/rules/index.js.map +1 -0
- package/dist/rules/probe-ladder.d.ts +33 -0
- package/dist/rules/probe-ladder.d.ts.map +1 -0
- package/dist/rules/probe-ladder.js +156 -0
- package/dist/rules/probe-ladder.js.map +1 -0
- package/dist/{detectors → rules}/response-metadata.d.ts +11 -14
- package/dist/rules/response-metadata.d.ts.map +1 -0
- package/dist/{detectors → rules}/response-metadata.js +32 -15
- package/dist/rules/response-metadata.js.map +1 -0
- package/dist/rules/table.d.ts +27 -0
- package/dist/rules/table.d.ts.map +1 -0
- package/dist/rules/table.js +47 -0
- package/dist/rules/table.js.map +1 -0
- package/dist/{detectors → rules}/thinking-floor.d.ts +14 -29
- package/dist/rules/thinking-floor.d.ts.map +1 -0
- package/dist/rules/thinking-floor.js +119 -0
- package/dist/rules/thinking-floor.js.map +1 -0
- package/dist/rules/thinking-signature.d.ts +43 -0
- package/dist/rules/thinking-signature.d.ts.map +1 -0
- package/dist/rules/thinking-signature.js +72 -0
- package/dist/rules/thinking-signature.js.map +1 -0
- package/dist/{detectors → rules}/throughput.d.ts +7 -10
- package/dist/rules/throughput.d.ts.map +1 -0
- package/dist/{detectors → rules}/throughput.js +32 -11
- package/dist/rules/throughput.js.map +1 -0
- package/dist/{detectors → rules}/token-truth.d.ts +7 -13
- package/dist/rules/token-truth.d.ts.map +1 -0
- package/dist/rules/token-truth.js +112 -0
- package/dist/rules/token-truth.js.map +1 -0
- package/dist/rules/tokenizer-fingerprint.d.ts +46 -0
- package/dist/rules/tokenizer-fingerprint.d.ts.map +1 -0
- package/dist/rules/tokenizer-fingerprint.js +110 -0
- package/dist/rules/tokenizer-fingerprint.js.map +1 -0
- package/dist/rules/types.d.ts +57 -0
- package/dist/rules/types.d.ts.map +1 -0
- package/dist/rules/types.js +2 -0
- package/dist/rules/types.js.map +1 -0
- package/dist/transport.d.ts +10 -2
- package/dist/transport.d.ts.map +1 -1
- package/dist/transport.js +40 -32
- package/dist/transport.js.map +1 -1
- package/dist/types.d.ts +70 -31
- package/dist/types.d.ts.map +1 -1
- package/dist/vendors/anthropic.d.ts +44 -0
- package/dist/vendors/anthropic.d.ts.map +1 -0
- package/dist/vendors/anthropic.js +114 -0
- package/dist/vendors/anthropic.js.map +1 -0
- package/dist/vendors/base-url.d.ts +3 -0
- package/dist/vendors/base-url.d.ts.map +1 -0
- package/dist/vendors/base-url.js +6 -0
- package/dist/vendors/base-url.js.map +1 -0
- package/dist/vendors/gemini.d.ts +44 -0
- package/dist/vendors/gemini.d.ts.map +1 -0
- package/dist/vendors/gemini.js +77 -0
- package/dist/vendors/gemini.js.map +1 -0
- package/dist/vendors/index.d.ts +9 -0
- package/dist/vendors/index.d.ts.map +1 -0
- package/dist/vendors/index.js +7 -0
- package/dist/vendors/index.js.map +1 -0
- package/dist/vendors/openai.d.ts +46 -0
- package/dist/vendors/openai.d.ts.map +1 -0
- package/dist/vendors/openai.js +77 -0
- package/dist/vendors/openai.js.map +1 -0
- package/dist/vendors/table.d.ts +113 -0
- package/dist/vendors/table.d.ts.map +1 -0
- package/dist/vendors/table.js +14 -0
- package/dist/vendors/table.js.map +1 -0
- package/dist/vendors/types.d.ts +93 -0
- package/dist/vendors/types.d.ts.map +1 -0
- package/dist/vendors/types.js +2 -0
- package/dist/vendors/types.js.map +1 -0
- package/dist/verify.d.ts +30 -0
- package/dist/verify.d.ts.map +1 -0
- package/dist/verify.js +185 -0
- package/dist/verify.js.map +1 -0
- package/package.json +25 -10
- package/dist/detectors/response-metadata.d.ts.map +0 -1
- package/dist/detectors/response-metadata.js.map +0 -1
- package/dist/detectors/thinking-floor.d.ts.map +0 -1
- package/dist/detectors/thinking-floor.js +0 -135
- package/dist/detectors/thinking-floor.js.map +0 -1
- package/dist/detectors/thinking-signature.d.ts +0 -70
- package/dist/detectors/thinking-signature.d.ts.map +0 -1
- package/dist/detectors/thinking-signature.js +0 -177
- package/dist/detectors/thinking-signature.js.map +0 -1
- package/dist/detectors/throughput.d.ts.map +0 -1
- package/dist/detectors/throughput.js.map +0 -1
- package/dist/detectors/token-truth.d.ts.map +0 -1
- package/dist/detectors/token-truth.js +0 -142
- package/dist/detectors/token-truth.js.map +0 -1
- package/dist/detectors/tokenizer-fingerprint.d.ts +0 -65
- package/dist/detectors/tokenizer-fingerprint.d.ts.map +0 -1
- package/dist/detectors/tokenizer-fingerprint.js +0 -135
- package/dist/detectors/tokenizer-fingerprint.js.map +0 -1
- package/dist/handshake.d.ts +0 -23
- package/dist/handshake.d.ts.map +0 -1
- package/dist/handshake.js.map +0 -1
- package/dist/internal/fixed-text.d.ts +0 -13
- package/dist/internal/fixed-text.d.ts.map +0 -1
- package/dist/internal/fixed-text.js +0 -13
- package/dist/internal/fixed-text.js.map +0 -1
- package/dist/models.d.ts +0 -5
- package/dist/models.d.ts.map +0 -1
- package/dist/models.js +0 -54
- package/dist/models.js.map +0 -1
- package/dist/nonce.d.ts +0 -4
- package/dist/nonce.d.ts.map +0 -1
- package/dist/nonce.js +0 -12
- package/dist/nonce.js.map +0 -1
- package/dist/patterns.d.ts.map +0 -1
- package/dist/patterns.js.map +0 -1
- package/dist/probes.d.ts +0 -10
- package/dist/probes.d.ts.map +0 -1
- package/dist/probes.js +0 -54
- package/dist/probes.js.map +0 -1
- package/dist/providers/anthropic.d.ts +0 -3
- package/dist/providers/anthropic.d.ts.map +0 -1
- package/dist/providers/anthropic.js +0 -67
- package/dist/providers/anthropic.js.map +0 -1
- package/dist/providers/config.d.ts +0 -34
- package/dist/providers/config.d.ts.map +0 -1
- package/dist/providers/config.js +0 -14
- package/dist/providers/config.js.map +0 -1
- package/dist/providers/gemini.d.ts +0 -3
- package/dist/providers/gemini.d.ts.map +0 -1
- package/dist/providers/gemini.js +0 -58
- package/dist/providers/gemini.js.map +0 -1
- package/dist/providers/openai.d.ts +0 -3
- package/dist/providers/openai.d.ts.map +0 -1
- package/dist/providers/openai.js +0 -56
- package/dist/providers/openai.js.map +0 -1
- package/dist/rules.d.ts +0 -6
- package/dist/rules.d.ts.map +0 -1
- package/dist/rules.js +0 -28
- package/dist/rules.js.map +0 -1
- package/dist/runner.d.ts +0 -39
- package/dist/runner.d.ts.map +0 -1
- package/dist/runner.js +0 -352
- package/dist/runner.js.map +0 -1
- package/dist/signals.d.ts +0 -11
- package/dist/signals.d.ts.map +0 -1
- package/dist/signals.js.map +0 -1
- package/dist/substitution.d.ts.map +0 -1
- package/dist/substitution.js.map +0 -1
- package/dist/verdict.d.ts +0 -26
- package/dist/verdict.d.ts.map +0 -1
- package/dist/verdict.js +0 -84
- package/dist/verdict.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-model-verifier",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "Verify an AI API endpoint really serves the model it claims: model substitution, Claude thinking signatures, token
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "Verify an AI API endpoint really serves the model it claims: a vendor adapter per wire, a rule table over shared evidence, and per model facts. Model substitution, Claude thinking signatures, token billing accuracy, envelope and throughput evidence.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"repository": {
|
|
@@ -21,7 +21,9 @@
|
|
|
21
21
|
"claude",
|
|
22
22
|
"openai",
|
|
23
23
|
"gemini",
|
|
24
|
-
"token-billing"
|
|
24
|
+
"token-billing",
|
|
25
|
+
"rule-engine",
|
|
26
|
+
"adapter"
|
|
25
27
|
],
|
|
26
28
|
"sideEffects": false,
|
|
27
29
|
"files": [
|
|
@@ -32,19 +34,32 @@
|
|
|
32
34
|
"types": "./dist/index.d.ts",
|
|
33
35
|
"default": "./dist/index.js"
|
|
34
36
|
},
|
|
35
|
-
"./
|
|
36
|
-
"types": "./dist/
|
|
37
|
-
"default": "./dist/
|
|
37
|
+
"./vendors": {
|
|
38
|
+
"types": "./dist/vendors/index.d.ts",
|
|
39
|
+
"default": "./dist/vendors/index.js"
|
|
38
40
|
},
|
|
39
|
-
"
|
|
40
|
-
"types": "./dist
|
|
41
|
-
"default": "./dist
|
|
41
|
+
"./rules": {
|
|
42
|
+
"types": "./dist/rules/index.d.ts",
|
|
43
|
+
"default": "./dist/rules/index.js"
|
|
44
|
+
},
|
|
45
|
+
"./models": {
|
|
46
|
+
"types": "./dist/models/index.d.ts",
|
|
47
|
+
"default": "./dist/models/index.js"
|
|
48
|
+
},
|
|
49
|
+
"./transport": {
|
|
50
|
+
"types": "./dist/transport.d.ts",
|
|
51
|
+
"default": "./dist/transport.js"
|
|
52
|
+
},
|
|
53
|
+
"./highlight": {
|
|
54
|
+
"types": "./dist/highlight.d.ts",
|
|
55
|
+
"default": "./dist/highlight.js"
|
|
42
56
|
}
|
|
43
57
|
},
|
|
44
58
|
"scripts": {
|
|
45
59
|
"build": "tsc -p tsconfig.build.json",
|
|
46
60
|
"typecheck": "tsc --noEmit",
|
|
47
|
-
"
|
|
61
|
+
"test": "bun test",
|
|
62
|
+
"prepublishOnly": "bun run typecheck && bun run build"
|
|
48
63
|
},
|
|
49
64
|
"devDependencies": {
|
|
50
65
|
"typescript": "^7.0.2"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"response-metadata.d.ts","sourceRoot":"","sources":["../../src/detectors/response-metadata.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,oEAAoE;AACpE,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1E,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qEAAqE;IACrE,KAAK,EAAE,aAAa,CAAC;IACrB,6CAA6C;IAC7C,QAAQ,EAAE,aAAa,CAAC;IACxB,iEAAiE;IACjE,UAAU,EAAE,OAAO,CAAC;IACpB,4EAA4E;IAC5E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,yEAAyE;IACzE,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,8EAA8E;IAC9E,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,kEAAkE;IAClE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAuCF,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,aAAa,GACtB,gBAAgB,CAyClB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"response-metadata.js","sourceRoot":"","sources":["../../src/detectors/response-metadata.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAsBH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,cAAc;IACd,eAAe;IACf,yBAAyB;IACzB,6BAA6B;CAC9B,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,eAAe;IACf,mBAAmB;IACnB,cAAc;CACf,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,kBAAkB;IAClB,sBAAsB;IACtB,iBAAiB;IACjB,oBAAoB;CACrB,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,EAAW;IAC7B,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3D,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC;QAC/D,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACjC,sEAAsE;IACtE,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACnD,CAAC;AAED,SAAS,OAAO,CAAC,IAA6B,EAAE,SAAmB;IACjE,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChF,OAAO,QAAiB,CAAC;IAC3B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,OAAO,QAAiB,CAAC;IAC3B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChF,OAAO,WAAoB,CAAC;IAC9B,OAAO,SAAkB,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAa,EACb,QAAuB;IAEvB,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC9D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,wEAAwE;IACxE,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,KAAK,WAAW,CAAC;QAC1D,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACvD,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,WAAW,CAAC;QACzD,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACnD,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,QAAQ,CAAC;IAC7D,IAAI,UAAU;QACZ,KAAK,CAAC,IAAI,CACR,eAAe,KAAK,iBAAiB,QAAQ,iDAAiD,CAC/F,CAAC;IACJ,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,IAAI,QAAQ,KAAK,MAAM;QAC7D,KAAK,CAAC,IAAI,CACR,cAAc,QAAQ,oEAAoE,CAC3F,CAAC;IACJ,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR,iBAAiB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC,CACrF,CAAC;IAEJ,OAAO;QACL,KAAK;QACL,QAAQ;QACR,UAAU;QACV,QAAQ;QACR,SAAS;QACT,gBAAgB;QAChB,KAAK;KACN,CAAC;AACJ,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"thinking-floor.d.ts","sourceRoot":"","sources":["../../src/detectors/thinking-floor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;OAGG;IACH,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3B,0FAA0F;IAC1F,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,SAAS,MAAM,EAAuB,CAAC;AAC3E,eAAO,MAAM,6BAA6B,KAAK,CAAC;AAMhD,MAAM,MAAM,kBAAkB;AAC5B,6EAA6E;AAC3E,SAAS;AACX,2EAA2E;GACzE,aAAa;AACf,+CAA+C;GAC7C,YAAY;AACd,+CAA+C;GAC7C,SAAS,CAAC;AAEd,MAAM,MAAM,mBAAmB,GAAG;IAChC,KAAK,EAAE,kBAAkB,CAAC;IAC1B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAKF,wBAAgB,eAAe,CAC7B,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAKT;AAYD,0FAA0F;AAC1F,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,OAAO,GACZ;IAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAWhE;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,OAAO,EACb,OAAO,CAAC,EAAE,oBAAoB,GAC7B,mBAAmB,CA8CrB;AAED,8FAA8F;AAC9F,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CA0C/B"}
|
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Thinking-floor check for models that cannot switch reasoning off.
|
|
3
|
-
*
|
|
4
|
-
* Gemini 2.5 Pro has a thinking budget floor (128 tokens): even "Reply with
|
|
5
|
-
* only the word ok." produces hidden thought tokens, reported as
|
|
6
|
-
* `completion_tokens_details.reasoning_tokens` on an OpenAI-shaped hop, folded
|
|
7
|
-
* into `completion_tokens` when the relay strips the breakdown, or as
|
|
8
|
-
* `usageMetadata.thoughtsTokenCount` on the native endpoint. Measured across 66
|
|
9
|
-
* historical probes of genuine 2.5 Pro: the lowest completion count for that
|
|
10
|
-
* prompt was 12, the native thoughts count 99 to 158.
|
|
11
|
-
*
|
|
12
|
-
* A flash-class backend sold under the pro name answers the same prompt with
|
|
13
|
-
* `completion_tokens: 1` (a6 merchant 3623, 2026-09-09), and it still echoes
|
|
14
|
-
* `model: gemini-2.5-pro`, so the envelope substitution check never fires.
|
|
15
|
-
* The token count is the only free evidence left.
|
|
16
|
-
*
|
|
17
|
-
* Not applied to flash tiers: those may legitimately run with thinking off.
|
|
18
|
-
*/
|
|
19
|
-
export const DEFAULT_ALWAYS_THINKS = ["gemini-2-5-pro"];
|
|
20
|
-
export const DEFAULT_MIN_COMPLETION_TOKENS = 10;
|
|
21
|
-
const PROBE_PROMPT = "Reply with only the word ok.";
|
|
22
|
-
/** Room for the thoughts plus the word; too small truncates thinking and reads as a fake. */
|
|
23
|
-
const MAX_TOKENS = 2048;
|
|
24
|
-
const normalize = (model) => model.toLowerCase().replace(/[._]/g, "-").split("/").pop() ?? "";
|
|
25
|
-
export function mustAlwaysThink(model, options) {
|
|
26
|
-
const m = normalize(model);
|
|
27
|
-
return (options?.models ?? DEFAULT_ALWAYS_THINKS).some((p) => m.startsWith(normalize(p)));
|
|
28
|
-
}
|
|
29
|
-
const num = (v) => typeof v === "number" && Number.isFinite(v) ? v : null;
|
|
30
|
-
/** Completion and reasoning counts from an OpenAI `usage` or a Gemini `usageMetadata`. */
|
|
31
|
-
export function readThinkingUsage(data) {
|
|
32
|
-
if (!data || typeof data !== "object")
|
|
33
|
-
return null;
|
|
34
|
-
const d = data;
|
|
35
|
-
const u = (d.usage ?? d.usageMetadata);
|
|
36
|
-
if (!u || typeof u !== "object")
|
|
37
|
-
return null;
|
|
38
|
-
const details = u.completion_tokens_details;
|
|
39
|
-
const reasoning = num(details && typeof details === "object" ? details.reasoning_tokens : null) ??
|
|
40
|
-
num(u.thoughtsTokenCount);
|
|
41
|
-
const completion = num(u.completion_tokens) ?? num(u.candidatesTokenCount);
|
|
42
|
-
return { completion, reasoning };
|
|
43
|
-
}
|
|
44
|
-
/** Judge one reply to the short probe. Pure: pass any response body already collected. */
|
|
45
|
-
export function checkThinkingFloor(model, data, options) {
|
|
46
|
-
if (!mustAlwaysThink(model, options))
|
|
47
|
-
return { state: "skipped", completionTokens: null, reasoningTokens: null };
|
|
48
|
-
const usage = readThinkingUsage(data);
|
|
49
|
-
if (!usage)
|
|
50
|
-
return {
|
|
51
|
-
state: "unmeasured",
|
|
52
|
-
completionTokens: null,
|
|
53
|
-
reasoningTokens: null,
|
|
54
|
-
reason: "no usage in reply",
|
|
55
|
-
};
|
|
56
|
-
const floor = options?.minCompletionTokens ?? DEFAULT_MIN_COMPLETION_TOKENS;
|
|
57
|
-
if (usage.reasoning !== null && usage.reasoning > 0)
|
|
58
|
-
return {
|
|
59
|
-
state: "thought",
|
|
60
|
-
completionTokens: usage.completion,
|
|
61
|
-
reasoningTokens: usage.reasoning,
|
|
62
|
-
};
|
|
63
|
-
if (usage.completion === null)
|
|
64
|
-
return {
|
|
65
|
-
state: "unmeasured",
|
|
66
|
-
completionTokens: null,
|
|
67
|
-
reasoningTokens: usage.reasoning,
|
|
68
|
-
reason: "no completion count in usage",
|
|
69
|
-
};
|
|
70
|
-
// An empty reply carries no evidence: nothing was generated, so nothing was
|
|
71
|
-
// skipped. Truncation and refusals land here, not in "no-thinking".
|
|
72
|
-
if (usage.completion === 0)
|
|
73
|
-
return {
|
|
74
|
-
state: "unmeasured",
|
|
75
|
-
completionTokens: 0,
|
|
76
|
-
reasoningTokens: usage.reasoning,
|
|
77
|
-
reason: "empty reply",
|
|
78
|
-
};
|
|
79
|
-
if (usage.completion < floor)
|
|
80
|
-
return {
|
|
81
|
-
state: "no-thinking",
|
|
82
|
-
completionTokens: usage.completion,
|
|
83
|
-
reasoningTokens: usage.reasoning,
|
|
84
|
-
reason: `completion_tokens ${usage.completion} below floor ${floor} with no reasoning tokens`,
|
|
85
|
-
};
|
|
86
|
-
return {
|
|
87
|
-
state: "thought",
|
|
88
|
-
completionTokens: usage.completion,
|
|
89
|
-
reasoningTokens: usage.reasoning,
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
/** One extra generation: the short prompt through the endpoint, then `checkThinkingFloor`. */
|
|
93
|
-
export async function probeThinkingFloor(opts) {
|
|
94
|
-
if (!mustAlwaysThink(opts.model, opts.options))
|
|
95
|
-
return { state: "skipped", completionTokens: null, reasoningTokens: null };
|
|
96
|
-
const base = opts.baseUrl.replace(/\/+$/, "");
|
|
97
|
-
const req = opts.provider === "gemini"
|
|
98
|
-
? {
|
|
99
|
-
url: `${base}/v1beta/models/${opts.model}:generateContent`,
|
|
100
|
-
headers: {
|
|
101
|
-
"content-type": "application/json",
|
|
102
|
-
"x-goog-api-key": opts.apiKey,
|
|
103
|
-
},
|
|
104
|
-
reqBody: {
|
|
105
|
-
contents: [{ role: "user", parts: [{ text: PROBE_PROMPT }] }],
|
|
106
|
-
generationConfig: { maxOutputTokens: MAX_TOKENS },
|
|
107
|
-
},
|
|
108
|
-
}
|
|
109
|
-
: {
|
|
110
|
-
url: `${base}/v1/chat/completions`,
|
|
111
|
-
headers: {
|
|
112
|
-
"content-type": "application/json",
|
|
113
|
-
authorization: `Bearer ${opts.apiKey}`,
|
|
114
|
-
},
|
|
115
|
-
reqBody: {
|
|
116
|
-
model: opts.model,
|
|
117
|
-
max_tokens: MAX_TOKENS,
|
|
118
|
-
messages: [{ role: "user", content: PROBE_PROMPT }],
|
|
119
|
-
},
|
|
120
|
-
};
|
|
121
|
-
const res = await opts.transport({
|
|
122
|
-
mode: "direct",
|
|
123
|
-
...req,
|
|
124
|
-
timeoutMs: opts.timeoutMs,
|
|
125
|
-
});
|
|
126
|
-
if (res.status !== 200 || !res.data)
|
|
127
|
-
return {
|
|
128
|
-
state: "unmeasured",
|
|
129
|
-
completionTokens: null,
|
|
130
|
-
reasoningTokens: null,
|
|
131
|
-
reason: res.error ?? `http ${res.status ?? "none"}`,
|
|
132
|
-
};
|
|
133
|
-
return checkThinkingFloor(opts.model, res.data, opts.options);
|
|
134
|
-
}
|
|
135
|
-
//# sourceMappingURL=thinking-floor.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"thinking-floor.js","sourceRoot":"","sources":["../../src/detectors/thinking-floor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAcH,MAAM,CAAC,MAAM,qBAAqB,GAAsB,CAAC,gBAAgB,CAAC,CAAC;AAC3E,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEhD,MAAM,YAAY,GAAG,8BAA8B,CAAC;AACpD,6FAA6F;AAC7F,MAAM,UAAU,GAAG,IAAI,CAAC;AAmBxB,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAClC,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;AAEnE,MAAM,UAAU,eAAe,CAC7B,KAAa,EACb,OAA8B;IAE9B,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3B,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,qBAAqB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3D,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAC3B,CAAC;AACJ,CAAC;AASD,MAAM,GAAG,GAAG,CAAC,CAAU,EAAiB,EAAE,CACxC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAEzD,0FAA0F;AAC1F,MAAM,UAAU,iBAAiB,CAC/B,IAAa;IAEb,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,CAAC,GAAG,IAAoD,CAAC;IAC/D,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,aAAa,CAA2B,CAAC;IACjE,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7C,MAAM,OAAO,GAAG,CAAC,CAAC,yBAAyB,CAAC;IAC5C,MAAM,SAAS,GACb,GAAG,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7E,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAC5B,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;IAC3E,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;AACnC,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,kBAAkB,CAChC,KAAa,EACb,IAAa,EACb,OAA8B;IAE9B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC;QAClC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC7E,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,CAAC,KAAK;QACR,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,mBAAmB;SAC5B,CAAC;IACJ,MAAM,KAAK,GAAG,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;IAC5E,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC;QACjD,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,gBAAgB,EAAE,KAAK,CAAC,UAAU;YAClC,eAAe,EAAE,KAAK,CAAC,SAAS;SACjC,CAAC;IACJ,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI;QAC3B,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,KAAK,CAAC,SAAS;YAChC,MAAM,EAAE,8BAA8B;SACvC,CAAC;IACJ,4EAA4E;IAC5E,oEAAoE;IACpE,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC;QACxB,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,gBAAgB,EAAE,CAAC;YACnB,eAAe,EAAE,KAAK,CAAC,SAAS;YAChC,MAAM,EAAE,aAAa;SACtB,CAAC;IACJ,IAAI,KAAK,CAAC,UAAU,GAAG,KAAK;QAC1B,OAAO;YACL,KAAK,EAAE,aAAa;YACpB,gBAAgB,EAAE,KAAK,CAAC,UAAU;YAClC,eAAe,EAAE,KAAK,CAAC,SAAS;YAChC,MAAM,EAAE,qBAAqB,KAAK,CAAC,UAAU,gBAAgB,KAAK,2BAA2B;SAC9F,CAAC;IACJ,OAAO;QACL,KAAK,EAAE,SAAS;QAChB,gBAAgB,EAAE,KAAK,CAAC,UAAU;QAClC,eAAe,EAAE,KAAK,CAAC,SAAS;KACjC,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IASxC;IACC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC;QAC5C,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,GAAG,GACP,IAAI,CAAC,QAAQ,KAAK,QAAQ;QACxB,CAAC,CAAC;YACE,GAAG,EAAE,GAAG,IAAI,kBAAkB,IAAI,CAAC,KAAK,kBAAkB;YAC1D,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,gBAAgB,EAAE,IAAI,CAAC,MAAM;aAC9B;YACD,OAAO,EAAE;gBACP,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;gBAC7D,gBAAgB,EAAE,EAAE,eAAe,EAAE,UAAU,EAAE;aAClD;SACF;QACH,CAAC,CAAC;YACE,GAAG,EAAE,GAAG,IAAI,sBAAsB;YAClC,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,OAAO,EAAE;gBACP,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;aACpD;SACF,CAAC;IACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,GAAG;QACN,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI;QACjC,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,QAAQ,GAAG,CAAC,MAAM,IAAI,MAAM,EAAE;SACpD,CAAC;IACJ,OAAO,kBAAkB,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Anthropic thinking-signature check.
|
|
3
|
-
*
|
|
4
|
-
* Claude attaches a server-generated `signature` to every thinking block. A
|
|
5
|
-
* relay serving some other model cannot produce one, so this is the only
|
|
6
|
-
* evidence here that does not rest on asking the model who it is: identity and
|
|
7
|
-
* behaviour probes can be coached with a system prompt, a signature cannot.
|
|
8
|
-
*
|
|
9
|
-
* What it does NOT prove: which tier answered. Sonnet returns a perfectly valid
|
|
10
|
-
* signature when sold as Opus, so a pass here still needs the tier checks.
|
|
11
|
-
*
|
|
12
|
-
* Ported from veridrop (AGPL-3.0), whose detector stops at "a signature-shaped
|
|
13
|
-
* string is present".
|
|
14
|
-
*
|
|
15
|
-
* Their DESIGN.md also specifies a replay step (hand the thinking block back so
|
|
16
|
-
* Anthropic re-validates the signature, rejecting a forgery with 400/422) which
|
|
17
|
-
* they never implemented. We built and measured it: **it does not work through a
|
|
18
|
-
* relay**. Tested against two live upstreams (a6 merchant 3542 and pol) by
|
|
19
|
-
* replaying a genuine block and then the same block with 308 bytes of random
|
|
20
|
-
* base64 in place of the signature. Both upstreams returned 200 for both. A
|
|
21
|
-
* relay re-issues the turn to its own backend rather than passing our thinking
|
|
22
|
-
* block through, so nothing ever validates it.
|
|
23
|
-
*
|
|
24
|
-
* `strict` therefore stays off by default and is documented as proving only
|
|
25
|
-
* that the endpoint accepts the block, not that the signature is real. Do not
|
|
26
|
-
* present it as cryptographic proof.
|
|
27
|
-
*/
|
|
28
|
-
import type { TransportFn } from "../transport";
|
|
29
|
-
export type SignatureState =
|
|
30
|
-
/** Signed thinking block: a genuine Anthropic path answered. */
|
|
31
|
-
"signed"
|
|
32
|
-
/**
|
|
33
|
-
* Thought, but no signature. Expected when the lane crosses an
|
|
34
|
-
* OpenAI-shaped hop, which has no field to carry one, so this is
|
|
35
|
-
* unprovable rather than fake. Never a hard fail on its own.
|
|
36
|
-
*/
|
|
37
|
-
| "unsigned"
|
|
38
|
-
/** Thinking was requested and no block came back at all. */
|
|
39
|
-
| "no-thinking"
|
|
40
|
-
/** Model or endpoint cannot be asked; carries no evidence either way. */
|
|
41
|
-
| "skipped";
|
|
42
|
-
export type SignatureResult = {
|
|
43
|
-
state: SignatureState;
|
|
44
|
-
signatureLength: number;
|
|
45
|
-
/** First 32 chars, for the report; never the whole signature. */
|
|
46
|
-
signaturePrefix: string | null;
|
|
47
|
-
thinkingChars: number;
|
|
48
|
-
/** Only set when the replay check actually ran. */
|
|
49
|
-
replayVerified?: boolean;
|
|
50
|
-
reason?: string;
|
|
51
|
-
};
|
|
52
|
-
export declare function thinkingModeFor(model: string): {
|
|
53
|
-
kind: "adaptive" | "extended";
|
|
54
|
-
} | null;
|
|
55
|
-
export declare function checkThinkingSignature(opts: {
|
|
56
|
-
transport: TransportFn;
|
|
57
|
-
baseUrl: string;
|
|
58
|
-
apiKey: string;
|
|
59
|
-
model: string;
|
|
60
|
-
timeoutMs: number;
|
|
61
|
-
/**
|
|
62
|
-
* Replay the thinking block to the same endpoint. Measured as NOT proving
|
|
63
|
-
* authenticity through a relay (see the module docstring): both a genuine and
|
|
64
|
-
* a forged signature come back 200, because the relay re-issues the turn
|
|
65
|
-
* upstream instead of forwarding our block. Off by default; it only tells you
|
|
66
|
-
* the endpoint accepts a thinking block in the request.
|
|
67
|
-
*/
|
|
68
|
-
strict?: boolean;
|
|
69
|
-
}): Promise<SignatureResult>;
|
|
70
|
-
//# sourceMappingURL=thinking-signature.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"thinking-signature.d.ts","sourceRoot":"","sources":["../../src/detectors/thinking-signature.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAqBhD,MAAM,MAAM,cAAc;AACxB,gEAAgE;AAC9D,QAAQ;AACV;;;;GAIG;GACD,UAAU;AACZ,4DAA4D;GAC1D,aAAa;AACf,yEAAyE;GACvE,SAAS,CAAC;AAEd,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,cAAc,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAkBF,wBAAgB,eAAe,CAC7B,KAAK,EAAE,MAAM,GACZ;IAAE,IAAI,EAAE,UAAU,GAAG,UAAU,CAAA;CAAE,GAAG,IAAI,CAK1C;AA4BD,wBAAsB,sBAAsB,CAAC,IAAI,EAAE;IACjD,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,GAAG,OAAO,CAAC,eAAe,CAAC,CA0E3B"}
|
|
@@ -1,177 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Anthropic thinking-signature check.
|
|
3
|
-
*
|
|
4
|
-
* Claude attaches a server-generated `signature` to every thinking block. A
|
|
5
|
-
* relay serving some other model cannot produce one, so this is the only
|
|
6
|
-
* evidence here that does not rest on asking the model who it is: identity and
|
|
7
|
-
* behaviour probes can be coached with a system prompt, a signature cannot.
|
|
8
|
-
*
|
|
9
|
-
* What it does NOT prove: which tier answered. Sonnet returns a perfectly valid
|
|
10
|
-
* signature when sold as Opus, so a pass here still needs the tier checks.
|
|
11
|
-
*
|
|
12
|
-
* Ported from veridrop (AGPL-3.0), whose detector stops at "a signature-shaped
|
|
13
|
-
* string is present".
|
|
14
|
-
*
|
|
15
|
-
* Their DESIGN.md also specifies a replay step (hand the thinking block back so
|
|
16
|
-
* Anthropic re-validates the signature, rejecting a forgery with 400/422) which
|
|
17
|
-
* they never implemented. We built and measured it: **it does not work through a
|
|
18
|
-
* relay**. Tested against two live upstreams (a6 merchant 3542 and pol) by
|
|
19
|
-
* replaying a genuine block and then the same block with 308 bytes of random
|
|
20
|
-
* base64 in place of the signature. Both upstreams returned 200 for both. A
|
|
21
|
-
* relay re-issues the turn to its own backend rather than passing our thinking
|
|
22
|
-
* block through, so nothing ever validates it.
|
|
23
|
-
*
|
|
24
|
-
* `strict` therefore stays off by default and is documented as proving only
|
|
25
|
-
* that the endpoint accepts the block, not that the signature is real. Do not
|
|
26
|
-
* present it as cryptographic proof.
|
|
27
|
-
*/
|
|
28
|
-
/**
|
|
29
|
-
* Multi-step GCD: hard enough that adaptive models decide to think, and
|
|
30
|
-
* deliberately not the 1071/462 pair from Anthropic's own docs, which a relay
|
|
31
|
-
* could special-case.
|
|
32
|
-
*/
|
|
33
|
-
const PROBE_PROMPT = "Find the greatest common divisor of 2378 and 1547 using the Euclidean algorithm.";
|
|
34
|
-
const THINKING_BUDGET_TOKENS = 2000;
|
|
35
|
-
/**
|
|
36
|
-
* Covers thinking + answer together. Too small and an adaptive model skips
|
|
37
|
-
* thinking entirely to fit, which reads as a fake when it is our own fault.
|
|
38
|
-
*/
|
|
39
|
-
const MAX_TOKENS = 16000;
|
|
40
|
-
/** Real signatures run to hundreds of chars; this only rejects a stub. */
|
|
41
|
-
const SIGNATURE_MIN_LEN = 50;
|
|
42
|
-
/** Adaptive-thinking models take `effort` at the top level, not inside `thinking`. */
|
|
43
|
-
const ADAPTIVE_MODELS = ["claude-opus-4-7", "claude-opus-4-8"];
|
|
44
|
-
/** Extended-thinking models take a token budget instead. */
|
|
45
|
-
const EXTENDED_MODELS = [
|
|
46
|
-
"claude-opus-4-6",
|
|
47
|
-
"claude-opus-4-5",
|
|
48
|
-
"claude-opus-4-1",
|
|
49
|
-
"claude-sonnet-4-6",
|
|
50
|
-
"claude-sonnet-4-5",
|
|
51
|
-
"claude-haiku-4-5",
|
|
52
|
-
];
|
|
53
|
-
const normalize = (model) => model.toLowerCase().replace(/[._]/g, "-").split("/").pop() ?? "";
|
|
54
|
-
export function thinkingModeFor(model) {
|
|
55
|
-
const m = normalize(model);
|
|
56
|
-
if (ADAPTIVE_MODELS.some((p) => m.startsWith(p)))
|
|
57
|
-
return { kind: "adaptive" };
|
|
58
|
-
if (EXTENDED_MODELS.some((p) => m.startsWith(p)))
|
|
59
|
-
return { kind: "extended" };
|
|
60
|
-
return null;
|
|
61
|
-
}
|
|
62
|
-
function readThinking(data) {
|
|
63
|
-
const content = data?.content;
|
|
64
|
-
if (!Array.isArray(content))
|
|
65
|
-
return { block: null, signature: "", chars: 0 };
|
|
66
|
-
for (const raw of content) {
|
|
67
|
-
if (!raw || typeof raw !== "object")
|
|
68
|
-
continue;
|
|
69
|
-
const b = raw;
|
|
70
|
-
if (b.type !== "thinking" && b.type !== "redacted_thinking")
|
|
71
|
-
continue;
|
|
72
|
-
return {
|
|
73
|
-
block: b,
|
|
74
|
-
signature: typeof b.signature === "string" ? b.signature : "",
|
|
75
|
-
chars: typeof b.thinking === "string" ? b.thinking.length : 0,
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
return { block: null, signature: "", chars: 0 };
|
|
79
|
-
}
|
|
80
|
-
export async function checkThinkingSignature(opts) {
|
|
81
|
-
const mode = thinkingModeFor(opts.model);
|
|
82
|
-
const empty = { signatureLength: 0, signaturePrefix: null, thinkingChars: 0 };
|
|
83
|
-
if (!mode)
|
|
84
|
-
return { state: "skipped", ...empty, reason: "model-has-no-thinking-mode" };
|
|
85
|
-
const url = `${opts.baseUrl.replace(/\/+$/, "")}/v1/messages`;
|
|
86
|
-
const headers = {
|
|
87
|
-
"Content-Type": "application/json",
|
|
88
|
-
"x-api-key": opts.apiKey,
|
|
89
|
-
"anthropic-version": "2023-06-01",
|
|
90
|
-
};
|
|
91
|
-
const messages = [{ role: "user", content: PROBE_PROMPT }];
|
|
92
|
-
// Non-streaming on purpose: streaming an adaptive model silently drops the
|
|
93
|
-
// thinking block, so a real Claude would look unsigned.
|
|
94
|
-
const body = {
|
|
95
|
-
model: opts.model,
|
|
96
|
-
max_tokens: MAX_TOKENS,
|
|
97
|
-
messages,
|
|
98
|
-
thinking: mode.kind === "extended"
|
|
99
|
-
? { type: "enabled", budget_tokens: THINKING_BUDGET_TOKENS }
|
|
100
|
-
: { type: "adaptive", display: "summarized" },
|
|
101
|
-
};
|
|
102
|
-
// Sibling of `thinking`, never nested inside it: nesting is a 400.
|
|
103
|
-
if (mode.kind === "adaptive")
|
|
104
|
-
body["output_config"] = { effort: "xhigh" };
|
|
105
|
-
const res = await opts.transport({
|
|
106
|
-
mode: "direct",
|
|
107
|
-
url,
|
|
108
|
-
headers,
|
|
109
|
-
reqBody: body,
|
|
110
|
-
timeoutMs: opts.timeoutMs,
|
|
111
|
-
});
|
|
112
|
-
if (res.status === null || res.status >= 400)
|
|
113
|
-
return {
|
|
114
|
-
state: "skipped",
|
|
115
|
-
...empty,
|
|
116
|
-
reason: `probe-failed:${res.status ?? "network"}`,
|
|
117
|
-
};
|
|
118
|
-
const found = readThinking(res.data);
|
|
119
|
-
if (!found.block)
|
|
120
|
-
return { state: "no-thinking", ...empty, reason: "no-thinking-block" };
|
|
121
|
-
const sig = found.signature;
|
|
122
|
-
const base = {
|
|
123
|
-
signatureLength: sig.length,
|
|
124
|
-
signaturePrefix: sig ? sig.slice(0, 32) : null,
|
|
125
|
-
thinkingChars: found.chars,
|
|
126
|
-
};
|
|
127
|
-
if (!sig)
|
|
128
|
-
return { state: "unsigned", ...base, reason: "no-signature-field" };
|
|
129
|
-
if (sig.length < SIGNATURE_MIN_LEN)
|
|
130
|
-
return { state: "unsigned", ...base, reason: "signature-too-short" };
|
|
131
|
-
if (!opts.strict)
|
|
132
|
-
return { state: "signed", ...base };
|
|
133
|
-
const replayVerified = await verifySignature({
|
|
134
|
-
transport: opts.transport,
|
|
135
|
-
url,
|
|
136
|
-
headers,
|
|
137
|
-
model: opts.model,
|
|
138
|
-
block: found.block,
|
|
139
|
-
timeoutMs: opts.timeoutMs,
|
|
140
|
-
});
|
|
141
|
-
// A rejection is real evidence (the endpoint checked and refused); an
|
|
142
|
-
// acceptance is not, so it never upgrades or downgrades the state on its own.
|
|
143
|
-
return {
|
|
144
|
-
state: replayVerified ? "signed" : "unsigned",
|
|
145
|
-
...base,
|
|
146
|
-
replayVerified,
|
|
147
|
-
...(replayVerified ? {} : { reason: "signature-rejected-on-replay" }),
|
|
148
|
-
};
|
|
149
|
-
}
|
|
150
|
-
/**
|
|
151
|
-
* Hand the thinking block back to the endpoint. Against api.anthropic.com this
|
|
152
|
-
* would re-validate the signature; against a relay it does not (measured: a
|
|
153
|
-
* forged 308-byte signature returned 200 on both upstreams tested), so the
|
|
154
|
-
* result is reported as `replayVerified` evidence rather than a verdict.
|
|
155
|
-
*/
|
|
156
|
-
async function verifySignature(opts) {
|
|
157
|
-
const res = await opts.transport({
|
|
158
|
-
mode: "direct",
|
|
159
|
-
url: opts.url,
|
|
160
|
-
headers: opts.headers,
|
|
161
|
-
reqBody: {
|
|
162
|
-
model: opts.model,
|
|
163
|
-
max_tokens: 1,
|
|
164
|
-
messages: [
|
|
165
|
-
{ role: "user", content: PROBE_PROMPT },
|
|
166
|
-
{ role: "assistant", content: [opts.block] },
|
|
167
|
-
],
|
|
168
|
-
},
|
|
169
|
-
timeoutMs: opts.timeoutMs,
|
|
170
|
-
});
|
|
171
|
-
// A network failure proves nothing, so treat only an explicit rejection as
|
|
172
|
-
// a forged signature.
|
|
173
|
-
if (res.status === null)
|
|
174
|
-
return true;
|
|
175
|
-
return res.status < 400;
|
|
176
|
-
}
|
|
177
|
-
//# sourceMappingURL=thinking-signature.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"thinking-signature.js","sourceRoot":"","sources":["../../src/detectors/thinking-signature.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAIH;;;;GAIG;AACH,MAAM,YAAY,GAChB,kFAAkF,CAAC;AAErF,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC;;;GAGG;AACH,MAAM,UAAU,GAAG,KAAK,CAAC;AAEzB,0EAA0E;AAC1E,MAAM,iBAAiB,GAAG,EAAE,CAAC;AA2B7B,sFAAsF;AACtF,MAAM,eAAe,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;AAE/D,4DAA4D;AAC5D,MAAM,eAAe,GAAG;IACtB,iBAAiB;IACjB,iBAAiB;IACjB,iBAAiB;IACjB,mBAAmB;IACnB,mBAAmB;IACnB,kBAAkB;CACnB,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAClC,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;AAEnE,MAAM,UAAU,eAAe,CAC7B,KAAa;IAEb,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC9E,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC9E,OAAO,IAAI,CAAC;AACd,CAAC;AAQD,SAAS,YAAY,CAAC,IAAa;IAKjC,MAAM,OAAO,GAAI,IAA8B,EAAE,OAAO,CAAC;IACzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC7E,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,SAAS;QAC9C,MAAM,CAAC,GAAG,GAAoB,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,mBAAmB;YAAE,SAAS;QACtE,OAAO;YACL,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;YAC7D,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9D,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAc5C;IACC,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;IAC9E,IAAI,CAAC,IAAI;QACP,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IAE9E,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC;IAC9D,MAAM,OAAO,GAAG;QACd,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,mBAAmB,EAAE,YAAY;KAClC,CAAC;IACF,MAAM,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;IAC3D,2EAA2E;IAC3E,wDAAwD;IACxD,MAAM,IAAI,GAA4B;QACpC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,UAAU,EAAE,UAAU;QACtB,QAAQ;QACR,QAAQ,EACN,IAAI,CAAC,IAAI,KAAK,UAAU;YACtB,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,sBAAsB,EAAE;YAC5D,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE;KAClD,CAAC;IACF,mEAAmE;IACnE,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU;QAAE,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAE1E,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG;QACH,OAAO;QACP,OAAO,EAAE,IAAI;QACb,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG;QAC1C,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,KAAK;YACR,MAAM,EAAE,gBAAgB,GAAG,CAAC,MAAM,IAAI,SAAS,EAAE;SAClD,CAAC;IAEJ,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK,CAAC,KAAK;QACd,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAEzE,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC;IAC5B,MAAM,IAAI,GAAG;QACX,eAAe,EAAE,GAAG,CAAC,MAAM;QAC3B,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;QAC9C,aAAa,EAAE,KAAK,CAAC,KAAK;KAC3B,CAAC;IACF,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC9E,IAAI,GAAG,CAAC,MAAM,GAAG,iBAAiB;QAChC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;IAEvE,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;IAEtD,MAAM,cAAc,GAAG,MAAM,eAAe,CAAC;QAC3C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG;QACH,OAAO;QACP,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,sEAAsE;IACtE,8EAA8E;IAC9E,OAAO;QACL,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU;QAC7C,GAAG,IAAI;QACP,cAAc;QACd,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;KACtE,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,IAO9B;IACC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,OAAO,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE;gBACvC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;aAC7C;SACF;QACD,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,2EAA2E;IAC3E,sBAAsB;IACtB,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;AAC1B,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"throughput.d.ts","sourceRoot":"","sources":["../../src/detectors/throughput.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,gBAAgB,GAAG,IAAI,CAa1B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,QAAQ,GAAG,QAAQ,CAAC;KAChC,EAAE,CAAC;IACJ,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAQF;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,IAAI,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/B,oBAAoB,EAAE,CA8CxB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"throughput.js","sourceRoot":"","sources":["../../src/detectors/throughput.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAWH,MAAM,UAAU,gBAAgB,CAAC,IAKhC;IACC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;IACjC,wEAAwE;IACxE,wDAAwD;IACxD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACzE,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,YAAY,EAAE,MAAM;QACpB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,eAAe,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;KAChE,CAAC;AACJ,CAAC;AAgBD,MAAM,MAAM,GAAG,CAAC,EAAY,EAAE,EAAE;IAC9B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,CAAC,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAC;AAC9D,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAA2B,EAC3B,IAAgC;IAEhC,MAAM,KAAK,GAAG,IAAI,EAAE,YAAY,IAAI,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YAClB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,wEAAwE;QACxE,6DAA6D;QAC7D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACxD,MAAM,QAAQ,GACZ,KAAK,CAAC,MAAM,IAAI,CAAC;YACf,CAAC,CAAC,KAAK;iBACF,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,eAAe,IAAI,GAAG,GAAG,KAAK;gBAChC,CAAC,CAAC,eAAe,IAAI,GAAG,GAAG,KAAK,CACnC;iBACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,eAAe,EAAE,CAAC,CAAC,eAAe;gBAClC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;gBACpD,SAAS,EACP,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,CAAE,QAAkB,CAAC,CAAC,CAAE,QAAkB;aACtE,CAAC,CAAC;iBACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC;YAC1D,CAAC,CAAC,EAAE,CAAC;QAET,GAAG,CAAC,IAAI,CAAC;YACP,KAAK;YACL,WAAW,EAAE,KAAK,CAAC,MAAM;YACzB,SAAS,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1B,QAAQ;YACR,IAAI,EACF,KAAK,CAAC,MAAM,GAAG,CAAC;gBACd,CAAC,CAAC,0BAA0B;gBAC5B,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,sCAAsC;oBACxC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,oHAAoH;SAC/I,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"token-truth.d.ts","sourceRoot":"","sources":["../../src/detectors/token-truth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAuBhD,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,eAAe,GAAG,aAAa,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACxE,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,0EAA0E;IAC1E,EAAE,EAAE,OAAO,GAAG,IAAI,CAAC;IACnB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAKF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAK9E;AAcD,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA2G5B"}
|
|
@@ -1,142 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Token-accounting checks for Anthropic endpoints.
|
|
3
|
-
*
|
|
4
|
-
* The signature check answers "is this really Claude". These answer a different
|
|
5
|
-
* question the signature cannot: "are the numbers you are billing me true".
|
|
6
|
-
* A relay can forward genuine Claude and still inflate `input_tokens` on every
|
|
7
|
-
* call, and nothing in an identity probe would notice.
|
|
8
|
-
*
|
|
9
|
-
* Ported from veridrop (AGPL-3.0) `token_usage.py` / `integrity.py`, whose
|
|
10
|
-
* tolerances were calibrated against the official API. Do not tighten them
|
|
11
|
-
* without re-measuring: the loose-looking bounds exist because real Anthropic
|
|
12
|
-
* responses sat near them.
|
|
13
|
-
*/
|
|
14
|
-
import { COUNT_PROBE_MAX_TOKENS as MAX_TOKENS, LONG_PROMPT, SHORT_PROMPT, } from "../internal/fixed-text";
|
|
15
|
-
/**
|
|
16
|
-
* Expected `input_tokens` growth between the short and long prompt.
|
|
17
|
-
*
|
|
18
|
-
* Two bands because Anthropic changed tokenizer: opus-4-7/4-8 encode that same
|
|
19
|
-
* text into materially more tokens. A relay claiming a new-tokenizer model
|
|
20
|
-
* while serving an older one lands in the wrong band, which makes this a tier
|
|
21
|
-
* check as well as a billing check.
|
|
22
|
-
*/
|
|
23
|
-
const DELTA_OLD = { min: 45, max: 140 };
|
|
24
|
-
const DELTA_NEW = { min: 90, max: 230 };
|
|
25
|
-
const NEW_TOKENIZER_MODELS = ["claude-opus-4-7", "claude-opus-4-8"];
|
|
26
|
-
/** `count_tokens` should agree with the billed `input_tokens` this closely. */
|
|
27
|
-
const countTolerance = (n) => Math.max(4, Math.round(n * 0.2));
|
|
28
|
-
const normalize = (model) => model.toLowerCase().replace(/[._]/g, "-").split("/").pop() ?? "";
|
|
29
|
-
export function expectedInputDelta(model) {
|
|
30
|
-
const m = normalize(model);
|
|
31
|
-
return NEW_TOKENIZER_MODELS.some((p) => m.startsWith(p))
|
|
32
|
-
? DELTA_NEW
|
|
33
|
-
: DELTA_OLD;
|
|
34
|
-
}
|
|
35
|
-
/** Rejects booleans, which are `typeof "number"`-adjacent traps in JS. */
|
|
36
|
-
function intOf(v) {
|
|
37
|
-
return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : null;
|
|
38
|
-
}
|
|
39
|
-
function usageOf(data) {
|
|
40
|
-
const u = data?.usage;
|
|
41
|
-
return { input: intOf(u?.input_tokens), output: intOf(u?.output_tokens) };
|
|
42
|
-
}
|
|
43
|
-
export async function checkTokenTruth(opts) {
|
|
44
|
-
const base = opts.baseUrl.replace(/\/+$/, "");
|
|
45
|
-
const headers = {
|
|
46
|
-
"Content-Type": "application/json",
|
|
47
|
-
"x-api-key": opts.apiKey,
|
|
48
|
-
"anthropic-version": "2023-06-01",
|
|
49
|
-
};
|
|
50
|
-
const ask = (prompt) => opts.transport({
|
|
51
|
-
mode: "direct",
|
|
52
|
-
url: `${base}/v1/messages`,
|
|
53
|
-
headers,
|
|
54
|
-
reqBody: {
|
|
55
|
-
model: opts.model,
|
|
56
|
-
max_tokens: MAX_TOKENS,
|
|
57
|
-
messages: [{ role: "user", content: prompt }],
|
|
58
|
-
},
|
|
59
|
-
timeoutMs: opts.timeoutMs,
|
|
60
|
-
});
|
|
61
|
-
const [shortRes, longRes] = await Promise.all([
|
|
62
|
-
ask(SHORT_PROMPT),
|
|
63
|
-
ask(LONG_PROMPT),
|
|
64
|
-
]);
|
|
65
|
-
const noCounts = {
|
|
66
|
-
shortInputTokens: null,
|
|
67
|
-
longInputTokens: null,
|
|
68
|
-
countTokens: null,
|
|
69
|
-
};
|
|
70
|
-
if (shortRes.status === null || shortRes.status >= 400)
|
|
71
|
-
return {
|
|
72
|
-
ok: null,
|
|
73
|
-
checks: [],
|
|
74
|
-
...noCounts,
|
|
75
|
-
reason: `probe-failed:${shortRes.status ?? "network"}`,
|
|
76
|
-
};
|
|
77
|
-
const s = usageOf(shortRes.data);
|
|
78
|
-
const l = usageOf(longRes.data);
|
|
79
|
-
const checks = [];
|
|
80
|
-
const usagePresent = s.input !== null && l.input !== null;
|
|
81
|
-
checks.push({
|
|
82
|
-
id: "usage-present",
|
|
83
|
-
pass: usagePresent,
|
|
84
|
-
detail: usagePresent
|
|
85
|
-
? `short=${s.input}, long=${l.input}`
|
|
86
|
-
: "endpoint reported no usage counts",
|
|
87
|
-
});
|
|
88
|
-
// Without usage there is nothing to verify; a missing field is not fraud.
|
|
89
|
-
if (!usagePresent)
|
|
90
|
-
return { ok: null, checks, ...noCounts, reason: "no-usage-reported" };
|
|
91
|
-
const band = expectedInputDelta(opts.model);
|
|
92
|
-
const delta = l.input - s.input;
|
|
93
|
-
const deltaOk = delta >= band.min && delta <= band.max;
|
|
94
|
-
checks.push({
|
|
95
|
-
id: "input-delta",
|
|
96
|
-
pass: deltaOk,
|
|
97
|
-
detail: `delta=${delta}, expected ${band.min}-${band.max}`,
|
|
98
|
-
});
|
|
99
|
-
// Output above the cap we set means the count is invented, not merely padded.
|
|
100
|
-
const outputs = [s.output, l.output].filter((v) => v !== null);
|
|
101
|
-
const outputOk = outputs.every((v) => v > 0 && v <= MAX_TOKENS + 4);
|
|
102
|
-
checks.push({
|
|
103
|
-
id: "output-bounded",
|
|
104
|
-
pass: outputOk,
|
|
105
|
-
detail: `outputs=${outputs.join(",") || "none"} (cap ${MAX_TOKENS})`,
|
|
106
|
-
});
|
|
107
|
-
// The same upstream pricing the same request without generating: the
|
|
108
|
-
// strongest anti-inflation signal available, when the endpoint exposes it.
|
|
109
|
-
let counted = null;
|
|
110
|
-
const countRes = await opts.transport({
|
|
111
|
-
mode: "direct",
|
|
112
|
-
url: `${base}/v1/messages/count_tokens`,
|
|
113
|
-
headers,
|
|
114
|
-
reqBody: {
|
|
115
|
-
model: opts.model,
|
|
116
|
-
messages: [{ role: "user", content: SHORT_PROMPT }],
|
|
117
|
-
},
|
|
118
|
-
timeoutMs: opts.timeoutMs,
|
|
119
|
-
});
|
|
120
|
-
if (countRes.status !== null && countRes.status < 400) {
|
|
121
|
-
counted = intOf(countRes.data?.input_tokens);
|
|
122
|
-
if (counted !== null) {
|
|
123
|
-
const drift = Math.abs(counted - s.input);
|
|
124
|
-
const tol = countTolerance(s.input);
|
|
125
|
-
checks.push({
|
|
126
|
-
id: "count-tokens",
|
|
127
|
-
pass: drift <= tol,
|
|
128
|
-
detail: `count=${counted} vs billed=${s.input}, drift=${drift} (tolerance ${tol})`,
|
|
129
|
-
});
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
// A missing count_tokens endpoint is common on relays and proves nothing, so
|
|
133
|
-
// it is simply absent from the checks rather than counted as a failure.
|
|
134
|
-
return {
|
|
135
|
-
ok: checks.every((c) => c.pass),
|
|
136
|
-
checks,
|
|
137
|
-
shortInputTokens: s.input,
|
|
138
|
-
longInputTokens: l.input,
|
|
139
|
-
countTokens: counted,
|
|
140
|
-
};
|
|
141
|
-
}
|
|
142
|
-
//# sourceMappingURL=token-truth.js.map
|