ai-model-verifier 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +40 -0
- package/dist/detectors/response-metadata.d.ts +36 -0
- package/dist/detectors/response-metadata.d.ts.map +1 -0
- package/dist/detectors/response-metadata.js +91 -0
- package/dist/detectors/response-metadata.js.map +1 -0
- package/dist/detectors/thinking-signature.d.ts +70 -0
- package/dist/detectors/thinking-signature.d.ts.map +1 -0
- package/dist/detectors/thinking-signature.js +177 -0
- package/dist/detectors/thinking-signature.js.map +1 -0
- package/dist/detectors/throughput.d.ts +51 -0
- package/dist/detectors/throughput.d.ts.map +1 -0
- package/dist/detectors/throughput.js +83 -0
- package/dist/detectors/throughput.js.map +1 -0
- package/dist/detectors/token-truth.d.ts +40 -0
- package/dist/detectors/token-truth.d.ts.map +1 -0
- package/dist/detectors/token-truth.js +145 -0
- package/dist/detectors/token-truth.js.map +1 -0
- package/dist/handshake.d.ts +23 -0
- package/dist/handshake.d.ts.map +1 -0
- package/dist/handshake.js +90 -0
- package/dist/handshake.js.map +1 -0
- package/dist/highlight.d.ts +8 -0
- package/dist/highlight.d.ts.map +1 -0
- package/dist/highlight.js +63 -0
- package/dist/highlight.js.map +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/utils.d.ts +6 -0
- package/dist/internal/utils.d.ts.map +1 -0
- package/dist/internal/utils.js +19 -0
- package/dist/internal/utils.js.map +1 -0
- package/dist/models.d.ts +5 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +54 -0
- package/dist/models.js.map +1 -0
- package/dist/nonce.d.ts +4 -0
- package/dist/nonce.d.ts.map +1 -0
- package/dist/nonce.js +12 -0
- package/dist/nonce.js.map +1 -0
- package/dist/patterns.d.ts +16 -0
- package/dist/patterns.d.ts.map +1 -0
- package/dist/patterns.js +63 -0
- package/dist/patterns.js.map +1 -0
- package/dist/probes.d.ts +10 -0
- package/dist/probes.d.ts.map +1 -0
- package/dist/probes.js +54 -0
- package/dist/probes.js.map +1 -0
- package/dist/providers/anthropic.d.ts +3 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +70 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/config.d.ts +34 -0
- package/dist/providers/config.d.ts.map +1 -0
- package/dist/providers/config.js +14 -0
- package/dist/providers/config.js.map +1 -0
- package/dist/providers/gemini.d.ts +3 -0
- package/dist/providers/gemini.d.ts.map +1 -0
- package/dist/providers/gemini.js +58 -0
- package/dist/providers/gemini.js.map +1 -0
- package/dist/providers/openai.d.ts +3 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +56 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/rules.d.ts +6 -0
- package/dist/rules.d.ts.map +1 -0
- package/dist/rules.js +28 -0
- package/dist/rules.js.map +1 -0
- package/dist/runner.d.ts +22 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +298 -0
- package/dist/runner.js.map +1 -0
- package/dist/signals.d.ts +11 -0
- package/dist/signals.d.ts.map +1 -0
- package/dist/signals.js +64 -0
- package/dist/signals.js.map +1 -0
- package/dist/substitution.d.ts +24 -0
- package/dist/substitution.d.ts.map +1 -0
- package/dist/substitution.js +35 -0
- package/dist/substitution.js.map +1 -0
- package/dist/transport.d.ts +17 -0
- package/dist/transport.d.ts.map +1 -0
- package/dist/transport.js +79 -0
- package/dist/transport.js.map +1 -0
- package/dist/types.d.ts +55 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/verdict.d.ts +26 -0
- package/dist/verdict.d.ts.map +1 -0
- package/dist/verdict.js +84 -0
- package/dist/verdict.js.map +1 -0
- package/package.json +56 -0
package/README.md
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# ai-model-verifier
|
|
2
|
+
|
|
3
|
+
**Is that API endpoint really serving the model it claims?**
|
|
4
|
+
|
|
5
|
+
Resellers and relay stations sell "Claude Opus" or "GPT-5" and quietly serve
|
|
6
|
+
something cheaper. Given a `base_url`, `api_key` and `model`, this library
|
|
7
|
+
probes the endpoint and reports what it actually found: a substituted model, a
|
|
8
|
+
downgraded Claude tier, a stripped thinking signature, inflated token billing,
|
|
9
|
+
or a translation layer sitting in front of the real vendor.
|
|
10
|
+
|
|
11
|
+
Extracted from [unorouter](https://unorouter.com)'s model tester so the same
|
|
12
|
+
engine runs in the browser, on a server, and inside our sync pipeline.
|
|
13
|
+
|
|
14
|
+
## Design
|
|
15
|
+
|
|
16
|
+
- **Isomorphic.** Web-standard APIs only (`fetch`, `TextEncoder`,
|
|
17
|
+
`AbortSignal`); no `node:` imports. The HTTP call itself is injected as a
|
|
18
|
+
`TransportFn`, so a browser can route around CORS and a server can route
|
|
19
|
+
through an SSRF-safe fetch.
|
|
20
|
+
- **Opt-in.** Providers and detectors have their own entry points, so a consumer
|
|
21
|
+
that only checks Anthropic never bundles the Gemini config.
|
|
22
|
+
- **Zero required dependencies.**
|
|
23
|
+
|
|
24
|
+
## Use
|
|
25
|
+
|
|
26
|
+
```ts
|
|
27
|
+
import { runVerification } from "ai-model-verifier";
|
|
28
|
+
import { anthropicConfig } from "ai-model-verifier/providers/anthropic";
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## What it detects
|
|
32
|
+
|
|
33
|
+
`coding-tool` refusals (Kiro/Amazon Q wearing a Claude badge), scam pages,
|
|
34
|
+
CJK language leaks from a substituted Chinese model, response-mixing proxies,
|
|
35
|
+
foreign vendor identity, Claude tier substitution (opus billed, sonnet served),
|
|
36
|
+
and envelope-level model substitution.
|
|
37
|
+
|
|
38
|
+
## License
|
|
39
|
+
|
|
40
|
+
AGPL-3.0-only.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Passive response metadata: what the envelope reveals about who really
|
|
3
|
+
* answered, read from responses the run already made.
|
|
4
|
+
*
|
|
5
|
+
* Costs nothing (no extra requests) and asserts nothing. Everything here is
|
|
6
|
+
* reported as observation, because each signal has an honest explanation as
|
|
7
|
+
* well as a dishonest one:
|
|
8
|
+
*
|
|
9
|
+
* - a `chatcmpl-` id inside an Anthropic-shaped reply means a translation
|
|
10
|
+
* layer, which is what our own gateway is;
|
|
11
|
+
* - OpenAI-style `usage` keys in a Messages reply mean the same;
|
|
12
|
+
* - a missing id means a sloppy relay, not a fake model.
|
|
13
|
+
*
|
|
14
|
+
* The value is in the aggregate: over many lanes, "which of my Claude lanes
|
|
15
|
+
* answer with a chatcmpl id" is a question worth being able to ask.
|
|
16
|
+
*/
|
|
17
|
+
/** Vendor whose response shape the envelope actually looks like. */
|
|
18
|
+
export type EnvelopeShape = "anthropic" | "openai" | "gemini" | "unknown";
|
|
19
|
+
export type ResponseMetadata = {
|
|
20
|
+
/** Shape the payload matches, regardless of which API was called. */
|
|
21
|
+
shape: EnvelopeShape;
|
|
22
|
+
/** Shape the endpoint was asked to speak. */
|
|
23
|
+
expected: EnvelopeShape;
|
|
24
|
+
/** True when a translating hop sits between us and the model. */
|
|
25
|
+
translated: boolean;
|
|
26
|
+
/** e.g. "msg_", "chatcmpl-", "resp_"; null when the id is absent or odd. */
|
|
27
|
+
idPrefix: string | null;
|
|
28
|
+
/** Usage field names present, sorted. A cross-vendor mix is the tell. */
|
|
29
|
+
usageKeys: string[];
|
|
30
|
+
/** Foreign-vendor residue in the usage object, e.g. claude_* under OpenAI. */
|
|
31
|
+
foreignUsageKeys: string[];
|
|
32
|
+
/** Facts worth surfacing, in plain words. No verdict attached. */
|
|
33
|
+
notes: string[];
|
|
34
|
+
};
|
|
35
|
+
export declare function readResponseMetadata(data: unknown, expected: EnvelopeShape): ResponseMetadata;
|
|
36
|
+
//# sourceMappingURL=response-metadata.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"response-metadata.d.ts","sourceRoot":"","sources":["../../src/detectors/response-metadata.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,oEAAoE;AACpE,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1E,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qEAAqE;IACrE,KAAK,EAAE,aAAa,CAAC;IACrB,6CAA6C;IAC7C,QAAQ,EAAE,aAAa,CAAC;IACxB,iEAAiE;IACjE,UAAU,EAAE,OAAO,CAAC;IACpB,4EAA4E;IAC5E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,yEAAyE;IACzE,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,8EAA8E;IAC9E,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,kEAAkE;IAClE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAuCF,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,aAAa,GACtB,gBAAgB,CAyClB"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Passive response metadata: what the envelope reveals about who really
|
|
3
|
+
* answered, read from responses the run already made.
|
|
4
|
+
*
|
|
5
|
+
* Costs nothing (no extra requests) and asserts nothing. Everything here is
|
|
6
|
+
* reported as observation, because each signal has an honest explanation as
|
|
7
|
+
* well as a dishonest one:
|
|
8
|
+
*
|
|
9
|
+
* - a `chatcmpl-` id inside an Anthropic-shaped reply means a translation
|
|
10
|
+
* layer, which is what our own gateway is;
|
|
11
|
+
* - OpenAI-style `usage` keys in a Messages reply mean the same;
|
|
12
|
+
* - a missing id means a sloppy relay, not a fake model.
|
|
13
|
+
*
|
|
14
|
+
* The value is in the aggregate: over many lanes, "which of my Claude lanes
|
|
15
|
+
* answer with a chatcmpl id" is a question worth being able to ask.
|
|
16
|
+
*/
|
|
17
|
+
const ANTHROPIC_USAGE = new Set([
|
|
18
|
+
"input_tokens",
|
|
19
|
+
"output_tokens",
|
|
20
|
+
"cache_read_input_tokens",
|
|
21
|
+
"cache_creation_input_tokens",
|
|
22
|
+
]);
|
|
23
|
+
const OPENAI_USAGE = new Set([
|
|
24
|
+
"prompt_tokens",
|
|
25
|
+
"completion_tokens",
|
|
26
|
+
"total_tokens",
|
|
27
|
+
]);
|
|
28
|
+
const GEMINI_USAGE = new Set([
|
|
29
|
+
"promptTokenCount",
|
|
30
|
+
"candidatesTokenCount",
|
|
31
|
+
"totalTokenCount",
|
|
32
|
+
"thoughtsTokenCount",
|
|
33
|
+
]);
|
|
34
|
+
function idPrefixOf(id) {
|
|
35
|
+
if (typeof id !== "string" || id.length === 0)
|
|
36
|
+
return null;
|
|
37
|
+
for (const p of ["msg_", "chatcmpl-", "resp_", "call_", "toolu_"])
|
|
38
|
+
if (id.startsWith(p))
|
|
39
|
+
return p;
|
|
40
|
+
// An unrecognised shape (a bare UUID, say) is itself worth reporting.
|
|
41
|
+
const cut = id.search(/[-_]/);
|
|
42
|
+
return cut > 0 ? id.slice(0, cut + 1) : "(none)";
|
|
43
|
+
}
|
|
44
|
+
function shapeOf(data, usageKeys) {
|
|
45
|
+
if (data["candidates"] !== undefined || usageKeys.some((k) => GEMINI_USAGE.has(k)))
|
|
46
|
+
return "gemini";
|
|
47
|
+
if (data["choices"] !== undefined || usageKeys.some((k) => OPENAI_USAGE.has(k)))
|
|
48
|
+
return "openai";
|
|
49
|
+
if (data["content"] !== undefined || usageKeys.some((k) => ANTHROPIC_USAGE.has(k)))
|
|
50
|
+
return "anthropic";
|
|
51
|
+
return "unknown";
|
|
52
|
+
}
|
|
53
|
+
export function readResponseMetadata(data, expected) {
|
|
54
|
+
const obj = (data ?? {});
|
|
55
|
+
const usage = (obj["usage"] ?? {});
|
|
56
|
+
const usageKeys = Object.keys(usage).sort();
|
|
57
|
+
const shape = shapeOf(obj, usageKeys);
|
|
58
|
+
const idPrefix = idPrefixOf(obj["id"]);
|
|
59
|
+
const notes = [];
|
|
60
|
+
// Usage keys belonging to a vendor other than the shape we are reading.
|
|
61
|
+
const foreignUsageKeys = usageKeys.filter((k) => {
|
|
62
|
+
if (k.startsWith("claude_"))
|
|
63
|
+
return shape !== "anthropic";
|
|
64
|
+
if (k.startsWith("gemini_"))
|
|
65
|
+
return shape !== "gemini";
|
|
66
|
+
if (ANTHROPIC_USAGE.has(k))
|
|
67
|
+
return shape !== "anthropic";
|
|
68
|
+
if (OPENAI_USAGE.has(k))
|
|
69
|
+
return shape !== "openai";
|
|
70
|
+
if (GEMINI_USAGE.has(k))
|
|
71
|
+
return shape !== "gemini";
|
|
72
|
+
return false;
|
|
73
|
+
});
|
|
74
|
+
const translated = shape !== "unknown" && shape !== expected;
|
|
75
|
+
if (translated)
|
|
76
|
+
notes.push(`answered in ${shape} shape though ${expected} was requested: a translating hop sits in front`);
|
|
77
|
+
if (expected === "anthropic" && idPrefix && idPrefix !== "msg_")
|
|
78
|
+
notes.push(`id starts "${idPrefix}" where Anthropic uses "msg_": the id was minted by something else`);
|
|
79
|
+
if (foreignUsageKeys.length > 0)
|
|
80
|
+
notes.push(`usage carries ${foreignUsageKeys.join(", ")}, which belong to another vendor's API`);
|
|
81
|
+
return {
|
|
82
|
+
shape,
|
|
83
|
+
expected,
|
|
84
|
+
translated,
|
|
85
|
+
idPrefix,
|
|
86
|
+
usageKeys,
|
|
87
|
+
foreignUsageKeys,
|
|
88
|
+
notes,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=response-metadata.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"response-metadata.js","sourceRoot":"","sources":["../../src/detectors/response-metadata.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAsBH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,cAAc;IACd,eAAe;IACf,yBAAyB;IACzB,6BAA6B;CAC9B,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,eAAe;IACf,mBAAmB;IACnB,cAAc;CACf,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,kBAAkB;IAClB,sBAAsB;IACtB,iBAAiB;IACjB,oBAAoB;CACrB,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,EAAW;IAC7B,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3D,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC;QAC/D,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACjC,sEAAsE;IACtE,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACnD,CAAC;AAED,SAAS,OAAO,CAAC,IAA6B,EAAE,SAAmB;IACjE,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChF,OAAO,QAAiB,CAAC;IAC3B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,OAAO,QAAiB,CAAC;IAC3B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChF,OAAO,WAAoB,CAAC;IAC9B,OAAO,SAAkB,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAa,EACb,QAAuB;IAEvB,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC9D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,wEAAwE;IACxE,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,KAAK,WAAW,CAAC;QAC1D,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACvD,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,WAAW,CAAC;QACzD,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACnD,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,QAAQ,CAAC;IAC7D,IAAI,UAAU;QACZ,KAAK,CAAC,IAAI,CACR,eAAe,KAAK,iBAAiB,QAAQ,iDAAiD,CAC/F,CAAC;IACJ,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,IAAI,QAAQ,KAAK,MAAM;QAC7D,KAAK,CAAC,IAAI,CACR,cAAc,QAAQ,oEAAoE,CAC3F,CAAC;IACJ,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR,iBAAiB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC,CACrF,CAAC;IAEJ,OAAO;QACL,KAAK;QACL,QAAQ;QACR,UAAU;QACV,QAAQ;QACR,SAAS;QACT,gBAAgB;QAChB,KAAK;KACN,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anthropic thinking-signature check.
|
|
3
|
+
*
|
|
4
|
+
* Claude attaches a server-generated `signature` to every thinking block. A
|
|
5
|
+
* relay serving some other model cannot produce one, so this is the only
|
|
6
|
+
* evidence here that does not rest on asking the model who it is: identity and
|
|
7
|
+
* behaviour probes can be coached with a system prompt, a signature cannot.
|
|
8
|
+
*
|
|
9
|
+
* What it does NOT prove: which tier answered. Sonnet returns a perfectly valid
|
|
10
|
+
* signature when sold as Opus, so a pass here still needs the tier checks.
|
|
11
|
+
*
|
|
12
|
+
* Ported from veridrop (AGPL-3.0), whose detector stops at "a signature-shaped
|
|
13
|
+
* string is present".
|
|
14
|
+
*
|
|
15
|
+
* Their DESIGN.md also specifies a replay step (hand the thinking block back so
|
|
16
|
+
* Anthropic re-validates the signature, rejecting a forgery with 400/422) which
|
|
17
|
+
* they never implemented. We built and measured it: **it does not work through a
|
|
18
|
+
* relay**. Tested against two live upstreams (a6 merchant 3542 and pol) by
|
|
19
|
+
* replaying a genuine block and then the same block with 308 bytes of random
|
|
20
|
+
* base64 in place of the signature. Both upstreams returned 200 for both. A
|
|
21
|
+
* relay re-issues the turn to its own backend rather than passing our thinking
|
|
22
|
+
* block through, so nothing ever validates it.
|
|
23
|
+
*
|
|
24
|
+
* `strict` therefore stays off by default and is documented as proving only
|
|
25
|
+
* that the endpoint accepts the block, not that the signature is real. Do not
|
|
26
|
+
* present it as cryptographic proof.
|
|
27
|
+
*/
|
|
28
|
+
import type { TransportFn } from "../transport";
|
|
29
|
+
export type SignatureState =
|
|
30
|
+
/** Signed thinking block: a genuine Anthropic path answered. */
|
|
31
|
+
"signed"
|
|
32
|
+
/**
|
|
33
|
+
* Thought, but no signature. Expected when the lane crosses an
|
|
34
|
+
* OpenAI-shaped hop, which has no field to carry one, so this is
|
|
35
|
+
* unprovable rather than fake. Never a hard fail on its own.
|
|
36
|
+
*/
|
|
37
|
+
| "unsigned"
|
|
38
|
+
/** Thinking was requested and no block came back at all. */
|
|
39
|
+
| "no-thinking"
|
|
40
|
+
/** Model or endpoint cannot be asked; carries no evidence either way. */
|
|
41
|
+
| "skipped";
|
|
42
|
+
export type SignatureResult = {
|
|
43
|
+
state: SignatureState;
|
|
44
|
+
signatureLength: number;
|
|
45
|
+
/** First 32 chars, for the report; never the whole signature. */
|
|
46
|
+
signaturePrefix: string | null;
|
|
47
|
+
thinkingChars: number;
|
|
48
|
+
/** Only set when the replay check actually ran. */
|
|
49
|
+
replayVerified?: boolean;
|
|
50
|
+
reason?: string;
|
|
51
|
+
};
|
|
52
|
+
export declare function thinkingModeFor(model: string): {
|
|
53
|
+
kind: "adaptive" | "extended";
|
|
54
|
+
} | null;
|
|
55
|
+
export declare function checkThinkingSignature(opts: {
|
|
56
|
+
transport: TransportFn;
|
|
57
|
+
baseUrl: string;
|
|
58
|
+
apiKey: string;
|
|
59
|
+
model: string;
|
|
60
|
+
timeoutMs: number;
|
|
61
|
+
/**
|
|
62
|
+
* Replay the thinking block to the same endpoint. Measured as NOT proving
|
|
63
|
+
* authenticity through a relay (see the module docstring): both a genuine and
|
|
64
|
+
* a forged signature come back 200, because the relay re-issues the turn
|
|
65
|
+
* upstream instead of forwarding our block. Off by default; it only tells you
|
|
66
|
+
* the endpoint accepts a thinking block in the request.
|
|
67
|
+
*/
|
|
68
|
+
strict?: boolean;
|
|
69
|
+
}): Promise<SignatureResult>;
|
|
70
|
+
//# sourceMappingURL=thinking-signature.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"thinking-signature.d.ts","sourceRoot":"","sources":["../../src/detectors/thinking-signature.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAqBhD,MAAM,MAAM,cAAc;AACxB,gEAAgE;AAC9D,QAAQ;AACV;;;;GAIG;GACD,UAAU;AACZ,4DAA4D;GAC1D,aAAa;AACf,yEAAyE;GACvE,SAAS,CAAC;AAEd,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,cAAc,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAkBF,wBAAgB,eAAe,CAC7B,KAAK,EAAE,MAAM,GACZ;IAAE,IAAI,EAAE,UAAU,GAAG,UAAU,CAAA;CAAE,GAAG,IAAI,CAK1C;AA4BD,wBAAsB,sBAAsB,CAAC,IAAI,EAAE;IACjD,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,GAAG,OAAO,CAAC,eAAe,CAAC,CA0E3B"}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anthropic thinking-signature check.
|
|
3
|
+
*
|
|
4
|
+
* Claude attaches a server-generated `signature` to every thinking block. A
|
|
5
|
+
* relay serving some other model cannot produce one, so this is the only
|
|
6
|
+
* evidence here that does not rest on asking the model who it is: identity and
|
|
7
|
+
* behaviour probes can be coached with a system prompt, a signature cannot.
|
|
8
|
+
*
|
|
9
|
+
* What it does NOT prove: which tier answered. Sonnet returns a perfectly valid
|
|
10
|
+
* signature when sold as Opus, so a pass here still needs the tier checks.
|
|
11
|
+
*
|
|
12
|
+
* Ported from veridrop (AGPL-3.0), whose detector stops at "a signature-shaped
|
|
13
|
+
* string is present".
|
|
14
|
+
*
|
|
15
|
+
* Their DESIGN.md also specifies a replay step (hand the thinking block back so
|
|
16
|
+
* Anthropic re-validates the signature, rejecting a forgery with 400/422) which
|
|
17
|
+
* they never implemented. We built and measured it: **it does not work through a
|
|
18
|
+
* relay**. Tested against two live upstreams (a6 merchant 3542 and pol) by
|
|
19
|
+
* replaying a genuine block and then the same block with 308 bytes of random
|
|
20
|
+
* base64 in place of the signature. Both upstreams returned 200 for both. A
|
|
21
|
+
* relay re-issues the turn to its own backend rather than passing our thinking
|
|
22
|
+
* block through, so nothing ever validates it.
|
|
23
|
+
*
|
|
24
|
+
* `strict` therefore stays off by default and is documented as proving only
|
|
25
|
+
* that the endpoint accepts the block, not that the signature is real. Do not
|
|
26
|
+
* present it as cryptographic proof.
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* Multi-step GCD: hard enough that adaptive models decide to think, and
|
|
30
|
+
* deliberately not the 1071/462 pair from Anthropic's own docs, which a relay
|
|
31
|
+
* could special-case.
|
|
32
|
+
*/
|
|
33
|
+
const PROBE_PROMPT = "Find the greatest common divisor of 2378 and 1547 using the Euclidean algorithm.";
|
|
34
|
+
const THINKING_BUDGET_TOKENS = 2000;
|
|
35
|
+
/**
|
|
36
|
+
* Covers thinking + answer together. Too small and an adaptive model skips
|
|
37
|
+
* thinking entirely to fit, which reads as a fake when it is our own fault.
|
|
38
|
+
*/
|
|
39
|
+
const MAX_TOKENS = 16000;
|
|
40
|
+
/** Real signatures run to hundreds of chars; this only rejects a stub. */
|
|
41
|
+
const SIGNATURE_MIN_LEN = 50;
|
|
42
|
+
/** Adaptive-thinking models take `effort` at the top level, not inside `thinking`. */
|
|
43
|
+
const ADAPTIVE_MODELS = ["claude-opus-4-7", "claude-opus-4-8"];
|
|
44
|
+
/** Extended-thinking models take a token budget instead. */
|
|
45
|
+
const EXTENDED_MODELS = [
|
|
46
|
+
"claude-opus-4-6",
|
|
47
|
+
"claude-opus-4-5",
|
|
48
|
+
"claude-opus-4-1",
|
|
49
|
+
"claude-sonnet-4-6",
|
|
50
|
+
"claude-sonnet-4-5",
|
|
51
|
+
"claude-haiku-4-5",
|
|
52
|
+
];
|
|
53
|
+
const normalize = (model) => model.toLowerCase().replace(/[._]/g, "-").split("/").pop() ?? "";
|
|
54
|
+
export function thinkingModeFor(model) {
|
|
55
|
+
const m = normalize(model);
|
|
56
|
+
if (ADAPTIVE_MODELS.some((p) => m.startsWith(p)))
|
|
57
|
+
return { kind: "adaptive" };
|
|
58
|
+
if (EXTENDED_MODELS.some((p) => m.startsWith(p)))
|
|
59
|
+
return { kind: "extended" };
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
function readThinking(data) {
|
|
63
|
+
const content = data?.content;
|
|
64
|
+
if (!Array.isArray(content))
|
|
65
|
+
return { block: null, signature: "", chars: 0 };
|
|
66
|
+
for (const raw of content) {
|
|
67
|
+
if (!raw || typeof raw !== "object")
|
|
68
|
+
continue;
|
|
69
|
+
const b = raw;
|
|
70
|
+
if (b.type !== "thinking" && b.type !== "redacted_thinking")
|
|
71
|
+
continue;
|
|
72
|
+
return {
|
|
73
|
+
block: b,
|
|
74
|
+
signature: typeof b.signature === "string" ? b.signature : "",
|
|
75
|
+
chars: typeof b.thinking === "string" ? b.thinking.length : 0,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
return { block: null, signature: "", chars: 0 };
|
|
79
|
+
}
|
|
80
|
+
export async function checkThinkingSignature(opts) {
|
|
81
|
+
const mode = thinkingModeFor(opts.model);
|
|
82
|
+
const empty = { signatureLength: 0, signaturePrefix: null, thinkingChars: 0 };
|
|
83
|
+
if (!mode)
|
|
84
|
+
return { state: "skipped", ...empty, reason: "model-has-no-thinking-mode" };
|
|
85
|
+
const url = `${opts.baseUrl.replace(/\/+$/, "")}/v1/messages`;
|
|
86
|
+
const headers = {
|
|
87
|
+
"Content-Type": "application/json",
|
|
88
|
+
"x-api-key": opts.apiKey,
|
|
89
|
+
"anthropic-version": "2023-06-01",
|
|
90
|
+
};
|
|
91
|
+
const messages = [{ role: "user", content: PROBE_PROMPT }];
|
|
92
|
+
// Non-streaming on purpose: streaming an adaptive model silently drops the
|
|
93
|
+
// thinking block, so a real Claude would look unsigned.
|
|
94
|
+
const body = {
|
|
95
|
+
model: opts.model,
|
|
96
|
+
max_tokens: MAX_TOKENS,
|
|
97
|
+
messages,
|
|
98
|
+
thinking: mode.kind === "extended"
|
|
99
|
+
? { type: "enabled", budget_tokens: THINKING_BUDGET_TOKENS }
|
|
100
|
+
: { type: "adaptive", display: "summarized" },
|
|
101
|
+
};
|
|
102
|
+
// Sibling of `thinking`, never nested inside it: nesting is a 400.
|
|
103
|
+
if (mode.kind === "adaptive")
|
|
104
|
+
body["output_config"] = { effort: "xhigh" };
|
|
105
|
+
const res = await opts.transport({
|
|
106
|
+
mode: "direct",
|
|
107
|
+
url,
|
|
108
|
+
headers,
|
|
109
|
+
reqBody: body,
|
|
110
|
+
timeoutMs: opts.timeoutMs,
|
|
111
|
+
});
|
|
112
|
+
if (res.status === null || res.status >= 400)
|
|
113
|
+
return {
|
|
114
|
+
state: "skipped",
|
|
115
|
+
...empty,
|
|
116
|
+
reason: `probe-failed:${res.status ?? "network"}`,
|
|
117
|
+
};
|
|
118
|
+
const found = readThinking(res.data);
|
|
119
|
+
if (!found.block)
|
|
120
|
+
return { state: "no-thinking", ...empty, reason: "no-thinking-block" };
|
|
121
|
+
const sig = found.signature;
|
|
122
|
+
const base = {
|
|
123
|
+
signatureLength: sig.length,
|
|
124
|
+
signaturePrefix: sig ? sig.slice(0, 32) : null,
|
|
125
|
+
thinkingChars: found.chars,
|
|
126
|
+
};
|
|
127
|
+
if (!sig)
|
|
128
|
+
return { state: "unsigned", ...base, reason: "no-signature-field" };
|
|
129
|
+
if (sig.length < SIGNATURE_MIN_LEN)
|
|
130
|
+
return { state: "unsigned", ...base, reason: "signature-too-short" };
|
|
131
|
+
if (!opts.strict)
|
|
132
|
+
return { state: "signed", ...base };
|
|
133
|
+
const replayVerified = await verifySignature({
|
|
134
|
+
transport: opts.transport,
|
|
135
|
+
url,
|
|
136
|
+
headers,
|
|
137
|
+
model: opts.model,
|
|
138
|
+
block: found.block,
|
|
139
|
+
timeoutMs: opts.timeoutMs,
|
|
140
|
+
});
|
|
141
|
+
// A rejection is real evidence (the endpoint checked and refused); an
|
|
142
|
+
// acceptance is not, so it never upgrades or downgrades the state on its own.
|
|
143
|
+
return {
|
|
144
|
+
state: replayVerified ? "signed" : "unsigned",
|
|
145
|
+
...base,
|
|
146
|
+
replayVerified,
|
|
147
|
+
...(replayVerified ? {} : { reason: "signature-rejected-on-replay" }),
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Hand the thinking block back to the endpoint. Against api.anthropic.com this
|
|
152
|
+
* would re-validate the signature; against a relay it does not (measured: a
|
|
153
|
+
* forged 308-byte signature returned 200 on both upstreams tested), so the
|
|
154
|
+
* result is reported as `replayVerified` evidence rather than a verdict.
|
|
155
|
+
*/
|
|
156
|
+
async function verifySignature(opts) {
|
|
157
|
+
const res = await opts.transport({
|
|
158
|
+
mode: "direct",
|
|
159
|
+
url: opts.url,
|
|
160
|
+
headers: opts.headers,
|
|
161
|
+
reqBody: {
|
|
162
|
+
model: opts.model,
|
|
163
|
+
max_tokens: 1,
|
|
164
|
+
messages: [
|
|
165
|
+
{ role: "user", content: PROBE_PROMPT },
|
|
166
|
+
{ role: "assistant", content: [opts.block] },
|
|
167
|
+
],
|
|
168
|
+
},
|
|
169
|
+
timeoutMs: opts.timeoutMs,
|
|
170
|
+
});
|
|
171
|
+
// A network failure proves nothing, so treat only an explicit rejection as
|
|
172
|
+
// a forged signature.
|
|
173
|
+
if (res.status === null)
|
|
174
|
+
return true;
|
|
175
|
+
return res.status < 400;
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=thinking-signature.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"thinking-signature.js","sourceRoot":"","sources":["../../src/detectors/thinking-signature.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAIH;;;;GAIG;AACH,MAAM,YAAY,GAChB,kFAAkF,CAAC;AAErF,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC;;;GAGG;AACH,MAAM,UAAU,GAAG,KAAK,CAAC;AAEzB,0EAA0E;AAC1E,MAAM,iBAAiB,GAAG,EAAE,CAAC;AA2B7B,sFAAsF;AACtF,MAAM,eAAe,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;AAE/D,4DAA4D;AAC5D,MAAM,eAAe,GAAG;IACtB,iBAAiB;IACjB,iBAAiB;IACjB,iBAAiB;IACjB,mBAAmB;IACnB,mBAAmB;IACnB,kBAAkB;CACnB,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAClC,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;AAEnE,MAAM,UAAU,eAAe,CAC7B,KAAa;IAEb,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC9E,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC9E,OAAO,IAAI,CAAC;AACd,CAAC;AAQD,SAAS,YAAY,CAAC,IAAa;IAKjC,MAAM,OAAO,GAAI,IAA8B,EAAE,OAAO,CAAC;IACzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC7E,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,SAAS;QAC9C,MAAM,CAAC,GAAG,GAAoB,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,mBAAmB;YAAE,SAAS;QACtE,OAAO;YACL,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;YAC7D,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9D,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAc5C;IACC,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;IAC9E,IAAI,CAAC,IAAI;QACP,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IAE9E,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC;IAC9D,MAAM,OAAO,GAAG;QACd,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,mBAAmB,EAAE,YAAY;KAClC,CAAC;IACF,MAAM,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;IAC3D,2EAA2E;IAC3E,wDAAwD;IACxD,MAAM,IAAI,GAA4B;QACpC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,UAAU,EAAE,UAAU;QACtB,QAAQ;QACR,QAAQ,EACN,IAAI,CAAC,IAAI,KAAK,UAAU;YACtB,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,sBAAsB,EAAE;YAC5D,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE;KAClD,CAAC;IACF,mEAAmE;IACnE,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU;QAAE,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAE1E,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG;QACH,OAAO;QACP,OAAO,EAAE,IAAI;QACb,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG;QAC1C,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,KAAK;YACR,MAAM,EAAE,gBAAgB,GAAG,CAAC,MAAM,IAAI,SAAS,EAAE;SAClD,CAAC;IAEJ,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK,CAAC,KAAK;QACd,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAEzE,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC;IAC5B,MAAM,IAAI,GAAG;QACX,eAAe,EAAE,GAAG,CAAC,MAAM;QAC3B,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;QAC9C,aAAa,EAAE,KAAK,CAAC,KAAK;KAC3B,CAAC;IACF,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC9E,IAAI,GAAG,CAAC,MAAM,GAAG,iBAAiB;QAChC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;IAEvE,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;IAEtD,MAAM,cAAc,GAAG,MAAM,eAAe,CAAC;QAC3C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG;QACH,OAAO;QACP,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,sEAAsE;IACtE,8EAA8E;IAC9E,OAAO;QACL,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU;QAC7C,GAAG,IAAI;QACP,cAAc;QACd,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;KACtE,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,IAO9B;IACC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,OAAO,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE;gBACvC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;aAC7C;SACF;QACD,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,2EAA2E;IAC3E,sBAAsB;IACtB,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Throughput metadata: tokens per second, derived from data the run already
|
|
3
|
+
* collected.
|
|
4
|
+
*
|
|
5
|
+
* Motivation: bigger models are slower per token, so a lane billed as Opus but
|
|
6
|
+
* served by Sonnet tends to run fast. That makes throughput the one tier signal
|
|
7
|
+
* that survives a relay refusing to name its model.
|
|
8
|
+
*
|
|
9
|
+
* It is metadata, not a verdict, and deliberately has no thresholds. Rate
|
|
10
|
+
* varies with load, batching, quantisation and time of day, so a single
|
|
11
|
+
* measurement cannot separate "cheaper model" from "quiet hour". The useful
|
|
12
|
+
* comparison is across lanes serving the SAME model, which only the caller can
|
|
13
|
+
* assemble: `compareThroughput` does that part and still only describes what it
|
|
14
|
+
* sees.
|
|
15
|
+
*/
|
|
16
|
+
export type ThroughputSample = {
|
|
17
|
+
/** Identifies the lane in the caller's own terms. */
|
|
18
|
+
lane: string;
|
|
19
|
+
model: string;
|
|
20
|
+
outputTokens: number;
|
|
21
|
+
elapsedMs: number;
|
|
22
|
+
tokensPerSecond: number;
|
|
23
|
+
};
|
|
24
|
+
export declare function sampleThroughput(args: {
|
|
25
|
+
lane: string;
|
|
26
|
+
model: string;
|
|
27
|
+
outputTokens: number | null | undefined;
|
|
28
|
+
elapsedMs: number;
|
|
29
|
+
}): ThroughputSample | null;
|
|
30
|
+
export type ThroughputComparison = {
|
|
31
|
+
model: string;
|
|
32
|
+
sampleCount: number;
|
|
33
|
+
medianTps: number;
|
|
34
|
+
/** Lanes running far from the median for this model, fastest first. */
|
|
35
|
+
outliers: {
|
|
36
|
+
lane: string;
|
|
37
|
+
tokensPerSecond: number;
|
|
38
|
+
ratioToMedian: number;
|
|
39
|
+
direction: "faster" | "slower";
|
|
40
|
+
}[];
|
|
41
|
+
note: string;
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Describe how lanes serving one model compare. `outlierRatio` is how far from
|
|
45
|
+
* the median a lane must sit to be worth mentioning; the default of 2 is a
|
|
46
|
+
* starting point for eyeballing, not a calibrated threshold.
|
|
47
|
+
*/
|
|
48
|
+
export declare function compareThroughput(samples: ThroughputSample[], opts?: {
|
|
49
|
+
outlierRatio?: number;
|
|
50
|
+
}): ThroughputComparison[];
|
|
51
|
+
//# sourceMappingURL=throughput.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throughput.d.ts","sourceRoot":"","sources":["../../src/detectors/throughput.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,gBAAgB,GAAG,IAAI,CAa1B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,QAAQ,GAAG,QAAQ,CAAC;KAChC,EAAE,CAAC;IACJ,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAQF;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,IAAI,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/B,oBAAoB,EAAE,CA8CxB"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Throughput metadata: tokens per second, derived from data the run already
|
|
3
|
+
* collected.
|
|
4
|
+
*
|
|
5
|
+
* Motivation: bigger models are slower per token, so a lane billed as Opus but
|
|
6
|
+
* served by Sonnet tends to run fast. That makes throughput the one tier signal
|
|
7
|
+
* that survives a relay refusing to name its model.
|
|
8
|
+
*
|
|
9
|
+
* It is metadata, not a verdict, and deliberately has no thresholds. Rate
|
|
10
|
+
* varies with load, batching, quantisation and time of day, so a single
|
|
11
|
+
* measurement cannot separate "cheaper model" from "quiet hour". The useful
|
|
12
|
+
* comparison is across lanes serving the SAME model, which only the caller can
|
|
13
|
+
* assemble: `compareThroughput` does that part and still only describes what it
|
|
14
|
+
* sees.
|
|
15
|
+
*/
|
|
16
|
+
export function sampleThroughput(args) {
|
|
17
|
+
const tokens = args.outputTokens;
|
|
18
|
+
// A handful of tokens over a noisy connection says nothing; the rate is
|
|
19
|
+
// dominated by connection setup rather than generation.
|
|
20
|
+
if (typeof tokens !== "number" || tokens < 20)
|
|
21
|
+
return null;
|
|
22
|
+
if (!Number.isFinite(args.elapsedMs) || args.elapsedMs <= 0)
|
|
23
|
+
return null;
|
|
24
|
+
return {
|
|
25
|
+
lane: args.lane,
|
|
26
|
+
model: args.model,
|
|
27
|
+
outputTokens: tokens,
|
|
28
|
+
elapsedMs: args.elapsedMs,
|
|
29
|
+
tokensPerSecond: +(tokens / (args.elapsedMs / 1000)).toFixed(1),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
const median = (xs) => {
|
|
33
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
34
|
+
const mid = Math.floor(s.length / 2);
|
|
35
|
+
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Describe how lanes serving one model compare. `outlierRatio` is how far from
|
|
39
|
+
* the median a lane must sit to be worth mentioning; the default of 2 is a
|
|
40
|
+
* starting point for eyeballing, not a calibrated threshold.
|
|
41
|
+
*/
|
|
42
|
+
export function compareThroughput(samples, opts) {
|
|
43
|
+
const ratio = opts?.outlierRatio ?? 2;
|
|
44
|
+
const byModel = new Map();
|
|
45
|
+
for (const s of samples) {
|
|
46
|
+
const list = byModel.get(s.model);
|
|
47
|
+
if (list)
|
|
48
|
+
list.push(s);
|
|
49
|
+
else
|
|
50
|
+
byModel.set(s.model, [s]);
|
|
51
|
+
}
|
|
52
|
+
const out = [];
|
|
53
|
+
for (const [model, group] of byModel) {
|
|
54
|
+
// Below three lanes there is no distribution to speak of, so report the
|
|
55
|
+
// median and no outliers rather than inventing a comparison.
|
|
56
|
+
const med = median(group.map((s) => s.tokensPerSecond));
|
|
57
|
+
const outliers = group.length >= 3
|
|
58
|
+
? group
|
|
59
|
+
.filter((s) => s.tokensPerSecond >= med * ratio ||
|
|
60
|
+
s.tokensPerSecond <= med / ratio)
|
|
61
|
+
.map((s) => ({
|
|
62
|
+
lane: s.lane,
|
|
63
|
+
tokensPerSecond: s.tokensPerSecond,
|
|
64
|
+
ratioToMedian: +(s.tokensPerSecond / med).toFixed(2),
|
|
65
|
+
direction: s.tokensPerSecond > med ? "faster" : "slower",
|
|
66
|
+
}))
|
|
67
|
+
.sort((a, b) => b.tokensPerSecond - a.tokensPerSecond)
|
|
68
|
+
: [];
|
|
69
|
+
out.push({
|
|
70
|
+
model,
|
|
71
|
+
sampleCount: group.length,
|
|
72
|
+
medianTps: +med.toFixed(1),
|
|
73
|
+
outliers,
|
|
74
|
+
note: group.length < 3
|
|
75
|
+
? "too few lanes to compare"
|
|
76
|
+
: outliers.length === 0
|
|
77
|
+
? "all lanes within range of each other"
|
|
78
|
+
: `${outliers.length} lane(s) far from the median; a much faster lane can mean a smaller model, but load and quantisation move this too`,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=throughput.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throughput.js","sourceRoot":"","sources":["../../src/detectors/throughput.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAWH,MAAM,UAAU,gBAAgB,CAAC,IAKhC;IACC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;IACjC,wEAAwE;IACxE,wDAAwD;IACxD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACzE,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,YAAY,EAAE,MAAM;QACpB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,eAAe,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;KAChE,CAAC;AACJ,CAAC;AAgBD,MAAM,MAAM,GAAG,CAAC,EAAY,EAAE,EAAE;IAC9B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,CAAC,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAC;AAC9D,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAA2B,EAC3B,IAAgC;IAEhC,MAAM,KAAK,GAAG,IAAI,EAAE,YAAY,IAAI,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YAClB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,wEAAwE;QACxE,6DAA6D;QAC7D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACxD,MAAM,QAAQ,GACZ,KAAK,CAAC,MAAM,IAAI,CAAC;YACf,CAAC,CAAC,KAAK;iBACF,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,eAAe,IAAI,GAAG,GAAG,KAAK;gBAChC,CAAC,CAAC,eAAe,IAAI,GAAG,GAAG,KAAK,CACnC;iBACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,eAAe,EAAE,CAAC,CAAC,eAAe;gBAClC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;gBACpD,SAAS,EACP,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,CAAE,QAAkB,CAAC,CAAC,CAAE,QAAkB;aACtE,CAAC,CAAC;iBACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC;YAC1D,CAAC,CAAC,EAAE,CAAC;QAET,GAAG,CAAC,IAAI,CAAC;YACP,KAAK;YACL,WAAW,EAAE,KAAK,CAAC,MAAM;YACzB,SAAS,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1B,QAAQ;YACR,IAAI,EACF,KAAK,CAAC,MAAM,GAAG,CAAC;gBACd,CAAC,CAAC,0BAA0B;gBAC5B,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,sCAAsC;oBACxC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,oHAAoH;SAC/I,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token-accounting checks for Anthropic endpoints.
|
|
3
|
+
*
|
|
4
|
+
* The signature check answers "is this really Claude". These answer a different
|
|
5
|
+
* question the signature cannot: "are the numbers you are billing me true".
|
|
6
|
+
* A relay can forward genuine Claude and still inflate `input_tokens` on every
|
|
7
|
+
* call, and nothing in an identity probe would notice.
|
|
8
|
+
*
|
|
9
|
+
* Ported from veridrop (AGPL-3.0) `token_usage.py` / `integrity.py`, whose
|
|
10
|
+
* tolerances were calibrated against the official API. Do not tighten them
|
|
11
|
+
* without re-measuring: the loose-looking bounds exist because real Anthropic
|
|
12
|
+
* responses sat near them.
|
|
13
|
+
*/
|
|
14
|
+
import type { TransportFn } from "../transport";
|
|
15
|
+
export type TokenTruthCheck = {
|
|
16
|
+
id: "usage-present" | "input-delta" | "output-bounded" | "count-tokens";
|
|
17
|
+
pass: boolean;
|
|
18
|
+
detail: string;
|
|
19
|
+
};
|
|
20
|
+
export type TokenTruthResult = {
|
|
21
|
+
/** null when nothing could be measured (endpoint refused every probe). */
|
|
22
|
+
ok: boolean | null;
|
|
23
|
+
checks: TokenTruthCheck[];
|
|
24
|
+
shortInputTokens: number | null;
|
|
25
|
+
longInputTokens: number | null;
|
|
26
|
+
countTokens: number | null;
|
|
27
|
+
reason?: string;
|
|
28
|
+
};
|
|
29
|
+
export declare function expectedInputDelta(model: string): {
|
|
30
|
+
min: number;
|
|
31
|
+
max: number;
|
|
32
|
+
};
|
|
33
|
+
export declare function checkTokenTruth(opts: {
|
|
34
|
+
transport: TransportFn;
|
|
35
|
+
baseUrl: string;
|
|
36
|
+
apiKey: string;
|
|
37
|
+
model: string;
|
|
38
|
+
timeoutMs: number;
|
|
39
|
+
}): Promise<TokenTruthResult>;
|
|
40
|
+
//# sourceMappingURL=token-truth.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-truth.d.ts","sourceRoot":"","sources":["../../src/detectors/token-truth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAsBhD,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,eAAe,GAAG,aAAa,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACxE,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,0EAA0E;IAC1E,EAAE,EAAE,OAAO,GAAG,IAAI,CAAC;IACnB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAKF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAK9E;AAcD,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA2G5B"}
|