ai-model-verifier 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/LICENSE +661 -0
  2. package/README.md +40 -0
  3. package/dist/detectors/response-metadata.d.ts +36 -0
  4. package/dist/detectors/response-metadata.d.ts.map +1 -0
  5. package/dist/detectors/response-metadata.js +91 -0
  6. package/dist/detectors/response-metadata.js.map +1 -0
  7. package/dist/detectors/thinking-signature.d.ts +70 -0
  8. package/dist/detectors/thinking-signature.d.ts.map +1 -0
  9. package/dist/detectors/thinking-signature.js +177 -0
  10. package/dist/detectors/thinking-signature.js.map +1 -0
  11. package/dist/detectors/throughput.d.ts +51 -0
  12. package/dist/detectors/throughput.d.ts.map +1 -0
  13. package/dist/detectors/throughput.js +83 -0
  14. package/dist/detectors/throughput.js.map +1 -0
  15. package/dist/detectors/token-truth.d.ts +40 -0
  16. package/dist/detectors/token-truth.d.ts.map +1 -0
  17. package/dist/detectors/token-truth.js +145 -0
  18. package/dist/detectors/token-truth.js.map +1 -0
  19. package/dist/handshake.d.ts +23 -0
  20. package/dist/handshake.d.ts.map +1 -0
  21. package/dist/handshake.js +90 -0
  22. package/dist/handshake.js.map +1 -0
  23. package/dist/highlight.d.ts +8 -0
  24. package/dist/highlight.d.ts.map +1 -0
  25. package/dist/highlight.js +63 -0
  26. package/dist/highlight.js.map +1 -0
  27. package/dist/index.d.ts +7 -0
  28. package/dist/index.d.ts.map +1 -0
  29. package/dist/index.js +7 -0
  30. package/dist/index.js.map +1 -0
  31. package/dist/internal/utils.d.ts +6 -0
  32. package/dist/internal/utils.d.ts.map +1 -0
  33. package/dist/internal/utils.js +19 -0
  34. package/dist/internal/utils.js.map +1 -0
  35. package/dist/models.d.ts +5 -0
  36. package/dist/models.d.ts.map +1 -0
  37. package/dist/models.js +54 -0
  38. package/dist/models.js.map +1 -0
  39. package/dist/nonce.d.ts +4 -0
  40. package/dist/nonce.d.ts.map +1 -0
  41. package/dist/nonce.js +12 -0
  42. package/dist/nonce.js.map +1 -0
  43. package/dist/patterns.d.ts +16 -0
  44. package/dist/patterns.d.ts.map +1 -0
  45. package/dist/patterns.js +63 -0
  46. package/dist/patterns.js.map +1 -0
  47. package/dist/probes.d.ts +10 -0
  48. package/dist/probes.d.ts.map +1 -0
  49. package/dist/probes.js +54 -0
  50. package/dist/probes.js.map +1 -0
  51. package/dist/providers/anthropic.d.ts +3 -0
  52. package/dist/providers/anthropic.d.ts.map +1 -0
  53. package/dist/providers/anthropic.js +70 -0
  54. package/dist/providers/anthropic.js.map +1 -0
  55. package/dist/providers/config.d.ts +34 -0
  56. package/dist/providers/config.d.ts.map +1 -0
  57. package/dist/providers/config.js +14 -0
  58. package/dist/providers/config.js.map +1 -0
  59. package/dist/providers/gemini.d.ts +3 -0
  60. package/dist/providers/gemini.d.ts.map +1 -0
  61. package/dist/providers/gemini.js +58 -0
  62. package/dist/providers/gemini.js.map +1 -0
  63. package/dist/providers/openai.d.ts +3 -0
  64. package/dist/providers/openai.d.ts.map +1 -0
  65. package/dist/providers/openai.js +56 -0
  66. package/dist/providers/openai.js.map +1 -0
  67. package/dist/rules.d.ts +6 -0
  68. package/dist/rules.d.ts.map +1 -0
  69. package/dist/rules.js +28 -0
  70. package/dist/rules.js.map +1 -0
  71. package/dist/runner.d.ts +22 -0
  72. package/dist/runner.d.ts.map +1 -0
  73. package/dist/runner.js +298 -0
  74. package/dist/runner.js.map +1 -0
  75. package/dist/signals.d.ts +11 -0
  76. package/dist/signals.d.ts.map +1 -0
  77. package/dist/signals.js +64 -0
  78. package/dist/signals.js.map +1 -0
  79. package/dist/substitution.d.ts +24 -0
  80. package/dist/substitution.d.ts.map +1 -0
  81. package/dist/substitution.js +35 -0
  82. package/dist/substitution.js.map +1 -0
  83. package/dist/transport.d.ts +17 -0
  84. package/dist/transport.d.ts.map +1 -0
  85. package/dist/transport.js +79 -0
  86. package/dist/transport.js.map +1 -0
  87. package/dist/types.d.ts +55 -0
  88. package/dist/types.d.ts.map +1 -0
  89. package/dist/types.js +2 -0
  90. package/dist/types.js.map +1 -0
  91. package/dist/verdict.d.ts +26 -0
  92. package/dist/verdict.d.ts.map +1 -0
  93. package/dist/verdict.js +84 -0
  94. package/dist/verdict.js.map +1 -0
  95. package/package.json +56 -0
package/README.md ADDED
@@ -0,0 +1,40 @@
1
+ # ai-model-verifier
2
+
3
+ **Is that API endpoint really serving the model it claims?**
4
+
5
+ Resellers and relay stations sell "Claude Opus" or "GPT-5" and quietly serve
6
+ something cheaper. Given a `base_url`, `api_key` and `model`, this library
7
+ probes the endpoint and reports what it actually found: a substituted model, a
8
+ downgraded Claude tier, a stripped thinking signature, inflated token billing,
9
+ or a translation layer sitting in front of the real vendor.
10
+
11
+ Extracted from [unorouter](https://unorouter.com)'s model tester so the same
12
+ engine runs in the browser, on a server, and inside our sync pipeline.
13
+
14
+ ## Design
15
+
16
+ - **Isomorphic.** Web-standard APIs only (`fetch`, `TextEncoder`,
17
+ `AbortSignal`); no `node:` imports. The HTTP call itself is injected as a
18
+ `TransportFn`, so a browser can route around CORS and a server can route
19
+ through an SSRF-safe fetch.
20
+ - **Opt-in.** Providers and detectors have their own entry points, so a consumer
21
+ that only checks Anthropic never bundles the Gemini config.
22
+ - **Zero required dependencies.**
23
+
24
+ ## Use
25
+
26
+ ```ts
27
+ import { runVerification } from "ai-model-verifier";
28
+ import { anthropicConfig } from "ai-model-verifier/providers/anthropic";
29
+ ```
30
+
31
+ ## What it detects
32
+
33
+ `coding-tool` refusals (Kiro/Amazon Q wearing a Claude badge), scam pages,
34
+ CJK language leaks from a substituted Chinese model, response-mixing proxies,
35
+ foreign vendor identity, Claude tier substitution (opus billed, sonnet served),
36
+ and envelope-level model substitution.
37
+
38
+ ## License
39
+
40
+ AGPL-3.0-only.
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Passive response metadata: what the envelope reveals about who really
3
+ * answered, read from responses the run already made.
4
+ *
5
+ * Costs nothing (no extra requests) and asserts nothing. Everything here is
6
+ * reported as observation, because each signal has an honest explanation as
7
+ * well as a dishonest one:
8
+ *
9
+ * - a `chatcmpl-` id inside an Anthropic-shaped reply means a translation
10
+ * layer, which is what our own gateway is;
11
+ * - OpenAI-style `usage` keys in a Messages reply mean the same;
12
+ * - a missing id means a sloppy relay, not a fake model.
13
+ *
14
+ * The value is in the aggregate: over many lanes, "which of my Claude lanes
15
+ * answer with a chatcmpl id" is a question worth being able to ask.
16
+ */
17
+ /** Vendor whose response shape the envelope actually looks like. */
18
+ export type EnvelopeShape = "anthropic" | "openai" | "gemini" | "unknown";
19
+ export type ResponseMetadata = {
20
+ /** Shape the payload matches, regardless of which API was called. */
21
+ shape: EnvelopeShape;
22
+ /** Shape the endpoint was asked to speak. */
23
+ expected: EnvelopeShape;
24
+ /** True when a translating hop sits between us and the model. */
25
+ translated: boolean;
26
+ /** e.g. "msg_", "chatcmpl-", "resp_"; null when the id is absent or odd. */
27
+ idPrefix: string | null;
28
+ /** Usage field names present, sorted. A cross-vendor mix is the tell. */
29
+ usageKeys: string[];
30
+ /** Foreign-vendor residue in the usage object, e.g. claude_* under OpenAI. */
31
+ foreignUsageKeys: string[];
32
+ /** Facts worth surfacing, in plain words. No verdict attached. */
33
+ notes: string[];
34
+ };
35
+ export declare function readResponseMetadata(data: unknown, expected: EnvelopeShape): ResponseMetadata;
36
+ //# sourceMappingURL=response-metadata.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"response-metadata.d.ts","sourceRoot":"","sources":["../../src/detectors/response-metadata.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,oEAAoE;AACpE,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1E,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qEAAqE;IACrE,KAAK,EAAE,aAAa,CAAC;IACrB,6CAA6C;IAC7C,QAAQ,EAAE,aAAa,CAAC;IACxB,iEAAiE;IACjE,UAAU,EAAE,OAAO,CAAC;IACpB,4EAA4E;IAC5E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,yEAAyE;IACzE,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,8EAA8E;IAC9E,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,kEAAkE;IAClE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAuCF,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,aAAa,GACtB,gBAAgB,CAyClB"}
@@ -0,0 +1,91 @@
1
+ /**
2
+ * Passive response metadata: what the envelope reveals about who really
3
+ * answered, read from responses the run already made.
4
+ *
5
+ * Costs nothing (no extra requests) and asserts nothing. Everything here is
6
+ * reported as observation, because each signal has an honest explanation as
7
+ * well as a dishonest one:
8
+ *
9
+ * - a `chatcmpl-` id inside an Anthropic-shaped reply means a translation
10
+ * layer, which is what our own gateway is;
11
+ * - OpenAI-style `usage` keys in a Messages reply mean the same;
12
+ * - a missing id means a sloppy relay, not a fake model.
13
+ *
14
+ * The value is in the aggregate: over many lanes, "which of my Claude lanes
15
+ * answer with a chatcmpl id" is a question worth being able to ask.
16
+ */
17
+ const ANTHROPIC_USAGE = new Set([
18
+ "input_tokens",
19
+ "output_tokens",
20
+ "cache_read_input_tokens",
21
+ "cache_creation_input_tokens",
22
+ ]);
23
+ const OPENAI_USAGE = new Set([
24
+ "prompt_tokens",
25
+ "completion_tokens",
26
+ "total_tokens",
27
+ ]);
28
+ const GEMINI_USAGE = new Set([
29
+ "promptTokenCount",
30
+ "candidatesTokenCount",
31
+ "totalTokenCount",
32
+ "thoughtsTokenCount",
33
+ ]);
34
+ function idPrefixOf(id) {
35
+ if (typeof id !== "string" || id.length === 0)
36
+ return null;
37
+ for (const p of ["msg_", "chatcmpl-", "resp_", "call_", "toolu_"])
38
+ if (id.startsWith(p))
39
+ return p;
40
+ // An unrecognised shape (a bare UUID, say) is itself worth reporting.
41
+ const cut = id.search(/[-_]/);
42
+ return cut > 0 ? id.slice(0, cut + 1) : "(none)";
43
+ }
44
+ function shapeOf(data, usageKeys) {
45
+ if (data["candidates"] !== undefined || usageKeys.some((k) => GEMINI_USAGE.has(k)))
46
+ return "gemini";
47
+ if (data["choices"] !== undefined || usageKeys.some((k) => OPENAI_USAGE.has(k)))
48
+ return "openai";
49
+ if (data["content"] !== undefined || usageKeys.some((k) => ANTHROPIC_USAGE.has(k)))
50
+ return "anthropic";
51
+ return "unknown";
52
+ }
53
+ export function readResponseMetadata(data, expected) {
54
+ const obj = (data ?? {});
55
+ const usage = (obj["usage"] ?? {});
56
+ const usageKeys = Object.keys(usage).sort();
57
+ const shape = shapeOf(obj, usageKeys);
58
+ const idPrefix = idPrefixOf(obj["id"]);
59
+ const notes = [];
60
+ // Usage keys belonging to a vendor other than the shape we are reading.
61
+ const foreignUsageKeys = usageKeys.filter((k) => {
62
+ if (k.startsWith("claude_"))
63
+ return shape !== "anthropic";
64
+ if (k.startsWith("gemini_"))
65
+ return shape !== "gemini";
66
+ if (ANTHROPIC_USAGE.has(k))
67
+ return shape !== "anthropic";
68
+ if (OPENAI_USAGE.has(k))
69
+ return shape !== "openai";
70
+ if (GEMINI_USAGE.has(k))
71
+ return shape !== "gemini";
72
+ return false;
73
+ });
74
+ const translated = shape !== "unknown" && shape !== expected;
75
+ if (translated)
76
+ notes.push(`answered in ${shape} shape though ${expected} was requested: a translating hop sits in front`);
77
+ if (expected === "anthropic" && idPrefix && idPrefix !== "msg_")
78
+ notes.push(`id starts "${idPrefix}" where Anthropic uses "msg_": the id was minted by something else`);
79
+ if (foreignUsageKeys.length > 0)
80
+ notes.push(`usage carries ${foreignUsageKeys.join(", ")}, which belong to another vendor's API`);
81
+ return {
82
+ shape,
83
+ expected,
84
+ translated,
85
+ idPrefix,
86
+ usageKeys,
87
+ foreignUsageKeys,
88
+ notes,
89
+ };
90
+ }
91
+ //# sourceMappingURL=response-metadata.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"response-metadata.js","sourceRoot":"","sources":["../../src/detectors/response-metadata.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAsBH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,cAAc;IACd,eAAe;IACf,yBAAyB;IACzB,6BAA6B;CAC9B,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,eAAe;IACf,mBAAmB;IACnB,cAAc;CACf,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,kBAAkB;IAClB,sBAAsB;IACtB,iBAAiB;IACjB,oBAAoB;CACrB,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,EAAW;IAC7B,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3D,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC;QAC/D,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACjC,sEAAsE;IACtE,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACnD,CAAC;AAED,SAAS,OAAO,CAAC,IAA6B,EAAE,SAAmB;IACjE,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChF,OAAO,QAAiB,CAAC;IAC3B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,OAAO,QAAiB,CAAC;IAC3B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChF,OAAO,WAAoB,CAAC;IAC9B,OAAO,SAAkB,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAa,EACb,QAAuB;IAEvB,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC9D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,wEAAwE;IACxE,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,KAAK,WAAW,CAAC;QAC1D,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACvD,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,WAAW,CAAC;QACzD,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACnD,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,KAAK,QAAQ,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,QAAQ,CAAC;IAC7D,IAAI,UAAU;QACZ,KAAK,CAAC,IAAI,CACR,eAAe,KAAK,iBAAiB,QAAQ,iDAAiD,CAC/F,CAAC;IACJ,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,IAAI,QAAQ,KAAK,MAAM;QAC7D,KAAK,CAAC,IAAI,CACR,cAAc,QAAQ,oEAAoE,CAC3F,CAAC;IACJ,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR,iBAAiB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC,CACrF,CAAC;IAEJ,OAAO;QACL,KAAK;QACL,QAAQ;QACR,UAAU;QACV,QAAQ;QACR,SAAS;QACT,gBAAgB;QAChB,KAAK;KACN,CAAC;AACJ,CAAC"}
@@ -0,0 +1,70 @@
1
+ /**
2
+ * Anthropic thinking-signature check.
3
+ *
4
+ * Claude attaches a server-generated `signature` to every thinking block. A
5
+ * relay serving some other model cannot produce one, so this is the only
6
+ * evidence here that does not rest on asking the model who it is: identity and
7
+ * behaviour probes can be coached with a system prompt, a signature cannot.
8
+ *
9
+ * What it does NOT prove: which tier answered. Sonnet returns a perfectly valid
10
+ * signature when sold as Opus, so a pass here still needs the tier checks.
11
+ *
12
+ * Ported from veridrop (AGPL-3.0), whose detector stops at "a signature-shaped
13
+ * string is present".
14
+ *
15
+ * Their DESIGN.md also specifies a replay step (hand the thinking block back so
16
+ * Anthropic re-validates the signature, rejecting a forgery with 400/422) which
17
+ * they never implemented. We built and measured it: **it does not work through a
18
+ * relay**. Tested against two live upstreams (a6 merchant 3542 and pol) by
19
+ * replaying a genuine block and then the same block with 308 bytes of random
20
+ * base64 in place of the signature. Both upstreams returned 200 for both. A
21
+ * relay re-issues the turn to its own backend rather than passing our thinking
22
+ * block through, so nothing ever validates it.
23
+ *
24
+ * `strict` therefore stays off by default and is documented as proving only
25
+ * that the endpoint accepts the block, not that the signature is real. Do not
26
+ * present it as cryptographic proof.
27
+ */
28
+ import type { TransportFn } from "../transport";
29
+ export type SignatureState =
30
+ /** Signed thinking block: a genuine Anthropic path answered. */
31
+ "signed"
32
+ /**
33
+ * Thought, but no signature. Expected when the lane crosses an
34
+ * OpenAI-shaped hop, which has no field to carry one, so this is
35
+ * unprovable rather than fake. Never a hard fail on its own.
36
+ */
37
+ | "unsigned"
38
+ /** Thinking was requested and no block came back at all. */
39
+ | "no-thinking"
40
+ /** Model or endpoint cannot be asked; carries no evidence either way. */
41
+ | "skipped";
42
+ export type SignatureResult = {
43
+ state: SignatureState;
44
+ signatureLength: number;
45
+ /** First 32 chars, for the report; never the whole signature. */
46
+ signaturePrefix: string | null;
47
+ thinkingChars: number;
48
+ /** Only set when the replay check actually ran. */
49
+ replayVerified?: boolean;
50
+ reason?: string;
51
+ };
52
+ export declare function thinkingModeFor(model: string): {
53
+ kind: "adaptive" | "extended";
54
+ } | null;
55
+ export declare function checkThinkingSignature(opts: {
56
+ transport: TransportFn;
57
+ baseUrl: string;
58
+ apiKey: string;
59
+ model: string;
60
+ timeoutMs: number;
61
+ /**
62
+ * Replay the thinking block to the same endpoint. Measured as NOT proving
63
+ * authenticity through a relay (see the module docstring): both a genuine and
64
+ * a forged signature come back 200, because the relay re-issues the turn
65
+ * upstream instead of forwarding our block. Off by default; it only tells you
66
+ * the endpoint accepts a thinking block in the request.
67
+ */
68
+ strict?: boolean;
69
+ }): Promise<SignatureResult>;
70
+ //# sourceMappingURL=thinking-signature.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"thinking-signature.d.ts","sourceRoot":"","sources":["../../src/detectors/thinking-signature.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAqBhD,MAAM,MAAM,cAAc;AACxB,gEAAgE;AAC9D,QAAQ;AACV;;;;GAIG;GACD,UAAU;AACZ,4DAA4D;GAC1D,aAAa;AACf,yEAAyE;GACvE,SAAS,CAAC;AAEd,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,cAAc,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAkBF,wBAAgB,eAAe,CAC7B,KAAK,EAAE,MAAM,GACZ;IAAE,IAAI,EAAE,UAAU,GAAG,UAAU,CAAA;CAAE,GAAG,IAAI,CAK1C;AA4BD,wBAAsB,sBAAsB,CAAC,IAAI,EAAE;IACjD,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,GAAG,OAAO,CAAC,eAAe,CAAC,CA0E3B"}
@@ -0,0 +1,177 @@
1
+ /**
2
+ * Anthropic thinking-signature check.
3
+ *
4
+ * Claude attaches a server-generated `signature` to every thinking block. A
5
+ * relay serving some other model cannot produce one, so this is the only
6
+ * evidence here that does not rest on asking the model who it is: identity and
7
+ * behaviour probes can be coached with a system prompt, a signature cannot.
8
+ *
9
+ * What it does NOT prove: which tier answered. Sonnet returns a perfectly valid
10
+ * signature when sold as Opus, so a pass here still needs the tier checks.
11
+ *
12
+ * Ported from veridrop (AGPL-3.0), whose detector stops at "a signature-shaped
13
+ * string is present".
14
+ *
15
+ * Their DESIGN.md also specifies a replay step (hand the thinking block back so
16
+ * Anthropic re-validates the signature, rejecting a forgery with 400/422) which
17
+ * they never implemented. We built and measured it: **it does not work through a
18
+ * relay**. Tested against two live upstreams (a6 merchant 3542 and pol) by
19
+ * replaying a genuine block and then the same block with 308 bytes of random
20
+ * base64 in place of the signature. Both upstreams returned 200 for both. A
21
+ * relay re-issues the turn to its own backend rather than passing our thinking
22
+ * block through, so nothing ever validates it.
23
+ *
24
+ * `strict` therefore stays off by default and is documented as proving only
25
+ * that the endpoint accepts the block, not that the signature is real. Do not
26
+ * present it as cryptographic proof.
27
+ */
28
+ /**
29
+ * Multi-step GCD: hard enough that adaptive models decide to think, and
30
+ * deliberately not the 1071/462 pair from Anthropic's own docs, which a relay
31
+ * could special-case.
32
+ */
33
+ const PROBE_PROMPT = "Find the greatest common divisor of 2378 and 1547 using the Euclidean algorithm.";
34
+ const THINKING_BUDGET_TOKENS = 2000;
35
+ /**
36
+ * Covers thinking + answer together. Too small and an adaptive model skips
37
+ * thinking entirely to fit, which reads as a fake when it is our own fault.
38
+ */
39
+ const MAX_TOKENS = 16000;
40
+ /** Real signatures run to hundreds of chars; this only rejects a stub. */
41
+ const SIGNATURE_MIN_LEN = 50;
42
+ /** Adaptive-thinking models take `effort` at the top level, not inside `thinking`. */
43
+ const ADAPTIVE_MODELS = ["claude-opus-4-7", "claude-opus-4-8"];
44
+ /** Extended-thinking models take a token budget instead. */
45
+ const EXTENDED_MODELS = [
46
+ "claude-opus-4-6",
47
+ "claude-opus-4-5",
48
+ "claude-opus-4-1",
49
+ "claude-sonnet-4-6",
50
+ "claude-sonnet-4-5",
51
+ "claude-haiku-4-5",
52
+ ];
53
+ const normalize = (model) => model.toLowerCase().replace(/[._]/g, "-").split("/").pop() ?? "";
54
+ export function thinkingModeFor(model) {
55
+ const m = normalize(model);
56
+ if (ADAPTIVE_MODELS.some((p) => m.startsWith(p)))
57
+ return { kind: "adaptive" };
58
+ if (EXTENDED_MODELS.some((p) => m.startsWith(p)))
59
+ return { kind: "extended" };
60
+ return null;
61
+ }
62
+ function readThinking(data) {
63
+ const content = data?.content;
64
+ if (!Array.isArray(content))
65
+ return { block: null, signature: "", chars: 0 };
66
+ for (const raw of content) {
67
+ if (!raw || typeof raw !== "object")
68
+ continue;
69
+ const b = raw;
70
+ if (b.type !== "thinking" && b.type !== "redacted_thinking")
71
+ continue;
72
+ return {
73
+ block: b,
74
+ signature: typeof b.signature === "string" ? b.signature : "",
75
+ chars: typeof b.thinking === "string" ? b.thinking.length : 0,
76
+ };
77
+ }
78
+ return { block: null, signature: "", chars: 0 };
79
+ }
80
+ export async function checkThinkingSignature(opts) {
81
+ const mode = thinkingModeFor(opts.model);
82
+ const empty = { signatureLength: 0, signaturePrefix: null, thinkingChars: 0 };
83
+ if (!mode)
84
+ return { state: "skipped", ...empty, reason: "model-has-no-thinking-mode" };
85
+ const url = `${opts.baseUrl.replace(/\/+$/, "")}/v1/messages`;
86
+ const headers = {
87
+ "Content-Type": "application/json",
88
+ "x-api-key": opts.apiKey,
89
+ "anthropic-version": "2023-06-01",
90
+ };
91
+ const messages = [{ role: "user", content: PROBE_PROMPT }];
92
+ // Non-streaming on purpose: streaming an adaptive model silently drops the
93
+ // thinking block, so a real Claude would look unsigned.
94
+ const body = {
95
+ model: opts.model,
96
+ max_tokens: MAX_TOKENS,
97
+ messages,
98
+ thinking: mode.kind === "extended"
99
+ ? { type: "enabled", budget_tokens: THINKING_BUDGET_TOKENS }
100
+ : { type: "adaptive", display: "summarized" },
101
+ };
102
+ // Sibling of `thinking`, never nested inside it: nesting is a 400.
103
+ if (mode.kind === "adaptive")
104
+ body["output_config"] = { effort: "xhigh" };
105
+ const res = await opts.transport({
106
+ mode: "direct",
107
+ url,
108
+ headers,
109
+ reqBody: body,
110
+ timeoutMs: opts.timeoutMs,
111
+ });
112
+ if (res.status === null || res.status >= 400)
113
+ return {
114
+ state: "skipped",
115
+ ...empty,
116
+ reason: `probe-failed:${res.status ?? "network"}`,
117
+ };
118
+ const found = readThinking(res.data);
119
+ if (!found.block)
120
+ return { state: "no-thinking", ...empty, reason: "no-thinking-block" };
121
+ const sig = found.signature;
122
+ const base = {
123
+ signatureLength: sig.length,
124
+ signaturePrefix: sig ? sig.slice(0, 32) : null,
125
+ thinkingChars: found.chars,
126
+ };
127
+ if (!sig)
128
+ return { state: "unsigned", ...base, reason: "no-signature-field" };
129
+ if (sig.length < SIGNATURE_MIN_LEN)
130
+ return { state: "unsigned", ...base, reason: "signature-too-short" };
131
+ if (!opts.strict)
132
+ return { state: "signed", ...base };
133
+ const replayVerified = await verifySignature({
134
+ transport: opts.transport,
135
+ url,
136
+ headers,
137
+ model: opts.model,
138
+ block: found.block,
139
+ timeoutMs: opts.timeoutMs,
140
+ });
141
+ // A rejection is real evidence (the endpoint checked and refused); an
142
+ // acceptance is not, so it never upgrades or downgrades the state on its own.
143
+ return {
144
+ state: replayVerified ? "signed" : "unsigned",
145
+ ...base,
146
+ replayVerified,
147
+ ...(replayVerified ? {} : { reason: "signature-rejected-on-replay" }),
148
+ };
149
+ }
150
+ /**
151
+ * Hand the thinking block back to the endpoint. Against api.anthropic.com this
152
+ * would re-validate the signature; against a relay it does not (measured: a
153
+ * forged 308-byte signature returned 200 on both upstreams tested), so the
154
+ * result is reported as `replayVerified` evidence rather than a verdict.
155
+ */
156
+ async function verifySignature(opts) {
157
+ const res = await opts.transport({
158
+ mode: "direct",
159
+ url: opts.url,
160
+ headers: opts.headers,
161
+ reqBody: {
162
+ model: opts.model,
163
+ max_tokens: 1,
164
+ messages: [
165
+ { role: "user", content: PROBE_PROMPT },
166
+ { role: "assistant", content: [opts.block] },
167
+ ],
168
+ },
169
+ timeoutMs: opts.timeoutMs,
170
+ });
171
+ // A network failure proves nothing, so treat only an explicit rejection as
172
+ // a forged signature.
173
+ if (res.status === null)
174
+ return true;
175
+ return res.status < 400;
176
+ }
177
+ //# sourceMappingURL=thinking-signature.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"thinking-signature.js","sourceRoot":"","sources":["../../src/detectors/thinking-signature.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAIH;;;;GAIG;AACH,MAAM,YAAY,GAChB,kFAAkF,CAAC;AAErF,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC;;;GAGG;AACH,MAAM,UAAU,GAAG,KAAK,CAAC;AAEzB,0EAA0E;AAC1E,MAAM,iBAAiB,GAAG,EAAE,CAAC;AA2B7B,sFAAsF;AACtF,MAAM,eAAe,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;AAE/D,4DAA4D;AAC5D,MAAM,eAAe,GAAG;IACtB,iBAAiB;IACjB,iBAAiB;IACjB,iBAAiB;IACjB,mBAAmB;IACnB,mBAAmB;IACnB,kBAAkB;CACnB,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAClC,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;AAEnE,MAAM,UAAU,eAAe,CAC7B,KAAa;IAEb,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC9E,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC9E,OAAO,IAAI,CAAC;AACd,CAAC;AAQD,SAAS,YAAY,CAAC,IAAa;IAKjC,MAAM,OAAO,GAAI,IAA8B,EAAE,OAAO,CAAC;IACzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC7E,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,SAAS;QAC9C,MAAM,CAAC,GAAG,GAAoB,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,mBAAmB;YAAE,SAAS;QACtE,OAAO;YACL,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;YAC7D,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9D,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAc5C;IACC,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;IAC9E,IAAI,CAAC,IAAI;QACP,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IAE9E,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC;IAC9D,MAAM,OAAO,GAAG;QACd,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,mBAAmB,EAAE,YAAY;KAClC,CAAC;IACF,MAAM,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;IAC3D,2EAA2E;IAC3E,wDAAwD;IACxD,MAAM,IAAI,GAA4B;QACpC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,UAAU,EAAE,UAAU;QACtB,QAAQ;QACR,QAAQ,EACN,IAAI,CAAC,IAAI,KAAK,UAAU;YACtB,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,sBAAsB,EAAE;YAC5D,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE;KAClD,CAAC;IACF,mEAAmE;IACnE,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU;QAAE,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAE1E,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG;QACH,OAAO;QACP,OAAO,EAAE,IAAI;QACb,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG;QAC1C,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,GAAG,KAAK;YACR,MAAM,EAAE,gBAAgB,GAAG,CAAC,MAAM,IAAI,SAAS,EAAE;SAClD,CAAC;IAEJ,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK,CAAC,KAAK;QACd,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAEzE,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC;IAC5B,MAAM,IAAI,GAAG;QACX,eAAe,EAAE,GAAG,CAAC,MAAM;QAC3B,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;QAC9C,aAAa,EAAE,KAAK,CAAC,KAAK;KAC3B,CAAC;IACF,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC9E,IAAI,GAAG,CAAC,MAAM,GAAG,iBAAiB;QAChC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;IAEvE,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;IAEtD,MAAM,cAAc,GAAG,MAAM,eAAe,CAAC;QAC3C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG;QACH,OAAO;QACP,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,sEAAsE;IACtE,8EAA8E;IAC9E,OAAO;QACL,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU;QAC7C,GAAG,IAAI;QACP,cAAc;QACd,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;KACtE,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,IAO9B;IACC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,OAAO,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE;gBACvC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;aAC7C;SACF;QACD,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,2EAA2E;IAC3E,sBAAsB;IACtB,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;AAC1B,CAAC"}
@@ -0,0 +1,51 @@
1
+ /**
2
+ * Throughput metadata: tokens per second, derived from data the run already
3
+ * collected.
4
+ *
5
+ * Motivation: bigger models are slower per token, so a lane billed as Opus but
6
+ * served by Sonnet tends to run fast. That makes throughput the one tier signal
7
+ * that survives a relay refusing to name its model.
8
+ *
9
+ * It is metadata, not a verdict, and deliberately has no thresholds. Rate
10
+ * varies with load, batching, quantisation and time of day, so a single
11
+ * measurement cannot separate "cheaper model" from "quiet hour". The useful
12
+ * comparison is across lanes serving the SAME model, which only the caller can
13
+ * assemble: `compareThroughput` does that part and still only describes what it
14
+ * sees.
15
+ */
16
+ export type ThroughputSample = {
17
+ /** Identifies the lane in the caller's own terms. */
18
+ lane: string;
19
+ model: string;
20
+ outputTokens: number;
21
+ elapsedMs: number;
22
+ tokensPerSecond: number;
23
+ };
24
+ export declare function sampleThroughput(args: {
25
+ lane: string;
26
+ model: string;
27
+ outputTokens: number | null | undefined;
28
+ elapsedMs: number;
29
+ }): ThroughputSample | null;
30
+ export type ThroughputComparison = {
31
+ model: string;
32
+ sampleCount: number;
33
+ medianTps: number;
34
+ /** Lanes running far from the median for this model, fastest first. */
35
+ outliers: {
36
+ lane: string;
37
+ tokensPerSecond: number;
38
+ ratioToMedian: number;
39
+ direction: "faster" | "slower";
40
+ }[];
41
+ note: string;
42
+ };
43
+ /**
44
+ * Describe how lanes serving one model compare. `outlierRatio` is how far from
45
+ * the median a lane must sit to be worth mentioning; the default of 2 is a
46
+ * starting point for eyeballing, not a calibrated threshold.
47
+ */
48
+ export declare function compareThroughput(samples: ThroughputSample[], opts?: {
49
+ outlierRatio?: number;
50
+ }): ThroughputComparison[];
51
+ //# sourceMappingURL=throughput.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"throughput.d.ts","sourceRoot":"","sources":["../../src/detectors/throughput.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,gBAAgB,GAAG,IAAI,CAa1B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,QAAQ,GAAG,QAAQ,CAAC;KAChC,EAAE,CAAC;IACJ,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAQF;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,IAAI,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/B,oBAAoB,EAAE,CA8CxB"}
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Throughput metadata: tokens per second, derived from data the run already
3
+ * collected.
4
+ *
5
+ * Motivation: bigger models are slower per token, so a lane billed as Opus but
6
+ * served by Sonnet tends to run fast. That makes throughput the one tier signal
7
+ * that survives a relay refusing to name its model.
8
+ *
9
+ * It is metadata, not a verdict, and deliberately has no thresholds. Rate
10
+ * varies with load, batching, quantisation and time of day, so a single
11
+ * measurement cannot separate "cheaper model" from "quiet hour". The useful
12
+ * comparison is across lanes serving the SAME model, which only the caller can
13
+ * assemble: `compareThroughput` does that part and still only describes what it
14
+ * sees.
15
+ */
16
+ export function sampleThroughput(args) {
17
+ const tokens = args.outputTokens;
18
+ // A handful of tokens over a noisy connection says nothing; the rate is
19
+ // dominated by connection setup rather than generation.
20
+ if (typeof tokens !== "number" || tokens < 20)
21
+ return null;
22
+ if (!Number.isFinite(args.elapsedMs) || args.elapsedMs <= 0)
23
+ return null;
24
+ return {
25
+ lane: args.lane,
26
+ model: args.model,
27
+ outputTokens: tokens,
28
+ elapsedMs: args.elapsedMs,
29
+ tokensPerSecond: +(tokens / (args.elapsedMs / 1000)).toFixed(1),
30
+ };
31
+ }
32
+ const median = (xs) => {
33
+ const s = [...xs].sort((a, b) => a - b);
34
+ const mid = Math.floor(s.length / 2);
35
+ return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
36
+ };
37
+ /**
38
+ * Describe how lanes serving one model compare. `outlierRatio` is how far from
39
+ * the median a lane must sit to be worth mentioning; the default of 2 is a
40
+ * starting point for eyeballing, not a calibrated threshold.
41
+ */
42
+ export function compareThroughput(samples, opts) {
43
+ const ratio = opts?.outlierRatio ?? 2;
44
+ const byModel = new Map();
45
+ for (const s of samples) {
46
+ const list = byModel.get(s.model);
47
+ if (list)
48
+ list.push(s);
49
+ else
50
+ byModel.set(s.model, [s]);
51
+ }
52
+ const out = [];
53
+ for (const [model, group] of byModel) {
54
+ // Below three lanes there is no distribution to speak of, so report the
55
+ // median and no outliers rather than inventing a comparison.
56
+ const med = median(group.map((s) => s.tokensPerSecond));
57
+ const outliers = group.length >= 3
58
+ ? group
59
+ .filter((s) => s.tokensPerSecond >= med * ratio ||
60
+ s.tokensPerSecond <= med / ratio)
61
+ .map((s) => ({
62
+ lane: s.lane,
63
+ tokensPerSecond: s.tokensPerSecond,
64
+ ratioToMedian: +(s.tokensPerSecond / med).toFixed(2),
65
+ direction: s.tokensPerSecond > med ? "faster" : "slower",
66
+ }))
67
+ .sort((a, b) => b.tokensPerSecond - a.tokensPerSecond)
68
+ : [];
69
+ out.push({
70
+ model,
71
+ sampleCount: group.length,
72
+ medianTps: +med.toFixed(1),
73
+ outliers,
74
+ note: group.length < 3
75
+ ? "too few lanes to compare"
76
+ : outliers.length === 0
77
+ ? "all lanes within range of each other"
78
+ : `${outliers.length} lane(s) far from the median; a much faster lane can mean a smaller model, but load and quantisation move this too`,
79
+ });
80
+ }
81
+ return out;
82
+ }
83
+ //# sourceMappingURL=throughput.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"throughput.js","sourceRoot":"","sources":["../../src/detectors/throughput.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAWH,MAAM,UAAU,gBAAgB,CAAC,IAKhC;IACC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;IACjC,wEAAwE;IACxE,wDAAwD;IACxD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACzE,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,YAAY,EAAE,MAAM;QACpB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,eAAe,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;KAChE,CAAC;AACJ,CAAC;AAgBD,MAAM,MAAM,GAAG,CAAC,EAAY,EAAE,EAAE;IAC9B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,CAAC,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAC;AAC9D,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAA2B,EAC3B,IAAgC;IAEhC,MAAM,KAAK,GAAG,IAAI,EAAE,YAAY,IAAI,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YAClB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,wEAAwE;QACxE,6DAA6D;QAC7D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACxD,MAAM,QAAQ,GACZ,KAAK,CAAC,MAAM,IAAI,CAAC;YACf,CAAC,CAAC,KAAK;iBACF,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,eAAe,IAAI,GAAG,GAAG,KAAK;gBAChC,CAAC,CAAC,eAAe,IAAI,GAAG,GAAG,KAAK,CACnC;iBACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,eAAe,EAAE,CAAC,CAAC,eAAe;gBAClC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;gBACpD,SAAS,EACP,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,CAAE,QAAkB,CAAC,CAAC,CAAE,QAAkB;aACtE,CAAC,CAAC;iBACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC;YAC1D,CAAC,CAAC,EAAE,CAAC;QAET,GAAG,CAAC,IAAI,CAAC;YACP,KAAK;YACL,WAAW,EAAE,KAAK,CAAC,MAAM;YACzB,SAAS,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1B,QAAQ;YACR,IAAI,EACF,KAAK,CAAC,MAAM,GAAG,CAAC;gBACd,CAAC,CAAC,0BAA0B;gBAC5B,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,sCAAsC;oBACxC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,oHAAoH;SAC/I,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Token-accounting checks for Anthropic endpoints.
3
+ *
4
+ * The signature check answers "is this really Claude". These answer a different
5
+ * question the signature cannot: "are the numbers you are billing me true".
6
+ * A relay can forward genuine Claude and still inflate `input_tokens` on every
7
+ * call, and nothing in an identity probe would notice.
8
+ *
9
+ * Ported from veridrop (AGPL-3.0) `token_usage.py` / `integrity.py`, whose
10
+ * tolerances were calibrated against the official API. Do not tighten them
11
+ * without re-measuring: the loose-looking bounds exist because real Anthropic
12
+ * responses sat near them.
13
+ */
14
+ import type { TransportFn } from "../transport";
15
+ export type TokenTruthCheck = {
16
+ id: "usage-present" | "input-delta" | "output-bounded" | "count-tokens";
17
+ pass: boolean;
18
+ detail: string;
19
+ };
20
+ export type TokenTruthResult = {
21
+ /** null when nothing could be measured (endpoint refused every probe). */
22
+ ok: boolean | null;
23
+ checks: TokenTruthCheck[];
24
+ shortInputTokens: number | null;
25
+ longInputTokens: number | null;
26
+ countTokens: number | null;
27
+ reason?: string;
28
+ };
29
+ export declare function expectedInputDelta(model: string): {
30
+ min: number;
31
+ max: number;
32
+ };
33
+ export declare function checkTokenTruth(opts: {
34
+ transport: TransportFn;
35
+ baseUrl: string;
36
+ apiKey: string;
37
+ model: string;
38
+ timeoutMs: number;
39
+ }): Promise<TokenTruthResult>;
40
+ //# sourceMappingURL=token-truth.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"token-truth.d.ts","sourceRoot":"","sources":["../../src/detectors/token-truth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAsBhD,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,eAAe,GAAG,aAAa,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACxE,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,0EAA0E;IAC1E,EAAE,EAAE,OAAO,GAAG,IAAI,CAAC;IACnB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAKF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAK9E;AAcD,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA2G5B"}