ai-model-verifier 1.0.0

package/dist/verdict.d.ts

@@ -0,0 +1,26 @@
+import type { ResponseMetadata } from "./detectors/response-metadata";
+import type { ProviderConfig } from "./providers/config";
+import type { ProbeOutcome, ProbeSignal, VerifyVerdict } from "./types";
+/**
+ * A probe's outcome plus the working values the runner needs but never stores:
+ * the reply text, and the envelope facts read while the raw payload was still
+ * in scope.
+ */
+export type ProbeEval = ProbeOutcome & {
+    text: string | undefined;
+    envelope?: ResponseMetadata;
+};
+export type VerdictResult = {
+    verdict: VerifyVerdict;
+    reasons: string[];
+    versionUnverifiable: boolean;
+};
+export declare function aggregateVerdict(args: {
+    model: string;
+    cfg: ProviderConfig;
+    results: ProbeEval[];
+    /** Model id echoed by the endpoint, when it reported one. */
+    detectedModel?: string | null;
+}): VerdictResult;
+export declare function probeReason(pass: boolean, signal: ProbeSignal, muxFailure: boolean, transient: boolean): string;
+//# sourceMappingURL=verdict.d.ts.map

package/dist/verdict.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../src/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAGtE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAEV,YAAY,EACZ,WAAW,EACX,aAAa,EACd,MAAM,SAAS,CAAC;AAEjB;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG,YAAY,GAAG;IACrC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,aAAa,CAAC;IACvB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,mBAAmB,EAAE,OAAO,CAAC;CAC9B,CAAC;AAWF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,cAAc,CAAC;IACpB,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,6DAA6D;IAC7D,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,GAAG,aAAa,CAwEhB;AAUD,wBAAgB,WAAW,CACzB,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,OAAO,EACnB,SAAS,EAAE,OAAO,GACjB,MAAM,CAUR"}

package/dist/verdict.js

@@ -0,0 +1,84 @@
+import { detectTierMismatch } from "./signals";
+import { detectSubstitution } from "./substitution";
+const labelsWith = (results, sig) => results
+    .filter((r) => r.signal === sig)
+    .map((r) => r.label)
+    .join(", ");
+const firstSignal = (results, sig) => results.some((r) => r.signal === sig) ? labelsWith(results, sig) : null;
+export function aggregateVerdict(args) {
+    const results = args.results;
+    const reasons = [];
+    const codingTool = firstSignal(results, "coding-tool");
+    if (codingTool)
+        return suspicious(`coding-tool-refusal: ${codingTool}`);
+    const scam = firstSignal(results, "scam");
+    if (scam)
+        return suspicious(`scam-page: ${scam}`);
+    const cjk = firstSignal(results, "cjk-leak");
+    if (cjk)
+        return suspicious(`cjk-language-leak: ${cjk}`);
+    const muxLabels = results.filter((r) => r.muxFailure).map((r) => r.label);
+    if (muxLabels.length >= 2)
+        return suspicious(`unsafe-proxy: response-mixing on ${muxLabels.join(", ")}`);
+    const foreignOnIdentity = results.some((r) => r.signal === "foreign" &&
+        (r.label === "model-name" || r.label === "identity"));
+    if (foreignOnIdentity)
+        return suspicious(`foreign-identity: ${labelsWith(results, "foreign")}`);
+    if (args.cfg.tiers) {
+        const modelNameText = results.find((r) => r.label === "model-name")?.text;
+        const served = detectTierMismatch(args.model, modelNameText, args.cfg.tiers);
+        if (served)
+            return suspicious(`tier-mismatch: requested ${args.model}, served ${served}`);
+    }
+    // Envelope check, free: the endpoint told us which model answered, and it is
+    // not the one asked for. Runs after tier-mismatch because that names the
+    // cheaper tier specifically, which reads better than a raw id diff.
+    const substituted = detectSubstitution(args.model, args.detectedModel);
+    if (substituted)
+        return suspicious(`substituted: requested ${args.model}, served ${substituted}`);
+    const passed = results.filter((r) => r.pass).length;
+    const failed = results.filter((r) => !r.pass);
+    const passing = passed >= 3;
+    if (passing)
+        return {
+            verdict: "genuine",
+            reasons,
+            versionUnverifiable: true,
+        };
+    const transientFails = failed.filter((r) => r.transient).length;
+    const nonTransientFails = failed.filter((r) => !r.transient && r.signal !== null).length;
+    if (transientFails > 0 && nonTransientFails === 0)
+        return {
+            verdict: "unverified",
+            reasons: [`transient: ${failed.map((r) => r.label).join(", ")}`],
+            versionUnverifiable: false,
+        };
+    return suspicious(`failed: ${failed.map((r) => r.label).join(", ")}`);
+}
+function suspicious(reason) {
+    return {
+        verdict: "suspicious",
+        reasons: [reason],
+        versionUnverifiable: false,
+    };
+}
+export function probeReason(pass, signal, muxFailure, transient) {
+    if (signal === "coding-tool")
+        return "coding-tool wrapper detected";
+    if (signal === "scam")
+        return "scam page detected";
+    if (signal === "cjk-leak")
+        return "CJK leaked into an English reply";
+    if (signal === "foreign")
+        return "foreign vendor named";
+    if (signal === "cloud-host")
+        return "cloud host named (accepted)";
+    if (signal === "blank")
+        return "blank response";
+    if (muxFailure)
+        return "response mixing (nonce mismatch)";
+    if (transient)
+        return "transient upstream error";
+    return pass ? "passed" : "failed quality check";
+}
+//# sourceMappingURL=verdict.js.map

package/dist/verdict.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../src/verdict.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAyBpD,MAAM,UAAU,GAAG,CAAC,OAAoB,EAAE,GAAgB,EAAE,EAAE,CAC5D,OAAO;KACJ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC;KAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;KACnB,IAAI,CAAC,IAAI,CAAC,CAAC;AAEhB,MAAM,WAAW,GAAG,CAAC,OAAoB,EAAE,GAAgB,EAAE,EAAE,CAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAE1E,MAAM,UAAU,gBAAgB,CAAC,IAMhC;IACC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IACvD,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1C,IAAI,IAAI;QAAE,OAAO,UAAU,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAC7C,IAAI,GAAG;QAAE,OAAO,UAAU,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;IAExD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1E,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC;QACvB,OAAO,UAAU,CACf,oCAAoC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEJ,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CACpC,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,MAAM,KAAK,SAAS;QACtB,CAAC,CAAC,CAAC,KAAK,KAAK,YAAY,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU,CAAC,CACvD,CAAC;IACF,IAAI,iBAAiB;QACnB,OAAO,UAAU,CAAC,qBAAqB,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAE3E,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,aAAa,GAAG,OAAO,CAAC,IAAI,CAChC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAM,YAAkC,CACvD,EAAE,IAAI,CAAC;QACR,MAAM,MAAM,GAAG,kBAAkB,CAC/B,IAAI,CAAC,KAAK,EACV,aAAa,EACb,IAAI,CAAC,GAAG,CAAC,KAAK,CACf,CAAC;QACF,IAAI,MAAM;YACR,OAAO,UAAU,CACf,4BAA4B,IAAI,CAAC,KAAK,YAAY,MAAM,EAAE,CAC3D,CAAC;IACN,CAAC;IAED,6EAA6E;IAC7E,yEAAyE;IACzE,oEAAoE;IACpE,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;IACvE,IAAI,WAAW;QACb,OAAO,UAAU,CACf,0BAA0B,IAAI,CAAC,KAAK,YAAY,WAAW,EAAE,CAC9D,CAAC;IAEJ,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,CAAC;IAE5B,IAAI,OAAO;QACT,OAAO;YACL,OAAO,EAAE,SAAS;YAClB,OAAO;YACP,mBAAmB,EAAE,IAAI;SAC1B,CAAC;IAEJ,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CACzC,CAAC,MAAM,CAAC;IACT,IAAI,cAAc,GAAG,CAAC,IAAI,iBAAiB,KAAK,CAAC;QAC/C,OAAO;YACL,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,CAAC,cAAc,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChE,mBAAmB,EAAE,KAAK;SAC3B,CAAC;IAEJ,OAAO,UAAU,CAAC,WAAW,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,UAAU,CAAC,MAAc;IAChC,OAAO;QACL,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,mBAAmB,EAAE,KAAK;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,IAAa,EACb,MAAmB,EACnB,UAAmB,EACnB,SAAkB;IAElB,IAAI,MAAM,KAAK,aAAa;QAAE,OAAO,8BAA8B,CAAC;IACpE,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,oBAAoB,CAAC;IACnD,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,kCAAkC,CAAC;IACrE,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,sBAAsB,CAAC;IACxD,IAAI,MAAM,KAAK,YAAY;QAAE,OAAO,6BAA6B,CAAC;IAClE,IAAI,MAAM,KAAK,OAAO;QAAE,OAAO,gBAAgB,CAAC;IAChD,IAAI,UAAU;QAAE,OAAO,kCAAkC,CAAC;IAC1D,IAAI,SAAS;QAAE,OAAO,0BAA0B,CAAC;IACjD,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC;AAClD,CAAC"}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "ai-model-verifier",
+  "version": "1.0.0",
+  "description": "Verify an AI API endpoint really serves the model it claims: model substitution, Claude thinking signatures, token-billing accuracy, response-envelope and throughput evidence.",
+  "license": "AGPL-3.0-only",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/unorouter/ai-model-verifier.git"
+  },
+  "keywords": [
+    "ai",
+    "llm",
+    "relay",
+    "proxy",
+    "verify",
+    "authenticity",
+    "model-verification",
+    "fake-model",
+    "anthropic",
+    "claude",
+    "openai",
+    "gemini",
+    "token-billing"
+  ],
+  "sideEffects": false,
+  "files": [
+    "dist"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./providers/*": {
+      "types": "./dist/providers/*.d.ts",
+      "default": "./dist/providers/*.js"
+    },
+    "./*": {
+      "types": "./dist/*.d.ts",
+      "default": "./dist/*.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "bun run build"
+  },
+  "devDependencies": {
+    "typescript": "^7.0.2"
+  },
+  "homepage": "https://github.com/unorouter/ai-model-verifier",
+  "bugs": {
+    "url": "https://github.com/unorouter/ai-model-verifier/issues"
+  }
+}