@pi-in-go/pigpen-pi-typesafe 0.1.0

package/port/oracle/tests/calibrate.test.ts

@@ -0,0 +1,144 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import {
+  auc, calibrate, defaultThresholds, formatCalibration, metricsAt, pickThreshold, replay, samplesOf, sweep,
+} from "../src/calibrate.js";
+import { TypeSafeIntegrationError } from "../src/errors.js";
+
+const sample = (score: number, label: boolean, id?: string) => ({ score, label, ...(id === undefined ? {} : { id }) });
+
+/** Eight observations: two clear positives, two clear negatives, and a muddy middle. */
+const muddy = [
+  sample(0.95, true, "p1"), sample(0.8, true, "p2"), sample(0.6, true, "p3"), sample(0.4, true, "p4"),
+  sample(0.7, false, "n1"), sample(0.5, false, "n2"), sample(0.2, false, "n3"), sample(0.05, false, "n4"),
+];
+
+test("AUC is the rank-based probability that a positive outranks a negative", () => {
+  assert.equal(auc([sample(1, true), sample(0, false)]), 1);
+  assert.equal(auc([sample(0, true), sample(1, false)]), 0);
+  assert.equal(auc([sample(0.5, true), sample(0.5, false)]), 0.5);
+  assert.equal(auc([sample(0.5, true), sample(0.5, true), sample(0.5, false), sample(0.5, false)]), 0.5);
+  // One class only: nothing to rank against.
+  assert.equal(auc([sample(0.5, true), sample(0.9, true)]), undefined);
+  assert.equal(auc([]), undefined);
+  // Three positives above one negative and one below: 3 wins of 4 pairs.
+  assert.equal(auc([sample(0.9, true), sample(0.8, true), sample(0.7, true), sample(0.6, false), sample(0.1, true)]) ?? 0, 0.75);
+});
+
+test("a threshold row counts every case once", () => {
+  const row = metricsAt(muddy, 0.6);
+  assert.deepEqual({ ...row, precision: row.precision, recall: row.recall }, { threshold: 0.6, flagged: 4, tp: 3, fp: 1, fn: 1, tn: 3, precision: 0.75, recall: 0.75, flagRate: 0.5 });
+  // At the top of the range nothing is flagged, so precision has no denominator.
+  const strict = metricsAt(muddy, 1);
+  assert.equal(strict.flagged, 0);
+  assert.equal(strict.precision, undefined);
+  assert.equal(strict.recall, 0);
+  assert.deepEqual(sweep(muddy, [0.6]).map(entry => entry.tp), [3]);
+});
+
+test("the default threshold grid is every distinct score, ascending", () => {
+  assert.deepEqual(defaultThresholds(muddy), [0.05, 0.2, 0.4, 0.5, 0.6, 0.7, 0.8, 0.95]);
+  assert.equal(defaultThresholds(muddy, 3).length, 3);
+  assert.deepEqual(defaultThresholds([sample(0.5, true), sample(0.5, false)]), [0.5]);
+});
+
+test("a recommendation honours the floors, and the report names what it misses and flags", () => {
+  const calibration = calibrate("muddy", muddy, { thresholds: [0.5, 0.6, 0.7, 0.8], minPrecision: 0.75, minRecall: 0.5 });
+  assert.equal(calibration.recommended?.threshold, 0.6);
+  assert.equal(calibration.scored, 8);
+  assert.equal(calibration.positives, 4);
+  assert.equal(calibration.negatives, 4);
+  assert.equal(calibration.errors, 0);
+  assert.ok((calibration.auc ?? 0) > 0.7);
+  assert.deepEqual(calibration.missed.map(entry => entry.id), ["p4"]);
+  assert.deepEqual(calibration.flagged.map(entry => entry.id), ["n1"]);
+  const text = formatCalibration(calibration);
+  assert.ok(text.includes("muddy: 8 scored, 4 positives, 4 negatives"));
+  assert.ok(text.includes("recommended 0.60: precision 75%, recall 75%"));
+  assert.ok(text.includes("missed positives (1): p4"));
+  assert.ok(text.includes("flagged negatives (1): n1"));
+
+  // Floors nothing can meet leave no recommendation, and the report says so.
+  const impossible = calibrate("strict", muddy, { thresholds: [0.5], minPrecision: 0.99, minRecall: 0.99 });
+  assert.equal(impossible.recommended, undefined);
+  assert.ok(formatCalibration(impossible).includes("recommended: none"));
+});
+
+test("with no floors, the recommendation is the best F1 among the rows", () => {
+  const rows = sweep(muddy, [0.4, 0.5, 0.6, 0.7]);
+  // At 0.4 every positive is caught for two false alarms: F1 0.8 beats the tighter rows.
+  assert.deepEqual(rows.map(row => row.tp), [4, 3, 3, 2]);
+  assert.equal(pickThreshold(rows)?.threshold, 0.4);
+  assert.equal(pickThreshold([]), undefined);
+  // No positives: precision has no denominator, so no row can be ranked.
+  assert.equal(pickThreshold(sweep([sample(0.5, false)], [0.5])), undefined);
+});
+
+test("replay scores labelled cases with bounded concurrency and keeps per-case order", async () => {
+  let inFlight = 0;
+  let peak = 0;
+  const cases = [
+    { id: "a", label: true, data: 1 },
+    { id: "b", label: false, data: 2 },
+    { id: "c", label: true, data: 3 },
+    { id: "d", label: false, data: 4 },
+  ];
+  const results = await replay(cases, async data => {
+    inFlight++;
+    peak = Math.max(peak, inFlight);
+    await new Promise(resolve => setTimeout(resolve, 5));
+    inFlight--;
+    return data;
+  }, { concurrency: 2 });
+  assert.equal(peak, 2);
+  assert.deepEqual(results.map(result => result.id), ["a", "b", "c", "d"]);
+  assert.deepEqual(results.map(result => result.score), [1, 2, 3, 4]);
+  assert.equal(results.every(result => !result.skipped && result.error === undefined), true);
+  const { samples, errors } = samplesOf(results);
+  assert.equal(errors, 0);
+  assert.equal(samples.length, 4);
+});
+
+test("a budget failure stops the replay and leaves the rest unsubmitted", async () => {
+  const started: string[] = [];
+  const cases = [
+    { id: "a", label: true, data: 1 },
+    { id: "b", label: false, data: 2 },
+    { id: "c", label: true, data: 3 },
+    { id: "d", label: false, data: 4 },
+  ];
+  const results = await replay(cases, async data => {
+    started.push(String(data));
+    if (data === 2) throw new TypeSafeIntegrationError("budget", "TypeSafe request limit reached (1 attempts per client instance).");
+    return 0.1;
+  }, { concurrency: 1, stopOn: error => error instanceof TypeSafeIntegrationError && error.code === "budget" });
+  assert.deepEqual(started, ["1", "2"]);
+  assert.equal(results[1]?.error, "TypeSafe request limit reached (1 attempts per client instance).");
+  assert.equal(results[1]?.skipped, false);
+  // The budget stop kept every later case from being submitted at all.
+  assert.equal(results[2]?.skipped, true);
+  assert.equal(results[3]?.skipped, true);
+  assert.equal(results[3]?.score, undefined);
+  const { samples, errors } = samplesOf(results);
+  assert.equal(errors, 3);
+  assert.deepEqual(samples, [{ label: true, score: 0.1, id: "a" }]);
+  // A replay that lost most of its cases still yields honest numbers.
+  const calibration = calibrate("replay", samples, { errors });
+  assert.equal(calibration.errors, 3);
+  assert.equal(calibration.scored, 1);
+  assert.equal(calibration.auc, undefined);
+});
+
+test("replay reports scorer failures with the caller's own message", async () => {
+  const thrown = new Error("upstream body with a key: sk-secret");
+  // The default is the scorer's own message, which the caller controls; a supplied describer replaces it.
+  const plain = await replay([{ id: "boom", label: true, data: "case" }], async () => { throw thrown; });
+  assert.equal(plain[0]?.error, "upstream body with a key: sk-secret");
+  assert.equal(plain[0]?.score, undefined);
+  const described = await replay([{ id: "boom", label: true, data: "case" }], async () => { throw thrown; }, {
+    describeError: () => "The scorer failed.",
+  });
+  assert.equal(described[0]?.error, "The scorer failed.");
+  const nonError = await replay([{ id: "boom", label: false, data: "case" }], async () => { throw "not an error"; });
+  assert.equal(nonError[0]?.error, "The scorer failed.");
+});

package/port/oracle/tests/client.test.ts

@@ -0,0 +1,499 @@
+import assert from "node:assert/strict";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, before, test } from "node:test";
+import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, authState, clearAuthState, TypeSafeIntegrationError } from "../src/index.js";
+import { safeError } from "../src/errors.js";
+import { APIError } from "@typesafe-ai/sdk";
+import type { Questions, SystemOneRequest } from "../src/index.js";
+
+export function responseFor(questions: Questions): Response {
+  const answers = Object.fromEntries(Object.entries(questions).map(([id, q]) => {
+    if (q.type === "noul") return [id, { type: "noul", noul: 0.9 }];
+    if (q.type === "choice") {
+      const keys = Object.keys(q.criteria);
+      return [id, { type: "choice", choice: keys[0], confidence: 1, probabilities: Object.fromEntries(keys.map((key, i) => [key, i === 0 ? 1 : 0])) }];
+    }
+    return [id, { type: "score", score: 0, confidence: 1, probabilities: Object.fromEntries(q.criteria.map((_, i) => [i, i === 0 ? 1 : 0])), legend: Object.fromEntries(q.criteria.map((level, i) => [i, level])) }];
+  }));
+  return Response.json({ model: "jev-test", answers, usage: { input_tokens: 42, output_tokens: 0 } });
+}
+const sample = () => ({ state: "synthetic", questions: { yes: noul("Is this synthetic?") } });
+const savedAgentDir = process.env.PI_CODING_AGENT_DIR;
+const savedKey = process.env.TYPESAFE_API_KEY;
+before(() => {
+  // Keep the developer's real stored key and environment out of these tests.
+  process.env.PI_CODING_AGENT_DIR = mkdtempSync(join(tmpdir(), "pi-typesafe-client-"));
+  delete process.env.TYPESAFE_API_KEY;
+});
+after(() => {
+  if (savedAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; else process.env.PI_CODING_AGENT_DIR = savedAgentDir;
+  if (savedKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = savedKey;
+});
+const hasCode = (code: string) => (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === code;
+
+test("official SDK helpers, typed answers, metadata, and one batched network call", async () => {
+  let calls = 0;
+  const questions = {
+    category: choice("Which?", { billing: "Charges", other: null }),
+    urgent: noul("Urgent?"),
+    frustration: score("Frustration?", ["Calm", "Angry"]),
+  };
+  const client = createTypeSafe({ apiKey: "test-key", fetch: async (url, init) => {
+    calls++;
+    assert.equal(url, "https://api.typesafe.ai/v1/systemone");
+    assert.equal(init?.method, "POST");
+    const request = JSON.parse(String(init?.body));
+    assert.deepEqual(request.questions, JSON.parse(JSON.stringify(questions)));
+    assert.equal(request.model, "jev-latest");
+    assert.ok(Object.values(Object.fromEntries(new Headers(init?.headers))).some(value => value.includes("test-key")));
+    return responseFor(questions);
+  } });
+  const result = await client.evaluate({ state: { message: "Example" }, questions });
+  const category: "billing" | "other" = result.answers.category.choice;
+  const yes: number = result.answers.urgent.noul;
+  const value: number = result.answers.frustration.score;
+  assert.equal(category, "billing");
+  assert.equal(yes, 0.9);
+  assert.equal(value, 0);
+  assert.equal(calls, 1);
+  assert.ok(result.elapsedMs >= 0);
+  assert.deepEqual(client.getUsage(), { requestsStarted: 1, requestsSucceeded: 1, requestsFailed: 0, inputTokens: 42, outputTokens: 0, estimatedUsd: 0.000002 });
+  const spend = client.getSpend();
+  assert.equal(spend.session.requestsStarted, 1);
+  assert.equal(spend.today.inputTokens, 42);
+  assert.equal(spend.blocked, undefined);
+});
+
+test("structured state, descriptions, and null values are supported", () => {
+  assert.doesNotThrow(() => parseEvaluationRequest({ state: null, questions: {
+    yes: noul({ goal: "Check" }, { true: ["a"], false: null }),
+    pick: choice(["Choose"], { a: { detail: "a" }, b: null }),
+    rating: score(null, [null, { level: "high" }]),
+  } }));
+});
+
+test("invalid questions are rejected before network submission without echoing state", async () => {
+  const secret = "private-user-content";
+  let calls = 0;
+  const client = createTypeSafe({ apiKey: "test-key", fetch: async () => { calls++; throw new Error(secret); } });
+  const invalid = [
+    { state: secret, questions: {} },
+    { state: secret, questions: { q: { type: "score", criteria: ["one"] } } },
+    { state: secret, questions: { q: { type: "choice", criteria: {} } } },
+    { state: secret, questions: { q: { type: "unsupported" } } },
+    { state: secret, questions: { q: noul("yes") }, apiKey: secret },
+    { state: secret, questions: Object.fromEntries(Array.from({ length: 33 }, (_, i) => [`q${i}`, noul("yes")])) },
+  ];
+  for (const request of invalid) {
+    await assert.rejects(() => Reflect.apply(client.evaluate, client, [request]), error => {
+      assert.ok(error instanceof TypeSafeIntegrationError);
+      assert.equal(error.code, "validation");
+      assert.equal(error.message.includes(secret), false);
+      return true;
+    });
+  }
+  assert.equal(calls, 0);
+  assert.equal(client.getUsage().requestsStarted, 0);
+});
+
+test("validation errors name the offending path, never the submitted value", () => {
+  const secret = "private-user-content";
+  const cases: Array<[unknown, RegExp]> = [
+    [{ state: secret, questions: { q: { type: "choice", instructions: secret, criteria: ["a", 1] } } }, /questions\.q/],
+    [{ state: secret, questions: { q: { type: "score", criteria: [secret] } } }, /questions\.q/],
+    [{ state: secret, questions: {} }, /questions/],
+    [{ state: secret }, /questions|request/],
+    [{ state: secret, questions: { q: noul("ok") }, extra: secret }, /extra/],
+  ];
+  for (const [request, path] of cases) {
+    assert.throws(() => parseEvaluationRequest(request), (error: unknown) => {
+      assert.ok(error instanceof TypeSafeIntegrationError);
+      assert.match(error.message, path);
+      assert.match(error.message, /Expected \{ state, questions/);
+      assert.equal(error.message.includes(secret), false);
+      return true;
+    });
+  }
+});
+
+test("normalization accepts common model near-misses without loosening the schema", () => {
+  const normalized = normalizeEvaluationRequest({
+    state: "s",
+    questions: {
+      team: { type: "choice", instructions: "Which team?", options: ["frontend", "backend"] },
+      level: { type: "score", instructions: "How bad?", levels: ["fine", "bad"] },
+      flag: { type: "noul", instructions: "Is it?", criteria: "Yes when stated" },
+      keep: choice("Already valid", { a: "A", b: null }),
+    },
+  });
+  const request = parseEvaluationRequest(normalized);
+  assert.deepEqual(request.questions.team, { type: "choice", instructions: "Which team?", criteria: { frontend: null, backend: null } });
+  assert.deepEqual(request.questions.level, { type: "score", instructions: "How bad?", criteria: ["fine", "bad"] });
+  assert.deepEqual(request.questions.flag, { type: "noul", instructions: "Is it?", criteria: { true: "Yes when stated" } });
+  assert.deepEqual(request.questions.keep, JSON.parse(JSON.stringify(choice("Already valid", { a: "A", b: null }))));
+  for (const value of [null, "text", [], { questions: [] }, { questions: { q: "text" } }]) {
+    assert.deepEqual(normalizeEvaluationRequest(value), value);
+  }
+  assert.throws(() => parseEvaluationRequest(normalizeEvaluationRequest({ state: "s", questions: { q: { type: "choice", options: ["a", 2] } } })), (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === "validation");
+});
+
+test("non-JSON state, getters, cycles, and excessive nesting are rejected", () => {
+  const cycle: Record<string, unknown> = {};
+  cycle.self = cycle;
+  let nested: unknown = null;
+  for (let i = 0; i < 70; i++) nested = { nested };
+  const invalid = [cycle, { n: NaN }, { n: Infinity }, { n: 1n }, { fn: () => null }, new Date(), nested,
+    Object.defineProperty({}, "secret", { enumerable: true, get() { throw new Error("must not read"); } })];
+  for (const state of invalid) assert.throws(() => parseEvaluationRequest({ state, questions: sample().questions }), hasCode("validation"));
+});
+
+test("UTF-8 byte limit applies before submission", async () => {
+  const client = createTypeSafe({ apiKey: "test-key", maxInputBytes: 180, fetch: async () => { throw new Error("must not run"); } });
+  await assert.rejects(client.evaluate({ ...sample(), state: "🙂".repeat(50) }), hasCode("validation"));
+  assert.equal(client.getUsage().requestsStarted, 0);
+});
+
+test("request budget is shared across concurrent calls and failures", async () => {
+  let calls = 0;
+  const client = createTypeSafe({ apiKey: "test-key", maxRequests: 2, fetch: async () => {
+    calls++;
+    return Response.json({ error: "do not display this upstream body" }, { status: 500 });
+  } });
+  const results = await Promise.allSettled([client.evaluate(sample()), client.evaluate(sample()), client.evaluate(sample())]);
+  assert.equal(calls, 2);
+  assert.equal(results.filter(result => result.status === "rejected").length, 3);
+  await assert.rejects(client.evaluate(sample()), hasCode("budget"));
+  assert.equal(client.getUsage().requestsStarted, 2);
+  assert.equal(client.getUsage().requestsSucceeded, 0);
+});
+
+test("HTTP errors are classified, never retried, and do not expose response secrets", async () => {
+  for (const status of [400, 401, 403, 422, 429, 500, 503]) {
+    let calls = 0;
+    const client = createTypeSafe({ apiKey: "test-key", fetch: async () => {
+      calls++;
+      return Response.json({ secret: "never-print-me" }, { status });
+    } });
+    await assert.rejects(client.evaluate(sample()), error => {
+      assert.ok(error instanceof TypeSafeIntegrationError);
+      assert.equal(error.code, "http");
+      assert.equal(error.status, status);
+      assert.equal(JSON.stringify(error).includes("never-print-me"), false);
+      assert.equal(String(error).includes("never-print-me"), false);
+      return true;
+    });
+    assert.equal(calls, 1);
+  }
+});
+
+test("HTTP advice names the backend's key, covers 402, and quotes a numeric Retry-After", async () => {
+  const savedTypesafeKey = process.env.TYPESAFE_API_KEY;
+  const savedOpenRouterKey = process.env.OPENROUTER_API_KEY;
+  // authState({ backend }).usable needs a key present for each backend; the client itself takes its key below.
+  process.env.TYPESAFE_API_KEY = "offline-env-key-0123456789abcdef";
+  process.env.OPENROUTER_API_KEY = "offline-or-key-0123456789abcdef";
+  const cases = [
+    { backend: "typesafe", status: 401, retryAfter: true, usable: false, message: "TypeSafe returned HTTP 401. Check TYPESAFE_API_KEY. No automatic retry was made." },
+    { backend: "openrouter", status: 401, retryAfter: true, usable: false, message: "TypeSafe returned HTTP 401. Check OPENROUTER_API_KEY. No automatic retry was made." },
+    { backend: "typesafe", status: 402, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 402. Check your account balance. No automatic retry was made." },
+    { backend: "openrouter", status: 402, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 402. Insufficient credits. Add credits at https://openrouter.ai/credits. No automatic retry was made." },
+    { backend: "typesafe", status: 429, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. Retry after 7 seconds. No automatic retry was made." },
+    { backend: "openrouter", status: 429, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. Retry after 7 seconds. No automatic retry was made." },
+    { backend: "typesafe", status: 429, retryAfter: false, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. No automatic retry was made." },
+  ] as const;
+  try {
+    for (const c of cases) {
+      clearAuthState();
+      const client = createTypeSafe({
+        apiKey: "test-key",
+        backend: c.backend,
+        fetch: async () => Response.json({ error: { code: "x", message: "never-print-me" } }, { status: c.status, headers: c.retryAfter ? { "Retry-After": "7" } : {} }),
+      });
+      await assert.rejects(client.evaluate(sample()), error => {
+        assert.ok(error instanceof TypeSafeIntegrationError);
+        assert.equal(error.code, "http");
+        assert.equal(error.status, c.status);
+        assert.equal(error.message, c.message);
+        assert.equal(error.message.includes("never-print-me"), false);
+        return true;
+      });
+      // A 401 rejects the key; a 402 is billing, so it must leave it usable (REJECTED_STATUSES stays {401, 403}).
+      assert.equal(authState({ backend: c.backend }).usable, c.usable, `${c.backend} ${c.status}`);
+    }
+  } finally {
+    clearAuthState();
+    if (savedTypesafeKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = savedTypesafeKey;
+    if (savedOpenRouterKey === undefined) delete process.env.OPENROUTER_API_KEY; else process.env.OPENROUTER_API_KEY = savedOpenRouterKey;
+  }
+});
+
+test("safeError keeps its one-argument form and defaults the key advice", () => {
+  const own = new TypeSafeIntegrationError("http", "kept", 401);
+  assert.equal(safeError(own), own);
+  assert.equal(safeError(new APIError(401, undefined, new Headers())).message, "TypeSafe returned HTTP 401. Check TYPESAFE_API_KEY. No automatic retry was made.");
+});
+
+test("cancellation before submission does not consume an attempt", async () => {
+  const client = createTypeSafe({ apiKey: "test-key", fetch: async () => { throw new Error("must not run"); } });
+  await assert.rejects(client.evaluate(sample(), { signal: AbortSignal.abort() }), hasCode("aborted"));
+  assert.equal(client.getUsage().requestsStarted, 0);
+});
+
+test("in-flight cancellation and timeout reach the transport", async () => {
+  const pending = async (_url: string, init?: RequestInit): Promise<Response> => new Promise((_resolve, reject) => {
+    init?.signal?.addEventListener("abort", () => reject(new DOMException("Cancelled", "AbortError")), { once: true });
+  });
+  const controller = new AbortController();
+  const client = createTypeSafe({ apiKey: "test-key", fetch: pending, timeoutMs: 1000 });
+  const request = client.evaluate(sample(), { signal: controller.signal });
+  controller.abort();
+  await assert.rejects(request, hasCode("aborted"));
+  const timed = createTypeSafe({ apiKey: "test-key", fetch: pending, timeoutMs: 10 });
+  await assert.rejects(timed.evaluate(sample()), hasCode("timeout"));
+});
+
+test("malformed successful responses fail safely", async () => {
+  for (const response of [Response.json({ secret: "private" }), new Response("not-json"), Response.json({ model: "test", usage: { input_tokens: 1, output_tokens: 0 }, answers: { yes: { type: "noul", noul: 5 } } })]) {
+    const client = createTypeSafe({ apiKey: "test-key", fetch: async () => response });
+    await assert.rejects(client.evaluate(sample()), hasCode("response"));
+    assert.equal(client.getUsage().requestsSucceeded, 0);
+  }
+});
+
+test("client ignores SDK endpoint and logging environment overrides", async () => {
+  const originalUrl = process.env.TYPESAFE_BASE_URL;
+  const originalLog = process.env.TYPESAFE_LOG_LEVEL;
+  process.env.TYPESAFE_BASE_URL = "https://untrusted.invalid";
+  process.env.TYPESAFE_LOG_LEVEL = "debug";
+  try {
+    const client = createTypeSafe({ apiKey: "test-key", fetch: async (url) => {
+      assert.ok(url.startsWith("https://api.typesafe.ai/"));
+      return responseFor(sample().questions);
+    } });
+    await client.evaluate(sample());
+  } finally {
+    if (originalUrl === undefined) delete process.env.TYPESAFE_BASE_URL; else process.env.TYPESAFE_BASE_URL = originalUrl;
+    if (originalLog === undefined) delete process.env.TYPESAFE_LOG_LEVEL; else process.env.TYPESAFE_LOG_LEVEL = originalLog;
+  }
+});
+
+test("configuration errors are early and usage snapshots are detached", async () => {
+  assert.throws(() => createTypeSafe({ apiKey: " " }), hasCode("configuration"));
+  for (const options of [{ timeoutMs: 0 }, { maxRequests: -1 }, { maxInputBytes: NaN }, { model: "" }, { model: "x".repeat(101) }]) {
+    assert.throws(() => createTypeSafe({ apiKey: "test-key", ...options }), hasCode("configuration"));
+  }
+  const client = createTypeSafe({ apiKey: "test-key", fetch: async () => responseFor(sample().questions) });
+  const snapshot = client.getUsage();
+  await client.evaluate(sample());
+  assert.equal(snapshot.requestsStarted, 0);
+  assert.equal(client.getUsage().requestsStarted, 1);
+});
+
+test("request data is snapshotted before asynchronous work", async () => {
+  let sent: SystemOneRequest | undefined;
+  const client = createTypeSafe({ apiKey: "test-key", fetch: async (_url, init) => {
+    sent = JSON.parse(String(init?.body));
+    return responseFor(sample().questions);
+  } });
+  const request = sample();
+  const result = client.evaluate(request);
+  request.state = "changed";
+  await result;
+  assert.equal(sent?.state, "synthetic");
+});
+
+test("evaluate admits the same near-miss aliases as the agent tool", async () => {
+  let sentCriteria: unknown;
+  const client = createTypeSafe({ apiKey: "test-key", fetch: async (_url, init) => {
+    const body = JSON.parse(String(init?.body)) as { questions: Record<string, { criteria?: unknown }> };
+    sentCriteria = body.questions.yes?.criteria;
+    return responseFor(sample().questions);
+  } });
+  const nearMiss = { state: "synthetic", questions: { yes: { type: "noul", instructions: "Is this synthetic?", criteria: "Is this synthetic data?" } } };
+  const result = await client.evaluate(nearMiss as unknown as SystemOneRequest);
+  assert.equal((result.answers.yes as { noul: number }).noul, 0.9);
+  assert.deepEqual(sentCriteria, { true: "Is this synthetic data?" });
+});
+
+test("unknown backend throws configuration error", () => {
+  assert.throws(() => createTypeSafe({ apiKey: "test-key", backend: "bogus" as never }), (error: unknown) => {
+    assert.ok(error instanceof TypeSafeIntegrationError);
+    assert.equal(error.code, "configuration");
+    assert.match(error.message, /Unknown judgment backend/);
+    assert.match(error.message, /bogus/);
+    return true;
+  });
+});
+
+test("known backend is called at its full request URL", async () => {
+  // A host does not identify the endpoint: openrouter.ai answers the SDK's own /v1/systemone with an HTML page and status 200.
+  const backends = [
+    ["typesafe", "https://api.typesafe.ai/v1/systemone"],
+    ["openrouter", "https://openrouter.ai/api/alpha/decisions"],
+  ] as const;
+  for (const [backend, expected] of backends) {
+    let capturedUrl = "";
+    const client = createTypeSafe({
+      apiKey: "test-key",
+      backend,
+      fetch: async (url) => { capturedUrl = String(url); return responseFor(sample().questions); },
+    });
+    await client.evaluate(sample());
+    assert.equal(capturedUrl, expected);
+  }
+});
+
+test("listModels asks each backend for its own model list", async () => {
+  const backends = [
+    ["typesafe", "https://api.typesafe.ai/v1/models", { models: [{ name: "jev-latest" }] }, ["jev-latest"]],
+    ["openrouter", "https://openrouter.ai/api/v1/models", { data: [{ id: "vendor/model", name: "Vendor: Model" }] }, ["vendor/model"]],
+  ] as const;
+  for (const [backend, expectedUrl, wire, expected] of backends) {
+    let capturedUrl = "";
+    const client = createTypeSafe({
+      apiKey: "test-key",
+      backend,
+      fetch: async (url) => { capturedUrl = String(url); return Response.json(wire); },
+    });
+    assert.deepEqual(await client.listModels(), expected);
+    assert.equal(capturedUrl, expectedUrl);
+  }
+});
+
+test("a model list without the backend's declared field keeps the SDK's own shape error", async () => {
+  const client = createTypeSafe({
+    apiKey: "test-key",
+    backend: "openrouter",
+    fetch: async () => Response.json({ items: [{ name: "not the declared field" }] }),
+  });
+  await assert.rejects(client.listModels(), (error: unknown) => {
+    assert.ok(error instanceof TypeSafeIntegrationError);
+    assert.equal(error.code, "response");
+    return true;
+  });
+});
+
+test("the model list is renamed only for the backend that declares another field", async () => {
+  const client = createTypeSafe({
+    apiKey: "test-key",
+    backend: "typesafe",
+    fetch: async () => Response.json({ data: [{ name: "not the SDK's field" }] }),
+  });
+  await assert.rejects(client.listModels(), (error: unknown) => {
+    assert.ok(error instanceof TypeSafeIntegrationError);
+    assert.equal(error.code, "response");
+    return true;
+  });
+});
+
+test("a public model list leaves the auth state unverified", async () => {
+  clearAuthState();
+  const client = createTypeSafe({
+    apiKey: "garbage-key",
+    backend: "openrouter",
+    fetch: async () => Response.json({ data: [{ id: "vendor/model", name: "Vendor: Model" }] }),
+  });
+  assert.deepEqual(await client.listModels(), ["vendor/model"]);
+  // openrouter.ai serves this list to anyone, so a success says nothing about the key.
+  assert.equal(authState({ backend: "openrouter" }).verified, false);
+});
+
+test("the model in the request body is the backend's own id form", async () => {
+  const cases = [
+    ["openrouter", undefined, "typesafe/jev-1.13"],
+    ["openrouter", "jev-latest", "~typesafe/jev-latest"],
+    ["openrouter", "jev-1.13", "typesafe/jev-1.13"],
+    ["openrouter", "jev-1.13.0", "typesafe/jev-1.13"],
+    ["openrouter", "vendor/other", "vendor/other"],
+    ["typesafe", "jev-latest", "jev-latest"],
+  ] as const;
+  for (const [backend, requested, expected] of cases) {
+    let sentUrl = "";
+    let sentModel: string | undefined;
+    const client = createTypeSafe({
+      apiKey: "test-key",
+      backend,
+      ...(requested === undefined ? {} : { model: requested }),
+      fetch: async (url, init) => {
+        sentUrl = String(url);
+        const body = JSON.parse(String(init?.body)) as { model?: string };
+        sentModel = body.model;
+        return responseFor(sample().questions);
+      },
+    });
+    await client.evaluate(sample());
+    assert.equal(sentModel, expected);
+    // OpenRouter judgments travel to its own decisions path, so the mapped id is proven on the wire that uses it.
+    assert.equal(sentUrl, backend === "openrouter" ? "https://openrouter.ai/api/alpha/decisions" : "https://api.typesafe.ai/v1/systemone");
+  }
+});
+
+test("a per-request model gets the same mapping as the client default", async () => {
+  const cases = [
+    ["openrouter", "jev-latest", "~typesafe/jev-latest"],
+    ["openrouter", "jev-1.13", "typesafe/jev-1.13"],
+    ["openrouter", "vendor/other", "vendor/other"],
+    ["typesafe", "jev-latest", "jev-latest"],
+  ] as const;
+  for (const [backend, requested, expected] of cases) {
+    let sentModel: string | undefined;
+    const client = createTypeSafe({
+      apiKey: "test-key",
+      backend,
+      fetch: async (_url, init) => {
+        const body = JSON.parse(String(init?.body)) as { model?: string };
+        sentModel = body.model;
+        return responseFor(sample().questions);
+      },
+    });
+    await client.evaluate({ ...sample(), model: requested });
+    assert.equal(sentModel, expected);
+  }
+});
+
+test("a TypeSafe key is never sent to another backend", () => {
+  const originalKey = process.env.TYPESAFE_API_KEY;
+  const originalOR = process.env.OPENROUTER_API_KEY;
+  process.env.TYPESAFE_API_KEY = "ts-test-key-1234567890123456";
+  delete process.env.OPENROUTER_API_KEY;
+  try {
+    assert.throws(() => createTypeSafe({ backend: "openrouter" }), (error: unknown) => {
+      assert.ok(error instanceof TypeSafeIntegrationError);
+      assert.equal(error.code, "configuration");
+      assert.match(error.message, /OPENROUTER_API_KEY/);
+      assert.doesNotMatch(error.message, /typesafe login/);
+      return true;
+    });
+  } finally {
+    if (originalKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = originalKey;
+    if (originalOR === undefined) delete process.env.OPENROUTER_API_KEY; else process.env.OPENROUTER_API_KEY = originalOR;
+  }
+});
+
+test("key resolution picks the right env var per backend", () => {
+  // With no key set, openrouter backend should complain about OPENROUTER_API_KEY.
+  const originalKey = process.env.TYPESAFE_API_KEY;
+  const originalOR = process.env.OPENROUTER_API_KEY;
+  delete process.env.TYPESAFE_API_KEY;
+  delete process.env.OPENROUTER_API_KEY;
+  try {
+    assert.throws(() => createTypeSafe({ backend: "openrouter" }), (error: unknown) => {
+      assert.ok(error instanceof TypeSafeIntegrationError);
+      assert.match(error.message, /OPENROUTER_API_KEY/);
+      return true;
+    });
+    // Setting the env var for the right backend should get past key resolution.
+    process.env.OPENROUTER_API_KEY = "or-test-key-1234567890123456";
+    let called = false;
+    createTypeSafe({
+      backend: "openrouter",
+      fetch: async () => { called = true; return responseFor(sample().questions); },
+    });
+    assert.equal(called, false); // no evaluate yet, just construction
+  } finally {
+    if (originalKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = originalKey;
+    if (originalOR === undefined) delete process.env.OPENROUTER_API_KEY; else process.env.OPENROUTER_API_KEY = originalOR;
+  }
+});