@bytesbrains/cruise 0.0.0-stage → 0.12.1

package/dist/conformance-NPFDYCQL.mjs

@@ -0,0 +1,922 @@
+#!/usr/bin/env node
+// Built by packages/cli/build.ts.
+import {
+  listFlag,
+  planShard,
+  selectModels
+} from "./chunk-WS6NFRIU.mjs";
+import {
+  usageFromJson
+} from "./chunk-ICJVJSTC.mjs";
+import "./chunk-LALZUKEC.mjs";
+import {
+  parseKeyPrefix
+} from "./chunk-5TVD3ZAI.mjs";
+import {
+  connect,
+  environmentOf,
+  parseArgs,
+  requireString,
+  run
+} from "./chunk-7NQHBXVZ.mjs";
+import "./chunk-JZPUTXM7.mjs";
+
+// src/conformance-tools.ts
+var TOOL_NAME = "get_weather";
+var isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+var firstChoice = (body) => {
+  const choices = isObject(body) && Array.isArray(body.choices) ? body.choices : [];
+  return isObject(choices[0]) ? choices[0] : void 0;
+};
+var terminator = (choice) => String(choice?.finish_reason ?? choice?.stop_reason ?? "<absent>");
+function callsOf(body) {
+  const calls = firstChoice(body)?.message?.tool_calls;
+  if (Array.isArray(calls) && calls.length > 0) {
+    return calls.map((call) => {
+      const fn = isObject(call) && isObject(call.function) ? call.function : {};
+      return { name: fn.name, args: fn.arguments };
+    });
+  }
+  const output = isObject(body) && Array.isArray(body.output) ? body.output : [];
+  return output.filter((item) => isObject(item) && item.type === "function_call").map((item) => ({ name: item.name, args: item.arguments }));
+}
+function malformed(call) {
+  if (call.name !== TOOL_NAME) return `Called '${String(call.name).slice(0, 80)}', which is not the offered tool '${TOOL_NAME}'.`;
+  let args = call.args;
+  try {
+    if (typeof args === "string") args = JSON.parse(args);
+  } catch {
+  }
+  if (!isObject(args)) return `Arguments are not a JSON object: ${String(call.args).slice(0, 80)}`;
+  return typeof args.city === "string" ? null : `Arguments omit the required string 'city': ${JSON.stringify(args).slice(0, 80)}`;
+}
+function checkToolCall(body, name = "tool_call") {
+  const choice = firstChoice(body);
+  const finish = isObject(body) && Array.isArray(body.output) && choice === void 0 ? "completed" : terminator(choice);
+  const calls = callsOf(body);
+  if (calls.length > 0) {
+    const problem = calls.map(malformed).find((p) => p !== null);
+    return problem === void 0 ? { check: name, outcome: "pass", observed: `${finish}, ${calls.length} call(s)` } : { check: name, outcome: "fail", observed: `${finish}, ${calls.length} call(s)`, detail: `Malformed tool call. ${problem}` };
+  }
+  const content = typeof choice?.message?.content === "string" ? choice.message.content : "";
+  const looksLikeProse = /"?(name|function|get_weather)"?\s*[:=]/i.test(content);
+  return {
+    check: name,
+    outcome: "fail",
+    observed: finish,
+    detail: looksLikeProse ? "Tool call emitted as text content, with no tool_calls array." : "No tool_calls array returned."
+  };
+}
+function checkParallelToolCalls(body) {
+  const calls = callsOf(body);
+  const problem = calls.map(malformed).find((p) => p !== null);
+  const count = calls.filter((call) => malformed(call) === null).length;
+  if (count > 1) return { check: "parallel_tool_calls", outcome: "pass", observed: `${count} call(s)` };
+  return {
+    check: "parallel_tool_calls",
+    outcome: "fail",
+    observed: `${count} call(s), ${terminator(firstChoice(body))}`,
+    detail: problem !== void 0 ? `Returned ${calls.length} call(s), of which ${count} could be run. ${problem}` : count === 1 ? "Returned a single tool call. Consistent with either an inability to parallelise or a choice not to, so this only records absence after every attempt agrees." : "Returned no tool calls at all, despite tool calling being supported."
+  };
+}
+
+// src/conformance-search.ts
+var WEB_SEARCH_CHECK = {
+  name: "web_search",
+  decides: "web_search",
+  attempts: 2,
+  request: {
+    messages: [
+      {
+        role: "user",
+        content: "Search the web: what is the latest Node.js LTS release, and when was it published? Answer in one sentence."
+      }
+    ],
+    // One search is the whole question, and `max_uses` bounds it where the
+    // host honours one. Perplexity does not (0101), and the conformance key
+    // is the one key served there anyway.
+    tools: [{ type: "web_search", max_uses: 1 }],
+    max_tokens: 512
+  }
+};
+function checkWebSearch(body) {
+  const searches = usageFromJson(body)?.web_search_requests;
+  if (searches === void 0) {
+    return { check: "web_search", outcome: "fail", observed: "<unreported>", detail: "No search count in usage: a search the ledger cannot see cannot be billed." };
+  }
+  return searches >= 1 ? { check: "web_search", outcome: "pass", observed: `searches=${searches}` } : { check: "web_search", outcome: "fail", observed: "searches=0", detail: "The host reported no search for a request that asked for one." };
+}
+
+// src/conformance.ts
+var WEATHER_TOOL = {
+  type: "function",
+  function: {
+    name: TOOL_NAME,
+    description: "Get the current weather for a city.",
+    parameters: {
+      type: "object",
+      properties: { city: { type: "string" } },
+      required: ["city"]
+    }
+  }
+};
+var RED_PNG = "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAJ0lEQVR42u3NsQkAAAjAsP7/tF7hIASyp6lTCQQCgUAgEAgEgi/BAjLD/C5w/SM9AAAAAElFTkSuQmCC";
+var CHECKS = [
+  {
+    name: "chat_completion",
+    decides: null,
+    request: {
+      messages: [{ role: "user", content: "Reply with exactly: ok" }],
+      // Generous on purpose. A reasoning model spends completion tokens thinking
+      // before it emits any content, so a tight cap truncates it to empty output
+      // and the check records a working model as broken — measured against
+      // deepseek-v4-flash, which used 11 of 13 completion tokens reasoning.
+      max_tokens: 256
+    }
+  },
+  {
+    name: "usage_reported",
+    decides: "usage_reported",
+    request: {
+      messages: [{ role: "user", content: "Reply with exactly: ok" }],
+      max_tokens: 256
+    }
+  },
+  {
+    name: "tool_call",
+    decides: "tools",
+    // OpenAI's gpt-6-astra (and gpt-5.6-*) refuse function tools on the legacy
+    // /v1/chat/completions endpoint, requiring /v1/responses. Cruise's translation
+    // adapter (#545) transparently routes tool calls for these models to
+    // /v1/responses upstream, allowing tools=1 to be measured honestly.
+    //
+    // Three attempts since #549. On 2026-09-30 gemini-3.1-pro-preview passed on
+    // production at 04:10 UTC and answered `MALFORMED_FUNCTION_CALL` twice within
+    // the hour, passing on staging both times between. One sample recorded
+    // tools = 0 and took it out of all four coding lanes; absence refuses
+    // callers, so it needs every attempt to agree (see `attempts`).
+    attempts: 3,
+    request: {
+      messages: [{ role: "user", content: "What is the weather in Paris? Use the tool." }],
+      tools: [WEATHER_TOOL]
+    }
+  },
+  {
+    name: "parallel_tool_calls",
+    decides: "parallel_tool_calls",
+    // Undefined without tool calling at all, and retried before concluding
+    // absence — see `dependsOn` and `attempts`.
+    dependsOn: "tool_call",
+    attempts: 3,
+    request: {
+      messages: [
+        {
+          role: "system",
+          content: "You call tools. get_weather accepts exactly one city per call, so covering several cities requires several separate calls in one response. Never ask the user to wait, and never combine cities into one call."
+        },
+        {
+          role: "user",
+          content: "I need the current weather for all three of Paris, Tokyo and Cairo right now. Call get_weather once per city."
+        }
+      ],
+      tools: [WEATHER_TOOL],
+      parallel_tool_calls: true
+    }
+  },
+  {
+    name: "tool_choice_forced",
+    decides: "tool_choice_forced",
+    // Allowed and forced come apart (#570): Claude Fable 5.1, Opus 5.5 and
+    // Sonnet 5.5 call a tool when allowed and answer 400 when `tool_choice`
+    // forces one, and `tool_call` sends no `tool_choice`, so it measured them
+    // `tools = 1` and lanes promised what a forced request then refused.
+    // `"required"` stands for both forcing shapes — a host that refuses one
+    // refuses the other, since both become Anthropic's `any`/`tool` — and is
+    // the form `messages-translate.ts` emits for `any`. Same evaluator as
+    // `tool_call`: forced or not, a malformed call is not a call.
+    dependsOn: "tool_call",
+    attempts: 2,
+    request: {
+      messages: [{ role: "user", content: "What is the weather in Paris?" }],
+      tools: [WEATHER_TOOL],
+      tool_choice: "required"
+    }
+  },
+  {
+    name: "streaming",
+    decides: "streaming",
+    streaming: true,
+    request: {
+      messages: [{ role: "user", content: "Count from one to five." }],
+      stream: true,
+      max_tokens: 64
+    }
+  },
+  {
+    name: "json_schema",
+    decides: "json_schema",
+    // A fail here is the provider refusing `response_format` or the model
+    // answering outside the schema. Retried once: a strict schema is a thing a
+    // model can get right on the second sample.
+    attempts: 2,
+    request: {
+      messages: [{ role: "user", content: "Where is the Eiffel Tower? Answer with the city and the country." }],
+      response_format: {
+        type: "json_schema",
+        json_schema: {
+          name: "place",
+          strict: true,
+          schema: {
+            type: "object",
+            properties: { city: { type: "string" }, country: { type: "string" } },
+            required: ["city", "country"],
+            additionalProperties: false
+          }
+        }
+      },
+      max_tokens: 256
+    }
+  },
+  {
+    name: "vision",
+    decides: "vision",
+    // A 32×32 solid red PNG, inline, so the check depends on no host. A model
+    // without image input refuses the part or describes what it cannot see;
+    // either is a fail, and the second is why the answer is checked for the
+    // colour rather than for being non-empty.
+    attempts: 2,
+    request: {
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: "What colour is this image? Answer with one word." },
+            { type: "image_url", image_url: { url: `data:image/png;base64,${RED_PNG}` } }
+          ]
+        }
+      ],
+      // Generous for the same reason as chat_completion: gemma-4 spent 86
+      // completion tokens reasoning before saying "Red", and a 64-token cap
+      // recorded a model that can see as one that cannot. Measured 2026-08-22.
+      max_tokens: 256
+    }
+  },
+  WEB_SEARCH_CHECK,
+  {
+    name: "truncation",
+    decides: null,
+    request: {
+      messages: [{ role: "user", content: "Write a long essay about the sea." }],
+      max_tokens: 8
+    }
+  }
+];
+var terminator2 = (choice) => String(choice?.finish_reason ?? choice?.stop_reason ?? "<absent>");
+function checkChatCompletion(body) {
+  const choice = body.choices?.[0];
+  if (!choice) return { check: "chat_completion", outcome: "fail", detail: "No choices returned." };
+  const content = typeof choice.message?.content === "string" ? choice.message.content : "";
+  const finish = terminator2(choice);
+  if (choice.message?.refusal != null || finish.toLowerCase().includes("refus")) {
+    return { check: "chat_completion", outcome: "fail", observed: finish, detail: "Refusal returned as success." };
+  }
+  if (content.trim() === "") {
+    const reasoning = body.usage?.completion_tokens_details?.reasoning_tokens;
+    const spentThinking = typeof reasoning === "number" && reasoning > 0;
+    return {
+      check: "chat_completion",
+      outcome: "fail",
+      observed: finish,
+      detail: spentThinking ? `Truncated before any content: ${reasoning} completion tokens went to reasoning. Raise max_tokens for this model.` : "Empty content returned as success."
+    };
+  }
+  return { check: "chat_completion", outcome: "pass", observed: finish };
+}
+function checkUsageReported(body) {
+  const usage = body.usage;
+  const ok = typeof usage?.prompt_tokens === "number" && typeof usage?.completion_tokens === "number";
+  return {
+    check: "usage_reported",
+    outcome: ok ? "pass" : "fail",
+    observed: usage ? JSON.stringify(usage) : "<absent>",
+    ...ok ? {} : { detail: "No usable usage block; spend cannot be attributed." }
+  };
+}
+function checkTruncation(body) {
+  const finish = terminator2(body.choices?.[0]);
+  const truncated = /length|max_tokens|truncat/i.test(finish);
+  return {
+    check: "truncation",
+    outcome: truncated ? "pass" : "fail",
+    observed: finish,
+    ...truncated ? {} : { detail: "Truncation not distinguishable from a clean completion by its terminator." }
+  };
+}
+function checkStreaming(sse) {
+  const dataLines = sse.split("\n").filter((line) => line.trim().startsWith("data:"));
+  const terminated = dataLines.some((line) => line.includes("[DONE]"));
+  let assembled = "";
+  let usageSeen = false;
+  for (const line of dataLines) {
+    const payload = line.trim().slice(5).trim();
+    if (payload === "" || payload === "[DONE]") continue;
+    try {
+      const chunk = JSON.parse(payload);
+      const delta = chunk.choices?.[0]?.delta?.content;
+      if (typeof delta === "string") assembled += delta;
+      if (chunk.usage) usageSeen = true;
+    } catch {
+      return { check: "streaming", outcome: "fail", detail: "Chunk was not valid JSON." };
+    }
+  }
+  const problems = [];
+  if (dataLines.length === 0) problems.push("no data chunks");
+  if (!terminated) problems.push("no [DONE] terminator");
+  if (assembled.trim() === "") problems.push("no content assembled");
+  return {
+    check: "streaming",
+    outcome: problems.length === 0 ? "pass" : "fail",
+    observed: `${dataLines.length} chunk(s), usage ${usageSeen ? "present" : "absent"}`,
+    ...problems.length === 0 ? {} : { detail: problems.join("; ") }
+  };
+}
+function contentText(body) {
+  const content = body.choices?.[0]?.message?.content;
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    return content.map((part) => typeof part?.text === "string" ? part.text : "").join("");
+  }
+  return null;
+}
+function checkJsonSchema(body) {
+  const text = contentText(body);
+  if (text === null || text.trim() === "") {
+    return { check: "json_schema", outcome: "fail", detail: "No content.", observed: terminator2(body.choices?.[0]) };
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return { check: "json_schema", outcome: "fail", detail: "Content is not JSON.", observed: text.slice(0, 80) };
+  }
+  const object = parsed;
+  const keys = Object.keys(object ?? {}).sort();
+  const shaped = typeof object === "object" && object !== null && typeof object.city === "string" && typeof object.country === "string" && keys.join(",") === "city,country";
+  return shaped ? { check: "json_schema", outcome: "pass", observed: text.slice(0, 80) } : { check: "json_schema", outcome: "fail", detail: "JSON does not match the schema.", observed: text.slice(0, 80) };
+}
+function checkVision(body) {
+  const text = contentText(body);
+  if (text === null || text.trim() === "") {
+    return { check: "vision", outcome: "fail", detail: "No content.", observed: terminator2(body.choices?.[0]) };
+  }
+  const observed = text.trim().slice(0, 80);
+  if (/\b(not|no|never|cannot|can't|unable|isn't|doesn't)\b/i.test(text)) {
+    return { check: "vision", outcome: "fail", detail: "Hedged or negated rather than answered.", observed };
+  }
+  return /\bred(dish)?\b/i.test(text) ? { check: "vision", outcome: "pass", observed } : { check: "vision", outcome: "fail", detail: "Did not identify the colour.", observed };
+}
+
+// src/conformance-image.ts
+var IMAGE_PROMPT = "A plain grey square on a white background.";
+var INTRODUCTION_SIZES = ["1024x1024", "1536x1024", "1024x1536"];
+var INTRODUCTION_QUALITIES = ["low", "medium", "high"];
+function checkImageGeneration(body) {
+  const first = body.data?.[0];
+  if (first === void 0) {
+    return { check: "image_generation", outcome: "fail", detail: "No data returned." };
+  }
+  const b64 = typeof first.b64_json === "string" ? first.b64_json : "";
+  const url = typeof first.url === "string" ? first.url : "";
+  if (b64.trim() === "" && url.trim() === "") {
+    return {
+      check: "image_generation",
+      outcome: "fail",
+      observed: Object.keys(first).join(",") || "<empty object>",
+      detail: "An entry was returned carrying neither b64_json nor url."
+    };
+  }
+  return {
+    check: "image_generation",
+    outcome: "pass",
+    observed: b64 !== "" ? `b64_json, ${b64.length} chars` : "url"
+  };
+}
+function checkImageUsage(body) {
+  const usage = body.usage;
+  const input = usage?.input_tokens;
+  const output = usage?.output_tokens;
+  const ok = typeof input === "number" && typeof output === "number";
+  return {
+    check: "image_usage",
+    outcome: ok ? "pass" : "fail",
+    observed: usage ? JSON.stringify(usage) : "<absent>",
+    ...ok ? {} : { detail: "No usable usage block; spend cannot be attributed." }
+  };
+}
+var IMAGE_CHECKS = [
+  { name: "image_generation", decides: null, evaluate: checkImageGeneration },
+  { name: "image_usage", decides: "usage_reported", evaluate: checkImageUsage }
+];
+var COMPONENT_IMAGE_CHECKS = [
+  { name: "image_generation", decides: null, evaluate: checkImageGeneration }
+];
+function outputTokensOf(body) {
+  const output = body.usage?.output_tokens;
+  return typeof output === "number" && Number.isInteger(output) && output > 0 ? output : null;
+}
+
+// src/conformance-verdict.ts
+var HOST_FEATURE_CHECKS = /* @__PURE__ */ new Set(["web_search", "tool_choice_forced"]);
+function answeredByModel(results) {
+  return results.filter(
+    (result) => result.outcome === "pass" || result.outcome === "fail" && !(HOST_FEATURE_CHECKS.has(result.check) && /^HTTP \d{3}\b/.test(result.observed ?? ""))
+  );
+}
+function unservableReason(results) {
+  if (answeredByModel(results).length > 0) return null;
+  const errors = results.filter((result) => result.outcome === "error");
+  if (errors.length === 0) return null;
+  const refusals = errors.filter((result) => /^HTTP 40[13]\b/.test(result.observed ?? ""));
+  if (refusals.length !== errors.length) return null;
+  const codes = [...new Set(refusals.map((result) => result.observed))].sort();
+  return `The provider refused every check: ${codes.join(", ")}. This is an entitlement \u2014 a plan, a tier or a licence \u2014 not a property of the model, and no further run will change it.`;
+}
+function failedEveryCheckReason(results) {
+  const answered = answeredByModel(results);
+  if (answered.length === 0) return null;
+  if (answered.some((result) => result.outcome === "pass")) return null;
+  const statuses = (of) => [...new Set(of.map((result) => result.observed).filter((o) => !!o))].sort();
+  const observed = statuses(answered);
+  const saw = observed.length > 0 ? ` (${observed.join(", ")})` : "";
+  const errors = results.filter((result) => result.outcome === "error");
+  const erroredStatuses = statuses(errors);
+  const errored = errors.length > 0 ? ` ${errors.length} more errored${erroredStatuses.length > 0 ? ` (${erroredStatuses.join(", ")})` : ""} and say nothing either way.` : "";
+  return `Every check the model answered failed${saw}.${errored} A model that has not been seen to serve is unmeasured whatever the date says; the next run in which any check passes will measure it.`;
+}
+function flagsFromResults(results, checks = CHECKS) {
+  const decidedBy = new Map(checks.filter((c) => c.decides).map((c) => [c.name, c.decides]));
+  const flags = {};
+  for (const result of results) {
+    const column = decidedBy.get(result.check);
+    if (!column) continue;
+    if (result.outcome === "pass") flags[column] = 1;
+    else if (result.outcome === "fail") flags[column] = 0;
+  }
+  return flags;
+}
+function errorCode(raw) {
+  try {
+    return String(JSON.parse(raw).error?.code ?? "");
+  } catch {
+    return "";
+  }
+}
+function errorParam(raw) {
+  try {
+    return String(JSON.parse(raw).error?.param ?? "");
+  } catch {
+    return "";
+  }
+}
+var REQUEST_SHAPE_PARAMS = /* @__PURE__ */ new Set([
+  "model",
+  "temperature",
+  "top_p",
+  "max_tokens",
+  "max_completion_tokens",
+  "n",
+  "user"
+]);
+var OUR_GATES = /* @__PURE__ */ new Set([
+  "unsupported_capability",
+  "model_unmeasured",
+  "measurement_stale",
+  "model_not_found",
+  "upstream_not_supported",
+  "limit_exceeded"
+]);
+function refusalIsOurs(status, raw) {
+  const code = errorCode(raw);
+  const clientRefusal = status === 400 || status === 422;
+  if (!OUR_GATES.has(code) && clientRefusal && (errorParam(raw) === "tool_choice" || /tool_choice|web_search/i.test(raw))) return false;
+  const providerSaysModelCannot = status === 403 && /doesn't support|not supported/i.test(raw);
+  return status >= 500 || status === 429 || status === 401 || status === 402 || status === 403 && !providerSaysModelCannot || status === 400 && REQUEST_SHAPE_PARAMS.has(errorParam(raw)) || code === "unsupported_parameter" || OUR_GATES.has(code);
+}
+var REFUSAL_HEADERS = [
+  "content-type",
+  "server-timing",
+  "cf-aig-cache-status",
+  "x-cruise-upstream-status",
+  "x-cruise-model",
+  "x-cruise-fell-back-from"
+];
+var DETAIL_LIMIT = 300;
+function refusalOf(status, raw, headers) {
+  const code = errorCode(raw);
+  const named = REFUSAL_HEADERS.flatMap((name) => {
+    const value = headers.get(name);
+    return value === null ? [] : [`${name}: ${value}`];
+  });
+  const detail = raw.length > 0 ? raw : `Empty body. ${named.join("; ")}`.trim();
+  return {
+    outcome: refusalIsOurs(status, raw) ? "error" : "fail",
+    observed: `HTTP ${status}${code ? ` ${code}` : ""}`,
+    detail: detail.slice(0, DETAIL_LIMIT)
+  };
+}
+
+// scripts/conformance-image.ts
+function variantsToMeasure(measured, full, sizes = INTRODUCTION_SIZES, qualities = INTRODUCTION_QUALITIES) {
+  if (full || measured.length === 0) {
+    return sizes.flatMap((size) => qualities.map((quality) => ({ size, quality })));
+  }
+  const cheapest = measured.reduce((low, row) => row.output_tokens < low.output_tokens ? row : low);
+  return [{ size: cheapest.size, quality: cheapest.quality }];
+}
+async function generate(base, key, model, variant) {
+  try {
+    const response = await fetch(`${base}/v1/images/generations`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+      body: JSON.stringify({ model, prompt: IMAGE_PROMPT, n: 1, ...variant })
+    });
+    const raw = await response.text();
+    if (!response.ok) {
+      return { body: null, failure: refusalOf(response.status, raw, response.headers) };
+    }
+    return { body: JSON.parse(raw), failure: null };
+  } catch (cause) {
+    return { body: null, failure: { outcome: "error", detail: String(cause).slice(0, 300) } };
+  }
+}
+async function measureImageModel(db, base, key, model, runId, options, log = console.log) {
+  if (options.pricedBy === "component") {
+    const attempt = await generate(base, key, model, {});
+    if (attempt.body !== null) log(`  ${model}  component-priced generation ok`);
+    return attempt.body === null ? COMPONENT_IMAGE_CHECKS.map((check) => ({ check: check.name, ...attempt.failure })) : COMPONENT_IMAGE_CHECKS.map((check) => check.evaluate(attempt.body));
+  }
+  const { results: measured } = await db.prepare(`SELECT size, quality, output_tokens FROM model_image_variants WHERE model = ?`).bind(model).all();
+  const wanted = variantsToMeasure(measured, options.full, options.sizes, options.qualities);
+  const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+  let results = null;
+  for (const variant of wanted) {
+    const attempt = await generate(base, key, model, variant);
+    if (attempt.body !== null) {
+      const tokens = outputTokensOf(attempt.body);
+      if (tokens !== null) {
+        await db.prepare(
+          `INSERT INTO model_image_variants (model, size, quality, output_tokens, measured_at, measured_by, source)
+             VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'conformance')
+             ON CONFLICT (model, size, quality) DO UPDATE
+               SET output_tokens = ?4, measured_at = ?5, measured_by = ?6, source = 'conformance'`
+        ).bind(model, variant.size, variant.quality, tokens, today, `conformance ${runId}`).run();
+        log(`  ${model}  ${variant.size} ${variant.quality.padEnd(6)} ${tokens} output token(s)`);
+      } else {
+        log(`  ${model}  ${variant.size} ${variant.quality.padEnd(6)} no usable output token count \u2014 not recorded`);
+      }
+    }
+    if (results === null) {
+      results = attempt.body === null ? IMAGE_CHECKS.map((check) => ({ check: check.name, ...attempt.failure })) : IMAGE_CHECKS.map((check) => check.evaluate(attempt.body));
+    }
+  }
+  return results ?? [];
+}
+
+// src/conformance-speech.ts
+var SPEECH_INPUT = "The quick brown fox jumps over the lazy dog.";
+var SPEECH_FORMAT = "mp3";
+var SPEECH_MIME_TYPES = {
+  mp3: ["audio/mpeg", "audio/mp3"]
+};
+var SPEECH_VOICES = {
+  openai: "alloy",
+  elevenlabs: "JBFqnCBsd6RMkjVDRZzb"
+};
+function checkSpeechGeneration(response, format = SPEECH_FORMAT) {
+  const raw = (response.contentType ?? "").toLowerCase();
+  const type = raw.split(";")[0].trim();
+  if (response.bytes === 0) {
+    return { check: "speech_generation", outcome: "fail", observed: type || "<no content-type>", detail: "Empty body returned as success." };
+  }
+  const observed = `${type || "<no content-type>"}, ${response.bytes} byte(s)`;
+  if (!type.startsWith("audio/")) {
+    return { check: "speech_generation", outcome: "fail", observed, detail: `Body is not audio; asked for ${format}.` };
+  }
+  const expected = SPEECH_MIME_TYPES[format] ?? [];
+  if (!expected.includes(type)) {
+    return {
+      check: "speech_generation",
+      outcome: "fail",
+      observed,
+      detail: `Audio in a format that was not asked for: wanted ${expected.join(" or ")} for ${format}.`
+    };
+  }
+  return { check: "speech_generation", outcome: "pass", observed };
+}
+var SPEECH_CHECKS = [
+  { name: "speech_generation", decides: null, evaluate: checkSpeechGeneration }
+];
+
+// scripts/conformance-speech.ts
+function voiceFor(model, overrides = {}) {
+  const upstream = model.slice(0, model.indexOf("/"));
+  return overrides[upstream] ?? SPEECH_VOICES[upstream] ?? null;
+}
+async function measureSpeechModel(base, key, model, voices = {}) {
+  const voice = voiceFor(model, voices);
+  if (voice === null) {
+    return SPEECH_CHECKS.map((check) => ({
+      check: check.name,
+      outcome: "error",
+      detail: `No voice is known for this model's host; pass --voice-<upstream> <id>.`
+    }));
+  }
+  let observed;
+  try {
+    const response = await fetch(`${base}/v1/audio/speech`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+      body: JSON.stringify({ model, input: SPEECH_INPUT, voice, response_format: SPEECH_FORMAT })
+    });
+    if (!response.ok) {
+      const failure = refusalOf(response.status, await response.text(), response.headers);
+      return SPEECH_CHECKS.map((check) => ({ check: check.name, ...failure }));
+    }
+    const bytes = (await response.arrayBuffer()).byteLength;
+    observed = { contentType: response.headers.get("content-type"), bytes };
+  } catch (cause) {
+    return SPEECH_CHECKS.map((check) => ({ check: check.name, outcome: "error", detail: String(cause).slice(0, 300) }));
+  }
+  return SPEECH_CHECKS.map((check) => check.evaluate(observed));
+}
+
+// src/conformance-embedding.ts
+var EMBEDDING_INPUT = "A plain sentence, embedded to prove the route works.";
+function checkEmbeddingVector(body, dimensions) {
+  const first = body.data?.[0];
+  if (first === void 0) {
+    return { check: "embedding_vector", outcome: "fail", detail: "No data returned." };
+  }
+  const vector = first.embedding;
+  if (!Array.isArray(vector) || vector.length === 0) {
+    return {
+      check: "embedding_vector",
+      outcome: "fail",
+      observed: typeof vector === "string" ? `a ${vector.length}-character string` : String(vector),
+      detail: "An entry was returned carrying no vector."
+    };
+  }
+  if (!vector.every((value) => typeof value === "number" && Number.isFinite(value))) {
+    return { check: "embedding_vector", outcome: "fail", observed: `${vector.length} value(s)`, detail: "The vector holds something that is not a number." };
+  }
+  const observed = `${vector.length} dimension(s)`;
+  if (dimensions !== null && vector.length !== dimensions) {
+    return {
+      check: "embedding_vector",
+      outcome: "fail",
+      observed,
+      detail: `The catalogue publishes ${dimensions} dimensions for this model, which is the figure a caller has sized storage by.`
+    };
+  }
+  return { check: "embedding_vector", outcome: "pass", observed };
+}
+function checkEmbeddingUsage(body) {
+  const usage = body.usage;
+  const ok = typeof usage?.prompt_tokens === "number";
+  return {
+    check: "embedding_usage",
+    outcome: ok ? "pass" : "fail",
+    observed: usage ? JSON.stringify(usage) : "<absent>",
+    ...ok ? {} : { detail: "No usable prompt token count; spend cannot be attributed." }
+  };
+}
+var EMBEDDING_CHECKS = [
+  { name: "embedding_vector", decides: null, evaluate: checkEmbeddingVector },
+  { name: "embedding_usage", decides: "usage_reported", evaluate: (body) => checkEmbeddingUsage(body) }
+];
+
+// scripts/conformance-embedding.ts
+async function measureEmbeddingModel(base, key, model, dimensions) {
+  let body;
+  try {
+    const response = await fetch(`${base}/v1/embeddings`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+      body: JSON.stringify({ model, input: EMBEDDING_INPUT })
+    });
+    const raw = await response.text();
+    if (!response.ok) {
+      const failure = refusalOf(response.status, raw, response.headers);
+      return EMBEDDING_CHECKS.map((check) => ({ check: check.name, ...failure }));
+    }
+    body = JSON.parse(raw);
+  } catch (cause) {
+    return EMBEDDING_CHECKS.map((check) => ({ check: check.name, outcome: "error", detail: String(cause).slice(0, 300) }));
+  }
+  return EMBEDDING_CHECKS.map((check) => check.evaluate(body, dimensions));
+}
+
+// src/conformance-fold.ts
+async function foldFlags(db, model, results, checks, runId, today) {
+  const flags = flagsFromResults(results, checks);
+  const columns = Object.keys(flags);
+  const assignments = columns.map((column) => `${column} = ?`);
+  const values = columns.map((column) => flags[column]);
+  const by = `conformance ${runId}`;
+  const cleared = unservableReason(results) ?? failedEveryCheckReason(results);
+  if (cleared !== null) {
+    await db.prepare(
+      `UPDATE models
+            SET ${[...assignments, "measured_at = NULL", "measurement_note = ?"].join(", ")},
+                measured_by = ?, measurement_source = 'conformance'
+          WHERE id = ?`
+    ).bind(...values, cleared, by, model).run();
+    return { outcome: "cleared", reason: cleared };
+  }
+  const decidesNothing = checks.every((check) => check.decides === null);
+  const passed = results.some((result) => result.outcome === "pass");
+  if (passed && (columns.length > 0 || decidesNothing)) {
+    await db.prepare(
+      `UPDATE models
+            SET ${[...assignments, "measured_at = ?"].join(", ")}, measured_by = ?,
+                measurement_source = 'conformance', measurement_note = NULL
+          WHERE id = ?`
+    ).bind(...values, today, by, model).run();
+    return { outcome: "stamped" };
+  }
+  return { outcome: "left" };
+}
+
+// scripts/conformance.ts
+function evaluate(definition, raw) {
+  if (definition.streaming) return checkStreaming(raw);
+  let body;
+  try {
+    body = JSON.parse(raw);
+  } catch {
+    return { check: definition.name, outcome: "error", detail: "Response was not valid JSON." };
+  }
+  switch (definition.name) {
+    case "chat_completion":
+      return checkChatCompletion(body);
+    case "usage_reported":
+      return checkUsageReported(body);
+    case "tool_call":
+      return checkToolCall(body);
+    case "parallel_tool_calls":
+      return checkParallelToolCalls(body);
+    case "tool_choice_forced":
+      return checkToolCall(body, "tool_choice_forced");
+    case "json_schema":
+      return checkJsonSchema(body);
+    case "vision":
+      return checkVision(body);
+    case "web_search":
+      return checkWebSearch(body);
+    case "truncation":
+      return checkTruncation(body);
+    default:
+      return { check: definition.name, outcome: "skip", detail: "No evaluator." };
+  }
+}
+async function runCheck(base, key, model, definition) {
+  try {
+    const path = typeof definition.path === "function" ? definition.path(model) : definition.path ?? "/v1/chat/completions";
+    const response = await fetch(`${base}${path}`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+      body: JSON.stringify({ ...definition.request, model })
+    });
+    const raw = await response.text();
+    if (!response.ok) {
+      return { check: definition.name, ...refusalOf(response.status, raw, response.headers) };
+    }
+    return evaluate(definition, raw);
+  } catch (cause) {
+    return { check: definition.name, outcome: "error", detail: String(cause).slice(0, 300) };
+  }
+}
+function parseShard(raw) {
+  if (raw === void 0) return { index: 1, total: 1 };
+  const match = /^(\d+)\/(\d+)$/.exec(String(raw));
+  if (!match) throw new Error(`--shard must look like 2/3, got '${String(raw)}'`);
+  return { index: Number(match[1]), total: Number(match[2]) };
+}
+run(async () => {
+  const args = parseArgs(process.argv.slice(2));
+  const environment = environmentOf(args);
+  const base = requireString(args, "base").replace(/\/$/, "");
+  const key = requireString(args, "key");
+  const db = connect(environment);
+  const prefix = parseKeyPrefix(key);
+  const holder = prefix ? await db.prepare(`SELECT conformance, revoked_at FROM virtual_keys WHERE prefix = ?`).bind(prefix).first() : null;
+  if (prefix === null || holder === null) {
+    throw new Error(`--key is not a virtual key this environment knows (${prefix ?? "unparseable"}).`);
+  }
+  if (holder.revoked_at !== null) {
+    throw new Error(
+      `--key ${prefix} was revoked ${holder.revoked_at}. Every request would answer 401, and a run of uniform 401s is read as the providers refusing us \u2014 it would clear measurements rather than take them.`
+    );
+  }
+  if (holder.conformance !== 1) {
+    throw new Error(
+      `--key ${prefix} is not a conformance key. Only a conformance key is exempt from the unmeasured gate, so an unmeasured model would answer 'model_unmeasured' and never be reached. Issue one with: npm run key:issue -- --env <env> --project <p> --actor <a> --conformance`
+    );
+  }
+  const shard = parseShard(args.shard);
+  const { results: models } = await db.prepare(`SELECT m.id, m.measured_at, m.modality, m.priced_by, m.dimensions FROM models m JOIN upstreams u ON u.id = m.upstream_id
+               WHERE m.enabled = 1 AND u.enabled = 1
+               ORDER BY m.id`).all();
+  const selected = selectModels(models, { ids: listFlag(args.models, "models"), modalities: listFlag(args.modality, "modality") });
+  if (selected.length < models.length) {
+    console.log(`selected ${selected.length} of ${models.length} model(s)`);
+  }
+  const mine = planShard(selected, shard.index, shard.total);
+  if (mine.length === 0) {
+    console.log(shard.total === 1 ? "no routed models" : `no routed models in shard ${shard.index}/${shard.total}`);
+    return;
+  }
+  if (shard.total > 1) {
+    console.log(`shard ${shard.index}/${shard.total}: ${mine.length} of ${selected.length} models`);
+  }
+  const runId = crypto.randomUUID();
+  await db.prepare(`INSERT INTO conformance_runs (id, environment, note) VALUES (?, ?, ?)`).bind(runId, environment, typeof args.note === "string" ? args.note : null).run();
+  const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+  let failures = 0;
+  const fold = async (model, results, checks) => {
+    const verdict = await foldFlags(db, model, results, checks, runId, today);
+    if (verdict.outcome === "cleared") console.log(`  ${model}: measurement cleared \u2014 ${verdict.reason}`);
+  };
+  const record = async (model, result, label) => {
+    await db.prepare(
+      `INSERT INTO conformance_results (run_id, model, check_name, outcome, observed, detail)
+         VALUES (?, ?, ?, ?, ?, ?)`
+    ).bind(runId, model, result.check, result.outcome, result.observed ?? null, result.detail ?? null).run();
+    const mark = { pass: "ok", fail: "FAIL", skip: "skip", error: "ERROR" }[result.outcome];
+    console.log(`${model}  ${label.padEnd(20)} ${mark}${result.observed ? "  " + result.observed : ""}`);
+    if (result.outcome === "fail" || result.outcome === "error") failures++;
+  };
+  for (const { id: model, modality, priced_by: pricedBy, dimensions } of mine) {
+    if (modality === "image") {
+      const checks = pricedBy === "component" ? COMPONENT_IMAGE_CHECKS : IMAGE_CHECKS;
+      const results2 = await measureImageModel(db, base, key, model, runId, {
+        full: args["image-variants"] === true || args["image-variants"] === "true",
+        sizes: listFlag(args.sizes, "sizes"),
+        qualities: listFlag(args.qualities, "qualities"),
+        pricedBy
+      });
+      for (const result of results2) await record(model, result, result.check);
+      await fold(model, results2, checks);
+      continue;
+    }
+    if (modality === "embedding") {
+      const results2 = await measureEmbeddingModel(base, key, model, dimensions);
+      for (const result of results2) await record(model, result, result.check);
+      await fold(model, results2, EMBEDDING_CHECKS);
+      continue;
+    }
+    if (modality === "speech") {
+      const voices = typeof args["voice-elevenlabs"] === "string" ? { elevenlabs: args["voice-elevenlabs"] } : {};
+      const results2 = await measureSpeechModel(base, key, model, voices);
+      for (const result of results2) await record(model, result, result.check);
+      await fold(model, results2, SPEECH_CHECKS);
+      continue;
+    }
+    const results = [];
+    const outcomeOf = /* @__PURE__ */ new Map();
+    for (const definition of CHECKS) {
+      let result;
+      if (definition.dependsOn && outcomeOf.get(definition.dependsOn) !== "pass") {
+        result = {
+          check: definition.name,
+          outcome: "skip",
+          detail: `Skipped: '${definition.dependsOn}' did not pass, so this cannot be measured.`
+        };
+      } else {
+        const attempts = Math.max(1, definition.attempts ?? 1);
+        result = await runCheck(base, key, model, definition);
+        for (let attempt = 2; attempt <= attempts && result.outcome === "fail"; attempt++) {
+          const retry = await runCheck(base, key, model, definition);
+          if (retry.outcome === "pass") {
+            result = { ...retry, detail: `Passed on attempt ${attempt} of ${attempts}.` };
+            break;
+          }
+          result = retry;
+        }
+        if (result.outcome === "fail" && attempts > 1) {
+          result = {
+            ...result,
+            detail: `${result.detail ?? ""} Failed all ${attempts} attempts.`.trim()
+          };
+        }
+      }
+      outcomeOf.set(definition.name, result.outcome);
+      results.push(result);
+      await record(model, result, definition.name);
+    }
+    await fold(model, results, CHECKS);
+  }
+  await db.prepare(`UPDATE conformance_runs SET finished_at = datetime('now') WHERE id = ?`).bind(runId).run();
+  console.log(`
+run ${runId}: ${mine.length} model(s), ${failures} failing check(s)`);
+});