@bytesbrains/cruise 0.12.1 → 0.13.0

package/dist/doctor-LVMXEX35.mjs

@@ -0,0 +1,802 @@
+#!/usr/bin/env node
+// Built by packages/cli/build.ts.
+import {
+  HEALTH_BURN_DAYS
+} from "./chunk-TJR2U73N.mjs";
+import {
+  prepaidOn,
+  unprepaidSignups
+} from "./chunk-5AFYJTLS.mjs";
+import {
+  DEMO_ENVIRONMENT,
+  DEMO_SERVABLE_MODALITIES
+} from "./chunk-D3K25SBO.mjs";
+import "./chunk-CDBZLSL7.mjs";
+import "./chunk-O7TNRKSA.mjs";
+import {
+  hitBasisPointsInForce,
+  hitCharge,
+  priceUsage
+} from "./chunk-7ZUKADIS.mjs";
+import {
+  CREDENTIAL_FREE_ENVIRONMENTS
+} from "./chunk-NR4K7AUA.mjs";
+import {
+  MICROS_PER_USD,
+  formatUsd
+} from "./chunk-UBM65ZNR.mjs";
+import {
+  FRESHLY_MEASURED_SQL
+} from "./chunk-A4DIVNYV.mjs";
+import "./chunk-5TVD3ZAI.mjs";
+import {
+  connect,
+  environmentOf,
+  parseArgs,
+  run,
+  workerScript
+} from "./chunk-Y2I46CTD.mjs";
+
+// src/doctor-balance.ts
+var RUNWAY_WARN_DAYS = 7;
+var ok = (detail) => ({ check: "upstream_balance", severity: "ok", detail });
+function usdMicros(credits) {
+  const balances = credits.balances;
+  if (balances === void 0) return null;
+  let total = 0;
+  let saw = false;
+  for (const b of balances) {
+    if (b.currency !== "USD") continue;
+    if (typeof b.total_micros === "number" && Number.isFinite(b.total_micros)) {
+      total += b.total_micros;
+      saw = true;
+    }
+  }
+  return saw ? total : null;
+}
+function otherMoney(credits) {
+  const balances = credits.balances;
+  if (balances === void 0) return null;
+  for (const b of balances) {
+    if (typeof b.currency === "string" && b.currency !== "USD" && typeof b.total_micros === "number") {
+      return b;
+    }
+  }
+  return null;
+}
+function characterRemaining(credits) {
+  const balances = credits.balances;
+  if (balances === void 0) return null;
+  for (const b of balances) {
+    if (b.unit === "characters" && typeof b.remaining === "number") return b.remaining;
+  }
+  return null;
+}
+function daysOfRunway(totalMicros, burn) {
+  if (burn.daily_list_micros <= 0) return null;
+  return Math.floor(totalMicros / burn.daily_list_micros);
+}
+function formatUsdLine(id, micros, burn) {
+  const dollars = `$${formatUsd(micros)}`;
+  const days = daysOfRunway(micros, burn);
+  if (days === null) {
+    return `${id}: ${dollars} left, no burn in the last ${HEALTH_BURN_DAYS} days`;
+  }
+  return `${id}: ${dollars} left, ~${days} day${days === 1 ? "" : "s"} at the last ${HEALTH_BURN_DAYS} days' burn`;
+}
+function formatOtherMoneyLine(id, balance) {
+  const amount = ((balance.total_micros ?? 0) / MICROS_PER_USD).toFixed(2);
+  return `${id}: ${amount} ${balance.currency} left (runway not computed \u2014 ledger burn is USD)`;
+}
+function upstreamBalance(probe) {
+  if (!probe.ok) {
+    return {
+      check: "upstream_balance",
+      severity: "warn",
+      detail: `not checked \u2014 ${probe.reason}`
+    };
+  }
+  const unavailable = [];
+  const short = [];
+  const healthy = [];
+  let exposed = 0;
+  for (const row of probe.upstreams) {
+    if (row.credits.status !== "ok") continue;
+    exposed += 1;
+    if (row.credits.available === false) {
+      unavailable.push(`${row.id}: unavailable (host reports available: false)`);
+      continue;
+    }
+    const micros = usdMicros(row.credits);
+    if (micros !== null) {
+      const line = formatUsdLine(row.id, micros, row.burn);
+      const days = daysOfRunway(micros, row.burn);
+      if (days !== null && days < RUNWAY_WARN_DAYS) short.push(line);
+      else healthy.push(line);
+      continue;
+    }
+    const other = otherMoney(row.credits);
+    if (other !== null) {
+      healthy.push(formatOtherMoneyLine(row.id, other));
+      continue;
+    }
+    const chars = characterRemaining(row.credits);
+    if (chars !== null) {
+      healthy.push(`${row.id}: ${chars} characters left`);
+    }
+  }
+  if (unavailable.length > 0) {
+    return {
+      check: "upstream_balance",
+      severity: "fail",
+      detail: unavailable.join("; ")
+    };
+  }
+  if (short.length > 0) {
+    return {
+      check: "upstream_balance",
+      severity: "warn",
+      detail: short.join("; ")
+    };
+  }
+  if (exposed === 0) {
+    return ok("no prepaid balance exposed on any upstream (see credits.status)");
+  }
+  if (healthy.length === 0) {
+    return ok(
+      `${exposed} prepaid host(s) answered ok without a usable balance figure`
+    );
+  }
+  return ok(healthy.join("; "));
+}
+function isBurn(value) {
+  if (value === null || typeof value !== "object") return false;
+  const b = value;
+  return typeof b.window_days === "number" && typeof b.requests === "number" && typeof b.list_micros === "number" && typeof b.cost_micros === "number" && typeof b.daily_list_micros === "number";
+}
+function isCreditsReport(value) {
+  return value !== null && typeof value === "object" && typeof value.status === "string";
+}
+function parseCreditsRows(upstreams) {
+  const rows = [];
+  for (let i = 0; i < upstreams.length; i++) {
+    const row = upstreams[i];
+    if (row === null || typeof row !== "object") {
+      return { ok: false, reason: `upstreams[${i}] is not an object` };
+    }
+    const r = row;
+    if (typeof r.id !== "string" || r.id === "") {
+      return { ok: false, reason: `upstreams[${i}].id missing` };
+    }
+    if (!isCreditsReport(r.credits)) {
+      return { ok: false, reason: `upstreams[${i}].credits missing or invalid` };
+    }
+    if (!isBurn(r.burn)) {
+      return { ok: false, reason: `upstreams[${i}].burn missing or invalid` };
+    }
+    rows.push({ id: r.id, credits: r.credits, burn: r.burn });
+  }
+  return { ok: true, upstreams: rows };
+}
+async function fetchCreditsProbe(baseUrl, token, fetcher = fetch) {
+  if (baseUrl === null || baseUrl === "") {
+    return { ok: false, reason: "no base URL given" };
+  }
+  if (token === null || token === "") {
+    return {
+      ok: false,
+      reason: "no CRUISE_DOCTOR_TOKEN (operator admin Bearer; machine credential is #91)"
+    };
+  }
+  const url = `${baseUrl.replace(/\/$/, "")}/admin/upstreams/credits`;
+  let response;
+  try {
+    response = await fetcher(url, { headers: { authorization: `Bearer ${token}`, accept: "application/json" } });
+  } catch (cause) {
+    return { ok: false, reason: String(cause).slice(0, 80) };
+  }
+  if (!response.ok) {
+    return { ok: false, reason: `HTTP ${response.status} from ${url}` };
+  }
+  let body;
+  try {
+    body = await response.json();
+  } catch {
+    return { ok: false, reason: "credits response was not JSON" };
+  }
+  const upstreams = body.upstreams;
+  if (!Array.isArray(upstreams)) {
+    return { ok: false, reason: "credits response missing upstreams[]" };
+  }
+  return parseCreditsRows(upstreams);
+}
+
+// src/doctor-cache.ts
+var ok2 = (check, detail) => ({ check, severity: "ok", detail });
+var fail = (check, detail) => ({ check, severity: "fail", detail });
+var unchecked = (check, detail) => ({ check, severity: "warn", detail: `not checked \u2014 ${detail}` });
+async function cloudflare(api, path) {
+  try {
+    const response = await (api.fetcher ?? fetch)(`https://api.cloudflare.com/client/v4/accounts/${api.accountId}${path}`, {
+      headers: { authorization: `Bearer ${api.token}` }
+    });
+    const body = await response.json();
+    if (!response.ok || body.success !== true || body.result === void 0) {
+      return { ok: false, status: response.status, detail: body.errors?.map((e) => e.message).join("; ") ?? `HTTP ${response.status}` };
+    }
+    return { ok: true, result: body.result, info: body.result_info ?? null };
+  } catch (cause) {
+    return { ok: false, status: 0, detail: String(cause).slice(0, 80) };
+  }
+}
+var PAGE = 50;
+var MAX_PAGES = 20;
+async function cloudflareAll(api, path) {
+  const all = [];
+  for (let page = 1; page <= MAX_PAGES; page += 1) {
+    const got = await cloudflare(api, `${path}?per_page=${PAGE}&page=${page}`);
+    if (!got.ok) return got;
+    all.push(...got.result);
+    const more = got.info?.total_pages !== void 0 ? page < got.info.total_pages : got.result.length === PAGE;
+    if (!more) return { ok: true, result: all, info: null };
+  }
+  return { ok: false, status: 0, detail: `more than ${MAX_PAGES * PAGE} entries at ${path}; not read to the end` };
+}
+var plainText = (bindings, name) => bindings.find((b) => b.type === "plain_text" && b.name === name)?.text ?? null;
+async function cacheSwitches(api) {
+  if (api === null) {
+    return [unchecked("cache_mode", "no API context"), unchecked("cache_store", "no API context")];
+  }
+  const settings = await cloudflare(api, `/workers/scripts/${api.scriptName}/settings`);
+  if (!settings.ok) {
+    return [unchecked("cache_mode", settings.detail), unchecked("cache_store", settings.detail)];
+  }
+  const bindings = settings.result.bindings ?? [];
+  const mode = plainText(bindings, "CACHE_MODE") ?? "unset";
+  const store = bindings.find((b) => b.type === "kv_namespace" && b.name === "CACHE") ?? null;
+  const namespace = store?.namespace_id ?? null;
+  let modeFinding;
+  if (mode !== "off" && mode !== "model" && mode !== "lane") {
+    modeFinding = fail("cache_mode", `CACHE_MODE=${mode} is not a mode the Worker knows, so it reads as off`);
+  } else if (mode !== "off" && namespace === null) {
+    modeFinding = fail(
+      "cache_mode",
+      `CACHE_MODE=${mode} with no CACHE binding \u2014 the mode is read, nothing is served or stored, and every response says miss`
+    );
+  } else {
+    modeFinding = ok2("cache_mode", `CACHE_MODE=${mode}, ${namespace === null ? "no CACHE binding" : `CACHE bound to ${namespace}`}`);
+  }
+  let storeFinding;
+  if (namespace === null) {
+    storeFinding = unchecked("cache_store", "no CACHE binding on this deployment");
+  } else {
+    const found = await cloudflare(api, `/storage/kv/namespaces/${namespace}`);
+    storeFinding = found.ok ? ok2("cache_store", `namespace ${namespace}${found.result.title ? ` (${found.result.title})` : ""} exists`) : found.status === 404 ? fail("cache_store", `CACHE names namespace ${namespace}, which the account does not have`) : unchecked("cache_store", found.detail);
+  }
+  return [modeFinding, storeFinding];
+}
+async function compressSwitch(api, db) {
+  if (api === null) return unchecked("compress_mode", "no API context");
+  const settings = await cloudflare(api, `/workers/scripts/${api.scriptName}/settings`);
+  if (!settings.ok) return unchecked("compress_mode", settings.detail);
+  const mode = plainText(settings.result.bindings ?? [], "COMPRESS_MODE");
+  const row = await db.prepare(
+    `SELECT (SELECT count(*) FROM projects WHERE compress = 1) AS projects,
+              (SELECT count(*) FROM lanes WHERE compress = 1 AND enabled = 1 AND archived_at IS NULL) AS lanes`
+  ).first();
+  const projects = `${row?.projects ?? 0} project(s) and ${row?.lanes ?? 0} lane(s) switched on`;
+  if (mode !== null && mode !== "off" && mode !== "on") {
+    return fail("compress_mode", `COMPRESS_MODE=${mode} is not a mode the Worker knows, so it reads as off; ${projects}`);
+  }
+  return ok2("compress_mode", `COMPRESS_MODE=${mode ?? "unset (reads as off)"}; ${projects}`);
+}
+async function gatewayCacheOff(api, environment = null) {
+  if (api === null) return unchecked("gateway_cache", "no API context");
+  const reachesNoGateway = CREDENTIAL_FREE_ENVIRONMENTS.includes(environment ?? "");
+  const settings = await cloudflare(api, `/workers/scripts/${api.scriptName}/settings`);
+  if (!settings.ok) return unchecked("gateway_cache", settings.detail);
+  const named = ["GATEWAY_SENSITIVE", "GATEWAY_INTERNAL"].map((v) => plainText(settings.result.bindings ?? [], v)).filter((g) => g !== null);
+  const gateways = await cloudflareAll(api, "/ai-gateway/gateways");
+  if (!gateways.ok) return unchecked("gateway_cache", gateways.detail);
+  const unreadable = gateways.result.filter((g) => typeof g.cache_ttl !== "number" || !Number.isFinite(g.cache_ttl)).map((g) => g.id);
+  if (unreadable.length > 0) return unchecked("gateway_cache", `no numeric cache_ttl on ${unreadable.join(", ")}`);
+  const byId = new Map(gateways.result.map((g) => [g.id, g.cache_ttl]));
+  const missing = named.filter((id) => !byId.has(id));
+  const caching = [...byId].filter(([, ttl]) => ttl !== 0).map(([id, ttl]) => `${id} cache_ttl=${ttl}`);
+  if (missing.length > 0 && !reachesNoGateway) {
+    return fail("gateway_cache", `the deployment names a gateway the account does not have: ${missing.join(", ")}`);
+  }
+  if (caching.length > 0) return fail("gateway_cache", `a second cache is on: ${caching.join(", ")} \u2014 one cache, ours (#275)`);
+  const ttls = `cache_ttl 0 on ${[...byId.keys()].join(", ") || "no gateways"}`;
+  return ok2(
+    "gateway_cache",
+    missing.length > 0 ? `${ttls}; this deployment sends to none of them (${missing.join(", ")} name no gateway)` : ttls
+  );
+}
+async function misplacedCachePools(db) {
+  const { results } = await db.prepare(
+    `SELECT k.cache_pool AS pool, count(*) AS keys,
+              sum(CASE WHEN k.cache_pool = 'platform' AND t.operator <> 1 THEN 1 ELSE 0 END) AS misplaced
+         FROM virtual_keys k
+         JOIN projects p ON p.id = k.project_id
+         JOIN tenants t ON t.id = p.tenant_id
+        WHERE k.revoked_at IS NULL
+        GROUP BY k.cache_pool ORDER BY k.cache_pool`
+  ).all();
+  const counts = results.map((r) => `${r.pool} ${r.keys}`).join(", ");
+  const misplaced = results.reduce((total, r) => total + r.misplaced, 0);
+  return misplaced === 0 ? ok2("cache_pools", `live keys by cache pool: ${counts === "" ? "none issued" : counts}`) : {
+    check: "cache_pools",
+    severity: "fail",
+    detail: `${misplaced} live key(s) marked 'platform' in a tenant that does not operate Cruise, so one project's cached answers reach another's`
+  };
+}
+
+// src/doctor-gateway.ts
+async function customerGateways(db) {
+  const { results } = await db.prepare(
+    `SELECT p.id FROM projects p JOIN tenants t ON t.id = p.tenant_id
+        WHERE p.gateway = 'gw-internal' AND t.operator <> 1
+        ORDER BY p.id`
+  ).all();
+  return results.length === 0 ? { check: "customer_gateways", severity: "ok", detail: "no project outside the operator's tenant is on gw-internal" } : {
+    check: "customer_gateways",
+    severity: "fail",
+    detail: `on gw-internal, which logs payloads, in a tenant that does not operate Cruise: ${results.map((r) => r.id).join(", ")}`
+  };
+}
+
+// src/doctor-demo.ts
+function names(ids, limit = 8) {
+  return ids.length <= limit ? ids.join(", ") : `${ids.slice(0, limit).join(", ")} and ${ids.length - limit} more`;
+}
+var routable = async (db) => (await db.prepare(`SELECT id, modality FROM models WHERE enabled = 1 AND ${FRESHLY_MEASURED_SQL} ORDER BY id`).all()).results;
+async function catalogueParity(demo, mirror) {
+  if (mirror.db === null) {
+    return [
+      {
+        check: "catalogue_parity",
+        severity: "warn",
+        detail: `not checked \u2014 could not open ${mirror.name}, so nothing compared this mirror`
+      }
+    ];
+  }
+  const [here, there] = await Promise.all([routable(demo), routable(mirror.db)]);
+  const servable = new Set(DEMO_SERVABLE_MODALITIES);
+  const demoServable = here.filter((r) => servable.has(r.modality));
+  const productionServable = there.filter((r) => servable.has(r.modality));
+  const demoIds = new Set(demoServable.map((r) => r.id));
+  const productionIds = new Set(productionServable.map((r) => r.id));
+  const missing = productionServable.filter((r) => !demoIds.has(r.id)).map((r) => r.id);
+  const extra = [...demoIds].filter((id) => !productionIds.has(id));
+  const unroutable = here.filter((r) => !servable.has(r.modality)).map((r) => `${r.id} (${r.modality})`);
+  const wrong = [
+    missing.length > 0 ? `${mirror.name} serves and the demo does not: ${names(missing)}` : "",
+    extra.length > 0 ? `the demo serves and ${mirror.name} does not: ${names(extra)}` : "",
+    unroutable.length > 0 ? `routable here with no emulator route to answer on: ${names(unroutable)}` : ""
+  ].filter((part) => part !== "");
+  const parity = wrong.length === 0 ? {
+    check: "catalogue_parity",
+    // Says what it compared, not more. The set is equal **within the
+    // modalities the emulator can serve**, which is the whole claim this
+    // check is entitled to make while #336 is open; `demo_modalities`
+    // carries the rest. An earlier wording said "the same set production
+    // serves" flat, and review was right that it read as a guarantee
+    // about the full catalogue.
+    severity: "ok",
+    detail: `${demoIds.size} routable model id(s) in ${[...servable].join(", ")}, the same set ${mirror.name} serves \u2014 other modalities are demo_modalities, not this check`
+  } : {
+    check: "catalogue_parity",
+    severity: "fail",
+    detail: `${wrong.join("; ")} \u2014 run migrate:demo then demo:seed`
+  };
+  const unservable = there.filter((r) => !servable.has(r.modality));
+  const modalities = unservable.length === 0 ? { check: "demo_modalities", severity: "ok", detail: `the emulator has a route for every modality ${mirror.name} serves` } : {
+    check: "demo_modalities",
+    severity: "warn",
+    detail: `no emulator route, so unmeasured here: ${names(unservable.map((r) => `${r.id} (${r.modality})`))}`
+  };
+  return [parity, modalities];
+}
+
+// src/doctor-upstreams.ts
+var ok3 = (check, detail) => ({ check, severity: "ok", detail });
+var ANSWERED = "CASE WHEN http_status < 400 OR http_status = 429 THEN 1 ELSE 0 END";
+var REFUSED = "CASE WHEN http_status >= 400 AND http_status < 500 AND http_status NOT IN (401, 402, 408, 429) THEN 1 ELSE 0 END";
+async function unansweredRuns(db, runLength) {
+  const { results } = await db.prepare(
+    `WITH worked AS (
+         SELECT model FROM ledger
+          GROUP BY model
+         HAVING sum(${ANSWERED}) > 0
+       ),
+       -- Every attempt, not only the served one (#647). A chain that falls
+       -- over hides a failing first candidate behind the one that answered,
+       -- so an upstream fallback was rescuing every request from read as
+       -- healthy here. A failed attempt carries its request's time, and one
+       -- that never answered (NULL status) counts as dark, as a 5xx does.
+       attempts AS (
+         -- The served attempt came last in its chain, so among rows sharing
+         -- its time it is the most recent: served sorts it first, with no
+         -- position sentinel tied to MAX_CHAIN (review of PR #672).
+         SELECT m.upstream_id AS upstream, l.http_status, l.at, 1 AS served, 0 AS position
+           FROM ledger l
+           JOIN models m ON m.id = l.model
+           JOIN worked w ON w.model = l.model
+         UNION ALL
+         SELECT a.upstream_id, a.http_status, l.at, 0 AS served, a.position
+           FROM ledger_attempts a
+           JOIN ledger l ON l.id = a.ledger_id
+           JOIN worked w ON w.model = a.model
+       ),
+       recent AS (
+         SELECT upstream, http_status,
+                row_number() OVER (PARTITION BY upstream ORDER BY at DESC, served DESC, position DESC) AS rn
+           FROM attempts
+       )
+       SELECT upstream, count(*) AS n,
+              sum(${ANSWERED}) AS answered,
+              sum(${REFUSED}) AS refused,
+              group_concat(DISTINCT CASE WHEN ${REFUSED} THEN http_status END) AS statuses
+         FROM recent
+        WHERE rn <= ?1
+        GROUP BY upstream
+       HAVING n >= ?1 AND answered = 0
+        ORDER BY upstream`
+  ).bind(runLength).all();
+  return results;
+}
+var DARK = "5xx, or 401/402/408 \u2014 our credential, our account or our clock, not the request";
+async function failingUpstreams(db, runLength = 6) {
+  const dark = (await unansweredRuns(db, runLength)).filter((r) => (r.n - r.refused) * 2 >= r.n);
+  return dark.length === 0 ? ok3("upstream_health", "every upstream has answered something recently") : {
+    check: "upstream_health",
+    severity: "fail",
+    detail: dark.map(
+      (r) => r.refused === 0 ? `${r.upstream}: last ${r.n} requests all failed upstream (${DARK})` : `${r.upstream}: ${r.n - r.refused} of the last ${r.n} requests failed upstream (${DARK}), ${r.refused} refused`
+    ).join("; ")
+  };
+}
+async function refusedRequests(db, runLength = 6) {
+  const refused = (await unansweredRuns(db, runLength)).filter((r) => r.refused > 0);
+  return refused.length === 0 ? ok3("upstream_refusals", "no upstream is refusing our requests") : {
+    check: "upstream_refusals",
+    severity: "fail",
+    detail: refused.map(
+      (r) => `${r.upstream}: ${r.refused} of the last ${r.n} requests refused (${r.statuses ?? "4xx"}) \u2014 the request, the model id or the entitlement, not the provider` + (r.refused === r.n ? "" : `; the other ${r.n - r.refused} failed upstream`)
+    ).join("; ")
+  };
+}
+async function neverAnswered(db, attempts = 3) {
+  const { results } = await db.prepare(
+    `SELECT m.upstream_id AS upstream, count(DISTINCT l.model) AS models, count(*) AS tried
+         FROM ledger l JOIN models m ON m.id = l.model
+        WHERE l.model IN (
+                SELECT model FROM ledger
+                 GROUP BY model
+                HAVING count(*) >= ?1
+                   AND sum(CASE WHEN http_status < 400 OR http_status = 429 THEN 1 ELSE 0 END) = 0)
+        GROUP BY m.upstream_id
+        ORDER BY m.upstream_id`
+  ).bind(attempts).all();
+  return results.length === 0 ? ok3("never_answered", "every catalogued model has succeeded at least once") : {
+    check: "never_answered",
+    severity: "warn",
+    detail: results.map((r) => `${r.upstream}: ${r.models} model(s) have never succeeded in ${r.tried} attempts`).join("; ")
+  };
+}
+
+// src/doctor-components.ts
+async function componentDrift(db, days = 7, limit = 500) {
+  const { results } = await db.prepare(
+    `SELECT l.id, l.model, l.at, l.prompt_tokens, l.completion_tokens, l.cached_input_tokens,
+              l.list_micros, l.cost_micros, l.cache_status,
+              COALESCE(t.billable, 0) AS billable,
+              COALESCE((SELECT sum(c.micros) FROM ledger_components c WHERE c.ledger_id = l.id), 0) AS components
+         FROM ledger l
+         LEFT JOIN tenants t ON t.id = l.tenant_id
+        WHERE l.priced = 1 AND l.at >= datetime('now', ?1)
+          AND (EXISTS (SELECT 1 FROM ledger_components c WHERE c.ledger_id = l.id)
+               OR EXISTS (SELECT 1 FROM price_components p WHERE p.model = l.model))
+        ORDER BY l.at DESC
+        LIMIT ?2`
+  ).bind(`-${days} days`, limit).all();
+  const drifted = [];
+  for (const row of results) {
+    const at = /* @__PURE__ */ new Date(`${row.at.replace(" ", "T")}Z`);
+    const tokens = await priceUsage(
+      db,
+      row.model,
+      { prompt_tokens: row.prompt_tokens, completion_tokens: row.completion_tokens, cached_input_tokens: row.cached_input_tokens },
+      at
+    );
+    const expected = tokens.micros + row.components;
+    if (expected !== row.list_micros) {
+      drifted.push(`${row.id} (${row.model}): list ${row.list_micros}, tokens ${tokens.micros} + components ${row.components}`);
+      continue;
+    }
+    if (row.billable === 0 && row.cache_status !== "HIT" && row.cost_micros !== row.list_micros) {
+      drifted.push(`${row.id} (${row.model}): cost ${row.cost_micros} against list ${row.list_micros}, with no discount or markup in play`);
+    } else if (row.cache_status === "HIT" && row.billable === 0) {
+      const charge = hitCharge(row.list_micros, await hitBasisPointsInForce(db, at));
+      if (row.cost_micros !== charge) {
+        drifted.push(`${row.id} (${row.model}): cache hit charged ${row.cost_micros}, published fraction of ${row.list_micros} is ${charge}`);
+      }
+    }
+  }
+  return drifted.length === 0 ? { check: "ledger_components", severity: "ok", detail: `${results.length} row(s) re-derive exactly from tokens, lines and the rate in force` } : {
+    check: "ledger_components",
+    severity: "fail",
+    detail: `${drifted.length} row(s) whose total is not what the rules produce: ${drifted.slice(0, 5).join("; ")}${drifted.length > 5 ? "; \u2026" : ""}`
+  };
+}
+
+// src/doctor-calendar.ts
+var CHECK = "peak_calendar";
+var NOTICE_MONTH = 10;
+async function peakCalendar(db, now = /* @__PURE__ */ new Date()) {
+  const { results } = await db.prepare(
+    `SELECT upstream_id, max(substr(day, 1, 4)) AS last_year
+         FROM peak_exempt_days GROUP BY upstream_id ORDER BY upstream_id`
+  ).all();
+  if (results.length === 0) {
+    return { check: CHECK, severity: "ok", detail: "no host has a peak-exempt calendar to run out" };
+  }
+  const year = now.getUTCFullYear();
+  const owed = now.getUTCMonth() >= NOTICE_MONTH ? year + 1 : year;
+  const lapsed = results.filter((r) => Number(r.last_year) < year);
+  const due = results.filter((r) => Number(r.last_year) < owed && Number(r.last_year) >= year);
+  if (lapsed.length > 0) {
+    return {
+      check: CHECK,
+      severity: "fail",
+      detail: `no exempt days for ${year}, so its holidays bill at peak: ${lapsed.map((r) => `${r.upstream_id} (last ${r.last_year})`).join(", ")} \u2014 add the year's notice as a migration (0092's shape)`
+    };
+  }
+  if (due.length > 0) {
+    return {
+      check: CHECK,
+      severity: "warn",
+      detail: `no exempt days for ${owed} yet: ${due.map((r) => r.upstream_id).join(", ")} \u2014 its notice is due; until a migration adds it, ${owed}'s holidays bill at peak`
+    };
+  }
+  return {
+    check: CHECK,
+    severity: "ok",
+    detail: `exempt days cover ${owed}: ${results.map((r) => `${r.upstream_id} through ${r.last_year}`).join(", ")}`
+  };
+}
+
+// src/doctor-wallet.ts
+var ok4 = (check, detail) => ({ check, severity: "ok", detail });
+async function walletCannotWarn(db) {
+  const { results } = await db.prepare(
+    `SELECT subject FROM budgets
+        WHERE scope = 'tenant' AND hard_micros IS NOT NULL AND soft_micros IS NULL
+        ORDER BY subject`
+  ).all();
+  return results.length === 0 ? ok4("wallet_warning", "every capped wallet has a soft threshold to warn at") : {
+    check: "wallet_warning",
+    severity: "warn",
+    detail: `no soft threshold, so the wallet goes from serving to refusing with no 'low' state: ${results.map((r) => r.subject).join(", ")}`
+  };
+}
+async function signupPrepaid(db, environment) {
+  if (environment !== null && !prepaidOn(environment)) return [];
+  const pending = await unprepaidSignups(db);
+  if (pending.length === 0) return [ok4("signup_prepaid", "every signup tenant holds a wallet")];
+  return [
+    {
+      check: "signup_prepaid",
+      severity: "fail",
+      detail: `no wallet, so serving without credit: ${pending.map((t) => t.id).join(", ")} \u2014 npm run tenant:prepay`
+    }
+  ];
+}
+async function walletDrift(db) {
+  const { results } = await db.prepare(
+    `SELECT b.subject, b.hard_micros,
+              COALESCE((SELECT sum(micros) FROM credits c WHERE c.tenant_id = b.subject), 0) AS granted
+         FROM budgets b
+        WHERE b.scope = 'tenant' AND b.hard_micros IS NOT NULL
+          AND b.hard_micros != COALESCE((SELECT sum(micros) FROM credits c WHERE c.tenant_id = b.subject), 0)`
+  ).all();
+  return results.length === 0 ? ok4("wallet_counter", "every wallet cap equals the sum of its grants") : {
+    check: "wallet_counter",
+    severity: "fail",
+    detail: results.map((r) => `${r.subject}: cap ${r.hard_micros} against ${r.granted} granted`).join("; ")
+  };
+}
+
+// src/doctor.ts
+var ok5 = (check, detail) => ({ check, severity: "ok", detail });
+async function uncappedProjects(db) {
+  const { results } = await db.prepare(
+    `SELECT p.id FROM projects p
+        LEFT JOIN budgets b ON b.scope = 'project' AND b.subject = p.id
+        WHERE b.subject IS NULL`
+  ).all();
+  return results.length === 0 ? ok5("project_budgets", "every project has a budget row") : {
+    check: "project_budgets",
+    severity: "fail",
+    detail: `uncapped and serving: ${results.map((r) => r.id).join(", ")}`
+  };
+}
+async function ownerlessTenants(db) {
+  const { results } = await db.prepare(
+    `SELECT t.id FROM tenants t
+        LEFT JOIN memberships m ON m.tenant_id = t.id AND m.role = 'owner'
+        WHERE m.uid IS NULL`
+  ).all();
+  return results.length === 0 ? ok5("tenant_owners", "every tenant has an owner") : {
+    check: "tenant_owners",
+    severity: "fail",
+    detail: `no owner, so nobody can grant one: ${results.map((r) => r.id).join(", ")}`
+  };
+}
+async function unpricedRoutableModels(db) {
+  const { results } = await db.prepare(
+    `SELECT m.id FROM models m
+        WHERE m.enabled = 1 AND m.measured_at IS NOT NULL
+          AND NOT EXISTS (SELECT 1 FROM model_prices p WHERE p.model = m.id)
+          -- A component-priced row has no token part and is priced by the
+          -- component its bill requires (0080, src/billing-block.ts); the log
+          -- that holds it is the one this must read for that shape.
+          AND NOT (m.priced_by = 'component'
+                   AND EXISTS (SELECT 1 FROM model_components c
+                                WHERE c.model = m.id
+                                  AND c.component = CASE m.modality
+                                    WHEN 'speech' THEN 'character'
+                                    WHEN 'image' THEN 'image'
+                                    ELSE NULL
+                                  END))`
+  ).all();
+  return results.length === 0 ? ok5("model_prices", "every routable model has a price") : {
+    check: "model_prices",
+    severity: "fail",
+    detail: `routable and unpriced, so reserved at a flat rate: ${results.map((r) => r.id).join(", ")}`
+  };
+}
+async function expiringMeasurements(db, graceDays = 7) {
+  const { results } = await db.prepare(
+    `SELECT count(*) AS n, min(measured_at) AS oldest FROM models
+        WHERE enabled = 1 AND measured_at IS NOT NULL
+          AND julianday('now') - julianday(measured_at) > ?`
+  ).bind(30 - graceDays).first().then((row2) => ({ results: row2 ? [row2] : [] }));
+  const row = results[0];
+  return !row || row.n === 0 ? ok5("measurement_age", "no measurement is within a week of expiring") : {
+    check: "measurement_age",
+    severity: "warn",
+    detail: `${row.n} model(s) expire within ${graceDays} days, oldest measured ${row.oldest} \u2014 conformance has that long to run green`
+  };
+}
+async function emptyLanes(db) {
+  const { results } = await db.prepare(
+    `SELECT l.id FROM lanes l
+        WHERE l.archived_at IS NULL
+          AND NOT EXISTS (
+            SELECT 1 FROM lane_members lm
+              JOIN models m ON m.id = lm.model_id
+             WHERE lm.lane_id = l.id AND m.enabled = 1 AND m.measured_at IS NOT NULL
+          )`
+  ).all();
+  return results.length === 0 ? ok5("lane_members", "every lane has at least one routable member") : {
+    check: "lane_members",
+    severity: "fail",
+    detail: `no routable member, so every request refuses: ${results.map((r) => r.id).join(", ")}`
+  };
+}
+async function saltIsLoadBearing(db, baseUrl, fetcher = fetch) {
+  if (baseUrl === null || baseUrl === "") {
+    return { check: "salt_in_use", severity: "warn", detail: "not checked \u2014 no base URL given" };
+  }
+  const before = await db.prepare(`SELECT count(*) AS n FROM rate_buckets WHERE scope = 'admin_subject'`).first();
+  const sub = `doctor-salt-probe-${crypto.randomUUID()}`;
+  const b64 = (o) => btoa(JSON.stringify(o)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+  const token = `${b64({ alg: "RS256" })}.${b64({ sub })}.not-a-signature`;
+  try {
+    await fetcher(`${baseUrl.replace(/\/$/, "")}/admin/me`, {
+      headers: { authorization: `Bearer ${token}` }
+    });
+  } catch (cause) {
+    return { check: "salt_in_use", severity: "warn", detail: `not checked \u2014 ${String(cause).slice(0, 80)}` };
+  }
+  const digest = async (salt) => [...new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(`${salt}:sub:${sub}`)))].map((b) => b.toString(16).padStart(2, "0")).join("").slice(0, 32);
+  const [absent, empty] = await Promise.all([digest("undefined"), digest("")]);
+  const row = await db.prepare(`SELECT 1 AS hit FROM rate_buckets WHERE scope = 'admin_subject' AND subject IN (?, ?) LIMIT 1`).bind(absent, empty).first();
+  if (row !== null) {
+    return {
+      check: "salt_in_use",
+      severity: "fail",
+      detail: "ACCOUNT_ID_SALT is absent or empty \u2014 every digest is computed under a publicly known salt"
+    };
+  }
+  const after = await db.prepare(`SELECT count(*) AS n FROM rate_buckets WHERE scope = 'admin_subject'`).first();
+  return (after?.n ?? 0) <= (before?.n ?? 0) ? {
+    check: "salt_in_use",
+    severity: "warn",
+    detail: "not checked \u2014 the probe reached something that charged no bucket"
+  } : ok5("salt_in_use", "the salt is set and not the empty string");
+}
+async function mixedLanes(db) {
+  const { results } = await db.prepare(
+    `SELECT lm.lane_id AS id, count(DISTINCT m.modality) AS modalities
+         FROM lane_members lm JOIN models m ON m.id = lm.model_id
+         JOIN lanes l ON l.id = lm.lane_id
+        WHERE l.archived_at IS NULL
+        GROUP BY lm.lane_id HAVING modalities > 1`
+  ).all();
+  return results.length === 0 ? ok5("lane_modalities", "no lane mixes modalities") : {
+    check: "lane_modalities",
+    severity: "fail",
+    detail: `members of two modalities, so one side is never allocated: ${results.map((r) => r.id).join(", ")}`
+  };
+}
+async function diagnose(db, baseUrl = null, api = null, peer = null, environment = null, credits = null) {
+  const findings = (await Promise.all([
+    ...peer === null ? [] : [catalogueParity(db, peer)],
+    uncappedProjects(db),
+    saltIsLoadBearing(db, baseUrl),
+    ownerlessTenants(db),
+    unpricedRoutableModels(db),
+    emptyLanes(db),
+    mixedLanes(db),
+    failingUpstreams(db),
+    refusedRequests(db),
+    neverAnswered(db),
+    // Always a finding: null means the CLI could not ask the Worker.
+    Promise.resolve(upstreamBalance(credits ?? { ok: false, reason: "no credits probe supplied" })),
+    walletDrift(db),
+    walletCannotWarn(db),
+    signupPrepaid(db, environment),
+    componentDrift(db),
+    misplacedCachePools(db),
+    customerGateways(db),
+    cacheSwitches(api),
+    compressSwitch(api, db),
+    gatewayCacheOff(api, environment),
+    expiringMeasurements(db),
+    peakCalendar(db)
+  ])).flat();
+  return {
+    findings,
+    failed: findings.filter((f) => f.severity === "fail").length,
+    warned: findings.filter((f) => f.severity === "warn").length
+  };
+}
+
+// scripts/doctor.ts
+function mirrorOf(environment) {
+  if (environment !== DEMO_ENVIRONMENT) return null;
+  try {
+    return { name: "production", db: connect("production") };
+  } catch {
+    return { name: "production", db: null };
+  }
+}
+var MARK = { ok: "ok  ", warn: "warn", fail: "FAIL" };
+run(async () => {
+  const args = parseArgs(process.argv.slice(2));
+  const environment = environmentOf(args);
+  const baseUrl = typeof args.base === "string" ? args.base : process.env.CRUISE_BASE_URL ?? null;
+  const doctorToken = process.env.CRUISE_DOCTOR_TOKEN ?? null;
+  const credits = await fetchCreditsProbe(baseUrl, doctorToken);
+  const { findings, failed, warned } = await diagnose(
+    connect(environment),
+    baseUrl,
+    workerScript(environment),
+    mirrorOf(environment),
+    environment,
+    credits
+  );
+  console.log(`Cruise \u2014 invariants, ${environment}
+`);
+  for (const f of findings) {
+    console.log(`  ${MARK[f.severity]}  ${f.check.padEnd(18)} ${f.detail}`);
+  }
+  console.log();
+  if (failed > 0) {
+    throw new Error(`${failed} invariant(s) do not hold on ${environment}. See above.`);
+  }
+  console.log(warned > 0 ? `  ${warned} warning(s), nothing failing.` : "  All invariants hold.");
+});