@axlsdk/studio 0.23.2 → 0.24.0

package/dist/cli.cjs

@@ -131,42 +131,177 @@ function sanitizeRichInputFailure(event) {
   }
   return event;
 }
+var MEASURED_METADATA_KEYS = /* @__PURE__ */ new Set([
+  "models",
+  "modelCallCounts",
+  "workflows",
+  "workflowCallCounts",
+  "tokens",
+  "agentCalls"
+]);
+function redactItemMetadata(metadata) {
+  if (!metadata) return metadata;
+  const out = {};
+  for (const [key, value] of Object.entries(metadata)) {
+    out[key] = MEASURED_METADATA_KEYS.has(key) ? value : REDACTED;
+  }
+  return out;
+}
+function projectItemFailure(failure) {
+  if (!failure || typeof failure !== "object") return void 0;
+  const f = failure;
+  if (typeof f.name !== "string") return void 0;
+  return {
+    name: f.name,
+    ...typeof f.provider === "string" ? { provider: f.provider } : {},
+    ...typeof f.status === "number" && Number.isFinite(f.status) ? { status: f.status } : {},
+    ...typeof f.retryable === "boolean" ? { retryable: f.retryable } : {},
+    ...typeof f.requestId === "string" ? { requestId: f.requestId } : {}
+  };
+}
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function redactScorerDetail(detail) {
+  if (!isPlainObject(detail)) return REDACTED;
+  const d = detail;
+  return {
+    score: d.score,
+    ...d.duration !== void 0 ? { duration: d.duration } : {},
+    ...d.cost !== void 0 ? { cost: d.cost } : {},
+    // `skipped` is a structural boolean (the `applies` predicate verdict),
+    // not user/LLM content — preserve it so the client's N/A chip renders.
+    ...d.skipped !== void 0 ? { skipped: d.skipped } : {},
+    // `outcome` and `accounting` are structural (a classification and a set
+    // of counts), so they survive redaction the way `skipped` does — without
+    // them a compliance-mode reader cannot tell a judge that was stopped on
+    // budget from one that scored 0.
+    ...d.outcome !== void 0 ? { outcome: d.outcome } : {},
+    ...d.accounting !== void 0 ? { accounting: d.accounting } : {},
+    // `diagnostics` survives for the same reason `accounting` does: it is a
+    // list of operation ids and statuses, not content.
+    ...d.diagnostics !== void 0 ? { diagnostics: d.diagnostics } : {}
+    // metadata deliberately omitted — may contain LLM scorer reasoning
+  };
+}
 function redactEvalItem(item) {
+  if (!isPlainObject(item)) return REDACTED;
+  const { failure, ...rest } = item;
+  const typed = item;
+  const projectedFailure = failure !== void 0 ? projectItemFailure(failure) : void 0;
   const scrubbed = {
-    ...item,
+    ...rest,
+    ...projectedFailure ? { failure: projectedFailure } : {},
     input: REDACTED,
     output: REDACTED,
-    ...item.annotations !== void 0 ? { annotations: REDACTED } : {},
-    ...item.error !== void 0 ? { error: REDACTED } : {},
-    ...item.scorerErrors !== void 0 ? { scorerErrors: item.scorerErrors.map(() => REDACTED) } : {}
-  };
-  if (item.scoreDetails) {
-    const detailsOut = {};
-    for (const [name, detail] of Object.entries(item.scoreDetails)) {
-      detailsOut[name] = {
-        score: detail.score,
-        ...detail.duration !== void 0 ? { duration: detail.duration } : {},
-        ...detail.cost !== void 0 ? { cost: detail.cost } : {},
-        // `skipped` is a structural boolean (the `applies` predicate verdict),
-        // not user/LLM content — preserve it so the client's N/A chip renders.
-        ...detail.skipped !== void 0 ? { skipped: detail.skipped } : {}
-        // metadata deliberately omitted — may contain LLM scorer reasoning
-      };
+    // `diagnostics` is spread through untouched on purpose: it holds operation
+    // IDs, kinds, turn/attempt indexes and a status — pointers into an artifact,
+    // never content. The records themselves are redacted by the core rule at
+    // both write time and delivery time (`redactRecordLine`).
+    ...typed.metadata !== void 0 ? {
+      metadata: isPlainObject(typed.metadata) ? redactItemMetadata(typed.metadata) : REDACTED
+    } : {},
+    ...typed.annotations !== void 0 ? { annotations: REDACTED } : {},
+    ...typed.error !== void 0 ? { error: REDACTED } : {},
+    ...typed.scorerErrors !== void 0 ? {
+      scorerErrors: Array.isArray(typed.scorerErrors) ? typed.scorerErrors.map(() => REDACTED) : REDACTED
+    } : {},
+    // `callerReport.metadata` is whatever the workflow callback returned — free-
+    // form user content, exactly like `output`, so it is dropped rather than
+    // masked (same treatment as scorer metadata below). Its sibling `cost` is a
+    // plain number and stays. (`outcome` and `accounting` are spread through
+    // untouched: both are structural counts and classifications, no content.)
+    ...typed.callerReport !== void 0 ? {
+      callerReport: isPlainObject(typed.callerReport) ? typeof typed.callerReport.cost === "number" ? { cost: typed.callerReport.cost } : {} : REDACTED
+    } : {}
+  };
+  if (typed.scoreDetails !== void 0) {
+    if (isPlainObject(typed.scoreDetails)) {
+      const detailsOut = {};
+      for (const [name, detail] of Object.entries(typed.scoreDetails)) {
+        detailsOut[name] = redactScorerDetail(detail);
+      }
+      scrubbed.scoreDetails = detailsOut;
+    } else {
+      scrubbed.scoreDetails = REDACTED;
     }
-    scrubbed.scoreDetails = detailsOut;
   }
   return scrubbed;
 }
 function redactEvalResult(result, redact) {
   if (!redact) return result;
-  const meta3 = result.metadata;
-  const scrubbedMetadata = meta3 && typeof meta3.batchFailure === "string" ? { ...meta3, batchFailure: REDACTED } : result.metadata;
+  return redactResultShape(result);
+}
+function redactResultShape(result) {
+  const scrubbedMetadata = redactResultMetadata(result.metadata);
+  const hasMultiRun = "_multiRun" in result;
+  const multiRun = result._multiRun;
   return {
     ...result,
     metadata: scrubbedMetadata,
-    items: result.items.map(redactEvalItem)
+    items: Array.isArray(result.items) ? result.items.map(redactEvalItem) : REDACTED,
+    ...hasMultiRun && multiRun !== void 0 ? {
+      _multiRun: isPlainObject(multiRun) ? redactMultiRun(multiRun) : REDACTED
+    } : {}
+  };
+}
+function redactResultMetadata(metadata) {
+  const meta3 = metadata;
+  return isPlainObject(meta3) && typeof meta3.batchFailure === "string" ? { ...meta3, batchFailure: REDACTED } : metadata;
+}
+function redactMultiRun(multiRun) {
+  const { allRuns } = multiRun;
+  return {
+    ...multiRun,
+    ...allRuns !== void 0 ? {
+      allRuns: Array.isArray(allRuns) ? allRuns.map(
+        (run) => isPlainObject(run) && Array.isArray(run.items) ? redactResultShape(run) : REDACTED
+      ) : REDACTED
+    } : {},
+    ...typeof multiRun.batchFailure === "string" ? { batchFailure: REDACTED } : {}
+  };
+}
+function redactEvalComparison(comparison, redact) {
+  if (!redact) return comparison;
+  const maskInput = (r) => ({ ...r, input: REDACTED });
+  const side = (s) => ({
+    ...s,
+    metadata: redactResultMetadata(s.metadata)
+  });
+  return {
+    ...comparison,
+    baseline: side(comparison.baseline),
+    candidate: side(comparison.candidate),
+    regressions: comparison.regressions.map(maskInput),
+    improvements: comparison.improvements.map(maskInput)
   };
 }
+function redactRecordLine(line, redact) {
+  if (!redact) return line;
+  let parsed;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    const stub = {
+      v: 1,
+      phase: "end",
+      operationId: "unknown",
+      kind: "chat",
+      transportAttempts: 1,
+      provider: "unknown",
+      model: "unknown",
+      termination: "the stored record could not be parsed",
+      captured: {
+        fidelity: "runtime_request",
+        redacted: true,
+        truncated: true,
+        omitted: ["record"]
+      }
+    };
+    return JSON.stringify(stub);
+  }
+  return JSON.stringify((0, import_axl.redactCapturedRequest)(parsed));
+}
 function redactEvalHistoryEntry(entry, redact) {
   if (!redact) return entry;
   return {
@@ -943,7 +1078,50 @@ function reduceCost(acc, event) {
   };
 }
 function emptyEvalTrendData() {
-  return { byEval: {}, totalRuns: 0, totalCost: 0 };
+  return { byEval: {}, totalRuns: 0, totalCost: 0, totalCostCompleteness: "complete" };
+}
+function worse(a, b) {
+  if (a === "unverified" || b === "unverified") return "unverified";
+  if (a === "incomplete" || b === "incomplete") return "incomplete";
+  return "complete";
+}
+function extractAccounting(data) {
+  const result = data ?? {};
+  const accounting = result.accounting;
+  if (!accounting || typeof accounting !== "object") {
+    return { cost: usable(extractCost(data)), completeness: "unverified", budgetStopped: false };
+  }
+  const completeness = accounting.completeness === "complete" || accounting.completeness === "incomplete" ? accounting.completeness : "unverified";
+  const knownCost = usable(accounting.knownCost);
+  const summary = result.summary;
+  return {
+    cost: knownCost,
+    completeness,
+    budgetStopped: isBudgetStopped(accounting.budget?.status, summary?.coverage)
+  };
+}
+function usable(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0;
+}
+function refusedWork(coverage) {
+  if (!coverage || typeof coverage !== "object") return 0;
+  const { items, scorers } = coverage;
+  let refused = count(items, "budget_skipped") + count(items, "budget_interrupted");
+  if (scorers && typeof scorers === "object") {
+    for (const s of Object.values(scorers)) {
+      refused += count(s, "budget_skipped") + count(s, "budget_interrupted");
+    }
+  }
+  return refused;
+}
+function count(block, key) {
+  if (!block || typeof block !== "object") return 0;
+  const value = block[key];
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
+}
+function isBudgetStopped(status, coverage) {
+  if (status !== "closed") return false;
+  return refusedWork(coverage) > 0;
 }
 function extractScores(data) {
   if (!data || typeof data !== "object") return {};
@@ -1010,7 +1188,7 @@ function computeScoreStats(runs) {
 }
 function reduceEvalTrends(acc, entry) {
   const scores = extractScores(entry.data);
-  const cost = extractCost(entry.data);
+  const { cost, completeness, budgetStopped } = extractAccounting(entry.data);
   const model = extractModel(entry.data);
   const duration3 = extractDuration(entry.data);
   const metadata = entry.data?.metadata;
@@ -1021,6 +1199,8 @@ function reduceEvalTrends(acc, entry) {
     id: entry.id,
     scores,
     cost,
+    completeness,
+    ...budgetStopped ? { budgetStopped: true } : {},
     ...model !== void 0 ? { model } : {},
     ...duration3 !== void 0 ? { duration: duration3 } : {},
     ...runGroupId !== void 0 ? { runGroupId } : {},
@@ -1039,12 +1219,15 @@ function reduceEvalTrends(acc, entry) {
     scoreMean: mean,
     scoreStd: std,
     costTotal: (prev?.costTotal ?? 0) + cost,
+    costCompleteness: worse(prev?.costCompleteness ?? "complete", completeness),
+    budgetStoppedRuns: (prev?.budgetStoppedRuns ?? 0) + (budgetStopped ? 1 : 0),
     runCount: (prev?.runCount ?? 0) + 1
   };
   return {
     byEval,
     totalRuns: acc.totalRuns + 1,
-    totalCost: acc.totalCost + cost
+    totalCost: acc.totalCost + cost,
+    totalCostCompleteness: worse(acc.totalCostCompleteness ?? "complete", completeness)
   };
 }
 var MAX_DURATIONS = 200;
@@ -1366,21 +1549,32 @@ var executions_default = createExecutionRoutes();
 
 // src/server/routes/sessions.ts
 var import_hono4 = require("hono");
+function mediaFields(source) {
+  const bytes = source.type === "bytes" ? source.data.byteLength : source.type === "base64" ? Math.floor(source.data.length * 3 / 4) - (source.data.endsWith("==") ? 2 : source.data.endsWith("=") ? 1 : 0) : void 0;
+  return {
+    source: source.type,
+    ...source.mediaType ? { mediaType: source.mediaType } : {},
+    ...bytes !== void 0 ? { bytes } : {}
+  };
+}
 function describeStudioInput(content) {
   if (typeof content === "string") return void 0;
   return {
     parts: content.map((part) => {
-      if (part.type === "text") return { type: "text", characters: part.text.length };
-      const { source } = part;
-      const bytes = source.type === "bytes" ? source.data.byteLength : source.type === "base64" ? Math.floor(source.data.length * 3 / 4) - (source.data.endsWith("==") ? 2 : source.data.endsWith("=") ? 1 : 0) : void 0;
-      return {
-        type: "image",
-        source: source.type,
-        ...source.mediaType ? { mediaType: source.mediaType } : {},
-        ...bytes !== void 0 ? { bytes } : {}
-        // URLs, provider-file references, and labels are intentionally not
-        // included: this is a Studio-safe descriptor, not a replay payload.
-      };
+      switch (part.type) {
+        case "text":
+          return { type: "text", characters: part.text.length };
+        case "image":
+          return { type: "image", ...mediaFields(part.source) };
+        case "audio":
+          return { type: "audio", ...mediaFields(part.source) };
+        default: {
+          const unreachable = part;
+          throw new Error(
+            `unsupported model input part: ${String(unreachable.type)}`
+          );
+        }
+      }
     })
   };
 }
@@ -1734,6 +1928,184 @@ function createCostRoutes(costAggregator) {
 // src/server/routes/evals.ts
 var import_node_crypto = require("crypto");
 var import_hono10 = require("hono");
+
+// src/server/eval-import.ts
+var ITEM_OUTCOMES = [
+  "completed",
+  "failed",
+  "cancelled",
+  "budget_skipped",
+  "budget_interrupted"
+];
+var SCORER_OUTCOMES = [
+  "scored",
+  "failed",
+  "skipped",
+  "cancelled",
+  "budget_skipped",
+  "budget_interrupted"
+];
+var EPSILON = 1e-9;
+function isFiniteNonNegative(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+function isCount(value) {
+  return isFiniteNonNegative(value) && Number.isInteger(value);
+}
+function isValidImportedBudget(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const b = value;
+  if (!isFiniteNonNegative(b.limit)) return false;
+  if (!isFiniteNonNegative(b.knownSpend)) return false;
+  if (!isFiniteNonNegative(b.knownOvershoot)) return false;
+  if (b.status !== "open" && b.status !== "closed") return false;
+  if (b.closedBy !== void 0 && b.closedBy !== "case" && b.closedBy !== "scorer" && b.closedBy !== "operation") {
+    return false;
+  }
+  if (b.knownSpend >= b.limit !== (b.status === "closed")) return false;
+  const expected = Math.max(0, b.knownSpend - b.limit);
+  return Math.abs(b.knownOvershoot - expected) <= EPSILON;
+}
+function isValidImportedCoverage(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const c = value;
+  if (!c.items || typeof c.items !== "object" || Array.isArray(c.items)) return false;
+  const items = c.items;
+  for (const key of ITEM_OUTCOMES) {
+    if (!isCount(items[key])) return false;
+  }
+  if (!c.scorers || typeof c.scorers !== "object" || Array.isArray(c.scorers)) return false;
+  for (const bucket of Object.values(c.scorers)) {
+    if (!bucket || typeof bucket !== "object" || Array.isArray(bucket)) return false;
+    const counts = bucket;
+    for (const key of SCORER_OUTCOMES) {
+      if (!isCount(counts[key])) return false;
+    }
+  }
+  return true;
+}
+function isValidImportedItemErrorRate(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const r = value;
+  const { failed, attempted, rate, limit, exceeded } = r;
+  if (!isCount(failed) || !isCount(attempted) || failed > attempted) return false;
+  if (!isFiniteNonNegative(limit) || limit > 1) return false;
+  if (typeof rate !== "number" || typeof exceeded !== "boolean") return false;
+  const expectedRate = attempted > 0 ? failed / attempted : 0;
+  if (Math.abs(rate - expectedRate) > EPSILON) return false;
+  return exceeded === (attempted > 0 && rate > limit);
+}
+function isValidImportedMultiRun(value) {
+  if (!isPlainRecord(value)) return false;
+  const { allRuns } = value;
+  if (allRuns === void 0) return true;
+  if (!Array.isArray(allRuns)) return false;
+  return allRuns.every(
+    (run) => isPlainRecord(run) && Array.isArray(run.items) && run.items.every(isPlainRecord)
+  );
+}
+function isPlainRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function sumOf(record2) {
+  if (record2 === void 0) return 0;
+  if (record2 === null || typeof record2 !== "object" || Array.isArray(record2)) return void 0;
+  let total = 0;
+  for (const value of Object.values(record2)) {
+    if (!isFiniteNonNegative(value)) return void 0;
+    total += value;
+  }
+  return total;
+}
+function allNumeric(record2) {
+  if (record2 === void 0) return true;
+  if (record2 === null || typeof record2 !== "object" || Array.isArray(record2)) return false;
+  return Object.values(record2).every(
+    (value) => typeof value === "number" && Number.isFinite(value)
+  );
+}
+function isValidImportedAccounting(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const a = value;
+  if (a.version !== 1) return false;
+  if (a.currency !== "USD") return false;
+  if (!isFiniteNonNegative(a.knownCost)) return false;
+  if (a.completeness !== "complete" && a.completeness !== "incomplete" && a.completeness !== "unverified") {
+    return false;
+  }
+  if (!allNumeric(a.reasons)) return false;
+  if (!allNumeric(a.usage)) return false;
+  const ops = a.operations;
+  if (!ops || typeof ops !== "object") return false;
+  for (const key of ["total", "settled", "unknown", "denied"]) {
+    if (!isFiniteNonNegative(ops[key])) return false;
+  }
+  if (!allNumeric(ops.byKind)) return false;
+  if (ops.total !== ops.settled + ops.unknown) return false;
+  if (a.completeness === "unverified") return true;
+  const provenanceTotal = sumOf(a.provenance);
+  if (provenanceTotal === void 0) return false;
+  if (Math.abs(provenanceTotal - a.knownCost) > EPSILON) return false;
+  const breakdown = a.breakdown;
+  if (!breakdown || typeof breakdown !== "object") return false;
+  const breakdownTotal = sumOf({
+    generation: breakdown.generation,
+    judging: breakdown.judging,
+    external: breakdown.external
+  });
+  if (breakdownTotal === void 0) return false;
+  if (Math.abs(breakdownTotal - a.knownCost) > EPSILON) return false;
+  return true;
+}
+function importedAccountingIsTrustworthy(result) {
+  if (!result || typeof result !== "object") return false;
+  const r = result;
+  if (r.accounting !== void 0 && !isValidImportedAccounting(r.accounting)) return false;
+  const budget = r.accounting?.budget;
+  if (budget !== void 0 && !isValidImportedBudget(budget)) return false;
+  const coverage = r.summary?.coverage;
+  if (coverage !== void 0 && !isValidImportedCoverage(coverage)) return false;
+  if (!Array.isArray(r.items)) return true;
+  for (const item of r.items) {
+    if (!item || typeof item !== "object") continue;
+    const i = item;
+    if (i.accounting !== void 0 && !isValidImportedAccounting(i.accounting)) return false;
+    if (i.scoreDetails && typeof i.scoreDetails === "object") {
+      for (const detail of Object.values(i.scoreDetails)) {
+        const d = detail;
+        if (d?.accounting !== void 0 && !isValidImportedAccounting(d.accounting)) return false;
+      }
+    }
+  }
+  return true;
+}
+function stripAccounting(result) {
+  const copy = structuredClone(result);
+  delete copy.accounting;
+  const summary = copy.summary;
+  if (summary && typeof summary === "object") {
+    delete summary.coverage;
+  }
+  const items = copy.items;
+  if (Array.isArray(items)) {
+    for (const item of items) {
+      if (!item || typeof item !== "object") continue;
+      const i = item;
+      delete i.accounting;
+      const details = i.scoreDetails;
+      if (details && typeof details === "object") {
+        for (const detail of Object.values(details)) {
+          if (detail && typeof detail === "object") {
+            delete detail.accounting;
+          }
+        }
+      }
+    }
+  }
+  return copy;
+}
+
+// src/server/routes/evals.ts
 function createEvalRoutes(connMgr, evalLoader) {
   const app5 = new import_hono10.Hono();
   const activeRuns = /* @__PURE__ */ new Map();
@@ -1781,6 +2153,7 @@ function createEvalRoutes(connMgr, evalLoader) {
     let runs = 1;
     let stream = false;
     let captureTraces = false;
+    let captureRequests = false;
     try {
       const body = await c.req.json().catch(() => ({}));
       if (typeof body.runs === "number" && Number.isFinite(body.runs) && body.runs > 1) {
@@ -1792,6 +2165,9 @@ function createEvalRoutes(connMgr, evalLoader) {
       if (body.captureTraces === true) {
         captureTraces = true;
       }
+      if (body.captureRequests === true) {
+        captureRequests = true;
+      }
     } catch {
     }
     if (stream) {
@@ -1815,6 +2191,7 @@ function createEvalRoutes(connMgr, evalLoader) {
                   metadata: { runGroupId, runIndex: r, batchAttempted: runs },
                   signal: ac.signal,
                   captureTraces,
+                  captureRequests,
                   onProgress: (event) => {
                     if (event.type === "run_done") return;
                     connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
@@ -1887,6 +2264,7 @@ function createEvalRoutes(connMgr, evalLoader) {
             const result = await runtime.runRegisteredEval(name, {
               signal: ac.signal,
               captureTraces,
+              captureRequests,
               onProgress: (event) => {
                 if (event.type === "run_done") return;
                 connMgr.broadcastWithWildcard(`eval:${evalRunId}`, event);
@@ -1918,7 +2296,8 @@ function createEvalRoutes(connMgr, evalLoader) {
           try {
             const result2 = await runtime.runRegisteredEval(name, {
               metadata: { runGroupId, runIndex: r, batchAttempted: runs },
-              captureTraces
+              captureTraces,
+              captureRequests
             });
             results.push(result2);
           } catch (err) {
@@ -1934,11 +2313,14 @@ function createEvalRoutes(connMgr, evalLoader) {
         const partial2 = results.length < runs;
         const failureMsg = runFailure ? redactErrorMessage(runFailure, redactOn) || String(runFailure) || void 0 : void 0;
         const aggDegraded = unionDegradedScorers(results);
+        const worstItemRate = worstItemErrorRate(results);
+        const { modelTiming: _runOneModelTiming, ...firstSummary } = first.summary;
         const result = {
           ...first,
           summary: {
-            ...first.summary,
-            ...aggDegraded.length > 0 ? { degraded: aggDegraded } : {}
+            ...firstSummary,
+            ...aggDegraded.length > 0 ? { degraded: aggDegraded } : {},
+            ...worstItemRate ? { itemErrorRate: worstItemRate } : {}
           },
           _multiRun: {
             aggregate,
@@ -1956,7 +2338,10 @@ function createEvalRoutes(connMgr, evalLoader) {
           data: redactEvalResult(result, redactOn)
         });
       } else {
-        const result = await runtime.runRegisteredEval(name, { captureTraces });
+        const result = await runtime.runRegisteredEval(name, {
+          captureTraces,
+          captureRequests
+        });
         return c.json({
           ok: true,
           data: redactEvalResult(result, redactOn)
@@ -1969,6 +2354,106 @@ function createEvalRoutes(connMgr, evalLoader) {
       );
     }
   });
+  async function resolveDiagnostics(runtime, id) {
+    const entry = await runtime.getEvalResult(id);
+    const artifactId = entry?.data?.diagnostics?.artifactId;
+    return typeof artifactId === "string" && artifactId !== "" ? { artifactId } : void 0;
+  }
+  app5.get("/evals/:id/diagnostics", async (c) => {
+    const runtime = c.get("runtime");
+    const id = c.req.param("id");
+    const resolved = await resolveDiagnostics(runtime, id);
+    if (!resolved) {
+      return c.json(
+        {
+          ok: false,
+          error: {
+            code: "NOT_FOUND",
+            message: `Eval result "${id}" has no captured requests`
+          }
+        },
+        404
+      );
+    }
+    const opened = await runtime.openDiagnosticArtifact(resolved.artifactId);
+    if (!opened) {
+      return c.json(
+        {
+          ok: false,
+          error: {
+            code: "NOT_FOUND",
+            message: `Captured requests for eval result "${id}" are no longer available`
+          }
+        },
+        404
+      );
+    }
+    return c.json({
+      ok: true,
+      data: {
+        artifactId: opened.manifest.artifactId,
+        status: opened.manifest.status,
+        ...opened.manifest.reason !== void 0 ? { reason: opened.manifest.reason } : {},
+        records: opened.manifest.records,
+        bytes: opened.manifest.bytes,
+        fidelity: opened.manifest.fidelity,
+        redaction: opened.manifest.redaction,
+        ...opened.manifest.expiresAt !== void 0 ? { expiresAt: opened.manifest.expiresAt } : {},
+        ...opened.manifest.copiedFrom !== void 0 ? { copiedFrom: opened.manifest.copiedFrom } : {}
+      }
+    });
+  });
+  app5.get("/evals/:id/diagnostics/records", async (c) => {
+    const runtime = c.get("runtime");
+    const redactOn = runtime.isRedactEnabled();
+    const id = c.req.param("id");
+    const resolved = await resolveDiagnostics(runtime, id);
+    const opened = resolved ? await runtime.openDiagnosticArtifact(resolved.artifactId) : void 0;
+    if (!opened) {
+      return c.json(
+        {
+          ok: false,
+          error: {
+            code: "NOT_FOUND",
+            message: `Captured requests for eval result "${id}" are not available`
+          }
+        },
+        404
+      );
+    }
+    let cancelled = false;
+    const lines = opened.lines[Symbol.asyncIterator]();
+    return new Response(
+      new ReadableStream({
+        async start(controller) {
+          const encoder = new TextEncoder();
+          try {
+            for (; ; ) {
+              const next = await lines.next();
+              if (next.done || cancelled) break;
+              controller.enqueue(encoder.encode(`${redactRecordLine(next.value, redactOn)}
+`));
+            }
+            if (!cancelled) controller.close();
+          } catch (error48) {
+            if (!cancelled) controller.error(error48);
+          } finally {
+            await lines.return?.().catch(() => void 0);
+          }
+        },
+        cancel() {
+          cancelled = true;
+        }
+      }),
+      {
+        status: 200,
+        headers: {
+          "Content-Type": "application/x-ndjson; charset=utf-8",
+          "Cache-Control": "no-store"
+        }
+      }
+    );
+  });
   app5.post("/evals/runs/:evalRunId/cancel", (c) => {
     const evalRunId = c.req.param("evalRunId");
     const ac = activeRuns.get(evalRunId);
@@ -1988,6 +2473,7 @@ function createEvalRoutes(connMgr, evalLoader) {
     const redactOn = runtime.isRedactEnabled();
     const name = c.req.param("name");
     const body = await c.req.json();
+    const captureRequests = body.captureRequests === true;
     if (!body.resultId || typeof body.resultId !== "string") {
       return c.json(
         { ok: false, error: { code: "BAD_REQUEST", message: "resultId is required" } },
@@ -2001,8 +2487,7 @@ function createEvalRoutes(connMgr, evalLoader) {
         404
       );
     }
-    const history = await runtime.getEvalHistory();
-    const historyEntry = history.find((h) => h.id === body.resultId);
+    const historyEntry = await runtime.getEvalResult(body.resultId);
     if (!historyEntry) {
       return c.json(
         { ok: false, error: { code: "NOT_FOUND", message: `Result "${body.resultId}" not found` } },
@@ -2015,7 +2500,8 @@ function createEvalRoutes(connMgr, evalLoader) {
       const result = await rescore(
         historyEntry.data,
         config2.scorers,
-        runtime
+        runtime,
+        captureRequests ? { captureRequests: true } : void 0
       );
       await runtime.saveEvalResult({
         id: result.id,
@@ -2069,8 +2555,13 @@ function createEvalRoutes(connMgr, evalLoader) {
         400
       );
     }
-    const history = await runtime.getEvalHistory();
-    const byId = new Map(history.map((h) => [h.id, h.data]));
+    const idsOf = (v) => Array.isArray(v) ? v : [v];
+    const requested = /* @__PURE__ */ new Set([...idsOf(body.baselineId), ...idsOf(body.candidateId)]);
+    const byId = /* @__PURE__ */ new Map();
+    for (const id of requested) {
+      const entry = await runtime.getEvalResult(id);
+      if (entry) byId.set(id, entry.data);
+    }
     const missing = [];
     const resolveOne = (id) => {
       const data = byId.get(id);
@@ -2106,7 +2597,10 @@ function createEvalRoutes(connMgr, evalLoader) {
     }
     try {
       const result = await runtime.evalCompare(baseline, candidate, body.options);
-      return c.json({ ok: true, data: result });
+      return c.json({
+        ok: true,
+        data: redactEvalComparison(result, redactOn)
+      });
     } catch (err) {
       return c.json(
         {
@@ -2174,6 +2668,26 @@ function createEvalRoutes(connMgr, evalLoader) {
     const workflowsFromMeta = Array.isArray(metadataObj.workflows) ? metadataObj.workflows : [];
     const primaryWorkflow = workflowsFromMeta.find((w) => typeof w === "string");
     const evalName = trim(body.eval) ?? trim(primaryWorkflow) ?? trim(firstResult.workflow) ?? "imported";
+    let readAccounting;
+    let validateRequestSidecar;
+    try {
+      ({ readAccounting, validateRequestSidecar } = await import("@axlsdk/eval"));
+    } catch {
+      readAccounting = void 0;
+      validateRequestSidecar = void 0;
+    }
+    let sidecarLines;
+    if (body.requests !== void 0 && body.requests !== null) {
+      if (!validateRequestSidecar) {
+        return bad("importing captured requests requires @axlsdk/eval to be installed");
+      }
+      const validation = validateRequestSidecar(body.requests);
+      if (!validation.ok) return bad(validation.reason);
+      sidecarLines = validation.lines;
+    }
+    if (sidecarLines && validatedResults.length > 1) {
+      return bad("a requests sidecar can only accompany a single result");
+    }
     const timestamp = Date.now();
     const imported = [];
     for (const r of validatedResults) {
@@ -2183,6 +2697,42 @@ function createEvalRoutes(connMgr, evalLoader) {
         id,
         metadata: typeof r.metadata === "object" && r.metadata != null ? r.metadata : {}
       };
+      if (entry.accounting === void 0) {
+        if (entry.summary?.coverage !== void 0 && !isValidImportedCoverage(entry.summary.coverage)) {
+          delete entry.summary.coverage;
+          entry.metadata.importedAccounting = "invalid";
+        }
+        if (readAccounting) {
+          entry.accounting = readAccounting(entry);
+        }
+      } else if (importedAccountingIsTrustworthy(entry)) {
+        entry.metadata.importedAccounting = "declared";
+      } else {
+        const stripped = stripAccounting(entry);
+        entry.accounting = readAccounting ? readAccounting(stripped) : void 0;
+        entry.items = stripped.items;
+        entry.summary = stripped.summary;
+        entry.metadata.importedAccounting = "invalid";
+      }
+      if (entry.summary?.itemErrorRate !== void 0 && !isValidImportedItemErrorRate(entry.summary.itemErrorRate)) {
+        delete entry.summary.itemErrorRate;
+        entry.metadata.importedItemErrorRate = "invalid";
+      }
+      const multiRun = entry._multiRun;
+      if (multiRun !== void 0 && !isValidImportedMultiRun(multiRun)) {
+        delete entry._multiRun;
+        entry.metadata.importedMultiRun = "invalid";
+      }
+      if (sidecarLines) {
+        entry.diagnostics = await restageImportedRequests(runtime, id, sidecarLines);
+      } else if (entry.diagnostics) {
+        entry.diagnostics = {
+          ...entry.diagnostics,
+          artifactId: "",
+          status: "unavailable",
+          reason: "imported without its captured-request sidecar"
+        };
+      }
       await runtime.saveEvalResult({ id, eval: evalName, timestamp, data: entry });
       imported.push({ id, eval: evalName, timestamp });
     }
@@ -2191,12 +2741,72 @@ function createEvalRoutes(connMgr, evalLoader) {
     }
     return c.json({ ok: true, data: { imported } });
   });
+  async function restageImportedRequests(runtime, ownerId, lines) {
+    let stagedId;
+    try {
+      const staged = await runtime.stageDiagnosticArtifact({ kind: "eval", id: ownerId });
+      stagedId = staged.artifactId;
+      let bytes = 0;
+      let allRedacted = lines.length > 0;
+      for (const line of lines) {
+        await staged.sink.append(line);
+        bytes += Buffer.byteLength(line, "utf-8");
+        if (allRedacted) {
+          try {
+            allRedacted = JSON.parse(line).captured?.redacted === true;
+          } catch {
+            allRedacted = false;
+          }
+        }
+      }
+      const manifest = await runtime.finalizeDiagnosticArtifact(
+        staged.artifactId,
+        "complete",
+        void 0,
+        allRedacted ? "applied" : "none"
+      );
+      return {
+        version: 1,
+        artifactId: manifest.artifactId,
+        fidelity: "runtime_request",
+        status: manifest.status,
+        records: lines.length,
+        bytes,
+        redaction: manifest.redaction
+      };
+    } catch (error48) {
+      if (stagedId !== void 0) {
+        await runtime.rollbackDiagnosticArtifact(stagedId).catch(() => void 0);
+      }
+      return {
+        version: 1,
+        artifactId: "",
+        fidelity: "runtime_request",
+        status: "unavailable",
+        reason: `captured requests could not be stored: ${error48 instanceof Error ? error48.message : String(error48)}`,
+        records: 0,
+        bytes: 0,
+        redaction: "none"
+      };
+    }
+  }
   function closeActiveRuns() {
     for (const ac of activeRuns.values()) ac.abort();
     activeRuns.clear();
   }
   return { app: app5, closeActiveRuns };
 }
+function worstItemErrorRate(results) {
+  let worst;
+  let runsExceeded = 0;
+  for (const run of results) {
+    const r = run.summary?.itemErrorRate;
+    if (!r) continue;
+    if (r.exceeded) runsExceeded++;
+    if (!worst || r.rate > worst.rate) worst = r;
+  }
+  return worst ? { ...worst, runsExceeded } : void 0;
+}
 function unionDegradedScorers(results) {
   const byScorer = /* @__PURE__ */ new Map();
   for (const run of results) {
@@ -2207,8 +2817,8 @@ function unionDegradedScorers(results) {
       if (!existing) {
         byScorer.set(d.scorer, { ...d, runsAffected: 1 });
       } else {
-        const worse = d.rate > existing.rate ? d : existing;
-        byScorer.set(d.scorer, { ...worse, runsAffected: existing.runsAffected + 1 });
+        const worse2 = d.rate > existing.rate ? d : existing;
+        byScorer.set(d.scorer, { ...worse2, runsAffected: existing.runsAffected + 1 });
       }
     }
   }
@@ -2848,7 +3458,7 @@ __export(util_exports, {
   getSizableOrigin: () => getSizableOrigin,
   hexToUint8Array: () => hexToUint8Array,
   isObject: () => isObject,
-  isPlainObject: () => isPlainObject,
+  isPlainObject: () => isPlainObject2,
   issue: () => issue,
   joinValues: () => joinValues,
   jsonStringifyReplacer: () => jsonStringifyReplacer,
@@ -3032,7 +3642,7 @@ var allowsEval = cached(() => {
     return false;
   }
 });
-function isPlainObject(o) {
+function isPlainObject2(o) {
   if (isObject(o) === false)
     return false;
   const ctor = o.constructor;
@@ -3049,7 +3659,7 @@ function isPlainObject(o) {
   return true;
 }
 function shallowClone(o) {
-  if (isPlainObject(o))
+  if (isPlainObject2(o))
     return { ...o };
   if (Array.isArray(o))
     return [...o];
@@ -3242,7 +3852,7 @@ function omit(schema, mask) {
   return clone(schema, def);
 }
 function extend(schema, shape) {
-  if (!isPlainObject(shape)) {
+  if (!isPlainObject2(shape)) {
     throw new Error("Invalid input to extend: expected a plain object");
   }
   const checks = schema._zod.def.checks;
@@ -3265,7 +3875,7 @@ function extend(schema, shape) {
   return clone(schema, def);
 }
 function safeExtend(schema, shape) {
-  if (!isPlainObject(shape)) {
+  if (!isPlainObject2(shape)) {
     throw new Error("Invalid input to safeExtend: expected a plain object");
   }
   const def = mergeDefs(schema._zod.def, {
@@ -5540,7 +6150,7 @@ function mergeValues(a, b) {
   if (a instanceof Date && b instanceof Date && +a === +b) {
     return { valid: true, data: a };
   }
-  if (isPlainObject(a) && isPlainObject(b)) {
+  if (isPlainObject2(a) && isPlainObject2(b)) {
     const bKeys = Object.keys(b);
     const sharedKeys = Object.keys(a).filter((key) => bKeys.indexOf(key) !== -1);
     const newObj = { ...a, ...b };
@@ -5694,7 +6304,7 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
   $ZodType.init(inst, def);
   inst._zod.parse = (payload, ctx) => {
     const input = payload.value;
-    if (!isPlainObject(input)) {
+    if (!isPlainObject2(input)) {
       payload.issues.push({
         expected: "record",
         code: "invalid_type",
@@ -6702,8 +7312,8 @@ function az_default() {
 }
 
 // ../../node_modules/.pnpm/zod@4.3.6/node_modules/zod/v4/locales/be.js
-function getBelarusianPlural(count, one, few, many) {
-  const absCount = Math.abs(count);
+function getBelarusianPlural(count2, one, few, many) {
+  const absCount = Math.abs(count2);
   const lastDigit = absCount % 10;
   const lastTwoDigits = absCount % 100;
   if (lastTwoDigits >= 11 && lastTwoDigits <= 19) {
@@ -8527,8 +9137,8 @@ function hu_default() {
 }
 
 // ../../node_modules/.pnpm/zod@4.3.6/node_modules/zod/v4/locales/hy.js
-function getArmenianPlural(count, one, many) {
-  return Math.abs(count) === 1 ? one : many;
+function getArmenianPlural(count2, one, many) {
+  return Math.abs(count2) === 1 ? one : many;
 }
 function withDefiniteArticle(word) {
   if (!word)
@@ -10536,8 +11146,8 @@ function pt_default() {
 }
 
 // ../../node_modules/.pnpm/zod@4.3.6/node_modules/zod/v4/locales/ru.js
-function getRussianPlural(count, one, few, many) {
-  const absCount = Math.abs(count);
+function getRussianPlural(count2, one, few, many) {
+  const absCount = Math.abs(count2);
   const lastDigit = absCount % 10;
   const lastTwoDigits = absCount % 100;
   if (lastTwoDigits >= 11 && lastTwoDigits <= 19) {