@tangle-network/agent-eval 0.49.0 → 0.50.0

package/dist/chunk-SHTXZ4O2.js

@@ -0,0 +1,113 @@
+import {
+  llmSpans
+} from "./chunk-47X6LRCE.js";
+
+// src/contamination-guard.ts
+function checkCanaries(output, scenarios) {
+  const leaks = [];
+  for (const s of scenarios) {
+    if (!s.canary) continue;
+    if (output.includes(s.canary)) {
+      leaks.push({ scenarioId: s.id, canary: s.canary, evidence: excerpt(output, s.canary) });
+    }
+  }
+  return leaks;
+}
+function checkBehavioralCanary(output, scenario) {
+  const pattern = scenario.forbiddenPattern ?? scenario.canary;
+  if (!pattern) return null;
+  const hit = matchForbidden(output, pattern);
+  if (!hit) return null;
+  return {
+    scenarioId: scenario.id,
+    canary: pattern,
+    evidence: excerpt(output, hit)
+  };
+}
+function runBehavioralCanaries(cases) {
+  const leaks = [];
+  for (const c of cases) {
+    const leak = checkBehavioralCanary(c.output, c.scenario);
+    if (leak) leaks.push({ ...leak, runId: c.runId ?? leak.runId });
+  }
+  return leaks;
+}
+function matchForbidden(output, pattern) {
+  const re = tryParseRegex(pattern);
+  if (re) {
+    const m = output.match(re);
+    return m && m[0].length > 0 ? m[0] : null;
+  }
+  return output.includes(pattern) ? pattern : null;
+}
+function tryParseRegex(pattern) {
+  if (pattern.length < 2 || pattern[0] !== "/") return null;
+  const last = pattern.lastIndexOf("/");
+  if (last <= 0) return null;
+  const body = pattern.slice(1, last);
+  const flags = pattern.slice(last + 1);
+  if (!/^[gimsuy]*$/.test(flags)) return null;
+  try {
+    return new RegExp(body, flags);
+  } catch {
+    return null;
+  }
+}
+async function canaryLeakView(store, scenarios) {
+  const targets = scenarios.filter((s) => !!s.canary);
+  if (targets.length === 0) return [];
+  const spans = await llmSpans(store);
+  const leaks = [];
+  for (const span of spans) {
+    const output = span.output ?? "";
+    for (const s of targets) {
+      if (s.canary && output.includes(s.canary)) {
+        leaks.push({
+          scenarioId: s.id,
+          canary: s.canary,
+          runId: span.runId,
+          evidence: excerpt(output, s.canary)
+        });
+      }
+    }
+  }
+  return leaks;
+}
+var HoldoutAuditor = class {
+  scenarios;
+  accessLog = [];
+  constructor(scenarios) {
+    this.scenarios = scenarios;
+  }
+  /** Retrieve a holdout scenario for a declared purpose. Non-'evaluation' throws. */
+  get(scenarioId, purpose) {
+    if (purpose !== "evaluation" && purpose !== "debugging") {
+      throw new Error(
+        `HoldoutAuditor.get: purpose must be 'evaluation' or 'debugging', got ${purpose}`
+      );
+    }
+    const s = this.scenarios.find((x) => x.id === scenarioId);
+    if (!s) throw new Error(`holdout scenario "${scenarioId}" not found`);
+    this.accessLog.push({ scenarioId, purpose, at: Date.now() });
+    return s;
+  }
+  getAccessLog() {
+    return this.accessLog;
+  }
+};
+function excerpt(source, needle) {
+  const at = source.indexOf(needle);
+  if (at < 0) return "";
+  const start = Math.max(0, at - 30);
+  const end = Math.min(source.length, at + needle.length + 30);
+  return (start > 0 ? "\u2026" : "") + source.slice(start, end) + (end < source.length ? "\u2026" : "");
+}
+
+export {
+  checkCanaries,
+  checkBehavioralCanary,
+  runBehavioralCanaries,
+  canaryLeakView,
+  HoldoutAuditor
+};
+//# sourceMappingURL=chunk-SHTXZ4O2.js.map

package/dist/chunk-SHTXZ4O2.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/contamination-guard.ts"],"sourcesContent":["/**\n * ContaminationGuard — ensures held-out scenarios don't leak into\n * training/prompt paths, and flags model memorization.\n *\n * Three probes:\n *   1. `checkCanaries(output, scenario)` — if the scenario carries a\n *      canary token, it MUST NOT appear in the agent's output.\n *      Canaries are strings that are statistically impossible to\n *      reconstruct from the scenario description alone — so if they\n *      echo back, the model memorized them.\n *   2. `canaryLeakView(store)` — cross-corpus view of every run whose\n *      output contained a canary, with the offending scenario + run.\n *   3. `HoldoutAuditor` — wraps a Dataset and emits a structured error\n *      on any code path that reads holdout scenarios but doesn't flag\n *      `purpose: 'evaluation'`. Keeps engineers honest.\n */\n\nimport type { DatasetScenario } from './dataset'\nimport { llmSpans } from './trace/query'\nimport type { TraceStore } from './trace/store'\n\nexport interface CanaryLeak {\n  scenarioId: string\n  canary: string\n  runId?: string\n  evidence: string\n}\n\nexport function checkCanaries(output: string, scenarios: DatasetScenario[]): CanaryLeak[] {\n  const leaks: CanaryLeak[] = []\n  for (const s of scenarios) {\n    if (!s.canary) continue\n    if (output.includes(s.canary)) {\n      leaks.push({ scenarioId: s.id, canary: s.canary, evidence: excerpt(output, s.canary) })\n    }\n  }\n  return leaks\n}\n\n/**\n * Behavioral canary: tests the AGENT, not the eval grep.\n *\n *   - `forbiddenPattern` PRESENT in output ⇒ AGENT EMITTED BAD CONTENT ⇒ LEAK.\n *   - `forbiddenPattern` ABSENT from output ⇒ AGENT HELD THE LINE       ⇒ PASS.\n *\n * Use when running known-bad-prompt scenarios against the agent under\n * test and you want to know if the agent misbehaved. The classical\n * {@link checkCanaries} / {@link import('./canary').runCanaries | runCanaries}\n * test whether the eval check fires when the bad output is forced\n * into the eval flow — different question, different answer.\n *\n * Pattern resolution order (first match wins):\n *   1. `scenario.forbiddenPattern` — if it parses as `/body/flags`,\n *      treated as a regex; otherwise a literal substring.\n *   2. `scenario.canary` — literal substring fallback so the helper\n *      works on existing scenario fixtures.\n *\n * Returns `null` when nothing forbidden was found OR the scenario\n * declared no pattern.\n */\nexport function checkBehavioralCanary(\n  output: string,\n  scenario: DatasetScenario,\n): CanaryLeak | null {\n  const pattern = scenario.forbiddenPattern ?? scenario.canary\n  if (!pattern) return null\n  const hit = matchForbidden(output, pattern)\n  if (!hit) return null\n  return {\n    scenarioId: scenario.id,\n    canary: pattern,\n    evidence: excerpt(output, hit),\n  }\n}\n\n/**\n * Behavioral canary over many (scenario, output) pairs. Sibling to\n * {@link import('./canary').runCanaries | runCanaries} — same idea\n * (run-many → report) but the question being answered is \"did the\n * AGENT misbehave?\" rather than \"did the EVAL grep fire?\".\n *\n * Returns one `CanaryLeak` per pair where the agent's output\n * contained its scenario's `forbiddenPattern` (or `canary` fallback).\n */\nexport function runBehavioralCanaries(\n  cases: Array<{ scenario: DatasetScenario; output: string; runId?: string }>,\n): CanaryLeak[] {\n  const leaks: CanaryLeak[] = []\n  for (const c of cases) {\n    const leak = checkBehavioralCanary(c.output, c.scenario)\n    if (leak) leaks.push({ ...leak, runId: c.runId ?? leak.runId })\n  }\n  return leaks\n}\n\n/**\n * Resolve a forbidden-pattern string to the matched substring inside\n * `output`. `/body/flags` notation is interpreted as a regex; anything\n * else is a literal substring.\n */\nfunction matchForbidden(output: string, pattern: string): string | null {\n  const re = tryParseRegex(pattern)\n  if (re) {\n    const m = output.match(re)\n    return m && m[0].length > 0 ? m[0] : null\n  }\n  return output.includes(pattern) ? pattern : null\n}\n\nfunction tryParseRegex(pattern: string): RegExp | null {\n  if (pattern.length < 2 || pattern[0] !== '/') return null\n  const last = pattern.lastIndexOf('/')\n  if (last <= 0) return null\n  const body = pattern.slice(1, last)\n  const flags = pattern.slice(last + 1)\n  if (!/^[gimsuy]*$/.test(flags)) return null\n  try {\n    return new RegExp(body, flags)\n  } catch {\n    return null\n  }\n}\n\n/**\n * Scan the LLM-output history in a corpus; returns every case where a\n * canary from a known scenario appeared in agent output. Pass the full\n * set of scenarios whose canaries you care about (typically the whole\n * held-out slice).\n */\nexport async function canaryLeakView(\n  store: TraceStore,\n  scenarios: DatasetScenario[],\n): Promise<CanaryLeak[]> {\n  const targets = scenarios.filter((s) => !!s.canary)\n  if (targets.length === 0) return []\n  const spans = await llmSpans(store)\n  const leaks: CanaryLeak[] = []\n  for (const span of spans) {\n    const output = span.output ?? ''\n    for (const s of targets) {\n      if (s.canary && output.includes(s.canary)) {\n        leaks.push({\n          scenarioId: s.id,\n          canary: s.canary,\n          runId: span.runId,\n          evidence: excerpt(output, s.canary),\n        })\n      }\n    }\n  }\n  return leaks\n}\n\nexport class HoldoutAuditor {\n  private scenarios: DatasetScenario[]\n  private accessLog: Array<{ scenarioId: string; purpose: string; at: number }> = []\n\n  constructor(scenarios: DatasetScenario[]) {\n    this.scenarios = scenarios\n  }\n\n  /** Retrieve a holdout scenario for a declared purpose. Non-'evaluation' throws. */\n  get(scenarioId: string, purpose: 'evaluation' | 'debugging'): DatasetScenario {\n    if (purpose !== 'evaluation' && purpose !== 'debugging') {\n      throw new Error(\n        `HoldoutAuditor.get: purpose must be 'evaluation' or 'debugging', got ${purpose}`,\n      )\n    }\n    const s = this.scenarios.find((x) => x.id === scenarioId)\n    if (!s) throw new Error(`holdout scenario \"${scenarioId}\" not found`)\n    this.accessLog.push({ scenarioId, purpose, at: Date.now() })\n    return s\n  }\n\n  getAccessLog(): ReadonlyArray<{ scenarioId: string; purpose: string; at: number }> {\n    return this.accessLog\n  }\n}\n\nfunction excerpt(source: string, needle: string): string {\n  const at = source.indexOf(needle)\n  if (at < 0) return ''\n  const start = Math.max(0, at - 30)\n  const end = Math.min(source.length, at + needle.length + 30)\n  return (start > 0 ? '…' : '') + source.slice(start, end) + (end < source.length ? '…' : '')\n}\n"],"mappings":";;;;;AA4BO,SAAS,cAAc,QAAgB,WAA4C;AACxF,QAAM,QAAsB,CAAC;AAC7B,aAAW,KAAK,WAAW;AACzB,QAAI,CAAC,EAAE,OAAQ;AACf,QAAI,OAAO,SAAS,EAAE,MAAM,GAAG;AAC7B,YAAM,KAAK,EAAE,YAAY,EAAE,IAAI,QAAQ,EAAE,QAAQ,UAAU,QAAQ,QAAQ,EAAE,MAAM,EAAE,CAAC;AAAA,IACxF;AAAA,EACF;AACA,SAAO;AACT;AAuBO,SAAS,sBACd,QACA,UACmB;AACnB,QAAM,UAAU,SAAS,oBAAoB,SAAS;AACtD,MAAI,CAAC,QAAS,QAAO;AACrB,QAAM,MAAM,eAAe,QAAQ,OAAO;AAC1C,MAAI,CAAC,IAAK,QAAO;AACjB,SAAO;AAAA,IACL,YAAY,SAAS;AAAA,IACrB,QAAQ;AAAA,IACR,UAAU,QAAQ,QAAQ,GAAG;AAAA,EAC/B;AACF;AAWO,SAAS,sBACd,OACc;AACd,QAAM,QAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,sBAAsB,EAAE,QAAQ,EAAE,QAAQ;AACvD,QAAI,KAAM,OAAM,KAAK,EAAE,GAAG,MAAM,OAAO,EAAE,SAAS,KAAK,MAAM,CAAC;AAAA,EAChE;AACA,SAAO;AACT;AAOA,SAAS,eAAe,QAAgB,SAAgC;AACtE,QAAM,KAAK,cAAc,OAAO;AAChC,MAAI,IAAI;AACN,UAAM,IAAI,OAAO,MAAM,EAAE;AACzB,WAAO,KAAK,EAAE,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC,IAAI;AAAA,EACvC;AACA,SAAO,OAAO,SAAS,OAAO,IAAI,UAAU;AAC9C;AAEA,SAAS,cAAc,SAAgC;AACrD,MAAI,QAAQ,SAAS,KAAK,QAAQ,CAAC,MAAM,IAAK,QAAO;AACrD,QAAM,OAAO,QAAQ,YAAY,GAAG;AACpC,MAAI,QAAQ,EAAG,QAAO;AACtB,QAAM,OAAO,QAAQ,MAAM,GAAG,IAAI;AAClC,QAAM,QAAQ,QAAQ,MAAM,OAAO,CAAC;AACpC,MAAI,CAAC,cAAc,KAAK,KAAK,EAAG,QAAO;AACvC,MAAI;AACF,WAAO,IAAI,OAAO,MAAM,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQA,eAAsB,eACpB,OACA,WACuB;AACvB,QAAM,UAAU,UAAU,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM;AAClD,MAAI,QAAQ,WAAW,EAAG,QAAO,CAAC;AAClC,QAAM,QAAQ,MAAM,SAAS,KAAK;AAClC,QAAM,QAAsB,CAAC;AAC7B,aAAW,QAAQ,OAAO;AACxB,UAAM,SAAS,KAAK,UAAU;AAC9B,eAAW,KAAK,SAAS;AACvB,UAAI,EAAE,UAAU,OAAO,SAAS,EAAE,MAAM,GAAG;AACzC,cAAM,KAAK;AAAA,UACT,YAAY,EAAE;AAAA,UACd,QAAQ,EAAE;AAAA,UACV,OAAO,KAAK;AAAA,UACZ,UAAU,QAAQ,QAAQ,EAAE,MAAM;AAAA,QACpC,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,iBAAN,MAAqB;AAAA,EAClB;AAAA,EACA,YAAwE,CAAC;AAAA,EAEjF,YAAY,WAA8B;AACxC,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAAoB,SAAsD;AAC5E,QAAI,YAAY,gBAAgB,YAAY,aAAa;AACvD,YAAM,IAAI;AAAA,QACR,wEAAwE,OAAO;AAAA,MACjF;AAAA,IACF;AACA,UAAM,IAAI,KAAK,UAAU,KAAK,CAAC,MAAM,EAAE,OAAO,UAAU;AACxD,QAAI,CAAC,EAAG,OAAM,IAAI,MAAM,qBAAqB,UAAU,aAAa;AACpE,SAAK,UAAU,KAAK,EAAE,YAAY,SAAS,IAAI,KAAK,IAAI,EAAE,CAAC;AAC3D,WAAO;AAAA,EACT;AAAA,EAEA,eAAmF;AACjF,WAAO,KAAK;AAAA,EACd;AACF;AAEA,SAAS,QAAQ,QAAgB,QAAwB;AACvD,QAAM,KAAK,OAAO,QAAQ,MAAM;AAChC,MAAI,KAAK,EAAG,QAAO;AACnB,QAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,EAAE;AACjC,QAAM,MAAM,KAAK,IAAI,OAAO,QAAQ,KAAK,OAAO,SAAS,EAAE;AAC3D,UAAQ,QAAQ,IAAI,WAAM,MAAM,OAAO,MAAM,OAAO,GAAG,KAAK,MAAM,OAAO,SAAS,WAAM;AAC1F;","names":[]}

package/dist/{chunk-KQ26DYTQ.js → chunk-UBQGWD3O.js}

@@ -1,6 +1,6 @@
 import {
   summaryTable
-} from "./chunk-MNL6LXGQ.js";
+} from "./chunk-EGIPWXHL.js";
 import {
   VerificationError
 } from "./chunk-QYJT52YW.js";
@@ -574,4 +574,4 @@ export {
   judgeReplayGate,
   renderReleaseReport
 };
-//# sourceMappingURL=chunk-KQ26DYTQ.js.map
+//# sourceMappingURL=chunk-UBQGWD3O.js.map

package/dist/contract/index.d.ts

@@ -1,16 +1,24 @@
-import { S as Scenario, M as MutableSurface, D as DispatchContext, J as JudgeConfig, I as ImprovementDriver, G as Gate } from '../types-8u72Gc76.js';
-export { C as CampaignAggregates, a as CampaignArtifactWriter, b as CampaignCellResult, c as CampaignCostMeter, d as CampaignResult, e as CampaignTraceWriter, f as CodeSurface, g as Dispatch, h as GateContext, i as GateDecision, j as GateResult, k as GenerationCandidate, l as GenerationRecord, m as JudgeDimension, n as JudgeScore, o as Mutator, O as OptimizerConfig, p as SessionScript } from '../types-8u72Gc76.js';
-import { C as CampaignStorage, R as RunImprovementLoopResult } from '../run-improvement-loop-B-L8GgpW.js';
-export { D as DefaultProductionGateOptions, E as EvolutionaryDriverOptions, G as GepaDriverOptions, H as HeldOutGateOptions, a as RunCampaignOptions, b as RunEvalOptions, c as RunImprovementLoopOptions, d as composeGate, e as defaultProductionGate, f as evolutionaryDriver, g as fsCampaignStorage, h as gepaDriver, i as heldOutGate, j as inMemoryCampaignStorage, r as runCampaign, k as runEval, l as runImprovementLoop } from '../run-improvement-loop-B-L8GgpW.js';
-export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, O as OutcomeStore } from '../outcome-store-BxJ3DQKJ.js';
-import { HostedTenant } from '../hosted/index.js';
+import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, I as ImprovementDriver, G as Gate } from '../types-Dbj5gu8n.js';
+export { f as CampaignAggregates, g as CampaignArtifactWriter, h as CampaignCellResult, i as CampaignCostMeter, j as CampaignResult, k as CampaignTraceWriter, C as CodeSurface, D as Dispatch, l as GateContext, m as GateDecision, n as GateResult, o as GenerationCandidate, p as GenerationRecord, r as JudgeDimension, J as JudgeScore, t as Mutator, O as OptimizerConfig, v as SessionScript } from '../types-Dbj5gu8n.js';
+import { C as CampaignStorage, d as RunImprovementLoopResult } from '../run-improvement-loop-BPMjNKMJ.js';
+export { D as DefaultProductionGateOptions, E as EvolutionaryDriverOptions, G as GepaDriverOptions, H as HeldOutGateOptions, R as RunCampaignOptions, b as RunEvalOptions, c as RunImprovementLoopOptions, g as composeGate, h as defaultProductionGate, i as evolutionaryDriver, j as fsCampaignStorage, k as gepaDriver, l as heldOutGate, m as inMemoryCampaignStorage, r as runCampaign, n as runEval, p as runImprovementLoop } from '../run-improvement-loop-BPMjNKMJ.js';
+export { D as DeploymentOutcome, F as FileSystemOutcomeStore, b as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, O as OutcomeStore } from '../outcome-store-D6KWmYvj.js';
+import { a as HostedTenant, I as InsightReport, T as TraceSpanEvent } from '../index-BRxz6qov.js';
+export { F as FailureClusterInsight, b as InterRaterInsight, J as JudgeInsight, L as LiftInsight, O as OutcomeCorrelationInsight, R as Recommendation, c as ReleaseSummary, S as ScalarDistribution } from '../index-BRxz6qov.js';
+import { A as AnalystRegistry } from '../registry-8KAs18kY.js';
+import { a as DatasetScenario } from '../dataset-BlwAtYYf.js';
+import { R as RunRecord, a as RunSplitTag } from '../run-record-BGY6bHRh.js';
 import '../llm-client-BXVRUZyX.js';
 import '../errors-mje_cKOs.js';
 import '../raw-provider-sink-C46HDghv.js';
 import '../red-team-30II1T4o.js';
-import '../dataset-BlwAtYYf.js';
 import '../store-Db2Bv8Cf.js';
-import '../run-record-BGY6bHRh.js';
+import '../summary-report-B7gNRX-r.js';
+import '../failure-cluster-Cw65_5FY.js';
+import '../judge-calibration-DilmB3Ml.js';
+import '../store-CJbzDxZ2.js';
+import '../types-DhqpAi_z.js';
+import '@tangle-network/tcloud';
 
 /**
  * # `selfImprove()` — the LAND-tier one-shot.
@@ -188,6 +196,13 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
     durationMs: number;
     /** Total cost across baseline + every generation. */
     totalCostUsd: number;
+    /**
+     * Rigor packet: distributional summary, paired-bootstrap lift CI,
+     * judge stats, contamination check, recommendations. Wired through
+     * `analyzeRuns()` on the baseline + winner cells of the campaign.
+     * Hosted-tier dashboards render this as the v3-vs-v4 decision view.
+     */
+    insight: InsightReport;
     /**
      * Raw substrate result for advanced inspection — full per-generation
      * candidates, full campaign artifacts, all judge scores. Useful for
@@ -222,4 +237,186 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
  */
 declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveOptions<TScenario, TArtifact>): Promise<SelfImproveResult<TScenario, TArtifact>>;
 
-export { CampaignStorage, DispatchContext, Gate, ImprovementDriver, JudgeConfig, MutableSurface, RunImprovementLoopResult, Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, selfImprove };
+/**
+ * # `analyzeRuns()` — turn a set of agent runs into an actionable decision packet.
+ *
+ * Wires the substrate's statistical, calibration, clustering, Pareto, and
+ * release-confidence primitives into one `InsightReport`. Two top-level
+ * entry points use this function:
+ *
+ *   - `selfImprove()` calls it on the campaign output to attach a packet
+ *     to every run.
+ *   - Consumers with observed `RunRecord[]` (production traces, gold
+ *     corpora, approve/reject tables) call it directly via `analyzeRuns()`
+ *     for analysis without a closed loop.
+ *
+ * Every section is opt-in based on what the input data supports — the
+ * function never invents signal. If runs carry no judge scores, `judges`
+ * is empty. If there's no baseline/candidate split, `lift` is undefined.
+ * If no `analyst` is wired, `failureClusters` is undefined.
+ *
+ * The `recommendations` array is the human-readable layer; everything
+ * else is the evidence backing each recommendation.
+ */
+
+interface AnalyzeRunsOptions {
+    /** The runs to analyze. */
+    runs: RunRecord[];
+    /** Which split to score against when reading composite from RunOutcome.
+     *  Default: holdout when ANY run has a `holdoutScore`, else search. */
+    split?: 'search' | 'holdout' | 'auto';
+    /** Pairwise analysis configuration. When both `baselineCandidateId` and
+     *  `candidateCandidateId` are present, lift is computed on paired
+     *  (experimentId, seed) tuples shared between the two sides. */
+    baselineCandidateId?: string;
+    candidateCandidateId?: string;
+    /** Canary scenarios — checked against every run's raw output for
+     *  holdout contamination. */
+    canaryScenarios?: DatasetScenario[];
+    /** Analyst registry for failure clustering. When omitted, the
+     *  `failureClusters` section is left undefined. */
+    analyst?: AnalystRegistry;
+    /** Downstream outcome metric per run (e.g. engagement rate, approval
+     *  rate, downstream pass rate). When present, the report includes
+     *  `outcomeCorrelation` + a simple linear reward model fit. */
+    outcomeSignal?: {
+        metric: string;
+        valueByRunId: Record<string, number>;
+    };
+    /** Multi-rater feedback for inter-rater agreement. Each entry is one
+     *  rater's score for one run. Two or more raters → kappa + disagreement
+     *  triage list. */
+    raterScores?: Array<{
+        runId: string;
+        rater: string;
+        score: number;
+    }>;
+    /** Number of histogram bins for distributional summaries. Default 12. */
+    histogramBins?: number;
+    /** Decision threshold — the smallest composite lift the caller cares
+     *  about. Used by the recommendations engine to call ship vs hold.
+     *  Default 0.02. */
+    decisionThreshold?: number;
+}
+declare function analyzeRuns(opts: AnalyzeRunsOptions): Promise<InsightReport>;
+
+/**
+ * # `intake/feedback-table` — multi-rater approve/reject corpus → `RunRecord[]`.
+ *
+ * The generic shape behind Obsidian's `#approved` / `#rejected` tags, a
+ * Google Sheet, a Postgres `feedback` table, or any CSV with ratings.
+ *
+ * Caller supplies one row per (run, rater) tuple plus per-run metadata; the
+ * adapter rolls them up into the substrate-canonical `RunRecord` shape so
+ * `analyzeRuns({ runs, raterScores })` can produce inter-rater agreement,
+ * disagreement triage, and downstream recommendations.
+ *
+ * Per-run `RunRecord.outcome.searchScore` is the rater-mean rating
+ * (normalised to 0..1 when scale is supplied); `outcome.raw` carries the
+ * per-rater scores keyed by rater id for downstream attribution.
+ */
+
+interface FeedbackTableRow {
+    /** Stable id for this run — the unit a rater scored. Drives pairing
+     *  across analysis primitives. */
+    runId: string;
+    /** Identifier of the rater that produced this rating. */
+    rater: string;
+    /** The rating itself. Accepts boolean (approve/reject), 0..1 scalar,
+     *  or any numeric scale — see `scale`. */
+    rating: number | boolean;
+    /** Optional metadata carried through to `RunRecord.outcome.raw` and the
+     *  custom-shape metadata bag. */
+    metadata?: Record<string, unknown>;
+}
+interface FeedbackTableMeta {
+    runId: string;
+    /** When omitted, defaults to `'feedback-corpus'`. Used to group related
+     *  runs in `analyzeRuns()` lift analysis. */
+    experimentId?: string;
+    /** When omitted, defaults to `runId` — each run is its own candidate. */
+    candidateId?: string;
+    /** Cost in USD, when available. Set to 0 when unknown — the consumer's
+     *  cost analysis sections will collapse gracefully. */
+    costUsd?: number;
+    /** Wall-clock ms, when available. Defaults to 0. */
+    wallMs?: number;
+    /** Model identifier including snapshot. Default `unknown@unknown`. */
+    model?: string;
+    /** Optional sha256 of the prompt; default `'sha256:unknown'`. */
+    promptHash?: string;
+    /** Default `'sha256:unknown'`. */
+    configHash?: string;
+    /** Default `'unknown'`. */
+    commitSha?: string;
+    /** Default `'holdout'` — feedback corpora are by nature the holdout
+     *  signal a closed-loop improvement aims at. */
+    splitTag?: RunSplitTag;
+    /** Free-form metadata available to consumers via the cast-out path on
+     *  the resulting RunRecord. */
+    extras?: Record<string, unknown>;
+}
+interface FromFeedbackTableOptions {
+    /** Per-(run, rater) ratings. */
+    ratings: FeedbackTableRow[];
+    /** Per-run metadata. When a runId appears in `ratings` but not here, the
+     *  adapter synthesises minimal metadata with defaults documented above. */
+    meta?: FeedbackTableMeta[];
+    /** Rating scale. Provide `{ min, max }` for non-0..1 numeric scales.
+     *  Booleans are normalised: true → 1, false → 0. Default: assumes
+     *  ratings are already 0..1. */
+    scale?: {
+        min: number;
+        max: number;
+    };
+    /** When true, the rater scores are emitted into `raterScores` (a sibling
+     *  array `analyzeRuns()` accepts) instead of being averaged into the
+     *  run's `outcome.searchScore`. Default `true` — preserves rater-level
+     *  signal for inter-rater analysis. */
+    emitRaterScores?: boolean;
+}
+interface FromFeedbackTableResult {
+    runs: RunRecord[];
+    /** Rater-level scores ready to pass into `analyzeRuns({ raterScores })`
+     *  for inter-rater agreement + disagreement triage. */
+    raterScores: Array<{
+        runId: string;
+        rater: string;
+        score: number;
+    }>;
+}
+declare function fromFeedbackTable(opts: FromFeedbackTableOptions): FromFeedbackTableResult;
+
+/**
+ * # `intake/otel-spans` — OTel `TraceSpanEvent[]` → `RunRecord[]`.
+ *
+ * Turns an existing observability stream into the substrate-canonical
+ * `RunRecord` shape so consumers with logs but no eval discipline can
+ * call `analyzeRuns()` against their production traffic immediately.
+ *
+ * Pivot rule: spans are grouped by `tangle.runId` (the same attribute the
+ * hosted-tier wire format uses) or, when absent, by `traceId`. One group
+ * becomes one `RunRecord`. The root span (no `parentSpanId`) supplies:
+ *
+ *   - `runId` (the group key)
+ *   - `wallMs` from `endTimeUnixNano - startTimeUnixNano`
+ *   - `model` from `gen_ai.request.model` / `llm.model` / `tangle.model`
+ *   - cost from `cost.usd` / `gen_ai.usage.cost_usd` / `tangle.cost.usd`
+ *   - token usage from `gen_ai.usage.{input,output}_tokens`
+ *   - `outcome.searchScore` from `tangle.score` / `eval.score` when
+ *     present; `outcome.raw` collects every numeric attribute.
+ *
+ * Spans that ERRORed (`status.code === 'ERROR'`) populate `failureMode`
+ * with their `name` so `analyzeRuns()`'s failure clustering sees them.
+ */
+
+interface FromOtelSpansOptions {
+    spans: TraceSpanEvent[];
+    /** Default split tag for synthesized records. Defaults to `'holdout'`. */
+    defaultSplit?: RunSplitTag;
+    /** Default `experimentId` when not present on any span. */
+    experimentId?: string;
+}
+declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
+
+export { type AnalyzeRunsOptions, CampaignStorage, DispatchContext, type FeedbackTableMeta, type FeedbackTableRow, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, Gate, ImprovementDriver, InsightReport, JudgeConfig, MutableSurface, RunImprovementLoopResult, Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, analyzeRuns, fromFeedbackTable, fromOtelSpans, selfImprove };