@tangle-network/agent-eval 0.95.1 → 0.96.1

package/dist/adapters/http.d.ts

@@ -1,4 +1,4 @@
-import { S as Scenario, D as DispatchFn, b as DispatchContext } from '../types-DQRY8ZT-.js';
+import { S as Scenario, D as DispatchFn, b as DispatchContext } from '../types-BMahhhio.js';
 import '../run-record-CP2ObebC.js';
 import '@tangle-network/agent-interface';
 import '../errors-CzMUYo7b.js';

package/dist/adapters/langchain.d.ts

@@ -1,4 +1,4 @@
-import { S as Scenario, J as JudgeScore, D as DispatchFn, a as JudgeConfig } from '../types-DQRY8ZT-.js';
+import { S as Scenario, J as JudgeScore, D as DispatchFn, a as JudgeConfig } from '../types-BMahhhio.js';
 import '../run-record-CP2ObebC.js';
 import '@tangle-network/agent-interface';
 import '../errors-CzMUYo7b.js';

package/dist/adapters/otel.d.ts

@@ -1,5 +1,5 @@
 import { TraceSpanEvent, HostedClient } from '../hosted/index.js';
-import '../types-DQRY8ZT-.js';
+import '../types-BMahhhio.js';
 import '../run-record-CP2ObebC.js';
 import '@tangle-network/agent-interface';
 import '../errors-CzMUYo7b.js';

package/dist/analyst/index.js

@@ -16,7 +16,7 @@ import {
   behavioralAnalyst,
   buildDefaultAnalystRegistry,
   deriveEfficiencyFindings
-} from "../chunk-AQ5WQAIV.js";
+} from "../chunk-3NHEO6ZC.js";
 import "../chunk-DJWX3GVS.js";
 import {
   ANALYST_SEVERITIES,

package/dist/belief-state/index.js

@@ -17,7 +17,7 @@ import {
 } from "../chunk-4DIJWVUT.js";
 import {
   confidenceInterval
-} from "../chunk-56GC6VYO.js";
+} from "../chunk-HRGUJTER.js";
 import {
   ValidationError
 } from "../chunk-3BFEG2F6.js";

package/dist/builder-eval/index.js

@@ -2,6 +2,10 @@ import {
   SandboxHarness,
   runTestGradedScenario
 } from "../chunk-ZOPLCJSY.js";
+import {
+  pearsonR,
+  spearmanR
+} from "../chunk-HRGUJTER.js";
 import {
   judgeSpans
 } from "../chunk-UFSG7ACU.js";
@@ -227,35 +231,6 @@ function pairwise(reports, a, b) {
     spearman: spearmanR(xs, ys)
   };
 }
-function pearsonR(a, b) {
-  const mA = a.reduce((s, v) => s + v, 0) / a.length;
-  const mB = b.reduce((s, v) => s + v, 0) / b.length;
-  let num = 0, dA = 0, dB = 0;
-  for (let i = 0; i < a.length; i++) {
-    const da = a[i] - mA;
-    const db = b[i] - mB;
-    num += da * db;
-    dA += da * da;
-    dB += db * db;
-  }
-  if (dA === 0 || dB === 0) return dA === 0 && dB === 0 ? 1 : 0;
-  return num / Math.sqrt(dA * dB);
-}
-function spearmanR(a, b) {
-  return pearsonR(ranks(a), ranks(b));
-}
-function ranks(xs) {
-  const indexed = xs.map((v, i) => ({ v, i })).sort((x, y) => x.v - y.v);
-  const r = new Array(xs.length);
-  for (let i = 0; i < indexed.length; i++) {
-    let j = i;
-    while (j + 1 < indexed.length && indexed[j + 1].v === indexed[i].v) j++;
-    const avg = (i + j + 2) / 2;
-    for (let k = i; k <= j; k++) r[indexed[k].i] = avg;
-    i = j;
-  }
-  return r;
-}
 
 // src/builder-eval/project-registry.ts
 var ProjectRegistry = class {

package/dist/builder-eval/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/builder-eval/builder-session.ts","../../src/builder-eval/correlation.ts","../../src/builder-eval/project-registry.ts","../../src/builder-eval/three-layer-eval.ts"],"sourcesContent":["/**\n * BuilderSession — ties a builder-of-builders workflow together.\n *\n * Models agent-builder's shape: Project → Chat → Edit → Ship → App →\n * AppAgent. Each layer is a Run (linked via parentRunId). The\n * framework-enforced invariants:\n *\n *   - One Project → many Chats; chatId scopes runs within a project.\n *   - One Chat = one builder Run with `layer='builder'`.\n *   - One Ship = one child Run with `layer='app-build'` + SandboxHarness.\n *   - One AppScenario = one grandchild Run with `layer='app-runtime'`.\n *\n * Consumers obtain a BuilderSession, call `startChat`, drive the\n * builder agent (emitting spans), and call `ship` / `runAppScenario`\n * as the workflow progresses. The session reconstructs itself from\n * trace data via `resume(store, projectId)`.\n */\n\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from '../sandbox-harness'\nimport { SandboxHarness } from '../sandbox-harness'\nimport type { TestGradedRunResult, TestGradedScenario } from '../test-graded-scenario'\nimport { runTestGradedScenario } from '../test-graded-scenario'\nimport { TraceEmitter } from '../trace/emitter'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface BuilderSessionInit {\n  projectId: string\n  chatId?: string\n  /** Free-form: user's task description, project name, etc. Stored on the builder Run. */\n  tags?: Record<string, string>\n}\n\nexport interface ShipOptions {\n  harness: HarnessConfig\n  driver?: SandboxDriver\n  /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */\n  scenarioId?: string\n}\n\nexport interface RunAppScenarioOptions {\n  scenario: TestGradedScenario\n  /** Harness driver override; defaults to the one the session was created with. */\n  driver?: SandboxDriver\n}\n\nexport class BuilderSession {\n  private store: TraceStore\n  private builderEmitter: TraceEmitter\n  readonly projectId: string\n  readonly chatId: string\n  private builderRunId?: string\n  private lastBuildRunId?: string\n  private defaultDriver?: SandboxDriver\n\n  constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver) {\n    this.store = store\n    this.projectId = init.projectId\n    this.chatId = init.chatId ?? cryptoId()\n    this.defaultDriver = driver\n    this.builderEmitter = new TraceEmitter(store)\n  }\n\n  /** Start the builder (L0) run for this chat. Returns the runId. */\n  async startChat(scenarioId = `${this.projectId}/chat`): Promise<string> {\n    await this.builderEmitter.startRun({\n      scenarioId,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      layer: 'builder',\n    })\n    this.builderRunId = this.builderEmitter.runId\n    return this.builderRunId\n  }\n\n  /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */\n  get emitter(): TraceEmitter {\n    if (!this.builderRunId) throw new Error('BuilderSession.emitter: call startChat() first')\n    return this.builderEmitter\n  }\n\n  /**\n   * Ship the project's generated app: run the sandbox harness as a child\n   * Run (`layer='app-build'`). Returns the build result + runId.\n   */\n  async ship(options: ShipOptions): Promise<{ runId: string; result: SandboxHarnessResult }> {\n    if (!this.builderRunId) throw new Error('BuilderSession.ship: call startChat() first')\n    const buildEmitter = new TraceEmitter(this.store)\n    await buildEmitter.startRun({\n      scenarioId: options.scenarioId ?? `${this.projectId}/build`,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      parentRunId: this.builderRunId,\n      layer: 'app-build',\n    })\n    const harness = new SandboxHarness(options.driver ?? this.defaultDriver)\n    const result = await harness.run(options.harness, buildEmitter)\n    await buildEmitter.endRun({\n      pass: result.passed,\n      score: result.score,\n      failureClass: result.passed ? 'success' : 'sandbox_failure',\n    })\n    this.lastBuildRunId = buildEmitter.runId\n    return { runId: buildEmitter.runId, result }\n  }\n\n  /**\n   * Run a domain scenario against the just-built app as a grandchild Run\n   * (`layer='app-runtime'`). The `ship` call must precede this so the\n   * parent is set correctly; if no build exists yet the session attaches\n   * directly to the builder run (useful for prototypes).\n   */\n  async runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult> {\n    const parentRunId = this.lastBuildRunId ?? this.builderRunId\n    if (!parentRunId)\n      throw new Error('BuilderSession.runAppScenario: call startChat() + ship() first')\n    const { scenario, driver } = options\n    const result = await runTestGradedScenario(scenario, this.store, {\n      driver: driver ?? this.defaultDriver,\n      provenance: { codeSha: undefined, promptSha: undefined, modelFingerprint: undefined },\n    })\n    // Attach to the parent chain by updating the stored Run in place.\n    await this.store.updateRun(result.runId, {\n      parentRunId,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      layer: 'app-runtime',\n    })\n    return result\n  }\n\n  /** Record an end-of-chat meta score (judge verdict on whether the builder\n   *  served the user's intent). Accepts a numeric score + optional rationale. */\n  async recordMetaScore(score: number, rationale?: string): Promise<void> {\n    if (!this.builderRunId)\n      throw new Error('BuilderSession.recordMetaScore: call startChat() first')\n    await this.builderEmitter.recordJudge({\n      judgeId: 'builder-meta',\n      targetSpanId: this.builderRunId, // attach to the builder run itself\n      dimension: 'user_intent_satisfaction',\n      score,\n      rationale,\n      name: 'builder-meta',\n    })\n  }\n\n  /** Close the builder Run with a final outcome. */\n  async endChat(outcome: { pass: boolean; score?: number; notes?: string }): Promise<void> {\n    await this.builderEmitter.endRun({\n      pass: outcome.pass,\n      score: outcome.score,\n      notes: outcome.notes,\n    })\n  }\n\n  /**\n   * Inline app-runtime run — for cases where the \"scenario\" isn't a\n   * SWE-bench-style test suite but a live agent interaction (LLM chat,\n   * domain flow). Returns an emitter bound to a fresh Run in the\n   * `app-runtime` layer; caller emits spans inside and calls\n   * `.endRun()` with the final verdict.\n   */\n  async startAppRuntime(scenarioId: string): Promise<TraceEmitter> {\n    const parentRunId = this.lastBuildRunId ?? this.builderRunId\n    if (!parentRunId)\n      throw new Error(\n        'BuilderSession.startAppRuntime: call startChat() + (optionally) ship() first',\n      )\n    const emitter = new TraceEmitter(this.store)\n    await emitter.startRun({\n      scenarioId,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      parentRunId,\n      layer: 'app-runtime',\n    })\n    return emitter\n  }\n\n  /**\n   * Lightweight \"ship marker\" — record an app-build Run with a caller-\n   * provided verdict. Use when there isn't a sandbox harness to run but\n   * you still want to mark the build state at publish time.\n   */\n  async recordShipMarker(args: {\n    pass: boolean\n    score: number\n    scenarioId?: string\n    notes?: string\n  }): Promise<string> {\n    if (!this.builderRunId)\n      throw new Error('BuilderSession.recordShipMarker: call startChat() first')\n    const emitter = new TraceEmitter(this.store)\n    await emitter.startRun({\n      scenarioId: args.scenarioId ?? `${this.projectId}/ship`,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      parentRunId: this.builderRunId,\n      layer: 'app-build',\n    })\n    await emitter.endRun({\n      pass: args.pass,\n      score: args.score,\n      failureClass: args.pass ? 'success' : 'sandbox_failure',\n      notes: args.notes,\n    })\n    this.lastBuildRunId = emitter.runId\n    return emitter.runId\n  }\n\n  get lastBuildRunIdValue(): string | undefined {\n    return this.lastBuildRunId\n  }\n  get builderRunIdValue(): string | undefined {\n    return this.builderRunId\n  }\n}\n\n/**\n * Reconstruct the most recent BuilderSession state for a given project —\n * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs\n * this is how a resumed session finds its place in the edit history.\n */\nexport async function resumeBuilderSession(\n  store: TraceStore,\n  projectId: string,\n): Promise<{\n  projectId: string\n  chatRuns: Run[]\n  lastBuilderRun?: Run\n  lastBuildRun?: Run\n  lastAppRuntimeRuns: Run[]\n}> {\n  const runs = await store.listRuns({ projectId })\n  const chatRuns = runs\n    .filter((r) => r.layer === 'builder')\n    .sort((a, b) => b.startedAt - a.startedAt)\n  const buildRuns = runs\n    .filter((r) => r.layer === 'app-build')\n    .sort((a, b) => b.startedAt - a.startedAt)\n  const appRuntimeRuns = runs\n    .filter((r) => r.layer === 'app-runtime')\n    .sort((a, b) => b.startedAt - a.startedAt)\n  return {\n    projectId,\n    chatRuns,\n    lastBuilderRun: chatRuns[0],\n    lastBuildRun: buildRuns[0],\n    lastAppRuntimeRuns: appRuntimeRuns,\n  }\n}\n\nfunction cryptoId(): string {\n  if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()\n  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`\n}\n","/**\n * Meta-eval correlation — the highest-leverage signal in the framework.\n *\n * Given a corpus of three-layer project reports, compute how well each\n * pair of layers correlates. The question we care about most:\n *\n *   Does `metaScore` (what the builder thinks it did) predict\n *   `runtimeScore` (what the user actually gets)?\n *\n * If r < ~0.4, the builder's self-scoring is broken — it's optimizing\n * for something other than real-world success. If r > 0.7, meta_score\n * is a usable proxy and can drive CI gates cheaply.\n *\n * Non-parametric rank correlation (Spearman) is also reported because\n * meta scores are often ordinal-ish.\n */\n\nimport type { ThreeLayerProjectReport } from './three-layer-eval'\n\nexport interface LayerCorrelation {\n  n: number\n  pearson: number\n  spearman: number\n}\n\nexport interface CorrelationReport {\n  /** Pairs present in the corpus (layers with ≥ 2 matched data points). */\n  metaVsBuild?: LayerCorrelation\n  metaVsRuntime?: LayerCorrelation\n  buildVsRuntime?: LayerCorrelation\n  /** Number of complete projects (all 3 scores present). */\n  completeProjects: number\n}\n\nexport function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport {\n  const completeProjects = reports.filter((r) => r.complete).length\n  return {\n    metaVsBuild: pairwise(\n      reports,\n      (r) => r.metaScore,\n      (r) => r.buildScore,\n    ),\n    metaVsRuntime: pairwise(\n      reports,\n      (r) => r.metaScore,\n      (r) => r.runtimeScore,\n    ),\n    buildVsRuntime: pairwise(\n      reports,\n      (r) => r.buildScore,\n      (r) => r.runtimeScore,\n    ),\n    completeProjects,\n  }\n}\n\nfunction pairwise(\n  reports: ThreeLayerProjectReport[],\n  a: (r: ThreeLayerProjectReport) => number | null,\n  b: (r: ThreeLayerProjectReport) => number | null,\n): LayerCorrelation | undefined {\n  const xs: number[] = []\n  const ys: number[] = []\n  for (const r of reports) {\n    const x = a(r)\n    const y = b(r)\n    if (x !== null && y !== null && Number.isFinite(x) && Number.isFinite(y)) {\n      xs.push(x)\n      ys.push(y)\n    }\n  }\n  if (xs.length < 2) return undefined\n  return {\n    n: xs.length,\n    pearson: pearsonR(xs, ys),\n    spearman: spearmanR(xs, ys),\n  }\n}\n\nfunction pearsonR(a: number[], b: number[]): number {\n  const mA = a.reduce((s, v) => s + v, 0) / a.length\n  const mB = b.reduce((s, v) => s + v, 0) / b.length\n  let num = 0,\n    dA = 0,\n    dB = 0\n  for (let i = 0; i < a.length; i++) {\n    const da = a[i]! - mA\n    const db = b[i]! - mB\n    num += da * db\n    dA += da * da\n    dB += db * db\n  }\n  if (dA === 0 || dB === 0) return dA === 0 && dB === 0 ? 1 : 0\n  return num / Math.sqrt(dA * dB)\n}\n\nfunction spearmanR(a: number[], b: number[]): number {\n  return pearsonR(ranks(a), ranks(b))\n}\n\nfunction ranks(xs: number[]): number[] {\n  const indexed = xs.map((v, i) => ({ v, i })).sort((x, y) => x.v - y.v)\n  const r = new Array<number>(xs.length)\n  for (let i = 0; i < indexed.length; i++) {\n    // Average rank for ties\n    let j = i\n    while (j + 1 < indexed.length && indexed[j + 1]!.v === indexed[i]!.v) j++\n    const avg = (i + j + 2) / 2\n    for (let k = i; k <= j; k++) r[indexed[k]!.i] = avg\n    i = j\n  }\n  return r\n}\n","/**\n * ProjectRegistry — project-level aggregation over the trace corpus.\n *\n * Thin reader over TraceStore that answers the questions a chat-first,\n * resumable UI needs:\n *   - listProjects() → project IDs with latest activity\n *   - projectTimeline(id) → chats + builds + runtime runs, chronological\n *   - projectChats(id) → chat-level summaries (turn count, outcome)\n *\n * All queries are pure reads; no state duplication.\n */\n\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface ProjectSummary {\n  projectId: string\n  chatCount: number\n  buildCount: number\n  appRuntimeCount: number\n  lastActivityAt: number\n  latestChatId?: string\n  latestOutcome?: { pass: boolean; score?: number }\n}\n\nexport interface ChatSummary {\n  chatId: string\n  projectId: string\n  builderRunId: string\n  startedAt: number\n  endedAt?: number\n  status: Run['status']\n  outcome?: Run['outcome']\n  /** Counts of spans emitted during the chat. */\n  llmTurns?: number\n  toolCalls?: number\n  buildRunId?: string\n  appRuntimeRunIds: string[]\n}\n\nexport interface ProjectTimelineEntry {\n  run: Run\n  layerBucket: 'chat' | 'build' | 'runtime' | 'other'\n}\n\nexport class ProjectRegistry {\n  constructor(private store: TraceStore) {}\n\n  async listProjects(): Promise<ProjectSummary[]> {\n    const runs = await this.store.listRuns()\n    const byProject = new Map<string, Run[]>()\n    for (const r of runs) {\n      if (!r.projectId) continue\n      const arr = byProject.get(r.projectId) ?? []\n      arr.push(r)\n      byProject.set(r.projectId, arr)\n    }\n    const summaries: ProjectSummary[] = []\n    for (const [projectId, projectRuns] of byProject) {\n      const sorted = projectRuns.slice().sort((a, b) => b.startedAt - a.startedAt)\n      const chats = projectRuns.filter((r) => r.layer === 'builder')\n      const builds = projectRuns.filter((r) => r.layer === 'app-build')\n      const runtimes = projectRuns.filter((r) => r.layer === 'app-runtime')\n      const latest = sorted[0]\n      if (!latest) continue\n      summaries.push({\n        projectId,\n        chatCount: chats.length,\n        buildCount: builds.length,\n        appRuntimeCount: runtimes.length,\n        lastActivityAt: latest.startedAt,\n        latestChatId: chats[0]?.chatId,\n        latestOutcome: latest.outcome\n          ? { pass: latest.outcome.pass ?? false, score: latest.outcome.score }\n          : undefined,\n      })\n    }\n    return summaries.sort((a, b) => b.lastActivityAt - a.lastActivityAt)\n  }\n\n  async projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]> {\n    const runs = await this.store.listRuns({ projectId })\n    const ordered = runs.slice().sort((a, b) => a.startedAt - b.startedAt)\n    return ordered.map((run) => ({\n      run,\n      layerBucket:\n        run.layer === 'builder'\n          ? 'chat'\n          : run.layer === 'app-build'\n            ? 'build'\n            : run.layer === 'app-runtime'\n              ? 'runtime'\n              : 'other',\n    }))\n  }\n\n  async projectChats(projectId: string): Promise<ChatSummary[]> {\n    const builderRuns = (await this.store.listRuns({ projectId, layer: 'builder' })).sort(\n      (a, b) => b.startedAt - a.startedAt,\n    )\n    const childrenFor = async (runId: string) => this.store.listRuns({ parentRunId: runId })\n    const out: ChatSummary[] = []\n    for (const run of builderRuns) {\n      const spans = await this.store.spans({ runId: run.runId })\n      const children = await childrenFor(run.runId)\n      const build = children.find((c) => c.layer === 'app-build')\n      const runtime: string[] = []\n      // Runtime runs can be grandchildren (attached to the build) or siblings\n      // when shipped skipped.\n      if (build) {\n        const grands = await childrenFor(build.runId)\n        for (const g of grands) if (g.layer === 'app-runtime') runtime.push(g.runId)\n      }\n      for (const c of children) if (c.layer === 'app-runtime') runtime.push(c.runId)\n      out.push({\n        chatId: run.chatId ?? run.runId,\n        projectId,\n        builderRunId: run.runId,\n        startedAt: run.startedAt,\n        endedAt: run.endedAt,\n        status: run.status,\n        outcome: run.outcome,\n        llmTurns: spans.filter((s) => s.kind === 'llm').length,\n        toolCalls: spans.filter((s) => s.kind === 'tool').length,\n        buildRunId: build?.runId,\n        appRuntimeRunIds: runtime,\n      })\n    }\n    return out\n  }\n}\n","/**\n * Three-layer evaluation — the canonical scoring breakdown for\n * builder-of-builders workflows.\n *\n *   meta_score:    did the builder understand + satisfy user intent?\n *                  (judge verdict attached to the builder run)\n *   build_score:   did the generated scaffold build + pass its own tests?\n *                  (outcome.score on the app-build child run)\n *   runtime_score: did the generated agent pass its domain scenarios?\n *                  (mean outcome.score over app-runtime grandchild runs)\n *\n * Returns a structured report per project. The cross-layer correlation\n * is the highest-leverage signal the framework computes — if\n * meta_score doesn't predict runtime_score, the builder's self-scoring\n * is broken.\n *\n * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a\n * scaffold-builder eval that grades compose + build without driving a\n * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures\n * meta + build only. Consumers can tell the two apart without having to\n * interpret null-runtime as either \"not yet computed\" or \"N/A for this\n * project shape\".\n */\n\nimport { judgeSpans } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport type ProjectKind = 'full' | 'scaffold-only'\n\nexport interface ThreeLayerProjectReport {\n  projectId: string\n  /**\n   * `'full'` when the project has at least one `app-runtime` run;\n   * `'scaffold-only'` when it only has meta + build layers. Lets\n   * downstream consumers treat a null runtime score as expected\n   * (scaffold-only) vs. missing (full, pipeline broke).\n   */\n  kind: ProjectKind\n  builderRunId?: string\n  /** Judge-verdict score on the builder run (0..1 after normalization). */\n  metaScore: number | null\n  buildRunId?: string\n  /** 0..1 from the sandbox harness (testsPassed / testsTotal). */\n  buildScore: number | null\n  appRuntimeRunIds: string[]\n  /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */\n  runtimeScore: number | null\n  runtimePassRate: number | null\n  /**\n   * Layer-aware completeness:\n   *   - `kind='full'`: all three layers scored\n   *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)\n   */\n  complete: boolean\n}\n\nexport async function scoreProject(\n  store: TraceStore,\n  projectId: string,\n): Promise<ThreeLayerProjectReport> {\n  const allRuns = await store.listRuns({ projectId })\n  const builder = latestByLayer(allRuns, 'builder')\n  const build = latestByLayer(allRuns, 'app-build')\n  const runtime = allRuns.filter((r) => r.layer === 'app-runtime')\n\n  const metaScore = builder ? await extractMetaScore(store, builder.runId) : null\n  const buildScore = build?.outcome?.score ?? null\n  const runtimeScores = runtime\n    .map((r) => r.outcome?.score)\n    .filter((s): s is number => typeof s === 'number')\n  const runtimeScore =\n    runtimeScores.length > 0\n      ? runtimeScores.reduce((a, b) => a + b, 0) / runtimeScores.length\n      : null\n  const runtimePassed = runtime.filter((r) => r.outcome?.pass === true).length\n  const runtimePassRate = runtime.length > 0 ? runtimePassed / runtime.length : null\n\n  const kind: ProjectKind = runtime.length === 0 ? 'scaffold-only' : 'full'\n  const complete =\n    kind === 'scaffold-only'\n      ? metaScore !== null && buildScore !== null\n      : metaScore !== null && buildScore !== null && runtimeScore !== null\n\n  return {\n    projectId,\n    kind,\n    builderRunId: builder?.runId,\n    metaScore,\n    buildRunId: build?.runId,\n    buildScore,\n    appRuntimeRunIds: runtime.map((r) => r.runId),\n    runtimeScore,\n    runtimePassRate,\n    complete,\n  }\n}\n\n/** Aggregate scoring across every project in a corpus. */\nexport async function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]> {\n  const runs = await store.listRuns()\n  const projectIds = [...new Set(runs.map((r) => r.projectId).filter((p): p is string => !!p))]\n  return Promise.all(projectIds.map((p) => scoreProject(store, p)))\n}\n\nfunction latestByLayer(runs: Run[], layer: Run['layer']): Run | undefined {\n  const filtered = runs.filter((r) => r.layer === layer).sort((a, b) => b.startedAt - a.startedAt)\n  return filtered[0]\n}\n\nasync function extractMetaScore(store: TraceStore, builderRunId: string): Promise<number | null> {\n  const js = await judgeSpans(store, builderRunId)\n  const meta = js.find(\n    (s) => s.judgeId === 'builder-meta' && s.dimension === 'user_intent_satisfaction',\n  )\n  if (!meta) return null\n  // Normalize score to 0..1. Accept 0-1 natively; 0-10 scale is also common.\n  if (meta.score >= 0 && meta.score <= 1) return meta.score\n  if (meta.score >= 0 && meta.score <= 10) return meta.score / 10\n  return null\n}\n"],"mappings":";;;;;;;;;;;;;;;AA8CO,IAAM,iBAAN,MAAqB;AAAA,EAClB;AAAA,EACA;AAAA,EACC;AAAA,EACA;AAAA,EACD;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,OAAmB,MAA0B,QAAwB;AAC/E,SAAK,QAAQ;AACb,SAAK,YAAY,KAAK;AACtB,SAAK,SAAS,KAAK,UAAU,SAAS;AACtC,SAAK,gBAAgB;AACrB,SAAK,iBAAiB,IAAI,aAAa,KAAK;AAAA,EAC9C;AAAA;AAAA,EAGA,MAAM,UAAU,aAAa,GAAG,KAAK,SAAS,SAA0B;AACtE,UAAM,KAAK,eAAe,SAAS;AAAA,MACjC;AAAA,MACA,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,OAAO;AAAA,IACT,CAAC;AACD,SAAK,eAAe,KAAK,eAAe;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAwB;AAC1B,QAAI,CAAC,KAAK,aAAc,OAAM,IAAI,MAAM,gDAAgD;AACxF,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,KAAK,SAAgF;AACzF,QAAI,CAAC,KAAK,aAAc,OAAM,IAAI,MAAM,6CAA6C;AACrF,UAAM,eAAe,IAAI,aAAa,KAAK,KAAK;AAChD,UAAM,aAAa,SAAS;AAAA,MAC1B,YAAY,QAAQ,cAAc,GAAG,KAAK,SAAS;AAAA,MACnD,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,OAAO;AAAA,IACT,CAAC;AACD,UAAM,UAAU,IAAI,eAAe,QAAQ,UAAU,KAAK,aAAa;AACvE,UAAM,SAAS,MAAM,QAAQ,IAAI,QAAQ,SAAS,YAAY;AAC9D,UAAM,aAAa,OAAO;AAAA,MACxB,MAAM,OAAO;AAAA,MACb,OAAO,OAAO;AAAA,MACd,cAAc,OAAO,SAAS,YAAY;AAAA,IAC5C,CAAC;AACD,SAAK,iBAAiB,aAAa;AACnC,WAAO,EAAE,OAAO,aAAa,OAAO,OAAO;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,eAAe,SAA8D;AACjF,UAAM,cAAc,KAAK,kBAAkB,KAAK;AAChD,QAAI,CAAC;AACH,YAAM,IAAI,MAAM,gEAAgE;AAClF,UAAM,EAAE,UAAU,OAAO,IAAI;AAC7B,UAAM,SAAS,MAAM,sBAAsB,UAAU,KAAK,OAAO;AAAA,MAC/D,QAAQ,UAAU,KAAK;AAAA,MACvB,YAAY,EAAE,SAAS,QAAW,WAAW,QAAW,kBAAkB,OAAU;AAAA,IACtF,CAAC;AAED,UAAM,KAAK,MAAM,UAAU,OAAO,OAAO;AAAA,MACvC;AAAA,MACA,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,OAAO;AAAA,IACT,CAAC;AACD,WAAO;AAAA,EACT;AAAA;AAAA;AAAA,EAIA,MAAM,gBAAgB,OAAe,WAAmC;AACtE,QAAI,CAAC,KAAK;AACR,YAAM,IAAI,MAAM,wDAAwD;AAC1E,UAAM,KAAK,eAAe,YAAY;AAAA,MACpC,SAAS;AAAA,MACT,cAAc,KAAK;AAAA;AAAA,MACnB,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAM,QAAQ,SAA2E;AACvF,UAAM,KAAK,eAAe,OAAO;AAAA,MAC/B,MAAM,QAAQ;AAAA,MACd,OAAO,QAAQ;AAAA,MACf,OAAO,QAAQ;AAAA,IACjB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,gBAAgB,YAA2C;AAC/D,UAAM,cAAc,KAAK,kBAAkB,KAAK;AAChD,QAAI,CAAC;AACH,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AACF,UAAM,UAAU,IAAI,aAAa,KAAK,KAAK;AAC3C,UAAM,QAAQ,SAAS;AAAA,MACrB;AAAA,MACA,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb;AAAA,MACA,OAAO;AAAA,IACT,CAAC;AACD,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,iBAAiB,MAKH;AAClB,QAAI,CAAC,KAAK;AACR,YAAM,IAAI,MAAM,yDAAyD;AAC3E,UAAM,UAAU,IAAI,aAAa,KAAK,KAAK;AAC3C,UAAM,QAAQ,SAAS;AAAA,MACrB,YAAY,KAAK,cAAc,GAAG,KAAK,SAAS;AAAA,MAChD,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,OAAO;AAAA,IACT,CAAC;AACD,UAAM,QAAQ,OAAO;AAAA,MACnB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,cAAc,KAAK,OAAO,YAAY;AAAA,MACtC,OAAO,KAAK;AAAA,IACd,CAAC;AACD,SAAK,iBAAiB,QAAQ;AAC9B,WAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,IAAI,sBAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EACA,IAAI,oBAAwC;AAC1C,WAAO,KAAK;AAAA,EACd;AACF;AAOA,eAAsB,qBACpB,OACA,WAOC;AACD,QAAM,OAAO,MAAM,MAAM,SAAS,EAAE,UAAU,CAAC;AAC/C,QAAM,WAAW,KACd,OAAO,CAAC,MAAM,EAAE,UAAU,SAAS,EACnC,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC3C,QAAM,YAAY,KACf,OAAO,CAAC,MAAM,EAAE,UAAU,WAAW,EACrC,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC3C,QAAM,iBAAiB,KACpB,OAAO,CAAC,MAAM,EAAE,UAAU,aAAa,EACvC,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC3C,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,gBAAgB,SAAS,CAAC;AAAA,IAC1B,cAAc,UAAU,CAAC;AAAA,IACzB,oBAAoB;AAAA,EACtB;AACF;AAEA,SAAS,WAAmB;AAC1B,MAAI,OAAO,WAAW,QAAQ,eAAe,WAAY,QAAO,WAAW,OAAO,WAAW;AAC7F,SAAO,GAAG,KAAK,IAAI,EAAE,SAAS,EAAE,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,GAAG,EAAE,CAAC;AAC9E;;;AC7NO,SAAS,gBAAgB,SAAuD;AACrF,QAAM,mBAAmB,QAAQ,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE;AAC3D,SAAO;AAAA,IACL,aAAa;AAAA,MACX;AAAA,MACA,CAAC,MAAM,EAAE;AAAA,MACT,CAAC,MAAM,EAAE;AAAA,IACX;AAAA,IACA,eAAe;AAAA,MACb;AAAA,MACA,CAAC,MAAM,EAAE;AAAA,MACT,CAAC,MAAM,EAAE;AAAA,IACX;AAAA,IACA,gBAAgB;AAAA,MACd;AAAA,MACA,CAAC,MAAM,EAAE;AAAA,MACT,CAAC,MAAM,EAAE;AAAA,IACX;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,SACP,SACA,GACA,GAC8B;AAC9B,QAAM,KAAe,CAAC;AACtB,QAAM,KAAe,CAAC;AACtB,aAAW,KAAK,SAAS;AACvB,UAAM,IAAI,EAAE,CAAC;AACb,UAAM,IAAI,EAAE,CAAC;AACb,QAAI,MAAM,QAAQ,MAAM,QAAQ,OAAO,SAAS,CAAC,KAAK,OAAO,SAAS,CAAC,GAAG;AACxE,SAAG,KAAK,CAAC;AACT,SAAG,KAAK,CAAC;AAAA,IACX;AAAA,EACF;AACA,MAAI,GAAG,SAAS,EAAG,QAAO;AAC1B,SAAO;AAAA,IACL,GAAG,GAAG;AAAA,IACN,SAAS,SAAS,IAAI,EAAE;AAAA,IACxB,UAAU,UAAU,IAAI,EAAE;AAAA,EAC5B;AACF;AAEA,SAAS,SAAS,GAAa,GAAqB;AAClD,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,MAAI,MAAM,GACR,KAAK,GACL,KAAK;AACP,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,WAAO,KAAK;AACZ,UAAM,KAAK;AACX,UAAM,KAAK;AAAA,EACb;AACA,MAAI,OAAO,KAAK,OAAO,EAAG,QAAO,OAAO,KAAK,OAAO,IAAI,IAAI;AAC5D,SAAO,MAAM,KAAK,KAAK,KAAK,EAAE;AAChC;AAEA,SAAS,UAAU,GAAa,GAAqB;AACnD,SAAO,SAAS,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC;AACpC;AAEA,SAAS,MAAM,IAAwB;AACrC,QAAM,UAAU,GAAG,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC;AACrE,QAAM,IAAI,IAAI,MAAc,GAAG,MAAM;AACrC,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AAEvC,QAAI,IAAI;AACR,WAAO,IAAI,IAAI,QAAQ,UAAU,QAAQ,IAAI,CAAC,EAAG,MAAM,QAAQ,CAAC,EAAG,EAAG;AACtE,UAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,aAAS,IAAI,GAAG,KAAK,GAAG,IAAK,GAAE,QAAQ,CAAC,EAAG,CAAC,IAAI;AAChD,QAAI;AAAA,EACN;AACA,SAAO;AACT;;;ACnEO,IAAM,kBAAN,MAAsB;AAAA,EAC3B,YAAoB,OAAmB;AAAnB;AAAA,EAAoB;AAAA,EAApB;AAAA,EAEpB,MAAM,eAA0C;AAC9C,UAAM,OAAO,MAAM,KAAK,MAAM,SAAS;AACvC,UAAM,YAAY,oBAAI,IAAmB;AACzC,eAAW,KAAK,MAAM;AACpB,UAAI,CAAC,EAAE,UAAW;AAClB,YAAM,MAAM,UAAU,IAAI,EAAE,SAAS,KAAK,CAAC;AAC3C,UAAI,KAAK,CAAC;AACV,gBAAU,IAAI,EAAE,WAAW,GAAG;AAAA,IAChC;AACA,UAAM,YAA8B,CAAC;AACrC,eAAW,CAAC,WAAW,WAAW,KAAK,WAAW;AAChD,YAAM,SAAS,YAAY,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC3E,YAAM,QAAQ,YAAY,OAAO,CAAC,MAAM,EAAE,UAAU,SAAS;AAC7D,YAAM,SAAS,YAAY,OAAO,CAAC,MAAM,EAAE,UAAU,WAAW;AAChE,YAAM,WAAW,YAAY,OAAO,CAAC,MAAM,EAAE,UAAU,aAAa;AACpE,YAAM,SAAS,OAAO,CAAC;AACvB,UAAI,CAAC,OAAQ;AACb,gBAAU,KAAK;AAAA,QACb;AAAA,QACA,WAAW,MAAM;AAAA,QACjB,YAAY,OAAO;AAAA,QACnB,iBAAiB,SAAS;AAAA,QAC1B,gBAAgB,OAAO;AAAA,QACvB,cAAc,MAAM,CAAC,GAAG;AAAA,QACxB,eAAe,OAAO,UAClB,EAAE,MAAM,OAAO,QAAQ,QAAQ,OAAO,OAAO,OAAO,QAAQ,MAAM,IAClE;AAAA,MACN,CAAC;AAAA,IACH;AACA,WAAO,UAAU,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAAA,EACrE;AAAA,EAEA,MAAM,gBAAgB,WAAoD;AACxE,UAAM,OAAO,MAAM,KAAK,MAAM,SAAS,EAAE,UAAU,CAAC;AACpD,UAAM,UAAU,KAAK,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AACrE,WAAO,QAAQ,IAAI,CAAC,SAAS;AAAA,MAC3B;AAAA,MACA,aACE,IAAI,UAAU,YACV,SACA,IAAI,UAAU,cACZ,UACA,IAAI,UAAU,gBACZ,YACA;AAAA,IACZ,EAAE;AAAA,EACJ;AAAA,EAEA,MAAM,aAAa,WAA2C;AAC5D,UAAM,eAAe,MAAM,KAAK,MAAM,SAAS,EAAE,WAAW,OAAO,UAAU,CAAC,GAAG;AAAA,MAC/E,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE;AAAA,IAC5B;AACA,UAAM,cAAc,OAAO,UAAkB,KAAK,MAAM,SAAS,EAAE,aAAa,MAAM,CAAC;AACvF,UAAM,MAAqB,CAAC;AAC5B,eAAW,OAAO,aAAa;AAC7B,YAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;AACzD,YAAM,WAAW,MAAM,YAAY,IAAI,KAAK;AAC5C,YAAM,QAAQ,SAAS,KAAK,CAAC,MAAM,EAAE,UAAU,WAAW;AAC1D,YAAM,UAAoB,CAAC;AAG3B,UAAI,OAAO;AACT,cAAM,SAAS,MAAM,YAAY,MAAM,KAAK;AAC5C,mBAAW,KAAK,OAAQ,KAAI,EAAE,UAAU,cAAe,SAAQ,KAAK,EAAE,KAAK;AAAA,MAC7E;AACA,iBAAW,KAAK,SAAU,KAAI,EAAE,UAAU,cAAe,SAAQ,KAAK,EAAE,KAAK;AAC7E,UAAI,KAAK;AAAA,QACP,QAAQ,IAAI,UAAU,IAAI;AAAA,QAC1B;AAAA,QACA,cAAc,IAAI;AAAA,QAClB,WAAW,IAAI;AAAA,QACf,SAAS,IAAI;AAAA,QACb,QAAQ,IAAI;AAAA,QACZ,SAAS,IAAI;AAAA,QACb,UAAU,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE;AAAA,QAChD,WAAW,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE;AAAA,QAClD,YAAY,OAAO;AAAA,QACnB,kBAAkB;AAAA,MACpB,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AACF;;;ACzEA,eAAsB,aACpB,OACA,WACkC;AAClC,QAAM,UAAU,MAAM,MAAM,SAAS,EAAE,UAAU,CAAC;AAClD,QAAM,UAAU,cAAc,SAAS,SAAS;AAChD,QAAM,QAAQ,cAAc,SAAS,WAAW;AAChD,QAAM,UAAU,QAAQ,OAAO,CAAC,MAAM,EAAE,UAAU,aAAa;AAE/D,QAAM,YAAY,UAAU,MAAM,iBAAiB,OAAO,QAAQ,KAAK,IAAI;AAC3E,QAAM,aAAa,OAAO,SAAS,SAAS;AAC5C,QAAM,gBAAgB,QACnB,IAAI,CAAC,MAAM,EAAE,SAAS,KAAK,EAC3B,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACnD,QAAM,eACJ,cAAc,SAAS,IACnB,cAAc,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc,SACzD;AACN,QAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS,SAAS,IAAI,EAAE;AACtE,QAAM,kBAAkB,QAAQ,SAAS,IAAI,gBAAgB,QAAQ,SAAS;AAE9E,QAAM,OAAoB,QAAQ,WAAW,IAAI,kBAAkB;AACnE,QAAM,WACJ,SAAS,kBACL,cAAc,QAAQ,eAAe,OACrC,cAAc,QAAQ,eAAe,QAAQ,iBAAiB;AAEpE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,SAAS;AAAA,IACvB;AAAA,IACA,YAAY,OAAO;AAAA,IACnB;AAAA,IACA,kBAAkB,QAAQ,IAAI,CAAC,MAAM,EAAE,KAAK;AAAA,IAC5C;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGA,eAAsB,iBAAiB,OAAuD;AAC5F,QAAM,OAAO,MAAM,MAAM,SAAS;AAClC,QAAM,aAAa,CAAC,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,MAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5F,SAAO,QAAQ,IAAI,WAAW,IAAI,CAAC,MAAM,aAAa,OAAO,CAAC,CAAC,CAAC;AAClE;AAEA,SAAS,cAAc,MAAa,OAAsC;AACxE,QAAM,WAAW,KAAK,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC/F,SAAO,SAAS,CAAC;AACnB;AAEA,eAAe,iBAAiB,OAAmB,cAA8C;AAC/F,QAAM,KAAK,MAAM,WAAW,OAAO,YAAY;AAC/C,QAAM,OAAO,GAAG;AAAA,IACd,CAAC,MAAM,EAAE,YAAY,kBAAkB,EAAE,cAAc;AAAA,EACzD;AACA,MAAI,CAAC,KAAM,QAAO;AAElB,MAAI,KAAK,SAAS,KAAK,KAAK,SAAS,EAAG,QAAO,KAAK;AACpD,MAAI,KAAK,SAAS,KAAK,KAAK,SAAS,GAAI,QAAO,KAAK,QAAQ;AAC7D,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../../src/builder-eval/builder-session.ts","../../src/builder-eval/correlation.ts","../../src/builder-eval/project-registry.ts","../../src/builder-eval/three-layer-eval.ts"],"sourcesContent":["/**\n * BuilderSession — ties a builder-of-builders workflow together.\n *\n * Models agent-builder's shape: Project → Chat → Edit → Ship → App →\n * AppAgent. Each layer is a Run (linked via parentRunId). The\n * framework-enforced invariants:\n *\n *   - One Project → many Chats; chatId scopes runs within a project.\n *   - One Chat = one builder Run with `layer='builder'`.\n *   - One Ship = one child Run with `layer='app-build'` + SandboxHarness.\n *   - One AppScenario = one grandchild Run with `layer='app-runtime'`.\n *\n * Consumers obtain a BuilderSession, call `startChat`, drive the\n * builder agent (emitting spans), and call `ship` / `runAppScenario`\n * as the workflow progresses. The session reconstructs itself from\n * trace data via `resume(store, projectId)`.\n */\n\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from '../sandbox-harness'\nimport { SandboxHarness } from '../sandbox-harness'\nimport type { TestGradedRunResult, TestGradedScenario } from '../test-graded-scenario'\nimport { runTestGradedScenario } from '../test-graded-scenario'\nimport { TraceEmitter } from '../trace/emitter'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface BuilderSessionInit {\n  projectId: string\n  chatId?: string\n  /** Free-form: user's task description, project name, etc. Stored on the builder Run. */\n  tags?: Record<string, string>\n}\n\nexport interface ShipOptions {\n  harness: HarnessConfig\n  driver?: SandboxDriver\n  /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */\n  scenarioId?: string\n}\n\nexport interface RunAppScenarioOptions {\n  scenario: TestGradedScenario\n  /** Harness driver override; defaults to the one the session was created with. */\n  driver?: SandboxDriver\n}\n\nexport class BuilderSession {\n  private store: TraceStore\n  private builderEmitter: TraceEmitter\n  readonly projectId: string\n  readonly chatId: string\n  private builderRunId?: string\n  private lastBuildRunId?: string\n  private defaultDriver?: SandboxDriver\n\n  constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver) {\n    this.store = store\n    this.projectId = init.projectId\n    this.chatId = init.chatId ?? cryptoId()\n    this.defaultDriver = driver\n    this.builderEmitter = new TraceEmitter(store)\n  }\n\n  /** Start the builder (L0) run for this chat. Returns the runId. */\n  async startChat(scenarioId = `${this.projectId}/chat`): Promise<string> {\n    await this.builderEmitter.startRun({\n      scenarioId,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      layer: 'builder',\n    })\n    this.builderRunId = this.builderEmitter.runId\n    return this.builderRunId\n  }\n\n  /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */\n  get emitter(): TraceEmitter {\n    if (!this.builderRunId) throw new Error('BuilderSession.emitter: call startChat() first')\n    return this.builderEmitter\n  }\n\n  /**\n   * Ship the project's generated app: run the sandbox harness as a child\n   * Run (`layer='app-build'`). Returns the build result + runId.\n   */\n  async ship(options: ShipOptions): Promise<{ runId: string; result: SandboxHarnessResult }> {\n    if (!this.builderRunId) throw new Error('BuilderSession.ship: call startChat() first')\n    const buildEmitter = new TraceEmitter(this.store)\n    await buildEmitter.startRun({\n      scenarioId: options.scenarioId ?? `${this.projectId}/build`,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      parentRunId: this.builderRunId,\n      layer: 'app-build',\n    })\n    const harness = new SandboxHarness(options.driver ?? this.defaultDriver)\n    const result = await harness.run(options.harness, buildEmitter)\n    await buildEmitter.endRun({\n      pass: result.passed,\n      score: result.score,\n      failureClass: result.passed ? 'success' : 'sandbox_failure',\n    })\n    this.lastBuildRunId = buildEmitter.runId\n    return { runId: buildEmitter.runId, result }\n  }\n\n  /**\n   * Run a domain scenario against the just-built app as a grandchild Run\n   * (`layer='app-runtime'`). The `ship` call must precede this so the\n   * parent is set correctly; if no build exists yet the session attaches\n   * directly to the builder run (useful for prototypes).\n   */\n  async runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult> {\n    const parentRunId = this.lastBuildRunId ?? this.builderRunId\n    if (!parentRunId)\n      throw new Error('BuilderSession.runAppScenario: call startChat() + ship() first')\n    const { scenario, driver } = options\n    const result = await runTestGradedScenario(scenario, this.store, {\n      driver: driver ?? this.defaultDriver,\n      provenance: { codeSha: undefined, promptSha: undefined, modelFingerprint: undefined },\n    })\n    // Attach to the parent chain by updating the stored Run in place.\n    await this.store.updateRun(result.runId, {\n      parentRunId,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      layer: 'app-runtime',\n    })\n    return result\n  }\n\n  /** Record an end-of-chat meta score (judge verdict on whether the builder\n   *  served the user's intent). Accepts a numeric score + optional rationale. */\n  async recordMetaScore(score: number, rationale?: string): Promise<void> {\n    if (!this.builderRunId)\n      throw new Error('BuilderSession.recordMetaScore: call startChat() first')\n    await this.builderEmitter.recordJudge({\n      judgeId: 'builder-meta',\n      targetSpanId: this.builderRunId, // attach to the builder run itself\n      dimension: 'user_intent_satisfaction',\n      score,\n      rationale,\n      name: 'builder-meta',\n    })\n  }\n\n  /** Close the builder Run with a final outcome. */\n  async endChat(outcome: { pass: boolean; score?: number; notes?: string }): Promise<void> {\n    await this.builderEmitter.endRun({\n      pass: outcome.pass,\n      score: outcome.score,\n      notes: outcome.notes,\n    })\n  }\n\n  /**\n   * Inline app-runtime run — for cases where the \"scenario\" isn't a\n   * SWE-bench-style test suite but a live agent interaction (LLM chat,\n   * domain flow). Returns an emitter bound to a fresh Run in the\n   * `app-runtime` layer; caller emits spans inside and calls\n   * `.endRun()` with the final verdict.\n   */\n  async startAppRuntime(scenarioId: string): Promise<TraceEmitter> {\n    const parentRunId = this.lastBuildRunId ?? this.builderRunId\n    if (!parentRunId)\n      throw new Error(\n        'BuilderSession.startAppRuntime: call startChat() + (optionally) ship() first',\n      )\n    const emitter = new TraceEmitter(this.store)\n    await emitter.startRun({\n      scenarioId,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      parentRunId,\n      layer: 'app-runtime',\n    })\n    return emitter\n  }\n\n  /**\n   * Lightweight \"ship marker\" — record an app-build Run with a caller-\n   * provided verdict. Use when there isn't a sandbox harness to run but\n   * you still want to mark the build state at publish time.\n   */\n  async recordShipMarker(args: {\n    pass: boolean\n    score: number\n    scenarioId?: string\n    notes?: string\n  }): Promise<string> {\n    if (!this.builderRunId)\n      throw new Error('BuilderSession.recordShipMarker: call startChat() first')\n    const emitter = new TraceEmitter(this.store)\n    await emitter.startRun({\n      scenarioId: args.scenarioId ?? `${this.projectId}/ship`,\n      projectId: this.projectId,\n      chatId: this.chatId,\n      parentRunId: this.builderRunId,\n      layer: 'app-build',\n    })\n    await emitter.endRun({\n      pass: args.pass,\n      score: args.score,\n      failureClass: args.pass ? 'success' : 'sandbox_failure',\n      notes: args.notes,\n    })\n    this.lastBuildRunId = emitter.runId\n    return emitter.runId\n  }\n\n  get lastBuildRunIdValue(): string | undefined {\n    return this.lastBuildRunId\n  }\n  get builderRunIdValue(): string | undefined {\n    return this.builderRunId\n  }\n}\n\n/**\n * Reconstruct the most recent BuilderSession state for a given project —\n * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs\n * this is how a resumed session finds its place in the edit history.\n */\nexport async function resumeBuilderSession(\n  store: TraceStore,\n  projectId: string,\n): Promise<{\n  projectId: string\n  chatRuns: Run[]\n  lastBuilderRun?: Run\n  lastBuildRun?: Run\n  lastAppRuntimeRuns: Run[]\n}> {\n  const runs = await store.listRuns({ projectId })\n  const chatRuns = runs\n    .filter((r) => r.layer === 'builder')\n    .sort((a, b) => b.startedAt - a.startedAt)\n  const buildRuns = runs\n    .filter((r) => r.layer === 'app-build')\n    .sort((a, b) => b.startedAt - a.startedAt)\n  const appRuntimeRuns = runs\n    .filter((r) => r.layer === 'app-runtime')\n    .sort((a, b) => b.startedAt - a.startedAt)\n  return {\n    projectId,\n    chatRuns,\n    lastBuilderRun: chatRuns[0],\n    lastBuildRun: buildRuns[0],\n    lastAppRuntimeRuns: appRuntimeRuns,\n  }\n}\n\nfunction cryptoId(): string {\n  if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()\n  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`\n}\n","/**\n * Meta-eval correlation — the highest-leverage signal in the framework.\n *\n * Given a corpus of three-layer project reports, compute how well each\n * pair of layers correlates. The question we care about most:\n *\n *   Does `metaScore` (what the builder thinks it did) predict\n *   `runtimeScore` (what the user actually gets)?\n *\n * If r < ~0.4, the builder's self-scoring is broken — it's optimizing\n * for something other than real-world success. If r > 0.7, meta_score\n * is a usable proxy and can drive CI gates cheaply.\n *\n * Non-parametric rank correlation (Spearman) is also reported because\n * meta scores are often ordinal-ish.\n */\n\nimport { pearsonR, spearmanR } from '../statistics'\nimport type { ThreeLayerProjectReport } from './three-layer-eval'\n\nexport interface LayerCorrelation {\n  n: number\n  pearson: number\n  spearman: number\n}\n\nexport interface CorrelationReport {\n  /** Pairs present in the corpus (layers with ≥ 2 matched data points). */\n  metaVsBuild?: LayerCorrelation\n  metaVsRuntime?: LayerCorrelation\n  buildVsRuntime?: LayerCorrelation\n  /** Number of complete projects (all 3 scores present). */\n  completeProjects: number\n}\n\nexport function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport {\n  const completeProjects = reports.filter((r) => r.complete).length\n  return {\n    metaVsBuild: pairwise(\n      reports,\n      (r) => r.metaScore,\n      (r) => r.buildScore,\n    ),\n    metaVsRuntime: pairwise(\n      reports,\n      (r) => r.metaScore,\n      (r) => r.runtimeScore,\n    ),\n    buildVsRuntime: pairwise(\n      reports,\n      (r) => r.buildScore,\n      (r) => r.runtimeScore,\n    ),\n    completeProjects,\n  }\n}\n\nfunction pairwise(\n  reports: ThreeLayerProjectReport[],\n  a: (r: ThreeLayerProjectReport) => number | null,\n  b: (r: ThreeLayerProjectReport) => number | null,\n): LayerCorrelation | undefined {\n  const xs: number[] = []\n  const ys: number[] = []\n  for (const r of reports) {\n    const x = a(r)\n    const y = b(r)\n    if (x !== null && y !== null && Number.isFinite(x) && Number.isFinite(y)) {\n      xs.push(x)\n      ys.push(y)\n    }\n  }\n  if (xs.length < 2) return undefined\n  return {\n    n: xs.length,\n    pearson: pearsonR(xs, ys),\n    spearman: spearmanR(xs, ys),\n  }\n}\n","/**\n * ProjectRegistry — project-level aggregation over the trace corpus.\n *\n * Thin reader over TraceStore that answers the questions a chat-first,\n * resumable UI needs:\n *   - listProjects() → project IDs with latest activity\n *   - projectTimeline(id) → chats + builds + runtime runs, chronological\n *   - projectChats(id) → chat-level summaries (turn count, outcome)\n *\n * All queries are pure reads; no state duplication.\n */\n\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface ProjectSummary {\n  projectId: string\n  chatCount: number\n  buildCount: number\n  appRuntimeCount: number\n  lastActivityAt: number\n  latestChatId?: string\n  latestOutcome?: { pass: boolean; score?: number }\n}\n\nexport interface ChatSummary {\n  chatId: string\n  projectId: string\n  builderRunId: string\n  startedAt: number\n  endedAt?: number\n  status: Run['status']\n  outcome?: Run['outcome']\n  /** Counts of spans emitted during the chat. */\n  llmTurns?: number\n  toolCalls?: number\n  buildRunId?: string\n  appRuntimeRunIds: string[]\n}\n\nexport interface ProjectTimelineEntry {\n  run: Run\n  layerBucket: 'chat' | 'build' | 'runtime' | 'other'\n}\n\nexport class ProjectRegistry {\n  constructor(private store: TraceStore) {}\n\n  async listProjects(): Promise<ProjectSummary[]> {\n    const runs = await this.store.listRuns()\n    const byProject = new Map<string, Run[]>()\n    for (const r of runs) {\n      if (!r.projectId) continue\n      const arr = byProject.get(r.projectId) ?? []\n      arr.push(r)\n      byProject.set(r.projectId, arr)\n    }\n    const summaries: ProjectSummary[] = []\n    for (const [projectId, projectRuns] of byProject) {\n      const sorted = projectRuns.slice().sort((a, b) => b.startedAt - a.startedAt)\n      const chats = projectRuns.filter((r) => r.layer === 'builder')\n      const builds = projectRuns.filter((r) => r.layer === 'app-build')\n      const runtimes = projectRuns.filter((r) => r.layer === 'app-runtime')\n      const latest = sorted[0]\n      if (!latest) continue\n      summaries.push({\n        projectId,\n        chatCount: chats.length,\n        buildCount: builds.length,\n        appRuntimeCount: runtimes.length,\n        lastActivityAt: latest.startedAt,\n        latestChatId: chats[0]?.chatId,\n        latestOutcome: latest.outcome\n          ? { pass: latest.outcome.pass ?? false, score: latest.outcome.score }\n          : undefined,\n      })\n    }\n    return summaries.sort((a, b) => b.lastActivityAt - a.lastActivityAt)\n  }\n\n  async projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]> {\n    const runs = await this.store.listRuns({ projectId })\n    const ordered = runs.slice().sort((a, b) => a.startedAt - b.startedAt)\n    return ordered.map((run) => ({\n      run,\n      layerBucket:\n        run.layer === 'builder'\n          ? 'chat'\n          : run.layer === 'app-build'\n            ? 'build'\n            : run.layer === 'app-runtime'\n              ? 'runtime'\n              : 'other',\n    }))\n  }\n\n  async projectChats(projectId: string): Promise<ChatSummary[]> {\n    const builderRuns = (await this.store.listRuns({ projectId, layer: 'builder' })).sort(\n      (a, b) => b.startedAt - a.startedAt,\n    )\n    const childrenFor = async (runId: string) => this.store.listRuns({ parentRunId: runId })\n    const out: ChatSummary[] = []\n    for (const run of builderRuns) {\n      const spans = await this.store.spans({ runId: run.runId })\n      const children = await childrenFor(run.runId)\n      const build = children.find((c) => c.layer === 'app-build')\n      const runtime: string[] = []\n      // Runtime runs can be grandchildren (attached to the build) or siblings\n      // when shipped skipped.\n      if (build) {\n        const grands = await childrenFor(build.runId)\n        for (const g of grands) if (g.layer === 'app-runtime') runtime.push(g.runId)\n      }\n      for (const c of children) if (c.layer === 'app-runtime') runtime.push(c.runId)\n      out.push({\n        chatId: run.chatId ?? run.runId,\n        projectId,\n        builderRunId: run.runId,\n        startedAt: run.startedAt,\n        endedAt: run.endedAt,\n        status: run.status,\n        outcome: run.outcome,\n        llmTurns: spans.filter((s) => s.kind === 'llm').length,\n        toolCalls: spans.filter((s) => s.kind === 'tool').length,\n        buildRunId: build?.runId,\n        appRuntimeRunIds: runtime,\n      })\n    }\n    return out\n  }\n}\n","/**\n * Three-layer evaluation — the canonical scoring breakdown for\n * builder-of-builders workflows.\n *\n *   meta_score:    did the builder understand + satisfy user intent?\n *                  (judge verdict attached to the builder run)\n *   build_score:   did the generated scaffold build + pass its own tests?\n *                  (outcome.score on the app-build child run)\n *   runtime_score: did the generated agent pass its domain scenarios?\n *                  (mean outcome.score over app-runtime grandchild runs)\n *\n * Returns a structured report per project. The cross-layer correlation\n * is the highest-leverage signal the framework computes — if\n * meta_score doesn't predict runtime_score, the builder's self-scoring\n * is broken.\n *\n * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a\n * scaffold-builder eval that grades compose + build without driving a\n * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures\n * meta + build only. Consumers can tell the two apart without having to\n * interpret null-runtime as either \"not yet computed\" or \"N/A for this\n * project shape\".\n */\n\nimport { judgeSpans } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport type ProjectKind = 'full' | 'scaffold-only'\n\nexport interface ThreeLayerProjectReport {\n  projectId: string\n  /**\n   * `'full'` when the project has at least one `app-runtime` run;\n   * `'scaffold-only'` when it only has meta + build layers. Lets\n   * downstream consumers treat a null runtime score as expected\n   * (scaffold-only) vs. missing (full, pipeline broke).\n   */\n  kind: ProjectKind\n  builderRunId?: string\n  /** Judge-verdict score on the builder run (0..1 after normalization). */\n  metaScore: number | null\n  buildRunId?: string\n  /** 0..1 from the sandbox harness (testsPassed / testsTotal). */\n  buildScore: number | null\n  appRuntimeRunIds: string[]\n  /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */\n  runtimeScore: number | null\n  runtimePassRate: number | null\n  /**\n   * Layer-aware completeness:\n   *   - `kind='full'`: all three layers scored\n   *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)\n   */\n  complete: boolean\n}\n\nexport async function scoreProject(\n  store: TraceStore,\n  projectId: string,\n): Promise<ThreeLayerProjectReport> {\n  const allRuns = await store.listRuns({ projectId })\n  const builder = latestByLayer(allRuns, 'builder')\n  const build = latestByLayer(allRuns, 'app-build')\n  const runtime = allRuns.filter((r) => r.layer === 'app-runtime')\n\n  const metaScore = builder ? await extractMetaScore(store, builder.runId) : null\n  const buildScore = build?.outcome?.score ?? null\n  const runtimeScores = runtime\n    .map((r) => r.outcome?.score)\n    .filter((s): s is number => typeof s === 'number')\n  const runtimeScore =\n    runtimeScores.length > 0\n      ? runtimeScores.reduce((a, b) => a + b, 0) / runtimeScores.length\n      : null\n  const runtimePassed = runtime.filter((r) => r.outcome?.pass === true).length\n  const runtimePassRate = runtime.length > 0 ? runtimePassed / runtime.length : null\n\n  const kind: ProjectKind = runtime.length === 0 ? 'scaffold-only' : 'full'\n  const complete =\n    kind === 'scaffold-only'\n      ? metaScore !== null && buildScore !== null\n      : metaScore !== null && buildScore !== null && runtimeScore !== null\n\n  return {\n    projectId,\n    kind,\n    builderRunId: builder?.runId,\n    metaScore,\n    buildRunId: build?.runId,\n    buildScore,\n    appRuntimeRunIds: runtime.map((r) => r.runId),\n    runtimeScore,\n    runtimePassRate,\n    complete,\n  }\n}\n\n/** Aggregate scoring across every project in a corpus. */\nexport async function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]> {\n  const runs = await store.listRuns()\n  const projectIds = [...new Set(runs.map((r) => r.projectId).filter((p): p is string => !!p))]\n  return Promise.all(projectIds.map((p) => scoreProject(store, p)))\n}\n\nfunction latestByLayer(runs: Run[], layer: Run['layer']): Run | undefined {\n  const filtered = runs.filter((r) => r.layer === layer).sort((a, b) => b.startedAt - a.startedAt)\n  return filtered[0]\n}\n\nasync function extractMetaScore(store: TraceStore, builderRunId: string): Promise<number | null> {\n  const js = await judgeSpans(store, builderRunId)\n  const meta = js.find(\n    (s) => s.judgeId === 'builder-meta' && s.dimension === 'user_intent_satisfaction',\n  )\n  if (!meta) return null\n  // Normalize score to 0..1. Accept 0-1 natively; 0-10 scale is also common.\n  if (meta.score >= 0 && meta.score <= 1) return meta.score\n  if (meta.score >= 0 && meta.score <= 10) return meta.score / 10\n  return null\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA8CO,IAAM,iBAAN,MAAqB;AAAA,EAClB;AAAA,EACA;AAAA,EACC;AAAA,EACA;AAAA,EACD;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,OAAmB,MAA0B,QAAwB;AAC/E,SAAK,QAAQ;AACb,SAAK,YAAY,KAAK;AACtB,SAAK,SAAS,KAAK,UAAU,SAAS;AACtC,SAAK,gBAAgB;AACrB,SAAK,iBAAiB,IAAI,aAAa,KAAK;AAAA,EAC9C;AAAA;AAAA,EAGA,MAAM,UAAU,aAAa,GAAG,KAAK,SAAS,SAA0B;AACtE,UAAM,KAAK,eAAe,SAAS;AAAA,MACjC;AAAA,MACA,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,OAAO;AAAA,IACT,CAAC;AACD,SAAK,eAAe,KAAK,eAAe;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAwB;AAC1B,QAAI,CAAC,KAAK,aAAc,OAAM,IAAI,MAAM,gDAAgD;AACxF,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,KAAK,SAAgF;AACzF,QAAI,CAAC,KAAK,aAAc,OAAM,IAAI,MAAM,6CAA6C;AACrF,UAAM,eAAe,IAAI,aAAa,KAAK,KAAK;AAChD,UAAM,aAAa,SAAS;AAAA,MAC1B,YAAY,QAAQ,cAAc,GAAG,KAAK,SAAS;AAAA,MACnD,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,OAAO;AAAA,IACT,CAAC;AACD,UAAM,UAAU,IAAI,eAAe,QAAQ,UAAU,KAAK,aAAa;AACvE,UAAM,SAAS,MAAM,QAAQ,IAAI,QAAQ,SAAS,YAAY;AAC9D,UAAM,aAAa,OAAO;AAAA,MACxB,MAAM,OAAO;AAAA,MACb,OAAO,OAAO;AAAA,MACd,cAAc,OAAO,SAAS,YAAY;AAAA,IAC5C,CAAC;AACD,SAAK,iBAAiB,aAAa;AACnC,WAAO,EAAE,OAAO,aAAa,OAAO,OAAO;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,eAAe,SAA8D;AACjF,UAAM,cAAc,KAAK,kBAAkB,KAAK;AAChD,QAAI,CAAC;AACH,YAAM,IAAI,MAAM,gEAAgE;AAClF,UAAM,EAAE,UAAU,OAAO,IAAI;AAC7B,UAAM,SAAS,MAAM,sBAAsB,UAAU,KAAK,OAAO;AAAA,MAC/D,QAAQ,UAAU,KAAK;AAAA,MACvB,YAAY,EAAE,SAAS,QAAW,WAAW,QAAW,kBAAkB,OAAU;AAAA,IACtF,CAAC;AAED,UAAM,KAAK,MAAM,UAAU,OAAO,OAAO;AAAA,MACvC;AAAA,MACA,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,OAAO;AAAA,IACT,CAAC;AACD,WAAO;AAAA,EACT;AAAA;AAAA;AAAA,EAIA,MAAM,gBAAgB,OAAe,WAAmC;AACtE,QAAI,CAAC,KAAK;AACR,YAAM,IAAI,MAAM,wDAAwD;AAC1E,UAAM,KAAK,eAAe,YAAY;AAAA,MACpC,SAAS;AAAA,MACT,cAAc,KAAK;AAAA;AAAA,MACnB,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAM,QAAQ,SAA2E;AACvF,UAAM,KAAK,eAAe,OAAO;AAAA,MAC/B,MAAM,QAAQ;AAAA,MACd,OAAO,QAAQ;AAAA,MACf,OAAO,QAAQ;AAAA,IACjB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,gBAAgB,YAA2C;AAC/D,UAAM,cAAc,KAAK,kBAAkB,KAAK;AAChD,QAAI,CAAC;AACH,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AACF,UAAM,UAAU,IAAI,aAAa,KAAK,KAAK;AAC3C,UAAM,QAAQ,SAAS;AAAA,MACrB;AAAA,MACA,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb;AAAA,MACA,OAAO;AAAA,IACT,CAAC;AACD,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,iBAAiB,MAKH;AAClB,QAAI,CAAC,KAAK;AACR,YAAM,IAAI,MAAM,yDAAyD;AAC3E,UAAM,UAAU,IAAI,aAAa,KAAK,KAAK;AAC3C,UAAM,QAAQ,SAAS;AAAA,MACrB,YAAY,KAAK,cAAc,GAAG,KAAK,SAAS;AAAA,MAChD,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,OAAO;AAAA,IACT,CAAC;AACD,UAAM,QAAQ,OAAO;AAAA,MACnB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,cAAc,KAAK,OAAO,YAAY;AAAA,MACtC,OAAO,KAAK;AAAA,IACd,CAAC;AACD,SAAK,iBAAiB,QAAQ;AAC9B,WAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,IAAI,sBAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EACA,IAAI,oBAAwC;AAC1C,WAAO,KAAK;AAAA,EACd;AACF;AAOA,eAAsB,qBACpB,OACA,WAOC;AACD,QAAM,OAAO,MAAM,MAAM,SAAS,EAAE,UAAU,CAAC;AAC/C,QAAM,WAAW,KACd,OAAO,CAAC,MAAM,EAAE,UAAU,SAAS,EACnC,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC3C,QAAM,YAAY,KACf,OAAO,CAAC,MAAM,EAAE,UAAU,WAAW,EACrC,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC3C,QAAM,iBAAiB,KACpB,OAAO,CAAC,MAAM,EAAE,UAAU,aAAa,EACvC,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC3C,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,gBAAgB,SAAS,CAAC;AAAA,IAC1B,cAAc,UAAU,CAAC;AAAA,IACzB,oBAAoB;AAAA,EACtB;AACF;AAEA,SAAS,WAAmB;AAC1B,MAAI,OAAO,WAAW,QAAQ,eAAe,WAAY,QAAO,WAAW,OAAO,WAAW;AAC7F,SAAO,GAAG,KAAK,IAAI,EAAE,SAAS,EAAE,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,GAAG,EAAE,CAAC;AAC9E;;;AC5NO,SAAS,gBAAgB,SAAuD;AACrF,QAAM,mBAAmB,QAAQ,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE;AAC3D,SAAO;AAAA,IACL,aAAa;AAAA,MACX;AAAA,MACA,CAAC,MAAM,EAAE;AAAA,MACT,CAAC,MAAM,EAAE;AAAA,IACX;AAAA,IACA,eAAe;AAAA,MACb;AAAA,MACA,CAAC,MAAM,EAAE;AAAA,MACT,CAAC,MAAM,EAAE;AAAA,IACX;AAAA,IACA,gBAAgB;AAAA,MACd;AAAA,MACA,CAAC,MAAM,EAAE;AAAA,MACT,CAAC,MAAM,EAAE;AAAA,IACX;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,SACP,SACA,GACA,GAC8B;AAC9B,QAAM,KAAe,CAAC;AACtB,QAAM,KAAe,CAAC;AACtB,aAAW,KAAK,SAAS;AACvB,UAAM,IAAI,EAAE,CAAC;AACb,UAAM,IAAI,EAAE,CAAC;AACb,QAAI,MAAM,QAAQ,MAAM,QAAQ,OAAO,SAAS,CAAC,KAAK,OAAO,SAAS,CAAC,GAAG;AACxE,SAAG,KAAK,CAAC;AACT,SAAG,KAAK,CAAC;AAAA,IACX;AAAA,EACF;AACA,MAAI,GAAG,SAAS,EAAG,QAAO;AAC1B,SAAO;AAAA,IACL,GAAG,GAAG;AAAA,IACN,SAAS,SAAS,IAAI,EAAE;AAAA,IACxB,UAAU,UAAU,IAAI,EAAE;AAAA,EAC5B;AACF;;;ACjCO,IAAM,kBAAN,MAAsB;AAAA,EAC3B,YAAoB,OAAmB;AAAnB;AAAA,EAAoB;AAAA,EAApB;AAAA,EAEpB,MAAM,eAA0C;AAC9C,UAAM,OAAO,MAAM,KAAK,MAAM,SAAS;AACvC,UAAM,YAAY,oBAAI,IAAmB;AACzC,eAAW,KAAK,MAAM;AACpB,UAAI,CAAC,EAAE,UAAW;AAClB,YAAM,MAAM,UAAU,IAAI,EAAE,SAAS,KAAK,CAAC;AAC3C,UAAI,KAAK,CAAC;AACV,gBAAU,IAAI,EAAE,WAAW,GAAG;AAAA,IAChC;AACA,UAAM,YAA8B,CAAC;AACrC,eAAW,CAAC,WAAW,WAAW,KAAK,WAAW;AAChD,YAAM,SAAS,YAAY,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC3E,YAAM,QAAQ,YAAY,OAAO,CAAC,MAAM,EAAE,UAAU,SAAS;AAC7D,YAAM,SAAS,YAAY,OAAO,CAAC,MAAM,EAAE,UAAU,WAAW;AAChE,YAAM,WAAW,YAAY,OAAO,CAAC,MAAM,EAAE,UAAU,aAAa;AACpE,YAAM,SAAS,OAAO,CAAC;AACvB,UAAI,CAAC,OAAQ;AACb,gBAAU,KAAK;AAAA,QACb;AAAA,QACA,WAAW,MAAM;AAAA,QACjB,YAAY,OAAO;AAAA,QACnB,iBAAiB,SAAS;AAAA,QAC1B,gBAAgB,OAAO;AAAA,QACvB,cAAc,MAAM,CAAC,GAAG;AAAA,QACxB,eAAe,OAAO,UAClB,EAAE,MAAM,OAAO,QAAQ,QAAQ,OAAO,OAAO,OAAO,QAAQ,MAAM,IAClE;AAAA,MACN,CAAC;AAAA,IACH;AACA,WAAO,UAAU,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAAA,EACrE;AAAA,EAEA,MAAM,gBAAgB,WAAoD;AACxE,UAAM,OAAO,MAAM,KAAK,MAAM,SAAS,EAAE,UAAU,CAAC;AACpD,UAAM,UAAU,KAAK,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AACrE,WAAO,QAAQ,IAAI,CAAC,SAAS;AAAA,MAC3B;AAAA,MACA,aACE,IAAI,UAAU,YACV,SACA,IAAI,UAAU,cACZ,UACA,IAAI,UAAU,gBACZ,YACA;AAAA,IACZ,EAAE;AAAA,EACJ;AAAA,EAEA,MAAM,aAAa,WAA2C;AAC5D,UAAM,eAAe,MAAM,KAAK,MAAM,SAAS,EAAE,WAAW,OAAO,UAAU,CAAC,GAAG;AAAA,MAC/E,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE;AAAA,IAC5B;AACA,UAAM,cAAc,OAAO,UAAkB,KAAK,MAAM,SAAS,EAAE,aAAa,MAAM,CAAC;AACvF,UAAM,MAAqB,CAAC;AAC5B,eAAW,OAAO,aAAa;AAC7B,YAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;AACzD,YAAM,WAAW,MAAM,YAAY,IAAI,KAAK;AAC5C,YAAM,QAAQ,SAAS,KAAK,CAAC,MAAM,EAAE,UAAU,WAAW;AAC1D,YAAM,UAAoB,CAAC;AAG3B,UAAI,OAAO;AACT,cAAM,SAAS,MAAM,YAAY,MAAM,KAAK;AAC5C,mBAAW,KAAK,OAAQ,KAAI,EAAE,UAAU,cAAe,SAAQ,KAAK,EAAE,KAAK;AAAA,MAC7E;AACA,iBAAW,KAAK,SAAU,KAAI,EAAE,UAAU,cAAe,SAAQ,KAAK,EAAE,KAAK;AAC7E,UAAI,KAAK;AAAA,QACP,QAAQ,IAAI,UAAU,IAAI;AAAA,QAC1B;AAAA,QACA,cAAc,IAAI;AAAA,QAClB,WAAW,IAAI;AAAA,QACf,SAAS,IAAI;AAAA,QACb,QAAQ,IAAI;AAAA,QACZ,SAAS,IAAI;AAAA,QACb,UAAU,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE;AAAA,QAChD,WAAW,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE;AAAA,QAClD,YAAY,OAAO;AAAA,QACnB,kBAAkB;AAAA,MACpB,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AACF;;;ACzEA,eAAsB,aACpB,OACA,WACkC;AAClC,QAAM,UAAU,MAAM,MAAM,SAAS,EAAE,UAAU,CAAC;AAClD,QAAM,UAAU,cAAc,SAAS,SAAS;AAChD,QAAM,QAAQ,cAAc,SAAS,WAAW;AAChD,QAAM,UAAU,QAAQ,OAAO,CAAC,MAAM,EAAE,UAAU,aAAa;AAE/D,QAAM,YAAY,UAAU,MAAM,iBAAiB,OAAO,QAAQ,KAAK,IAAI;AAC3E,QAAM,aAAa,OAAO,SAAS,SAAS;AAC5C,QAAM,gBAAgB,QACnB,IAAI,CAAC,MAAM,EAAE,SAAS,KAAK,EAC3B,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACnD,QAAM,eACJ,cAAc,SAAS,IACnB,cAAc,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc,SACzD;AACN,QAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS,SAAS,IAAI,EAAE;AACtE,QAAM,kBAAkB,QAAQ,SAAS,IAAI,gBAAgB,QAAQ,SAAS;AAE9E,QAAM,OAAoB,QAAQ,WAAW,IAAI,kBAAkB;AACnE,QAAM,WACJ,SAAS,kBACL,cAAc,QAAQ,eAAe,OACrC,cAAc,QAAQ,eAAe,QAAQ,iBAAiB;AAEpE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,SAAS;AAAA,IACvB;AAAA,IACA,YAAY,OAAO;AAAA,IACnB;AAAA,IACA,kBAAkB,QAAQ,IAAI,CAAC,MAAM,EAAE,KAAK;AAAA,IAC5C;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGA,eAAsB,iBAAiB,OAAuD;AAC5F,QAAM,OAAO,MAAM,MAAM,SAAS;AAClC,QAAM,aAAa,CAAC,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,MAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5F,SAAO,QAAQ,IAAI,WAAW,IAAI,CAAC,MAAM,aAAa,OAAO,CAAC,CAAC,CAAC;AAClE;AAEA,SAAS,cAAc,MAAa,OAAsC;AACxE,QAAM,WAAW,KAAK,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAC/F,SAAO,SAAS,CAAC;AACnB;AAEA,eAAe,iBAAiB,OAAmB,cAA8C;AAC/F,QAAM,KAAK,MAAM,WAAW,OAAO,YAAY;AAC/C,QAAM,OAAO,GAAG;AAAA,IACd,CAAC,MAAM,EAAE,YAAY,kBAAkB,EAAE,cAAc;AAAA,EACzD;AACA,MAAI,CAAC,KAAM,QAAO;AAElB,MAAI,KAAK,SAAS,KAAK,KAAK,SAAS,EAAG,QAAO,KAAK;AACpD,MAAI,KAAK,SAAS,KAAK,KAAK,SAAS,GAAI,QAAO,KAAK,QAAQ;AAC7D,SAAO;AACT;","names":[]}

package/dist/campaign/index.d.ts

@@ -1,11 +1,11 @@
 import { A as AnalyzeTracesOptions, a as AnalyzeTracesInput, b as AnalyzeTracesResult } from '../analyst-C8HHvfJp.js';
-import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, n as GenerationRecord, G as Gate, J as JudgeScore, L as LabeledScenarioStore, s as LabeledScenarioWrite, t as LabeledScenarioSampleArgs, u as LabeledScenarioRecord, v as LabelTrust, c as SurfaceProposer, w as ProposedCandidate, x as ProposeContext, y as LabeledScenarioSource, C as CampaignResult, j as CodeSurface } from '../types-DQRY8ZT-.js';
-export { e as CampaignAggregates, f as CampaignArtifactWriter, g as CampaignCellResult, h as CampaignCostMeter, z as CampaignTokenUsage, i as CampaignTraceWriter, D as DispatchFn, k as GateContext, d as GateDecision, l as GateResult, m as GenerationCandidate, A as JudgeAggregate, o as JudgeDimension, p as Mutator, O as OptimizationProposer, q as OptimizerConfig, P as ParetoParent, R as RedactionStatus, B as ScenarioAggregate, r as SessionScript, T as TraceSpan, E as isProposedCandidate, F as labelTrustRank } from '../types-DQRY8ZT-.js';
-import { b as RunCampaignOptions, c as RunImprovementLoopOptions, C as CampaignStorage } from '../gepa-C1NCIZ9o.js';
-export { e as GepaProposerConstraints, G as GepaProposerOptions, O as OpenAutoPrOptions, h as OpenAutoPrResult, a as RunImprovementLoopResult, R as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaProposer, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, d as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../gepa-C1NCIZ9o.js';
-export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryProposer, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-CncDq9qE.js';
+import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, d as GenerationRecord, f as Gate, J as JudgeScore, L as LabeledScenarioStore, s as LabeledScenarioWrite, t as LabeledScenarioSampleArgs, u as LabeledScenarioRecord, v as LabelTrust, e as SurfaceProposer, w as ProposedCandidate, x as ProposeContext, y as LabeledScenarioSource, C as CampaignResult, n as CodeSurface } from '../types-BMahhhio.js';
+export { j as CampaignAggregates, k as CampaignArtifactWriter, l as CampaignCellResult, m as CampaignCostMeter, z as CampaignTokenUsage, c as CampaignTraceWriter, D as DispatchFn, g as GateContext, i as GateDecision, G as GateResult, o as GenerationCandidate, A as JudgeAggregate, p as JudgeDimension, h as Mutator, O as OptimizationProposer, q as OptimizerConfig, P as ParetoParent, R as RedactionStatus, B as ScenarioAggregate, r as SessionScript, T as TraceSpan, E as isProposedCandidate, F as labelTrustRank } from '../types-BMahhhio.js';
+import { R as RunCampaignOptions, c as RunImprovementLoopOptions, C as CampaignStorage } from '../gepa-Dprxvz8r.js';
+export { e as GepaProposerConstraints, G as GepaProposerOptions, O as OpenAutoPrOptions, h as OpenAutoPrResult, b as RunImprovementLoopResult, a as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaProposer, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, d as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../gepa-Dprxvz8r.js';
+export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryProposer, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-DJ--Jcbg.js';
 import { S as SignedManifest, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../pre-registration-nfUdc9EQ.js';
-import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-CCJpTGOS.js';
+import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-xP-cWc5k.js';
 import { L as LlmClientOptions } from '../llm-client-Bj7g0rqu.js';
 import { AgentProfile } from '@tangle-network/agent-interface';
 import { A as AgentEvalError } from '../errors-CzMUYo7b.js';

package/dist/campaign/index.js

@@ -10,7 +10,7 @@ import {
   paretoPolicy,
   paretoSignificanceGate,
   runEval
-} from "../chunk-KKWJD5E6.js";
+} from "../chunk-RPZMBW27.js";
 import {
   agentProfileHash,
   agentProfileId,
@@ -39,14 +39,14 @@ import {
   runOptimization,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-7QTQKIDD.js";
+} from "../chunk-JMIUM2HE.js";
 import {
   assertRealBackend,
   fsCampaignStorage,
   inMemoryCampaignStorage,
   runCampaign,
   summarizeBackendIntegrity
-} from "../chunk-2T4EZACH.js";
+} from "../chunk-HRGTA6U5.js";
 import {
   estimateCost,
   isModelPriced
@@ -56,7 +56,7 @@ import {
   DEFAULT_TRACE_ANALYST_KINDS,
   createTraceAnalystKind
 } from "../chunk-S4SYLDFX.js";
-import "../chunk-5HRORJQY.js";
+import "../chunk-AIGWQEME.js";
 import {
   callLlm
 } from "../chunk-CWNP4DV4.js";
@@ -65,7 +65,7 @@ import {
   eProcess,
   mulberry32,
   pairedBootstrap
-} from "../chunk-56GC6VYO.js";
+} from "../chunk-HRGUJTER.js";
 import {
   analyzeTraces
 } from "../chunk-IZCEK2HR.js";

package/dist/{chunk-AQ5WQAIV.js → chunk-3NHEO6ZC.js}

@@ -15,7 +15,7 @@ import {
 // src/trace-analyst/behavioral-metrics.ts
 var INPUT_GROWTH_FACTOR = 3;
 var MIN_TOOL_CALLS = 3;
-var VERIFY_RE = /verif|eval|inspect|check|assert|validat|review|confirm/i;
+var VERIFY_RE = /verif|eval|inspect|check|assert|validat|review|confirm|read|grep|glob|search|view|\blist\b|\bls\b|\bcat\b|\bfind\b|diff|status|\btest|lint|typecheck/i;
 function num(v) {
   return typeof v === "number" && Number.isFinite(v) ? v : null;
 }
@@ -204,4 +204,4 @@ export {
   behavioralAnalyst,
   buildDefaultAnalystRegistry
 };
-//# sourceMappingURL=chunk-AQ5WQAIV.js.map
+//# sourceMappingURL=chunk-3NHEO6ZC.js.map

package/dist/chunk-3NHEO6ZC.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/default-registry.ts"],"sourcesContent":["/**\n * Deterministic behavioral metrics over OTLP spans — pure arithmetic, no LLM.\n *\n * These are the model-independent multiplier: the four trace-quality signals a\n * tolerant analyzer (e.g. HALO) re-derives per run inside the model — token\n * growth, output decay, tool monoculture, missing self-verification — computed\n * here once, in TypeScript, with zero model judgment. A finding that falls out\n * of arithmetic is trivially model-agnostic and cannot hallucinate the trend.\n *\n * General, not trace-specific: the detectors key off token trajectories and\n * tool usage present in any agentic OTLP trace, not any one benchmark.\n */\n\nimport {\n  LLM_INPUT_TOKEN_ATTR_KEYS,\n  LLM_OUTPUT_TOKEN_ATTR_KEYS,\n  TOOL_NAME_ATTR_KEYS,\n} from '../trace/otlp-attributes'\nimport type { TraceAnalystSpan } from './types'\n\nexport type SuboptimalCode =\n  | 'monotonic-input-growth'\n  | 'output-length-decay'\n  | 'single-tool-dependency'\n  | 'no-self-verification'\n\nexport interface SuboptimalSignal {\n  code: SuboptimalCode\n  severity: 'high' | 'medium' | 'low'\n  /** Human-readable claim, with the backing numbers inlined. */\n  detail: string\n  /** The exact figures the detector fired on — auditable, no model in the loop. */\n  evidence: Record<string, number | string | boolean>\n}\n\nexport interface BehavioralMetrics {\n  llmCallCount: number\n  inputTokenTrajectory: number[]\n  outputTokenTrajectory: number[]\n  toolHistogram: Record<string, number>\n  totalToolCalls: number\n  distinctTools: number\n  /** distinct/total tool calls; 1.0 when there are no tool calls. */\n  toolDiversityRatio: number\n  hasSelfVerification: boolean\n  signals: SuboptimalSignal[]\n}\n\n/** ≥ this input-token growth ratio across a run, with no compression, fires. */\nconst INPUT_GROWTH_FACTOR = 3\n/** Tool-usage signals need at least this many calls to be meaningful. */\nconst MIN_TOOL_CALLS = 3\n/** Tool names that read or check state count as self-verification, not mutation.\n *  Covers the inspect verbs plus the read/search tools real harnesses use to\n *  verify (Claude Code Read/Grep/Glob, codex read_file/ls/cat, git status/diff,\n *  test/lint). A pure shell tool (Bash/exec_command) is intentionally NOT matched\n *  — its name can't tell a `pytest` from an `rm`. */\nconst VERIFY_RE =\n  /verif|eval|inspect|check|assert|validat|review|confirm|read|grep|glob|search|view|\\blist\\b|\\bls\\b|\\bcat\\b|\\bfind\\b|diff|status|\\btest|lint|typecheck/i\n\nfunction num(v: unknown): number | null {\n  return typeof v === 'number' && Number.isFinite(v) ? v : null\n}\nfunction numAttr(attrs: Record<string, unknown>, keys: readonly string[]): number | null {\n  for (const key of keys) {\n    const value = num(attrs[key])\n    if (value !== null) return value\n  }\n  return null\n}\nfunction inputTokensOf(s: TraceAnalystSpan): number | null {\n  return (\n    numAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS) ?? num(s.attributes['llm.usage.input_tokens'])\n  )\n}\nfunction outputTokensOf(s: TraceAnalystSpan): number | null {\n  return (\n    numAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS) ??\n    num(s.attributes['llm.usage.output_tokens'])\n  )\n}\nfunction stepOf(s: TraceAnalystSpan): number | null {\n  return num(s.attributes.step)\n}\nfunction toolNameOf(s: TraceAnalystSpan): string | null {\n  if (s.tool_name) return s.tool_name\n  for (const key of TOOL_NAME_ATTR_KEYS) {\n    const t = s.attributes[key]\n    if (typeof t === 'string' && t.length > 0) return t\n  }\n  return null\n}\n\n/**\n * Reduce a span list to behavioral metrics + fired suboptimality signals.\n * Pure + deterministic: same spans → same output, on any machine, no model.\n */\nexport function computeTraceMetrics(spans: readonly TraceAnalystSpan[]): BehavioralMetrics {\n  // Order by step (when present) then start_time so trajectories reflect run order.\n  const ordered = [...spans].sort((a, b) => {\n    const sa = stepOf(a)\n    const sb = stepOf(b)\n    if (sa !== null && sb !== null && sa !== sb) return sa - sb\n    return a.start_time.localeCompare(b.start_time)\n  })\n\n  const inputTokenTrajectory: number[] = []\n  const outputTokenTrajectory: number[] = []\n  const toolHistogram: Record<string, number> = {}\n  let hasSelfVerification = false\n\n  for (const s of ordered) {\n    const inT = inputTokensOf(s)\n    if (inT !== null) inputTokenTrajectory.push(inT)\n    const outT = outputTokensOf(s)\n    if (outT !== null) outputTokenTrajectory.push(outT)\n    const tool = toolNameOf(s)\n    if (tool) {\n      toolHistogram[tool] = (toolHistogram[tool] ?? 0) + 1\n      if (VERIFY_RE.test(tool)) hasSelfVerification = true\n    }\n  }\n\n  const totalToolCalls = Object.values(toolHistogram).reduce((a, b) => a + b, 0)\n  const distinctTools = Object.keys(toolHistogram).length\n  const toolDiversityRatio = totalToolCalls === 0 ? 1 : distinctTools / totalToolCalls\n\n  const signals: SuboptimalSignal[] = []\n\n  if (inputTokenTrajectory.length >= 3) {\n    const first = inputTokenTrajectory[0]!\n    const last = inputTokenTrajectory[inputTokenTrajectory.length - 1]!\n    // first === 0 with later growth is an unbounded ratio (0→huge context blowup);\n    // treat it as infinite so the signal fires, and report it as such instead of\n    // dividing by zero for the displayed factor.\n    const growthFromZero = first === 0 && last > 0\n    const growth = growthFromZero ? Infinity : first > 0 ? last / first : 0\n    if (last > first && growth >= INPUT_GROWTH_FACTOR) {\n      const growthLabel = growthFromZero ? '0→nonzero (unbounded)' : `${growth.toFixed(1)}x`\n      signals.push({\n        code: 'monotonic-input-growth',\n        severity: 'high',\n        detail: `LLM input tokens grew ${growthLabel} (${first}→${last}) across ${inputTokenTrajectory.length} calls — full history re-sent each step with no compression.`,\n        evidence: {\n          first,\n          last,\n          growth_x: growthFromZero ? 'unbounded' : Number(growth.toFixed(2)),\n          calls: inputTokenTrajectory.length,\n        },\n      })\n    }\n  }\n\n  if (outputTokenTrajectory.length >= 3) {\n    const first = outputTokenTrajectory[0]!\n    const last = outputTokenTrajectory[outputTokenTrajectory.length - 1]!\n    if (last < first) {\n      signals.push({\n        code: 'output-length-decay',\n        severity: 'medium',\n        detail: `LLM output tokens shrank ${first}→${last} over ${outputTokenTrajectory.length} calls — less planning/reasoning per step as context grows.`,\n        evidence: { first, last, calls: outputTokenTrajectory.length },\n      })\n    }\n  }\n\n  if (totalToolCalls >= MIN_TOOL_CALLS && distinctTools === 1) {\n    const only = Object.keys(toolHistogram)[0]!\n    signals.push({\n      code: 'single-tool-dependency',\n      severity: 'medium',\n      detail: `All ${totalToolCalls} tool calls are \\`${only}\\` — no tool diversity and no fallback path.`,\n      evidence: { tool: only, calls: totalToolCalls, distinct_tools: 1 },\n    })\n  }\n\n  if (totalToolCalls >= MIN_TOOL_CALLS && !hasSelfVerification) {\n    signals.push({\n      code: 'no-self-verification',\n      severity: 'medium',\n      detail: `${totalToolCalls} tool calls and none verify/inspect/check state — the agent never validates its own actions.`,\n      evidence: { tool_calls: totalToolCalls, verification_calls: 0 },\n    })\n  }\n\n  return {\n    llmCallCount: inputTokenTrajectory.length,\n    inputTokenTrajectory,\n    outputTokenTrajectory,\n    toolHistogram,\n    totalToolCalls,\n    distinctTools,\n    toolDiversityRatio,\n    hasSelfVerification,\n    signals,\n  }\n}\n","/**\n * `behavioralAnalyst` — a DETERMINISTIC analyst (cost.kind = 'deterministic',\n * never calls the LLM). It produces the efficiency/behavioral findings a\n * tolerant agentic analyzer (HALO) re-derives per run inside the model —\n * context bloat, output decay, tool monoculture, missing self-verification —\n * directly from arithmetic over spans (`computeTraceMetrics`).\n *\n * Why it matters: these findings are model-agnostic BY CONSTRUCTION (no model\n * in the loop), so they cannot return 0 on a weak model the way the Ax-RLM\n * does — and they are strictly more reliable than HALO, which spends tokens\n * re-deriving the same numbers and can hallucinate the trend. The agentic\n * RLM kinds remain for SEMANTIC findings that genuinely need a model; this\n * analyst owns the behavioral class.\n */\n\nimport {\n  type BehavioralMetrics,\n  computeTraceMetrics,\n  type SuboptimalCode,\n} from '../trace-analyst/behavioral-metrics'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { TraceAnalystSpan } from '../trace-analyst/types'\nimport { type Analyst, type AnalystFinding, makeFinding } from './types'\n\nconst RECOMMENDED_ACTION: Record<SuboptimalCode, string> = {\n  'monotonic-input-growth':\n    'Add a context-budget instruction: once prior context exceeds a threshold, summarize earlier steps into a short status line instead of re-sending full history.',\n  'output-length-decay':\n    'Require a minimum planning/reasoning budget per step so late steps do not degrade into terse, error-prone commands.',\n  'single-tool-dependency':\n    'Direct the agent to use the full toolset (verify / inspect / alternate actions), not a single execute call, and to plan a fallback when a call returns an unexpected result.',\n  'no-self-verification':\n    'After every state-mutating action, verify the result (eval / inspect / assert) before proceeding.',\n}\n\nconst ANALYST_ID = 'efficiency-behavioral'\n\n/**\n * Map computed signals → structured AnalystFindings. Pure: no LLM, no clock\n * dependence beyond `produced_at` (overridable for deterministic tests).\n */\nexport function deriveEfficiencyFindings(\n  metrics: BehavioralMetrics,\n  opts: { analystId?: string; producedAt?: string } = {},\n): AnalystFinding[] {\n  const analystId = opts.analystId ?? ANALYST_ID\n  return metrics.signals.map((sig) =>\n    makeFinding({\n      analyst_id: analystId,\n      area: 'efficiency',\n      subject: sig.code, // kebab — passes the cluster grammar; stable key for diffFindings\n      claim: sig.detail,\n      severity: sig.severity,\n      // Deterministic arithmetic over spans, not a model judgment → certain.\n      confidence: 1,\n      evidence_refs: [\n        {\n          kind: 'metric',\n          uri: `metric://efficiency/${sig.code}`,\n          excerpt: JSON.stringify(sig.evidence),\n        },\n      ],\n      recommended_action: RECOMMENDED_ACTION[sig.code],\n      metadata: { deterministic: true, evidence: sig.evidence },\n      ...(opts.producedAt ? { produced_at: opts.producedAt } : {}),\n    }),\n  )\n}\n\n/** The deterministic behavioral/efficiency analyst (no LLM, any-model). */\nexport function behavioralAnalyst(): Analyst<TraceAnalysisStore> {\n  return {\n    id: ANALYST_ID,\n    description:\n      'Deterministic behavioral/efficiency findings over OTLP spans — token-growth, output-decay, tool-monoculture, missing self-verification. Zero LLM; model-agnostic by construction.',\n    inputKind: 'trace-store',\n    cost: { kind: 'deterministic' },\n    version: '1.0.0',\n    async analyze(store) {\n      const overview = await store.getOverview()\n      const spans: TraceAnalystSpan[] = []\n      for (const traceId of overview.sample_trace_ids) {\n        const viewed = await store.viewTrace({ trace_id: traceId })\n        if (viewed.spans) spans.push(...viewed.spans)\n      }\n      return deriveEfficiencyFindings(computeTraceMetrics(spans))\n    },\n  }\n}\n","/**\n * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers\n * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`.\n *\n * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no\n * model and is model-agnostic by construction). The agentic RLM kinds are\n * registered only when an `ai` service is supplied — so a caller with no LLM\n * still gets the full behavioral/efficiency diagnosis, and the substrate's\n * \"any model (including no model)\" guarantee holds at the suite level.\n */\n\nimport type { AxAIService } from '@ax-llm/ax'\nimport { behavioralAnalyst } from './behavioral-analyst'\nimport { createTraceAnalystKind, type TraceAnalystKindSpec } from './kind-factory'\nimport { DEFAULT_TRACE_ANALYST_KINDS } from './kinds'\nimport { AnalystRegistry, type AnalystRegistryOptions } from './registry'\n\nexport interface DefaultAnalystRegistryOptions {\n  /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */\n  ai?: AxAIService\n  /** Model for the agentic kinds (falls back to the ai service default). */\n  model?: string\n  /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */\n  kinds?: readonly TraceAnalystKindSpec[]\n  /** Set false to omit the deterministic behavioral analyst (default: include). */\n  includeBehavioral?: boolean\n  /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */\n  registry?: AnalystRegistryOptions\n}\n\nexport function buildDefaultAnalystRegistry(\n  opts: DefaultAnalystRegistryOptions = {},\n): AnalystRegistry {\n  const registry = new AnalystRegistry(opts.registry)\n  if (opts.includeBehavioral !== false) {\n    registry.register(behavioralAnalyst())\n  }\n  if (opts.ai) {\n    const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS\n    for (const spec of kinds) {\n      registry.register(createTraceAnalystKind(spec, { ai: opts.ai, model: opts.model }))\n    }\n  }\n  return registry\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiDA,IAAM,sBAAsB;AAE5B,IAAM,iBAAiB;AAMvB,IAAM,YACJ;AAEF,SAAS,IAAI,GAA2B;AACtC,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,IAAI,IAAI;AAC3D;AACA,SAAS,QAAQ,OAAgC,MAAwC;AACvF,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,IAAI,MAAM,GAAG,CAAC;AAC5B,QAAI,UAAU,KAAM,QAAO;AAAA,EAC7B;AACA,SAAO;AACT;AACA,SAAS,cAAc,GAAoC;AACzD,SACE,QAAQ,EAAE,YAAY,yBAAyB,KAAK,IAAI,EAAE,WAAW,wBAAwB,CAAC;AAElG;AACA,SAAS,eAAe,GAAoC;AAC1D,SACE,QAAQ,EAAE,YAAY,0BAA0B,KAChD,IAAI,EAAE,WAAW,yBAAyB,CAAC;AAE/C;AACA,SAAS,OAAO,GAAoC;AAClD,SAAO,IAAI,EAAE,WAAW,IAAI;AAC9B;AACA,SAAS,WAAW,GAAoC;AACtD,MAAI,EAAE,UAAW,QAAO,EAAE;AAC1B,aAAW,OAAO,qBAAqB;AACrC,UAAM,IAAI,EAAE,WAAW,GAAG;AAC1B,QAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,EACpD;AACA,SAAO;AACT;AAMO,SAAS,oBAAoB,OAAuD;AAEzF,QAAM,UAAU,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM;AACxC,UAAM,KAAK,OAAO,CAAC;AACnB,UAAM,KAAK,OAAO,CAAC;AACnB,QAAI,OAAO,QAAQ,OAAO,QAAQ,OAAO,GAAI,QAAO,KAAK;AACzD,WAAO,EAAE,WAAW,cAAc,EAAE,UAAU;AAAA,EAChD,CAAC;AAED,QAAM,uBAAiC,CAAC;AACxC,QAAM,wBAAkC,CAAC;AACzC,QAAM,gBAAwC,CAAC;AAC/C,MAAI,sBAAsB;AAE1B,aAAW,KAAK,SAAS;AACvB,UAAM,MAAM,cAAc,CAAC;AAC3B,QAAI,QAAQ,KAAM,sBAAqB,KAAK,GAAG;AAC/C,UAAM,OAAO,eAAe,CAAC;AAC7B,QAAI,SAAS,KAAM,uBAAsB,KAAK,IAAI;AAClD,UAAM,OAAO,WAAW,CAAC;AACzB,QAAI,MAAM;AACR,oBAAc,IAAI,KAAK,cAAc,IAAI,KAAK,KAAK;AACnD,UAAI,UAAU,KAAK,IAAI,EAAG,uBAAsB;AAAA,IAClD;AAAA,EACF;AAEA,QAAM,iBAAiB,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAC7E,QAAM,gBAAgB,OAAO,KAAK,aAAa,EAAE;AACjD,QAAM,qBAAqB,mBAAmB,IAAI,IAAI,gBAAgB;AAEtE,QAAM,UAA8B,CAAC;AAErC,MAAI,qBAAqB,UAAU,GAAG;AACpC,UAAM,QAAQ,qBAAqB,CAAC;AACpC,UAAM,OAAO,qBAAqB,qBAAqB,SAAS,CAAC;AAIjE,UAAM,iBAAiB,UAAU,KAAK,OAAO;AAC7C,UAAM,SAAS,iBAAiB,WAAW,QAAQ,IAAI,OAAO,QAAQ;AACtE,QAAI,OAAO,SAAS,UAAU,qBAAqB;AACjD,YAAM,cAAc,iBAAiB,+BAA0B,GAAG,OAAO,QAAQ,CAAC,CAAC;AACnF,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ,yBAAyB,WAAW,KAAK,KAAK,SAAI,IAAI,YAAY,qBAAqB,MAAM;AAAA,QACrG,UAAU;AAAA,UACR;AAAA,UACA;AAAA,UACA,UAAU,iBAAiB,cAAc,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,UACjE,OAAO,qBAAqB;AAAA,QAC9B;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,sBAAsB,UAAU,GAAG;AACrC,UAAM,QAAQ,sBAAsB,CAAC;AACrC,UAAM,OAAO,sBAAsB,sBAAsB,SAAS,CAAC;AACnE,QAAI,OAAO,OAAO;AAChB,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ,4BAA4B,KAAK,SAAI,IAAI,SAAS,sBAAsB,MAAM;AAAA,QACtF,UAAU,EAAE,OAAO,MAAM,OAAO,sBAAsB,OAAO;AAAA,MAC/D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,kBAAkB,kBAAkB,kBAAkB,GAAG;AAC3D,UAAM,OAAO,OAAO,KAAK,aAAa,EAAE,CAAC;AACzC,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,OAAO,cAAc,qBAAqB,IAAI;AAAA,MACtD,UAAU,EAAE,MAAM,MAAM,OAAO,gBAAgB,gBAAgB,EAAE;AAAA,IACnE,CAAC;AAAA,EACH;AAEA,MAAI,kBAAkB,kBAAkB,CAAC,qBAAqB;AAC5D,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,GAAG,cAAc;AAAA,MACzB,UAAU,EAAE,YAAY,gBAAgB,oBAAoB,EAAE;AAAA,IAChE,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,cAAc,qBAAqB;AAAA,IACnC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;AC5KA,IAAM,qBAAqD;AAAA,EACzD,0BACE;AAAA,EACF,uBACE;AAAA,EACF,0BACE;AAAA,EACF,wBACE;AACJ;AAEA,IAAM,aAAa;AAMZ,SAAS,yBACd,SACA,OAAoD,CAAC,GACnC;AAClB,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO,QAAQ,QAAQ;AAAA,IAAI,CAAC,QAC1B,YAAY;AAAA,MACV,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,IAAI;AAAA;AAAA,MACb,OAAO,IAAI;AAAA,MACX,UAAU,IAAI;AAAA;AAAA,MAEd,YAAY;AAAA,MACZ,eAAe;AAAA,QACb;AAAA,UACE,MAAM;AAAA,UACN,KAAK,uBAAuB,IAAI,IAAI;AAAA,UACpC,SAAS,KAAK,UAAU,IAAI,QAAQ;AAAA,QACtC;AAAA,MACF;AAAA,MACA,oBAAoB,mBAAmB,IAAI,IAAI;AAAA,MAC/C,UAAU,EAAE,eAAe,MAAM,UAAU,IAAI,SAAS;AAAA,MACxD,GAAI,KAAK,aAAa,EAAE,aAAa,KAAK,WAAW,IAAI,CAAC;AAAA,IAC5D,CAAC;AAAA,EACH;AACF;AAGO,SAAS,oBAAiD;AAC/D,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS;AAAA,IACT,MAAM,QAAQ,OAAO;AACnB,YAAM,WAAW,MAAM,MAAM,YAAY;AACzC,YAAM,QAA4B,CAAC;AACnC,iBAAW,WAAW,SAAS,kBAAkB;AAC/C,cAAM,SAAS,MAAM,MAAM,UAAU,EAAE,UAAU,QAAQ,CAAC;AAC1D,YAAI,OAAO,MAAO,OAAM,KAAK,GAAG,OAAO,KAAK;AAAA,MAC9C;AACA,aAAO,yBAAyB,oBAAoB,KAAK,CAAC;AAAA,IAC5D;AAAA,EACF;AACF;;;AC1DO,SAAS,4BACd,OAAsC,CAAC,GACtB;AACjB,QAAM,WAAW,IAAI,gBAAgB,KAAK,QAAQ;AAClD,MAAI,KAAK,sBAAsB,OAAO;AACpC,aAAS,SAAS,kBAAkB,CAAC;AAAA,EACvC;AACA,MAAI,KAAK,IAAI;AACX,UAAM,QAAQ,KAAK,SAAS;AAC5B,eAAW,QAAQ,OAAO;AACxB,eAAS,SAAS,uBAAuB,MAAM,EAAE,IAAI,KAAK,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC;AAAA,IACpF;AAAA,EACF;AACA,SAAO;AACT;","names":[]}

package/dist/{chunk-URWVKRCS.js → chunk-6GT4NI4V.js}

@@ -5,7 +5,7 @@ import {
   pairedBootstrap,
   pairedMde,
   wilcoxonSignedRank
-} from "./chunk-56GC6VYO.js";
+} from "./chunk-HRGUJTER.js";
 import {
   canonicalize,
   hashJson
@@ -877,4 +877,4 @@ export {
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
   researchReport
 };
-//# sourceMappingURL=chunk-URWVKRCS.js.map
+//# sourceMappingURL=chunk-6GT4NI4V.js.map

package/dist/{chunk-5HRORJQY.js → chunk-AIGWQEME.js}

@@ -1,3 +1,7 @@
+import {
+  pearsonR
+} from "./chunk-HRGUJTER.js";
+
 // src/rl/verifiable-reward.ts
 var DEFAULT_DETERMINISTIC_LAYERS = /* @__PURE__ */ new Set([
   "install",
@@ -280,21 +284,6 @@ function clamp012(x) {
   if (!Number.isFinite(x)) return 0;
   return Math.max(0, Math.min(1, x));
 }
-function pearsonR(a, b) {
-  if (a.length !== b.length || a.length < 2) return 0;
-  const ma = mean(a);
-  const mb = mean(b);
-  let num = 0, da = 0, db = 0;
-  for (let i = 0; i < a.length; i++) {
-    const xa = a[i] - ma;
-    const xb = b[i] - mb;
-    num += xa * xb;
-    da += xa * xa;
-    db += xb * xb;
-  }
-  if (da === 0 || db === 0) return 0;
-  return num / Math.sqrt(da * db);
-}
 function ksStatistic(a, b) {
   const sortedA = [...a].sort((x, y) => x - y);
   const sortedB = [...b].sort((x, y) => x - y);
@@ -320,4 +309,4 @@ export {
   filterDeterministicallyRewarded,
   detectRewardHacking
 };
-//# sourceMappingURL=chunk-5HRORJQY.js.map
+//# sourceMappingURL=chunk-AIGWQEME.js.map

package/dist/{chunk-5HRORJQY.js.map → chunk-AIGWQEME.js.map}

@@ -1 +1 @@
-{"version":3,"sources":["../src/rl/verifiable-reward.ts","../src/rl/reward-hacking.ts"],"sourcesContent":["/**\n * Verifiable reward channel.\n *\n * For RL on coding / math / theorem-proving / structured-output tasks, the\n * reward signal is *decidable* — a test passes or fails, a proof checks or\n * doesn't, an output validates against a schema or doesn't. These rewards\n * are dramatically more useful for RL training than LLM-judge scores\n * because they don't drift, can't be Goodhart-gamed by the policy in the\n * same way, and don't require a separate calibration loop.\n *\n * The `MultiLayerVerifier` already produces this signal — it just doesn't\n * surface it in a shape that's clean enough for RL training. This module\n * wraps the verifier output so consumers can:\n *\n *   1. Extract a clean `VerifiableReward` from a `VerificationReport`\n *   2. Distinguish *deterministic* rewards (compile, test, schema) from\n *      *probabilistic* rewards (judge) so they can be weighted differently\n *      in the RL training step\n *   3. Filter `RunRecord[]` to only those with a verifiable reward,\n *      producing the clean training set that DeepSeek-R1-style GRPO and\n *      AlphaProof-style search both depend on\n *\n * Why this matters: every credible 2025-2026 frontier RL result on coding\n * agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,\n * o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing\n * judge scores into the reward signal poisons the gradient. This module\n * is the seam.\n */\n\nimport type { LayerResult, VerificationReport } from '../multi-layer-verifier'\nimport type { RunRecord } from '../run-record'\n\nexport type VerifiableRewardSource =\n  | 'compile' // typecheck / build / lint passed\n  | 'test' // unit / integration test pass-rate\n  | 'schema' // structured output validates\n  | 'sandbox' // sandbox exec exit code\n  | 'judge' // LLM judge — probabilistic, included for completeness\n  | 'composite' // weighted blend across multiple of the above\n\nexport interface VerifiableReward {\n  /** Scalar in [0, 1]. The RL training signal. */\n  value: number\n  /** What produced the reward — different sources have different determinism. */\n  source: VerifiableRewardSource\n  /**\n   * Determinism class. `'deterministic'` rewards are repeatable byte-for-byte\n   * given the same inputs (compile, test, schema validation, sandbox exit code).\n   * `'probabilistic'` rewards depend on a stochastic component (LLM judge).\n   * Mixing these in the same training batch without separation is a known\n   * footgun in production RLHF pipelines.\n   */\n  determinism: 'deterministic' | 'probabilistic'\n  /**\n   * Confidence in the reward value. For deterministic sources this is 1.0\n   * (the bit either flipped or didn't). For judge sources this is the\n   * judge-reported confidence or — when missing — a calibrated prior.\n   */\n  confidence: number\n  /** The layer / judge id that produced the signal, for provenance. */\n  origin: string\n  /**\n   * Per-source contribution to `value`, keyed by layer/judge id. Single-source\n   * rewards carry one entry (`{ [origin]: value }`); composite rewards carry\n   * every contributing layer's score — the anti-scalar-collapse surface RL\n   * consumers weight per-source instead of trusting one blended number.\n   */\n  components: Record<string, number>\n  /**\n   * @deprecated Read `components` for per-source reward values. Kept for\n   * published-API compatibility: single-source rewards carry the layer's\n   * diagnostics here (e.g. `{ tests_passed: 7 }`); composite rewards carry\n   * the same per-layer scores `components` now holds.\n   */\n  breakdown?: Record<string, number>\n}\n\nexport interface VerifiableRewardExtractionOptions {\n  /**\n   * Which layers count as deterministic-reward sources. The verifier doesn't\n   * tag layers as \"this is verifiable\"; the caller declares it via this list\n   * (or via the layer name → source mapping). Default treats common names\n   * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,\n   * `sandbox`) as deterministic.\n   */\n  deterministicLayers?: string[]\n  /**\n   * Map layer name → reward source. Defaults to a sensible string-match.\n   */\n  sourceFor?: (layerName: string) => VerifiableRewardSource\n  /**\n   * Whether to fall back to a probabilistic (judge) reward when no\n   * deterministic layer produced a numeric score. Default `true`. Set to\n   * `false` for \"deterministic-only\" training pipelines that should\n   * discard runs without a verifiable signal.\n   */\n  fallbackToJudge?: boolean\n  /**\n   * Default confidence for probabilistic (judge) rewards when the judge\n   * doesn't report one. Default `0.7`.\n   */\n  judgeConfidenceFloor?: number\n}\n\nconst DEFAULT_DETERMINISTIC_LAYERS = new Set([\n  'install',\n  'typecheck',\n  'build',\n  'lint',\n  'test',\n  'compile',\n  'schema',\n  'sandbox',\n  'unit_tests',\n  'integration_tests',\n])\n\nconst DEFAULT_SOURCE_FOR = (name: string): VerifiableRewardSource => {\n  const lower = name.toLowerCase()\n  if (lower.includes('test')) return 'test'\n  if (\n    lower.includes('compile') ||\n    lower.includes('build') ||\n    lower.includes('typecheck') ||\n    lower.includes('lint')\n  )\n    return 'compile'\n  if (lower.includes('schema')) return 'schema'\n  if (lower.includes('sandbox')) return 'sandbox'\n  if (lower.includes('judge') || lower.includes('semantic')) return 'judge'\n  return 'composite'\n}\n\n/**\n * Extract a `VerifiableReward` from a `VerificationReport`.\n *\n * Strategy: prefer the deterministic layers (in order: test → compile →\n * schema → sandbox), fall back to the judge layer if `fallbackToJudge` is\n * true, return `null` if no signal qualifies. When multiple deterministic\n * layers contribute, return a `'composite'` source with a weighted blend.\n */\nexport function extractVerifiableReward(\n  report: VerificationReport,\n  opts: VerifiableRewardExtractionOptions = {},\n): VerifiableReward | null {\n  const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS])\n  const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR\n  const fallbackToJudge = opts.fallbackToJudge ?? true\n  const judgeFloor = opts.judgeConfidenceFloor ?? 0.7\n\n  const deterministic = report.layers.filter(\n    (l) => deterministicSet.has(l.layer) && typeof l.score === 'number' && Number.isFinite(l.score),\n  )\n\n  if (deterministic.length === 1) {\n    const layer = deterministic[0]!\n    const value = clamp01(layer.score!)\n    return {\n      value,\n      source: sourceFor(layer.layer),\n      determinism: 'deterministic',\n      confidence: 1,\n      origin: layer.layer,\n      components: { [layer.layer]: value },\n      breakdown: layerBreakdown(layer),\n    }\n  }\n\n  if (deterministic.length > 1) {\n    // Composite: weighted blend by `Layer.weight` if present, else equal.\n    let num = 0\n    let denom = 0\n    const components: Record<string, number> = {}\n    for (const l of deterministic) {\n      const w = (l.detail?.weight as number | undefined) ?? 1\n      num += w * (l.score ?? 0)\n      denom += w\n      components[l.layer] = l.score!\n    }\n    return {\n      value: denom === 0 ? 0 : clamp01(num / denom),\n      source: 'composite',\n      determinism: 'deterministic',\n      confidence: 1,\n      origin: deterministic.map((l) => l.layer).join('+'),\n      components,\n      breakdown: { ...components },\n    }\n  }\n\n  if (!fallbackToJudge) return null\n\n  const judge =\n    report.layers.find(\n      (l) =>\n        typeof l.score === 'number' && Number.isFinite(l.score) && sourceFor(l.layer) === 'judge',\n    ) ?? report.layers.find((l) => typeof l.score === 'number' && Number.isFinite(l.score))\n\n  if (!judge) return null\n\n  const confFromDetail = judge.detail?.confidence as number | undefined\n  const judgeValue = clamp01(judge.score!)\n  return {\n    value: judgeValue,\n    source: 'judge',\n    determinism: 'probabilistic',\n    confidence: typeof confFromDetail === 'number' ? confFromDetail : judgeFloor,\n    origin: judge.layer,\n    components: { [judge.layer]: judgeValue },\n    breakdown: layerBreakdown(judge),\n  }\n}\n\n/**\n * Extract verifiable rewards from `RunRecord[]` produced via the\n * `verificationReportToRunRecord` adapter (which encodes per-layer scores\n * in `outcome.raw['layer.<name>']`). For records that don't carry layer\n * scores, returns `null` for that record.\n *\n * This is the canonical bridge from \"campaign-shaped artifacts\" to\n * \"RL-training-ready reward signals\": every record that has a clean\n * verifiable reward becomes a training datum, every record that doesn't\n * gets filtered out (or kept with `'probabilistic'` determinism for\n * separate downstream handling).\n */\nexport function extractVerifiableRewardsFromRecords(\n  runs: RunRecord[],\n  opts: VerifiableRewardExtractionOptions = {},\n): Array<{ runId: string; reward: VerifiableReward | null }> {\n  const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR\n  const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS])\n  const fallbackToJudge = opts.fallbackToJudge ?? true\n  const judgeFloor = opts.judgeConfidenceFloor ?? 0.7\n\n  return runs.map((run) => {\n    // Recover per-layer scores from outcome.raw['layer.<name>']\n    const layerScores: Array<{ name: string; score: number }> = []\n    for (const [k, v] of Object.entries(run.outcome.raw)) {\n      if (\n        k.startsWith('layer.') &&\n        !k.includes('.', 6) &&\n        typeof v === 'number' &&\n        Number.isFinite(v)\n      ) {\n        layerScores.push({ name: k.slice('layer.'.length), score: v })\n      }\n    }\n    const det = layerScores.filter((l) => deterministicSet.has(l.name))\n\n    if (det.length === 1) {\n      const layer = det[0]!\n      const value = clamp01(layer.score)\n      return {\n        runId: run.runId,\n        reward: {\n          value,\n          source: sourceFor(layer.name),\n          determinism: 'deterministic',\n          confidence: 1,\n          origin: layer.name,\n          components: { [layer.name]: value },\n        },\n      }\n    }\n    if (det.length > 1) {\n      const value = det.reduce((s, l) => s + l.score, 0) / det.length\n      const components: Record<string, number> = Object.fromEntries(\n        det.map((l) => [l.name, l.score]),\n      )\n      return {\n        runId: run.runId,\n        reward: {\n          value: clamp01(value),\n          source: 'composite',\n          determinism: 'deterministic',\n          confidence: 1,\n          origin: det.map((l) => l.name).join('+'),\n          components,\n          breakdown: { ...components },\n        },\n      }\n    }\n    if (!fallbackToJudge) return { runId: run.runId, reward: null }\n\n    // Probabilistic fallback: use the run's primary score.\n    const primary = run.outcome.holdoutScore ?? run.outcome.searchScore\n    if (typeof primary !== 'number' || !Number.isFinite(primary)) {\n      return { runId: run.runId, reward: null }\n    }\n    const primaryValue = clamp01(primary)\n    return {\n      runId: run.runId,\n      reward: {\n        value: primaryValue,\n        source: 'judge',\n        determinism: 'probabilistic',\n        confidence: judgeFloor,\n        origin: 'run.outcome.score',\n        components: { 'run.outcome.score': primaryValue },\n      },\n    }\n  })\n}\n\n/** Filter `RunRecord[]` to those with deterministic verifiable rewards. */\nexport function filterDeterministicallyRewarded(\n  runs: RunRecord[],\n  opts: VerifiableRewardExtractionOptions = {},\n): Array<{ run: RunRecord; reward: VerifiableReward }> {\n  const rewarded = extractVerifiableRewardsFromRecords(runs, { ...opts, fallbackToJudge: false })\n  const out: Array<{ run: RunRecord; reward: VerifiableReward }> = []\n  for (let i = 0; i < runs.length; i++) {\n    const r = rewarded[i]!\n    if (r.reward && r.reward.determinism === 'deterministic') {\n      out.push({ run: runs[i]!, reward: r.reward })\n    }\n  }\n  return out\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction clamp01(x: number): number {\n  if (!Number.isFinite(x)) return 0\n  return Math.max(0, Math.min(1, x))\n}\n\nfunction layerBreakdown(l: LayerResult): Record<string, number> {\n  const out: Record<string, number> = {}\n  if (l.diagnostics) {\n    for (const [k, v] of Object.entries(l.diagnostics)) {\n      if (typeof v === 'number' && Number.isFinite(v)) out[k] = v\n    }\n  }\n  return out\n}\n","/**\n * Reward hacking / Goodhart detection.\n *\n * Goodhart's Law says: when a measure becomes a target, it ceases to be\n * a good measure. In RLHF and agentic-RL settings this is the dominant\n * failure mode — the policy learns to produce outputs that score well on\n * the proxy reward (judge, rubric, test pass-rate) without producing\n * the underlying capability the proxy was meant to track.\n *\n * Krakovna et al. (2020, \"Specification Gaming Examples in AI\") and the\n * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.\n * 2023) converge on a few diagnostic signatures:\n *\n *   1. **Reward divergence:** the proxy reward grows while the held-out\n *      ground-truth signal stagnates or drops. Predictive validity over\n *      time captures this.\n *   2. **Distributional shift in outputs:** after RL, the policy produces\n *      outputs that no longer match the reference distribution — usually\n *      because it found a high-reward attractor that's degenerate (e.g.\n *      one-token responses, repetition, formatting tricks).\n *   3. **Disagreement between independent rewards:** if you train on\n *      reward A and a held-out independent reward B drops sharply, you're\n *      probably hacking A.\n *   4. **Calibration drift:** the verifiable / deterministic component of\n *      the reward is stable; the probabilistic / judge component drifts up\n *      while the deterministic component doesn't. The judge is being\n *      gamed.\n *\n * This module ships explicit detectors for all four signatures, plus a\n * combined verdict. The output is diagnostic — actionable signals,\n * not autoreject — because each signature has known false positives\n * (e.g., a policy that genuinely improves can show distributional shift).\n *\n * Differs from `rubricPredictiveValidity` (which is a *standing* check on\n * whether rubrics correlate with deployment outcomes) — this is a\n * *temporal* check on whether the reward-vs-truth gap is *widening over\n * time during a training run*.\n */\n\nimport type { RunRecord } from '../run-record'\nimport {\n  filterDeterministicallyRewarded,\n  type VerifiableRewardExtractionOptions,\n} from './verifiable-reward'\n\nexport type RewardHackingSignal =\n  | 'reward_divergence'\n  | 'distribution_shift'\n  | 'reward_disagreement'\n  | 'judge_drift'\n\nexport interface RewardHackingFinding {\n  signal: RewardHackingSignal\n  /** Severity in [0, 1]. >0.5 = strong signal. */\n  severity: number\n  message: string\n  /** Numeric evidence the consumer can render. */\n  detail: Record<string, number>\n}\n\nexport interface RewardHackingReport {\n  findings: RewardHackingFinding[]\n  /**\n   * Composite verdict. `'clean'` if every signal severity < 0.3;\n   * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6; `'gaming'` if any ≥ 0.6.\n   */\n  verdict: 'clean' | 'suspect' | 'gaming'\n  /** Rationale for the verdict, ready to paste into an audit log. */\n  rationale: string[]\n  /** Number of paired (proxy, truth) data points the report saw. */\n  n: number\n}\n\nexport interface DetectRewardHackingInput {\n  /**\n   * Run records ordered by recency (oldest first). The detector segments\n   * them into prefix/suffix windows to compute \"did the gap widen.\"\n   */\n  runs: RunRecord[]\n  /**\n   * The metric the policy was trained to optimize. Should be present on\n   * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.\n   */\n  proxyOf?: (run: RunRecord) => number | null\n  /**\n   * The held-out ground-truth metric. For RL on coding, this is typically\n   * test pass-rate. For RLHF, it's downstream task performance or human\n   * preference. For knowledge tasks, it's an independently-graded score.\n   */\n  truthOf?: (run: RunRecord) => number | null\n  /**\n   * Independent secondary reward. Used for the `reward_disagreement`\n   * signal. Default uses the verifiable reward extractor (deterministic\n   * sources only).\n   */\n  secondaryRewardOf?: (run: RunRecord) => number | null\n  /**\n   * Window size — how many of the most recent runs count as the \"after\"\n   * cohort. Default min(50, half the runs).\n   */\n  windowSize?: number\n  /**\n   * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6\n   * (gaming).\n   */\n  thresholds?: { suspect?: number; gaming?: number }\n  /**\n   * Verifiable-reward options used for the secondary-reward fallback.\n   */\n  verifiableRewardOptions?: VerifiableRewardExtractionOptions\n}\n\nconst DEFAULT_PROXY = (r: RunRecord): number | null => {\n  const v = r.outcome.holdoutScore ?? r.outcome.searchScore\n  return typeof v === 'number' && Number.isFinite(v) ? v : null\n}\n\nexport function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport {\n  const proxyOf = input.proxyOf ?? DEFAULT_PROXY\n  const truthOf = input.truthOf\n  const sus = input.thresholds?.suspect ?? 0.3\n  const gam = input.thresholds?.gaming ?? 0.6\n\n  const runs = input.runs.filter((r) => proxyOf(r) !== null)\n  const n = runs.length\n  if (n < 4) {\n    return {\n      findings: [],\n      verdict: 'clean',\n      n,\n      rationale: [`fewer than 4 runs with proxy reward (n=${n}); insufficient evidence`],\n    }\n  }\n  const windowSize = Math.max(1, input.windowSize ?? Math.min(50, Math.floor(n / 2)))\n  const before = runs.slice(0, n - windowSize)\n  const after = runs.slice(n - windowSize)\n\n  const findings: RewardHackingFinding[] = []\n\n  // ── Signal 1: reward divergence (proxy ↑ while truth flat or ↓) ──────\n  if (truthOf) {\n    const beforeProxy = before.map(proxyOf).filter((v): v is number => typeof v === 'number')\n    const afterProxy = after.map(proxyOf).filter((v): v is number => typeof v === 'number')\n    const beforeTruth = before.map(truthOf).filter((v): v is number => typeof v === 'number')\n    const afterTruth = after.map(truthOf).filter((v): v is number => typeof v === 'number')\n    if (\n      beforeProxy.length >= 2 &&\n      afterProxy.length >= 2 &&\n      beforeTruth.length >= 2 &&\n      afterTruth.length >= 2\n    ) {\n      const proxyDelta = mean(afterProxy) - mean(beforeProxy)\n      const truthDelta = mean(afterTruth) - mean(beforeTruth)\n      // Divergence: proxy goes up while truth goes flat or down.\n      // Severity = max(0, (proxyDelta - truthDelta)) — bigger gap = bigger signal.\n      const gap = Math.max(0, proxyDelta - truthDelta)\n      const severity = clamp01(gap * 5) // scale: 0.2 absolute gap → severity 1.0\n      findings.push({\n        signal: 'reward_divergence',\n        severity,\n        message:\n          severity >= sus\n            ? `proxy reward rose by ${proxyDelta.toFixed(3)} while truth changed by ${truthDelta.toFixed(3)} — potential Goodhart`\n            : `proxy and truth moved together (proxy ${proxyDelta.toFixed(3)}, truth ${truthDelta.toFixed(3)})`,\n        detail: {\n          proxyDelta,\n          truthDelta,\n          gap,\n          beforeN: beforeProxy.length,\n          afterN: afterProxy.length,\n        },\n      })\n    }\n  }\n\n  // ── Signal 2: distributional shift in outputs (KS on score distributions) ──\n  {\n    const beforeP = before.map(proxyOf).filter((v): v is number => typeof v === 'number')\n    const afterP = after.map(proxyOf).filter((v): v is number => typeof v === 'number')\n    if (beforeP.length >= 4 && afterP.length >= 4) {\n      const ks = ksStatistic(beforeP, afterP)\n      // KS statistic: bigger = more shift. We're agnostic about direction;\n      // genuine improvement ALSO produces shift, so this signal is\n      // contributory rather than load-bearing.\n      const severity = clamp01(ks - 0.2)\n      findings.push({\n        signal: 'distribution_shift',\n        severity,\n        message:\n          severity >= sus\n            ? `KS=${ks.toFixed(3)} between before/after windows — distributional shift large`\n            : `KS=${ks.toFixed(3)} between before/after windows — within-distribution drift`,\n        detail: { ks, beforeN: beforeP.length, afterN: afterP.length },\n      })\n    }\n  }\n\n  // ── Signal 3: reward disagreement (proxy vs independent secondary) ────\n  {\n    const secondaryOf = input.secondaryRewardOf ?? defaultSecondary(input.verifiableRewardOptions)\n    const aligned = runs\n      .map((r) => ({ p: proxyOf(r), s: secondaryOf(r) }))\n      .filter(\n        (x): x is { p: number; s: number } => typeof x.p === 'number' && typeof x.s === 'number',\n      )\n    if (aligned.length >= 4) {\n      const ps = aligned.map((x) => x.p)\n      const ss = aligned.map((x) => x.s)\n      const r = pearsonR(ps, ss)\n      // Disagreement: low or negative correlation between primary proxy\n      // reward and an independent secondary signal.\n      const severity = clamp01(0.5 - Math.max(0, r))\n      findings.push({\n        signal: 'reward_disagreement',\n        severity,\n        message:\n          severity >= sus\n            ? `proxy and independent secondary reward correlate ρ=${r.toFixed(3)} — possibly hacking proxy`\n            : `proxy and secondary reward correlate ρ=${r.toFixed(3)}`,\n        detail: { pearson: r, n: aligned.length },\n      })\n    }\n  }\n\n  // ── Signal 4: judge drift (probabilistic up while deterministic flat) ─\n  {\n    const detRuns = filterDeterministicallyRewarded(runs, input.verifiableRewardOptions ?? {})\n    if (detRuns.length >= 4) {\n      const detBefore = detRuns.slice(0, Math.floor(detRuns.length / 2))\n      const detAfter = detRuns.slice(Math.floor(detRuns.length / 2))\n      const detDelta =\n        mean(detAfter.map((r) => r.reward.value)) - mean(detBefore.map((r) => r.reward.value))\n      const proxyDelta =\n        mean(after.map(proxyOf).filter((v): v is number => typeof v === 'number')) -\n        mean(before.map(proxyOf).filter((v): v is number => typeof v === 'number'))\n      const driftGap = Math.max(0, proxyDelta - detDelta)\n      const severity = clamp01(driftGap * 5)\n      findings.push({\n        signal: 'judge_drift',\n        severity,\n        message:\n          severity >= sus\n            ? `judge proxy +${proxyDelta.toFixed(3)} while deterministic reward +${detDelta.toFixed(3)} — judge drifting up without verifiable backing`\n            : `judge and deterministic rewards move in step (judge ${proxyDelta.toFixed(3)}, det ${detDelta.toFixed(3)})`,\n        detail: { proxyDelta, detDelta, driftGap, n: detRuns.length },\n      })\n    }\n  }\n\n  const maxSev = findings.reduce((m, f) => Math.max(m, f.severity), 0)\n  const verdict: RewardHackingReport['verdict'] =\n    maxSev >= gam ? 'gaming' : maxSev >= sus ? 'suspect' : 'clean'\n  const rationale = findings\n    .filter((f) => f.severity >= sus)\n    .map((f) => `${f.signal}: severity ${f.severity.toFixed(2)} — ${f.message}`)\n  if (rationale.length === 0) rationale.push('no signals fired above suspect threshold')\n\n  return { findings, verdict, rationale, n }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction mean(xs: number[]): number {\n  if (xs.length === 0) return 0\n  return xs.reduce((s, x) => s + x, 0) / xs.length\n}\n\nfunction clamp01(x: number): number {\n  if (!Number.isFinite(x)) return 0\n  return Math.max(0, Math.min(1, x))\n}\n\nfunction pearsonR(a: number[], b: number[]): number {\n  if (a.length !== b.length || a.length < 2) return 0\n  const ma = mean(a)\n  const mb = mean(b)\n  let num = 0,\n    da = 0,\n    db = 0\n  for (let i = 0; i < a.length; i++) {\n    const xa = a[i]! - ma\n    const xb = b[i]! - mb\n    num += xa * xb\n    da += xa * xa\n    db += xb * xb\n  }\n  if (da === 0 || db === 0) return 0\n  return num / Math.sqrt(da * db)\n}\n\nfunction ksStatistic(a: number[], b: number[]): number {\n  // Two-sample Kolmogorov-Smirnov statistic.\n  const sortedA = [...a].sort((x, y) => x - y)\n  const sortedB = [...b].sort((x, y) => x - y)\n  const all = [...new Set([...sortedA, ...sortedB])].sort((x, y) => x - y)\n  let max = 0\n  for (const v of all) {\n    const fa = sortedA.filter((x) => x <= v).length / sortedA.length\n    const fb = sortedB.filter((x) => x <= v).length / sortedB.length\n    max = Math.max(max, Math.abs(fa - fb))\n  }\n  return max\n}\n\nfunction defaultSecondary(\n  verifiableOpts?: VerifiableRewardExtractionOptions,\n): (run: RunRecord) => number | null {\n  return (run: RunRecord) => {\n    const filtered = filterDeterministicallyRewarded([run], verifiableOpts ?? {})\n    return filtered.length === 1 ? filtered[0]!.reward.value : null\n  }\n}\n"],"mappings":";AAwGA,IAAM,+BAA+B,oBAAI,IAAI;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,qBAAqB,CAAC,SAAyC;AACnE,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,SAAS,MAAM,EAAG,QAAO;AACnC,MACE,MAAM,SAAS,SAAS,KACxB,MAAM,SAAS,OAAO,KACtB,MAAM,SAAS,WAAW,KAC1B,MAAM,SAAS,MAAM;AAErB,WAAO;AACT,MAAI,MAAM,SAAS,QAAQ,EAAG,QAAO;AACrC,MAAI,MAAM,SAAS,SAAS,EAAG,QAAO;AACtC,MAAI,MAAM,SAAS,OAAO,KAAK,MAAM,SAAS,UAAU,EAAG,QAAO;AAClE,SAAO;AACT;AAUO,SAAS,wBACd,QACA,OAA0C,CAAC,GAClB;AACzB,QAAM,mBAAmB,IAAI,IAAI,KAAK,uBAAuB,CAAC,GAAG,4BAA4B,CAAC;AAC9F,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,QAAM,aAAa,KAAK,wBAAwB;AAEhD,QAAM,gBAAgB,OAAO,OAAO;AAAA,IAClC,CAAC,MAAM,iBAAiB,IAAI,EAAE,KAAK,KAAK,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK;AAAA,EAChG;AAEA,MAAI,cAAc,WAAW,GAAG;AAC9B,UAAM,QAAQ,cAAc,CAAC;AAC7B,UAAM,QAAQ,QAAQ,MAAM,KAAM;AAClC,WAAO;AAAA,MACL;AAAA,MACA,QAAQ,UAAU,MAAM,KAAK;AAAA,MAC7B,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ,MAAM;AAAA,MACd,YAAY,EAAE,CAAC,MAAM,KAAK,GAAG,MAAM;AAAA,MACnC,WAAW,eAAe,KAAK;AAAA,IACjC;AAAA,EACF;AAEA,MAAI,cAAc,SAAS,GAAG;AAE5B,QAAI,MAAM;AACV,QAAI,QAAQ;AACZ,UAAM,aAAqC,CAAC;AAC5C,eAAW,KAAK,eAAe;AAC7B,YAAM,IAAK,EAAE,QAAQ,UAAiC;AACtD,aAAO,KAAK,EAAE,SAAS;AACvB,eAAS;AACT,iBAAW,EAAE,KAAK,IAAI,EAAE;AAAA,IAC1B;AACA,WAAO;AAAA,MACL,OAAO,UAAU,IAAI,IAAI,QAAQ,MAAM,KAAK;AAAA,MAC5C,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ,cAAc,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG;AAAA,MAClD;AAAA,MACA,WAAW,EAAE,GAAG,WAAW;AAAA,IAC7B;AAAA,EACF;AAEA,MAAI,CAAC,gBAAiB,QAAO;AAE7B,QAAM,QACJ,OAAO,OAAO;AAAA,IACZ,CAAC,MACC,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK,KAAK,UAAU,EAAE,KAAK,MAAM;AAAA,EACtF,KAAK,OAAO,OAAO,KAAK,CAAC,MAAM,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK,CAAC;AAExF,MAAI,CAAC,MAAO,QAAO;AAEnB,QAAM,iBAAiB,MAAM,QAAQ;AACrC,QAAM,aAAa,QAAQ,MAAM,KAAM;AACvC,SAAO;AAAA,IACL,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,YAAY,OAAO,mBAAmB,WAAW,iBAAiB;AAAA,IAClE,QAAQ,MAAM;AAAA,IACd,YAAY,EAAE,CAAC,MAAM,KAAK,GAAG,WAAW;AAAA,IACxC,WAAW,eAAe,KAAK;AAAA,EACjC;AACF;AAcO,SAAS,oCACd,MACA,OAA0C,CAAC,GACgB;AAC3D,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,mBAAmB,IAAI,IAAI,KAAK,uBAAuB,CAAC,GAAG,4BAA4B,CAAC;AAC9F,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,QAAM,aAAa,KAAK,wBAAwB;AAEhD,SAAO,KAAK,IAAI,CAAC,QAAQ;AAEvB,UAAM,cAAsD,CAAC;AAC7D,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,IAAI,QAAQ,GAAG,GAAG;AACpD,UACE,EAAE,WAAW,QAAQ,KACrB,CAAC,EAAE,SAAS,KAAK,CAAC,KAClB,OAAO,MAAM,YACb,OAAO,SAAS,CAAC,GACjB;AACA,oBAAY,KAAK,EAAE,MAAM,EAAE,MAAM,SAAS,MAAM,GAAG,OAAO,EAAE,CAAC;AAAA,MAC/D;AAAA,IACF;AACA,UAAM,MAAM,YAAY,OAAO,CAAC,MAAM,iBAAiB,IAAI,EAAE,IAAI,CAAC;AAElE,QAAI,IAAI,WAAW,GAAG;AACpB,YAAM,QAAQ,IAAI,CAAC;AACnB,YAAM,QAAQ,QAAQ,MAAM,KAAK;AACjC,aAAO;AAAA,QACL,OAAO,IAAI;AAAA,QACX,QAAQ;AAAA,UACN;AAAA,UACA,QAAQ,UAAU,MAAM,IAAI;AAAA,UAC5B,aAAa;AAAA,UACb,YAAY;AAAA,UACZ,QAAQ,MAAM;AAAA,UACd,YAAY,EAAE,CAAC,MAAM,IAAI,GAAG,MAAM;AAAA,QACpC;AAAA,MACF;AAAA,IACF;AACA,QAAI,IAAI,SAAS,GAAG;AAClB,YAAM,QAAQ,IAAI,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI;AACzD,YAAM,aAAqC,OAAO;AAAA,QAChD,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC;AAAA,MAClC;AACA,aAAO;AAAA,QACL,OAAO,IAAI;AAAA,QACX,QAAQ;AAAA,UACN,OAAO,QAAQ,KAAK;AAAA,UACpB,QAAQ;AAAA,UACR,aAAa;AAAA,UACb,YAAY;AAAA,UACZ,QAAQ,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG;AAAA,UACvC;AAAA,UACA,WAAW,EAAE,GAAG,WAAW;AAAA,QAC7B;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,gBAAiB,QAAO,EAAE,OAAO,IAAI,OAAO,QAAQ,KAAK;AAG9D,UAAM,UAAU,IAAI,QAAQ,gBAAgB,IAAI,QAAQ;AACxD,QAAI,OAAO,YAAY,YAAY,CAAC,OAAO,SAAS,OAAO,GAAG;AAC5D,aAAO,EAAE,OAAO,IAAI,OAAO,QAAQ,KAAK;AAAA,IAC1C;AACA,UAAM,eAAe,QAAQ,OAAO;AACpC,WAAO;AAAA,MACL,OAAO,IAAI;AAAA,MACX,QAAQ;AAAA,QACN,OAAO;AAAA,QACP,QAAQ;AAAA,QACR,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,YAAY,EAAE,qBAAqB,aAAa;AAAA,MAClD;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAGO,SAAS,gCACd,MACA,OAA0C,CAAC,GACU;AACrD,QAAM,WAAW,oCAAoC,MAAM,EAAE,GAAG,MAAM,iBAAiB,MAAM,CAAC;AAC9F,QAAM,MAA2D,CAAC;AAClE,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,IAAI,SAAS,CAAC;AACpB,QAAI,EAAE,UAAU,EAAE,OAAO,gBAAgB,iBAAiB;AACxD,UAAI,KAAK,EAAE,KAAK,KAAK,CAAC,GAAI,QAAQ,EAAE,OAAO,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,SAAO;AACT;AAIA,SAAS,QAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,eAAe,GAAwC;AAC9D,QAAM,MAA8B,CAAC;AACrC,MAAI,EAAE,aAAa;AACjB,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,EAAE,WAAW,GAAG;AAClD,UAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,EAAG,KAAI,CAAC,IAAI;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;;;AC/NA,IAAM,gBAAgB,CAAC,MAAgC;AACrD,QAAM,IAAI,EAAE,QAAQ,gBAAgB,EAAE,QAAQ;AAC9C,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,IAAI,IAAI;AAC3D;AAEO,SAAS,oBAAoB,OAAsD;AACxF,QAAM,UAAU,MAAM,WAAW;AACjC,QAAM,UAAU,MAAM;AACtB,QAAM,MAAM,MAAM,YAAY,WAAW;AACzC,QAAM,MAAM,MAAM,YAAY,UAAU;AAExC,QAAM,OAAO,MAAM,KAAK,OAAO,CAAC,MAAM,QAAQ,CAAC,MAAM,IAAI;AACzD,QAAM,IAAI,KAAK;AACf,MAAI,IAAI,GAAG;AACT,WAAO;AAAA,MACL,UAAU,CAAC;AAAA,MACX,SAAS;AAAA,MACT;AAAA,MACA,WAAW,CAAC,0CAA0C,CAAC,0BAA0B;AAAA,IACnF;AAAA,EACF;AACA,QAAM,aAAa,KAAK,IAAI,GAAG,MAAM,cAAc,KAAK,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC;AAClF,QAAM,SAAS,KAAK,MAAM,GAAG,IAAI,UAAU;AAC3C,QAAM,QAAQ,KAAK,MAAM,IAAI,UAAU;AAEvC,QAAM,WAAmC,CAAC;AAG1C,MAAI,SAAS;AACX,UAAM,cAAc,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACxF,UAAM,aAAa,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACtF,UAAM,cAAc,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACxF,UAAM,aAAa,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACtF,QACE,YAAY,UAAU,KACtB,WAAW,UAAU,KACrB,YAAY,UAAU,KACtB,WAAW,UAAU,GACrB;AACA,YAAM,aAAa,KAAK,UAAU,IAAI,KAAK,WAAW;AACtD,YAAM,aAAa,KAAK,UAAU,IAAI,KAAK,WAAW;AAGtD,YAAM,MAAM,KAAK,IAAI,GAAG,aAAa,UAAU;AAC/C,YAAM,WAAWA,SAAQ,MAAM,CAAC;AAChC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,wBAAwB,WAAW,QAAQ,CAAC,CAAC,2BAA2B,WAAW,QAAQ,CAAC,CAAC,+BAC7F,yCAAyC,WAAW,QAAQ,CAAC,CAAC,WAAW,WAAW,QAAQ,CAAC,CAAC;AAAA,QACpG,QAAQ;AAAA,UACN;AAAA,UACA;AAAA,UACA;AAAA,UACA,SAAS,YAAY;AAAA,UACrB,QAAQ,WAAW;AAAA,QACrB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,UAAU,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACpF,UAAM,SAAS,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AAClF,QAAI,QAAQ,UAAU,KAAK,OAAO,UAAU,GAAG;AAC7C,YAAM,KAAK,YAAY,SAAS,MAAM;AAItC,YAAM,WAAWA,SAAQ,KAAK,GAAG;AACjC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,MAAM,GAAG,QAAQ,CAAC,CAAC,oEACnB,MAAM,GAAG,QAAQ,CAAC,CAAC;AAAA,QACzB,QAAQ,EAAE,IAAI,SAAS,QAAQ,QAAQ,QAAQ,OAAO,OAAO;AAAA,MAC/D,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,cAAc,MAAM,qBAAqB,iBAAiB,MAAM,uBAAuB;AAC7F,UAAM,UAAU,KACb,IAAI,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,GAAG,GAAG,YAAY,CAAC,EAAE,EAAE,EACjD;AAAA,MACC,CAAC,MAAqC,OAAO,EAAE,MAAM,YAAY,OAAO,EAAE,MAAM;AAAA,IAClF;AACF,QAAI,QAAQ,UAAU,GAAG;AACvB,YAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;AACjC,YAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;AACjC,YAAM,IAAI,SAAS,IAAI,EAAE;AAGzB,YAAM,WAAWA,SAAQ,MAAM,KAAK,IAAI,GAAG,CAAC,CAAC;AAC7C,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,2DAAsD,EAAE,QAAQ,CAAC,CAAC,mCAClE,+CAA0C,EAAE,QAAQ,CAAC,CAAC;AAAA,QAC5D,QAAQ,EAAE,SAAS,GAAG,GAAG,QAAQ,OAAO;AAAA,MAC1C,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,UAAU,gCAAgC,MAAM,MAAM,2BAA2B,CAAC,CAAC;AACzF,QAAI,QAAQ,UAAU,GAAG;AACvB,YAAM,YAAY,QAAQ,MAAM,GAAG,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;AACjE,YAAM,WAAW,QAAQ,MAAM,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;AAC7D,YAAM,WACJ,KAAK,SAAS,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,CAAC;AACvF,YAAM,aACJ,KAAK,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,CAAC,IACzE,KAAK,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,CAAC;AAC5E,YAAM,WAAW,KAAK,IAAI,GAAG,aAAa,QAAQ;AAClD,YAAM,WAAWA,SAAQ,WAAW,CAAC;AACrC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,gBAAgB,WAAW,QAAQ,CAAC,CAAC,gCAAgC,SAAS,QAAQ,CAAC,CAAC,yDACxF,uDAAuD,WAAW,QAAQ,CAAC,CAAC,SAAS,SAAS,QAAQ,CAAC,CAAC;AAAA,QAC9G,QAAQ,EAAE,YAAY,UAAU,UAAU,GAAG,QAAQ,OAAO;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,SAAS,SAAS,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,GAAG,EAAE,QAAQ,GAAG,CAAC;AACnE,QAAM,UACJ,UAAU,MAAM,WAAW,UAAU,MAAM,YAAY;AACzD,QAAM,YAAY,SACf,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,EAC/B,IAAI,CAAC,MAAM,GAAG,EAAE,MAAM,cAAc,EAAE,SAAS,QAAQ,CAAC,CAAC,WAAM,EAAE,OAAO,EAAE;AAC7E,MAAI,UAAU,WAAW,EAAG,WAAU,KAAK,0CAA0C;AAErF,SAAO,EAAE,UAAU,SAAS,WAAW,EAAE;AAC3C;AAIA,SAAS,KAAK,IAAsB;AAClC,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAEA,SAASA,SAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,SAAS,GAAa,GAAqB;AAClD,MAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAG,QAAO;AAClD,QAAM,KAAK,KAAK,CAAC;AACjB,QAAM,KAAK,KAAK,CAAC;AACjB,MAAI,MAAM,GACR,KAAK,GACL,KAAK;AACP,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,WAAO,KAAK;AACZ,UAAM,KAAK;AACX,UAAM,KAAK;AAAA,EACb;AACA,MAAI,OAAO,KAAK,OAAO,EAAG,QAAO;AACjC,SAAO,MAAM,KAAK,KAAK,KAAK,EAAE;AAChC;AAEA,SAAS,YAAY,GAAa,GAAqB;AAErD,QAAM,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,QAAM,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,QAAM,MAAM,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACvE,MAAI,MAAM;AACV,aAAW,KAAK,KAAK;AACnB,UAAM,KAAK,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,SAAS,QAAQ;AAC1D,UAAM,KAAK,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,SAAS,QAAQ;AAC1D,UAAM,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EACvC;AACA,SAAO;AACT;AAEA,SAAS,iBACP,gBACmC;AACnC,SAAO,CAAC,QAAmB;AACzB,UAAM,WAAW,gCAAgC,CAAC,GAAG,GAAG,kBAAkB,CAAC,CAAC;AAC5E,WAAO,SAAS,WAAW,IAAI,SAAS,CAAC,EAAG,OAAO,QAAQ;AAAA,EAC7D;AACF;","names":["clamp01"]}
+{"version":3,"sources":["../src/rl/verifiable-reward.ts","../src/rl/reward-hacking.ts"],"sourcesContent":["/**\n * Verifiable reward channel.\n *\n * For RL on coding / math / theorem-proving / structured-output tasks, the\n * reward signal is *decidable* — a test passes or fails, a proof checks or\n * doesn't, an output validates against a schema or doesn't. These rewards\n * are dramatically more useful for RL training than LLM-judge scores\n * because they don't drift, can't be Goodhart-gamed by the policy in the\n * same way, and don't require a separate calibration loop.\n *\n * The `MultiLayerVerifier` already produces this signal — it just doesn't\n * surface it in a shape that's clean enough for RL training. This module\n * wraps the verifier output so consumers can:\n *\n *   1. Extract a clean `VerifiableReward` from a `VerificationReport`\n *   2. Distinguish *deterministic* rewards (compile, test, schema) from\n *      *probabilistic* rewards (judge) so they can be weighted differently\n *      in the RL training step\n *   3. Filter `RunRecord[]` to only those with a verifiable reward,\n *      producing the clean training set that DeepSeek-R1-style GRPO and\n *      AlphaProof-style search both depend on\n *\n * Why this matters: every credible 2025-2026 frontier RL result on coding\n * agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,\n * o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing\n * judge scores into the reward signal poisons the gradient. This module\n * is the seam.\n */\n\nimport type { LayerResult, VerificationReport } from '../multi-layer-verifier'\nimport type { RunRecord } from '../run-record'\n\nexport type VerifiableRewardSource =\n  | 'compile' // typecheck / build / lint passed\n  | 'test' // unit / integration test pass-rate\n  | 'schema' // structured output validates\n  | 'sandbox' // sandbox exec exit code\n  | 'judge' // LLM judge — probabilistic, included for completeness\n  | 'composite' // weighted blend across multiple of the above\n\nexport interface VerifiableReward {\n  /** Scalar in [0, 1]. The RL training signal. */\n  value: number\n  /** What produced the reward — different sources have different determinism. */\n  source: VerifiableRewardSource\n  /**\n   * Determinism class. `'deterministic'` rewards are repeatable byte-for-byte\n   * given the same inputs (compile, test, schema validation, sandbox exit code).\n   * `'probabilistic'` rewards depend on a stochastic component (LLM judge).\n   * Mixing these in the same training batch without separation is a known\n   * footgun in production RLHF pipelines.\n   */\n  determinism: 'deterministic' | 'probabilistic'\n  /**\n   * Confidence in the reward value. For deterministic sources this is 1.0\n   * (the bit either flipped or didn't). For judge sources this is the\n   * judge-reported confidence or — when missing — a calibrated prior.\n   */\n  confidence: number\n  /** The layer / judge id that produced the signal, for provenance. */\n  origin: string\n  /**\n   * Per-source contribution to `value`, keyed by layer/judge id. Single-source\n   * rewards carry one entry (`{ [origin]: value }`); composite rewards carry\n   * every contributing layer's score — the anti-scalar-collapse surface RL\n   * consumers weight per-source instead of trusting one blended number.\n   */\n  components: Record<string, number>\n  /**\n   * @deprecated Read `components` for per-source reward values. Kept for\n   * published-API compatibility: single-source rewards carry the layer's\n   * diagnostics here (e.g. `{ tests_passed: 7 }`); composite rewards carry\n   * the same per-layer scores `components` now holds.\n   */\n  breakdown?: Record<string, number>\n}\n\nexport interface VerifiableRewardExtractionOptions {\n  /**\n   * Which layers count as deterministic-reward sources. The verifier doesn't\n   * tag layers as \"this is verifiable\"; the caller declares it via this list\n   * (or via the layer name → source mapping). Default treats common names\n   * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,\n   * `sandbox`) as deterministic.\n   */\n  deterministicLayers?: string[]\n  /**\n   * Map layer name → reward source. Defaults to a sensible string-match.\n   */\n  sourceFor?: (layerName: string) => VerifiableRewardSource\n  /**\n   * Whether to fall back to a probabilistic (judge) reward when no\n   * deterministic layer produced a numeric score. Default `true`. Set to\n   * `false` for \"deterministic-only\" training pipelines that should\n   * discard runs without a verifiable signal.\n   */\n  fallbackToJudge?: boolean\n  /**\n   * Default confidence for probabilistic (judge) rewards when the judge\n   * doesn't report one. Default `0.7`.\n   */\n  judgeConfidenceFloor?: number\n}\n\nconst DEFAULT_DETERMINISTIC_LAYERS = new Set([\n  'install',\n  'typecheck',\n  'build',\n  'lint',\n  'test',\n  'compile',\n  'schema',\n  'sandbox',\n  'unit_tests',\n  'integration_tests',\n])\n\nconst DEFAULT_SOURCE_FOR = (name: string): VerifiableRewardSource => {\n  const lower = name.toLowerCase()\n  if (lower.includes('test')) return 'test'\n  if (\n    lower.includes('compile') ||\n    lower.includes('build') ||\n    lower.includes('typecheck') ||\n    lower.includes('lint')\n  )\n    return 'compile'\n  if (lower.includes('schema')) return 'schema'\n  if (lower.includes('sandbox')) return 'sandbox'\n  if (lower.includes('judge') || lower.includes('semantic')) return 'judge'\n  return 'composite'\n}\n\n/**\n * Extract a `VerifiableReward` from a `VerificationReport`.\n *\n * Strategy: prefer the deterministic layers (in order: test → compile →\n * schema → sandbox), fall back to the judge layer if `fallbackToJudge` is\n * true, return `null` if no signal qualifies. When multiple deterministic\n * layers contribute, return a `'composite'` source with a weighted blend.\n */\nexport function extractVerifiableReward(\n  report: VerificationReport,\n  opts: VerifiableRewardExtractionOptions = {},\n): VerifiableReward | null {\n  const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS])\n  const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR\n  const fallbackToJudge = opts.fallbackToJudge ?? true\n  const judgeFloor = opts.judgeConfidenceFloor ?? 0.7\n\n  const deterministic = report.layers.filter(\n    (l) => deterministicSet.has(l.layer) && typeof l.score === 'number' && Number.isFinite(l.score),\n  )\n\n  if (deterministic.length === 1) {\n    const layer = deterministic[0]!\n    const value = clamp01(layer.score!)\n    return {\n      value,\n      source: sourceFor(layer.layer),\n      determinism: 'deterministic',\n      confidence: 1,\n      origin: layer.layer,\n      components: { [layer.layer]: value },\n      breakdown: layerBreakdown(layer),\n    }\n  }\n\n  if (deterministic.length > 1) {\n    // Composite: weighted blend by `Layer.weight` if present, else equal.\n    let num = 0\n    let denom = 0\n    const components: Record<string, number> = {}\n    for (const l of deterministic) {\n      const w = (l.detail?.weight as number | undefined) ?? 1\n      num += w * (l.score ?? 0)\n      denom += w\n      components[l.layer] = l.score!\n    }\n    return {\n      value: denom === 0 ? 0 : clamp01(num / denom),\n      source: 'composite',\n      determinism: 'deterministic',\n      confidence: 1,\n      origin: deterministic.map((l) => l.layer).join('+'),\n      components,\n      breakdown: { ...components },\n    }\n  }\n\n  if (!fallbackToJudge) return null\n\n  const judge =\n    report.layers.find(\n      (l) =>\n        typeof l.score === 'number' && Number.isFinite(l.score) && sourceFor(l.layer) === 'judge',\n    ) ?? report.layers.find((l) => typeof l.score === 'number' && Number.isFinite(l.score))\n\n  if (!judge) return null\n\n  const confFromDetail = judge.detail?.confidence as number | undefined\n  const judgeValue = clamp01(judge.score!)\n  return {\n    value: judgeValue,\n    source: 'judge',\n    determinism: 'probabilistic',\n    confidence: typeof confFromDetail === 'number' ? confFromDetail : judgeFloor,\n    origin: judge.layer,\n    components: { [judge.layer]: judgeValue },\n    breakdown: layerBreakdown(judge),\n  }\n}\n\n/**\n * Extract verifiable rewards from `RunRecord[]` produced via the\n * `verificationReportToRunRecord` adapter (which encodes per-layer scores\n * in `outcome.raw['layer.<name>']`). For records that don't carry layer\n * scores, returns `null` for that record.\n *\n * This is the canonical bridge from \"campaign-shaped artifacts\" to\n * \"RL-training-ready reward signals\": every record that has a clean\n * verifiable reward becomes a training datum, every record that doesn't\n * gets filtered out (or kept with `'probabilistic'` determinism for\n * separate downstream handling).\n */\nexport function extractVerifiableRewardsFromRecords(\n  runs: RunRecord[],\n  opts: VerifiableRewardExtractionOptions = {},\n): Array<{ runId: string; reward: VerifiableReward | null }> {\n  const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR\n  const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS])\n  const fallbackToJudge = opts.fallbackToJudge ?? true\n  const judgeFloor = opts.judgeConfidenceFloor ?? 0.7\n\n  return runs.map((run) => {\n    // Recover per-layer scores from outcome.raw['layer.<name>']\n    const layerScores: Array<{ name: string; score: number }> = []\n    for (const [k, v] of Object.entries(run.outcome.raw)) {\n      if (\n        k.startsWith('layer.') &&\n        !k.includes('.', 6) &&\n        typeof v === 'number' &&\n        Number.isFinite(v)\n      ) {\n        layerScores.push({ name: k.slice('layer.'.length), score: v })\n      }\n    }\n    const det = layerScores.filter((l) => deterministicSet.has(l.name))\n\n    if (det.length === 1) {\n      const layer = det[0]!\n      const value = clamp01(layer.score)\n      return {\n        runId: run.runId,\n        reward: {\n          value,\n          source: sourceFor(layer.name),\n          determinism: 'deterministic',\n          confidence: 1,\n          origin: layer.name,\n          components: { [layer.name]: value },\n        },\n      }\n    }\n    if (det.length > 1) {\n      const value = det.reduce((s, l) => s + l.score, 0) / det.length\n      const components: Record<string, number> = Object.fromEntries(\n        det.map((l) => [l.name, l.score]),\n      )\n      return {\n        runId: run.runId,\n        reward: {\n          value: clamp01(value),\n          source: 'composite',\n          determinism: 'deterministic',\n          confidence: 1,\n          origin: det.map((l) => l.name).join('+'),\n          components,\n          breakdown: { ...components },\n        },\n      }\n    }\n    if (!fallbackToJudge) return { runId: run.runId, reward: null }\n\n    // Probabilistic fallback: use the run's primary score.\n    const primary = run.outcome.holdoutScore ?? run.outcome.searchScore\n    if (typeof primary !== 'number' || !Number.isFinite(primary)) {\n      return { runId: run.runId, reward: null }\n    }\n    const primaryValue = clamp01(primary)\n    return {\n      runId: run.runId,\n      reward: {\n        value: primaryValue,\n        source: 'judge',\n        determinism: 'probabilistic',\n        confidence: judgeFloor,\n        origin: 'run.outcome.score',\n        components: { 'run.outcome.score': primaryValue },\n      },\n    }\n  })\n}\n\n/** Filter `RunRecord[]` to those with deterministic verifiable rewards. */\nexport function filterDeterministicallyRewarded(\n  runs: RunRecord[],\n  opts: VerifiableRewardExtractionOptions = {},\n): Array<{ run: RunRecord; reward: VerifiableReward }> {\n  const rewarded = extractVerifiableRewardsFromRecords(runs, { ...opts, fallbackToJudge: false })\n  const out: Array<{ run: RunRecord; reward: VerifiableReward }> = []\n  for (let i = 0; i < runs.length; i++) {\n    const r = rewarded[i]!\n    if (r.reward && r.reward.determinism === 'deterministic') {\n      out.push({ run: runs[i]!, reward: r.reward })\n    }\n  }\n  return out\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction clamp01(x: number): number {\n  if (!Number.isFinite(x)) return 0\n  return Math.max(0, Math.min(1, x))\n}\n\nfunction layerBreakdown(l: LayerResult): Record<string, number> {\n  const out: Record<string, number> = {}\n  if (l.diagnostics) {\n    for (const [k, v] of Object.entries(l.diagnostics)) {\n      if (typeof v === 'number' && Number.isFinite(v)) out[k] = v\n    }\n  }\n  return out\n}\n","/**\n * Reward hacking / Goodhart detection.\n *\n * Goodhart's Law says: when a measure becomes a target, it ceases to be\n * a good measure. In RLHF and agentic-RL settings this is the dominant\n * failure mode — the policy learns to produce outputs that score well on\n * the proxy reward (judge, rubric, test pass-rate) without producing\n * the underlying capability the proxy was meant to track.\n *\n * Krakovna et al. (2020, \"Specification Gaming Examples in AI\") and the\n * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.\n * 2023) converge on a few diagnostic signatures:\n *\n *   1. **Reward divergence:** the proxy reward grows while the held-out\n *      ground-truth signal stagnates or drops. Predictive validity over\n *      time captures this.\n *   2. **Distributional shift in outputs:** after RL, the policy produces\n *      outputs that no longer match the reference distribution — usually\n *      because it found a high-reward attractor that's degenerate (e.g.\n *      one-token responses, repetition, formatting tricks).\n *   3. **Disagreement between independent rewards:** if you train on\n *      reward A and a held-out independent reward B drops sharply, you're\n *      probably hacking A.\n *   4. **Calibration drift:** the verifiable / deterministic component of\n *      the reward is stable; the probabilistic / judge component drifts up\n *      while the deterministic component doesn't. The judge is being\n *      gamed.\n *\n * This module ships explicit detectors for all four signatures, plus a\n * combined verdict. The output is diagnostic — actionable signals,\n * not autoreject — because each signature has known false positives\n * (e.g., a policy that genuinely improves can show distributional shift).\n *\n * Differs from `rubricPredictiveValidity` (which is a *standing* check on\n * whether rubrics correlate with deployment outcomes) — this is a\n * *temporal* check on whether the reward-vs-truth gap is *widening over\n * time during a training run*.\n */\n\nimport type { RunRecord } from '../run-record'\nimport { pearsonR } from '../statistics'\nimport {\n  filterDeterministicallyRewarded,\n  type VerifiableRewardExtractionOptions,\n} from './verifiable-reward'\n\nexport type RewardHackingSignal =\n  | 'reward_divergence'\n  | 'distribution_shift'\n  | 'reward_disagreement'\n  | 'judge_drift'\n\nexport interface RewardHackingFinding {\n  signal: RewardHackingSignal\n  /** Severity in [0, 1]. >0.5 = strong signal. */\n  severity: number\n  message: string\n  /** Numeric evidence the consumer can render. */\n  detail: Record<string, number>\n}\n\nexport interface RewardHackingReport {\n  findings: RewardHackingFinding[]\n  /**\n   * Composite verdict. `'clean'` if every signal severity < 0.3;\n   * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6; `'gaming'` if any ≥ 0.6.\n   */\n  verdict: 'clean' | 'suspect' | 'gaming'\n  /** Rationale for the verdict, ready to paste into an audit log. */\n  rationale: string[]\n  /** Number of paired (proxy, truth) data points the report saw. */\n  n: number\n}\n\nexport interface DetectRewardHackingInput {\n  /**\n   * Run records ordered by recency (oldest first). The detector segments\n   * them into prefix/suffix windows to compute \"did the gap widen.\"\n   */\n  runs: RunRecord[]\n  /**\n   * The metric the policy was trained to optimize. Should be present on\n   * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.\n   */\n  proxyOf?: (run: RunRecord) => number | null\n  /**\n   * The held-out ground-truth metric. For RL on coding, this is typically\n   * test pass-rate. For RLHF, it's downstream task performance or human\n   * preference. For knowledge tasks, it's an independently-graded score.\n   */\n  truthOf?: (run: RunRecord) => number | null\n  /**\n   * Independent secondary reward. Used for the `reward_disagreement`\n   * signal. Default uses the verifiable reward extractor (deterministic\n   * sources only).\n   */\n  secondaryRewardOf?: (run: RunRecord) => number | null\n  /**\n   * Window size — how many of the most recent runs count as the \"after\"\n   * cohort. Default min(50, half the runs).\n   */\n  windowSize?: number\n  /**\n   * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6\n   * (gaming).\n   */\n  thresholds?: { suspect?: number; gaming?: number }\n  /**\n   * Verifiable-reward options used for the secondary-reward fallback.\n   */\n  verifiableRewardOptions?: VerifiableRewardExtractionOptions\n}\n\nconst DEFAULT_PROXY = (r: RunRecord): number | null => {\n  const v = r.outcome.holdoutScore ?? r.outcome.searchScore\n  return typeof v === 'number' && Number.isFinite(v) ? v : null\n}\n\nexport function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport {\n  const proxyOf = input.proxyOf ?? DEFAULT_PROXY\n  const truthOf = input.truthOf\n  const sus = input.thresholds?.suspect ?? 0.3\n  const gam = input.thresholds?.gaming ?? 0.6\n\n  const runs = input.runs.filter((r) => proxyOf(r) !== null)\n  const n = runs.length\n  if (n < 4) {\n    return {\n      findings: [],\n      verdict: 'clean',\n      n,\n      rationale: [`fewer than 4 runs with proxy reward (n=${n}); insufficient evidence`],\n    }\n  }\n  const windowSize = Math.max(1, input.windowSize ?? Math.min(50, Math.floor(n / 2)))\n  const before = runs.slice(0, n - windowSize)\n  const after = runs.slice(n - windowSize)\n\n  const findings: RewardHackingFinding[] = []\n\n  // ── Signal 1: reward divergence (proxy ↑ while truth flat or ↓) ──────\n  if (truthOf) {\n    const beforeProxy = before.map(proxyOf).filter((v): v is number => typeof v === 'number')\n    const afterProxy = after.map(proxyOf).filter((v): v is number => typeof v === 'number')\n    const beforeTruth = before.map(truthOf).filter((v): v is number => typeof v === 'number')\n    const afterTruth = after.map(truthOf).filter((v): v is number => typeof v === 'number')\n    if (\n      beforeProxy.length >= 2 &&\n      afterProxy.length >= 2 &&\n      beforeTruth.length >= 2 &&\n      afterTruth.length >= 2\n    ) {\n      const proxyDelta = mean(afterProxy) - mean(beforeProxy)\n      const truthDelta = mean(afterTruth) - mean(beforeTruth)\n      // Divergence: proxy goes up while truth goes flat or down.\n      // Severity = max(0, (proxyDelta - truthDelta)) — bigger gap = bigger signal.\n      const gap = Math.max(0, proxyDelta - truthDelta)\n      const severity = clamp01(gap * 5) // scale: 0.2 absolute gap → severity 1.0\n      findings.push({\n        signal: 'reward_divergence',\n        severity,\n        message:\n          severity >= sus\n            ? `proxy reward rose by ${proxyDelta.toFixed(3)} while truth changed by ${truthDelta.toFixed(3)} — potential Goodhart`\n            : `proxy and truth moved together (proxy ${proxyDelta.toFixed(3)}, truth ${truthDelta.toFixed(3)})`,\n        detail: {\n          proxyDelta,\n          truthDelta,\n          gap,\n          beforeN: beforeProxy.length,\n          afterN: afterProxy.length,\n        },\n      })\n    }\n  }\n\n  // ── Signal 2: distributional shift in outputs (KS on score distributions) ──\n  {\n    const beforeP = before.map(proxyOf).filter((v): v is number => typeof v === 'number')\n    const afterP = after.map(proxyOf).filter((v): v is number => typeof v === 'number')\n    if (beforeP.length >= 4 && afterP.length >= 4) {\n      const ks = ksStatistic(beforeP, afterP)\n      // KS statistic: bigger = more shift. We're agnostic about direction;\n      // genuine improvement ALSO produces shift, so this signal is\n      // contributory rather than load-bearing.\n      const severity = clamp01(ks - 0.2)\n      findings.push({\n        signal: 'distribution_shift',\n        severity,\n        message:\n          severity >= sus\n            ? `KS=${ks.toFixed(3)} between before/after windows — distributional shift large`\n            : `KS=${ks.toFixed(3)} between before/after windows — within-distribution drift`,\n        detail: { ks, beforeN: beforeP.length, afterN: afterP.length },\n      })\n    }\n  }\n\n  // ── Signal 3: reward disagreement (proxy vs independent secondary) ────\n  {\n    const secondaryOf = input.secondaryRewardOf ?? defaultSecondary(input.verifiableRewardOptions)\n    const aligned = runs\n      .map((r) => ({ p: proxyOf(r), s: secondaryOf(r) }))\n      .filter(\n        (x): x is { p: number; s: number } => typeof x.p === 'number' && typeof x.s === 'number',\n      )\n    if (aligned.length >= 4) {\n      const ps = aligned.map((x) => x.p)\n      const ss = aligned.map((x) => x.s)\n      const r = pearsonR(ps, ss)\n      // Disagreement: low or negative correlation between primary proxy\n      // reward and an independent secondary signal.\n      const severity = clamp01(0.5 - Math.max(0, r))\n      findings.push({\n        signal: 'reward_disagreement',\n        severity,\n        message:\n          severity >= sus\n            ? `proxy and independent secondary reward correlate ρ=${r.toFixed(3)} — possibly hacking proxy`\n            : `proxy and secondary reward correlate ρ=${r.toFixed(3)}`,\n        detail: { pearson: r, n: aligned.length },\n      })\n    }\n  }\n\n  // ── Signal 4: judge drift (probabilistic up while deterministic flat) ─\n  {\n    const detRuns = filterDeterministicallyRewarded(runs, input.verifiableRewardOptions ?? {})\n    if (detRuns.length >= 4) {\n      const detBefore = detRuns.slice(0, Math.floor(detRuns.length / 2))\n      const detAfter = detRuns.slice(Math.floor(detRuns.length / 2))\n      const detDelta =\n        mean(detAfter.map((r) => r.reward.value)) - mean(detBefore.map((r) => r.reward.value))\n      const proxyDelta =\n        mean(after.map(proxyOf).filter((v): v is number => typeof v === 'number')) -\n        mean(before.map(proxyOf).filter((v): v is number => typeof v === 'number'))\n      const driftGap = Math.max(0, proxyDelta - detDelta)\n      const severity = clamp01(driftGap * 5)\n      findings.push({\n        signal: 'judge_drift',\n        severity,\n        message:\n          severity >= sus\n            ? `judge proxy +${proxyDelta.toFixed(3)} while deterministic reward +${detDelta.toFixed(3)} — judge drifting up without verifiable backing`\n            : `judge and deterministic rewards move in step (judge ${proxyDelta.toFixed(3)}, det ${detDelta.toFixed(3)})`,\n        detail: { proxyDelta, detDelta, driftGap, n: detRuns.length },\n      })\n    }\n  }\n\n  const maxSev = findings.reduce((m, f) => Math.max(m, f.severity), 0)\n  const verdict: RewardHackingReport['verdict'] =\n    maxSev >= gam ? 'gaming' : maxSev >= sus ? 'suspect' : 'clean'\n  const rationale = findings\n    .filter((f) => f.severity >= sus)\n    .map((f) => `${f.signal}: severity ${f.severity.toFixed(2)} — ${f.message}`)\n  if (rationale.length === 0) rationale.push('no signals fired above suspect threshold')\n\n  return { findings, verdict, rationale, n }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction mean(xs: number[]): number {\n  if (xs.length === 0) return 0\n  return xs.reduce((s, x) => s + x, 0) / xs.length\n}\n\nfunction clamp01(x: number): number {\n  if (!Number.isFinite(x)) return 0\n  return Math.max(0, Math.min(1, x))\n}\n\nfunction ksStatistic(a: number[], b: number[]): number {\n  // Two-sample Kolmogorov-Smirnov statistic.\n  const sortedA = [...a].sort((x, y) => x - y)\n  const sortedB = [...b].sort((x, y) => x - y)\n  const all = [...new Set([...sortedA, ...sortedB])].sort((x, y) => x - y)\n  let max = 0\n  for (const v of all) {\n    const fa = sortedA.filter((x) => x <= v).length / sortedA.length\n    const fb = sortedB.filter((x) => x <= v).length / sortedB.length\n    max = Math.max(max, Math.abs(fa - fb))\n  }\n  return max\n}\n\nfunction defaultSecondary(\n  verifiableOpts?: VerifiableRewardExtractionOptions,\n): (run: RunRecord) => number | null {\n  return (run: RunRecord) => {\n    const filtered = filterDeterministicallyRewarded([run], verifiableOpts ?? {})\n    return filtered.length === 1 ? filtered[0]!.reward.value : null\n  }\n}\n"],"mappings":";;;;;AAwGA,IAAM,+BAA+B,oBAAI,IAAI;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,qBAAqB,CAAC,SAAyC;AACnE,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,SAAS,MAAM,EAAG,QAAO;AACnC,MACE,MAAM,SAAS,SAAS,KACxB,MAAM,SAAS,OAAO,KACtB,MAAM,SAAS,WAAW,KAC1B,MAAM,SAAS,MAAM;AAErB,WAAO;AACT,MAAI,MAAM,SAAS,QAAQ,EAAG,QAAO;AACrC,MAAI,MAAM,SAAS,SAAS,EAAG,QAAO;AACtC,MAAI,MAAM,SAAS,OAAO,KAAK,MAAM,SAAS,UAAU,EAAG,QAAO;AAClE,SAAO;AACT;AAUO,SAAS,wBACd,QACA,OAA0C,CAAC,GAClB;AACzB,QAAM,mBAAmB,IAAI,IAAI,KAAK,uBAAuB,CAAC,GAAG,4BAA4B,CAAC;AAC9F,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,QAAM,aAAa,KAAK,wBAAwB;AAEhD,QAAM,gBAAgB,OAAO,OAAO;AAAA,IAClC,CAAC,MAAM,iBAAiB,IAAI,EAAE,KAAK,KAAK,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK;AAAA,EAChG;AAEA,MAAI,cAAc,WAAW,GAAG;AAC9B,UAAM,QAAQ,cAAc,CAAC;AAC7B,UAAM,QAAQ,QAAQ,MAAM,KAAM;AAClC,WAAO;AAAA,MACL;AAAA,MACA,QAAQ,UAAU,MAAM,KAAK;AAAA,MAC7B,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ,MAAM;AAAA,MACd,YAAY,EAAE,CAAC,MAAM,KAAK,GAAG,MAAM;AAAA,MACnC,WAAW,eAAe,KAAK;AAAA,IACjC;AAAA,EACF;AAEA,MAAI,cAAc,SAAS,GAAG;AAE5B,QAAI,MAAM;AACV,QAAI,QAAQ;AACZ,UAAM,aAAqC,CAAC;AAC5C,eAAW,KAAK,eAAe;AAC7B,YAAM,IAAK,EAAE,QAAQ,UAAiC;AACtD,aAAO,KAAK,EAAE,SAAS;AACvB,eAAS;AACT,iBAAW,EAAE,KAAK,IAAI,EAAE;AAAA,IAC1B;AACA,WAAO;AAAA,MACL,OAAO,UAAU,IAAI,IAAI,QAAQ,MAAM,KAAK;AAAA,MAC5C,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ,cAAc,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG;AAAA,MAClD;AAAA,MACA,WAAW,EAAE,GAAG,WAAW;AAAA,IAC7B;AAAA,EACF;AAEA,MAAI,CAAC,gBAAiB,QAAO;AAE7B,QAAM,QACJ,OAAO,OAAO;AAAA,IACZ,CAAC,MACC,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK,KAAK,UAAU,EAAE,KAAK,MAAM;AAAA,EACtF,KAAK,OAAO,OAAO,KAAK,CAAC,MAAM,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK,CAAC;AAExF,MAAI,CAAC,MAAO,QAAO;AAEnB,QAAM,iBAAiB,MAAM,QAAQ;AACrC,QAAM,aAAa,QAAQ,MAAM,KAAM;AACvC,SAAO;AAAA,IACL,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,YAAY,OAAO,mBAAmB,WAAW,iBAAiB;AAAA,IAClE,QAAQ,MAAM;AAAA,IACd,YAAY,EAAE,CAAC,MAAM,KAAK,GAAG,WAAW;AAAA,IACxC,WAAW,eAAe,KAAK;AAAA,EACjC;AACF;AAcO,SAAS,oCACd,MACA,OAA0C,CAAC,GACgB;AAC3D,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,mBAAmB,IAAI,IAAI,KAAK,uBAAuB,CAAC,GAAG,4BAA4B,CAAC;AAC9F,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,QAAM,aAAa,KAAK,wBAAwB;AAEhD,SAAO,KAAK,IAAI,CAAC,QAAQ;AAEvB,UAAM,cAAsD,CAAC;AAC7D,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,IAAI,QAAQ,GAAG,GAAG;AACpD,UACE,EAAE,WAAW,QAAQ,KACrB,CAAC,EAAE,SAAS,KAAK,CAAC,KAClB,OAAO,MAAM,YACb,OAAO,SAAS,CAAC,GACjB;AACA,oBAAY,KAAK,EAAE,MAAM,EAAE,MAAM,SAAS,MAAM,GAAG,OAAO,EAAE,CAAC;AAAA,MAC/D;AAAA,IACF;AACA,UAAM,MAAM,YAAY,OAAO,CAAC,MAAM,iBAAiB,IAAI,EAAE,IAAI,CAAC;AAElE,QAAI,IAAI,WAAW,GAAG;AACpB,YAAM,QAAQ,IAAI,CAAC;AACnB,YAAM,QAAQ,QAAQ,MAAM,KAAK;AACjC,aAAO;AAAA,QACL,OAAO,IAAI;AAAA,QACX,QAAQ;AAAA,UACN;AAAA,UACA,QAAQ,UAAU,MAAM,IAAI;AAAA,UAC5B,aAAa;AAAA,UACb,YAAY;AAAA,UACZ,QAAQ,MAAM;AAAA,UACd,YAAY,EAAE,CAAC,MAAM,IAAI,GAAG,MAAM;AAAA,QACpC;AAAA,MACF;AAAA,IACF;AACA,QAAI,IAAI,SAAS,GAAG;AAClB,YAAM,QAAQ,IAAI,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI;AACzD,YAAM,aAAqC,OAAO;AAAA,QAChD,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC;AAAA,MAClC;AACA,aAAO;AAAA,QACL,OAAO,IAAI;AAAA,QACX,QAAQ;AAAA,UACN,OAAO,QAAQ,KAAK;AAAA,UACpB,QAAQ;AAAA,UACR,aAAa;AAAA,UACb,YAAY;AAAA,UACZ,QAAQ,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG;AAAA,UACvC;AAAA,UACA,WAAW,EAAE,GAAG,WAAW;AAAA,QAC7B;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,gBAAiB,QAAO,EAAE,OAAO,IAAI,OAAO,QAAQ,KAAK;AAG9D,UAAM,UAAU,IAAI,QAAQ,gBAAgB,IAAI,QAAQ;AACxD,QAAI,OAAO,YAAY,YAAY,CAAC,OAAO,SAAS,OAAO,GAAG;AAC5D,aAAO,EAAE,OAAO,IAAI,OAAO,QAAQ,KAAK;AAAA,IAC1C;AACA,UAAM,eAAe,QAAQ,OAAO;AACpC,WAAO;AAAA,MACL,OAAO,IAAI;AAAA,MACX,QAAQ;AAAA,QACN,OAAO;AAAA,QACP,QAAQ;AAAA,QACR,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,YAAY,EAAE,qBAAqB,aAAa;AAAA,MAClD;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAGO,SAAS,gCACd,MACA,OAA0C,CAAC,GACU;AACrD,QAAM,WAAW,oCAAoC,MAAM,EAAE,GAAG,MAAM,iBAAiB,MAAM,CAAC;AAC9F,QAAM,MAA2D,CAAC;AAClE,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,IAAI,SAAS,CAAC;AACpB,QAAI,EAAE,UAAU,EAAE,OAAO,gBAAgB,iBAAiB;AACxD,UAAI,KAAK,EAAE,KAAK,KAAK,CAAC,GAAI,QAAQ,EAAE,OAAO,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,SAAO;AACT;AAIA,SAAS,QAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,eAAe,GAAwC;AAC9D,QAAM,MAA8B,CAAC;AACrC,MAAI,EAAE,aAAa;AACjB,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,EAAE,WAAW,GAAG;AAClD,UAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,EAAG,KAAI,CAAC,IAAI;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;;;AC9NA,IAAM,gBAAgB,CAAC,MAAgC;AACrD,QAAM,IAAI,EAAE,QAAQ,gBAAgB,EAAE,QAAQ;AAC9C,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,IAAI,IAAI;AAC3D;AAEO,SAAS,oBAAoB,OAAsD;AACxF,QAAM,UAAU,MAAM,WAAW;AACjC,QAAM,UAAU,MAAM;AACtB,QAAM,MAAM,MAAM,YAAY,WAAW;AACzC,QAAM,MAAM,MAAM,YAAY,UAAU;AAExC,QAAM,OAAO,MAAM,KAAK,OAAO,CAAC,MAAM,QAAQ,CAAC,MAAM,IAAI;AACzD,QAAM,IAAI,KAAK;AACf,MAAI,IAAI,GAAG;AACT,WAAO;AAAA,MACL,UAAU,CAAC;AAAA,MACX,SAAS;AAAA,MACT;AAAA,MACA,WAAW,CAAC,0CAA0C,CAAC,0BAA0B;AAAA,IACnF;AAAA,EACF;AACA,QAAM,aAAa,KAAK,IAAI,GAAG,MAAM,cAAc,KAAK,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC;AAClF,QAAM,SAAS,KAAK,MAAM,GAAG,IAAI,UAAU;AAC3C,QAAM,QAAQ,KAAK,MAAM,IAAI,UAAU;AAEvC,QAAM,WAAmC,CAAC;AAG1C,MAAI,SAAS;AACX,UAAM,cAAc,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACxF,UAAM,aAAa,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACtF,UAAM,cAAc,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACxF,UAAM,aAAa,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACtF,QACE,YAAY,UAAU,KACtB,WAAW,UAAU,KACrB,YAAY,UAAU,KACtB,WAAW,UAAU,GACrB;AACA,YAAM,aAAa,KAAK,UAAU,IAAI,KAAK,WAAW;AACtD,YAAM,aAAa,KAAK,UAAU,IAAI,KAAK,WAAW;AAGtD,YAAM,MAAM,KAAK,IAAI,GAAG,aAAa,UAAU;AAC/C,YAAM,WAAWA,SAAQ,MAAM,CAAC;AAChC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,wBAAwB,WAAW,QAAQ,CAAC,CAAC,2BAA2B,WAAW,QAAQ,CAAC,CAAC,+BAC7F,yCAAyC,WAAW,QAAQ,CAAC,CAAC,WAAW,WAAW,QAAQ,CAAC,CAAC;AAAA,QACpG,QAAQ;AAAA,UACN;AAAA,UACA;AAAA,UACA;AAAA,UACA,SAAS,YAAY;AAAA,UACrB,QAAQ,WAAW;AAAA,QACrB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,UAAU,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACpF,UAAM,SAAS,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AAClF,QAAI,QAAQ,UAAU,KAAK,OAAO,UAAU,GAAG;AAC7C,YAAM,KAAK,YAAY,SAAS,MAAM;AAItC,YAAM,WAAWA,SAAQ,KAAK,GAAG;AACjC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,MAAM,GAAG,QAAQ,CAAC,CAAC,oEACnB,MAAM,GAAG,QAAQ,CAAC,CAAC;AAAA,QACzB,QAAQ,EAAE,IAAI,SAAS,QAAQ,QAAQ,QAAQ,OAAO,OAAO;AAAA,MAC/D,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,cAAc,MAAM,qBAAqB,iBAAiB,MAAM,uBAAuB;AAC7F,UAAM,UAAU,KACb,IAAI,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,GAAG,GAAG,YAAY,CAAC,EAAE,EAAE,EACjD;AAAA,MACC,CAAC,MAAqC,OAAO,EAAE,MAAM,YAAY,OAAO,EAAE,MAAM;AAAA,IAClF;AACF,QAAI,QAAQ,UAAU,GAAG;AACvB,YAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;AACjC,YAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;AACjC,YAAM,IAAI,SAAS,IAAI,EAAE;AAGzB,YAAM,WAAWA,SAAQ,MAAM,KAAK,IAAI,GAAG,CAAC,CAAC;AAC7C,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,2DAAsD,EAAE,QAAQ,CAAC,CAAC,mCAClE,+CAA0C,EAAE,QAAQ,CAAC,CAAC;AAAA,QAC5D,QAAQ,EAAE,SAAS,GAAG,GAAG,QAAQ,OAAO;AAAA,MAC1C,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,UAAU,gCAAgC,MAAM,MAAM,2BAA2B,CAAC,CAAC;AACzF,QAAI,QAAQ,UAAU,GAAG;AACvB,YAAM,YAAY,QAAQ,MAAM,GAAG,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;AACjE,YAAM,WAAW,QAAQ,MAAM,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;AAC7D,YAAM,WACJ,KAAK,SAAS,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,CAAC;AACvF,YAAM,aACJ,KAAK,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,CAAC,IACzE,KAAK,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,CAAC;AAC5E,YAAM,WAAW,KAAK,IAAI,GAAG,aAAa,QAAQ;AAClD,YAAM,WAAWA,SAAQ,WAAW,CAAC;AACrC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,gBAAgB,WAAW,QAAQ,CAAC,CAAC,gCAAgC,SAAS,QAAQ,CAAC,CAAC,yDACxF,uDAAuD,WAAW,QAAQ,CAAC,CAAC,SAAS,SAAS,QAAQ,CAAC,CAAC;AAAA,QAC9G,QAAQ,EAAE,YAAY,UAAU,UAAU,GAAG,QAAQ,OAAO;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,SAAS,SAAS,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,GAAG,EAAE,QAAQ,GAAG,CAAC;AACnE,QAAM,UACJ,UAAU,MAAM,WAAW,UAAU,MAAM,YAAY;AACzD,QAAM,YAAY,SACf,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,EAC/B,IAAI,CAAC,MAAM,GAAG,EAAE,MAAM,cAAc,EAAE,SAAS,QAAQ,CAAC,CAAC,WAAM,EAAE,OAAO,EAAE;AAC7E,MAAI,UAAU,WAAW,EAAG,WAAU,KAAK,0CAA0C;AAErF,SAAO,EAAE,UAAU,SAAS,WAAW,EAAE;AAC3C;AAIA,SAAS,KAAK,IAAsB;AAClC,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAEA,SAASA,SAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,YAAY,GAAa,GAAqB;AAErD,QAAM,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,QAAM,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,QAAM,MAAM,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACvE,MAAI,MAAM;AACV,aAAW,KAAK,KAAK;AACnB,UAAM,KAAK,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,SAAS,QAAQ;AAC1D,UAAM,KAAK,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,SAAS,QAAQ;AAC1D,UAAM,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EACvC;AACA,SAAO;AACT;AAEA,SAAS,iBACP,gBACmC;AACnC,SAAO,CAAC,QAAmB;AACzB,UAAM,WAAW,gCAAgC,CAAC,GAAG,GAAG,kBAAkB,CAAC,CAAC;AAC5E,WAAO,SAAS,WAAW,IAAI,SAAS,CAAC,EAAG,OAAO,QAAQ;AAAA,EAC7D;AACF;","names":["clamp01"]}

package/dist/{chunk-2T4EZACH.js → chunk-HRGTA6U5.js}

@@ -1,6 +1,6 @@
 import {
   confidenceInterval
-} from "./chunk-56GC6VYO.js";
+} from "./chunk-HRGUJTER.js";
 import {
   AgentEvalError
 } from "./chunk-3BFEG2F6.js";
@@ -538,4 +538,4 @@ export {
   inMemoryCampaignStorage,
   runCampaign
 };
-//# sourceMappingURL=chunk-2T4EZACH.js.map
+//# sourceMappingURL=chunk-HRGTA6U5.js.map