@tangle-network/agent-eval 0.125.0 → 0.126.0

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-SJT4OBVL.js";
+} from "./chunk-SDPM6554.js";
 import {
   mintRolloutRows,
   rolloutReward
@@ -46,7 +46,7 @@ import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
   deterministicSplit
-} from "./chunk-JKDNAOF5.js";
+} from "./chunk-W4L6C2XT.js";
 import {
   DEFAULT_RULES,
   classifyFailure,
@@ -87,7 +87,7 @@ import {
   pairArms,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-A62YMFWA.js";
+} from "./chunk-KO2PZOGP.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -98,7 +98,6 @@ import {
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
   adversarialJudge,
   buildReflectionPrompt,
-  campaignMeanComposite,
   codeExecutionJudge,
   coherenceJudge,
   costReceiptFromTCloud,
@@ -108,9 +107,7 @@ import {
   crowdingDistance,
   defaultJudges,
   dominates,
-  gepaProposer,
   hashScenarios,
-  heldOutGate,
   llmJudge,
   maximumChargeForTCloudRequest,
   paretoFrontier,
@@ -119,13 +116,12 @@ import {
   redTeamDataset,
   redTeamReport,
   runCanaries,
-  runImprovementLoop,
   runReferenceEquivalenceJudge,
   scalarScore,
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-PMITBABE.js";
+} from "./chunk-NTOV7RU5.js";
 import {
   BackendIntegrityError,
   assertRealAgentReceipts,
@@ -137,7 +133,7 @@ import {
   inMemoryVerdictCache,
   summarizeAgentReceiptIntegrity,
   summarizeBackendIntegrity
-} from "./chunk-D5JZ7UDZ.js";
+} from "./chunk-UCLVDLCH.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   FindingsStore,
@@ -150,47 +146,32 @@ import {
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-W5B3ZGP3.js";
-import {
-  buildDefaultAnalystRegistry,
-  computeTraceMetrics,
-  createChatClient
-} from "./chunk-A6GT67HT.js";
-import "./chunk-HHWE3POT.js";
+} from "./chunk-4B7ZZHPX.js";
 import {
   AnalystRegistry,
-  DEFAULT_RUN_SCORE_WEIGHTS,
   DEFAULT_TRACE_ANALYST_KINDS,
   FAILURE_MODE_KIND_SPEC,
   IMPROVEMENT_KIND_SPEC,
   KNOWLEDGE_GAP_KIND_SPEC,
   KNOWLEDGE_POISONING_KIND_SPEC,
-  Mutex,
-  POLICY_EDIT_AXES,
-  POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
-  POLICY_EDIT_TARGET_SURFACES,
-  PolicyEditValidationError,
-  admitPolicyEdit,
-  aggregateRunScore,
-  applyPolicyEditToSurface,
-  clamp01,
+  buildDefaultAnalystRegistry,
   computeFindingId,
-  computePolicyEditId,
+  computeTraceMetrics,
   createAnalystAi,
+  createChatClient,
   createTraceAnalystKind,
-  isPolicyEdit,
   makeFinding,
-  makePolicyEdit,
-  makePolicyEditCandidateRecord,
-  mapConcurrent,
-  policyEditFromFinding,
-  policyEditsFromFindings,
   renderPriorFindings,
-  renderUpstreamFindings,
-  scorePolicyEditReadiness,
-  validatePolicyEdit,
-  validatePolicyEditCandidateRecord
-} from "./chunk-DT7OXY3C.js";
+  renderUpstreamFindings
+} from "./chunk-CM4OILD2.js";
+import "./chunk-HHWE3POT.js";
+import {
+  DEFAULT_RUN_SCORE_WEIGHTS,
+  Mutex,
+  aggregateRunScore,
+  clamp01,
+  mapConcurrent
+} from "./chunk-UI4YMIN2.js";
 import {
   allCriticalPassed,
   controlFailureClassFromVerification,
@@ -211,7 +192,7 @@ import {
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-R226UZOI.js";
+} from "./chunk-474LBSOX.js";
 import {
   assertReleaseConfidence,
   bootstrapCi,
@@ -221,7 +202,7 @@ import {
 } from "./chunk-MOXWMGPC.js";
 import {
   runEvalCampaign
-} from "./chunk-GID26AN4.js";
+} from "./chunk-P22LJ3Y2.js";
 import "./chunk-ARU2PZFM.js";
 import {
   LlmCallError,
@@ -238,7 +219,7 @@ import {
   maximumChargeForLlmRequest,
   probeLlm,
   stripFencedJson
-} from "./chunk-EQUK3RFS.js";
+} from "./chunk-J5SQWP6Y.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -301,7 +282,7 @@ import {
   costForTokenPricing,
   costForUsage,
   modelPriceKey
-} from "./chunk-GRCDRKII.js";
+} from "./chunk-WS3NZZQQ.js";
 import {
   MODEL_PRICING,
   MetricsCollector,
@@ -340,7 +321,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-QFQZ3U3X.js";
+} from "./chunk-OCFJACJU.js";
 import "./chunk-H5UD2323.js";
 import {
   extractUsage,
@@ -405,14 +386,26 @@ import {
   llmSpanFromProvider
 } from "./chunk-VQMK5FMP.js";
 import {
+  AGENT_PROFILE_KINDS,
+  AgentProfileCellValidationError,
   RunRecordValidationError,
+  agentProfileCellHashMaterial,
+  agentProfileCellKey,
+  assertRunAgentProfileCell,
+  buildAgentInterfaceProfileCell,
+  buildAgentProfileCell,
+  groupRunsByAgentProfileCell,
   isRunRecord,
   modelHasSnapshot,
   parseRunRecordSafe,
+  requireAgentProfileCell,
   resolveRunCostProvenance,
   roundTripRunRecord,
-  validateRunRecord
-} from "./chunk-LOW3U7JZ.js";
+  toAgentProfileJson,
+  validateAgentProfileCell,
+  validateRunRecord,
+  verifyAgentProfileCell
+} from "./chunk-IILEIWGW.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
@@ -422,20 +415,6 @@ import {
   isSandboxSpan,
   isToolSpan
 } from "./chunk-MA6HLL3S.js";
-import {
-  AGENT_PROFILE_KINDS,
-  AgentProfileCellValidationError,
-  agentProfileCellHashMaterial,
-  agentProfileCellKey,
-  assertRunAgentProfileCell,
-  buildAgentInterfaceProfileCell,
-  buildAgentProfileCell,
-  groupRunsByAgentProfileCell,
-  requireAgentProfileCell,
-  toAgentProfileJson,
-  validateAgentProfileCell,
-  verifyAgentProfileCell
-} from "./chunk-GC4ATIKK.js";
 import {
   canonicalize,
   evaluateHypothesis,
@@ -10066,414 +10045,6 @@ function isOtelConfigured() {
   return !!(typeof process !== "undefined" && process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
 }
 
-// src/traced-analyst.ts
-async function tracedAnalyzeTraces(input, options, traceOpts) {
-  const parentSpan = await traceOpts.emitter.span({
-    kind: "custom",
-    name: "analyst:analyze-traces",
-    parentSpanId: traceOpts.parentSpanId,
-    attributes: {
-      "analyst.question_length": input.question.length,
-      "analyst.max_turns": options.maxTurns ?? 12,
-      "analyst.max_subqueries": options.maxSubqueries ?? 4,
-      "eval.phase": "analyst"
-    }
-  });
-  const originalOnTurn = options.onTurn;
-  const wrappedOptions = {
-    ...options,
-    onTurn: async (turn) => {
-      const turnSpan = await traceOpts.emitter.span({
-        kind: "custom",
-        name: `analyst:turn-${turn.turn}`,
-        parentSpanId: parentSpan.span.spanId,
-        attributes: {
-          "analyst.stage": turn.stage,
-          "analyst.turn": turn.turn,
-          "analyst.is_error": turn.isError,
-          "analyst.code_length": turn.code.length,
-          "analyst.output_length": turn.output.length,
-          "eval.phase": "analyst"
-        }
-      });
-      if (turn.isError) {
-        await turnSpan.fail("Turn produced an error");
-      } else {
-        await turnSpan.end();
-      }
-      if (originalOnTurn) await originalOnTurn(turn);
-    }
-  };
-  try {
-    const result = await analyzeTraces(input, wrappedOptions);
-    await parentSpan.end({
-      attributes: {
-        "analyst.question_length": input.question.length,
-        "analyst.turn_count": result.turnCount,
-        "analyst.finding_count": result.findings.length,
-        "analyst.answer_length": result.answer.length,
-        "eval.phase": "analyst"
-      }
-    });
-    return result;
-  } catch (err) {
-    await parentSpan.fail(err instanceof Error ? err : String(err));
-    throw err;
-  }
-}
-
-// src/traced-judges.ts
-function traceJudge(judge, judgeName, opts) {
-  return async (tc, input) => {
-    const span = await opts.emitter.span({
-      kind: "llm",
-      name: `judge:${judgeName}`,
-      parentSpanId: opts.parentSpanId,
-      attributes: {
-        "judge.name": judgeName,
-        "eval.phase": "judge"
-      }
-    });
-    try {
-      const scores2 = await judge(tc, input);
-      const composite = scores2.length > 0 ? scores2.reduce((sum4, s) => sum4 + s.score, 0) / scores2.length : 0;
-      await span.end({
-        attributes: {
-          "judge.name": judgeName,
-          "judge.composite_score": composite,
-          "judge.dimension_count": scores2.length,
-          "eval.phase": "judge"
-        }
-      });
-      return scores2;
-    } catch (err) {
-      await span.fail(err instanceof Error ? err : String(err));
-      throw err;
-    }
-  };
-}
-function traceJudgeEnsemble(judges, judgeNames, opts) {
-  return async (tc, input) => {
-    const ensembleSpan = await opts.emitter.span({
-      kind: "custom",
-      name: "judge:ensemble",
-      parentSpanId: opts.parentSpanId,
-      attributes: {
-        "judge.ensemble_size": judges.length,
-        "eval.phase": "judge"
-      }
-    });
-    try {
-      const allScores = [];
-      let failedJudges = 0;
-      for (let i = 0; i < judges.length; i++) {
-        const judge = judges[i];
-        const name = judgeNames[i] ?? `judge_${i}`;
-        const tracedFn = traceJudge(judge, name, {
-          emitter: opts.emitter,
-          parentSpanId: ensembleSpan.span.spanId
-        });
-        try {
-          const scores2 = await tracedFn(tc, input);
-          allScores.push(...scores2);
-        } catch (err) {
-          if (!(err instanceof JudgeParseError)) throw err;
-          failedJudges++;
-        }
-      }
-      const composite = allScores.length > 0 ? allScores.reduce((sum4, s) => sum4 + s.score, 0) / allScores.length : 0;
-      await ensembleSpan.end({
-        attributes: {
-          "judge.ensemble_size": judges.length,
-          "judge.composite_score": composite,
-          "judge.total_dimensions": allScores.length,
-          "judge.failed_judges": failedJudges,
-          "eval.phase": "judge"
-        }
-      });
-      return allScores;
-    } catch (err) {
-      await ensembleSpan.fail(err instanceof Error ? err : String(err));
-      throw err;
-    }
-  };
-}
-
-// src/campaign/distillation/agreement-judge.ts
-var AGREEMENT_DIM = "agreement";
-function buildAgreementJudge(options) {
-  const name = options.name ?? "gold-agreement";
-  const goldOnly = options.goldOnly ?? true;
-  const declaredDims = options.dimensionKeys ?? [AGREEMENT_DIM];
-  return {
-    name,
-    dimensions: declaredDims.map((key) => ({
-      key,
-      description: `Per-field agreement between the produced label and the gold label on '${key}'`
-    })),
-    appliesTo: goldOnly ? (scenario) => scenario.kind === "gold" : void 0,
-    score({ artifact, scenario }) {
-      const { score, dimensions } = options.compareLabels(artifact, scenario.label);
-      if (!Number.isFinite(score) || score < 0 || score > 1) {
-        throw new Error(
-          `buildAgreementJudge: comparator returned out-of-range score ${score} for scenario '${scenario.id}' (must be in [0,1])`
-        );
-      }
-      const outDims = { [AGREEMENT_DIM]: score, ...dimensions };
-      const weakest = Object.entries(dimensions).sort((a, b) => a[1] - b[1])[0];
-      const notes = weakest ? `agreement ${score.toFixed(3)}; weakest field '${weakest[0]}' (${weakest[1].toFixed(3)})` : `agreement ${score.toFixed(3)}`;
-      return { composite: score, dimensions: outDims, notes };
-    }
-  };
-}
-function fieldAgreement(spec) {
-  const categorical = spec.categorical ?? [];
-  const array = spec.array ?? [];
-  if (categorical.length === 0 && array.length === 0) {
-    throw new Error("fieldAgreement: at least one categorical or array field is required");
-  }
-  return (produced, gold) => {
-    const p = produced ?? {};
-    const g = gold ?? {};
-    const dimensions = {};
-    for (const field of categorical) {
-      dimensions[field] = categoricalAgreement(p[field], g[field]);
-    }
-    for (const field of array) {
-      dimensions[field] = jaccard(asArray(p[field]), asArray(g[field]));
-    }
-    const values = Object.values(dimensions);
-    const score = values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
-    return { score, dimensions };
-  };
-}
-function categoricalAgreement(produced, gold) {
-  if (produced === void 0 && gold === void 0) return 1;
-  return normalizeScalar(produced) === normalizeScalar(gold) ? 1 : 0;
-}
-function normalizeScalar(value) {
-  if (value === void 0) return "__undefined__";
-  if (value === null) return "__null__";
-  return JSON.stringify(value);
-}
-function asArray(value) {
-  if (Array.isArray(value)) return value;
-  if (value === void 0 || value === null) return [];
-  return [value];
-}
-function jaccard(a, b) {
-  const sa = new Set(a.map((x) => JSON.stringify(x)));
-  const sb = new Set(b.map((x) => JSON.stringify(x)));
-  if (sa.size === 0 && sb.size === 0) return 1;
-  let inter = 0;
-  for (const x of sa) if (sb.has(x)) inter++;
-  const union = sa.size + sb.size - inter;
-  return union === 0 ? 1 : inter / union;
-}
-
-// src/campaign/distillation/gold-scenarios.ts
-import { readFileSync as readFileSync5 } from "fs";
-function loadGoldScenarios(jsonlPath) {
-  const text = readFileSync5(jsonlPath, "utf8");
-  return parseGoldJsonl(text, jsonlPath);
-}
-function parseGoldJsonl(text, sourceLabel = "<inline>") {
-  const out = [];
-  const lines = text.split("\n");
-  for (let i = 0; i < lines.length; i++) {
-    const raw = lines[i].trim();
-    if (raw.length === 0) continue;
-    let parsed;
-    try {
-      parsed = JSON.parse(raw);
-    } catch (err) {
-      throw new Error(
-        `loadGoldScenarios: ${sourceLabel}:${i + 1} is not valid JSON \u2014 ${err instanceof Error ? err.message : String(err)}`
-      );
-    }
-    const rawId = parsed.scenarioId ?? parsed.id;
-    if (typeof rawId !== "string" || rawId.length === 0) {
-      throw new Error(
-        `loadGoldScenarios: ${sourceLabel}:${i + 1} missing string \`scenarioId\`/\`id\``
-      );
-    }
-    const id = rawId.replace(/:/g, "__");
-    if (parsed.input === void 0) {
-      throw new Error(`loadGoldScenarios: ${sourceLabel}:${i + 1} (${rawId}) missing \`input\``);
-    }
-    if (parsed.label === void 0) {
-      throw new Error(`loadGoldScenarios: ${sourceLabel}:${i + 1} (${rawId}) missing \`label\``);
-    }
-    const scenario = {
-      id,
-      kind: "gold",
-      input: parsed.input,
-      label: parsed.label
-    };
-    const tags = [];
-    if (id !== rawId) tags.push(`gold-id:${rawId}`);
-    if (parsed.split !== void 0) tags.push(`split:${parsed.split}`);
-    if (tags.length > 0) scenario.tags = tags;
-    out.push(scenario);
-  }
-  if (out.length === 0) {
-    throw new Error(`loadGoldScenarios: ${sourceLabel} contained no gold records`);
-  }
-  return out;
-}
-function splitGold(scenarios, options = {}) {
-  const testEveryNth = options.testEveryNth ?? 4;
-  if (!Number.isInteger(testEveryNth) || testEveryNth < 2) {
-    throw new Error("splitGold: testEveryNth must be an integer \u2265 2 (else train or test is empty)");
-  }
-  const train = [];
-  const test = [];
-  let implicitIndex = 0;
-  for (const scenario of scenarios) {
-    const explicit = explicitSplit(scenario);
-    if (explicit === "train") {
-      train.push(scenario);
-    } else if (explicit === "test") {
-      test.push(scenario);
-    } else {
-      if (implicitIndex % testEveryNth === 0) test.push(scenario);
-      else train.push(scenario);
-      implicitIndex += 1;
-    }
-  }
-  return { train, test };
-}
-function explicitSplit(scenario) {
-  for (const tag of scenario.tags ?? []) {
-    if (tag === "split:train") return "train";
-    if (tag === "split:test") return "test";
-  }
-  return void 0;
-}
-
-// src/campaign/distillation/run-distillation.ts
-async function runDistillation(opts) {
-  if (opts.train.length === 0) throw new Error("runDistillation: train split is empty");
-  if (opts.holdout.length === 0) throw new Error("runDistillation: holdout split is empty");
-  const chat = createChatClient(opts.llm);
-  const render = opts.renderStudentPrompt ?? defaultRenderStudentPrompt;
-  const parse = opts.parseStudentLabel ?? defaultParseStudentLabel;
-  const runDir = opts.runDir ?? `.evolve/distillation/${Date.now()}`;
-  const studentTemperature = opts.studentTemperature ?? 0;
-  const studentMaxTokens = opts.studentMaxTokens ?? 1024;
-  const proposer = gepaProposer({
-    llm: opts.reflectionLlm,
-    model: opts.optimizerModel,
-    target: "a cheap single-shot analyst system prompt that reproduces an expensive workflow gold verdict",
-    mutationPrimitives: opts.mutationPrimitives ?? DEFAULT_MUTATION_PRIMITIVES2,
-    constraints: opts.constraints
-  });
-  const gate = opts.gate ?? heldOutGate({
-    scenarios: opts.holdout,
-    deltaThreshold: opts.deltaThreshold ?? 0
-  });
-  const loop = await runImprovementLoop({
-    baselineSurface: opts.baselinePrompt,
-    scenarios: opts.train,
-    holdoutScenarios: opts.holdout,
-    judges: [opts.judge],
-    proposer,
-    gate,
-    autoOnPromote: "none",
-    // the loop NEVER opens a PR — the caller decides
-    populationSize: opts.populationSize ?? 4,
-    maxGenerations: opts.maxGenerations ?? 3,
-    reps: opts.reps ?? 1,
-    runDir,
-    // The student spends tokens; tracing must stay on (the proposer is wired and
-    // runImprovementLoop refuses tracing='off' with a proposer).
-    tracing: "on",
-    dispatchWithSurface: async (surface, scenario, ctx) => {
-      const prompt = render({
-        surface: typeof surface === "string" ? surface : JSON.stringify(surface),
-        input: scenario.input,
-        scenarioId: scenario.id
-      });
-      const request = {
-        model: opts.studentModel,
-        messages: prompt,
-        jsonMode: true,
-        temperature: studentTemperature,
-        maxTokens: studentMaxTokens
-      };
-      const paid = await ctx.cost.runPaidCall({
-        actor: "distillation-student",
-        model: opts.studentModel,
-        maximumCharge: chat.maximumAttempts === void 0 ? void 0 : maximumChargeForLlmRequest(request, { maxRetries: chat.maximumAttempts }),
-        execute: (signal, callId) => chat.chat(request, { signal, idempotencyKey: callId }),
-        receipt: costReceiptFromLlm,
-        receiptFromError: costReceiptFromLlmError
-      });
-      if (!paid.succeeded) throw paid.error;
-      return parse(paid.value.content, scenario.id);
-    }
-  });
-  const winnerPrompt = typeof loop.winnerSurface === "string" ? loop.winnerSurface : opts.baselinePrompt;
-  const baseline = campaignMeanComposite(loop.baselineOnHoldout);
-  const winner = campaignMeanComposite(loop.winnerOnHoldout);
-  return {
-    ...loop,
-    winnerPrompt,
-    holdoutAgreement: { baseline, winner, delta: winner - baseline }
-  };
-}
-var DEFAULT_MUTATION_PRIMITIVES2 = [
-  "Add an explicit output-schema instruction so the model emits exactly the gold label fields as JSON.",
-  "Add a one-line decision rule for each verdict field the student keeps getting wrong.",
-  "Add a worked example mapping a representative input to its correct gold label.",
-  "Tighten ambiguous phrasing that lets the student hedge instead of committing to a verdict.",
-  "Add a guardrail that forces the student to set boolean risk flags (e.g. leak risk) when the triggering condition is present."
-];
-function defaultRenderStudentPrompt(args) {
-  return [
-    { role: "system", content: args.surface },
-    {
-      role: "user",
-      content: `Input:
-${stableStringify(args.input)}
-
-Respond with ONLY a single JSON object \u2014 the verdict. No prose, no code fences.`
-    }
-  ];
-}
-function defaultParseStudentLabel(rawContent, scenarioId) {
-  const stripped = stripFence(rawContent).trim();
-  if (stripped.length === 0) {
-    throw new Error(`distillation student returned empty output for scenario '${scenarioId}'`);
-  }
-  try {
-    return JSON.parse(stripped);
-  } catch (err) {
-    throw new Error(
-      `distillation student returned non-JSON for scenario '${scenarioId}': ${err instanceof Error ? err.message : String(err)} \u2014 raw: ${stripped.slice(0, 200)}`
-    );
-  }
-}
-function stripFence(text) {
-  const fenced = /```(?:json)?\s*([\s\S]*?)\s*```/.exec(text);
-  return fenced ? fenced[1] ?? text : text;
-}
-function stableStringify(value) {
-  return JSON.stringify(value, replacerSortKeys(), 2);
-}
-function replacerSortKeys() {
-  return (_key, value) => {
-    if (value && typeof value === "object" && !Array.isArray(value)) {
-      const sorted = {};
-      for (const k of Object.keys(value).sort()) {
-        sorted[k] = value[k];
-      }
-      return sorted;
-    }
-    return value;
-  };
-}
-
 // src/profile/index.ts
 var profile_exports = {};
 __export(profile_exports, {
@@ -10615,6 +10186,139 @@ function sectionHash(section) {
   return surfaceContentHash(JSON.stringify({ title: section.title, body: section.body }));
 }
 
+// src/traced-analyst.ts
+async function tracedAnalyzeTraces(input, options, traceOpts) {
+  const parentSpan = await traceOpts.emitter.span({
+    kind: "custom",
+    name: "analyst:analyze-traces",
+    parentSpanId: traceOpts.parentSpanId,
+    attributes: {
+      "analyst.question_length": input.question.length,
+      "analyst.max_turns": options.maxTurns ?? 12,
+      "analyst.max_subqueries": options.maxSubqueries ?? 4,
+      "eval.phase": "analyst"
+    }
+  });
+  const originalOnTurn = options.onTurn;
+  const wrappedOptions = {
+    ...options,
+    onTurn: async (turn) => {
+      const turnSpan = await traceOpts.emitter.span({
+        kind: "custom",
+        name: `analyst:turn-${turn.turn}`,
+        parentSpanId: parentSpan.span.spanId,
+        attributes: {
+          "analyst.stage": turn.stage,
+          "analyst.turn": turn.turn,
+          "analyst.is_error": turn.isError,
+          "analyst.code_length": turn.code.length,
+          "analyst.output_length": turn.output.length,
+          "eval.phase": "analyst"
+        }
+      });
+      if (turn.isError) {
+        await turnSpan.fail("Turn produced an error");
+      } else {
+        await turnSpan.end();
+      }
+      if (originalOnTurn) await originalOnTurn(turn);
+    }
+  };
+  try {
+    const result = await analyzeTraces(input, wrappedOptions);
+    await parentSpan.end({
+      attributes: {
+        "analyst.question_length": input.question.length,
+        "analyst.turn_count": result.turnCount,
+        "analyst.finding_count": result.findings.length,
+        "analyst.answer_length": result.answer.length,
+        "eval.phase": "analyst"
+      }
+    });
+    return result;
+  } catch (err) {
+    await parentSpan.fail(err instanceof Error ? err : String(err));
+    throw err;
+  }
+}
+
+// src/traced-judges.ts
+function traceJudge(judge, judgeName, opts) {
+  return async (tc, input) => {
+    const span = await opts.emitter.span({
+      kind: "llm",
+      name: `judge:${judgeName}`,
+      parentSpanId: opts.parentSpanId,
+      attributes: {
+        "judge.name": judgeName,
+        "eval.phase": "judge"
+      }
+    });
+    try {
+      const scores2 = await judge(tc, input);
+      const composite = scores2.length > 0 ? scores2.reduce((sum4, s) => sum4 + s.score, 0) / scores2.length : 0;
+      await span.end({
+        attributes: {
+          "judge.name": judgeName,
+          "judge.composite_score": composite,
+          "judge.dimension_count": scores2.length,
+          "eval.phase": "judge"
+        }
+      });
+      return scores2;
+    } catch (err) {
+      await span.fail(err instanceof Error ? err : String(err));
+      throw err;
+    }
+  };
+}
+function traceJudgeEnsemble(judges, judgeNames, opts) {
+  return async (tc, input) => {
+    const ensembleSpan = await opts.emitter.span({
+      kind: "custom",
+      name: "judge:ensemble",
+      parentSpanId: opts.parentSpanId,
+      attributes: {
+        "judge.ensemble_size": judges.length,
+        "eval.phase": "judge"
+      }
+    });
+    try {
+      const allScores = [];
+      let failedJudges = 0;
+      for (let i = 0; i < judges.length; i++) {
+        const judge = judges[i];
+        const name = judgeNames[i] ?? `judge_${i}`;
+        const tracedFn = traceJudge(judge, name, {
+          emitter: opts.emitter,
+          parentSpanId: ensembleSpan.span.spanId
+        });
+        try {
+          const scores2 = await tracedFn(tc, input);
+          allScores.push(...scores2);
+        } catch (err) {
+          if (!(err instanceof JudgeParseError)) throw err;
+          failedJudges++;
+        }
+      }
+      const composite = allScores.length > 0 ? allScores.reduce((sum4, s) => sum4 + s.score, 0) / allScores.length : 0;
+      await ensembleSpan.end({
+        attributes: {
+          "judge.ensemble_size": judges.length,
+          "judge.composite_score": composite,
+          "judge.total_dimensions": allScores.length,
+          "judge.failed_judges": failedJudges,
+          "eval.phase": "judge"
+        }
+      });
+      return allScores;
+    } catch (err) {
+      await ensembleSpan.fail(err instanceof Error ? err : String(err));
+      throw err;
+    }
+  };
+}
+
 // src/cost-report.ts
 function costReport(ledger) {
   const summary = ledger.summary();
@@ -10735,13 +10439,13 @@ function verifyAttestation(report, attested) {
 }
 
 // src/product-benchmark/index.ts
-import { existsSync as existsSync6, readFileSync as readFileSync7, statSync as statSync3 } from "fs";
+import { existsSync as existsSync6, readFileSync as readFileSync6, statSync as statSync3 } from "fs";
 import { dirname as dirname4, join as join5 } from "path";
 
 // src/product-benchmark/export.ts
 import { spawnSync as spawnSync2 } from "child_process";
 import { createHash } from "crypto";
-import { cpSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readFileSync as readFileSync6, writeFileSync } from "fs";
+import { cpSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readFileSync as readFileSync5, writeFileSync } from "fs";
 import { basename as basename2, dirname as dirname3, isAbsolute, join as join4, relative, resolve } from "path";
 var productBenchmarkMutableSurfaces = [
   "prompt",
@@ -10789,19 +10493,19 @@ function productBenchmarkRepoIdentity() {
   };
 }
 function packageVersion(name) {
-  const pkg = JSON.parse(readFileSync6(resolve("package.json"), "utf8"));
+  const pkg = JSON.parse(readFileSync5(resolve("package.json"), "utf8"));
   if (pkg.name === name && pkg.version) return pkg.version;
   const declared = pkg.dependencies?.[name] ?? pkg.devDependencies?.[name];
   if (declared) return declared;
   const installed = resolve("node_modules", name, "package.json");
   if (existsSync5(installed)) {
-    const installedPkg = JSON.parse(readFileSync6(installed, "utf8"));
+    const installedPkg = JSON.parse(readFileSync5(installed, "utf8"));
     if (installedPkg.version) return installedPkg.version;
   }
   return "unknown";
 }
 function readRunRecords(path) {
-  const lines = readFileSync6(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
+  const lines = readFileSync5(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
   return lines.map((line, index) => {
     let parsed;
     try {
@@ -11452,7 +11156,7 @@ function productBenchmarkIntegrityFailures(record) {
   return failures;
 }
 function readProductBenchmarkRecords(path) {
-  const lines = readFileSync7(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
+  const lines = readFileSync6(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
   const records = [];
   for (const [index, line] of lines.entries()) {
     try {
@@ -11465,7 +11169,7 @@ function readProductBenchmarkRecords(path) {
 }
 function readProductBenchmarkManifest(path) {
   try {
-    return validateProductBenchmarkManifest(JSON.parse(readFileSync7(path, "utf8")));
+    return validateProductBenchmarkManifest(JSON.parse(readFileSync6(path, "utf8")));
   } catch (err) {
     wrapValidationError(path, err);
   }
@@ -11668,11 +11372,7 @@ export {
   OTEL_AGENT_EVAL_SCOPE,
   OUTPUT_VALUE,
   OtlpFileTraceStore,
-  POLICY_EDIT_AXES,
-  POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
-  POLICY_EDIT_TARGET_SURFACES,
   PairwiseSteeringOptimizer,
-  PolicyEditValidationError,
   ProductClient,
   PromptRegistry,
   REDACTION_VERSION,
@@ -11718,7 +11418,6 @@ export {
   ValidationError,
   VerificationError,
   acquisitionPlansForKnowledgeGaps,
-  admitPolicyEdit,
   adversarialJudge,
   agentProfileCellHashMaterial,
   agentProfileCellKey,
@@ -11738,7 +11437,6 @@ export {
   analyzeTraces,
   appendScorecard,
   applyLlmSpanOtlpAttributes,
-  applyPolicyEditToSurface,
   applyToolSpanOtlpAttributes,
   argHash,
   asNumber,
@@ -11772,7 +11470,6 @@ export {
   bootstrapCi,
   buildAgentInterfaceProfileCell,
   buildAgentProfileCell,
-  buildAgreementJudge,
   buildDefaultAnalystRegistry,
   buildDriverSystemPrompt,
   buildProductBenchmarkManifest,
@@ -11820,7 +11517,6 @@ export {
   composeValidators,
   computeExperimentStats,
   computeFindingId,
-  computePolicyEditId,
   computeToolUseMetrics,
   computeTraceMetrics,
   confidenceInterval,
@@ -11867,10 +11563,8 @@ export {
   defaultBlendWeights,
   defaultIsMaterial,
   defaultJudges,
-  defaultParseStudentLabel,
   defaultProviderRedactor,
   defaultReferenceReplayMatcher,
-  defaultRenderStudentPrompt,
   defaultTraceInsightPanel,
   deployGateLayer,
   describeTraceInsightScope,
@@ -11910,7 +11604,6 @@ export {
   feedbackTrajectoriesToOptimizerRows,
   feedbackTrajectoryToDatasetScenario,
   feedbackTrajectoryToOptimizerRow,
-  fieldAgreement,
   fileContains,
   fileExists,
   fileExperimentStore,
@@ -11968,7 +11661,6 @@ export {
   isLlmSpan,
   isModelPriced,
   isOtelConfigured,
-  isPolicyEdit,
   isRetrievalSpan,
   isRolloutLine,
   isRunRecord,
@@ -11994,15 +11686,12 @@ export {
   llmJudge,
   llmSpanFromProvider,
   llmSpans,
-  loadGoldScenarios,
   loadScorecard,
   loadScorerFromGrader,
   localCommandRunner,
   lowercaseMutator,
   makeEvalTools,
   makeFinding,
-  makePolicyEdit,
-  makePolicyEditCandidateRecord,
   mannWhitneyU,
   mapConcurrent,
   matchGoldens,
@@ -12043,7 +11732,6 @@ export {
   paretoFrontierWithCrowding,
   parseCorrectnessResponse,
   parseFeedbackTrajectoriesJsonl,
-  parseGoldJsonl,
   parseReflectionResponse,
   parseRunRecordSafe,
   parseRuntimeTrajectoryHookEvent,
@@ -12054,8 +11742,6 @@ export {
   pearsonR,
   pixelDeltaRatio,
   planTraceInsightQuestions,
-  policyEditFromFinding,
-  policyEditsFromFindings,
   politenessPrefixMutator,
   positionalBias,
   preflightModels,
@@ -12118,7 +11804,6 @@ export {
   runBehavioralCanaries,
   runCanaries,
   runCounterfactual,
-  runDistillation,
   runE2EWorkflow,
   runEvalCampaign,
   runExpectations,
@@ -12144,7 +11829,6 @@ export {
   scoreContinuity,
   scoreFromEvals,
   scoreKnowledgeReadiness,
-  scorePolicyEditReadiness,
   scorePrReviewComments,
   scorePrReviewSource,
   scoreRedTeamOutput,
@@ -12159,7 +11843,6 @@ export {
   showMeasured,
   signManifest,
   spearmanR,
-  splitGold,
   statusAdvanced,
   stopOnNoProgress,
   stopOnRepeatedAction,
@@ -12197,8 +11880,6 @@ export {
   urlContains,
   userQuestionsForKnowledgeGaps,
   validateAgentProfileCell,
-  validatePolicyEdit,
-  validatePolicyEditCandidateRecord,
   validateProductBenchmarkManifest,
   validateProductBenchmarkRecord,
   validateProductBenchmarkRun,