@tangle-network/agent-eval 0.108.1 → 0.110.0

package/dist/index.js

@@ -9,52 +9,34 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-GDZAWO2I.js";
-import {
-  classifyEuAiRisk,
-  euAiActReport,
-  nistAiRmfReport,
-  renderMarkdown,
-  soc2Report,
-  summarize
-} from "./chunk-KKHDIONI.js";
-import {
-  acquisitionPlansForKnowledgeGaps,
-  blockingKnowledgeEval,
-  knowledgeReadinessTracePayload,
-  scoreKnowledgeReadiness,
-  userQuestionsForKnowledgeGaps
-} from "./chunk-Q2JRAWRI.js";
-import {
-  assertRecordIntegrity,
-  checkRecordIntegrity,
-  expandMatrix,
-  gatePerf,
-  scenarioKey,
-  summarizeRecords
-} from "./chunk-STGVSCDH.js";
-import {
-  assertProductBenchmarkRun,
-  buildProductBenchmarkManifest,
-  exportProductBenchmark,
-  exportProductBenchmarkRuns,
-  findProductBenchmarkArtifacts,
-  productBenchmarkIntegrityFailures,
-  productBenchmarkMutableSurfaces,
-  productBenchmarkRepoIdentity,
-  productBenchmarkSplits,
-  readProductBenchmarkManifest,
-  readProductBenchmarkRecords,
-  runRecordToProductBenchmarkRecord,
-  validateProductBenchmarkManifest,
-  validateProductBenchmarkRecord,
-  validateProductBenchmarkRun
-} from "./chunk-FOUG2VVS.js";
+} from "./chunk-QFGTU7MT.js";
 import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
   deterministicSplit
 } from "./chunk-T6W5ADLG.js";
+import {
+  DEFAULT_RULES,
+  buildTrajectory,
+  classifyFailure,
+  compareToBaseline,
+  computeToolUseMetrics,
+  iqr,
+  welchsTTest
+} from "./chunk-PLOMR3HP.js";
+import {
+  analyzeSeries
+} from "./chunk-BOD4O7OF.js";
+import {
+  DockerSandboxDriver,
+  SandboxHarness,
+  SubprocessSandboxDriver,
+  composeParsers,
+  jestTestParser,
+  pytestTestParser,
+  runTestGradedScenario,
+  vitestTestParser
+} from "./chunk-HZHNRYHK.js";
 import {
   CODING_HARNESSES,
   HARNESS_NATIVE_MODEL,
@@ -77,7 +59,7 @@ import {
   llmJudge,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-6SKVFBTR.js";
+} from "./chunk-RNB2NICW.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -101,17 +83,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-OVPVM4JC.js";
-import {
-  BackendIntegrityError,
-  assertRealBackend,
-  cachedJudge,
-  canonicalJson,
-  contentHash,
-  fileVerdictCache,
-  inMemoryVerdictCache,
-  summarizeBackendIntegrity
-} from "./chunk-3PFZBGMR.js";
+} from "./chunk-GTERJI6Q.js";
 import {
   MODEL_PRICING,
   MetricsCollector,
@@ -122,33 +94,20 @@ import {
   resolveModelPricing
 } from "./chunk-VI2UW6B6.js";
 import {
-  DEFAULT_RULES,
-  classifyFailure,
-  compareToBaseline,
-  computeToolUseMetrics,
-  iqr,
-  welchsTTest
-} from "./chunk-DBDRR6GF.js";
-import {
-  analyzeSeries
-} from "./chunk-BOD4O7OF.js";
-import {
-  exportTrainingData,
-  toNdjson
-} from "./chunk-KMPRBJK4.js";
-import {
-  DockerSandboxDriver,
-  SandboxHarness,
-  SubprocessSandboxDriver,
-  composeParsers,
-  jestTestParser,
-  pytestTestParser,
-  runTestGradedScenario,
-  vitestTestParser
-} from "./chunk-HZHNRYHK.js";
+  BackendIntegrityError,
+  assertRealBackend,
+  cachedJudge,
+  canonicalJson,
+  contentHash,
+  fileVerdictCache,
+  inMemoryVerdictCache,
+  summarizeBackendIntegrity
+} from "./chunk-3PFZBGMR.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   FindingsStore,
+  LockedJsonlAppender,
+  Mutex,
   RunCritic,
   SEMANTIC_CONCEPT_JUDGE_VERSION,
   SKILL_USAGE_ANALYST,
@@ -159,15 +118,11 @@ import {
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-5PK3626Q.js";
-import {
-  LockedJsonlAppender,
-  Mutex
-} from "./chunk-DJWX3GVS.js";
+} from "./chunk-XRGOKCMO.js";
 import {
   buildDefaultAnalystRegistry,
   computeTraceMetrics
-} from "./chunk-QRVS7MX4.js";
+} from "./chunk-OW47B5WA.js";
 import {
   DEFAULT_RUN_SCORE_WEIGHTS,
   POLICY_EDIT_AXES,
@@ -184,7 +139,7 @@ import {
   policyEditsFromFindings,
   scorePolicyEditReadiness,
   validatePolicyEdit
-} from "./chunk-LOJ2QVCE.js";
+} from "./chunk-2IY4ILP4.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -192,32 +147,32 @@ import {
   IMPROVEMENT_KIND_SPEC,
   KNOWLEDGE_GAP_KIND_SPEC,
   KNOWLEDGE_POISONING_KIND_SPEC,
+  computeFindingId,
   createTraceAnalystKind,
+  makeFinding,
   renderPriorFindings
-} from "./chunk-2OGPXHOB.js";
+} from "./chunk-7NX6ZSBG.js";
 import {
+  allCriticalPassed,
   controlFailureClassFromVerification,
   controlRunToRunRecord,
   createLlmReviewer,
+  errorStreakDetector,
   evaluateActionPolicy,
   inMemoryReviewStore,
   jsonlReviewStore,
-  runProposeReview,
-  runProposeReviewAsControlLoop,
-  scoreFromEvals
-} from "./chunk-K7QEIHHJ.js";
-import {
-  allCriticalPassed,
-  errorStreakDetector,
   noProgressDetector,
   objectiveEval,
   observeAll,
   repeatedActionDetector,
   runAgentControlLoop,
+  runProposeReview,
+  runProposeReviewAsControlLoop,
+  scoreFromEvals,
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-YEHAEDUD.js";
+} from "./chunk-IMWDSFUM.js";
 import {
   assertReleaseConfidence,
   bootstrapCi,
@@ -227,20 +182,8 @@ import {
 } from "./chunk-6HFYGEZ3.js";
 import {
   runEvalCampaign
-} from "./chunk-LIEJUH2I.js";
+} from "./chunk-6PL5MGDL.js";
 import "./chunk-N22ZO7FV.js";
-import {
-  LlmCallError,
-  LlmClient,
-  LlmRouteAssertionError,
-  assertLlmRoute,
-  backoffMs,
-  callLlm,
-  callLlmJson,
-  isTransientLlmError,
-  probeLlm,
-  stripFencedJson
-} from "./chunk-FUCQVFMU.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -252,17 +195,6 @@ import {
   researchReport,
   summaryTable
 } from "./chunk-6MFFKPZ4.js";
-import {
-  attributeCounterfactuals,
-  runCounterfactual
-} from "./chunk-6SK5VFYK.js";
-import {
-  buildTrajectory
-} from "./chunk-RZTMDUO7.js";
-import {
-  computeFindingId,
-  makeFinding
-} from "./chunk-45EEMHTC.js";
 import {
   benjaminiHochberg,
   bonferroni,
@@ -331,7 +263,24 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-V7HNA47Z.js";
+} from "./chunk-RSVSSZKF.js";
+import {
+  FAILURE_CLASSES,
+  TRACE_SCHEMA_VERSION,
+  aggregateLlm,
+  argHash,
+  groupBy,
+  isJudgeSpan,
+  isLlmSpan,
+  isRetrievalSpan,
+  isSandboxSpan,
+  isToolSpan,
+  judgeSpans,
+  llmSpans,
+  runFailureClass,
+  runsForScenario,
+  toolSpans
+} from "./chunk-MHNQWM4I.js";
 import {
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
@@ -344,25 +293,6 @@ import {
   redactString,
   redactValue
 } from "./chunk-GGE4NNQT.js";
-import {
-  aggregateLlm,
-  argHash,
-  groupBy,
-  judgeSpans,
-  llmSpans,
-  runFailureClass,
-  runsForScenario,
-  toolSpans
-} from "./chunk-JZXGWLK5.js";
-import {
-  FAILURE_CLASSES,
-  TRACE_SCHEMA_VERSION,
-  isJudgeSpan,
-  isLlmSpan,
-  isRetrievalSpan,
-  isSandboxSpan,
-  isToolSpan
-} from "./chunk-5BKGXME7.js";
 import {
   DEFAULT_TRACE_ANALYST_BUDGETS,
   LLM_CACHED_TOKENS,
@@ -403,12 +333,9 @@ import {
   throwIfRunIncomplete
 } from "./chunk-TT4KNT67.js";
 import {
-  FileSystemRawProviderSink,
-  InMemoryRawProviderSink,
-  NoopRawProviderSink,
-  defaultProviderRedactor,
-  providerFromBaseUrl
-} from "./chunk-PC4UYEBM.js";
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-TVVP3ZZQ.js";
 import {
   RunRecordValidationError,
   isRunRecord,
@@ -417,10 +344,6 @@ import {
   roundTripRunRecord,
   validateRunRecord
 } from "./chunk-VK6HBGAE.js";
-import {
-  TraceEmitter,
-  llmSpanFromProvider
-} from "./chunk-TVVP3ZZQ.js";
 import {
   AGENT_PROFILE_KINDS,
   AgentProfileCellValidationError,
@@ -442,6 +365,25 @@ import {
   signManifest,
   verifyManifest
 } from "./chunk-VSMTAMNK.js";
+import {
+  LlmCallError,
+  LlmClient,
+  LlmRouteAssertionError,
+  assertLlmRoute,
+  backoffMs,
+  callLlm,
+  callLlmJson,
+  isTransientLlmError,
+  probeLlm,
+  stripFencedJson
+} from "./chunk-FUCQVFMU.js";
+import {
+  FileSystemRawProviderSink,
+  InMemoryRawProviderSink,
+  NoopRawProviderSink,
+  defaultProviderRedactor,
+  providerFromBaseUrl
+} from "./chunk-PC4UYEBM.js";
 import {
   AgentEvalError,
   CaptureIntegrityError,
@@ -654,12 +596,12 @@ function ghCliClient(opts = {}) {
       await exec("git", ["branch", "-D", input.branchName], { cwd });
       await run("git", ["checkout", "-b", input.branchName]);
       const { mkdir, writeFile } = await import("fs/promises");
-      const { dirname: dirname3, join: join4, resolve } = await import("path");
+      const { dirname: dirname5, join: join6, resolve: resolve2 } = await import("path");
       for (const change of input.fileChanges) {
-        const abs = resolve(cwd, change.path);
-        await mkdir(dirname3(abs), { recursive: true });
+        const abs = resolve2(cwd, change.path);
+        await mkdir(dirname5(abs), { recursive: true });
         await writeFile(abs, change.contents, "utf8");
-        await run("git", ["add", join4(change.path)]);
+        await run("git", ["add", join6(change.path)]);
       }
       const env = {};
       if (input.authorName) env.GIT_AUTHOR_NAME = input.authorName;
@@ -1121,7 +1063,7 @@ function capabilityHeadroom(rows, opts = {}) {
     tasksWithGap: tasks.filter((t) => t.headroom === "gap").length,
     tasksSaturated: tasks.filter((t) => t.headroom === "saturated").length,
     tasksUnknown: tasks.filter((t) => t.headroom === "unknown").length,
-    repsUnknown: tasks.reduce((sum3, t) => sum3 + (t.n - t.nKnown), 0)
+    repsUnknown: tasks.reduce((sum4, t) => sum4 + (t.n - t.nKnown), 0)
   };
   return { tasks, summary };
 }
@@ -1711,10 +1653,10 @@ var FileSystemFeedbackTrajectoryStore = class {
   }
   async append(record) {
     const { appendFile, mkdir } = await import("fs/promises");
-    const { join: join4 } = await import("path");
+    const { join: join6 } = await import("path");
     await mkdir(this.dir, { recursive: true });
     await appendFile(
-      join4(this.dir, "feedback-trajectories.ndjson"),
+      join6(this.dir, "feedback-trajectories.ndjson"),
       `${JSON.stringify(record)}
 `,
       "utf8"
@@ -1723,8 +1665,8 @@ var FileSystemFeedbackTrajectoryStore = class {
   async load() {
     if (this.loaded) return;
     const { readFile: readFile2 } = await import("fs/promises");
-    const { join: join4 } = await import("path");
-    const file = join4(this.dir, "feedback-trajectories.ndjson");
+    const { join: join6 } = await import("path");
+    const file = join6(this.dir, "feedback-trajectories.ndjson");
     try {
       const raw = await readFile2(file, "utf8");
       for (const line of raw.split("\n")) {
@@ -1971,7 +1913,7 @@ function scoreFromLabels(labels) {
     return void 0;
   }).filter((value) => typeof value === "number");
   if (!scored.length) return void 0;
-  return Math.round(scored.reduce((sum3, value) => sum3 + value, 0) / scored.length * 1e3) / 1e3;
+  return Math.round(scored.reduce((sum4, value) => sum4 + value, 0) / scored.length * 1e3) / 1e3;
 }
 function instructionFromLabel(trajectory, label) {
   if (label.kind === "reject" && label.reason)
@@ -2260,6 +2202,190 @@ function assertCrossFamily(models, opts = {}) {
   return list;
 }
 
+// src/knowledge/readiness.ts
+function scoreKnowledgeReadiness(options) {
+  const now = options.now ?? /* @__PURE__ */ new Date();
+  const requirements = options.requirements.map(normalizeRequirement);
+  const missing = requirements.filter((requirement) => isRequirementMissing(requirement, now));
+  const blockingMissingRequirements = missing.filter(isBlockingGap);
+  const nonBlockingGaps = missing.filter((requirement) => !isBlockingGap(requirement));
+  const readinessScore = weightedReadinessAt(requirements, now);
+  const bundle = {
+    taskId: options.taskId,
+    requirements,
+    evidenceIds: unique([
+      ...options.evidenceIds ?? [],
+      ...requirements.flatMap((r) => r.evidenceIds)
+    ]),
+    claimIds: unique(options.claimIds ?? []),
+    wikiPageIds: unique(options.wikiPageIds ?? []),
+    userAnswers: options.userAnswers ?? {},
+    missing,
+    readinessScore,
+    metadata: options.metadata
+  };
+  const recommendedAction = chooseRecommendedAction(blockingMissingRequirements, nonBlockingGaps);
+  const severity = blockingMissingRequirements.length > 0 ? "critical" : nonBlockingGaps.some((gap) => gap.importance === "high") ? "warning" : "info";
+  const reason = blockingMissingRequirements.length > 0 ? `${blockingMissingRequirements.length} blocking knowledge requirement(s) are missing.` : nonBlockingGaps.length > 0 ? `${nonBlockingGaps.length} non-blocking knowledge gap(s) remain.` : "All declared knowledge requirements are ready.";
+  return {
+    taskId: options.taskId,
+    readinessScore,
+    blockingMissingRequirements,
+    nonBlockingGaps,
+    recommendedAction,
+    bundle,
+    severity,
+    reason
+  };
+}
+function blockingKnowledgeEval(report, options = {}) {
+  const minimumScore = options.minimumScore ?? 0.7;
+  const passed = report.blockingMissingRequirements.length === 0 && report.readinessScore >= minimumScore;
+  if (options.emitter) {
+    void options.emitter.emit({
+      kind: "custom",
+      payload: knowledgeReadinessTracePayload(report, { passed, minimumScore })
+    }).catch(() => void 0);
+  }
+  return objectiveEval({
+    id: options.id ?? "knowledge-ready",
+    passed,
+    score: report.readinessScore,
+    severity: passed ? "info" : report.severity,
+    detail: report.reason,
+    evidence: report.blockingMissingRequirements.map((r) => r.id).join(", ") || void 0,
+    metadata: { knowledgeReadiness: report }
+  });
+}
+function knowledgeReadinessTracePayload(report, options = {}) {
+  return {
+    kind: "readiness_scored",
+    taskId: report.taskId,
+    passed: options.passed ?? report.blockingMissingRequirements.length === 0,
+    readinessScore: report.readinessScore,
+    minimumScore: options.minimumScore,
+    blockingRequirementIds: report.blockingMissingRequirements.map((r) => r.id),
+    nonBlockingRequirementIds: report.nonBlockingGaps.map((r) => r.id),
+    recommendedAction: report.recommendedAction,
+    severity: report.severity,
+    reason: report.reason
+  };
+}
+function userQuestionsForKnowledgeGaps(gaps) {
+  return gaps.filter((gap) => gap.acquisitionMode === "ask_user" || gap.fallbackPolicy === "ask").map((gap) => ({
+    id: `question_${gap.id}`,
+    question: `Please provide: ${gap.description}`,
+    reason: `Required for ${gap.requiredFor.join(", ") || "the task"}.`,
+    requirementId: gap.id,
+    importance: gap.importance,
+    answerType: gap.sensitivity === "secret" ? "credential" : "free_text",
+    impactIfUnknown: impactFor(gap)
+  }));
+}
+function acquisitionPlansForKnowledgeGaps(gaps) {
+  const byMode = /* @__PURE__ */ new Map();
+  for (const gap of gaps) {
+    const mode = planMode(gap.acquisitionMode);
+    if (!mode) continue;
+    const bucket = byMode.get(mode) ?? [];
+    bucket.push(gap);
+    byMode.set(mode, bucket);
+  }
+  return [...byMode.entries()].map(([mode, requirements]) => ({
+    id: `acquire_${mode}`,
+    requirementIds: requirements.map((r) => r.id),
+    mode,
+    description: descriptionForPlan(mode, requirements),
+    priority: maxImportance(requirements.map((r) => r.importance)),
+    questions: mode === "ask_user" ? userQuestionsForKnowledgeGaps(requirements) : void 0
+  }));
+}
+function normalizeRequirement(requirement) {
+  return {
+    ...requirement,
+    confidenceNeeded: clamp012(requirement.confidenceNeeded),
+    currentConfidence: clamp012(requirement.currentConfidence),
+    evidenceIds: unique(requirement.evidenceIds)
+  };
+}
+function weightedReadinessAt(requirements, now) {
+  if (requirements.length === 0) return 1;
+  let weightSum = 0;
+  let scoreSum = 0;
+  for (const requirement of requirements) {
+    const weight = importanceWeight(requirement.importance);
+    const score = isExpired(requirement, now) ? 0 : requirement.confidenceNeeded <= 0 ? 1 : Math.min(1, requirement.currentConfidence / requirement.confidenceNeeded);
+    weightSum += weight;
+    scoreSum += weight * score;
+  }
+  return clamp012(scoreSum / weightSum);
+}
+function isRequirementMissing(requirement, now) {
+  return isExpired(requirement, now) || requirement.currentConfidence < requirement.confidenceNeeded;
+}
+function isExpired(requirement, now) {
+  if (!requirement.validUntil) return false;
+  const deadline = Date.parse(requirement.validUntil);
+  if (!Number.isFinite(deadline)) return true;
+  return deadline <= now.getTime();
+}
+function isBlockingGap(requirement) {
+  return requirement.importance === "blocking" || requirement.fallbackPolicy === "block" || requirement.sensitivity === "secret";
+}
+function chooseRecommendedAction(blocking, nonBlocking) {
+  const gaps = blocking.length > 0 ? blocking : nonBlocking;
+  if (gaps.length === 0) return "run_agent";
+  if (gaps.some((gap) => gap.acquisitionMode === "ask_user" || gap.fallbackPolicy === "ask"))
+    return "ask_user";
+  if (gaps.some((gap) => gap.acquisitionMode === "query_connector")) return "query_connectors";
+  if (gaps.some(
+    (gap) => gap.acquisitionMode === "inspect_repo" || gap.acquisitionMode === "run_command"
+  ))
+    return "inspect_repo";
+  if (gaps.some((gap) => gap.acquisitionMode === "search_web")) return "collect_web_data";
+  if (gaps.some((gap) => gap.acquisitionMode === "not_available")) return "abort_or_rescope";
+  if (nonBlocking.some((gap) => gap.importance === "high")) return "build_domain_wiki";
+  return "continue_with_caveat";
+}
+function planMode(mode) {
+  if (mode === "infer_low_confidence" || mode === "not_available") return null;
+  return mode;
+}
+function descriptionForPlan(mode, requirements) {
+  const labels = requirements.map((r) => r.description).join("; ");
+  if (mode === "ask_user") return `Ask the user for: ${labels}`;
+  if (mode === "search_web") return `Search web or documentation sources for: ${labels}`;
+  if (mode === "query_connector") return `Query configured connectors for: ${labels}`;
+  if (mode === "inspect_repo") return `Inspect repository context for: ${labels}`;
+  if (mode === "run_command") return `Run local commands to collect: ${labels}`;
+  return `Build domain wiki evidence for: ${labels}`;
+}
+function impactFor(requirement) {
+  if (requirement.fallbackPolicy === "block") return "The agent should not run until this is known.";
+  if (requirement.fallbackPolicy === "continue_with_caveat")
+    return "The agent may continue, but must disclose uncertainty.";
+  if (requirement.fallbackPolicy === "use_default")
+    return "The agent will use the configured default if skipped.";
+  return "The agent should ask before continuing.";
+}
+function maxImportance(values) {
+  const order = ["blocking", "high", "medium", "low"];
+  return order.find((value) => values.includes(value)) ?? "low";
+}
+function importanceWeight(importance) {
+  if (importance === "blocking") return 8;
+  if (importance === "high") return 4;
+  if (importance === "medium") return 2;
+  return 1;
+}
+function clamp012(value) {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+function unique(items) {
+  return [...new Set(items)];
+}
+
 // src/live-proof.ts
 async function runLiveProof(config) {
   const startedAt = Date.now();
@@ -3520,7 +3646,7 @@ async function mapLimit(items, limit, fn) {
   return results;
 }
 function mean2(values) {
-  return values.length ? values.reduce((sum3, value) => sum3 + value, 0) / values.length : 0;
+  return values.length ? values.reduce((sum4, value) => sum4 + value, 0) / values.length : 0;
 }
 function meanRunScore(scores2) {
   return {
@@ -3612,7 +3738,7 @@ function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
 var DEFAULT_MAX_ATTEMPTS = 3;
 var DEFAULT_TIMEOUT_MS = 3e5;
 function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
 }
 async function withJudgeRetry(judgeFn, policy = {}) {
   const maxAttempts = policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
@@ -4041,7 +4167,7 @@ function rankRows(rows, weights) {
   }
   return [...buckets.entries()].map(([variantId, values]) => ({
     variantId,
-    mean: values.reduce((sum3, value) => sum3 + value, 0) / values.length,
+    mean: values.reduce((sum4, value) => sum4 + value, 0) / values.length,
     runs: values.length
   })).sort((a, b) => b.mean - a.mean);
 }
@@ -4295,13 +4421,13 @@ var defaultBlendWeights = { heldout: 0.7, judge: 0.3 };
 function normalizeWeights(weights) {
   const h = Number.isFinite(weights.heldout) && weights.heldout >= 0 ? weights.heldout : 0;
   const j = Number.isFinite(weights.judge) && weights.judge >= 0 ? weights.judge : 0;
-  const sum3 = h + j;
-  if (sum3 <= 0) {
+  const sum4 = h + j;
+  if (sum4 <= 0) {
     throw new ValidationError(
       "blend weights must have a positive sum (got heldout+judge <= 0) \u2014 cannot weight a composite by zero"
     );
   }
-  return { heldout: h / sum3, judge: j / sum3 };
+  return { heldout: h / sum4, judge: j / sum4 };
 }
 function blendHeldout(heldoutPassRate, judgeScore, weights = defaultBlendWeights) {
   const w = normalizeWeights(weights);
@@ -6705,7 +6831,7 @@ async function commitBisect(options) {
 }
 async function promptBisect(options) {
   const split = options.paragraphSplitter ?? ((p) => p.split(/\n\s*\n/));
-  const join4 = (paragraphs) => paragraphs.join("\n\n");
+  const join6 = (paragraphs) => paragraphs.join("\n\n");
   const goodParas = split(options.good);
   const badParas = split(options.bad);
   if (goodParas.length !== badParas.length) {
@@ -6725,7 +6851,7 @@ async function promptBisect(options) {
   const result = await bisect({
     good: goodMask,
     bad: badMask,
-    runEval: (mask) => options.runEval(join4(paragraphsFor(mask))),
+    runEval: (mask) => options.runEval(join6(paragraphsFor(mask))),
     maxIterations: options.maxIterations ?? n + 5,
     halfway: (g, b) => {
       for (let i = 0; i < g.length; i++) {
@@ -6756,12 +6882,12 @@ async function promptBisect(options) {
     }
   }
   const materializedPath = result.path.map((s) => ({
-    state: join4(paragraphsFor(s.state)),
+    state: join6(paragraphsFor(s.state)),
     score: s.score,
     pass: s.pass
   }));
   return {
-    culprit: join4(paragraphsFor(culprit)),
+    culprit: join6(paragraphsFor(culprit)),
     path: materializedPath,
     converged: result.converged,
     inputInconsistent: result.inputInconsistent,
@@ -6769,6 +6895,90 @@ async function promptBisect(options) {
   };
 }
 
+// src/counterfactual.ts
+async function runCounterfactual(store, originalRunId, mutation, runner) {
+  const originalRun = await store.getRun(originalRunId);
+  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`);
+  const trajectory = await buildTrajectory(store, originalRunId);
+  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {
+    throw new ValidationError(
+      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`
+    );
+  }
+  const targetStep = trajectory.steps[mutation.at];
+  const mutatedStep = applyMutation(targetStep, mutation);
+  const cfEmitter = new TraceEmitter(store);
+  await cfEmitter.startRun({
+    scenarioId: originalRun.scenarioId,
+    variantId: originalRun.variantId ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}` : `cf:${mutation.kind}@${mutation.at}`,
+    projectId: originalRun.projectId,
+    parentRunId: originalRunId,
+    layer: "meta",
+    tags: { counterfactual: "true", mutationKind: mutation.kind, mutationAt: String(mutation.at) }
+  });
+  await runner.executeFrom(
+    {
+      originalRunId,
+      originalTrajectory: trajectory,
+      prefix: trajectory.steps.slice(0, mutation.at),
+      mutation,
+      mutatedStep
+    },
+    cfEmitter
+  );
+  const counterfactual = await store.getRun(cfEmitter.runId);
+  const delta = {
+    originalOutcomeScore: originalRun.outcome?.score ?? null,
+    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,
+    deltaScore: originalRun.outcome?.score !== void 0 && counterfactual?.outcome?.score !== void 0 ? counterfactual.outcome.score - originalRun.outcome.score : null
+  };
+  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta };
+}
+function applyMutation(step, mutation) {
+  if (mutation.kind === "swap-model" && step.span.kind === "llm") {
+    const llm = step.span;
+    return { ...step, span: { ...llm, model: mutation.newModel } };
+  }
+  if (mutation.kind === "swap-tool-result" && step.span.kind === "tool") {
+    const tool = step.span;
+    return { ...step, span: { ...tool, result: mutation.newResult } };
+  }
+  if (mutation.kind === "inject-system-message" && step.span.kind === "llm") {
+    const llm = step.span;
+    return {
+      ...step,
+      span: {
+        ...llm,
+        messages: [{ role: "system", content: mutation.content }, ...llm.messages]
+      }
+    };
+  }
+  if (mutation.kind === "custom") return mutation.apply(step);
+  return step;
+}
+function attributeCounterfactuals(results) {
+  const grouped = /* @__PURE__ */ new Map();
+  for (const r of results) {
+    const arr = grouped.get(r.mutation.kind) ?? [];
+    arr.push(r);
+    grouped.set(r.mutation.kind, arr);
+  }
+  const out = [];
+  for (const [kind, items] of grouped) {
+    const deltas = items.map((i) => i.delta.deltaScore).filter((d) => typeof d === "number");
+    if (deltas.length === 0) continue;
+    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length;
+    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length;
+    out.push({
+      mutationKind: kind,
+      n: deltas.length,
+      meanAbsDelta: meanAbs,
+      meanSignedDelta: meanSigned
+    });
+  }
+  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta);
+}
+
 // src/cross-trace-diff.ts
 async function crossTraceDiff(store, runA, runB, options = {}) {
   const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)]);
@@ -7051,6 +7261,67 @@ function groupBy2(items, key) {
   return m;
 }
 
+// src/prm/training-export.ts
+async function exportTrainingData(store, graded, options = {}) {
+  const window = options.contextWindow ?? 5;
+  const out = [];
+  for (const g of graded) {
+    const trajectory = await buildTrajectory(store, g.runId);
+    const spanById = new Map(trajectory.steps.map((s) => [s.span.spanId, s]));
+    for (const gs of g.steps) {
+      const node = spanById.get(gs.spanId);
+      if (!node) continue;
+      const idx = trajectory.steps.indexOf(node);
+      const priorSpans = trajectory.steps.slice(Math.max(0, idx - window), idx).map((s) => s.span);
+      out.push({
+        runId: g.runId,
+        spanId: gs.spanId,
+        rubricId: gs.rubricId,
+        score: gs.score,
+        context: {
+          priorTurns: priorSpans.map(spanToTurn).filter((t) => t !== null),
+          step: { kind: node.span.kind, text: spanToText(node.span) }
+        },
+        rationale: gs.rationale,
+        evidence: gs.evidence
+      });
+    }
+  }
+  return out;
+}
+function toNdjson(samples) {
+  return `${samples.map((s) => JSON.stringify(s)).join("\n")}
+`;
+}
+function spanToTurn(span) {
+  if (isLlmSpan(span)) {
+    const text = span.output ?? span.messages.map((m) => `${m.role}: ${m.content}`).join("\n");
+    return { role: "assistant", content: text };
+  }
+  if (isToolSpan(span)) {
+    return {
+      role: "tool",
+      content: `${span.toolName}(${safeStringify(span.args)}) \u2192 ${safeStringify(span.result)}`
+    };
+  }
+  return null;
+}
+function spanToText(span) {
+  if (isLlmSpan(span)) return span.output ?? "";
+  if (isToolSpan(span))
+    return `${span.toolName}(${safeStringify(span.args)}) \u2192 ${safeStringify(span.result)}`;
+  return span.name;
+}
+function safeStringify(v) {
+  if (v === null || v === void 0) return "";
+  if (typeof v === "string") return v;
+  try {
+    return JSON.stringify(v);
+  } catch {
+    return String(v);
+  }
+}
+
 // src/reward-model-export.ts
 async function exportRewardModel(store, grader, runIds) {
   const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)));
@@ -7481,7 +7752,7 @@ function extractErrorCount(text, opts = {}) {
   for (const p of patterns) {
     const matches2 = Array.from(text.matchAll(p.regex));
     if (matches2.length === 0) continue;
-    const count = p.transform ? matches2.reduce((sum3, m) => sum3 + p.transform(m), 0) : matches2.length;
+    const count = p.transform ? matches2.reduce((sum4, m) => sum4 + p.transform(m), 0) : matches2.length;
     return {
       count,
       matched: p.name,
@@ -8419,7 +8690,7 @@ function defaultReferenceReplayMatcher(reference, candidate) {
   const textScore = tokenJaccard(referenceText, candidateText);
   const severityScore = reference.severity && candidate.severity ? normalize(reference.severity) === normalize(candidate.severity) ? 0.1 : -0.05 : 0;
   const tagScore = tagOverlap(reference.tags, candidate.tags) * 0.15;
-  const score = clamp012(textScore * 0.85 + tagScore + severityScore);
+  const score = clamp013(textScore * 0.85 + tagScore + severityScore);
   return {
     score,
     reason: `token=${textScore.toFixed(2)} tags=${tagScore.toFixed(2)} severity=${severityScore.toFixed(2)}`
@@ -8525,13 +8796,13 @@ function scorePair(scenario, matcher, reference, candidate) {
       `reference replay matcher returned non-finite score for ${scenario.id}:${reference.id}:${candidate.id}`
     );
   }
-  return { score: clamp012(result.score), reason: result.reason ?? "" };
+  return { score: clamp013(result.score), reason: result.reason ?? "" };
 }
 function buildScenarioScore(scenario, matches2, falsePositives) {
   const matched = matches2.filter((match) => match.matched).length;
   const total = scenario.references.length;
-  const matchedWeight = matches2.filter((match) => match.matched).reduce((sum3, match) => sum3 + match.weight, 0);
-  const totalWeight = matches2.reduce((sum3, match) => sum3 + match.weight, 0);
+  const matchedWeight = matches2.filter((match) => match.matched).reduce((sum4, match) => sum4 + match.weight, 0);
+  const totalWeight = matches2.reduce((sum4, match) => sum4 + match.weight, 0);
   const precision2 = ratio(matched, matched + falsePositives);
   const recall = ratio(matched, total);
   return {
@@ -8624,7 +8895,7 @@ function tokens(text) {
 function normalize(text) {
   return text.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
 }
-function clamp012(value) {
+function clamp013(value) {
   if (!Number.isFinite(value)) return 0;
   return Math.max(0, Math.min(1, value));
 }
@@ -9428,8 +9699,8 @@ function createSandboxPool(opts) {
       });
       return state;
     }
-    return new Promise((resolve, reject) => {
-      waiters.push({ resolve, reject });
+    return new Promise((resolve2, reject) => {
+      waiters.push({ resolve: resolve2, reject });
     });
   }
   function handOffCleanSlot(state) {
@@ -9617,7 +9888,7 @@ function traceJudge(judge, judgeName, opts) {
     });
     try {
       const scores2 = await judge(tc, input);
-      const composite = scores2.length > 0 ? scores2.reduce((sum3, s) => sum3 + s.score, 0) / scores2.length : 0;
+      const composite = scores2.length > 0 ? scores2.reduce((sum4, s) => sum4 + s.score, 0) / scores2.length : 0;
       await span.end({
         attributes: {
           "judge.name": judgeName,
@@ -9662,7 +9933,7 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
           failedJudges++;
         }
       }
-      const composite = allScores.length > 0 ? allScores.reduce((sum3, s) => sum3 + s.score, 0) / allScores.length : 0;
+      const composite = allScores.length > 0 ? allScores.reduce((sum4, s) => sum4 + s.score, 0) / allScores.length : 0;
       await ensembleSpan.end({
         attributes: {
           "judge.ensemble_size": judges.length,
@@ -10119,7 +10390,7 @@ function costReport(ledger) {
     perChannel: summary.byChannel,
     total: {
       usd: summary.totalCostUsd,
-      unknownEntries: summary.byChannel.reduce((sum3, c) => sum3 + c.unpricedCalls, 0)
+      unknownEntries: summary.byChannel.reduce((sum4, c) => sum4 + c.unpricedCalls, 0)
     },
     perModel: [...perModel.values()].sort((a, b) => a.model.localeCompare(b.model))
   };
@@ -10217,6 +10488,839 @@ function verifyAttestation(report, attested) {
   }
   return { valid: true };
 }
+
+// src/product-benchmark/index.ts
+import { existsSync as existsSync6, readFileSync as readFileSync7, statSync as statSync3 } from "fs";
+import { dirname as dirname4, join as join5 } from "path";
+
+// src/product-benchmark/export.ts
+import { spawnSync as spawnSync2 } from "child_process";
+import { createHash } from "crypto";
+import { cpSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readFileSync as readFileSync6, writeFileSync } from "fs";
+import { basename as basename2, dirname as dirname3, isAbsolute, join as join4, relative, resolve } from "path";
+var productBenchmarkMutableSurfaces = [
+  "prompt",
+  "resources.files",
+  "tools",
+  "mcp",
+  "hooks",
+  "subagents"
+];
+function sha256(value) {
+  return `sha256:${createHash("sha256").update(value).digest("hex")}`;
+}
+function safePathPart(value) {
+  return value.replace(/[^a-zA-Z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "artifact";
+}
+function git(args, fallback) {
+  const res = spawnSync2("git", args, { cwd: process.cwd(), encoding: "utf8" });
+  const value = res.status === 0 ? res.stdout.trim() : "";
+  return value.length > 0 ? value : fallback;
+}
+function knownEnv(name) {
+  const value = process.env[name]?.trim();
+  return value && value !== "unknown" ? value : void 0;
+}
+function ciBranchName() {
+  const direct = knownEnv("GITHUB_HEAD_REF") ?? knownEnv("GITHUB_REF_NAME") ?? knownEnv("VERCEL_GIT_COMMIT_REF");
+  if (direct) return direct;
+  const ref = knownEnv("GITHUB_REF");
+  return ref?.startsWith("refs/heads/") ? ref.slice("refs/heads/".length) : void 0;
+}
+function repoBranch() {
+  const ci = ciBranchName();
+  if (ci) return ci;
+  const branch = git(["branch", "--show-current"], "");
+  if (branch) return branch;
+  const name = git(["name-rev", "--name-only", "--exclude=tags/*", "HEAD"], "");
+  if (name && name !== "undefined") return name;
+  return `detached:${git(["rev-parse", "--short", "HEAD"], "unknown")}`;
+}
+function productBenchmarkRepoIdentity() {
+  return {
+    url: git(["config", "--get", "remote.origin.url"], "unknown"),
+    commit: git(["rev-parse", "HEAD"], "unknown"),
+    branch: repoBranch()
+  };
+}
+function packageVersion(name) {
+  const pkg = JSON.parse(readFileSync6(resolve("package.json"), "utf8"));
+  if (pkg.name === name && pkg.version) return pkg.version;
+  const declared = pkg.dependencies?.[name] ?? pkg.devDependencies?.[name];
+  if (declared) return declared;
+  const installed = resolve("node_modules", name, "package.json");
+  if (existsSync5(installed)) {
+    const installedPkg = JSON.parse(readFileSync6(installed, "utf8"));
+    if (installedPkg.version) return installedPkg.version;
+  }
+  return "unknown";
+}
+function readRunRecords(path) {
+  const lines = readFileSync6(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
+  return lines.map((line, index) => {
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch (err) {
+      throw new ValidationError(
+        `${path}:${index + 1}: ${err instanceof Error ? err.message : String(err)}`,
+        { cause: err }
+      );
+    }
+    return expectRunRecordShape(parsed, `${path}:${index + 1}`);
+  });
+}
+function expectRunRecordShape(value, path) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new ValidationError(`${path}: run record must be an object`);
+  }
+  const obj = value;
+  for (const key of ["runId", "model"]) {
+    if (typeof obj[key] !== "string" || obj[key].length === 0) {
+      throw new ValidationError(`${path}: run record ${key} must be a non-empty string`);
+    }
+  }
+  for (const key of ["costUsd", "wallMs"]) {
+    if (typeof obj[key] !== "number" || !Number.isFinite(obj[key])) {
+      throw new ValidationError(`${path}: run record ${key} must be a finite number`);
+    }
+  }
+  const tokenUsage = obj.tokenUsage;
+  if (!tokenUsage || typeof tokenUsage.input !== "number" || typeof tokenUsage.output !== "number") {
+    throw new ValidationError(`${path}: run record tokenUsage.input/output must be numbers`);
+  }
+  const outcome = obj.outcome;
+  if (!outcome || !outcome.raw || typeof outcome.raw !== "object") {
+    throw new ValidationError(`${path}: run record outcome.raw must be an object`);
+  }
+  return value;
+}
+function clamp014(value) {
+  return Math.max(0, Math.min(1, value));
+}
+function splitOf(record, opts) {
+  const custom = opts.classifySplit?.(record);
+  if (custom) return custom;
+  if (record.outcome.raw.safety === 1) return "safety";
+  if (record.splitTag === "holdout") return "holdout";
+  if (record.splitTag === "dev") return "dev";
+  return "practice";
+}
+function scoreOf(record) {
+  const score = record.outcome.holdoutScore ?? record.outcome.searchScore;
+  if (typeof score === "number" && Number.isFinite(score)) return clamp014(score);
+  const rawScore = record.outcome.raw.score ?? record.outcome.raw.composite;
+  return typeof rawScore === "number" && Number.isFinite(rawScore) ? clamp014(rawScore) : 0;
+}
+function rawPassOf(record) {
+  const rawPass = record.outcome.raw.pass;
+  if (typeof rawPass === "boolean") return rawPass;
+  if (typeof rawPass === "number" && Number.isFinite(rawPass)) return rawPass >= 1;
+  return null;
+}
+function passOf(record, score, threshold) {
+  const rawPass = rawPassOf(record);
+  if (rawPass !== null) return rawPass && !record.failureMode;
+  return score >= threshold && !record.failureMode;
+}
+function failureModeOf(record, score, threshold) {
+  if (record.failureMode) return record.failureMode;
+  const belowThreshold = `quality-below-threshold: ${Math.round(score * 100)}% < ${Math.round(threshold * 100)}%`;
+  if (rawPassOf(record) === false) {
+    return score < threshold ? belowThreshold : "product-pass-failed";
+  }
+  if (passOf(record, score, threshold)) return null;
+  return belowThreshold;
+}
+function numericDimensions(record) {
+  const dimensions = {};
+  for (const [key, value] of Object.entries(record.outcome.raw ?? {})) {
+    if (typeof value === "number" && Number.isFinite(value)) dimensions[key] = value;
+  }
+  if (Object.keys(dimensions).length === 0) dimensions.score = scoreOf(record);
+  return dimensions;
+}
+function armIdOf(record) {
+  if (record.candidateId) return record.candidateId;
+  const variant = record.agentProfile?.dimensions?.variant;
+  if (typeof variant === "string" && variant.length > 0) return variant;
+  const variantId = record.agentProfile?.dimensions?.variantId;
+  if (typeof variantId === "string" && variantId.length > 0) return variantId;
+  return "production-profile";
+}
+function backendOf(record) {
+  const backend = record.agentProfile?.dimensions?.backend;
+  return typeof backend === "string" && backend.length > 0 ? backend : "unknown";
+}
+function modelProvider(record) {
+  const backend = backendOf(record);
+  if (backend === "cli-bridge") return "cli-bridge";
+  if (backend === "sandbox") return "router";
+  if (record.model.startsWith("router/")) return "router";
+  return backend === "unknown" ? "router" : backend;
+}
+function runtimeResolution(record, opts) {
+  const reasoningEffort = record.agentProfile?.dimensions?.reasoningLevel;
+  return {
+    model: record.agentProfile?.model ?? record.model,
+    harness: record.agentProfile?.harness?.id ?? `${opts.projectId}-canonical-eval`,
+    backend: backendOf(record),
+    ...typeof reasoningEffort === "string" && reasoningEffort.length > 0 ? { reasoningEffort } : {}
+  };
+}
+function sourceProfileHash(record) {
+  return record.agentProfile?.sourceProfile?.hash ?? record.agentProfile?.cellId ?? sha256(
+    JSON.stringify({
+      candidateId: record.candidateId,
+      promptHash: record.promptHash,
+      configHash: record.configHash
+    })
+  );
+}
+function materializedProfileHash(record, runtime) {
+  return sha256(
+    JSON.stringify({ sourceProfileHash: sourceProfileHash(record), model: runtime.model })
+  );
+}
+function profileIdOf(record, armId, runtime, opts) {
+  const base = record.agentProfile?.profileId ?? opts.fallbackProfileId ?? armId;
+  const withArm = base.endsWith(`:${armId}`) ? base : `${base}:${armId}`;
+  return `${withArm}:${runtime.model}`;
+}
+function toolCallsOf(record, runDir, opts) {
+  const raw = record.outcome.raw;
+  const primary = Number(raw.tool_call_count ?? raw.tool_calls ?? raw.toolCalls ?? 0);
+  if (Number.isFinite(primary) && primary > 0) return primary;
+  const fallback = opts.toolCallFallback?.(record, runDir);
+  if (fallback !== void 0 && Number.isFinite(fallback) && fallback > 0) return fallback;
+  return Number.isFinite(primary) ? primary : 0;
+}
+var TRACE_CANDIDATES = ["traces.jsonl", "traces", "trace", "trace-store"];
+var RAW_CANDIDATES = ["raws.jsonl", "raw-events"];
+var SCORE_CANDIDATES = ["scores.json", "manifest.json"];
+function firstExisting(runDir, candidates) {
+  return candidates.find((candidate) => existsSync5(join4(runDir, candidate))) ?? candidates[0];
+}
+function relativeArtifacts(runDir) {
+  return {
+    records: "records.jsonl",
+    traces: firstExisting(runDir, TRACE_CANDIDATES),
+    raws: firstExisting(runDir, RAW_CANDIDATES),
+    scores: firstExisting(runDir, SCORE_CANDIDATES),
+    workspace: existsSync5(join4(runDir, "workspace")) ? "workspace" : "."
+  };
+}
+function absoluteArtifacts(runDir) {
+  const rel = relativeArtifacts(runDir);
+  return {
+    records: resolve(runDir, rel.records),
+    traces: resolve(runDir, rel.traces),
+    raws: resolve(runDir, rel.raws),
+    scores: resolve(runDir, rel.scores),
+    workspace: resolve(runDir, rel.workspace)
+  };
+}
+function materializeRunArtifacts(runDir, outDir, index) {
+  if (resolve(runDir) === resolve(outDir)) return relativeArtifacts(runDir);
+  const rel = relative(runDir, outDir);
+  if (rel && !rel.startsWith("..") && !isAbsolute(rel)) {
+    throw new ValidationError(`outDir must not be nested inside runDir: ${outDir}`);
+  }
+  const id = `${String(index + 1).padStart(2, "0")}-${safePathPart(basename2(runDir))}-${sha256(runDir).slice(7, 19)}`;
+  const destRel = join4("source-runs", id);
+  const destAbs = join4(outDir, destRel);
+  mkdirSync3(dirname3(destAbs), { recursive: true });
+  cpSync(runDir, destAbs, { recursive: true, force: true });
+  const source = relativeArtifacts(runDir);
+  return {
+    records: join4(destRel, source.records),
+    traces: join4(destRel, source.traces),
+    raws: join4(destRel, source.raws),
+    scores: join4(destRel, source.scores),
+    workspace: source.workspace === "." ? destRel : join4(destRel, source.workspace)
+  };
+}
+function existsArtifact(artifactRoot, value) {
+  return existsSync5(resolve(artifactRoot, value));
+}
+function runRecordToProductBenchmarkRecord(record, runDir, artifactRoot, artifacts, options) {
+  const opts = resolveOptions(options);
+  const runtime = runtimeResolution(record, opts);
+  const score = scoreOf(record);
+  const armId = armIdOf(record);
+  const inputTokens = record.tokenUsage.input;
+  const outputTokens = record.tokenUsage.output;
+  const toolCallCount = toolCallsOf(record, runDir, opts);
+  const dimensions = numericDimensions(record);
+  if (!("tool_calls" in dimensions)) dimensions.tool_calls = toolCallCount;
+  const product = {
+    schemaVersion: 1,
+    projectId: opts.projectId,
+    benchmarkId: opts.benchmarkId,
+    runId: record.runId,
+    scenarioId: record.scenarioId ?? record.experimentId,
+    split: splitOf(record, opts),
+    armId,
+    rep: Number(record.seed ?? 0) + 1,
+    agentProfile: {
+      id: profileIdOf(record, armId, runtime, opts),
+      hash: materializedProfileHash(record, runtime),
+      path: opts.agentProfilePath,
+      declared: runtime,
+      resolved: runtime
+    },
+    model: { provider: modelProvider(record), id: record.model },
+    backend: { kind: backendOf(record), version: opts.backendVersion },
+    outcome: {
+      pass: passOf(record, score, opts.passThreshold),
+      score,
+      dimensions,
+      failureMode: failureModeOf(record, score, opts.passThreshold)
+    },
+    usage: {
+      inputTokens,
+      outputTokens,
+      costUsd: record.costUsd,
+      // Rounded: the bundle contract requires integer milliseconds.
+      wallMs: Math.round(record.wallMs),
+      toolCalls: toolCallCount
+    },
+    integrity: {
+      realBackend: inputTokens + outputTokens > 0,
+      rawCapture: existsArtifact(artifactRoot, artifacts.raws),
+      traceCapture: existsArtifact(artifactRoot, artifacts.traces),
+      noStubRows: inputTokens + outputTokens > 0,
+      priced: record.costUsd > 0,
+      profileMaterialized: Boolean(record.agentProfile?.cellId)
+    },
+    artifacts
+  };
+  return validateProductBenchmarkRecord(product);
+}
+function uniqueArmRecords(records) {
+  const byArm = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    const prior = byArm.get(record.armId);
+    if (!prior) {
+      byArm.set(record.armId, record);
+      continue;
+    }
+    const fields = [
+      ["profileId", prior.agentProfile.id, record.agentProfile.id],
+      ["model", prior.agentProfile.resolved.model, record.agentProfile.resolved.model],
+      ["backend", prior.agentProfile.resolved.backend, record.agentProfile.resolved.backend],
+      ["harness", prior.agentProfile.resolved.harness, record.agentProfile.resolved.harness],
+      [
+        "reasoningEffort",
+        prior.agentProfile.resolved.reasoningEffort,
+        record.agentProfile.resolved.reasoningEffort
+      ]
+    ];
+    for (const [name, a, b] of fields) {
+      if (a !== b) {
+        throw new ValidationError(
+          `records for arm '${record.armId}' disagree on ${name} (${String(a)} vs ${String(b)}) \u2014 one arm id must map to one policy`
+        );
+      }
+    }
+  }
+  return [...byArm.values()];
+}
+function uniqueScenarioRecords(records) {
+  const byScenario = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    const prior = byScenario.get(record.scenarioId);
+    if (!prior) {
+      byScenario.set(record.scenarioId, record);
+      continue;
+    }
+    if (prior.split !== record.split) {
+      throw new ValidationError(
+        `records for scenario '${record.scenarioId}' disagree on split (${prior.split} vs ${record.split})`
+      );
+    }
+  }
+  return [...byScenario.values()];
+}
+function buildProductBenchmarkManifest(records, options) {
+  if (records.length === 0) {
+    throw new ValidationError("cannot build a product benchmark manifest from zero records");
+  }
+  const scenarioTagPrefix = options.scenarioTagPrefix ?? defaultScenarioTagPrefix(options.projectId);
+  const mutableSurfaces = options.mutableSurfaces ?? productBenchmarkMutableSurfaces;
+  const byProfile = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    byProfile.set(record.agentProfile.id, {
+      id: record.agentProfile.id,
+      profileHash: record.agentProfile.hash,
+      agentProfilePath: record.agentProfile.path
+    });
+  }
+  const manifest = {
+    schemaVersion: 1,
+    projectId: options.projectId,
+    benchmarkId: options.benchmarkId,
+    repo: productBenchmarkRepoIdentity(),
+    substrate: {
+      agentEval: packageVersion("@tangle-network/agent-eval"),
+      agentRuntime: packageVersion("@tangle-network/agent-runtime"),
+      agentInterface: packageVersion("@tangle-network/agent-interface"),
+      sandbox: packageVersion("@tangle-network/sandbox"),
+      ...options.substrate
+    },
+    profiles: [...byProfile.values()],
+    arms: uniqueArmRecords(records).map((record) => ({
+      id: record.armId,
+      profileId: record.agentProfile.id,
+      mutableSurfaces: [...mutableSurfaces],
+      policyAxes: {
+        carrier: record.armId.includes("policy") ? "resource-file" : "profile",
+        model: record.agentProfile.resolved.model,
+        backend: record.agentProfile.resolved.backend,
+        harness: record.agentProfile.resolved.harness,
+        ...record.agentProfile.resolved.reasoningEffort !== void 0 ? { reasoningEffort: record.agentProfile.resolved.reasoningEffort } : {}
+      }
+    })),
+    scenarios: uniqueScenarioRecords(records).map((record) => ({
+      id: record.scenarioId,
+      split: record.split,
+      tags: [scenarioTagPrefix, options.benchmarkId, record.split],
+      sourceAllowedForSynthesis: false
+    })),
+    budgets: {
+      maxUsd: records.reduce((sum4, record) => sum4 + record.usage.costUsd, 0),
+      maxCells: records.length,
+      maxWallMs: records.reduce((sum4, record) => sum4 + record.usage.wallMs, 0)
+    },
+    expectedArtifactDir: resolve(options.outDir)
+  };
+  return validateProductBenchmarkManifest(manifest);
+}
+function exportProductBenchmark(options) {
+  const { runDir, ...rest } = options;
+  return exportProductBenchmarkRuns({ ...rest, runDirs: [runDir] });
+}
+function exportProductBenchmarkRuns(options) {
+  const opts = resolveOptions(options);
+  const outDir = resolve(options.outDir);
+  const runDirs = options.runDirs.map((runDir) => resolve(runDir));
+  if (runDirs.length === 0) throw new ValidationError("export requires at least one run directory");
+  const materialize = options.materializeSourceRuns !== false;
+  const rows = runDirs.flatMap((runDir) => {
+    const sourceRecordsPath = join4(runDir, "records.jsonl");
+    if (!existsSync5(sourceRecordsPath)) throw new ValidationError(`missing ${sourceRecordsPath}`);
+    const records = readRunRecords(sourceRecordsPath);
+    if (records.length === 0) throw new ValidationError(`${sourceRecordsPath} is empty`);
+    return records.map((record) => ({ record, runDir }));
+  });
+  mkdirSync3(outDir, { recursive: true });
+  const artifactsByRunDir = /* @__PURE__ */ new Map();
+  for (const [index, runDir] of runDirs.entries()) {
+    artifactsByRunDir.set(
+      runDir,
+      materialize ? materializeRunArtifacts(runDir, outDir, index) : absoluteArtifacts(runDir)
+    );
+  }
+  const artifactRoot = materialize ? outDir : "/";
+  const normalized = rows.map(
+    ({ record, runDir }) => runRecordToProductBenchmarkRecord(
+      record,
+      runDir,
+      artifactRoot,
+      artifactsByRunDir.get(runDir),
+      { ...options, runDirs, backendVersion: opts.backendVersion }
+    )
+  );
+  const manifest = buildProductBenchmarkManifest(normalized, {
+    outDir,
+    projectId: opts.projectId,
+    benchmarkId: opts.benchmarkId,
+    scenarioTagPrefix: opts.scenarioTagPrefix,
+    mutableSurfaces: opts.mutableSurfaces,
+    ...options.substrate !== void 0 ? { substrate: options.substrate } : {}
+  });
+  const manifestPath = join4(outDir, "product-benchmark-manifest.json");
+  const recordsPath = join4(outDir, "product-benchmark-records.jsonl");
+  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}
+`);
+  writeFileSync(recordsPath, `${normalized.map((record) => JSON.stringify(record)).join("\n")}
+`);
+  return { manifestPath, recordsPath, records: normalized.length };
+}
+function defaultScenarioTagPrefix(projectId) {
+  return projectId.replace(/-agent$/, "") || projectId;
+}
+function resolveOptions(options) {
+  return {
+    projectId: options.projectId,
+    benchmarkId: options.benchmarkId,
+    agentProfilePath: options.agentProfilePath,
+    passThreshold: options.passThreshold ?? 0.7,
+    scenarioTagPrefix: options.scenarioTagPrefix ?? defaultScenarioTagPrefix(options.projectId),
+    ...options.fallbackProfileId !== void 0 ? { fallbackProfileId: options.fallbackProfileId } : {},
+    mutableSurfaces: options.mutableSurfaces ?? productBenchmarkMutableSurfaces,
+    ...options.classifySplit !== void 0 ? { classifySplit: options.classifySplit } : {},
+    ...options.toolCallFallback !== void 0 ? { toolCallFallback: options.toolCallFallback } : {},
+    backendVersion: options.backendVersion ?? packageVersion("@tangle-network/sandbox")
+  };
+}
+
+// src/product-benchmark/index.ts
+var productBenchmarkSplits = ["practice", "dev", "holdout", "safety", "sentinel"];
+function isObject2(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function fail(path, message) {
+  throw new ValidationError(`${path}: ${message}`);
+}
+function wrapValidationError(path, err) {
+  if (err instanceof ValidationError) {
+    throw new ValidationError(`${path}: ${err.message}`, { cause: err });
+  }
+  throw new ValidationError(`${path}: ${err instanceof Error ? err.message : String(err)}`, {
+    cause: err
+  });
+}
+function expectObject(value, path) {
+  if (!isObject2(value)) fail(path, "must be an object");
+  return value;
+}
+function expectString(value, path) {
+  if (typeof value !== "string" || value.trim().length === 0)
+    fail(path, "must be a non-empty string");
+  return value;
+}
+function expectBoolean(value, path) {
+  if (typeof value !== "boolean") fail(path, "must be a boolean");
+  return value;
+}
+function expectNumber(value, path, opts = {}) {
+  if (typeof value !== "number" || !Number.isFinite(value)) fail(path, "must be a finite number");
+  if (opts.integer && !Number.isInteger(value)) fail(path, "must be an integer");
+  if (opts.min !== void 0 && value < opts.min) fail(path, `must be >= ${opts.min}`);
+  if (opts.max !== void 0 && value > opts.max) fail(path, `must be <= ${opts.max}`);
+  return value;
+}
+function expectStringArray(value, path) {
+  if (!Array.isArray(value)) fail(path, "must be an array");
+  return value.map((entry, index) => expectString(entry, `${path}[${index}]`));
+}
+function expectObjectArray(value, path) {
+  if (!Array.isArray(value)) fail(path, "must be an array");
+  return value.map((entry, index) => expectObject(entry, `${path}[${index}]`));
+}
+function expectSplit(value, path) {
+  const split = expectString(value, path);
+  if (!productBenchmarkSplits.includes(split)) {
+    fail(path, `must be one of ${productBenchmarkSplits.join(", ")}`);
+  }
+  return split;
+}
+function optionalString(value, path) {
+  if (value === void 0) return void 0;
+  return expectString(value, path);
+}
+function expectDimensions(value, path) {
+  const obj = expectObject(value, path);
+  const out = {};
+  for (const [key, raw] of Object.entries(obj)) out[key] = expectNumber(raw, `${path}.${key}`);
+  return out;
+}
+function expectRuntimeResolution(value, path) {
+  const obj = expectObject(value, path);
+  return {
+    model: expectString(obj.model, `${path}.model`),
+    harness: expectString(obj.harness, `${path}.harness`),
+    backend: expectString(obj.backend, `${path}.backend`),
+    ...obj.reasoningEffort !== void 0 ? { reasoningEffort: optionalString(obj.reasoningEffort, `${path}.reasoningEffort`) } : {}
+  };
+}
+function validateProductBenchmarkManifest(value) {
+  const obj = expectObject(value, "manifest");
+  if (obj.schemaVersion !== 1) fail("manifest.schemaVersion", "must be 1");
+  const repo = expectObject(obj.repo, "manifest.repo");
+  const substrate = expectObject(obj.substrate, "manifest.substrate");
+  const profiles = expectObjectArray(obj.profiles, "manifest.profiles").map((profile, index) => ({
+    id: expectString(profile.id, `manifest.profiles[${index}].id`),
+    profileHash: expectString(profile.profileHash, `manifest.profiles[${index}].profileHash`),
+    agentProfilePath: expectString(
+      profile.agentProfilePath,
+      `manifest.profiles[${index}].agentProfilePath`
+    )
+  }));
+  const arms = expectObjectArray(obj.arms, "manifest.arms").map((arm, index) => ({
+    id: expectString(arm.id, `manifest.arms[${index}].id`),
+    profileId: expectString(arm.profileId, `manifest.arms[${index}].profileId`),
+    mutableSurfaces: expectStringArray(
+      arm.mutableSurfaces,
+      `manifest.arms[${index}].mutableSurfaces`
+    ),
+    policyAxes: expectObject(arm.policyAxes, `manifest.arms[${index}].policyAxes`)
+  }));
+  const scenarios = expectObjectArray(obj.scenarios, "manifest.scenarios").map(
+    (scenario, index) => ({
+      id: expectString(scenario.id, `manifest.scenarios[${index}].id`),
+      split: expectSplit(scenario.split, `manifest.scenarios[${index}].split`),
+      tags: expectStringArray(scenario.tags, `manifest.scenarios[${index}].tags`),
+      sourceAllowedForSynthesis: expectBoolean(
+        scenario.sourceAllowedForSynthesis,
+        `manifest.scenarios[${index}].sourceAllowedForSynthesis`
+      )
+    })
+  );
+  const budgets = expectObject(obj.budgets, "manifest.budgets");
+  if (profiles.length === 0) fail("manifest.profiles", "must contain at least one profile");
+  if (arms.length === 0) fail("manifest.arms", "must contain at least one arm");
+  if (scenarios.length === 0) fail("manifest.scenarios", "must contain at least one scenario");
+  assertUnique(
+    profiles.map((profile) => profile.id),
+    "manifest.profiles.id"
+  );
+  assertUnique(
+    arms.map((arm) => arm.id),
+    "manifest.arms.id"
+  );
+  assertUnique(
+    scenarios.map((scenario) => scenario.id),
+    "manifest.scenarios.id"
+  );
+  const profileIds = new Set(profiles.map((profile) => profile.id));
+  for (const arm of arms) {
+    if (!profileIds.has(arm.profileId))
+      fail(`manifest.arms.${arm.id}.profileId`, `unknown profile ${arm.profileId}`);
+  }
+  return {
+    schemaVersion: 1,
+    projectId: expectString(obj.projectId, "manifest.projectId"),
+    benchmarkId: expectString(obj.benchmarkId, "manifest.benchmarkId"),
+    repo: {
+      url: expectString(repo.url, "manifest.repo.url"),
+      commit: expectString(repo.commit, "manifest.repo.commit"),
+      branch: expectString(repo.branch, "manifest.repo.branch")
+    },
+    substrate: {
+      agentEval: expectString(substrate.agentEval, "manifest.substrate.agentEval"),
+      agentRuntime: expectString(substrate.agentRuntime, "manifest.substrate.agentRuntime"),
+      agentInterface: expectString(substrate.agentInterface, "manifest.substrate.agentInterface"),
+      sandbox: expectString(substrate.sandbox, "manifest.substrate.sandbox"),
+      ...substrate.agentBench !== void 0 ? { agentBench: expectString(substrate.agentBench, "manifest.substrate.agentBench") } : {}
+    },
+    profiles,
+    arms,
+    scenarios,
+    budgets: {
+      maxUsd: expectNumber(budgets.maxUsd, "manifest.budgets.maxUsd", { min: 0 }),
+      maxCells: expectNumber(budgets.maxCells, "manifest.budgets.maxCells", {
+        min: 0,
+        integer: true
+      }),
+      maxWallMs: expectNumber(budgets.maxWallMs, "manifest.budgets.maxWallMs", {
+        min: 0,
+        integer: true
+      })
+    },
+    expectedArtifactDir: expectString(obj.expectedArtifactDir, "manifest.expectedArtifactDir")
+  };
+}
+function validateProductBenchmarkRecord(value) {
+  const obj = expectObject(value, "record");
+  if (obj.schemaVersion !== 1) fail("record.schemaVersion", "must be 1");
+  const agentProfile = expectObject(obj.agentProfile, "record.agentProfile");
+  const model = expectObject(obj.model, "record.model");
+  const backend = expectObject(obj.backend, "record.backend");
+  const outcome = expectObject(obj.outcome, "record.outcome");
+  const usage = expectObject(obj.usage, "record.usage");
+  const integrity = expectObject(obj.integrity, "record.integrity");
+  const artifacts = expectObject(obj.artifacts, "record.artifacts");
+  const record = {
+    schemaVersion: 1,
+    projectId: expectString(obj.projectId, "record.projectId"),
+    benchmarkId: expectString(obj.benchmarkId, "record.benchmarkId"),
+    runId: expectString(obj.runId, "record.runId"),
+    scenarioId: expectString(obj.scenarioId, "record.scenarioId"),
+    split: expectSplit(obj.split, "record.split"),
+    armId: expectString(obj.armId, "record.armId"),
+    rep: expectNumber(obj.rep, "record.rep", { min: 1, integer: true }),
+    agentProfile: {
+      id: expectString(agentProfile.id, "record.agentProfile.id"),
+      hash: expectString(agentProfile.hash, "record.agentProfile.hash"),
+      path: expectString(agentProfile.path, "record.agentProfile.path"),
+      declared: expectRuntimeResolution(agentProfile.declared, "record.agentProfile.declared"),
+      resolved: expectRuntimeResolution(agentProfile.resolved, "record.agentProfile.resolved")
+    },
+    model: {
+      provider: expectString(model.provider, "record.model.provider"),
+      id: expectString(model.id, "record.model.id")
+    },
+    backend: {
+      kind: expectString(backend.kind, "record.backend.kind"),
+      version: expectString(backend.version, "record.backend.version")
+    },
+    outcome: {
+      pass: expectBoolean(outcome.pass, "record.outcome.pass"),
+      score: expectNumber(outcome.score, "record.outcome.score", { min: 0, max: 1 }),
+      dimensions: expectDimensions(outcome.dimensions, "record.outcome.dimensions"),
+      failureMode: outcome.failureMode === null ? null : expectString(outcome.failureMode, "record.outcome.failureMode")
+    },
+    usage: {
+      inputTokens: expectNumber(usage.inputTokens, "record.usage.inputTokens", {
+        min: 0,
+        integer: true
+      }),
+      outputTokens: expectNumber(usage.outputTokens, "record.usage.outputTokens", {
+        min: 0,
+        integer: true
+      }),
+      costUsd: expectNumber(usage.costUsd, "record.usage.costUsd", { min: 0 }),
+      wallMs: expectNumber(usage.wallMs, "record.usage.wallMs", { min: 0, integer: true }),
+      toolCalls: expectNumber(usage.toolCalls, "record.usage.toolCalls", { min: 0, integer: true })
+    },
+    integrity: {
+      realBackend: expectBoolean(integrity.realBackend, "record.integrity.realBackend"),
+      rawCapture: expectBoolean(integrity.rawCapture, "record.integrity.rawCapture"),
+      traceCapture: expectBoolean(integrity.traceCapture, "record.integrity.traceCapture"),
+      noStubRows: expectBoolean(integrity.noStubRows, "record.integrity.noStubRows"),
+      priced: expectBoolean(integrity.priced, "record.integrity.priced"),
+      profileMaterialized: expectBoolean(
+        integrity.profileMaterialized,
+        "record.integrity.profileMaterialized"
+      )
+    },
+    artifacts: {
+      records: expectString(artifacts.records, "record.artifacts.records"),
+      traces: expectString(artifacts.traces, "record.artifacts.traces"),
+      raws: expectString(artifacts.raws, "record.artifacts.raws"),
+      scores: expectString(artifacts.scores, "record.artifacts.scores"),
+      workspace: expectString(artifacts.workspace, "record.artifacts.workspace")
+    }
+  };
+  if (record.integrity.realBackend && record.usage.inputTokens + record.usage.outputTokens === 0) {
+    fail("record.usage", "realBackend rows must carry non-zero token usage");
+  }
+  return record;
+}
+function productBenchmarkIntegrityFailures(record) {
+  const failures = [];
+  for (const [key, value] of Object.entries(record.integrity)) {
+    if (!value) failures.push(`${record.runId}:${key}=false`);
+  }
+  return failures;
+}
+function readProductBenchmarkRecords(path) {
+  const lines = readFileSync7(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
+  const records = [];
+  for (const [index, line] of lines.entries()) {
+    try {
+      records.push(validateProductBenchmarkRecord(JSON.parse(line)));
+    } catch (err) {
+      wrapValidationError(`${path}:${index + 1}`, err);
+    }
+  }
+  return records;
+}
+function readProductBenchmarkManifest(path) {
+  try {
+    return validateProductBenchmarkManifest(JSON.parse(readFileSync7(path, "utf8")));
+  } catch (err) {
+    wrapValidationError(path, err);
+  }
+}
+function validateProductBenchmarkRun(input) {
+  const manifest = readProductBenchmarkManifest(input.manifestPath);
+  const records = readProductBenchmarkRecords(input.recordsPath);
+  const manifestProjectBench = `${manifest.projectId}/${manifest.benchmarkId}`;
+  const integrityFailures = records.flatMap(productBenchmarkIntegrityFailures);
+  const missingArtifacts = input.checkArtifacts === false ? [] : records.flatMap(
+    (record) => missingArtifactsForRecord(record, input.artifactRoot ?? dirname4(input.recordsPath))
+  );
+  for (const [index, record] of records.entries()) {
+    const recordProjectBench = `${record.projectId}/${record.benchmarkId}`;
+    if (recordProjectBench !== manifestProjectBench) {
+      fail(
+        `records[${index}]`,
+        `project/benchmark ${recordProjectBench} does not match manifest ${manifestProjectBench}`
+      );
+    }
+    if (!manifest.arms.some((arm) => arm.id === record.armId))
+      fail(`records[${index}].armId`, `unknown arm ${record.armId}`);
+    if (!manifest.scenarios.some((scenario) => scenario.id === record.scenarioId)) {
+      fail(`records[${index}].scenarioId`, `unknown scenario ${record.scenarioId}`);
+    }
+  }
+  const repoFailures = ["url", "commit", "branch"].filter((key) => manifest.repo[key].trim().length === 0 || manifest.repo[key] === "unknown").map((key) => `manifest.repo.${key}`);
+  const substrateFailures = ["agentEval", "agentRuntime", "agentInterface", "sandbox"].filter(
+    (key) => manifest.substrate[key].trim().length === 0 || manifest.substrate[key] === "unknown"
+  ).map((key) => `manifest.substrate.${key}`);
+  return {
+    manifestPath: input.manifestPath,
+    recordsPath: input.recordsPath,
+    records: records.length,
+    repoFailures,
+    substrateFailures,
+    projects: sortedUnique(records.map((record) => record.projectId)),
+    benchmarks: sortedUnique(records.map((record) => record.benchmarkId)),
+    arms: sortedUnique(records.map((record) => record.armId)),
+    scenarios: sortedUnique(records.map((record) => record.scenarioId)),
+    passed: records.filter((record) => record.outcome.pass).length,
+    failed: records.filter((record) => !record.outcome.pass).length,
+    inputTokens: sum3(records, (record) => record.usage.inputTokens),
+    outputTokens: sum3(records, (record) => record.usage.outputTokens),
+    costUsd: sum3(records, (record) => record.usage.costUsd),
+    wallMs: sum3(records, (record) => record.usage.wallMs),
+    integrityFailures,
+    missingArtifacts
+  };
+}
+function missingArtifactsForRecord(record, artifactRoot) {
+  const missing = [];
+  for (const [key, value] of Object.entries(record.artifacts)) {
+    const path = resolveArtifactPath(value, artifactRoot);
+    if (!existsSync6(path)) missing.push(`${record.runId}:${key}:${value}`);
+  }
+  return missing;
+}
+function resolveArtifactPath(value, artifactRoot) {
+  return value.startsWith("/") ? value : join5(artifactRoot, value);
+}
+function assertUnique(values, path) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const value of values) {
+    if (seen.has(value)) fail(path, `duplicate ${value}`);
+    seen.add(value);
+  }
+}
+function sortedUnique(values) {
+  return [...new Set(values)].sort();
+}
+function sum3(items, fn) {
+  return items.reduce((total, item) => total + fn(item), 0);
+}
+function findProductBenchmarkArtifacts(runDir) {
+  const manifestPath = join5(runDir, "product-benchmark-manifest.json");
+  const recordsPath = join5(runDir, "product-benchmark-records.jsonl");
+  if (existsSync6(manifestPath) && statSync3(manifestPath).isFile() && existsSync6(recordsPath) && statSync3(recordsPath).isFile()) {
+    return { manifestPath, recordsPath };
+  }
+  return null;
+}
+function assertProductBenchmarkRun(runDir) {
+  const artifacts = findProductBenchmarkArtifacts(runDir);
+  if (!artifacts) {
+    fail(runDir, "missing product-benchmark-manifest.json or product-benchmark-records.jsonl");
+  }
+  const report = validateProductBenchmarkRun({ ...artifacts, artifactRoot: runDir });
+  const failures = [
+    ...report.repoFailures,
+    ...report.substrateFailures,
+    ...report.integrityFailures,
+    ...report.missingArtifacts
+  ];
+  if (failures.length > 0) {
+    fail(runDir, `product benchmark validation failed:
+${failures.join("\n")}`);
+  }
+  return report;
+}
 export {
   AGENT_PROFILE_KINDS,
   ATTESTATION_ALGORITHM,
@@ -10375,7 +11479,6 @@ export {
   assertNoHiddenLeak,
   assertProductBenchmarkRun,
   assertRealBackend,
-  assertRecordIntegrity,
   assertReleaseConfidence,
   assertRunAgentProfileCell,
   assertRunCaptured,
@@ -10421,11 +11524,9 @@ export {
   causalAttribution,
   checkBehavioralCanary,
   checkCanaries,
-  checkRecordIntegrity,
   checkSlos,
   checkTraceContracts,
   clamp01,
-  classifyEuAiRisk,
   classifyFailure,
   classifyTreatment,
   cliffsDelta,
@@ -10504,7 +11605,6 @@ export {
   errorStreakDetector,
   estimateCost,
   estimateTokens,
-  euAiActReport,
   evaluateActionPolicy,
   evaluateContract,
   evaluateHypothesis,
@@ -10513,7 +11613,6 @@ export {
   evaluateReleaseConfidence,
   evaluateTraceContract,
   executeScenario,
-  expandMatrix,
   expandProfileAxes,
   expectAgent,
   exportProductBenchmark,
@@ -10552,7 +11651,6 @@ export {
   formatFindings,
   formatScorecardDiff,
   gainHistogram,
-  gatePerf,
   gateTreatmentApplied,
   gateTreatmentFromMetrics,
   gateTreatmentFromSpans,
@@ -10632,7 +11730,6 @@ export {
   modelPriceKey,
   mulberry32,
   multiToolchainLayer,
-  nistAiRmfReport,
   noProgressDetector,
   normalizeScores,
   notBlocked,
@@ -10697,7 +11794,6 @@ export {
   referenceReplayScenarioToRunScore,
   regexMatch,
   regexMatches,
-  renderMarkdown,
   renderMarkdownReport,
   renderPlaybookMarkdown,
   renderPreferenceMemoryMarkdown,
@@ -10745,7 +11841,6 @@ export {
   runsForScenario,
   scalarScore,
   scanForMuffledGates,
-  scenarioKey,
   scoreContinuity,
   scoreFromEvals,
   scoreKnowledgeReadiness,
@@ -10762,7 +11857,6 @@ export {
   sentenceReorderMutator,
   serializeFeedbackTrajectoriesJsonl,
   signManifest,
-  soc2Report,
   spearmanR,
   splitGold,
   statusAdvanced,
@@ -10771,12 +11865,10 @@ export {
   stringField,
   stripFencedJson,
   subjectiveEval,
-  summarize,
   summarizeBackendIntegrity,
   summarizeHarnessResults,
   summarizePrReviewBenchmark,
   summarizePreferenceMemory,
-  summarizeRecords,
   summaryTable,
   testJudge,
   textInSnapshot,