@tangle-network/agent-eval 0.109.1 → 0.110.0

package/dist/prm/index.d.ts

@@ -1,104 +0,0 @@
-import { P as PrmGradedTrace, S as StepRubric, a as PrmGrader } from '../rubric-Cc6UHvUb.js';
-export { G as GradedStep, b as StepContext, i as isPrmVerdict } from '../rubric-Cc6UHvUb.js';
-import { T as TraceStore } from '../store-BcFXE6LG.js';
-import { S as Span } from '../schema-m0gsnbt3.js';
-import '../trajectory-2TkpSEVh.js';
-
-/**
- * Export PRM-graded traces as training data for downstream reward-model
- * fine-tuning. Canonical format is NDJSON of
- * `{ trajectory_text, step_index, rubric, score }` so a small model can
- * learn to predict step rewards from step context.
- *
- * The framework doesn't train the model — we emit the data; callers
- * plug it into their preferred trainer (TRL, Unsloth, custom).
- */
-
-interface PrmTrainingSample {
-    runId: string;
-    spanId: string;
-    rubricId: string;
-    score: number;
-    /** Serialized step context — step + surrounding conversation. */
-    context: {
-        priorTurns: Array<{
-            role: string;
-            content: string;
-        }>;
-        step: {
-            kind: Span['kind'];
-            text: string;
-        };
-    };
-    /** Optional evidence + rationale for auditability. */
-    rationale?: string;
-    evidence?: string;
-}
-declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
-    contextWindow?: number;
-}): Promise<PrmTrainingSample[]>;
-/** NDJSON serialization — write to file or stream directly to a trainer. */
-declare function toNdjson(samples: PrmTrainingSample[]): string;
-
-/**
- * Built-in reference rubrics. Consumers combine these with domain
- * rubrics. All are deterministic, rule-based — cheap to run + easy
- * to unit-test. LLM-based rubrics are trivially authored by
- * following the StepRubric contract.
- */
-
-/** Penalize very short or very long assistant outputs. */
-declare function outputLengthRubric(args?: {
-    minChars?: number;
-    maxChars?: number;
-    weight?: number;
-}): StepRubric;
-/** Reward tool calls that succeeded (status='ok') with an informative result. */
-declare function toolSuccessRubric(args?: {
-    weight?: number;
-}): StepRubric;
-/** Penalize tool calls that duplicate a prior call with identical args. */
-declare function toolNonRedundantRubric(args?: {
-    weight?: number;
-}): StepRubric;
-/** Penalize LLM outputs that contain common refusal markers when a refusal
- *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */
-declare function nonRefusalRubric(args?: {
-    markers?: RegExp[];
-    weight?: number;
-}): StepRubric;
-/** Reward outputs that invoke the next-step tool the trajectory actually uses
- *  (i.e. the LLM span announced "I will call X" and the following tool span IS X). */
-declare function toolIntentAlignmentRubric(args?: {
-    weight?: number;
-}): StepRubric;
-
-/**
- * Inference-time PRM scoring — pick the best of N candidate trajectories
- * using a trained reward model (or a rule-based PRM as a proxy).
- *
- * The canonical Best-of-N pattern: generate N completions, score each
- * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
- * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
- */
-
-interface BestOfNResult {
-    winner: PrmGradedTrace;
-    ranked: PrmGradedTrace[];
-    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
-    stdDev: number;
-}
-interface PrmBestOfNOptions {
-    /** Max concurrent `grader.grade` calls. Default 4. */
-    concurrency?: number;
-}
-declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
-/**
- * Weighted vote across multiple graders — use when you want a PRM ensemble
- * (e.g. rule-based + LLM-based + trained model). Each grader produces its
- * own ranking; we aggregate via rank-sum (Borda count) so no single grader
- * dominates via a different score scale.
- */
-declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
-
-export { type BestOfNResult, type PrmBestOfNOptions, PrmGradedTrace, PrmGrader, type PrmTrainingSample, StepRubric, exportTrainingData, nonRefusalRubric, outputLengthRubric, prmBestOfN, prmEnsembleBestOfN, toNdjson, toolIntentAlignmentRubric, toolNonRedundantRubric, toolSuccessRubric };

package/dist/prm/index.js

@@ -1,265 +0,0 @@
-import {
-  exportTrainingData,
-  toNdjson
-} from "../chunk-KMPRBJK4.js";
-import {
-  buildTrajectory
-} from "../chunk-RZTMDUO7.js";
-import "../chunk-5BKGXME7.js";
-import {
-  TraceEmitter
-} from "../chunk-TVVP3ZZQ.js";
-import "../chunk-PZ5AY32C.js";
-
-// src/prm/builtin-rubrics.ts
-function outputLengthRubric(args = {}) {
-  const min = args.minChars ?? 20;
-  const max = args.maxChars ?? 8e3;
-  return {
-    id: "output-length",
-    kinds: ["llm"],
-    weight: args.weight ?? 0.5,
-    async grade({ step }) {
-      const llm = step.span;
-      const len = (llm.output ?? "").length;
-      if (len === 0) return { score: 0, rationale: "empty output" };
-      if (len < min)
-        return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` };
-      if (len > max)
-        return {
-          score: Math.max(0, 1 - (len - max) / max),
-          rationale: `above max (${len} > ${max})`
-        };
-      return { score: 1, rationale: `${len} chars in bounds` };
-    }
-  };
-}
-function toolSuccessRubric(args = {}) {
-  return {
-    id: "tool-success",
-    kinds: ["tool"],
-    weight: args.weight ?? 1,
-    async grade({ step }) {
-      const tool = step.span;
-      if (tool.status === "error")
-        return { score: 0, rationale: `error: ${tool.error ?? "unknown"}` };
-      const r = tool.result;
-      if (r === null || r === void 0) return { score: 0.3, rationale: "empty result" };
-      const asText = typeof r === "string" ? r : JSON.stringify(r);
-      if (asText.length < 4) return { score: 0.5, rationale: "tiny result" };
-      return { score: 1, rationale: `${tool.toolName} ok` };
-    }
-  };
-}
-function toolNonRedundantRubric(args = {}) {
-  const weight = args.weight ?? 0.5;
-  return {
-    id: "tool-non-redundant",
-    kinds: ["tool"],
-    weight,
-    async grade({ step, prior }) {
-      const tool = step.span;
-      const priorMatches = prior.filter((p) => {
-        if (p.span.kind !== "tool") return false;
-        const pt = p.span;
-        return pt.toolName === tool.toolName && stableStringify(pt.args) === stableStringify(tool.args);
-      });
-      if (priorMatches.length === 0) return { score: 1, rationale: "novel call" };
-      return {
-        score: Math.max(0, 1 - priorMatches.length * 0.5),
-        rationale: `${priorMatches.length} duplicate(s)`
-      };
-    }
-  };
-}
-function nonRefusalRubric(args = {}) {
-  const weight = args.weight ?? 1;
-  const markers = args.markers ?? [
-    /\bi\s+(?:can(?:not|'t)|won't|will\s+not)\b/i,
-    /\b(?:as\s+an?\s+)?ai\b.*?\b(?:can't|cannot)\b/i
-  ];
-  return {
-    id: "non-refusal",
-    kinds: ["llm"],
-    weight,
-    async grade({ step }) {
-      const llm = step.span;
-      const out = llm.output ?? "";
-      const refused = markers.some((re) => re.test(out));
-      return refused ? { score: 0, rationale: "refusal marker present" } : { score: 1, rationale: "no refusal" };
-    }
-  };
-}
-function toolIntentAlignmentRubric(args = {}) {
-  return {
-    id: "tool-intent-alignment",
-    kinds: ["llm"],
-    weight: args.weight ?? 0.5,
-    async grade({ step, next }) {
-      const llm = step.span;
-      const nextTool = next.find((s) => s.span.kind === "tool");
-      if (!nextTool) return null;
-      const toolName = nextTool.span.toolName;
-      const out = (llm.output ?? "").toLowerCase();
-      const mentioned = out.includes(toolName.toLowerCase());
-      return mentioned ? { score: 1, rationale: `mentioned "${toolName}" before calling it` } : { score: 0.5, rationale: `called "${toolName}" without announcing it` };
-    }
-  };
-}
-function stableStringify(value) {
-  if (value === null || typeof value !== "object") return JSON.stringify(value);
-  if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
-  const keys = Object.keys(value).sort();
-  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(",")}}`;
-}
-
-// src/prm/inference.ts
-var DEFAULT_PRM_CONCURRENCY = 4;
-async function prmBestOfN(store, grader, runIds, options = {}) {
-  if (runIds.length === 0) throw new Error("prmBestOfN: at least 1 candidate required");
-  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY);
-  const graded = new Array(runIds.length);
-  let cursor = 0;
-  async function worker() {
-    while (true) {
-      const i = cursor++;
-      if (i >= runIds.length) return;
-      graded[i] = await grader.grade(store, runIds[i]);
-    }
-  }
-  await Promise.all(Array.from({ length: Math.min(concurrency, runIds.length) }, () => worker()));
-  const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore);
-  const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
-  const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length;
-  return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance) };
-}
-async function prmEnsembleBestOfN(store, graders, runIds, options = {}) {
-  if (graders.length === 0) throw new Error("prmEnsembleBestOfN: at least 1 grader");
-  if (runIds.length === 0) throw new Error("prmEnsembleBestOfN: at least 1 candidate required");
-  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY);
-  const jobs = [];
-  for (let gi = 0; gi < graders.length; gi++) {
-    for (const runId of runIds) jobs.push({ graderIdx: gi, runId });
-  }
-  const settled = new Array(jobs.length);
-  let cursor = 0;
-  async function worker() {
-    while (true) {
-      const i = cursor++;
-      if (i >= jobs.length) return;
-      const job = jobs[i];
-      settled[i] = await graders[job.graderIdx].grade(store, job.runId).then((value) => ({ status: "fulfilled", value })).catch((reason) => ({ status: "rejected", reason }));
-    }
-  }
-  await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, () => worker()));
-  const perGrader = graders.map(() => []);
-  const failures = [];
-  for (let i = 0; i < jobs.length; i++) {
-    const job = jobs[i];
-    const result = settled[i];
-    if (result.status === "fulfilled") perGrader[job.graderIdx].push(result.value);
-    else {
-      const error = result.reason instanceof Error ? result.reason.message : String(result.reason);
-      failures.push({ graderIdx: job.graderIdx, runId: job.runId, error });
-    }
-  }
-  const survivingGraders = perGrader.filter((ranking) => ranking.length > 0);
-  if (survivingGraders.length === 0) {
-    throw new Error(
-      `prmEnsembleBestOfN: every grader failed on every candidate (${failures.length} call(s)). First error: ${failures[0]?.error ?? "unknown"}`
-    );
-  }
-  for (const ranking of survivingGraders)
-    ranking.sort((a, b) => b.aggregateScore - a.aggregateScore);
-  const bordaScores = /* @__PURE__ */ new Map();
-  for (const ranking of survivingGraders) {
-    ranking.forEach((g, rank) => {
-      bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank));
-    });
-  }
-  const byRun = /* @__PURE__ */ new Map();
-  for (const ranking of survivingGraders) {
-    for (const g of ranking) if (!byRun.has(g.runId)) byRun.set(g.runId, g);
-  }
-  const ranked = [...byRun.values()].sort(
-    (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0)
-  );
-  const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length;
-  const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length;
-  return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance) };
-}
-
-// src/prm/rubric.ts
-var PrmGrader = class {
-  constructor(rubrics) {
-    this.rubrics = rubrics;
-    if (rubrics.length === 0) throw new Error("PrmGrader: at least 1 rubric required");
-  }
-  rubrics;
-  /**
-   * Grade every eligible span in a run. Emits a JudgeVerdict span for each
-   * (rubric × span) verdict so the result is visible to downstream pipelines
-   * (judgeAgreementView, etc.) — PRM is just "a judge that runs per span."
-   */
-  async grade(store, runId) {
-    const trajectory = await buildTrajectory(store, runId);
-    const emitter = new TraceEmitter(store, { runId });
-    const steps = [];
-    let ungraded = 0;
-    for (let i = 0; i < trajectory.steps.length; i++) {
-      const step = trajectory.steps[i];
-      const ctx = {
-        trajectory,
-        step,
-        prior: trajectory.steps.slice(0, i),
-        next: trajectory.steps.slice(i + 1)
-      };
-      let gradedThis = false;
-      for (const rubric of this.rubrics) {
-        if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue;
-        const verdict = await rubric.grade(ctx);
-        if (verdict === null) continue;
-        const weight = rubric.weight ?? 1;
-        steps.push({
-          spanId: step.span.spanId,
-          rubricId: rubric.id,
-          score: verdict.score,
-          weight,
-          rationale: verdict.rationale,
-          evidence: verdict.evidence
-        });
-        gradedThis = true;
-        await emitter.recordJudge({
-          judgeId: `prm:${rubric.id}`,
-          targetSpanId: step.span.spanId,
-          dimension: "step_quality",
-          score: verdict.score,
-          rationale: verdict.rationale,
-          evidence: verdict.evidence,
-          name: `prm:${rubric.id}`
-        });
-      }
-      if (!gradedThis) ungraded++;
-    }
-    const totalWeight = steps.reduce((a, s) => a + s.weight, 0);
-    const aggregateScore = totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight;
-    return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded };
-  }
-};
-function isPrmVerdict(verdict) {
-  return verdict.judgeId.startsWith("prm:");
-}
-export {
-  PrmGrader,
-  exportTrainingData,
-  isPrmVerdict,
-  nonRefusalRubric,
-  outputLengthRubric,
-  prmBestOfN,
-  prmEnsembleBestOfN,
-  toNdjson,
-  toolIntentAlignmentRubric,
-  toolNonRedundantRubric,
-  toolSuccessRubric
-};
-//# sourceMappingURL=index.js.map

package/dist/prm/index.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/prm/builtin-rubrics.ts","../../src/prm/inference.ts","../../src/prm/rubric.ts"],"sourcesContent":["/**\n * Built-in reference rubrics. Consumers combine these with domain\n * rubrics. All are deterministic, rule-based — cheap to run + easy\n * to unit-test. LLM-based rubrics are trivially authored by\n * following the StepRubric contract.\n */\n\nimport type { LlmSpan, ToolSpan } from '../trace/schema'\nimport type { StepRubric } from './rubric'\n\n/** Penalize very short or very long assistant outputs. */\nexport function outputLengthRubric(\n  args: { minChars?: number; maxChars?: number; weight?: number } = {},\n): StepRubric {\n  const min = args.minChars ?? 20\n  const max = args.maxChars ?? 8000\n  return {\n    id: 'output-length',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const len = (llm.output ?? '').length\n      if (len === 0) return { score: 0, rationale: 'empty output' }\n      if (len < min)\n        return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` }\n      if (len > max)\n        return {\n          score: Math.max(0, 1 - (len - max) / max),\n          rationale: `above max (${len} > ${max})`,\n        }\n      return { score: 1, rationale: `${len} chars in bounds` }\n    },\n  }\n}\n\n/** Reward tool calls that succeeded (status='ok') with an informative result. */\nexport function toolSuccessRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-success',\n    kinds: ['tool'],\n    weight: args.weight ?? 1,\n    async grade({ step }) {\n      const tool = step.span as ToolSpan\n      if (tool.status === 'error')\n        return { score: 0, rationale: `error: ${tool.error ?? 'unknown'}` }\n      const r = tool.result\n      if (r === null || r === undefined) return { score: 0.3, rationale: 'empty result' }\n      const asText = typeof r === 'string' ? r : JSON.stringify(r)\n      if (asText.length < 4) return { score: 0.5, rationale: 'tiny result' }\n      return { score: 1, rationale: `${tool.toolName} ok` }\n    },\n  }\n}\n\n/** Penalize tool calls that duplicate a prior call with identical args. */\nexport function toolNonRedundantRubric(args: { weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 0.5\n  return {\n    id: 'tool-non-redundant',\n    kinds: ['tool'],\n    weight,\n    async grade({ step, prior }) {\n      const tool = step.span as ToolSpan\n      const priorMatches = prior.filter((p) => {\n        if (p.span.kind !== 'tool') return false\n        const pt = p.span as ToolSpan\n        return (\n          pt.toolName === tool.toolName && stableStringify(pt.args) === stableStringify(tool.args)\n        )\n      })\n      if (priorMatches.length === 0) return { score: 1, rationale: 'novel call' }\n      return {\n        score: Math.max(0, 1 - priorMatches.length * 0.5),\n        rationale: `${priorMatches.length} duplicate(s)`,\n      }\n    },\n  }\n}\n\n/** Penalize LLM outputs that contain common refusal markers when a refusal\n *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */\nexport function nonRefusalRubric(args: { markers?: RegExp[]; weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 1\n  const markers = args.markers ?? [\n    /\\bi\\s+(?:can(?:not|'t)|won't|will\\s+not)\\b/i,\n    /\\b(?:as\\s+an?\\s+)?ai\\b.*?\\b(?:can't|cannot)\\b/i,\n  ]\n  return {\n    id: 'non-refusal',\n    kinds: ['llm'],\n    weight,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const out = llm.output ?? ''\n      const refused = markers.some((re) => re.test(out))\n      return refused\n        ? { score: 0, rationale: 'refusal marker present' }\n        : { score: 1, rationale: 'no refusal' }\n    },\n  }\n}\n\n/** Reward outputs that invoke the next-step tool the trajectory actually uses\n *  (i.e. the LLM span announced \"I will call X\" and the following tool span IS X). */\nexport function toolIntentAlignmentRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-intent-alignment',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step, next }) {\n      const llm = step.span as LlmSpan\n      const nextTool = next.find((s) => s.span.kind === 'tool')\n      if (!nextTool) return null\n      const toolName = (nextTool.span as ToolSpan).toolName\n      const out = (llm.output ?? '').toLowerCase()\n      const mentioned = out.includes(toolName.toLowerCase())\n      return mentioned\n        ? { score: 1, rationale: `mentioned \"${toolName}\" before calling it` }\n        : { score: 0.5, rationale: `called \"${toolName}\" without announcing it` }\n    },\n  }\n}\n\nfunction stableStringify(value: unknown): string {\n  if (value === null || typeof value !== 'object') return JSON.stringify(value)\n  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n  const keys = Object.keys(value as Record<string, unknown>).sort()\n  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify((value as Record<string, unknown>)[k])}`).join(',')}}`\n}\n","/**\n * Inference-time PRM scoring — pick the best of N candidate trajectories\n * using a trained reward model (or a rule-based PRM as a proxy).\n *\n * The canonical Best-of-N pattern: generate N completions, score each\n * with a PRM, pick the winner. Here the scoring loop is framework-agnostic\n * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.\n */\n\nimport type { TraceStore } from '../trace/store'\nimport type { PrmGradedTrace, PrmGrader } from './rubric'\n\nexport interface BestOfNResult {\n  winner: PrmGradedTrace\n  ranked: PrmGradedTrace[]\n  /** Standard deviation of aggregate scores — small = candidates were homogenous. */\n  stdDev: number\n}\n\n/** Default max concurrent grader calls. Bounds LLM fan-out so a wide ensemble\n *  doesn't trigger a provider rate-limit storm. */\nconst DEFAULT_PRM_CONCURRENCY = 4\n\nexport interface PrmBestOfNOptions {\n  /** Max concurrent `grader.grade` calls. Default 4. */\n  concurrency?: number\n}\n\nexport async function prmBestOfN(\n  store: TraceStore,\n  grader: PrmGrader,\n  runIds: string[],\n  options: PrmBestOfNOptions = {},\n): Promise<BestOfNResult> {\n  if (runIds.length === 0) throw new Error('prmBestOfN: at least 1 candidate required')\n  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY)\n  const graded: PrmGradedTrace[] = new Array(runIds.length)\n  let cursor = 0\n  async function worker(): Promise<void> {\n    while (true) {\n      const i = cursor++\n      if (i >= runIds.length) return\n      graded[i] = await grader.grade(store, runIds[i]!)\n    }\n  }\n  await Promise.all(Array.from({ length: Math.min(concurrency, runIds.length) }, () => worker()))\n  const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore)\n  const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length\n  const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n\n/**\n * Weighted vote across multiple graders — use when you want a PRM ensemble\n * (e.g. rule-based + LLM-based + trained model). Each grader produces its\n * own ranking; we aggregate via rank-sum (Borda count) so no single grader\n * dominates via a different score scale.\n */\nexport async function prmEnsembleBestOfN(\n  store: TraceStore,\n  graders: PrmGrader[],\n  runIds: string[],\n  options: PrmBestOfNOptions = {},\n): Promise<BestOfNResult> {\n  if (graders.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 grader')\n  if (runIds.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 candidate required')\n  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY)\n\n  // Flatten the (grader, runId) product and run it through a single bounded\n  // pool. Nested unbounded fan-out (graders × runIds) would launch every LLM\n  // call at once. allSettled isolates failures: one grader (or one runId)\n  // failing doesn't void the whole ensemble.\n  type Job = { graderIdx: number; runId: string }\n  const jobs: Job[] = []\n  for (let gi = 0; gi < graders.length; gi++) {\n    for (const runId of runIds) jobs.push({ graderIdx: gi, runId })\n  }\n  const settled: PromiseSettledResult<PrmGradedTrace>[] = new Array(jobs.length)\n  let cursor = 0\n  async function worker(): Promise<void> {\n    while (true) {\n      const i = cursor++\n      if (i >= jobs.length) return\n      const job = jobs[i]!\n      settled[i] = await graders[job.graderIdx]!.grade(store, job.runId)\n        .then((value): PromiseSettledResult<PrmGradedTrace> => ({ status: 'fulfilled', value }))\n        .catch((reason): PromiseSettledResult<PrmGradedTrace> => ({ status: 'rejected', reason }))\n    }\n  }\n  await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, () => worker()))\n\n  // Regroup fulfilled results into per-grader rankings. A grader contributes\n  // only the candidates it successfully graded; a grader that graded nothing\n  // is dropped from the vote rather than skewing it with phantom zeros.\n  const perGrader: PrmGradedTrace[][] = graders.map(() => [])\n  const failures: { graderIdx: number; runId: string; error: string }[] = []\n  for (let i = 0; i < jobs.length; i++) {\n    const job = jobs[i]!\n    const result = settled[i]!\n    if (result.status === 'fulfilled') perGrader[job.graderIdx]!.push(result.value)\n    else {\n      const error = result.reason instanceof Error ? result.reason.message : String(result.reason)\n      failures.push({ graderIdx: job.graderIdx, runId: job.runId, error })\n    }\n  }\n  const survivingGraders = perGrader.filter((ranking) => ranking.length > 0)\n  if (survivingGraders.length === 0) {\n    throw new Error(\n      `prmEnsembleBestOfN: every grader failed on every candidate (${failures.length} call(s)). First error: ${failures[0]?.error ?? 'unknown'}`,\n    )\n  }\n  for (const ranking of survivingGraders)\n    ranking.sort((a, b) => b.aggregateScore - a.aggregateScore)\n\n  // Borda: rank-sum across surviving graders.\n  const bordaScores = new Map<string, number>()\n  for (const ranking of survivingGraders) {\n    ranking.forEach((g, rank) => {\n      bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank))\n    })\n  }\n  // Synthesize a ranking from the union of every successfully-graded trace,\n  // ordered by Borda score. aggregateScore field kept for UX. Using the union\n  // (not just the first grader) keeps a candidate that one grader dropped but\n  // another graded.\n  const byRun = new Map<string, PrmGradedTrace>()\n  for (const ranking of survivingGraders) {\n    for (const g of ranking) if (!byRun.has(g.runId)) byRun.set(g.runId, g)\n  }\n  const ranked = [...byRun.values()].sort(\n    (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0),\n  )\n  const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length\n  const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n","/**\n * Process Reward Modeling — per-step rubric grading.\n *\n * A StepRubric inspects one span and returns a score + rationale.\n * PrmGrader applies an array of rubrics to every LLM span in a\n * trajectory (consumers can broaden to tool/retrieval spans via the\n * `kind` filter on each rubric).\n *\n * Why this matters: outcome-only eval (did the final artifact work?)\n * gives sparse reward — most agent turns are unattributable. PRMs\n * densify the signal so optimizers and RL fine-tuning can assign\n * credit per turn.\n */\n\nimport { TraceEmitter } from '../trace/emitter'\nimport type { JudgeSpan, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface StepContext {\n  trajectory: Trajectory\n  step: TrajectoryStep\n  /** Steps preceding `step` in trajectory order. */\n  prior: TrajectoryStep[]\n  /** Steps following `step`. */\n  next: TrajectoryStep[]\n}\n\nexport interface StepRubric {\n  id: string\n  /** Only grade spans of these kinds (default: all). */\n  kinds?: Array<Span['kind']>\n  /** Weight in the aggregate score. Default 1. */\n  weight?: number\n  /** Returns score in 0..1 + optional rationale/evidence. Return `null` to\n   *  skip grading (rubric doesn't apply to this step). */\n  grade: (\n    ctx: StepContext,\n  ) => Promise<{ score: number; rationale?: string; evidence?: string } | null>\n}\n\nexport interface GradedStep {\n  spanId: string\n  rubricId: string\n  score: number\n  weight: number\n  rationale?: string\n  evidence?: string\n}\n\nexport interface PrmGradedTrace {\n  runId: string\n  steps: GradedStep[]\n  /** Weighted mean of all graded steps; 0..1. */\n  aggregateScore: number\n  /** Number of spans graded — useful for sanity-checking coverage. */\n  gradedCount: number\n  /** Number of spans in the trajectory that no rubric matched. */\n  ungradedCount: number\n}\n\nexport class PrmGrader {\n  constructor(private rubrics: StepRubric[]) {\n    if (rubrics.length === 0) throw new Error('PrmGrader: at least 1 rubric required')\n  }\n\n  /**\n   * Grade every eligible span in a run. Emits a JudgeVerdict span for each\n   * (rubric × span) verdict so the result is visible to downstream pipelines\n   * (judgeAgreementView, etc.) — PRM is just \"a judge that runs per span.\"\n   */\n  async grade(store: TraceStore, runId: string): Promise<PrmGradedTrace> {\n    const trajectory = await buildTrajectory(store, runId)\n    const emitter = new TraceEmitter(store, { runId })\n    const steps: GradedStep[] = []\n    let ungraded = 0\n    for (let i = 0; i < trajectory.steps.length; i++) {\n      const step = trajectory.steps[i]!\n      const ctx: StepContext = {\n        trajectory,\n        step,\n        prior: trajectory.steps.slice(0, i),\n        next: trajectory.steps.slice(i + 1),\n      }\n      let gradedThis = false\n      for (const rubric of this.rubrics) {\n        if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue\n        const verdict = await rubric.grade(ctx)\n        if (verdict === null) continue\n        const weight = rubric.weight ?? 1\n        steps.push({\n          spanId: step.span.spanId,\n          rubricId: rubric.id,\n          score: verdict.score,\n          weight,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n        })\n        gradedThis = true\n        // Persist the verdict as a JudgeSpan so the query pipelines see it\n        await emitter.recordJudge({\n          judgeId: `prm:${rubric.id}`,\n          targetSpanId: step.span.spanId,\n          dimension: 'step_quality',\n          score: verdict.score,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n          name: `prm:${rubric.id}`,\n        })\n      }\n      if (!gradedThis) ungraded++\n    }\n\n    const totalWeight = steps.reduce((a, s) => a + s.weight, 0)\n    const aggregateScore =\n      totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight\n\n    return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded }\n  }\n}\n\n/** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines\n *  can distinguish PRM verdicts from human or top-level LLM judges. */\nexport function isPrmVerdict(verdict: JudgeSpan): boolean {\n  return verdict.judgeId.startsWith('prm:')\n}\n"],"mappings":";;;;;;;;;;;;;;AAWO,SAAS,mBACd,OAAkE,CAAC,GACvD;AACZ,QAAM,MAAM,KAAK,YAAY;AAC7B,QAAM,MAAM,KAAK,YAAY;AAC7B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,OAAO,IAAI,UAAU,IAAI;AAC/B,UAAI,QAAQ,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,eAAe;AAC5D,UAAI,MAAM;AACR,eAAO,EAAE,OAAO,KAAK,IAAI,GAAG,MAAM,GAAG,GAAG,WAAW,cAAc,GAAG,MAAM,GAAG,IAAI;AACnF,UAAI,MAAM;AACR,eAAO;AAAA,UACL,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,GAAG;AAAA,UACxC,WAAW,cAAc,GAAG,MAAM,GAAG;AAAA,QACvC;AACF,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,GAAG,mBAAmB;AAAA,IACzD;AAAA,EACF;AACF;AAGO,SAAS,kBAAkB,OAA4B,CAAC,GAAe;AAC5E,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,OAAO,KAAK;AAClB,UAAI,KAAK,WAAW;AAClB,eAAO,EAAE,OAAO,GAAG,WAAW,UAAU,KAAK,SAAS,SAAS,GAAG;AACpE,YAAM,IAAI,KAAK;AACf,UAAI,MAAM,QAAQ,MAAM,OAAW,QAAO,EAAE,OAAO,KAAK,WAAW,eAAe;AAClF,YAAM,SAAS,OAAO,MAAM,WAAW,IAAI,KAAK,UAAU,CAAC;AAC3D,UAAI,OAAO,SAAS,EAAG,QAAO,EAAE,OAAO,KAAK,WAAW,cAAc;AACrE,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,KAAK,QAAQ,MAAM;AAAA,IACtD;AAAA,EACF;AACF;AAGO,SAAS,uBAAuB,OAA4B,CAAC,GAAe;AACjF,QAAM,SAAS,KAAK,UAAU;AAC9B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd;AAAA,IACA,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG;AAC3B,YAAM,OAAO,KAAK;AAClB,YAAM,eAAe,MAAM,OAAO,CAAC,MAAM;AACvC,YAAI,EAAE,KAAK,SAAS,OAAQ,QAAO;AACnC,cAAM,KAAK,EAAE;AACb,eACE,GAAG,aAAa,KAAK,YAAY,gBAAgB,GAAG,IAAI,MAAM,gBAAgB,KAAK,IAAI;AAAA,MAE3F,CAAC;AACD,UAAI,aAAa,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,aAAa;AAC1E,aAAO;AAAA,QACL,OAAO,KAAK,IAAI,GAAG,IAAI,aAAa,SAAS,GAAG;AAAA,QAChD,WAAW,GAAG,aAAa,MAAM;AAAA,MACnC;AAAA,IACF;AAAA,EACF;AACF;AAIO,SAAS,iBAAiB,OAAgD,CAAC,GAAe;AAC/F,QAAM,SAAS,KAAK,UAAU;AAC9B,QAAM,UAAU,KAAK,WAAW;AAAA,IAC9B;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb;AAAA,IACA,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,MAAM,IAAI,UAAU;AAC1B,YAAM,UAAU,QAAQ,KAAK,CAAC,OAAO,GAAG,KAAK,GAAG,CAAC;AACjD,aAAO,UACH,EAAE,OAAO,GAAG,WAAW,yBAAyB,IAChD,EAAE,OAAO,GAAG,WAAW,aAAa;AAAA,IAC1C;AAAA,EACF;AACF;AAIO,SAAS,0BAA0B,OAA4B,CAAC,GAAe;AACpF,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,MAAM,KAAK,GAAG;AAC1B,YAAM,MAAM,KAAK;AACjB,YAAM,WAAW,KAAK,KAAK,CAAC,MAAM,EAAE,KAAK,SAAS,MAAM;AACxD,UAAI,CAAC,SAAU,QAAO;AACtB,YAAM,WAAY,SAAS,KAAkB;AAC7C,YAAM,OAAO,IAAI,UAAU,IAAI,YAAY;AAC3C,YAAM,YAAY,IAAI,SAAS,SAAS,YAAY,CAAC;AACrD,aAAO,YACH,EAAE,OAAO,GAAG,WAAW,cAAc,QAAQ,sBAAsB,IACnE,EAAE,OAAO,KAAK,WAAW,WAAW,QAAQ,0BAA0B;AAAA,IAC5E;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,OAAwB;AAC/C,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK;AAC5E,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AACzE,QAAM,OAAO,OAAO,KAAK,KAAgC,EAAE,KAAK;AAChE,SAAO,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAiB,MAAkC,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC;AACxH;;;AC5GA,IAAM,0BAA0B;AAOhC,eAAsB,WACpB,OACA,QACA,QACA,UAA6B,CAAC,GACN;AACxB,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,2CAA2C;AACpF,QAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,eAAe,uBAAuB;AAC9E,QAAM,SAA2B,IAAI,MAAM,OAAO,MAAM;AACxD,MAAI,SAAS;AACb,iBAAe,SAAwB;AACrC,WAAO,MAAM;AACX,YAAM,IAAI;AACV,UAAI,KAAK,OAAO,OAAQ;AACxB,aAAO,CAAC,IAAI,MAAM,OAAO,MAAM,OAAO,OAAO,CAAC,CAAE;AAAA,IAClD;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAC9F,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAC7E,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;AAQA,eAAsB,mBACpB,OACA,SACA,QACA,UAA6B,CAAC,GACN;AACxB,MAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AACjF,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,mDAAmD;AAC5F,QAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,eAAe,uBAAuB;AAO9E,QAAM,OAAc,CAAC;AACrB,WAAS,KAAK,GAAG,KAAK,QAAQ,QAAQ,MAAM;AAC1C,eAAW,SAAS,OAAQ,MAAK,KAAK,EAAE,WAAW,IAAI,MAAM,CAAC;AAAA,EAChE;AACA,QAAM,UAAkD,IAAI,MAAM,KAAK,MAAM;AAC7E,MAAI,SAAS;AACb,iBAAe,SAAwB;AACrC,WAAO,MAAM;AACX,YAAM,IAAI;AACV,UAAI,KAAK,KAAK,OAAQ;AACtB,YAAM,MAAM,KAAK,CAAC;AAClB,cAAQ,CAAC,IAAI,MAAM,QAAQ,IAAI,SAAS,EAAG,MAAM,OAAO,IAAI,KAAK,EAC9D,KAAK,CAAC,WAAiD,EAAE,QAAQ,aAAa,MAAM,EAAE,EACtF,MAAM,CAAC,YAAkD,EAAE,QAAQ,YAAY,OAAO,EAAE;AAAA,IAC7F;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,KAAK,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAK5F,QAAM,YAAgC,QAAQ,IAAI,MAAM,CAAC,CAAC;AAC1D,QAAM,WAAkE,CAAC;AACzE,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,UAAM,SAAS,QAAQ,CAAC;AACxB,QAAI,OAAO,WAAW,YAAa,WAAU,IAAI,SAAS,EAAG,KAAK,OAAO,KAAK;AAAA,SACzE;AACH,YAAM,QAAQ,OAAO,kBAAkB,QAAQ,OAAO,OAAO,UAAU,OAAO,OAAO,MAAM;AAC3F,eAAS,KAAK,EAAE,WAAW,IAAI,WAAW,OAAO,IAAI,OAAO,MAAM,CAAC;AAAA,IACrE;AAAA,EACF;AACA,QAAM,mBAAmB,UAAU,OAAO,CAAC,YAAY,QAAQ,SAAS,CAAC;AACzE,MAAI,iBAAiB,WAAW,GAAG;AACjC,UAAM,IAAI;AAAA,MACR,+DAA+D,SAAS,MAAM,2BAA2B,SAAS,CAAC,GAAG,SAAS,SAAS;AAAA,IAC1I;AAAA,EACF;AACA,aAAW,WAAW;AACpB,YAAQ,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAG5D,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,WAAW,kBAAkB;AACtC,YAAQ,QAAQ,CAAC,GAAG,SAAS;AAC3B,kBAAY,IAAI,EAAE,QAAQ,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,QAAQ,SAAS,KAAK;AAAA,IACpF,CAAC;AAAA,EACH;AAKA,QAAM,QAAQ,oBAAI,IAA4B;AAC9C,aAAW,WAAW,kBAAkB;AACtC,eAAW,KAAK,QAAS,KAAI,CAAC,MAAM,IAAI,EAAE,KAAK,EAAG,OAAM,IAAI,EAAE,OAAO,CAAC;AAAA,EACxE;AACA,QAAM,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE;AAAA,IACjC,CAAC,GAAG,OAAO,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,YAAY,IAAI,EAAE,KAAK,KAAK;AAAA,EAC3E;AACA,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;;;AC1EO,IAAM,YAAN,MAAgB;AAAA,EACrB,YAAoB,SAAuB;AAAvB;AAClB,QAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AAAA,EACnF;AAAA,EAFoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASpB,MAAM,MAAM,OAAmB,OAAwC;AACrE,UAAM,aAAa,MAAM,gBAAgB,OAAO,KAAK;AACrD,UAAM,UAAU,IAAI,aAAa,OAAO,EAAE,MAAM,CAAC;AACjD,UAAM,QAAsB,CAAC;AAC7B,QAAI,WAAW;AACf,aAAS,IAAI,GAAG,IAAI,WAAW,MAAM,QAAQ,KAAK;AAChD,YAAM,OAAO,WAAW,MAAM,CAAC;AAC/B,YAAM,MAAmB;AAAA,QACvB;AAAA,QACA;AAAA,QACA,OAAO,WAAW,MAAM,MAAM,GAAG,CAAC;AAAA,QAClC,MAAM,WAAW,MAAM,MAAM,IAAI,CAAC;AAAA,MACpC;AACA,UAAI,aAAa;AACjB,iBAAW,UAAU,KAAK,SAAS;AACjC,YAAI,OAAO,SAAS,CAAC,OAAO,MAAM,SAAS,KAAK,KAAK,IAAI,EAAG;AAC5D,cAAM,UAAU,MAAM,OAAO,MAAM,GAAG;AACtC,YAAI,YAAY,KAAM;AACtB,cAAM,SAAS,OAAO,UAAU;AAChC,cAAM,KAAK;AAAA,UACT,QAAQ,KAAK,KAAK;AAAA,UAClB,UAAU,OAAO;AAAA,UACjB,OAAO,QAAQ;AAAA,UACf;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,QACpB,CAAC;AACD,qBAAa;AAEb,cAAM,QAAQ,YAAY;AAAA,UACxB,SAAS,OAAO,OAAO,EAAE;AAAA,UACzB,cAAc,KAAK,KAAK;AAAA,UACxB,WAAW;AAAA,UACX,OAAO,QAAQ;AAAA,UACf,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,UAClB,MAAM,OAAO,OAAO,EAAE;AAAA,QACxB,CAAC;AAAA,MACH;AACA,UAAI,CAAC,WAAY;AAAA,IACnB;AAEA,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,CAAC;AAC1D,UAAM,iBACJ,gBAAgB,IAAI,IAAI,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI;AAE9E,WAAO,EAAE,OAAO,OAAO,gBAAgB,aAAa,MAAM,QAAQ,eAAe,SAAS;AAAA,EAC5F;AACF;AAIO,SAAS,aAAa,SAA6B;AACxD,SAAO,QAAQ,QAAQ,WAAW,MAAM;AAC1C;","names":[]}

package/dist/product-benchmark/index.d.ts

@@ -1,247 +0,0 @@
-import { R as RunRecord } from '../run-record-I-Z3JNvO.js';
-import '@tangle-network/agent-interface';
-import '../errors-oeQrLqXC.js';
-import '../schema-m0gsnbt3.js';
-
-/**
- * Export side of the product benchmark bundle contract: convert product
- * eval run directories (`records.jsonl` of `RunRecord` rows + trace/raw
- * artifacts) into a portable `product-benchmark-manifest.json` +
- * `product-benchmark-records.jsonl` bundle that
- * `validateProductBenchmarkRun` accepts.
- *
- * Product-specific policy (safety-split detection, tool-call recovery,
- * profile id fallback, artifact materialization) enters through explicit
- * options; everything else is the shared union of the tax/legal/creative
- * exporters. Scenario catalogs, smoke runners, and CLIs stay in the
- * products.
- *
- * Input rows are checked structurally, not with `validateRunRecord`:
- * product harnesses record bare model aliases and partial provenance, and
- * the bundle contract's own validators re-check every field that matters
- * on the way out.
- */
-
-/** Full mutable-surface superset a product arm may declare. */
-declare const productBenchmarkMutableSurfaces: readonly ["prompt", "resources.files", "tools", "mcp", "hooks", "subagents"];
-interface ProductBenchmarkExportOptions {
-    /** Source eval run directories, each containing a `records.jsonl` of RunRecord rows. */
-    readonly runDirs: readonly string[];
-    /** Destination directory for the bundle (manifest + records + materialized source runs). */
-    readonly outDir: string;
-    readonly projectId: string;
-    readonly benchmarkId: string;
-    /** Repo-relative path of the product's canonical agent profile source. */
-    readonly agentProfilePath: string;
-    /** Pass threshold applied when a row carries no explicit `outcome.raw.pass`. Default 0.7. */
-    readonly passThreshold?: number;
-    /**
-     * First scenario tag. Defaults to `projectId` with a trailing `-agent`
-     * stripped (`tax-agent` → `tax`), matching the product exporters.
-     */
-    readonly scenarioTagPrefix?: string;
-    /** Profile id used when a row has no `agentProfile.profileId`. Defaults to the row's arm id. */
-    readonly fallbackProfileId?: string;
-    /** Arm mutable surfaces recorded in the manifest. Defaults to the full superset. */
-    readonly mutableSurfaces?: readonly string[];
-    /**
-     * Copy each run dir into `<outDir>/source-runs/` and record
-     * bundle-relative artifact paths (portable, self-contained). When false,
-     * artifacts keep absolute paths into the original run dirs. Default true.
-     */
-    readonly materializeSourceRuns?: boolean;
-    /**
-     * Override split classification for a row. Return undefined to fall back
-     * to the default (`outcome.raw.safety === 1` → safety, then splitTag).
-     */
-    readonly classifySplit?: (record: RunRecord) => ProductBenchmarkSplit | undefined;
-    /** Recovers a tool-call count when the row's raw bag carries none (e.g. from turn artifacts). */
-    readonly toolCallFallback?: (record: RunRecord, runDir: string) => number;
-    /** Backend version recorded per row. Defaults to the cwd package.json's `@tangle-network/sandbox` range. */
-    readonly backendVersion?: string;
-    /**
-     * Explicit substrate versions for the manifest, merged over what the cwd
-     * package.json / node_modules resolve. Use when a substrate package is not
-     * installed where the export runs — the validator refuses an `'unknown'`
-     * version, so provide the real one rather than shipping the sentinel.
-     */
-    readonly substrate?: Partial<ProductBenchmarkManifest['substrate']>;
-}
-interface ProductBenchmarkSingleRunExportOptions extends Omit<ProductBenchmarkExportOptions, 'runDirs'> {
-    readonly runDir: string;
-}
-interface ProductBenchmarkExportResult {
-    readonly manifestPath: string;
-    readonly recordsPath: string;
-    readonly records: number;
-}
-/** Repo identity from the exporting process's cwd. `'unknown'` values are flagged by `validateProductBenchmarkRun`. */
-declare function productBenchmarkRepoIdentity(): ProductBenchmarkManifest['repo'];
-/** Map one RunRecord row to a validated product benchmark record. */
-declare function runRecordToProductBenchmarkRecord(record: RunRecord, runDir: string, artifactRoot: string, artifacts: ProductBenchmarkRecord['artifacts'], options: ProductBenchmarkExportOptions | ProductBenchmarkSingleRunExportOptions): ProductBenchmarkRecord;
-/** Derive the bundle manifest from already-normalized records. */
-declare function buildProductBenchmarkManifest(records: readonly ProductBenchmarkRecord[], options: Pick<ProductBenchmarkExportOptions, 'outDir' | 'projectId' | 'benchmarkId' | 'scenarioTagPrefix' | 'mutableSurfaces' | 'substrate'>): ProductBenchmarkManifest;
-/** Single-run convenience wrapper over `exportProductBenchmarkRuns`. */
-declare function exportProductBenchmark(options: ProductBenchmarkSingleRunExportOptions): ProductBenchmarkExportResult;
-/**
- * Export one or more product eval run dirs into a validated product
- * benchmark bundle at `outDir`. Both the manifest and every record are
- * run through the contract validators before anything is written.
- */
-declare function exportProductBenchmarkRuns(options: ProductBenchmarkExportOptions): ProductBenchmarkExportResult;
-
-declare const productBenchmarkSplits: readonly ["practice", "dev", "holdout", "safety", "sentinel"];
-type ProductBenchmarkSplit = (typeof productBenchmarkSplits)[number];
-interface ProductBenchmarkRepoRef {
-    readonly url: string;
-    readonly commit: string;
-    readonly branch: string;
-}
-interface ProductBenchmarkSubstrateVersions {
-    readonly agentEval: string;
-    readonly agentRuntime: string;
-    readonly agentInterface: string;
-    readonly sandbox: string;
-    readonly agentBench?: string;
-}
-interface ProductBenchmarkProfileRef {
-    readonly id: string;
-    readonly profileHash: string;
-    readonly agentProfilePath: string;
-}
-interface ProductBenchmarkArm {
-    readonly id: string;
-    readonly profileId: string;
-    readonly mutableSurfaces: readonly string[];
-    readonly policyAxes: Record<string, unknown>;
-}
-interface ProductBenchmarkScenario {
-    readonly id: string;
-    readonly split: ProductBenchmarkSplit;
-    readonly tags: readonly string[];
-    readonly sourceAllowedForSynthesis: boolean;
-}
-interface ProductBenchmarkBudgets {
-    readonly maxUsd: number;
-    readonly maxCells: number;
-    readonly maxWallMs: number;
-}
-interface ProductBenchmarkManifest {
-    readonly schemaVersion: 1;
-    readonly projectId: string;
-    readonly benchmarkId: string;
-    readonly repo: ProductBenchmarkRepoRef;
-    readonly substrate: ProductBenchmarkSubstrateVersions;
-    readonly profiles: readonly ProductBenchmarkProfileRef[];
-    readonly arms: readonly ProductBenchmarkArm[];
-    readonly scenarios: readonly ProductBenchmarkScenario[];
-    readonly budgets: ProductBenchmarkBudgets;
-    readonly expectedArtifactDir: string;
-}
-interface AgentProfileRuntimeReceipt {
-    readonly model: string;
-    readonly harness: string;
-    readonly backend: string;
-    readonly reasoningEffort?: string;
-}
-type RuntimeResolution = AgentProfileRuntimeReceipt;
-interface ProductBenchmarkRecord {
-    readonly schemaVersion: 1;
-    readonly projectId: string;
-    readonly benchmarkId: string;
-    readonly runId: string;
-    readonly scenarioId: string;
-    readonly split: ProductBenchmarkSplit;
-    readonly armId: string;
-    readonly rep: number;
-    readonly agentProfile: {
-        readonly id: string;
-        readonly hash: string;
-        readonly path: string;
-        readonly declared: RuntimeResolution;
-        readonly resolved: RuntimeResolution;
-    };
-    readonly model: {
-        readonly provider: string;
-        readonly id: string;
-    };
-    readonly backend: {
-        readonly kind: string;
-        readonly version: string;
-    };
-    readonly outcome: {
-        readonly pass: boolean;
-        readonly score: number;
-        readonly dimensions: Record<string, number>;
-        readonly failureMode: string | null;
-    };
-    readonly usage: {
-        readonly inputTokens: number;
-        readonly outputTokens: number;
-        readonly costUsd: number;
-        readonly wallMs: number;
-        readonly toolCalls: number;
-    };
-    readonly integrity: {
-        readonly realBackend: boolean;
-        readonly rawCapture: boolean;
-        readonly traceCapture: boolean;
-        readonly noStubRows: boolean;
-        readonly priced: boolean;
-        readonly profileMaterialized: boolean;
-    };
-    readonly artifacts: {
-        readonly records: string;
-        readonly traces: string;
-        readonly raws: string;
-        readonly scores: string;
-        readonly workspace: string;
-    };
-}
-interface ProductBenchmarkRunInput {
-    readonly manifestPath: string;
-    readonly recordsPath: string;
-    readonly artifactRoot?: string;
-    readonly checkArtifacts?: boolean;
-}
-interface ProductBenchmarkValidationReport {
-    readonly manifestPath: string;
-    readonly recordsPath: string;
-    readonly records: number;
-    /** Manifest repo fields that are empty or the `'unknown'` export sentinel. */
-    readonly repoFailures: readonly string[];
-    /** Manifest substrate versions that are empty or the `'unknown'` export
-     *  sentinel — a bundle without substrate identity is not reproducible. */
-    readonly substrateFailures: readonly string[];
-    readonly projects: readonly string[];
-    readonly benchmarks: readonly string[];
-    readonly arms: readonly string[];
-    readonly scenarios: readonly string[];
-    readonly passed: number;
-    readonly failed: number;
-    readonly inputTokens: number;
-    readonly outputTokens: number;
-    readonly costUsd: number;
-    readonly wallMs: number;
-    readonly integrityFailures: readonly string[];
-    readonly missingArtifacts: readonly string[];
-}
-interface ProductBenchmarkArtifactPaths {
-    readonly manifestPath: string;
-    readonly recordsPath: string;
-}
-declare function validateProductBenchmarkManifest(value: unknown): ProductBenchmarkManifest;
-declare function validateProductBenchmarkRecord(value: unknown): ProductBenchmarkRecord;
-declare function productBenchmarkIntegrityFailures(record: ProductBenchmarkRecord): string[];
-declare function readProductBenchmarkRecords(path: string): ProductBenchmarkRecord[];
-declare function readProductBenchmarkManifest(path: string): ProductBenchmarkManifest;
-declare function validateProductBenchmarkRun(input: ProductBenchmarkRunInput): ProductBenchmarkValidationReport;
-declare function findProductBenchmarkArtifacts(runDir: string): ProductBenchmarkArtifactPaths | null;
-/**
- * Fail-loud gate over a bundle directory: locates the manifest + records,
- * runs `validateProductBenchmarkRun`, and throws with every repo,
- * integrity, and artifact failure listed. Returns the report when clean.
- */
-declare function assertProductBenchmarkRun(runDir: string): ProductBenchmarkValidationReport;
-
-export { type AgentProfileRuntimeReceipt, type ProductBenchmarkArm, type ProductBenchmarkArtifactPaths, type ProductBenchmarkBudgets, type ProductBenchmarkExportOptions, type ProductBenchmarkExportResult, type ProductBenchmarkManifest, type ProductBenchmarkProfileRef, type ProductBenchmarkRecord, type ProductBenchmarkRepoRef, type ProductBenchmarkRunInput, type ProductBenchmarkScenario, type ProductBenchmarkSingleRunExportOptions, type ProductBenchmarkSplit, type ProductBenchmarkSubstrateVersions, type ProductBenchmarkValidationReport, type RuntimeResolution, assertProductBenchmarkRun, buildProductBenchmarkManifest, exportProductBenchmark, exportProductBenchmarkRuns, findProductBenchmarkArtifacts, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, readProductBenchmarkManifest, readProductBenchmarkRecords, runRecordToProductBenchmarkRecord, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun };