@tangle-network/agent-eval 0.123.7 → 0.124.0

package/dist/index.js

@@ -10,6 +10,36 @@ import {
   checkCanaries,
   runBehavioralCanaries
 } from "./chunk-SJT4OBVL.js";
+import {
+  mintRolloutRows,
+  rolloutReward
+} from "./chunk-MGGFVCJ7.js";
+import {
+  SUPERVISOR_RUN_SCHEMA,
+  analyzeSupervisorRun,
+  analyzeSupervisorRunSources,
+  isUnavailable,
+  renderSupervisorRunHeadline,
+  renderSupervisorRunMarkdown,
+  rollupSupervisorRuns,
+  showMeasured,
+  supervisorRunRolloutLines,
+  writeSupervisorRunReport
+} from "./chunk-4Y7AAATF.js";
+import "./chunk-R7ZRE2KV.js";
+import {
+  toJsonl,
+  toRewardRows,
+  toSftRows
+} from "./chunk-IPYXE555.js";
+import {
+  ROLLOUT_FORMAT,
+  ROLLOUT_SCHEMA,
+  assertRolloutLine,
+  isRolloutLine,
+  isTrainableSplit,
+  validateRolloutLine
+} from "./chunk-MAX3TN3C.js";
 import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
@@ -17,13 +47,15 @@ import {
 } from "./chunk-JKDNAOF5.js";
 import {
   DEFAULT_RULES,
-  buildTrajectory,
   classifyFailure,
   compareToBaseline,
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-J3LHTAAB.js";
+} from "./chunk-QOTFXW5L.js";
+import {
+  buildTrajectory
+} from "./chunk-RZTMDUO7.js";
 import {
   analyzeSeries
 } from "./chunk-BOD4O7OF.js";
@@ -53,7 +85,7 @@ import {
   pairArms,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-VPDOSN3L.js";
+} from "./chunk-5PVZVCZB.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -91,7 +123,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-KKPPFIDS.js";
+} from "./chunk-PMITBABE.js";
 import {
   BackendIntegrityError,
   assertRealAgentReceipts,
@@ -116,12 +148,12 @@ import {
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-QNVBALEX.js";
+} from "./chunk-W5B3ZGP3.js";
 import {
   buildDefaultAnalystRegistry,
   computeTraceMetrics,
   createChatClient
-} from "./chunk-WP3GEXKI.js";
+} from "./chunk-A6GT67HT.js";
 import "./chunk-HHWE3POT.js";
 import {
   AnalystRegistry,
@@ -156,7 +188,7 @@ import {
   scorePolicyEditReadiness,
   validatePolicyEdit,
   validatePolicyEditCandidateRecord
-} from "./chunk-MHPEGJHC.js";
+} from "./chunk-DT7OXY3C.js";
 import {
   allCriticalPassed,
   controlFailureClassFromVerification,
@@ -187,7 +219,7 @@ import {
 } from "./chunk-MOXWMGPC.js";
 import {
   runEvalCampaign
-} from "./chunk-N7ZS6PEF.js";
+} from "./chunk-GID26AN4.js";
 import "./chunk-ARU2PZFM.js";
 import {
   LlmCallError,
@@ -204,7 +236,7 @@ import {
   maximumChargeForLlmRequest,
   probeLlm,
   stripFencedJson
-} from "./chunk-MK3Q6GCL.js";
+} from "./chunk-EQUK3RFS.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -3204,117 +3236,6 @@ function printDriverSummary(results) {
   console.log(`${completedCount}/${results.length} personas completed`);
 }
 
-// src/rollout-export.ts
-var ROLLOUT_FORMAT = "tangle.rollout.v1";
-var asText = (v, scrub) => {
-  const s = typeof v === "string" ? v : JSON.stringify(v);
-  return scrub(s ?? "");
-};
-function projectStep(span, scrub) {
-  const base = {
-    kind: span.kind,
-    name: scrub(span.name),
-    status: span.status,
-    durationMs: span.endedAt !== void 0 ? span.endedAt - span.startedAt : void 0
-  };
-  if (span.kind === "llm") {
-    const llm = span;
-    const last = llm.messages[llm.messages.length - 1];
-    if (last) base.input = scrub(last.content);
-    if (llm.output !== void 0) base.output = scrub(llm.output);
-  } else if (span.kind === "tool") {
-    const tool = span;
-    base.input = asText(tool.args, scrub);
-    if (tool.result !== void 0) base.output = asText(tool.result, scrub);
-  }
-  return base;
-}
-function scrubMessages(messages, scrub) {
-  return messages.map((m) => ({ ...m, content: scrub(m.content) }));
-}
-function finalConversation(spans, scrub) {
-  const llms = spans.filter((s) => s.kind === "llm");
-  const last = llms[llms.length - 1];
-  if (!last) return [];
-  const messages = scrubMessages(last.messages, scrub);
-  if (last.output !== void 0 && last.output !== "") {
-    messages.push({ role: "assistant", content: scrub(last.output) });
-  }
-  return messages;
-}
-function rolloutReward(record) {
-  const gated = record.outcome.realness?.gated === true;
-  const raw = record.outcome.holdoutScore ?? record.outcome.searchScore ?? 0;
-  return { reward: gated ? 0 : raw, gated };
-}
-async function mintRolloutRows(records, store, options = {}) {
-  const scrub = options.scrub ?? ((t) => t);
-  const rows = [];
-  const missingTraces = [];
-  for (const record of records) {
-    const trajectory = await buildTrajectory(store, record.runId);
-    if (trajectory.steps.length === 0) {
-      missingTraces.push(record.runId);
-      continue;
-    }
-    let steps = trajectory.steps.map((s) => projectStep(s.span, scrub));
-    if (options.maxSteps !== void 0 && steps.length > options.maxSteps) {
-      const head = Math.ceil(options.maxSteps / 2);
-      const tail = options.maxSteps - head;
-      steps = [...steps.slice(0, head), ...steps.slice(steps.length - tail)];
-    }
-    const { reward, gated } = rolloutReward(record);
-    rows.push({
-      format: ROLLOUT_FORMAT,
-      runId: record.runId,
-      experimentId: record.experimentId,
-      candidateId: record.candidateId,
-      scenarioId: record.scenarioId,
-      splitTag: record.splitTag,
-      model: record.model,
-      agentProfileCellId: record.agentProfile?.cellId,
-      reward,
-      realnessGated: gated,
-      costUsd: record.costUsd,
-      totalTokens: record.tokenUsage.input + record.tokenUsage.output,
-      steps,
-      conversation: finalConversation(
-        trajectory.steps.map((s) => s.span),
-        scrub
-      )
-    });
-  }
-  return { rows, missingTraces };
-}
-function toSftRows(rows, options = {}) {
-  const minReward = options.minReward ?? 1;
-  return rows.filter((r) => r.reward >= minReward && !r.realnessGated && r.conversation.length > 0).map((r) => ({
-    messages: r.conversation,
-    metadata: {
-      runId: r.runId,
-      candidateId: r.candidateId,
-      scenarioId: r.scenarioId,
-      reward: r.reward
-    }
-  }));
-}
-function toRewardRows(rows) {
-  return rows.map((r) => ({
-    prompt: r.conversation.find((m) => m.role === "user")?.content ?? "",
-    steps: r.steps,
-    reward: r.reward,
-    metadata: {
-      runId: r.runId,
-      candidateId: r.candidateId,
-      scenarioId: r.scenarioId,
-      splitTag: r.splitTag
-    }
-  }));
-}
-function toJsonl(rows) {
-  return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length ? "\n" : "");
-}
-
 // src/treatment-gate.ts
 function gateTreatmentApplied(input, opts = {}) {
   const failOpen = opts.failOpenWhenNoTelemetry ?? true;
@@ -11757,6 +11678,7 @@ export {
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
   ROLLOUT_FORMAT,
+  ROLLOUT_SCHEMA,
   RUN_COST_ATTR_KEYS,
   ReplayCache,
   ReplayCacheMissError,
@@ -11767,6 +11689,7 @@ export {
   SEMANTIC_CONCEPT_JUDGE_VERSION,
   SKILL_USAGE_ANALYST,
   SPAN_KIND_ATTR_KEYS,
+  SUPERVISOR_RUN_SCHEMA,
   SandboxHarness,
   ScenarioRegistry,
   SeatUnsetError,
@@ -11808,6 +11731,8 @@ export {
   allCriticalPassed,
   analyzeAntiSlop,
   analyzeSeries,
+  analyzeSupervisorRun,
+  analyzeSupervisorRunSources,
   analyzeTraces,
   appendScorecard,
   applyLlmSpanOtlpAttributes,
@@ -11825,6 +11750,7 @@ export {
   assertRealAgentReceipts,
   assertRealBackend,
   assertReleaseConfidence,
+  assertRolloutLine,
   assertRunAgentProfileCell,
   assertRunCaptured,
   assertSingleBackend,
@@ -12041,10 +11967,13 @@ export {
   isOtelConfigured,
   isPolicyEdit,
   isRetrievalSpan,
+  isRolloutLine,
   isRunRecord,
   isSandboxSpan,
   isToolSpan,
+  isTrainableSplit,
   isTransientLlmError,
+  isUnavailable,
   iterateRawCalls,
   jestTestParser,
   jsonHasKeys,
@@ -12160,6 +12089,8 @@ export {
   renderPriorFindings,
   renderReleaseReport,
   renderSteeringText,
+  renderSupervisorRunHeadline,
+  renderSupervisorRunMarkdown,
   renderUpstreamFindings,
   repeatedActionDetector,
   replayFeedbackTrajectories,
@@ -12173,6 +12104,7 @@ export {
   resolveRunCostProvenance,
   resolveSeat,
   rolloutReward,
+  rollupSupervisorRuns,
   roundTripRunRecord,
   routeFields,
   rowCount,
@@ -12220,6 +12152,7 @@ export {
   selfPreference,
   sentenceReorderMutator,
   serializeFeedbackTrajectoriesJsonl,
+  showMeasured,
   signManifest,
   spearmanR,
   splitGold,
@@ -12235,6 +12168,7 @@ export {
   summarizePrReviewBenchmark,
   summarizePreferenceMemory,
   summaryTable,
+  supervisorRunRolloutLines,
   testJudge,
   textInSnapshot,
   throwIfRunIncomplete,
@@ -12264,6 +12198,7 @@ export {
   validateProductBenchmarkManifest,
   validateProductBenchmarkRecord,
   validateProductBenchmarkRun,
+  validateRolloutLine,
   validateRunRecord,
   verbosityBias,
   verifyAgentProfileCell,
@@ -12284,6 +12219,7 @@ export {
   withHeldoutBlend,
   withJudgeRetry,
   withOtelPipeline,
-  wranglerDeployRunner
+  wranglerDeployRunner,
+  writeSupervisorRunReport
 };
 //# sourceMappingURL=index.js.map