@tangle-network/agent-eval 0.120.1 → 0.120.2

package/dist/chunk-T4SQEITX.js

@@ -1,95 +0,0 @@
-// src/runtime-trajectory.ts
-var DEFAULT_SPLIT_TAG = "search";
-function projectRuntimeTrajectoryEvidence(options) {
-  const diagnostics = [];
-  const runsById = /* @__PURE__ */ new Map();
-  const events = [];
-  let recordWithRuntimeEventsCount = 0;
-  let defaultedSplitCount = 0;
-  for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {
-    const record = options.records[recordIndex];
-    const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf);
-    const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG;
-    if (record.splitTag === void 0) defaultedSplitCount += 1;
-    const rawEvents = record.runtimeEvents;
-    if (!Array.isArray(rawEvents)) {
-      diagnostics.push(
-        `${key}: runtimeEvents is not an array; no runtime run join can be extracted`
-      );
-      continue;
-    }
-    if (rawEvents.length === 0) {
-      diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`);
-      continue;
-    }
-    recordWithRuntimeEventsCount += 1;
-    for (let index = 0; index < rawEvents.length; index += 1) {
-      const event = parseRuntimeTrajectoryHookEvent(rawEvents[index]);
-      if (!event) {
-        diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`);
-        continue;
-      }
-      events.push(event);
-      const scenarioId = event.scenarioId ?? stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ?? stringOrUndefined(record.scenarioId);
-      const prior = runsById.get(event.runId);
-      if (!prior) {
-        runsById.set(event.runId, { runId: event.runId, scenarioId, splitTag });
-        continue;
-      }
-      if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) {
-        diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`);
-      }
-    }
-  }
-  const runs = [...runsById.values()];
-  return {
-    runs,
-    events,
-    summary: {
-      recordCount: options.records.length,
-      recordWithRuntimeEventsCount,
-      runtimeRunCount: runs.length,
-      lifecycleEventCount: events.length,
-      defaultedSplitCount
-    },
-    diagnostics
-  };
-}
-function parseRuntimeTrajectoryHookEvent(input) {
-  if (!isRecord(input)) return null;
-  if (typeof input.id !== "string" || input.id.length === 0) return null;
-  if (typeof input.runId !== "string" || input.runId.length === 0) return null;
-  if (typeof input.target !== "string" || input.target.length === 0) return null;
-  if (typeof input.phase !== "string" || input.phase.length === 0) return null;
-  if (typeof input.timestamp !== "number" || !Number.isFinite(input.timestamp)) return null;
-  return {
-    id: input.id,
-    runId: input.runId,
-    scenarioId: stringOrUndefined(input.scenarioId),
-    target: input.target,
-    phase: input.phase,
-    timestamp: input.timestamp,
-    stepIndex: finiteNumberOrUndefined(input.stepIndex),
-    parentId: stringOrUndefined(input.parentId),
-    payload: input.payload,
-    metadata: isRecord(input.metadata) ? { ...input.metadata } : void 0
-  };
-}
-function runtimeTrajectoryRecordKey(record, index, recordIdOf) {
-  return stringOrUndefined(recordIdOf?.(record, index)) ?? stringOrUndefined(record.id) ?? `record[${index}]`;
-}
-function isRecord(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function stringOrUndefined(value) {
-  return typeof value === "string" && value.length > 0 ? value : void 0;
-}
-function finiteNumberOrUndefined(value) {
-  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
-}
-
-export {
-  projectRuntimeTrajectoryEvidence,
-  parseRuntimeTrajectoryHookEvent
-};
-//# sourceMappingURL=chunk-T4SQEITX.js.map

package/dist/chunk-T4SQEITX.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/runtime-trajectory.ts"],"sourcesContent":["import type { RunSplitTag } from './run-record'\n\nexport interface RuntimeTrajectoryHookEvent {\n  id: string\n  runId: string\n  scenarioId?: string\n  target: string\n  phase: string\n  timestamp: number\n  stepIndex?: number\n  parentId?: string\n  payload?: unknown\n  metadata?: Record<string, unknown>\n}\n\nexport interface RuntimeTrajectoryRecord {\n  id?: string\n  scenarioId?: string\n  splitTag?: RunSplitTag\n  runtimeEvents?: unknown\n  [key: string]: unknown\n}\n\nexport interface RuntimeTrajectoryRunRecord {\n  runId: string\n  scenarioId?: string\n  splitTag: RunSplitTag\n}\n\nexport interface RuntimeTrajectoryEvidenceSummary {\n  recordCount: number\n  recordWithRuntimeEventsCount: number\n  runtimeRunCount: number\n  lifecycleEventCount: number\n  defaultedSplitCount: number\n}\n\nexport interface RuntimeTrajectoryEvidenceProjection {\n  runs: RuntimeTrajectoryRunRecord[]\n  events: RuntimeTrajectoryHookEvent[]\n  summary: RuntimeTrajectoryEvidenceSummary\n  diagnostics: string[]\n}\n\nexport interface ProjectRuntimeTrajectoryEvidenceOptions<\n  TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord,\n> {\n  records: TRecord[]\n  defaultSplitTag?: RunSplitTag\n  recordIdOf?: (record: TRecord, index: number) => string | undefined\n  scenarioIdOf?: (record: TRecord, index: number) => string | undefined\n}\n\nconst DEFAULT_SPLIT_TAG: RunSplitTag = 'search'\n\nexport function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(\n  options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>,\n): RuntimeTrajectoryEvidenceProjection {\n  const diagnostics: string[] = []\n  const runsById = new Map<string, RuntimeTrajectoryRunRecord>()\n  const events: RuntimeTrajectoryHookEvent[] = []\n  let recordWithRuntimeEventsCount = 0\n  let defaultedSplitCount = 0\n\n  for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {\n    const record = options.records[recordIndex]!\n    const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf)\n    const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG\n    if (record.splitTag === undefined) defaultedSplitCount += 1\n\n    const rawEvents = record.runtimeEvents\n    if (!Array.isArray(rawEvents)) {\n      diagnostics.push(\n        `${key}: runtimeEvents is not an array; no runtime run join can be extracted`,\n      )\n      continue\n    }\n    if (rawEvents.length === 0) {\n      diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`)\n      continue\n    }\n    recordWithRuntimeEventsCount += 1\n\n    for (let index = 0; index < rawEvents.length; index += 1) {\n      const event = parseRuntimeTrajectoryHookEvent(rawEvents[index])\n      if (!event) {\n        diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`)\n        continue\n      }\n      events.push(event)\n\n      const scenarioId =\n        event.scenarioId ??\n        stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ??\n        stringOrUndefined(record.scenarioId)\n      const prior = runsById.get(event.runId)\n      if (!prior) {\n        runsById.set(event.runId, { runId: event.runId, scenarioId, splitTag })\n        continue\n      }\n      if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) {\n        diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`)\n      }\n    }\n  }\n\n  const runs = [...runsById.values()]\n  return {\n    runs,\n    events,\n    summary: {\n      recordCount: options.records.length,\n      recordWithRuntimeEventsCount,\n      runtimeRunCount: runs.length,\n      lifecycleEventCount: events.length,\n      defaultedSplitCount,\n    },\n    diagnostics,\n  }\n}\n\nexport function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null {\n  if (!isRecord(input)) return null\n  if (typeof input.id !== 'string' || input.id.length === 0) return null\n  if (typeof input.runId !== 'string' || input.runId.length === 0) return null\n  if (typeof input.target !== 'string' || input.target.length === 0) return null\n  if (typeof input.phase !== 'string' || input.phase.length === 0) return null\n  if (typeof input.timestamp !== 'number' || !Number.isFinite(input.timestamp)) return null\n\n  return {\n    id: input.id,\n    runId: input.runId,\n    scenarioId: stringOrUndefined(input.scenarioId),\n    target: input.target,\n    phase: input.phase,\n    timestamp: input.timestamp,\n    stepIndex: finiteNumberOrUndefined(input.stepIndex),\n    parentId: stringOrUndefined(input.parentId),\n    payload: input.payload,\n    metadata: isRecord(input.metadata) ? { ...input.metadata } : undefined,\n  }\n}\n\nfunction runtimeTrajectoryRecordKey<TRecord extends RuntimeTrajectoryRecord>(\n  record: TRecord,\n  index: number,\n  recordIdOf?: (record: TRecord, index: number) => string | undefined,\n): string {\n  return (\n    stringOrUndefined(recordIdOf?.(record, index)) ??\n    stringOrUndefined(record.id) ??\n    `record[${index}]`\n  )\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n\nfunction stringOrUndefined(value: unknown): string | undefined {\n  return typeof value === 'string' && value.length > 0 ? value : undefined\n}\n\nfunction finiteNumberOrUndefined(value: unknown): number | undefined {\n  return typeof value === 'number' && Number.isFinite(value) ? value : undefined\n}\n"],"mappings":";AAqDA,IAAM,oBAAiC;AAEhC,SAAS,iCACd,SACqC;AACrC,QAAM,cAAwB,CAAC;AAC/B,QAAM,WAAW,oBAAI,IAAwC;AAC7D,QAAM,SAAuC,CAAC;AAC9C,MAAI,+BAA+B;AACnC,MAAI,sBAAsB;AAE1B,WAAS,cAAc,GAAG,cAAc,QAAQ,QAAQ,QAAQ,eAAe,GAAG;AAChF,UAAM,SAAS,QAAQ,QAAQ,WAAW;AAC1C,UAAM,MAAM,2BAA2B,QAAQ,aAAa,QAAQ,UAAU;AAC9E,UAAM,WAAW,OAAO,YAAY,QAAQ,mBAAmB;AAC/D,QAAI,OAAO,aAAa,OAAW,wBAAuB;AAE1D,UAAM,YAAY,OAAO;AACzB,QAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;AAC7B,kBAAY;AAAA,QACV,GAAG,GAAG;AAAA,MACR;AACA;AAAA,IACF;AACA,QAAI,UAAU,WAAW,GAAG;AAC1B,kBAAY,KAAK,GAAG,GAAG,0DAA0D;AACjF;AAAA,IACF;AACA,oCAAgC;AAEhC,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,GAAG;AACxD,YAAM,QAAQ,gCAAgC,UAAU,KAAK,CAAC;AAC9D,UAAI,CAAC,OAAO;AACV,oBAAY,KAAK,GAAG,GAAG,mBAAmB,KAAK,6BAA6B;AAC5E;AAAA,MACF;AACA,aAAO,KAAK,KAAK;AAEjB,YAAM,aACJ,MAAM,cACN,kBAAkB,QAAQ,eAAe,QAAQ,WAAW,CAAC,KAC7D,kBAAkB,OAAO,UAAU;AACrC,YAAM,QAAQ,SAAS,IAAI,MAAM,KAAK;AACtC,UAAI,CAAC,OAAO;AACV,iBAAS,IAAI,MAAM,OAAO,EAAE,OAAO,MAAM,OAAO,YAAY,SAAS,CAAC;AACtE;AAAA,MACF;AACA,UAAI,MAAM,eAAe,cAAc,MAAM,aAAa,UAAU;AAClE,oBAAY,KAAK,GAAG,GAAG,WAAW,MAAM,KAAK,0CAA0C;AAAA,MACzF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC;AAClC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS;AAAA,MACP,aAAa,QAAQ,QAAQ;AAAA,MAC7B;AAAA,MACA,iBAAiB,KAAK;AAAA,MACtB,qBAAqB,OAAO;AAAA,MAC5B;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,gCAAgC,OAAmD;AACjG,MAAI,CAAC,SAAS,KAAK,EAAG,QAAO;AAC7B,MAAI,OAAO,MAAM,OAAO,YAAY,MAAM,GAAG,WAAW,EAAG,QAAO;AAClE,MAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,EAAG,QAAO;AACxE,MAAI,OAAO,MAAM,WAAW,YAAY,MAAM,OAAO,WAAW,EAAG,QAAO;AAC1E,MAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,EAAG,QAAO;AACxE,MAAI,OAAO,MAAM,cAAc,YAAY,CAAC,OAAO,SAAS,MAAM,SAAS,EAAG,QAAO;AAErF,SAAO;AAAA,IACL,IAAI,MAAM;AAAA,IACV,OAAO,MAAM;AAAA,IACb,YAAY,kBAAkB,MAAM,UAAU;AAAA,IAC9C,QAAQ,MAAM;AAAA,IACd,OAAO,MAAM;AAAA,IACb,WAAW,MAAM;AAAA,IACjB,WAAW,wBAAwB,MAAM,SAAS;AAAA,IAClD,UAAU,kBAAkB,MAAM,QAAQ;AAAA,IAC1C,SAAS,MAAM;AAAA,IACf,UAAU,SAAS,MAAM,QAAQ,IAAI,EAAE,GAAG,MAAM,SAAS,IAAI;AAAA,EAC/D;AACF;AAEA,SAAS,2BACP,QACA,OACA,YACQ;AACR,SACE,kBAAkB,aAAa,QAAQ,KAAK,CAAC,KAC7C,kBAAkB,OAAO,EAAE,KAC3B,UAAU,KAAK;AAEnB;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,kBAAkB,OAAoC;AAC7D,SAAO,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ;AACjE;AAEA,SAAS,wBAAwB,OAAoC;AACnE,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE;","names":[]}

package/dist/chunk-TT4KNT67.js

@@ -1,124 +0,0 @@
-import {
-  CaptureIntegrityError
-} from "./chunk-ONWEPEDO.js";
-
-// src/trace/integrity.ts
-var RunIntegrityError = class extends CaptureIntegrityError {
-  constructor(report) {
-    super(
-      `Run ${report.runId} failed integrity check: ${report.issues.map((i) => i.code).join(", ")}`
-    );
-    this.report = report;
-  }
-  report;
-};
-async function assertRunCaptured(store, runId, expectations = {}) {
-  const issues = [];
-  const run = await store.getRun(runId);
-  if (!run) {
-    return {
-      ok: false,
-      runId,
-      llmSpanCount: 0,
-      judgeSpanCount: 0,
-      toolSpanCount: 0,
-      rawProviderEventCount: 0,
-      rawSpanCoverage: { covered: 0, total: 0 },
-      issues: [{ code: "no_run", message: `Run ${runId} not found in store.` }]
-    };
-  }
-  const spans = await store.spans({ runId });
-  const llmSpans = spans.filter((s) => s.kind === "llm");
-  const judgeSpans = spans.filter((s) => s.kind === "judge");
-  const toolSpans = spans.filter((s) => s.kind === "tool");
-  const llmMin = expectations.llmSpansMin ?? 0;
-  const judgeMin = expectations.judgeSpansMin ?? 0;
-  const toolMin = expectations.toolSpansMin ?? 0;
-  if (llmSpans.length < llmMin) {
-    issues.push({
-      code: "missing_llm_spans",
-      message: `Expected \u2265 ${llmMin} LLM spans, found ${llmSpans.length}.`,
-      detail: { expected: llmMin, found: llmSpans.length }
-    });
-  }
-  if (judgeSpans.length < judgeMin) {
-    issues.push({
-      code: "missing_judge_spans",
-      message: `Expected \u2265 ${judgeMin} judge spans, found ${judgeSpans.length}.`,
-      detail: { expected: judgeMin, found: judgeSpans.length }
-    });
-  }
-  if (toolSpans.length < toolMin) {
-    issues.push({
-      code: "missing_tool_spans",
-      message: `Expected \u2265 ${toolMin} tool spans, found ${toolSpans.length}.`,
-      detail: { expected: toolMin, found: toolSpans.length }
-    });
-  }
-  let rawEventCount = 0;
-  let coverage = { covered: 0, total: llmSpans.length };
-  if (expectations.rawSink) {
-    if (!expectations.rawSink.list) {
-      issues.push({
-        code: "no_raw_sink",
-        message: "Provided rawSink does not implement list(); cannot verify capture."
-      });
-    } else {
-      const events = await expectations.rawSink.list({ runId });
-      rawEventCount = events.length;
-      const rawMin = expectations.rawProviderEventsMin ?? 1;
-      if (rawEventCount < rawMin) {
-        issues.push({
-          code: "missing_raw_events",
-          message: `Expected \u2265 ${rawMin} raw provider events, found ${rawEventCount}.`,
-          detail: { expected: rawMin, found: rawEventCount }
-        });
-      }
-      if (expectations.requireRawCoverageOfLlmSpans) {
-        const requestEventsBySpan = new Set(
-          events.filter((e) => e.direction === "request" && e.spanId).map((e) => e.spanId)
-        );
-        const orphaned = llmSpans.filter((s) => !requestEventsBySpan.has(s.spanId));
-        coverage = { covered: llmSpans.length - orphaned.length, total: llmSpans.length };
-        if (orphaned.length > 0) {
-          issues.push({
-            code: "orphan_llm_span",
-            message: `${orphaned.length} LLM span(s) have no matching raw provider request event.`,
-            detail: { orphanedSpanIds: orphaned.map((s) => s.spanId) }
-          });
-        }
-      }
-    }
-  } else if (expectations.requireRawCoverageOfLlmSpans || expectations.rawProviderEventsMin) {
-    issues.push({
-      code: "no_raw_sink",
-      message: "Raw coverage required but no rawSink supplied to the integrity check."
-    });
-  }
-  if (expectations.requireOutcome && (run.outcome === void 0 || run.outcome === null)) {
-    issues.push({
-      code: "missing_outcome",
-      message: `Run ${runId} has no outcome recorded.`
-    });
-  }
-  return {
-    ok: issues.length === 0,
-    runId,
-    llmSpanCount: llmSpans.length,
-    judgeSpanCount: judgeSpans.length,
-    toolSpanCount: toolSpans.length,
-    rawProviderEventCount: rawEventCount,
-    rawSpanCoverage: coverage,
-    issues
-  };
-}
-function throwIfRunIncomplete(report) {
-  if (!report.ok) throw new RunIntegrityError(report);
-}
-
-export {
-  RunIntegrityError,
-  assertRunCaptured,
-  throwIfRunIncomplete
-};
-//# sourceMappingURL=chunk-TT4KNT67.js.map

package/dist/chunk-TT4KNT67.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/trace/integrity.ts"],"sourcesContent":["/**\n * Run-completion integrity check — at end of run, verify the expected event\n * types were actually captured. The point is the launch-review failure mode:\n * a run *appears* successful but the raw provider events were never written,\n * so a downstream reviewer can't reconstruct what happened.\n *\n * Pattern:\n *\n *   const report = await assertRunCaptured(store, runId, {\n *     llmSpansMin: 1,\n *     judgeSpansMin: 1,\n *     rawSink: providerSink,                  // must have ≥ 1 event for this run\n *     requireRawCoverageOfLlmSpans: true,     // every llm span has matching raw events\n *   })\n *   if (!report.ok) throwIfRunIncomplete(report)  // or mark run failed and continue\n *\n * The function is read-only on the store and returns a structured report;\n * the caller chooses the failure mode (throw, mark run failed, log warning).\n * `throwIfRunIncomplete` is the convenient strict mode.\n */\n\nimport { CaptureIntegrityError } from '../errors'\nimport type { RawProviderSink } from './raw-provider-sink'\nimport type { TraceStore } from './store'\n\nexport interface RunIntegrityExpectations {\n  /** Minimum LLM span count. Default 0 (no requirement). */\n  llmSpansMin?: number\n  /** Minimum judge span count. Default 0. */\n  judgeSpansMin?: number\n  /** Minimum tool span count. Default 0. */\n  toolSpansMin?: number\n  /**\n   * Raw provider sink to consult for capture verification. When present,\n   * the check requires at least one raw event for the run.\n   */\n  rawSink?: RawProviderSink\n  /** Minimum raw provider event count. Default 0; ignored when `rawSink` absent. */\n  rawProviderEventsMin?: number\n  /**\n   * Every LLM span must have at least one matching raw `request` event\n   * (matched by spanId). Catches the common bug where the structured span\n   * was emitted but the raw HTTP capture was wired to a different sink.\n   */\n  requireRawCoverageOfLlmSpans?: boolean\n  /** Run outcome must be set (not null/undefined). Default false. */\n  requireOutcome?: boolean\n}\n\nexport type RunIntegrityIssueCode =\n  | 'no_run'\n  | 'missing_llm_spans'\n  | 'missing_judge_spans'\n  | 'missing_tool_spans'\n  | 'missing_raw_events'\n  | 'no_raw_sink'\n  | 'orphan_llm_span'\n  | 'missing_outcome'\n\nexport interface RunIntegrityIssue {\n  code: RunIntegrityIssueCode\n  message: string\n  detail?: Record<string, unknown>\n}\n\nexport interface RunIntegrityReport {\n  ok: boolean\n  runId: string\n  llmSpanCount: number\n  judgeSpanCount: number\n  toolSpanCount: number\n  rawProviderEventCount: number\n  /**\n   * Coverage of LLM spans by raw provider events keyed on spanId.\n   * `total` is the number of LLM spans; `covered` is the count with at\n   * least one matching `request` raw event.\n   */\n  rawSpanCoverage: { covered: number; total: number }\n  issues: RunIntegrityIssue[]\n}\n\nexport class RunIntegrityError extends CaptureIntegrityError {\n  constructor(public readonly report: RunIntegrityReport) {\n    super(\n      `Run ${report.runId} failed integrity check: ${report.issues.map((i) => i.code).join(', ')}`,\n    )\n  }\n}\n\nexport async function assertRunCaptured(\n  store: TraceStore,\n  runId: string,\n  expectations: RunIntegrityExpectations = {},\n): Promise<RunIntegrityReport> {\n  const issues: RunIntegrityIssue[] = []\n  const run = await store.getRun(runId)\n  if (!run) {\n    return {\n      ok: false,\n      runId,\n      llmSpanCount: 0,\n      judgeSpanCount: 0,\n      toolSpanCount: 0,\n      rawProviderEventCount: 0,\n      rawSpanCoverage: { covered: 0, total: 0 },\n      issues: [{ code: 'no_run', message: `Run ${runId} not found in store.` }],\n    }\n  }\n\n  const spans = await store.spans({ runId })\n  const llmSpans = spans.filter((s) => s.kind === 'llm')\n  const judgeSpans = spans.filter((s) => s.kind === 'judge')\n  const toolSpans = spans.filter((s) => s.kind === 'tool')\n\n  const llmMin = expectations.llmSpansMin ?? 0\n  const judgeMin = expectations.judgeSpansMin ?? 0\n  const toolMin = expectations.toolSpansMin ?? 0\n\n  if (llmSpans.length < llmMin) {\n    issues.push({\n      code: 'missing_llm_spans',\n      message: `Expected ≥ ${llmMin} LLM spans, found ${llmSpans.length}.`,\n      detail: { expected: llmMin, found: llmSpans.length },\n    })\n  }\n  if (judgeSpans.length < judgeMin) {\n    issues.push({\n      code: 'missing_judge_spans',\n      message: `Expected ≥ ${judgeMin} judge spans, found ${judgeSpans.length}.`,\n      detail: { expected: judgeMin, found: judgeSpans.length },\n    })\n  }\n  if (toolSpans.length < toolMin) {\n    issues.push({\n      code: 'missing_tool_spans',\n      message: `Expected ≥ ${toolMin} tool spans, found ${toolSpans.length}.`,\n      detail: { expected: toolMin, found: toolSpans.length },\n    })\n  }\n\n  let rawEventCount = 0\n  let coverage = { covered: 0, total: llmSpans.length }\n\n  if (expectations.rawSink) {\n    if (!expectations.rawSink.list) {\n      issues.push({\n        code: 'no_raw_sink',\n        message: 'Provided rawSink does not implement list(); cannot verify capture.',\n      })\n    } else {\n      const events = await expectations.rawSink.list({ runId })\n      rawEventCount = events.length\n      const rawMin = expectations.rawProviderEventsMin ?? 1\n      if (rawEventCount < rawMin) {\n        issues.push({\n          code: 'missing_raw_events',\n          message: `Expected ≥ ${rawMin} raw provider events, found ${rawEventCount}.`,\n          detail: { expected: rawMin, found: rawEventCount },\n        })\n      }\n      if (expectations.requireRawCoverageOfLlmSpans) {\n        const requestEventsBySpan = new Set(\n          events.filter((e) => e.direction === 'request' && e.spanId).map((e) => e.spanId!),\n        )\n        const orphaned = llmSpans.filter((s) => !requestEventsBySpan.has(s.spanId))\n        coverage = { covered: llmSpans.length - orphaned.length, total: llmSpans.length }\n        if (orphaned.length > 0) {\n          issues.push({\n            code: 'orphan_llm_span',\n            message: `${orphaned.length} LLM span(s) have no matching raw provider request event.`,\n            detail: { orphanedSpanIds: orphaned.map((s) => s.spanId) },\n          })\n        }\n      }\n    }\n  } else if (expectations.requireRawCoverageOfLlmSpans || expectations.rawProviderEventsMin) {\n    issues.push({\n      code: 'no_raw_sink',\n      message: 'Raw coverage required but no rawSink supplied to the integrity check.',\n    })\n  }\n\n  if (expectations.requireOutcome && (run.outcome === undefined || run.outcome === null)) {\n    issues.push({\n      code: 'missing_outcome',\n      message: `Run ${runId} has no outcome recorded.`,\n    })\n  }\n\n  return {\n    ok: issues.length === 0,\n    runId,\n    llmSpanCount: llmSpans.length,\n    judgeSpanCount: judgeSpans.length,\n    toolSpanCount: toolSpans.length,\n    rawProviderEventCount: rawEventCount,\n    rawSpanCoverage: coverage,\n    issues,\n  }\n}\n\n/** Strict mode: throws `RunIntegrityError` when the report isn't ok. */\nexport function throwIfRunIncomplete(report: RunIntegrityReport): void {\n  if (!report.ok) throw new RunIntegrityError(report)\n}\n"],"mappings":";;;;;AAiFO,IAAM,oBAAN,cAAgC,sBAAsB;AAAA,EAC3D,YAA4B,QAA4B;AACtD;AAAA,MACE,OAAO,OAAO,KAAK,4BAA4B,OAAO,OAAO,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,IAC5F;AAH0B;AAAA,EAI5B;AAAA,EAJ4B;AAK9B;AAEA,eAAsB,kBACpB,OACA,OACA,eAAyC,CAAC,GACb;AAC7B,QAAM,SAA8B,CAAC;AACrC,QAAM,MAAM,MAAM,MAAM,OAAO,KAAK;AACpC,MAAI,CAAC,KAAK;AACR,WAAO;AAAA,MACL,IAAI;AAAA,MACJ;AAAA,MACA,cAAc;AAAA,MACd,gBAAgB;AAAA,MAChB,eAAe;AAAA,MACf,uBAAuB;AAAA,MACvB,iBAAiB,EAAE,SAAS,GAAG,OAAO,EAAE;AAAA,MACxC,QAAQ,CAAC,EAAE,MAAM,UAAU,SAAS,OAAO,KAAK,uBAAuB,CAAC;AAAA,IAC1E;AAAA,EACF;AAEA,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,CAAC;AACzC,QAAM,WAAW,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK;AACrD,QAAM,aAAa,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO;AACzD,QAAM,YAAY,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM;AAEvD,QAAM,SAAS,aAAa,eAAe;AAC3C,QAAM,WAAW,aAAa,iBAAiB;AAC/C,QAAM,UAAU,aAAa,gBAAgB;AAE7C,MAAI,SAAS,SAAS,QAAQ;AAC5B,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,SAAS,mBAAc,MAAM,qBAAqB,SAAS,MAAM;AAAA,MACjE,QAAQ,EAAE,UAAU,QAAQ,OAAO,SAAS,OAAO;AAAA,IACrD,CAAC;AAAA,EACH;AACA,MAAI,WAAW,SAAS,UAAU;AAChC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,SAAS,mBAAc,QAAQ,uBAAuB,WAAW,MAAM;AAAA,MACvE,QAAQ,EAAE,UAAU,UAAU,OAAO,WAAW,OAAO;AAAA,IACzD,CAAC;AAAA,EACH;AACA,MAAI,UAAU,SAAS,SAAS;AAC9B,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,SAAS,mBAAc,OAAO,sBAAsB,UAAU,MAAM;AAAA,MACpE,QAAQ,EAAE,UAAU,SAAS,OAAO,UAAU,OAAO;AAAA,IACvD,CAAC;AAAA,EACH;AAEA,MAAI,gBAAgB;AACpB,MAAI,WAAW,EAAE,SAAS,GAAG,OAAO,SAAS,OAAO;AAEpD,MAAI,aAAa,SAAS;AACxB,QAAI,CAAC,aAAa,QAAQ,MAAM;AAC9B,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL,YAAM,SAAS,MAAM,aAAa,QAAQ,KAAK,EAAE,MAAM,CAAC;AACxD,sBAAgB,OAAO;AACvB,YAAM,SAAS,aAAa,wBAAwB;AACpD,UAAI,gBAAgB,QAAQ;AAC1B,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,SAAS,mBAAc,MAAM,+BAA+B,aAAa;AAAA,UACzE,QAAQ,EAAE,UAAU,QAAQ,OAAO,cAAc;AAAA,QACnD,CAAC;AAAA,MACH;AACA,UAAI,aAAa,8BAA8B;AAC7C,cAAM,sBAAsB,IAAI;AAAA,UAC9B,OAAO,OAAO,CAAC,MAAM,EAAE,cAAc,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAO;AAAA,QAClF;AACA,cAAM,WAAW,SAAS,OAAO,CAAC,MAAM,CAAC,oBAAoB,IAAI,EAAE,MAAM,CAAC;AAC1E,mBAAW,EAAE,SAAS,SAAS,SAAS,SAAS,QAAQ,OAAO,SAAS,OAAO;AAChF,YAAI,SAAS,SAAS,GAAG;AACvB,iBAAO,KAAK;AAAA,YACV,MAAM;AAAA,YACN,SAAS,GAAG,SAAS,MAAM;AAAA,YAC3B,QAAQ,EAAE,iBAAiB,SAAS,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE;AAAA,UAC3D,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF,WAAW,aAAa,gCAAgC,aAAa,sBAAsB;AACzF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI,aAAa,mBAAmB,IAAI,YAAY,UAAa,IAAI,YAAY,OAAO;AACtF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,SAAS,OAAO,KAAK;AAAA,IACvB,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,IAAI,OAAO,WAAW;AAAA,IACtB;AAAA,IACA,cAAc,SAAS;AAAA,IACvB,gBAAgB,WAAW;AAAA,IAC3B,eAAe,UAAU;AAAA,IACzB,uBAAuB;AAAA,IACvB,iBAAiB;AAAA,IACjB;AAAA,EACF;AACF;AAGO,SAAS,qBAAqB,QAAkC;AACrE,MAAI,CAAC,OAAO,GAAI,OAAM,IAAI,kBAAkB,MAAM;AACpD;","names":[]}

package/dist/chunk-U5CHZ5M3.js

@@ -1,357 +0,0 @@
-import {
-  assertLlmRoute
-} from "./chunk-NJC7U437.js";
-import {
-  researchReport
-} from "./chunk-DPZAEKA6.js";
-import {
-  RunIntegrityError,
-  assertRunCaptured
-} from "./chunk-TT4KNT67.js";
-import {
-  FileSystemRawProviderSink
-} from "./chunk-PC4UYEBM.js";
-import {
-  TraceEmitter
-} from "./chunk-VQMK5FMP.js";
-import {
-  validateRunRecord
-} from "./chunk-S3UZOQ5Y.js";
-import {
-  buildAgentProfileCell,
-  verifyAgentProfileCell
-} from "./chunk-XJYR7XFV.js";
-import {
-  canonicalize,
-  hashJson
-} from "./chunk-VSMTAMNK.js";
-
-// src/eval-campaign.ts
-var DEFAULT_INTEGRITY = {
-  llmSpansMin: 1,
-  requireRawCoverageOfLlmSpans: true,
-  requireOutcome: true
-};
-var DEFAULT_ROUTE = {
-  requireExplicitBaseUrl: true,
-  requireAuth: true
-};
-async function runEvalCampaign(opts) {
-  assertLlmRoute(opts.llmOpts, opts.routeRequirements ?? DEFAULT_ROUTE);
-  if (opts.variants.length === 0) {
-    throw new Error("runEvalCampaign: variants must be non-empty.");
-  }
-  if (opts.scenarios.length === 0) {
-    throw new Error("runEvalCampaign: scenarios must be non-empty.");
-  }
-  const variantIds = /* @__PURE__ */ new Set();
-  for (const v of opts.variants) {
-    if (variantIds.has(v.id)) {
-      throw new Error(`runEvalCampaign: duplicate variant id "${v.id}".`);
-    }
-    variantIds.add(v.id);
-  }
-  const scenarioIds = /* @__PURE__ */ new Set();
-  for (const s of opts.scenarios) {
-    if (scenarioIds.has(s.scenarioId)) {
-      throw new Error(`runEvalCampaign: duplicate scenarioId "${s.scenarioId}".`);
-    }
-    scenarioIds.add(s.scenarioId);
-  }
-  if (opts.report?.comparator && !variantIds.has(opts.report.comparator)) {
-    throw new Error(
-      `runEvalCampaign: report.comparator "${opts.report.comparator}" is not a configured variantId.`
-    );
-  }
-  if (!opts.commitSha) {
-    throw new Error("runEvalCampaign: commitSha is required (every RunRecord needs it).");
-  }
-  const seeds = opts.seeds ?? [0, 1, 2];
-  const splitTag = opts.splitTag ?? "holdout";
-  const concurrency = Math.max(1, opts.concurrency ?? 1);
-  const integrity = { ...DEFAULT_INTEGRITY, ...opts.integrity ?? {} };
-  const onIntegrityFailure = opts.onIntegrityFailure ?? "mark_failed";
-  const now = opts.now ?? (() => Date.now());
-  const baseUrl = (opts.llmOpts.baseUrl ?? "").replace(/\/+$/, "");
-  const provider = opts.llmOpts.provider ?? null;
-  const preregistrationHash = opts.preregistrationHash ?? null;
-  const rawSinkFactory = opts.rawSinkFactory ?? defaultRawSinkFactory(opts.workDir);
-  const campaignFingerprint = await hashJson(
-    canonicalize({
-      campaignId: opts.campaignId,
-      variants: opts.variants.map((v) => v.id).sort(),
-      scenarios: opts.scenarios.map((s) => s.scenarioId).sort(),
-      seeds: [...seeds].sort((a, b) => a - b),
-      splitTag,
-      comparator: opts.report?.comparator ?? null,
-      baseUrl,
-      provider,
-      preregistrationHash
-    })
-  );
-  const cells = [];
-  for (const variant of opts.variants) {
-    for (const scenario of opts.scenarios) {
-      for (const seed of seeds) {
-        cells.push({ variant, scenario, seed });
-      }
-    }
-  }
-  const startedAt = new Date(now()).toISOString();
-  const runs = [];
-  const integrityReports = [];
-  const failedRuns = [];
-  let cursor = 0;
-  let aborting = false;
-  const genuineErrors = [];
-  const openRuns = /* @__PURE__ */ new Map();
-  async function worker() {
-    while (!aborting) {
-      const i = cursor++;
-      if (i >= cells.length) return;
-      const cell = cells[i];
-      try {
-        const result = await runOneCell(cell);
-        runs.push(result.record);
-        integrityReports.push(result.integrity);
-      } catch (err) {
-        if (err instanceof CellExecutionError) {
-          failedRuns.push(err.failed);
-          if (err.integrity) integrityReports.push(err.integrity);
-        } else {
-          genuineErrors.push(err);
-          aborting = true;
-          return;
-        }
-      }
-    }
-  }
-  async function runOneCell(cell) {
-    const runId = (opts.runId ?? defaultRunId)({
-      campaignId: opts.campaignId,
-      runId: "",
-      // unused by default generator
-      variantId: cell.variant.id,
-      scenarioId: cell.scenario.scenarioId,
-      seed: cell.seed
-    });
-    const factoryParams = {
-      campaignId: opts.campaignId,
-      runId,
-      variantId: cell.variant.id,
-      scenarioId: cell.scenario.scenarioId,
-      seed: cell.seed
-    };
-    const store = opts.storeFactory(factoryParams);
-    const rawSink = rawSinkFactory(factoryParams);
-    const emitter = new TraceEmitter(store, {
-      runId,
-      now: opts.now,
-      onRunComplete: opts.onRunComplete
-    });
-    openRuns.set(runId, emitter);
-    const llmOpts = {
-      ...opts.llmOpts,
-      rawSink,
-      traceContext: { runId }
-    };
-    const ctx = {
-      runId,
-      experimentId: opts.campaignId,
-      variant: cell.variant.payload,
-      variantId: cell.variant.id,
-      scenarioId: cell.scenario.scenarioId,
-      scenarioTags: cell.scenario.tags ?? {},
-      seed: cell.seed,
-      splitTag,
-      emitter,
-      store,
-      rawSink,
-      llmOpts
-    };
-    try {
-      const wallStart = now();
-      let outcome;
-      try {
-        outcome = await opts.runner(ctx);
-      } catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        await finalizeAbort(emitter, runId, message);
-        throw new CellExecutionError({
-          runId,
-          variantId: cell.variant.id,
-          scenarioId: cell.scenario.scenarioId,
-          seed: cell.seed,
-          reason: "runner_threw",
-          error: message
-        });
-      }
-      const wallMs = now() - wallStart;
-      const integrityReport = await assertRunCaptured(store, runId, { ...integrity, rawSink });
-      if (!integrityReport.ok) {
-        switch (onIntegrityFailure) {
-          case "throw":
-            throw new RunIntegrityError(integrityReport);
-          case "mark_failed":
-            throw new CellExecutionError(
-              {
-                runId,
-                variantId: cell.variant.id,
-                scenarioId: cell.scenario.scenarioId,
-                seed: cell.seed,
-                reason: "integrity_failed",
-                error: integrityReport.issues.map((i) => i.code).join(", ")
-              },
-              integrityReport
-            );
-          case "log":
-            break;
-        }
-      }
-      const recordOutcome = {
-        raw: outcome.raw ?? {}
-      };
-      if (splitTag === "holdout") recordOutcome.holdoutScore = outcome.score;
-      else recordOutcome.searchScore = outcome.score;
-      if (outcome.judgeScores !== void 0) recordOutcome.judgeScores = outcome.judgeScores;
-      const record = {
-        runId,
-        experimentId: opts.campaignId,
-        candidateId: cell.variant.id,
-        seed: cell.seed,
-        model: outcome.model,
-        promptHash: outcome.promptHash,
-        configHash: outcome.configHash,
-        commitSha: opts.commitSha,
-        wallMs,
-        costUsd: outcome.costUsd,
-        tokenUsage: outcome.tokenUsage,
-        judgeMetadata: outcome.judgeMetadata,
-        outcome: recordOutcome,
-        ...outcome.failureClass ? { failureClass: outcome.failureClass } : {},
-        failureMode: outcome.failureMode,
-        splitTag,
-        scenarioId: cell.scenario.scenarioId
-      };
-      const profileSource = outcome.agentProfile ?? (typeof opts.agentProfile === "function" ? await opts.agentProfile({
-        campaignId: opts.campaignId,
-        runId,
-        variantId: cell.variant.id,
-        scenarioId: cell.scenario.scenarioId,
-        seed: cell.seed,
-        variant: cell.variant.payload,
-        scenarioTags: cell.scenario.tags ?? {}
-      }) : opts.agentProfile);
-      if (profileSource !== void 0) {
-        const agentProfile = await resolveAgentProfileCell(profileSource);
-        assertAgentProfileMatchesRun(agentProfile, outcome.model, outcome.promptHash);
-        record.agentProfile = agentProfile;
-      }
-      return { record: validateRunRecord(record), integrity: integrityReport };
-    } finally {
-      openRuns.delete(runId);
-    }
-  }
-  const workers = Array.from({ length: Math.min(concurrency, cells.length) }, () => worker());
-  await Promise.allSettled(workers);
-  for (const [runId, emitter] of openRuns) {
-    await finalizeAbort(emitter, runId, "campaign aborted: genuine error in a sibling run");
-  }
-  openRuns.clear();
-  if (genuineErrors.length > 0) {
-    throw genuineErrors.length === 1 ? genuineErrors[0] : new AggregateError(
-      genuineErrors,
-      `runEvalCampaign: ${genuineErrors.length} runs failed with genuine (non-run-level) errors`
-    );
-  }
-  let report;
-  if (opts.report) {
-    const reportOpts = {
-      ...opts.report,
-      comparator: opts.report.comparator,
-      split: splitTag === "dev" ? "search" : splitTag,
-      generatedAt: new Date(now()).toISOString(),
-      preregistrationHash: preregistrationHash ?? void 0
-    };
-    report = await researchReport(runs, reportOpts);
-  }
-  const endedAt = new Date(now()).toISOString();
-  return {
-    campaignId: opts.campaignId,
-    campaignFingerprint,
-    preregistrationHash,
-    runs,
-    integrityReports,
-    failedRuns,
-    report,
-    startedAt,
-    endedAt
-  };
-}
-var CellExecutionError = class extends Error {
-  failed;
-  integrity;
-  constructor(failed, integrity) {
-    super(`cell ${failed.variantId}/${failed.scenarioId}@${failed.seed} failed: ${failed.reason}`);
-    this.failed = failed;
-    this.integrity = integrity;
-  }
-};
-async function finalizeAbort(emitter, runId, reason) {
-  const existing = await emitter.traceStore.getRun(runId);
-  if (existing === void 0) return;
-  if (existing.status !== "running") return;
-  await emitter.abortRun(reason);
-}
-function defaultRawSinkFactory(workDir) {
-  return (params) => {
-    if (!workDir) {
-      throw new Error(
-        "runEvalCampaign: rawSinkFactory not supplied and workDir not set. Pass either to enable raw provider capture, or pass `new NoopRawProviderSink()` via rawSinkFactory to opt out explicitly."
-      );
-    }
-    return new FileSystemRawProviderSink({
-      dir: `${workDir}/raw-events/${params.runId}`
-    });
-  };
-}
-async function resolveAgentProfileCell(input) {
-  if (isAgentProfileCell(input)) {
-    if (!await verifyAgentProfileCell(input)) {
-      throw new Error(`runEvalCampaign: agentProfile.cellId does not match its content`);
-    }
-    return input;
-  }
-  return buildAgentProfileCell(input);
-}
-function isAgentProfileCell(input) {
-  return "schemaVersion" in input && "cellId" in input;
-}
-function assertAgentProfileMatchesRun(profile, model, promptHash) {
-  if (profile.model !== void 0 && profile.model !== model) {
-    throw new Error(
-      `runEvalCampaign: agentProfile.model "${profile.model}" does not match outcome.model "${model}"`
-    );
-  }
-  if (profile.promptHash !== void 0 && profile.promptHash !== promptHash) {
-    throw new Error(
-      `runEvalCampaign: agentProfile.promptHash "${profile.promptHash}" does not match outcome.promptHash "${promptHash}"`
-    );
-  }
-}
-function defaultRunId(params) {
-  const base = `${params.campaignId}::${params.variantId}::${params.scenarioId}::${params.seed}`;
-  let h1 = 2166136261;
-  let h2 = 305419896;
-  for (let i = 0; i < base.length; i++) {
-    const c = base.charCodeAt(i);
-    h1 = Math.imul(h1 ^ c, 16777619) >>> 0;
-    h2 = Math.imul(h2 ^ c, 2654435761) >>> 0;
-  }
-  return `run-${h1.toString(16).padStart(8, "0")}${h2.toString(16).padStart(8, "0")}`;
-}
-
-export {
-  runEvalCampaign
-};
-//# sourceMappingURL=chunk-U5CHZ5M3.js.map