@tangle-network/agent-eval 0.84.0 → 0.86.0

package/CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.86.0] — 2026-06-09 — fleet-rebuilt eval primitives
+
+One clean, canonical version of five generic patterns the fleet kept hand-rolling across 2–4 product agents each. All additive — no existing export changed.
+
+### Added
+
+- **`ExperimentTracker` + `improvementVerdict` + `computeExperimentStats` (root).** Git-provenanced experiment log with N-rep stats (`median` / `mean` / `min` / `max` / `iqr` / `stddev` / `passRate` / `n` + a `stable` flag) and a `KEEP` / `REGRESSION` / `NOISE` / `ITERATE` verdict against a parent. Provenance (`ProvenanceReader`, default `gitProvenanceReader`) and persistence (`ExperimentStore`, with `inMemoryExperimentStore` + `fileExperimentStore`) are injected, so the stats + verdict are pure and unit-testable without a repo or disk. Thresholds (`keepThreshold` / `regressionThreshold` / `iqrUnstableAbove` / `stddevUnstableAbove` / `minRepsForVerdict`) are configurable. Replaces the per-agent `tests/eval/lib/experiment-tracker.ts` copies (tax / insurance / legal).
+- **`EvalTraceStore` + `runScore` (root).** JSONL save / query / compare over the analysis-time `RunRecord` row: `query(filter)`, `getBest(scenarioId)`, and `compareRuns(a, b)` (paired on matched scenarios, best-score-per-scenario). Persistence is injected via `RunRecordBackend` (`inMemoryRunRecordBackend` / `jsonlRunRecordBackend`, which fail loud on a malformed line). Does NOT fork `FileSystemTraceStore` (the rich TraceSchema-v1 span store) — it is the analysis projection beside it. Replaces the hand-rolled `tests/eval/lib/trace-store.ts` copies.
+- **`CostLedger` + `costForUsage` + `modelPriceKey` (root).** Per-run token + USD accounting folded over the substrate's `resolveModelPricing` / `isModelPriced`, with an explicit `costUnknown` axis so a $0 from an unpriced model is never mistaken for a measured free run. Classifies spend by channel (`agent` / `judge` / `verifier` / …), surfaces `unpricedModels` + `fullyPriced`, and computes `costPerCompletedTask`. Generalizes physim's `costForUsage` / `modelPriceKey` and the tax / gtm / agent-builder copies.
+- **`extractUsage` + `extractUsageFromSse` + `extractUsageFromResponse` (`/traces` + root).** Token-usage extraction from a chat-completions response or an SSE stream — OpenAI / Anthropic / camelCase shapes, cache-read tokens, and per-chunk SSE accumulation — returning `null` (not a silent zero) when no usage is present. `captureFetchToRawSink` gains an optional `onUsage` callback that emits the parsed usage off each response (reusing the body it already reads — no extra clone), so a caller folds usage → cost without re-cloning. Replaces the insurance / legal / gtm `raw-capture.ts` copies.
+- **`partitionHeldOut` + `assignHeldOutTag` + `hashToUnit` + `fnv1a32` (root).** Deterministic FNV-1a id+seed held-out splitter. `assignHeldOutTag` stamps a single id; `partitionHeldOut` splits a whole id list into disjoint search / holdout sets and fails loud on duplicate ids, empty input, an under-floor holdout (`minHoldout` / `minSearch` significance floor), or an out-of-range `holdoutFraction`. Generalizes agent-builder's `deterministicSplit` and the frontier persona-splitter; complements the existing 3-way benchmark `deterministicSplit` in `/benchmarks`.
+
+---
+
 ## [0.83.0] — 2026-06-05 — hostedTenantFromEnv
 
 ### Added

package/dist/belief-state/index.d.ts

@@ -577,6 +577,7 @@ type RuntimeBenchmarkTrajectoryRecord = RuntimeTrajectoryRecord & {
     benchmark?: unknown;
     condition?: unknown;
     instanceId?: unknown;
+    runtimeDecisionPoints?: unknown;
 };
 interface BuildRuntimeBenchmarkBeliefPhase0MeasurementOptions extends Omit<BuildRuntimeBeliefPhase0MeasurementOptions, 'runs' | 'events' | 'decisions' | 'labels'> {
     records: RuntimeBenchmarkTrajectoryRecord[];

package/dist/belief-state/index.js

@@ -1720,6 +1720,20 @@ function compactMetadata2(values) {
 }
 
 // src/belief-state/runtime-benchmark-corpus.ts
+var MAX_STRING_LENGTH = 12e3;
+var MAX_CONTEXT_LENGTH = 2e4;
+var MAX_EVIDENCE_DETAIL_LENGTH = 2e3;
+var MAX_CANDIDATE_ACTIONS = 50;
+var MAX_EVIDENCE_REFS = 50;
+var MAX_METADATA_DEPTH = 4;
+var MAX_METADATA_KEYS = 100;
+var SENSITIVE_KEY_RE = /(?:authorization|api[_-]?key|token|secret|password|cookie|credential|bearer)/i;
+var SENSITIVE_VALUE_RES = [
+  /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi,
+  /\b(?:sk|gh[pousr])_[A-Za-z0-9_]{20,}\b/g,
+  /\b(?:sk|ghp|gho|ghu|ghs|ghr)-[A-Za-z0-9_-]{20,}\b/g
+];
+var SENSITIVE_ASSIGNMENT_RE = /\b(api[_-]?key|token|secret|password|cookie)\s*[:=]\s*["']?[^"'\s,;}]+/gi;
 function buildRuntimeBenchmarkBeliefPhase0Measurement(options) {
   const diagnostics = [];
   const trajectory = projectRuntimeTrajectoryEvidence({
@@ -1728,11 +1742,11 @@ function buildRuntimeBenchmarkBeliefPhase0Measurement(options) {
     recordIdOf: runtimeBenchmarkRecordId,
     scenarioIdOf: runtimeBenchmarkScenarioId
   });
-  const decisions = options.decisions ?? [];
+  const decisions = options.decisions ?? runtimeBenchmarkDecisionPoints(options.records, diagnostics);
   const labels = options.labels ?? [];
   if (decisions.length === 0) {
     diagnostics.push(
-      "no runtime decision points supplied; benchmark lifecycle events alone cannot produce belief decision rows"
+      "no runtime decision points supplied or found on records; benchmark lifecycle events alone cannot produce belief decision rows"
     );
   }
   if (labels.length === 0 && decisions.length > 0) {
@@ -1772,6 +1786,125 @@ function runtimeBenchmarkRecordId(record2) {
 function runtimeBenchmarkScenarioId(record2) {
   return nonEmptyString(record2.instanceId);
 }
+function runtimeBenchmarkDecisionPoints(records, diagnostics) {
+  const decisions = [];
+  for (let recordIndex = 0; recordIndex < records.length; recordIndex += 1) {
+    const record2 = records[recordIndex];
+    const raw = record2.runtimeDecisionPoints;
+    if (raw === void 0) continue;
+    const recordId = runtimeBenchmarkRecordId(record2) ?? `record[${recordIndex}]`;
+    if (!Array.isArray(raw)) {
+      diagnostics.push(`${recordId}: runtimeDecisionPoints is not an array`);
+      continue;
+    }
+    for (let pointIndex = 0; pointIndex < raw.length; pointIndex += 1) {
+      const point = runtimeBenchmarkDecisionPoint(raw[pointIndex], {
+        diagnostics,
+        path: `${recordId}: runtimeDecisionPoints[${pointIndex}]`
+      });
+      if (!point) {
+        diagnostics.push(
+          `${recordId}: runtimeDecisionPoints[${pointIndex}] is not a RuntimeDecisionPoint`
+        );
+        continue;
+      }
+      decisions.push(point);
+    }
+  }
+  return decisions;
+}
+function runtimeBenchmarkDecisionPoint(input, context) {
+  if (!isRecord3(input)) return null;
+  if (typeof input.id !== "string" || input.id.length === 0) return null;
+  if (typeof input.runId !== "string" || input.runId.length === 0) return null;
+  if (typeof input.stepIndex !== "number" || !Number.isInteger(input.stepIndex) || input.stepIndex < 0) {
+    return null;
+  }
+  if (typeof input.kind !== "string" || input.kind.length === 0) return null;
+  return {
+    id: sanitizeString(input.id, MAX_STRING_LENGTH),
+    runId: sanitizeString(input.runId, MAX_STRING_LENGTH),
+    scenarioId: sanitizeOptionalString(input.scenarioId, MAX_STRING_LENGTH),
+    stepIndex: input.stepIndex,
+    kind: sanitizeString(input.kind, MAX_STRING_LENGTH),
+    candidateActions: stringArray(input.candidateActions, {
+      ...context,
+      maxItems: MAX_CANDIDATE_ACTIONS,
+      label: "candidateActions"
+    }),
+    context: sanitizeOptionalString(input.context, MAX_CONTEXT_LENGTH),
+    evidence: runtimeBenchmarkEvidence(input.evidence, context),
+    metadata: sanitizeMetadataRecord(input.metadata)
+  };
+}
+function runtimeBenchmarkEvidence(input, context) {
+  if (!Array.isArray(input)) return [];
+  if (input.length > MAX_EVIDENCE_REFS) {
+    context.diagnostics.push(`${context.path}: evidence truncated to ${MAX_EVIDENCE_REFS} refs`);
+  }
+  return input.slice(0, MAX_EVIDENCE_REFS).flatMap((item) => {
+    if (!isRecord3(item)) return [];
+    const source = sanitizeOptionalString(item.source, MAX_STRING_LENGTH);
+    const id = sanitizeOptionalString(item.id, MAX_STRING_LENGTH);
+    if (!source || !id) return [];
+    return [
+      {
+        source,
+        id,
+        detail: sanitizeOptionalString(item.detail, MAX_EVIDENCE_DETAIL_LENGTH),
+        metadata: sanitizeMetadataRecord(item.metadata)
+      }
+    ];
+  });
+}
+function stringArray(input, context) {
+  if (!Array.isArray(input)) return void 0;
+  if (input.length > context.maxItems) {
+    context.diagnostics.push(`${context.path}: ${context.label} truncated to ${context.maxItems}`);
+  }
+  const values = input.slice(0, context.maxItems).filter((value) => typeof value === "string" && value.length > 0).map((value) => sanitizeString(value, MAX_STRING_LENGTH));
+  return values.length > 0 ? values : void 0;
+}
+function sanitizeMetadataRecord(metadata) {
+  if (!isRecord3(metadata)) return void 0;
+  const sanitized = sanitizeMetadata(metadata);
+  if (!sanitized || typeof sanitized !== "object" || Array.isArray(sanitized)) return void 0;
+  return sanitized;
+}
+function sanitizeMetadata(value, depth = 0) {
+  if (value == null) return value;
+  if (typeof value === "string") return sanitizeString(value, MAX_STRING_LENGTH);
+  if (typeof value === "number" || typeof value === "boolean") return value;
+  if (Array.isArray(value)) {
+    if (depth >= MAX_METADATA_DEPTH) return "[MaxDepth]";
+    return value.slice(0, MAX_METADATA_KEYS).map((item) => sanitizeMetadata(item, depth + 1));
+  }
+  if (!isRecord3(value)) return void 0;
+  if (depth >= MAX_METADATA_DEPTH) return "[MaxDepth]";
+  const sanitized = {};
+  for (const [key, nested] of Object.entries(value).slice(0, MAX_METADATA_KEYS)) {
+    sanitized[key] = SENSITIVE_KEY_RE.test(key) ? "[REDACTED]" : sanitizeMetadata(nested, depth + 1);
+  }
+  return sanitized;
+}
+function sanitizeOptionalString(value, maxLength) {
+  return typeof value === "string" && value.length > 0 ? sanitizeString(value, maxLength) : void 0;
+}
+function sanitizeString(value, maxLength) {
+  let sanitized = value;
+  for (const pattern of SENSITIVE_VALUE_RES) {
+    sanitized = sanitized.replace(pattern, "[REDACTED]");
+  }
+  sanitized = sanitized.replace(
+    SENSITIVE_ASSIGNMENT_RE,
+    (_match, key) => `${key}=[REDACTED]`
+  );
+  if (sanitized.length <= maxLength) return sanitized;
+  return sanitized.slice(0, maxLength);
+}
+function isRecord3(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
 function nonEmptyString(value) {
   return typeof value === "string" && value.length > 0 ? value : void 0;
 }