@tangle-network/agent-eval 0.80.0 → 0.81.0

package/dist/index.js

@@ -1,17 +1,11 @@
 import {
-  MODEL_PRICING,
-  MetricsCollector,
-  TokenCounter,
   agentProfileHash,
   createLlmCorrectnessChecker,
-  estimateCost,
-  estimateTokens,
+  createTokenRecallChecker,
   extractProducedState,
-  isModelPriced,
   parseCorrectnessResponse,
-  resolveModelPricing,
   verifyCompletion
-} from "./chunk-LB2UOI5F.js";
+} from "./chunk-YGYXHNAQ.js";
 import {
   HoldoutAuditor,
   canaryLeakView,
@@ -42,12 +36,21 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-RPLZ4OIB.js";
+} from "./chunk-BABOZOSN.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
   summarizeBackendIntegrity
-} from "./chunk-ZPSKPT3V.js";
+} from "./chunk-ZZ2HOPME.js";
+import {
+  MODEL_PRICING,
+  MetricsCollector,
+  TokenCounter,
+  estimateCost,
+  estimateTokens,
+  isModelPriced,
+  resolveModelPricing
+} from "./chunk-VI2UW6B6.js";
 import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
@@ -60,7 +63,7 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-GWGO2K6Y.js";
+} from "./chunk-RBNA5AZT.js";
 import {
   exportTrainingData,
   toNdjson
@@ -114,7 +117,7 @@ import {
   diffFindings,
   resetLockedAppendersForTesting,
   runSemanticConceptJudge
-} from "./chunk-5LVWPNS5.js";
+} from "./chunk-L5G7OUKD.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -126,7 +129,7 @@ import {
   createTraceAnalystKind,
   makeFinding,
   renderPriorFindings
-} from "./chunk-CF67I6QY.js";
+} from "./chunk-VIDQF3F5.js";
 import {
   controlFailureClassFromVerification,
   controlRunToRunRecord,
@@ -152,10 +155,10 @@ import {
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-B26KI423.js";
+} from "./chunk-FZWAFVAA.js";
 import {
   runEvalCampaign
-} from "./chunk-XXNIODOM.js";
+} from "./chunk-WJL2NJXN.js";
 import {
   LlmCallError,
   LlmClient,
@@ -167,7 +170,7 @@ import {
   isTransientLlmError,
   probeLlm,
   stripFencedJson
-} from "./chunk-IHDHUN2X.js";
+} from "./chunk-CVVHBFGN.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -178,7 +181,7 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-KX6F6NCG.js";
+} from "./chunk-Z7VFTS2J.js";
 import {
   benjaminiHochberg,
   bonferroni,
@@ -205,7 +208,7 @@ import {
   weightedComposite,
   weightedMean,
   wilcoxonSignedRank
-} from "./chunk-ITBRCT73.js";
+} from "./chunk-IDVBLYCY.js";
 import {
   FileSystemTraceStore,
   InMemoryTraceStore,
@@ -2895,391 +2898,6 @@ var DualAgentBench = class {
   }
 };
 
-// src/experiment-tracker.ts
-var InMemoryExperimentStore = class {
-  experiments = /* @__PURE__ */ new Map();
-  runs = /* @__PURE__ */ new Map();
-  async saveExperiment(exp) {
-    this.experiments.set(exp.id, { ...exp });
-  }
-  async getExperiment(id) {
-    const e = this.experiments.get(id);
-    return e ? { ...e } : null;
-  }
-  async listExperiments() {
-    return [...this.experiments.values()].sort((a, b) => b.createdAt.localeCompare(a.createdAt));
-  }
-  async saveRun(run) {
-    this.runs.set(run.id, structuredClone(run));
-  }
-  async getRun(id) {
-    const r = this.runs.get(id);
-    return r ? structuredClone(r) : null;
-  }
-  async listRuns(experimentId) {
-    return [...this.runs.values()].filter((r) => r.experimentId === experimentId).sort((a, b) => b.startedAt.localeCompare(a.startedAt)).map((r) => structuredClone(r));
-  }
-};
-var ExperimentTracker = class {
-  constructor(store) {
-    this.store = store;
-  }
-  store;
-  async startExperiment(name, metadata) {
-    const exp = {
-      id: `exp_${rand(8)}`,
-      name,
-      createdAt: (/* @__PURE__ */ new Date()).toISOString(),
-      metadata
-    };
-    await this.store.saveExperiment(exp);
-    return exp;
-  }
-  async startRun(config) {
-    const exp = await this.store.getExperiment(config.experimentId);
-    if (!exp) throw new Error(`Experiment ${config.experimentId} not found`);
-    const run = {
-      id: `run_${rand(10)}`,
-      experimentId: config.experimentId,
-      name: config.name,
-      config,
-      startedAt: (/* @__PURE__ */ new Date()).toISOString(),
-      status: "running"
-    };
-    await this.store.saveRun(run);
-    return run;
-  }
-  async completeRun(runId, report) {
-    const run = await this.store.getRun(runId);
-    if (!run) throw new Error(`Run ${runId} not found`);
-    run.status = "completed";
-    run.completedAt = (/* @__PURE__ */ new Date()).toISOString();
-    run.report = report;
-    await this.store.saveRun(run);
-  }
-  async failRun(runId, error) {
-    const run = await this.store.getRun(runId);
-    if (!run) throw new Error(`Run ${runId} not found`);
-    run.status = "failed";
-    run.completedAt = (/* @__PURE__ */ new Date()).toISOString();
-    run.error = error;
-    await this.store.saveRun(run);
-  }
-  /**
-   * Diff two completed runs. Returns per-scenario deltas, aggregate delta,
-   * and config changes that may explain the movement.
-   */
-  async diff(runIdA, runIdB) {
-    const [a, b] = await Promise.all([this.store.getRun(runIdA), this.store.getRun(runIdB)]);
-    if (!a || !b) throw new Error("Both runs must exist");
-    if (!a.report || !b.report) throw new Error("Both runs must be completed with reports");
-    const byScenarioA = new Map(a.report.results.map((r) => [r.scenarioId, r.overallScore]));
-    const byScenarioB = new Map(b.report.results.map((r) => [r.scenarioId, r.overallScore]));
-    const scenarioIds = /* @__PURE__ */ new Set([...byScenarioA.keys(), ...byScenarioB.keys()]);
-    const scenarios = [];
-    for (const id of scenarioIds) {
-      const aScore = byScenarioA.get(id);
-      const bScore = byScenarioB.get(id);
-      if (aScore === void 0) {
-        scenarios.push({
-          scenarioId: id,
-          before: null,
-          after: bScore,
-          delta: null,
-          status: "added"
-        });
-      } else if (bScore === void 0) {
-        scenarios.push({
-          scenarioId: id,
-          before: aScore,
-          after: null,
-          delta: null,
-          status: "removed"
-        });
-      } else {
-        scenarios.push({
-          scenarioId: id,
-          before: aScore,
-          after: bScore,
-          delta: bScore - aScore,
-          status: bScore > aScore ? "improved" : bScore < aScore ? "regressed" : "unchanged"
-        });
-      }
-    }
-    scenarios.sort((x, y) => (y.delta ?? 0) - (x.delta ?? 0));
-    const aggregateDelta = b.report.summary.overallAvg - a.report.summary.overallAvg;
-    const configChanges = {};
-    const keys = /* @__PURE__ */ new Set([...Object.keys(a.config), ...Object.keys(b.config)]);
-    const aCfg = a.config;
-    const bCfg = b.config;
-    for (const k of keys) {
-      if (JSON.stringify(aCfg[k]) !== JSON.stringify(bCfg[k])) {
-        configChanges[k] = { before: aCfg[k], after: bCfg[k] };
-      }
-    }
-    return {
-      before: { runId: runIdA, name: a.name, startedAt: a.startedAt },
-      after: { runId: runIdB, name: b.name, startedAt: b.startedAt },
-      aggregateDelta,
-      scenarios,
-      configChanges
-    };
-  }
-  /** Timeline of aggregate scores for an experiment. */
-  async timeline(experimentId) {
-    const runs = await this.store.listRuns(experimentId);
-    return runs.slice().sort((a, b) => a.startedAt.localeCompare(b.startedAt)).map((r) => ({
-      runId: r.id,
-      startedAt: r.startedAt,
-      overall: r.report?.summary.overallAvg ?? null
-    }));
-  }
-};
-function rand(bytes) {
-  const arr = new Uint8Array(bytes);
-  crypto.getRandomValues(arr);
-  return Array.from(arr).map((b) => b.toString(16).padStart(2, "0")).join("");
-}
-
-// src/experiment-tracker-d1.ts
-var SCHEMA_VERSION = 1;
-var D1ExperimentStore = class {
-  db;
-  experimentsTable;
-  runsTable;
-  metaTable;
-  schemaReady = false;
-  constructor(options) {
-    this.db = options.db;
-    const prefix = options.tablePrefix ?? "agent_eval_";
-    this.experimentsTable = `${prefix}experiments`;
-    this.runsTable = `${prefix}runs`;
-    this.metaTable = `${prefix}meta`;
-  }
-  /**
-   * Idempotent schema setup. Safe to call before every operation; the second
-   * call short-circuits via `schemaReady`. Most consumers will call it once
-   * during Worker bootstrap.
-   */
-  async ensureSchema() {
-    if (this.schemaReady) return;
-    const ddl = `
-      CREATE TABLE IF NOT EXISTS ${this.experimentsTable} (
-        id TEXT PRIMARY KEY,
-        name TEXT NOT NULL,
-        created_at TEXT NOT NULL,
-        metadata_json TEXT
-      );
-      CREATE TABLE IF NOT EXISTS ${this.runsTable} (
-        id TEXT PRIMARY KEY,
-        experiment_id TEXT NOT NULL,
-        name TEXT,
-        status TEXT NOT NULL,
-        started_at TEXT NOT NULL,
-        completed_at TEXT,
-        config_json TEXT NOT NULL,
-        report_json TEXT,
-        error TEXT
-      );
-      CREATE INDEX IF NOT EXISTS idx_${this.runsTable}_experiment ON ${this.runsTable}(experiment_id);
-      CREATE INDEX IF NOT EXISTS idx_${this.runsTable}_started ON ${this.runsTable}(started_at);
-      CREATE TABLE IF NOT EXISTS ${this.metaTable} (
-        key TEXT PRIMARY KEY,
-        value TEXT NOT NULL
-      );
-      INSERT OR REPLACE INTO ${this.metaTable}(key, value) VALUES ('schema_version', '${SCHEMA_VERSION}');
-    `;
-    await this.db.exec(ddl.trim().replace(/\s+/g, " "));
-    this.schemaReady = true;
-  }
-  async saveExperiment(exp) {
-    await this.ensureSchema();
-    await this.db.prepare(
-      `INSERT INTO ${this.experimentsTable}(id, name, created_at, metadata_json)
-         VALUES (?1, ?2, ?3, ?4)
-         ON CONFLICT(id) DO UPDATE SET
-           name = excluded.name,
-           created_at = excluded.created_at,
-           metadata_json = excluded.metadata_json`
-    ).bind(exp.id, exp.name, exp.createdAt, exp.metadata ? JSON.stringify(exp.metadata) : null).run();
-  }
-  async getExperiment(id) {
-    await this.ensureSchema();
-    const row = await this.db.prepare(
-      `SELECT id, name, created_at, metadata_json
-         FROM ${this.experimentsTable}
-         WHERE id = ?1`
-    ).bind(id).first();
-    return row ? rowToExperiment(row) : null;
-  }
-  async listExperiments() {
-    await this.ensureSchema();
-    const { results } = await this.db.prepare(
-      `SELECT id, name, created_at, metadata_json
-         FROM ${this.experimentsTable}
-         ORDER BY created_at DESC`
-    ).all();
-    return results.map(rowToExperiment);
-  }
-  async saveRun(run) {
-    await this.ensureSchema();
-    await this.db.prepare(
-      `INSERT INTO ${this.runsTable}(id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error)
-         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
-         ON CONFLICT(id) DO UPDATE SET
-           experiment_id = excluded.experiment_id,
-           name = excluded.name,
-           status = excluded.status,
-           started_at = excluded.started_at,
-           completed_at = excluded.completed_at,
-           config_json = excluded.config_json,
-           report_json = excluded.report_json,
-           error = excluded.error`
-    ).bind(
-      run.id,
-      run.experimentId,
-      run.name ?? null,
-      run.status,
-      run.startedAt,
-      run.completedAt ?? null,
-      JSON.stringify(run.config),
-      run.report ? JSON.stringify(run.report) : null,
-      run.error ?? null
-    ).run();
-  }
-  async getRun(id) {
-    await this.ensureSchema();
-    const row = await this.db.prepare(
-      `SELECT id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error
-         FROM ${this.runsTable}
-         WHERE id = ?1`
-    ).bind(id).first();
-    return row ? rowToRun(row) : null;
-  }
-  async listRuns(experimentId) {
-    await this.ensureSchema();
-    const { results } = await this.db.prepare(
-      `SELECT id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error
-         FROM ${this.runsTable}
-         WHERE experiment_id = ?1
-         ORDER BY started_at DESC`
-    ).bind(experimentId).all();
-    return results.map(rowToRun);
-  }
-};
-function rowToExperiment(row) {
-  return {
-    id: row.id,
-    name: row.name,
-    createdAt: row.created_at,
-    ...row.metadata_json ? { metadata: JSON.parse(row.metadata_json) } : {}
-  };
-}
-function rowToRun(row) {
-  return {
-    id: row.id,
-    experimentId: row.experiment_id,
-    ...row.name ? { name: row.name } : {},
-    status: row.status,
-    startedAt: row.started_at,
-    ...row.completed_at ? { completedAt: row.completed_at } : {},
-    config: JSON.parse(row.config_json),
-    ...row.report_json ? { report: JSON.parse(row.report_json) } : {},
-    ...row.error ? { error: row.error } : {}
-  };
-}
-
-// src/experiment-tracker-fs.ts
-var FileSystemExperimentStore = class {
-  dir;
-  maxBytes;
-  index;
-  loaded = false;
-  constructor(options) {
-    this.dir = options.dir;
-    this.maxBytes = options.maxBytes ?? 32 * 1024 * 1024;
-  }
-  async saveExperiment(exp) {
-    const idx = await this.load();
-    await idx.saveExperiment(exp);
-    await this.append("experiments", exp);
-  }
-  async getExperiment(id) {
-    const idx = await this.load();
-    return idx.getExperiment(id);
-  }
-  async listExperiments() {
-    const idx = await this.load();
-    return idx.listExperiments();
-  }
-  async saveRun(run) {
-    const idx = await this.load();
-    await idx.saveRun(run);
-    await this.append("runs", run);
-  }
-  async getRun(id) {
-    const idx = await this.load();
-    return idx.getRun(id);
-  }
-  async listRuns(experimentId) {
-    const idx = await this.load();
-    return idx.listRuns(experimentId);
-  }
-  async ensureDir() {
-    const fs2 = await import("fs/promises");
-    await fs2.mkdir(this.dir, { recursive: true });
-  }
-  async append(name, record) {
-    await this.ensureDir();
-    const fs2 = await import("fs/promises");
-    const path = await import("path");
-    const active = path.join(this.dir, `${name}.ndjson`);
-    try {
-      const stat = await fs2.stat(active);
-      if (stat.size >= this.maxBytes) {
-        const rolled = path.join(this.dir, `${name}.${Date.now()}.ndjson`);
-        await fs2.rename(active, rolled);
-      }
-    } catch {
-    }
-    await fs2.appendFile(active, `${JSON.stringify(record)}
-`, "utf8");
-  }
-  async load() {
-    if (this.loaded && this.index) return this.index;
-    const fs2 = await import("fs/promises");
-    const path = await import("path");
-    const store = new InMemoryExperimentStore();
-    try {
-      const entries = await fs2.readdir(this.dir);
-      const sorted = entries.filter((f) => f.endsWith(".ndjson")).sort((a, b) => a.localeCompare(b));
-      for (const file of sorted) {
-        const full = path.join(this.dir, file);
-        const content = await fs2.readFile(full, "utf8");
-        const base = file.split(".")[0];
-        for (const line of content.split("\n")) {
-          if (!line.trim()) continue;
-          let record;
-          try {
-            record = JSON.parse(line);
-          } catch {
-            continue;
-          }
-          if (base === "experiments") {
-            await store.saveExperiment(record);
-          } else if (base === "runs") {
-            await store.saveRun(record);
-          }
-        }
-      }
-    } catch {
-    }
-    this.index = store;
-    this.loaded = true;
-    return store;
-  }
-};
-
 // src/harness-optimizer.ts
 var DEFAULT_HARNESS_OBJECTIVES = [
   { name: "aggregate", direction: "maximize", value: (r) => r.aggregateMean },
@@ -3387,6 +3005,67 @@ function meanRunScore(scores2) {
   };
 }
 
+// src/judge-ensemble.ts
+function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
+  if (verdicts.length === 0) {
+    throw new Error("aggregateJudgeVerdicts: no verdicts to aggregate");
+  }
+  if (dimensionKeys.length === 0) {
+    throw new Error("aggregateJudgeVerdicts: dimensionKeys is empty");
+  }
+  const perJudge = {};
+  const failedJudges = [];
+  const dimAcc = {};
+  for (const d of dimensionKeys) dimAcc[d] = [];
+  let rationale = "";
+  let costUsd = 0;
+  for (const v of verdicts) {
+    costUsd += v.costUsd ?? 0;
+    if (!v.perDimension) {
+      failedJudges.push(v.model);
+      continue;
+    }
+    const dims = {};
+    for (const d of dimensionKeys) dims[d] = clamp01(Number(v.perDimension[d]));
+    perJudge[v.model] = dims;
+    for (const d of dimensionKeys) dimAcc[d].push(dims[d]);
+    if (!rationale && typeof v.rationale === "string" && v.rationale) rationale = v.rationale;
+  }
+  if (failedJudges.length === verdicts.length) {
+    throw new Error(`aggregateJudgeVerdicts: all ${verdicts.length} judges failed`);
+  }
+  const perDimension = {};
+  for (const d of dimensionKeys) {
+    const vals = dimAcc[d];
+    perDimension[d] = vals.length === 0 ? 0 : vals.reduce((s, x) => s + x, 0) / vals.length;
+  }
+  let maxDisagreement = 0;
+  for (const d of dimensionKeys) {
+    const vals = dimAcc[d];
+    if (vals.length < 2) continue;
+    const spread = Math.max(...vals) - Math.min(...vals);
+    if (spread > maxDisagreement) maxDisagreement = spread;
+  }
+  const w = {};
+  if (weights) {
+    for (const [d, weight] of Object.entries(weights)) {
+      if (weight !== void 0) w[d] = weight;
+    }
+  } else {
+    for (const d of dimensionKeys) w[d] = 1 / dimensionKeys.length;
+  }
+  const composite = weightedComposite({ dims: perDimension, weights: w }).composite;
+  return {
+    perDimension,
+    composite,
+    perJudge,
+    maxDisagreement,
+    failedJudges,
+    costUsd,
+    rationale: rationale || "llm-judge"
+  };
+}
+
 // src/judge-runner.ts
 var JudgeRunner = class {
   driver;
@@ -6208,7 +5887,7 @@ function flowLayer(input) {
 // src/intent-match-judge.ts
 var INTENT_MATCH_JUDGE_VERSION = "intent-match-judge-v1-2026-04-24";
 var DEFAULT_MODEL = "claude-sonnet-4-6";
-var DEFAULT_TIMEOUT = 9e4;
+var DEFAULT_TIMEOUT = 3e5;
 var DEFAULT_MAX_SOURCE = 25e3;
 var DEFAULT_MAX_PER_FILE = 12e3;
 var DEFAULT_MAX_HTML = 2e4;
@@ -7364,7 +7043,7 @@ function createDefaultReviewer(options) {
     ...options.softFailDefaults ?? {}
   };
   const promptBuilder = options.promptBuilder ?? buildReviewerPrompt;
-  const timeoutMs = options.timeoutMs ?? 18e4;
+  const timeoutMs = options.timeoutMs ?? 3e5;
   return async (input) => {
     const start = Date.now();
     const { system, user } = promptBuilder(input);
@@ -7718,7 +7397,7 @@ function fmt(x) {
 
 // src/judge-retry.ts
 var DEFAULT_MAX_ATTEMPTS = 3;
-var DEFAULT_TIMEOUT_MS = 9e4;
+var DEFAULT_TIMEOUT_MS = 3e5;
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -8620,7 +8299,6 @@ export {
   ConvergenceTracker,
   CostTracker,
   CrossFamilyError,
-  D1ExperimentStore,
   DEFAULT_AGENT_SLOS,
   DEFAULT_COMPLEXITY_WEIGHTS,
   DEFAULT_RULES as DEFAULT_FAILURE_RULES,
@@ -8639,10 +8317,8 @@ export {
   DockerSandboxDriver,
   DualAgentBench,
   ERROR_COUNT_PATTERNS,
-  ExperimentTracker,
   FAILURE_CLASSES,
   FAILURE_MODE_KIND_SPEC,
-  FileSystemExperimentStore,
   FileSystemFeedbackTrajectoryStore,
   FileSystemRawProviderSink,
   FileSystemTraceStore,
@@ -8652,7 +8328,6 @@ export {
   HoldoutLockedError,
   IMPROVEMENT_KIND_SPEC,
   INTENT_MATCH_JUDGE_VERSION,
-  InMemoryExperimentStore,
   InMemoryFeedbackTrajectoryStore,
   InMemoryRawProviderSink,
   InMemoryTraceStore,
@@ -8712,6 +8387,7 @@ export {
   agentProfileCellHashMaterial,
   agentProfileCellKey,
   agentProfileHash,
+  aggregateJudgeVerdicts,
   aggregateLlm,
   aggregatePrReviewScore,
   aggregateRunScore,
@@ -8805,6 +8481,7 @@ export {
   createReplayFetch,
   createSandboxPool,
   createSemanticConceptJudge,
+  createTokenRecallChecker,
   createTraceAnalystKind,
   crossTraceDiff,
   crowdingDistance,