@tangle-network/agent-eval 0.1.0 → 0.2.0

package/dist/index.js

@@ -506,6 +506,123 @@ function partialCredit(current, target) {
   if (target <= 0) return 1;
   return Math.min(1, Math.max(0, current / target));
 }
+function pairedTTest(before, after) {
+  if (before.length !== after.length) {
+    throw new Error(`pairedTTest: unequal sample sizes (${before.length} vs ${after.length})`);
+  }
+  const n = before.length;
+  if (n < 2) return { t: 0, df: 0, p: 1 };
+  const diffs = before.map((b, i) => after[i] - b);
+  const mean = diffs.reduce((a, b) => a + b, 0) / n;
+  const variance = diffs.reduce((acc, d) => acc + (d - mean) ** 2, 0) / (n - 1);
+  const se = Math.sqrt(variance / n);
+  if (se === 0) return { t: mean === 0 ? 0 : Infinity, df: n - 1, p: mean === 0 ? 1 : 0 };
+  const t = mean / se;
+  const df = n - 1;
+  const p = 2 * (1 - studentTCdf(Math.abs(t), df));
+  return { t, df, p };
+}
+function wilcoxonSignedRank(before, after) {
+  if (before.length !== after.length) {
+    throw new Error(`wilcoxonSignedRank: unequal sample sizes (${before.length} vs ${after.length})`);
+  }
+  const diffs = before.map((b, i2) => after[i2] - b).filter((d) => d !== 0);
+  const n = diffs.length;
+  if (n < 6) return { w: 0, p: 1 };
+  const absRanks = diffs.map((d, i2) => ({ abs: Math.abs(d), sign: Math.sign(d), i: i2 })).sort((a, b) => a.abs - b.abs);
+  const ranks = new Array(n);
+  let i = 0;
+  while (i < n) {
+    let j = i;
+    while (j < n && absRanks[j].abs === absRanks[i].abs) j++;
+    const avg = (i + 1 + j) / 2;
+    for (let k = i; k < j; k++) ranks[absRanks[k].i] = avg;
+    i = j;
+  }
+  let wPlus = 0;
+  for (let k = 0; k < n; k++) if (diffs[k] > 0) wPlus += ranks[k];
+  const mean = n * (n + 1) / 4;
+  const variance = n * (n + 1) * (2 * n + 1) / 24;
+  const z = (wPlus - mean) / Math.sqrt(variance);
+  const p = 2 * (1 - normalCdf(Math.abs(z)));
+  return { w: wPlus, p };
+}
+function cohensD(a, b) {
+  if (a.length < 2 || b.length < 2) return 0;
+  const meanA = a.reduce((x, y) => x + y, 0) / a.length;
+  const meanB = b.reduce((x, y) => x + y, 0) / b.length;
+  const varA = a.reduce((acc, x) => acc + (x - meanA) ** 2, 0) / (a.length - 1);
+  const varB = b.reduce((acc, x) => acc + (x - meanB) ** 2, 0) / (b.length - 1);
+  const pooled = Math.sqrt(
+    ((a.length - 1) * varA + (b.length - 1) * varB) / (a.length + b.length - 2)
+  );
+  if (pooled === 0) return 0;
+  return (meanB - meanA) / pooled;
+}
+function studentTCdf(t, df) {
+  if (df <= 0) return 0.5;
+  if (df > 100) return normalCdf(t);
+  const x = df / (df + t * t);
+  const a = df / 2;
+  const b = 0.5;
+  const ib = incompleteBeta(x, a, b);
+  return t >= 0 ? 1 - 0.5 * ib : 0.5 * ib;
+}
+function incompleteBeta(x, a, b) {
+  if (x <= 0) return 0;
+  if (x >= 1) return 1;
+  const lnBeta = lnGamma(a) + lnGamma(b) - lnGamma(a + b);
+  const front = Math.exp(Math.log(x) * a + Math.log(1 - x) * b - lnBeta) / a;
+  const maxIter = 200;
+  const eps = 3e-7;
+  let c = 1;
+  let d = 1 - (a + b) * x / (a + 1);
+  if (Math.abs(d) < 1e-30) d = 1e-30;
+  d = 1 / d;
+  let f = d;
+  for (let m = 1; m <= maxIter; m++) {
+    const m2 = 2 * m;
+    let num = m * (b - m) * x / ((a + m2 - 1) * (a + m2));
+    d = 1 + num * d;
+    if (Math.abs(d) < 1e-30) d = 1e-30;
+    c = 1 + num / c;
+    if (Math.abs(c) < 1e-30) c = 1e-30;
+    d = 1 / d;
+    f *= d * c;
+    num = -((a + m) * (a + b + m) * x) / ((a + m2) * (a + m2 + 1));
+    d = 1 + num * d;
+    if (Math.abs(d) < 1e-30) d = 1e-30;
+    c = 1 + num / c;
+    if (Math.abs(c) < 1e-30) c = 1e-30;
+    d = 1 / d;
+    const delta = d * c;
+    f *= delta;
+    if (Math.abs(delta - 1) < eps) break;
+  }
+  return front * f;
+}
+function lnGamma(z) {
+  const g = 7;
+  const coefs = [
+    0.9999999999998099,
+    676.5203681218851,
+    -1259.1392167224028,
+    771.3234287776531,
+    -176.6150291621406,
+    12.507343278686905,
+    -0.13857109526572012,
+    9984369578019572e-21,
+    15056327351493116e-23
+  ];
+  if (z < 0.5) {
+    return Math.log(Math.PI / Math.sin(Math.PI * z)) - lnGamma(1 - z);
+  }
+  z -= 1;
+  let x = coefs[0];
+  for (let i = 1; i < g + 2; i++) x += coefs[i] / (z + i);
+  const t = z + g + 0.5;
+  return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x);
+}
 function normalCdf(x) {
   const a1 = 0.254829592;
   const a2 = -0.284496736;
@@ -1261,33 +1378,926 @@ function printDriverSummary(results) {
   const completedCount = results.filter((r) => r.completed).length;
   console.log(`${completedCount}/${results.length} personas completed`);
 }
+
+// src/prompt-registry.ts
+var PromptRegistry = class {
+  entries = /* @__PURE__ */ new Map();
+  // `${id}@${version}` → handle
+  /**
+   * Register a prompt. Re-registering the same id+version with DIFFERENT
+   * content throws — versions are immutable. Re-registering with the SAME
+   * content is a no-op (idempotent).
+   */
+  async register(id, version, content) {
+    validateId(id);
+    validateVersion(version);
+    const key = makeKey(id, version);
+    const hash = await hashContent(content);
+    const existing = this.entries.get(key);
+    if (existing) {
+      if (existing.hash !== hash) {
+        throw new Error(
+          `Prompt ${key} already registered with a different hash (${existing.hash} vs ${hash}). Bump the version.`
+        );
+      }
+      return existing;
+    }
+    const handle = { id, version, hash, content };
+    this.entries.set(key, handle);
+    return handle;
+  }
+  /** Look up a registered prompt. Throws if unknown — no implicit defaults. */
+  get(id, version) {
+    const key = makeKey(id, version);
+    const handle = this.entries.get(key);
+    if (!handle) throw new Error(`Prompt ${key} not registered`);
+    return handle;
+  }
+  /** Return all versions of an id, newest-first (lex-descending on version). */
+  listVersions(id) {
+    return [...this.entries.values()].filter((h) => h.id === id).sort((a, b) => b.version.localeCompare(a.version));
+  }
+  /** Snapshot the whole registry — useful for including in reports. */
+  list() {
+    return [...this.entries.values()];
+  }
+  /** Verify a hash against registered content. Returns null if not found. */
+  verifyHash(id, version, expectedHash) {
+    const handle = this.entries.get(makeKey(id, version));
+    if (!handle) return null;
+    return handle.hash === expectedHash;
+  }
+};
+async function hashContent(content) {
+  const bytes = new TextEncoder().encode(content);
+  const digest = await crypto.subtle.digest("SHA-256", bytes);
+  const full = Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join("");
+  return full.slice(0, 12);
+}
+function makeKey(id, version) {
+  return `${id}@${version}`;
+}
+var ID_RE = /^[a-z][a-z0-9._-]*$/i;
+function validateId(id) {
+  if (!ID_RE.test(id)) {
+    throw new Error(`Invalid prompt id "${id}": must match ${ID_RE}`);
+  }
+}
+function validateVersion(version) {
+  if (!version || version.length > 64) {
+    throw new Error(`Invalid version "${version}": must be 1\u201364 chars`);
+  }
+}
+
+// src/trace-store.ts
+var MemoryTraceStore = class {
+  traces = [];
+  async record(trace) {
+    this.traces.push(trace);
+  }
+  async query(query) {
+    let result = this.filter(query);
+    if (query.limit !== void 0) result = result.slice(0, query.limit);
+    return result;
+  }
+  async count(query) {
+    return query ? this.filter(query).length : this.traces.length;
+  }
+  /** Clear the store — test helper. */
+  reset() {
+    this.traces = [];
+  }
+  filter(query) {
+    return this.traces.filter((t) => {
+      if (query.runId && t.runId !== query.runId) return false;
+      if (query.scenarioId && t.scenarioId !== query.scenarioId) return false;
+      if (query.role && t.role !== query.role) return false;
+      if (query.model && t.model !== query.model) return false;
+      if (query.sinceMs !== void 0) {
+        const ts = Date.parse(t.timestamp);
+        if (Number.isFinite(ts) && ts < query.sinceMs) return false;
+      }
+      return true;
+    });
+  }
+};
+var FileSystemTraceStore = class {
+  opts;
+  constructor(opts) {
+    this.opts = {
+      rolloverBytes: 32 * 1024 * 1024,
+      append: defaultAppend,
+      read: defaultRead,
+      list: defaultList,
+      stat: defaultStat,
+      mkdir: defaultMkdir,
+      ...opts
+    };
+  }
+  async record(trace) {
+    const file = await this.currentSegment();
+    await this.opts.append(file, JSON.stringify(trace) + "\n");
+  }
+  async query(query) {
+    const files = await this.segments();
+    const out = [];
+    for (const file of files) {
+      const contents = await this.opts.read(file).catch(() => "");
+      for (const line of contents.split("\n")) {
+        if (!line) continue;
+        try {
+          const t = JSON.parse(line);
+          if (!matches(t, query)) continue;
+          out.push(t);
+          if (query.limit !== void 0 && out.length >= query.limit) return out;
+        } catch {
+        }
+      }
+    }
+    return out;
+  }
+  async count(query) {
+    if (!query) {
+      const files = await this.segments();
+      let total = 0;
+      for (const file of files) {
+        const contents = await this.opts.read(file).catch(() => "");
+        total += contents.split("\n").filter(Boolean).length;
+      }
+      return total;
+    }
+    return (await this.query(query)).length;
+  }
+  async segments() {
+    try {
+      const all = await this.opts.list(this.opts.dir);
+      return all.filter((f) => f.endsWith(".ndjson")).sort();
+    } catch {
+      return [];
+    }
+  }
+  async currentSegment() {
+    await this.opts.mkdir(this.opts.dir);
+    const existing = await this.segments();
+    if (existing.length === 0) return pathJoin(this.opts.dir, `traces-000.ndjson`);
+    const latest = existing[existing.length - 1];
+    try {
+      const s = await this.opts.stat(latest);
+      if (s.size < this.opts.rolloverBytes) return latest;
+    } catch {
+      return latest;
+    }
+    const n = existing.length;
+    return pathJoin(this.opts.dir, `traces-${String(n).padStart(3, "0")}.ndjson`);
+  }
+};
+function matches(t, query) {
+  if (query.runId && t.runId !== query.runId) return false;
+  if (query.scenarioId && t.scenarioId !== query.scenarioId) return false;
+  if (query.role && t.role !== query.role) return false;
+  if (query.model && t.model !== query.model) return false;
+  if (query.sinceMs !== void 0) {
+    const ts = Date.parse(t.timestamp);
+    if (Number.isFinite(ts) && ts < query.sinceMs) return false;
+  }
+  return true;
+}
+function pathJoin(dir, file) {
+  return dir.endsWith("/") ? `${dir}${file}` : `${dir}/${file}`;
+}
+async function defaultAppend(path, data) {
+  const fs = await import("fs/promises");
+  await fs.appendFile(path, data);
+}
+async function defaultRead(path) {
+  const fs = await import("fs/promises");
+  return fs.readFile(path, "utf8");
+}
+async function defaultList(dir) {
+  const fs = await import("fs/promises");
+  const p = await import("path");
+  try {
+    const entries = await fs.readdir(dir);
+    return entries.map((e) => p.join(dir, e));
+  } catch {
+    return [];
+  }
+}
+async function defaultStat(path) {
+  const fs = await import("fs/promises");
+  const s = await fs.stat(path);
+  return { size: s.size };
+}
+async function defaultMkdir(dir) {
+  const fs = await import("fs/promises");
+  await fs.mkdir(dir, { recursive: true });
+}
+
+// src/anti-slop.ts
+var DEFAULT_HEDGES = [
+  /\bi\s+could\s+be\s+wrong\b/i,
+  /\bi\s+think\s+maybe\b/i,
+  /\bit\s+might\s+be\s+that\b/i,
+  /\bperhaps\s+(?:you\s+)?could\b/i
+];
+var DEFAULT_APOLOGIES = [
+  /\bi\s+(?:apologize|apologise)\s+(?:for|if)\b/i,
+  /\bi'?m\s+(?:so\s+|really\s+)?sorry\s+(?:for|if|about)\b/i,
+  /\bmy\s+apologies\b/i
+];
+function createAntiSlopJudge(config = {}) {
+  const conf = {
+    domain: config.domain ?? "general",
+    bannedPhrases: config.bannedPhrases ?? [],
+    bannedOpenings: config.bannedOpenings ?? [],
+    hedgingPatterns: config.hedgingPatterns ?? DEFAULT_HEDGES,
+    apologyPatterns: config.apologyPatterns ?? DEFAULT_APOLOGIES,
+    repetitionThreshold: config.repetitionThreshold ?? 0.15,
+    minLength: config.minLength ?? 20,
+    maxLength: config.maxLength ?? 8e3,
+    penaltyWeights: {
+      banned_phrase: 1,
+      banned_opening: 1,
+      hedging: 0.5,
+      apology: 0.5,
+      repetition: 0.75,
+      length: 0.5,
+      ...config.penaltyWeights
+    }
+  };
+  const judge = async (_tc, input) => {
+    const outputs = input.turns.map((t) => t.agentResponse ?? "");
+    const report = analyzeAntiSlop(outputs, conf);
+    return [
+      {
+        judgeName: `anti-slop(${conf.domain})`,
+        dimension: "anti_slop",
+        score: report.score,
+        reasoning: report.issues.length ? report.issues.slice(0, 5).map((i) => `${i.category}: ${i.detail}`).join("; ") : "No slop patterns detected.",
+        evidence: report.issues[0]?.example
+      }
+    ];
+  };
+  return judge;
+}
+function analyzeAntiSlop(outputs, config) {
+  const issues = [];
+  const counts = {
+    banned_phrase: 0,
+    banned_opening: 0,
+    hedging: 0,
+    apology: 0,
+    repetition: 0,
+    length: 0
+  };
+  for (const output of outputs) {
+    if (!output) continue;
+    const lower = output.toLowerCase();
+    for (const phrase of config.bannedPhrases) {
+      const needle = phrase.toLowerCase();
+      let idx = 0;
+      while ((idx = lower.indexOf(needle, idx)) !== -1) {
+        counts.banned_phrase += 1;
+        if (issues.length < 20) {
+          issues.push({
+            category: "banned_phrase",
+            detail: `"${phrase}"`,
+            example: snippet(output, idx, phrase.length)
+          });
+        }
+        idx += needle.length;
+      }
+    }
+    for (const re of config.bannedOpenings) {
+      if (re.test(output)) {
+        counts.banned_opening += 1;
+        issues.push({ category: "banned_opening", detail: re.source, example: output.slice(0, 80) });
+      }
+    }
+    for (const re of config.hedgingPatterns) {
+      const matches2 = output.match(new RegExp(re, re.flags.includes("g") ? re.flags : re.flags + "g"));
+      if (matches2) {
+        counts.hedging += matches2.length;
+        issues.push({
+          category: "hedging",
+          detail: `${matches2.length}x ${re.source}`,
+          example: matches2[0]
+        });
+      }
+    }
+    for (const re of config.apologyPatterns) {
+      const matches2 = output.match(new RegExp(re, re.flags.includes("g") ? re.flags : re.flags + "g"));
+      if (matches2) {
+        counts.apology += matches2.length;
+        issues.push({
+          category: "apology",
+          detail: `${matches2.length}x ${re.source}`,
+          example: matches2[0]
+        });
+      }
+    }
+    const sentences = splitSentences(output);
+    if (sentences.length >= 4) {
+      const seen = /* @__PURE__ */ new Map();
+      for (const s of sentences) {
+        const key = normalizeForDupe(s);
+        if (!key) continue;
+        seen.set(key, (seen.get(key) ?? 0) + 1);
+      }
+      let dupes = 0;
+      for (const n of seen.values()) if (n > 1) dupes += n - 1;
+      const ratio = dupes / sentences.length;
+      if (ratio > config.repetitionThreshold) {
+        counts.repetition += 1;
+        issues.push({
+          category: "repetition",
+          detail: `${(ratio * 100).toFixed(0)}% duplicated (threshold ${(config.repetitionThreshold * 100).toFixed(0)}%)`
+        });
+      }
+    }
+    if (output.length < config.minLength) {
+      counts.length += 1;
+      issues.push({ category: "length", detail: `too short (${output.length} < ${config.minLength})` });
+    } else if (output.length > config.maxLength) {
+      counts.length += 1;
+      issues.push({ category: "length", detail: `too long (${output.length} > ${config.maxLength})` });
+    }
+  }
+  let penalty = 0;
+  for (const cat of Object.keys(counts)) {
+    penalty += counts[cat] * (config.penaltyWeights[cat] ?? 1);
+  }
+  const score = Math.max(0, Math.min(10, 10 - penalty));
+  return { score, issues, counts };
+}
+function snippet(source, at, len) {
+  const pad = 24;
+  const start = Math.max(0, at - pad);
+  const end = Math.min(source.length, at + len + pad);
+  return (start > 0 ? "\u2026" : "") + source.slice(start, end) + (end < source.length ? "\u2026" : "");
+}
+function splitSentences(text) {
+  return text.split(/[.!?\n]+/).map((s) => s.trim()).filter((s) => s.length > 0);
+}
+function normalizeForDupe(s) {
+  return s.toLowerCase().replace(/\s+/g, " ").replace(/[^a-z0-9 ]/g, "").trim();
+}
+
+// src/artifact-validator.ts
+function composeValidators(validators, options) {
+  const weights = options?.weights ?? validators.map(() => 1);
+  if (weights.length !== validators.length) {
+    throw new Error("composeValidators: weights length mismatch");
+  }
+  const totalWeight = weights.reduce((a, b) => a + b, 0) || 1;
+  return {
+    name: options?.name ?? validators.map((v) => v.name).join("+"),
+    async validate(artifact, ctx) {
+      const results = await Promise.all(validators.map((v) => v.validate(artifact, ctx)));
+      const pass = results.every((r) => r.pass);
+      const score = results.reduce((acc, r, i) => acc + r.score * weights[i], 0) / totalWeight;
+      return {
+        pass,
+        score,
+        issues: results.flatMap(
+          (r, i) => r.issues.map((issue) => ({
+            ...issue,
+            locus: issue.locus ? `${validators[i].name}:${issue.locus}` : validators[i].name
+          }))
+        ),
+        evidence: Object.fromEntries(results.map((r, i) => [validators[i].name, r.evidence]))
+      };
+    }
+  };
+}
+function regexMatch(name, pattern) {
+  return {
+    name,
+    async validate(artifact) {
+      const body = artifact.content ?? "";
+      const ok = pattern.test(body);
+      return {
+        pass: ok,
+        score: ok ? 1 : 0,
+        issues: ok ? [] : [{ severity: "error", message: `Artifact content did not match ${pattern}` }]
+      };
+    }
+  };
+}
+function jsonHasKeys(name, requiredPaths) {
+  return {
+    name,
+    async validate(artifact) {
+      const body = artifact.content ?? "";
+      let parsed;
+      try {
+        parsed = JSON.parse(body);
+      } catch (err) {
+        return {
+          pass: false,
+          score: 0,
+          issues: [{ severity: "error", message: `Invalid JSON: ${err instanceof Error ? err.message : err}` }]
+        };
+      }
+      const missing = [];
+      for (const path of requiredPaths) {
+        if (!pathExists(parsed, path)) missing.push(path);
+      }
+      const pass = missing.length === 0;
+      return {
+        pass,
+        score: 1 - missing.length / Math.max(1, requiredPaths.length),
+        issues: missing.map((p) => ({ severity: "error", message: `Missing path: ${p}`, locus: p }))
+      };
+    }
+  };
+}
+function byteLengthRange(name, min, max) {
+  return {
+    name,
+    async validate(artifact) {
+      const size = artifact.bytes?.byteLength ?? new TextEncoder().encode(artifact.content ?? "").byteLength;
+      const pass = size >= min && size <= max;
+      const score = pass ? 1 : size < min ? Math.max(0, size / min) : Math.max(0, max / size);
+      return {
+        pass,
+        score,
+        issues: pass ? [] : [{ severity: "error", message: `Size ${size} outside [${min}, ${max}]` }]
+      };
+    }
+  };
+}
+function containsAll(name, required, options) {
+  const cs = options?.caseSensitive ?? false;
+  return {
+    name,
+    async validate(artifact) {
+      const body = cs ? artifact.content ?? "" : (artifact.content ?? "").toLowerCase();
+      const missing = [];
+      for (const needle of required) {
+        const probe = cs ? needle : needle.toLowerCase();
+        if (!body.includes(probe)) missing.push(needle);
+      }
+      const pass = missing.length === 0;
+      return {
+        pass,
+        score: 1 - missing.length / Math.max(1, required.length),
+        issues: missing.map((m) => ({ severity: "error", message: `Missing substring: ${m}` }))
+      };
+    }
+  };
+}
+function pathExists(obj, path) {
+  const parts = path.split(".");
+  let current = obj;
+  for (const part of parts) {
+    if (current === null || typeof current !== "object") return false;
+    const key = /^\d+$/.test(part) ? Number(part) : part;
+    current = current[key];
+    if (current === void 0) return false;
+  }
+  return true;
+}
+
+// src/workspace-inspector.ts
+var InMemoryWorkspaceInspector = class {
+  name = "in-memory";
+  snapshots = /* @__PURE__ */ new Map();
+  set(scopeId, snapshot) {
+    this.snapshots.set(scopeId, snapshot);
+  }
+  async snapshot(context) {
+    return this.snapshots.get(context.scopeId) ?? { files: {}, rows: {}, kv: {} };
+  }
+};
+function fileExists(path) {
+  return {
+    name: `file_exists:${path}`,
+    check(snapshot) {
+      const pass = path in snapshot.files;
+      return {
+        pass,
+        score: pass ? 1 : 0,
+        detail: pass ? void 0 : `No file at ${path}`
+      };
+    }
+  };
+}
+function fileContains(path, needle) {
+  return {
+    name: `file_contains:${path}:${needle}`,
+    check(snapshot) {
+      const content = snapshot.files[path];
+      if (content === void 0) {
+        return { pass: false, score: 0, detail: `File ${path} missing` };
+      }
+      const pass = content.includes(needle);
+      return { pass, score: pass ? 1 : 0, detail: pass ? void 0 : `File ${path} missing substring "${needle}"` };
+    }
+  };
+}
+function rowCount(table, min, max) {
+  return {
+    name: `row_count:${table}:[${min},${max ?? "\u221E"}]`,
+    check(snapshot) {
+      const rows = snapshot.rows[table] ?? [];
+      const count = rows.length;
+      const upper = max ?? Infinity;
+      const pass = count >= min && count <= upper;
+      const score = pass ? 1 : count < min ? Math.max(0, count / min) : Math.max(0, upper / count);
+      return {
+        pass,
+        score,
+        detail: pass ? void 0 : `Table ${table} has ${count} rows, expected [${min}, ${max ?? "\u221E"}]`
+      };
+    }
+  };
+}
+function rowWhere(table, predicate, options) {
+  const min = options?.min ?? 1;
+  return {
+    name: `row_where:${table}`,
+    check(snapshot) {
+      const rows = snapshot.rows[table] ?? [];
+      const matching = rows.filter(predicate).length;
+      const pass = matching >= min;
+      return {
+        pass,
+        score: pass ? 1 : Math.max(0, matching / min),
+        detail: pass ? void 0 : `Table ${table} has ${matching} matching rows, expected \u2265 ${min}`
+      };
+    }
+  };
+}
+function runAssertions(snapshot, assertions) {
+  const results = assertions.map((a) => ({ assertion: a.name, result: a.check(snapshot) }));
+  const pass = results.every((r) => r.result.pass);
+  const score = results.length ? results.reduce((acc, r) => acc + r.result.score, 0) / results.length : 1;
+  return { pass, score, results };
+}
+
+// src/experiment-tracker.ts
+var InMemoryExperimentStore = class {
+  experiments = /* @__PURE__ */ new Map();
+  runs = /* @__PURE__ */ new Map();
+  async saveExperiment(exp) {
+    this.experiments.set(exp.id, { ...exp });
+  }
+  async getExperiment(id) {
+    const e = this.experiments.get(id);
+    return e ? { ...e } : null;
+  }
+  async listExperiments() {
+    return [...this.experiments.values()].sort((a, b) => b.createdAt.localeCompare(a.createdAt));
+  }
+  async saveRun(run) {
+    this.runs.set(run.id, structuredClone(run));
+  }
+  async getRun(id) {
+    const r = this.runs.get(id);
+    return r ? structuredClone(r) : null;
+  }
+  async listRuns(experimentId) {
+    return [...this.runs.values()].filter((r) => r.experimentId === experimentId).sort((a, b) => b.startedAt.localeCompare(a.startedAt)).map((r) => structuredClone(r));
+  }
+};
+var ExperimentTracker = class {
+  constructor(store) {
+    this.store = store;
+  }
+  store;
+  async startExperiment(name, metadata) {
+    const exp = {
+      id: `exp_${rand(8)}`,
+      name,
+      createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+      metadata
+    };
+    await this.store.saveExperiment(exp);
+    return exp;
+  }
+  async startRun(config) {
+    const exp = await this.store.getExperiment(config.experimentId);
+    if (!exp) throw new Error(`Experiment ${config.experimentId} not found`);
+    const run = {
+      id: `run_${rand(10)}`,
+      experimentId: config.experimentId,
+      name: config.name,
+      config,
+      startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      status: "running"
+    };
+    await this.store.saveRun(run);
+    return run;
+  }
+  async completeRun(runId, report) {
+    const run = await this.store.getRun(runId);
+    if (!run) throw new Error(`Run ${runId} not found`);
+    run.status = "completed";
+    run.completedAt = (/* @__PURE__ */ new Date()).toISOString();
+    run.report = report;
+    await this.store.saveRun(run);
+  }
+  async failRun(runId, error) {
+    const run = await this.store.getRun(runId);
+    if (!run) throw new Error(`Run ${runId} not found`);
+    run.status = "failed";
+    run.completedAt = (/* @__PURE__ */ new Date()).toISOString();
+    run.error = error;
+    await this.store.saveRun(run);
+  }
+  /**
+   * Diff two completed runs. Returns per-scenario deltas, aggregate delta,
+   * and config changes that may explain the movement.
+   */
+  async diff(runIdA, runIdB) {
+    const [a, b] = await Promise.all([this.store.getRun(runIdA), this.store.getRun(runIdB)]);
+    if (!a || !b) throw new Error("Both runs must exist");
+    if (!a.report || !b.report) throw new Error("Both runs must be completed with reports");
+    const byScenarioA = new Map(a.report.results.map((r) => [r.scenarioId, r.overallScore]));
+    const byScenarioB = new Map(b.report.results.map((r) => [r.scenarioId, r.overallScore]));
+    const scenarioIds = /* @__PURE__ */ new Set([...byScenarioA.keys(), ...byScenarioB.keys()]);
+    const scenarios = [];
+    for (const id of scenarioIds) {
+      const aScore = byScenarioA.get(id);
+      const bScore = byScenarioB.get(id);
+      if (aScore === void 0) {
+        scenarios.push({ scenarioId: id, before: null, after: bScore, delta: null, status: "added" });
+      } else if (bScore === void 0) {
+        scenarios.push({ scenarioId: id, before: aScore, after: null, delta: null, status: "removed" });
+      } else {
+        scenarios.push({
+          scenarioId: id,
+          before: aScore,
+          after: bScore,
+          delta: bScore - aScore,
+          status: bScore > aScore ? "improved" : bScore < aScore ? "regressed" : "unchanged"
+        });
+      }
+    }
+    scenarios.sort((x, y) => (y.delta ?? 0) - (x.delta ?? 0));
+    const aggregateDelta = b.report.summary.overallAvg - a.report.summary.overallAvg;
+    const configChanges = {};
+    const keys = /* @__PURE__ */ new Set([...Object.keys(a.config), ...Object.keys(b.config)]);
+    const aCfg = a.config;
+    const bCfg = b.config;
+    for (const k of keys) {
+      if (JSON.stringify(aCfg[k]) !== JSON.stringify(bCfg[k])) {
+        configChanges[k] = { before: aCfg[k], after: bCfg[k] };
+      }
+    }
+    return {
+      before: { runId: runIdA, name: a.name, startedAt: a.startedAt },
+      after: { runId: runIdB, name: b.name, startedAt: b.startedAt },
+      aggregateDelta,
+      scenarios,
+      configChanges
+    };
+  }
+  /** Timeline of aggregate scores for an experiment. */
+  async timeline(experimentId) {
+    const runs = await this.store.listRuns(experimentId);
+    return runs.slice().sort((a, b) => a.startedAt.localeCompare(b.startedAt)).map((r) => ({
+      runId: r.id,
+      startedAt: r.startedAt,
+      overall: r.report?.summary.overallAvg ?? null
+    }));
+  }
+};
+function rand(bytes) {
+  const arr = new Uint8Array(bytes);
+  crypto.getRandomValues(arr);
+  return Array.from(arr).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+// src/prompt-optimizer.ts
+var PromptOptimizer = class {
+  async run(config) {
+    const trials = config.trialsPerScenario ?? 3;
+    const alpha = config.significanceLevel ?? 0.05;
+    if (config.variants.length < 2) {
+      throw new Error("PromptOptimizer requires at least 2 variants");
+    }
+    if (config.scenarioIds.length === 0) {
+      throw new Error("PromptOptimizer requires at least 1 scenario");
+    }
+    const rawScores = /* @__PURE__ */ new Map();
+    for (const variant of config.variants) {
+      const scenarioMap = /* @__PURE__ */ new Map();
+      rawScores.set(variant.id, scenarioMap);
+      for (const scenarioId of config.scenarioIds) {
+        const samples = [];
+        for (let t = 0; t < trials; t++) {
+          const score = await config.scoreVariant({
+            variant,
+            scenarioId,
+            trialIndex: t
+          });
+          if (!Number.isFinite(score)) {
+            throw new Error(`scoreVariant returned non-finite: variant=${variant.id} scenario=${scenarioId} trial=${t}`);
+          }
+          samples.push(score);
+        }
+        scenarioMap.set(scenarioId, samples);
+        config.onScenarioComplete?.({
+          variantId: variant.id,
+          scenarioId,
+          scores: samples
+        });
+      }
+    }
+    const scores = config.variants.map((variant) => {
+      const scenarioMap = rawScores.get(variant.id);
+      const allSamples = [];
+      const perScenario = {};
+      for (const scenarioId of config.scenarioIds) {
+        const samples = scenarioMap.get(scenarioId) ?? [];
+        allSamples.push(...samples);
+        perScenario[scenarioId] = {
+          mean: samples.length ? samples.reduce((a, b) => a + b, 0) / samples.length : 0,
+          n: samples.length,
+          samples
+        };
+      }
+      const ci = confidenceInterval(allSamples, 0.95);
+      return {
+        variantId: variant.id,
+        mean: ci.mean,
+        ci95: { lower: ci.lower, upper: ci.upper },
+        n: allSamples.length,
+        perScenario
+      };
+    });
+    const pairwise = [];
+    for (let i = 0; i < scores.length; i++) {
+      for (let j = i + 1; j < scores.length; j++) {
+        const a = scores[i];
+        const b = scores[j];
+        const samplesA = flatSamples(a);
+        const samplesB = flatSamples(b);
+        const { p } = mannWhitneyU(samplesA, samplesB);
+        pairwise.push({
+          variantA: a.variantId,
+          variantB: b.variantId,
+          pValue: p,
+          significant: p < alpha,
+          meanDelta: b.mean - a.mean
+        });
+      }
+    }
+    const sorted = scores.slice().sort((x, y) => y.mean - x.mean);
+    const winner = sorted[0];
+    const second = sorted[1];
+    const winnerComparisons = pairwise.filter(
+      (c) => c.variantA === winner.variantId || c.variantB === winner.variantId
+    );
+    const significantOverAll = winnerComparisons.every((c) => c.significant);
+    const ciLowerBoundExceedsSecondMean = winner.ci95.lower > second.mean;
+    return {
+      winner: {
+        variantId: winner.variantId,
+        significant: significantOverAll,
+        ciLowerBoundExceedsSecondMean
+      },
+      scores,
+      pairwise,
+      config: {
+        trialsPerScenario: trials,
+        significanceLevel: alpha,
+        variants: config.variants.map((v) => v.id),
+        scenarios: config.scenarioIds
+      }
+    };
+  }
+};
+function flatSamples(score) {
+  const out = [];
+  for (const s of Object.values(score.perScenario)) out.push(...s.samples);
+  return out;
+}
+
+// src/dual-agent-bench.ts
+var DualAgentBench = class {
+  async run(config) {
+    const maxRounds = config.maxRounds ?? 5;
+    const threshold = config.convergenceThreshold ?? 0.85;
+    if (config.scenarios.length === 0) {
+      throw new Error("DualAgentBench requires at least 1 scenario");
+    }
+    const results = [];
+    for (const scenario of config.scenarios) {
+      const history = [];
+      let converged = false;
+      let roundsToConverge = null;
+      let finalProposal = "";
+      let lastScore = 0;
+      let priorCritique;
+      for (let r = 0; r < maxRounds; r++) {
+        const priorProposal = history[history.length - 1]?.proposal;
+        const proposal = await config.propose({
+          scenario,
+          roundIndex: r,
+          priorProposal,
+          priorCritique
+        });
+        const { critique, convergenceScore } = await config.critique({
+          scenario,
+          roundIndex: r,
+          proposal
+        });
+        if (!Number.isFinite(convergenceScore) || convergenceScore < 0 || convergenceScore > 1) {
+          throw new Error(
+            `critique must return convergenceScore in [0,1]; got ${convergenceScore} for scenario ${scenario.id} round ${r}`
+          );
+        }
+        const round = {
+          roundIndex: r,
+          proposal,
+          critique,
+          convergenceScore
+        };
+        history.push(round);
+        config.onRoundComplete?.({ scenarioId: scenario.id, round });
+        finalProposal = proposal;
+        lastScore = convergenceScore;
+        priorCritique = critique;
+        if (convergenceScore >= threshold) {
+          converged = true;
+          roundsToConverge = r + 1;
+          break;
+        }
+      }
+      results.push({
+        scenarioId: scenario.id,
+        converged,
+        roundsToConverge,
+        finalProposal,
+        history,
+        finalScore: lastScore
+      });
+    }
+    const convergedResults = results.filter((r) => r.converged);
+    const convergenceRate = results.length ? convergedResults.length / results.length : 0;
+    const avgRoundsToConverge = convergedResults.length ? convergedResults.reduce((acc, r) => acc + (r.roundsToConverge ?? 0), 0) / convergedResults.length : null;
+    const avgFinalScore = results.length ? results.reduce((acc, r) => acc + r.finalScore, 0) / results.length : 0;
+    return {
+      scenarios: results,
+      aggregate: { convergenceRate, avgRoundsToConverge, avgFinalScore },
+      config: { maxRounds, convergenceThreshold: threshold }
+    };
+  }
+};
 export {
   AgentDriver,
   BenchmarkRunner,
   ConvergenceTracker,
+  DualAgentBench,
+  ExperimentTracker,
+  FileSystemTraceStore,
+  InMemoryExperimentStore,
+  InMemoryWorkspaceInspector,
   MODEL_PRICING,
+  MemoryTraceStore,
   MetricsCollector,
   ProductClient,
+  PromptOptimizer,
+  PromptRegistry,
   ScenarioRegistry,
   TokenCounter,
   adversarialJudge,
+  analyzeAntiSlop,
+  byteLengthRange,
   codeExecutionJudge,
+  cohensD,
   coherenceJudge,
+  composeValidators,
   confidenceInterval,
+  containsAll,
+  createAntiSlopJudge,
   createCustomJudge,
   createDomainExpertJudge,
   defaultJudges,
   estimateCost,
   estimateTokens,
   executeScenario,
+  fileContains,
+  fileExists,
   formatBenchmarkReport,
   formatDriverReport,
+  hashContent,
   interRaterReliability,
+  jsonHasKeys,
   mannWhitneyU,
   normalizeScores,
+  pairedTTest,
   partialCredit,
   printDriverSummary,
+  regexMatch,
+  rowCount,
+  rowWhere,
+  runAssertions,
   runE2EWorkflow,
-  weightedMean
+  weightedMean,
+  wilcoxonSignedRank
 };
 //# sourceMappingURL=index.js.map