@tangle-network/agent-eval 0.25.0 → 0.27.0

package/dist/index.js

@@ -96,7 +96,7 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-EDUKQ5AM.js";
+} from "./chunk-WHZMVFUV.js";
 import {
   RunRecordValidationError,
   isRunRecord,
@@ -425,12 +425,12 @@ function ghCliClient(opts = {}) {
       await exec("git", ["branch", "-D", input.branchName], { cwd });
       await run("git", ["checkout", "-b", input.branchName]);
       const { mkdir, writeFile } = await import("fs/promises");
-      const { dirname: dirname5, join: join3, resolve } = await import("path");
+      const { dirname: dirname5, join: join4, resolve } = await import("path");
       for (const change of input.fileChanges) {
         const abs = resolve(cwd, change.path);
         await mkdir(dirname5(abs), { recursive: true });
         await writeFile(abs, change.contents, "utf8");
-        await run("git", ["add", join3(change.path)]);
+        await run("git", ["add", join4(change.path)]);
       }
       const env = {};
       if (input.authorName) env.GIT_AUTHOR_NAME = input.authorName;
@@ -3073,36 +3073,36 @@ var FileSystemExperimentStore = class {
     return idx.listRuns(experimentId);
   }
   async ensureDir() {
-    const fs = await import("fs/promises");
-    await fs.mkdir(this.dir, { recursive: true });
+    const fs2 = await import("fs/promises");
+    await fs2.mkdir(this.dir, { recursive: true });
   }
   async append(name, record) {
     await this.ensureDir();
-    const fs = await import("fs/promises");
+    const fs2 = await import("fs/promises");
     const path = await import("path");
     const active = path.join(this.dir, `${name}.ndjson`);
     try {
-      const stat = await fs.stat(active);
+      const stat = await fs2.stat(active);
       if (stat.size >= this.maxBytes) {
         const rolled = path.join(this.dir, `${name}.${Date.now()}.ndjson`);
-        await fs.rename(active, rolled);
+        await fs2.rename(active, rolled);
       }
     } catch {
     }
-    await fs.appendFile(active, `${JSON.stringify(record)}
+    await fs2.appendFile(active, `${JSON.stringify(record)}
 `, "utf8");
   }
   async load() {
     if (this.loaded && this.index) return this.index;
-    const fs = await import("fs/promises");
+    const fs2 = await import("fs/promises");
     const path = await import("path");
     const store = new InMemoryExperimentStore();
     try {
-      const entries = await fs.readdir(this.dir);
+      const entries = await fs2.readdir(this.dir);
       const sorted = entries.filter((f) => f.endsWith(".ndjson")).sort((a, b) => a.localeCompare(b));
       for (const file of sorted) {
         const full = path.join(this.dir, file);
-        const content = await fs.readFile(full, "utf8");
+        const content = await fs2.readFile(full, "utf8");
         const base = file.split(".")[0];
         for (const line of content.split("\n")) {
           if (!line.trim()) continue;
@@ -5063,6 +5063,218 @@ function weightedKappa(a, b) {
   if (den === 0) return 1;
   return 1 - num / den;
 }
+function continuousAgreement(scores, opts = {}) {
+  const bootstrap = opts.bootstrap ?? 1e3;
+  const weights = opts.weights ?? "quadratic";
+  const seed = opts.seed ?? 12648430;
+  const ciLevel = opts.ciLevel ?? 0.95;
+  const matrix = scores.filter((row) => row.length >= 2 && row.every((v) => Number.isFinite(v)));
+  const raters = matrix[0]?.length ?? 0;
+  const clean = matrix.filter((row) => row.length === raters);
+  const nClean = clean.length;
+  if (nClean < 2 || raters < 2) {
+    return {
+      weightedKappa: NaN,
+      icc: NaN,
+      pearson: NaN,
+      spearman: NaN,
+      ci: { icc: [NaN, NaN], weightedKappa: [NaN, NaN] },
+      n: nClean,
+      raters
+    };
+  }
+  const kappa = continuousWeightedKappa(clean, weights);
+  const icc = icc21(clean);
+  const pearson = avgPairwise(clean, pearsonR);
+  const spearman = avgPairwise(clean, spearmanR);
+  const ciIcc = [NaN, NaN];
+  const ciKappa = [NaN, NaN];
+  if (bootstrap > 0) {
+    const rng = mulberry32(seed);
+    const iccs = [];
+    const kappas = [];
+    for (let b = 0; b < bootstrap; b++) {
+      const sample = new Array(nClean);
+      for (let i = 0; i < nClean; i++) {
+        sample[i] = clean[Math.floor(rng() * nClean)];
+      }
+      const iccB = icc21(sample);
+      const kB = continuousWeightedKappa(sample, weights);
+      if (Number.isFinite(iccB)) iccs.push(iccB);
+      if (Number.isFinite(kB)) kappas.push(kB);
+    }
+    const [lo, hi] = percentileBounds(ciLevel);
+    if (iccs.length > 0) {
+      iccs.sort((a, b) => a - b);
+      ciIcc[0] = quantile(iccs, lo);
+      ciIcc[1] = quantile(iccs, hi);
+    }
+    if (kappas.length > 0) {
+      kappas.sort((a, b) => a - b);
+      ciKappa[0] = quantile(kappas, lo);
+      ciKappa[1] = quantile(kappas, hi);
+    }
+  }
+  return {
+    weightedKappa: kappa,
+    icc,
+    pearson,
+    spearman,
+    ci: { icc: ciIcc, weightedKappa: ciKappa },
+    n: nClean,
+    raters
+  };
+}
+function calibrateJudgeContinuous(golden, candidate, opts = {}) {
+  const base = calibrateJudge(golden, candidate);
+  const map = /* @__PURE__ */ new Map();
+  for (const g of golden) map.set(g.itemId, { h: g.humanScore, j: NaN });
+  for (const c of candidate) {
+    const entry = map.get(c.itemId);
+    if (entry) entry.j = c.score;
+  }
+  const rows = [];
+  for (const v of map.values()) {
+    if (Number.isFinite(v.j)) rows.push([v.h, v.j]);
+  }
+  const agreement = continuousAgreement(rows, opts);
+  return {
+    ...base,
+    weightedKappaContinuous: agreement.weightedKappa,
+    icc: agreement.icc,
+    spearman: agreement.spearman,
+    ci: agreement.ci
+  };
+}
+function continuousWeightedKappa(rows, scheme) {
+  if (rows.length === 0) return NaN;
+  const raters = rows[0].length;
+  if (raters < 2) return NaN;
+  const wFn = scheme === "linear" ? (x, y) => Math.abs(x - y) : (x, y) => (x - y) ** 2;
+  let sum2 = 0;
+  let pairs = 0;
+  for (let r1 = 0; r1 < raters; r1++) {
+    for (let r2 = r1 + 1; r2 < raters; r2++) {
+      const a = rows.map((row) => row[r1]);
+      const b = rows.map((row) => row[r2]);
+      const n = a.length;
+      let obs = 0;
+      for (let i = 0; i < n; i++) obs += wFn(a[i], b[i]);
+      obs /= n;
+      let exp = 0;
+      for (let i = 0; i < n; i++) {
+        for (let j = 0; j < n; j++) exp += wFn(a[i], b[j]);
+      }
+      exp /= n * n;
+      if (exp === 0) {
+        sum2 += obs === 0 ? 1 : 0;
+      } else {
+        sum2 += 1 - obs / exp;
+      }
+      pairs++;
+    }
+  }
+  return pairs === 0 ? NaN : sum2 / pairs;
+}
+function icc21(rows) {
+  const n = rows.length;
+  if (n < 2) return NaN;
+  const k = rows[0].length;
+  if (k < 2) return NaN;
+  const rowMeans = rows.map((row) => row.reduce((s, v) => s + v, 0) / k);
+  const colMeans = new Array(k).fill(0);
+  for (let j = 0; j < k; j++) {
+    let s = 0;
+    for (let i = 0; i < n; i++) s += rows[i][j];
+    colMeans[j] = s / n;
+  }
+  let grand = 0;
+  for (let i = 0; i < n; i++) grand += rowMeans[i];
+  grand /= n;
+  let ssR = 0;
+  for (let i = 0; i < n; i++) ssR += (rowMeans[i] - grand) ** 2;
+  ssR *= k;
+  let ssC = 0;
+  for (let j = 0; j < k; j++) ssC += (colMeans[j] - grand) ** 2;
+  ssC *= n;
+  let ssT = 0;
+  for (let i = 0; i < n; i++) {
+    for (let j = 0; j < k; j++) ssT += (rows[i][j] - grand) ** 2;
+  }
+  const ssE = ssT - ssR - ssC;
+  const dfR = n - 1;
+  const dfC = k - 1;
+  const dfE = (n - 1) * (k - 1);
+  const msR = ssR / dfR;
+  const msC = ssC / dfC;
+  const msE = dfE > 0 ? ssE / dfE : 0;
+  const denom = msR + (k - 1) * msE + k * (msC - msE) / n;
+  if (denom === 0) {
+    return msR === 0 && msE === 0 ? 1 : 0;
+  }
+  return (msR - msE) / denom;
+}
+function avgPairwise(rows, fn) {
+  const k = rows[0]?.length ?? 0;
+  if (k < 2) return NaN;
+  let sum2 = 0;
+  let pairs = 0;
+  for (let i = 0; i < k; i++) {
+    for (let j = i + 1; j < k; j++) {
+      const a = rows.map((row) => row[i]);
+      const b = rows.map((row) => row[j]);
+      const r = fn(a, b);
+      if (Number.isFinite(r)) {
+        sum2 += r;
+        pairs++;
+      }
+    }
+  }
+  return pairs === 0 ? NaN : sum2 / pairs;
+}
+function spearmanR(a, b) {
+  if (a.length !== b.length || a.length < 2) return NaN;
+  return pearsonR(rankWithTies(a), rankWithTies(b));
+}
+function rankWithTies(xs) {
+  const n = xs.length;
+  const indexed = xs.map((v, i2) => ({ v, i: i2 }));
+  indexed.sort((x, y) => x.v - y.v);
+  const ranks = new Array(n).fill(0);
+  let i = 0;
+  while (i < n) {
+    let j = i;
+    while (j + 1 < n && indexed[j + 1].v === indexed[i].v) j++;
+    const avg = (i + j) / 2 + 1;
+    for (let k = i; k <= j; k++) ranks[indexed[k].i] = avg;
+    i = j + 1;
+  }
+  return ranks;
+}
+function mulberry32(seed) {
+  let a = seed >>> 0;
+  return () => {
+    a = a + 1831565813 >>> 0;
+    let t = a;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+function percentileBounds(ciLevel) {
+  const tail = (1 - ciLevel) / 2;
+  return [tail, 1 - tail];
+}
+function quantile(sorted, q) {
+  if (sorted.length === 0) return NaN;
+  if (sorted.length === 1) return sorted[0];
+  const pos = q * (sorted.length - 1);
+  const lo = Math.floor(pos);
+  const hi = Math.ceil(pos);
+  if (lo === hi) return sorted[lo];
+  const frac = pos - lo;
+  return sorted[lo] * (1 - frac) + sorted[hi] * frac;
+}
 
 // src/observability.ts
 async function toLangfuseEnvelope(store, runId) {
@@ -5564,7 +5776,7 @@ async function commitBisect(options) {
 }
 async function promptBisect(options) {
   const split = options.paragraphSplitter ?? ((p) => p.split(/\n\s*\n/));
-  const join3 = (paragraphs) => paragraphs.join("\n\n");
+  const join4 = (paragraphs) => paragraphs.join("\n\n");
   const goodParas = split(options.good);
   const badParas = split(options.bad);
   if (goodParas.length !== badParas.length) {
@@ -5584,7 +5796,7 @@ async function promptBisect(options) {
   const result = await bisect({
     good: goodMask,
     bad: badMask,
-    runEval: (mask) => options.runEval(join3(paragraphsFor(mask))),
+    runEval: (mask) => options.runEval(join4(paragraphsFor(mask))),
     maxIterations: options.maxIterations ?? n + 5,
     halfway: (g, b) => {
       for (let i = 0; i < g.length; i++) {
@@ -5615,12 +5827,12 @@ async function promptBisect(options) {
     }
   }
   const materializedPath = result.path.map((s) => ({
-    state: join3(paragraphsFor(s.state)),
+    state: join4(paragraphsFor(s.state)),
     score: s.score,
     pass: s.pass
   }));
   return {
-    culprit: join3(paragraphsFor(culprit)),
+    culprit: join4(paragraphsFor(culprit)),
     path: materializedPath,
     converged: result.converged,
     inputInconsistent: result.inputInconsistent,
@@ -5865,7 +6077,7 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
     runCountByScenario.set(r.scenarioId, (runCountByScenario.get(r.scenarioId) ?? 0) + 1);
   }
   const runCounts = [...runCountByScenario.values()];
-  const p25 = runCounts.length > 0 ? quantile(runCounts, 0.25) : 0;
+  const p25 = runCounts.length > 0 ? quantile2(runCounts, 0.25) : 0;
   for (const s of scenarios) {
     const count = runCountByScenario.get(s.id) ?? 0;
     if (count <= p25 && count < 3) {
@@ -5919,7 +6131,7 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
   }
   return targets.sort((a, b) => b.priority - a.priority).slice(0, topK);
 }
-function quantile(xs, p) {
+function quantile2(xs, p) {
   const sorted = [...xs].sort((a, b) => a - b);
   const idx = p * (sorted.length - 1);
   const lo = Math.floor(idx);
@@ -8308,6 +8520,52 @@ function createCompositeMutator(opts) {
   };
 }
 
+// src/discover-personas.ts
+import { promises as fs } from "fs";
+import { basename, extname, join as join3 } from "path";
+var DEFAULT_PATTERN = /^\d{2}-.+\.(yaml|yml|json|md)$/;
+async function discoverPersonas(dir, opts = {}) {
+  const pattern = opts.pattern ?? DEFAULT_PATTERN;
+  const exclude = new Set(opts.exclude ?? []);
+  const include = opts.include;
+  async function walk(d) {
+    let entries;
+    try {
+      const raw = await fs.readdir(d, { withFileTypes: true });
+      entries = raw.map((e) => ({ name: e.name, isDir: e.isDirectory() }));
+    } catch (err) {
+      const code = err.code;
+      if (code === "ENOENT") return [];
+      throw err;
+    }
+    const out = [];
+    for (const entry of entries) {
+      const full = join3(d, entry.name);
+      if (entry.isDir) {
+        if (opts.recursive) out.push(...await walk(full));
+        continue;
+      }
+      if (!pattern.test(entry.name)) continue;
+      if (exclude.has(entry.name) || exclude.has(basename(entry.name, extname(entry.name))))
+        continue;
+      if (include && include.length > 0) {
+        const id = basename(entry.name, extname(entry.name));
+        const matched = include.some((needle) => entry.name.includes(needle) || id.includes(needle));
+        if (!matched) continue;
+      }
+      out.push({
+        path: full,
+        filename: entry.name,
+        id: basename(entry.name, extname(entry.name))
+      });
+    }
+    return out;
+  }
+  const results = await walk(dir);
+  results.sort((a, b) => a.filename.localeCompare(b.filename));
+  return results;
+}
+
 // src/evolution-telemetry.ts
 import { appendFileSync as appendFileSync3, existsSync as existsSync5, mkdirSync as mkdirSync3, readFileSync as readFileSync4, writeFileSync } from "fs";
 import { dirname as dirname3 } from "path";
@@ -8697,6 +8955,90 @@ var JsonlTrialCache = class {
   }
 };
 
+// src/judge-retry.ts
+var DEFAULT_MAX_ATTEMPTS = 3;
+var DEFAULT_TIMEOUT_MS = 9e4;
+var DEFAULT_BACKOFF = (attempt) => Math.min(500 * 2 ** attempt, 16e3);
+var ABORT_PATTERNS = [
+  /AbortError/i,
+  /TimeoutError/i,
+  /fetch failed/i,
+  /ECONNRESET/i,
+  /ETIMEDOUT/i,
+  /EAI_AGAIN/i,
+  /this operation was aborted/i,
+  /stream.*ended.*unexpectedly/i,
+  /socket hang up/i
+];
+var RETRYABLE_HTTP_STATUS = /* @__PURE__ */ new Set([429, 502, 503, 504]);
+function defaultIsRetryable(err) {
+  if (err instanceof Error) {
+    if (ABORT_PATTERNS.some((p) => p.test(err.message) || p.test(err.name))) return true;
+    const status = err.status;
+    if (typeof status === "number" && RETRYABLE_HTTP_STATUS.has(status)) return true;
+  }
+  return false;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function withJudgeRetry(judgeFn, policy = {}) {
+  const maxAttempts = policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+  const timeoutMs = policy.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const backoff = policy.backoffMs ?? DEFAULT_BACKOFF;
+  const isRetryable = policy.isRetryable ?? defaultIsRetryable;
+  const models = policy.models && policy.models.length > 0 ? policy.models : [void 0];
+  let totalAttempts = 0;
+  const attemptErrors = [];
+  let lastError;
+  for (const model of models) {
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      totalAttempts += 1;
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(new Error("TimeoutError")), timeoutMs);
+      try {
+        const value = await judgeFn(model, controller.signal);
+        clearTimeout(timer);
+        return {
+          value,
+          succeeded: true,
+          attempts: totalAttempts,
+          modelUsed: model,
+          attemptErrors
+        };
+      } catch (err) {
+        clearTimeout(timer);
+        const errObj = err instanceof Error ? err : new Error(String(err));
+        lastError = errObj;
+        attemptErrors.push({
+          attempt: totalAttempts,
+          model: model ?? "(default)",
+          error: errObj.message
+        });
+        if (!isRetryable(errObj)) {
+          return {
+            value: null,
+            succeeded: false,
+            attempts: totalAttempts,
+            error: errObj,
+            attemptErrors
+          };
+        }
+        if (attempt < maxAttempts - 1) {
+          await sleep(backoff(attempt));
+        }
+      }
+    }
+  }
+  return {
+    value: null,
+    succeeded: false,
+    attempts: totalAttempts,
+    error: lastError,
+    attemptErrors
+  };
+}
+
 // src/orthogonality.ts
 function passOrthogonality(input) {
   const passes = input.passes;
@@ -8914,6 +9256,55 @@ function createSandboxPool(opts) {
     utilization
   };
 }
+
+// src/trial-aggregator.ts
+function meanOf(xs) {
+  if (xs.length === 0) return 0;
+  return xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+function meanMetrics(rows) {
+  if (rows.length === 0) return {};
+  const keys = /* @__PURE__ */ new Set();
+  for (const row of rows) for (const k of Object.keys(row)) keys.add(k);
+  const out = {};
+  for (const k of keys) {
+    const xs = rows.map((r) => r[k]).filter((x) => typeof x === "number");
+    if (xs.length > 0) out[k] = meanOf(xs);
+  }
+  return out;
+}
+function aggregateTrialsByMode(trials, opts) {
+  const gradedTrials = trials.filter((t) => !t.error);
+  const judgeOk = gradedTrials.filter((t) => t.judgeSucceeded !== false);
+  const judgeFailed = gradedTrials.filter((t) => t.judgeSucceeded === false);
+  if (opts.mode === "strict-fail" && judgeFailed.length > 0) {
+    return {
+      meanScore: 0,
+      meanCost: 0,
+      meanDurationMs: 0,
+      okRate: 0,
+      countedTrials: 0,
+      excludedFailedTrials: judgeFailed.length,
+      totalTrials: trials.length,
+      metrics: {},
+      strictFailure: {
+        failedCount: judgeFailed.length,
+        firstError: judgeFailed.find((t) => t.judgeError)?.judgeError
+      }
+    };
+  }
+  const counted = opts.mode === "exclude-failed" ? judgeOk : gradedTrials;
+  return {
+    meanScore: meanOf(counted.map((t) => t.score)),
+    meanCost: meanOf(counted.map((t) => t.cost ?? 0)),
+    meanDurationMs: meanOf(counted.map((t) => t.durationMs ?? 0)),
+    okRate: gradedTrials.length === 0 ? 0 : gradedTrials.filter((t) => t.ok).length / gradedTrials.length,
+    countedTrials: counted.length,
+    excludedFailedTrials: judgeFailed.length,
+    totalTrials: trials.length,
+    metrics: meanMetrics(counted.map((t) => t.metrics ?? {}))
+  };
+}
 export {
   AgentDriver,
   AgentEvalError,
@@ -9003,6 +9394,7 @@ export {
   adversarialJudge,
   aggregateLlm,
   aggregateRunScore,
+  aggregateTrialsByMode,
   allCriticalPassed,
   analyzeAntiSlop,
   analyzeSeries,
@@ -9025,6 +9417,7 @@ export {
   buildTrajectory,
   byteLengthRange,
   calibrateJudge,
+  calibrateJudgeContinuous,
   callLlm,
   callLlmJson,
   canaryLeakView,
@@ -9049,6 +9442,7 @@ export {
   computeToolUseMetrics,
   confidenceInterval,
   containsAll,
+  continuousAgreement,
   controlFailureClassFromVerification,
   controlRunToFeedbackTrajectory,
   controlRunToRunRecord,
@@ -9073,6 +9467,7 @@ export {
   defaultProviderRedactor,
   defaultReferenceReplayMatcher,
   deployGateLayer,
+  discoverPersonas,
   distillPlaybook,
   dominates,
   estimateCost,
@@ -9275,6 +9670,7 @@ export {
   whitespaceCollapseMutator,
   wilcoxonSignedRank,
   withAssignedFeedbackSplit,
+  withJudgeRetry,
   wranglerDeployRunner
 };
 //# sourceMappingURL=index.js.map