@tangle-network/agent-eval 0.13.0 → 0.14.1

package/dist/index.js

@@ -9538,6 +9538,49 @@ function extractErrorCount(text, opts = {}) {
 // src/reference-replay.ts
 import { appendFileSync as appendFileSync2, existsSync as existsSync4, mkdirSync as mkdirSync2, readFileSync as readFileSync4 } from "fs";
 import { dirname as dirname2 } from "path";
+
+// src/concurrency.ts
+var Mutex = class {
+  locked = false;
+  waiters = [];
+  async acquire() {
+    if (!this.locked) {
+      this.locked = true;
+      return () => this.release();
+    }
+    return new Promise((resolve) => {
+      this.waiters.push(() => {
+        resolve(() => this.release());
+      });
+    });
+  }
+  release() {
+    const next = this.waiters.shift();
+    if (next) {
+      next();
+    } else {
+      this.locked = false;
+    }
+  }
+  async runExclusive(fn) {
+    const release = await this.acquire();
+    try {
+      return await fn();
+    } finally {
+      release();
+    }
+  }
+  /** True iff someone holds the lock right now. Diagnostics only. */
+  get isLocked() {
+    return this.locked;
+  }
+  /** Pending waiter count. Diagnostics only. */
+  get pending() {
+    return this.waiters.length;
+  }
+};
+
+// src/reference-replay.ts
 var DEFAULT_MATCH_THRESHOLD = 0.55;
 var ALL_SPLITS = ["train", "dev", "test", "holdout"];
 async function runReferenceReplay(cases, options) {
@@ -9638,15 +9681,29 @@ function inMemoryReferenceReplayStore(initial = []) {
     }
   };
 }
+var jsonlStoreLocks = /* @__PURE__ */ new Map();
+function getJsonlStoreLock(path) {
+  let m = jsonlStoreLocks.get(path);
+  if (!m) {
+    m = new Mutex();
+    jsonlStoreLocks.set(path, m);
+  }
+  return m;
+}
 function jsonlReferenceReplayStore(path) {
+  const lock = getJsonlStoreLock(path);
   return {
     async save(run) {
-      mkdirSync2(dirname2(path), { recursive: true });
-      appendFileSync2(path, JSON.stringify(run) + "\n");
+      await lock.runExclusive(() => {
+        mkdirSync2(dirname2(path), { recursive: true });
+        appendFileSync2(path, JSON.stringify(run) + "\n");
+      });
     },
     async list() {
-      if (!existsSync4(path)) return [];
-      return readJsonl(path);
+      return lock.runExclusive(() => {
+        if (!existsSync4(path)) return [];
+        return readJsonl(path);
+      });
     }
   };
 }
@@ -10195,13 +10252,14 @@ function aggregateTrials(population, scenarioIds, trials) {
     const scenarios = scenarioIds.map((sid) => {
       const scenarioTrials = variantTrials.filter((t) => t.scenarioId === sid);
       const okTrials = scenarioTrials.filter((t) => t.ok);
-      const metrics = aggregateMetrics(okTrials.map((t) => t.metrics ?? {}));
+      const gradedTrials = scenarioTrials.filter((t) => !t.error);
+      const metrics = aggregateMetrics(gradedTrials.map((t) => t.metrics ?? {}));
       return {
         variantId: variant.id,
         scenarioId: sid,
-        meanScore: mean5(okTrials.map((t) => t.score)),
-        meanCost: mean5(okTrials.map((t) => t.cost ?? 0)),
-        meanDurationMs: mean5(okTrials.map((t) => t.durationMs ?? 0)),
+        meanScore: mean5(gradedTrials.map((t) => t.score)),
+        meanCost: mean5(gradedTrials.map((t) => t.cost ?? 0)),
+        meanDurationMs: mean5(gradedTrials.map((t) => t.durationMs ?? 0)),
         okRate: scenarioTrials.length === 0 ? 0 : okTrials.length / scenarioTrials.length,
         trials: scenarioTrials.length,
         metrics
@@ -10272,6 +10330,562 @@ function samePopulation(a, b) {
   return b.every((id) => setA.has(id));
 }
 
+// src/jsonl-trial-cache.ts
+import { appendFileSync as appendFileSync4, existsSync as existsSync6, mkdirSync as mkdirSync4, readFileSync as readFileSync5 } from "fs";
+import { dirname as dirname4 } from "path";
+
+// src/locked-jsonl-appender.ts
+import { appendFileSync as appendFileSync3, existsSync as existsSync5, mkdirSync as mkdirSync3 } from "fs";
+import { dirname as dirname3 } from "path";
+var mutexes = /* @__PURE__ */ new Map();
+function getMutex(path) {
+  let m = mutexes.get(path);
+  if (!m) {
+    m = new Mutex();
+    mutexes.set(path, m);
+  }
+  return m;
+}
+var LockedJsonlAppender = class {
+  constructor(path) {
+    this.path = path;
+    this.mutex = getMutex(path);
+    if (!existsSync5(dirname3(path))) {
+      mkdirSync3(dirname3(path), { recursive: true });
+    }
+  }
+  path;
+  mutex;
+  async append(entry) {
+    const line = `${JSON.stringify(entry)}
+`;
+    await this.mutex.runExclusive(() => {
+      appendFileSync3(this.path, line);
+    });
+  }
+};
+function resetLockedAppendersForTesting() {
+  mutexes.clear();
+}
+
+// src/jsonl-trial-cache.ts
+var JsonlTrialCache = class {
+  map = /* @__PURE__ */ new Map();
+  path;
+  appender;
+  constructor(path) {
+    this.path = path;
+    if (existsSync6(path)) {
+      for (const line of readFileSync5(path, "utf-8").split("\n")) {
+        if (!line.trim()) continue;
+        try {
+          const entry = JSON.parse(line);
+          this.map.set(entry.key, entry.result);
+        } catch {
+        }
+      }
+    } else {
+      mkdirSync4(dirname4(path), { recursive: true });
+    }
+    this.appender = new LockedJsonlAppender(path);
+  }
+  get(key) {
+    return this.map.get(key);
+  }
+  set(key, value) {
+    this.map.set(key, value);
+    const line = { key, result: value, writtenAt: Date.now() };
+    void this.appender.append(line);
+  }
+  size() {
+    return this.map.size;
+  }
+  /**
+   * Synchronous fallback path for tests / CLI tools that want to be sure
+   * the line is on disk before returning. Bypasses the mutex (single-
+   * threaded callers only).
+   */
+  setSync(key, value) {
+    this.map.set(key, value);
+    const line = { key, result: value, writtenAt: Date.now() };
+    appendFileSync4(this.path, `${JSON.stringify(line)}
+`);
+  }
+};
+
+// src/evolution-telemetry.ts
+import { appendFileSync as appendFileSync5, existsSync as existsSync7, mkdirSync as mkdirSync5, readFileSync as readFileSync6, writeFileSync } from "fs";
+import { dirname as dirname5 } from "path";
+var MutationTelemetry = class {
+  appender;
+  constructor(path) {
+    this.appender = new LockedJsonlAppender(path);
+  }
+  async record(attempt) {
+    await this.appender.append(attempt);
+  }
+};
+var TrialTelemetry = class {
+  appender;
+  constructor(path) {
+    this.appender = new LockedJsonlAppender(path);
+  }
+  async record(attempt) {
+    await this.appender.append(attempt);
+  }
+};
+var LineageRecorder = class {
+  path;
+  snapshotPath;
+  mutex = new Mutex();
+  nodes = /* @__PURE__ */ new Map();
+  kindOf;
+  constructor(path, kindOf) {
+    this.path = path;
+    this.snapshotPath = `${path}.snapshot`;
+    this.kindOf = kindOf ?? defaultKindOf;
+    mkdirSync5(dirname5(path), { recursive: true });
+    if (existsSync7(this.snapshotPath)) {
+      try {
+        const parsed = JSON.parse(readFileSync6(this.snapshotPath, "utf-8"));
+        for (const n of parsed) this.nodes.set(n.id, n);
+      } catch {
+      }
+    }
+    if (existsSync7(path)) {
+      try {
+        for (const line of readFileSync6(path, "utf-8").split("\n")) {
+          if (!line.trim()) continue;
+          try {
+            const entry = JSON.parse(line);
+            const prev = this.nodes.get(entry.id);
+            this.nodes.set(entry.id, { ...prev, ...entry });
+          } catch {
+          }
+        }
+      } catch {
+      }
+    }
+    if (existsSync7(path) && this.nodes.size === 0) {
+      try {
+        const raw = readFileSync6(path, "utf-8").trim();
+        if (raw.startsWith("[")) {
+          const parsed = JSON.parse(raw);
+          for (const n of parsed) this.nodes.set(n.id, n);
+        }
+      } catch {
+      }
+    }
+  }
+  async upsert(node) {
+    await this.mutex.runExclusive(() => {
+      const prev = this.nodes.get(node.id);
+      this.nodes.set(node.id, { ...prev, ...node });
+      try {
+        if (existsSync7(this.path)) {
+          const head = readFileSync6(this.path, { encoding: "utf-8", flag: "r" }).slice(0, 1);
+          if (head === "[") {
+            writeFileSync(this.path, "");
+          }
+        }
+      } catch {
+      }
+      appendFileSync5(this.path, `${JSON.stringify(this.nodes.get(node.id))}
+`);
+    });
+  }
+  async upsertVariant(variant, opts = {}) {
+    await this.upsert({
+      id: variant.id,
+      parentId: variant.parentId ?? null,
+      generation: variant.generation,
+      kind: this.kindOf(variant),
+      ...variant.rationale ? { rationale: variant.rationale } : {},
+      ...opts.omitPayload || variant.payload === void 0 ? {} : { payload: variant.payload }
+    });
+  }
+  snapshot() {
+    return [...this.nodes.values()];
+  }
+  /**
+   * Write the current consolidated state to `<path>.snapshot` so external
+   * tools can read it without replaying the event log. Idempotent.
+   */
+  async compact() {
+    await this.mutex.runExclusive(() => {
+      writeFileSync(this.snapshotPath, JSON.stringify([...this.nodes.values()], null, 2));
+    });
+  }
+};
+function defaultKindOf(variant) {
+  if (variant.parentId === void 0) return "seed";
+  const payload = variant.payload;
+  if (payload && typeof payload === "object" && payload.codeMutation) return "code";
+  return "prompt";
+}
+function emptyGenBucket() {
+  return {
+    mutatorPromptUsd: 0,
+    mutatorCodeUsd: 0,
+    scorerPromptUsd: 0,
+    scorerCodeUsd: 0,
+    trialsCounted: 0,
+    cachedTrials: 0
+  };
+}
+var CostLedger = class {
+  totals = {
+    mutatorPromptUsd: 0,
+    mutatorCodeUsd: 0,
+    scorerPromptUsd: 0,
+    scorerCodeUsd: 0,
+    trialsCounted: 0,
+    cachedTrials: 0,
+    poolBusyMs: 0,
+    poolUtilizationPct: 0,
+    byGeneration: {}
+  };
+  path;
+  mutex = new Mutex();
+  constructor(path) {
+    this.path = path;
+    if (existsSync7(path)) {
+      try {
+        const loaded = JSON.parse(readFileSync6(path, "utf-8"));
+        for (const k of Object.keys(this.totals)) {
+          if (k === "byGeneration") {
+            if (loaded.byGeneration && typeof loaded.byGeneration === "object") {
+              this.totals.byGeneration = loaded.byGeneration;
+            }
+            continue;
+          }
+          const v = loaded[k];
+          if (typeof v === "number" && Number.isFinite(v)) {
+            this.totals[k] = v;
+          }
+        }
+      } catch {
+      }
+    } else {
+      mkdirSync5(dirname5(path), { recursive: true });
+    }
+  }
+  genBucket(generation) {
+    if (generation === void 0) return null;
+    const key = String(generation);
+    if (!this.totals.byGeneration[key]) {
+      this.totals.byGeneration[key] = emptyGenBucket();
+    }
+    return this.totals.byGeneration[key];
+  }
+  async addMutation(channel, usd, opts = {}) {
+    await this.mutex.runExclusive(() => {
+      const bucket = this.genBucket(opts.generation);
+      if (channel === "prompt") {
+        this.totals.mutatorPromptUsd += usd;
+        if (bucket) bucket.mutatorPromptUsd += usd;
+      } else {
+        this.totals.mutatorCodeUsd += usd;
+        if (bucket) bucket.mutatorCodeUsd += usd;
+      }
+      this.persist();
+    });
+  }
+  async addTrial(channel, usd, cached, opts = {}) {
+    await this.mutex.runExclusive(() => {
+      const bucket = this.genBucket(opts.generation);
+      if (cached) {
+        this.totals.cachedTrials++;
+        this.totals.trialsCounted++;
+        if (bucket) {
+          bucket.cachedTrials++;
+          bucket.trialsCounted++;
+        }
+        this.persist();
+        return;
+      }
+      if (channel === "prompt") {
+        this.totals.scorerPromptUsd += usd;
+        if (bucket) bucket.scorerPromptUsd += usd;
+      } else {
+        this.totals.scorerCodeUsd += usd;
+        if (bucket) bucket.scorerCodeUsd += usd;
+      }
+      this.totals.trialsCounted++;
+      if (bucket) bucket.trialsCounted++;
+      this.persist();
+    });
+  }
+  async setPoolUtilization(busyMs, totalMs) {
+    await this.mutex.runExclusive(() => {
+      this.totals.poolBusyMs = busyMs;
+      this.totals.poolUtilizationPct = totalMs > 0 ? 100 * busyMs / totalMs : 0;
+      this.persist();
+    });
+  }
+  snapshot() {
+    const totalUsd = this.totals.mutatorPromptUsd + this.totals.mutatorCodeUsd + this.totals.scorerPromptUsd + this.totals.scorerCodeUsd;
+    const byGeneration = Object.entries(this.totals.byGeneration).map(([g, b]) => ({ generation: Number(g), ...b })).sort((a, b) => a.generation - b.generation);
+    return {
+      totalUsd,
+      mutatorPromptUsd: this.totals.mutatorPromptUsd,
+      mutatorCodeUsd: this.totals.mutatorCodeUsd,
+      scorerPromptUsd: this.totals.scorerPromptUsd,
+      scorerCodeUsd: this.totals.scorerCodeUsd,
+      trialsCounted: this.totals.trialsCounted,
+      cachedTrials: this.totals.cachedTrials,
+      poolBusyMs: this.totals.poolBusyMs,
+      poolUtilizationPct: this.totals.poolUtilizationPct,
+      byGeneration
+    };
+  }
+  persist() {
+    writeFileSync(this.path, JSON.stringify(this.totals, null, 2));
+  }
+};
+
+// src/composite-mutator.ts
+function createCompositeMutator(opts) {
+  const recentScores = [];
+  const plateauThreshold = opts.plateauThreshold ?? 0.02;
+  const plateauPatience = opts.plateauPatience ?? 2;
+  function pickMode(args) {
+    recentScores.push(args.parentAggregate.meanScore);
+    switch (opts.policy) {
+      case "primary-only":
+        return { mode: "primary", reason: "policy=primary-only" };
+      case "secondary-only":
+        if (!opts.secondary) return { mode: "primary", reason: "secondary-only requested but no secondary mutator wired" };
+        return { mode: "secondary", reason: "policy=secondary-only" };
+      case "alternate":
+        if (!opts.secondary) return { mode: "primary", reason: "alternate requested but no secondary mutator wired" };
+        return args.generation % 2 === 1 ? { mode: "secondary", reason: `alternate: gen${args.generation} odd \u2192 secondary` } : { mode: "primary", reason: `alternate: gen${args.generation} even \u2192 primary` };
+      case "plateau": {
+        if (!opts.secondary) return { mode: "primary", reason: "plateau requested but no secondary mutator wired" };
+        if (recentScores.length <= plateauPatience) {
+          return { mode: "primary", reason: "plateau: warming up with primary mutations" };
+        }
+        const window = recentScores.slice(-plateauPatience - 1);
+        const deltas = window.slice(1).map((v, i) => v - window[i]);
+        const stagnant = deltas.every((d) => d < plateauThreshold);
+        if (stagnant) {
+          return {
+            mode: "split",
+            reason: `plateau detected (${deltas.map((d) => d.toFixed(3)).join(", ")}) \u2192 split`
+          };
+        }
+        return {
+          mode: "primary",
+          reason: `plateau: still improving (${deltas[deltas.length - 1].toFixed(3)})`
+        };
+      }
+    }
+  }
+  return {
+    async mutate(args) {
+      const { mode, reason } = pickMode(args);
+      opts.onPolicyDecision?.({ generation: args.generation, chose: mode, reason });
+      if (mode === "primary") return opts.primary.mutate(args);
+      if (mode === "secondary" && opts.secondary) return opts.secondary.mutate(args);
+      if (mode === "split" && opts.secondary) {
+        const secondaryShare = Math.ceil(args.childCount / 2);
+        const primaryShare = args.childCount - secondaryShare;
+        const [primaryChildren, secondaryChildren] = await Promise.all([
+          opts.primary.mutate({ ...args, childCount: primaryShare }),
+          opts.secondary.mutate({ ...args, childCount: secondaryShare })
+        ]);
+        return [...primaryChildren, ...secondaryChildren];
+      }
+      return opts.primary.mutate(args);
+    }
+  };
+}
+
+// src/sandbox-pool.ts
+function createSandboxPool(opts) {
+  if (opts.size < 1) throw new Error(`sandbox pool size must be >= 1 (got ${opts.size})`);
+  const slots = [];
+  const waiters = [];
+  const mutex = new Mutex();
+  let nextSlotId = 0;
+  let totalCheckouts = 0;
+  let busyMs = 0;
+  const startedAt = Date.now();
+  async function acquireSlot() {
+    let mintId;
+    const ready = await mutex.runExclusive(async () => {
+      const idle = slots.find((s) => !s.busy);
+      if (idle) {
+        idle.busy = true;
+        return idle;
+      }
+      if (slots.length < opts.size) {
+        mintId = `slot_${nextSlotId++}`;
+        return null;
+      }
+      return null;
+    });
+    if (ready) return ready;
+    if (mintId !== void 0) {
+      const resource = await opts.factory.create(mintId);
+      const state = {
+        slot: { id: mintId, resource },
+        busy: true
+      };
+      await mutex.runExclusive(() => {
+        slots.push(state);
+      });
+      return state;
+    }
+    return new Promise((resolve) => {
+      waiters.push((s) => {
+        s.busy = true;
+        resolve(s);
+      });
+    });
+  }
+  function releaseSlot(state) {
+    void (async () => {
+      try {
+        if (opts.factory.reset) await opts.factory.reset(state.slot);
+      } catch (err) {
+        console.warn(`[sandbox-pool] reset failed for slot ${state.slot.id}:`, err);
+      }
+      state.busy = false;
+      const next = waiters.shift();
+      if (next) next(state);
+    })();
+  }
+  async function checkout() {
+    const state = await acquireSlot();
+    const checkoutStart = Date.now();
+    totalCheckouts++;
+    return {
+      slot: state.slot,
+      release: () => {
+        busyMs += Date.now() - checkoutStart;
+        releaseSlot(state);
+      }
+    };
+  }
+  async function withSlot(fn) {
+    const { slot, release } = await checkout();
+    try {
+      return await fn(slot);
+    } finally {
+      release();
+    }
+  }
+  async function drain() {
+    const snapshot = await mutex.runExclusive(() => {
+      const taken = slots.splice(0, slots.length);
+      for (const w of waiters.splice(0, waiters.length)) {
+        void w;
+      }
+      return taken;
+    });
+    await Promise.allSettled(snapshot.map((s) => opts.factory.destroy(s.slot)));
+  }
+  function utilization() {
+    return {
+      busyMs,
+      totalMs: Date.now() - startedAt,
+      checkouts: totalCheckouts
+    };
+  }
+  return {
+    checkout,
+    withSlot,
+    drain,
+    poolSize: () => slots.length,
+    activeCheckouts: () => slots.filter((s) => s.busy).length,
+    utilization
+  };
+}
+
+// src/code-mutator.ts
+function createSandboxCodeMutator(opts) {
+  const childIdFor = opts.childIdFor ?? ((parent, generation, index) => `${parent.id}.g${generation}.code.${index}`);
+  const labelFor = opts.labelFor ?? ((outcome, parent, _generation, index) => outcome.description?.slice(0, 80) ?? `${parent.label} \u2192 code.${index}`);
+  return {
+    async mutate(args) {
+      const { parent, parentAggregate, topTrials, bottomTrials, childCount, generation } = args;
+      const startedAt = Date.now();
+      const outcomes = await opts.pool.withSlot(async (slot) => {
+        try {
+          return await opts.runner({
+            slot,
+            parent,
+            parentAggregate,
+            topTrials,
+            bottomTrials,
+            childCount,
+            generation
+          });
+        } catch (err) {
+          return [{
+            ok: false,
+            failureReason: "runner_error",
+            description: err instanceof Error ? err.message : String(err),
+            latencyMs: Date.now() - startedAt
+          }];
+        }
+      });
+      const variants = [];
+      let index = 0;
+      for (const outcome of outcomes) {
+        const childId = outcome.childId ?? childIdFor(parent, generation, index);
+        if (opts.mutationTelemetry) {
+          await opts.mutationTelemetry.record({
+            ts: Date.now(),
+            channel: "code",
+            generation,
+            parentId: parent.id,
+            childId: outcome.ok ? childId : null,
+            ok: outcome.ok,
+            failureReason: outcome.failureReason,
+            description: outcome.description,
+            latencyMs: outcome.latencyMs,
+            diffBytes: outcome.diffBytes,
+            filesTouched: outcome.filesTouched,
+            agentSteps: outcome.agentSteps,
+            costUsd: outcome.costUsd
+          });
+        }
+        if (opts.costLedger && outcome.costUsd !== void 0) {
+          await opts.costLedger.addMutation("code", outcome.costUsd, { generation });
+        }
+        if (outcome.ok) {
+          const variant = {
+            id: childId,
+            payload: opts.toVariantPayload(outcome, parent),
+            generation,
+            parentId: parent.id,
+            label: labelFor(outcome, parent, generation, index),
+            ...outcome.rationale ? { rationale: outcome.rationale } : {}
+          };
+          variants.push(variant);
+          if (opts.lineage) {
+            await opts.lineage.upsert({
+              id: variant.id,
+              parentId: variant.parentId ?? null,
+              generation: variant.generation,
+              kind: "code",
+              ...variant.rationale ? { rationale: variant.rationale } : {}
+            });
+          }
+        }
+        index++;
+      }
+      if (opts.costLedger) {
+        const u = opts.pool.utilization();
+        await opts.costLedger.setPoolUtilization(u.busyMs, u.totalMs);
+      }
+      return variants;
+    }
+  };
+}
+
 // src/golden-matcher.ts
 function matchGoldens(goldens, candidates, options = {}) {
   const extract = options.text ?? defaultExtract5;
@@ -10634,6 +11248,7 @@ export {
   BudgetGuard,
   BuilderSession,
   ConvergenceTracker,
+  CostLedger,
   CostTracker,
   D1ExperimentStore,
   DEFAULT_AGENT_SLOS,
@@ -10664,12 +11279,17 @@ export {
   InMemoryTraceStore,
   InMemoryTrialCache,
   InMemoryWorkspaceInspector,
+  JsonlTrialCache,
   JudgeRunner,
+  LineageRecorder,
   LlmCallError,
   LlmClient,
+  LockedJsonlAppender,
   MODEL_PRICING,
   MetricsCollector,
   MultiLayerVerifier,
+  MutationTelemetry,
+  Mutex,
   OTEL_AGENT_EVAL_SCOPE,
   OptimizationLoop,
   PairwiseSteeringOptimizer,
@@ -10687,6 +11307,7 @@ export {
   TRACE_SCHEMA_VERSION,
   TokenCounter,
   TraceEmitter,
+  TrialTelemetry,
   UNIVERSAL_FINDERS,
   adversarialJudge,
   aggregateLlm,
@@ -10731,11 +11352,14 @@ export {
   correlateLayers,
   correlationStudy,
   createAntiSlopJudge,
+  createCompositeMutator,
   createCustomJudge,
   createDefaultReviewer,
   createDomainExpertJudge,
   createIntentMatchJudge,
   createLlmReviewer,
+  createSandboxCodeMutator,
+  createSandboxPool,
   createSemanticConceptJudge,
   crossTraceDiff,
   crowdingDistance,
@@ -10846,6 +11470,7 @@ export {
   replayScorerOverCorpus,
   replayTraceThroughJudge,
   requiredSampleSize,
+  resetLockedAppendersForTesting,
   resumeBuilderSession,
   rowCount,
   rowWhere,