@tangle-network/agent-eval 0.23.1 → 0.24.0

package/dist/{chunk-VQQSPGSM.js → chunk-VRJVTXRV.js}

@@ -1,10 +1,10 @@
 import {
   validateRunRecord
-} from "./chunk-QBW3YBTR.js";
+} from "./chunk-NLMNWKVM.js";
 import {
   pairedBootstrap,
   pairedWilcoxon
-} from "./chunk-IOXMGMHQ.js";
+} from "./chunk-2A5XJB43.js";
 
 // src/feedback-trajectory.ts
 var DEFAULT_SPLIT_POLICY = {
@@ -27,7 +27,8 @@ var InMemoryFeedbackTrajectoryStore = class {
   }
   async appendAttempt(id, attempt) {
     const trajectory = this.trajectories.get(id);
-    if (!trajectory) throw new Error(`FeedbackTrajectoryStore.appendAttempt: unknown trajectory "${id}"`);
+    if (!trajectory)
+      throw new Error(`FeedbackTrajectoryStore.appendAttempt: unknown trajectory "${id}"`);
     const next = cloneTrajectory({
       ...trajectory,
       attempts: [...trajectory.attempts, attempt],
@@ -38,8 +39,11 @@ var InMemoryFeedbackTrajectoryStore = class {
   }
   async appendLabel(id, label, attemptId) {
     const trajectory = this.trajectories.get(id);
-    if (!trajectory) throw new Error(`FeedbackTrajectoryStore.appendLabel: unknown trajectory "${id}"`);
-    const attempts = attemptId ? trajectory.attempts.map((attempt) => attempt.id === attemptId ? { ...attempt, feedback: [...attempt.feedback ?? [], label] } : attempt) : trajectory.attempts;
+    if (!trajectory)
+      throw new Error(`FeedbackTrajectoryStore.appendLabel: unknown trajectory "${id}"`);
+    const attempts = attemptId ? trajectory.attempts.map(
+      (attempt) => attempt.id === attemptId ? { ...attempt, feedback: [...attempt.feedback ?? [], label] } : attempt
+    ) : trajectory.attempts;
     const next = cloneTrajectory({
       ...trajectory,
       attempts,
@@ -86,7 +90,12 @@ var FileSystemFeedbackTrajectoryStore = class {
     const { appendFile, mkdir } = await import("fs/promises");
     const { join } = await import("path");
     await mkdir(this.dir, { recursive: true });
-    await appendFile(join(this.dir, "feedback-trajectories.ndjson"), JSON.stringify(record) + "\n", "utf8");
+    await appendFile(
+      join(this.dir, "feedback-trajectories.ndjson"),
+      `${JSON.stringify(record)}
+`,
+      "utf8"
+    );
   }
   async load() {
     if (this.loaded) return;
@@ -100,8 +109,10 @@ var FileSystemFeedbackTrajectoryStore = class {
         try {
           const record = JSON.parse(line);
           if (record.op === "save") await this.memory.save(record.trajectory);
-          if (record.op === "appendAttempt") await this.memory.appendAttempt(record.id, record.attempt);
-          if (record.op === "appendLabel") await this.memory.appendLabel(record.id, record.label, record.attemptId);
+          if (record.op === "appendAttempt")
+            await this.memory.appendAttempt(record.id, record.attempt);
+          if (record.op === "appendLabel")
+            await this.memory.appendLabel(record.id, record.label, record.attemptId);
         } catch {
         }
       }
@@ -131,7 +142,9 @@ function assignFeedbackSplit(trajectory, policy = {}) {
   const split = { ...DEFAULT_SPLIT_POLICY, ...policy };
   const total = split.trainPct + split.devPct + split.testPct + split.holdoutPct;
   if (total <= 0) throw new Error("assignFeedbackSplit: split percentages must sum above zero");
-  const bucket = stableHash(`${trajectory.projectId ?? ""}|${trajectory.scenarioId ?? ""}|${trajectory.id}|${trajectory.task.intent}`) % total;
+  const bucket = stableHash(
+    `${trajectory.projectId ?? ""}|${trajectory.scenarioId ?? ""}|${trajectory.id}|${trajectory.task.intent}`
+  ) % total;
   if (bucket < split.trainPct) return "train";
   if (bucket < split.trainPct + split.devPct) return "dev";
   if (bucket < split.trainPct + split.devPct + split.testPct) return "test";
@@ -192,14 +205,16 @@ async function replayFeedbackTrajectory(trajectory, adapter) {
     return {
       trajectoryId: trajectory.id,
       pass: false,
-      labels: [{
-        source: "system",
-        kind: "reject",
-        value: false,
-        reason: message,
-        severity: "error",
-        createdAt
-      }],
+      labels: [
+        {
+          source: "system",
+          kind: "reject",
+          value: false,
+          reason: message,
+          severity: "error",
+          createdAt
+        }
+      ],
       outcome: {
         success: false,
         score: 0,
@@ -250,10 +265,12 @@ function renderPreferenceMemoryMarkdown(entries) {
     lines.push(`  Source: ${entry.sourceTrajectoryId}`);
     lines.push("");
   }
-  return lines.join("\n").trim() + "\n";
+  return `${lines.join("\n").trim()}
+`;
 }
 function serializeFeedbackTrajectoriesJsonl(trajectories) {
-  return trajectories.slice().sort((a, b) => a.id.localeCompare(b.id)).map((trajectory) => JSON.stringify(canonicalize(trajectory))).join("\n") + "\n";
+  return `${trajectories.slice().sort((a, b) => a.id.localeCompare(b.id)).map((trajectory) => JSON.stringify(canonicalize(trajectory))).join("\n")}
+`;
 }
 function parseFeedbackTrajectoriesJsonl(jsonl) {
   const trajectories = [];
@@ -326,17 +343,22 @@ function scoreFromLabels(labels) {
   const scored = labels.map((label) => {
     if (label.kind === "approve" || label.kind === "select") return 1;
     if (label.kind === "reject" || label.kind === "policy_block") return 0;
-    if (label.kind === "rate" && typeof label.value === "number") return Math.max(0, Math.min(1, label.value));
+    if (label.kind === "rate" && typeof label.value === "number")
+      return Math.max(0, Math.min(1, label.value));
     return void 0;
   }).filter((value) => typeof value === "number");
   if (!scored.length) return void 0;
   return Math.round(scored.reduce((sum, value) => sum + value, 0) / scored.length * 1e3) / 1e3;
 }
 function instructionFromLabel(trajectory, label) {
-  if (label.kind === "reject" && label.reason) return `Avoid outputs like "${compact(trajectory.task.intent, 80)}" when: ${label.reason}`;
-  if (label.kind === "revision_request" && label.reason) return `Revise similar work by applying: ${label.reason}`;
-  if (label.kind === "select" && label.reason) return `Prefer selected options for "${compact(trajectory.task.intent, 80)}" because: ${label.reason}`;
-  if (label.kind === "approve" && label.reason) return `Repeat the pattern approved for "${compact(trajectory.task.intent, 80)}": ${label.reason}`;
+  if (label.kind === "reject" && label.reason)
+    return `Avoid outputs like "${compact(trajectory.task.intent, 80)}" when: ${label.reason}`;
+  if (label.kind === "revision_request" && label.reason)
+    return `Revise similar work by applying: ${label.reason}`;
+  if (label.kind === "select" && label.reason)
+    return `Prefer selected options for "${compact(trajectory.task.intent, 80)}" because: ${label.reason}`;
+  if (label.kind === "approve" && label.reason)
+    return `Repeat the pattern approved for "${compact(trajectory.task.intent, 80)}": ${label.reason}`;
   if (label.kind === "comment" && label.reason) return label.reason;
   return void 0;
 }
@@ -398,9 +420,7 @@ function paretoFrontier(candidates, objectives) {
   if (objectives.length === 0) {
     throw new Error("paretoFrontier: at least 1 objective required");
   }
-  const valid = candidates.filter(
-    (c) => objectives.every((o) => Number.isFinite(o.value(c)))
-  );
+  const valid = candidates.filter((c) => objectives.every((o) => Number.isFinite(o.value(c))));
   const frontier = [];
   const dominated = [];
   for (const c of valid) {
@@ -624,44 +644,6 @@ function fmt(x) {
   return x.toFixed(4);
 }
 
-// src/researcher.ts
-var CallbackResearcher = class {
-  constructor(callbacks) {
-    this.callbacks = callbacks;
-  }
-  callbacks;
-  inspectFailures(runs) {
-    return this.callbacks.inspectFailures(runs);
-  }
-  proposeChange(failures) {
-    return this.callbacks.proposeChange(failures);
-  }
-  applyChange(changes, baseline) {
-    return this.callbacks.applyChange(changes, baseline);
-  }
-  evaluateChange(plan) {
-    return this.callbacks.evaluateChange(plan);
-  }
-};
-var NoopResearcher = class {
-  hint;
-  constructor(hint = "NoopResearcher: no implementation wired") {
-    this.hint = hint;
-  }
-  async inspectFailures(_runs) {
-    throw new Error(`${this.hint} (inspectFailures not implemented)`);
-  }
-  async proposeChange(_failures) {
-    throw new Error(`${this.hint} (proposeChange not implemented)`);
-  }
-  async applyChange(_changes, _baseline) {
-    throw new Error(`${this.hint} (applyChange not implemented)`);
-  }
-  async evaluateChange(_plan) {
-    throw new Error(`${this.hint} (evaluateChange not implemented)`);
-  }
-};
-
 // src/prompt-evolution.ts
 var InMemoryTrialCache = class {
   store = /* @__PURE__ */ new Map();
@@ -714,7 +696,11 @@ async function runPromptEvolution(config) {
       const prev = generations[generations.length - 2];
       const noChange = prev.winnerId === winnerId && samePopulation(prev.paretoFrontIds, [...frontIds]);
       if (noChange) {
-        config.onProgress?.({ type: "converged", generation, reason: "no improvement vs previous generation" });
+        config.onProgress?.({
+          type: "converged",
+          generation,
+          reason: "no improvement vs previous generation"
+        });
         break;
       }
     }
@@ -726,7 +712,9 @@ async function runPromptEvolution(config) {
     target: config.target,
     generations,
     bestVariant,
-    bestAggregate: bestAggregate ?? aggregateTrials(population, config.scenarioIds, []).find((a) => a.variantId === bestVariant.id)
+    bestAggregate: bestAggregate ?? aggregateTrials(population, config.scenarioIds, []).find(
+      (a) => a.variantId === bestVariant.id
+    )
   };
 }
 async function scorePopulation(population, config, generation) {
@@ -834,7 +822,9 @@ function mean2(xs) {
 async function nextPopulation(current, aggregates, trials, front, config, nextGeneration) {
   const survivorIds = new Set(front.map((c) => c.candidate.variantId));
   const survivors = current.filter((v) => survivorIds.has(v.id));
-  const ranked = scalarScore(aggregates, config.objectives, { weights: config.scalarWeights }).sort((a, b) => b.score - a.score);
+  const ranked = scalarScore(aggregates, config.objectives, { weights: config.scalarWeights }).sort(
+    (a, b) => b.score - a.score
+  );
   const parentId = ranked[0]?.candidate.variantId ?? current[0].id;
   const parent = current.find((v) => v.id === parentId) ?? current[0];
   const parentAggregate = aggregates.find((a) => a.variantId === parent.id) ?? aggregates[0];
@@ -947,8 +937,12 @@ async function evaluateMultiShotGate(config, baseline, candidate) {
       const seed = seedFor(config, scenarioId, rep);
       const baseTrial = await scoreOne(config, baseline, scenarioId, rep, "search");
       const candTrial = await scoreOne(config, candidate, scenarioId, rep, "search");
-      baselineRuns.push(toValidatedRecord(config, baseline, scenarioId, rep, "search", seed, baseTrial));
-      candidateRuns.push(toValidatedRecord(config, candidate, scenarioId, rep, "search", seed, candTrial));
+      baselineRuns.push(
+        toValidatedRecord(config, baseline, scenarioId, rep, "search", seed, baseTrial)
+      );
+      candidateRuns.push(
+        toValidatedRecord(config, candidate, scenarioId, rep, "search", seed, candTrial)
+      );
     }
   }
   for (const scenarioId of gateConfig.holdoutScenarioIds) {
@@ -956,8 +950,12 @@ async function evaluateMultiShotGate(config, baseline, candidate) {
       const seed = seedFor(config, scenarioId, rep);
       const baseTrial = await scoreOne(config, baseline, scenarioId, rep, "holdout");
       const candTrial = await scoreOne(config, candidate, scenarioId, rep, "holdout");
-      baselineRuns.push(toValidatedRecord(config, baseline, scenarioId, rep, "holdout", seed, baseTrial));
-      candidateRuns.push(toValidatedRecord(config, candidate, scenarioId, rep, "holdout", seed, candTrial));
+      baselineRuns.push(
+        toValidatedRecord(config, baseline, scenarioId, rep, "holdout", seed, baseTrial)
+      );
+      candidateRuns.push(
+        toValidatedRecord(config, candidate, scenarioId, rep, "holdout", seed, candTrial)
+      );
     }
   }
   const decision = new HeldOutGate(gateConfig.gate).evaluate(candidateRuns, baselineRuns);
@@ -1002,11 +1000,13 @@ async function scoreOne(config, variant, scenarioId, rep, split) {
       error: err instanceof Error ? err.message : String(err),
       split,
       seed,
-      asi: [{
-        severity: "critical",
-        message: err instanceof Error ? err.message : String(err),
-        responsibleSurface: config.target
-      }],
+      asi: [
+        {
+          severity: "critical",
+          message: err instanceof Error ? err.message : String(err),
+          responsibleSurface: config.target
+        }
+      ],
       emitted: ""
     };
   }
@@ -1027,11 +1027,15 @@ function validateConfig(config) {
   requirePositiveInteger(config.reps, "reps");
   requirePositiveInteger(config.generations, "generations");
   requirePositiveInteger(config.populationSize, "populationSize");
-  if (config.scoreConcurrency !== void 0) requirePositiveInteger(config.scoreConcurrency, "scoreConcurrency");
+  if (config.scoreConcurrency !== void 0)
+    requirePositiveInteger(config.scoreConcurrency, "scoreConcurrency");
   if (config.populationSize < config.seedVariants.length) {
     throw new Error("runMultiShotOptimization: populationSize must be >= seedVariants.length");
   }
-  assertUnique(config.seedVariants.map((v) => v.id), "seedVariants.id");
+  assertUnique(
+    config.seedVariants.map((v) => v.id),
+    "seedVariants.id"
+  );
   assertUnique(config.searchScenarioIds, "searchScenarioIds");
   if (config.gate) {
     if (config.gate.holdoutScenarioIds.length === 0) {
@@ -1039,11 +1043,14 @@ function validateConfig(config) {
     }
     if (config.gate.reps !== void 0) requirePositiveInteger(config.gate.reps, "gate.reps");
     assertUnique(config.gate.holdoutScenarioIds, "gate.holdoutScenarioIds");
-    if (config.gate.searchScenarioIds) assertUnique(config.gate.searchScenarioIds, "gate.searchScenarioIds");
+    if (config.gate.searchScenarioIds)
+      assertUnique(config.gate.searchScenarioIds, "gate.searchScenarioIds");
     const searchIds = new Set(config.searchScenarioIds);
     for (const id of config.gate.holdoutScenarioIds) {
       if (searchIds.has(id)) {
-        throw new Error(`runMultiShotOptimization: holdout scenario "${id}" also appears in searchScenarioIds`);
+        throw new Error(
+          `runMultiShotOptimization: holdout scenario "${id}" also appears in searchScenarioIds`
+        );
       }
     }
     const baselineId = config.seedVariants[0].id;
@@ -1062,7 +1069,8 @@ function requirePositiveInteger(value, name) {
 function assertUnique(values, name) {
   const seen = /* @__PURE__ */ new Set();
   for (const value of values) {
-    if (!value.trim()) throw new Error(`runMultiShotOptimization: ${name} must not contain empty values`);
+    if (!value.trim())
+      throw new Error(`runMultiShotOptimization: ${name} must not contain empty values`);
     if (seen.has(value)) throw new Error(`runMultiShotOptimization: duplicate ${name} "${value}"`);
     seen.add(value);
   }
@@ -1149,7 +1157,9 @@ function buildReflectionPrompt(ctx) {
   const sections = [];
   sections.push(`# Mutation target: ${ctx.target}`);
   sections.push("");
-  sections.push(`You are tuning the prompt component named \`${ctx.target}\`. The current variant is shown below; you have ${ctx.topTrials.length} top trials and ${ctx.bottomTrials.length} bottom trials as evidence. Propose ${ctx.childCount} mutation${ctx.childCount === 1 ? "" : "s"} that fix specific weaknesses visible in the bottom trials. Avoid blank rephrasings.`);
+  sections.push(
+    `You are tuning the prompt component named \`${ctx.target}\`. The current variant is shown below; you have ${ctx.topTrials.length} top trials and ${ctx.bottomTrials.length} bottom trials as evidence. Propose ${ctx.childCount} mutation${ctx.childCount === 1 ? "" : "s"} that fix specific weaknesses visible in the bottom trials. Avoid blank rephrasings.`
+  );
   sections.push("");
   sections.push("## Current variant");
   sections.push("```json");
@@ -1160,7 +1170,9 @@ function buildReflectionPrompt(ctx) {
     sections.push("## Failures (bottom trials) \u2014 what went wrong");
     sections.push("");
     for (const trial of ctx.bottomTrials) {
-      sections.push(`### Trial \`${trial.id}\` \u2014 score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`);
+      sections.push(
+        `### Trial \`${trial.id}\` \u2014 score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`
+      );
       const missed = (trial.expectations ?? []).filter((e) => !e.matched);
       if (missed.length > 0) {
         sections.push("");
@@ -1183,7 +1195,9 @@ function buildReflectionPrompt(ctx) {
     sections.push("## Successes (top trials) \u2014 what to preserve");
     sections.push("");
     for (const trial of ctx.topTrials) {
-      sections.push(`- \`${trial.id}\`: score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`);
+      sections.push(
+        `- \`${trial.id}\`: score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`
+      );
     }
     sections.push("");
   }
@@ -1195,25 +1209,27 @@ function buildReflectionPrompt(ctx) {
   sections.push("");
   sections.push("Respond with a JSON object \u2014 no prose, no markdown fences:");
   sections.push("```json");
-  sections.push(JSON.stringify(
-    {
-      proposals: [
-        {
-          label: "<short label, \u2264 40 chars>",
-          rationale: "<which failure this targets and which primitive you used>",
-          payload: "<full payload of the new variant \u2014 same shape as the current variant>"
-        }
-      ]
-    },
-    null,
-    2
-  ));
+  sections.push(
+    JSON.stringify(
+      {
+        proposals: [
+          {
+            label: "<short label, \u2264 40 chars>",
+            rationale: "<which failure this targets and which primitive you used>",
+            payload: "<full payload of the new variant \u2014 same shape as the current variant>"
+          }
+        ]
+      },
+      null,
+      2
+    )
+  );
   sections.push("```");
   return sections.join("\n");
 }
 function truncate(s, max) {
   if (s.length <= max) return s;
-  return s.slice(0, max) + "\u2026 [truncated]";
+  return `${s.slice(0, max)}\u2026 [truncated]`;
 }
 function quote(s) {
   return s.replace(/`/g, "\\`");
@@ -1221,15 +1237,15 @@ function quote(s) {
 function autoCloseTruncatedJson(raw) {
   const stack = [];
   let inString = false;
-  let escape = false;
+  let escaped = false;
   for (const c of raw) {
-    if (escape) {
-      escape = false;
+    if (escaped) {
+      escaped = false;
       continue;
     }
     if (inString) {
       if (c === "\\") {
-        escape = true;
+        escaped = true;
         continue;
       }
       if (c === '"') {
@@ -1269,11 +1285,15 @@ function parseReflectionResponse(raw, maxProposals) {
   const tryObjectFirst = objectStart >= 0 && (arrayStart < 0 || objectStart < arrayStart);
   const candidates = [];
   if (tryObjectFirst) {
-    if (objectStart >= 0 && objectEnd > objectStart) candidates.push(text.slice(objectStart, objectEnd + 1));
-    if (arrayStart >= 0 && arrayEnd > arrayStart) candidates.push(text.slice(arrayStart, arrayEnd + 1));
+    if (objectStart >= 0 && objectEnd > objectStart)
+      candidates.push(text.slice(objectStart, objectEnd + 1));
+    if (arrayStart >= 0 && arrayEnd > arrayStart)
+      candidates.push(text.slice(arrayStart, arrayEnd + 1));
   } else {
-    if (arrayStart >= 0 && arrayEnd > arrayStart) candidates.push(text.slice(arrayStart, arrayEnd + 1));
-    if (objectStart >= 0 && objectEnd > objectStart) candidates.push(text.slice(objectStart, objectEnd + 1));
+    if (arrayStart >= 0 && arrayEnd > arrayStart)
+      candidates.push(text.slice(arrayStart, arrayEnd + 1));
+    if (objectStart >= 0 && objectEnd > objectStart)
+      candidates.push(text.slice(objectStart, objectEnd + 1));
   }
   for (const slice of candidates) {
     try {
@@ -1317,6 +1337,44 @@ function parseReflectionResponse(raw, maxProposals) {
   return out;
 }
 
+// src/researcher.ts
+var CallbackResearcher = class {
+  constructor(callbacks) {
+    this.callbacks = callbacks;
+  }
+  callbacks;
+  inspectFailures(runs) {
+    return this.callbacks.inspectFailures(runs);
+  }
+  proposeChange(failures) {
+    return this.callbacks.proposeChange(failures);
+  }
+  applyChange(changes, baseline) {
+    return this.callbacks.applyChange(changes, baseline);
+  }
+  evaluateChange(plan) {
+    return this.callbacks.evaluateChange(plan);
+  }
+};
+var NoopResearcher = class {
+  hint;
+  constructor(hint = "NoopResearcher: no implementation wired") {
+    this.hint = hint;
+  }
+  async inspectFailures(_runs) {
+    throw new Error(`${this.hint} (inspectFailures not implemented)`);
+  }
+  async proposeChange(_failures) {
+    throw new Error(`${this.hint} (proposeChange not implemented)`);
+  }
+  async applyChange(_changes, _baseline) {
+    throw new Error(`${this.hint} (applyChange not implemented)`);
+  }
+  async evaluateChange(_plan) {
+    throw new Error(`${this.hint} (evaluateChange not implemented)`);
+  }
+};
+
 export {
   InMemoryFeedbackTrajectoryStore,
   FileSystemFeedbackTrajectoryStore,
@@ -1340,8 +1398,6 @@ export {
   crowdingDistance,
   paretoFrontierWithCrowding,
   HeldOutGate,
-  CallbackResearcher,
-  NoopResearcher,
   InMemoryTrialCache,
   runPromptEvolution,
   runMultiShotOptimization,
@@ -1349,6 +1405,8 @@ export {
   trialTraceFromMultiShotTrial,
   DEFAULT_MUTATION_PRIMITIVES,
   buildReflectionPrompt,
-  parseReflectionResponse
+  parseReflectionResponse,
+  CallbackResearcher,
+  NoopResearcher
 };
-//# sourceMappingURL=chunk-VQQSPGSM.js.map
+//# sourceMappingURL=chunk-VRJVTXRV.js.map