@tangle-network/agent-eval 0.17.2 → 0.18.0

package/dist/index.js

@@ -2252,8 +2252,6 @@ async function finish(emitter, result) {
 }
 
 // src/feedback-trajectory.ts
-import { appendFile, mkdir, readFile } from "fs/promises";
-import { join } from "path";
 var DEFAULT_SPLIT_POLICY = {
   trainPct: 70,
   devPct: 15,
@@ -2330,12 +2328,16 @@ var FileSystemFeedbackTrajectoryStore = class {
     return next;
   }
   async append(record) {
+    const { appendFile, mkdir } = await import("fs/promises");
+    const { join: join3 } = await import("path");
     await mkdir(this.dir, { recursive: true });
-    await appendFile(join(this.dir, "feedback-trajectories.ndjson"), JSON.stringify(record) + "\n", "utf8");
+    await appendFile(join3(this.dir, "feedback-trajectories.ndjson"), JSON.stringify(record) + "\n", "utf8");
   }
   async load() {
     if (this.loaded) return;
-    const file = join(this.dir, "feedback-trajectories.ndjson");
+    const { readFile } = await import("fs/promises");
+    const { join: join3 } = await import("path");
+    const file = join3(this.dir, "feedback-trajectories.ndjson");
     try {
       const raw = await readFile(file, "utf8");
       for (const line of raw.split("\n")) {
@@ -2422,6 +2424,44 @@ function feedbackTrajectoryToOptimizerRow(trajectory) {
 function feedbackTrajectoriesToOptimizerRows(trajectories) {
   return trajectories.map(feedbackTrajectoryToOptimizerRow);
 }
+async function replayFeedbackTrajectory(trajectory, adapter2) {
+  try {
+    const result = await adapter2.replay(trajectory);
+    return {
+      trajectoryId: trajectory.id,
+      ...result
+    };
+  } catch (err) {
+    const createdAt = (/* @__PURE__ */ new Date()).toISOString();
+    const message = err instanceof Error ? err.message : String(err);
+    return {
+      trajectoryId: trajectory.id,
+      pass: false,
+      labels: [{
+        source: "system",
+        kind: "reject",
+        value: false,
+        reason: message,
+        severity: "error",
+        createdAt
+      }],
+      outcome: {
+        success: false,
+        score: 0,
+        detail: message,
+        observedAt: createdAt
+      },
+      metadata: { replayError: true }
+    };
+  }
+}
+async function replayFeedbackTrajectories(trajectories, adapter2) {
+  const results = [];
+  for (const trajectory of trajectories) {
+    results.push(await replayFeedbackTrajectory(trajectory, adapter2));
+  }
+  return results;
+}
 function summarizePreferenceMemory(trajectories, options = {}) {
   const maxEntries = options.maxEntries ?? 20;
   const entries = [];
@@ -2585,6 +2625,69 @@ function canonicalize(value) {
   return out;
 }
 
+// src/action-policy.ts
+function evaluateActionPolicy(action, policy = {}, options = {}) {
+  const reasons = [];
+  let blocked = false;
+  let requiresApproval = Boolean(action.requiresApproval);
+  if (policy.allowedTypes?.length && !policy.allowedTypes.includes(action.type)) {
+    blocked = true;
+    reasons.push(`action type "${action.type}" is not allowed`);
+  }
+  if (policy.blockedTypes?.includes(action.type)) {
+    blocked = true;
+    reasons.push(`action type "${action.type}" is blocked`);
+  }
+  if (policy.alwaysRequireApprovalTypes?.includes(action.type)) {
+    requiresApproval = true;
+    reasons.push(`action type "${action.type}" requires approval`);
+  }
+  if (policy.requireApprovalForExternalSideEffects && action.externalSideEffect) {
+    requiresApproval = true;
+    reasons.push("external side effect requires approval");
+  }
+  if (policy.requireApprovalAboveCostUsd !== void 0 && (action.costUsd ?? 0) > policy.requireApprovalAboveCostUsd) {
+    requiresApproval = true;
+    reasons.push(`cost ${action.costUsd} exceeds approval threshold ${policy.requireApprovalAboveCostUsd}`);
+  }
+  if (policy.maxActionCostUsd !== void 0 && (action.costUsd ?? 0) > policy.maxActionCostUsd) {
+    blocked = true;
+    reasons.push(`cost ${action.costUsd} exceeds max action cost ${policy.maxActionCostUsd}`);
+  }
+  if (policy.remainingBudgetUsd !== void 0 && (action.costUsd ?? 0) > policy.remainingBudgetUsd) {
+    blocked = true;
+    reasons.push(`cost ${action.costUsd} exceeds remaining budget ${policy.remainingBudgetUsd}`);
+  }
+  if (policy.expectedOutcomeRequired && !action.metadata?.expectedOutcome) {
+    blocked = true;
+    reasons.push("expected outcome is required");
+  }
+  if (policy.killCriteriaRequired && !action.metadata?.killCriteria) {
+    blocked = true;
+    reasons.push("kill criteria are required");
+  }
+  if (policy.autoApproveTypes?.includes(action.type) && requiresApproval) {
+    reasons.push(`action type "${action.type}" is auto-approved only when no approval policy applies`);
+  }
+  if (!reasons.length) reasons.push(requiresApproval ? "approval required" : "action allowed");
+  const label = blocked || requiresApproval ? {
+    source: "policy",
+    kind: blocked ? "policy_block" : "comment",
+    value: { actionType: action.type, blocked, requiresApproval },
+    reason: reasons.join("; "),
+    severity: blocked ? "critical" : "warning",
+    createdAt: options.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+    metadata: { action, policy }
+  } : void 0;
+  return {
+    allowed: !blocked,
+    blocked,
+    requiresApproval: !blocked && requiresApproval,
+    reasons,
+    label
+  };
+}
+
 // src/prompt-registry.ts
 var PromptRegistry = class {
   entries = /* @__PURE__ */ new Map();
@@ -6382,7 +6485,7 @@ function assertNonNegative(n, name) {
 
 // src/muffled-gate-scanner.ts
 import { readFileSync as readFileSync2, existsSync as existsSync2, readdirSync, statSync } from "fs";
-import { join as join2 } from "path";
+import { join } from "path";
 function codeOf(line) {
   return line.replace(/\/\/.*$/, "").replace(/^\s*\*.*$/, "");
 }
@@ -6486,11 +6589,11 @@ var UNIVERSAL_FINDERS = [
 function autoDeriveImporters(repoRoot, roots, extensions, importsContain) {
   const matches2 = [];
   const walk = (rel) => {
-    const abs = join2(repoRoot, rel);
+    const abs = join(repoRoot, rel);
     if (!existsSync2(abs)) return;
     for (const entry of readdirSync(abs)) {
-      const sub = join2(rel, entry);
-      const subAbs = join2(repoRoot, sub);
+      const sub = join(rel, entry);
+      const subAbs = join(repoRoot, sub);
       let st;
       try {
         st = statSync(subAbs);
@@ -6519,7 +6622,7 @@ function scanForMuffledGates(opts) {
   const findings = [];
   const scanned = /* @__PURE__ */ new Set();
   for (const file of opts.scanFiles) {
-    const abs = join2(opts.repoRoot, file);
+    const abs = join(opts.repoRoot, file);
     if (!existsSync2(abs)) continue;
     const text = readFileSync2(abs, "utf8");
     for (const find of opts.finders) findings.push(...find(file, text));
@@ -6534,7 +6637,7 @@ function scanForMuffledGates(opts) {
     );
     for (const file of importers) {
       if (scanned.has(file)) continue;
-      const abs = join2(opts.repoRoot, file);
+      const abs = join(opts.repoRoot, file);
       if (!existsSync2(abs)) continue;
       const text = readFileSync2(abs, "utf8");
       for (const find of opts.autoDerive.universalFinders) findings.push(...find(file, text));
@@ -8522,7 +8625,7 @@ async function commitBisect(options) {
 }
 async function promptBisect(options) {
   const split = options.paragraphSplitter ?? ((p) => p.split(/\n\s*\n/));
-  const join4 = (paragraphs) => paragraphs.join("\n\n");
+  const join3 = (paragraphs) => paragraphs.join("\n\n");
   const goodParas = split(options.good);
   const badParas = split(options.bad);
   if (goodParas.length !== badParas.length) {
@@ -8540,7 +8643,7 @@ async function promptBisect(options) {
   const result = await bisect({
     good: goodMask,
     bad: badMask,
-    runEval: (mask) => options.runEval(join4(paragraphsFor(mask))),
+    runEval: (mask) => options.runEval(join3(paragraphsFor(mask))),
     maxIterations: options.maxIterations ?? n + 5,
     halfway: (g, b) => {
       for (let i = 0; i < g.length; i++) {
@@ -8571,12 +8674,12 @@ async function promptBisect(options) {
     }
   }
   const materializedPath = result.path.map((s) => ({
-    state: join4(paragraphsFor(s.state)),
+    state: join3(paragraphsFor(s.state)),
     score: s.score,
     pass: s.pass
   }));
   return {
-    culprit: join4(paragraphsFor(culprit)),
+    culprit: join3(paragraphsFor(culprit)),
     path: materializedPath,
     converged: result.converged,
     inputInconsistent: result.inputInconsistent,
@@ -9631,7 +9734,7 @@ function mergeSignals(a, b) {
 // src/command-runner.ts
 import { spawnSync } from "child_process";
 import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync as readdirSync2, statSync as statSync2 } from "fs";
-import { join as join3 } from "path";
+import { join as join2 } from "path";
 var localCommandRunner = {
   name: "local",
   async run(input) {
@@ -9678,7 +9781,7 @@ var localCommandRunner = {
     const out = [];
     for (const name of entries) {
       try {
-        const st = statSync2(join3(path, name));
+        const st = statSync2(join2(path, name));
         out.push({
           name,
           isDirectory: st.isDirectory(),
@@ -12589,6 +12692,274 @@ function samePopulation(a, b) {
   return b.every((id) => setA.has(id));
 }
 
+// src/multi-shot-optimization.ts
+async function runMultiShotOptimization(config) {
+  validateConfig(config);
+  const scoreAdapter = {
+    score: (args) => scoreOne(config, args.variant, args.scenarioId, args.rep, "search")
+  };
+  const evolution = await runPromptEvolution({
+    runId: config.runId,
+    target: config.target,
+    seedVariants: config.seedVariants,
+    scenarioIds: config.searchScenarioIds,
+    reps: config.reps,
+    generations: config.generations,
+    populationSize: config.populationSize,
+    scoreConcurrency: config.scoreConcurrency ?? 1,
+    scoreAdapter,
+    mutateAdapter: {
+      mutate: (args) => config.mutateAdapter.mutate({
+        ...args,
+        topTrials: args.topTrials,
+        bottomTrials: args.bottomTrials
+      })
+    },
+    objectives: config.objectives ?? defaultMultiShotObjectives(),
+    scalarWeights: config.scalarWeights,
+    earlyStopOnNoImprovement: config.earlyStopOnNoImprovement,
+    cache: config.cache,
+    onProgress: config.onProgress
+  });
+  let gate = null;
+  const baseline = config.seedVariants[0];
+  let promotedVariant = evolution.bestVariant;
+  let promotedAggregate = evolution.bestAggregate;
+  if (config.gate && evolution.bestVariant.id !== baseline.id) {
+    gate = await evaluateMultiShotGate(config, baseline, evolution.bestVariant);
+    if (!gate.decision.promote) {
+      promotedVariant = baseline;
+      promotedAggregate = aggregateFor(evolution, baseline.id);
+    }
+  }
+  return {
+    evolution,
+    searchBestVariant: evolution.bestVariant,
+    searchBestAggregate: evolution.bestAggregate,
+    promotedVariant,
+    promotedAggregate,
+    gate
+  };
+}
+function defaultMultiShotObjectives() {
+  return [
+    { name: "score", direction: "maximize", value: (a) => a.meanScore },
+    { name: "cost", direction: "minimize", value: (a) => a.meanCost }
+  ];
+}
+function trialTraceFromMultiShotTrial(trial) {
+  return {
+    id: `${trial.variantId}/${trial.scenarioId}/r${trial.rep}`,
+    score: trial.score,
+    inputName: trial.scenarioId,
+    expectations: (trial.asi ?? []).map((item, i) => ({
+      id: item.expectationId ?? `asi-${i}`,
+      phrase: item.message,
+      matched: item.matched ?? false
+    })),
+    emitted: trial.emitted ?? traceExcerpt(trial.trace),
+    metrics: trial.metrics
+  };
+}
+async function evaluateMultiShotGate(config, baseline, candidate) {
+  const gateConfig = config.gate;
+  const reps = gateConfig.reps ?? config.reps;
+  const candidateRuns = [];
+  const baselineRuns = [];
+  const searchIds = gateConfig.searchScenarioIds ?? config.searchScenarioIds;
+  for (const scenarioId of searchIds) {
+    for (let rep = 0; rep < reps; rep++) {
+      const seed = seedFor(config, scenarioId, rep);
+      const baseTrial = await scoreOne(config, baseline, scenarioId, rep, "search");
+      const candTrial = await scoreOne(config, candidate, scenarioId, rep, "search");
+      baselineRuns.push(toValidatedRecord(config, baseline, scenarioId, rep, "search", seed, baseTrial));
+      candidateRuns.push(toValidatedRecord(config, candidate, scenarioId, rep, "search", seed, candTrial));
+    }
+  }
+  for (const scenarioId of gateConfig.holdoutScenarioIds) {
+    for (let rep = 0; rep < reps; rep++) {
+      const seed = seedFor(config, scenarioId, rep);
+      const baseTrial = await scoreOne(config, baseline, scenarioId, rep, "holdout");
+      const candTrial = await scoreOne(config, candidate, scenarioId, rep, "holdout");
+      baselineRuns.push(toValidatedRecord(config, baseline, scenarioId, rep, "holdout", seed, baseTrial));
+      candidateRuns.push(toValidatedRecord(config, candidate, scenarioId, rep, "holdout", seed, candTrial));
+    }
+  }
+  const decision = new HeldOutGate(gateConfig.gate).evaluate(candidateRuns, baselineRuns);
+  return { decision, candidateRuns, baselineRuns };
+}
+async function scoreOne(config, variant, scenarioId, rep, split) {
+  const seed = seedFor(config, scenarioId, rep);
+  const input = { variant, scenarioId, rep, split, seed };
+  try {
+    const run = await config.runner.run(input);
+    const scored = await config.scorer.score({ ...input, run });
+    const asi = scored.asi ?? [];
+    return {
+      variantId: variant.id,
+      scenarioId,
+      rep,
+      ok: scored.ok ?? true,
+      score: clamp013(scored.score),
+      cost: scored.costUsd ?? run.costUsd ?? 0,
+      durationMs: scored.durationMs ?? run.durationMs ?? 0,
+      metrics: {
+        ...numericMetrics(scored.metrics),
+        ...asiMetrics(asi)
+      },
+      split,
+      seed,
+      trace: run.trace,
+      asi,
+      emitted: scored.emitted ?? traceExcerpt(run.trace),
+      metadata: scored.metadata
+    };
+  } catch (err) {
+    return {
+      variantId: variant.id,
+      scenarioId,
+      rep,
+      ok: false,
+      score: 0,
+      cost: 0,
+      durationMs: 0,
+      metrics: { error: 1 },
+      error: err instanceof Error ? err.message : String(err),
+      split,
+      seed,
+      asi: [{
+        severity: "critical",
+        message: err instanceof Error ? err.message : String(err),
+        responsibleSurface: config.target
+      }],
+      emitted: ""
+    };
+  }
+}
+function toValidatedRecord(config, variant, scenarioId, rep, split, seed, trial) {
+  const record = config.gate.toRunRecord({ variant, scenarioId, rep, split, seed, trial });
+  return validateRunRecord(record);
+}
+function validateConfig(config) {
+  if (!config.runId.trim()) throw new Error("runMultiShotOptimization: runId must not be empty");
+  if (!config.target.trim()) throw new Error("runMultiShotOptimization: target must not be empty");
+  if (config.seedVariants.length === 0) {
+    throw new Error("runMultiShotOptimization: seedVariants must not be empty");
+  }
+  if (config.searchScenarioIds.length === 0) {
+    throw new Error("runMultiShotOptimization: searchScenarioIds must not be empty");
+  }
+  requirePositiveInteger(config.reps, "reps");
+  requirePositiveInteger(config.generations, "generations");
+  requirePositiveInteger(config.populationSize, "populationSize");
+  if (config.scoreConcurrency !== void 0) requirePositiveInteger(config.scoreConcurrency, "scoreConcurrency");
+  if (config.populationSize < config.seedVariants.length) {
+    throw new Error("runMultiShotOptimization: populationSize must be >= seedVariants.length");
+  }
+  assertUnique(config.seedVariants.map((v) => v.id), "seedVariants.id");
+  assertUnique(config.searchScenarioIds, "searchScenarioIds");
+  if (config.gate) {
+    if (config.gate.holdoutScenarioIds.length === 0) {
+      throw new Error("runMultiShotOptimization: gate.holdoutScenarioIds must not be empty");
+    }
+    if (config.gate.reps !== void 0) requirePositiveInteger(config.gate.reps, "gate.reps");
+    assertUnique(config.gate.holdoutScenarioIds, "gate.holdoutScenarioIds");
+    if (config.gate.searchScenarioIds) assertUnique(config.gate.searchScenarioIds, "gate.searchScenarioIds");
+    const searchIds = new Set(config.searchScenarioIds);
+    for (const id of config.gate.holdoutScenarioIds) {
+      if (searchIds.has(id)) {
+        throw new Error(`runMultiShotOptimization: holdout scenario "${id}" also appears in searchScenarioIds`);
+      }
+    }
+    const baselineId = config.seedVariants[0].id;
+    if (config.gate.gate.baselineKey !== baselineId) {
+      throw new Error(
+        `runMultiShotOptimization: gate.gate.baselineKey must match first seed variant id "${baselineId}"`
+      );
+    }
+  }
+}
+function requirePositiveInteger(value, name) {
+  if (!Number.isInteger(value) || value <= 0) {
+    throw new Error(`runMultiShotOptimization: ${name} must be a positive integer`);
+  }
+}
+function assertUnique(values, name) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const value of values) {
+    if (!value.trim()) throw new Error(`runMultiShotOptimization: ${name} must not contain empty values`);
+    if (seen.has(value)) throw new Error(`runMultiShotOptimization: duplicate ${name} "${value}"`);
+    seen.add(value);
+  }
+}
+function aggregateFor(evolution, variantId) {
+  const final = evolution.generations[evolution.generations.length - 1];
+  const aggregate2 = final?.aggregates.find((a) => a.variantId === variantId);
+  if (!aggregate2) {
+    throw new Error(`runMultiShotOptimization: missing aggregate for variant "${variantId}"`);
+  }
+  return aggregate2;
+}
+function seedFor(config, scenarioId, rep) {
+  const base = config.seedBase ?? 0;
+  return (base + stableHash2(`${scenarioId}${rep}`)) % Number.MAX_SAFE_INTEGER;
+}
+function stableHash2(input) {
+  let h = 2166136261;
+  for (let i = 0; i < input.length; i++) {
+    h ^= input.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return h >>> 0;
+}
+function clamp013(n) {
+  if (!Number.isFinite(n)) return 0;
+  return Math.max(0, Math.min(1, n));
+}
+function numericMetrics(metrics) {
+  const out = {};
+  for (const [k, v] of Object.entries(metrics ?? {})) {
+    if (Number.isFinite(v)) out[k] = v;
+  }
+  return out;
+}
+function asiMetrics(asi) {
+  const out = { asi: asi.length };
+  for (const item of asi.slice(0, 1e3)) {
+    const sev = normalizeSeverity(item.severity);
+    out[`asi.${sev}`] = (out[`asi.${sev}`] ?? 0) + 1;
+    if (item.responsibleSurface) {
+      const key = `surface.${metricKeySegment(item.responsibleSurface)}`;
+      out[key] = (out[key] ?? 0) + 1;
+    }
+  }
+  return out;
+}
+function normalizeSeverity(severity) {
+  if (severity === "info" || severity === "warning" || severity === "error" || severity === "critical") {
+    return severity;
+  }
+  return "error";
+}
+function metricKeySegment(raw) {
+  return raw.trim().replace(/[^a-zA-Z0-9._-]+/g, "_").slice(0, 80) || "unknown";
+}
+function traceExcerpt(trace) {
+  if (!trace) return void 0;
+  if (typeof trace.output === "string") return trace.output;
+  if (trace.transcript) return trace.transcript;
+  if (trace.turns) {
+    try {
+      const clipped = trace.turns.slice(0, 20);
+      const suffix = trace.turns.length > clipped.length ? ` ... ${trace.turns.length - clipped.length} more turn(s)` : "";
+      return `${JSON.stringify(clipped).slice(0, 2e3)}${suffix}`;
+    } catch {
+      return "[unserializable trace turns]";
+    }
+  }
+  return void 0;
+}
+
 // src/jsonl-trial-cache.ts
 import { appendFileSync as appendFileSync4, existsSync as existsSync6, mkdirSync as mkdirSync4, readFileSync as readFileSync5 } from "fs";
 import { dirname as dirname4 } from "path";
@@ -13708,6 +14079,7 @@ export {
   decideReferenceReplayPromotion,
   decideReferenceReplayRunPromotion,
   defaultJudges,
+  defaultMultiShotObjectives,
   defaultReferenceReplayMatcher,
   deployGateLayer,
   distillPlaybook,
@@ -13715,6 +14087,7 @@ export {
   estimateCost,
   estimateTokens,
   euAiActReport,
+  evaluateActionPolicy,
   evaluateContract,
   evaluateHypothesis,
   evaluateOracles,
@@ -13822,6 +14195,8 @@ export {
   renderPlaybookMarkdown,
   renderPreferenceMemoryMarkdown,
   renderSteeringText,
+  replayFeedbackTrajectories,
+  replayFeedbackTrajectory,
   replayScorerOverCorpus,
   replayTraceThroughJudge,
   requiredSampleSize,
@@ -13842,6 +14217,7 @@ export {
   runJudgeFleet,
   runKeywordCoverageJudge,
   runKeywordCoverageJudgeUrl,
+  runMultiShotOptimization,
   runPromptEvolution,
   runProposeReview,
   runProposeReviewAsControlLoop,
@@ -13885,6 +14261,7 @@ export {
   toolSpans,
   toolSuccessRubric,
   toolWasteView,
+  trialTraceFromMultiShotTrial,
   typoMutator,
   urlContains,
   validateRunRecord,