@tangle-network/agent-eval 0.23.0 → 0.24.0

package/dist/chunk-ZN274SWR.js

@@ -0,0 +1,613 @@
+import {
+  objectiveEval,
+  runAgentControlLoop
+} from "./chunk-LSH4MMOZ.js";
+import {
+  validateRunRecord
+} from "./chunk-NLMNWKVM.js";
+import {
+  TraceEmitter
+} from "./chunk-TVVP3ZZQ.js";
+
+// src/action-policy.ts
+function evaluateActionPolicy(action, policy = {}, options = {}) {
+  const reasons = [];
+  let blocked = false;
+  let requiresApproval = Boolean(action.requiresApproval);
+  if (policy.allowedTypes?.length && !policy.allowedTypes.includes(action.type)) {
+    blocked = true;
+    reasons.push(`action type "${action.type}" is not allowed`);
+  }
+  if (policy.blockedTypes?.includes(action.type)) {
+    blocked = true;
+    reasons.push(`action type "${action.type}" is blocked`);
+  }
+  if (policy.alwaysRequireApprovalTypes?.includes(action.type)) {
+    requiresApproval = true;
+    reasons.push(`action type "${action.type}" requires approval`);
+  }
+  if (policy.requireApprovalForExternalSideEffects && action.externalSideEffect) {
+    requiresApproval = true;
+    reasons.push("external side effect requires approval");
+  }
+  if (policy.requireApprovalAboveCostUsd !== void 0 && (action.costUsd ?? 0) > policy.requireApprovalAboveCostUsd) {
+    requiresApproval = true;
+    reasons.push(
+      `cost ${action.costUsd} exceeds approval threshold ${policy.requireApprovalAboveCostUsd}`
+    );
+  }
+  if (policy.maxActionCostUsd !== void 0 && (action.costUsd ?? 0) > policy.maxActionCostUsd) {
+    blocked = true;
+    reasons.push(`cost ${action.costUsd} exceeds max action cost ${policy.maxActionCostUsd}`);
+  }
+  if (policy.remainingBudgetUsd !== void 0 && (action.costUsd ?? 0) > policy.remainingBudgetUsd) {
+    blocked = true;
+    reasons.push(`cost ${action.costUsd} exceeds remaining budget ${policy.remainingBudgetUsd}`);
+  }
+  if (policy.expectedOutcomeRequired && !action.metadata?.expectedOutcome) {
+    blocked = true;
+    reasons.push("expected outcome is required");
+  }
+  if (policy.killCriteriaRequired && !action.metadata?.killCriteria) {
+    blocked = true;
+    reasons.push("kill criteria are required");
+  }
+  if (policy.autoApproveTypes?.includes(action.type) && requiresApproval) {
+    reasons.push(
+      `action type "${action.type}" is auto-approved only when no approval policy applies`
+    );
+  }
+  if (!reasons.length) reasons.push(requiresApproval ? "approval required" : "action allowed");
+  const label = blocked || requiresApproval ? {
+    source: "policy",
+    kind: blocked ? "policy_block" : "comment",
+    value: { actionType: action.type, blocked, requiresApproval },
+    reason: reasons.join("; "),
+    severity: blocked ? "critical" : "warning",
+    createdAt: options.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+    metadata: { action, policy }
+  } : void 0;
+  return {
+    allowed: !blocked,
+    blocked,
+    requiresApproval: !blocked && requiresApproval,
+    reasons,
+    label
+  };
+}
+
+// src/run-evidence.ts
+function controlRunToRunRecord(run, options) {
+  const score = clampScore(
+    options.score ?? run.score ?? scoreFromEvals(run.finalEvals) ?? (run.pass ? 1 : 0)
+  );
+  const outcome = options.splitTag === "holdout" ? { holdoutScore: score, raw: normalizeRawMetrics(options.raw, run, score) } : { searchScore: score, raw: normalizeRawMetrics(options.raw, run, score) };
+  return validateRunRecord({
+    runId: options.runId ?? run.runId ?? `control:${options.experimentId}:${options.candidateId}:${options.seed}:${options.splitTag}`,
+    experimentId: options.experimentId,
+    candidateId: options.candidateId,
+    seed: options.seed,
+    model: options.model,
+    promptHash: options.promptHash,
+    configHash: options.configHash,
+    commitSha: options.commitSha,
+    wallMs: run.wallMs,
+    ...options.queueMs !== void 0 ? { queueMs: options.queueMs } : {},
+    costUsd: run.spentCostUsd,
+    tokenUsage: options.tokenUsage,
+    ...options.judgeMetadata ? { judgeMetadata: options.judgeMetadata } : {},
+    outcome,
+    failureMode: options.failureMode ?? failureModeFromRun(run),
+    splitTag: options.splitTag
+  });
+}
+function scoreFromEvals(evals) {
+  const scores = evals.map((e) => e.score).filter((score) => typeof score === "number" && Number.isFinite(score));
+  if (scores.length === 0) return void 0;
+  return clampScore(scores.reduce((sum, score) => sum + score, 0) / scores.length);
+}
+function normalizeRawMetrics(raw, run, score) {
+  return {
+    ...finiteOnly(raw ?? {}),
+    score,
+    pass: run.pass ? 1 : 0,
+    completed: run.completed ? 1 : 0,
+    steps: run.steps.length,
+    runtimeErrors: run.runtimeErrors.length
+  };
+}
+function finiteOnly(values) {
+  const out = {};
+  for (const [key, value] of Object.entries(values)) {
+    if (Number.isFinite(value)) out[key] = value;
+  }
+  return out;
+}
+function failureModeFromRun(run) {
+  if (run.pass) return void 0;
+  return run.failureClass ?? "unknown";
+}
+function clampScore(value) {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+
+// src/propose-review.ts
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "fs";
+import { dirname } from "path";
+function inMemoryReviewStore(initial = []) {
+  const entries = [...initial];
+  return {
+    async load() {
+      return [...entries];
+    },
+    async append(entry) {
+      entries.push(entry);
+    }
+  };
+}
+function jsonlReviewStore(path) {
+  return {
+    async load() {
+      if (!existsSync(path)) return [];
+      const raw = readFileSync(path, "utf8");
+      const out = [];
+      for (const line of raw.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+        try {
+          out.push(JSON.parse(trimmed));
+        } catch {
+        }
+      }
+      return out;
+    },
+    async append(entry) {
+      mkdirSync(dirname(path), { recursive: true });
+      appendFileSync(path, `${JSON.stringify(entry)}
+`);
+    }
+  };
+}
+var DEFAULT_FALLBACK_INSTRUCTION = "Inspect the verification failures above. Fix the critical issues first, then the major ones. Do not restate the failures \u2014 act on them.";
+async function runProposeReview(config) {
+  const maxShots = config.maxShots ?? 10;
+  const maxWallMs = config.maxWallMs ?? 10 * 60 * 1e3;
+  const confidenceFloor = config.confidenceFloor ?? 0.3;
+  const confidenceFloorWindow = config.confidenceFloorWindow ?? 2;
+  const memory = config.memory ?? inMemoryReviewStore();
+  const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION;
+  const emitter = config.store ? new TraceEmitter(config.store) : null;
+  if (emitter) {
+    await emitter.startRun({
+      scenarioId: config.scenarioId ?? "propose-review",
+      projectId: config.projectId,
+      variantId: config.variantId,
+      layer: "meta",
+      tags: {
+        goal: config.goal.slice(0, 120),
+        maxShots: String(maxShots)
+      }
+    });
+  }
+  const abort = new AbortController();
+  const wallStart = Date.now();
+  const wallTimer = setTimeout(
+    () => abort.abort(new Error("propose-review wall timeout")),
+    maxWallMs
+  );
+  const shots = [];
+  let state = config.initialState;
+  let priorReview = null;
+  let lastVerification = { pass: false };
+  let failureClass;
+  let completed = false;
+  let lowConfidenceStreak = 0;
+  try {
+    for (let shot = 1; shot <= maxShots; shot++) {
+      if (abort.signal.aborted) {
+        failureClass = "timeout";
+        break;
+      }
+      const shotStart = Date.now();
+      const shotHandle = emitter ? await emitter.span({ kind: "tool", name: `shot-${shot}` }) : null;
+      let proposeOut;
+      try {
+        proposeOut = await config.propose({
+          shot,
+          goal: config.goal,
+          state,
+          priorReview,
+          abortSignal: abort.signal,
+          emitter: emitter ?? void 0
+        });
+      } catch (err) {
+        await shotHandle?.fail(err instanceof Error ? err : String(err));
+        failureClass = "unknown";
+        throw err;
+      }
+      state = proposeOut.state;
+      const traceSummary = proposeOut.traceSummary;
+      let verification;
+      try {
+        verification = await config.verify(state);
+      } catch (err) {
+        await shotHandle?.fail(err instanceof Error ? err : String(err));
+        failureClass = "unknown";
+        throw err;
+      }
+      lastVerification = verification;
+      const memorySnapshot = await memory.load();
+      const verificationDigest = {
+        pass: verification.pass,
+        score: verification.score,
+        failingLayers: verification.failingLayers ?? []
+      };
+      let review;
+      let reviewAvailable = true;
+      let reviewError;
+      if (verification.pass) {
+        review = {
+          observations: "verification passed \u2014 skipping reviewer LLM call",
+          diagnosis: "no failures to diagnose",
+          nextShotInstruction: "(done)",
+          shouldContinue: false,
+          confidence: 1
+        };
+      } else {
+        try {
+          review = await config.review({
+            shot,
+            goal: config.goal,
+            state,
+            verification,
+            traceSummary,
+            memory: memorySnapshot
+          });
+          review = coerceReview(review);
+        } catch (err) {
+          reviewAvailable = false;
+          reviewError = err instanceof Error ? err.message : String(err);
+          const lastInstruction = memorySnapshot.length > 0 ? memorySnapshot[memorySnapshot.length - 1].nextShotInstruction : fallbackInstruction;
+          review = {
+            observations: "(reviewer unavailable \u2014 using last-known instruction)",
+            diagnosis: reviewError,
+            nextShotInstruction: lastInstruction,
+            shouldContinue: true,
+            confidence: 0.3
+          };
+        }
+      }
+      const entry = {
+        shot,
+        timestamp: Date.now(),
+        ...review,
+        verification: verificationDigest
+      };
+      await memory.append(entry);
+      const shotRecord = {
+        shot,
+        state,
+        verification,
+        traceSummary,
+        review,
+        reviewAvailable,
+        reviewError,
+        durationMs: Date.now() - shotStart
+      };
+      shots.push(shotRecord);
+      await shotHandle?.end({
+        attributes: {
+          verificationPass: verification.pass,
+          verificationScore: verification.score ?? null,
+          reviewShouldContinue: review.shouldContinue,
+          reviewConfidence: review.confidence,
+          reviewAvailable
+        }
+      });
+      if (verification.pass) {
+        completed = true;
+        break;
+      }
+      if (!review.shouldContinue) {
+        break;
+      }
+      if (confidenceFloorWindow > 0 && review.confidence <= confidenceFloor) {
+        lowConfidenceStreak += 1;
+        if (lowConfidenceStreak >= confidenceFloorWindow) break;
+      } else {
+        lowConfidenceStreak = 0;
+      }
+      priorReview = review;
+    }
+    if (!completed && !failureClass) {
+      failureClass = shots.length >= maxShots ? "budget_exceeded" : "unknown";
+    }
+  } finally {
+    clearTimeout(wallTimer);
+  }
+  const score = lastVerification.pass ? 1 : typeof lastVerification.score === "number" ? lastVerification.score : 0;
+  if (emitter) {
+    await emitter.endRun({
+      pass: completed,
+      score,
+      failureClass,
+      notes: `${shots.length} shot(s); final pass=${lastVerification.pass}`
+    });
+  }
+  return {
+    runId: emitter?.runId ?? null,
+    completed,
+    shots,
+    finalState: state,
+    finalVerification: lastVerification,
+    failureClass,
+    wallMs: Date.now() - wallStart,
+    score
+  };
+}
+var REVIEWER_SYSTEM_PROMPT = `You are a senior reviewer directing a multi-shot build loop.
+You do NOT grade \u2014 the verifier already did. Your job is to direct the worker's next shot.
+You are blind to the worker's inner monologue. You see what it DID, not what it thought.
+Return STRICT JSON matching the schema. No prose outside the JSON.`;
+function createLlmReviewer(cfg) {
+  const renderState = cfg.renderState ?? ((s) => safeJson(s));
+  const renderTraceSummary = cfg.renderTraceSummary ?? ((s) => s === void 0 ? "(none)" : safeJson(s));
+  const system = cfg.systemPromptAddendum ? `${REVIEWER_SYSTEM_PROMPT}
+
+${cfg.systemPromptAddendum}` : REVIEWER_SYSTEM_PROMPT;
+  return async (input) => {
+    const memoryBlock = input.memory.length === 0 ? "(no prior shots \u2014 this is shot 1)" : input.memory.map(
+      (m) => [
+        `shot ${m.shot} \u2014 verification.pass=${m.verification.pass}` + (typeof m.verification.score === "number" ? ` score=${m.verification.score.toFixed(2)}` : "") + ` confidence=${m.confidence.toFixed(2)} failing=[${(m.verification.failingLayers ?? []).join(",")}]`,
+        `  observations: ${m.observations.slice(0, 400)}`,
+        `  diagnosis: ${m.diagnosis.slice(0, 400)}`,
+        `  instruction given: ${m.nextShotInstruction.slice(0, 400)}`
+      ].join("\n")
+    ).join("\n\n");
+    const user = [
+      `=== GOAL ===`,
+      input.goal,
+      ``,
+      `=== SHOT NUMBER ===`,
+      String(input.shot),
+      ``,
+      `=== CURRENT STATE ===`,
+      renderState(input.state),
+      ``,
+      `=== TRACE SUMMARY ===`,
+      renderTraceSummary(input.traceSummary),
+      ``,
+      `=== VERIFICATION ===`,
+      summarizeVerification(input.verification),
+      ``,
+      `=== REVIEWER MEMORY (prior shots) ===`,
+      memoryBlock,
+      ``,
+      `=== YOUR TASK ===`,
+      `Return STRICT JSON:`,
+      `{`,
+      `  "observations": string (20..2000 chars, first-person worker behavior \u2014 quote counts, errors, loops)`,
+      `  "diagnosis": string (20..1500 chars, root cause, NOT a restatement of verification)`,
+      `  "nextShotInstruction": string (40..3000 chars, concrete directive to the worker)`,
+      `  "shouldContinue": boolean (false if verification.pass, or if thrashing, or unachievable)`,
+      `  "confidence": number in [0,1]`,
+      `}`
+    ].join("\n");
+    const raw = await cfg.callJson({ system, user });
+    return coerceReview(raw);
+  };
+}
+function coerceReview(raw) {
+  if (!raw || typeof raw !== "object") {
+    throw new Error("reviewer returned non-object");
+  }
+  const observations = typeof raw.observations === "string" ? raw.observations : "";
+  const diagnosis = typeof raw.diagnosis === "string" ? raw.diagnosis : "";
+  const nextShotInstruction = typeof raw.nextShotInstruction === "string" ? raw.nextShotInstruction : "";
+  if (!observations || !diagnosis || !nextShotInstruction) {
+    throw new Error("reviewer missing required string fields");
+  }
+  if (typeof raw.shouldContinue !== "boolean") {
+    throw new Error("reviewer missing shouldContinue boolean");
+  }
+  const confidenceRaw = Number(raw.confidence);
+  if (!Number.isFinite(confidenceRaw)) {
+    throw new Error("reviewer confidence not finite");
+  }
+  return {
+    observations,
+    diagnosis,
+    nextShotInstruction,
+    shouldContinue: raw.shouldContinue,
+    confidence: Math.max(0, Math.min(1, confidenceRaw))
+  };
+}
+function summarizeVerification(v) {
+  const header = `pass=${v.pass}` + (typeof v.score === "number" ? ` score=${v.score.toFixed(3)}` : "") + (v.failingLayers && v.failingLayers.length > 0 ? ` failing=[${v.failingLayers.join(", ")}]` : "");
+  const details = v.details === void 0 ? "" : `
+${safeJson(v.details).slice(0, 1500)}`;
+  return header + details;
+}
+function safeJson(x) {
+  try {
+    return JSON.stringify(x, null, 2);
+  } catch {
+    return String(x);
+  }
+}
+
+// src/propose-review-control.ts
+var DEFAULT_FALLBACK_INSTRUCTION2 = "Inspect the verification failures above. Fix the critical issues first, then the major ones. Do not restate the failures \u2014 act on them.";
+async function runProposeReviewAsControlLoop(config) {
+  const maxShots = config.maxShots ?? 10;
+  const confidenceFloor = config.confidenceFloor ?? 0.3;
+  const confidenceFloorWindow = config.confidenceFloorWindow ?? 2;
+  const memory = config.memory ?? inMemoryReviewStore();
+  const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION2;
+  const failureClassFromVerification = config.failureClassFromVerification ?? controlFailureClassFromVerification;
+  let lowConfidenceStreak = 0;
+  let current = {
+    shot: 0,
+    state: config.initialState,
+    priorReview: null,
+    verification: { pass: false },
+    memory: await memory.load(),
+    completed: false,
+    reviewAvailable: false
+  };
+  return runAgentControlLoop({
+    intent: config.goal,
+    budget: { maxSteps: maxShots, maxWallMs: config.maxWallMs },
+    store: config.store,
+    scenarioId: config.scenarioId ?? "propose-review-control",
+    projectId: config.projectId,
+    variantId: config.variantId,
+    actionFailure: config.actionFailure ?? "stop",
+    observe: () => current,
+    validate: ({ state }) => [
+      objectiveEval({
+        id: "verification",
+        passed: state.verification.pass,
+        score: state.verification.score,
+        severity: "critical",
+        detail: state.verification.pass ? "verification passed" : `verification failed${state.verification.failingLayers?.length ? `: ${state.verification.failingLayers.join(", ")}` : ""}`
+      })
+    ],
+    shouldStop: ({ state }) => {
+      if (state.verification.pass) {
+        return {
+          stop: true,
+          pass: true,
+          reason: "verification passed",
+          score: state.verification.score
+        };
+      }
+      if (state.completed) {
+        return {
+          stop: true,
+          pass: false,
+          reason: "reviewer stopped continuation",
+          score: state.verification.score,
+          failureClass: failureClassFromVerification(state.verification)
+        };
+      }
+      return {
+        stop: false,
+        pass: false,
+        reason: "verification still failing",
+        score: state.verification.score
+      };
+    },
+    decide: ({ state }) => ({
+      type: "continue",
+      action: { type: "propose-review-shot", shot: state.shot + 1 },
+      reason: state.priorReview?.nextShotInstruction ?? fallbackInstruction
+    }),
+    act: async (action, ctx) => {
+      const shot = action.shot;
+      const proposeOut = await config.propose({
+        shot,
+        goal: config.goal,
+        state: current.state,
+        priorReview: current.priorReview,
+        abortSignal: ctx.abortSignal,
+        emitter: ctx.emitter
+      });
+      const nextState = proposeOut.state;
+      const verification = await config.verify(nextState);
+      let review = null;
+      let reviewAvailable = false;
+      let reviewError;
+      let shouldContinue = !verification.pass;
+      if (!verification.pass) {
+        try {
+          review = await config.review({
+            shot,
+            goal: config.goal,
+            state: nextState,
+            verification,
+            traceSummary: proposeOut.traceSummary,
+            memory: await memory.load()
+          });
+          reviewAvailable = true;
+          shouldContinue = review.shouldContinue;
+          lowConfidenceStreak = review.confidence <= confidenceFloor ? lowConfidenceStreak + 1 : 0;
+          if (confidenceFloorWindow > 0 && lowConfidenceStreak >= confidenceFloorWindow)
+            shouldContinue = false;
+        } catch (err) {
+          reviewError = err instanceof Error ? err.message : String(err);
+          review = current.priorReview ?? {
+            observations: "Reviewer unavailable.",
+            diagnosis: reviewError,
+            nextShotInstruction: fallbackInstruction,
+            shouldContinue: true,
+            confidence: 0
+          };
+          shouldContinue = true;
+        }
+      } else {
+        review = {
+          observations: "Verification passed.",
+          diagnosis: "No further revision needed.",
+          nextShotInstruction: "",
+          shouldContinue: false,
+          confidence: 1
+        };
+      }
+      const entry = {
+        ...review ?? {
+          observations: "No review.",
+          diagnosis: "",
+          nextShotInstruction: fallbackInstruction,
+          shouldContinue,
+          confidence: 0
+        },
+        shot,
+        timestamp: Date.now(),
+        verification: {
+          pass: verification.pass,
+          score: verification.score,
+          failingLayers: verification.failingLayers
+        }
+      };
+      await memory.append(entry);
+      current = {
+        shot,
+        state: nextState,
+        priorReview: review,
+        verification,
+        traceSummary: proposeOut.traceSummary,
+        memory: await memory.load(),
+        completed: verification.pass || !shouldContinue,
+        reviewAvailable,
+        reviewError
+      };
+      return {
+        state: nextState,
+        verification,
+        traceSummary: proposeOut.traceSummary,
+        review,
+        reviewAvailable,
+        reviewError
+      };
+    }
+  });
+}
+function controlFailureClassFromVerification(verification) {
+  if (verification.pass) return void 0;
+  return verification.failingLayers?.length ? "instruction_following" : "unknown";
+}
+
+export {
+  evaluateActionPolicy,
+  controlRunToRunRecord,
+  scoreFromEvals,
+  inMemoryReviewStore,
+  jsonlReviewStore,
+  runProposeReview,
+  createLlmReviewer,
+  runProposeReviewAsControlLoop,
+  controlFailureClassFromVerification
+};
+//# sourceMappingURL=chunk-ZN274SWR.js.map