@theaiteam/promptdiff 1.0.0-rc.1

package/src/engine/compare.ts

@@ -0,0 +1,563 @@
+import { createHash } from "node:crypto";
+import { assembleSystemPrompt } from "../prompt";
+import type { Runner, RunnerRunOptions, RunResult } from "../types";
+import { buildCacheKey, loadCachedArm, storeCachedArm } from "./cache";
+import type { ArmConfig, CompareConfig, EvalCaseConfig, ScenarioKind } from "./config";
+import { renderStrict, type RenderVars } from "./render";
+import { fisherExactTwoTailedP } from "./stats";
+import { gradeRun, type GradeResult } from "./grader";
+import { assertJudgeCalibrated } from "./judge";
+import { prepareSandbox } from "./sandbox";
+import { installSkills } from "./skill-install";
+
+export interface ArmRunSummary {
+  run: number;
+  pass: boolean;
+  grade: GradeResult;
+  costUsd: number;
+  turns: number;
+  durationMs: number;
+  sandboxDir: string;
+  output: string;
+}
+
+export interface ArmSummary {
+  name: "baseline" | "proposed";
+  passes: number;
+  totalRuns: number;
+  passRate: number;
+  totalCostUsd: number;
+  runs: ArmRunSummary[];
+  /** True when this summary was served from the baseline cache instead of fresh runs. */
+  cached?: boolean;
+}
+
+export interface CaseSummary {
+  name: string;
+  kind: ScenarioKind;
+  baseline: ArmSummary;
+  proposed: ArmSummary;
+  assertions: string[];
+  /** Two-tailed Fisher exact p for the pass/fail table — how easily noise explains the delta. */
+  samplingP?: number;
+  /** Hashes of each arm's rendered system prompt — pins a result to exact prompt content. */
+  promptSha256?: { baseline: string; proposed: string };
+}
+
+export interface CompareSummary {
+  name: string;
+  arms: { baseline: ArmConfig; proposed: ArmConfig };
+  productionModel?: string;
+  cases: CaseSummary[];
+  failedAssertions: string[];
+  totalCostUsd: number;
+}
+
+export interface CompareRunOptions {
+  config: CompareConfig;
+  runners: { baseline: Runner; proposed: Runner };
+  onProgress?: (message: string) => void;
+  /** Opt-in baseline-arm result cache; hits skip the baseline runs entirely. */
+  cache?: { dir: string };
+}
+
+export async function runCompare(options: CompareRunOptions): Promise<CompareSummary> {
+  const { config, runners, onProgress, cache } = options;
+  // Validate each arm against its own runner — a mixed comparison fails on the
+  // violating arm before either arm's paid runs.
+  validateRunnerSupport(config, runners.baseline);
+  validateRunnerSupport(config, runners.proposed);
+  assertJudgeGradersCalibrated(config);
+  // Install delivery keeps skill text out of the system prompt entirely — the
+  // arms differ only by which skill directory lands in the sandbox registry.
+  const inline = config.delivery !== "install";
+  const baselinePrompt = assembleSystemPrompt(config.agent, inline ? config.baselineSkills : []);
+  const proposedPrompt = assembleSystemPrompt(config.agent, inline ? config.proposedSkills : []);
+  // Rendering all cases up front means an unbound placeholder in scenario 3
+  // fails here — before scenario 1 spends anything.
+  const renderedCases = config.cases.map((evalCase) => renderCase(evalCase, baselinePrompt, proposedPrompt, config));
+  const cases: CaseSummary[] = [];
+
+  for (const { evalCase, baselineSystem, proposedSystem } of renderedCases) {
+    onProgress?.(`scenario ${evalCase.name} (${evalCase.kind})`);
+    const baseline = await runBaselineArm(config, evalCase, baselineSystem, runners.baseline, cache, onProgress);
+    const proposed = await runArm(config, evalCase, "proposed", proposedSystem, runners.proposed, onProgress);
+    cases.push({
+      name: evalCase.name,
+      kind: evalCase.kind,
+      baseline,
+      proposed,
+      assertions: evaluateAssertions(evalCase, baseline, proposed),
+      samplingP: fisherExactTwoTailedP(baseline.passes, baseline.totalRuns, proposed.passes, proposed.totalRuns),
+      promptSha256: { baseline: sha256(baselineSystem), proposed: sha256(proposedSystem) },
+    });
+  }
+
+  const failedAssertions = cases.flatMap((caseSummary) =>
+    caseSummary.assertions.map((assertion) => `${caseSummary.name}: ${assertion}`),
+  );
+  const totalCostUsd = cases.reduce(
+    (sum, caseSummary) => sum + caseSummary.baseline.totalCostUsd + caseSummary.proposed.totalCostUsd,
+    0,
+  );
+
+  return {
+    name: config.name,
+    arms: config.arms,
+    productionModel: config.productionModel,
+    cases,
+    failedAssertions,
+    totalCostUsd,
+  };
+}
+
+export function formatCompareSummary(summary: CompareSummary): string {
+  const lines = [`promptdiff compare: ${summary.name}`];
+  const baselineLabel = armLabel("baseline", summary.arms);
+  const proposedLabel = armLabel("proposed", summary.arms);
+
+  for (const caseSummary of summary.cases) {
+    const deltaPass = caseSummary.proposed.passRate - caseSummary.baseline.passRate;
+    const deltaCost = caseSummary.proposed.totalCostUsd - caseSummary.baseline.totalCostUsd;
+    lines.push(
+      "",
+      `${caseSummary.name} (${caseSummary.kind})`,
+      `  ${baselineLabel}: ${caseSummary.baseline.passes}/${caseSummary.baseline.totalRuns} pass (${formatPct(caseSummary.baseline.passRate)}) | $${caseSummary.baseline.totalCostUsd.toFixed(4)}${caseSummary.baseline.cached ? " (cached)" : ""}`,
+      `  ${proposedLabel}: ${caseSummary.proposed.passes}/${caseSummary.proposed.totalRuns} pass (${formatPct(caseSummary.proposed.passRate)}) | $${caseSummary.proposed.totalCostUsd.toFixed(4)}`,
+      `  delta: ${deltaPass >= 0 ? "+" : ""}${formatPct(deltaPass)} pass | ${deltaCost >= 0 ? "+" : ""}$${deltaCost.toFixed(4)}`,
+    );
+
+    if (caseSummary.assertions.length > 0) {
+      for (const assertion of caseSummary.assertions) {
+        lines.push(`  FAIL: ${assertion}`);
+      }
+    } else if (caseSummary.kind === "compare") {
+      lines.push("  INFO: no assertion (kind \"compare\")");
+    } else {
+      lines.push("  PASS: assertions satisfied");
+    }
+
+    // A pass-rate delta that noise explains must not read like a receipt:
+    // 1/3 → 2/3 passes the target assertion but is close to a coin flip.
+    if (
+      caseSummary.samplingP !== undefined &&
+      caseSummary.baseline.passes !== caseSummary.proposed.passes &&
+      caseSummary.samplingP > 0.05
+    ) {
+      lines.push(
+        `  NOTE: delta could be sampling noise (Fisher exact p=${caseSummary.samplingP.toFixed(2)}) — consider more runs`,
+      );
+    }
+
+    // Failing runs carry their grader evidence into the summary — diagnosing
+    // WHICH check missed used to require re-running with --keep-sandbox.
+    for (const arm of [caseSummary.baseline, caseSummary.proposed]) {
+      for (const run of arm.runs) {
+        if (run.pass) continue;
+        lines.push(`  ${arm.name} run ${run.run} failed: ${run.grade.message}`);
+        lines.push(...graderEvidence(run.grade));
+      }
+    }
+  }
+
+  lines.push("", `total cost: $${summary.totalCostUsd.toFixed(4)}`);
+  if ((summary.arms.baseline.runner === "openai") !== (summary.arms.proposed.runner === "openai")) {
+    lines.push('note: cost columns may not be comparable — openai arms report $0 unless "pricing" is set');
+  }
+  // A pass on the wrong model validates prompt logic, not production behavior —
+  // that divergence must be on the receipt, not in a README caveat.
+  for (const armName of ["baseline", "proposed"] as const) {
+    const model = summary.arms[armName].model;
+    if (summary.productionModel !== undefined && model !== summary.productionModel) {
+      lines.push(
+        `warning: ${armName} tests "${model}" but production model is "${summary.productionModel}" — this validates prompt logic, not production behavior`,
+      );
+    }
+  }
+  if (summary.failedAssertions.length > 0) {
+    lines.push("failed assertions:");
+    for (const assertion of summary.failedAssertions) {
+      lines.push(`  - ${assertion}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+function sha256(text: string): string {
+  return createHash("sha256").update(text).digest("hex");
+}
+
+const EVIDENCE_TAIL_LINES = 6;
+const EVIDENCE_MAX_CHARS = 700;
+
+/** Last few lines of a failing run's grader stdout/stderr, indented for the summary. */
+function graderEvidence(grade: GradeResult): string[] {
+  const streams: Array<[string, string | undefined]> = [
+    ["stdout", grade.stdout],
+    ["stderr", grade.stderr],
+  ];
+  const lines: string[] = [];
+  for (const [stream, content] of streams) {
+    const trimmed = content?.trim();
+    if (!trimmed) continue;
+    const tail = trimmed.split("\n").slice(-EVIDENCE_TAIL_LINES).join("\n").slice(-EVIDENCE_MAX_CHARS);
+    lines.push(`    grader ${stream}:`);
+    lines.push(...tail.split("\n").map((line) => `      ${line}`));
+  }
+  return lines;
+}
+
+/** Bare arm name when the arms match; annotated with model (and runner when runners differ) otherwise. */
+function armLabel(name: "baseline" | "proposed", arms: { baseline: ArmConfig; proposed: ArmConfig }): string {
+  const runnersDiffer = arms.baseline.runner !== arms.proposed.runner;
+  if (!runnersDiffer && arms.baseline.model === arms.proposed.model) return name;
+  const arm = arms[name];
+  return runnersDiffer ? `${name} (${arm.model} via ${arm.runner})` : `${name} (${arm.model})`;
+}
+
+/**
+ * The judge calibration gate, run alongside validateRunnerSupport — BEFORE any
+ * paid run. Missing, stale, mismatched, or below-bar calibration refuses the
+ * whole comparison; an unproven judge must never certify a result.
+ */
+function assertJudgeGradersCalibrated(config: CompareConfig): void {
+  for (const evalCase of config.cases) {
+    if (evalCase.grader.type === "judge") {
+      assertJudgeCalibrated(evalCase.grader);
+    }
+  }
+}
+
+/**
+ * Rejects scenario demands the runner cannot honor — before any paid run,
+ * instead of mid-comparison or (worse) via a silently tool-less arm.
+ */
+export function validateRunnerSupport(config: CompareConfig, runner: Runner): void {
+  if (config.delivery === "install" && !runner.capabilities.skillRegistry) {
+    throw new Error(
+      `delivery "install" needs a runner with a skill registry; runner "${runner.name}" has none (use claude-p)`,
+    );
+  }
+  for (const evalCase of config.cases) {
+    if (evalCase.images.length > 0 && !runner.capabilities.images) {
+      throw new Error(
+        `runner "${runner.name}" cannot attach images, but scenario "${evalCase.name}" has \`images\` — use the openai runner with a vision model`,
+      );
+    }
+  }
+  if (runner.capabilities.sandboxTools) return;
+  for (const evalCase of config.cases) {
+    if (effectiveTools(config, evalCase) !== "") {
+      throw new Error(
+        `runner "${runner.name}" is text-only, but scenario "${evalCase.name}" needs sandbox tools ` +
+          `(artifact mode, a command grader, or an explicit tools list) — use claude-p or make the scenario text-graded`,
+      );
+    }
+  }
+}
+
+export interface MeasureCaseSummary {
+  name: string;
+  kind: ScenarioKind;
+  result: ArmSummary;
+  /** Hash of the rendered system prompt this case measured. */
+  promptSha256: string;
+}
+
+export interface MeasureSummary {
+  name: string;
+  arm: ArmConfig;
+  productionModel?: string;
+  cases: MeasureCaseSummary[];
+  totalCostUsd: number;
+}
+
+export interface MeasureRunOptions {
+  config: CompareConfig;
+  runner: Runner;
+  onProgress?: (message: string) => void;
+}
+
+/**
+ * Characterizes ONE instruction set: per-case pass rates, no delta and no
+ * assertions. The "know the current survival rate before you change anything"
+ * half of prompt testing — faking it with identical compare arms produces
+ * nonsense verdicts from sampling noise.
+ */
+export async function runMeasure(options: MeasureRunOptions): Promise<MeasureSummary> {
+  const { config, runner, onProgress } = options;
+  validateRunnerSupport(config, runner);
+  assertJudgeGradersCalibrated(config);
+  const inline = config.delivery !== "install";
+  const basePrompt = assembleSystemPrompt(config.agent, inline ? config.baselineSkills : []);
+  // Same fail-before-any-paid-run guarantee as compare: render everything first.
+  const rendered = config.cases.map((evalCase) => {
+    const vars = mergedRenderVars(config, evalCase);
+    if (vars === undefined) return { evalCase, systemPrompt: basePrompt };
+    return {
+      evalCase: { ...evalCase, prompt: renderStrict(evalCase.prompt, vars, `scenario "${evalCase.name}" prompt`) },
+      systemPrompt: renderStrict(basePrompt, vars, `scenario "${evalCase.name}" system prompt`),
+    };
+  });
+
+  const cases: MeasureCaseSummary[] = [];
+  for (const { evalCase, systemPrompt } of rendered) {
+    onProgress?.(`scenario ${evalCase.name}`);
+    const result = await runArm(config, evalCase, "baseline", systemPrompt, runner, onProgress, "measure");
+    cases.push({ name: evalCase.name, kind: evalCase.kind, result, promptSha256: sha256(systemPrompt) });
+  }
+
+  return {
+    name: config.name,
+    arm: config.arms.baseline,
+    productionModel: config.productionModel,
+    cases,
+    totalCostUsd: cases.reduce((sum, caseSummary) => sum + caseSummary.result.totalCostUsd, 0),
+  };
+}
+
+export function formatMeasureSummary(summary: MeasureSummary): string {
+  const lines = [`promptdiff measure: ${summary.name} (${summary.arm.model} via ${summary.arm.runner})`];
+
+  for (const caseSummary of summary.cases) {
+    const result = caseSummary.result;
+    lines.push(
+      "",
+      caseSummary.name,
+      `  ${result.passes}/${result.totalRuns} pass (${formatPct(result.passRate)}) | $${result.totalCostUsd.toFixed(4)}`,
+    );
+    for (const run of result.runs) {
+      if (run.pass) continue;
+      lines.push(`  run ${run.run} failed: ${run.grade.message}`);
+      lines.push(...graderEvidence(run.grade));
+    }
+  }
+
+  lines.push("", `total cost: $${summary.totalCostUsd.toFixed(4)}`);
+  if (summary.productionModel !== undefined && summary.arm.model !== summary.productionModel) {
+    lines.push(
+      `warning: measured "${summary.arm.model}" but production model is "${summary.productionModel}" — this characterizes prompt logic, not production behavior`,
+    );
+  }
+  return lines.join("\n");
+}
+
+interface RenderedCase {
+  evalCase: EvalCaseConfig;
+  baselineSystem: string;
+  proposedSystem: string;
+}
+
+/** Applies merged render vars (scenario over top-level) to both arms' system prompts and the scenario prompt. */
+function renderCase(
+  evalCase: EvalCaseConfig,
+  baselinePrompt: string,
+  proposedPrompt: string,
+  config: CompareConfig,
+): RenderedCase {
+  const vars = mergedRenderVars(config, evalCase);
+  if (vars === undefined) {
+    return { evalCase, baselineSystem: baselinePrompt, proposedSystem: proposedPrompt };
+  }
+  return {
+    evalCase: { ...evalCase, prompt: renderStrict(evalCase.prompt, vars, `scenario "${evalCase.name}" prompt`) },
+    baselineSystem: renderStrict(baselinePrompt, vars, `scenario "${evalCase.name}" baseline system prompt`),
+    proposedSystem: renderStrict(proposedPrompt, vars, `scenario "${evalCase.name}" proposed system prompt`),
+  };
+}
+
+/** Undefined when neither level defines render — rendering (and its strictness) is opt-in. */
+function mergedRenderVars(config: CompareConfig, evalCase: EvalCaseConfig): RenderVars | undefined {
+  if (config.renderVars === undefined && evalCase.renderVars === undefined) return undefined;
+  return { ...config.renderVars, ...evalCase.renderVars };
+}
+
+/**
+ * Baseline runs through the opt-in cache when one is configured: a hit reuses
+ * the recorded ArmSummary (real graded runs, just not re-paid), a miss runs
+ * normally and records the result. Cache mechanics stay out of runArm.
+ */
+async function runBaselineArm(
+  config: CompareConfig,
+  evalCase: EvalCaseConfig,
+  systemPrompt: string,
+  runner: Runner,
+  cache: { dir: string } | undefined,
+  onProgress?: (message: string) => void,
+): Promise<ArmSummary> {
+  if (cache === undefined) {
+    return runArm(config, evalCase, "baseline", systemPrompt, runner, onProgress);
+  }
+  const key = buildCacheKey({
+    systemPrompt,
+    casePrompt: evalCase.prompt,
+    arm: config.arms.baseline,
+    runs: evalCase.runs ?? config.runs,
+    tools: effectiveTools(config, evalCase),
+    mode: effectiveMode(config, evalCase),
+    delivery: config.delivery,
+    grader: evalCase.grader,
+    images: evalCase.images,
+    seedDir: evalCase.seed ?? config.sandboxSeed,
+    baselineSkills: config.baselineSkills,
+  });
+  const hit = loadCachedArm(cache.dir, key);
+  if (hit !== undefined) {
+    onProgress?.(`  baseline: cache hit (${key.slice(0, 12)})`);
+    return { ...hit, cached: true };
+  }
+  const summary = await runArm(config, evalCase, "baseline", systemPrompt, runner, onProgress);
+  storeCachedArm(cache.dir, key, summary);
+  return summary;
+}
+
+async function runArm(
+  config: CompareConfig,
+  evalCase: EvalCaseConfig,
+  arm: "baseline" | "proposed",
+  systemPrompt: string,
+  runner: Runner,
+  onProgress?: (message: string) => void,
+  label: string = arm,
+): Promise<ArmSummary> {
+  const runs = evalCase.runs ?? config.runs;
+  const summaries: ArmRunSummary[] = [];
+  const armSkills = arm === "baseline" ? config.baselineSkills : config.proposedSkills;
+
+  for (let index = 0; index < runs; index += 1) {
+    const sandbox = prepareSandbox({
+      root: config.sandboxRoot,
+      seed: evalCase.seed ?? config.sandboxSeed,
+      prefix: `${evalCase.name}-${label}-${index + 1}`,
+      keep: config.keepSandbox,
+    });
+
+    try {
+      onProgress?.(`  ${label} run ${index + 1}/${runs}`);
+      if (config.delivery === "install") {
+        const { installed, warnings } = installSkills(armSkills, sandbox.dir);
+        if (index === 0) {
+          onProgress?.(`  ${label} installed skills: ${installed.map((skill) => skill.name).join(", ")}`);
+          for (const warning of warnings) {
+            onProgress?.(`  WARNING: ${warning}`);
+          }
+        }
+      }
+      const run = await runner.run(buildRunnerOptions(config, evalCase, arm, systemPrompt, sandbox.dir));
+      const grade = await gradeRun(evalCase.grader, {
+        run,
+        sandboxDir: sandbox.dir,
+        timeoutMs: config.timeoutMs,
+        maxBudgetUsd: config.maxBudgetUsd,
+      });
+      summaries.push(toArmRunSummary(index + 1, run, grade, sandbox.dir));
+    } finally {
+      sandbox.cleanup();
+    }
+  }
+
+  const passes = summaries.filter((summary) => summary.pass).length;
+  const totalCostUsd = summaries.reduce((sum, summary) => sum + summary.costUsd, 0);
+  return {
+    name: arm,
+    passes,
+    totalRuns: runs,
+    passRate: passes / runs,
+    totalCostUsd,
+    runs: summaries,
+  };
+}
+
+function buildRunnerOptions(
+  config: CompareConfig,
+  evalCase: EvalCaseConfig,
+  arm: "baseline" | "proposed",
+  systemPrompt: string,
+  cwd: string,
+): RunnerRunOptions {
+  return {
+    systemPrompt,
+    systemPromptMode: config.delivery === "install" ? "append" : "replace",
+    userPrompt: evalCase.prompt,
+    images: evalCase.images,
+    requestParams: config.requestParams,
+    model: config.arms[arm].model,
+    cwd,
+    addDirs: [...config.addDirs, ...evalCase.addDirs],
+    tools: effectiveTools(config, evalCase),
+    timeoutMs: config.timeoutMs,
+    maxBudgetUsd: config.maxBudgetUsd,
+  };
+}
+
+function effectiveTools(config: CompareConfig, evalCase: EvalCaseConfig): string {
+  // Install delivery always needs tools (the Skill tool does the triggering),
+  // even when the grader is text-only and inline delivery would disable them.
+  const fallbackTools = config.delivery === "install" ? "default" : defaultTools(effectiveMode(config, evalCase));
+  return evalCase.tools ?? config.tools ?? fallbackTools;
+}
+
+function effectiveMode(config: CompareConfig, evalCase: EvalCaseConfig): "text" | "artifact" {
+  return evalCase.mode ?? config.mode ?? inferMode(evalCase);
+}
+
+function toArmRunSummary(
+  runNumber: number,
+  run: RunResult,
+  grade: GradeResult,
+  sandboxDir: string,
+): ArmRunSummary {
+  return {
+    run: runNumber,
+    pass: grade.pass,
+    grade,
+    // Judge graders bill their own model call; totals stay honest only if it counts.
+    costUsd: run.costUsd + (grade.costUsd ?? 0),
+    turns: run.turns,
+    durationMs: run.durationMs,
+    sandboxDir,
+    output: run.output,
+  };
+}
+
+function evaluateAssertions(
+  evalCase: EvalCaseConfig,
+  baseline: ArmSummary,
+  proposed: ArmSummary,
+): string[] {
+  // "compare" scenarios report pass rates and the delta with no directional claim.
+  if (evalCase.kind === "compare") {
+    return [];
+  }
+
+  if (evalCase.kind === "target") {
+    const failures = [];
+    if (baseline.passRate >= 1) {
+      failures.push("baseline fully passed; the target gap was not reproduced");
+    }
+    if (proposed.passRate <= baseline.passRate) {
+      failures.push("proposed did not improve the target pass rate");
+    }
+    return failures;
+  }
+
+  if (proposed.passRate < baseline.passRate) {
+    return ["proposed regressed below baseline pass rate"];
+  }
+  return [];
+}
+
+function inferMode(evalCase: EvalCaseConfig): "text" | "artifact" {
+  // text, json, and judge graders inspect the run's output only, so they
+  // demand no sandbox tools and stay valid on text-only runners (openai).
+  return evalCase.grader.type === "command" ? "artifact" : "text";
+}
+
+function defaultTools(mode: "text" | "artifact"): string {
+  return mode === "text" ? "" : "default";
+}
+
+function formatPct(value: number): string {
+  return `${(value * 100).toFixed(0)}%`;
+}