haansi 0.1.54 → 0.1.56

package/dist/haansi.js

@@ -95,7 +95,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "haansi",
-      version: "0.1.54",
+      version: "0.1.56",
       description: "Haansi CLI - Session collector and MCP server for Claude Code & OpenAI Codex",
       bin: {
         haansi: "./dist/haansi.js"
@@ -19220,6 +19220,16 @@ var init_exclusions = __esm({
         model: ["operational", "The model name the replay ran with."],
         run_ref: ["not-personal", "A link to the customer's CI run."]
       }),
+      ...exchangeExclusions(INTEL, "replay_task_results", {
+        inputs_fingerprint: [
+          "derived",
+          "A SHA-256 of the task's inputs the replay ran on; not reversible."
+        ],
+        arm: ["operational", "Closed CHECK vocabulary (baseline, haansi)."],
+        agent: ["operational", "The agent CLI's name, e.g. claude-code."],
+        model: ["operational", "The model name the replay ran with."],
+        run_ref: ["not-personal", "A link to the customer's CI run."]
+      }),
       ...exchangeExclusions(INTEL, "replay_tasks", {
         process_id: [
           "join-key",
@@ -19236,6 +19246,77 @@ var init_exclusions = __esm({
           "Actions workflow names from CI config that passed on the fix."
         ]
       }),
+      ...exchangeExclusions(INTEL, "stored_replay_tasks", {
+        task_key: [
+          "join-key",
+          "The task's kind with the workstream id or owner/repo#PR numbers it was cut from; upserted by equality."
+        ],
+        kind: [
+          "operational",
+          "Closed CHECK vocabulary (full, milestone, recovery)."
+        ],
+        process_id: [
+          "join-key",
+          "The workstream's process_registry.process_id, joined by equality."
+        ],
+        repository: [
+          "join-key",
+          "owner/repo of the task's PRs, as in workstream_attempts."
+        ],
+        base_sha: ["not-personal", "A commit SHA."],
+        target_sha: ["not-personal", "A commit SHA."],
+        checks: [
+          "not-personal",
+          "Actions workflow names from CI config that passed on the target."
+        ],
+        score_parts: [
+          "not-personal",
+          "Counts and flags the replayability score rests on (files, areas, context records, gradable by tests or checks); no text."
+        ],
+        inputs_fingerprint: [
+          "derived",
+          "A SHA-256 of the task's inputs; not reversible."
+        ],
+        card_state: [
+          "operational",
+          "Closed CHECK vocabulary (ready, dropped, invalid)."
+        ],
+        card_dropped_reason: [
+          "operational",
+          "Closed CHECK vocabulary (leak, insufficient)."
+        ],
+        card_fingerprint: [
+          "derived",
+          "A SHA-256 of what the card was written from; not reversible."
+        ],
+        card_model: ["not-personal", "The model id that wrote the card."],
+        card_prompt_tag: [
+          "not-personal",
+          "The prompt version that wrote the card (replay-task-card-vN)."
+        ],
+        calibration_state: [
+          "operational",
+          "Closed CHECK vocabulary (valid, invalid, flaky, error)."
+        ],
+        calibration_reason: [
+          "operational",
+          "Closed CHECK vocabulary (passes_at_base, fails_at_target, \u2026)."
+        ],
+        calibration_inputs_fingerprint: [
+          "derived",
+          "A SHA-256 of the task's inputs; not reversible."
+        ],
+        calibration_env_fingerprint: [
+          "derived",
+          "A SHA-256 of the CI environment; not reversible."
+        ],
+        calibration_environment: [
+          "not-personal",
+          "The CI's setup and test commands an org admin or CI owner typed, runtime versions, the runner image and the CLI version."
+        ],
+        calibration_runs: ["operational", "Exit codes of the test runs."],
+        calibration_run_ref: ["not-personal", "A link to the customer's CI run."]
+      }),
       ...exchangeExclusions(INTEL, "problem_family_members", {
         process_id: [
           "join-key",
@@ -26495,6 +26576,28 @@ var init_table_registry = __esm({
       entry("intelligence", INTELLIGENCE_DB, "replay_tasks", "direct", {
         scopingColumn: "organization_id"
       }),
+      // Stored replay tasks proposed by the ledger (replay tasks v2 plan §4).
+      entry("intelligence", INTELLIGENCE_DB, "stored_replay_tasks", "direct", {
+        scopingColumn: "organization_id"
+      }),
+      entry("intelligence", INTELLIGENCE_DB, "replay_task_results", "direct", {
+        scopingColumn: "organization_id"
+      }),
+      // A person's revert verdicts, the checks that validate a change and the
+      // replay commands, per repository (outcome evidence v2 plan).
+      entry("intelligence", INTELLIGENCE_DB, "revert_labels", "direct", {
+        scopingColumn: "organization_id"
+      }),
+      entry("intelligence", INTELLIGENCE_DB, "repository_checks", "direct", {
+        scopingColumn: "organization_id"
+      }),
+      entry(
+        "intelligence",
+        INTELLIGENCE_DB,
+        "repository_replay_settings",
+        "direct",
+        { scopingColumn: "organization_id" }
+      ),
       entry("intelligence", INTELLIGENCE_DB, "problem_family_feedback", "direct", {
         scopingColumn: "organization_id"
       }),
@@ -77377,7 +77480,7 @@ var require_view = __commonJS({
     var dirname5 = path2.dirname;
     var basename4 = path2.basename;
     var extname = path2.extname;
-    var join16 = path2.join;
+    var join17 = path2.join;
     var resolve5 = path2.resolve;
     module2.exports = View;
     function View(name, options) {
@@ -77439,12 +77542,12 @@ var require_view = __commonJS({
     };
     View.prototype.resolve = function resolve6(dir, file2) {
       var ext = this.ext;
-      var path3 = join16(dir, file2);
+      var path3 = join17(dir, file2);
       var stat = tryStat(path3);
       if (stat && stat.isFile()) {
         return path3;
       }
-      path3 = join16(dir, basename4(file2, ext), "index" + ext);
+      path3 = join17(dir, basename4(file2, ext), "index" + ext);
       stat = tryStat(path3);
       if (stat && stat.isFile()) {
         return path3;
@@ -81089,7 +81192,7 @@ var require_send = __commonJS({
     var Stream = require("stream");
     var util2 = require("util");
     var extname = path2.extname;
-    var join16 = path2.join;
+    var join17 = path2.join;
     var normalize = path2.normalize;
     var resolve5 = path2.resolve;
     var sep2 = path2.sep;
@@ -81261,7 +81364,7 @@ var require_send = __commonJS({
           return res;
         }
         parts = path3.split(sep2);
-        path3 = normalize(join16(root, path3));
+        path3 = normalize(join17(root, path3));
       } else {
         if (UP_PATH_REGEXP.test(path3)) {
           debug('malicious path "%s"', path3);
@@ -81394,7 +81497,7 @@ var require_send = __commonJS({
           if (err2) return self.onStatError(err2);
           return self.error(404);
         }
-        var p = join16(path3, self._index[i]);
+        var p = join17(path3, self._index[i]);
         debug('stat "%s"', p);
         fs.stat(p, function(err3, stat) {
           if (err3) return next(err3);
@@ -85105,10 +85208,540 @@ var init_skills = __esm({
   }
 });
 
+// src/commands/replay-calibrate.ts
+var replay_calibrate_exports = {};
+__export(replay_calibrate_exports, {
+  RUNTIMES: () => RUNTIMES,
+  calibrate: () => calibrate,
+  calibrateTask: () => calibrateTask,
+  calibrateTaskOptions: () => calibrateTaskOptions,
+  cliVersion: () => cliVersion,
+  environmentOf: () => environmentOf,
+  parseCalibrateArgs: () => parseCalibrateArgs,
+  repositoryOf: () => repositoryOf,
+  runCalibration: () => runCalibration,
+  runnerOf: () => runnerOf
+});
+function parseCalibrateArgs(argv) {
+  const options = {
+    setupCmd: null,
+    testCmd: "",
+    repoCommands: false,
+    runs: 2,
+    force: false,
+    limit: null,
+    timeoutMs: 30 * 6e4,
+    runRef: null,
+    dryRun: false
+  };
+  const errors = [];
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    const next = () => argv[++i] ?? "";
+    if (a === "--test-cmd") options.testCmd = next();
+    else if (a === "--setup-cmd") options.setupCmd = next() || null;
+    else if (a === "--repo-commands") options.repoCommands = true;
+    else if (a === "--runs") options.runs = Number(next());
+    else if (a === "--force") options.force = true;
+    else if (a === "--limit") options.limit = Number(next());
+    else if (a === "--timeout-min") options.timeoutMs = Number(next()) * 6e4;
+    else if (a === "--run-ref") options.runRef = next();
+    else if (a === "--dry-run") options.dryRun = true;
+    else errors.push(`Unknown option: ${a}`);
+  }
+  if (!options.testCmd && !options.repoCommands)
+    errors.push(
+      "--test-cmd is required (or --repo-commands, to use the repository's from Haansi)."
+    );
+  if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 5)
+    errors.push("--runs must be 1 to 5.");
+  if (options.limit !== null && (!Number.isInteger(options.limit) || options.limit < 1))
+    errors.push("--limit must be a positive whole number.");
+  if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0)
+    errors.push("--timeout-min must be a positive number.");
+  return { options, errors };
+}
+function calibrateTaskOptions(run3, commands) {
+  return {
+    setupCmd: commands.setupCmd,
+    testCmd: commands.testCmd,
+    repoCommands: false,
+    runs: 2,
+    force: false,
+    limit: run3.limit,
+    timeoutMs: run3.timeoutMs,
+    runRef: run3.runRef,
+    dryRun: run3.dryRun
+  };
+}
+function runnerOf(env) {
+  if (env.ImageOS && env.ImageVersion)
+    return `${env.ImageOS}-${env.ImageVersion}`;
+  if (env.RUNNER_OS)
+    return [env.RUNNER_OS, env.RUNNER_ARCH].filter(Boolean).join("/");
+  return null;
+}
+async function environmentOf(deps, commands) {
+  const runtimes = {};
+  for (const name of RUNTIMES) {
+    const out = await deps.shell(
+      `command -v ${name} >/dev/null 2>&1 && ${name} --version 2>&1 | head -n 1`,
+      deps.repoDir,
+      {},
+      3e4
+    );
+    const version3 = out.output.trim().split("\n")[0]?.trim();
+    if (out.code === 0 && version3) runtimes[name] = version3.slice(0, 200);
+  }
+  const described = {
+    setupCommand: commands.setupCmd,
+    testCommand: commands.testCmd,
+    runtimes,
+    runner: runnerOf(deps.env),
+    cliVersion: deps.cliVersion
+  };
+  return {
+    fingerprint: (0, import_node_crypto2.createHash)("sha256").update(JSON.stringify(described)).digest("hex"),
+    ...described
+  };
+}
+function verdict(codes) {
+  if (codes.every((c) => c === 0)) return "pass";
+  if (codes.every((c) => c !== 0)) return "fail";
+  return "flaky";
+}
+async function calibrateTask(deps, options, commands, task) {
+  const label = task.name ?? task.taskKey;
+  const runs = { base: [], target: [] };
+  const done = (reason) => ({
+    taskId: task.taskId,
+    inputsFingerprint: task.inputsFingerprint,
+    state: STATE[reason],
+    reason,
+    runs
+  });
+  for (const side of ["base", "target"]) {
+    const dir = (0, import_node_path9.join)(
+      deps.workDir(),
+      `${task.taskId.replace(/[^a-zA-Z0-9-]/g, "")}-${side}`
+    );
+    try {
+      try {
+        await cleanClone(
+          deps,
+          dir,
+          side === "base" ? task.baseSha : task.targetSha,
+          side === "base" ? task.targetSha : null
+        );
+      } catch (err2) {
+        deps.log(
+          `\u2026 ${label}: ${err2 instanceof Error ? err2.message : String(err2)}`
+        );
+        return done("checkout_failed");
+      }
+      if (commands.setupCmd) {
+        const setup = await deps.shell(
+          commands.setupCmd,
+          dir,
+          {},
+          options.timeoutMs
+        );
+        if (setup.code !== 0) {
+          deps.log(
+            `\u2026 ${label} [${side}]: setup failed: ${setup.output.trim().split("\n").slice(-3).join(" | ")}`
+          );
+          return done("setup_failed");
+        }
+      }
+      if (side === "base") {
+        for (const f of task.testFiles) {
+          const content = await deps.showFile(task.targetSha, f);
+          if (content === null) deps.removeFile((0, import_node_path9.join)(dir, f));
+          else deps.writeFile((0, import_node_path9.join)(dir, f), content);
+        }
+      }
+      for (let i = 0; i < options.runs; i++) {
+        const test = await deps.shell(
+          commands.testCmd,
+          dir,
+          { HAANSI_REPLAY_TEST_FILES: task.testFiles.join(" ") },
+          options.timeoutMs
+        );
+        runs[side].push(test.code);
+        if (side === "target" && test.code !== 0)
+          for (const line of test.output.trim().split("\n").slice(-FAILURE_LINES))
+            deps.log(`    ${line}`);
+      }
+    } finally {
+      deps.removeDir(dir);
+    }
+    const seen = verdict(runs[side]);
+    if (side === "base") {
+      if (seen === "flaky") return done("flaky_at_base");
+      if (seen === "pass") return done("passes_at_base");
+    } else {
+      if (seen === "flaky") return done("flaky_at_target");
+      if (seen === "fail") return done("fails_at_target");
+    }
+  }
+  return done("fails_at_base_passes_at_target");
+}
+async function repositoryOf(deps) {
+  const remote = await deps.git(["remote", "get-url", "origin"], deps.repoDir);
+  const repository = remote.code === 0 ? repositoryFromRemote(remote.output) : null;
+  if (!repository)
+    throw new Error("Run from a clone of the repository (no origin remote).");
+  return repository;
+}
+async function runCalibration(options, deps, preloaded) {
+  const repository = await repositoryOf(deps);
+  const set2 = preloaded ?? await deps.getReplaySet(repository);
+  const repo = options.repoCommands ? set2.repoCommands : null;
+  const setupCmd = options.setupCmd ?? repo?.setupCommand ?? null;
+  const testCmd = options.testCmd || repo?.testCommand || "";
+  if (!testCmd)
+    throw new Error(
+      `No test command: pass --test-cmd, or set one for ${repository} in Haansi.`
+    );
+  const environment = await environmentOf(deps, { setupCmd, testCmd });
+  const calibrations = [];
+  const states = /* @__PURE__ */ new Map();
+  let reused = 0;
+  let skipped = 0;
+  const tasks = options.limit ? set2.tasks.slice(0, options.limit) : set2.tasks;
+  for (const task of tasks) {
+    const label = task.name ?? task.taskKey;
+    if (!SHA.test(task.baseSha) || !SHA.test(task.targetSha)) {
+      skipped += 1;
+      deps.log(`skip ${label}: its commits are not commit ids`);
+      continue;
+    }
+    if (task.testFiles.length === 0) {
+      skipped += 1;
+      deps.log(
+        `skip ${label}: graded by checks (calibrating by checks comes later)`
+      );
+      continue;
+    }
+    if (!task.testFiles.every(safeTestFile)) {
+      skipped += 1;
+      deps.log(`skip ${label}: a test path is not usable`);
+      continue;
+    }
+    const c = task.calibration;
+    if (!options.force && c && c.inputsFingerprint === task.inputsFingerprint && c.envFingerprint === environment.fingerprint) {
+      reused += 1;
+      states.set(task.taskId, c.state);
+      deps.log(`= ${c.state.padEnd(7)} ${label} (calibrated already)`);
+      continue;
+    }
+    if (!await ensureCommit(deps, task.baseSha) || !await ensureCommit(deps, task.targetSha)) {
+      skipped += 1;
+      deps.log(`skip ${label}: its commits are not reachable from origin`);
+      continue;
+    }
+    const result = await calibrateTask(
+      deps,
+      options,
+      { setupCmd, testCmd },
+      task
+    );
+    calibrations.push(result);
+    states.set(task.taskId, result.state);
+    deps.log(
+      `${result.state.padEnd(7)} ${label}: ${result.reason.replace(/_/g, " ")} (base ${result.runs.base.join(",") || "-"}; target ${result.runs.target.join(",") || "-"})`
+    );
+  }
+  if (options.dryRun || calibrations.length === 0)
+    return { calibrations, reused, skipped, recorded: null, states };
+  let recorded = 0;
+  for (let i = 0; i < calibrations.length; i += REPORT_CHUNK) {
+    const out = await deps.report({
+      runRef: options.runRef,
+      environment,
+      results: calibrations.slice(i, i + REPORT_CHUNK)
+    });
+    recorded += out.recorded;
+    if (out.refused.length)
+      deps.log(
+        `Haansi did not record ${out.refused.length} calibration(s): their task changed since it was listed.`
+      );
+  }
+  return { calibrations, reused, skipped, recorded, states };
+}
+async function calibrate(argv) {
+  if (argv.includes("--help")) {
+    console.log(USAGE);
+    return;
+  }
+  const { options, errors } = parseCalibrateArgs(argv);
+  if (errors.length) {
+    for (const e of errors) console.error(e);
+    console.log(USAGE);
+    process.exitCode = 1;
+    return;
+  }
+  options.runRef ??= githubRunRef(process.env);
+  const token = resolveToken5();
+  const apiUrl2 = apiUrl();
+  const work = (0, import_node_fs10.mkdtempSync)((0, import_node_path9.join)((0, import_node_os9.tmpdir)(), "haansi-calibrate-"));
+  try {
+    const { calibrations, reused, skipped, recorded } = await runCalibration(
+      options,
+      {
+        ...localDeps(work),
+        getReplaySet: (repository) => apiGet3(
+          `/memory/replays/tasks?repository=${encodeURIComponent(repository)}`,
+          token,
+          apiUrl2
+        ),
+        report: async (body) => await apiPost2(
+          "/memory/replays/tasks/calibration",
+          body,
+          token,
+          apiUrl2
+        ),
+        env: process.env,
+        cliVersion: cliVersion()
+      }
+    );
+    const count = (state) => calibrations.filter((c) => c.state === state).length;
+    console.log(
+      `
+Valid: ${count("valid")} \xB7 Invalid: ${count("invalid")} \xB7 Flaky: ${count("flaky")} \xB7 Errors: ${count("error")} \xB7 ${reused} already calibrated \xB7 ${skipped} skipped`
+    );
+    console.log(
+      recorded === null ? options.dryRun ? "Dry run: nothing reported." : "Nothing to report." : `Reported ${recorded} calibration(s) to Haansi.`
+    );
+  } catch (err2) {
+    console.error(err2 instanceof Error ? err2.message : String(err2));
+    process.exitCode = 1;
+  } finally {
+    (0, import_node_fs10.rmSync)(work, { recursive: true, force: true });
+  }
+}
+function cliVersion() {
+  try {
+    return String(require_package().version);
+  } catch {
+    return null;
+  }
+}
+var import_node_crypto2, import_node_fs10, import_node_os9, import_node_path9, RUNTIMES, REPORT_CHUNK, FAILURE_LINES, STATE, USAGE;
+var init_replay_calibrate = __esm({
+  "src/commands/replay-calibrate.ts"() {
+    "use strict";
+    import_node_crypto2 = require("node:crypto");
+    import_node_fs10 = require("node:fs");
+    import_node_os9 = require("node:os");
+    import_node_path9 = require("node:path");
+    init_paths();
+    init_api2();
+    init_replay();
+    RUNTIMES = ["node", "python3", "go", "java", "ruby", "rustc"];
+    REPORT_CHUNK = 100;
+    FAILURE_LINES = 5;
+    STATE = {
+      fails_at_base_passes_at_target: "valid",
+      passes_at_base: "invalid",
+      fails_at_target: "invalid",
+      flaky_at_base: "flaky",
+      flaky_at_target: "flaky",
+      setup_failed: "error",
+      checkout_failed: "error"
+    };
+    USAGE = `Usage: haansi replay calibrate --test-cmd "<cmd>" [--setup-cmd "<cmd>"] [--repo-commands]
+                               [--runs 2] [--force] [--limit n] [--timeout-min 30]
+                               [--run-ref <url>] [--dry-run]
+
+Checks each task of this repository's replay set with no agent: the tests of
+its target must fail at its base and pass at its target, --runs times each.
+Already calibrated tasks are not run again until their inputs or the
+environment (commands, runtimes, runner, CLI version) change; --force runs
+them all.`;
+  }
+});
+
+// src/commands/replay-tasks.ts
+var replay_tasks_exports = {};
+__export(replay_tasks_exports, {
+  areaOf: () => areaOf,
+  cardPrompt: () => cardPrompt,
+  changedFiles: () => changedFiles,
+  overlapOf: () => overlapOf,
+  runStoredReplays: () => runStoredReplays
+});
+function cardPrompt(task, feedback) {
+  return [
+    "# Task",
+    "",
+    task.card.problem,
+    "",
+    "## Done when",
+    "",
+    task.card.acceptance,
+    ...task.card.area ? ["", "## Part of the code", "", task.card.area] : [],
+    "",
+    ...feedback ? [FEEDBACK_HEADER, "", "```", feedback, "```", ""] : [],
+    "Work in this repository as it is now. Do not look at later commits or other branches."
+  ].join("\n");
+}
+function areaOf(path2) {
+  const dirs = path2.split("/").slice(0, -1);
+  return dirs.length ? dirs.slice(0, 2).join("/") : ".";
+}
+async function changedFiles(deps, dir, baseSha) {
+  const lines = (out) => out.code === 0 ? out.output.split("\n").map((l) => l.trim()).filter(Boolean) : [];
+  const diff = await deps.git(
+    ["diff", "--name-only", "--no-renames", baseSha],
+    dir
+  );
+  const added = await deps.git(
+    ["ls-files", "--others", "--exclude-standard"],
+    dir
+  );
+  return [.../* @__PURE__ */ new Set([...lines(diff), ...lines(added)])].filter((f) => !f.startsWith(".haansi-replay/")).sort();
+}
+function overlapOf(changed, task) {
+  const tests = new Set(task.testFiles);
+  const mine = new Set(changed.filter((f) => !tests.has(f)));
+  const theirs = new Set(task.targetFiles.filter((f) => !tests.has(f)));
+  const both = [...mine].filter((f) => theirs.has(f)).length;
+  const myAreas = new Set([...mine].map(areaOf));
+  const theirAreas = new Set([...theirs].map(areaOf));
+  const bothAreas = [...myAreas].filter((a) => theirAreas.has(a)).length;
+  const known = mine.size > 0 && theirs.size > 0;
+  return {
+    filePrecision: known ? share(both, mine.size) : null,
+    fileRecall: known ? share(both, theirs.size) : null,
+    areaPrecision: known ? share(bothAreas, myAreas.size) : null,
+    areaRecall: known ? share(bothAreas, theirAreas.size) : null,
+    changedFiles: mine.size
+  };
+}
+async function runStoredReplays(options, deps) {
+  const repository = await repositoryOf(deps);
+  const set2 = await deps.getReplaySet(repository);
+  const repo = options.repoCommands ? set2.repoCommands : null;
+  const setupCmd = options.setupCmd ?? repo?.setupCommand ?? null;
+  const testCmd = options.testCmd || repo?.testCommand || "";
+  if (!testCmd)
+    throw new Error(
+      `No test command: pass --test-cmd, or set one for ${repository} in Haansi.`
+    );
+  if (set2.tasks.length === 0) {
+    deps.log(
+      `No replay tasks for ${repository} yet: they need a card (replay task cards on for the organization).`
+    );
+    return { results: [], notValid: 0, recorded: null };
+  }
+  const { states } = await runCalibration(
+    calibrateTaskOptions(options, { setupCmd, testCmd }),
+    { ...deps, report: deps.reportCalibration },
+    set2
+  );
+  const valid = set2.tasks.filter(
+    (t) => states.get(t.taskId) === "valid" && t.testFiles.length > 0 && t.testFiles.every(safeTestFile) && SHA.test(t.baseSha) && SHA.test(t.targetSha)
+  );
+  const notValid = set2.tasks.length - valid.length;
+  let arms = options.arms;
+  if (arms.includes("haansi")) {
+    deps.log(
+      "The with-Haansi arm on stored tasks needs Haansi answering as of each task's start; running the baseline arm only."
+    );
+    arms = arms.filter((a) => a !== "haansi");
+  }
+  const chosen = options.limit ? valid.slice(0, options.limit) : valid;
+  const results = [];
+  for (const task of chosen) {
+    const label = task.name ?? task.taskKey;
+    const asTask = {
+      taskId: task.taskId,
+      // Names the clone's directory.
+      processId: task.taskId,
+      name: label,
+      repository: task.repository,
+      baseSha: task.baseSha,
+      targetPr: task.targetPr,
+      targetSha: task.targetSha,
+      testFiles: task.testFiles,
+      context: []
+    };
+    for (const arm of arms) {
+      let changed = [];
+      try {
+        const r = await replayArm(
+          deps,
+          { ...options, setupCmd, testCmd },
+          asTask,
+          arm,
+          null,
+          (feedback) => cardPrompt(task, feedback),
+          async (dir) => {
+            changed = await changedFiles(deps, dir, task.baseSha);
+          }
+        );
+        const overlap = overlapOf(changed, task);
+        results.push({
+          taskId: task.taskId,
+          cardVersion: task.card.version,
+          inputsFingerprint: task.inputsFingerprint,
+          arm,
+          passed: r.passed,
+          attempts: r.attempts,
+          maxAttempts: r.maxAttempts,
+          durationMs: r.durationMs,
+          overlap
+        });
+        deps.log(
+          `${r.passed ? "pass" : "fail"} ${arm.padEnd(8)} ${label} (${r.attempts}/${r.maxAttempts} attempts; files ${pct(overlap.fileRecall)} of the target's, ${pct(overlap.filePrecision)} of its own on target)`
+        );
+      } catch (err2) {
+        deps.log(
+          `error ${arm} ${label}: ${err2 instanceof Error ? err2.message : String(err2)}`
+        );
+      }
+    }
+  }
+  if (options.dryRun || results.length === 0)
+    return { results, notValid, recorded: null };
+  let recorded = 0;
+  for (let i = 0; i < results.length; i += REPORT_CHUNK2) {
+    const out = await deps.reportResults({
+      agent: options.agentName,
+      model: options.model,
+      runRef: options.runRef,
+      results: results.slice(i, i + REPORT_CHUNK2)
+    });
+    recorded += out.recorded;
+    if (out.refused.length)
+      deps.log(
+        `Haansi did not record ${out.refused.length} result(s): their task's card or inputs changed during the run.`
+      );
+  }
+  return { results, notValid, recorded };
+}
+var FEEDBACK_HEADER, REPORT_CHUNK2, share, pct;
+var init_replay_tasks = __esm({
+  "src/commands/replay-tasks.ts"() {
+    "use strict";
+    init_replay_calibrate();
+    init_replay();
+    FEEDBACK_HEADER = "Your previous attempt did not pass the tests. Their output ends with:";
+    REPORT_CHUNK2 = 200;
+    share = (part, whole) => whole === 0 ? null : Math.round(part / whole * 1e3) / 1e3;
+    pct = (v) => v === null ? "n/a" : `${Math.round(v * 100)}%`;
+  }
+});
+
 // src/commands/replay.ts
 var replay_exports = {};
 __export(replay_exports, {
+  SHA: () => SHA,
+  cleanClone: () => cleanClone,
+  ensureCommit: () => ensureCommit,
   githubRunRef: () => githubRunRef,
+  localDeps: () => localDeps,
   parseReplayArgs: () => parseReplayArgs,
   replay: () => replay,
   replayArm: () => replayArm,
@@ -85160,8 +85793,8 @@ function parseReplayArgs(argv) {
     else if (!sub && !a.startsWith("-")) sub = a;
     else errors.push(`Unknown option: ${a}`);
   }
-  if (!/^[0-9a-f-]{36}$/i.test(options.familyId))
-    errors.push("--family <id> is required (the family's id).");
+  if (options.familyId && !/^[0-9a-f-]{36}$/i.test(options.familyId))
+    errors.push("--family must be a family's id.");
   if (!options.agentCmd) errors.push("--agent-cmd is required.");
   if (!options.testCmd && !options.repoCommands)
     errors.push(
@@ -85238,7 +85871,7 @@ async function ensureCommit(deps, sha) {
     return true;
   return (await deps.git(["fetch", "--quiet", "origin", sha], deps.repoDir)).code === 0;
 }
-async function cleanCheckout(deps, dir, task) {
+async function cleanClone(deps, dir, sha, unreachable) {
   const steps = [
     [["init", "--quiet", dir], deps.repoDir],
     [
@@ -85248,7 +85881,7 @@ async function cleanCheckout(deps, dir, task) {
         "--no-tags",
         "--no-write-fetch-head",
         deps.repoDir,
-        `${task.baseSha}:refs/heads/${REPLAY_BRANCH}`
+        `${sha}:refs/heads/${REPLAY_BRANCH}`
       ],
       dir
     ],
@@ -85256,18 +85889,22 @@ async function cleanCheckout(deps, dir, task) {
   ];
   for (const [args, cwd] of steps) {
     if ((await deps.git(args, cwd)).code !== 0)
-      throw new Error(`could not check out ${task.baseSha}`);
+      throw new Error(`could not check out ${sha}`);
   }
+  if (unreachable === null) return;
   const leak = await deps.git(
-    ["cat-file", "-e", `${task.targetSha}^{commit}`],
+    ["cat-file", "-e", `${unreachable}^{commit}`],
     dir
   );
   if (leak.code === 0)
     throw new Error("the fix's commit is readable in the clone: run refused");
 }
-async function replayArm(deps, options, task, arm, playbook) {
+function cleanCheckout(deps, dir, task) {
+  return cleanClone(deps, dir, task.baseSha, task.targetSha);
+}
+async function replayArm(deps, options, task, arm, playbook, promptOf = (feedback) => replayPrompt(task, arm, feedback), inspect) {
   const started = deps.now();
-  const dir = (0, import_node_path9.join)(
+  const dir = (0, import_node_path10.join)(
     deps.workDir(),
     `${task.processId.replace(/[^a-zA-Z0-9_-]/g, "")}-${arm}`
   );
@@ -85293,15 +85930,15 @@ async function replayArm(deps, options, task, arm, playbook) {
       }
       deps.log(`\u2026 ${label}: set up in ${since(at)}`);
     }
-    const promptFile = (0, import_node_path9.join)(dir, ".haansi-replay", "prompt.md");
+    const promptFile = (0, import_node_path10.join)(dir, ".haansi-replay", "prompt.md");
     if (arm === "haansi" && playbook)
       deps.writeFile(
-        (0, import_node_path9.join)(dir, ".haansi-replay", "playbook.md"),
+        (0, import_node_path10.join)(dir, ".haansi-replay", "playbook.md"),
         playbook.skillMd
       );
     let feedback = null;
     for (let attempt = 1; attempt <= options.attempts; attempt++) {
-      const prompt2 = replayPrompt(task, arm, feedback);
+      const prompt2 = promptOf(feedback);
       deps.writeFile(promptFile, prompt2);
       const agentAt = deps.now();
       deps.log(
@@ -85313,11 +85950,11 @@ async function replayArm(deps, options, task, arm, playbook) {
         { HAANSI_REPLAY_PROMPT: prompt2, HAANSI_REPLAY_PROMPT_FILE: promptFile },
         options.timeoutMs
       );
-      const own = new Map(files.map((f) => [f, deps.readFile((0, import_node_path9.join)(dir, f))]));
+      const own = new Map(files.map((f) => [f, deps.readFile((0, import_node_path10.join)(dir, f))]));
       for (const f of files) {
         const content = await deps.showFile(task.targetSha, f);
-        if (content === null) deps.removeFile((0, import_node_path9.join)(dir, f));
-        else deps.writeFile((0, import_node_path9.join)(dir, f), content);
+        if (content === null) deps.removeFile((0, import_node_path10.join)(dir, f));
+        else deps.writeFile((0, import_node_path10.join)(dir, f), content);
       }
       const testAt = deps.now();
       deps.log(`\u2026 ${label}: agent done in ${since(agentAt)}, grading`);
@@ -85331,10 +85968,11 @@ async function replayArm(deps, options, task, arm, playbook) {
         `\u2026 ${label}: tests ${test.code === 0 ? "passed" : "failed"} in ${since(testAt)}`
       );
       if (test.code !== 0) {
-        for (const line of test.output.trim().split("\n").slice(-FAILURE_LINES))
+        for (const line of test.output.trim().split("\n").slice(-FAILURE_LINES2))
           deps.log(`    ${line}`);
       }
       if (test.code === 0) {
+        if (inspect) await inspect(dir);
         return {
           processId: task.processId,
           arm,
@@ -85346,11 +85984,12 @@ async function replayArm(deps, options, task, arm, playbook) {
         };
       }
       for (const [file2, content] of own) {
-        if (content === null) deps.removeFile((0, import_node_path9.join)(dir, file2));
-        else deps.writeFile((0, import_node_path9.join)(dir, file2), content);
+        if (content === null) deps.removeFile((0, import_node_path10.join)(dir, file2));
+        else deps.writeFile((0, import_node_path10.join)(dir, file2), content);
       }
       feedback = test.output.slice(-FEEDBACK_CHARS);
     }
+    if (inspect) await inspect(dir);
     return {
       processId: task.processId,
       arm,
@@ -85477,17 +86116,54 @@ function runCommand(file2, args, cwd, env, timeoutMs, stdoutOnly = false) {
     });
   });
 }
+function localDeps(work) {
+  return {
+    git: (args, cwd) => runCommand("git", args, cwd, {}, 10 * 6e4),
+    showFile: async (sha, path2) => {
+      const shown = await runCommand(
+        "git",
+        ["show", `${sha}:${path2}`],
+        process.cwd(),
+        {},
+        6e4,
+        true
+      );
+      return shown.code === 0 ? shown.output : null;
+    },
+    shell: (command2, cwd, env, timeoutMs) => runCommand("sh", ["-c", command2], cwd, env, timeoutMs),
+    repoDir: process.cwd(),
+    workDir: () => work,
+    writeFile: (path2, content) => {
+      (0, import_node_fs11.mkdirSync)((0, import_node_path10.dirname)(path2), { recursive: true });
+      (0, import_node_fs11.writeFileSync)(path2, content);
+    },
+    readFile: (path2) => (0, import_node_fs11.existsSync)(path2) ? (0, import_node_fs11.readFileSync)(path2, "utf-8") : null,
+    removeFile: (path2) => (0, import_node_fs11.rmSync)(path2, { force: true }),
+    removeDir: (path2) => (0, import_node_fs11.rmSync)(path2, { recursive: true, force: true }),
+    now: () => Date.now(),
+    log: (line) => console.log(line)
+  };
+}
 async function replay(argv) {
+  if (argv[0] === "calibrate") {
+    const { calibrate: calibrate2 } = await Promise.resolve().then(() => (init_replay_calibrate(), replay_calibrate_exports));
+    await calibrate2(argv.slice(1));
+    return;
+  }
   const { sub, options, errors } = parseReplayArgs(argv);
   if (sub !== "run" || errors.length) {
     for (const e of sub === "run" ? errors : []) console.error(e);
-    console.log(USAGE);
+    console.log(USAGE2);
     if (sub === "run") process.exitCode = 1;
     return;
   }
   const token = resolveToken5();
   const apiUrl2 = apiUrl();
-  const work = (0, import_node_fs10.mkdtempSync)((0, import_node_path9.join)((0, import_node_os9.tmpdir)(), "haansi-replay-"));
+  if (!options.familyId) {
+    await replayStored(options, token, apiUrl2);
+    return;
+  }
+  const work = (0, import_node_fs11.mkdtempSync)((0, import_node_path10.join)((0, import_node_os10.tmpdir)(), "haansi-replay-"));
   try {
     const { results, skipped, recorded } = await runReplays(options, {
       getTasks: async (familyId, cutoff) => (await apiGet3(
@@ -85506,30 +86182,7 @@ async function replay(argv) {
         token,
         apiUrl2
       ),
-      git: (args, cwd) => runCommand("git", args, cwd, {}, 10 * 6e4),
-      showFile: async (sha, path2) => {
-        const shown = await runCommand(
-          "git",
-          ["show", `${sha}:${path2}`],
-          process.cwd(),
-          {},
-          6e4,
-          true
-        );
-        return shown.code === 0 ? shown.output : null;
-      },
-      shell: (command2, cwd, env, timeoutMs) => runCommand("sh", ["-c", command2], cwd, env, timeoutMs),
-      repoDir: process.cwd(),
-      workDir: () => work,
-      writeFile: (path2, content) => {
-        (0, import_node_fs10.mkdirSync)((0, import_node_path9.dirname)(path2), { recursive: true });
-        (0, import_node_fs10.writeFileSync)(path2, content);
-      },
-      readFile: (path2) => (0, import_node_fs10.existsSync)(path2) ? (0, import_node_fs10.readFileSync)(path2, "utf-8") : null,
-      removeFile: (path2) => (0, import_node_fs10.rmSync)(path2, { force: true }),
-      removeDir: (path2) => (0, import_node_fs10.rmSync)(path2, { recursive: true, force: true }),
-      now: () => Date.now(),
-      log: (line) => console.log(line)
+      ...localDeps(work)
     });
     const passed = (arm) => results.filter((r) => r.arm === arm && r.passed).length;
     const ran = (arm) => results.filter((r) => r.arm === arm).length;
@@ -85541,30 +86194,79 @@ Baseline: ${passed("baseline")}/${ran("baseline")} passed \xB7 With Haansi: ${pa
       recorded === null ? options.dryRun ? "Dry run: nothing reported." : "Nothing to report." : `Reported ${recorded} result(s) to Haansi.`
     );
   } finally {
-    (0, import_node_fs10.rmSync)(work, { recursive: true, force: true });
+    (0, import_node_fs11.rmSync)(work, { recursive: true, force: true });
   }
 }
-var import_node_child_process4, import_node_fs10, import_node_os9, import_node_path9, SHA, ARMS, FEEDBACK_CHARS, FAILURE_LINES, REPLAY_BRANCH, USAGE;
+async function replayStored(options, token, apiUrl2) {
+  const { runStoredReplays: runStoredReplays2 } = await Promise.resolve().then(() => (init_replay_tasks(), replay_tasks_exports));
+  const { cliVersion: cliVersion2 } = await Promise.resolve().then(() => (init_replay_calibrate(), replay_calibrate_exports));
+  const work = (0, import_node_fs11.mkdtempSync)((0, import_node_path10.join)((0, import_node_os10.tmpdir)(), "haansi-replay-"));
+  try {
+    const { results, notValid, recorded } = await runStoredReplays2(options, {
+      ...localDeps(work),
+      getReplaySet: (repository) => apiGet3(
+        `/memory/replays/tasks?repository=${encodeURIComponent(repository)}`,
+        token,
+        apiUrl2
+      ),
+      reportCalibration: async (body) => await apiPost2(
+        "/memory/replays/tasks/calibration",
+        body,
+        token,
+        apiUrl2
+      ),
+      reportResults: async (body) => await apiPost2(
+        "/memory/replays/tasks/results",
+        body,
+        token,
+        apiUrl2
+      ),
+      env: process.env,
+      cliVersion: cliVersion2()
+    });
+    const passed = results.filter((r) => r.passed).length;
+    console.log(
+      `
+Baseline: ${passed}/${results.length} passed \xB7 ${notValid} task(s) not valid to replay`
+    );
+    console.log(
+      recorded === null ? options.dryRun ? "Dry run: nothing reported." : "Nothing to report." : `Reported ${recorded} result(s) to Haansi.`
+    );
+  } catch (err2) {
+    console.error(err2 instanceof Error ? err2.message : String(err2));
+    process.exitCode = 1;
+  } finally {
+    (0, import_node_fs11.rmSync)(work, { recursive: true, force: true });
+  }
+}
+var import_node_child_process4, import_node_fs11, import_node_os10, import_node_path10, SHA, ARMS, FEEDBACK_CHARS, FAILURE_LINES2, REPLAY_BRANCH, USAGE2;
 var init_replay = __esm({
   "src/commands/replay.ts"() {
     "use strict";
     import_node_child_process4 = require("node:child_process");
-    import_node_fs10 = require("node:fs");
-    import_node_os9 = require("node:os");
-    import_node_path9 = require("node:path");
+    import_node_fs11 = require("node:fs");
+    import_node_os10 = require("node:os");
+    import_node_path10 = require("node:path");
     init_paths();
     init_api2();
     SHA = /^[0-9a-f]{7,40}$/i;
     ARMS = ["baseline", "haansi"];
     FEEDBACK_CHARS = 6e3;
-    FAILURE_LINES = 8;
+    FAILURE_LINES2 = 8;
     REPLAY_BRANCH = "replay";
-    USAGE = `Usage: haansi replay run --family <id> --agent-cmd "<cmd>" --test-cmd "<cmd>"
+    USAGE2 = `Usage: haansi replay run [--family <id>] --agent-cmd "<cmd>" --test-cmd "<cmd>"
                          [--setup-cmd "<cmd>"] [--attempts 3] [--arms baseline,haansi] [--limit n]
                          [--cutoff <date>] [--timeout-min 30]
                          [--agent-name <name>] [--model <name>] [--run-ref <url>] [--dry-run]
                          [--repo-commands]
 
+       haansi replay calibrate --test-cmd "<cmd>" [--setup-cmd "<cmd>"] (see --help)
+
+Without --family, runs this repository's stored replay tasks: calibrates them
+first, then replays the valid ones with their task card as the prompt, graded
+by the target's tests and where the change landed. With --family, replays that
+problem family's tasks with and without its playbook.
+
 --repo-commands uses the setup and test commands an org admin set for the
 repository in Haansi (Settings \u2192 Data sources) for any not given here.
 
@@ -85790,9 +86492,13 @@ Commands:
                              digest. [--date YYYY-MM-DD] [--workstream <id>]
   digest [--week|--month|--since YYYY-MM-DD]
                              Your weekly digest (default: the latest week)
-  replay run --family <id> --agent-cmd "<cmd>" --test-cmd "<cmd>"
-                             Replay a problem family's tasks in your CI, with and
-                             without its playbook, and report the results
+  replay run --agent-cmd "<cmd>" --test-cmd "<cmd>" [--family <id>]
+                             Replay this repository's tasks in your CI (or a
+                             problem family's, with and without its playbook)
+                             and report the results
+  replay calibrate --test-cmd "<cmd>"
+                             Check this repository's replay tasks are valid: their
+                             tests fail at the start and pass at the fix
   skills sync [--agent <sel>] [--project] [--dir <path>]
                              Copy your team's published skills into Claude Code / Codex
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "haansi",
-  "version": "0.1.54",
+  "version": "0.1.56",
   "description": "Haansi CLI - Session collector and MCP server for Claude Code & OpenAI Codex",
   "bin": {
     "haansi": "./dist/haansi.js"