haansi 0.1.55 → 0.1.56

package/dist/haansi.js

@@ -95,7 +95,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "haansi",
-      version: "0.1.55",
+      version: "0.1.56",
       description: "Haansi CLI - Session collector and MCP server for Claude Code & OpenAI Codex",
       bin: {
         haansi: "./dist/haansi.js"
@@ -19220,6 +19220,16 @@ var init_exclusions = __esm({
         model: ["operational", "The model name the replay ran with."],
         run_ref: ["not-personal", "A link to the customer's CI run."]
       }),
+      ...exchangeExclusions(INTEL, "replay_task_results", {
+        inputs_fingerprint: [
+          "derived",
+          "A SHA-256 of the task's inputs the replay ran on; not reversible."
+        ],
+        arm: ["operational", "Closed CHECK vocabulary (baseline, haansi)."],
+        agent: ["operational", "The agent CLI's name, e.g. claude-code."],
+        model: ["operational", "The model name the replay ran with."],
+        run_ref: ["not-personal", "A link to the customer's CI run."]
+      }),
       ...exchangeExclusions(INTEL, "replay_tasks", {
         process_id: [
           "join-key",
@@ -26570,6 +26580,9 @@ var init_table_registry = __esm({
       entry("intelligence", INTELLIGENCE_DB, "stored_replay_tasks", "direct", {
         scopingColumn: "organization_id"
       }),
+      entry("intelligence", INTELLIGENCE_DB, "replay_task_results", "direct", {
+        scopingColumn: "organization_id"
+      }),
       // A person's revert verdicts, the checks that validate a change and the
       // replay commands, per repository (outcome evidence v2 plan).
       entry("intelligence", INTELLIGENCE_DB, "revert_labels", "direct", {
@@ -85201,8 +85214,11 @@ __export(replay_calibrate_exports, {
   RUNTIMES: () => RUNTIMES,
   calibrate: () => calibrate,
   calibrateTask: () => calibrateTask,
+  calibrateTaskOptions: () => calibrateTaskOptions,
+  cliVersion: () => cliVersion,
   environmentOf: () => environmentOf,
   parseCalibrateArgs: () => parseCalibrateArgs,
+  repositoryOf: () => repositoryOf,
   runCalibration: () => runCalibration,
   runnerOf: () => runnerOf
 });
@@ -85245,6 +85261,19 @@ function parseCalibrateArgs(argv) {
     errors.push("--timeout-min must be a positive number.");
   return { options, errors };
 }
+function calibrateTaskOptions(run3, commands) {
+  return {
+    setupCmd: commands.setupCmd,
+    testCmd: commands.testCmd,
+    repoCommands: false,
+    runs: 2,
+    force: false,
+    limit: run3.limit,
+    timeoutMs: run3.timeoutMs,
+    runRef: run3.runRef,
+    dryRun: run3.dryRun
+  };
+}
 function runnerOf(env) {
   if (env.ImageOS && env.ImageVersion)
     return `${env.ImageOS}-${env.ImageVersion}`;
@@ -85357,12 +85386,16 @@ async function calibrateTask(deps, options, commands, task) {
   }
   return done("fails_at_base_passes_at_target");
 }
-async function runCalibration(options, deps) {
+async function repositoryOf(deps) {
   const remote = await deps.git(["remote", "get-url", "origin"], deps.repoDir);
   const repository = remote.code === 0 ? repositoryFromRemote(remote.output) : null;
   if (!repository)
     throw new Error("Run from a clone of the repository (no origin remote).");
-  const set2 = await deps.getReplaySet(repository);
+  return repository;
+}
+async function runCalibration(options, deps, preloaded) {
+  const repository = await repositoryOf(deps);
+  const set2 = preloaded ?? await deps.getReplaySet(repository);
   const repo = options.repoCommands ? set2.repoCommands : null;
   const setupCmd = options.setupCmd ?? repo?.setupCommand ?? null;
   const testCmd = options.testCmd || repo?.testCommand || "";
@@ -85372,17 +85405,12 @@ async function runCalibration(options, deps) {
     );
   const environment = await environmentOf(deps, { setupCmd, testCmd });
   const calibrations = [];
+  const states = /* @__PURE__ */ new Map();
   let reused = 0;
   let skipped = 0;
   const tasks = options.limit ? set2.tasks.slice(0, options.limit) : set2.tasks;
   for (const task of tasks) {
     const label = task.name ?? task.taskKey;
-    const c = task.calibration;
-    if (!options.force && c && c.inputsFingerprint === task.inputsFingerprint && c.envFingerprint === environment.fingerprint) {
-      reused += 1;
-      deps.log(`= ${c.state.padEnd(7)} ${label} (calibrated already)`);
-      continue;
-    }
     if (!SHA.test(task.baseSha) || !SHA.test(task.targetSha)) {
       skipped += 1;
       deps.log(`skip ${label}: its commits are not commit ids`);
@@ -85400,6 +85428,13 @@ async function runCalibration(options, deps) {
       deps.log(`skip ${label}: a test path is not usable`);
       continue;
     }
+    const c = task.calibration;
+    if (!options.force && c && c.inputsFingerprint === task.inputsFingerprint && c.envFingerprint === environment.fingerprint) {
+      reused += 1;
+      states.set(task.taskId, c.state);
+      deps.log(`= ${c.state.padEnd(7)} ${label} (calibrated already)`);
+      continue;
+    }
     if (!await ensureCommit(deps, task.baseSha) || !await ensureCommit(deps, task.targetSha)) {
       skipped += 1;
       deps.log(`skip ${label}: its commits are not reachable from origin`);
@@ -85412,12 +85447,13 @@ async function runCalibration(options, deps) {
       task
     );
     calibrations.push(result);
+    states.set(task.taskId, result.state);
     deps.log(
       `${result.state.padEnd(7)} ${label}: ${result.reason.replace(/_/g, " ")} (base ${result.runs.base.join(",") || "-"}; target ${result.runs.target.join(",") || "-"})`
     );
   }
   if (options.dryRun || calibrations.length === 0)
-    return { calibrations, reused, skipped, recorded: null };
+    return { calibrations, reused, skipped, recorded: null, states };
   let recorded = 0;
   for (let i = 0; i < calibrations.length; i += REPORT_CHUNK) {
     const out = await deps.report({
@@ -85431,7 +85467,7 @@ async function runCalibration(options, deps) {
         `Haansi did not record ${out.refused.length} calibration(s): their task changed since it was listed.`
       );
   }
-  return { calibrations, reused, skipped, recorded };
+  return { calibrations, reused, skipped, recorded, states };
 }
 async function calibrate(argv) {
   if (argv.includes("--help")) {
@@ -85526,6 +85562,178 @@ them all.`;
   }
 });
 
+// src/commands/replay-tasks.ts
+var replay_tasks_exports = {};
+__export(replay_tasks_exports, {
+  areaOf: () => areaOf,
+  cardPrompt: () => cardPrompt,
+  changedFiles: () => changedFiles,
+  overlapOf: () => overlapOf,
+  runStoredReplays: () => runStoredReplays
+});
+function cardPrompt(task, feedback) {
+  return [
+    "# Task",
+    "",
+    task.card.problem,
+    "",
+    "## Done when",
+    "",
+    task.card.acceptance,
+    ...task.card.area ? ["", "## Part of the code", "", task.card.area] : [],
+    "",
+    ...feedback ? [FEEDBACK_HEADER, "", "```", feedback, "```", ""] : [],
+    "Work in this repository as it is now. Do not look at later commits or other branches."
+  ].join("\n");
+}
+function areaOf(path2) {
+  const dirs = path2.split("/").slice(0, -1);
+  return dirs.length ? dirs.slice(0, 2).join("/") : ".";
+}
+async function changedFiles(deps, dir, baseSha) {
+  const lines = (out) => out.code === 0 ? out.output.split("\n").map((l) => l.trim()).filter(Boolean) : [];
+  const diff = await deps.git(
+    ["diff", "--name-only", "--no-renames", baseSha],
+    dir
+  );
+  const added = await deps.git(
+    ["ls-files", "--others", "--exclude-standard"],
+    dir
+  );
+  return [.../* @__PURE__ */ new Set([...lines(diff), ...lines(added)])].filter((f) => !f.startsWith(".haansi-replay/")).sort();
+}
+function overlapOf(changed, task) {
+  const tests = new Set(task.testFiles);
+  const mine = new Set(changed.filter((f) => !tests.has(f)));
+  const theirs = new Set(task.targetFiles.filter((f) => !tests.has(f)));
+  const both = [...mine].filter((f) => theirs.has(f)).length;
+  const myAreas = new Set([...mine].map(areaOf));
+  const theirAreas = new Set([...theirs].map(areaOf));
+  const bothAreas = [...myAreas].filter((a) => theirAreas.has(a)).length;
+  const known = mine.size > 0 && theirs.size > 0;
+  return {
+    filePrecision: known ? share(both, mine.size) : null,
+    fileRecall: known ? share(both, theirs.size) : null,
+    areaPrecision: known ? share(bothAreas, myAreas.size) : null,
+    areaRecall: known ? share(bothAreas, theirAreas.size) : null,
+    changedFiles: mine.size
+  };
+}
+async function runStoredReplays(options, deps) {
+  const repository = await repositoryOf(deps);
+  const set2 = await deps.getReplaySet(repository);
+  const repo = options.repoCommands ? set2.repoCommands : null;
+  const setupCmd = options.setupCmd ?? repo?.setupCommand ?? null;
+  const testCmd = options.testCmd || repo?.testCommand || "";
+  if (!testCmd)
+    throw new Error(
+      `No test command: pass --test-cmd, or set one for ${repository} in Haansi.`
+    );
+  if (set2.tasks.length === 0) {
+    deps.log(
+      `No replay tasks for ${repository} yet: they need a card (replay task cards on for the organization).`
+    );
+    return { results: [], notValid: 0, recorded: null };
+  }
+  const { states } = await runCalibration(
+    calibrateTaskOptions(options, { setupCmd, testCmd }),
+    { ...deps, report: deps.reportCalibration },
+    set2
+  );
+  const valid = set2.tasks.filter(
+    (t) => states.get(t.taskId) === "valid" && t.testFiles.length > 0 && t.testFiles.every(safeTestFile) && SHA.test(t.baseSha) && SHA.test(t.targetSha)
+  );
+  const notValid = set2.tasks.length - valid.length;
+  let arms = options.arms;
+  if (arms.includes("haansi")) {
+    deps.log(
+      "The with-Haansi arm on stored tasks needs Haansi answering as of each task's start; running the baseline arm only."
+    );
+    arms = arms.filter((a) => a !== "haansi");
+  }
+  const chosen = options.limit ? valid.slice(0, options.limit) : valid;
+  const results = [];
+  for (const task of chosen) {
+    const label = task.name ?? task.taskKey;
+    const asTask = {
+      taskId: task.taskId,
+      // Names the clone's directory.
+      processId: task.taskId,
+      name: label,
+      repository: task.repository,
+      baseSha: task.baseSha,
+      targetPr: task.targetPr,
+      targetSha: task.targetSha,
+      testFiles: task.testFiles,
+      context: []
+    };
+    for (const arm of arms) {
+      let changed = [];
+      try {
+        const r = await replayArm(
+          deps,
+          { ...options, setupCmd, testCmd },
+          asTask,
+          arm,
+          null,
+          (feedback) => cardPrompt(task, feedback),
+          async (dir) => {
+            changed = await changedFiles(deps, dir, task.baseSha);
+          }
+        );
+        const overlap = overlapOf(changed, task);
+        results.push({
+          taskId: task.taskId,
+          cardVersion: task.card.version,
+          inputsFingerprint: task.inputsFingerprint,
+          arm,
+          passed: r.passed,
+          attempts: r.attempts,
+          maxAttempts: r.maxAttempts,
+          durationMs: r.durationMs,
+          overlap
+        });
+        deps.log(
+          `${r.passed ? "pass" : "fail"} ${arm.padEnd(8)} ${label} (${r.attempts}/${r.maxAttempts} attempts; files ${pct(overlap.fileRecall)} of the target's, ${pct(overlap.filePrecision)} of its own on target)`
+        );
+      } catch (err2) {
+        deps.log(
+          `error ${arm} ${label}: ${err2 instanceof Error ? err2.message : String(err2)}`
+        );
+      }
+    }
+  }
+  if (options.dryRun || results.length === 0)
+    return { results, notValid, recorded: null };
+  let recorded = 0;
+  for (let i = 0; i < results.length; i += REPORT_CHUNK2) {
+    const out = await deps.reportResults({
+      agent: options.agentName,
+      model: options.model,
+      runRef: options.runRef,
+      results: results.slice(i, i + REPORT_CHUNK2)
+    });
+    recorded += out.recorded;
+    if (out.refused.length)
+      deps.log(
+        `Haansi did not record ${out.refused.length} result(s): their task's card or inputs changed during the run.`
+      );
+  }
+  return { results, notValid, recorded };
+}
+var FEEDBACK_HEADER, REPORT_CHUNK2, share, pct;
+var init_replay_tasks = __esm({
+  "src/commands/replay-tasks.ts"() {
+    "use strict";
+    init_replay_calibrate();
+    init_replay();
+    FEEDBACK_HEADER = "Your previous attempt did not pass the tests. Their output ends with:";
+    REPORT_CHUNK2 = 200;
+    share = (part, whole) => whole === 0 ? null : Math.round(part / whole * 1e3) / 1e3;
+    pct = (v) => v === null ? "n/a" : `${Math.round(v * 100)}%`;
+  }
+});
+
 // src/commands/replay.ts
 var replay_exports = {};
 __export(replay_exports, {
@@ -85585,8 +85793,8 @@ function parseReplayArgs(argv) {
     else if (!sub && !a.startsWith("-")) sub = a;
     else errors.push(`Unknown option: ${a}`);
   }
-  if (!/^[0-9a-f-]{36}$/i.test(options.familyId))
-    errors.push("--family <id> is required (the family's id).");
+  if (options.familyId && !/^[0-9a-f-]{36}$/i.test(options.familyId))
+    errors.push("--family must be a family's id.");
   if (!options.agentCmd) errors.push("--agent-cmd is required.");
   if (!options.testCmd && !options.repoCommands)
     errors.push(
@@ -85694,7 +85902,7 @@ async function cleanClone(deps, dir, sha, unreachable) {
 function cleanCheckout(deps, dir, task) {
   return cleanClone(deps, dir, task.baseSha, task.targetSha);
 }
-async function replayArm(deps, options, task, arm, playbook) {
+async function replayArm(deps, options, task, arm, playbook, promptOf = (feedback) => replayPrompt(task, arm, feedback), inspect) {
   const started = deps.now();
   const dir = (0, import_node_path10.join)(
     deps.workDir(),
@@ -85730,7 +85938,7 @@ async function replayArm(deps, options, task, arm, playbook) {
       );
     let feedback = null;
     for (let attempt = 1; attempt <= options.attempts; attempt++) {
-      const prompt2 = replayPrompt(task, arm, feedback);
+      const prompt2 = promptOf(feedback);
       deps.writeFile(promptFile, prompt2);
       const agentAt = deps.now();
       deps.log(
@@ -85764,6 +85972,7 @@ async function replayArm(deps, options, task, arm, playbook) {
           deps.log(`    ${line}`);
       }
       if (test.code === 0) {
+        if (inspect) await inspect(dir);
         return {
           processId: task.processId,
           arm,
@@ -85780,6 +85989,7 @@ async function replayArm(deps, options, task, arm, playbook) {
       }
       feedback = test.output.slice(-FEEDBACK_CHARS);
     }
+    if (inspect) await inspect(dir);
     return {
       processId: task.processId,
       arm,
@@ -85949,6 +86159,10 @@ async function replay(argv) {
   }
   const token = resolveToken5();
   const apiUrl2 = apiUrl();
+  if (!options.familyId) {
+    await replayStored(options, token, apiUrl2);
+    return;
+  }
   const work = (0, import_node_fs11.mkdtempSync)((0, import_node_path10.join)((0, import_node_os10.tmpdir)(), "haansi-replay-"));
   try {
     const { results, skipped, recorded } = await runReplays(options, {
@@ -85983,6 +86197,48 @@ Baseline: ${passed("baseline")}/${ran("baseline")} passed \xB7 With Haansi: ${pa
     (0, import_node_fs11.rmSync)(work, { recursive: true, force: true });
   }
 }
+async function replayStored(options, token, apiUrl2) {
+  const { runStoredReplays: runStoredReplays2 } = await Promise.resolve().then(() => (init_replay_tasks(), replay_tasks_exports));
+  const { cliVersion: cliVersion2 } = await Promise.resolve().then(() => (init_replay_calibrate(), replay_calibrate_exports));
+  const work = (0, import_node_fs11.mkdtempSync)((0, import_node_path10.join)((0, import_node_os10.tmpdir)(), "haansi-replay-"));
+  try {
+    const { results, notValid, recorded } = await runStoredReplays2(options, {
+      ...localDeps(work),
+      getReplaySet: (repository) => apiGet3(
+        `/memory/replays/tasks?repository=${encodeURIComponent(repository)}`,
+        token,
+        apiUrl2
+      ),
+      reportCalibration: async (body) => await apiPost2(
+        "/memory/replays/tasks/calibration",
+        body,
+        token,
+        apiUrl2
+      ),
+      reportResults: async (body) => await apiPost2(
+        "/memory/replays/tasks/results",
+        body,
+        token,
+        apiUrl2
+      ),
+      env: process.env,
+      cliVersion: cliVersion2()
+    });
+    const passed = results.filter((r) => r.passed).length;
+    console.log(
+      `
+Baseline: ${passed}/${results.length} passed \xB7 ${notValid} task(s) not valid to replay`
+    );
+    console.log(
+      recorded === null ? options.dryRun ? "Dry run: nothing reported." : "Nothing to report." : `Reported ${recorded} result(s) to Haansi.`
+    );
+  } catch (err2) {
+    console.error(err2 instanceof Error ? err2.message : String(err2));
+    process.exitCode = 1;
+  } finally {
+    (0, import_node_fs11.rmSync)(work, { recursive: true, force: true });
+  }
+}
 var import_node_child_process4, import_node_fs11, import_node_os10, import_node_path10, SHA, ARMS, FEEDBACK_CHARS, FAILURE_LINES2, REPLAY_BRANCH, USAGE2;
 var init_replay = __esm({
   "src/commands/replay.ts"() {
@@ -85998,7 +86254,7 @@ var init_replay = __esm({
     FEEDBACK_CHARS = 6e3;
     FAILURE_LINES2 = 8;
     REPLAY_BRANCH = "replay";
-    USAGE2 = `Usage: haansi replay run --family <id> --agent-cmd "<cmd>" --test-cmd "<cmd>"
+    USAGE2 = `Usage: haansi replay run [--family <id>] --agent-cmd "<cmd>" --test-cmd "<cmd>"
                          [--setup-cmd "<cmd>"] [--attempts 3] [--arms baseline,haansi] [--limit n]
                          [--cutoff <date>] [--timeout-min 30]
                          [--agent-name <name>] [--model <name>] [--run-ref <url>] [--dry-run]
@@ -86006,6 +86262,11 @@ var init_replay = __esm({
 
        haansi replay calibrate --test-cmd "<cmd>" [--setup-cmd "<cmd>"] (see --help)
 
+Without --family, runs this repository's stored replay tasks: calibrates them
+first, then replays the valid ones with their task card as the prompt, graded
+by the target's tests and where the change landed. With --family, replays that
+problem family's tasks with and without its playbook.
+
 --repo-commands uses the setup and test commands an org admin set for the
 repository in Haansi (Settings \u2192 Data sources) for any not given here.
 
@@ -86231,9 +86492,10 @@ Commands:
                              digest. [--date YYYY-MM-DD] [--workstream <id>]
   digest [--week|--month|--since YYYY-MM-DD]
                              Your weekly digest (default: the latest week)
-  replay run --family <id> --agent-cmd "<cmd>" --test-cmd "<cmd>"
-                             Replay a problem family's tasks in your CI, with and
-                             without its playbook, and report the results
+  replay run --agent-cmd "<cmd>" --test-cmd "<cmd>" [--family <id>]
+                             Replay this repository's tasks in your CI (or a
+                             problem family's, with and without its playbook)
+                             and report the results
   replay calibrate --test-cmd "<cmd>"
                              Check this repository's replay tasks are valid: their
                              tests fail at the start and pass at the fix

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "haansi",
-  "version": "0.1.55",
+  "version": "0.1.56",
   "description": "Haansi CLI - Session collector and MCP server for Claude Code & OpenAI Codex",
   "bin": {
     "haansi": "./dist/haansi.js"