haansi 0.1.57 → 0.1.58

package/dist/haansi.js

@@ -95,7 +95,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "haansi",
-      version: "0.1.57",
+      version: "0.1.58",
       description: "Haansi CLI - Session collector and MCP server for Claude Code & OpenAI Codex",
       bin: {
         haansi: "./dist/haansi.js"
@@ -19246,7 +19246,15 @@ var init_exclusions = __esm({
         arm: ["operational", "Closed CHECK vocabulary (baseline, haansi)."],
         agent: ["operational", "The agent CLI's name, e.g. claude-code."],
         model: ["operational", "The model name the replay ran with."],
-        run_ref: ["not-personal", "A link to the customer's CI run."]
+        run_ref: ["not-personal", "A link to the customer's CI run."],
+        haansi_used: [
+          "operational",
+          "Closed CHECK vocabulary (used, partly, not_used)."
+        ],
+        haansi_judge_tag: [
+          "not-personal",
+          "The prompt version that judged it (replay-use-judge-vN)."
+        ]
       }),
       ...exchangeExclusions(INTEL, "replay_tasks", {
         process_id: [
@@ -77508,7 +77516,7 @@ var require_view = __commonJS({
     var dirname5 = path2.dirname;
     var basename4 = path2.basename;
     var extname = path2.extname;
-    var join17 = path2.join;
+    var join18 = path2.join;
     var resolve5 = path2.resolve;
     module2.exports = View;
     function View(name, options) {
@@ -77570,12 +77578,12 @@ var require_view = __commonJS({
     };
     View.prototype.resolve = function resolve6(dir, file2) {
       var ext = this.ext;
-      var path3 = join17(dir, file2);
+      var path3 = join18(dir, file2);
       var stat = tryStat(path3);
       if (stat && stat.isFile()) {
         return path3;
       }
-      path3 = join17(dir, basename4(file2, ext), "index" + ext);
+      path3 = join18(dir, basename4(file2, ext), "index" + ext);
       stat = tryStat(path3);
       if (stat && stat.isFile()) {
         return path3;
@@ -81220,7 +81228,7 @@ var require_send = __commonJS({
     var Stream = require("stream");
     var util2 = require("util");
     var extname = path2.extname;
-    var join17 = path2.join;
+    var join18 = path2.join;
     var normalize = path2.normalize;
     var resolve5 = path2.resolve;
     var sep2 = path2.sep;
@@ -81392,7 +81400,7 @@ var require_send = __commonJS({
           return res;
         }
         parts = path3.split(sep2);
-        path3 = normalize(join17(root, path3));
+        path3 = normalize(join18(root, path3));
       } else {
         if (UP_PATH_REGEXP.test(path3)) {
           debug('malicious path "%s"', path3);
@@ -81525,7 +81533,7 @@ var require_send = __commonJS({
           if (err2) return self.onStatError(err2);
           return self.error(404);
         }
-        var p = join17(path3, self._index[i]);
+        var p = join18(path3, self._index[i]);
         debug('stat "%s"', p);
         fs.stat(p, function(err3, stat) {
           if (err3) return next(err3);
@@ -85603,6 +85611,111 @@ them all.`;
   }
 });
 
+// src/commands/replay-judge.ts
+function renderTemplate(template, vars) {
+  return template.replace(
+    /\{\{(\w+)\}\}/g,
+    (all, key) => key in vars ? vars[key] : all
+  );
+}
+async function changeOf(deps, dir, baseSha) {
+  await deps.git(["add", "--all", "--intent-to-add"], dir);
+  const diff = await deps.git(
+    ["diff", "--no-color", baseSha, "--", ".", ":(exclude).haansi-replay"],
+    dir
+  );
+  return diff.code === 0 ? clip(diff.output, MAX_CHANGE_CHARS) : "";
+}
+function parseJudgement(text, returnedIds) {
+  const match = /\{[\s\S]*\}/.exec(text);
+  if (!match) return null;
+  let v;
+  try {
+    v = JSON.parse(match[0]);
+  } catch {
+    return null;
+  }
+  if (!VERDICTS.includes(v.verdict)) return null;
+  const items = Array.isArray(v.used_items) ? new Set(
+    v.used_items.filter(
+      (id) => typeof id === "string" && returnedIds.has(id)
+    )
+  ) : /* @__PURE__ */ new Set();
+  const reason = typeof v.reason === "string" && v.reason.trim() ? v.reason.trim().slice(0, MAX_REASON) : null;
+  return { used: v.verdict, reason, usedItems: items.size };
+}
+async function judgeUse(deps, input) {
+  const returnedIds = new Set(input.calls.flatMap((c) => c.returnedIds));
+  if (input.calls.length === 0)
+    return {
+      used: "not_used",
+      reason: "The agent did not call Haansi.",
+      usedItems: 0,
+      judgeTag: null
+    };
+  if (returnedIds.size === 0)
+    return {
+      used: "not_used",
+      reason: "Haansi returned nothing.",
+      usedItems: 0,
+      judgeTag: null
+    };
+  if (!input.prompt)
+    return { used: null, reason: null, usedItems: null, judgeTag: null };
+  const tag = input.prompt.edited ? `${input.prompt.tag}+edited` : input.prompt.tag;
+  const dir = (0, import_node_path10.join)(deps.workDir(), `${input.taskId}-judge`);
+  const verdictFile = (0, import_node_path10.join)(dir, "verdict.json");
+  const promptFile = (0, import_node_path10.join)(dir, "prompt.md");
+  const configFile = (0, import_node_path10.join)(dir, "mcp.json");
+  const returned = input.calls.map((c) => c.returnedText ?? "").filter(Boolean).join("\n\n");
+  const prompt2 = [
+    input.prompt.system,
+    "",
+    renderTemplate(input.prompt.user, {
+      task: input.card,
+      returned: clip(returned || "(only ids came back)", MAX_RETURNED_CHARS),
+      change: input.change || "(no change)",
+      verdict_file: verdictFile
+    })
+  ].join("\n");
+  try {
+    deps.writeFile(promptFile, prompt2);
+    deps.writeFile(configFile, JSON.stringify({ mcpServers: {} }));
+    const ran = await deps.shell(
+      input.agentCmd,
+      dir,
+      {
+        HAANSI_REPLAY_PROMPT: prompt2,
+        HAANSI_REPLAY_PROMPT_FILE: promptFile,
+        HAANSI_REPLAY_MCP_CONFIG: configFile,
+        HAANSI_REPLAY_VERDICT_FILE: verdictFile
+      },
+      input.timeoutMs
+    );
+    const judged = parseJudgement(deps.readFile(verdictFile) ?? "", returnedIds) ?? parseJudgement(ran.output, returnedIds);
+    if (!judged) {
+      deps.log(`\u2026 ${input.label} [haansi]: the judge gave no verdict`);
+      return { used: null, reason: null, usedItems: null, judgeTag: tag };
+    }
+    return { ...judged, judgeTag: tag };
+  } finally {
+    deps.removeDir(dir);
+  }
+}
+var import_node_path10, MAX_CHANGE_CHARS, MAX_RETURNED_CHARS, MAX_REASON, VERDICTS, clip;
+var init_replay_judge = __esm({
+  "src/commands/replay-judge.ts"() {
+    "use strict";
+    import_node_path10 = require("node:path");
+    MAX_CHANGE_CHARS = 6e4;
+    MAX_RETURNED_CHARS = 6e4;
+    MAX_REASON = 300;
+    VERDICTS = ["used", "partly", "not_used"];
+    clip = (text, max2) => text.length > max2 ? `${text.slice(0, max2)}
+\u2026 (trimmed)` : text;
+  }
+});
+
 // src/commands/replay-tasks.ts
 var replay_tasks_exports = {};
 __export(replay_tasks_exports, {
@@ -85721,6 +85834,12 @@ async function runStoredReplays(options, deps) {
   const notValid = set2.tasks.length - valid.length;
   const arms = options.arms;
   const chosen = options.limit ? valid.slice(0, options.limit) : valid;
+  let judgePrompt;
+  const promptForJudge = async () => {
+    if (judgePrompt === void 0)
+      judgePrompt = await deps.getJudgePrompt().catch(() => null);
+    return judgePrompt;
+  };
   const results = [];
   for (const task of chosen) {
     const label = task.name ?? task.taskKey;
@@ -85738,6 +85857,7 @@ async function runStoredReplays(options, deps) {
     };
     for (const arm of arms) {
       let changed = [];
+      let change = "";
       try {
         const session = arm === "haansi" ? {
           ...await deps.openSession(task.taskId),
@@ -85755,11 +85875,29 @@ async function runStoredReplays(options, deps) {
           (feedback) => cardPrompt(task, feedback, arm),
           async (dir) => {
             changed = await changedFiles(deps, dir, task.baseSha);
+            if (arm === "haansi")
+              change = await changeOf(deps, dir, task.baseSha);
           },
           { HAANSI_REPLAY_MCP_CONFIG: configPath }
         );
         deps.removeFile(configPath);
-        const haansi = session ? haansiUseOf((await deps.getCalls(session.sessionId)).calls) : null;
+        let haansi = null;
+        if (session) {
+          const { calls } = await deps.getCalls(session.sessionId);
+          haansi = {
+            ...haansiUseOf(calls),
+            ...await judgeUse(deps, {
+              label,
+              taskId: task.taskId,
+              agentCmd: options.agentCmd,
+              timeoutMs: options.timeoutMs,
+              card: cardPrompt(task, null),
+              calls,
+              change,
+              prompt: await promptForJudge()
+            })
+          };
+        }
         const overlap = overlapOf(changed, task);
         results.push({
           taskId: task.taskId,
@@ -85808,13 +85946,17 @@ function haansiUseOf(calls) {
   };
 }
 function haansiText(use) {
-  return use.calls === 0 ? "did not call Haansi" : `called Haansi ${use.calls}\xD7, ${use.returned} item${use.returned === 1 ? "" : "s"} returned`;
+  if (use.calls === 0) return "did not call Haansi";
+  const base = `called Haansi ${use.calls}\xD7, ${use.returned} item${use.returned === 1 ? "" : "s"} returned`;
+  if (!use.used) return base;
+  return use.used === "not_used" ? `${base}, not used` : `${base}, ${use.used === "partly" ? "partly " : ""}used${use.usedItems ? ` (${use.usedItems})` : ""}`;
 }
 var FEEDBACK_HEADER, REPORT_CHUNK2, HAANSI_LINE, share, pct;
 var init_replay_tasks = __esm({
   "src/commands/replay-tasks.ts"() {
     "use strict";
     init_replay_calibrate();
+    init_replay_judge();
     init_replay();
     FEEDBACK_HEADER = "Your previous attempt did not pass the tests. Their output ends with:";
     REPORT_CHUNK2 = 200;
@@ -85994,7 +86136,7 @@ function cleanCheckout(deps, dir, task) {
 }
 async function replayArm(deps, options, task, arm, playbook, promptOf = (feedback) => replayPrompt(task, arm, feedback), inspect, agentEnv = {}) {
   const started = deps.now();
-  const dir = (0, import_node_path10.join)(
+  const dir = (0, import_node_path11.join)(
     deps.workDir(),
     `${task.processId.replace(/[^a-zA-Z0-9_-]/g, "")}-${arm}`
   );
@@ -86020,7 +86162,7 @@ async function replayArm(deps, options, task, arm, playbook, promptOf = (feedbac
       }
       deps.log(`\u2026 ${label}: set up in ${since(at)}`);
     }
-    const promptFile = (0, import_node_path10.join)(dir, ".haansi-replay", "prompt.md");
+    const promptFile = (0, import_node_path11.join)(dir, ".haansi-replay", "prompt.md");
     let mcpConfigFile = agentEnv.HAANSI_REPLAY_MCP_CONFIG;
     if (!mcpConfigFile) {
       mcpConfigFile = `${dir}.mcp.json`;
@@ -86028,7 +86170,7 @@ async function replayArm(deps, options, task, arm, playbook, promptOf = (feedbac
     }
     if (arm === "haansi" && playbook)
       deps.writeFile(
-        (0, import_node_path10.join)(dir, ".haansi-replay", "playbook.md"),
+        (0, import_node_path11.join)(dir, ".haansi-replay", "playbook.md"),
         playbook.skillMd
       );
     let feedback = null;
@@ -86050,11 +86192,11 @@ async function replayArm(deps, options, task, arm, playbook, promptOf = (feedbac
         },
         options.timeoutMs
       );
-      const own = new Map(files.map((f) => [f, deps.readFile((0, import_node_path10.join)(dir, f))]));
+      const own = new Map(files.map((f) => [f, deps.readFile((0, import_node_path11.join)(dir, f))]));
       for (const f of files) {
         const content = await deps.showFile(task.targetSha, f);
-        if (content === null) deps.removeFile((0, import_node_path10.join)(dir, f));
-        else deps.writeFile((0, import_node_path10.join)(dir, f), content);
+        if (content === null) deps.removeFile((0, import_node_path11.join)(dir, f));
+        else deps.writeFile((0, import_node_path11.join)(dir, f), content);
       }
       const testAt = deps.now();
       deps.log(`\u2026 ${label}: agent done in ${since(agentAt)}, grading`);
@@ -86084,8 +86226,8 @@ async function replayArm(deps, options, task, arm, playbook, promptOf = (feedbac
         };
       }
       for (const [file2, content] of own) {
-        if (content === null) deps.removeFile((0, import_node_path10.join)(dir, file2));
-        else deps.writeFile((0, import_node_path10.join)(dir, file2), content);
+        if (content === null) deps.removeFile((0, import_node_path11.join)(dir, file2));
+        else deps.writeFile((0, import_node_path11.join)(dir, file2), content);
       }
       feedback = test.output.slice(-FEEDBACK_CHARS);
     }
@@ -86235,7 +86377,7 @@ function localDeps(work) {
     repoDir: process.cwd(),
     workDir: () => work,
     writeFile: (path2, content) => {
-      (0, import_node_fs11.mkdirSync)((0, import_node_path10.dirname)(path2), { recursive: true });
+      (0, import_node_fs11.mkdirSync)((0, import_node_path11.dirname)(path2), { recursive: true });
       (0, import_node_fs11.writeFileSync)(path2, content);
     },
     readFile: (path2) => (0, import_node_fs11.existsSync)(path2) ? (0, import_node_fs11.readFileSync)(path2, "utf-8") : null,
@@ -86264,7 +86406,7 @@ async function replay(argv) {
     await replayStored(options, token, apiUrl2);
     return;
   }
-  const work = (0, import_node_fs11.mkdtempSync)((0, import_node_path10.join)((0, import_node_os10.tmpdir)(), "haansi-replay-"));
+  const work = (0, import_node_fs11.mkdtempSync)((0, import_node_path11.join)((0, import_node_os10.tmpdir)(), "haansi-replay-"));
   try {
     const { results, skipped, recorded } = await runReplays(options, {
       getTasks: async (familyId, cutoff) => (await apiGet3(
@@ -86301,7 +86443,7 @@ Baseline: ${passed("baseline")}/${ran("baseline")} passed \xB7 With Haansi: ${pa
 async function replayStored(options, token, apiUrl2) {
   const { runStoredReplays: runStoredReplays2 } = await Promise.resolve().then(() => (init_replay_tasks(), replay_tasks_exports));
   const { cliVersion: cliVersion2 } = await Promise.resolve().then(() => (init_replay_calibrate(), replay_calibrate_exports));
-  const work = (0, import_node_fs11.mkdtempSync)((0, import_node_path10.join)((0, import_node_os10.tmpdir)(), "haansi-replay-"));
+  const work = (0, import_node_fs11.mkdtempSync)((0, import_node_path11.join)((0, import_node_os10.tmpdir)(), "haansi-replay-"));
   try {
     const { results, notValid, recorded } = await runStoredReplays2(options, {
       ...localDeps(work),
@@ -86331,10 +86473,11 @@ async function replayStored(options, token, apiUrl2) {
         apiUrl2
       ),
       getCalls: (sessionId) => apiGet3(
-        `/memory/replays/sessions/${encodeURIComponent(sessionId)}/calls`,
+        `/memory/replays/sessions/${encodeURIComponent(sessionId)}/calls?include=text`,
         token,
         apiUrl2
       ),
+      getJudgePrompt: () => apiGet3("/memory/replays/prompts/use-judge", token, apiUrl2),
       apiUrl: apiUrl2,
       token
     });
@@ -86356,14 +86499,14 @@ Baseline: ${arm("baseline")} passed \xB7 With Haansi: ${arm("haansi")} passed \x
     (0, import_node_fs11.rmSync)(work, { recursive: true, force: true });
   }
 }
-var import_node_child_process4, import_node_fs11, import_node_os10, import_node_path10, SHA, ARMS, FEEDBACK_CHARS, FAILURE_LINES2, REPLAY_BRANCH, USAGE2;
+var import_node_child_process4, import_node_fs11, import_node_os10, import_node_path11, SHA, ARMS, FEEDBACK_CHARS, FAILURE_LINES2, REPLAY_BRANCH, USAGE2;
 var init_replay = __esm({
   "src/commands/replay.ts"() {
     "use strict";
     import_node_child_process4 = require("node:child_process");
     import_node_fs11 = require("node:fs");
     import_node_os10 = require("node:os");
-    import_node_path10 = require("node:path");
+    import_node_path11 = require("node:path");
     init_paths();
     init_api2();
     SHA = /^[0-9a-f]{7,40}$/i;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "haansi",
-  "version": "0.1.57",
+  "version": "0.1.58",
   "description": "Haansi CLI - Session collector and MCP server for Claude Code & OpenAI Codex",
   "bin": {
     "haansi": "./dist/haansi.js"