@rafinery/cli 0.8.0 → 0.8.2

package/lib/distill.mjs

@@ -14,9 +14,21 @@
 // the checkers itself (trust-but-verify), pushes, and resolves. The agent
 // judges; it never ships.
 //
+// Collection is branch-match PLUS ancestry sweep (capture-engine spec P1,
+// 2026-07-17): any still-active row on ANY branch whose capturedAtSha is
+// reachable from this checkout's HEAD *off the first-parent mainline* is
+// provably-merged backlog — an earlier merge whose distill never ran — and
+// folds into this run. Mainline-grounded rows (a checkpoint before the
+// branch's first commit) and squash/rebase-merged commits are NOT provable;
+// those rows stay active for their own merge event or the platform
+// reconciler. Same-path collisions: the merged branch's copy is judged
+// (backlog never displaces the triggering merge); other copies stay active
+// for their own later judgment — a verdict never applies to unjudged content.
+//
 // Fallback chain when this can't run (no key, no CI): the dev-session flow —
 // /rafa distill <branch> in Claude Code (the bootstrap digest offers it).
 
+import { execSync } from "node:child_process";
 import {
   existsSync,
   mkdirSync,
@@ -74,6 +86,62 @@ async function loadAgentSdk(cwd) {
 
 const VERDICTS = ["distilled", "refuted", "needs-adjudication"];
 
+// ── collection helpers (pure; exported for tests) ───────────────────────────
+
+// Same-path collision across branches. The MERGED branch's own row always
+// wins — this run exists to judge the merge that triggered it; a swept
+// backlog row must never displace it. Between swept rows, newest updatedAt
+// wins with a deterministic branch-name tiebreak (platform iteration order
+// must never pick the winner). Superseded copies are NOT resolved here —
+// each copy is judged on its own merits in a later run (a verdict never
+// applies to content the judge did not see).
+export function latestWinsByPath(rows, preferredBranch) {
+  const beats = (a, b) => {
+    const aPref = a.branch === preferredBranch,
+      bPref = b.branch === preferredBranch;
+    if (aPref !== bPref) return aPref;
+    const aAt = a.updatedAt ?? 0,
+      bAt = b.updatedAt ?? 0;
+    if (aAt !== bAt) return aAt > bAt;
+    return String(a.branch) > String(b.branch);
+  };
+  const byPath = new Map();
+  for (const r of rows) {
+    const prev = byPath.get(r.path);
+    if (!prev) byPath.set(r.path, { winner: r, superseded: [] });
+    else if (beats(r, prev.winner)) {
+      prev.superseded.push(prev.winner);
+      prev.winner = r;
+    } else prev.superseded.push(r);
+  }
+  return [...byPath.values()];
+}
+
+// Rows whose capture commit git can PROVE belongs to merged branch work:
+// reachable from HEAD AND off the first-parent mainline. Reachability alone
+// is not proof — a session that checkpoints before its branch's first commit
+// grounds at a MAINLINE sha (the branch point), which is reachable forever;
+// sweeping it would steal in-flight rows from open branches. Off-mainline
+// reachable = the commit arrived via a merge. Rebase/squash-merged work is
+// unprovable here and stays active (safety over recall — the platform
+// reconciler and the session digest remain its path). Only full shas match
+// (checkpoint stamps `git rev-parse HEAD`); anything else counts unproven,
+// never guessed.
+export function sweepByAncestry(rows, provenMerged) {
+  const swept = [];
+  let unproven = 0;
+  for (const r of rows) {
+    if (
+      typeof r.capturedAtSha === "string" &&
+      /^[0-9a-f]{40,64}$/i.test(r.capturedAtSha) &&
+      provenMerged(r.capturedAtSha.toLowerCase())
+    )
+      swept.push(r);
+    else unproven++;
+  }
+  return { swept, unproven };
+}
+
 export default async function distill(args = []) {
   // U7 recursion guard, belt-and-braces: the Agent SDK worker must never fire
   // the M5 session sensors (a worker's own tool calls re-triggering hooks would
@@ -99,31 +167,200 @@ export default async function distill(args = []) {
   // log answers "where is it, and where did the time go" at a glance.
   const t0 = Date.now();
   const secs = (t) => ((Date.now() - t) / 1000).toFixed(1);
+  let agentModel = "unreported";
 
-  // 1 · collect the branch working set. Adjudication-flagged rows are LEFT for
-  // a human session (CI can flag divergence, never resolve it).
+  // Run record (capture-engine P5): the Reconciliations tab renders CI runs
+  // too — a terminal report_distill_run per NON-EMPTY run, keyed by the merge
+  // commit (RAFA_MERGE_SHA from the workflow; HEAD of the merged checkout as
+  // the fallback). Telemetry — best-effort, never fatal, and empty merges
+  // write no row (honest no-op stays row-less).
+  const mergeSha = (() => {
+    const env = process.env.RAFA_MERGE_SHA ?? "";
+    if (/^[0-9a-f]{7,64}$/i.test(env)) return env;
+    try {
+      return execSync("git rev-parse HEAD", {
+        cwd: ROOT,
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+      }).trim();
+    } catch {
+      return "";
+    }
+  })();
+  const reportRun = async (status, summary) => {
+    if (!mergeSha) return;
+    try {
+      await callTool(ROOT, "report_distill_run", {
+        branch,
+        toBranch: process.env.RAFA_TARGET_BRANCH || "main",
+        mergeCommitSha: mergeSha,
+        status,
+        note: summary,
+        actorMeta: { agent: "rafa-distill", runner: "ci", model: agentModel },
+      });
+    } catch {
+      /* the run itself is the deliverable; its record is telemetry */
+    }
+  };
+  const failRun = async (m) => {
+    await reportRun("failed", m);
+    die(m);
+  };
+
+  // Merged-work oracle: TWO subprocesses total — every commit reachable from
+  // HEAD, and the first-parent mainline. provenMerged = reachable ∧
+  // off-mainline (see sweepByAncestry). A shallow checkout truncates both
+  // sets, which only ever UNDER-sweeps (rows stay active) — safe.
+  const revList = (args) => {
+    try {
+      return new Set(
+        execSync(`git rev-list ${args} HEAD`, {
+          cwd: ROOT,
+          encoding: "utf8",
+          stdio: ["ignore", "pipe", "ignore"],
+          maxBuffer: 256 * 1024 * 1024,
+        })
+          .trim()
+          .split("\n")
+          .filter(Boolean),
+      );
+    } catch {
+      return new Set(); // no git history readable → sweep proves nothing
+    }
+  };
+  const reachable = revList("");
+  const mainline = revList("--first-parent");
+  const provenMerged = (sha) => reachable.has(sha) && !mainline.has(sha);
+  let shallow = false;
+  try {
+    shallow =
+      execSync("git rev-parse --is-shallow-repository", {
+        cwd: ROOT,
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+      }).trim() === "true";
+  } catch {
+    /* not a git repo would have failed checkout long before this */
+  }
+
+  // 1 · collect: the merged branch's own rows PLUS the ancestry sweep over
+  // every other branch's active rows (provably-merged backlog folds in).
+  // Adjudication-flagged rows are LEFT for a human session (CI can flag
+  // divergence, never resolve it). The sweep is AUXILIARY: its failure warns
+  // and narrows the run to the merged branch — it never kills the primary
+  // distillation the merge event exists for.
   console.log(`• collecting working set for ${branch} …`);
-  let active, flagged;
+  let branchAll;
   try {
-    active =
-      (await callTool(ROOT, "get_working_set", { branch, status: "active" }))
-        .files ?? [];
-    flagged =
-      (
-        await callTool(ROOT, "get_working_set", {
-          branch,
-          status: "needs-adjudication",
-        })
-      ).files ?? [];
+    branchAll =
+      (await callTool(ROOT, "get_working_set", { branch })).files ?? [];
   } catch (e) {
     die(e instanceof Error ? e.message : String(e));
   }
+  const branchRows = branchAll.filter((r) => r.status === "active");
+  const flagged = branchAll.filter((r) => r.status === "needs-adjudication");
+  const sweptRows = [];
+  let unproven = 0,
+    sweptBranches = 0;
+  try {
+    const { branches } = await callTool(ROOT, "list_working_sets", {});
+    const others = (branches ?? [])
+      .map((b) => (typeof b === "string" ? b : b.branch))
+      .filter((other) => other && other !== branch);
+    const perBranch = await Promise.all(
+      others.map(async (other) => {
+        const rows =
+          (
+            await callTool(ROOT, "get_working_set", {
+              branch: other,
+              status: "active",
+            })
+          ).files ?? [];
+        return sweepByAncestry(rows, provenMerged);
+      }),
+    );
+    for (const { swept, unproven: u } of perBranch) {
+      if (swept.length) sweptBranches++;
+      sweptRows.push(...swept);
+      unproven += u;
+    }
+  } catch (e) {
+    console.log(
+      `  ! ancestry sweep skipped (${e instanceof Error ? e.message : e}) — distilling ${branch}'s own rows only`,
+    );
+  }
   if (flagged.length)
     console.log(
       `  ! ${flagged.length} file(s) already need adjudication — left for the next dev session's digest`,
     );
+  if (sweptRows.length)
+    console.log(
+      `  + ancestry sweep: ${sweptRows.length} provably-merged row(s) from ${sweptBranches} other branch(es) folded into this run`,
+    );
+  if (unproven > 0)
+    console.log(
+      `  · ${unproven} row(s) on other branches not provably merged — left intact` +
+        (shallow
+          ? `\n    (shallow checkout limits the proof — re-run \`npx @rafinery/cli ci-setup --overwrite\` for full history)`
+          : ""),
+    );
+
+  const allGroups = latestWinsByPath([...branchRows, ...sweptRows], branch);
+  // P2 typed candidates: intent records are PROVENANCE, never truth claims —
+  // they resolve as consumed without staging or judgment; notes and
+  // improvements are the judged kinds (per-kind question in the prompt).
+  const kindOf = (p) =>
+    p.startsWith("improve/improvements/")
+      ? "improvement"
+      : p.startsWith("intent/")
+        ? "intent"
+        : "note";
+  const intentGroups = allGroups.filter((g) => kindOf(g.winner.path) === "intent");
+  const groups = allGroups.filter((g) => kindOf(g.winner.path) !== "intent");
+  const active = groups.map((g) => g.winner);
+  if (intentGroups.length)
+    console.log(
+      `  · ${intentGroups.length} intent record(s) — provenance, consumed without judgment`,
+    );
+  let intentConsumed = 0;
+  for (const g of intentGroups) {
+    for (const row of [g.winner, ...g.superseded]) {
+      try {
+        await callTool(ROOT, "resolve_working_file", {
+          branch: row.branch ?? branch,
+          path: row.path,
+          resolution: "distilled",
+          actorMeta: { agent: "rafa-distill", runner: "ci", model: agentModel },
+          note: "intent record — provenance consumed by the merge run (never a truth claim)",
+          ...(typeof row.version === "number" ? { baseVersion: row.version } : {}),
+        });
+        intentConsumed++;
+      } catch {
+        /* stays active for a later run — the brain branch keeps the trail */
+      }
+    }
+  }
+
   if (active.length === 0) {
-    console.log(`✓ nothing to distill on ${branch}`);
+    // Diagnostic empty (owner catch 2026-07-17): silence must explain itself —
+    // "never captured" and "already reconciled" are different states.
+    if (branchAll.length === 0) {
+      console.log(
+        `✓ nothing to distill on ${branch} — 0 rows were ever captured on this branch.\n` +
+          `  Capture never fired during its dev sessions (see \`rafa status\` → SENSORS; ` +
+          `candidates land via \`rafa checkpoint\`).`,
+      );
+    } else {
+      const counts = {};
+      for (const r of branchAll) counts[r.status] = (counts[r.status] ?? 0) + 1;
+      const parts = Object.entries(counts)
+        .map(([s, n]) => `${n} ${s}`)
+        .join(" · ");
+      console.log(
+        `✓ nothing to distill on ${branch} — ${branchAll.length} row(s) exist, none active: ${parts}\n` +
+          `  (distilled/refuted = already reconciled by an earlier run · folded = moved to the parent branch)`,
+      );
+    }
     return;
   }
   console.log(`  ${active.length} file(s) to distill`);
@@ -142,8 +379,9 @@ export default async function distill(args = []) {
     mkdirSync(dirname(abs), { recursive: true });
     writeFileSync(
       abs,
-      `<!-- incoming working-set file · branch: ${branch} · author: ${f.lastAuthor} · v${f.version}` +
-        `${f.baseBrainSha ? ` · base: ${f.baseBrainSha}` : ""} -->\n` +
+      `<!-- incoming working-set file · branch: ${f.branch ?? branch} · author: ${f.lastAuthor} · v${f.version}` +
+        `${f.baseBrainSha ? ` · base: ${f.baseBrainSha}` : ""}` +
+        `${f.capturedAtSha ? ` · captured-at: ${f.capturedAtSha}` : ""} -->\n` +
         f.content,
     );
   }
@@ -153,18 +391,25 @@ export default async function distill(args = []) {
   // 4 · the judgment pass (org's own key; gates stay deterministic below).
   const sdk = await loadAgentSdk(ROOT);
   const fileList = active
-    .map((f) => `- ${f.path} (author: ${f.lastAuthor})`)
+    .map(
+      (f) =>
+        `- ${f.path} (kind: ${kindOf(f.path)} · branch: ${f.branch ?? branch} · author: ${f.lastAuthor})`,
+    )
     .join("\n");
   const prompt = `You are running rafa's CI DISTILLATION — merge-time reconciliation of branch "${branch}"'s
-working set into the org brain. The checked-out repo IS merged main (the ground truth).
+working set into the org brain. Some staged files were swept in from OTHER already-merged
+branches (their provenance comments name the source branch) — judge them identically.
+The checked-out repo IS merged main (the ground truth).
 The current org brain is mirrored at .rafa/brain/. The incoming candidate-grade files are
 staged under .rafa/distill-incoming/ (brain-relative paths mirrored; a provenance comment
 tops each file).
 
 Read .claude/skills/rafa-distill/SKILL.md (the SOP) and .claude/rafa/contract.md §2 (the
-note schema) first.
+note schema) + §3 (the improvement schema) first.
+
+Each staged file names its KIND in the list below — the judge question differs:
 
-For EACH staged file:
+For each kind:note file:
 1. VALIDATE like prism — skeptical, code-grounded: does every claim hold against the code
    as it now stands on main? Confirm each citation resolves (the cited line contains the
    token; run \`git grep\` yourself). A claim you cannot confirm with a file:line is
@@ -174,8 +419,20 @@ For EACH staged file:
    per the contract: valid frontmatter, real cites into main, \`anchor:\` on contracts;
    fold into an existing note when one already covers the topic (supersede, never
    duplicate). Never copy the provenance comment into the brain file.
-3. When all survivors are authored, run \`npx @rafinery/cli verify-citations\` and
-   \`npx @rafinery/cli compile\` and fix your authored files until BOTH exit 0.
+
+For each kind:improvement file — the LIFECYCLE question, not the claim question:
+1. If it reports a FIX (status: fixed): is the cited defect actually GONE on merged main?
+   Gone → verdict "distilled", note "confirmed-fixed — <cite>"; author the status change
+   into .rafa/improve/improvements/<id>.md. Still present → verdict "refuted", note
+   "still-open — <cite>" (the ledger keeps saying open, truthfully).
+2. If it proposes a NEW improvement: does the defect actually EXIST on merged main, at the
+   cited lines? Exists → verdict "distilled", note "opened"; author the file under
+   .rafa/improve/improvements/. Not reproducible → "refuted" with the cited reason.
+3. After ANY improvement change, regenerate .rafa/improve/ledger.md counts to match the
+   rows — the ledger is DERIVED and compile cross-checks it.
+
+When all survivors are authored, run \`npx @rafinery/cli verify-citations\` and
+\`npx @rafinery/cli compile\` and fix your authored files until BOTH exit 0.
 
 Finally write .rafa/distill-verdicts.json:
   {"verdicts":[{"path":"<staged brain-relative path>","verdict":"distilled|refuted|needs-adjudication","note":"<cited reason — required for refuted and needs-adjudication>"}]}
@@ -202,7 +459,6 @@ ${fileList}`;
   // Stream the agent's narration lines — a silent multi-minute CI step reads
   // as hung; a narrated one reads as working (and shows WHERE it hangs).
   let lastLine = "";
-  let agentModel = "unreported";
   for await (const message of run) {
     if (
       message.type === "assistant" &&
@@ -222,7 +478,7 @@ ${fileList}`;
     }
     if (message.type === "result") {
       if (message.subtype !== "success")
-        die(
+        await failRun(
           `distillation agent did not complete (${message.subtype}) — nothing resolved, nothing pushed`,
         );
       console.log(
@@ -252,42 +508,42 @@ ${fileList}`;
 
   // 5 · verdicts must be complete — a missing entry is a hard stop, never a guess.
   if (!existsSync(verdictsPath))
-    die(
+    await failRun(
       "agent produced no .rafa/distill-verdicts.json — nothing resolved, nothing pushed; working set left intact",
     );
   let verdicts;
   try {
     verdicts = JSON.parse(readFileSync(verdictsPath, "utf8")).verdicts;
   } catch {
-    die(
+    await failRun(
       "distill-verdicts.json is not valid JSON — nothing resolved, nothing pushed",
     );
   }
   if (!Array.isArray(verdicts))
-    die("distill-verdicts.json has no verdicts[] — aborting");
+    await failRun("distill-verdicts.json has no verdicts[] — aborting");
   const byPath = new Map(verdicts.map((v) => [v.path, v]));
   for (const f of active) {
     const v = byPath.get(f.path);
     if (!v || !VERDICTS.includes(v.verdict))
-      die(
+      await failRun(
         `no valid verdict for ${f.path} — nothing resolved, nothing pushed; working set left intact`,
       );
     if (
       v.verdict !== "distilled" &&
       (typeof v.note !== "string" || v.note.trim() === "")
     )
-      die(`${f.path}: verdict "${v.verdict}" requires a cited note — aborting`);
+      await failRun(`${f.path}: verdict "${v.verdict}" requires a cited note — aborting`);
   }
 
   // 6 · trust-but-verify: OUR gates re-run regardless of what the agent claims.
   console.log("• re-running the gates (trust-but-verify) …");
   const tGates = Date.now();
   if (runVerifyCitations([]) !== 0)
-    die(
+    await failRun(
       "citation checker failed on the authored brain — nothing resolved, nothing pushed",
     );
   if (runCompile([]) !== 0)
-    die(
+    await failRun(
       "compile gate failed on the authored brain — nothing resolved, nothing pushed",
     );
   console.log(`  ⏱ gates: ${secs(tGates)}s`);
@@ -298,39 +554,59 @@ ${fileList}`;
     (f) => byPath.get(f.path).verdict === "distilled",
   ).length;
   if (distilledCount > 0) {
-    await push([]);
+    await push(["--verb=distill"]);
   } else {
     console.log("• no survivors — nothing to push to the brain repo");
   }
 
   // 8 · resolve the rows (CAS — a racing session's resolve loses loudly).
+  // ONLY judged winners resolve, each under ITS OWN branch key and fenced on
+  // the version this run READ — a checkpoint that landed mid-judgment wins,
+  // never the stale verdict. Superseded same-path copies stay active: a
+  // verdict must never apply to content the judge did not see; each copy is
+  // judged on its own merits in a later run.
   let refuted = 0,
     adjudication = 0;
-  for (const f of active) {
+  for (const g of groups) {
+    const f = g.winner;
     const v = byPath.get(f.path);
     if (v.verdict === "refuted") refuted++;
     if (v.verdict === "needs-adjudication") adjudication++;
     try {
       await callTool(ROOT, "resolve_working_file", {
-        branch,
+        branch: f.branch ?? branch,
         path: f.path,
         resolution: v.verdict,
         actorMeta: { agent: "rafa-distill", runner: "ci", model: agentModel },
         ...(v.note ? { note: v.note } : {}),
+        ...(typeof f.version === "number" ? { baseVersion: f.version } : {}),
       });
       console.log(
-        `  ${v.verdict === "distilled" ? "✓" : "!"} ${f.path} → ${v.verdict}${v.note ? ` (${v.note})` : ""}`,
+        `  ${v.verdict === "distilled" ? "✓" : "!"} ${f.branch ?? branch} · ${f.path} → ${v.verdict}${v.note ? ` (${v.note})` : ""}`,
       );
     } catch (e) {
       console.log(
-        `  ✗ ${f.path} — resolve failed: ${e instanceof Error ? e.message : e}`,
+        `  ✗ ${f.branch ?? branch} · ${f.path} — resolve failed: ${e instanceof Error ? e.message : e}`,
       );
     }
+    for (const s of g.superseded)
+      console.log(
+        `  · ${s.branch ?? branch} · ${s.path} — older same-path copy left active for its own judgment`,
+      );
   }
   rmSync(verdictsPath, { force: true });
 
+  await reportRun(
+    "succeeded",
+    `${distilledCount} banked · ${refuted} refuted · ${adjudication} needs-adjudication` +
+      (sweptRows.length ? ` · ${sweptRows.length} swept from earlier merges` : "") +
+      (intentConsumed ? ` · ${intentConsumed} intent record(s) consumed` : ""),
+  );
   console.log(
     `✓ distillation: ${distilledCount} into the org brain · ${refuted} refuted (reported, cited) · ` +
-      `${adjudication} flagged needs-adjudication (next session's digest) · total ${secs(t0)}s`,
+      `${adjudication} flagged needs-adjudication (next session's digest)` +
+      (sweptRows.length ? ` · ${sweptRows.length} swept from earlier merges` : "") +
+      (intentConsumed ? ` · ${intentConsumed} intent consumed` : "") +
+      ` · total ${secs(t0)}s`,
   );
 }