@rafinery/cli 0.8.1 → 0.8.3

package/lib/distiller/doctrine.mjs

@@ -0,0 +1,379 @@
+// The arbitration DOCTRINE (reconciliation orchestrator, spec §2.5) — the
+// mechanical core of the distiller. Everything here is a pure function of the
+// candidate working set + the checked-out merged main (git). The ONLY non-
+// mechanical step in the whole run is claim-level truth ("does this normative
+// claim still hold?"), and that is injected as a `judge` seam by the loop — it
+// never lives here. So the doctrine is fully unit-testable against temp-dir git
+// fixtures with a deterministic fake judge (refinery-arc-p1-distiller cases 1,3,4,5).
+//
+// Vocabulary (the per-row outcome the spec mandates, stamped with source branch
+// + capturedAtSha + reconciliation id):
+//   banked    — a survivor authored fresh into the org brain
+//   rewritten — a survivor whose citation moved on main and was re-grounded
+//   folded    — a collision loser, superseded by the latest-merge winner
+//   refuted   — a claim that no longer holds against main (cited reason)
+//   pruned    — a note citing code the merge deleted, unrecoverably (deletion record)
+//
+// The citation machinery is REUSED verbatim from the gate (verify-citations):
+// one implementation of "does this token still live in the code," gate + doctrine.
+
+import { join } from "node:path";
+import { execSync } from "node:child_process";
+import { parseFrontmatter, parseCites } from "@rafinery/okf";
+import { citeResolves, gitGrep } from "../gate/verify-citations.mjs";
+
+// ── cite model ────────────────────────────────────────────────────────────────
+// parseCites yields { file, line, token } where line is "55" or "14-18". Normalize
+// to explicit start/end so citeResolves (the gate predicate) can consume it.
+export function normalizeCite(c) {
+  const [a, b] = String(c.line).split("-");
+  const start = Number(a);
+  const end = b ? Number(b) : start;
+  return { file: c.file, start, end, token: c.token, loc: `${c.file}:${c.line}` };
+}
+
+// Parse a candidate note's frontmatter `cites:` DSL → normalized cites.
+export function citesOf(content) {
+  const { data, error } = parseFrontmatter(content);
+  if (error || !data) return [];
+  return parseCites(data.cites).map(normalizeCite);
+}
+
+// The note id a candidate carries (frontmatter `id:`), else its path stem — the
+// stable key an outcome/delta row is stamped with.
+export function idOf(content, path = "") {
+  const { data } = parseFrontmatter(content) || {};
+  if (data && data.id) return String(data.id);
+  return path.split("/").pop()?.replace(/\.md$/, "") ?? "";
+}
+
+// ── single-cite arbitration against merged main ────────────────────────────────
+// resolved     — the cite still resolves as authored (grounded)
+// regrounded   — the exact line moved but the token still lives elsewhere → re-cite
+// deleted      — the token is nowhere in the code → a deletion (prune candidate)
+export function arbitrateCite(cite, cwd = process.cwd()) {
+  const abs = join(cwd, cite.file);
+  if (citeResolves(abs, cite.start, cite.end, cite.token).ok)
+    return { status: "resolved", cite };
+  // The authored location is stale — is the knowledge merely MOVED, or GONE? One
+  // grep of the token over the merged tree settles it (the gate's gitGrep, .rafa
+  // excluded). A hit anywhere → the code moved, re-ground; no hit → deleted.
+  const hits = gitGrep(cite.token, cwd);
+  if (hits.length) {
+    const to = hits[0];
+    return {
+      status: "regrounded",
+      cite,
+      to: { file: to.file, start: to.line, end: to.line, token: cite.token, loc: `${to.file}:${to.line}` },
+    };
+  }
+  return {
+    status: "deleted",
+    cite,
+    record: `cited \`${cite.token}\` at ${cite.loc} — deleted on main (token nowhere in the merged tree)`,
+  };
+}
+
+// ── candidate-level arbitration (mechanical layer + the judge seam) ─────────────
+// Returns a disposition the loop turns into an outcome + a delta entry.
+//   prune     — the note cites code the merge deleted, no re-grounding possible
+//   refute    — the citations ground but the claim no longer holds (judge said so)
+//   rewrite   — a survivor whose citations were re-grounded (moved code)
+//   bank      — a clean survivor (all cites resolve, claim holds)
+// `judge` is the ONLY LLM touch-point; it is injected. A candidate whose cites
+// don't even ground is decided WITHOUT the judge (mechanical refutation/prune).
+// The judge is inherently async (a network LLM call), so this awaits it; a sync
+// fake judge in tests satisfies `await` too. A verdict may report `tokens` — the
+// loop's metering wrapper reads it off the return to accumulate meter.tokens.
+export async function arbitrateCandidate(candidate, cwd, { judge }) {
+  const cites = citesOf(candidate.content);
+  const noteId = idOf(candidate.content, candidate.path);
+  const dispositions = cites.map((c) => arbitrateCite(c, cwd));
+
+  // A deletion the doctrine cannot re-ground → PRUNE with a citation-backed record
+  // (recoverable: the brain repo is git-versioned; a prune is a commit).
+  const deleted = dispositions.filter((d) => d.status === "deleted");
+  if (deleted.length) {
+    return {
+      outcome: "pruned",
+      noteId,
+      cites,
+      detail: deleted.map((d) => d.record).join("; "),
+      deletionRecords: deleted.map((d) => d.record),
+    };
+  }
+
+  const regroundings = dispositions.filter((d) => d.status === "regrounded");
+  const grounded = dispositions.map((d) => (d.status === "regrounded" ? d.to : d.cite));
+
+  // All citations now ground (as authored or re-grounded). Claim-level truth is
+  // the judge's call — the single LLM step, injected. Mechanical everywhere else.
+  const verdict = await judge(candidate, { cwd, grounded, regroundings });
+  if (!verdict || verdict.hold !== true) {
+    return {
+      outcome: "refuted",
+      noteId,
+      cites: grounded,
+      regroundings,
+      detail: verdict?.reason ?? "claim does not hold against merged main",
+    };
+  }
+  if (regroundings.length) {
+    return {
+      outcome: "rewritten",
+      noteId,
+      cites: grounded,
+      regroundings,
+      detail: `re-grounded ${regroundings.length} citation(s) that moved on main: ` +
+        regroundings.map((r) => `${r.cite.loc} → ${r.to.loc}`).join(", "),
+    };
+  }
+  return { outcome: "banked", noteId, cites: grounded, regroundings, detail: `banked as ${noteId}` };
+}
+
+// Rewrite a survivor's frontmatter `cites:` DSL so a re-grounded citation points
+// at where the token NOW lives on main — otherwise banking it as authored would
+// fail the checker (the stale line no longer resolves). Mechanical string
+// substitution on the exact `file:line :: token` locator; untouched lines pass
+// through. Returns the rewritten content.
+export function rewriteCites(content, regroundings) {
+  let out = content;
+  for (const r of regroundings ?? []) {
+    const fromLoc = r.cite.loc; // file:line[-end]
+    const toLoc = r.to.loc; // file:line
+    // Match the DSL line `  - <fromLoc> :: <token>` and swap the locator only.
+    const esc = fromLoc.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = new RegExp(`(-\\s+)${esc}(\\s*::\\s*)`, "g");
+    out = out.replace(re, `$1${toLoc}$2`);
+  }
+  return out;
+}
+
+// ── folding N branch working sets into ONE run (case 1 · case 5) ────────────────
+// Input: [{ branch, capturedAtSha, mergedAt, files: [{ path, content }] }].
+// Output: candidates keyed by path; a path carried by >1 branch is a COLLISION
+// (same-note collision across folded branches — claim-level arbitration + a
+// latest-merge-wins tiebreak resolve it, spec §2.5).
+export function foldCandidates(branchSets) {
+  const byPath = new Map();
+  for (const set of branchSets) {
+    for (const f of set.files) {
+      const cand = {
+        path: f.path,
+        branch: set.branch,
+        capturedAtSha: set.capturedAtSha,
+        mergedAt: set.mergedAt ?? 0,
+        content: f.content,
+      };
+      if (!byPath.has(f.path)) byPath.set(f.path, []);
+      byPath.get(f.path).push(cand);
+    }
+  }
+  const candidates = [];
+  const collisions = [];
+  for (const [path, versions] of byPath) {
+    if (versions.length === 1) candidates.push(versions[0]);
+    else collisions.push({ path, versions });
+  }
+  return { candidates, collisions };
+}
+
+// Deterministic latest-merge-wins ordering: LATEST mergedAt wins, branch name is
+// the stable tiebreak. Never a coin-flip — CI has no one to ask (epic ADR: this
+// supersedes needs-adjudication at merge time). Returns { winner, losers }.
+export function latestMergeWins(versions) {
+  const ordered = [...versions].sort((a, b) => {
+    if (a.mergedAt !== b.mergedAt) return a.mergedAt - b.mergedAt;
+    return a.branch < b.branch ? -1 : a.branch > b.branch ? 1 : 0;
+  });
+  const winner = ordered[ordered.length - 1];
+  const losers = ordered.slice(0, -1);
+  return { winner, losers };
+}
+
+// Resolve a same-note collision (case 5): every version is judged (claim-level
+// arbitration), THEN latest-merge-wins decides which SURVIVOR is banked. The other
+// survivors become `folded, superseded by <winner branch>` (recorded in provenance);
+// versions that did not survive keep their own outcome (refuted/pruned). Returns
+// dispositions (each carrying its `candidate`) ready for the loop's outcome rows.
+export async function resolveCollision(versions, cwd, { judge }) {
+  const judged = [];
+  for (const v of versions) judged.push({ candidate: v, ...(await arbitrateCandidate(v, cwd, { judge })) });
+  const survivors = judged.filter((d) => d.outcome === "banked" || d.outcome === "rewritten");
+  if (survivors.length <= 1) return judged; // 0 or 1 survivor → no tiebreak needed
+  const { winner } = latestMergeWins(survivors.map((s) => s.candidate));
+  return judged.map((d) => {
+    if ((d.outcome === "banked" || d.outcome === "rewritten") && d.candidate !== winner) {
+      return {
+        ...d,
+        outcome: "folded",
+        supersededBy: winner.branch,
+        detail: `folded, superseded by ${winner.branch} (latest merge @ ${winner.capturedAtSha})`,
+      };
+    }
+    return d;
+  });
+}
+
+// ── merge-range citation sweep (case 4, second half) ────────────────────────────
+// The changed files since the reconciliation cursor — bounded, mechanical, never
+// a re-scan. `git diff --name-only <cursor>..<merge>`.
+export function changedFilesSince(cursorSha, mergeSha, cwd = process.cwd()) {
+  if (!cursorSha) return [];
+  try {
+    const out = execSync(
+      `git diff --name-only ${JSON.stringify(cursorSha)} ${JSON.stringify(mergeSha)}`,
+      { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] },
+    );
+    return out.split("\n").filter(Boolean);
+  } catch {
+    return null; // unreachable range → caller treats as a transient sweep skip
+  }
+}
+
+// Re-resolve the citations of EXISTING org-brain notes that touch a changed file
+// (never every note — only those in the blast radius of this merge range). Each
+// touched cite re-grounds or prunes by the SAME rule as a candidate's. Returns
+// [{ notePath, noteId, cite, disposition }] — the loop turns prunes into deletion
+// records + delta.pruned rows and re-groundings into rewrite rows.
+export function mergeRangeSweep({ changedFiles, orgNotes, cwd = process.cwd() }) {
+  const changed = new Set(changedFiles ?? []);
+  const results = [];
+  for (const note of orgNotes) {
+    const cites = citesOf(note.content);
+    const noteId = idOf(note.content, note.path);
+    for (const cite of cites) {
+      if (!changed.has(cite.file)) continue; // out of the merge range → untouched
+      const disposition = arbitrateCite(cite, cwd);
+      if (disposition.status !== "resolved")
+        results.push({ notePath: note.path, noteId, cite, disposition });
+    }
+  }
+  return results;
+}
+
+// ── failure classification (case 3) ─────────────────────────────────────────────
+// Deterministic (config / gate rejection / schema): retrying only burns spend →
+// reconcile_report failure class "deterministic" → needs-attention immediately.
+// Transient (network / boot / OOM / clone / timeout): backoff retry while a later
+// branch proceeds. An error may carry an explicit `errorClass`; else heuristics.
+const TRANSIENT_RE =
+  /\b(fetch failed|ECONNREFUSED|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|network|timed? ?out|timeout|rate limit|429|50[0-9]\b|oom|out of memory|clone failed|unreachable)\b/i;
+export function classifyError(err) {
+  const explicit = err && err.errorClass;
+  if (explicit === "deterministic" || explicit === "transient") return explicit;
+  const msg = String((err && err.message) || err || "");
+  return TRANSIENT_RE.test(msg) ? "transient" : "deterministic";
+}
+
+// Tagged-error helpers so the loop can raise a failure of a known class and the
+// reconcile_report path carries it verbatim (never a re-guess at the boundary).
+export function deterministicFailure(message) {
+  const e = new Error(message);
+  e.errorClass = "deterministic";
+  return e;
+}
+export function transientFailure(message) {
+  const e = new Error(message);
+  e.errorClass = "transient";
+  return e;
+}
+
+// ── per-run provenance assembly ─────────────────────────────────────────────────
+// One outcome row per candidate/collision-member, stamped with its source branch,
+// capturedAtSha, and the reconciliation id (spec §2.5 provenance requirement).
+export function outcomeRow({ candidate, outcome, detail, reconciliationId, supersededBy }) {
+  return {
+    path: candidate.path,
+    branch: candidate.branch,
+    capturedAtSha: candidate.capturedAtSha,
+    reconciliationId,
+    outcome, // banked | rewritten | folded | refuted | pruned
+    detail: supersededBy ? `folded, superseded by ${supersededBy}` : detail,
+  };
+}
+
+// The reconcile_report `delta` shape (knowledgeGraph.deltaValidator): four buckets
+// of { noteId, cites }. Built from the run's dispositions so the node the pointer
+// advance mints carries exactly what this run refined. `folded` losers do not
+// enter the delta (they were superseded, not banked) but ARE in the outcomes.
+export function buildDelta(dispositions) {
+  const delta = { banked: [], rewritten: [], pruned: [], refuted: [] };
+  for (const d of dispositions) {
+    const entry = { noteId: d.noteId, cites: toWireCites(d.cites) };
+    if (d.outcome === "banked") delta.banked.push(entry);
+    else if (d.outcome === "rewritten") delta.rewritten.push(entry);
+    else if (d.outcome === "pruned") delta.pruned.push(entry);
+    else if (d.outcome === "refuted") delta.refuted.push(entry);
+  }
+  return delta;
+}
+
+// Normalized cites → the citeValidator wire shape ({ file, line, token }).
+function toWireCites(cites = []) {
+  return cites.map((c) => ({
+    file: c.file,
+    line: c.start === c.end ? String(c.start) : `${c.start}-${c.end}`,
+    token: c.token,
+  }));
+}
+
+// Per-branch outcome rows for reconcile_report (perBranchOutcomeValidator:
+// { branch, capturedAtSha, verdict }). One row per source branch, the verdict a
+// short mechanical summary of that branch's fate in the fold.
+export function perBranchOutcomes(branchSets, outcomeRows) {
+  return branchSets.map((set) => {
+    const rows = outcomeRows.filter((r) => r.branch === set.branch);
+    const tally = {};
+    for (const r of rows) tally[r.outcome] = (tally[r.outcome] ?? 0) + 1;
+    const verdict = rows.length
+      ? Object.entries(tally).map(([k, n]) => `${n} ${k}`).join(" · ")
+      : "no candidates";
+    return { branch: set.branch, capturedAtSha: set.capturedAtSha, verdict };
+  });
+}
+
+// ── the reconciliation-report OKF concept (contract §11 file class) ─────────────
+// A per-run outcome/provenance record authored into brain/reconciliations/. type/
+// title/description/tags are STAMPED by `rafa okf` at emit from run/outcome/tier —
+// authored here WITHOUT them so the emitter owns derivation (no guessed values).
+// Deletion + refutation records ride INSIDE the body (prose locators, NOT the
+// frontmatter `cites:` DSL — a deletion record cites code that is GONE, so it must
+// never enter the machine cite surface the checker re-verifies). ONE class, three
+// record kinds as sections.
+export function reconReportId(mergeSha) {
+  return `run-${String(mergeSha).slice(0, 12)}`;
+}
+export function renderReconReport({ mergeSha, outcome, tier, reconciliationId, outcomeRows, meter }) {
+  const id = reconReportId(mergeSha);
+  const by = (k) => outcomeRows.filter((r) => r.outcome === k);
+  const section = (heading, rows, fmt) => [
+    `## ${heading}`,
+    ...(rows.length ? rows.map(fmt) : ["* none this run."]),
+    "",
+  ];
+  const line = (r) => `* \`${r.path}\` (branch ${r.branch} @ ${r.capturedAtSha}) — ${r.detail}`;
+  const body = [
+    `<!-- reconciliation ${reconciliationId} · merge ${mergeSha} -->`,
+    "",
+    ...section("Refined", [...by("banked"), ...by("rewritten")], line),
+    ...section("Refuted", by("refuted"), line),
+    ...section("Pruned", by("pruned"), line),
+    ...section("Folded (superseded, latest-merge-wins)", by("folded"), line),
+    "## Meter",
+    `* machineSeconds: ${meter?.machineSeconds ?? 0}`,
+    `* tokens: ${meter?.tokens ?? 0}`,
+    "",
+  ].join("\n");
+  const frontmatter = [
+    "---",
+    "schemaVersion: 1",
+    `id: ${id}`,
+    `run: ${mergeSha}`,
+    `outcome: ${outcome}`,
+    `tier: ${tier}`,
+    "---",
+    "",
+  ].join("\n");
+  return { id, path: `brain/reconciliations/${id}.md`, content: frontmatter + body };
+}

package/lib/distiller/state-plane.mjs

@@ -0,0 +1,159 @@
+// The reconcile_* state-plane protocol, client side (reconciliation orchestrator,
+// spec §2.6 / refinery-arc-p1-state-plane). boot.sh already won the claim and
+// handed us the attempt token; from here the distiller must:
+//   heartbeat  — extend the lease every ~2 min wall-clock (a dead run's lease
+//                expires and the queue sweep re-queues the row — completeness).
+//   log_append — batch narration ~1 s and ship it as chunks (the live tail).
+//   report     — the terminal write: node delta + per-branch outcomes + meter
+//                (success) OR errorClass + message (failure).
+//
+// EVERY write carries the attempt fence token: a write from a superseded attempt
+// (the run was killed and re-claimed → attempt++) is rejected by the queue as a
+// zombie (`stale-attempt`). That is the fencing half of exit-gate case 2, provable
+// at this boundary — we always send `attempt`, and a fenced write comes back
+// `ok:false`. The queue-side detection + retry is proven in reconcileStatePlane.test.
+//
+// Timers + the platform `call` are INJECTED so the whole surface is unit-testable
+// with a manual clock and a fake platform (no real 2-min waits, no real network).
+
+// Heartbeat: fire reconcile_heartbeat on an interval. `beat()` also fires on demand
+// (e.g. a phase transition). A beat that comes back fenced/not-running means WE are
+// the zombie — stop and surface it via onFenced; the loop aborts rather than push.
+export function startHeartbeat({
+  call,
+  reconciliationId,
+  attempt,
+  actor,
+  phase = "reconciling",
+  intervalMs = 120_000, // ~2 min wall-clock
+  setTimer = setInterval,
+  clearTimer = clearInterval,
+  onFenced = () => {},
+}) {
+  let currentPhase = phase;
+  let stopped = false;
+  let handle = null;
+
+  async function beat() {
+    if (stopped) return { ok: false, reason: "stopped" };
+    let res;
+    try {
+      res = await call("reconcile_heartbeat", {
+        reconciliationId,
+        attempt,
+        phase: currentPhase,
+        actorMeta: actor,
+      });
+    } catch (e) {
+      // Transport hiccup on a single beat is non-fatal — the lease still has slack;
+      // the NEXT beat retries. A persistent outage lets the lease expire (sweep
+      // re-queues), which is the correct completeness behaviour.
+      return { ok: false, reason: "transport", error: e instanceof Error ? e.message : String(e) };
+    }
+    if (res && res.ok === false && (res.reason === "stale-attempt" || res.reason === "not-running")) {
+      stop();
+      onFenced(res.reason);
+    }
+    return res;
+  }
+
+  function setPhase(p) {
+    currentPhase = p;
+  }
+
+  function stop() {
+    stopped = true;
+    if (handle) clearTimer(handle);
+    handle = null;
+  }
+
+  handle = setTimer(() => {
+    void beat();
+  }, intervalMs);
+
+  return { beat, setPhase, stop };
+}
+
+// Log batcher: buffer narration lines and flush them as chunks on a ~1 s cadence
+// (and on demand). A batch the queue rejects for a credential pattern is DROPPED
+// with a value-free warning (names are contracts, values are secrets — the queue
+// screens per line, we never echo the line). Attempt-fenced like every write.
+export function makeLogBatcher({
+  call,
+  reconciliationId,
+  attempt,
+  actor,
+  flushMs = 1000,
+  setTimer = setInterval,
+  clearTimer = clearInterval,
+  warn = () => {},
+}) {
+  let buffer = [];
+  let stopped = false;
+  let handle = null;
+
+  async function flush() {
+    if (buffer.length === 0) return { ok: true, appended: 0 };
+    const chunks = buffer;
+    buffer = [];
+    let res;
+    try {
+      res = await call("reconcile_log_append", {
+        reconciliationId,
+        attempt,
+        chunks,
+        actorMeta: actor,
+      });
+    } catch (e) {
+      // Never fatal: dropping a log batch loses narration, never correctness.
+      return { ok: false, reason: "transport", error: e instanceof Error ? e.message : String(e) };
+    }
+    if (res && res.ok === false && res.reason === "secret-detected") {
+      // The queue named the position, never the content — keep it that way.
+      warn(`log batch dropped: a line matched a credential pattern (chunk ${res.chunkIndex}) — not stored, value never echoed`);
+    }
+    return res;
+  }
+
+  function append(...lines) {
+    if (stopped) return;
+    for (const l of lines) if (l != null && String(l).length) buffer.push(String(l));
+  }
+
+  async function stop() {
+    stopped = true;
+    if (handle) clearTimer(handle);
+    handle = null;
+    return flush(); // final drain
+  }
+
+  handle = setTimer(() => {
+    void flush();
+  }, flushMs);
+
+  return { append, flush, stop };
+}
+
+// The terminal write. A SUCCESS payload carries the node delta + per-branch
+// outcomes + meter (the pointer advance composes atomically queue-side). A run
+// that refuted EVERYTHING is still a success — refutations are outcomes, not
+// infrastructure failures (exit-gate: refute-everything is GREEN). A FAILURE
+// payload carries the errorClass (deterministic → needs-attention now; transient
+// → backoff retry) + a message.
+export async function reportSuccess({ call, reconciliationId, attempt, actor, parents, delta, outcomes, meter }) {
+  return call("reconcile_report", {
+    reconciliationId,
+    attempt,
+    actorMeta: actor,
+    result: { kind: "success", parents, delta, outcomes, meter },
+  });
+}
+
+export async function reportFailure({ call, reconciliationId, attempt, actor, errorClass, message }) {
+  return call("reconcile_report", {
+    reconciliationId,
+    attempt,
+    actorMeta: actor,
+    result: { kind: "failure", errorClass, message },
+  });
+}