forgedock 1.0.21 → 1.1.1

package/bin/engine/phases.mjs

@@ -0,0 +1,148 @@
+/**
+ * Declarative phase table for the headless work-on pipeline. The ENGINE (not an
+ * LLM) chooses the next phase via pickPhase. Each phase's outcome is read from
+ * GitHub/git AFTER the run (detectOutcome); the runner's return is advisory.
+ * @typedef ... (see plan "Shared types")
+ */
+
+export const TERMINAL_REASONS = ["merged", "invalid", "needs-human", "decomposed"];
+
+/** Fetch the issue's comment bodies as one blob for marker checks. */
+async function issueMarkers(issue, io) {
+  const out = await io.gh(["api", `repos/{owner}/{repo}/issues/${issue}/comments`, "--jq", ".[].body"]);
+  return out || "";
+}
+/**
+ * Count commits on branch ahead of the lane base. On the first build the
+ * branch does not exist yet, so real git rejects the ref range — swallow
+ * that (and any other git failure) as "0 ahead" rather than letting it
+ * propagate and crash runIssue (C1).
+ */
+async function commitsAhead(state, io) {
+  try {
+    const n = await io.git(["rev-list", "--count", `origin/${state.lane}..${state.branch}`]);
+    return parseInt(String(n).trim(), 10) || 0;
+  } catch {
+    return 0;
+  }
+}
+function has(blob, marker) { return blob.includes(marker); }
+
+/** @type {Phase[]} */
+export const PHASES = [
+  {
+    id: "investigate",
+    command: "work-on/investigate",
+    entryCondition: () => true,
+    async detectOutcome(state, io) {
+      const m = await issueMarkers(state.issue, io);
+      if (has(m, "INVESTIGATION:INVALID"))
+        return { status: "committed", terminalReason: "invalid", outputs: { verdict: "INVALID" } };
+      if (has(m, "DECOMPOSE:YES"))
+        return { status: "committed", terminalReason: "decomposed", outputs: { decompose: true } };
+      if (has(m, "INVESTIGATION:COMPLETE"))
+        return { status: "committed", outputs: { verdict: "CONFIRMED" } };
+      return { status: "failed", detail: "no INVESTIGATION:COMPLETE marker" };
+    },
+    isTerminalAfter: (s) => s.terminalReason === "invalid" || s.terminalReason === "decomposed",
+  },
+  {
+    id: "context",
+    command: "work-on/build/context",
+    entryCondition: (s) => s.committed.includes("investigate"),
+    async detectOutcome(state, io) {
+      const m = await issueMarkers(state.issue, io);
+      // Context is non-critical: a missing marker is a VISIBLE skip, not a hard fail (spec §7).
+      if (has(m, "FORGE:CONTEXT")) return { status: "committed", outputs: {} };
+      return { status: "committed", outputs: { skipped: true, which: "context" } };
+    },
+  },
+  {
+    id: "architect",
+    command: "work-on/build/architect",
+    entryCondition: (s) => s.committed.includes("context"),
+    async detectOutcome(state, io) {
+      const m = await issueMarkers(state.issue, io);
+      return has(m, "FORGE:ARCHITECT")
+        ? { status: "committed", outputs: {} }
+        : { status: "failed", detail: "no FORGE:ARCHITECT" };
+    },
+  },
+  {
+    id: "build",
+    command: "work-on/build",
+    entryCondition: (s) => s.committed.includes("architect"),
+    async reconcile(state, io) {
+      // Idempotent resume: branch already ahead of base → treat as done, skip the LLM.
+      if (state.branch && (await commitsAhead(state, io)) > 0) {
+        const m = await issueMarkers(state.issue, io);
+        if (has(m, "FORGE:BUILDER:COMPLETE")) return { satisfied: true, outputs: { branch: state.branch } };
+      }
+      return { satisfied: false };
+    },
+    async detectOutcome(state, io) {
+      const m = await issueMarkers(state.issue, io);
+      const complete = has(m, "FORGE:BUILDER:COMPLETE");           // #1305: require :COMPLETE …
+      const ahead = state.branch ? await commitsAhead(state, io) : 0; // … AND real commits
+      if (complete && ahead > 0) return { status: "committed", outputs: { branch: state.branch } };
+      return { status: "failed", detail: `builder complete=${complete} commitsAhead=${ahead}` };
+    },
+  },
+  {
+    id: "review",
+    command: "work-on/review",
+    entryCondition: (s) => s.committed.includes("build"),
+    async reconcile(state, io) {
+      const pr = await openPrFor(state, io);   // adopt an existing PR instead of opening a second
+      return pr ? { satisfied: false, outputs: { pr } } : { satisfied: false };
+    },
+    async detectOutcome(state, io) {
+      const pr = await prStatusFor(state, io);
+      if (!pr) return { status: "failed", detail: "no PR created" };
+      if (pr.merged) return { status: "committed", outputs: { pr: pr.number } };
+      if (pr.needsHuman) return { status: "blocked", detail: "review escalated", outputs: { pr: pr.number } };
+      return { status: "failed", detail: "PR open, not merged" };
+    },
+  },
+  {
+    id: "close",
+    command: "work-on/close",
+    entryCondition: (s) => s.committed.includes("review"),
+    async detectOutcome(state, io) {
+      const out = await io.gh(["issue", "view", String(state.issue), "--json", "state,labels"]);
+      let j;
+      try { j = JSON.parse(out || "{}"); } catch { return { status: "failed", detail: "malformed gh response" }; }
+      const labels = (j.labels || []).map((l) => l.name || l);
+      if (j.state === "CLOSED" || labels.includes("workflow:merged"))
+        return { status: "committed", terminalReason: "merged", outputs: {} };
+      return { status: "failed", detail: "issue not closed" };
+    },
+    isTerminalAfter: () => true,
+  },
+];
+
+async function openPrFor(state, io) {
+  if (!state.branch) return null;
+  const out = await io.gh(["pr", "list", "--head", state.branch, "--json", "number", "--state", "all"]);
+  try { const a = JSON.parse(out || "[]"); return a[0]?.number ?? null; } catch { return null; }
+}
+async function prStatusFor(state, io) {
+  const n = await openPrFor(state, io);
+  if (!n) return null;
+  const out = await io.gh(["pr", "view", String(n), "--json", "number,state,labels,mergedAt"]);
+  let j;
+  try { j = JSON.parse(out || "{}"); } catch { return null; }
+  const labels = (j.labels || []).map((l) => l.name || l);
+  return { number: j.number, merged: !!j.mergedAt || j.state === "MERGED",
+           needsHuman: labels.includes("needs-human") };
+}
+
+/** The engine's transition function: first uncommitted phase whose gate holds. */
+export function pickPhase(state) {
+  if (state.terminal) return null;
+  for (const p of PHASES) {
+    if (state.committed.includes(p.id)) continue;
+    if (p.entryCondition(state)) return p;
+  }
+  return null;
+}

package/bin/engine/projector.mjs

@@ -0,0 +1,32 @@
+/**
+ * GitHub adapter: reads/writes the FORGE:STATE block and workflow labels.
+ * `io.gh(args)` is injected (async, returns stdout) so this is testable offline.
+ */
+import { parseState, upsertStateBlock } from "./state.mjs";
+
+export function makeProjector(io) {
+  const gh = io.gh;
+
+  async function getBody(issue) {
+    const out = await gh(["issue", "view", String(issue), "--json", "body"]);
+    try { return JSON.parse(out).body ?? ""; } catch { return ""; }
+  }
+
+  return {
+    /** @returns {Promise<import("./phases.mjs").RunState|null>} */
+    async readState(issue) { return parseState(await getBody(issue)); },
+
+    async writeState(issue, index) {
+      const body = upsertStateBlock(await getBody(issue), index);
+      await gh(["issue", "edit", String(issue), "--body", body]);
+    },
+
+    async setLabel(issue, label) {
+      await gh(["issue", "edit", String(issue), "--add-label", label]);
+    },
+
+    async removeLabel(issue, label) {
+      await gh(["issue", "edit", String(issue), "--remove-label", label]);
+    },
+  };
+}

package/bin/engine/reconcile.mjs

@@ -0,0 +1,11 @@
+/**
+ * Merge the local run-log state and the remote FORGE:STATE index.
+ * Rule: GitHub wins on divergence (spec §5.3).
+ */
+export function reconcileState(local, remote) {
+  if (!remote) return { state: local ?? null, action: local ? "local" : "fresh" };
+  if (!local) return { state: remote, action: "hydrate" };
+  if (remote.v > local.v) return { state: remote, action: "hydrate" };   // advanced elsewhere
+  if (remote.v < local.v) return { state: local, action: "remirror" };   // crashed pre-mirror
+  return { state: local, action: "local" };
+}

package/bin/engine/runlog.mjs

@@ -0,0 +1,79 @@
+/**
+ * Append-only per-issue run-log (the crash-safe local hot path).
+ * One JSON event per line at {dir}/{issue}.jsonl. seq is assigned monotonically.
+ */
+import { readFileSync, appendFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+
+function logPath(dir, issue) { return join(dir, `${issue}.jsonl`); }
+
+/** Append an event; assigns the next seq based on the current line count. */
+export function appendEvent(dir, issue, event) {
+  mkdirSync(dir, { recursive: true });
+  const path = logPath(dir, issue);
+  const seq = readLog(dir, issue).length + 1;
+  appendFileSync(path, JSON.stringify({ seq, ...event }) + "\n");
+}
+
+/**
+ * Overwrite the local run-log with a fresh set of events (fresh seq each).
+ * Used to reconstruct the local log from a remote FORGE:STATE index on
+ * hydrate (C2), so the local log and the GitHub-wins state stay consistent.
+ */
+export function rewriteLog(dir, issue, events) {
+  mkdirSync(dir, { recursive: true });
+  const path = logPath(dir, issue);
+  const lines = events.map((e, i) => JSON.stringify({ seq: i + 1, ...e }));
+  writeFileSync(path, lines.length ? lines.join("\n") + "\n" : "");
+}
+
+/** Read all complete events; a truncated final line (crash mid-write) is ignored. */
+export function readLog(dir, issue) {
+  const path = logPath(dir, issue);
+  if (!existsSync(path)) return [];
+  const lines = readFileSync(path, "utf-8").split("\n");
+  // Drop trailing empty lines to find the last non-empty line.
+  while (lines.length > 0 && !lines[lines.length - 1].trim()) lines.pop();
+
+  const out = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line.trim()) continue;
+    try {
+      out.push(JSON.parse(line));
+    } catch (err) {
+      // Only tolerate parse failure on the final non-empty line (crash mid-write).
+      if (i === lines.length - 1) {
+        break; // Ignore truncated final line.
+      }
+      throw new Error(`corrupt run-log line ${i + 1} in ${path}`);
+    }
+  }
+  return out;
+}
+
+/** Fold events into a RunState. The commit rule lives here. */
+export function deriveState(events) {
+  /** @type {import("./phases.mjs").RunState} */
+  const s = { v: 0, run: null, issue: null, lane: "staging", committed: [],
+              phase: null, branch: null, pr: null, terminal: false,
+              terminalReason: null, lease: null };
+  for (const e of events) {
+    switch (e.event) {
+      case "RUN_START":
+        s.run = e.run ?? s.run; s.issue = e.issue ?? s.issue; s.lane = e.lane ?? s.lane;
+        break;
+      case "PHASE_COMMIT":
+        if (!s.committed.includes(e.phase)) s.committed.push(e.phase);
+        s.v = e.seq;
+        if (e.outputs?.branch) s.branch = e.outputs.branch;
+        if (e.outputs?.pr != null) s.pr = e.outputs.pr;
+        break;
+      case "RUN_TERMINAL":
+        s.terminal = true; s.terminalReason = e.reason ?? "done"; s.v = e.seq;
+        break;
+      // PHASE_START / PHASE_FAILED do not change committed state
+    }
+  }
+  return s;
+}

package/bin/engine/state.mjs

@@ -0,0 +1,27 @@
+/**
+ * Codec for the single machine-readable FORGE:STATE HTML-comment block on an
+ * issue body. Carries the COMPACT INDEX only (no rich per-phase outputs).
+ */
+const OPEN = "<!-- FORGE:STATE";
+const CLOSE = "-->";
+// NOTE: a literal "-->" inside a serialized value would close the block early; terminalReason etc. should not contain it. Tracked as a follow-up.
+const BLOCK_RE = /<!-- FORGE:STATE\s*([\s\S]*?)-->/;
+
+/** @param {import("./phases.mjs").RunState} index */
+export function serializeState(index) {
+  return `${OPEN}\n${JSON.stringify(index)}\n${CLOSE}`;
+}
+
+/** @returns {import("./phases.mjs").RunState|null} */
+export function parseState(issueBody) {
+  const m = BLOCK_RE.exec(issueBody || "");
+  if (!m) return null;
+  try { return JSON.parse(m[1].trim()); } catch { return null; }
+}
+
+/** Replace the FORGE:STATE block in place, or append one if absent. */
+export function upsertStateBlock(body, index) {
+  const block = serializeState(index);
+  if (BLOCK_RE.test(body || "")) return body.replace(BLOCK_RE, () => block);
+  return `${body || ""}\n\n${block}`.trimStart();
+}

package/bin/engine-cli.mjs

@@ -0,0 +1,50 @@
+/**
+ * Headless entry point: `forgedock run-issue <issue>` drives one issue through the
+ * durable engine; scanStalls finds dead-lease issues for the orchestrator to resume.
+ */
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { runIssue } from "./engine.mjs";
+
+const pexec = promisify(execFile);
+
+/** Real gh/git accessors. */
+export function makeIo() {
+  const run = (bin) => async (args) => {
+    const { stdout } = await pexec(bin, args, { maxBuffer: 100 * 1024 * 1024 });
+    return stdout;
+  };
+  return { gh: run("gh"), git: run("git") };
+}
+
+export function runDir() { return join(homedir(), ".forge", "runs"); }
+
+/**
+ * @param {number[]} issues
+ * @param {{readState:(i:number)=>Promise<{terminal:boolean,lease:?{until:number}}|null>}} io
+ * @param {number} now
+ * @returns {Promise<number[]>} issues that appear stalled (expired lease, non-terminal)
+ */
+export async function scanStalls(issues, io, now) {
+  const stalled = [];
+  for (const i of issues) {
+    const s = await io.readState(i);
+    if (s && !s.terminal && s.lease && s.lease.until < now) stalled.push(i);
+  }
+  return stalled;
+}
+
+export async function runFromCli(argv) {
+  const issue = parseInt(argv[0], 10);
+  if (!Number.isInteger(issue)) throw new Error("usage: forgedock run-issue <issue-number>");
+  const lane = flag(argv, "--lane") || "staging";
+  const io = makeIo();
+  const agentId = `cli_${process.pid}`;
+  const res = await runIssue({ issue, dir: runDir(), agentId, lane, io,
+    runner: (await import("./runner.mjs")).runCommand, now: () => Date.now() });
+  console.log(`issue #${issue} → ${res.terminalReason}`);
+  return res;
+}
+function flag(argv, name) { const i = argv.indexOf(name); return i >= 0 ? argv[i + 1] : null; }

package/bin/engine.mjs

@@ -0,0 +1,138 @@
+/**
+ * Durable per-phase engine loop. Drives one pipeline phase at a time via an
+ * injected runner (runCommand-shaped), determining each phase's outcome from
+ * GitHub state (phase.detectOutcome). All effects are injected → fully testable.
+ */
+import { fileURLToPath } from "node:url";
+import { appendEvent, readLog, deriveState, rewriteLog } from "./engine/runlog.mjs";
+import { pickPhase, TERMINAL_REASONS } from "./engine/phases.mjs";
+import { reconcileState } from "./engine/reconcile.mjs";
+import { makeProjector } from "./engine/projector.mjs";
+
+const DEFAULT_MAX_ATTEMPTS = 3;
+
+export async function runIssue(opts) {
+  const { issue, dir, agentId, lane = "staging", io, runner,
+          now = () => Date.now(), maxAttempts = DEFAULT_MAX_ATTEMPTS,
+          commandsDir = fileURLToPath(new URL("../commands", import.meta.url)) } = opts;
+  const projector = makeProjector(io);
+
+  // 1. Load + reconcile (GitHub wins).
+  const local = readLog(dir, issue).length ? deriveState(readLog(dir, issue)) : null;
+  const remote = await projector.readState(issue);
+  let { state, action } = reconcileState(local, remote);
+
+  // I3 (best-effort): GitHub issue-edit is not an atomic CAS, so this is a
+  // courtesy check, not a hard mutual-exclusion guarantee — a concurrent
+  // start can still race between this read and the next lease write.
+  // MUST come before any projector.writeState() — remirror/hydrate branches
+  // must not overwrite GitHub state when another agent holds a valid lease.
+  if (remote?.lease && remote.lease.until > now() && remote.lease.by !== agentId) {
+    return { terminalReason: "deferred", detail: `issue ${issue} leased by ${remote.lease.by}` };
+  }
+
+  if (!state) {
+    state = freshState(issue, lane);
+    appendEvent(dir, issue, { event: "RUN_START", issue, run: state.run, lane });
+    await projector.writeState(issue, state);
+  } else if (action === "remirror") {
+    await projector.writeState(issue, state);
+  } else if (action === "hydrate") {
+    // GitHub is ahead of (or the only source of) local state. Rebuild the
+    // local run-log to match the remote compact index so downstream
+    // `deriveState(readLog(...))` calls stay consistent — otherwise the
+    // post-commit re-derive below would fold over an empty/stale local log
+    // and regress committed/issue/v (C2).
+    rewriteLog(dir, issue, eventsFromIndex(state));
+    state = deriveState(readLog(dir, issue));
+    await projector.writeState(issue, state);
+  }
+
+  // 2. Drive phases until terminal.
+  let phase;
+  while ((phase = pickPhase(state))) {
+    // Every issue's build works on a deterministic branch; set it before build runs.
+    if (phase.id === "build" && !state.branch) state.branch = `fix/pipeline-${issue}`;
+
+    let reconciled;
+    try {
+      reconciled = phase.reconcile ? await phase.reconcile(state, io) : { satisfied: false };
+    } catch {
+      // A reconcile probe (gh/git) failing must not crash the run — degrade
+      // to "not satisfied" so the phase runs normally instead (C1).
+      reconciled = { satisfied: false };
+    }
+    let outcome;
+    if (reconciled.satisfied) {
+      outcome = { status: "committed", outputs: reconciled.outputs || {} };
+    } else {
+      if (reconciled.outputs?.pr) state.pr = reconciled.outputs.pr;
+      outcome = await runPhaseWithRetry(phase, state, { io, runner, dir, issue, commandsDir, maxAttempts });
+    }
+
+    if (outcome.status === "blocked") return await terminate(state, "needs-human", outcome.detail);
+
+    // committed
+    appendEvent(dir, issue, { event: "PHASE_COMMIT", phase: phase.id, outputs: outcome.outputs || {} });
+    state = deriveState(readLog(dir, issue));
+    if (outcome.terminalReason) state.terminalReason = outcome.terminalReason;
+    await projector.writeState(issue, { ...state, lease: { by: agentId, until: now() + 600000 } });
+
+    if (outcome.terminalReason && TERMINAL_REASONS.includes(outcome.terminalReason))
+      return await terminate(state, outcome.terminalReason);
+    if (phase.isTerminalAfter && phase.isTerminalAfter(state))
+      return await terminate(state, state.terminalReason || "merged");
+  }
+  return await terminate(state, state.terminalReason || "merged");
+
+  async function terminate(s, reason, detail) {
+    appendEvent(dir, issue, { event: "RUN_TERMINAL", reason });
+    const final = { ...deriveState(readLog(dir, issue)), terminal: true, terminalReason: reason, lease: null };
+    await projector.writeState(issue, final);
+    if (reason === "needs-human") await projector.setLabel(issue, "needs-human");
+    return { terminalReason: reason, detail };
+  }
+}
+
+async function runPhaseWithRetry(phase, state, ctx) {
+  const { io, runner, dir, issue, commandsDir, maxAttempts } = ctx;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    appendEvent(dir, issue, { event: "PHASE_START", phase: phase.id, attempt });
+    try {
+      await runner({ commandsDir, commandName: phase.command, args: [String(issue)] });
+    } catch (e) {
+      // A missing API key / SDK is a config error, not a transient phase
+      // failure — surface it distinctly instead of burning retries and
+      // escalating to needs-human as if the LLM run itself misbehaved.
+      if (e.code === "NO_API_KEY" || e.code === "NO_SDK") throw e;
+      appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message });
+      continue;
+    }
+    const outcome = await phase.detectOutcome(state, io);
+    if (outcome.status === "committed" || outcome.status === "blocked") return outcome;
+    appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: outcome.detail });
+  }
+  // Exhausted transient retries → escalate (spec §7).
+  return { status: "blocked", detail: `phase ${phase.id} failed after ${maxAttempts} attempts` };
+}
+
+/**
+ * Build a run-log event sequence that reproduces a compact FORGE:STATE index,
+ * used to reconstruct the local run-log on hydrate (C2).
+ */
+function eventsFromIndex(idx) {
+  const events = [{ event: "RUN_START", issue: idx.issue, run: idx.run, lane: idx.lane }];
+  for (const phase of idx.committed) {
+    const outputs = {};
+    if (phase === "build" && idx.branch) outputs.branch = idx.branch;
+    if (phase === "review" && idx.pr != null) outputs.pr = idx.pr;
+    events.push({ event: "PHASE_COMMIT", phase, outputs });
+  }
+  if (idx.terminal) events.push({ event: "RUN_TERMINAL", reason: idx.terminalReason });
+  return events;
+}
+
+function freshState(issue, lane) {
+  return { v: 0, run: `r_${issue}_${lane}`, issue, lane, committed: [], phase: null,
+           branch: null, pr: null, terminal: false, terminalReason: null, lease: null };
+}