@opsee/cli 0.11.9

package/src/foreman/core/reconcile.ts

@@ -0,0 +1,227 @@
+/**
+ * Reconcile (see ../../../CONTEXT.md; story 51, ADR-0009): what the Foreman does on start and
+ * every tick. It compares the Process Table against live Worker processes and the Run Record
+ * against Tracker state, then resumes, restarts, or releases each in-flight Task:
+ *
+ * - The Worker's pid is alive: leave it. A daemon that died left the Worker running detached; it
+ *   finishes on its own, and the tick after it exits resumes its session to collect the report.
+ * - The Run Record already has the attempt event for this attempt: the turn was settled (the
+ *   attempt event is the commit) and only the row is stale, so release it: remove the row.
+ * - Dead, with a session id: resume the turn by session id in the same Workspace; the attempt
+ *   event it ends with carries a `resumed` marker.
+ * - Dead, no session id: the Worker never got as far as `started`, so nothing can be resumed;
+ *   dispatch the Task again, which is attempt+1.
+ * - A row under another Account is not this Run's to touch (half-finished work never changes
+ *   Account); it waits for a Run under its own.
+ *
+ * The human's steering marks (stories 53-56) come before the process facts: a `cancelled` row
+ * whose attempt is not yet recorded is settled as cancelled once its pid is dead (the owner of the
+ * turn died, or nobody owned it; while the pid lives, the owner settles it when the stream ends);
+ * an `attached` row is left alone whatever its pid, a human has the session; a `released` row
+ * resumes whatever its pid says, with the prompt saying an attended turn happened. And
+ * while the Initiative is paused (story 55) nothing new is dispatched: a row with no session to
+ * resume waits for the resume instead of being restarted.
+ *
+ * `planReconcile` is the pure decision, tested on its own; `reconcile` reads the Run Record for
+ * each row, applies the plan through the callbacks run.ts supplies, and never lets one row's
+ * failure stop the others: a Tracker that cannot be reached leaves every row for the next tick.
+ */
+import type { Account } from "../account.js";
+import type { TrackerAdapter } from "../tracker-adapter.js";
+import type { ProcessTableApi, WorkerRow } from "./process-table.js";
+import { pidIsAlive } from "./process-table.js";
+import { isTriageSummary } from "./triage.js";
+import { stopProcessByPid } from "../process-group.js";
+import { errorMessage, KILL_GRACE_MS } from "../worker-process.js";
+
+/** The Status Label a Task wears while a Worker has it (mirrors STATUS_LABELS.running in run.ts,
+ * kept here so reconcile does not import the loop). */
+const RUNNING_LABEL = "foreman:running";
+
+export type ReconcileDecision =
+  | { action: "leave"; row: WorkerRow; reason: string }
+  | { action: "skip"; row: WorkerRow; reason: string }
+  | { action: "release"; row: WorkerRow; reason: string; finish?: { outcome: string } }
+  /** `released` is true when a human had the session as an attended turn and handed it back. */
+  | { action: "resume"; row: WorkerRow; sessionId: string; released: boolean }
+  | { action: "restart"; row: WorkerRow }
+  /** `foreman cancel` stopped the Worker and nobody closed the attempt: record it as cancelled. */
+  | { action: "cancel"; row: WorkerRow };
+
+export interface RowFacts {
+  alive: boolean;
+  /** The outcome of the Run Record's attempt event for this row's attempt, when one exists. */
+  settledOutcome?: string;
+  /** The Task still wears `foreman:running`: the attempt event was written but the Foreman died
+   * before the comment and the board followed, so the release must finish that settlement. */
+  stillRunning?: boolean;
+  /** New dispatch is paused for the Initiative (story 55). */
+  paused?: boolean;
+  /** This Foreman has the turn in flight right now, in a Slot of the Run that is reconciling
+   * (OPS-276). Only true once dispatch became concurrent: with one Worker at a time, every row a
+   * tick saw belonged to some earlier process. */
+  owned?: boolean;
+}
+
+export function planReconcile(row: WorkerRow, account: Account, facts: RowFacts): ReconcileDecision {
+  if (row.account !== account.name) return { action: "skip", row, reason: `under Account "${row.account}", not "${account.name}"` };
+  // A turn this Foreman is driving is nobody else's to settle, whatever the row says: the pid may
+  // be unset (an adapter that has none), the session may not have arrived yet, and restarting it
+  // beside the live Worker would dispatch the Task twice.
+  if (facts.owned) return { action: "leave", row, reason: "this Foreman has the turn in flight" };
+  if (facts.settledOutcome !== undefined) {
+    return {
+      action: "release",
+      row,
+      reason: `attempt ${row.attempt} already recorded as ${facts.settledOutcome}${facts.stillRunning ? ", but the Task still says running: finishing its settlement" : ""}`,
+      ...(facts.stillRunning ? { finish: { outcome: facts.settledOutcome } } : {}),
+    };
+  }
+  if (row.control === "cancelled") {
+    // The cancel command marks the row before it stops the process; if it died in between, the
+    // Worker is still running and its owner settles the attempt as cancelled when the stream ends.
+    // Closing it here as well would record the attempt twice.
+    if (facts.alive) return { action: "leave", row, reason: `cancelled but pid ${row.pid} is still alive; the owner of the turn, or another foreman cancel, closes it` };
+    return { action: "cancel", row };
+  }
+  if (row.control === "attached") return { action: "leave", row, reason: "attached: a human has the session until foreman release" };
+  // A released row resumes whatever its pid says: attach stopped the unattended turn and cleared
+  // the pid, and a pid it might still carry (an attach from before that) is dead or recycled.
+  if (row.control === "released" && row.sessionId) return { action: "resume", row, sessionId: row.sessionId, released: true };
+  if (facts.alive) return { action: "leave", row, reason: `pid ${row.pid} is alive` };
+  if (row.sessionId) return { action: "resume", row, sessionId: row.sessionId, released: false };
+  if (facts.paused) return { action: "leave", row, reason: "no session to resume and the Run is paused: dispatched again after foreman resume" };
+  return { action: "restart", row };
+}
+
+export interface ReconcileDeps<T> {
+  table: Pick<ProcessTableApi, "liveWorkers" | "removeWorker"> & Partial<Pick<ProcessTableApi, "setAppPid">>;
+  tracker: Pick<TrackerAdapter, "readRunRecord" | "getTask">;
+  account: Account;
+  log: (line: string) => void;
+  /** Finishes a settlement the attempt event began and a crash cut short: the comment and the
+   * board. Absent, an unfinished settlement is only logged. */
+  finishSettlement?: (row: WorkerRow, outcome: string) => Promise<void>;
+  /** Runs the rest of the turn from the stored session and settles it; `released` says a human
+   * had it as an attended turn in between. Undefined when the turn could not be started at all —
+   * a Run that is stopping does not wait for one of its Slots — and the row is left as it is. */
+  resume: (row: WorkerRow, sessionId: string, released: boolean) => Promise<T | undefined>;
+  /** Dispatches the Task again, attempt+1. Undefined has the same meaning as on `resume`. */
+  restart: (row: WorkerRow) => Promise<T | undefined>;
+  /** Closes the attempt of a Worker `foreman cancel` stopped: the attempt event, the comment,
+   * the board, the row. Absent, a cancelled row is only logged. */
+  cancel?: (row: WorkerRow) => Promise<T | undefined>;
+  /** Whether new dispatch is paused for the Initiative; false by default. */
+  paused?: (initiativeId: number) => boolean;
+  /** Whether this Foreman has the Task's turn in flight in one of its Slots; false by default,
+   * which is what a Reconcile outside a Run (the daemon's idle tick) wants. */
+  owns?: (taskId: number) => boolean;
+  isAlive?: (pid: number | undefined) => boolean;
+  /** Stops the app a Verifier round left running (`WorkerRow.appPid`) when the Worker that owned
+   * the round is gone: a Foreman that died mid-round orphaned it. Resolves true when something was
+   * stopped. `stopProcessByPid` with the row's start time by default; a test scripts one. */
+  stopApp?: (row: WorkerRow) => Promise<boolean>;
+}
+
+const defaultStopApp = async (row: WorkerRow): Promise<boolean> => (row.appPid === undefined ? false : stopProcessByPid(row.appPid, KILL_GRACE_MS, { startedAt: row.startedAt }));
+
+export interface ReconcileOutcome<T> {
+  decisions: ReconcileDecision[];
+  /** What the resumed and restarted turns returned, in order; a turn that was not started at all
+   * (no Slot, the Run stopping) contributes nothing. */
+  results: T[];
+}
+
+/** Reconciles the Process Table rows of one Initiative. */
+export async function reconcile<T>(deps: ReconcileDeps<T>, initiativeId: number): Promise<ReconcileOutcome<T>> {
+  const isAlive = deps.isAlive ?? pidIsAlive;
+  const decisions: ReconcileDecision[] = [];
+  const results: T[] = [];
+  const rows = deps.table.liveWorkers({ initiativeId });
+  if (rows.length) deps.log(`reconcile: ${rows.length} Process Table ${rows.length === 1 ? "row" : "rows"} for Initiative ${initiativeId}`);
+  const paused = deps.paused?.(initiativeId) ?? false;
+  for (const row of rows) {
+    let facts: RowFacts;
+    try {
+      const settled = await settledOutcome(deps.tracker, row);
+      const stillRunning = settled === undefined ? false : (await deps.tracker.getTask(row.taskId)).task.labels.includes(RUNNING_LABEL);
+      facts = { alive: isAlive(row.pid), settledOutcome: settled, stillRunning, paused, owned: deps.owns?.(row.taskId) ?? false };
+    } catch (error) {
+      deps.log(`reconcile: ${row.identifier}: could not read the Run Record, leaving it for the next tick: ${errorMessage(error)}`);
+      continue;
+    }
+    const decision = planReconcile(row, deps.account, facts);
+    decisions.push(decision);
+    // A Verifier round's app outlives the Foreman that started it (spawned detached). Once the
+    // Worker of the row is gone, nothing owns that app any more, whatever else happens to the row.
+    if (row.appPid !== undefined && !facts.alive && decision.action !== "skip") {
+      try {
+        const stopped = await (deps.stopApp ?? defaultStopApp)(row);
+        deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: the Verifier's app (pid ${row.appPid}) ${stopped ? "was still running and has been stopped" : "is already gone"}`);
+      } catch (error) {
+        deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: could not stop the Verifier's app (pid ${row.appPid}): ${errorMessage(error)}`);
+      }
+      if ("setAppPid" in deps.table) (deps.table as Pick<ProcessTableApi, "setAppPid">).setAppPid(row.taskId, undefined);
+    }
+    switch (decision.action) {
+      case "leave":
+        deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: ${decision.reason}, leaving it`);
+        break;
+      case "skip":
+        deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: ${decision.reason}, skipping`);
+        break;
+      case "release":
+        if (decision.finish) {
+          if (deps.finishSettlement) await deps.finishSettlement(row, decision.finish.outcome);
+          else deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: settlement unfinished and nothing to finish it with`);
+        }
+        deps.table.removeWorker(row.taskId);
+        deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: ${decision.reason}, released`);
+        break;
+      case "resume":
+        deps.log(
+          decision.released
+            ? `reconcile: ${row.identifier} attempt ${row.attempt}: released by the human, resuming session ${decision.sessionId} unattended in ${row.workspace}`
+            : `reconcile: ${row.identifier} attempt ${row.attempt}: Worker gone (pid ${row.pid ?? "unknown"}), resuming session ${decision.sessionId} in ${row.workspace}`,
+        );
+        push(results, await deps.resume(row, decision.sessionId, decision.released));
+        break;
+      case "restart":
+        deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: Worker gone before it started a session, dispatching again`);
+        push(results, await deps.restart(row));
+        break;
+      case "cancel":
+        if (deps.cancel) {
+          deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: cancelled by the human and not yet recorded, closing the attempt`);
+          push(results, await deps.cancel(row));
+        } else {
+          deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: cancelled by the human and nothing to close the attempt with`);
+        }
+        break;
+    }
+  }
+  return { decisions, results };
+}
+
+function push<T>(results: T[], result: T | undefined): void {
+  if (result !== undefined) results.push(result);
+}
+
+/**
+ * Whether this row's attempt has already been settled on the Run Record, and how it ended.
+ *
+ * The Triage turn is skipped: it is given the attempt number the implementer turn is about to take
+ * and writes its event *before* the dispatch event, so an attempt can carry two attempt events with
+ * triage first (core/triage.ts `TRIAGE_MARKER`). Taking the first match would read a drafting turn
+ * that completed as the settlement of an implementer turn that is still running — the row would be
+ * dropped instead of resumed, its session lost, and the Task blocked with "the Foreman stopped
+ * before finishing". That is precisely the case Reconcile exists for.
+ */
+async function settledOutcome(tracker: Pick<TrackerAdapter, "readRunRecord">, row: WorkerRow): Promise<string | undefined> {
+  const { events } = await tracker.readRunRecord(row.initiativeId, { taskId: row.taskId, kinds: ["attempt"] });
+  for (const e of events) {
+    const p = e.payload?.payload;
+    if (p?.case === "attempt" && p.value.attempt === row.attempt && !isTriageSummary(p.value.summary)) return p.value.outcome;
+  }
+  return undefined;
+}

package/src/foreman/core/report.ts

@@ -0,0 +1,60 @@
+/**
+ * Completion Report -> Tracker writes (story 44, ADR-0008). A Worker's report becomes typed
+ * Initiative memory entries with the Task as source: one `outcome` entry carrying the summary,
+ * decisions and what was tried; one `learning` entry per Proposed Learning; one `blocker` entry
+ * per blocker. A one-line comment on the Task points a reader of the Task alone at them. Nothing
+ * here talks to a Tracker; the Run loop hands the entries to the adapter.
+ */
+import type { CompletionReport } from "../completion-report.js";
+import type { MemoryEntry, TrackerTask } from "../tracker-adapter.js";
+
+function bulleted(title: string, items: string[]): string {
+  if (items.length === 0) return "";
+  return `\n\n${title}:\n${items.map((item) => `- ${item}`).join("\n")}`;
+}
+
+/** The `outcome` entry: the report's summary with its decisions and attempts under it. */
+export function outcomeBody(task: Pick<TrackerTask, "identifier">, report: CompletionReport): string {
+  return (
+    `Completion Report for ${task.identifier} (outcome: ${report.outcome})\n\n${report.summary.trim()}` +
+    bulleted("Decisions", report.decisions) +
+    bulleted("Tried", report.tried) +
+    (report.handOff?.prUrl ? `\n\nHand-off: ${report.handOff.prUrl}` : report.handOff?.branch ? `\n\nBranch: ${report.handOff.branch}` : "")
+  );
+}
+
+/** Every memory entry a report produces, outcome first, all sourced to the Task. */
+export function reportToMemory(task: Pick<TrackerTask, "id" | "identifier">, report: CompletionReport): MemoryEntry[] {
+  const source = { sourceTaskId: task.id, sourceUrl: report.handOff?.prUrl };
+  return [
+    { kind: "outcome", body: outcomeBody(task, report), ...source },
+    ...report.proposedLearnings.map((body): MemoryEntry => ({ kind: "learning", body, ...source })),
+    ...report.blockers.map((body): MemoryEntry => ({ kind: "blocker", body, ...source })),
+  ];
+}
+
+/** The linking comment: one line, so a Task's thread stays readable while the detail lives in the
+ * memory log. Names the outcome entry by id and the other entries by count; `note` is what the
+ * Run loop adds about the Hand-off (incomplete, or not linked), already punctuated. */
+export function reportComment(report: CompletionReport, memoryIds: number[], note = ""): string {
+  const [outcomeId] = memoryIds;
+  const counts = [
+    report.proposedLearnings.length > 0 && `${report.proposedLearnings.length} proposed learning${report.proposedLearnings.length === 1 ? "" : "s"}`,
+    report.blockers.length > 0 && `${report.blockers.length} blocker${report.blockers.length === 1 ? "" : "s"}`,
+  ].filter((c): c is string => !!c);
+  const where = `Initiative memory (outcome entry #${outcomeId}${counts.length ? `, ${counts.join(", ")}` : ""})`;
+  const handOff = report.handOff?.prUrl ? `; Hand-off ${report.handOff.prUrl}` : "";
+  const reason = report.outcome === "done" ? "" : ` ${firstSentence(report.blockers[0] ?? report.summary)}`;
+  return `Foreman: Completion Report (${report.outcome}) recorded in ${where}${handOff}.${reason}${note}`;
+}
+
+/** The comment for a turn that ended without a report: the typed reason and the vendor's text. */
+export function failureComment(reason: string, message: string): string {
+  return `Foreman: Worker turn failed (${reason}): ${firstSentence(message)}`;
+}
+
+function firstSentence(text: string): string {
+  const trimmed = text.trim().replace(/\s+/g, " ");
+  const end = trimmed.search(/[.!?](\s|$)/);
+  return end === -1 ? trimmed : trimmed.slice(0, end + 1);
+}