@volter/supercode-orchestrator 0.4.17 → 0.4.19

package/board/dispatch.mjs

@@ -15,12 +15,13 @@
 // subscriber's cursor moves only after its notice was delivered, so two dispatchers over one board never both
 // launch and a crash between the two steps is answered by the next tick.
 import { spawn as spawnChild, spawnSync } from 'node:child_process';
-import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync } from 'node:fs';
+import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
 import { hostname, tmpdir } from 'node:os';
 import { basename, dirname, join } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
-import { event, json, listBoards, now, openBoard, tx } from './store.mjs';
+import { boardPath, event, json, listBoards, now, openBoard, tx } from './store.mjs';
+import { holdBoardTickLock } from '../hermes-locks.mjs';
 import {
   cardContext, claimIn, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
   recomputeReady, recordFailure, release, setRunSession, subscribeIn, subscriptions,
@@ -106,6 +107,16 @@ export function boardSettings(kanban = {}) {
     grace: pos(kanban.session_grace_seconds) ?? 120,
     machines: Object.fromEntries(Object.entries(kanban.machines ?? {}).map(([name, m]) => [normalMachine(name), { capacity: pos(m?.capacity) }])),
     interval: Math.max(Number(kanban.dispatch_interval_seconds) || 60, 1),
+    staleTimeout: pos(kanban.dispatch_stale_timeout_seconds),
+    autoDecompose: kanban.auto_decompose !== false,
+    autoDecomposePerTick: pos(kanban.auto_decompose_per_tick) ?? 3,
+    goalMaxTurns: pos(kanban.goal_max_turns) ?? 20,
+    maxSpawn: pos(kanban.max_spawn),
+    reconcileOrphans: kanban.reconcile_orphans !== false,
+    logRotateBytes: pos(kanban.worker_log_rotate_bytes) ?? 2 * 1024 * 1024,
+    logBackups: Number.isInteger(Number(kanban.worker_log_backup_count)) && Number(kanban.worker_log_backup_count) >= 0 ? Number(kanban.worker_log_backup_count) : 1,
+    hooks: kanban.hooks && typeof kanban.hooks === 'object' ? kanban.hooks : {},
+    rateLimitCooldown: Number.isInteger(Number(process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS)) && process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS !== '' ? Number(process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS) : 300,
   };
 }
 
@@ -132,6 +143,14 @@ function supervise(board, fleet, settings, report) {
       const runRow = currentRun(db, id);
       const session = runRow?.metadata?.supercode ?? null;
       const t = now();
+      // Hermes's orphan reconciliation: a running card whose claim bookkeeping is broken, with nothing live behind it
+      const live = session?.surface === 'headless' ? pidAlive(session.pid) : session?.session_id ? (fleet.ok ? sessionAlive(fleet, session) : true) : false;
+      if (settings.reconcileOrphans && (!task.claim_lock || !task.claim_expires) && !live) {
+        release(db, id, { outcome: 'reclaimed', error: 'orphaned running card (broken claim bookkeeping)', kind: 'reconciled', payload: { reason: 'orphaned_running', claim_lock: task.claim_lock, claim_expires: task.claim_expires } });
+        db.prepare('INSERT INTO task_comments (task_id, author, body, created_at) VALUES (?, ?, ?, ?)').run(id, 'dispatcher', 'Requeued: the card was running with no claim and nothing live behind it.', t);
+        report.reclaimed.push(id);
+        return;
+      }
       // a card past its max runtime is stopped and counted, as Hermes times its workers out
       if (task.max_runtime_seconds && runRow && t - Number(runRow.started_at) > Number(task.max_runtime_seconds)) {
         stop.push(session);
@@ -141,13 +160,32 @@ function supervise(board, fleet, settings, report) {
         return;
       }
       if (session?.surface === 'headless') {
+        // Hermes's stale check: a worker running past the timeout with no heartbeat in the last hour is stopped
+        if (settings.staleTimeout && runRow && t - Number(runRow.started_at) >= settings.staleTimeout && !(task.last_heartbeat_at && t - Number(task.last_heartbeat_at) < 3600)) {
+          stop.push(session);
+          release(db, id, { outcome: 'stale', error: `no heartbeat after ${t - Number(runRow.started_at)}s running`, kind: 'stale', payload: { timeout_seconds: settings.staleTimeout, pid: session.pid } });
+          report.reclaimed.push(id);
+          return;
+        }
         if (pidAlive(session.pid)) {
           if (Number(task.claim_expires) <= t + 60) extendClaim(db, id, { ttl: settings.ttl, reason: 'pid_alive', extra: { worker_pid: session.pid } });
           return;
         }
         if (t - Number(session.launched_at ?? 0) < 30) return;
-        release(db, id, { outcome: 'crashed', error: `worker ${session.pid} exited without completing the card`, kind: 'crashed', payload: { worker_pid: session.pid } });
-        if (recordFailure(db, id, `worker ${session.pid} exited without completing the card`, { failureLimit: settings.failureLimit })) report.auto_blocked.push(id);
+        // how the worker ended, from the exit code its wrapper wrote (Hermes reads its reaped children's)
+        const code = session.exit_file && existsSync(session.exit_file) ? Number(readFileSync(session.exit_file, 'utf8').trim()) : null;
+        if (code === 75) {
+          // a quota wall: back to ready with no failure counted; the respawn guard waits out the cooldown
+          release(db, id, { outcome: 'rate_limited', error: 'worker hit a rate limit (exit 75)', kind: 'rate_limited', payload: { worker_pid: session.pid, exit_code: 75 } });
+          report.crashed.push(id);
+          return;
+        }
+        const clean = code === 0;
+        const why = clean ? `worker ${session.pid} exited cleanly (rc=0) without completing or blocking the card` : `worker ${session.pid} exited${code === null ? '' : ` with ${code}`} without completing the card`;
+        release(db, id, { outcome: 'crashed', error: why, kind: 'crashed', payload: { worker_pid: session.pid, exit_code: code, protocol_violation: clean } });
+        // a clean exit that left the card running is a protocol violation: Hermes trips the breaker on the first
+        if (clean) db.prepare('UPDATE tasks SET consecutive_failures = ? WHERE id = ?').run(Math.max(0, (task.max_retries ?? settings.failureLimit) - 1), id);
+        if (recordFailure(db, id, why, { failureLimit: settings.failureLimit })) report.auto_blocked.push(id);
         report.crashed.push(id);
         return;
       }
@@ -190,7 +228,14 @@ function supervise(board, fleet, settings, report) {
 
 function stopSession(session) {
   if (!session) return;
-  if (session.surface === 'headless' && pidAlive(session.pid)) { try { process.kill(session.pid, 'SIGTERM'); } catch { /* gone */ } return; }
+  if (session.surface === 'headless' && pidAlive(session.pid)) {
+    // SIGTERM to the worker's process group, then SIGKILL if it is still there after a grace, as Hermes stops its own
+    const signal = (sig) => { try { process.kill(-session.pid, sig); } catch { try { process.kill(session.pid, sig); } catch { /* gone */ } } };
+    signal('SIGTERM');
+    for (let waited = 0; waited < 10 && pidAlive(session.pid); waited += 1) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500);
+    if (pidAlive(session.pid)) signal('SIGKILL');
+    return;
+  }
   if (session.pane) sc(['teams', 'panes', 'close', session.pane, ...(session.machine ? ['--machine', session.machine] : [])], { timeout: 60_000 });
 }
 
@@ -212,8 +257,8 @@ function machineLoad(db, machine, fleet) {
 }
 
 /** Why a card cannot start now, or null. */
-function guard(db, task, lane, fleet, settings) {
-  if (settings.maxInProgress && running(db, '') >= settings.maxInProgress) return `the board runs ${settings.maxInProgress} cards at most (kanban.max_in_progress)`;
+function guard(db, task, lane, fleet, settings, budget = null) {
+  if (budget && budget.left <= 0) return budget.why;
   const cap = lane.capacity ?? settings.perProfile;
   if (cap && running(db, 'AND t.assignee = ?', lane.name) >= cap) return `${lane.name} is at capacity (${cap}); the card starts when one of its sessions ends`;
   if (lane.surface !== 'pane') return null;
@@ -223,11 +268,45 @@ function guard(db, task, lane, fleet, settings) {
   if (mcap && machineLoad(db, machine, fleet) >= mcap) return `machine ${machine} holds ${mcap} of the board's sessions (its capacity); the card starts when one is closed`;
   if (!fleet.ok) return `supercode could not read the fleet: ${fleet.error}`;
   if (!fleet.online.has(machine)) return `machine ${machine} is offline; the card starts when it is back`;
-  if (task.workspace_kind !== 'dir' || !task.workspace_path) return `a pane session starts in a directory on ${machine}: the card's workspace is dir:<absolute path>, not ${task.workspace_kind}`;
+  // a card on this machine gets its workspace made here (scratch, dir or worktree, as Hermes makes it); another
+  // machine's card names its directory there
+  if (machine !== localMachine() && (task.workspace_kind !== 'dir' || !task.workspace_path)) return `a pane session on ${machine} starts in a directory there: the card's workspace is dir:<absolute path>, not ${task.workspace_kind}`;
   if (lane.harness !== 'claude-code') return `a pane session's id is chosen at launch, which ${lane.harness} does not take; the lane's worker is claude-code`;
   return null;
 }
 
+// Hermes's respawn guard, for a one-shot (headless) worker: no respawn while the last failure is an auth or quota
+// wall, after a completion nobody requeued, or while a pull request is open on the card. A pane session is resumed
+// rather than respawned, so it never runs twice and these do not apply.
+const BLOCKER = /\b(quota|rate[\s_-]?limit|429|403|auth\w*|unauthorized|forbidden|billing|subscription|access[\s_]denied|permission[\s_]denied|invalid[\s_]api[\s_]key)\b/i;
+const PR_URL = /https?:\/\/github\.com\/[^/\s]+\/[^/\s]+\/pull\/\d+/i;
+
+function respawnGuard(db, task, settings, lane) {
+  const t = now();
+  const last = db.prepare('SELECT outcome, ended_at FROM task_runs WHERE task_id = ? AND ended_at IS NOT NULL ORDER BY ended_at DESC LIMIT 1').get(task.id);
+  if (last?.outcome === 'rate_limited') return settings.rateLimitCooldown > 0 && t - Number(last.ended_at) < settings.rateLimitCooldown ? 'rate_limit_cooldown' : null;
+  if (task.last_failure_error && BLOCKER.test(task.last_failure_error)) return 'blocker_auth';
+  if (lane === 'review') return null;
+  const done = db.prepare("SELECT ended_at FROM task_runs WHERE task_id = ? AND outcome = 'completed' AND ended_at >= ? ORDER BY ended_at DESC LIMIT 1").get(task.id, t - 3600);
+  if (done && !db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND created_at >= ? AND kind IN ('status', 'promoted', 'unblocked', 'reclaimed') LIMIT 1").get(task.id, done.ended_at)) return 'recent_success';
+  for (const c of db.prepare('SELECT body FROM task_comments WHERE task_id = ? AND created_at >= ?').all(task.id, t - 86400)) if (PR_URL.test(c.body ?? '')) return 'active_pr';
+  return null;
+}
+
+/** This machine's memory pressure as Hermes reads it (Linux's MemAvailable; `unknown` elsewhere, which restricts nothing). */
+function memoryPressure() {
+  try {
+    const info = Object.fromEntries(readFileSync('/proc/meminfo', 'utf8').split('\n').map((l) => /^(\w+):\s+(\d+)/.exec(l)).filter(Boolean).map((m) => [m[1], Number(m[2])]));
+    const avail = info.MemAvailable;
+    const total = info.MemTotal;
+    if (!Number.isFinite(avail)) return 'unknown';
+    const fraction = total > 0 ? avail / total : null;
+    if (avail < 64 * 1024 || (fraction !== null && fraction < 0.05)) return 'critical';
+    if (avail < 128 * 1024 || (fraction !== null && fraction < 0.15)) return 'elevated';
+    return 'ok';
+  } catch { return 'unknown'; }
+}
+
 /** Record a guard reason once per change, not once per tick. */
 function noteGuard(db, id, reason) {
   const last = db.prepare("SELECT payload FROM task_events WHERE task_id = ? AND kind IN ('respawn_guarded', 'claimed') ORDER BY id DESC LIMIT 1").get(id);
@@ -235,10 +314,39 @@ function noteGuard(db, id, reason) {
   event(db, id, 'respawn_guarded', { reason });
 }
 
-function start(board, fleet, lanes, settings, report, { only = null } = {}) {
-  const lanesWanted = settings.reviewDispatch ? "('ready', 'review')" : "('ready')";
-  const rows = board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status IN ${lanesWanted} AND claim_lock IS NULL ORDER BY priority DESC, created_at ASC`).all();
-  for (const row of rows) {
+/** Hermes's emergency stop: the `ESTOP` sentinel in the home or in `~/.hermes` stops new starts (running ones go on). */
+export function paused(root) {
+  return [join(root, 'ESTOP'), join(process.env.HOME ?? '', '.hermes', 'ESTOP')].find((p) => existsSync(p)) ?? null;
+}
+
+/**
+ * The tick's spawn budget, as Hermes counts it: `kanban.max_spawn` against this board's running cards, and
+ * `kanban.max_in_progress` against every board of the home (workers share one machine's memory).
+ */
+function spawnBudget(board, settings, hostRunning) {
+  const here = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks WHERE status = 'running'").get().n);
+  const limits = [];
+  if (settings.maxSpawn) limits.push([settings.maxSpawn - here, `the board runs ${settings.maxSpawn} cards at most (kanban.max_spawn)`]);
+  if (settings.maxInProgress) limits.push([settings.maxInProgress - hostRunning, `the home runs ${settings.maxInProgress} cards at most across its boards (kanban.max_in_progress)`]);
+  if (!limits.length) return null;
+  const [left, why] = limits.sort((a, b) => a[0] - b[0])[0];
+  return { left, why };
+}
+
+function start(board, fleet, lanes, settings, report, { only = null, hostRunning = 0 } = {}) {
+  const stop = paused(board.root);
+  if (stop) { report.paused = stop; return; }
+  const pressure = memoryPressure();
+  let headlessStarted = 0;
+  const order = 'ORDER BY priority DESC, created_at ASC';
+  const ready = board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status = 'ready' AND claim_lock IS NULL ${order}`).all();
+  const review = settings.reviewDispatch ? board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status = 'review' AND claim_lock IS NULL ${order}`).all() : [];
+  const budget = spawnBudget(board, settings, hostRunning);
+  // Hermes's review reservation: while review work waits, one slot of the budget is held back from the ready lane
+  const reviewWaits = review.some((r) => lanes[r.assignee || settings.defaultAssignee]);
+  const readyBudget = budget && budget.left > 0 && reviewWaits ? { left: budget.left - 1, why: `${budget.why}; one slot is held for review` } : budget;
+  for (const row of [...ready, ...review]) {
+    const lanesBudget = row.status === 'review' ? budget : readyBudget;
     if (only && row.id !== only) continue;
     if (board.db.prepare('SELECT subtask_of FROM supercode_cards WHERE task_id = ?').get(row.id)?.subtask_of) continue; // worked by its arc's session
     const assignee = row.assignee || settings.defaultAssignee;
@@ -248,7 +356,12 @@ function start(board, fleet, lanes, settings, report, { only = null } = {}) {
     const claimed = tx(board, (db) => {
       const task = getTask(db, row.id);
       if (!task || task.status !== row.status || task.claim_lock) return null;
-      const why = guard(db, task, lane, fleet, settings);
+      let why = guard(db, task, lane, fleet, settings, lanesBudget);
+      if (!why && lane.surface !== 'pane') {
+        // a one-shot worker runs on this machine: its memory, and Hermes's respawn guard
+        if (pressure === 'critical' || (pressure === 'elevated' && headlessStarted >= 1)) why = `this machine's memory pressure is ${pressure}; the card starts when it eases`;
+        else why = respawnGuard(db, task, settings, row.status);
+      }
       if (why) { noteGuard(db, row.id, why); report.queued.push([row.id, why]); return null; }
       if (!row.assignee) {
         db.prepare('UPDATE tasks SET assignee = ? WHERE id = ?').run(assignee, row.id);
@@ -259,7 +372,12 @@ function start(board, fleet, lanes, settings, report, { only = null } = {}) {
     });
     if (!claimed) continue;
     try {
-      const session = lane.surface === 'pane' ? launchPane(board, claimed.task, claimed.runId, lane, fleet) : launchHeadless(board, claimed.task, claimed.runId, lane, settings);
+      const session = lane.surface === 'pane'
+        ? launchPane(board, claimed.task, claimed.runId, lane, fleet, { input: row.status === 'review' ? reviewAsk(board, claimed.task) : null })
+        : launchHeadless(board, claimed.task, claimed.runId, lane, settings);
+      if (lane.surface !== 'pane') headlessStarted += 1;
+      if (budget) budget.left -= 1;
+      if (readyBudget && readyBudget !== budget) readyBudget.left -= 1;
       report.started.push([row.id, session.address ?? `pid ${session.pid}`, session.mode]);
     } catch (error) {
       tx(board, (db) => {
@@ -271,6 +389,50 @@ function start(board, fleet, lanes, settings, report, { only = null } = {}) {
   }
 }
 
+/** What a session started from the review lane is asked to do: verify the handoff and give the verdict. */
+function reviewAsk(board, task) {
+  const local = normalMachine(task.machine) === localMachine();
+  const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
+  return `You are reviewing card ${task.id} on the ${board.label} board: its implementer asked for review. Verify the handoff against the card's acceptance, `
+    + `then give the verdict: \`supercode workflow complete ${task.id} --summary "<what you verified>" ${where}\` approves it, and `
+    + `\`supercode workflow request-changes ${task.id} "<what must change>" ${where}\` sends it back.\n\n${tx(board, (db) => cardContext(db, task.id))}`;
+}
+
+/** What a session started on a triage card is asked to do: write its spec, or split it into cards it waits on. */
+function triageAsk(board, task, label, purpose) {
+  const local = normalMachine(task.machine) === localMachine();
+  const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(label)}`;
+  const context = tx(board, (db) => cardContext(db, task.id));
+  return purpose === 'decompose'
+    ? `Card ${task.id} on the ${label} board is in triage: a rough idea to split. Read it, then file the cards it breaks into and make it wait on them: `
+      + `\`supercode workflow decompose ${task.id} --child "<title>[@<profile>]" --child "..." ${where}\` (each child gets this card's machine and directory). `
+      + `The card itself stays and wakes when they are done.\n\n${context}`
+    : `Card ${task.id} on the ${label} board is in triage: a rough idea. Write its spec (the goal, the approach, what will show it is done), then file it: `
+      + `\`supercode workflow specify ${task.id} --title "<a tightened title>" --body "<the spec>" ${where}\`. The card then waits for its parents and starts.\n\n${context}`;
+}
+
+/**
+ * Start a session on a triage card to specify or decompose it (where Hermes asks an auxiliary model, supercode asks a
+ * session in the card's lane, which then keeps the card's memory when the card starts). Answers the session.
+ */
+export function startTriageSession(board, id, lanes, settings, purpose) {
+  const fleet = observeFleet();
+  const task = tx(board, (db) => getTask(db, id));
+  if (!task) throw new Error(`no card ${id}`);
+  if (task.status !== 'triage') throw new Error(`${id} is ${task.status}; only a triage card is ${purpose === 'decompose' ? 'decomposed' : 'specified'}`);
+  const lane = lanes[task.assignee || settings.defaultAssignee];
+  if (!lane || lane.surface !== 'pane') throw new Error(`${id} has no pane lane to start a session in; give its spec directly (${purpose === 'decompose' ? '--child' : '--title/--body'})`);
+  const runId = tx(board, (db) => {
+    const why = guard(db, task, lane, fleet, settings);
+    if (why) throw new Error(`${id} cannot start a session now: ${why}`);
+    return claimIn(db, id, { ttl: settings.ttl, expectStatus: 'triage' });
+  });
+  if (!runId) throw new Error(`${id} could not be claimed`);
+  return launchPane(board, getTaskNow(board, id), runId, lane, fleet, { input: triageAsk(board, task, board.label, purpose) });
+}
+
+const getTaskNow = (board, id) => tx(board, (db) => getTask(db, id));
+
 function opening(board, task, label) {
   const context = tx(board, (db) => cardContext(db, task.id));
   const skills = task.skills?.length ? `\nSkills pinned to this card: ${task.skills.join(', ')}.` : '';
@@ -279,14 +441,19 @@ function opening(board, task, label) {
   const local = normalMachine(task.machine) === localMachine();
   const where = local ? at : `--at ${boardAddress(label)}`;
   const door = `\nYour card is yours to keep current: \`supercode workflow comment ${task.id} "<text>" ${where}\`, \`block ${task.id} "<why>" --kind <needs_input|capability|transient|dependency> ${where}\` `
-    + `and \`review ${task.id} --summary "<what>" ${where}\` update it; \`create "<title>" --subtask-of ${task.id} ${where}\` files a subtask, and \`complete <subtask> ${where}\` marks one done`
+    + `and \`request-review ${task.id} --summary "<what>" ${where}\` update it; \`create "<title>" --subtask-of ${task.id} ${where}\` files a subtask, and \`complete <subtask> ${where}\` marks one done`
     + `${local ? '' : ' (each is filed with the board and answered by a board notice)'}. Its reviewers, not you, mark the card itself done.`;
   return `You are the session working card ${task.id} on the ${label} board. The board started you; updates to the card `
     + `(comments, reassignment, blocks, completion) and to the cards it waits on reach you as board notices.${door}${skills}\n\n${context}`;
 }
 
-function launchPane(board, task, runId, lane, fleet) {
+function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
   const machine = normalMachine(task.machine);
+  if (machine === localMachine() && !(task.workspace_kind === 'dir' && task.workspace_path && existsSync(task.workspace_path))) {
+    const made = resolveLocalWorkspace(board, task);
+    tx(board, (db) => db.prepare('UPDATE tasks SET workspace_path = ? WHERE id = ?').run(made, task.id));
+    task = { ...task, workspace_path: made };
+  }
   const model = task.model_override ?? lane.model;
   const tail = model ? ['--model', model] : [];
   const base = ['open', '--new', 'claude', '--detach', '--on', task.machine, '--cwd', task.workspace_path];
@@ -309,7 +476,7 @@ function launchPane(board, task, runId, lane, fleet) {
   } else if (prev) replaced = { session_id: prev.session_id, why: `the card moved from ${prev.machine} to ${machine}` };
   if (!session) {
     const sid = randomUUID();
-    const r = sc([...base, '--key', `board-${task.id}-r${runId}-${sid.slice(0, 8)}`, '--input', opening(board, task, board.label), '--', '--session-id', sid, ...tail], { timeout: 180_000 });
+    const r = sc([...base, '--key', `board-${task.id}-r${runId}-${sid.slice(0, 8)}`, '--input', input ?? opening(board, task, board.label), '--', '--session-id', sid, ...tail], { timeout: 180_000 });
     if (r.code !== 0) throw new Error(`supercode open --on ${task.machine}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
     session = { surface: 'pane', session_id: sid, machine, pane: lastJson(r.stdout)?.pane ?? null, mode: 'fresh', launched_at: now(), ...(replaced ? { replaces: replaced.session_id } : {}) };
   }
@@ -342,15 +509,23 @@ function paneSurvives(machine, pane) {
 /** Hermes's own worker argv and task environment, run on this machine; watched by its pid. */
 function launchHeadless(board, task, runId, lane, settings) {
   const workspace = resolveLocalWorkspace(board, task);
-  const args = ['-p', lane.name, '--cli', '--accept-hooks', ...task.skills.flatMap((s) => ['--skills', s]),
+  const reviewing = tx(board, (db) => json(db.prepare("SELECT payload FROM task_events WHERE task_id = ? AND run_id = ? AND kind = 'claimed'").get(task.id, runId)?.payload, {})?.source_status === 'review');
+  const skills = reviewing ? [...new Set([...task.skills, 'sdlc-review'])] : task.skills;
+  const args = ['-p', lane.name, '--cli', '--accept-hooks', ...skills.flatMap((s) => ['--skills', s]),
     ...(task.model_override ? ['-m', task.model_override, ...(task.provider_override ? ['--provider', task.provider_override] : [])] : []),
     ...(task.reasoning_effort ? ['--reasoning', task.reasoning_effort] : []), 'chat', '-q', `work kanban task ${task.id}`];
   const logDir = join(dirname(board.path), board.slug === 'default' ? 'kanban' : '', 'logs');
   mkdirSync(logDir, { recursive: true });
-  const out = openSync(join(logDir, `${task.id}.log`), 'a');
-  const child = spawnChild(process.execPath, [WORKER, ...args], {
+  const logFile = join(logDir, `${task.id}.log`);
+  rotateLog(logFile, settings.logRotateBytes, settings.logBackups);
+  const out = openSync(logFile, 'a');
+  const exitFile = join(logDir, `${task.id}.r${runId}.exit`);
+  // the worker runs under a shell that records its exit code, so a later tick can tell a clean exit from a crash
+  const child = spawnChild('/bin/sh', ['-c', '"$0" "$@"; echo $? > "$SUPERCODE_BOARD_EXIT_FILE"', process.execPath, WORKER, ...args], {
     cwd: workspace, detached: true, stdio: ['ignore', out, out],
     env: {
+      SUPERCODE_BOARD_EXIT_FILE: exitFile,
+      ...(task.goal_mode ? { HERMES_KANBAN_GOAL_MODE: '1', ...(task.goal_max_turns ? { HERMES_KANBAN_GOAL_MAX_TURNS: String(task.goal_max_turns) } : {}) } : {}),
       ...process.env, HERMES_HOME: lane.dir, HERMES_PROFILE: lane.name, HERMES_KANBAN_TASK: task.id, HERMES_KANBAN_WORKSPACE: workspace,
       HERMES_KANBAN_RUN_ID: String(runId), HERMES_KANBAN_DB: board.path, HERMES_KANBAN_BOARD: board.slug, HERMES_SESSION_SOURCE: 'kanban',
       HERMES_KANBAN_CLAIM_LOCK: `${hostname()}:${process.pid}`, ...(task.tenant ? { HERMES_TENANT: task.tenant } : {}),
@@ -358,7 +533,7 @@ function launchHeadless(board, task, runId, lane, settings) {
     },
   });
   child.unref();
-  const session = { surface: 'headless', pid: child.pid, machine: normalMachine(hostname()), mode: 'fresh', launched_at: now() };
+  const session = { surface: 'headless', pid: child.pid, machine: localMachine(), mode: 'fresh', launched_at: now(), exit_file: exitFile };
   tx(board, (db) => {
     db.prepare('UPDATE tasks SET worker_pid = ? WHERE id = ?').run(child.pid, task.id);
     db.prepare('UPDATE task_runs SET worker_pid = ? WHERE id = ?').run(child.pid, runId);
@@ -368,6 +543,16 @@ function launchHeadless(board, task, runId, lane, settings) {
   return session;
 }
 
+/** Hermes's worker log rotation: past `max` bytes the log moves to `.1` (… `.n`), the oldest dropped. */
+function rotateLog(file, max, backups) {
+  try {
+    if (!existsSync(file) || statSync(file).size <= max) return;
+    if (backups === 0) { unlinkSync(file); return; }
+    for (let n = backups - 1; n >= 1; n -= 1) if (existsSync(`${file}.${n}`)) renameSync(`${file}.${n}`, `${file}.${n + 1}`);
+    renameSync(file, `${file}.1`);
+  } catch { /* a log that cannot rotate keeps growing */ }
+}
+
 /** A headless card's directory on this machine, made as Hermes makes it (scratch, dir, or a git worktree). */
 function resolveLocalWorkspace(board, task) {
   const root = board.root;
@@ -398,7 +583,7 @@ function resolveLocalWorkspace(board, task) {
  * never as the session describes itself. From then on the board watches it, resumes it when lost, and tells it of
  * its card. `who` is an address, a session id, or a `name@machine` the fleet lists.
  */
-export function attachSession(board, id, who, { workspace = null, ttl } = {}) {
+export function adoptSession(board, id, who, { workspace = null, ttl } = {}) {
   const fleet = observeFleet();
   if (!fleet.ok) throw new Error(`supercode could not read the fleet: ${fleet.error}`);
   const found = fleet.sessions.find((s) => s.address === who || s.address.endsWith(`:${who}`) || s.name === who);
@@ -483,6 +668,9 @@ function send(from, label, to, body, key) {
 /** The board's own address: where its notices come from and where card updates are mailed. */
 export const boardAddress = (label) => `sc:${localMachine()}:board:${label.replace(/[^A-Za-z0-9_.-]/g, '-')}`;
 
+/** Consecutive failed sends after which a subscription is dropped (Hermes's notifier drops a dead chat after 12). */
+const MAX_SEND_FAILURES = 12;
+
 function notify(board, fleet, report) {
   const from = boardAddress(board.label);
   for (const sub of subscriptions(board.db)) {
@@ -499,6 +687,7 @@ function notify(board, fleet, report) {
     if (!events.length) continue;
     const task = tx(board, (db) => getTask(db, sub.task_id));
     let cursor = Number(sub.last_event_id);
+    let failed = null;
     try {
       for (const ev of events) {
         const p = json(ev.payload, {}) ?? {};
@@ -515,20 +704,139 @@ function notify(board, fleet, report) {
         cursor = ev.id;
       }
     } catch (error) {
-      report.undelivered.push([sub.task_id, sub.chat_id, String(error.message).slice(-200)]);
+      failed = String(error.message).slice(-200);
+      report.undelivered.push([sub.task_id, sub.chat_id, failed]);
     }
     // the cursor moves past what was delivered (a failed delivery is retried next tick; the mail id dedupes)
     tx(board, (db) => {
-      db.prepare("UPDATE kanban_notify_subs SET last_event_id = ? WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND last_event_id < ?").run(cursor, sub.task_id, sub.chat_id, cursor);
-      if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?").run(sub.task_id, sub.chat_id);
+      const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?";
+      db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, sub.task_id, sub.chat_id, cursor);
+      const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
+      if (failures >= MAX_SEND_FAILURES) { db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(sub.task_id, sub.chat_id); return; }
+      if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), sub.task_id, sub.chat_id);
+      if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(sub.task_id, sub.chat_id);
     });
   }
 }
 
+// Hermes's terminal kinds for a chat subscriber, in its order of silence: archived and unblocked move the cursor and say nothing
+const CHAT_KINDS = ['completed', 'blocked', 'gave_up', 'crashed', 'timed_out', 'status', 'archived', 'unblocked', 'block_loop_detected', 'review_requested', 'changes_requested'];
+
+/**
+ * A chat-platform subscription's events, sent through the orchestrator's own channel adapters (the daemon hosts them;
+ * `serve` hosts none, as a Hermes CLI without its gateway delivers none). Answers what was sent and what failed.
+ */
+export async function notifyChats(root, adapters, log = () => {}) {
+  const sent = [];
+  for (const slug of listBoards(root)) {
+    const board = openBoard(root, slug);
+    board.label = boardLabel(root, slug);
+    try {
+      for (const sub of subscriptions(board.db)) {
+        const adapter = adapters?.[sub.platform];
+        if (sub.platform === 'supercode' || !adapter) continue;
+        const events = board.db.prepare(`SELECT id, kind, payload FROM task_events WHERE task_id = ? AND id > ? AND kind IN (${CHAT_KINDS.map(() => '?').join(', ')}) ORDER BY id`).all(sub.task_id, sub.last_event_id, ...CHAT_KINDS);
+        if (!events.length) continue;
+        const task = tx(board, (db) => getTask(db, sub.task_id));
+        let cursor = Number(sub.last_event_id);
+        let failed = false;
+        for (const ev of events) {
+          const payload = json(ev.payload, {}) ?? {};
+          const text = ['archived', 'unblocked'].includes(ev.kind) ? null : notice(ev.kind, task, payload, 'creator');
+          if (text) {
+            // a completed card's deliverables go with it as uploads, as Hermes's notifier sends them
+            const files = ev.kind === 'completed' ? artifactsOf(payload, task).map((path) => ({ path })) : [];
+            try {
+              await adapter.send({ kind: 'explicit', platform: sub.platform, chat_id: sub.chat_id, ...(sub.thread_id ? { thread_id: sub.thread_id } : {}) }, { text: `${slug !== 'default' ? `[${slug}] ` : ''}${text}`, ...(files.length ? { files } : {}) });
+              sent.push([sub.task_id, ev.kind, `${sub.platform}:${sub.chat_id}`]);
+            } catch (error) {
+              failed = true;
+              log('warn', `board ${slug}: a notice for ${sub.task_id} to ${sub.platform}:${sub.chat_id} failed: ${error.message}`);
+              break;
+            }
+          }
+          cursor = ev.id;
+        }
+        tx(board, (db) => {
+          const where = 'WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ?';
+          const key = [sub.task_id, sub.platform, sub.chat_id, sub.thread_id ?? ''];
+          db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...key, cursor);
+          const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
+          if (failures >= MAX_SEND_FAILURES) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...key);
+          else if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), ...key);
+          if (task?.status === 'archived' && !failed) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...key);
+        });
+      }
+    } finally {
+      board.close();
+    }
+  }
+  return sent;
+}
+
+/** A completed card's artifact files: its completion's `artifacts`, then absolute paths its summary or result names that exist. */
+function artifactsOf(payload, task) {
+  const seen = new Set();
+  const out = [];
+  const add = (p) => {
+    const path = String(p ?? '').replace(/^~(?=\/)/, process.env.HOME ?? '~');
+    if (!path.startsWith('/') || seen.has(path)) return;
+    try { if (!statSync(path).isFile()) return; } catch { return; }
+    seen.add(path);
+    out.push(path);
+  };
+  for (const p of Array.isArray(payload.artifacts) ? payload.artifacts : []) add(p);
+  for (const text of [payload.summary, task?.result]) for (const m of String(text ?? '').matchAll(/(?:^|\s)(~?\/[^\s'"`]+\.[A-Za-z0-9]{1,8})/g)) add(m[1]);
+  return out;
+}
+
+// ------------------------------------------------------------------ hooks
+
+// Hermes's lifecycle hooks, from the board's own events (so a change any writer made fires them, Hermes's CLI's too)
+const HOOK_OF = {
+  claimed: 'kanban_task_claimed', completed: 'kanban_task_completed', blocked: 'kanban_task_blocked',
+  spawned: 'on_kanban_worker_spawned', session_started: 'on_kanban_worker_spawned', session_resumed: 'on_kanban_worker_spawned', session_replaced: 'on_kanban_worker_spawned', session_adopted: 'on_kanban_worker_spawned',
+  crashed: 'on_kanban_worker_exited', rate_limited: 'on_kanban_worker_exited', session_lost: 'on_kanban_worker_exited', timed_out: 'on_kanban_worker_exited',
+  stale: 'on_kanban_worker_stale_claim', reclaimed: 'on_kanban_worker_stale_claim',
+  assigned: 'on_kanban_task_updated', edited: 'on_kanban_task_updated', model_override_set: 'on_kanban_task_updated', machine_set: 'on_kanban_task_updated',
+  linked: 'on_kanban_task_updated', unlinked: 'on_kanban_task_updated', unblocked: 'on_kanban_task_updated', promoted: 'on_kanban_task_updated',
+  archived: 'on_kanban_task_updated', review_requested: 'on_kanban_task_updated', changes_requested: 'on_kanban_task_updated',
+};
+
+/**
+ * `kanban.hooks.<hook>: <command>` (or a list): each command runs with the event as one JSON object on its stdin, and
+ * nothing waits for it. Hermes runs Python plugins at these points; supercode runs any program.
+ */
+function fireHooks(board, settings, report) {
+  const hooks = settings.hooks ?? {};
+  if (!Object.keys(hooks).length) return;
+  const run = (hook, payload) => {
+    for (const command of [hooks[hook]].flat().filter((c) => typeof c === 'string' && c.trim())) {
+      const child = spawnChild('/bin/sh', ['-c', command], { detached: true, stdio: ['pipe', 'ignore', 'ignore'], cwd: board.root });
+      child.on('error', () => {});
+      child.stdin.on('error', () => {});
+      child.stdin.end(JSON.stringify({ hook, board: board.slug, ...payload }));
+      child.unref();
+    }
+  };
+  tx(board, (db) => {
+    db.exec('CREATE TABLE IF NOT EXISTS supercode_board_state (key TEXT PRIMARY KEY, value TEXT)');
+    const max = Number(db.prepare('SELECT COALESCE(MAX(id), 0) AS m FROM task_events').get().m);
+    const row = db.prepare("SELECT value FROM supercode_board_state WHERE key = 'hooks_cursor'").get();
+    const cursor = row ? Number(row.value) : max; // the first tick starts at the present, never replays history
+    for (const ev of db.prepare('SELECT e.id, e.task_id, e.run_id, e.kind, e.payload, e.created_at, t.assignee FROM task_events e LEFT JOIN tasks t ON t.id = e.task_id WHERE e.id > ? ORDER BY e.id').all(cursor)) {
+      const hook = HOOK_OF[ev.kind];
+      if (hook && hooks[hook]) run(hook, { task_id: ev.task_id, run_id: ev.run_id, event: ev.kind, assignee: ev.assignee, payload: json(ev.payload, null), at: ev.created_at });
+    }
+    db.prepare("INSERT INTO supercode_board_state (key, value) VALUES ('hooks_cursor', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(String(max));
+  });
+  if (hooks.on_kanban_dispatch_tick) run('on_kanban_dispatch_tick', { started: report.started.length, reclaimed: report.reclaimed.length, crashed: report.crashed.length, timed_out: report.timed_out.length, promoted: report.promoted });
+}
+
 // ------------------------------------------------------------------ the board's mailbox
 
 const ENTRY = fileURLToPath(new URL('../bin/orchestrator.mjs', import.meta.url));
-const NOT_BY_MAIL = new Set(['serve', 'dispatch', 'init', 'attach']);
+const NOT_BY_MAIL = new Set(['serve', 'daemon', 'dispatch', 'init', 'adopt', 'boards', 'gc', 'repair', 'watch', 'tail', 'attach']);
 
 /** The board's mailbox on this machine: the directory supercode files mail for `address` in, or null. */
 function mailboxOf(address) {
@@ -572,6 +880,62 @@ function readMailbox(board, address, report) {
   }
 }
 
+// ------------------------------------------------------------------ goals and triage
+
+/**
+ * Hermes's goal loop for a pane session: a goal card whose session sits idle while the card is still running is told
+ * to keep going, up to the card's turn budget; when the budget is spent the card is blocked for review. Whether the
+ * work is done is the session's call through its card (`request-review`), not an auxiliary judge's.
+ */
+function pursueGoals(board, fleet, settings, report) {
+  if (!fleet.ok) return;
+  const cards = board.db.prepare("SELECT id FROM tasks WHERE status = 'running' AND goal_mode = 1").all();
+  for (const { id } of cards) {
+    const nudge = tx(board, (db) => {
+      const task = getTask(db, id);
+      const runRow = currentRun(db, id);
+      const session = runRow?.metadata?.supercode;
+      if (!session?.session_id || session.surface !== 'pane') return null;
+      const live = fleet.sessions.find((x) => x.address?.endsWith(`:${session.session_id}`));
+      if (live?.status !== 'idle') return null;
+      if (session.goal_nudged_at && now() - session.goal_nudged_at < settings.interval * 2) return null;
+      const turns = Number(session.goal_turns ?? 0);
+      const budget = task.goal_max_turns ?? settings.goalMaxTurns;
+      if (turns >= budget) {
+        db.prepare("UPDATE tasks SET status = 'blocked', claim_lock = NULL, claim_expires = NULL WHERE id = ?").run(id);
+        const ended = endRun(db, id, { outcome: 'blocked', summary: `goal turn budget (${budget}) spent` });
+        event(db, id, 'blocked', { reason: `goal turn budget (${budget}) spent; review it`, kind: 'needs_input', source_status: 'ready' }, ended);
+        return null;
+      }
+      setRunSession(db, runRow.id, { ...session, goal_turns: turns + 1, goal_nudged_at: now() });
+      event(db, id, 'goal_continued', { turn: turns + 1, budget }, runRow.id);
+      return { address: live.address, text: `Card ${id} (${task.title}) is not done yet: keep working toward it (goal turn ${turns + 1} of ${budget}). When it is done, \`request-review\` it; if you are stuck, \`block\` it with its kind.` };
+    });
+    if (nudge) {
+      try { send(boardAddress(board.label), board.label, nudge.address, nudge.text, `board-goal-${id}-${now()}`); report.notified.push([id, 'goal_continued', nudge.address]); }
+      catch (error) { report.undelivered.push([id, nudge.address, String(error.message).slice(-200)]); }
+    }
+  }
+}
+
+/** Hermes's auto-decompose: triage cards with a pane lane get a session that splits them, a few per tick. */
+function autoDecompose(board, lanes, settings, report) {
+  if (!settings.autoDecompose) return;
+  let started = 0;
+  for (const { id, assignee } of board.db.prepare("SELECT id, assignee FROM tasks WHERE status = 'triage' ORDER BY priority DESC, created_at ASC").all()) {
+    if (started >= settings.autoDecomposePerTick) break;
+    const lane = lanes[assignee || settings.defaultAssignee];
+    if (lane?.surface !== 'pane') continue;
+    try {
+      const session = startTriageSession(board, id, lanes, settings, 'decompose');
+      report.started.push([id, session.address, 'decompose']);
+      started += 1;
+    } catch (error) {
+      report.queued.push([id, String(error.message).slice(-300)]);
+    }
+  }
+}
+
 // ------------------------------------------------------------------ the tick
 
 /** The board's name in notices: the home's owner folder (`…/volter-manager/home` → `volter-manager`), and the slug. */
@@ -584,12 +948,32 @@ export function boardLabel(root, slug) {
  * One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
  * `only` limits starting to one card (a create that answers whether its card started). Answers one report per board.
  */
-export function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true }) {
+/** A board file's fingerprint: a corrupt board is skipped until the file changes (Hermes retries it then). */
+const fingerprint = (path) => { try { const st = statSync(path); return `${st.size}:${st.mtimeMs}`; } catch { return 'missing'; } };
+const CORRUPT = /malformed|not a database|corrupt|disk image/i;
+
+/**
+ * One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
+ * `only` limits starting to one card (a create that answers whether its card started). Each board is ticked under its
+ * per-tick lock (Hermes's `.dispatch.lock`), skipped when another dispatcher holds it. Answers one report per board.
+ */
+export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true }) {
   const needFleet = Object.values(lanes).some((l) => l.surface === 'pane');
-  const fleet = needFleet ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), error: 'no pane lanes' };
+  const fleet = needFleet ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), sessions: [], error: 'no pane lanes' };
+  const boards = slugs ?? listBoards(root);
+  // the running cards of every board of the home, for kanban.max_in_progress (host-wide, as Hermes counts it)
+  let hostRunning = 0;
+  for (const slug of boards) {
+    try { const b = openBoard(root, slug); try { hostRunning += Number(b.db.prepare("SELECT COUNT(*) AS n FROM tasks WHERE status = 'running'").get().n); } finally { b.close(); } } catch { /* its own tick reports it */ }
+  }
   const out = [];
-  for (const slug of slugs ?? listBoards(root)) {
+  for (const slug of boards) {
     const report = { board: slug, mailed: [], started: [], queued: [], failed: [], lost: [], crashed: [], timed_out: [], reclaimed: [], auto_blocked: [], promoted: 0, unassigned: [], nonspawnable: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
+    const path = boardPath(root, slug);
+    const marker = `${path}.corrupt-seen`;
+    if (existsSync(marker) && readFileSync(marker, 'utf8').trim() === fingerprint(path)) { report.error = 'skipped: the board file was corrupt and has not changed since (`supercode workflow repair`)'; out.push(report); continue; }
+    let lock = null;
+    try { lock = await holdBoardTickLock(path); } catch (error) { report.skipped_locked = error.message; out.push(report); continue; }
     let board;
     try {
       board = openBoard(root, slug);
@@ -597,15 +981,23 @@ export function tick({ root, lanes, settings, only = null, slugs = null, dispatc
       readMailbox(board, boardAddress(board.label), report);
       supervise(board, fleet, settings, report);
       report.promoted = recomputeReady(board, settings.failureLimit);
-      if (dispatch) start(board, fleet, lanes, settings, report, { only });
+      const before = report.started.length;
+      if (dispatch) start(board, fleet, lanes, settings, report, { only, hostRunning });
+      hostRunning += report.started.length - before;
+      if (dispatch && !only && !report.paused) { pursueGoals(board, fleet, settings, report); autoDecompose(board, lanes, settings, report); }
       notify(board, fleet, report);
+      fireHooks(board, settings, report);
+      if (String(board.db.prepare('PRAGMA journal_mode').get()?.journal_mode).toLowerCase() === 'wal') board.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
+      report.ready_waiting = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'ready' AND t.assignee IS NOT NULL AND c.subtask_of IS NULL").get().n);
       // what a Hermes gateway serves around its dispatcher and this one does not: said by the caller
       report.chat_subscriptions = Number(board.db.prepare("SELECT COUNT(*) AS n FROM kanban_notify_subs WHERE platform != 'supercode'").get().n);
-      report.triage = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks WHERE status = 'triage'").get().n);
+      if (existsSync(marker)) unlinkSync(marker);
     } catch (error) {
       report.error = error.message;
+      if (CORRUPT.test(error.message)) writeFileSync(marker, fingerprint(path));
     } finally {
       board?.close();
+      lock.release();
     }
     out.push(report);
   }