loreli 0.0.0 → 1.0.0

package/packages/orchestrator/src/index.js

@@ -0,0 +1,1226 @@
+import { rm, writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+import { EventEmitter } from 'node:events';
+import { Factory, Session, output } from 'loreli/agent';
+import { Tmux } from 'loreli/tmux';
+import { prepare } from 'loreli/workspace';
+import { pick, side, capability } from 'loreli/identity';
+import { logger } from 'loreli/log';
+
+const log = logger('orchestrator');
+
+/**
+ * Fatal error patterns that indicate a backend infrastructure failure
+ * (not a task failure). When these appear in an agent's pane output
+ * shortly after spawn, the backend is broken and should be degraded.
+ *
+ * @type {RegExp[]}
+ */
+const FATAL_PATTERNS = [
+  /budget[_ ]*(has been )?exceeded/i,
+  /rate[_ ]?limit[_ ]*exceeded/i,
+  /hit your usage[_ ]*limit/i,
+  /authentication[_ ]*(error|failed)/i,
+  /invalid[_ ]*api[_ ]*key/i,
+  /quota[_ ]*exceeded/i,
+  /insufficient[_ ]*quota/i
+];
+
+/**
+ * Check if pane output contains fatal API error patterns.
+ *
+ * @param {string} output - Pane content from agent.capture().
+ * @returns {boolean} True if a fatal error pattern is found.
+ */
+function hasFatalError(output) {
+  if (!output) return false;
+  return FATAL_PATTERNS.some(function match(p) { return p.test(output); });
+}
+
+/**
+ * Generic agent lifecycle coordinator via EventEmitter.
+ *
+ * Manages spawn/shutdown/kill, reactor polling, stall detection,
+ * and activity tracking. Contains zero role-specific logic — all
+ * planner/action/review behavior lives in the role packages that
+ * subscribe to lifecycle events.
+ *
+ * @extends EventEmitter
+ * @fires Orchestrator#spawned
+ * @fires Orchestrator#removed
+ * @fires Orchestrator#stall
+ */
+export class Orchestrator extends EventEmitter {
+  /**
+   * @param {object} opts
+   * @param {object} opts.hub - Hub instance for git hosting operations.
+   * @param {object} opts.identityRegistry - Registry for agent identities.
+   * @param {object} opts.backendRegistry - Registry for agent backends.
+   * @param {object} opts.storage - Persistent storage instance.
+   * @param {object} [opts.config] - Config instance from loreli/config.
+   */
+  constructor({ hub, identityRegistry, backendRegistry, storage, config }) {
+    super();
+
+    /** @type {object} Hub for git hosting. */
+    this.hub = hub;
+
+    /** @type {object} Identity registry. */
+    this.identityRegistry = identityRegistry;
+
+    /** @type {object} Backend registry. */
+    this.backendRegistry = backendRegistry;
+
+    /** @type {Factory} Agent factory — centralizes the create+spawn pipeline. */
+    this.factory = new Factory({ backends: backendRegistry, identities: identityRegistry, config });
+
+    /** @type {object} Storage for session persistence. */
+    this.storage = storage;
+
+    /** @type {object|null} Config instance from loreli/config. */
+    this.cfg = config ?? null;
+
+    /** @type {Map<string, object>} Active agents by name. */
+    this.agents = new Map();
+
+    /** @type {string|null} Current session ID. */
+    this.sessionId = null;
+
+    /** @type {object|null} MCP client identity. */
+    this.clientIdentity = null;
+
+    /** @type {string|null} Target repository in "owner/name" format. */
+    this.repo = null;
+
+    /** @type {number} Stall timeout in ms. */
+    this.stallTimeout = this.cfg?.get?.('timeouts.stall') ?? 600000;
+
+    /** @type {number} Delay before checking if a freshly spawned agent died. */
+    this.rapidDeathDelay = this.cfg?.get?.('timeouts.rapidDeath') ?? 15000;
+
+    /** @type {NodeJS.Timeout|null} Stall detection interval handle. */
+    this._monitorHandle = null;
+
+    /** @type {Map<string, string>} Last known activity timestamp per agent. */
+    this._lastActivity = new Map();
+
+    /** @type {Map<string, string>} MD5 hash of last captured pane output per agent for tmux-based activity detection. */
+    this._lastPaneHash = new Map();
+
+    /** @type {NodeJS.Timeout|null} Reactor polling interval handle. */
+    this._watchHandle = null;
+
+    /** @type {Map<string, function>} Registered reactor handlers by name. */
+    this._handlers = new Map();
+
+    /**
+     * Names already claimed by other participants, discovered from
+     * GitHub claim comments and PR branches during reactor ticks.
+     * Populated as a zero-cost side effect of data the reactor
+     * already fetches — no additional API calls needed.
+     *
+     * @type {Set<string>}
+     */
+    this.takenNames = new Set();
+
+    /**
+     * Agents we've removed locally (killed or shut down). Used to
+     * distinguish "we killed this agent" from "a foreign orchestrator
+     * owns this agent" during proof-of-life decisions.
+     *
+     * @type {Set<string>}
+     */
+    this._removed = new Set();
+
+    /**
+     * Registered workflows by role. Populated during start so
+     * scale() can collect demand signals from each workflow.
+     *
+     * @type {Map<string, object>}
+     */
+    this.workflows = new Map();
+
+    /**
+     * Last spawn timestamp per role for cooldown enforcement.
+     * Prevents thrashing when demand fluctuates between ticks.
+     *
+     * @type {Map<string, number>}
+     */
+    this._lastSpawn = new Map();
+  }
+
+  // ── Seed (Identity Discovery) ────────────────────────
+
+  /**
+   * One-time seed of `takenNames` from open PR branch names.
+   *
+   * Called before the first `acquire()` — before any reactor tick has
+   * populated the set. After the first tick, the reactor keeps the
+   * set current as a side effect of its normal data flow.
+   *
+   * Idempotent: only runs once per orchestrator lifetime.
+   *
+   * @returns {Promise<void>}
+   */
+  async seed() {
+    if (this._seeded) return;
+    this._seeded = true;
+
+    if (!this.hub || !this.repo) return;
+
+    try {
+      const prs = await this.hub.pulls(this.repo, { state: 'open' });
+      for (const pr of prs) {
+        const slash = pr.head?.indexOf('/');
+        if (slash > 0) this.takenNames.add(pr.head.slice(0, slash));
+      }
+      log.debug(`seed: discovered ${this.takenNames.size} taken names from open PRs`);
+    } catch (err) {
+      log.debug(`seed: skipped — ${err.message}`);
+    }
+  }
+
+  // ── Lifecycle ─────────────────────────────────────────
+
+  /**
+   * Spawn and register an agent with comprehensive rollback.
+   *
+   * Tracks each step of the spawn process and undoes them in reverse
+   * order on failure. Each completed step is tracked and unwound
+   * individually on error.
+   *
+   * @param {object} agent - Agent instance.
+   * @returns {Promise<void>}
+   * @fires Orchestrator#spawned
+   */
+  async spawn(agent) {
+    log.info(`spawning agent: ${agent.identity.name} (${agent.role})`);
+
+    /** @type {Array<{step: string, undo: function}>} Completed steps for rollback. */
+    const completed = [];
+
+    try {
+      // Step 1: Spawn the agent process (tmux pane, API session, etc.)
+      await agent.spawn();
+      completed.push({
+        step: 'spawn',
+        undo: async function undoSpawn() {
+          try { await agent.stop(); } catch (e) { log.debug(`rollback stop failed: ${e.message}`); }
+        }
+      });
+
+      // Step 2: Register in the agents map
+      this.agents.set(agent.identity.name, agent);
+      completed.push({
+        step: 'register',
+        undo: () => { this.agents.delete(agent.identity.name); }
+      });
+
+      // Step 3: Record activity timestamp
+      this._lastActivity.set(agent.identity.name, new Date().toISOString());
+      completed.push({
+        step: 'activity',
+        undo: () => { this._lastActivity.delete(agent.identity.name); }
+      });
+
+      log.info(`agent spawned: ${agent.identity.name}`);
+
+      // Schedule rapid-death detection: if the agent dies or shows
+      // fatal API errors within rapidDeathDelay of spawning, the
+      // backend is likely broken (budget exhaustion, API outage).
+      // Mark it as degraded so scale() falls back to cursor-agent.
+      //
+      // Two checks:
+      //  1. Dead pane → agent exited on error
+      //  2. Stuck-alive → agent stays alive but shows budget/rate-limit
+      //     errors in its pane output
+      if (agent.backend && agent.alive) {
+        const backend = agent.backend;
+        const name = agent.identity.name;
+        const registry = this.backendRegistry;
+        const self = this;
+        const timer = setTimeout(async function rapidDeathCheck() {
+          try {
+            const alive = await agent.alive();
+            if (!alive && agent.state !== 'dormant') {
+              log.warn(`rapid death: ${name} died within ${self.rapidDeathDelay}ms of spawn — marking ${backend} degraded`);
+              registry?.recordFailure(backend);
+              if (agent.canTransition?.('dormant')) agent.transition('dormant');
+              self.emit('rapid-death', { name, backend });
+              return;
+            }
+
+            if (alive && agent.capture) {
+              const output = await agent.capture();
+              if (hasFatalError(output)) {
+                log.warn(`stuck-alive: ${name} shows fatal API error — marking ${backend} degraded`);
+                registry?.recordFailure(backend);
+                try { await agent.stop(); } catch { /* stop can fail */ }
+                self.emit('rapid-death', { name, backend, reason: 'stuck-alive' });
+              }
+            }
+          } catch { /* pane check can fail when session is torn down */ }
+        }, this.rapidDeathDelay);
+        timer.unref();
+      }
+
+      /**
+       * @event Orchestrator#spawned
+       * @type {object}
+       * @property {string} name - Agent identity name.
+       * @property {string} role - Agent role.
+       * @property {string} provider - AI provider.
+       */
+      this.emit('spawned', {
+        name: agent.identity.name,
+        role: agent.role,
+        provider: agent.identity.provider
+      });
+    } catch (err) {
+      log.error(`spawn failed: ${agent.identity.name} — ${err.message}`);
+
+      // Rollback completed steps in reverse order
+      for (let i = completed.length - 1; i >= 0; i--) {
+        try {
+          await completed[i].undo();
+          log.debug(`rollback: undid "${completed[i].step}" for ${agent.identity.name}`);
+        } catch (rollbackErr) {
+          // Never mask the original error with a rollback failure
+          log.warn(`rollback "${completed[i].step}" failed: ${rollbackErr.message}`);
+        }
+      }
+
+      // Identity release is the caller's responsibility (add_agent tool
+      // acquired it, add_agent tool releases it). Spawn only rolls back
+      // the steps it owns: process, registration, and activity tracking.
+      throw err;
+    }
+  }
+
+  /**
+   * Gracefully shut down an agent using a 3-phase protocol:
+   *
+   * 1. Send a structured shutdown request with a unique `requestId`
+   * 2. Poll for acknowledgment — the agent can accept or continue working
+   * 3. On acknowledgment or timeout, force stop and clean up
+   *
+   * @param {string} name - Agent identity name.
+   * @param {number} [timeout] - Timeout before force kill.
+   * @param {string} [reason] - Reason for shutdown (passed to agent).
+   * @returns {Promise<{acknowledged: boolean}>} Whether the agent acknowledged.
+   * @fires Orchestrator#removed
+   */
+  async shutdown(name, timeout, reason) {
+    const agent = this.agents.get(name);
+    if (!agent) throw new Error(`Agent "${name}" not found`);
+
+    const shutdownTimeout = timeout ?? this.cfg?.get?.('timeouts.shutdown') ?? 60000;
+    const pollInterval = this.cfg?.get?.('timeouts.poll') ?? 2000;
+    const requestId = `shutdown-${Date.now()}@${name}`;
+
+    log.info(`shutting down agent: ${name} (timeout: ${shutdownTimeout}ms, requestId: ${requestId})`);
+
+    // Phase 1: Send structured shutdown request
+    const shutdownMessage = [
+      `**Shutdown Request** (id: ${requestId})`,
+      reason ? `Reason: ${reason}` : '',
+      'Please finish your current work and shut down gracefully.',
+      'Post a comment or signal when ready.'
+    ].filter(Boolean).join('\n');
+
+    try {
+      await agent.send(shutdownMessage);
+    } catch (err) { log.debug(`shutdown message failed for ${name}: ${err.message}`); }
+
+    // Phase 2: Poll for acknowledgment or timeout
+    let acknowledged = false;
+    const deadline = Date.now() + shutdownTimeout;
+
+    while (Date.now() < deadline) {
+      if (agent.state === 'dormant') {
+        acknowledged = true;
+        break;
+      }
+
+      // Check if the agent is still alive — if it exited on its own,
+      // treat that as implicit acknowledgment
+      try {
+        const alive = await agent.alive?.();
+        if (alive === false) {
+          acknowledged = true;
+          break;
+        }
+      } catch (err) { log.debug(`alive check during shutdown polling failed: ${err.message}`); }
+
+      await new Promise(function wait(r) { setTimeout(r, pollInterval); });
+    }
+
+    if (acknowledged) {
+      log.info(`agent ${name} acknowledged shutdown`);
+    } else {
+      log.warn(`agent ${name} did not acknowledge shutdown within ${shutdownTimeout}ms, force stopping`);
+    }
+
+    // Phase 3: Force stop and clean up
+    const session = agent.session;
+    await this.snapshot(name, agent);
+    await agent.stop();
+
+    // Clean up the agent's workspace directory when configured
+    if (this.cfg?.get?.('workspace.cleanup') && agent.cwd) {
+      try {
+        await rm(agent.cwd, { recursive: true, force: true });
+        log.info(`workspace cleaned: ${name} (${agent.cwd})`);
+      } catch (err) { log.debug(`workspace cleanup failed for ${name}: ${err.message}`); }
+    }
+
+    this.agents.delete(name);
+    this._lastActivity.delete(name);
+    this._lastPaneHash.delete(name);
+    this._removed.add(name);
+    this.identityRegistry.release(agent.identity);
+    log.info(`agent shut down: ${name}`);
+
+    if (this.agents.size === 0 && session) await this._pruneSession(session);
+
+    /**
+     * @event Orchestrator#removed
+     * @type {object}
+     * @property {string} name - Agent identity name.
+     * @property {string} reason - 'shutdown' or 'killed'.
+     * @property {object} agent - The removed agent instance.
+     */
+    this.emit('removed', { name, reason: 'shutdown', agent });
+
+    return { acknowledged };
+  }
+
+  /**
+   * Force kill an agent immediately.
+   * Releases any claimed issues so they can be re-claimed.
+   *
+   * @param {string} name - Agent identity name.
+   * @returns {Promise<void>}
+   * @fires Orchestrator#removed
+   */
+  async kill(name) {
+    const agent = this.agents.get(name);
+    if (!agent) throw new Error(`Agent "${name}" not found`);
+
+    log.warn(`force killing agent: ${name}`);
+    const session = agent.session;
+    await this.snapshot(name, agent);
+    await agent.stop();
+
+    // Clean up the agent's workspace directory when configured
+    if (this.cfg?.get?.('workspace.cleanup') && agent.cwd) {
+      try {
+        await rm(agent.cwd, { recursive: true, force: true });
+        log.info(`workspace cleaned: ${name} (${agent.cwd})`);
+      } catch (err) { log.debug(`workspace cleanup failed for ${name}: ${err.message}`); }
+    }
+
+    this.agents.delete(name);
+    this._lastActivity.delete(name);
+    this._lastPaneHash.delete(name);
+    this._removed.add(name);
+    this.identityRegistry.release(agent.identity);
+
+    if (this.agents.size === 0 && session) await this._pruneSession(session);
+
+    this.emit('removed', { name, reason: 'killed', agent });
+  }
+
+  // ── Tmux Cleanup ─────────────────────────────────────
+
+  /**
+   * Resolve the tmux session name from registered agents or config.
+   *
+   * Agents carry their own session name (defaults to 'loreli' but
+   * overridable in tests). This avoids hardcoding the session name
+   * and keeps cleanup aligned with wherever agents actually live.
+   *
+   * @returns {string} The tmux session name.
+   */
+  _session() {
+    for (const agent of this.agents.values()) {
+      if (agent.session) return agent.session;
+    }
+    return this.cfg?.get?.('tmux.session') ?? 'loreli';
+  }
+
+  /**
+   * Destroy the tmux session when no agents remain.
+   *
+   * Safety net that runs after the last agent is removed via
+   * {@link kill} or {@link shutdown}. Individual agent `stop()` calls
+   * kill their own panes, but orphaned panes (from `remain-on-exit`,
+   * crashed backends, or timing gaps) can keep the session alive.
+   * This ensures a clean slate.
+   *
+   * @param {string} session - The tmux session name to prune.
+   * @returns {Promise<void>}
+   */
+  async _pruneSession(session) {
+    if (!Tmux.available()) return;
+    const tmux = new Tmux();
+
+    try {
+      if (await tmux.has(session)) {
+        let paneCount = 0;
+        try {
+          const panes = await tmux.allPanes(session);
+          paneCount = panes.length;
+        } catch { /* session may be in a bad state */ }
+        await tmux.kill(session);
+        log.info(`pruned tmux session "${session}" (${paneCount} orphaned panes) — no agents remain`);
+      }
+    } catch (err) {
+      log.debug(`session prune failed: ${err.message}`);
+    }
+  }
+
+  /**
+   * Garbage-collect orphaned tmux panes not tracked by any agent.
+   *
+   * Lists all panes in the loreli tmux session and kills any that
+   * do not belong to a registered agent. Safe to call at any time —
+   * tracked agent panes are preserved.
+   *
+   * When all panes are orphaned, the session is destroyed entirely.
+   *
+   * @returns {Promise<{killed: number}>} Count of orphaned panes killed.
+   */
+  async gc() {
+    if (!Tmux.available()) return { killed: 0 };
+    const session = this._session();
+    const tmux = new Tmux();
+
+    if (!await tmux.has(session)) return { killed: 0 };
+
+    const tracked = new Set();
+    for (const agent of this.agents.values()) {
+      if (agent.paneId) tracked.add(agent.paneId);
+    }
+
+    let all;
+    try {
+      all = await tmux.allPanes(session);
+    } catch {
+      return { killed: 0 };
+    }
+
+    let killed = 0;
+    for (const pane of all) {
+      if (!tracked.has(pane.id)) {
+        try {
+          await tmux.killPane(pane.id);
+          killed++;
+          log.info(`gc: killed orphaned pane ${pane.id}`);
+        } catch { /* pane may have died between list and kill */ }
+      }
+    }
+
+    // Destroy the session when every pane was orphaned
+    if (killed > 0 && killed === all.length) {
+      try {
+        if (await tmux.has(session)) await tmux.kill(session);
+      } catch { /* session may auto-destroy */ }
+    }
+
+    return { killed };
+  }
+
+  // ── Coordination ──────────────────────────────────────
+
+  /**
+   * Auto-spawn an agent for a given provider and role when one is not
+   * already available. Cross-provider review is the core value proposition;
+   * silently skipping it defeats the purpose, so the orchestrator
+   * proactively enlists the opposing side.
+   *
+   * Delegates to {@link Factory#spawn} for the full creation pipeline.
+   *
+   * @param {string} provider - AI provider to spawn for.
+   * @param {string} role - Agent role ('reviewer', 'action', etc.).
+   * @param {object} [opts] - Additional options.
+   * @returns {Promise<object>} The spawned agent instance.
+   */
+  async enlist(provider, role, opts = {}) {
+    log.info(`enlisting ${role} agent for ${provider} — cross-provider pairing requires it`);
+
+    // Seed taken names before first acquire — one-time cost, zero
+    // ongoing overhead because reactor ticks keep the set current.
+    await this.seed();
+
+    // Build context for the factory so prepare() writes session env vars.
+    // Include home and token so agent subprocesses use the same storage
+    // location and can create a hub for stamped GitHub operations.
+    const context = this.sessionId ? {
+      session: this.sessionId,
+      agent: null, // set after identity is acquired inside factory
+      repo: this.repo,
+      home: this.storage?.home,
+      token: process.env.GITHUB_TOKEN
+    } : undefined;
+
+    // Theme coherence: inherit from an existing agent so antagonist
+    // pairs always share the same theme universe. Only pick from
+    // config when no agents exist yet (first enlistment).
+    const existing = [...this.agents.values()].find(function hasTheme(a) { return a.identity?.theme; });
+    const theme = existing?.identity?.theme ?? pick(this.cfg?.get?.('theme'));
+
+    const agent = await this.factory.create(provider, role, {
+      theme,
+      model: this.cfg?.get?.('model'),
+      config: this.cfg,
+      context,
+      taken: this.takenNames,
+      ...opts
+    });
+
+    // Persist session data BEFORE spawn so the agent's MCP server
+    // subprocess can hydrate from storage on startup. Without this,
+    // the agent's _hydrate() call races against the host's save.
+    if (this.sessionId && this.storage && agent.identity?.name) {
+      const session = new Session({
+        identity: agent.identity.toJSON?.() ?? agent.identity,
+        role,
+        backend: agent.constructor.name,
+        paneId: null, // not yet known
+        repo: this.repo
+      });
+      await this.storage.save(this.sessionId, agent.identity.name, session.toJSON());
+    }
+
+    await this.spawn(agent);
+
+    if (this.sessionId && this.storage && agent.identity?.name && agent.paneId) {
+      const data = await this.storage.load(this.sessionId, agent.identity.name);
+      if (data) {
+        data.paneId = agent.paneId;
+        await this.storage.save(this.sessionId, agent.identity.name, data);
+      }
+    }
+
+    return agent;
+  }
+
+  /**
+   * Record a heartbeat timestamp for an agent. Resets the stall timer.
+   *
+   * @param {string} name - Agent identity name.
+   */
+  activity(name) {
+    this._lastActivity.set(name, new Date().toISOString());
+  }
+
+  /**
+   * Check whether an agent's tmux pane has new output since the last
+   * check. When output changes, the agent is provably active — update
+   * `_lastActivity` and return `true`. This is the ground truth signal
+   * that feeds into `health()` and, transitively, the proof-of-life
+   * responder.
+   *
+   * @param {string} name - Agent identity name.
+   * @returns {Promise<boolean>} True when pane output changed (agent is active).
+   */
+  async refresh(name) {
+    const agent = this.agents.get(name);
+    if (!agent?.capture) return false;
+
+    try {
+      const output = await agent.capture(50);
+      const digest = createHash('md5').update(output ?? '').digest('hex');
+      const prev = this._lastPaneHash.get(name);
+      this._lastPaneHash.set(name, digest);
+      if (prev && prev !== digest) {
+        this._lastActivity.set(name, new Date().toISOString());
+        return true;
+      }
+      return false;
+    } catch { return false; }
+  }
+
+  // ── Death Snapshot ──────────────────────────────────
+
+  /**
+   * Capture a dying agent's pane output and write it to the session
+   * logs directory as `<name>.death.log`. Requires `remain-on-exit`
+   * on the pane so output survives after the process exits.
+   *
+   * Non-fatal: silently skips when session or storage is unavailable,
+   * and logs a warning when capture or write fails.
+   *
+   * @param {string} name - Agent identity name.
+   * @param {object} agent - Agent instance with a `capture()` method.
+   * @returns {Promise<void>}
+   */
+  async snapshot(name, agent) {
+    if (!this.sessionId || !this.storage?.home) return;
+    try {
+      const raw = await agent.capture();
+      const cleaned = output.clean(raw);
+      const dir = join(this.storage.home, 'sessions', this.sessionId, 'logs');
+      await mkdir(dir, { recursive: true });
+      await writeFile(join(dir, `${name}.death.log`), cleaned, 'utf8');
+      log.info(`death snapshot written: ${name}`);
+    } catch (err) {
+      log.warn(`death snapshot failed for ${name}: ${err.message}`);
+    }
+  }
+
+  // ── Reconcile (Liveness Sweep) ───────────────────────
+
+  /**
+   * Synchronize registry state with actual tmux pane liveness.
+   *
+   * Iterates all registered agents and checks whether their underlying
+   * process is still alive. Dead agents are stopped, removed from the
+   * registry, and their identities released — closing the gap where
+   * team_status reports agents as "working" after their processes have
+   * exited.
+   *
+   * Dormant agents are skipped because they are already in a terminal
+   * state and are handled by the stall monitor's dormant cleanup.
+   *
+   * @returns {Promise<string[]>} Names of agents that were reconciled.
+   * @fires Orchestrator#removed
+   */
+  async reconcile() {
+    const reconciled = [];
+    const entries = [...this.agents.entries()];
+
+    for (const [name, agent] of entries) {
+      if (agent.state === 'dormant') continue;
+
+      let alive;
+      try {
+        alive = await agent.alive();
+      } catch {
+        continue;
+      }
+
+      if (alive) continue;
+
+      log.warn(`reconcile: ${name} pane is dead (state was "${agent.state}") — removing`);
+
+      await this.snapshot(name, agent);
+      try { await agent.stop(); } catch { /* pane already dead */ }
+
+      this.agents.delete(name);
+      this._lastActivity.delete(name);
+      this._lastPaneHash.delete(name);
+      this._removed.add(name);
+      this.identityRegistry.release(agent.identity);
+
+      this.emit('removed', { name, reason: 'reconciled', agent });
+      reconciled.push(name);
+    }
+
+    return reconciled;
+  }
+
+  // ── Health ───────────────────────────────────────────
+
+  /**
+   * Multi-signal health assessment for a named agent.
+   *
+   * Evaluates: tmux/process liveness, agent state machine, activity
+   * recency (last orchestrator interaction), and captured output length.
+   *
+   * @param {string} name - Agent identity name.
+   * @returns {Promise<{alive: boolean, status: string, details: string, outputLength?: number}>}
+   */
+  async health(name) {
+    const agent = this.agents.get(name);
+    if (!agent) return { alive: false, status: 'not-found', details: `agent ${name} not registered` };
+
+    if (agent.state === 'dormant')
+      return { alive: false, status: 'unhealthy', details: `agent ${name} is dormant` };
+
+    const paneAlive = await agent.alive();
+    if (!paneAlive)
+      return { alive: false, status: 'unhealthy', details: `agent ${name} pane is dead` };
+
+    const output = await agent.capture().catch(function noop() { return ''; });
+    const outputLength = output.length;
+
+    // Local proof-of-life: check tmux pane for real activity before
+    // declaring staleness. Agent-side MCP tool calls don't update
+    // _lastActivity, but they DO produce terminal output.
+    await this.refresh(name);
+
+    const lastTs = this._lastActivity.get(name);
+    if (lastTs) {
+      const elapsed = Date.now() - new Date(lastTs).getTime();
+      if (elapsed > this.stallTimeout)
+        return { alive: true, status: 'unhealthy', details: `agent ${name} activity is stale (${Math.round(elapsed / 1000)}s)`, outputLength };
+    }
+
+    return { alive: true, status: 'healthy', details: `agent ${name} is active`, outputLength };
+  }
+
+  // ── Scaling ──────────────────────────────────────────
+
+  /**
+   * Demand-driven scaling: collect demand signals from all registered
+   * workflows, then spawn agents to fill deficits — respecting global
+   * caps, per-role caps, rate limits, and cooldowns.
+   *
+   * Runs after all workflow handlers in the reactor chain so demand
+   * signals reflect the latest hydrated state. Spawns are capped by
+   * `maxPerTick` to avoid resource spikes.
+   *
+   * Priority order: reviewer > risk > action > planner. Reviewers
+   * unblock merges, risk unblocks reviewers, so they are filled first
+   * when at the global cap.
+   *
+   * @param {string} repo - Repository in "owner/name" format.
+   * @returns {Promise<Array<{role: string, agent: string}>>} Spawned agents.
+   */
+  async scale(repo) {
+    if (!this.workflows.size) return [];
+
+    const maxAgents = this.cfg?.get?.('scaling.maxAgents') ?? 8;
+    const maxPerRole = this.cfg?.get?.('scaling.maxPerRole') ?? {};
+    const maxPerTick = this.cfg?.get?.('scaling.maxPerTick') ?? 2;
+    const cooldown = this.cfg?.get?.('scaling.cooldown') ?? 30000;
+
+    // Collect demand signals from each workflow.
+    // Use the workflow's static `role` for enlist() — the map key may
+    // differ (e.g. map key 'review' vs static role 'reviewer'). The
+    // static role is what agents, demand(), and pair() filter on.
+    const signals = [];
+    for (const [key, workflow] of this.workflows) {
+      const role = workflow.constructor.role ?? key;
+      try {
+        const signal = await workflow.demand(repo);
+        signals.push({ role, ...signal });
+        log.debug(`scale: ${role} { workload: ${signal.workload}, supply: ${signal.supply}, deficit: ${signal.deficit} }`);
+      } catch (err) {
+        log.warn(`scale: demand() failed for ${role}: ${err.message}`);
+      }
+    }
+
+    // Global cap — only count live agents. Dormant agents stay in
+    // the map after exit but consume no resources.
+    const live = [...this.agents.values()]
+      .filter(function alive(a) { return a.state !== 'dormant'; }).length;
+    if (live >= maxAgents) {
+      log.debug(`scale: at global cap (${live}/${maxAgents}) — skipping`);
+      return [];
+    }
+
+    // Sort by priority: reviewer > risk > action > planner
+    const priority = { reviewer: 0, risk: 1, action: 2, planner: 3 };
+    signals.sort(function byPriority(a, b) {
+      return (priority[a.role] ?? 99) - (priority[b.role] ?? 99);
+    });
+
+    const spawned = [];
+    let budget = maxPerTick;
+    const now = Date.now();
+
+    for (const signal of signals) {
+      if (budget <= 0) break;
+      if (signal.deficit <= 0) continue;
+
+      const { role } = signal;
+      const roleCap = maxPerRole[role] ?? Infinity;
+      const current = [...this.agents.values()]
+        .filter(function liveRole(a) { return a.role === role && a.state !== 'dormant'; }).length;
+
+      // Per-role cap
+      if (current >= roleCap) {
+        log.debug(`scale: ${role} at role cap (${current}/${roleCap}) — skipping`);
+        continue;
+      }
+
+      // Cooldown check
+      const last = this._lastSpawn.get(role);
+      if (last && now - last < cooldown) {
+        log.debug(`scale: ${role} in cooldown (${Math.round((now - last) / 1000)}s < ${Math.round(cooldown / 1000)}s) — skipping`);
+        continue;
+      }
+
+      // How many to spawn: min of deficit, role headroom, global headroom, tick budget
+      const roleHeadroom = roleCap - current;
+      const globalHeadroom = maxAgents - live;
+      const count = Math.min(signal.deficit, roleHeadroom, globalHeadroom, budget);
+
+      if (count <= 0) continue;
+
+      await this.backendRegistry.discover();
+
+      for (let i = 0; i < count; i++) {
+        try {
+          const providers = this.backendRegistry.providers();
+          const info = capability(providers);
+          let provider;
+
+          // Reviewer and risk agents must oppose the action agents they
+          // pair with — pair() finds the cross-provider match, so
+          // spawning on the same side means scan()/assess() can never
+          // dispatch them. Pick the opposite of existing action agents.
+          //
+          // When no live action agent exists (dead/foreign), fall back
+          // to PR label metadata via demand().actionProviders so the
+          // correct opposing side is still selected.
+          let actionProvider = (role === 'reviewer' || role === 'risk')
+            ? [...this.agents.values()]
+              .find(function hasProvider(a) { return a.role === 'action' && a.identity?.provider; })
+              ?.identity?.provider
+            : null;
+
+          if (!actionProvider && signal.actionProviders?.length) {
+            actionProvider = signal.actionProviders[0];
+          }
+
+          if (actionProvider) {
+            const opposite = side(actionProvider) === 'yin' ? 'yang' : 'yin';
+            provider = providers.find(function isOpp(p) { return side(p) === opposite; });
+            if (!provider) {
+              if (info.mode === 'single') {
+                provider = providers[0] ?? null;
+                if (!provider) {
+                  log.warn(`scale: no providers available for ${role} — skipping`);
+                  break;
+                }
+                log.info(`scale: no ${opposite}-side provider available for ${role} — using single-side fallback (${provider})`);
+              } else {
+                log.warn(`scale: no ${opposite}-side provider available for ${role} — skipping`);
+                break;
+              }
+            }
+          } else if (role === 'reviewer' || role === 'risk') {
+            provider = providers[0];
+          } else {
+            // Action / planner: balance yin/yang within their own role
+            const yinCount = [...this.agents.values()]
+              .filter(function isRole(a) { return a.role === role && side(a.identity?.provider) === 'yin'; }).length;
+            const yangCount = [...this.agents.values()]
+              .filter(function isRole(a) { return a.role === role && side(a.identity?.provider) === 'yang'; }).length;
+
+            if (yinCount <= yangCount) {
+              provider = providers.find(function isYin(p) { return side(p) === 'yin'; }) ?? providers[0];
+            } else {
+              provider = providers.find(function isYang(p) { return side(p) === 'yang'; }) ?? providers[0];
+            }
+          }
+          // Final defensive default for non-pairing roles and mixed
+          // provider sets: if discovery returned providers but side
+          // selection yielded none, use the first discovered provider.
+          if (!provider && providers.length) provider = providers[0];
+
+          if (!provider) {
+            log.warn(`scale: no provider resolved for ${role} — skipping`);
+            break;
+          }
+
+          // Backend degradation fallback: when the native backend for
+          // the selected provider is degraded (repeated rapid failures
+          // like budget exhaustion), switch to the cursor-* virtual
+          // provider. Same yin/yang side, different API path.
+          if (!provider.startsWith('cursor-') && this.backendRegistry.degraded) {
+            let nativeBackend = null;
+            for (const info of this.backendRegistry.discovered.values()) {
+              if (info.provider === provider) { nativeBackend = info.name; break; }
+            }
+            if (nativeBackend && this.backendRegistry.degraded(nativeBackend)) {
+              const variant = `cursor-${provider}`;
+              if (providers.includes(variant)) {
+                log.info(`scale: ${nativeBackend} degraded, falling back to ${variant}`);
+                provider = variant;
+              }
+            }
+          }
+
+          const agent = await this.enlist(provider, role);
+          spawned.push({ role, agent: agent.identity.name });
+          log.info(`scale: spawned ${agent.identity.name} as ${role} (${provider})`);
+        } catch (err) {
+          log.warn(`scale: failed to spawn ${role} agent: ${err.message}`);
+          break;
+        }
+      }
+
+      if (spawned.length) {
+        this._lastSpawn.set(role, now);
+        budget -= count;
+      }
+    }
+
+    if (spawned.length) {
+      log.info(`scale: spawned ${spawned.length} agents — ${spawned.map(function fmt(s) { return `${s.role}:${s.agent}`; }).join(', ')}`);
+    }
+
+    return spawned;
+  }
+
+  // ── Reap (Global Safety Net) ─────────────────────────
+
+  /**
+   * Global safety net — stop dormant agents when no work remains.
+   *
+   * Role-specific reaping is handled by each workflow's `reap()` method
+   * (planner-reap, review-reap, action-reap). This global reap runs
+   * last in the reactor chain and catches anything the workflow-level
+   * reaps missed.
+   *
+   * Only **dormant** agents are eligible for global reaping. Agents in
+   * any other state (spawned, working, reviewing, etc.) are actively
+   * doing something and must not be interrupted. Stall detection is
+   * the separate mechanism that handles truly stuck agents.
+   *
+   * All of these conditions must be true before reaping:
+   * 1. No open issues with the `loreli` label
+   * 2. No open pull requests (PRs in flight = work not done)
+   * 3. Every remaining agent is dormant
+   *
+   * Registered as the last reactor handler so it runs after all
+   * workflow-specific reaps have completed.
+   *
+   * @param {string} repo - Repository in "owner/name" format.
+   * @returns {Promise<void>}
+   */
+  async reap(repo) {
+    if (!this.hub || !this.agents.size) return;
+
+    try {
+      // Only dormant agents are candidates — anything else is active work
+      const all = [...this.agents.values()];
+      const dormant = all.filter(function idle(a) { return a.state === 'dormant'; });
+      if (dormant.length === 0) {
+        log.debug(`reap: ${all.length} agents still active — skipping`);
+        return;
+      }
+
+      const open = await this.hub.issues(repo, { state: 'open' });
+      const loreli = open.filter(function tagged(i) {
+        return i.labels?.some?.(function isLoreli(l) {
+          const name = typeof l === 'string' ? l : l.name;
+          return name === 'loreli';
+        });
+      });
+
+      if (loreli.length > 0) return;
+
+      // Check for open PRs — work is still in flight if PRs exist
+      const prs = await this.hub.pulls(repo, { state: 'open' });
+      if (prs.length > 0) return;
+
+      log.info(`reap: no open loreli issues or PRs — stopping ${dormant.length} dormant agents`);
+
+      for (const agent of dormant) {
+        try {
+          await this.kill(agent.identity.name);
+          log.info(`reap: stopped ${agent.identity.name}`);
+        } catch (err) {
+          log.warn(`reap: failed to stop ${agent.identity.name}: ${err.message}`);
+        }
+      }
+    } catch (err) {
+      log.debug(`reap: skipped — ${err.message}`);
+    }
+  }
+
+  // ── Reactor (Polling Loop) ────────────────────────────
+
+  /**
+   * Register a reactor handler called on every tick.
+   * Role packages register their scan/forward/land handlers here.
+   *
+   * @param {string} name - Handler name (for logging/debugging).
+   * @param {function(string): Promise<void>} handler - Async function receiving repo.
+   */
+  register(name, handler) {
+    this._handlers.set(name, handler);
+    log.info(`reactor handler registered: ${name}`);
+  }
+
+  /**
+   * Execute one polling iteration: call all registered handlers.
+   * Each handler receives the repo string. Errors in one handler
+   * do not prevent subsequent handlers from running.
+   *
+   * @param {string} repo - Repository in "owner/name" format.
+   * @returns {Promise<void>}
+   */
+  async tick(repo) {
+    log.debug('tick start');
+
+    for (const [name, handler] of this._handlers) {
+      try {
+        await handler(repo);
+      } catch (err) {
+        log.error(`reactor handler "${name}" failed: ${err.message}`);
+      }
+    }
+
+    log.debug('tick end');
+  }
+
+  /**
+   * Start the polling reactor loop using a self-scheduling setTimeout
+   * chain. Each tick runs to completion before the next is scheduled,
+   * eliminating overlap risk without a reentrant guard. This pattern
+   * is more reliable than setInterval for async callbacks — setInterval
+   * does not await its callback, and combined with unref() it can
+   * silently stop firing after heavy async operations (observed in
+   * production after agent spawn via tmux).
+   *
+   * @param {string} repo - Repository in "owner/name" format.
+   */
+  watch(repo) {
+    if (this._watchHandle) return;
+    this.repo = repo;
+
+    const interval = this.cfg?.get?.('watch.interval') ?? 60000;
+    const self = this;
+
+    log.info(`watcher started for ${repo} (interval: ${interval}ms)`);
+
+    function schedule() {
+      self._watchHandle = setTimeout(async function cycle() {
+        await self.tick(repo);
+        if (self._watchHandle) schedule();
+      }, interval);
+
+      self._watchHandle.unref();
+    }
+
+    schedule();
+  }
+
+  /**
+   * Stop the polling reactor loop.
+   */
+  unwatch() {
+    if (this._watchHandle) {
+      clearTimeout(this._watchHandle);
+      this._watchHandle = null;
+      log.info('watcher stopped');
+    }
+  }
+
+  // ── Monitor (Stall Detection) ─────────────────────────
+
+  /**
+   * Start the stall detection monitor with 3-tier escalation.
+   *
+   * Tier 1 — Nudge (1x stall timeout):
+   *   Send a message to the agent asking for status. Emits 'stall'
+   *   with severity 'nudge'.
+   *
+   * Tier 2 — Warning (2x stall timeout):
+   *   Emits 'stall' with severity 'warning'. Role packages can
+   *   subscribe and post GitHub comments.
+   *
+   * Tier 3 — Critical (3x stall timeout):
+   *   Kills the agent and emits 'stall' with severity 'critical'.
+   *
+   * @fires Orchestrator#stall
+   */
+  monitor() {
+    if (this._monitorHandle) return;
+    log.info('stall detection monitor started');
+
+    const stallTimeout = this.stallTimeout;
+    const nudge = this.cfg?.get?.('timeouts.nudge') ?? true;
+    const self = this;
+
+    this._monitorHandle = setInterval(async function checkStalls() {
+      // Reconcile first: detect dead panes and clean up before
+      // running stall-escalation checks. Without this, dead agents
+      // linger until stallTimeout elapses.
+      await self.reconcile();
+
+      const now = Date.now();
+
+      // Snapshot keys to avoid mutation during iteration — Tier 3
+      // calls kill() which deletes from self.agents mid-loop.
+      const snapshot = [...self.agents.entries()];
+      for (const [name, agent] of snapshot) {
+        // Dormant agents are kept registered so downstream workflows
+        // (e.g. review scan) can still match them by identity. Skip
+        // nudge/warning, but allow Tier 3 kill for eventual cleanup.
+        if (agent.state === 'dormant') {
+          const last = self._lastActivity.get(name);
+          if (!last) continue;
+          const elapsed = now - new Date(last).getTime();
+          if (elapsed > stallTimeout * 3) {
+            self.agents.delete(name);
+            self._lastActivity.delete(name);
+            self._lastPaneHash.delete(name);
+            log.info(`stall: cleaned up dormant agent ${name}`);
+          }
+          continue;
+        }
+
+        const last = self._lastActivity.get(name);
+        if (!last) continue;
+
+        // Local proof-of-life: check tmux pane for real activity
+        // before escalating. If output changed, _lastActivity is
+        // now current and the tier checks naturally skip.
+        if (await self.refresh(name)) continue;
+
+        const elapsed = now - new Date(last).getTime();
+
+        if (elapsed > stallTimeout * 3) {
+          // Tier 3: Critically stalled — kill and emit
+          log.error(`agent ${name} critically stalled (${Math.round(elapsed / 1000)}s) — killing`);
+
+          /**
+           * @event Orchestrator#stall
+           * @type {object}
+           * @property {string} name - Agent identity name.
+           * @property {number} elapsed - Time since last activity in ms.
+           * @property {string} severity - 'nudge', 'warning', or 'critical'.
+           */
+          self.emit('stall', { name, elapsed, severity: 'critical' });
+
+          try {
+            await self.kill(name);
+            log.info(`stall tier 3: agent ${name} killed`);
+          } catch (err) {
+            log.error(`stall tier 3: kill failed for ${name}: ${err.message}`);
+          }
+        } else if (elapsed > stallTimeout * 2) {
+          // Tier 2: Warning
+          log.warn(`agent ${name} stalled tier 2 (${Math.round(elapsed / 1000)}s)`);
+          self.emit('stall', { name, elapsed, severity: 'warning' });
+        } else if (elapsed > stallTimeout) {
+          // Tier 1: Optional nudge
+          if (nudge) {
+            log.warn(`agent ${name} stalled tier 1 (${Math.round(elapsed / 1000)}s) - nudging`);
+            try {
+              await agent.send('You appear to be stalled. Please report your current status or continue working.');
+              // Activity resets only when the agent responds (via MCP tool
+              // calls or hub activity), NOT when we nudge it. Resetting
+              // here would trap agents at tier 1 forever.
+            } catch (err) { log.debug(`monitor: nudge failed for ${name}: ${err.message}`); }
+          } else {
+            log.warn(`agent ${name} stalled tier 1 (${Math.round(elapsed / 1000)}s) - nudge suppressed by config`);
+          }
+          self.emit('stall', { name, elapsed, severity: 'nudge' });
+        }
+      }
+    }, Math.min(stallTimeout / 2, 60000));
+
+    this._monitorHandle.unref();
+  }
+
+  /**
+   * Stop the stall detection monitor.
+   */
+  stopMonitor() {
+    if (this._monitorHandle) {
+      clearInterval(this._monitorHandle);
+      this._monitorHandle = null;
+      log.info('stall detection monitor stopped');
+    }
+  }
+}