loreli 1.0.0 → 2.0.0

package/packages/orchestrator/src/index.js

@@ -6,6 +6,7 @@ import { Factory, Session, output } from 'loreli/agent';
 import { Tmux } from 'loreli/tmux';
 import { prepare } from 'loreli/workspace';
 import { pick, side, capability } from 'loreli/identity';
+import { classify } from 'loreli/classify';
 import { logger } from 'loreli/log';
 
 const log = logger('orchestrator');
@@ -24,9 +25,19 @@ const FATAL_PATTERNS = [
   /authentication[_ ]*(error|failed)/i,
   /invalid[_ ]*api[_ ]*key/i,
   /quota[_ ]*exceeded/i,
-  /insufficient[_ ]*quota/i
+  /insufficient[_ ]*quota/i,
+  /invalid model name/i,
+  /unable to connect to api/i,
+  /connection\s*refused/i
 ];
 
+/**
+ * Maximum pane characters logged in diagnostic debug output.
+ *
+ * @type {number}
+ */
+const PANE_DEBUG_LIMIT = 4000;
+
 /**
  * Check if pane output contains fatal API error patterns.
  *
@@ -38,6 +49,46 @@ function hasFatalError(output) {
   return FATAL_PATTERNS.some(function match(p) { return p.test(output); });
 }
 
+/**
+ * Format captured pane output for debug logging.
+ *
+ * Keeps logs readable while still preserving enough context to validate
+ * classifier and fallback decisions during stall/rapid-death diagnosis.
+ *
+ * @param {string} output - Raw pane output.
+ * @returns {string} Pane text, truncated when necessary.
+ */
+function paneDebug(output) {
+  if (!output) return '[empty pane output]';
+  if (output.length <= PANE_DEBUG_LIMIT) return output;
+  const rest = output.length - PANE_DEBUG_LIMIT;
+  return `${output.slice(0, PANE_DEBUG_LIMIT)}\n… [truncated ${rest} chars]`;
+}
+
+/**
+ * Normalize remedy instructions into tmux key names.
+ *
+ * Classifier prompts return remedies as space-delimited strings
+ * (`"Down Enter"`), while backend fallback diagnose methods return
+ * string arrays (`['Down', 'Enter']`). The orchestrator accepts both.
+ *
+ * @param {string|string[]|null|undefined} remedy - Remedy from diagnosis.
+ * @returns {string[]} Tmux key sequence.
+ */
+function remedy(remedy) {
+  if (Array.isArray(remedy)) {
+    const keys = remedy.filter(Boolean);
+    if (keys.length > 0) return keys;
+    return ['Enter'];
+  }
+  if (typeof remedy === 'string') {
+    const keys = remedy.split(/\s+/).filter(Boolean);
+    if (keys.length > 0) return keys;
+    return ['Enter'];
+  }
+  return ['Enter'];
+}
+
 /**
  * Generic agent lifecycle coordinator via EventEmitter.
  *
@@ -108,6 +159,9 @@ export class Orchestrator extends EventEmitter {
     /** @type {Map<string, string>} MD5 hash of last captured pane output per agent for tmux-based activity detection. */
     this._lastPaneHash = new Map();
 
+    /** @type {Map<string, number>} Consecutive classify failures per agent — safety net kill after threshold. */
+    this._classifyFails = new Map();
+
     /** @type {NodeJS.Timeout|null} Reactor polling interval handle. */
     this._watchHandle = null;
 
@@ -231,34 +285,121 @@ export class Orchestrator extends EventEmitter {
       // backend is likely broken (budget exhaustion, API outage).
       // Mark it as degraded so scale() falls back to cursor-agent.
       //
-      // Two checks:
-      //  1. Dead pane → agent exited on error
-      //  2. Stuck-alive → agent stays alive but shows budget/rate-limit
-      //     errors in its pane output
+      // Uses the pane-state classifier when pane output is available
+      // (remain-on-exit keeps dead panes capturable). Falls back to
+      // raw alive() when capture fails.
       if (agent.backend && agent.alive) {
         const backend = agent.backend;
         const name = agent.identity.name;
         const registry = this.backendRegistry;
         const self = this;
         const timer = setTimeout(async function rapidDeathCheck() {
+          if (agent.state === 'dormant') return;
+
           try {
             const alive = await agent.alive();
-            if (!alive && agent.state !== 'dormant') {
-              log.warn(`rapid death: ${name} died within ${self.rapidDeathDelay}ms of spawn — marking ${backend} degraded`);
+
+            // Agent is alive and healthy — no rapid death
+            if (alive && !agent.capture) return;
+
+            let output;
+            try {
+              output = agent.capture
+                ? await agent.capture(self.cfg?.get?.('classify.maxLines') ?? 100)
+                : null;
+            } catch { output = null; }
+            if (output !== null) {
+              log.debug(`rapid-death pane ${name} (${backend}, alive=${alive}):\n${paneDebug(output)}`);
+            } else {
+              log.debug(`rapid-death pane ${name} (${backend}, alive=${alive}): [capture unavailable]`);
+            }
+
+            // Classify the pane output to determine why the agent
+            // died or what error it hit while still alive.
+            let diagnosis;
+            if (output) {
+              try {
+                diagnosis = await classify('pane-state', output, {
+                  backends: self.backendRegistry,
+                  config: self.cfg,
+                  vars: { model: agent.model, backend, role: agent.role }
+                });
+                log.info(`rapid-death classify ${name}: ${diagnosis.category} — ${diagnosis.reasoning}`);
+              } catch (err) {
+                log.warn(`rapid-death classify failed for ${name}: ${err.message}`);
+              }
+            }
+
+            // When LLM classify fails, fall back to backend-specific
+            // regex detection. Each backend knows its CLI's dialog patterns.
+            let category = diagnosis?.category;
+            if (alive && output) {
+              const fallback = registry?.diagnose?.(backend, output);
+              const actionable = new Set(['option_dialog', 'waiting_for_input', 'fatal', 'dead']);
+              const fallbackActionable = actionable.has(fallback?.category);
+              const llmActionable = actionable.has(category);
+              const llmCategory = category;
+
+              if (!category && fallback) {
+                category = fallback.category;
+                diagnosis = fallback;
+                log.info(`rapid-death fallback diagnose ${name}: ${category} — ${fallback.reasoning}`);
+              } else if (fallbackActionable && !llmActionable) {
+                category = fallback.category;
+                diagnosis = fallback;
+                log.info(`rapid-death fallback override ${name}: ${fallback.category} over ${llmCategory ?? 'unknown'} — ${fallback.reasoning}`);
+              }
+            }
+
+            if (!alive) {
+              log.warn(`rapid death: ${name} died within ${self.rapidDeathDelay}ms of spawn (${category ?? 'unknown'}) — marking ${backend} degraded`);
               registry?.recordFailure(backend);
-              if (agent.canTransition?.('dormant')) agent.transition('dormant');
-              self.emit('rapid-death', { name, backend });
+              try { await self.kill(name); } catch { /* already dead */ }
+              self.emit('rapid-death', { name, backend, diagnosis });
               return;
             }
 
-            if (alive && agent.capture) {
-              const output = await agent.capture();
-              if (hasFatalError(output)) {
-                log.warn(`stuck-alive: ${name} shows fatal API error — marking ${backend} degraded`);
-                registry?.recordFailure(backend);
-                try { await agent.stop(); } catch { /* stop can fail */ }
-                self.emit('rapid-death', { name, backend, reason: 'stuck-alive' });
-              }
+            // Alive with recoverable dialog — send the appropriate
+            // input to dismiss it. Record a soft warning instead of a
+            // hard failure so the backend isn't blacklisted for a
+            // transient issue. Repeated warnings promote to failure.
+            if (category === 'option_dialog') {
+              const keys = remedy(diagnosis?.remedy);
+              log.info(`rapid-death remediation: ${name} has option dialog — sending ${keys.join('+')}`);
+              try {
+                const tmux = new Tmux();
+                await tmux.keys(agent.paneId, ...keys);
+              } catch (err) { log.debug(`rapid-death: keys failed for ${name}: ${err.message}`); }
+              registry?.recordWarning?.(backend);
+              self.emit('rapid-death', { name, backend, reason: 'remediated', diagnosis });
+              return;
+            }
+
+            if (category === 'waiting_for_input') {
+              log.info(`rapid-death remediation: ${name} waiting for input — sending continuation`);
+              try {
+                await agent.send('Please continue working or report your status.');
+              } catch (err) { log.debug(`rapid-death: send failed for ${name}: ${err.message}`); }
+              registry?.recordWarning?.(backend);
+              self.emit('rapid-death', { name, backend, reason: 'remediated', diagnosis });
+              return;
+            }
+
+            // Alive but classifier detected fatal state
+            if (category === 'fatal' || category === 'dead') {
+              log.warn(`stuck-alive: ${name} classified as ${category} — marking ${backend} degraded`);
+              registry?.recordFailure(backend);
+              try { await agent.stop(); } catch { /* stop can fail */ }
+              self.emit('rapid-death', { name, backend, reason: 'stuck-alive', diagnosis });
+              return;
+            }
+
+            // Alive but regex fallback for when classifier didn't detect fatal
+            if (alive && output && hasFatalError(output)) {
+              log.warn(`stuck-alive: ${name} shows fatal API error (regex) — marking ${backend} degraded`);
+              registry?.recordFailure(backend);
+              try { await agent.stop(); } catch { /* stop can fail */ }
+              self.emit('rapid-death', { name, backend, reason: 'stuck-alive' });
             }
           } catch { /* pane check can fail when session is torn down */ }
         }, this.rapidDeathDelay);
@@ -378,6 +519,7 @@ export class Orchestrator extends EventEmitter {
     this.agents.delete(name);
     this._lastActivity.delete(name);
     this._lastPaneHash.delete(name);
+    this._classifyFails.delete(name);
     this._removed.add(name);
     this.identityRegistry.release(agent.identity);
     log.info(`agent shut down: ${name}`);
@@ -424,6 +566,7 @@ export class Orchestrator extends EventEmitter {
     this.agents.delete(name);
     this._lastActivity.delete(name);
     this._lastPaneHash.delete(name);
+    this._classifyFails.delete(name);
     this._removed.add(name);
     this.identityRegistry.release(agent.identity);
 
@@ -573,7 +716,7 @@ export class Orchestrator extends EventEmitter {
 
     const agent = await this.factory.create(provider, role, {
       theme,
-      model: this.cfg?.get?.('model'),
+      model: this.cfg?.get?.(`workflows.${role}.model`) ?? this.cfg?.get?.('model'),
       config: this.cfg,
       context,
       taken: this.takenNames,
@@ -594,7 +737,14 @@ export class Orchestrator extends EventEmitter {
       await this.storage.save(this.sessionId, agent.identity.name, session.toJSON());
     }
 
-    await this.spawn(agent);
+    try {
+      await this.spawn(agent);
+    } catch (err) {
+      if (this.sessionId && this.storage && agent.identity?.name) {
+        try { await this.storage.remove(this.sessionId, agent.identity.name); } catch { /* best-effort */ }
+      }
+      throw err;
+    }
 
     if (this.sessionId && this.storage && agent.identity?.name && agent.paneId) {
       const data = await this.storage.load(this.sessionId, agent.identity.name);
@@ -636,6 +786,7 @@ export class Orchestrator extends EventEmitter {
       const prev = this._lastPaneHash.get(name);
       this._lastPaneHash.set(name, digest);
       if (prev && prev !== digest) {
+        if (hasFatalError(output)) return false;
         this._lastActivity.set(name, new Date().toISOString());
         return true;
       }
@@ -643,6 +794,29 @@ export class Orchestrator extends EventEmitter {
     } catch { return false; }
   }
 
+  /**
+   * Update the pane hash after an orchestrator-initiated interaction.
+   *
+   * Must be called after any action that changes the pane content
+   * (sending keys, messages, etc.) to prevent `refresh()` from
+   * misinterpreting the orchestrator's own output as agent activity
+   * on the next monitor cycle. The orchestrator also resets the
+   * stall timer here because a remediation attempt should buy the
+   * agent time to react before another nudge is sent.
+   *
+   * @param {string} name - Agent identity name.
+   * @param {object} agent - Agent instance with a `capture()` method.
+   * @returns {Promise<void>}
+   */
+  async _rehash(name, agent) {
+    try {
+      const content = await agent.capture(50);
+      const digest = createHash('md5').update(content ?? '').digest('hex');
+      this._lastPaneHash.set(name, digest);
+      this._lastActivity.set(name, new Date().toISOString());
+    } catch { /* capture can fail if pane died */ }
+  }
+
   // ── Death Snapshot ──────────────────────────────────
 
   /**
@@ -698,8 +872,9 @@ export class Orchestrator extends EventEmitter {
       let alive;
       try {
         alive = await agent.alive();
-      } catch {
-        continue;
+      } catch (err) {
+        log.warn(`reconcile: alive() threw for ${name}: ${err.message} — treating as dead`);
+        alive = false;
       }
 
       if (alive) continue;
@@ -712,6 +887,7 @@ export class Orchestrator extends EventEmitter {
       this.agents.delete(name);
       this._lastActivity.delete(name);
       this._lastPaneHash.delete(name);
+      this._classifyFails.delete(name);
       this._removed.add(name);
       this.identityRegistry.release(agent.identity);
 
@@ -784,7 +960,6 @@ export class Orchestrator extends EventEmitter {
     if (!this.workflows.size) return [];
 
     const maxAgents = this.cfg?.get?.('scaling.maxAgents') ?? 8;
-    const maxPerRole = this.cfg?.get?.('scaling.maxPerRole') ?? {};
     const maxPerTick = this.cfg?.get?.('scaling.maxPerTick') ?? 2;
     const cooldown = this.cfg?.get?.('scaling.cooldown') ?? 30000;
 
@@ -828,7 +1003,7 @@ export class Orchestrator extends EventEmitter {
       if (signal.deficit <= 0) continue;
 
       const { role } = signal;
-      const roleCap = maxPerRole[role] ?? Infinity;
+      const roleCap = this.cfg?.get?.(`workflows.${role}.maxAgents`) ?? Infinity;
       const current = [...this.agents.values()]
         .filter(function liveRole(a) { return a.role === role && a.state !== 'dormant'; }).length;
 
@@ -1108,18 +1283,22 @@ export class Orchestrator extends EventEmitter {
   // ── Monitor (Stall Detection) ─────────────────────────
 
   /**
-   * Start the stall detection monitor with 3-tier escalation.
+   * Start the stall detection monitor with LLM-powered classification.
    *
-   * Tier 1 — Nudge (1x stall timeout):
-   *   Send a message to the agent asking for status. Emits 'stall'
-   *   with severity 'nudge'.
+   * When an agent's pane output has not changed for longer than the
+   * stall timeout, the monitor captures the pane content, classifies
+   * it via `loreli/classify`, and dispatches the appropriate action:
    *
-   * Tier 2 — Warning (2x stall timeout):
-   *   Emits 'stall' with severity 'warning'. Role packages can
-   *   subscribe and post GitHub comments.
+   * - `working` — reset activity timer, leave the agent alone
+   * - `waiting_for_input` — send a continuation prompt
+   * - `option_dialog` — send the appropriate keystroke (Enter)
+   * - `error_loop` — emit 'stall' with diagnostic context
+   * - `idle` — transition the agent to dormant
+   * - `fatal` — kill the agent and mark the backend degraded
    *
-   * Tier 3 — Critical (3x stall timeout):
-   *   Kills the agent and emits 'stall' with severity 'critical'.
+   * Falls back to regex heuristics when no LLM backend is available.
+   * Consecutive classification failures trigger a safety-net kill
+   * (replaces the old tier 3 fixed-time kill).
    *
    * @fires Orchestrator#stall
    */
@@ -1128,24 +1307,23 @@ export class Orchestrator extends EventEmitter {
     log.info('stall detection monitor started');
 
     const stallTimeout = this.stallTimeout;
-    const nudge = this.cfg?.get?.('timeouts.nudge') ?? true;
+    const maxClassifyFails = this.cfg?.get?.('classify.maxRetries') ?? 5;
     const self = this;
 
+    /** @type {boolean} Re-entrancy guard for the monitor callback. */
+    this._monitoring = false;
+
     this._monitorHandle = setInterval(async function checkStalls() {
-      // Reconcile first: detect dead panes and clean up before
-      // running stall-escalation checks. Without this, dead agents
-      // linger until stallTimeout elapses.
+      if (self._monitoring) return;
+      self._monitoring = true;
+
+      try {
       await self.reconcile();
 
       const now = Date.now();
-
-      // Snapshot keys to avoid mutation during iteration — Tier 3
-      // calls kill() which deletes from self.agents mid-loop.
       const snapshot = [...self.agents.entries()];
+
       for (const [name, agent] of snapshot) {
-        // Dormant agents are kept registered so downstream workflows
-        // (e.g. review scan) can still match them by identity. Skip
-        // nudge/warning, but allow Tier 3 kill for eventual cleanup.
         if (agent.state === 'dormant') {
           const last = self._lastActivity.get(name);
           if (!last) continue;
@@ -1154,6 +1332,7 @@ export class Orchestrator extends EventEmitter {
             self.agents.delete(name);
             self._lastActivity.delete(name);
             self._lastPaneHash.delete(name);
+            self._classifyFails.delete(name);
             log.info(`stall: cleaned up dormant agent ${name}`);
           }
           continue;
@@ -1162,52 +1341,91 @@ export class Orchestrator extends EventEmitter {
         const last = self._lastActivity.get(name);
         if (!last) continue;
 
-        // Local proof-of-life: check tmux pane for real activity
-        // before escalating. If output changed, _lastActivity is
-        // now current and the tier checks naturally skip.
-        if (await self.refresh(name)) continue;
+        if (await self.refresh(name)) {
+          self._classifyFails.set(name, 0);
+          continue;
+        }
 
         const elapsed = now - new Date(last).getTime();
+        if (elapsed <= stallTimeout) continue;
 
-        if (elapsed > stallTimeout * 3) {
-          // Tier 3: Critically stalled — kill and emit
-          log.error(`agent ${name} critically stalled (${Math.round(elapsed / 1000)}s) — killing`);
+        // Stall detected — classify the pane content
+        const maxLines = self.cfg?.get?.('classify.maxLines') ?? 100;
+        let result;
 
-          /**
-           * @event Orchestrator#stall
-           * @type {object}
-           * @property {string} name - Agent identity name.
-           * @property {number} elapsed - Time since last activity in ms.
-           * @property {string} severity - 'nudge', 'warning', or 'critical'.
-           */
-          self.emit('stall', { name, elapsed, severity: 'critical' });
-
-          try {
-            await self.kill(name);
-            log.info(`stall tier 3: agent ${name} killed`);
-          } catch (err) {
-            log.error(`stall tier 3: kill failed for ${name}: ${err.message}`);
+        try {
+          const pane = await agent.capture(maxLines);
+          log.debug(`monitor pane ${name} (${agent.backend}, stale=${Math.round(elapsed / 1000)}s):\n${paneDebug(pane)}`);
+          result = await classify('pane-state', pane, {
+            backends: self.backendRegistry,
+            config: self.cfg,
+            vars: { model: agent.model, backend: agent.backend, role: agent.role }
+          });
+          self._classifyFails.set(name, 0);
+          log.info(`classify ${name}: ${result.category} — ${result.reasoning}`);
+        } catch (err) {
+          const fails = (self._classifyFails.get(name) ?? 0) + 1;
+          self._classifyFails.set(name, fails);
+          log.warn(`classify failed for ${name} (${fails}/${maxClassifyFails}): ${err.message}`);
+
+          if (fails >= maxClassifyFails) {
+            log.error(`agent ${name} unclassifiable after ${fails} attempts — killing as safety net`);
+            self.emit('stall', { name, elapsed, severity: 'critical' });
+            try { await self.kill(name); } catch (e) { log.error(`safety kill failed for ${name}: ${e.message}`); }
           }
-        } else if (elapsed > stallTimeout * 2) {
-          // Tier 2: Warning
-          log.warn(`agent ${name} stalled tier 2 (${Math.round(elapsed / 1000)}s)`);
-          self.emit('stall', { name, elapsed, severity: 'warning' });
-        } else if (elapsed > stallTimeout) {
-          // Tier 1: Optional nudge
-          if (nudge) {
-            log.warn(`agent ${name} stalled tier 1 (${Math.round(elapsed / 1000)}s) - nudging`);
+          continue;
+        }
+
+        switch (result.category) {
+          case 'working':
+            self._lastActivity.set(name, new Date().toISOString());
+            break;
+
+          case 'waiting_for_input':
             try {
-              await agent.send('You appear to be stalled. Please report your current status or continue working.');
-              // Activity resets only when the agent responds (via MCP tool
-              // calls or hub activity), NOT when we nudge it. Resetting
-              // here would trap agents at tier 1 forever.
-            } catch (err) { log.debug(`monitor: nudge failed for ${name}: ${err.message}`); }
-          } else {
-            log.warn(`agent ${name} stalled tier 1 (${Math.round(elapsed / 1000)}s) - nudge suppressed by config`);
+              await agent.send('Please continue working or report your status.');
+              await self._rehash(name, agent);
+            } catch (err) { log.debug(`monitor: send failed for ${name}: ${err.message}`); }
+            self.emit('stall', { name, elapsed, severity: 'nudge', diagnosis: result });
+            break;
+
+          case 'option_dialog': {
+            const keys = remedy(result.remedy);
+            try {
+              const tmux = new Tmux();
+              await tmux.keys(agent.paneId, ...keys);
+              await self._rehash(name, agent);
+            } catch (err) { log.debug(`monitor: keys failed for ${name}: ${err.message}`); }
+            self.emit('stall', { name, elapsed, severity: 'nudge', diagnosis: result });
+            break;
           }
-          self.emit('stall', { name, elapsed, severity: 'nudge' });
+
+          case 'error_loop':
+            self.emit('stall', { name, elapsed, severity: 'warning', diagnosis: result });
+            break;
+
+          case 'idle':
+            agent.transition?.('dormant');
+            self.emit('stall', { name, elapsed, severity: 'nudge', diagnosis: result });
+            break;
+
+          case 'fatal':
+            log.error(`agent ${name} hit fatal error — killing`);
+            self.emit('stall', { name, elapsed, severity: 'critical', diagnosis: result });
+            try {
+              await self.kill(name);
+              self.backendRegistry?.recordFailure?.(agent.backend);
+            } catch (err) { log.error(`fatal kill failed for ${name}: ${err.message}`); }
+            break;
+
+          default:
+            log.warn(`classify ${name}: unknown category "${result.category}"`);
+            break;
         }
       }
+      } finally {
+        self._monitoring = false;
+      }
     }, Math.min(stallTimeout / 2, 60000));
 
     this._monitorHandle.unref();
@@ -1223,4 +1441,52 @@ export class Orchestrator extends EventEmitter {
       log.info('stall detection monitor stopped');
     }
   }
+
+  // ── Halt (Full System Stop) ──────────────────────────
+
+  /**
+   * Stop the entire orchestrator: reactor loop, stall monitor, and
+   * all registered agents. The MCP server process stays alive so the
+   * user can call `start` again to resume.
+   *
+   * Composes {@link unwatch}, {@link stopMonitor}, and {@link kill}
+   * into a single atomic operation. Idempotent — safe to call when
+   * already halted.
+   *
+   * @returns {Promise<{reactor: boolean, monitor: boolean, agents: string[]}>}
+   *   Summary of what was stopped.
+   * @fires Orchestrator#halted
+   */
+  async halt() {
+    const reactor = Boolean(this._watchHandle);
+    const monitor = Boolean(this._monitorHandle);
+
+    this.unwatch();
+    this.stopMonitor();
+
+    const killed = [];
+    const entries = [...this.agents.entries()];
+
+    for (const [name] of entries) {
+      try {
+        await this.kill(name);
+        killed.push(name);
+      } catch (err) {
+        log.warn(`halt: failed to kill ${name}: ${err.message}`);
+      }
+    }
+
+    log.info(`halt: reactor=${reactor} monitor=${monitor} agents=${killed.length}`);
+
+    /**
+     * @event Orchestrator#halted
+     * @type {object}
+     * @property {boolean} reactor - Whether the reactor was running.
+     * @property {boolean} monitor - Whether the monitor was running.
+     * @property {string[]} agents - Names of agents that were killed.
+     */
+    this.emit('halted', { reactor, monitor, agents: killed });
+
+    return { reactor, monitor, agents: killed };
+  }
 }