iriai-build 0.1.0

package/v3/recovery.js

@@ -0,0 +1,508 @@
+// recovery.js — Startup recovery: orphan cleanup, stale signal processing.
+// Runs once at bridge startup to restore state from SQLite + filesystem.
+
+import fs from "node:fs";
+import path from "node:path";
+import * as queries from "./queries.js";
+import { PLANNING_ROLES, PIPELINE_ORDER, SIGNAL } from "./constants.js";
+
+export class Recovery {
+  constructor({ orchestrator, adapter }) {
+    this.orchestrator = orchestrator;
+    this.adapter = adapter;
+  }
+
+  /**
+   * Run selective recovery for a single feature.
+   * Used by CLI commands to resume a specific feature.
+   */
+  async runForFeature(featureId) {
+    const feature = queries.getFeatureById(featureId);
+    if (!feature) return;
+
+    console.log(`[recovery] Running selective recovery for feature: ${feature.slug} (${feature.phase})`);
+
+    if (feature.phase === "planning" && feature.active_planning_role) {
+      // Recover first (checks .done on disk), THEN set up watchers which emit stale signals
+      // (stale signal handlers may delete .done files via readSignal({ deleteAfter: true }))
+      await this._recoverPlanningFeature(feature);
+      await this._setupPlanningWatchersForFeature(feature);
+    } else if (feature.phase === "impl") {
+      await this._cleanDeadAgentsForFeature(featureId);
+      await this._recoverImplFeature(feature);
+    } else if (feature.phase === "plan-approval") {
+      const teamsDir = path.join(feature.signal_dir, "teams");
+      if (fs.existsSync(teamsDir) && fs.readdirSync(teamsDir).some(d => d.startsWith("team-"))) {
+        console.log(`[recovery] Feature ${feature.slug} phase is plan-approval but has impl infrastructure — fixing phase to impl`);
+        queries.updateFeaturePhase(feature.id, "impl");
+        await this._cleanDeadAgentsForFeature(featureId);
+        await this._recoverImplFeature({ ...feature, phase: "impl" });
+      } else {
+        console.log(`[recovery] Feature ${feature.slug} awaiting plan approval — re-notifying Operator`);
+        await this.orchestrator.repostPendingDecision(featureId);
+      }
+    }
+
+    // Re-present any pending decision that was lost across restart
+    // (skip phases already handled above to avoid double-notify)
+    if (feature.phase !== "plan-approval" && feature.phase !== "planning") {
+      const pendingDecision = queries.getPendingDecision(featureId);
+      if (pendingDecision) {
+        console.log(`[recovery] Re-presenting pending decision: ${pendingDecision.decision_id}`);
+        await this.orchestrator.repostPendingDecision(featureId);
+      }
+    }
+
+    // Recover any stuck relay entries for this feature
+    const stuckEntries = queries.getProcessingRelays(featureId);
+    for (const entry of stuckEntries) {
+      // Skip decision-needed entries — already rendered by postDecision()
+      if (entry.event_hint === "decision-needed") {
+        queries.updateRelayStatus(entry.id, "posted", { processedAt: new Date().toISOString() });
+        continue;
+      }
+      try {
+        await this.adapter.postAgentResponse(feature.id, entry.source_agent, entry.raw_content);
+        queries.updateRelayStatus(entry.id, "posted", { processedAt: new Date().toISOString() });
+      } catch (err) {
+        queries.updateRelayStatus(entry.id, "failed");
+      }
+    }
+  }
+
+  /**
+   * Set up signal watchers and process stale planning signals for a single feature.
+   * Mirrors _processExistingPlanningSignals() but scoped to one feature.
+   */
+  async _setupPlanningWatchersForFeature(feature) {
+    const featureDir = feature.signal_dir;
+    const planningDir = path.join(featureDir, "planning");
+    if (!fs.existsSync(planningDir)) return;
+
+    const planningTree = {};
+    for (const role of PLANNING_ROLES) {
+      const roleDir = path.join(planningDir, role);
+      if (fs.existsSync(roleDir)) planningTree[role] = roleDir;
+    }
+
+    const operatorDir = fs.existsSync(path.join(featureDir, "operator")) ? path.join(featureDir, "operator") : null;
+    this.orchestrator.fileIO.watchFeaturePlanningSignals(feature.slug, planningTree, operatorDir);
+
+    if (!this.orchestrator._signalTrees[feature.slug]) {
+      this.orchestrator._signalTrees[feature.slug] = {
+        featureDir,
+        featureLead: null,
+        operator: operatorDir,
+        featureReview: {},
+        teams: {},
+        planning: planningTree,
+        plansDir: fs.existsSync(path.join(featureDir, "plans")) ? path.join(featureDir, "plans") : null,
+      };
+    }
+
+    // Emit stale signals
+    for (const [role, dir] of Object.entries(planningTree)) {
+      for (const [sig, eventType] of [
+        [SIGNAL.DONE, "planning:done"],
+        [SIGNAL.AGENT_RESPONSE, "planning:response"],
+        [SIGNAL.QUESTION, "planning:question"],
+      ]) {
+        const filePath = path.join(dir, sig);
+        if (fs.existsSync(filePath)) {
+          console.log(`[recovery] Found unprocessed ${sig} for ${feature.slug}/${role}`);
+          this.orchestrator.fileIO.emit(eventType, { slug: feature.slug, role, filePath });
+        }
+      }
+    }
+
+    if (operatorDir) {
+      const opResponse = path.join(operatorDir, SIGNAL.AGENT_RESPONSE);
+      if (fs.existsSync(opResponse)) {
+        console.log(`[recovery] Found stale operator .agent-response for ${feature.slug} (planning)`);
+        this.orchestrator.fileIO.emit("impl:operatorResponse", { slug: feature.slug, filePath: opResponse });
+      }
+    }
+  }
+
+  /**
+   * Clean dead agents for a single feature (mirrors _cleanDeadAgents scoped to one feature).
+   */
+  async _cleanDeadAgentsForFeature(featureId) {
+    const runningAgents = queries.getRunningAgents(featureId);
+    for (const agent of runningAgents) {
+      if (agent.pid && !this.orchestrator.supervisor.isAlive(agent.pid)) {
+        console.log(`[recovery] Agent ${agent.agent_key} PID ${agent.pid} is dead — marking crashed`);
+        queries.updateAgentStatus(agent.id, "crashed");
+        queries.insertEvent(agent.feature_id, "agent-crashed", "bridge",
+          `${agent.agent_key} found dead on recovery (PID ${agent.pid})`);
+      }
+    }
+  }
+
+  /**
+   * Run full recovery sequence on startup.
+   */
+  /**
+   * Sync CLI-created features to Slack.
+   * Features created in terminal mode have slug-based feature_channel values
+   * (not real Slack channel IDs). Detect these and create real Slack channels.
+   */
+  async _syncCliFeaturesToSlack() {
+    const activeFeatures = queries.getActiveFeatures();
+    for (const feature of activeFeatures) {
+      const ch = feature.feature_channel;
+      // Slack channel IDs start with C/G/D followed by alphanumerics
+      if (ch && !/^[CGD][A-Z0-9]+$/.test(ch)) {
+        console.log(`[recovery] Syncing CLI feature "${feature.slug}" to Slack (channel was: ${ch})`);
+        try {
+          const channelId = await this.adapter.createFeatureChannel(feature.id, feature.slug);
+          if (channelId) {
+            queries.updateFeatureChannel(feature.id, channelId);
+            console.log(`[recovery] Created Slack channel ${channelId} for ${feature.slug}`);
+
+            // Announce in #planning channel with link to impl channel
+            await this.adapter.web.chat.postMessage({
+              channel: this.adapter.planningChannel,
+              text: `[FEATURE][CLI SYNC] ${feature.slug} → <#${channelId}>`,
+            });
+
+            // Post status as first message in the feature channel
+            const phase = feature.phase || "planning";
+            const role = feature.active_planning_role;
+            const meta = JSON.parse(feature.metadata || "{}");
+
+            const lines = [`*Feature:* \`${feature.slug}\``, `*Phase:* ${phase}`];
+            if (role) lines.push(`*Active role:* ${role}`);
+            if (meta.awaiting_phase_review) lines.push(`*Status:* Awaiting ${meta.phase_review_role} phase review`);
+            else if (phase === "plan-approval") lines.push("*Status:* Awaiting plan approval");
+            else if (phase === "impl") lines.push("*Status:* Implementation in progress");
+            else if (role) lines.push(`*Status:* ${role} agent running`);
+
+            const planDir = path.join(feature.signal_dir, "plans");
+            if (fs.existsSync(planDir)) {
+              const files = fs.readdirSync(planDir).filter(f => f.endsWith(".md") || f.endsWith(".yaml") || f.endsWith(".html"));
+              if (files.length > 0) lines.push(`*Planning artifacts:* ${files.join(", ")}`);
+            }
+
+            lines.push("_Synced from CLI._");
+            await this.adapter.postMessage(feature.id, lines.join("\n"));
+          }
+        } catch (err) {
+          console.error(`[recovery] Failed to sync CLI feature ${feature.slug} to Slack:`, err.message);
+        }
+      }
+    }
+  }
+
+  async run() {
+    console.log("[recovery] Starting recovery...");
+
+    // 0. Sync CLI-created features to Slack (create channels for slug-based feature_channels)
+    if (this.adapter.createFeatureChannel && typeof this.adapter.createFeatureChannel === "function") {
+      await this._syncCliFeaturesToSlack();
+    }
+
+    // 1. Process stale planning signals
+    await this._processExistingPlanningSignals();
+
+    // 2. Recover active features (re-read after sync since channels may have changed)
+    const activeFeatures = queries.getActiveFeatures();
+    console.log(`[recovery] Found ${activeFeatures.length} active feature(s)`);
+
+    for (const feature of activeFeatures) {
+      try {
+        if (feature.phase === "planning" && feature.active_planning_role) {
+          await this._recoverPlanningFeature(feature);
+        } else if (feature.phase === "impl") {
+          await this._recoverImplFeature(feature);
+        } else if (feature.phase === "plan-approval") {
+          // Check if impl infrastructure exists — feature was approved but phase
+          // never transitioned (e.g. old "go" confirmation step, or crash mid-launch).
+          const teamsDir = path.join(feature.signal_dir, "teams");
+          if (fs.existsSync(teamsDir) && fs.readdirSync(teamsDir).some(d => d.startsWith("team-"))) {
+            console.log(`[recovery] Feature ${feature.slug} phase is plan-approval but has impl infrastructure — fixing phase to impl`);
+            queries.updateFeaturePhase(feature.id, "impl");
+            await this._recoverImplFeature({ ...feature, phase: "impl" });
+          } else {
+            console.log(`[recovery] Feature ${feature.slug} awaiting plan approval — re-notifying Operator`);
+            await this.orchestrator.repostPendingDecision(feature.id);
+          }
+        } else if (feature.phase === "launching") {
+          console.log(`[recovery] Feature ${feature.slug} was launching — may need manual intervention`);
+        }
+      } catch (err) {
+        console.error(`[recovery] Error recovering feature ${feature.slug} (id=${feature.id}):`, err.message);
+      }
+    }
+
+    // 3. Recover stale relay queue entries
+    await this._recoverRelayQueue();
+
+    // 4. Check for running agents with dead PIDs
+    await this._cleanDeadAgents();
+
+    console.log("[recovery] Recovery complete");
+  }
+
+  /**
+   * Process existing planning signals for active planning-phase features.
+   * Scans per-feature planning/ subdirs instead of global ROLE_DIRS.
+   */
+  async _processExistingPlanningSignals() {
+    const activeFeatures = queries.getActiveFeatures();
+    for (const feature of activeFeatures) {
+      if (feature.phase !== "planning" || !feature.active_planning_role) continue;
+
+      const featureDir = feature.signal_dir;
+      const planningDir = path.join(featureDir, "planning");
+      if (!fs.existsSync(planningDir)) continue;
+
+      // Build planning tree for this feature
+      const planningTree = {};
+      for (const role of PLANNING_ROLES) {
+        const roleDir = path.join(planningDir, role);
+        if (fs.existsSync(roleDir)) planningTree[role] = roleDir;
+      }
+
+      // Set up watchers for this feature's planning dirs + operator
+      const operatorDir = fs.existsSync(path.join(featureDir, "operator")) ? path.join(featureDir, "operator") : null;
+      this.orchestrator.fileIO.watchFeaturePlanningSignals(feature.slug, planningTree, operatorDir);
+
+      // Cache in signal trees
+      if (!this.orchestrator._signalTrees[feature.slug]) {
+        this.orchestrator._signalTrees[feature.slug] = {
+          featureDir,
+          featureLead: null,
+          operator: operatorDir,
+          featureReview: {},
+          teams: {},
+          planning: planningTree,
+          plansDir: fs.existsSync(path.join(featureDir, "plans")) ? path.join(featureDir, "plans") : null,
+        };
+      }
+
+      // Emit stale signals
+      for (const [role, dir] of Object.entries(planningTree)) {
+        for (const [sig, eventType] of [
+          [SIGNAL.DONE, "planning:done"],
+          [SIGNAL.AGENT_RESPONSE, "planning:response"],
+          [SIGNAL.QUESTION, "planning:question"],
+        ]) {
+          const filePath = path.join(dir, sig);
+          if (fs.existsSync(filePath)) {
+            console.log(`[recovery] Found unprocessed ${sig} for ${feature.slug}/${role}`);
+            this.orchestrator.fileIO.emit(eventType, { slug: feature.slug, role, filePath });
+          }
+        }
+      }
+
+      // Emit stale operator signals during planning
+      if (operatorDir) {
+        const opResponse = path.join(operatorDir, SIGNAL.AGENT_RESPONSE);
+        if (fs.existsSync(opResponse)) {
+          console.log(`[recovery] Found stale operator .agent-response for ${feature.slug} (planning)`);
+          this.orchestrator.fileIO.emit("impl:operatorResponse", { slug: feature.slug, filePath: opResponse });
+        }
+      }
+    }
+  }
+
+  /**
+   * Recover a feature in planning phase.
+   */
+  async _recoverPlanningFeature(feature) {
+    const role = feature.active_planning_role;
+    if (!role || !PIPELINE_ORDER.includes(role)) return;
+
+    const featureDir = feature.signal_dir;
+    const roleDir = path.join(featureDir, "planning", role);
+
+    // If a .done file exists, defer to processExistingPlanningSignals
+    if (fs.existsSync(path.join(roleDir, SIGNAL.DONE))) {
+      console.log(`[recovery] Feature ${feature.slug}: stale ${role}.done found, deferring to signal processing`);
+      return;
+    }
+
+    // If already awaiting phase review, re-post the decision prompt (may have been lost)
+    const meta = JSON.parse(feature.metadata || "{}");
+    if (meta.awaiting_phase_review) {
+      console.log(`[recovery] Feature ${feature.slug}: awaiting phase review for ${meta.phase_review_role} — re-posting decision`);
+      await this.orchestrator.repostPendingPhaseReview(feature.id);
+      return;
+    }
+
+    // .handover exists → context was exhausted → fresh session with handover context
+    if (fs.existsSync(path.join(roleDir, SIGNAL.HANDOVER))) {
+      console.log(`[recovery] Feature ${feature.slug}: found .handover for ${role}, starting fresh with handover`);
+      this.orchestrator.dispatchPlanningRole(feature.id, role, { continue: false });
+      return;
+    }
+
+    // No .handover, no .done → crash or bridge died mid-session → --continue
+    console.log(`[recovery] Feature ${feature.slug}: resuming ${role} session with --continue`);
+    this.orchestrator.dispatchPlanningRole(feature.id, role, { continue: true });
+  }
+
+  /**
+   * Recover a feature in implementation phase.
+   */
+  async _recoverImplFeature(feature) {
+    console.log(`[recovery] Recovering impl feature: ${feature.slug}`);
+
+    // Discover signal tree
+    const tree = this.orchestrator.discoverSignalTree(feature.slug);
+
+    // Kill orphaned processes
+    this.orchestrator.supervisor.killOrphans(tree);
+
+    // Launch all agents (creates DB records, watches signals)
+    this.orchestrator.launchImplAgents(feature.id);
+
+    // Process stale signals that arrived while bridge was down
+    await this._processStaleImplSignals(feature, tree);
+  }
+
+  /**
+   * Process stale implementation signals.
+   */
+  async _processStaleImplSignals(feature, tree) {
+    const fileIO = this.orchestrator.fileIO;
+    const slug = feature.slug;
+
+    // Feature Lead signals
+    if (tree.featureLead) {
+      const flDir = tree.featureLead;
+      const staleChecks = [
+        [SIGNAL.AGENT_RESPONSE, "impl:response", { slug, agent: "feature-lead" }],
+        [SIGNAL.FEATURE_COMPLETE, "impl:featureComplete", { slug }],
+        [SIGNAL.PHASE_DONE, "impl:phaseDone", { slug }],
+        [SIGNAL.CONTEXT_REFRESH, "impl:contextRefresh", { slug }],
+        [SIGNAL.NEEDS_RESTART, "impl:needsRestart", { slug, dir: flDir }],
+      ];
+      for (const [sig, event, data] of staleChecks) {
+        const filePath = path.join(flDir, sig);
+        if (fs.existsSync(filePath)) {
+          console.log(`[recovery] Processing stale FL ${sig} for ${slug}`);
+          fileIO.emit(event, { ...data, filePath });
+        }
+      }
+    }
+
+    // Operator signals
+    if (tree.operator) {
+      const opDir = tree.operator;
+      const opResponse = path.join(opDir, SIGNAL.AGENT_RESPONSE);
+      if (fs.existsSync(opResponse)) {
+        console.log(`[recovery] Processing stale operator .agent-response for ${slug}`);
+        fileIO.emit("impl:operatorResponse", { slug, filePath: opResponse });
+      }
+      const opUserMsg = path.join(opDir, SIGNAL.USER_MESSAGE);
+      if (fs.existsSync(opUserMsg)) {
+        console.log(`[recovery] Processing stale operator .user-message for ${slug}`);
+        fileIO.emit("impl:userMessage", { slug, agent: "operator", filePath: opUserMsg });
+      }
+    }
+
+    // Team signals
+    for (const [teamNum, team] of Object.entries(tree.teams)) {
+      if (team.orchestrator) {
+        // Stale .task
+        const taskFile = path.join(team.orchestrator, SIGNAL.TASK);
+        if (fs.existsSync(taskFile)) {
+          console.log(`[recovery] Found existing .task for team-${teamNum} orchestrator`);
+          fileIO.emit("impl:orchTask", { slug, teamNum, filePath: taskFile });
+        }
+
+        // Stale .gate-ready
+        if (fs.existsSync(path.join(team.orchestrator, SIGNAL.GATE_READY))) {
+          console.log(`[recovery] Found stale .gate-ready for team-${teamNum}`);
+          fileIO.emit("impl:gateReady", { slug, teamNum, filePath: path.join(team.orchestrator, SIGNAL.GATE_READY) });
+        }
+
+        // Stale .question
+        if (fs.existsSync(path.join(team.orchestrator, SIGNAL.QUESTION))) {
+          console.log(`[recovery] Found stale .question for team-${teamNum}`);
+          fileIO.emit("impl:question", { slug, teamNum, filePath: path.join(team.orchestrator, SIGNAL.QUESTION) });
+        }
+
+        // Stale .crashed
+        if (fs.existsSync(path.join(team.orchestrator, SIGNAL.CRASHED))) {
+          console.log(`[recovery] Found stale .crashed for team-${teamNum}`);
+          fileIO.emit("impl:crashed", { slug, agent: `orch-${teamNum}`, teamNum, filePath: path.join(team.orchestrator, SIGNAL.CRASHED) });
+        }
+
+        // Stale .needs-restart
+        if (fs.existsSync(path.join(team.orchestrator, SIGNAL.NEEDS_RESTART))) {
+          console.log(`[recovery] Processing stale .needs-restart for team-${teamNum} orch`);
+          fileIO.emit("impl:needsRestart", { slug, dir: team.orchestrator, filePath: path.join(team.orchestrator, SIGNAL.NEEDS_RESTART) });
+        }
+      }
+
+      // Role agent stale signals
+      for (const [role, roleDir] of Object.entries(team.roles)) {
+        if (fs.existsSync(path.join(roleDir, SIGNAL.TASK))) {
+          console.log(`[recovery] Found existing .task for team-${teamNum} ${role}`);
+          fileIO.emit("impl:task", { slug, teamNum, role, filePath: path.join(roleDir, SIGNAL.TASK) });
+        }
+        if (fs.existsSync(path.join(roleDir, SIGNAL.DONE))) {
+          console.log(`[recovery] Found stale .done for team-${teamNum} ${role}`);
+          fileIO.emit("impl:done", { slug, teamNum, role, agent: `role-${teamNum}-${role}`, filePath: path.join(roleDir, SIGNAL.DONE) });
+        }
+      }
+    }
+
+    // Feature review stale signals
+    for (const [role, reviewDir] of Object.entries(tree.featureReview)) {
+      if (fs.existsSync(path.join(reviewDir, SIGNAL.DONE))) {
+        console.log(`[recovery] Processing stale review .done for ${slug}/${role}`);
+        fileIO.emit("impl:done", { slug, agent: `review-${role}`, role, filePath: path.join(reviewDir, SIGNAL.DONE) });
+      }
+    }
+  }
+
+  /**
+   * Recover relay queue entries stuck in 'processing' state (bridge died mid-relay).
+   * Fallback: post raw content directly.
+   */
+  async _recoverRelayQueue() {
+    const stuckEntries = queries.getAllProcessingRelays();
+    if (!stuckEntries.length) return;
+
+    console.log(`[recovery] Found ${stuckEntries.length} stuck relay queue entries`);
+    for (const entry of stuckEntries) {
+      // Skip decision-needed entries — already rendered by postDecision()
+      if (entry.event_hint === "decision-needed") {
+        queries.updateRelayStatus(entry.id, "posted", { processedAt: new Date().toISOString() });
+        continue;
+      }
+      try {
+        const feature = queries.getFeatureById(entry.feature_id);
+        if (!feature) {
+          queries.updateRelayStatus(entry.id, "failed");
+          continue;
+        }
+        console.log(`[recovery] Fallback posting relay ${entry.id} (${entry.source_agent}/${entry.event_hint})`);
+        await this.adapter.postAgentResponse(feature.id, entry.source_agent, entry.raw_content);
+        queries.updateRelayStatus(entry.id, "posted", { processedAt: new Date().toISOString() });
+      } catch (err) {
+        console.error(`[recovery] Failed to recover relay ${entry.id}:`, err.message);
+        queries.updateRelayStatus(entry.id, "failed");
+      }
+    }
+  }
+
+  /**
+   * Clean up agents that are marked 'running' in DB but whose PIDs are dead.
+   */
+  async _cleanDeadAgents() {
+    const runningAgents = queries.getAllRunningAgents();
+    for (const agent of runningAgents) {
+      if (agent.pid && !this.orchestrator.supervisor.isAlive(agent.pid)) {
+        console.log(`[recovery] Agent ${agent.agent_key} PID ${agent.pid} is dead — marking crashed`);
+        queries.updateAgentStatus(agent.id, "crashed");
+        queries.insertEvent(agent.feature_id, "agent-crashed", "bridge",
+          `${agent.agent_key} found dead on recovery (PID ${agent.pid})`);
+      }
+    }
+  }
+}