@stoneforge/smithy 1.4.1 → 1.6.0

package/dist/services/dispatch-daemon.js

@@ -22,6 +22,7 @@ import { InboxStatus, createTimestamp, TaskStatus, asEntityId, asElementId, Plan
 import { loadTriagePrompt, loadRolePrompt } from '../prompts/index.js';
 import { createLogger } from '../utils/logger.js';
 import { getAgentMetadata } from './agent-registry.js';
+import { createRateLimitTracker } from './rate-limit-tracker.js';
 import { getOrchestratorTaskMeta, updateOrchestratorTaskMeta, appendTaskSessionHistory, } from '../types/task-meta.js';
 const logger = createLogger('dispatch-daemon');
 // ============================================================================
@@ -55,12 +56,15 @@ export class DispatchDaemonImpl {
     stewardScheduler;
     inboxService;
     poolService;
+    settingsService;
+    rateLimitTracker;
     emitter;
     config;
     running = false;
     polling = false;
     pollIntervalHandle;
     currentPollCycle;
+    rateLimitSleepTimer;
     /**
      * Tracks inbox item IDs that are currently being forwarded to persistent agents.
      * Prevents duplicate message delivery when concurrent pollInboxes() calls
@@ -72,7 +76,7 @@ export class DispatchDaemonImpl {
      * Items are added before forwarding and removed after markAsRead() completes.
      */
     forwardingInboxItems = new Set();
-    constructor(api, agentRegistry, sessionManager, dispatchService, worktreeManager, taskAssignment, stewardScheduler, inboxService, config, poolService) {
+    constructor(api, agentRegistry, sessionManager, dispatchService, worktreeManager, taskAssignment, stewardScheduler, inboxService, config, poolService, settingsService) {
         this.api = api;
         this.agentRegistry = agentRegistry;
         this.sessionManager = sessionManager;
@@ -82,6 +86,8 @@ export class DispatchDaemonImpl {
         this.stewardScheduler = stewardScheduler;
         this.inboxService = inboxService;
         this.poolService = poolService;
+        this.settingsService = settingsService;
+        this.rateLimitTracker = createRateLimitTracker();
         this.emitter = new EventEmitter();
         this.config = this.normalizeConfig(config);
     }
@@ -134,6 +140,10 @@ export class DispatchDaemonImpl {
             clearInterval(this.pollIntervalHandle);
             this.pollIntervalHandle = undefined;
         }
+        if (this.rateLimitSleepTimer) {
+            clearTimeout(this.rateLimitSleepTimer);
+            this.rateLimitSleepTimer = undefined;
+        }
         // Wait for in-flight poll cycle to complete (M-8)
         if (this.currentPollCycle) {
             try {
@@ -150,6 +160,57 @@ export class DispatchDaemonImpl {
         return this.running;
     }
     // ----------------------------------------
+    // Rate Limiting
+    // ----------------------------------------
+    handleRateLimitDetected(executable, resetsAt) {
+        this.rateLimitTracker.markLimited(executable, resetsAt);
+        logger.info(`Rate limit detected for executable '${executable}', resets at ${resetsAt.toISOString()}`);
+    }
+    getRateLimitStatus() {
+        const fallbackChain = this.settingsService?.getAgentDefaults().fallbackChain ?? [];
+        const isPaused = fallbackChain.length > 0 && this.rateLimitTracker.isAllLimited(fallbackChain);
+        const allLimits = this.rateLimitTracker.getAllLimits();
+        const soonestReset = this.rateLimitTracker.getSoonestResetTime();
+        return {
+            isPaused,
+            limits: allLimits.map((entry) => ({
+                executable: entry.executable,
+                resetsAt: entry.resetsAt.toISOString(),
+            })),
+            soonestReset: soonestReset?.toISOString(),
+        };
+    }
+    sleepUntil(resetTime) {
+        const fallbackChain = this.settingsService?.getAgentDefaults().fallbackChain ?? [];
+        if (fallbackChain.length === 0) {
+            logger.warn('sleepUntil: No fallback chain configured — nothing to mark as limited');
+            return;
+        }
+        // Mark all executables in the fallback chain as rate-limited until the given time
+        for (const executable of fallbackChain) {
+            this.rateLimitTracker.markLimited(executable, resetTime);
+        }
+        // Clear any existing sleep timer and set a new one
+        if (this.rateLimitSleepTimer) {
+            clearTimeout(this.rateLimitSleepTimer);
+        }
+        const sleepMs = Math.max(0, resetTime.getTime() - Date.now());
+        logger.info(`Manual sleep: pausing dispatch for ${Math.round(sleepMs / 1000)}s (until ${resetTime.toISOString()})`);
+        this.rateLimitSleepTimer = setTimeout(() => {
+            this.rateLimitSleepTimer = undefined;
+        }, sleepMs);
+    }
+    wake() {
+        // Clear all rate limit entries
+        this.rateLimitTracker.clear();
+        // Clear the sleep timer
+        if (this.rateLimitSleepTimer) {
+            clearTimeout(this.rateLimitSleepTimer);
+            this.rateLimitSleepTimer = undefined;
+        }
+        logger.info('Manual wake: cleared all rate limits, dispatch will resume on next poll cycle');
+    }
+    // ----------------------------------------
     // Manual Poll Triggers
     // ----------------------------------------
     async pollWorkerAvailability() {
@@ -542,7 +603,33 @@ export class DispatchDaemonImpl {
                 if (stewardTasks.length === 0)
                     continue;
                 const orphanedAssignment = stewardTasks[0];
+                // Safety valve: cap steward recovery attempts to prevent infinite re-dispatch loops
+                const stewardRecoveryCount = orphanedAssignment.orchestratorMeta?.stewardRecoveryCount ?? 0;
+                if (stewardRecoveryCount >= 3) {
+                    logger.warn(`Steward recovery limit reached for task ${orphanedAssignment.task.id}, setting mergeStatus to 'failed'`);
+                    try {
+                        await this.api.update(orphanedAssignment.task.id, {
+                            assignee: undefined,
+                            metadata: updateOrchestratorTaskMeta(orphanedAssignment.task.metadata, {
+                                mergeStatus: 'failed',
+                                mergeFailureReason: `Steward recovery limit reached after ${stewardRecoveryCount} attempts`,
+                            }),
+                        });
+                        processed++;
+                    }
+                    catch (error) {
+                        errors++;
+                        const errorMessage = error instanceof Error ? error.message : String(error);
+                        errorMessages.push(`Steward recovery limit for ${steward.name}: ${errorMessage}`);
+                        logger.error(`Error setting mergeStatus to failed for task ${orphanedAssignment.task.id}:`, error);
+                    }
+                    continue;
+                }
                 try {
+                    // Increment stewardRecoveryCount before recovering
+                    await this.api.update(orphanedAssignment.task.id, {
+                        metadata: updateOrchestratorTaskMeta(orphanedAssignment.task.metadata, { stewardRecoveryCount: stewardRecoveryCount + 1 }),
+                    });
                     await this.recoverOrphanedStewardTask(steward, orphanedAssignment.task, orphanedAssignment.orchestratorMeta);
                     processed++;
                 }
@@ -891,6 +978,40 @@ export class DispatchDaemonImpl {
             return;
         this.polling = true;
         try {
+            // Check if all executables in the fallback chain are rate-limited.
+            // When paused, skip dispatch-related polls but still run non-dispatch work.
+            const fallbackChain = this.settingsService?.getAgentDefaults().fallbackChain ?? [];
+            const allLimited = fallbackChain.length > 0 && this.rateLimitTracker.isAllLimited(fallbackChain);
+            if (allLimited) {
+                // Schedule a wake-up timer so we re-check when the soonest limit expires
+                const soonestReset = this.rateLimitTracker.getSoonestResetTime();
+                if (soonestReset && !this.rateLimitSleepTimer) {
+                    const sleepMs = Math.max(0, soonestReset.getTime() - Date.now());
+                    logger.info(`All executables rate-limited. Pausing dispatch polls for ${Math.round(sleepMs / 1000)}s (until ${soonestReset.toISOString()})`);
+                    this.rateLimitSleepTimer = setTimeout(() => {
+                        this.rateLimitSleepTimer = undefined;
+                    }, sleepMs);
+                }
+                // Run non-dispatch polls only
+                if (this.config.inboxPollEnabled) {
+                    await this.pollInboxes();
+                }
+                if (this.config.closedUnmergedReconciliationEnabled) {
+                    await this.reconcileClosedUnmergedTasks();
+                }
+                if (this.config.stuckMergeRecoveryEnabled) {
+                    await this.recoverStuckMergeTasks();
+                }
+                if (this.config.planAutoCompleteEnabled) {
+                    await this.pollPlanAutoComplete();
+                }
+                return;
+            }
+            // Clear sleep timer if limits have expired
+            if (this.rateLimitSleepTimer) {
+                clearTimeout(this.rateLimitSleepTimer);
+                this.rateLimitSleepTimer = undefined;
+            }
             // Recover orphaned assignments first — workers with tasks but no session
             // (e.g. from mid-cycle crashes). Runs before availability polling so
             // orphans are handled before they'd be skipped.
@@ -933,6 +1054,48 @@ export class DispatchDaemonImpl {
             this.polling = false;
         }
     }
+    /**
+     * Resolves the executable path for a session, checking rate limits and applying
+     * fallback selection when the primary executable is rate-limited.
+     *
+     * @param agent - The agent to resolve the executable for
+     * @returns The executable path override if fallback was needed, undefined if primary is OK,
+     *          or 'all_limited' if all executables in the fallback chain are rate-limited.
+     */
+    resolveExecutableWithFallback(agent) {
+        const meta = getAgentMetadata(agent);
+        if (!meta)
+            return undefined;
+        // Get the configured executable for this agent
+        const agentExecutablePath = meta.executablePath;
+        const providerName = meta.provider ?? 'claude';
+        // Determine the effective executable path that would be used
+        // Priority: agent-specific → workspace-wide default → provider default
+        let effectiveExecutable = agentExecutablePath;
+        if (!effectiveExecutable && this.settingsService) {
+            const defaults = this.settingsService.getAgentDefaults();
+            effectiveExecutable = defaults.defaultExecutablePaths[providerName];
+        }
+        if (!effectiveExecutable) {
+            effectiveExecutable = providerName;
+        }
+        // Check if the effective executable is rate-limited
+        if (!this.rateLimitTracker.isLimited(effectiveExecutable)) {
+            return undefined; // Primary is fine, no override needed
+        }
+        // Primary is limited — try fallback chain
+        const fallbackChain = this.settingsService?.getAgentDefaults().fallbackChain ?? [];
+        if (fallbackChain.length === 0) {
+            // No fallback chain configured, can't dispatch
+            return 'all_limited';
+        }
+        const available = this.rateLimitTracker.getAvailableExecutable(fallbackChain);
+        if (!available) {
+            return 'all_limited';
+        }
+        logger.info(`Executable '${effectiveExecutable}' is rate-limited, falling back to '${available}'`);
+        return available;
+    }
     /**
      * Terminates sessions that have exceeded the configured max duration.
      * Prevents stuck workers from blocking their slot indefinitely.
@@ -1055,16 +1218,25 @@ export class DispatchDaemonImpl {
             }
             catch (error) {
                 logger.warn(`Failed to resume session ${previousSessionId} for worker ${worker.name}, falling back to fresh spawn:`, error);
+                // Clear stale session ID so next recovery cycle doesn't try to resume again
+                const clearedMeta = updateOrchestratorTaskMeta(task.metadata, { sessionId: undefined });
+                await this.api.update(task.id, { metadata: clearedMeta });
             }
         }
-        // 3. Fall back to fresh spawn
+        // 3. Fall back to fresh spawn (with rate limit fallback)
+        const orphanExecutableOverride = this.resolveExecutableWithFallback(worker);
+        if (orphanExecutableOverride === 'all_limited') {
+            logger.warn(`All executables rate-limited, deferring orphan recovery for worker ${worker.name}`);
+            return;
+        }
         const initialPrompt = await this.buildTaskPrompt(task, workerId);
         const { session, events } = await this.sessionManager.startSession(workerId, {
             workingDirectory: worktreePath,
             worktree: worktreePath,
             initialPrompt,
+            executablePathOverride: orphanExecutableOverride ?? undefined,
         });
-        // Record session history entry for fresh spawned worker session
+        // Record session history entry and new sessionId for fresh spawned worker session
         const freshSpawnHistoryEntry = {
             sessionId: session.id,
             providerSessionId: session.providerSessionId,
@@ -1076,7 +1248,9 @@ export class DispatchDaemonImpl {
         const taskAfterFreshSpawn = await this.api.get(task.id);
         if (taskAfterFreshSpawn) {
             const metadataWithHistory = appendTaskSessionHistory(taskAfterFreshSpawn.metadata, freshSpawnHistoryEntry);
-            await this.api.update(task.id, { metadata: metadataWithHistory });
+            // Write the new session ID so future recovery cycles can resume this session
+            const metadataWithSessionId = updateOrchestratorTaskMeta(metadataWithHistory, { sessionId: session.providerSessionId ?? session.id });
+            await this.api.update(task.id, { metadata: metadataWithSessionId });
         }
         if (this.config.onSessionStarted) {
             this.config.onSessionStarted(session, events, workerId, initialPrompt);
@@ -1139,6 +1313,9 @@ export class DispatchDaemonImpl {
             }
             catch (error) {
                 logger.warn(`Failed to resume steward session ${previousSessionId} for ${steward.name}, falling back to fresh spawn:`, error);
+                // Clear stale session ID so next recovery cycle doesn't try to resume again
+                const clearedMeta = updateOrchestratorTaskMeta(task.metadata, { sessionId: undefined });
+                await this.api.update(task.id, { metadata: clearedMeta });
             }
         }
         // 3. Fall back to fresh spawn (spawnMergeStewardForTask handles metadata update AND session history)
@@ -1178,6 +1355,12 @@ export class DispatchDaemonImpl {
                 }
             }
         }
+        // Check rate limits and determine executable path override if needed
+        const executableOverride = this.resolveExecutableWithFallback(worker);
+        if (executableOverride === 'all_limited') {
+            logger.warn(`All executables rate-limited, skipping dispatch for worker ${worker.name}`);
+            return false;
+        }
         // Check for existing worktree/branch in task metadata
         // Priority: handoff > existing assignment > create new
         const taskMeta = getOrchestratorTaskMeta(task.metadata);
@@ -1228,6 +1411,7 @@ export class DispatchDaemonImpl {
             workingDirectory: worktreePath,
             worktree: worktreePath,
             initialPrompt,
+            executablePathOverride: executableOverride ?? undefined,
         });
         // Session started successfully — now dispatch the task (assigns + sends message)
         const dispatchOptions = {
@@ -1421,6 +1605,12 @@ export class DispatchDaemonImpl {
                 return;
             }
         }
+        // Check rate limits and determine executable path override if needed
+        const stewardExecutableOverride = this.resolveExecutableWithFallback(steward);
+        if (stewardExecutableOverride === 'all_limited') {
+            logger.warn(`All executables rate-limited, skipping merge steward dispatch for ${steward.name}`);
+            return;
+        }
         // Get task metadata for worktree path
         const taskMeta = task.metadata;
         const orchestratorMeta = taskMeta?.orchestrator;
@@ -1486,6 +1676,7 @@ export class DispatchDaemonImpl {
             worktree: worktreePath,
             initialPrompt,
             interactive: false, // Stewards use headless mode
+            executablePathOverride: stewardExecutableOverride ?? undefined,
         });
         // Record steward assignment and session history on the task to prevent double-dispatch and enable recovery.
         // Setting task.assignee makes the steward visible in the UI and enables
@@ -1579,6 +1770,12 @@ export class DispatchDaemonImpl {
                 return;
             }
         }
+        // 2b. Check rate limits and determine executable path override
+        const recoveryExecutableOverride = this.resolveExecutableWithFallback(recoverySteward);
+        if (recoveryExecutableOverride === 'all_limited') {
+            logger.warn(`All executables rate-limited, deferring recovery steward spawn for ${recoverySteward.name}`);
+            return;
+        }
         // 3. Resolve worktree — reuse the worker's existing worktree
         let worktreePath = taskMeta?.worktree ?? taskMeta?.handoffWorktree;
         const branch = taskMeta?.branch ?? taskMeta?.handoffBranch;
@@ -1629,6 +1826,7 @@ export class DispatchDaemonImpl {
             worktree: worktreePath,
             initialPrompt,
             interactive: false, // Stewards use headless mode
+            executablePathOverride: recoveryExecutableOverride ?? undefined,
         });
         // 7. Record steward assignment and session history on the task
         const taskAfterUpdate = await this.api.get(task.id);
@@ -2155,7 +2353,7 @@ export class DispatchDaemonImpl {
 /**
  * Creates a DispatchDaemon instance
  */
-export function createDispatchDaemon(api, agentRegistry, sessionManager, dispatchService, worktreeManager, taskAssignment, stewardScheduler, inboxService, config, poolService) {
-    return new DispatchDaemonImpl(api, agentRegistry, sessionManager, dispatchService, worktreeManager, taskAssignment, stewardScheduler, inboxService, config, poolService);
+export function createDispatchDaemon(api, agentRegistry, sessionManager, dispatchService, worktreeManager, taskAssignment, stewardScheduler, inboxService, config, poolService, settingsService) {
+    return new DispatchDaemonImpl(api, agentRegistry, sessionManager, dispatchService, worktreeManager, taskAssignment, stewardScheduler, inboxService, config, poolService, settingsService);
 }
 //# sourceMappingURL=dispatch-daemon.js.map