crewly 1.3.31 → 1.4.0

package/dist/backend/backend/src/services/workflow/scheduler.service.js

@@ -8,6 +8,7 @@
  */
 import { EventEmitter } from 'events';
 import { v4 as uuidv4 } from 'uuid';
+import * as cron from 'node-cron';
 import { getSessionBackendSync, createSessionBackend, } from '../session/index.js';
 import { MessageDeliveryLogModel } from '../../models/ScheduledMessage.js';
 import { LoggerService } from '../core/logger.service.js';
@@ -97,6 +98,10 @@ export class SchedulerService extends EventEmitter {
     // 25+ concurrent Ctrl+C presses crash the runtime. This guard ensures only
     // one check delivers per session at a time; concurrent checks are dropped.
     deliveryInProgress = new Set();
+    /** #167: Dead-letter queue — messages for offline agents, delivered when they come online. */
+    deadLetterQueue = new Map();
+    /** #167: Active cron tasks keyed by check ID. */
+    cronTasks = new Map();
     /** Consecutive idle hits per recurring check ID for auto-cancel policy. */
     recurringIdleStreak = new Map();
     /** Auto-cancel recurring checks after this many consecutive idle observations. */
@@ -107,6 +112,8 @@ export class SchedulerService extends EventEmitter {
      * whose message mentions a recently-checked session are suppressed.
      */
     lastManualCheck = new Map();
+    /** Unique session ID generated on construction — tags all new checks so stale checks from previous sessions can be purged on restart (#169). */
+    sessionId;
     /**
      * Creates a new SchedulerService
      *
@@ -115,8 +122,17 @@ export class SchedulerService extends EventEmitter {
     constructor(storageService) {
         super();
         this.storageService = storageService;
+        this.sessionId = uuidv4();
         this.logger = LoggerService.getInstance().createComponentLogger('SchedulerService');
     }
+    /**
+     * Get the current session ID for this SchedulerService instance.
+     *
+     * @returns The unique session ID
+     */
+    getSessionId() {
+        return this.sessionId;
+    }
     /**
      * Record that the orchestrator manually checked an agent's status or logs.
      * Recurring checks whose message mentions this session will be suppressed
@@ -306,6 +322,7 @@ export class SchedulerService extends EventEmitter {
             timeOfDay: options?.timeOfDay,
             dayOfWeek: options?.dayOfWeek,
             createdAt: new Date().toISOString(),
+            sessionId: this.sessionId,
         };
         // Store enhanced message info
         const enhancedMessage = {
@@ -360,6 +377,24 @@ export class SchedulerService extends EventEmitter {
      * @returns Check ID
      */
     scheduleRecurringCheck(targetSession, intervalMinutes, message, type = 'progress-check', maxOccurrences, options) {
+        // #165: Enforce uniqueness — cancel existing recurring checks for the same target
+        // to prevent duplicate/overlapping checks from accumulating.
+        const existingCheckIds = [];
+        for (const [existingId, existingCheck] of this.recurringChecks.entries()) {
+            if (existingCheck.targetSession === targetSession) {
+                existingCheckIds.push(existingId);
+            }
+        }
+        if (existingCheckIds.length > 0) {
+            this.logger.info('Cancelling existing recurring checks for target before scheduling new one', {
+                targetSession,
+                cancelledCount: existingCheckIds.length,
+                cancelledIds: existingCheckIds,
+            });
+            for (const existingId of existingCheckIds) {
+                this.cancelCheck(existingId);
+            }
+        }
         const checkId = uuidv4();
         const firstExecution = new Date(Date.now() + intervalMinutes * 60 * 1000);
         const scheduledCheck = {
@@ -379,6 +414,7 @@ export class SchedulerService extends EventEmitter {
             currentOccurrence: 0,
             createdAt: new Date().toISOString(),
             taskId: options?.taskId,
+            sessionId: this.sessionId,
         };
         this.recurringChecks.set(checkId, scheduledCheck);
         // Persist to disk so checks survive restarts
@@ -414,6 +450,112 @@ export class SchedulerService extends EventEmitter {
         });
         return checkId;
     }
+    /**
+     * #167: Schedule a recurring check using a cron expression.
+     * Supports standard 5-field cron syntax (minute hour day-of-month month day-of-week).
+     * Falls back to intervalMinutes-based scheduling if cron expression is invalid.
+     *
+     * @param targetSession - Target session name
+     * @param cronExpression - Cron expression (e.g., '0 9 * * *' for daily at 9am)
+     * @param message - Message to send
+     * @param type - Type of scheduled message
+     * @param maxOccurrences - Maximum number of times the check fires
+     * @param options - Additional scheduling options
+     * @returns Check ID
+     */
+    scheduleCronCheck(targetSession, cronExpression, message, type = 'progress-check', maxOccurrences, options) {
+        // Validate cron expression
+        if (!cron.validate(cronExpression)) {
+            this.logger.error('Invalid cron expression, falling back to 30-min interval', {
+                cronExpression,
+                targetSession,
+            });
+            return this.scheduleRecurringCheck(targetSession, 30, message, type, maxOccurrences, options);
+        }
+        // Cancel existing recurring/cron checks for the same target (uniqueness constraint)
+        const existingCheckIds = [];
+        for (const [existingId, existingCheck] of this.recurringChecks.entries()) {
+            if (existingCheck.targetSession === targetSession) {
+                existingCheckIds.push(existingId);
+            }
+        }
+        for (const existingId of existingCheckIds) {
+            this.cancelCheck(existingId);
+        }
+        const checkId = uuidv4();
+        const scheduledCheck = {
+            id: checkId,
+            targetSession,
+            message,
+            scheduledFor: new Date().toISOString(),
+            isRecurring: true,
+            cronExpression,
+            maxOccurrences,
+            currentOccurrence: 0,
+            createdAt: new Date().toISOString(),
+            label: options?.label,
+            taskId: options?.taskId,
+            sessionId: this.sessionId,
+        };
+        this.recurringChecks.set(checkId, scheduledCheck);
+        // Persist to disk
+        this.storageService.saveRecurringCheck(scheduledCheck).catch(err => {
+            this.logger.error('Failed to persist cron check', {
+                checkId,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        });
+        // Store enhanced message info
+        const enhancedMessage = {
+            id: checkId,
+            sessionName: targetSession,
+            message,
+            scheduledFor: new Date(),
+            type,
+            recurring: {
+                interval: 0,
+                currentOccurrence: 0,
+                maxOccurrences,
+                cronExpression,
+            },
+            createdAt: new Date().toISOString(),
+        };
+        this.enhancedMessages.set(checkId, enhancedMessage);
+        // Create the cron task
+        const task = cron.schedule(cronExpression, async () => {
+            if (!this.recurringChecks.has(checkId)) {
+                task.stop();
+                this.cronTasks.delete(checkId);
+                return;
+            }
+            const enhanced = this.enhancedMessages.get(checkId);
+            const occurrence = (enhanced?.recurring?.currentOccurrence ?? 0) + 1;
+            // Check max occurrences
+            if (maxOccurrences && occurrence > maxOccurrences) {
+                this.cancelCheck(checkId);
+                return;
+            }
+            await this.executeCheck(targetSession, this.addFreshnessInstructions(message));
+            // Update occurrence count
+            if (enhanced?.recurring) {
+                enhanced.recurring.currentOccurrence = occurrence;
+                const rc = this.recurringChecks.get(checkId);
+                if (rc) {
+                    rc.currentOccurrence = occurrence;
+                    this.storageService.saveRecurringCheck(rc).catch(() => { });
+                }
+            }
+        });
+        this.cronTasks.set(checkId, task);
+        this.emit('recurring_check_scheduled', scheduledCheck);
+        this.logger.info('Scheduled cron check', {
+            checkId,
+            targetSession,
+            cronExpression,
+            type,
+        });
+        return checkId;
+    }
     /**
      * Schedule default check-ins for a new agent
      *
@@ -582,6 +724,12 @@ export class SchedulerService extends EventEmitter {
                 clearTimeout(recurringTimeout);
                 this.recurringTimeouts.delete(checkId);
             }
+            // #167: Stop cron task if this was a cron-based check
+            const cronTask = this.cronTasks.get(checkId);
+            if (cronTask) {
+                cronTask.stop();
+                this.cronTasks.delete(checkId);
+            }
             this.recurringChecks.delete(checkId);
             this.enhancedMessages.delete(checkId);
             this.recurringIdleStreak.delete(checkId);
@@ -843,7 +991,16 @@ export class SchedulerService extends EventEmitter {
                 });
             }
             else {
-                // Fallback: direct PTY write (should not happen in normal operation)
+                // Fallback: direct PTY write (should not happen in normal operation).
+                // Skip crewly-agent sessions — they have no PTY and require
+                // AgentRegistrationService for in-process delivery (Bug 3 fix).
+                const runtimeType = await this.resolveRuntimeType(targetSession);
+                if (runtimeType === RUNTIME_TYPES.CREWLY_AGENT) {
+                    this.logger.warn('Cannot deliver scheduled message to crewly-agent without AgentRegistrationService', {
+                        targetSession,
+                    });
+                    return;
+                }
                 this.logger.warn('AgentRegistrationService not available, using fallback PTY write', {
                     targetSession,
                 });
@@ -871,6 +1028,19 @@ export class SchedulerService extends EventEmitter {
             success = false;
             error = sendError instanceof Error ? sendError.message : 'Unknown error';
             this.logger.error('Error executing check-in', { targetSession, error });
+            // #167: Dead-letter queue — queue the message for delivery when agent comes online
+            if (!this.deadLetterQueue.has(targetSession)) {
+                this.deadLetterQueue.set(targetSession, []);
+            }
+            const dlq = this.deadLetterQueue.get(targetSession);
+            // Limit DLQ per session to 10 messages to prevent unbounded growth
+            if (dlq.length < 10) {
+                dlq.push({ message, queuedAt: new Date().toISOString() });
+                this.logger.info('Message queued in dead-letter queue for offline agent', {
+                    targetSession,
+                    queueSize: dlq.length,
+                });
+            }
             this.emit('check_execution_failed', {
                 targetSession,
                 message,
@@ -1017,7 +1187,11 @@ export class SchedulerService extends EventEmitter {
                 }
                 return;
             }
-            await this.executeCheck(targetSession, message);
+            // Enrich the recurring check message with freshness instructions so the
+            // agent queries current task status instead of relying on potentially
+            // stale task descriptions baked into the original scheduled message.
+            const enrichedMessage = this.addFreshnessInstructions(message);
+            await this.executeCheck(targetSession, enrichedMessage);
             // Log per-execution result
             const enhanced = this.enhancedMessages.get(checkId);
             const occurrence = (enhanced?.recurring?.currentOccurrence ?? 0) + 1;
@@ -1068,6 +1242,70 @@ export class SchedulerService extends EventEmitter {
         const firstTimeout = setTimeout(executeRecurring, intervalMinutes * 60 * 1000);
         this.recurringTimeouts.set(checkId, firstTimeout);
     }
+    /**
+     * Enrich a recurring check message with freshness instructions.
+     * Tells the agent to query current task status rather than relying on
+     * potentially stale task descriptions embedded in the original message.
+     *
+     * @param originalMessage - The original scheduled check message
+     * @returns Enriched message with freshness instructions
+     */
+    addFreshnessInstructions(originalMessage) {
+        return `${originalMessage}
+
+⚠️ NOTE: This is a recurring scheduled check. The task description above may be OUTDATED. Before responding, use get_tasks or get_agent_status to verify the CURRENT status. If the task is already completed, cancel this recurring check using cancel_schedule and report the task is done.`;
+    }
+    /**
+     * #167: Drain the dead-letter queue for a session that has come online.
+     * Delivers all queued messages and clears the queue for that session.
+     *
+     * @param sessionName - The session that came online
+     * @returns Number of messages delivered
+     */
+    async drainDeadLetterQueue(sessionName) {
+        const queued = this.deadLetterQueue.get(sessionName);
+        if (!queued || queued.length === 0) {
+            return 0;
+        }
+        this.logger.info('Draining dead-letter queue for session', {
+            sessionName,
+            messageCount: queued.length,
+        });
+        // Take all messages and clear the queue immediately to prevent double delivery
+        const messages = [...queued];
+        this.deadLetterQueue.delete(sessionName);
+        let delivered = 0;
+        for (const entry of messages) {
+            try {
+                await this.executeCheck(sessionName, entry.message);
+                delivered++;
+            }
+            catch (err) {
+                this.logger.warn('Failed to deliver dead-letter message', {
+                    sessionName,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        this.logger.info('Dead-letter queue drained', {
+            sessionName,
+            delivered,
+            total: messages.length,
+        });
+        return delivered;
+    }
+    /**
+     * Get dead-letter queue stats.
+     *
+     * @returns Map of session name to queued message count
+     */
+    getDeadLetterQueueStats() {
+        const stats = {};
+        for (const [session, messages] of this.deadLetterQueue) {
+            stats[session] = messages.length;
+        }
+        return stats;
+    }
     /**
      * Clean up all scheduled checks
      */
@@ -1085,6 +1323,11 @@ export class SchedulerService extends EventEmitter {
         this.recurringTimeouts.clear();
         this.recurringChecks.clear();
         this.recurringIdleStreak.clear();
+        // #167: Stop and clear all cron tasks
+        for (const task of this.cronTasks.values()) {
+            task.stop();
+        }
+        this.cronTasks.clear();
         // Clear persisted recurring checks
         this.storageService.clearRecurringChecks().catch(err => {
             this.logger.error('Failed to clear persisted recurring checks', {
@@ -1106,6 +1349,8 @@ export class SchedulerService extends EventEmitter {
         this.adaptiveChecks.clear();
         // Clear enhanced messages
         this.enhancedMessages.clear();
+        // Clear dead-letter queue
+        this.deadLetterQueue.clear();
         this.logger.info('Scheduler service cleaned up');
     }
     /**
@@ -1125,8 +1370,25 @@ export class SchedulerService extends EventEmitter {
             }
             let restored = 0;
             let purged = 0;
+            let staleSessionPurged = 0;
             for (const check of persisted) {
-                if (!check.isRecurring || !check.intervalMinutes) {
+                if (!check.isRecurring || (!check.intervalMinutes && !check.cronExpression)) {
+                    continue;
+                }
+                // #169: Skip and purge checks from previous sessions
+                if (check.sessionId && check.sessionId !== this.sessionId) {
+                    this.logger.info('Purging stale recurring check from previous session', {
+                        checkId: check.id,
+                        staleSessionId: check.sessionId,
+                        currentSessionId: this.sessionId,
+                    });
+                    this.storageService.deleteRecurringCheck(check.id).catch(err => {
+                        this.logger.error('Failed to delete stale session recurring check', {
+                            checkId: check.id,
+                            error: err instanceof Error ? err.message : String(err),
+                        });
+                    });
+                    staleSessionPurged++;
                     continue;
                 }
                 // Skip checks whose linked task is already completed
@@ -1153,23 +1415,51 @@ export class SchedulerService extends EventEmitter {
                     id: check.id,
                     sessionName: check.targetSession,
                     message: check.message,
-                    scheduledFor: new Date(Date.now() + check.intervalMinutes * 60 * 1000),
+                    scheduledFor: new Date(Date.now() + (check.intervalMinutes || 30) * 60 * 1000),
                     type: 'progress-check',
                     recurring: {
-                        interval: check.intervalMinutes,
+                        interval: check.intervalMinutes || 0,
                         currentOccurrence: check.currentOccurrence || 0,
                         maxOccurrences: check.maxOccurrences,
+                        cronExpression: check.cronExpression,
                     },
                     createdAt: check.createdAt,
                 };
                 this.enhancedMessages.set(check.id, enhancedMessage);
-                this.scheduleRecurringExecution(check.id, check.intervalMinutes, check.targetSession, check.message);
+                // #167: Restore cron-based checks using cron scheduler, interval-based using setTimeout
+                if (check.cronExpression && cron.validate(check.cronExpression)) {
+                    const task = cron.schedule(check.cronExpression, async () => {
+                        if (!this.recurringChecks.has(check.id)) {
+                            task.stop();
+                            this.cronTasks.delete(check.id);
+                            return;
+                        }
+                        await this.executeCheck(check.targetSession, this.addFreshnessInstructions(check.message));
+                        const enh = this.enhancedMessages.get(check.id);
+                        if (enh?.recurring) {
+                            enh.recurring.currentOccurrence = (enh.recurring.currentOccurrence || 0) + 1;
+                            const rc = this.recurringChecks.get(check.id);
+                            if (rc) {
+                                rc.currentOccurrence = enh.recurring.currentOccurrence;
+                                this.storageService.saveRecurringCheck(rc).catch(() => { });
+                            }
+                            if (enh.recurring.maxOccurrences && enh.recurring.currentOccurrence >= enh.recurring.maxOccurrences) {
+                                this.cancelCheck(check.id);
+                            }
+                        }
+                    });
+                    this.cronTasks.set(check.id, task);
+                }
+                else if (check.intervalMinutes) {
+                    this.scheduleRecurringExecution(check.id, check.intervalMinutes, check.targetSession, check.message);
+                }
                 restored++;
             }
             this.logger.info('Restored recurring checks from disk', {
                 total: persisted.length,
                 restored,
                 purgedCompletedTasks: purged,
+                purgedStaleSessions: staleSessionPurged,
             });
             return restored;
         }
@@ -1196,12 +1486,29 @@ export class SchedulerService extends EventEmitter {
                 return 0;
             }
             let restored = 0;
+            let staleSessionPurged = 0;
             const now = Date.now();
             for (const check of persisted) {
                 // Skip recurring checks that ended up here by mistake
                 if (check.isRecurring) {
                     continue;
                 }
+                // #169: Skip and purge checks from previous sessions
+                if (check.sessionId && check.sessionId !== this.sessionId) {
+                    this.logger.info('Purging stale one-time check from previous session', {
+                        checkId: check.id,
+                        staleSessionId: check.sessionId,
+                        currentSessionId: this.sessionId,
+                    });
+                    this.storageService.deleteOneTimeCheck(check.id).catch(err => {
+                        this.logger.error('Failed to delete stale session one-time check', {
+                            checkId: check.id,
+                            error: err instanceof Error ? err.message : String(err),
+                        });
+                    });
+                    staleSessionPurged++;
+                    continue;
+                }
                 const scheduledTime = new Date(check.scheduledFor).getTime();
                 const remainingMs = scheduledTime - now;
                 // If the scheduled time has already passed, discard as stale
@@ -1244,6 +1551,7 @@ export class SchedulerService extends EventEmitter {
             this.logger.info('Restored one-time checks from disk', {
                 total: persisted.length,
                 restored,
+                purgedStaleSessions: staleSessionPurged,
             });
             return restored;
         }
@@ -1269,12 +1577,17 @@ export class SchedulerService extends EventEmitter {
         for (const msg of this.enhancedMessages.values()) {
             activeSessions.add(msg.sessionName);
         }
+        let deadLetterMessages = 0;
+        for (const messages of this.deadLetterQueue.values()) {
+            deadLetterMessages += messages.length;
+        }
         return {
             oneTimeChecks: this.scheduledChecks.size,
             recurringChecks: this.recurringChecks.size,
             totalActiveSessions: activeSessions.size,
             continuationChecks: this.continuationChecks.size,
             adaptiveChecks: this.adaptiveChecks.size,
+            deadLetterMessages,
         };
     }
 }