crewly 1.20.106 → 1.20.108

package/dist/backend/backend/src/services/restart/interrupted-turns.js

@@ -0,0 +1,279 @@
+/**
+ * Interrupted Turns — persist on shutdown, resume on boot.
+ *
+ * When the restart drain gives up (timeout, second signal, drain disabled),
+ * the turns still in flight are written to `<CREWLY_HOME>/interrupted-turns.json`.
+ * On the next boot those sessions count as having work in hand (so
+ * auto-restore brings them back), and once each agent is up its message is
+ * re-delivered with a short notice through the normal delivery path:
+ * - messages that came through the message queue are re-enqueued with their
+ *   original source, conversation and Slack thread, so replies still route;
+ * - anything else goes through `sendMessageToAgent`, whose registration gate
+ *   holds the text until the agent has registered.
+ *
+ * Scheduler / status pings ([SYSTEM] markers) are not persisted: their
+ * producers fire again after a restart, and replaying a stale ping is noise.
+ *
+ * @module services/restart/interrupted-turns
+ */
+import { readFileSync, writeFileSync, renameSync, mkdirSync, unlinkSync } from 'fs';
+import * as path from 'path';
+import { SAFE_RESTART, MESSAGE_SOURCES } from '../../constants.js';
+const SOURCE_VALUES = new Set(Object.values(MESSAGE_SOURCES));
+/**
+ * Default location of the interrupted-turns file.
+ *
+ * @param crewlyHome - CREWLY_HOME directory
+ * @returns Absolute file path
+ */
+export function interruptedTurnsPath(crewlyHome) {
+    return path.join(crewlyHome, SAFE_RESTART.INTERRUPTED_TURNS_FILE);
+}
+/**
+ * Flatten in-flight turns into storable entries, dropping [SYSTEM] pings.
+ *
+ * @param turns - Turns still in flight when the drain ended
+ * @returns Entries to persist
+ */
+export function toInterruptedEntries(turns) {
+    const out = [];
+    for (const turn of turns) {
+        for (const m of turn.messages) {
+            if (m.systemEvent)
+                continue;
+            out.push({
+                sessionName: turn.sessionName,
+                deliveredAt: m.deliveredAt,
+                text: m.text,
+                preview: m.preview,
+                ...(m.messageId ? { messageId: m.messageId } : {}),
+                ...(m.source ? { source: m.source } : {}),
+                ...(m.conversationId ? { conversationId: m.conversationId } : {}),
+                ...(m.originalContent !== undefined ? { originalContent: m.originalContent } : {}),
+                ...(m.sourceMetadata ? { sourceMetadata: m.sourceMetadata } : {}),
+            });
+        }
+    }
+    return out;
+}
+/**
+ * Read the file without filtering. Missing or unreadable file → [].
+ *
+ * @param filePath - File path
+ * @returns Stored entries
+ */
+function readEntries(filePath) {
+    try {
+        const parsed = JSON.parse(readFileSync(filePath, 'utf-8'));
+        return Array.isArray(parsed.turns) ? parsed.turns : [];
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Write entries atomically (write-then-rename), or remove the file when empty.
+ *
+ * @param filePath - File path
+ * @param entries - Entries to store
+ * @param reason - Why they were interrupted
+ */
+export function writeInterruptedTurns(filePath, entries, reason) {
+    if (entries.length === 0) {
+        clearInterruptedTurns(filePath);
+        return;
+    }
+    const body = {
+        version: 1,
+        savedAt: new Date().toISOString(),
+        ...(reason ? { reason } : {}),
+        turns: [...entries],
+    };
+    mkdirSync(path.dirname(filePath), { recursive: true });
+    const tmp = `${filePath}.tmp`;
+    writeFileSync(tmp, JSON.stringify(body, null, 2), 'utf-8');
+    renameSync(tmp, filePath);
+}
+/**
+ * Persist interrupted turns at shutdown, merged with any entries a previous
+ * boot had not resumed yet (so a quick second restart cannot drop them).
+ *
+ * @param filePath - File path
+ * @param turns - Turns still in flight
+ * @param reason - Drain outcome / signal
+ * @returns Number of entries now on disk
+ */
+export function saveInterruptedTurns(filePath, turns, reason) {
+    const incoming = toInterruptedEntries(turns);
+    const existing = readEntries(filePath);
+    const key = (e) => `${e.sessionName}\u0000${e.deliveredAt}\u0000${e.text}`;
+    const seen = new Set();
+    const merged = [];
+    for (const e of [...existing, ...incoming]) {
+        const k = key(e);
+        if (seen.has(k))
+            continue;
+        seen.add(k);
+        merged.push(e);
+    }
+    if (merged.length === 0)
+        return 0;
+    writeInterruptedTurns(filePath, merged, reason);
+    return merged.length;
+}
+/**
+ * Load entries worth resuming.
+ *
+ * @param filePath - File path
+ * @param now - Current time (ms)
+ * @param maxAgeMs - Older entries are dropped
+ * @returns Fresh entries and the number dropped
+ */
+export function loadInterruptedTurns(filePath, now = Date.now(), maxAgeMs = SAFE_RESTART.INTERRUPTED_TURN_MAX_AGE_MS) {
+    const all = readEntries(filePath);
+    const fresh = [];
+    let dropped = 0;
+    for (const e of all) {
+        const valid = e &&
+            typeof e.sessionName === 'string' &&
+            e.sessionName.length > 0 &&
+            typeof e.text === 'string' &&
+            typeof e.deliveredAt === 'number' &&
+            now - e.deliveredAt <= maxAgeMs;
+        if (valid)
+            fresh.push(e);
+        else
+            dropped += 1;
+    }
+    return { fresh, dropped };
+}
+/**
+ * Remove the file (missing file is fine).
+ *
+ * @param filePath - File path
+ */
+export function clearInterruptedTurns(filePath) {
+    try {
+        unlinkSync(filePath);
+    }
+    catch {
+        // Already gone.
+    }
+}
+/**
+ * Prefix a message with the resume notice, without stacking notices when a
+ * resumed message is itself interrupted again.
+ *
+ * @param original - The message to resume
+ * @returns Notice + original
+ */
+export function buildResumeMessage(original) {
+    let body = original;
+    while (body.startsWith(SAFE_RESTART.RESUME_NOTICE)) {
+        body = body.slice(SAFE_RESTART.RESUME_NOTICE.length).replace(/^\s+/, '');
+    }
+    return `${SAFE_RESTART.RESUME_NOTICE}\n${body}`;
+}
+/**
+ * Whether an entry carries enough queue metadata to be re-enqueued.
+ *
+ * @param entry - Stored entry
+ * @returns True for message-queue deliveries with a known source
+ */
+function isQueueEntry(entry) {
+    return (typeof entry.source === 'string' &&
+        SOURCE_VALUES.has(entry.source) &&
+        entry.source !== MESSAGE_SOURCES.SYSTEM_EVENT &&
+        typeof entry.conversationId === 'string' &&
+        entry.conversationId.length > 0 &&
+        typeof entry.originalContent === 'string');
+}
+/**
+ * Re-deliver interrupted messages, one at a time.
+ *
+ * @param entries - Fresh entries from loadInterruptedTurns
+ * @param deps - Delivery hooks
+ * @returns Counts of redelivered / skipped / failed
+ *
+ * @example
+ * ```typescript
+ * const { fresh } = loadInterruptedTurns(file);
+ * await resumeInterruptedTurns(fresh, deps);
+ * clearInterruptedTurns(file);
+ * ```
+ */
+export async function resumeInterruptedTurns(entries, deps) {
+    const summary = { redelivered: 0, skipped: 0, failed: 0 };
+    const remaining = [...entries];
+    let orcReady = null;
+    for (const entry of entries) {
+        const isOrc = entry.sessionName === deps.orchestratorSession;
+        try {
+            if (!isOrc && !deps.isSessionRunning(entry.sessionName)) {
+                summary.skipped += 1;
+                deps.logger.warn('Interrupted turn not resumed: agent is not running after restart', {
+                    sessionName: entry.sessionName,
+                    messagePreview: entry.preview,
+                });
+            }
+            else if (isQueueEntry(entry)) {
+                deps.enqueue({
+                    content: buildResumeMessage(entry.originalContent),
+                    conversationId: entry.conversationId,
+                    source: entry.source,
+                    ...(entry.sourceMetadata ? { sourceMetadata: entry.sourceMetadata } : {}),
+                    ...(isOrc ? {} : { targetSession: entry.sessionName }),
+                });
+                summary.redelivered += 1;
+                deps.logger.info('Interrupted turn re-enqueued with its original source', {
+                    sessionName: entry.sessionName,
+                    source: entry.source,
+                    messagePreview: entry.preview,
+                });
+            }
+            else {
+                if (isOrc) {
+                    if (orcReady === null)
+                        orcReady = await deps.waitForOrchestratorActive();
+                    if (!orcReady) {
+                        summary.failed += 1;
+                        deps.logger.warn('Interrupted turn not resumed: orchestrator never became active', {
+                            messagePreview: entry.preview,
+                        });
+                        continue;
+                    }
+                }
+                const result = await deps.sendMessageToAgent(entry.sessionName, buildResumeMessage(entry.text));
+                if (result.success) {
+                    summary.redelivered += 1;
+                    deps.logger.info('Interrupted turn re-delivered', {
+                        sessionName: entry.sessionName,
+                        queuedUntilRegistered: result.queued === true,
+                        messagePreview: entry.preview,
+                    });
+                }
+                else {
+                    summary.failed += 1;
+                    deps.logger.warn('Interrupted turn re-delivery failed', {
+                        sessionName: entry.sessionName,
+                        error: result.error,
+                        messagePreview: entry.preview,
+                    });
+                }
+            }
+        }
+        catch (error) {
+            summary.failed += 1;
+            deps.logger.warn('Interrupted turn re-delivery threw', {
+                sessionName: entry.sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        finally {
+            remaining.splice(remaining.indexOf(entry), 1);
+            deps.onEntryHandled?.(entry, [...remaining]);
+        }
+    }
+    return summary;
+}
+//# sourceMappingURL=interrupted-turns.js.map

package/dist/backend/backend/src/services/restart/interrupted-turns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interrupted-turns.js","sourceRoot":"","sources":["../../../../../../backend/src/services/restart/interrupted-turns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACpF,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,eAAe,EAAsB,MAAM,oBAAoB,CAAC;AAuEvF,MAAM,aAAa,GAAwB,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;AAEnF;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACtD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,sBAAsB,CAAC,CAAC;AACnE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAA8B;IAClE,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,CAAC,WAAW;gBAAE,SAAS;YAC5B,GAAG,CAAC,IAAI,CAAC;gBACR,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClD,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,CAAC,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClF,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACjE,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,QAAgB;IACpC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAkC,CAAC;QAC5F,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,OAAwC,EAAE,MAAe;IAChH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QAChC,OAAO;IACR,CAAC;IACD,MAAM,IAAI,GAAyB;QAClC,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACjC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC;KACnB,CAAC;IACF,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,GAAG,QAAQ,MAAM,CAAC;IAC9B,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC3D,UAAU,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAgB,EAAE,KAA8B,EAAE,MAAe;IACrG,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,CAAC,CAAuB,EAAU,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW,SAAS,CAAC,CAAC,WAAW,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IACzG,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACZ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChB,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,MAAM,CAAC;AACtB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CACnC,QAAgB,EAChB,MAAc,IAAI,CAAC,GAAG,EAAE,EACxB,WAAmB,YAAY,CAAC,2BAA2B;IAE3D,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,KAAK,GACV,CAAC;YACD,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ;YACjC,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;YACxB,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;YAC1B,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ;YACjC,GAAG,GAAG,CAAC,CAAC,WAAW,IAAI,QAAQ,CAAC;QACjC,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YACpB,OAAO,IAAI,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACrD,IAAI,CAAC;QACJ,UAAU,CAAC,QAAQ,CAAC,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACR,gBAAgB;IACjB,CAAC;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IAClD,IAAI,IAAI,GAAG,QAAQ,CAAC;IACpB,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,CAAC;QACpD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,GAAG,YAAY,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;AACjD,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,KAA2B;IAChD,OAAO,CACN,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;QAChC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;QAC/B,KAAK,CAAC,MAAM,KAAK,eAAe,CAAC,YAAY;QAC7C,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ;QACxC,KAAK,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;QAC/B,OAAO,KAAK,CAAC,eAAe,KAAK,QAAQ,CACzC,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,OAAwC,EAAE,IAAgB;IACtG,MAAM,OAAO,GAAkB,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACzE,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;IAC/B,IAAI,QAAQ,GAAmB,IAAI,CAAC;IAEpC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,KAAK,IAAI,CAAC,mBAAmB,CAAC;QAC7D,IAAI,CAAC;YACJ,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;gBACzD,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;gBACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kEAAkE,EAAE;oBACpF,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,cAAc,EAAE,KAAK,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACJ,CAAC;iBAAM,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC;oBACZ,OAAO,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC;oBAClD,cAAc,EAAE,KAAK,CAAC,cAAc;oBACpC,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACzE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;iBACtD,CAAC,CAAC;gBACH,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,EAAE;oBACzE,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,cAAc,EAAE,KAAK,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,IAAI,KAAK,EAAE,CAAC;oBACX,IAAI,QAAQ,KAAK,IAAI;wBAAE,QAAQ,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;oBACzE,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;wBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gEAAgE,EAAE;4BAClF,cAAc,EAAE,KAAK,CAAC,OAAO;yBAC7B,CAAC,CAAC;wBACH,SAAS;oBACV,CAAC;gBACF,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,WAAW,EAAE,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACpB,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;oBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;wBACjD,WAAW,EAAE,KAAK,CAAC,WAAW;wBAC9B,qBAAqB,EAAE,MAAM,CAAC,MAAM,KAAK,IAAI;wBAC7C,cAAc,EAAE,KAAK,CAAC,OAAO;qBAC7B,CAAC,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACP,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE;wBACvD,WAAW,EAAE,KAAK,CAAC,WAAW;wBAC9B,KAAK,EAAE,MAAM,CAAC,KAAK;wBACnB,cAAc,EAAE,KAAK,CAAC,OAAO;qBAC7B,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBACtD,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC7D,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9C,IAAI,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAC9C,CAAC;IACF,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC"}

package/dist/backend/backend/src/services/restart/restart-drain.service.d.ts

@@ -0,0 +1,176 @@
+/**
+ * Restart Drain Service
+ *
+ * Owns the "safe restart" state of the process:
+ * - a delivery pause: once shutdown starts, nothing new is written into an
+ *   agent's PTY — messages stay on the persistent queues instead;
+ * - the drain: wait until no agent is mid-turn, capped by a timeout, and
+ *   interruptible by a second signal;
+ * - restart readiness for operators (`GET /api/system/restart-readiness`);
+ * - a hook through which the REST restart endpoint asks the server for the
+ *   same graceful, drained shutdown a SIGTERM gets.
+ *
+ * "Queue empty" is not "safe to restart": a delivered message is off the
+ * queue while the agent is still working on it (2026-09-24, Ella). Readiness
+ * therefore reports in-flight turns, not queue depth.
+ *
+ * @module services/restart/restart-drain
+ */
+import { type InFlightTurn } from './in-flight-turn-tracker.service.js';
+/** How a drain ended. */
+export type DrainOutcome = 'drained' | 'timed-out' | 'skipped' | 'disabled';
+/** Result of a drain. */
+export interface DrainResult {
+    /** How it ended */
+    outcome: DrainOutcome;
+    /** Time spent waiting (ms) */
+    waitedMs: number;
+    /** Turns still in flight when the drain ended (to be persisted and resumed) */
+    remaining: InFlightTurn[];
+}
+/** Options for {@link RestartDrainService.drain}. */
+export interface DrainOptions {
+    /** Maximum wait (ms); 0 skips the wait */
+    timeoutMs: number;
+    /** Re-check interval (ms) */
+    pollMs?: number;
+    /** Interval for the repeated "still waiting" log line (ms) */
+    logIntervalMs?: number;
+    /** Clock (tests) */
+    now?: () => number;
+    /** Sleep (tests) */
+    sleep?: (ms: number) => Promise<void>;
+}
+/** One agent the readiness endpoint reports as busy. */
+export interface BusyAgent {
+    /** Agent session name */
+    session: string;
+    /** ISO time of the oldest open delivery */
+    since: string;
+    /** Preview of the most recent open delivery */
+    messagePreview: string;
+}
+/** Body of `GET /api/system/restart-readiness`. */
+export interface RestartReadiness {
+    /** True when no agent is mid-turn */
+    safe: boolean;
+    /** Agents mid-turn */
+    busyAgents: BusyAgent[];
+    /** Messages waiting on the persistent queues (they survive a restart) */
+    queued: number;
+    /** True when a shutdown drain is already running */
+    draining: boolean;
+}
+/** Options for a graceful shutdown requested through the service. */
+export interface GracefulShutdownRequest {
+    /** Why (for logs) */
+    reason: string;
+    /** Process exit code at the end of shutdown */
+    exitCode?: number;
+}
+/**
+ * Resolve the drain timeout from the environment.
+ *
+ * @param env - Environment (defaults to process.env)
+ * @returns Timeout in ms; 0 disables the wait. Invalid or negative values fall back to the default.
+ *
+ * @example
+ * ```typescript
+ * resolveRestartDrainMs({ CREWLY_RESTART_DRAIN_MS: '0' }); // 0
+ * resolveRestartDrainMs({}); // 120000
+ * ```
+ */
+export declare function resolveRestartDrainMs(env?: NodeJS.ProcessEnv): number;
+/**
+ * Process-wide safe-restart state. Singleton.
+ */
+export declare class RestartDrainService {
+    private readonly tracker;
+    private static instance;
+    private readonly logger;
+    private paused;
+    private pauseReason;
+    private draining;
+    private skipRequested;
+    private wakeSleeper;
+    private queueCounter;
+    private shutdownHandler;
+    private constructor();
+    /**
+     * Get the singleton instance.
+     *
+     * @returns The service
+     */
+    static getInstance(): RestartDrainService;
+    /**
+     * Reset the singleton (tests only).
+     */
+    static resetInstance(): void;
+    /**
+     * Whether deliveries into agent PTYs are paused (shutdown in progress).
+     *
+     * @returns True while paused
+     */
+    isDeliveryPaused(): boolean;
+    /**
+     * Whether the drain wait is running right now.
+     *
+     * @returns True during the wait
+     */
+    isDraining(): boolean;
+    /**
+     * Stop writing new messages into agent PTYs. Idempotent; never undone in
+     * this process (the next boot starts unpaused).
+     *
+     * @param reason - Why (for logs)
+     */
+    pauseDelivery(reason: string): void;
+    /**
+     * Ask a running drain to stop waiting (second signal, supervisor escalation).
+     *
+     * @param reason - Why (for logs)
+     * @returns True if a drain was running and will now end
+     */
+    requestSkip(reason: string): boolean;
+    /**
+     * Register the counter used for `queued` in readiness.
+     *
+     * @param counter - Returns the number of messages on the persistent queues
+     */
+    setQueueCounter(counter: (() => number) | null): void;
+    /**
+     * Register the server's graceful-shutdown entry point.
+     *
+     * @param handler - Runs the drained shutdown and exits the process
+     */
+    setShutdownHandler(handler: ((request: GracefulShutdownRequest) => Promise<void>) | null): void;
+    /**
+     * Run the server's graceful (drained) shutdown, if one is registered.
+     *
+     * @param request - Reason and exit code
+     * @returns True if a handler was registered and has been started
+     */
+    requestGracefulShutdown(request: GracefulShutdownRequest): boolean;
+    /**
+     * Current restart readiness.
+     *
+     * @returns Busy agents, queued count, and whether a restart is safe now
+     */
+    getReadiness(): RestartReadiness;
+    /**
+     * Wait until no agent is mid-turn, the timeout passes, or a skip arrives.
+     * Pauses delivery first (idempotent) so the set of turns can only shrink.
+     *
+     * @param options - Timeout and test hooks
+     * @returns How it ended and the turns still in flight
+     */
+    drain(options: DrainOptions): Promise<DrainResult>;
+    /**
+     * Sleep, but wake early when a skip is requested.
+     *
+     * @param sleep - Injected sleep (tests), or null for a cancellable timer
+     * @param ms - Duration
+     */
+    private sleepUnlessSkipped;
+}
+//# sourceMappingURL=restart-drain.service.d.ts.map

package/dist/backend/backend/src/services/restart/restart-drain.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"restart-drain.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/restart/restart-drain.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,OAAO,EAAuB,KAAK,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAE7F,yBAAyB;AACzB,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,CAAC;AAE5E,yBAAyB;AACzB,MAAM,WAAW,WAAW;IAC3B,mBAAmB;IACnB,OAAO,EAAE,YAAY,CAAC;IACtB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,SAAS,EAAE,YAAY,EAAE,CAAC;CAC1B;AAED,qDAAqD;AACrD,MAAM,WAAW,YAAY;IAC5B,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oBAAoB;IACpB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,oBAAoB;IACpB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED,wDAAwD;AACxD,MAAM,WAAW,SAAS;IACzB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,cAAc,EAAE,MAAM,CAAC;CACvB;AAED,mDAAmD;AACnD,MAAM,WAAW,gBAAgB;IAChC,qCAAqC;IACrC,IAAI,EAAE,OAAO,CAAC;IACd,sBAAsB;IACtB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;IACf,oDAAoD;IACpD,QAAQ,EAAE,OAAO,CAAC;CAClB;AAED,qEAAqE;AACrE,MAAM,WAAW,uBAAuB;IACvC,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,MAAM,CAMlF;AAED;;GAEG;AACH,qBAAa,mBAAmB;IAWX,OAAO,CAAC,QAAQ,CAAC,OAAO;IAV5C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAoC;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,eAAe,CAAsE;IAE7F,OAAO;IAIP;;;;OAIG;IACH,MAAM,CAAC,WAAW,IAAI,mBAAmB;IAOzC;;OAEG;IACH,MAAM,CAAC,aAAa,IAAI,IAAI;IAI5B;;;;OAIG;IACH,gBAAgB,IAAI,OAAO;IAI3B;;;;OAIG;IACH,UAAU,IAAI,OAAO;IAIrB;;;;;OAKG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAOnC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAUpC;;;;OAIG;IACH,eAAe,CAAC,OAAO,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,IAAI,GAAG,IAAI;IAIrD;;;;OAIG;IACH,kBAAkB,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,uBAAuB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI;IAI/F;;;;;OAKG;IACH,uBAAuB,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO;IAWlE;;;;OAIG;IACH,YAAY,IAAI,gBAAgB;IAgBhC;;;;;;OAMG;IACG,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC;IAgExD;;;;;OAKG;YACW,kBAAkB;CAoBhC"}