@pasko70/pibo 1.11.1 → 1.11.3

package/dist/core/session-router.js

@@ -16,13 +16,14 @@ import { loadPiboUserSettings } from "./user-settings.js";
 import { resolvePiboSessionActiveModel } from "./session-model.js";
 import { isPiboThinkingLevel } from "./thinking.js";
 import { RuntimeSessionRegistry } from "../tools/runtime/registry.js";
-import { assertGatewayResourceAvailableForWork } from "./gateway-resource-guard.js";
+import { GatewayWorkAdmissionController } from "./gateway-resource-guard.js";
 import { withWorkflowSessionKind } from "../sessions/workflow-session-kind.js";
 import { PiboRuntimeTelemetryRecorder } from "./runtime-telemetry.js";
 import { createPiboProviderTelemetryExtension } from "./provider-telemetry.js";
 import { AsyncTelemetryWriter } from "../data/telemetry-writer.js";
 const DEFAULT_SUBAGENT_REPLY_TIMEOUT_MS = 10 * 60 * 1000;
 const DEFAULT_ROUTED_SESSION_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
+const DEFAULT_ROUTED_SESSION_DISPOSE_TIMEOUT_MS = 30 * 1000;
 export const LOOP_RUNTIME_RETRY_DEFAULTS = {
     enabled: true,
     maxRetries: 7,
@@ -115,7 +116,7 @@ function formatRunReminderMessage(notification) {
     ].join("\n");
 }
 function isRunReminderServiceMessage(event) {
-    return event.source === "service" && event.text.startsWith("<pibo_run_notification>");
+    return event.source === "service" && event.capabilityScope === "run-reminder";
 }
 function isTerminalRunStatus(status) {
     return status === "completed" || status === "failed" || status === "timed_out" || status === "cancelled";
@@ -130,6 +131,16 @@ function piboRoomIdFromMetadata(metadata) {
     const value = metadata?.chatRoomId;
     return typeof value === "string" && value.length > 0 ? value : undefined;
 }
+class PiboSessionDisposalTimeoutError extends Error {
+    piboSessionId;
+    timeoutMs;
+    constructor(piboSessionId, timeoutMs) {
+        super(`Timed out disposing Pibo session "${piboSessionId}" after ${timeoutMs}ms`);
+        this.piboSessionId = piboSessionId;
+        this.timeoutMs = timeoutMs;
+        this.name = "PiboSessionDisposalTimeoutError";
+    }
+}
 function telemetryStoreFromSessionStore(store) {
     return store.getTelemetryStore?.();
 }
@@ -143,11 +154,16 @@ export class PiboSessionRouter {
     pendingSessions = new Map();
     listeners = new Set();
     runRegistry;
+    gatewayWorkAdmission = new GatewayWorkAdmissionController();
     signalRegistry;
     runtimeRegistry;
     scheduledRunReminders = new Map();
+    runReminderGenerations = new Map();
+    quiescingSessions = new Set();
+    disposingSessions = new Map();
     idleSessionTimers = new Map();
     routedSessionIdleTimeoutMs;
+    routedSessionDisposeTimeoutMs;
     baseProfile;
     pluginRegistry;
     sessionStore;
@@ -175,6 +191,10 @@ export class PiboSessionRouter {
             : typeof idleTimeoutMs === "number" && Number.isFinite(idleTimeoutMs) && idleTimeoutMs > 0
                 ? idleTimeoutMs
                 : DEFAULT_ROUTED_SESSION_IDLE_TIMEOUT_MS;
+        const disposeTimeoutMs = options.routedSessionDisposeTimeoutMs;
+        this.routedSessionDisposeTimeoutMs = typeof disposeTimeoutMs === "number" && Number.isFinite(disposeTimeoutMs) && disposeTimeoutMs > 0
+            ? disposeTimeoutMs
+            : DEFAULT_ROUTED_SESSION_DISPOSE_TIMEOUT_MS;
         const defaultProfileName = selectDefaultPiboProfileName(this.pluginRegistry);
         this.baseProfile = options.profile ?? createPiboProfileFromRegistryOrDefault(this.pluginRegistry, defaultProfileName);
         this.reliabilityStore = options.reliabilityStore ?? (options.persistSession === false ? undefined : createDefaultPiboReliabilityStore());
@@ -195,6 +215,16 @@ export class PiboSessionRouter {
     async emit(event) {
         if (this.closing)
             throw new Error("Pibo session router is disposed.");
+        const teardownAction = event.type === "execution" && (event.action === "dispose" || event.action === "kill" || event.action === "kill_all");
+        const teardownIds = teardownAction
+            ? [event.piboSessionId, ...this.descendantSessionIds(event.piboSessionId)]
+            : [];
+        if (event.type === "execution" && event.action === "abort") {
+            this.invalidateRunReminders([event.piboSessionId]);
+        }
+        else if (teardownAction) {
+            this.invalidateRunReminders(teardownIds);
+        }
         if (event.type === "message" && event.id) {
             const stored = this.sessionStore.get(event.piboSessionId);
             if (stored)
@@ -220,6 +250,9 @@ export class PiboSessionRouter {
             throw error;
         }
         this.clearIdleSessionTimer(event.piboSessionId);
+        let teardownCompleted = false;
+        if (teardownAction)
+            this.beginSessionQuiescence(teardownIds);
         try {
             if (event.type === "message") {
                 return event.delivery === "steer"
@@ -232,12 +265,23 @@ export class PiboSessionRouter {
             else if (event.action === "dispose" || event.action === "kill" || event.action === "kill_all") {
                 this.signalRegistry.project({ type: "session_disposed", piboSessionId: event.piboSessionId, reason: `${event.action} action` });
             }
-            const output = await session.executeAction(event);
             if (event.action === "dispose") {
+                const output = {
+                    type: "execution_result",
+                    piboSessionId: event.piboSessionId,
+                    eventId: event.id,
+                    action: event.action,
+                    result: { disposed: true },
+                };
+                this.emitOutput(output);
                 await this.disposeSessionSubtree(event.piboSessionId, "dispose action", { cancelRuns: true });
+                teardownCompleted = true;
+                return output;
             }
-            else if (event.action === "kill" || event.action === "kill_all") {
+            const output = await session.executeAction(event);
+            if (event.action === "kill" || event.action === "kill_all") {
                 await this.disposeSessionSubtree(event.piboSessionId, `${event.action} action`, { cancelRuns: event.action === "kill_all" });
+                teardownCompleted = true;
             }
             else if (shouldResetSessionAfterAction(event.action)) {
                 await this.resetCachedSession(event.piboSessionId, "provider auth changed");
@@ -245,6 +289,9 @@ export class PiboSessionRouter {
             return output;
         }
         catch (error) {
+            if (teardownAction && !teardownCompleted) {
+                await this.disposeSessionSubtree(event.piboSessionId, `${event.action} action failed`, { cancelRuns: event.action === "dispose" || event.action === "kill_all" }).catch(() => { });
+            }
             if (event.type === "message" && event.id) {
                 this.signalRegistry.project({
                     type: "message_rejected",
@@ -266,46 +313,118 @@ export class PiboSessionRouter {
         }
     }
     async killSession(piboSessionId, options) {
+        const rootSession = this.sessions.get(piboSessionId);
+        if (!rootSession)
+            return { killed: [], cancelledRuns: [] };
+        const ids = [piboSessionId, ...this.descendantSessionIds(piboSessionId)];
+        this.beginSessionQuiescence(ids);
         const killed = [];
         const cancelledRuns = [];
-        const session = this.sessions.get(piboSessionId);
-        if (session) {
-            this.signalRegistry.project({ type: "session_disposed", piboSessionId, reason: "kill" });
-            killed.push(await session.kill());
+        const failures = [];
+        for (const id of ids) {
+            const session = this.sessions.get(id);
+            if (session) {
+                this.signalRegistry.project({ type: "session_disposed", piboSessionId: id, reason: "kill" });
+                try {
+                    killed.push(await session.kill());
+                }
+                catch (error) {
+                    failures.push(error);
+                }
+            }
             if (options?.includeRuns) {
-                const runs = this.runRegistry.cancelControllerRuns(piboSessionId);
-                cancelledRuns.push(...runs.map((r) => r.runId));
+                const runs = this.runRegistry.cancelControllerRuns(id);
+                cancelledRuns.push(...runs.map((run) => run.runId));
             }
-            const children = await this.killChildSessions(piboSessionId, options);
-            killed.push(...children.killed);
-            cancelledRuns.push(...children.cancelledRuns);
+        }
+        try {
             await this.disposeSessionSubtree(piboSessionId, "kill", { cancelRuns: false });
         }
+        catch (error) {
+            failures.push(error);
+        }
+        if (failures.length > 0)
+            throw new AggregateError(failures, `Failed to kill Pibo session subtree "${piboSessionId}"`);
         return { killed, cancelledRuns };
     }
+    async disposeRoutedSession(piboSessionId, session, reason) {
+        const disposal = Promise.resolve().then(() => session.dispose());
+        let timeout;
+        const timedOut = new Promise((_resolve, reject) => {
+            timeout = setTimeout(() => reject(new PiboSessionDisposalTimeoutError(piboSessionId, this.routedSessionDisposeTimeoutMs)), this.routedSessionDisposeTimeoutMs);
+            timeout.unref?.();
+        });
+        try {
+            await Promise.race([disposal, timedOut]);
+        }
+        catch (error) {
+            if (error instanceof PiboSessionDisposalTimeoutError) {
+                session.forceDispose(`${reason}; bounded disposal timeout`);
+                void disposal.catch(() => { });
+            }
+            throw error;
+        }
+        finally {
+            if (timeout)
+                clearTimeout(timeout);
+        }
+    }
     async disposeSessionSubtree(piboSessionId, reason, options) {
         const ids = [piboSessionId, ...this.descendantSessionIds(piboSessionId)];
-        const sessions = [];
-        for (const id of ids) {
-            if (options.cancelRuns)
+        const existingDisposals = [...new Set(ids.map((id) => this.disposingSessions.get(id)).filter((value) => Boolean(value)))];
+        if (existingDisposals.length > 0)
+            await Promise.all(existingDisposals);
+        this.beginSessionQuiescence(ids);
+        if (options.cancelRuns) {
+            for (const id of ids)
                 this.runRegistry.cancelControllerRuns(id);
-            this.clearIdleSessionTimer(id);
-            this.scheduledRunReminders.delete(id);
-            const cached = this.sessions.get(id);
-            if (cached)
-                sessions.push(cached);
-            this.sessions.delete(id);
         }
+        let releaseStart;
+        const startGate = new Promise((resolve) => {
+            releaseStart = resolve;
+        });
+        const operation = (async () => {
+            await startGate;
+            const pending = ids.map((id) => this.pendingSessions.get(id)).filter((value) => Boolean(value));
+            if (pending.length > 0)
+                await Promise.allSettled(pending);
+            const sessions = ids.flatMap((id) => {
+                const session = this.sessions.get(id);
+                return session ? [{ id, session }] : [];
+            });
+            const failures = [];
+            const closeResults = await Promise.allSettled(ids.map((id) => this.runtimeRegistry.closeControllerSessions(id, { force: true })));
+            for (const result of closeResults) {
+                if (result.status === "rejected")
+                    failures.push(result.reason);
+            }
+            const disposeResults = await Promise.allSettled(sessions.map(({ id, session }) => this.disposeRoutedSession(id, session, reason)));
+            for (const result of disposeResults) {
+                if (result.status === "rejected")
+                    failures.push(result.reason);
+            }
+            for (const { id, session } of sessions) {
+                if (this.sessions.get(id) === session)
+                    this.sessions.delete(id);
+            }
+            if (failures.length > 0)
+                throw new AggregateError(failures, `Failed to dispose Pibo session subtree "${piboSessionId}"`);
+        })();
+        for (const id of ids)
+            this.disposingSessions.set(id, operation);
+        releaseStart?.();
         try {
-            await Promise.all(ids.map((id) => this.runtimeRegistry.closeControllerSessions(id, { force: true })));
-            await Promise.all(sessions.map((session) => session.dispose()));
+            await operation;
         }
         finally {
+            for (const id of ids) {
+                if (this.disposingSessions.get(id) === operation)
+                    this.disposingSessions.delete(id);
+                this.quiescingSessions.delete(id);
+                this.signalRegistry.project({ type: "session_disposed", piboSessionId: id, reason });
+            }
             await this.telemetryWriter?.flush();
         }
-        for (const id of ids) {
-            this.signalRegistry.project({ type: "session_disposed", piboSessionId: id, reason });
-        }
     }
     descendantSessionIds(parentId) {
         const output = [];
@@ -456,18 +575,28 @@ export class PiboSessionRouter {
     }
     async disposeAllUnsafe() {
         try {
+            const initialIds = [...new Set([...this.sessions.keys(), ...this.pendingSessions.keys()])];
+            this.beginSessionQuiescence(initialIds);
             await Promise.allSettled([...this.pendingSessions.values()]);
-            const sessions = [...this.sessions.values()];
-            this.sessions.clear();
+            const sessions = [...this.sessions.entries()];
             for (const timer of this.idleSessionTimers.values())
                 clearTimeout(timer);
             this.idleSessionTimers.clear();
             this.runRegistry.cancelAll("Pibo session router was disposed.");
-            for (const session of sessions)
-                this.signalRegistry.project({ type: "session_disposed", piboSessionId: session.getStatus().piboSessionId, reason: "router disposed" });
             this.scheduledRunReminders.clear();
-            await this.runtimeRegistry.closeAll({ force: true });
-            await Promise.all(sessions.map((session) => session.dispose()));
+            const closeResult = await Promise.allSettled([this.runtimeRegistry.closeAll({ force: true })]);
+            const disposeResults = await Promise.allSettled(sessions.map(([id, session]) => this.disposeRoutedSession(id, session, "router disposed")));
+            for (const [id, session] of sessions) {
+                if (this.sessions.get(id) === session)
+                    this.sessions.delete(id);
+                this.signalRegistry.project({ type: "session_disposed", piboSessionId: id, reason: "router disposed" });
+            }
+            const failures = [
+                ...closeResult.filter((result) => result.status === "rejected").map((result) => result.reason),
+                ...disposeResults.filter((result) => result.status === "rejected").map((result) => result.reason),
+            ];
+            if (failures.length > 0)
+                throw new AggregateError(failures, "Failed to dispose all Pibo sessions");
         }
         finally {
             await this.telemetryWriter?.dispose();
@@ -518,6 +647,14 @@ export class PiboSessionRouter {
     async getOrCreateSession(piboSessionId) {
         if (this.closing)
             throw new Error("Pibo session router is disposed.");
+        if (this.quiescingSessions.has(piboSessionId)) {
+            throw new Error(`Pibo session "${piboSessionId}" is quiescing.`);
+        }
+        const disposing = this.disposingSessions.get(piboSessionId);
+        if (disposing) {
+            await disposing;
+            return await this.getOrCreateSession(piboSessionId);
+        }
         const existing = this.sessions.get(piboSessionId);
         if (existing) {
             this.clearIdleSessionTimer(piboSessionId);
@@ -537,6 +674,7 @@ export class PiboSessionRouter {
     }
     async createRoutedSession(piboSessionId) {
         const piboSession = this.resolvePiboSession(piboSessionId);
+        let session;
         this.signalRegistry.project({ type: "session_created", session: piboSession });
         const profile = createPiboProfileFromRegistryOrDefault(this.pluginRegistry, piboSession.profile);
         const parentPiSessionId = piboSession.parentId
@@ -568,10 +706,11 @@ export class PiboSessionRouter {
                 piboSessionId: piboSession.id,
                 piboRoomId: piboRoomIdFromMetadata(piboSession.metadata),
                 timezone: userSettings.timezone,
+                getActiveMessage: () => session?.getActiveMessage(),
             },
         });
         const initialFastMode = resolvePiboSessionInitialFastMode(piboSession) ?? selectRequestedFastMode(profileForSession(profile, piboSession.piSessionId, parentPiSessionId), modelDefaults) ?? false;
-        const session = new RoutedSession(piboSession.id, runtime, this.emitOutput, this.pluginRegistry, this.options.forwardPiEvents ?? false, this.telemetryRecorder
+        session = new RoutedSession(piboSession.id, runtime, this.emitOutput, this.pluginRegistry, this.options.forwardPiEvents ?? false, this.telemetryRecorder
             ? (id, event, context) => this.telemetryRecorder?.recordPiEvent(id, event, { session: this.sessionStore.get(id), status: context.status, activeEventId: context.activeEventId })
             : undefined, initialFastMode, (result, event) => this.handleSessionOperation(result, event), (id, opts) => this.killChildSessions(id, opts), (state) => {
             this.signalRegistry.project({ type: "session_processing_changed", piboSessionId: piboSession.id, processing: state.processing, queuedMessages: state.queuedMessages });
@@ -582,7 +721,7 @@ export class PiboSessionRouter {
         }, (messages, reason) => this.telemetryRecorder?.recordMessagesInterrupted(messages, {
             session: this.sessionStore.get(piboSession.id),
             status: this.sessions.get(piboSession.id)?.getStatus(),
-        }, reason));
+        }, reason), this.options.messagePreflight);
         this.sessions.set(piboSession.id, session);
         return session;
     }
@@ -640,14 +779,42 @@ export class PiboSessionRouter {
         });
     }
     async resetCachedSession(piboSessionId, reason) {
-        const cached = this.sessions.get(piboSessionId);
+        const existingDisposal = this.disposingSessions.get(piboSessionId);
+        if (existingDisposal)
+            await existingDisposal;
         this.clearIdleSessionTimer(piboSessionId);
-        this.sessions.delete(piboSessionId);
+        let releaseStart;
+        const startGate = new Promise((resolve) => {
+            releaseStart = resolve;
+        });
+        const operation = (async () => {
+            await startGate;
+            const pending = this.pendingSessions.get(piboSessionId);
+            if (pending)
+                await Promise.allSettled([pending]);
+            const cached = this.sessions.get(piboSessionId);
+            const failures = [];
+            const closeResult = await Promise.allSettled([this.runtimeRegistry.closeControllerSessions(piboSessionId, { force: true })]);
+            if (closeResult[0]?.status === "rejected")
+                failures.push(closeResult[0].reason);
+            if (cached) {
+                const disposeResult = await Promise.allSettled([this.disposeRoutedSession(piboSessionId, cached, reason ?? "session reset")]);
+                if (disposeResult[0]?.status === "rejected")
+                    failures.push(disposeResult[0].reason);
+                if (this.sessions.get(piboSessionId) === cached)
+                    this.sessions.delete(piboSessionId);
+            }
+            if (failures.length > 0)
+                throw new AggregateError(failures, `Failed to reset Pibo session "${piboSessionId}"`);
+        })();
+        this.disposingSessions.set(piboSessionId, operation);
+        releaseStart?.();
         try {
-            await this.runtimeRegistry.closeControllerSessions(piboSessionId, { force: true });
-            await cached?.dispose();
+            await operation;
         }
         finally {
+            if (this.disposingSessions.get(piboSessionId) === operation)
+                this.disposingSessions.delete(piboSessionId);
             await this.telemetryWriter?.flush();
         }
         if (reason)
@@ -697,23 +864,31 @@ export class PiboSessionRouter {
     createRunToolController(parentPiboSessionId) {
         return {
             startToolRun: ({ toolName, params, completionPolicy, retryable, maxAttempts, timeoutMs, serviceWarning, execute }) => {
-                assertGatewayResourceAvailableForWork(`yielded run ${toolName}`);
-                const run = this.runRegistry.startToolRun({
-                    controllerPiboSessionId: parentPiboSessionId,
-                    toolName,
-                    params,
-                    completionPolicy,
-                    retryable,
-                    maxAttempts,
-                    timeoutMs,
-                    serviceWarning,
-                });
+                const admission = this.gatewayWorkAdmission.reserve(`yielded run ${toolName}`);
+                const reminderGeneration = this.runReminderGeneration(parentPiboSessionId);
+                let run;
+                try {
+                    run = this.runRegistry.startToolRun({
+                        controllerPiboSessionId: parentPiboSessionId,
+                        toolName,
+                        params,
+                        completionPolicy,
+                        retryable,
+                        maxAttempts,
+                        timeoutMs,
+                        serviceWarning,
+                    });
+                }
+                catch (error) {
+                    admission.release();
+                    throw error;
+                }
                 void (async () => {
                     try {
                         const result = await execute();
                         const completed = this.runRegistry.complete(run.runId, result);
                         if (completed)
-                            this.scheduleRunReminder(parentPiboSessionId, false);
+                            this.handleTerminalRunReminder(parentPiboSessionId, completed.runId, reminderGeneration);
                     }
                     catch (error) {
                         const message = error instanceof Error ? error.message : String(error);
@@ -721,7 +896,10 @@ export class PiboSessionRouter {
                             ? this.runRegistry.timeOut(run.runId, message, error.timeoutPhase)
                             : this.runRegistry.fail(run.runId, message);
                         if (terminalRun)
-                            this.scheduleRunReminder(parentPiboSessionId, false);
+                            this.handleTerminalRunReminder(parentPiboSessionId, terminalRun.runId, reminderGeneration);
+                    }
+                    finally {
+                        admission.release();
                     }
                 })();
                 return run;
@@ -731,9 +909,8 @@ export class PiboSessionRouter {
             waitForRun: (runId, timeoutMs) => this.runRegistry.wait(parentPiboSessionId, runId, timeoutMs),
             readRun: (runId) => {
                 const run = this.runRegistry.read(parentPiboSessionId, runId);
-                if (run.consumed && isTerminalRunStatus(run.status)) {
+                if (run.consumed && isTerminalRunStatus(run.status))
                     this.refreshQueuedRunReminders(parentPiboSessionId);
-                }
                 return run;
             },
             cancelRun: async (runId) => {
@@ -846,17 +1023,54 @@ export class PiboSessionRouter {
         }
         this.signalRegistry.project({ type: "run_changed", run: event.run, previousStatus: "previousStatus" in event ? event.previousStatus : undefined, reason: "reason" in event ? event.reason : event.type });
     }
-    scheduleRunReminder(piboSessionId, includeAlreadyNotified) {
+    runReminderGeneration(piboSessionId) {
+        return this.runReminderGenerations.get(piboSessionId) ?? 0;
+    }
+    invalidateRunReminders(piboSessionIds) {
+        for (const piboSessionId of piboSessionIds) {
+            this.runReminderGenerations.set(piboSessionId, this.runReminderGeneration(piboSessionId) + 1);
+            this.scheduledRunReminders.delete(piboSessionId);
+            try {
+                this.sessions.get(piboSessionId)?.removeQueuedMessages(isRunReminderServiceMessage);
+            }
+            catch {
+                // A concurrently disposed RoutedSession is already quiescent.
+            }
+            this.runRegistry.suppressControllerNotifications(piboSessionId);
+        }
+    }
+    beginSessionQuiescence(piboSessionIds) {
+        this.invalidateRunReminders(piboSessionIds);
+        for (const piboSessionId of piboSessionIds) {
+            this.quiescingSessions.add(piboSessionId);
+            this.clearIdleSessionTimer(piboSessionId);
+        }
+    }
+    handleTerminalRunReminder(piboSessionId, runId, generation) {
+        if (generation !== this.runReminderGeneration(piboSessionId) || this.quiescingSessions.has(piboSessionId) || this.closing) {
+            this.runRegistry.suppressNotification(piboSessionId, runId);
+            return;
+        }
+        this.scheduleRunReminder(piboSessionId, false, generation);
+    }
+    scheduleRunReminder(piboSessionId, includeAlreadyNotified, expectedGeneration = this.runReminderGeneration(piboSessionId)) {
+        if (this.closing || this.quiescingSessions.has(piboSessionId))
+            return;
+        if (expectedGeneration !== this.runReminderGeneration(piboSessionId))
+            return;
         if (!this.runRegistry.hasPendingNotification(piboSessionId, { includeAlreadyNotified }))
             return;
         const previous = this.scheduledRunReminders.get(piboSessionId);
-        if (previous !== undefined) {
-            this.scheduledRunReminders.set(piboSessionId, previous || includeAlreadyNotified);
+        if (previous?.generation === expectedGeneration) {
+            this.scheduledRunReminders.set(piboSessionId, {
+                generation: expectedGeneration,
+                includeAlreadyNotified: previous.includeAlreadyNotified || includeAlreadyNotified,
+            });
             return;
         }
-        this.scheduledRunReminders.set(piboSessionId, includeAlreadyNotified);
+        this.scheduledRunReminders.set(piboSessionId, { generation: expectedGeneration, includeAlreadyNotified });
         queueMicrotask(() => {
-            void this.deliverRunReminder(piboSessionId);
+            void this.deliverRunReminder(piboSessionId, expectedGeneration);
         });
     }
     refreshQueuedRunReminders(piboSessionId) {
@@ -864,23 +1078,32 @@ export class PiboSessionRouter {
         if (removed > 0)
             this.scheduleRunReminder(piboSessionId, true);
     }
-    async deliverRunReminder(piboSessionId) {
-        const includeAlreadyNotified = this.scheduledRunReminders.get(piboSessionId) ?? false;
+    async deliverRunReminder(piboSessionId, expectedGeneration) {
+        const scheduled = this.scheduledRunReminders.get(piboSessionId);
+        if (!scheduled || scheduled.generation !== expectedGeneration)
+            return;
         this.scheduledRunReminders.delete(piboSessionId);
-        const notification = this.runRegistry.createNotification(piboSessionId, { includeAlreadyNotified });
+        if (this.closing || this.quiescingSessions.has(piboSessionId) || expectedGeneration !== this.runReminderGeneration(piboSessionId))
+            return;
+        const notification = this.runRegistry.createNotification(piboSessionId, { includeAlreadyNotified: scheduled.includeAlreadyNotified });
         if (!notification)
             return;
         try {
             const session = await this.getOrCreateSession(piboSessionId);
+            if (this.closing || this.quiescingSessions.has(piboSessionId) || expectedGeneration !== this.runReminderGeneration(piboSessionId))
+                return;
             session.enqueueMessage({
                 type: "message",
                 piboSessionId,
                 text: formatRunReminderMessage(notification),
                 source: "service",
+                capabilityScope: "run-reminder",
                 id: randomUUID(),
             });
         }
         catch (error) {
+            if (this.closing || this.quiescingSessions.has(piboSessionId) || expectedGeneration !== this.runReminderGeneration(piboSessionId))
+                return;
             const message = error instanceof Error ? error.message : String(error);
             this.emitOutput({
                 type: "session_error",

package/dist/debug/index.js

@@ -1058,11 +1058,13 @@ Reports:
   Gateway RSS/heap headroom, host free-memory reserve, direct child processes, and known heavy local daemons such as ComfyUI or Unity when process listing is available.
 
 Environment:
-  PIBO_GATEWAY_RESOURCE_GUARD=warn|block|off
+  PIBO_GATEWAY_RESOURCE_GUARD=block|warn|off (default: block)
   PIBO_GATEWAY_MIN_FREE_MEMORY_BYTES=<bytes>
   PIBO_GATEWAY_MIN_HEAP_AVAILABLE_BYTES=<bytes>
   PIBO_GATEWAY_MAX_RSS_BYTES=<bytes>
   PIBO_GATEWAY_KNOWN_DAEMON_WARNING_RSS_BYTES=<bytes>
+  PIBO_GATEWAY_MAX_CONCURRENT_YIELDED_RUNS=<count> (default: 1)
+  PIBO_GATEWAY_YIELDED_RUN_MEMORY_RESERVATION_BYTES=<bytes> (default: 2147483648)
 
 Next:
   pibo debug resources --json

package/dist/gateway/server.js

@@ -1,6 +1,7 @@
 import { createServer } from "node:net";
 import { createDefaultPiboPluginRegistry, createPiboProfileFromRegistryOrDefault, resolvePiboProfileNameFromRegistryOrDefault } from "../plugins/builtin.js";
 import { PiboSessionRouter } from "../core/session-router.js";
+import { createLoopMessagePreflight } from "../loops/store.js";
 import { loadPiboModelDefaults, selectRequestedModelProfile } from "../core/model-defaults.js";
 import { ResourceReaperService } from "../resources/reaper.js";
 import { DEFAULT_GATEWAY_HOST, DEFAULT_GATEWAY_PORT, encodeFrame, errorResponse, isGatewayRequestFrame, isGatewaySubscribeFrame, } from "./protocol.js";
@@ -147,6 +148,7 @@ export class PiboGatewayServer {
             persistSession: this.options.persistSession,
             pluginRegistry: this.pluginRegistry,
             sessionStore: this.sessionStore,
+            messagePreflight: createLoopMessagePreflight({ path: this.options.loopStorePath }),
         });
         this.unsubscribe = this.router.subscribe((event) => this.broadcastRouterEvent(event));
         this.server = createServer((socket) => this.handleSocket(socket));

package/dist/gateway/web.js

@@ -197,6 +197,7 @@ export async function runWebGatewayServer(options = {}) {
             ...resolvedOptions,
             pluginRegistry,
             resourceReaper: resolveGatewayResourceReaperOptions(resolvedOptions),
+            loopStorePath: resolvedOptions.chat?.ralphStorePath,
         });
         await server.start();
     }

package/dist/loops/accounting.js

@@ -13,7 +13,14 @@ export function goalElapsedWallClockSeconds(job, now = new Date()) {
 export function goalRemainingTokens(job) {
     return job.tokenBudget === undefined ? undefined : Math.max(0, job.tokenBudget - (job.state.tokensUsed ?? 0));
 }
-export function goalCanStartNextTurn(job) {
+export function goalCanStartNextTurn(job, now = new Date()) {
+    if (job.mode !== 'goal' || !job.enabled)
+        return false;
+    const status = job.state.goalStatus ?? 'paused';
+    if (status !== 'active')
+        return false;
+    if (job.state.nextAttemptAt && Date.parse(job.state.nextAttemptAt) > now.getTime())
+        return false;
     const remaining = goalRemainingTokens(job);
     return remaining === undefined || remaining > (job.tokenReserve ?? 0);
 }

package/dist/loops/cli.js

@@ -207,7 +207,7 @@ export async function runLoopCli(argv = process.argv, defaults = {}) {
         printJson(job);
     else
         console.log(`${job.id}\tcancel-requested\t${job.name}`); store.close(); });
-    program.command('remove').argument('<id>').description('Delete a Loop job').option('--json', 'Print JSON').action((id, options) => { const store = createDefaultPiboLoopStore({ path: program.opts().store }); const removed = store.removeJob(id); if (options.json)
+    program.command('remove').argument('<id>').description('Delete a Loop job and all of its runs and facts; active runs must be cancelled first').option('--json', 'Print JSON').action((id, options) => { const store = createDefaultPiboLoopStore({ path: program.opts().store }); const removed = store.removeJob(id); if (options.json)
         printJson({ removed });
     else
         console.log(removed ? 'removed' : 'not found'); store.close(); });