n8n 2.23.2 → 2.24.0

package/dist/modules/instance-ai/instance-ai.service.js

@@ -73,6 +73,7 @@ const db_iteration_log_storage_1 = require("./storage/db-iteration-log-storage")
 const typeorm_agent_checkpoint_store_1 = require("./storage/typeorm-agent-checkpoint-store");
 const typeorm_agent_memory_1 = require("./storage/typeorm-agent-memory");
 const proxy_token_manager_1 = require("../../services/proxy-token-manager");
+const instance_ai_pending_confirmation_repository_1 = require("./repositories/instance-ai-pending-confirmation.repository");
 const instance_ai_thread_repository_1 = require("./repositories/instance-ai-thread.repository");
 const trace_replay_state_1 = require("./trace-replay-state");
 const liveness_1 = require("./liveness");
@@ -86,6 +87,7 @@ function isTelemetryConfigurableAgent(agent) {
         typeof Reflect.get(agent, 'telemetry') === 'function');
 }
 const INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS = 30 * 1000;
+const INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS = 5 * 1000;
 function isTextMessagePart(part) {
     return (typeof part === 'object' &&
         part !== null &&
@@ -331,7 +333,7 @@ let InstanceAiService = class InstanceAiService {
         }
         return this._mcpClientManager;
     }
-    constructor(logger, globalConfig, instanceSettings, adapterService, eventBus, settingsService, agentMemory, checkpointStore, aiService, push, threadRepo, urlService, dbSnapshotStorage, dbIterationLogStorage, sourceControlPreferencesService, telemetry, userRepository, aiBuilderTemporaryWorkflowRepository, errorReporter, ssrfProtectionConfig, ssrfProtectionService, eventService) {
+    constructor(logger, globalConfig, instanceSettings, adapterService, eventBus, settingsService, agentMemory, checkpointStore, aiService, push, threadRepo, pendingConfirmationRepo, urlService, dbSnapshotStorage, dbIterationLogStorage, sourceControlPreferencesService, telemetry, userRepository, aiBuilderTemporaryWorkflowRepository, errorReporter, ssrfProtectionConfig, ssrfProtectionService, eventService) {
         this.instanceSettings = instanceSettings;
         this.adapterService = adapterService;
         this.eventBus = eventBus;
@@ -341,6 +343,7 @@ let InstanceAiService = class InstanceAiService {
         this.aiService = aiService;
         this.push = push;
         this.threadRepo = threadRepo;
+        this.pendingConfirmationRepo = pendingConfirmationRepo;
         this.urlService = urlService;
         this.dbSnapshotStorage = dbSnapshotStorage;
         this.dbIterationLogStorage = dbIterationLogStorage;
@@ -364,6 +367,9 @@ let InstanceAiService = class InstanceAiService {
         this.creditedThreads = new Set();
         this.traceReplay = new trace_replay_state_1.TraceReplayState();
         this.checkpointPruningStopped = true;
+        this.inFlightExecutions = new Set();
+        this.preserveHitlOnShutdown = new Set();
+        this.userMessagePersistenceByRun = new Map();
         this.logger = logger.scoped('instance-ai');
         this.instanceAiConfig = globalConfig.instanceAi;
         const livenessPolicyConfig = (0, instance_ai_1.createInstanceAiLivenessPolicyConfig)({
@@ -379,6 +385,9 @@ let InstanceAiService = class InstanceAiService {
             finalizeCancelledSuspendedRun: (suspended, reason) => {
                 void this.finalizeCancelledSuspendedRun(suspended, reason);
             },
+            onPendingConfirmationRejected: (requestId) => {
+                void this.dropPendingConfirmation(requestId);
+            },
         });
         this.defaultTimeZone = globalConfig.generic.timezone;
         const restEndpoint = globalConfig.endpoints.rest;
@@ -1036,7 +1045,12 @@ let InstanceAiService = class InstanceAiService {
         if (pushRef !== undefined) {
             this.threadPushRef.set(threadId, pushRef);
         }
-        void this.executeRun(user, threadId, runId, message, abortController, attachments, messageGroupId, timeZone);
+        const userMessageId = (0, nanoid_1.nanoid)();
+        this.userMessagePersistenceByRun.set(runId, {
+            userId: user.id,
+            message: { id: userMessageId, text: message },
+        });
+        this.startExecuteRun(user, threadId, runId, message, abortController, attachments, messageGroupId, timeZone, false, undefined, undefined);
         return runId;
     }
     getMessageGroupId(threadId) {
@@ -1079,6 +1093,7 @@ let InstanceAiService = class InstanceAiService {
             if (reason === liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON)
                 this.liveness.markRunTimedOut(active.runId);
             active.abortController.abort();
+            void this.dropPendingConfirmationsForThread(threadId);
             return;
         }
         if (suspended) {
@@ -1087,6 +1102,7 @@ let InstanceAiService = class InstanceAiService {
             suspended.abortController.abort();
             void this.finalizeCancelledSuspendedRun(suspended, reason);
         }
+        void this.dropPendingConfirmationsForThread(threadId);
     }
     sendCorrectionToTask(threadId, taskId, correction) {
         return this.backgroundTasks.queueCorrection(threadId, taskId, correction);
@@ -1317,30 +1333,44 @@ let InstanceAiService = class InstanceAiService {
         this.deleteTraceContextsForThread(threadId);
         await this.destroySandbox(threadId);
         await this.reapAiTemporaryForThreadCleanup(threadId);
+        await this.dropPendingConfirmationsForThread(threadId);
         this.eventBus.clearThread(threadId);
     }
     async shutdown() {
         this.stopCheckpointPruning();
         this.liveness.shutdown();
-        const { activeRuns, suspendedRuns } = this.runState.shutdown();
+        const { activeRuns, suspendedRuns, pendingThreadIds } = this.runState.shutdown();
+        const threadsWithPendingHitl = new Set(pendingThreadIds);
         for (const run of activeRuns) {
-            run.abortController.abort();
+            if (threadsWithPendingHitl.has(run.threadId)) {
+                await this.finalizeRunTracing(run.runId, run.tracing, {
+                    status: 'cancelled',
+                    reason: 'service_shutdown',
+                });
+                this.preserveHitlOnShutdown.add(run.runId);
+                run.abortController.abort();
+                continue;
+            }
+            this.publishRunFinish(run.threadId, run.runId, 'cancelled', 'service_shutdown');
+            await this.persistShutdownSnapshot(run.threadId, run.runId, run.messageGroupId);
             await this.finalizeRunTracing(run.runId, run.tracing, {
                 status: 'cancelled',
                 reason: 'service_shutdown',
             });
+            run.abortController.abort();
         }
         for (const run of suspendedRuns) {
-            run.abortController.abort();
             await this.finalizeRunTracing(run.runId, run.tracing, {
                 status: 'cancelled',
                 reason: 'service_shutdown',
             });
+            run.abortController.abort();
         }
         for (const task of this.backgroundTasks.cancelAll()) {
             task.abortController.abort();
             await this.finalizeBackgroundTaskTracing(task, 'cancelled');
         }
+        await this.drainInFlightExecutions(INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS);
         const threadsWithTraces = new Set([...this.traceContextsByRunId.values()].map((entry) => entry.threadId));
         for (const threadId of threadsWithTraces) {
             await this.finalizeRemainingMessageTraceRoots(threadId, {
@@ -1376,28 +1406,168 @@ let InstanceAiService = class InstanceAiService {
         }, delayMs);
         this.checkpointPruneTimer.unref();
     }
+    trackInFlightExecution(promise) {
+        const tracked = promise.finally(() => {
+            this.inFlightExecutions.delete(tracked);
+        });
+        this.inFlightExecutions.add(tracked);
+    }
+    startExecuteRun(...args) {
+        this.trackInFlightExecution(this.executeRun(...args));
+    }
+    startProcessResumedStream(...args) {
+        this.trackInFlightExecution(this.processResumedStream(...args));
+    }
+    shouldPreserveHitlOnShutdown(runId) {
+        return this.preserveHitlOnShutdown.has(runId);
+    }
+    async persistUserMessageOnFirstSuspend(threadId, runId) {
+        const entry = this.userMessagePersistenceByRun.get(runId);
+        if (!entry)
+            return;
+        const ok = await this.persistUserMessageOnSuspend(threadId, entry.userId, entry.message);
+        if (ok)
+            this.userMessagePersistenceByRun.delete(runId);
+    }
+    async drainInFlightExecutions(timeoutMs) {
+        if (this.inFlightExecutions.size === 0)
+            return;
+        const drain = Promise.allSettled([...this.inFlightExecutions]);
+        const timeout = new Promise((resolve) => {
+            setTimeout(() => resolve('timeout'), timeoutMs).unref();
+        });
+        const outcome = await Promise.race([drain.then(() => 'drained'), timeout]);
+        if (outcome === 'timeout') {
+            this.logger.warn('Timed out waiting for in-flight Instance AI runs to drain', {
+                timeoutMs,
+                stillInFlight: this.inFlightExecutions.size,
+            });
+        }
+    }
     async pruneStaleCheckpoints(now = Date.now()) {
         const olderThan = new Date(now - this.instanceAiConfig.snapshotRetention);
         try {
-            const count = await this.checkpointStore.deleteOlderThan(olderThan);
+            const count = await this.checkpointStore.markExpiredOlderThan(olderThan);
             if (count > 0) {
-                this.logger.info('Deleted stale Instance AI checkpoints', { count });
+                this.logger.info('Expired stale Instance AI checkpoints', { count });
             }
             else {
-                this.logger.debug('No stale Instance AI checkpoints to delete');
+                this.logger.debug('No stale Instance AI checkpoints to expire');
             }
+            await this.pruneStalePendingConfirmations(now);
             this.scheduleCheckpointPrune();
         }
         catch (error) {
-            this.logger.warn('Failed to delete stale Instance AI checkpoints', {
+            this.logger.warn('Failed to expire stale Instance AI checkpoints', {
                 error: getErrorMessage(error),
             });
             this.scheduleCheckpointPrune(INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS);
         }
     }
+    async pruneStalePendingConfirmations(now) {
+        try {
+            const count = await this.pendingConfirmationRepo.deleteExpired(new Date(now));
+            if (count === 0) {
+                this.logger.debug('No stale Instance AI pending confirmations to drop');
+                return;
+            }
+            this.logger.info('Dropped stale Instance AI pending confirmations', { count });
+        }
+        catch (error) {
+            this.logger.warn('Failed to drop stale Instance AI pending confirmations', {
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    computePendingConfirmationExpiresAt() {
+        const timeoutMs = this.instanceAiConfig.confirmationTimeout;
+        return timeoutMs > 0 ? new Date(Date.now() + timeoutMs) : null;
+    }
+    async persistPendingConfirmation(params) {
+        try {
+            await this.pendingConfirmationRepo.save(this.pendingConfirmationRepo.create({
+                requestId: params.requestId,
+                threadId: params.threadId,
+                userId: params.userId,
+                kind: params.kind,
+                runId: params.runId,
+                messageGroupId: params.messageGroupId ?? null,
+                toolCallId: params.toolCallId ?? null,
+                checkpointKey: params.checkpointKey ?? null,
+                checkpointTaskId: params.checkpointTaskId ?? null,
+                expiresAt: this.computePendingConfirmationExpiresAt(),
+            }));
+        }
+        catch (error) {
+            this.logger.warn('Failed to persist pending confirmation', {
+                requestId: params.requestId,
+                threadId: params.threadId,
+                kind: params.kind,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async dropPendingConfirmation(requestId) {
+        try {
+            await this.pendingConfirmationRepo.deleteByRequestId(requestId);
+        }
+        catch (error) {
+            this.logger.warn('Failed to drop pending confirmation', {
+                requestId,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async dropPendingConfirmationsForThread(threadId) {
+        try {
+            await this.pendingConfirmationRepo.deleteByThreadId(threadId);
+        }
+        catch (error) {
+            this.logger.warn('Failed to drop pending confirmations for thread', {
+                threadId,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async persistUserMessageOnSuspend(threadId, userId, message) {
+        try {
+            await this.agentMemory.saveMessages({
+                threadId,
+                resourceId: userId,
+                messages: [
+                    {
+                        id: message.id,
+                        role: 'user',
+                        content: [{ type: 'text', text: message.text }],
+                        createdAt: new Date(),
+                    },
+                ],
+            });
+            return true;
+        }
+        catch (error) {
+            this.logger.warn('Failed to persist user message on HITL suspend', {
+                threadId,
+                userId,
+                error: getErrorMessage(error),
+            });
+            return false;
+        }
+    }
+    async persistShutdownSnapshot(threadId, runId, messageGroupId) {
+        try {
+            await this.saveAgentTreeSnapshot(threadId, runId, this.dbSnapshotStorage, true, messageGroupId);
+        }
+        catch (error) {
+            this.logger.warn('Failed to persist shutdown snapshot', {
+                threadId,
+                runId,
+                error: getErrorMessage(error),
+            });
+        }
+    }
     createAgentMemoryOptions() {
         return {
-            lastMessages: this.instanceAiConfig.lastMessages,
             observationalMemory: {
                 observerThresholdTokens: this.instanceAiConfig.observerMessageTokens,
                 reflectorThresholdTokens: this.instanceAiConfig.reflectorObservationTokens,
@@ -1545,6 +1715,7 @@ let InstanceAiService = class InstanceAiService {
     }
     async finishInvalidConfirmationRun(args) {
         this.runState.cancelThread(args.threadId);
+        void this.dropPendingConfirmationsForThread(args.threadId);
         args.abortController.abort();
         await this.finalizeRunTracing(args.runId, args.tracing, {
             status: 'error',
@@ -1932,9 +2103,6 @@ let InstanceAiService = class InstanceAiService {
             abortSignal,
             taskStorage,
             timeZone: this.defaultTimeZone,
-            browserMcpConfig: this.instanceAiConfig.browserMcp
-                ? { name: 'chrome-devtools', command: 'npx', args: ['-y', 'chrome-devtools-mcp@latest'] }
-                : undefined,
             localMcpServer: context.localMcpServer,
             runtimeSkills,
             runtimeSkillCatalog: baseRuntimeSkills,
@@ -1943,6 +2111,7 @@ let InstanceAiService = class InstanceAiService {
             formBaseUrl: this.formBaseUrl,
             waitForConfirmation: async (requestId) => {
                 this.runState.touchActiveRun(threadId);
+                void this.persistUserMessageOnFirstSuspend(threadId, runId);
                 return await new Promise((resolve) => {
                     this.runState.registerPendingConfirmation(requestId, {
                         resolve,
@@ -1950,6 +2119,14 @@ let InstanceAiService = class InstanceAiService {
                         userId: user.id,
                         createdAt: Date.now(),
                     });
+                    void this.persistPendingConfirmation({
+                        requestId,
+                        threadId,
+                        userId: user.id,
+                        runId,
+                        messageGroupId,
+                        kind: 'inline',
+                    });
                     queueMicrotask(() => {
                         void this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage);
                     });
@@ -1963,6 +2140,10 @@ let InstanceAiService = class InstanceAiService {
             schedulePlannedTasks: async () => await this.schedulePlannedTasks(user, threadId),
             iterationLog,
             sendCorrectionToTask: (taskId, correction) => this.sendCorrectionToTask(threadId, taskId, correction),
+            findSubAgentResumeInfo: async (agentKind) => await this.checkpointStore.findSuspendedSubAgentResumeInfo((0, instance_ai_1.createSubAgentResourceId)(threadId, agentKind)),
+            persistInFlightUserMessage: async () => {
+                await this.persistUserMessageOnFirstSuspend(threadId, runId);
+            },
             workflowTaskService: workflowTasks,
             workspace: runtimeWorkspace,
             nodeDefinitionDirs: nodeDefDirs.length > 0 ? nodeDefDirs : undefined,
@@ -2102,7 +2283,7 @@ let InstanceAiService = class InstanceAiService {
             : isReplanFollowUp
                 ? 'replan'
                 : 'background_task_completed';
-        void this.executeRun(user, threadId, runId, message, abortController, undefined, messageGroupId, timeZone, isReplanFollowUp, checkpoint, resumeReason);
+        this.startExecuteRun(user, threadId, runId, message, abortController, undefined, messageGroupId, timeZone, isReplanFollowUp, checkpoint, resumeReason);
         return runId;
     }
     async schedulePlannedTasks(user, threadId) {
@@ -2178,6 +2359,7 @@ let InstanceAiService = class InstanceAiService {
         await this.doSchedulePlannedTasks(activeUser, threadId);
     }
     async executeRun(user, threadId, runId, message, abortController, attachments, messageGroupId, timeZone, isReplanFollowUp = false, checkpoint, resumeReason) {
+        const userMessagePersistence = this.userMessagePersistenceByRun.get(runId)?.message;
         const signal = abortController.signal;
         let tracing;
         let messageTraceFinalization;
@@ -2326,18 +2508,20 @@ let InstanceAiService = class InstanceAiService {
                 : undefined;
             let streamInput;
             try {
-                if (nonStructuredAttachments.length > 0) {
+                if (nonStructuredAttachments.length > 0 || userMessagePersistence) {
+                    const baseContent = [
+                        { type: 'text', text: fullMessage },
+                        ...nonStructuredAttachments.map((attachment) => ({
+                            type: 'file',
+                            data: attachment.data,
+                            mediaType: attachment.mimeType,
+                        })),
+                    ];
                     streamInput = [
                         {
+                            ...(userMessagePersistence ? { id: userMessagePersistence.id } : {}),
                             role: 'user',
-                            content: [
-                                { type: 'text', text: fullMessage },
-                                ...nonStructuredAttachments.map((attachment) => ({
-                                    type: 'file',
-                                    data: attachment.data,
-                                    mediaType: attachment.mimeType,
-                                })),
-                            ],
+                            content: baseContent,
                         },
                     ];
                 }
@@ -2451,6 +2635,17 @@ let InstanceAiService = class InstanceAiService {
                         modelId,
                         checkpoint,
                     });
+                    void this.persistPendingConfirmation({
+                        requestId: result.suspension.requestId,
+                        threadId,
+                        userId: user.id,
+                        runId,
+                        messageGroupId,
+                        kind: 'suspended',
+                        toolCallId: result.suspension.toolCallId,
+                        checkpointKey: result.agentRunId,
+                        checkpointTaskId: checkpoint?.checkpointTaskId,
+                    });
                 }
                 const intermediateText = await (result.text ?? Promise.resolve(''));
                 if (intermediateText) {
@@ -2518,6 +2713,9 @@ let InstanceAiService = class InstanceAiService {
                 };
                 return;
             }
+            if (result.status === 'cancelled' && this.shouldPreserveHitlOnShutdown(runId)) {
+                return;
+            }
             const outputText = await (result.text ?? Promise.resolve(''));
             this.evaluateTerminalResponse(threadId, runId, result.status, {
                 messageGroupId,
@@ -2555,6 +2753,9 @@ let InstanceAiService = class InstanceAiService {
         }
         catch (error) {
             if (signal.aborted) {
+                if (this.shouldPreserveHitlOnShutdown(runId)) {
+                    return;
+                }
                 const runTimeout = this.liveness.consumeRunTimeout(runId);
                 const cancellationReason = runTimeout.timedOut
                     ? liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON
@@ -2624,6 +2825,7 @@ let InstanceAiService = class InstanceAiService {
         }
         finally {
             this.runState.clearActiveRun(threadId);
+            this.userMessagePersistenceByRun.delete(runId);
             this.domainAccessTrackersByThread.get(threadId)?.clearRun(runId);
             if (messageTraceFinalization) {
                 await this.maybeFinalizeRunTraceRoot(runId, messageTraceFinalization);
@@ -2764,6 +2966,7 @@ let InstanceAiService = class InstanceAiService {
             return false;
         }
         if (this.runState.resolvePendingConfirmation(freshUser.id, requestId, data)) {
+            void this.dropPendingConfirmation(requestId);
             this.logger.debug('Resolved pending confirmation (sub-agent HITL)', {
                 requestId,
                 approved: data.approved,
@@ -2774,7 +2977,157 @@ let InstanceAiService = class InstanceAiService {
             requestId,
             approved: data.approved,
         });
-        return await this.resumeSuspendedRun(requestingUserId, requestId, data);
+        if (await this.resumeSuspendedRun(requestingUserId, requestId, data)) {
+            return true;
+        }
+        return await this.resolveOrphanedConfirmation(requestingUserId, requestId, data);
+    }
+    async resolveOrphanedConfirmation(userId, requestId, data) {
+        let orphan;
+        try {
+            orphan = await this.pendingConfirmationRepo.claim(requestId, userId);
+        }
+        catch (error) {
+            this.logger.warn('Failed to claim orphaned pending confirmation', {
+                requestId,
+                error: getErrorMessage(error),
+            });
+            return false;
+        }
+        if (!orphan)
+            return false;
+        this.logger.info('Reclaiming pending confirmation orphaned by a process restart', {
+            requestId,
+            threadId: orphan.threadId,
+            runId: orphan.runId,
+            kind: orphan.kind,
+            hasCheckpoint: Boolean(orphan.checkpointKey),
+        });
+        if (orphan.kind === 'suspended' && this.canResumeOrphan(orphan)) {
+            const resumed = await this.tryResumeFromOrphan(orphan, data);
+            if (resumed)
+                return true;
+        }
+        this.finalizeUnresumableOrphan(orphan);
+        throw new n8n_workflow_1.UserError('This confirmation was lost when the assistant restarted. Send a new message to continue.');
+    }
+    canResumeOrphan(orphan) {
+        return Boolean(orphan.toolCallId && orphan.checkpointKey);
+    }
+    finalizeUnresumableOrphan(orphan) {
+        try {
+            this.publishRunFinish(orphan.threadId, orphan.runId, 'cancelled', 'restart_lost_confirmation');
+            void this.dbSnapshotStorage
+                .markRunCancelled(orphan.threadId, orphan.runId)
+                .catch((error) => {
+                this.logger.warn('Failed to mark orphan snapshot as cancelled', {
+                    requestId: orphan.requestId,
+                    threadId: orphan.threadId,
+                    runId: orphan.runId,
+                    error: getErrorMessage(error),
+                });
+            });
+        }
+        catch (error) {
+            this.logger.warn('Failed to finalize orphaned confirmation snapshot', {
+                requestId: orphan.requestId,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async tryResumeFromOrphan(orphan, data) {
+        const outcome = await this.rebuildSuspendedRunFromCheckpoint(orphan);
+        switch (outcome.kind) {
+            case 'ready':
+                this.runState.suspendRun(orphan.threadId, outcome.state);
+                return await this.resumeSuspendedRun(orphan.userId, orphan.requestId, data);
+            case 'no-user':
+                this.logger.warn('Cannot resume orphaned run: user no longer authorized', {
+                    requestId: orphan.requestId,
+                    userId: orphan.userId,
+                });
+                return false;
+            case 'no-checkpoint':
+                this.logger.warn('Cannot resume orphaned run: checkpoint missing or unavailable', {
+                    requestId: orphan.requestId,
+                    checkpointKey: orphan.checkpointKey,
+                    ...(outcome.error ? { error: getErrorMessage(outcome.error) } : {}),
+                });
+                return false;
+            case 'env-failure':
+                this.logger.warn('Cannot resume orphaned run: failed to build execution environment', {
+                    requestId: orphan.requestId,
+                    threadId: orphan.threadId,
+                    error: getErrorMessage(outcome.error),
+                });
+                return false;
+            case 'agent-failure':
+                this.logger.warn('Cannot resume orphaned run: failed to build agent', {
+                    requestId: orphan.requestId,
+                    threadId: orphan.threadId,
+                    error: getErrorMessage(outcome.error),
+                });
+                return false;
+        }
+    }
+    async rebuildSuspendedRunFromCheckpoint(orphan) {
+        const user = await this.revalidateActiveUser(orphan.userId);
+        if (!user)
+            return { kind: 'no-user' };
+        try {
+            const state = await this.checkpointStore.load(orphan.checkpointKey);
+            if (!state)
+                return { kind: 'no-checkpoint' };
+        }
+        catch (error) {
+            return { kind: 'no-checkpoint', error };
+        }
+        const abortController = new AbortController();
+        let environment;
+        try {
+            environment = await this.createExecutionEnvironment(user, orphan.threadId, orphan.runId, abortController.signal, orphan.messageGroupId ?? undefined, this.threadPushRef.get(orphan.threadId));
+        }
+        catch (error) {
+            return { kind: 'env-failure', error };
+        }
+        const mcpServers = this.parseMcpServers(this.instanceAiConfig.mcpServers);
+        let agent;
+        try {
+            agent = await (0, instance_ai_1.createInstanceAgent)({
+                modelId: environment.modelId,
+                context: environment.context,
+                orchestrationContext: environment.orchestrationContext,
+                mcpServers,
+                mcpManager: this.mcpClientManager,
+                memoryConfig: this.createAgentMemoryOptions(),
+                memory: environment.memory,
+                checkpointStore: this.checkpointStore,
+                timeZone: this.runState.getTimeZone(orphan.threadId) ?? this.defaultTimeZone,
+            });
+        }
+        catch (error) {
+            return { kind: 'agent-failure', error };
+        }
+        return {
+            kind: 'ready',
+            state: {
+                runId: orphan.runId,
+                agentRunId: orphan.checkpointKey,
+                agent,
+                threadId: orphan.threadId,
+                user,
+                toolCallId: orphan.toolCallId,
+                requestId: orphan.requestId,
+                abortController,
+                messageGroupId: orphan.messageGroupId ?? undefined,
+                createdAt: Date.now(),
+                tracing: undefined,
+                modelId: environment.modelId,
+                checkpoint: orphan.checkpointTaskId
+                    ? { isCheckpointFollowUp: true, checkpointTaskId: orphan.checkpointTaskId }
+                    : undefined,
+            },
+        };
     }
     async revalidateActiveUser(userId) {
         try {
@@ -2818,6 +3171,7 @@ let InstanceAiService = class InstanceAiService {
             return false;
         }
         this.runState.activateSuspendedRun(threadId);
+        void this.dropPendingConfirmation(requestId);
         const credentialsPayload = data.nodeCredentials ?? data.credentials;
         const resumeData = {
             approved: data.approved,
@@ -2854,7 +3208,7 @@ let InstanceAiService = class InstanceAiService {
                     : {}),
             },
         });
-        void this.processResumedStream(agent, resumeData, {
+        this.startProcessResumedStream(agent, resumeData, {
             runId,
             agentRunId,
             threadId,
@@ -2921,6 +3275,7 @@ let InstanceAiService = class InstanceAiService {
                 });
             if (result.status === 'suspended') {
                 if (result.suspension) {
+                    const resumeMessageGroupId = this.traceContextsByRunId.get(opts.runId)?.messageGroupId;
                     this.runState.suspendRun(opts.threadId, {
                         runId: opts.runId,
                         agentRunId: result.agentRunId,
@@ -2930,12 +3285,23 @@ let InstanceAiService = class InstanceAiService {
                         toolCallId: result.suspension.toolCallId,
                         requestId: result.suspension.requestId,
                         abortController: opts.abortController,
-                        messageGroupId: this.traceContextsByRunId.get(opts.runId)?.messageGroupId,
+                        messageGroupId: resumeMessageGroupId,
                         createdAt: Date.now(),
                         tracing: opts.tracing,
                         ...(opts.modelId !== undefined ? { modelId: opts.modelId } : {}),
                         checkpoint: opts.checkpoint,
                     });
+                    void this.persistPendingConfirmation({
+                        requestId: result.suspension.requestId,
+                        threadId: opts.threadId,
+                        userId: opts.user.id,
+                        runId: opts.runId,
+                        messageGroupId: resumeMessageGroupId,
+                        kind: 'suspended',
+                        toolCallId: result.suspension.toolCallId,
+                        checkpointKey: result.agentRunId,
+                        checkpointTaskId: opts.checkpoint?.checkpointTaskId,
+                    });
                 }
                 const intermediateText = await (result.text ?? Promise.resolve(''));
                 if (intermediateText) {
@@ -3001,6 +3367,9 @@ let InstanceAiService = class InstanceAiService {
                 };
                 return;
             }
+            if (result.status === 'cancelled' && this.shouldPreserveHitlOnShutdown(opts.runId)) {
+                return;
+            }
             const outputText = await (result.text ?? Promise.resolve(''));
             const messageGroupId = this.traceContextsByRunId.get(opts.runId)?.messageGroupId;
             this.evaluateTerminalResponse(opts.threadId, opts.runId, result.status, {
@@ -3036,6 +3405,9 @@ let InstanceAiService = class InstanceAiService {
         }
         catch (error) {
             if (opts.signal.aborted) {
+                if (this.shouldPreserveHitlOnShutdown(opts.runId)) {
+                    return;
+                }
                 const messageGroupId = this.traceContextsByRunId.get(opts.runId)?.messageGroupId;
                 const runTimeout = this.liveness.consumeRunTimeout(opts.runId);
                 const cancellationReason = runTimeout.timedOut
@@ -3388,6 +3760,7 @@ let InstanceAiService = class InstanceAiService {
                 ...(runTimeout ? { runTimeout } : {}),
             }),
         });
+        void this.dropPendingConfirmation(suspended.requestId);
     }
     async reapAiTemporaryForThreadCleanup(threadId) {
         let markedWorkflows;
@@ -3656,6 +4029,7 @@ exports.InstanceAiService = InstanceAiService = __decorate([
         ai_service_1.AiService,
         push_1.Push,
         instance_ai_thread_repository_1.InstanceAiThreadRepository,
+        instance_ai_pending_confirmation_repository_1.InstanceAiPendingConfirmationRepository,
         url_service_1.UrlService,
         db_snapshot_storage_1.DbSnapshotStorage,
         db_iteration_log_storage_1.DbIterationLogStorage,