n8n 2.29.7 → 2.30.0

package/dist/modules/instance-ai/instance-ai.service.js

@@ -43,6 +43,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InstanceAiService = void 0;
+exports.getUserFacingErrorMessage = getUserFacingErrorMessage;
 const api_types_1 = require("@n8n/api-types");
 const backend_common_1 = require("@n8n/backend-common");
 const backend_network_1 = require("@n8n/backend-network");
@@ -58,6 +59,7 @@ const nanoid_1 = require("nanoid");
 const constants_1 = require("../../constants");
 const event_service_1 = require("../../events/event.service");
 const source_control_preferences_service_ee_1 = require("../../modules/source-control.ee/source-control-preferences.service.ee");
+const publisher_service_1 = require("../../scaling/pubsub/publisher.service");
 const ai_service_1 = require("../../services/ai.service");
 const proxy_token_manager_1 = require("../../services/proxy-token-manager");
 const url_service_1 = require("../../services/url.service");
@@ -78,6 +80,7 @@ const instance_ai_adapter_service_1 = require("./instance-ai.adapter.service");
 const internal_messages_1 = require("./internal-messages");
 const liveness_1 = require("./liveness");
 const mcp_1 = require("./mcp");
+const instance_ai_error_reporter_service_1 = require("./instance-ai-error-reporter.service");
 const observability_1 = require("./observability");
 const output_redaction_config_1 = require("./output-redaction-config");
 const planned_task_action_runner_1 = require("./planned-task-action-runner");
@@ -141,6 +144,7 @@ function isTelemetryConfigurableAgent(agent) {
 }
 const INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS = 30 * 1000;
 const WORKFLOW_SETUP_ROUTING_CLAIM_TTL_MS = 15 * 60 * 1000;
+const CONFIRMATION_EXPIRED_MESSAGE = 'This confirmation has expired. Send a new message to continue.';
 const INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS = 5 * 1000;
 function isTextMessagePart(part) {
     return (typeof part === 'object' &&
@@ -150,10 +154,16 @@ function isTextMessagePart(part) {
         'text' in part &&
         typeof part.text === 'string');
 }
+function isSandboxEndpointNotAllowedError(error) {
+    return getErrorMessage(error).toLowerCase().includes('endpoint not allowed');
+}
 function getUserFacingErrorMessage(error) {
     if (error instanceof n8n_workflow_1.UserError) {
         return error.message;
     }
+    if (isSandboxEndpointNotAllowedError(error)) {
+        return "I couldn't finish preparing the workspace sandbox. Please try again in a moment.";
+    }
     if (error instanceof n8n_workflow_1.OperationalError) {
         return 'I hit an operational error before I could finish that response. Please try again.';
     }
@@ -178,6 +188,7 @@ function getAbortReason(signal) {
     return typeof reason === 'string' ? reason : 'user_cancelled';
 }
 const MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD = 5;
+const TITLE_REFINE_HISTORY_LIMIT = 50;
 function stringifyForContextValue(value) {
     if (typeof value === 'string')
         return value;
@@ -241,6 +252,8 @@ function toConfirmationData(request) {
             return { approved: true, answers: request.answers };
         case 'credentialSelection':
             return { approved: true, credentials: request.credentials };
+        case 'credentialAutoSetup':
+            return { approved: true, autoSetup: { credentialType: request.credentialType } };
         case 'resourceDecision':
             return { approved: true, resourceDecision: request.resourceDecision };
         case 'setupWorkflowApply':
@@ -263,11 +276,11 @@ function toConfirmationData(request) {
 let InstanceAiService = class InstanceAiService {
     get mcpClientManager() {
         if (!this._mcpClientManager) {
-            this._mcpClientManager = new instance_ai_1.McpClientManager(this._ssrfProtectionConfig.enabled ? this._ssrfProtectionService : undefined);
+            this._mcpClientManager = new instance_ai_1.McpClientManager(this._ssrfProtectionConfig.enabled ? this._ssrfProtectionService : undefined, { onToolCallSettled: (event) => this.trackMcpToolCall(event) });
         }
         return this._mcpClientManager;
     }
-    constructor(logger, globalConfig, instanceSettings, adapterService, eventBus, settingsService, gatewayService, browserSessionService, memoryService, agentMemory, checkpointStore, aiService, threadGrantRepo, pendingConfirmationRepo, urlService, dbSnapshotStorage, dbIterationLogStorage, sourceControlPreferencesService, telemetry, mcpRegistryService, userRepository, temporaryWorkflowService, errorReporter, ssrfProtectionConfig, ssrfProtectionService, eventService, evalCredentialAllowlists, runProbe, modelService, creditService) {
+    constructor(logger, globalConfig, instanceSettings, adapterService, eventBus, settingsService, gatewayService, browserSessionService, memoryService, agentMemory, checkpointStore, aiService, threadGrantRepo, pendingConfirmationRepo, urlService, dbSnapshotStorage, dbIterationLogStorage, sourceControlPreferencesService, telemetry, mcpRegistryService, userRepository, temporaryWorkflowService, errorReporter, ssrfProtectionConfig, ssrfProtectionService, eventService, evalCredentialAllowlists, runProbe, modelService, creditService, publisher, instanceAiErrorReporter) {
         this.instanceSettings = instanceSettings;
         this.adapterService = adapterService;
         this.eventBus = eventBus;
@@ -293,10 +306,11 @@ let InstanceAiService = class InstanceAiService {
         this.evalCredentialAllowlists = evalCredentialAllowlists;
         this.modelService = modelService;
         this.creditService = creditService;
+        this.publisher = publisher;
+        this.instanceAiErrorReporter = instanceAiErrorReporter;
         this.runState = new instance_ai_1.RunStateRegistry();
         this.backgroundTasks = new instance_ai_1.BackgroundTaskManager(MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD);
         this.memoryTaskRegistry = new instance_ai_1.MemoryTaskRegistry();
-        this.reportedErrors = new WeakSet();
         this.domainAccessTrackersByThread = new Map();
         this.threadPushRef = new Map();
         this.planRequestsByThread = new Map();
@@ -497,39 +511,13 @@ let InstanceAiService = class InstanceAiService {
         agent.on("error", (event) => {
             if (event.type !== "error" || !event.source)
                 return;
-            this.reportInstanceAiError(event.error, {
+            this.instanceAiErrorReporter.report(event.error, {
                 component: `instance-ai-${event.source}`,
                 threadId,
                 runId,
             });
         });
     }
-    reportInstanceAiError(error, context) {
-        if (typeof error === 'object' && error !== null) {
-            if (this.reportedErrors.has(error))
-                return;
-            this.reportedErrors.add(error);
-        }
-        const observability = (0, observability_1.buildInstanceAiObservabilityContext)(context);
-        this.logger.error(`Instance AI error in ${context.component}`, {
-            error,
-            component: context.component,
-            ...observability,
-        });
-        this.errorReporter.error(error, {
-            tags: { component: context.component, ...observability },
-            extra: observability,
-        });
-    }
-    async withSetupBoundary(component, context, fn) {
-        try {
-            return await fn();
-        }
-        catch (error) {
-            this.reportInstanceAiError(error, { component, ...context });
-            throw error;
-        }
-    }
     isRunDebugEnabled() {
         return this.instanceAiConfig.runDebugEnabled;
     }
@@ -677,6 +665,70 @@ let InstanceAiService = class InstanceAiService {
             void this.handlePlannedTaskSettlement(user, task, 'cancelled');
         }
     }
+    broadcastTaskControl(payload) {
+        if (!this.instanceSettings.isMultiMain)
+            return;
+        void this.publisher
+            .publishCommand({ command: 'relay-instance-ai-task-control', payload })
+            .catch((error) => this.logger.error('Failed to relay Instance AI task-control to sibling mains', {
+            threadId: payload.threadId,
+            action: payload.action,
+            error,
+        }));
+    }
+    async applyTaskControlLocally({ threadId, taskId, action, correction, }) {
+        switch (action) {
+            case 'correct':
+                if (!taskId || correction === undefined)
+                    return true;
+                return this.sendCorrectionToTask(threadId, taskId, correction) !== 'task-not-found';
+            case 'cancel-task': {
+                if (!taskId)
+                    return true;
+                const isLocal = this.backgroundTasks
+                    .getTaskSnapshots(threadId)
+                    .some((task) => task.taskId === taskId);
+                this.cancelBackgroundTask(threadId, taskId);
+                return isLocal;
+            }
+            case 'cancel-thread':
+                this.cancelRun(threadId);
+                return false;
+            case 'clear-thread':
+                await this.clearThreadState(threadId);
+                return false;
+        }
+    }
+    async routeTaskControl(payload) {
+        const foundLocally = await this.applyTaskControlLocally(payload);
+        if (!foundLocally)
+            this.broadcastTaskControl(payload);
+    }
+    async routeCorrectionToTask(threadId, taskId, correction) {
+        await this.routeTaskControl({ threadId, taskId, action: 'correct', correction });
+    }
+    async routeCancelBackgroundTask(threadId, taskId) {
+        await this.routeTaskControl({ threadId, taskId, action: 'cancel-task' });
+    }
+    async routeCancelRun(threadId) {
+        await this.routeTaskControl({ threadId, action: 'cancel-thread' });
+    }
+    async routeClearThreadState(threadId) {
+        await this.routeTaskControl({ threadId, action: 'clear-thread' });
+    }
+    async handleRelayTaskControl(payload) {
+        try {
+            await this.applyTaskControlLocally(payload);
+        }
+        catch (error) {
+            this.logger.error('Failed to apply relayed Instance AI task-control', {
+                threadId: payload.threadId,
+                taskId: payload.taskId,
+                action: payload.action,
+                error,
+            });
+        }
+    }
     cancelAllBackgroundTasks() {
         const cancelled = this.backgroundTasks.cancelAll();
         for (const task of cancelled) {
@@ -880,6 +932,7 @@ let InstanceAiService = class InstanceAiService {
             await this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
         }
         await this.drainInFlightExecutions(INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS);
+        this.instanceAiErrorReporter.endAllRuns();
         const threadsWithTraces = new Set(this.tracing.getTrackedThreadIds());
         for (const threadId of threadsWithTraces) {
             await this.tracing.finalizeRemainingMessageTraceRoots(threadId, {
@@ -956,6 +1009,7 @@ let InstanceAiService = class InstanceAiService {
             else {
                 this.logger.debug('No stale Instance AI checkpoints to expire');
             }
+            await this.hardDeleteExpiredCheckpoints(now);
             await this.suspendedThreads.pruneStalePendingConfirmations(now);
             await this.pruneExpiredThreads();
             this.scheduleCheckpointPrune();
@@ -967,6 +1021,23 @@ let InstanceAiService = class InstanceAiService {
             this.scheduleCheckpointPrune(INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS);
         }
     }
+    async hardDeleteExpiredCheckpoints(now) {
+        const retention = this.instanceAiConfig.checkpointGcRetention;
+        if (retention <= 0)
+            return;
+        try {
+            const olderThan = new Date(now - retention);
+            const count = await this.checkpointStore.hardDeleteExpiredOlderThan(olderThan);
+            if (count > 0) {
+                this.logger.info('Hard-deleted expired Instance AI checkpoint tombstones', { count });
+            }
+        }
+        catch (error) {
+            this.logger.warn('Failed to hard-delete expired Instance AI checkpoint tombstones', {
+                error: getErrorMessage(error),
+            });
+        }
+    }
     async pruneExpiredThreads() {
         try {
             await this.memoryService.cleanupExpiredThreads(async (threadId) => await this.clearThreadState(threadId));
@@ -1184,19 +1255,20 @@ let InstanceAiService = class InstanceAiService {
             context.permissions = (0, api_types_1.applyBranchReadOnlyOverrides)(context.permissions);
             context.branchReadOnly = true;
         }
-        let domainTracker = this.domainAccessTrackersByThread.get(threadId);
-        if (!domainTracker) {
-            domainTracker = (0, instance_ai_1.createDomainAccessTracker)();
-            this.domainAccessTrackersByThread.set(threadId, domainTracker);
-        }
-        context.domainAccessTracker = domainTracker;
         context.runId = runId;
         const sessionGrants = await this.loadThreadSessionGrants(threadId, user.id);
         context.sessionApprovedToolKeys = sessionGrants;
-        context.grantSessionToolApproval = async (key) => {
+        const grantSessionToolApproval = async (key) => {
             await this.persistThreadSessionGrant(threadId, user.id, key);
             sessionGrants.add(key);
         };
+        context.grantSessionToolApproval = grantSessionToolApproval;
+        const domainTracker = (0, instance_ai_1.createDomainAccessTracker)({
+            grantedKeys: sessionGrants,
+            persistGrant: grantSessionToolApproval,
+        });
+        this.domainAccessTrackersByThread.set(threadId, domainTracker);
+        context.domainAccessTracker = domainTracker;
         if (this.isRunDebugEnabled()) {
             context.recordWorkflowCodeSnapshot = (snapshot) => {
                 this.runDebugBuffer.ensure(runId, threadId);
@@ -1253,7 +1325,7 @@ let InstanceAiService = class InstanceAiService {
         let workspaceRoot;
         const sandboxStatus = this.settingsService.getSandboxStatus();
         if (sandboxStatus.workflowBuilderAvailable) {
-            const sandboxConfig = await this.withSetupBoundary('instance-ai-sandbox-setup', { threadId, runId, userId: user.id, messageGroupId }, async () => await this.sandboxService.resolveSandboxConfig(user));
+            const sandboxConfig = await this.instanceAiErrorReporter.withBoundary('instance-ai-sandbox-setup', { threadId, runId, userId: user.id, messageGroupId }, async () => await this.sandboxService.resolveSandboxConfig(user));
             if (sandboxConfig.enabled) {
                 workspaceRoot = (0, instance_ai_1.getPromptWorkspaceRoot)(sandboxConfig.provider);
                 let sandboxEntryPromise;
@@ -1822,6 +1894,31 @@ let InstanceAiService = class InstanceAiService {
                 return;
         }
     }
+    async persistInterruptedUserMessage(threadId, userId, message, createdAt) {
+        if (!message)
+            return;
+        try {
+            await this.agentMemory.saveMessages({
+                threadId,
+                resourceId: userId,
+                messages: [
+                    {
+                        id: (0, nanoid_1.nanoid)(),
+                        createdAt,
+                        type: 'llm',
+                        role: 'user',
+                        content: [{ type: 'text', text: message }],
+                    },
+                ],
+            });
+        }
+        catch (error) {
+            this.logger.warn('Failed to persist user message on cancel', {
+                threadId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
     async executeRun(user, threadId, runId, message, abortController, attachments, handoffContext, messageGroupId, timeZone, isReplanFollowUp = false, checkpoint, resumeReason, plannedBuild) {
         const fileAttachments = (attachments ?? []).filter((attachment) => attachment.type === 'file');
         const workflowAttachments = (attachments ?? []).filter((attachment) => attachment.type === 'workflow');
@@ -1831,7 +1928,10 @@ let InstanceAiService = class InstanceAiService {
         let aiCreatedWorkflowIds;
         let activeSnapshotStorage;
         let messageId = '';
+        let streamReached = false;
+        const turnStartedAt = new Date();
         try {
+            this.instanceAiErrorReporter.beginRun(runId);
             messageId = (0, nanoid_1.nanoid)();
             const traceInput = {
                 message,
@@ -1890,6 +1990,7 @@ let InstanceAiService = class InstanceAiService {
                 payload: { messageId, messageGroupId, ...(traceId ? { traceId } : {}) },
             });
             if (signal.aborted) {
+                await this.persistInterruptedUserMessage(threadId, user.id, message, turnStartedAt);
                 this.terminalOutcome.evaluateTerminalResponse(threadId, runId, 'cancelled', {
                     messageGroupId,
                     correlationId: messageId,
@@ -1902,19 +2003,6 @@ let InstanceAiService = class InstanceAiService {
                 });
                 return;
             }
-            const staticMcpServers = this.parseMcpServers(this.instanceAiConfig.mcpServers);
-            const registryMcpServers = this.settingsService.isMcpAccessEnabled()
-                ? await this.withSetupBoundary('instance-ai-mcp-setup', {
-                    threadId,
-                    runId,
-                    tracing,
-                    agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
-                    userId: user.id,
-                    messageGroupId,
-                    messageId,
-                }, async () => await this.mcpRegistryService.getRegistryMcpServers(user))
-                : [];
-            const mcpServers = [...staticMcpServers, ...registryMcpServers];
             const executionPushRef = this.threadPushRef.get(threadId);
             const environment = await this.createExecutionEnvironment(user, threadId, runId, signal, messageGroupId, executionPushRef, proxyRunConfig);
             activeSnapshotStorage = environment.snapshotStorage;
@@ -1968,7 +2056,6 @@ let InstanceAiService = class InstanceAiService {
             if (fileAttachments.length > 0) {
                 context.currentUserAttachments = fileAttachments;
             }
-            const memoryConfig = this.createAgentMemoryOptions();
             if (!tracing && process.env.E2E_TESTS === 'true') {
                 const { createTraceReplayOnlyContext } = await Promise.resolve().then(() => __importStar(require('@n8n/instance-ai')));
                 tracing = createTraceReplayOnlyContext();
@@ -2097,20 +2184,9 @@ let InstanceAiService = class InstanceAiService {
             }
             const runControl = (0, instance_ai_1.createOrchestratorRunControl)(orchestrationContext);
             const stopSignal = () => runControl.getStopSignal();
-            const agent = await (0, instance_ai_1.createInstanceAgent)({
-                modelId,
-                context,
-                orchestrationContext,
-                mcpServers,
-                mcpManager: this.mcpClientManager,
-                memoryConfig,
-                memory,
-                checkpointStore: this.checkpointStore,
-                onMemoryTaskEvent: this.memoryTaskObserverFor(threadId),
-                thinkingEnabled: this.instanceAiConfig.thinkingEnabled,
-            });
-            this.subscribeToAgentErrors(agent, threadId, runId);
+            const agent = await this.createAgentFromEnvironment(environment, threadId, runId, user, tracing);
             const streamOptions = this.buildOrchestratorAgentStreamOptions(user, threadId, runId, signal);
+            streamReached = true;
             const result = tracing
                 ? await tracing.withActiveSpan(tracing.actorRun, async () => {
                     return await (0, instance_ai_1.streamAgentRun)(agent, streamInput, streamOptions, {
@@ -2137,6 +2213,11 @@ let InstanceAiService = class InstanceAiService {
                     outputRedaction: (0, output_redaction_config_1.resolveOutputRedaction)(this.instanceAiConfig),
                 });
             if (result.status === 'suspended') {
+                this.emitRunMetrics(threadId, 'suspended', {
+                    modelId,
+                    workSummary: result.workSummary,
+                    usage: result.usage,
+                });
                 if (result.suspension) {
                     this.runState.suspendRun(threadId, {
                         runId,
@@ -2243,7 +2324,7 @@ let InstanceAiService = class InstanceAiService {
             }
             const outputText = await (result.text ?? Promise.resolve(''));
             if (result.status === 'errored') {
-                this.reportInstanceAiError(result.error ?? new Error('Instance AI stream errored'), {
+                this.instanceAiErrorReporter.report(result.error ?? new Error('Instance AI stream errored'), {
                     component: 'instance-ai-stream',
                     threadId,
                     runId,
@@ -2254,11 +2335,13 @@ let InstanceAiService = class InstanceAiService {
                     messageId,
                 });
             }
+            const userFacingErrorMessage = result.status === 'errored' ? getUserFacingErrorMessage(result.error) : undefined;
             if (runControl.shouldEmitTerminalOutcome(result.stopReason)) {
                 this.terminalOutcome.evaluateTerminalResponse(threadId, runId, result.status, {
                     messageGroupId,
                     correlationId: messageId,
                     workSummary: result.workSummary,
+                    errorMessage: userFacingErrorMessage,
                     suppressCompletedFallback: checkpoint?.isCheckpointFollowUp === true ||
                         plannedBuild?.isPlannedBuildFollowUp === true,
                 });
@@ -2282,6 +2365,7 @@ let InstanceAiService = class InstanceAiService {
                 archivedWorkflowIds,
                 workSummary: result.workSummary,
                 usage: result.usage,
+                errorReason: userFacingErrorMessage,
             });
             await this.creditService.claimRunUsage(user, threadId, result.agentRunId || runId, result.usage?.usage ?? [], result.status);
             if (result.status === 'completed') {
@@ -2299,6 +2383,9 @@ let InstanceAiService = class InstanceAiService {
                 if (this.shouldPreserveHitlOnShutdown(runId)) {
                     return;
                 }
+                if (!streamReached) {
+                    await this.persistInterruptedUserMessage(threadId, user.id, message, turnStartedAt);
+                }
                 const runTimeout = this.liveness.consumeRunTimeout(runId);
                 const cancellationReason = runTimeout.timedOut
                     ? liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON
@@ -2345,7 +2432,7 @@ let InstanceAiService = class InstanceAiService {
                 error: errorMessage,
                 ...(0, observability_1.buildInstanceAiObservabilityContext)(errCtx),
             });
-            this.reportInstanceAiError(error, { component: 'instance-ai-run', ...errCtx });
+            this.instanceAiErrorReporter.report(error, { component: 'instance-ai-run', ...errCtx });
             this.terminalOutcome.evaluateTerminalResponse(threadId, runId, 'errored', {
                 messageGroupId,
                 correlationId: messageId,
@@ -2398,6 +2485,7 @@ let InstanceAiService = class InstanceAiService {
                 await this.taskProjector.syncFromWorkflowLoop(threadId, runId);
                 await this.maybeStartWorkflowSetupFollowUp(user, threadId);
             }
+            this.instanceAiErrorReporter.endRun(runId);
         }
     }
     queuePendingCheckpointReentry(threadId, checkpointTaskId) {
@@ -2555,25 +2643,80 @@ let InstanceAiService = class InstanceAiService {
                 userId: requestingUserId,
                 requestId,
             });
-            return false;
+            return null;
         }
-        if (this.runState.resolvePendingConfirmation(freshUser.id, requestId, data)) {
+        if (await this.pendingConfirmationRepo.isPastExpiry(requestId, freshUser.id, new Date())) {
+            this.logger.debug('Rejecting expired confirmation', { requestId });
+            throw new n8n_workflow_1.UserError(CONFIRMATION_EXPIRED_MESSAGE);
+        }
+        const pending = this.runState.getPendingConfirmation(requestId);
+        if (pending &&
+            pending.userId === freshUser.id &&
+            this.runState.resolvePendingConfirmation(freshUser.id, requestId, data)) {
             void this.suspendedThreads.dropPendingConfirmation(requestId);
             this.logger.debug('Resolved pending confirmation (sub-agent HITL)', {
                 requestId,
                 approved: data.approved,
             });
-            return true;
+            const runId = this.runState.getActiveRunId(pending.threadId);
+            return {
+                ok: true,
+                ...(runId ? { runId } : {}),
+            };
         }
         this.logger.debug('Pending confirmation not found, trying suspended run resume', {
             requestId,
             approved: data.approved,
         });
-        if (await this.resumeSuspendedRun(requestingUserId, requestId, data)) {
-            return true;
+        const resumed = await this.resumeSuspendedRun(requestingUserId, requestId, data);
+        if (resumed) {
+            return resumed;
         }
         return await this.suspendedRunRestorer.resolveOrphanedConfirmation(requestingUserId, requestId, data);
     }
+    async buildMcpServers(user, threadId, runId, tracing, messageGroupId) {
+        const staticMcpServers = this.parseMcpServers(this.instanceAiConfig.mcpServers);
+        const registryMcpServers = this.settingsService.isMcpAccessEnabled()
+            ? await this.instanceAiErrorReporter.withBoundary('instance-ai-mcp-setup', {
+                threadId,
+                runId,
+                tracing,
+                agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
+                userId: user.id,
+                messageGroupId,
+            }, async () => await this.mcpRegistryService.getRegistryMcpServers(user))
+            : [];
+        return [...staticMcpServers, ...registryMcpServers];
+    }
+    async createAgentFromEnvironment(environment, threadId, runId, user, tracing) {
+        if (tracing) {
+            environment.orchestrationContext.tracing = tracing;
+        }
+        const mcpServers = await this.buildMcpServers(user, threadId, runId, tracing, environment.orchestrationContext.messageGroupId);
+        const agent = await (0, instance_ai_1.createInstanceAgent)({
+            modelId: environment.modelId,
+            context: environment.context,
+            orchestrationContext: environment.orchestrationContext,
+            mcpServers,
+            mcpManager: this.mcpClientManager,
+            memoryConfig: this.createAgentMemoryOptions(),
+            memory: environment.memory,
+            checkpointStore: this.checkpointStore,
+            onMemoryTaskEvent: this.memoryTaskObserverFor(threadId),
+            thinkingEnabled: this.instanceAiConfig.thinkingEnabled,
+        });
+        this.subscribeToAgentErrors(agent, threadId, runId);
+        return agent;
+    }
+    async buildFreshInstanceAgent(user, threadId, runId, abortSignal, tracing, messageGroupId, pushRef) {
+        const environment = await this.createExecutionEnvironment(user, threadId, runId, abortSignal, messageGroupId, pushRef);
+        const agent = await this.createAgentFromEnvironment(environment, threadId, runId, user, tracing);
+        return {
+            agent,
+            modelId: environment.modelId,
+            orchestrationContext: environment.orchestrationContext,
+        };
+    }
     async rebuildSuspendedRunFromCheckpoint(orphan) {
         const user = await this.revalidateActiveUser(orphan.userId);
         if (!user)
@@ -2594,23 +2737,10 @@ let InstanceAiService = class InstanceAiService {
         catch (error) {
             return { kind: 'env-failure', error };
         }
-        const mcpServers = this.parseMcpServers(this.instanceAiConfig.mcpServers);
         const runControl = (0, instance_ai_1.createOrchestratorRunControl)(environment.orchestrationContext);
         let agent;
         try {
-            agent = await (0, instance_ai_1.createInstanceAgent)({
-                modelId: environment.modelId,
-                context: environment.context,
-                orchestrationContext: environment.orchestrationContext,
-                mcpServers,
-                mcpManager: this.mcpClientManager,
-                memoryConfig: this.createAgentMemoryOptions(),
-                memory: environment.memory,
-                checkpointStore: this.checkpointStore,
-                onMemoryTaskEvent: this.memoryTaskObserverFor(orphan.threadId),
-                thinkingEnabled: this.instanceAiConfig.thinkingEnabled,
-            });
-            this.subscribeToAgentErrors(agent, orphan.threadId, orphan.runId);
+            agent = await this.createAgentFromEnvironment(environment, orphan.threadId, orphan.runId, user, undefined);
         }
         catch (error) {
             return { kind: 'agent-failure', error };
@@ -2656,6 +2786,21 @@ let InstanceAiService = class InstanceAiService {
             return null;
         }
     }
+    async rebuildAgentForAutoSetupResume(user, threadId, runId, abortController, tracing, runHandoff, messageGroupId) {
+        try {
+            const rebuilt = await this.buildFreshInstanceAgent(user, threadId, runId, abortController.signal, tracing, messageGroupId, this.threadPushRef.get(threadId));
+            (0, instance_ai_1.createOrchestratorRunControl)(rebuilt.orchestrationContext, runHandoff ?? {});
+            return { agent: rebuilt.agent, modelId: rebuilt.modelId };
+        }
+        catch (error) {
+            this.logger.warn('Failed to rebuild agent for credential auto-setup resume', {
+                threadId,
+                runId,
+                error: getErrorMessage(error),
+            });
+            return undefined;
+        }
+    }
     async resumeSuspendedRun(requestingUserId, requestId, data) {
         const suspended = this.runState.findSuspendedByRequestId(requestId);
         if (!suspended) {
@@ -2663,11 +2808,11 @@ let InstanceAiService = class InstanceAiService {
                 requestId,
                 approved: data.approved,
             });
-            return false;
+            return null;
         }
         const { agent, runId, agentRunId, threadId, user, toolCallId, toolName, suspendPayload, abortController, tracing, modelId, messageGroupId, checkpoint, plannedBuild, runHandoff, } = suspended;
         if (user.id !== requestingUserId)
-            return false;
+            return null;
         const activeUser = await this.revalidateActiveUser(user.id);
         if (!activeUser) {
             this.logger.warn('Cancelling suspended run: user no longer authorized for AI Assistant', {
@@ -2676,7 +2821,7 @@ let InstanceAiService = class InstanceAiService {
                 requestId,
             });
             this.cancelRun(threadId);
-            return false;
+            return null;
         }
         this.runState.activateSuspendedRun(threadId);
         void this.suspendedThreads.dropPendingConfirmation(requestId);
@@ -2692,6 +2837,7 @@ let InstanceAiService = class InstanceAiService {
             ...(data.answers ? { answers: data.answers } : {}),
             ...(data.resourceDecision ? { resourceDecision: data.resourceDecision } : {}),
             ...(data.scope ? { scope: data.scope } : {}),
+            ...(data.autoSetup ? { autoSetup: data.autoSetup } : {}),
         };
         const resumeTracing = await this.tracing.createOrchestratorResumeTraceContext({
             baseTracing: tracing,
@@ -2720,7 +2866,19 @@ let InstanceAiService = class InstanceAiService {
                     : {}),
             },
         });
-        this.startProcessResumedStream(agent, resumeData, {
+        const effectiveTracing = resumeTracing ?? tracing;
+        let resumeAgent = agent;
+        let resumeModelId = modelId;
+        if (data.autoSetup) {
+            const rebuilt = await this.rebuildAgentForAutoSetupResume(activeUser, threadId, runId, abortController, effectiveTracing, runHandoff, messageGroupId);
+            if (!rebuilt) {
+                this.cancelRun(threadId, 'agent_rebuild_failed');
+                return null;
+            }
+            resumeAgent = rebuilt.agent;
+            resumeModelId = rebuilt.modelId;
+        }
+        this.startProcessResumedStream(resumeAgent, resumeData, {
             runId,
             agentRunId,
             threadId,
@@ -2731,18 +2889,19 @@ let InstanceAiService = class InstanceAiService {
             signal: abortController.signal,
             abortController,
             snapshotStorage: this.dbSnapshotStorage,
-            tracing: resumeTracing ?? tracing,
-            modelId,
+            tracing: effectiveTracing,
+            modelId: resumeModelId,
             checkpoint,
             plannedBuild,
             runHandoff,
         });
-        return true;
+        return { ok: true, runId };
     }
     async processResumedStream(agent, resumeData, opts) {
         let messageTraceFinalization;
         let completedSetupWorkflowId;
         try {
+            this.instanceAiErrorReporter.beginRun(opts.runId);
             if (opts.tracing?.getTelemetry && isTelemetryConfigurableAgent(agent)) {
                 try {
                     agent.telemetry(opts.tracing.getTelemetry({
@@ -2790,6 +2949,11 @@ let InstanceAiService = class InstanceAiService {
                     outputRedaction: (0, output_redaction_config_1.resolveOutputRedaction)(this.instanceAiConfig),
                 });
             if (result.status === 'suspended') {
+                this.emitRunMetrics(opts.threadId, 'suspended', {
+                    modelId: opts.modelId,
+                    workSummary: result.workSummary,
+                    usage: result.usage,
+                });
                 if (result.suspension) {
                     const resumeMessageGroupId = this.tracing.getMessageGroupId(opts.runId);
                     this.runState.suspendRun(opts.threadId, {
@@ -2896,7 +3060,7 @@ let InstanceAiService = class InstanceAiService {
             const outputText = await (result.text ?? Promise.resolve(''));
             const messageGroupId = this.tracing.getMessageGroupId(opts.runId);
             if (result.status === 'errored') {
-                this.reportInstanceAiError(result.error ?? new Error('Instance AI resumed stream errored'), {
+                this.instanceAiErrorReporter.report(result.error ?? new Error('Instance AI resumed stream errored'), {
                     component: 'instance-ai-stream',
                     threadId: opts.threadId,
                     runId: opts.runId,
@@ -2906,10 +3070,12 @@ let InstanceAiService = class InstanceAiService {
                     messageGroupId,
                 });
             }
+            const userFacingErrorMessage = result.status === 'errored' ? getUserFacingErrorMessage(result.error) : undefined;
             if (runControl.shouldEmitTerminalOutcome(result.stopReason)) {
                 this.terminalOutcome.evaluateTerminalResponse(opts.threadId, opts.runId, result.status, {
                     messageGroupId,
                     workSummary: result.workSummary,
+                    errorMessage: userFacingErrorMessage,
                     suppressCompletedFallback: opts.checkpoint?.isCheckpointFollowUp === true ||
                         opts.plannedBuild?.isPlannedBuildFollowUp === true,
                 });
@@ -2929,9 +3095,11 @@ let InstanceAiService = class InstanceAiService {
             const archivedWorkflowIds = await this.temporaryWorkflowService.reapForRun(opts.threadId, opts.user, undefined, this.backgroundTasks.getRunningTasks(opts.threadId).length);
             await this.finalizeRun(opts.threadId, opts.runId, result.status, opts.snapshotStorage, {
                 userId: opts.user.id,
+                ...(opts.modelId !== undefined ? { modelId: opts.modelId } : {}),
                 archivedWorkflowIds,
                 workSummary: result.workSummary,
                 usage: result.usage,
+                errorReason: userFacingErrorMessage,
             });
             await this.creditService.claimRunUsage(opts.user, opts.threadId, result.agentRunId || opts.runId, result.usage?.usage ?? [], result.status);
             if (result.status === 'completed') {
@@ -2994,7 +3162,7 @@ let InstanceAiService = class InstanceAiService {
                 error: errorMessage,
                 ...(0, observability_1.buildInstanceAiObservabilityContext)(errCtx),
             });
-            this.reportInstanceAiError(error, { component: 'instance-ai-run', ...errCtx });
+            this.instanceAiErrorReporter.report(error, { component: 'instance-ai-run', ...errCtx });
             this.terminalOutcome.evaluateTerminalResponse(opts.threadId, opts.runId, 'errored', {
                 messageGroupId,
                 errorMessage: userFacingErrorMessage,
@@ -3048,6 +3216,7 @@ let InstanceAiService = class InstanceAiService {
                 await this.taskProjector.syncFromWorkflowLoop(opts.threadId, opts.runId);
                 await this.maybeStartWorkflowSetupFollowUp(opts.user, opts.threadId);
             }
+            this.instanceAiErrorReporter.endRun(opts.runId);
         }
     }
     spawnBackgroundTask(runId, opts, snapshotStorage, messageGroupIdOverride) {
@@ -3105,7 +3274,7 @@ let InstanceAiService = class InstanceAiService {
             },
             onFailed: async (task) => {
                 await this.tracing.finalizeBackgroundTaskTracing(task, 'failed');
-                this.reportInstanceAiError(new Error(task.error ?? 'Instance AI background task failed'), {
+                this.instanceAiErrorReporter.report(new Error(task.error ?? 'Instance AI background task failed'), {
                     component: 'instance-ai-background-task',
                     threadId: opts.threadId,
                     runId,
@@ -3317,7 +3486,11 @@ let InstanceAiService = class InstanceAiService {
             agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
             payload: {
                 status: effectiveStatus,
-                ...(status === 'cancelled' ? { reason: reason ?? 'user_cancelled' } : {}),
+                ...(status === 'cancelled'
+                    ? { reason: reason ?? 'user_cancelled' }
+                    : status === 'errored' && reason
+                        ? { reason }
+                        : {}),
                 ...(hasArchived ? { archivedWorkflowIds } : {}),
             },
         });
@@ -3329,7 +3502,7 @@ let InstanceAiService = class InstanceAiService {
         });
     }
     async finalizeRun(threadId, runId, status, snapshotStorage, options) {
-        this.publishRunFinish(threadId, runId, status, undefined, options?.archivedWorkflowIds, options?.userId);
+        this.publishRunFinish(threadId, runId, status, options?.errorReason, options?.archivedWorkflowIds, options?.userId);
         this.emitRunMetrics(threadId, status, options);
         await this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage);
         if (status === 'completed' && options?.userId && options?.modelId) {
@@ -3340,7 +3513,7 @@ let InstanceAiService = class InstanceAiService {
         const startedAt = this.runState.getActiveRun(threadId)?.startedAt;
         this.eventService.emit('instance-ai-run-finished', {
             status: status === 'errored' ? 'error' : status,
-            durationMs: startedAt !== undefined ? Date.now() - startedAt : undefined,
+            durationMs: status !== 'suspended' && startedAt !== undefined ? Date.now() - startedAt : undefined,
             model: typeof options?.modelId === 'string' ? options.modelId : 'custom',
             toolCalls: options?.workSummary?.totalToolCalls ?? 0,
             toolErrors: options?.workSummary?.totalToolErrors ?? 0,
@@ -3355,13 +3528,17 @@ let InstanceAiService = class InstanceAiService {
                 return;
             if (thread.metadata?.titleRefined)
                 return;
-            const history = await memory.getMessages(threadId, { limit: 5 });
-            const userTexts = history.flatMap((m) => {
+            const history = await memory.getMessages(threadId, { limit: TITLE_REFINE_HISTORY_LIMIT });
+            const userTexts = [];
+            for (const m of history) {
                 if (!('role' in m) || m.role !== 'user')
-                    return [];
-                const text = this.extractStoredMessageText(m.content);
-                return text.length > 0 ? [text] : [];
-            });
+                    continue;
+                const text = (0, internal_messages_1.cleanStoredUserMessage)(this.extractStoredMessageText(m.content));
+                if (text && text.length > 0)
+                    userTexts.push(text);
+                if (userTexts.length >= 5)
+                    break;
+            }
             if (userTexts.length === 0)
                 return;
             const userText = userTexts.join('\n');
@@ -3510,8 +3687,26 @@ let InstanceAiService = class InstanceAiService {
             return { name: name.trim(), url: url?.trim() };
         });
     }
+    trackMcpToolCall({ server, toolName, success, }) {
+        const serverSlug = server.metadata?.serverSlug;
+        const userId = server.metadata?.userId;
+        if (serverSlug && userId) {
+            this.telemetry.track('Instance AI mcp tool called', {
+                user_id: userId,
+                server_slug: serverSlug,
+                tool_name: toolName,
+                success,
+            });
+        }
+    }
 };
 exports.InstanceAiService = InstanceAiService;
+__decorate([
+    (0, decorators_1.OnPubSubEvent)('relay-instance-ai-task-control', { instanceType: 'main' }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], InstanceAiService.prototype, "handleRelayTaskControl", null);
 __decorate([
     (0, decorators_1.OnLeaderTakeover)(),
     __metadata("design:type", Function),
@@ -3555,6 +3750,8 @@ exports.InstanceAiService = InstanceAiService = __decorate([
         thread_credential_allowlist_service_1.EvalThreadCredentialAllowlistService,
         instance_ai_run_probe_1.InstanceAiRunProbe,
         instance_ai_model_service_1.InstanceAiModelService,
-        instance_ai_credit_service_1.InstanceAiCreditService])
+        instance_ai_credit_service_1.InstanceAiCreditService,
+        publisher_service_1.Publisher,
+        instance_ai_error_reporter_service_1.InstanceAiErrorReporterService])
 ], InstanceAiService);
 //# sourceMappingURL=instance-ai.service.js.map