n8n 2.14.2 → 2.16.0

package/dist/modules/instance-ai/instance-ai.service.js

@@ -0,0 +1,1674 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InstanceAiService = void 0;
+const api_types_1 = require("@n8n/api-types");
+const backend_common_1 = require("@n8n/backend-common");
+const config_1 = require("@n8n/config");
+const constants_1 = require("@n8n/constants");
+const di_1 = require("@n8n/di");
+const url_service_1 = require("../../services/url.service");
+const instance_ai_1 = require("@n8n/instance-ai");
+const workflow_sdk_1 = require("@n8n/workflow-sdk");
+const nanoid_1 = require("nanoid");
+const ai_service_1 = require("../../services/ai.service");
+const push_1 = require("../../push");
+const in_process_event_bus_1 = require("./event-bus/in-process-event-bus");
+const filesystem_1 = require("./filesystem");
+const instance_ai_settings_service_1 = require("./instance-ai-settings.service");
+const instance_ai_adapter_service_1 = require("./instance-ai.adapter.service");
+const internal_messages_1 = require("./internal-messages");
+const typeorm_composite_store_1 = require("./storage/typeorm-composite-store");
+const db_snapshot_storage_1 = require("./storage/db-snapshot-storage");
+const db_iteration_log_storage_1 = require("./storage/db-iteration-log-storage");
+const compaction_service_1 = require("./compaction.service");
+const instance_ai_thread_repository_1 = require("./repositories/instance-ai-thread.repository");
+function getErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function createInertAbortSignal() {
+    return new AbortController().signal;
+}
+const ORCHESTRATOR_AGENT_ID = 'agent-001';
+const MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD = 5;
+let InstanceAiService = class InstanceAiService {
+    constructor(logger, globalConfig, adapterService, eventBus, settingsService, compositeStore, compactionService, aiService, push, threadRepo, urlService, dbSnapshotStorage, dbIterationLogStorage) {
+        this.adapterService = adapterService;
+        this.eventBus = eventBus;
+        this.settingsService = settingsService;
+        this.compositeStore = compositeStore;
+        this.compactionService = compactionService;
+        this.aiService = aiService;
+        this.push = push;
+        this.threadRepo = threadRepo;
+        this.urlService = urlService;
+        this.dbSnapshotStorage = dbSnapshotStorage;
+        this.dbIterationLogStorage = dbIterationLogStorage;
+        this.mcpClientManager = new instance_ai_1.McpClientManager();
+        this.runState = new instance_ai_1.RunStateRegistry();
+        this.backgroundTasks = new instance_ai_1.BackgroundTaskManager(MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD);
+        this.traceContextsByRunId = new Map();
+        this.sandboxes = new Map();
+        this.gatewayRegistry = new filesystem_1.LocalGatewayRegistry();
+        this.domainAccessTrackersByThread = new Map();
+        this.threadPushRef = new Map();
+        this.creditedThreads = new Set();
+        this.logger = logger.scoped('instance-ai');
+        this.instanceAiConfig = globalConfig.instanceAi;
+        this.defaultTimeZone = globalConfig.generic.timezone;
+        const editorBaseUrl = globalConfig.editorBaseUrl || `http://localhost:${globalConfig.port}`;
+        const restEndpoint = globalConfig.endpoints.rest;
+        this.oauth2CallbackUrl = `${editorBaseUrl.replace(/\/$/, '')}/${restEndpoint}/oauth2-credential/callback`;
+        this.webhookBaseUrl = `${this.urlService.getWebhookBaseUrl()}${globalConfig.endpoints.webhook}`;
+        this.startConfirmationTimeoutSweep();
+    }
+    startConfirmationTimeoutSweep() {
+        const timeoutMs = this.instanceAiConfig.confirmationTimeout;
+        if (timeoutMs <= 0)
+            return;
+        this.confirmationTimeoutInterval = setInterval(() => {
+            const { suspendedThreadIds, confirmationRequestIds } = this.runState.sweepTimedOut(timeoutMs);
+            for (const threadId of suspendedThreadIds) {
+                this.logger.debug('Auto-rejecting timed-out suspended run', { threadId });
+                this.cancelRun(threadId);
+            }
+            for (const reqId of confirmationRequestIds) {
+                this.logger.debug('Auto-rejecting timed-out sub-agent confirmation', {
+                    requestId: reqId,
+                });
+                this.runState.rejectPendingConfirmation(reqId);
+            }
+        }, constants_1.Time.minutes.toMilliseconds);
+    }
+    getSandboxConfigFromEnv() {
+        const { sandboxEnabled, sandboxProvider, daytonaApiUrl, daytonaApiKey, n8nSandboxServiceUrl, n8nSandboxServiceApiKey, sandboxImage, sandboxTimeout, } = this.instanceAiConfig;
+        if (!sandboxEnabled) {
+            return {
+                enabled: false,
+                provider: sandboxProvider === 'n8n-sandbox'
+                    ? 'n8n-sandbox'
+                    : sandboxProvider === 'daytona'
+                        ? 'daytona'
+                        : 'local',
+                timeout: sandboxTimeout,
+            };
+        }
+        if (sandboxProvider === 'daytona') {
+            return {
+                enabled: true,
+                provider: 'daytona',
+                daytonaApiUrl: daytonaApiUrl || undefined,
+                daytonaApiKey: daytonaApiKey || undefined,
+                image: sandboxImage || undefined,
+                timeout: sandboxTimeout,
+            };
+        }
+        if (sandboxProvider === 'n8n-sandbox') {
+            return {
+                enabled: true,
+                provider: 'n8n-sandbox',
+                serviceUrl: n8nSandboxServiceUrl || undefined,
+                apiKey: n8nSandboxServiceApiKey || undefined,
+                timeout: sandboxTimeout,
+            };
+        }
+        return {
+            enabled: true,
+            provider: 'local',
+            timeout: sandboxTimeout,
+        };
+    }
+    async resolveSandboxConfig(user) {
+        const base = this.getSandboxConfigFromEnv();
+        if (!base.enabled)
+            return base;
+        if (base.provider === 'daytona') {
+            if (this.aiService.isProxyEnabled()) {
+                const client = await this.aiService.getClient();
+                const proxyConfig = await client.getSandboxProxyConfig();
+                return {
+                    ...base,
+                    daytonaApiUrl: client.getSandboxProxyBaseUrl(),
+                    image: proxyConfig.image,
+                    getAuthToken: async () => {
+                        const token = await client.getBuilderApiProxyToken({ id: user.id }, { userMessageId: (0, nanoid_1.nanoid)() });
+                        return token.accessToken;
+                    },
+                };
+            }
+            const daytona = await this.settingsService.resolveDaytonaConfig(user);
+            return {
+                ...base,
+                daytonaApiUrl: daytona.apiUrl ?? base.daytonaApiUrl,
+                daytonaApiKey: daytona.apiKey ?? base.daytonaApiKey,
+            };
+        }
+        if (base.provider === 'n8n-sandbox') {
+            const sandbox = await this.settingsService.resolveN8nSandboxConfig(user);
+            return {
+                ...base,
+                serviceUrl: sandbox.serviceUrl ?? base.serviceUrl,
+                apiKey: sandbox.apiKey ?? base.apiKey,
+            };
+        }
+        return base;
+    }
+    async createBuilderFactory(user) {
+        const config = await this.resolveSandboxConfig(user);
+        if (!config.enabled)
+            return undefined;
+        if (config.provider === 'daytona') {
+            return new instance_ai_1.BuilderSandboxFactory(config, new instance_ai_1.SnapshotManager(config.image, this.logger));
+        }
+        return new instance_ai_1.BuilderSandboxFactory(config);
+    }
+    getLocalFsProvider() {
+        if (!this.localFsProvider) {
+            const basePath = this.instanceAiConfig.filesystemPath || undefined;
+            this.localFsProvider = new filesystem_1.LocalFilesystemProvider(basePath);
+        }
+        return this.localFsProvider;
+    }
+    async getOrCreateWorkspace(threadId, user) {
+        const existing = this.sandboxes.get(threadId);
+        if (existing)
+            return existing;
+        const config = await this.resolveSandboxConfig(user);
+        if (!config.enabled)
+            return undefined;
+        const sandbox = await (0, instance_ai_1.createSandbox)(config);
+        const workspace = (0, instance_ai_1.createWorkspace)(sandbox);
+        if (!sandbox || !workspace)
+            return undefined;
+        const entry = { sandbox, workspace };
+        this.sandboxes.set(threadId, entry);
+        return entry;
+    }
+    async destroySandbox(threadId) {
+        const entry = this.sandboxes.get(threadId);
+        if (!entry?.sandbox)
+            return;
+        this.sandboxes.delete(threadId);
+        try {
+            if ('destroy' in entry.sandbox && typeof entry.sandbox.destroy === 'function') {
+                await entry.sandbox.destroy();
+            }
+        }
+        catch (error) {
+            this.logger.warn('Failed to destroy sandbox', {
+                threadId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    async getProxyAuth(user) {
+        const client = await this.aiService.getClient();
+        const token = await client.getBuilderApiProxyToken({ id: user.id }, { userMessageId: (0, nanoid_1.nanoid)() });
+        return {
+            client,
+            headers: { Authorization: `${token.tokenType} ${token.accessToken}` },
+        };
+    }
+    async resolveModel(user) {
+        if (this.aiService.isProxyEnabled()) {
+            const { client, headers } = await this.getProxyAuth(user);
+            const modelName = await this.settingsService.resolveModelName(user);
+            const { createAnthropic } = await Promise.resolve().then(() => __importStar(require('@ai-sdk/anthropic')));
+            const provider = createAnthropic({
+                baseURL: client.getApiProxyBaseUrl() + '/anthropic/v1',
+                apiKey: 'proxy-managed',
+                headers,
+            });
+            return provider(modelName);
+        }
+        return await this.settingsService.resolveModelConfig(user);
+    }
+    async resolveSearchProxyConfig(user) {
+        if (!this.aiService.isProxyEnabled())
+            return undefined;
+        const { client, headers } = await this.getProxyAuth(user);
+        return { apiUrl: client.getApiProxyBaseUrl() + '/brave-search', headers };
+    }
+    async resolveTracingProxyConfig(user) {
+        if (!this.aiService.isProxyEnabled())
+            return undefined;
+        const { client, headers } = await this.getProxyAuth(user);
+        return { apiUrl: client.getApiProxyBaseUrl() + '/langsmith', headers };
+    }
+    async countCreditsIfFirst(user, threadId, runId) {
+        if (!this.aiService.isProxyEnabled())
+            return;
+        if (this.creditedThreads.has(threadId))
+            return;
+        const thread = await this.threadRepo.findOneBy({ id: threadId });
+        if (!thread)
+            return;
+        if (thread.metadata?.creditCounted) {
+            this.creditedThreads.add(threadId);
+            return;
+        }
+        try {
+            this.creditedThreads.add(threadId);
+            const { client, headers: authHeaders } = await this.getProxyAuth(user);
+            const info = await client.markBuilderSuccess({ id: user.id }, authHeaders);
+            if (info) {
+                thread.metadata = { ...thread.metadata, creditCounted: true };
+                await this.threadRepo.save(thread);
+                this.push.sendToUsers({
+                    type: 'updateInstanceAiCredits',
+                    data: { creditsQuota: info.creditsQuota, creditsClaimed: info.creditsClaimed },
+                }, [user.id]);
+            }
+        }
+        catch (error) {
+            this.creditedThreads.delete(threadId);
+            this.logger.warn('Failed to count Instance AI credits', {
+                error: getErrorMessage(error),
+                threadId,
+                runId,
+            });
+        }
+    }
+    isProxyEnabled() {
+        return this.aiService.isProxyEnabled();
+    }
+    async getCredits(user) {
+        if (!this.aiService.isProxyEnabled()) {
+            return { creditsQuota: api_types_1.UNLIMITED_CREDITS, creditsClaimed: 0 };
+        }
+        const client = await this.aiService.getClient();
+        return await client.getBuilderInstanceCredits({ id: user.id });
+    }
+    isEnabled() {
+        return !!this.instanceAiConfig.model;
+    }
+    isLocalFilesystemAvailable() {
+        return !!this.instanceAiConfig.filesystemPath?.trim();
+    }
+    getLocalFilesystemDirectory() {
+        const basePath = this.instanceAiConfig.filesystemPath?.trim();
+        return basePath || null;
+    }
+    hasActiveRun(threadId) {
+        return this.runState.hasLiveRun(threadId);
+    }
+    getThreadStatus(threadId) {
+        return this.runState.getThreadStatus(threadId, this.backgroundTasks.getTaskSnapshots(threadId));
+    }
+    storeTraceContext(runId, threadId, tracing, messageGroupId) {
+        this.traceContextsByRunId.set(runId, { threadId, messageGroupId, tracing });
+    }
+    getTraceContext(runId) {
+        return this.traceContextsByRunId.get(runId)?.tracing;
+    }
+    async finalizeMessageTraceRoot(runId, tracing, options) {
+        if (tracing.rootRun.endTime)
+            return;
+        const outputs = options.outputs ?? {
+            status: options.status,
+            runId,
+            ...(options.outputText ? { response: options.outputText } : {}),
+            ...(options.reason ? { reason: options.reason } : {}),
+        };
+        const metadata = {
+            final_status: options.status,
+            ...(options.modelId !== undefined ? { model_id: options.modelId } : {}),
+            ...options.metadata,
+        };
+        try {
+            await tracing.finishRun(tracing.rootRun, {
+                outputs,
+                metadata,
+                ...(options.error
+                    ? { error: options.error }
+                    : options.status === 'error' && options.reason
+                        ? { error: options.reason }
+                        : {}),
+            });
+        }
+        catch (error) {
+            this.logger.warn('Failed to finalize Instance AI message trace root', {
+                runId,
+                threadId: tracing.rootRun.metadata?.thread_id,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async maybeFinalizeRunTraceRoot(runId, options) {
+        const tracing = this.getTraceContext(runId);
+        if (!tracing)
+            return;
+        await this.finalizeMessageTraceRoot(runId, tracing, options);
+    }
+    async finalizeRemainingMessageTraceRoots(threadId, options) {
+        const finalizedMessageRuns = new Set();
+        for (const [runId, entry] of this.traceContextsByRunId) {
+            if (entry.threadId !== threadId)
+                continue;
+            if (finalizedMessageRuns.has(entry.tracing.rootRun.id))
+                continue;
+            finalizedMessageRuns.add(entry.tracing.rootRun.id);
+            await this.finalizeMessageTraceRoot(runId, entry.tracing, options);
+        }
+    }
+    deleteTraceContextsForThread(threadId) {
+        for (const [runId, entry] of this.traceContextsByRunId) {
+            if (entry.threadId === threadId) {
+                this.traceContextsByRunId.delete(runId);
+            }
+        }
+    }
+    async finalizeDetachedTraceRun(taskId, traceContext, options) {
+        if (!traceContext)
+            return;
+        try {
+            await traceContext.finishRun(traceContext.rootRun, {
+                outputs: {
+                    status: options.status,
+                    ...options.outputs,
+                },
+                metadata: {
+                    final_status: options.status,
+                    ...options.metadata,
+                },
+                ...(options.error ? { error: options.error } : {}),
+            });
+        }
+        catch (error) {
+            this.logger.warn('Failed to finalize Instance AI detached trace run', {
+                taskId,
+                traceRunId: traceContext.rootRun.id,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async finalizeRunTracing(runId, tracing, options) {
+        if (!tracing)
+            return;
+        const outputs = {
+            status: options.status,
+            runId,
+            ...(options.outputText ? { response: options.outputText } : {}),
+            ...(options.reason ? { reason: options.reason } : {}),
+        };
+        const metadata = {
+            final_status: options.status,
+            ...(options.modelId !== undefined ? { model_id: options.modelId } : {}),
+        };
+        try {
+            await tracing.finishRun(tracing.actorRun, {
+                outputs,
+                metadata,
+                ...(options.status === 'error' && options.reason ? { error: options.reason } : {}),
+            });
+        }
+        catch (error) {
+            this.logger.warn('Failed to finalize Instance AI run tracing', {
+                runId,
+                threadId: tracing.actorRun.metadata?.thread_id,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async finalizeBackgroundTaskTracing(task, status) {
+        await this.finalizeDetachedTraceRun(task.taskId, task.traceContext, {
+            status,
+            outputs: {
+                taskId: task.taskId,
+                agentId: task.agentId,
+                role: task.role,
+                ...(task.result ? { result: task.result } : {}),
+            },
+            ...(status === 'failed' && task.error ? { error: task.error } : {}),
+            metadata: {
+                ...(task.plannedTaskId ? { planned_task_id: task.plannedTaskId } : {}),
+                ...(task.workItemId ? { work_item_id: task.workItemId } : {}),
+            },
+        });
+    }
+    startRun(user, threadId, message, researchMode, attachments, timeZone, pushRef) {
+        const { runId, abortController, messageGroupId } = this.runState.startRun({
+            threadId,
+            user,
+            researchMode,
+        });
+        if (pushRef !== undefined) {
+            this.threadPushRef.set(threadId, pushRef);
+        }
+        void this.executeRun(user, threadId, runId, message, abortController, researchMode, attachments, messageGroupId, timeZone);
+        return runId;
+    }
+    getMessageGroupId(threadId) {
+        return this.runState.getMessageGroupId(threadId);
+    }
+    getLiveMessageGroupId(threadId) {
+        return this.runState.getLiveMessageGroupId(threadId, this.backgroundTasks.getTaskSnapshots(threadId));
+    }
+    getRunIdsForMessageGroup(messageGroupId) {
+        return this.runState.getRunIdsForMessageGroup(messageGroupId);
+    }
+    getActiveRunId(threadId) {
+        return this.runState.getActiveRunId(threadId);
+    }
+    cancelRun(threadId) {
+        const cancelledTasks = this.backgroundTasks.cancelThread(threadId);
+        const user = this.runState.getThreadUser(threadId);
+        for (const task of cancelledTasks) {
+            void this.finalizeBackgroundTaskTracing(task, 'cancelled');
+            this.eventBus.publish(threadId, {
+                type: 'agent-completed',
+                runId: task.runId,
+                agentId: task.agentId,
+                payload: { role: task.role, result: '', error: 'Cancelled by user' },
+            });
+            if (user) {
+                void this.handlePlannedTaskSettlement(user, task, 'cancelled');
+            }
+        }
+        const { active, suspended } = this.runState.cancelThread(threadId);
+        if (active) {
+            active.abortController.abort();
+            return;
+        }
+        if (suspended) {
+            suspended.abortController.abort();
+            void this.finalizeRunTracing(suspended.runId, suspended.tracing, {
+                status: 'cancelled',
+                reason: 'user_cancelled',
+            });
+            this.eventBus.publish(threadId, {
+                type: 'run-finish',
+                runId: suspended.runId,
+                agentId: ORCHESTRATOR_AGENT_ID,
+                payload: { status: 'cancelled', reason: 'user_cancelled' },
+            });
+            if (suspended.mastraRunId) {
+                void this.cleanupMastraSnapshots(suspended.mastraRunId);
+            }
+            void this.maybeFinalizeRunTraceRoot(suspended.runId, {
+                status: 'cancelled',
+                reason: 'user_cancelled',
+                metadata: { completion_source: 'orchestrator' },
+            });
+        }
+    }
+    sendCorrectionToTask(threadId, taskId, correction) {
+        return this.backgroundTasks.queueCorrection(threadId, taskId, correction);
+    }
+    cancelBackgroundTask(threadId, taskId) {
+        const task = this.backgroundTasks.cancelTask(threadId, taskId);
+        if (!task)
+            return;
+        void this.finalizeBackgroundTaskTracing(task, 'cancelled');
+        this.eventBus.publish(threadId, {
+            type: 'agent-completed',
+            runId: task.runId,
+            agentId: task.agentId,
+            payload: { role: task.role, result: '', error: 'Cancelled by user' },
+        });
+        const user = this.runState.getThreadUser(threadId);
+        if (user) {
+            void this.handlePlannedTaskSettlement(user, task, 'cancelled');
+        }
+    }
+    getUserIdForApiKey(key) {
+        return this.gatewayRegistry.getUserIdForApiKey(key);
+    }
+    generatePairingToken(userId) {
+        return this.gatewayRegistry.generatePairingToken(userId);
+    }
+    getPairingToken(userId) {
+        return this.gatewayRegistry.getPairingToken(userId);
+    }
+    consumePairingToken(userId, token) {
+        return this.gatewayRegistry.consumePairingToken(userId, token);
+    }
+    getActiveSessionKey(userId) {
+        return this.gatewayRegistry.getActiveSessionKey(userId);
+    }
+    clearActiveSessionKey(userId) {
+        this.gatewayRegistry.clearActiveSessionKey(userId);
+    }
+    getLocalGateway(userId) {
+        return this.gatewayRegistry.getGateway(userId);
+    }
+    initGateway(userId, data) {
+        this.gatewayRegistry.initGateway(userId, data);
+    }
+    resolveGatewayRequest(userId, requestId, result, error) {
+        return this.gatewayRegistry.resolveGatewayRequest(userId, requestId, result, error);
+    }
+    disconnectGateway(userId) {
+        this.gatewayRegistry.disconnectGateway(userId);
+    }
+    isLocalGatewayDisabled() {
+        return this.settingsService.isLocalGatewayDisabled();
+    }
+    getGatewayStatus(userId) {
+        return this.gatewayRegistry.getGatewayStatus(userId);
+    }
+    startDisconnectTimer(userId, onDisconnect) {
+        this.gatewayRegistry.startDisconnectTimer(userId, onDisconnect);
+    }
+    clearDisconnectTimer(userId) {
+        this.gatewayRegistry.clearDisconnectTimer(userId);
+    }
+    async clearThreadState(threadId) {
+        const { active, suspended } = this.runState.clearThread(threadId);
+        if (active) {
+            active.abortController.abort();
+            await this.finalizeRunTracing(active.runId, active.tracing, {
+                status: 'cancelled',
+                reason: 'thread_cleared',
+            });
+        }
+        if (suspended) {
+            suspended.abortController.abort();
+            await this.finalizeRunTracing(suspended.runId, suspended.tracing, {
+                status: 'cancelled',
+                reason: 'thread_cleared',
+            });
+        }
+        for (const task of this.backgroundTasks.cancelThread(threadId)) {
+            task.abortController.abort();
+            await this.finalizeBackgroundTaskTracing(task, 'cancelled');
+        }
+        await this.finalizeRemainingMessageTraceRoots(threadId, {
+            status: 'cancelled',
+            reason: 'thread_cleared',
+            metadata: { completion_source: 'service_cleanup' },
+        });
+        this.creditedThreads.delete(threadId);
+        this.domainAccessTrackersByThread.delete(threadId);
+        this.threadPushRef.delete(threadId);
+        this.deleteTraceContextsForThread(threadId);
+        await this.destroySandbox(threadId);
+        this.eventBus.clearThread(threadId);
+    }
+    async shutdown() {
+        if (this.confirmationTimeoutInterval) {
+            clearInterval(this.confirmationTimeoutInterval);
+            this.confirmationTimeoutInterval = undefined;
+        }
+        const { activeRuns, suspendedRuns } = this.runState.shutdown();
+        for (const run of activeRuns) {
+            run.abortController.abort();
+            await this.finalizeRunTracing(run.runId, run.tracing, {
+                status: 'cancelled',
+                reason: 'service_shutdown',
+            });
+        }
+        for (const run of suspendedRuns) {
+            run.abortController.abort();
+            await this.finalizeRunTracing(run.runId, run.tracing, {
+                status: 'cancelled',
+                reason: 'service_shutdown',
+            });
+        }
+        for (const task of this.backgroundTasks.cancelAll()) {
+            task.abortController.abort();
+            await this.finalizeBackgroundTaskTracing(task, 'cancelled');
+        }
+        const threadsWithTraces = new Set([...this.traceContextsByRunId.values()].map((entry) => entry.threadId));
+        for (const threadId of threadsWithTraces) {
+            await this.finalizeRemainingMessageTraceRoots(threadId, {
+                status: 'cancelled',
+                reason: 'service_shutdown',
+                metadata: { completion_source: 'service_cleanup' },
+            });
+        }
+        this.gatewayRegistry.disconnectAll();
+        const sandboxCleanups = [...this.sandboxes.keys()].map(async (threadId) => await this.destroySandbox(threadId));
+        await Promise.allSettled(sandboxCleanups);
+        this.domainAccessTrackersByThread.clear();
+        this.traceContextsByRunId.clear();
+        this.eventBus.clear();
+        await this.mcpClientManager.disconnect();
+        this.logger.debug('Instance AI service shut down');
+    }
+    createMemoryConfig() {
+        return {
+            storage: this.compositeStore,
+            embedderModel: this.instanceAiConfig.embedderModel || undefined,
+            lastMessages: this.instanceAiConfig.lastMessages,
+            semanticRecallTopK: this.instanceAiConfig.semanticRecallTopK,
+        };
+    }
+    async ensureThreadExists(memory, threadId, resourceId) {
+        const existingThread = await memory.getThreadById({ threadId });
+        if (existingThread)
+            return;
+        const now = new Date();
+        await memory.saveThread({
+            thread: {
+                id: threadId,
+                resourceId,
+                title: '',
+                createdAt: now,
+                updatedAt: now,
+            },
+        });
+    }
+    projectPlannedTaskList(graph) {
+        return {
+            tasks: graph.tasks.map((task) => ({
+                id: task.id,
+                description: task.title,
+                status: task.status === 'planned'
+                    ? 'todo'
+                    : task.status === 'running'
+                        ? 'in_progress'
+                        : task.status === 'succeeded'
+                            ? 'done'
+                            : task.status,
+            })),
+        };
+    }
+    buildPlannedTaskFollowUpMessage(type, graph, failedTask) {
+        const payload = {
+            tasks: graph.tasks.map((task) => ({
+                id: task.id,
+                title: task.title,
+                kind: task.kind,
+                status: task.status,
+                result: task.result,
+                error: task.error,
+                outcome: task.outcome,
+            })),
+        };
+        if (failedTask) {
+            payload.failedTask = {
+                id: failedTask.id,
+                title: failedTask.title,
+                kind: failedTask.kind,
+                error: failedTask.error,
+                result: failedTask.result,
+            };
+        }
+        return `<planned-task-follow-up type="${type}">\n${JSON.stringify(payload, null, 2)}\n</planned-task-follow-up>\n\n${internal_messages_1.AUTO_FOLLOW_UP_MESSAGE}`;
+    }
+    async createPlannedTaskState() {
+        const memory = (0, instance_ai_1.createMemory)(this.createMemoryConfig());
+        const taskStorage = new instance_ai_1.MastraTaskStorage(memory);
+        const plannedTaskStorage = new instance_ai_1.PlannedTaskStorage(memory);
+        const plannedTaskService = new instance_ai_1.PlannedTaskCoordinator(plannedTaskStorage);
+        return { memory, taskStorage, plannedTaskService };
+    }
+    async syncPlannedTasksToUi(threadId, graph) {
+        const { taskStorage } = await this.createPlannedTaskState();
+        const tasks = this.projectPlannedTaskList(graph);
+        await taskStorage.save(threadId, tasks);
+        this.eventBus.publish(threadId, {
+            type: 'tasks-update',
+            runId: graph.planRunId,
+            agentId: ORCHESTRATOR_AGENT_ID,
+            payload: { tasks },
+        });
+    }
+    async createExecutionEnvironment(user, threadId, runId, abortSignal, researchMode, messageGroupId, pushRef) {
+        const localGatewayDisabled = this.settingsService.isLocalGatewayDisabled();
+        const userGateway = this.gatewayRegistry.findGateway(user.id);
+        const localFilesystemService = !localGatewayDisabled && !userGateway?.isConnected && this.isLocalFilesystemAvailable()
+            ? this.getLocalFsProvider()
+            : undefined;
+        const searchProxyConfig = await this.resolveSearchProxyConfig(user);
+        const tracingProxyConfig = await this.resolveTracingProxyConfig(user);
+        const context = this.adapterService.createContext(user, {
+            filesystemService: localFilesystemService,
+            searchProxyConfig,
+            pushRef,
+        });
+        if (!localGatewayDisabled && userGateway?.isConnected) {
+            context.localMcpServer = userGateway;
+        }
+        context.permissions = this.settingsService.getPermissions();
+        let domainTracker = this.domainAccessTrackersByThread.get(threadId);
+        if (!domainTracker) {
+            domainTracker = (0, instance_ai_1.createDomainAccessTracker)();
+            this.domainAccessTrackersByThread.set(threadId, domainTracker);
+        }
+        context.domainAccessTracker = domainTracker;
+        context.runId = runId;
+        if (localGatewayDisabled) {
+            context.localGatewayStatus = { status: 'disabled' };
+        }
+        else if (userGateway?.isConnected) {
+            context.localGatewayStatus = { status: 'connected' };
+        }
+        else {
+            context.localGatewayStatus = {
+                status: 'disconnected',
+                capabilities: ['filesystem', 'browser'],
+            };
+        }
+        const modelId = await this.resolveModel(user);
+        const memory = (0, instance_ai_1.createMemory)(this.createMemoryConfig());
+        await this.ensureThreadExists(memory, threadId, user.id);
+        const taskStorage = new instance_ai_1.MastraTaskStorage(memory);
+        const iterationLog = this.dbIterationLogStorage;
+        const snapshotStorage = this.dbSnapshotStorage;
+        const workflowLoopStorage = new instance_ai_1.WorkflowLoopStorage(memory);
+        const workflowTasks = new instance_ai_1.WorkflowTaskCoordinator(threadId, workflowLoopStorage);
+        const plannedTaskStorage = new instance_ai_1.PlannedTaskStorage(memory);
+        const plannedTaskService = new instance_ai_1.PlannedTaskCoordinator(plannedTaskStorage);
+        const nodeDefDirs = this.adapterService.getNodeDefinitionDirs();
+        if (nodeDefDirs.length > 0) {
+            (0, workflow_sdk_1.setSchemaBaseDirs)(nodeDefDirs);
+        }
+        const domainTools = (0, instance_ai_1.createAllTools)(context);
+        const sandboxEntry = await this.getOrCreateWorkspace(threadId, user);
+        const orchestrationContext = {
+            threadId,
+            runId,
+            messageGroupId,
+            userId: user.id,
+            orchestratorAgentId: ORCHESTRATOR_AGENT_ID,
+            modelId,
+            storage: this.compositeStore,
+            subAgentMaxSteps: this.instanceAiConfig.subAgentMaxSteps,
+            eventBus: this.eventBus,
+            logger: this.logger,
+            domainTools,
+            abortSignal,
+            taskStorage,
+            researchMode,
+            browserMcpConfig: this.instanceAiConfig.browserMcp
+                ? { name: 'chrome-devtools', command: 'npx', args: ['-y', 'chrome-devtools-mcp@latest'] }
+                : undefined,
+            localMcpServer: context.localMcpServer,
+            oauth2CallbackUrl: this.oauth2CallbackUrl,
+            webhookBaseUrl: this.webhookBaseUrl,
+            waitForConfirmation: async (requestId) => {
+                return await new Promise((resolve) => {
+                    this.runState.registerPendingConfirmation(requestId, {
+                        resolve,
+                        threadId,
+                        userId: user.id,
+                        createdAt: Date.now(),
+                    });
+                });
+            },
+            cancelBackgroundTask: async (taskId) => this.cancelBackgroundTask(threadId, taskId),
+            spawnBackgroundTask: (opts) => this.spawnBackgroundTask(runId, opts, snapshotStorage, messageGroupId),
+            plannedTaskService,
+            schedulePlannedTasks: async () => await this.schedulePlannedTasks(user, threadId),
+            iterationLog,
+            sendCorrectionToTask: (taskId, correction) => this.sendCorrectionToTask(threadId, taskId, correction),
+            workflowTaskService: workflowTasks,
+            workspace: sandboxEntry?.workspace,
+            builderSandboxFactory: await this.createBuilderFactory(user),
+            nodeDefinitionDirs: nodeDefDirs.length > 0 ? nodeDefDirs : undefined,
+            domainContext: context,
+            tracingProxyConfig,
+        };
+        return {
+            context,
+            memory,
+            taskStorage,
+            iterationLog,
+            snapshotStorage,
+            workflowTasks,
+            plannedTaskService,
+            modelId,
+            orchestrationContext,
+            sandboxEntry,
+        };
+    }
+    async dispatchPlannedTask(task, context) {
+        const taskContext = this.createPlannedTaskContext(task.kind, context);
+        let started = null;
+        switch (task.kind) {
+            case 'build-workflow':
+                started = await (0, instance_ai_1.startBuildWorkflowAgentTask)(taskContext, {
+                    task: task.spec,
+                    workflowId: task.workflowId,
+                    plannedTaskId: task.id,
+                });
+                break;
+            case 'manage-data-tables':
+                started = await (0, instance_ai_1.startDataTableAgentTask)(taskContext, {
+                    task: task.spec,
+                    plannedTaskId: task.id,
+                });
+                break;
+            case 'research':
+                started = await (0, instance_ai_1.startResearchAgentTask)(taskContext, {
+                    goal: task.title,
+                    constraints: task.spec,
+                    plannedTaskId: task.id,
+                });
+                break;
+            case 'delegate':
+                started = await (0, instance_ai_1.startDetachedDelegateTask)(taskContext, {
+                    title: task.title,
+                    spec: task.spec,
+                    tools: task.tools ?? [],
+                    plannedTaskId: task.id,
+                });
+                break;
+        }
+        if (!started?.taskId) {
+            await context.plannedTaskService?.markFailed(context.threadId, task.id, {
+                error: started?.result || `Failed to start planned task "${task.title}"`,
+            });
+            return;
+        }
+        await context.plannedTaskService?.markRunning(context.threadId, task.id, {
+            agentId: started.agentId,
+            backgroundTaskId: started.taskId,
+        });
+        const nextGraph = await context.plannedTaskService?.getGraph(context.threadId);
+        if (nextGraph) {
+            await this.syncPlannedTasksToUi(context.threadId, nextGraph);
+        }
+    }
+    createPlannedTaskContext(kind, context) {
+        if (!context.domainContext)
+            return context;
+        const taskDomainContext = (0, instance_ai_1.applyPlannedTaskPermissions)(context.domainContext, kind);
+        if (taskDomainContext === context.domainContext)
+            return context;
+        return {
+            ...context,
+            domainContext: taskDomainContext,
+            domainTools: (0, instance_ai_1.createAllTools)(taskDomainContext),
+        };
+    }
+    async handlePlannedTaskSettlement(user, task, status) {
+        if (!task.plannedTaskId)
+            return;
+        const { plannedTaskService } = await this.createPlannedTaskState();
+        let graph = null;
+        if (status === 'succeeded') {
+            graph = await plannedTaskService.markSucceeded(task.threadId, task.plannedTaskId, {
+                result: task.result,
+                outcome: task.outcome,
+            });
+        }
+        else if (status === 'failed') {
+            graph = await plannedTaskService.markFailed(task.threadId, task.plannedTaskId, {
+                error: task.error,
+            });
+        }
+        else {
+            graph = await plannedTaskService.markCancelled(task.threadId, task.plannedTaskId, {
+                error: task.error,
+            });
+        }
+        if (graph) {
+            await this.syncPlannedTasksToUi(task.threadId, graph);
+        }
+        await this.schedulePlannedTasks(user, task.threadId);
+    }
+    async startInternalFollowUpRun(user, threadId, message, researchMode, messageGroupId) {
+        const { runId, abortController } = this.runState.startRun({
+            threadId,
+            user,
+            researchMode,
+            messageGroupId,
+        });
+        void this.executeRun(user, threadId, runId, message, abortController, researchMode, undefined, messageGroupId);
+        return runId;
+    }
+    async schedulePlannedTasks(user, threadId) {
+        const { plannedTaskService } = await this.createPlannedTaskState();
+        const graph = await plannedTaskService.getGraph(threadId);
+        if (!graph)
+            return;
+        await this.syncPlannedTasksToUi(threadId, graph);
+        const availableSlots = Math.max(0, MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD -
+            this.backgroundTasks.getRunningTasks(threadId).length);
+        const action = await plannedTaskService.tick(threadId, { availableSlots });
+        if (action.type === 'none')
+            return;
+        if (action.type === 'replan') {
+            await this.syncPlannedTasksToUi(threadId, action.graph);
+            await this.startInternalFollowUpRun(user, threadId, this.buildPlannedTaskFollowUpMessage('replan', action.graph, action.failedTask), this.runState.getThreadResearchMode(threadId), action.graph.messageGroupId);
+            return;
+        }
+        if (action.type === 'synthesize') {
+            await this.syncPlannedTasksToUi(threadId, action.graph);
+            await this.startInternalFollowUpRun(user, threadId, this.buildPlannedTaskFollowUpMessage('synthesize', action.graph), this.runState.getThreadResearchMode(threadId), action.graph.messageGroupId);
+            return;
+        }
+        const environment = await this.createExecutionEnvironment(user, threadId, action.graph.planRunId, createInertAbortSignal(), this.runState.getThreadResearchMode(threadId), action.graph.messageGroupId);
+        environment.orchestrationContext.tracing = this.getTraceContext(action.graph.planRunId);
+        for (const task of action.tasks) {
+            await this.dispatchPlannedTask(task, environment.orchestrationContext);
+        }
+        await this.schedulePlannedTasks(user, threadId);
+    }
+    async executeRun(user, threadId, runId, message, abortController, researchMode, attachments, messageGroupId, timeZone) {
+        const signal = abortController.signal;
+        let mastraRunId = '';
+        let tracing;
+        let messageTraceFinalization;
+        try {
+            const messageId = (0, nanoid_1.nanoid)();
+            this.eventBus.publish(threadId, {
+                type: 'run-start',
+                runId,
+                agentId: ORCHESTRATOR_AGENT_ID,
+                userId: user.id,
+                payload: { messageId, messageGroupId },
+            });
+            if (signal.aborted) {
+                this.eventBus.publish(threadId, {
+                    type: 'run-finish',
+                    runId,
+                    agentId: ORCHESTRATOR_AGENT_ID,
+                    payload: { status: 'cancelled', reason: 'user_cancelled' },
+                });
+                return;
+            }
+            const mcpServers = this.parseMcpServers(this.instanceAiConfig.mcpServers);
+            const executionPushRef = this.threadPushRef.get(threadId);
+            const { context, memory, taskStorage, snapshotStorage, modelId, orchestrationContext } = await this.createExecutionEnvironment(user, threadId, runId, signal, researchMode, messageGroupId, executionPushRef);
+            const memoryConfig = this.createMemoryConfig();
+            const traceInput = {
+                message,
+                ...(attachments?.length
+                    ? {
+                        attachments: attachments.map((attachment) => ({
+                            mimeType: attachment.mimeType,
+                            size: attachment.data.length,
+                        })),
+                    }
+                    : {}),
+                ...(researchMode !== undefined ? { researchMode } : {}),
+                ...(messageGroupId ? { messageGroupId } : {}),
+            };
+            tracing = await (0, instance_ai_1.createInstanceAiTraceContext)({
+                threadId,
+                messageId,
+                messageGroupId,
+                runId,
+                userId: user.id,
+                modelId,
+                input: traceInput,
+                proxyConfig: orchestrationContext.tracingProxyConfig,
+            });
+            if (tracing) {
+                orchestrationContext.tracing = tracing;
+                this.runState.attachTracing(threadId, tracing);
+                this.storeTraceContext(runId, threadId, tracing, messageGroupId);
+            }
+            const thread = await memory.getThreadById({ threadId });
+            if (thread && !thread.title) {
+                await (0, instance_ai_1.patchThread)(memory, {
+                    threadId,
+                    update: () => ({ title: (0, instance_ai_1.truncateToTitle)(message) }),
+                });
+            }
+            const existingTasks = await taskStorage.get(threadId);
+            if (existingTasks) {
+                this.eventBus.publish(threadId, {
+                    type: 'tasks-update',
+                    runId,
+                    agentId: ORCHESTRATOR_AGENT_ID,
+                    payload: { tasks: existingTasks },
+                });
+            }
+            const agent = await (0, instance_ai_1.createInstanceAgent)({
+                modelId,
+                context,
+                orchestrationContext,
+                mcpServers,
+                memoryConfig,
+                memory,
+                workspace: orchestrationContext.workspace,
+                disableDeferredTools: true,
+                timeZone: timeZone ?? this.defaultTimeZone,
+            });
+            this.eventBus.publish(threadId, {
+                type: 'status',
+                runId,
+                agentId: ORCHESTRATOR_AGENT_ID,
+                payload: { message: 'Recalling conversation...' },
+            });
+            const contextCompactionRun = tracing
+                ? await tracing.startChildRun(tracing.actorRun, {
+                    name: 'context_compaction',
+                    tags: ['context'],
+                    metadata: { agent_role: 'context_compaction' },
+                    inputs: {
+                        threadId,
+                        lastMessages: this.instanceAiConfig.lastMessages ?? 20,
+                    },
+                })
+                : undefined;
+            let conversationSummary;
+            try {
+                conversationSummary = await this.compactionService.prepareCompactedContext(threadId, memory, modelId, this.instanceAiConfig.lastMessages ?? 20);
+                if (contextCompactionRun && tracing) {
+                    await tracing.finishRun(contextCompactionRun, {
+                        outputs: {
+                            summarized: Boolean(conversationSummary),
+                            summary: conversationSummary ?? '',
+                        },
+                        metadata: { final_status: 'completed' },
+                    });
+                }
+            }
+            catch (error) {
+                if (contextCompactionRun && tracing) {
+                    await tracing.failRun(contextCompactionRun, error, {
+                        final_status: 'error',
+                    });
+                }
+                throw error;
+            }
+            this.eventBus.publish(threadId, {
+                type: 'status',
+                runId,
+                agentId: ORCHESTRATOR_AGENT_ID,
+                payload: { message: '' },
+            });
+            const promptBuildRun = tracing
+                ? await tracing.startChildRun(tracing.actorRun, {
+                    name: 'prompt_build',
+                    tags: ['prompt'],
+                    metadata: { agent_role: 'prompt_build' },
+                    inputs: {
+                        message,
+                        hasConversationSummary: Boolean(conversationSummary),
+                        attachmentCount: attachments?.length ?? 0,
+                    },
+                })
+                : undefined;
+            let streamInput;
+            try {
+                const enrichedMessage = await this.buildMessageWithRunningTasks(threadId, message);
+                const fullMessage = conversationSummary
+                    ? `${conversationSummary}\n\n${enrichedMessage}`
+                    : enrichedMessage;
+                streamInput =
+                    attachments && attachments.length > 0
+                        ? [
+                            {
+                                role: 'user',
+                                content: [
+                                    { type: 'text', text: fullMessage },
+                                    ...attachments.map((a) => ({
+                                        type: 'file',
+                                        data: a.data,
+                                        mimeType: a.mimeType,
+                                    })),
+                                ],
+                            },
+                        ]
+                        : fullMessage;
+                if (promptBuildRun && tracing) {
+                    await tracing.finishRun(promptBuildRun, {
+                        outputs: {
+                            fullMessage,
+                            streamInput,
+                        },
+                        metadata: { final_status: 'completed' },
+                    });
+                }
+            }
+            catch (error) {
+                if (promptBuildRun && tracing) {
+                    await tracing.failRun(promptBuildRun, error, {
+                        final_status: 'error',
+                    });
+                }
+                throw error;
+            }
+            const result = tracing
+                ? await tracing.withRunTree(tracing.actorRun, async () => {
+                    return await (0, instance_ai_1.streamAgentRun)(agent, streamInput, {
+                        abortSignal: signal,
+                        memory: {
+                            resource: user.id,
+                            thread: threadId,
+                        },
+                        providerOptions: {
+                            anthropic: { cacheControl: { type: 'ephemeral' } },
+                        },
+                    }, {
+                        threadId,
+                        runId,
+                        agentId: ORCHESTRATOR_AGENT_ID,
+                        signal,
+                        eventBus: this.eventBus,
+                        logger: this.logger,
+                    });
+                })
+                : await (0, instance_ai_1.streamAgentRun)(agent, streamInput, {
+                    abortSignal: signal,
+                    memory: {
+                        resource: user.id,
+                        thread: threadId,
+                    },
+                    providerOptions: {
+                        anthropic: { cacheControl: { type: 'ephemeral' } },
+                    },
+                }, {
+                    threadId,
+                    runId,
+                    agentId: ORCHESTRATOR_AGENT_ID,
+                    signal,
+                    eventBus: this.eventBus,
+                    logger: this.logger,
+                });
+            mastraRunId = result.mastraRunId;
+            if (result.status === 'suspended') {
+                if (result.suspension) {
+                    this.runState.suspendRun(threadId, {
+                        runId,
+                        mastraRunId: result.mastraRunId,
+                        agent,
+                        threadId,
+                        user,
+                        toolCallId: result.suspension.toolCallId,
+                        requestId: result.suspension.requestId,
+                        abortController,
+                        messageGroupId,
+                        createdAt: Date.now(),
+                        tracing,
+                    });
+                }
+                if (result.confirmationEvent) {
+                    this.eventBus.publish(threadId, result.confirmationEvent);
+                }
+                await this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage);
+                return;
+            }
+            const outputText = await (result.text ?? Promise.resolve(''));
+            const finalStatus = result.status === 'errored' ? 'error' : result.status;
+            await this.finalizeRunTracing(runId, tracing, {
+                status: finalStatus,
+                outputText,
+                modelId,
+            });
+            messageTraceFinalization = {
+                status: finalStatus,
+                outputText,
+                modelId,
+                metadata: { completion_source: 'orchestrator' },
+            };
+            await this.finalizeRun(threadId, runId, result.status, snapshotStorage, {
+                userId: user.id,
+                modelId,
+            });
+            if (result.status === 'completed') {
+                await this.countCreditsIfFirst(user, threadId, runId);
+            }
+        }
+        catch (error) {
+            if (signal.aborted) {
+                await this.finalizeRunTracing(runId, tracing, {
+                    status: 'cancelled',
+                    reason: 'user_cancelled',
+                });
+                messageTraceFinalization = {
+                    status: 'cancelled',
+                    reason: 'user_cancelled',
+                    metadata: { completion_source: 'orchestrator' },
+                };
+                this.publishRunFinish(threadId, runId, 'cancelled', 'user_cancelled');
+                return;
+            }
+            const errorMessage = getErrorMessage(error);
+            this.logger.error('Instance AI run error', {
+                error: errorMessage,
+                threadId,
+                runId,
+            });
+            await this.finalizeRunTracing(runId, tracing, {
+                status: 'error',
+                reason: errorMessage,
+            });
+            messageTraceFinalization = {
+                status: 'error',
+                reason: errorMessage,
+                metadata: { completion_source: 'orchestrator' },
+            };
+            this.eventBus.publish(threadId, {
+                type: 'run-finish',
+                runId,
+                agentId: ORCHESTRATOR_AGENT_ID,
+                payload: {
+                    status: 'error',
+                    reason: errorMessage,
+                },
+            });
+        }
+        finally {
+            this.runState.clearActiveRun(threadId);
+            this.threadPushRef.delete(threadId);
+            this.domainAccessTrackersByThread.get(threadId)?.clearRun(runId);
+            if (messageTraceFinalization) {
+                await this.maybeFinalizeRunTraceRoot(runId, messageTraceFinalization);
+            }
+            if (!this.runState.hasSuspendedRun(threadId) && mastraRunId) {
+                void this.cleanupMastraSnapshots(mastraRunId);
+            }
+        }
+    }
+    async resolveConfirmation(requestingUserId, requestId, data) {
+        if (this.runState.resolvePendingConfirmation(requestingUserId, requestId, data)) {
+            this.logger.debug('Resolved pending confirmation (sub-agent HITL)', {
+                requestId,
+                approved: data.approved,
+            });
+            return true;
+        }
+        this.logger.debug('Pending confirmation not found, trying suspended run resume', {
+            requestId,
+            approved: data.approved,
+        });
+        return await this.resumeSuspendedRun(requestingUserId, requestId, data);
+    }
+    async resumeSuspendedRun(requestingUserId, requestId, data) {
+        const suspended = this.runState.findSuspendedByRequestId(requestId);
+        if (!suspended) {
+            this.logger.warn('Confirmation target not found: no pending confirmation or suspended run', {
+                requestId,
+                approved: data.approved,
+            });
+            return false;
+        }
+        const { agent, runId, mastraRunId, threadId, user, toolCallId, abortController, tracing } = suspended;
+        if (user.id !== requestingUserId)
+            return false;
+        this.runState.activateSuspendedRun(threadId);
+        const credentialsPayload = data.nodeCredentials ?? data.credentials;
+        const resumeData = {
+            approved: data.approved,
+            ...(data.credentialId ? { credentialId: data.credentialId } : {}),
+            ...(credentialsPayload ? { credentials: credentialsPayload } : {}),
+            ...(data.autoSetup ? { autoSetup: data.autoSetup } : {}),
+            ...(data.userInput !== undefined ? { userInput: data.userInput } : {}),
+            ...(data.domainAccessAction ? { domainAccessAction: data.domainAccessAction } : {}),
+            ...(data.action ? { action: data.action } : {}),
+            ...(data.nodeParameters ? { nodeParameters: data.nodeParameters } : {}),
+            ...(data.testTriggerNode ? { testTriggerNode: data.testTriggerNode } : {}),
+            ...(data.answers ? { answers: data.answers } : {}),
+            ...(data.resourceDecision ? { resourceDecision: data.resourceDecision } : {}),
+        };
+        void this.processResumedStream(agent, resumeData, {
+            runId,
+            mastraRunId,
+            threadId,
+            user,
+            toolCallId,
+            signal: abortController.signal,
+            abortController,
+            snapshotStorage: this.dbSnapshotStorage,
+            tracing,
+        });
+        return true;
+    }
+    async processResumedStream(agent, resumeData, opts) {
+        let messageTraceFinalization;
+        try {
+            const result = opts.tracing
+                ? await opts.tracing.withRunTree(opts.tracing.actorRun, async () => {
+                    return await (0, instance_ai_1.resumeAgentRun)(agent, resumeData, {
+                        runId: opts.mastraRunId,
+                        toolCallId: opts.toolCallId,
+                        memory: { resource: opts.user.id, thread: opts.threadId },
+                    }, {
+                        threadId: opts.threadId,
+                        runId: opts.runId,
+                        agentId: ORCHESTRATOR_AGENT_ID,
+                        signal: opts.signal,
+                        eventBus: this.eventBus,
+                        logger: this.logger,
+                        mastraRunId: opts.mastraRunId,
+                    });
+                })
+                : await (0, instance_ai_1.resumeAgentRun)(agent, resumeData, {
+                    runId: opts.mastraRunId,
+                    toolCallId: opts.toolCallId,
+                    memory: { resource: opts.user.id, thread: opts.threadId },
+                }, {
+                    threadId: opts.threadId,
+                    runId: opts.runId,
+                    agentId: ORCHESTRATOR_AGENT_ID,
+                    signal: opts.signal,
+                    eventBus: this.eventBus,
+                    logger: this.logger,
+                    mastraRunId: opts.mastraRunId,
+                });
+            if (result.status === 'suspended') {
+                if (result.suspension) {
+                    this.runState.suspendRun(opts.threadId, {
+                        runId: opts.runId,
+                        mastraRunId: result.mastraRunId,
+                        agent,
+                        threadId: opts.threadId,
+                        user: opts.user,
+                        toolCallId: result.suspension.toolCallId,
+                        requestId: result.suspension.requestId,
+                        abortController: opts.abortController,
+                        messageGroupId: this.traceContextsByRunId.get(opts.runId)?.messageGroupId,
+                        createdAt: Date.now(),
+                        tracing: opts.tracing,
+                    });
+                }
+                if (result.confirmationEvent) {
+                    this.eventBus.publish(opts.threadId, result.confirmationEvent);
+                }
+                return;
+            }
+            const outputText = await (result.text ?? Promise.resolve(''));
+            const finalStatus = result.status === 'errored' ? 'error' : result.status;
+            await this.finalizeRunTracing(opts.runId, opts.tracing, {
+                status: finalStatus,
+                outputText,
+            });
+            messageTraceFinalization = {
+                status: finalStatus,
+                outputText,
+                metadata: { completion_source: 'orchestrator' },
+            };
+            await this.finalizeRun(opts.threadId, opts.runId, result.status, opts.snapshotStorage);
+        }
+        catch (error) {
+            if (opts.signal.aborted) {
+                await this.finalizeRunTracing(opts.runId, opts.tracing, {
+                    status: 'cancelled',
+                    reason: 'user_cancelled',
+                });
+                messageTraceFinalization = {
+                    status: 'cancelled',
+                    reason: 'user_cancelled',
+                    metadata: { completion_source: 'orchestrator' },
+                };
+                this.publishRunFinish(opts.threadId, opts.runId, 'cancelled', 'user_cancelled');
+                return;
+            }
+            const errorMessage = getErrorMessage(error);
+            this.logger.error('Instance AI resumed run error', {
+                error: errorMessage,
+                threadId: opts.threadId,
+                runId: opts.runId,
+            });
+            await this.finalizeRunTracing(opts.runId, opts.tracing, {
+                status: 'error',
+                reason: errorMessage,
+            });
+            messageTraceFinalization = {
+                status: 'error',
+                reason: errorMessage,
+                metadata: { completion_source: 'orchestrator' },
+            };
+            this.eventBus.publish(opts.threadId, {
+                type: 'run-finish',
+                runId: opts.runId,
+                agentId: ORCHESTRATOR_AGENT_ID,
+                payload: {
+                    status: 'error',
+                    reason: errorMessage,
+                },
+            });
+        }
+        finally {
+            this.runState.clearActiveRun(opts.threadId);
+            this.threadPushRef.delete(opts.threadId);
+            if (messageTraceFinalization) {
+                await this.maybeFinalizeRunTraceRoot(opts.runId, messageTraceFinalization);
+            }
+        }
+    }
+    spawnBackgroundTask(runId, opts, snapshotStorage, messageGroupIdOverride) {
+        this.backgroundTasks.spawn({
+            taskId: opts.taskId,
+            threadId: opts.threadId,
+            runId,
+            role: opts.role,
+            agentId: opts.agentId,
+            messageGroupId: messageGroupIdOverride ?? this.runState.getMessageGroupId(opts.threadId),
+            plannedTaskId: opts.plannedTaskId,
+            workItemId: opts.workItemId,
+            traceContext: opts.traceContext,
+            run: opts.run,
+            onLimitReached: async (errorMessage) => {
+                await this.finalizeDetachedTraceRun(opts.taskId, opts.traceContext, {
+                    status: 'failed',
+                    outputs: {
+                        taskId: opts.taskId,
+                        agentId: opts.agentId,
+                        role: opts.role,
+                    },
+                    error: errorMessage,
+                    metadata: {
+                        ...(opts.plannedTaskId ? { planned_task_id: opts.plannedTaskId } : {}),
+                        ...(opts.workItemId ? { work_item_id: opts.workItemId } : {}),
+                    },
+                });
+                this.eventBus.publish(opts.threadId, {
+                    type: 'agent-completed',
+                    runId,
+                    agentId: opts.agentId,
+                    payload: {
+                        role: opts.role,
+                        result: '',
+                        error: errorMessage,
+                    },
+                });
+            },
+            onCompleted: async (task) => {
+                await this.finalizeBackgroundTaskTracing(task, 'completed');
+                this.eventBus.publish(opts.threadId, {
+                    type: 'agent-completed',
+                    runId,
+                    agentId: opts.agentId,
+                    payload: { role: opts.role, result: task.result ?? '' },
+                });
+                const user = this.runState.getThreadUser(opts.threadId);
+                if (user) {
+                    await this.handlePlannedTaskSettlement(user, task, 'succeeded');
+                }
+            },
+            onFailed: async (task) => {
+                await this.finalizeBackgroundTaskTracing(task, 'failed');
+                this.eventBus.publish(opts.threadId, {
+                    type: 'agent-completed',
+                    runId,
+                    agentId: opts.agentId,
+                    payload: { role: opts.role, result: '', error: task.error ?? 'Unknown error' },
+                });
+                const user = this.runState.getThreadUser(opts.threadId);
+                if (user) {
+                    await this.handlePlannedTaskSettlement(user, task, 'failed');
+                }
+            },
+            onSettled: async (task) => {
+                await this.saveAgentTreeSnapshot(opts.threadId, runId, snapshotStorage, true, task.messageGroupId);
+                if (!task.plannedTaskId) {
+                    const remaining = this.backgroundTasks.getRunningTasks(opts.threadId);
+                    const hasActiveRun = !!this.runState.getActiveRunId(opts.threadId);
+                    const hasSuspendedRun = this.runState.hasSuspendedRun(opts.threadId);
+                    if (remaining.length === 0 && !hasActiveRun && !hasSuspendedRun) {
+                        const user = this.runState.getThreadUser(opts.threadId);
+                        if (user) {
+                            const payload = JSON.stringify({
+                                role: opts.role,
+                                status: task.result ? 'completed' : task.error ? 'failed' : 'finished',
+                                result: task.result ?? undefined,
+                                error: task.error ?? undefined,
+                            }, null, 2);
+                            await this.startInternalFollowUpRun(user, opts.threadId, `<background-task-completed>\n${payload}\n</background-task-completed>\n\n${internal_messages_1.AUTO_FOLLOW_UP_MESSAGE}`, this.runState.getThreadResearchMode(opts.threadId), task.messageGroupId);
+                        }
+                    }
+                }
+            },
+        });
+    }
+    async buildMessageWithRunningTasks(threadId, message) {
+        return await (0, instance_ai_1.enrichMessageWithBackgroundTasks)(message, this.backgroundTasks.getRunningTasks(threadId), {
+            formatTask: async (task) => `[Running task — ${task.role}]: taskId=${task.taskId}`,
+        });
+    }
+    publishRunFinish(threadId, runId, status, reason) {
+        const effectiveStatus = status === 'errored' ? 'error' : status;
+        this.eventBus.publish(threadId, {
+            type: 'run-finish',
+            runId,
+            agentId: ORCHESTRATOR_AGENT_ID,
+            payload: status === 'cancelled'
+                ? { status: effectiveStatus, reason: reason ?? 'user_cancelled' }
+                : { status: effectiveStatus },
+        });
+    }
+    async finalizeRun(threadId, runId, status, snapshotStorage, options) {
+        this.publishRunFinish(threadId, runId, status);
+        if (status === 'completed') {
+            await this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage);
+            if (options?.userId && options?.modelId) {
+                void this.refineTitleIfNeeded(threadId, options.userId, options.modelId);
+            }
+        }
+    }
+    async refineTitleIfNeeded(threadId, userId, modelId) {
+        try {
+            const memory = (0, instance_ai_1.createMemory)(this.createMemoryConfig());
+            const thread = await memory.getThreadById({ threadId });
+            if (!thread?.title)
+                return;
+            if (thread.metadata?.titleRefined)
+                return;
+            const result = await memory.recall({ threadId, resourceId: userId, perPage: 5 });
+            const firstUserMsg = result.messages.find((m) => m.role === 'user');
+            if (!firstUserMsg)
+                return;
+            const userText = typeof firstUserMsg.content === 'string'
+                ? firstUserMsg.content
+                : JSON.stringify(firstUserMsg.content);
+            const llmTitle = await (0, instance_ai_1.generateThreadTitle)(modelId, userText);
+            if (!llmTitle)
+                return;
+            await (0, instance_ai_1.patchThread)(memory, {
+                threadId,
+                update: ({ metadata }) => ({
+                    title: llmTitle,
+                    metadata: { ...metadata, titleRefined: true },
+                }),
+            });
+            this.eventBus.publish(threadId, {
+                type: 'thread-title-updated',
+                runId: '',
+                agentId: ORCHESTRATOR_AGENT_ID,
+                payload: { title: llmTitle },
+            });
+        }
+        catch (error) {
+            this.logger.warn('Failed to refine thread title', {
+                threadId,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async cleanupMastraSnapshots(mastraRunId) {
+        try {
+            const workflowsStorage = this.compositeStore.stores.workflows;
+            await workflowsStorage.deleteAllByRunId(mastraRunId);
+        }
+        catch (error) {
+            this.logger.warn('Failed to clean up Mastra workflow snapshots', {
+                mastraRunId,
+                error: getErrorMessage(error),
+            });
+        }
+    }
+    async saveAgentTreeSnapshot(threadId, runId, snapshotStorage, isUpdate = false, overrideMessageGroupId) {
+        try {
+            const messageGroupId = overrideMessageGroupId ?? this.runState.getMessageGroupId(threadId);
+            let events;
+            let groupRunIds;
+            if (messageGroupId) {
+                groupRunIds = this.getRunIdsForMessageGroup(messageGroupId);
+                events = this.eventBus.getEventsForRuns(threadId, groupRunIds);
+            }
+            else {
+                events = this.eventBus.getEventsForRun(threadId, runId);
+            }
+            const agentTree = (0, instance_ai_1.buildAgentTreeFromEvents)(events);
+            if (isUpdate) {
+                await snapshotStorage.updateLast(threadId, agentTree, runId, messageGroupId, groupRunIds);
+            }
+            else {
+                await snapshotStorage.save(threadId, agentTree, runId, messageGroupId, groupRunIds);
+            }
+        }
+        catch (error) {
+            this.logger.warn('Failed to save agent tree snapshot', {
+                threadId,
+                runId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    parseMcpServers(raw) {
+        if (!raw.trim())
+            return [];
+        return raw.split(',').map((entry) => {
+            const [name, url] = entry.trim().split('=');
+            return { name: name.trim(), url: url?.trim() };
+        });
+    }
+};
+exports.InstanceAiService = InstanceAiService;
+exports.InstanceAiService = InstanceAiService = __decorate([
+    (0, di_1.Service)(),
+    __metadata("design:paramtypes", [backend_common_1.Logger,
+        config_1.GlobalConfig,
+        instance_ai_adapter_service_1.InstanceAiAdapterService,
+        in_process_event_bus_1.InProcessEventBus,
+        instance_ai_settings_service_1.InstanceAiSettingsService,
+        typeorm_composite_store_1.TypeORMCompositeStore,
+        compaction_service_1.InstanceAiCompactionService,
+        ai_service_1.AiService,
+        push_1.Push,
+        instance_ai_thread_repository_1.InstanceAiThreadRepository,
+        url_service_1.UrlService,
+        db_snapshot_storage_1.DbSnapshotStorage,
+        db_iteration_log_storage_1.DbIterationLogStorage])
+], InstanceAiService);
+//# sourceMappingURL=instance-ai.service.js.map