n8n 2.22.3 → 2.23.0

package/dist/modules/instance-ai/instance-ai.service.js

@@ -46,11 +46,11 @@ exports.InstanceAiService = void 0;
 const api_types_1 = require("@n8n/api-types");
 const backend_common_1 = require("@n8n/backend-common");
 const config_1 = require("@n8n/config");
+const decorators_1 = require("@n8n/decorators");
 const n8n_core_1 = require("n8n-core");
 const ssrf_protection_service_1 = require("../../services/ssrf/ssrf-protection.service");
 const db_1 = require("@n8n/db");
 const di_1 = require("@n8n/di");
-const permissions_1 = require("@n8n/permissions");
 const url_service_1 = require("../../services/url.service");
 const instance_ai_1 = require("@n8n/instance-ai");
 const workflow_sdk_1 = require("@n8n/workflow-sdk");
@@ -68,10 +68,10 @@ const filesystem_1 = require("./filesystem");
 const instance_ai_settings_service_1 = require("./instance-ai-settings.service");
 const instance_ai_adapter_service_1 = require("./instance-ai.adapter.service");
 const internal_messages_1 = require("./internal-messages");
-const typeorm_composite_store_1 = require("./storage/typeorm-composite-store");
 const db_snapshot_storage_1 = require("./storage/db-snapshot-storage");
 const db_iteration_log_storage_1 = require("./storage/db-iteration-log-storage");
-const compaction_service_1 = require("./compaction.service");
+const typeorm_agent_checkpoint_store_1 = require("./storage/typeorm-agent-checkpoint-store");
+const typeorm_agent_memory_1 = require("./storage/typeorm-agent-memory");
 const proxy_token_manager_1 = require("../../services/proxy-token-manager");
 const instance_ai_thread_repository_1 = require("./repositories/instance-ai-thread.repository");
 const trace_replay_state_1 = require("./trace-replay-state");
@@ -80,7 +80,89 @@ const run_trace_metadata_1 = require("./run-trace-metadata");
 function getErrorMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }
+function isTelemetryConfigurableAgent(agent) {
+    return (typeof agent === 'object' &&
+        agent !== null &&
+        typeof Reflect.get(agent, 'telemetry') === 'function');
+}
+const INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS = 30 * 1000;
+function isTextMessagePart(part) {
+    return (typeof part === 'object' &&
+        part !== null &&
+        'type' in part &&
+        part.type === 'text' &&
+        'text' in part &&
+        typeof part.text === 'string');
+}
 const ORCHESTRATOR_AGENT_ID = 'agent-001';
+const SANDBOX_NAME_MAX_LEN = 63;
+const SANDBOX_LABEL_MAX_LEN = 63;
+const NAME_PREFIX_SLUG_MAX_LEN = 24;
+const SHORT_RUN_ID_LEN = 8;
+const DEFAULT_SANDBOX_TTL_MS = 15 * 60 * 1000;
+function slugifySandboxName(value, maxLen) {
+    const slug = value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-+|-+$/g, '');
+    return slug.slice(0, maxLen).replace(/-+$/, '');
+}
+function slugifySandboxLabel(value, maxLen) {
+    return value
+        .replace(/[^A-Za-z0-9_.-]+/g, '-')
+        .replace(/^[-.]+|[-.]+$/g, '')
+        .slice(0, maxLen)
+        .replace(/[-.]+$/, '');
+}
+function getThreadScopedSandboxName(threadId) {
+    return `instance-ai-thread-${threadId}`;
+}
+function buildThreadScopedSandboxName(threadId, namePrefix, runId) {
+    const parts = [];
+    if (namePrefix) {
+        const prefixSlug = slugifySandboxName(namePrefix, NAME_PREFIX_SLUG_MAX_LEN);
+        if (prefixSlug)
+            parts.push(prefixSlug);
+    }
+    if (runId) {
+        const runSlug = slugifySandboxName(runId, SHORT_RUN_ID_LEN);
+        if (runSlug)
+            parts.push(runSlug);
+    }
+    const threadSlug = slugifySandboxName(getThreadScopedSandboxName(threadId), SANDBOX_NAME_MAX_LEN);
+    if (threadSlug)
+        parts.push(threadSlug);
+    const name = slugifySandboxName(parts.join('-'), SANDBOX_NAME_MAX_LEN);
+    if (!name)
+        throw new n8n_workflow_1.UnexpectedError('Failed to build thread-scoped sandbox name');
+    return name;
+}
+function buildThreadScopedSandboxLabels(threadId, namePrefix, runId) {
+    const baseName = getThreadScopedSandboxName(threadId);
+    const labels = {
+        'n8n-builder': slugifySandboxLabel(baseName, SANDBOX_LABEL_MAX_LEN),
+        thread_id: slugifySandboxLabel(threadId, SANDBOX_LABEL_MAX_LEN),
+    };
+    if (namePrefix)
+        labels.name_prefix = slugifySandboxLabel(namePrefix, SANDBOX_LABEL_MAX_LEN);
+    if (runId)
+        labels.run_id = slugifySandboxLabel(runId, SANDBOX_LABEL_MAX_LEN);
+    return labels;
+}
+function withThreadScopedSandboxIdentity(config, threadId, runId) {
+    if (!config.enabled || config.provider !== 'daytona')
+        return config;
+    const name = buildThreadScopedSandboxName(threadId, config.namePrefix, runId);
+    return {
+        ...config,
+        id: name,
+        name,
+        labels: {
+            ...buildThreadScopedSandboxLabels(threadId, config.namePrefix, runId),
+            ...config.labels,
+        },
+    };
+}
 function getUserFacingErrorMessage(error) {
     if (error instanceof n8n_workflow_1.UserError) {
         return error.message;
@@ -149,9 +231,6 @@ function getAbortReason(signal) {
 }
 const INSTANCE_AI_FEEDBACK_NAMESPACE = 'c5be4c87-5b6e-49ed-afe1-9c5c1f99a5c0';
 const MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD = 5;
-function estimateTokens(text) {
-    return Math.ceil(text.length / 4);
-}
 function stringifyForContextValue(value) {
     if (typeof value === 'string')
         return value;
@@ -220,6 +299,8 @@ function toConfirmationData(request) {
             return { approved: true, domainAccessAction: request.domainAccessAction };
         case 'domainAccessDeny':
             return { approved: false };
+        case 'planDeny':
+            return { approved: false, denied: true };
         case 'questions':
             return { approved: true, answers: request.answers };
         case 'credentialSelection':
@@ -244,12 +325,19 @@ function toConfirmationData(request) {
     }
 }
 let InstanceAiService = class InstanceAiService {
-    constructor(logger, globalConfig, adapterService, eventBus, settingsService, compositeStore, compactionService, aiService, push, threadRepo, urlService, dbSnapshotStorage, dbIterationLogStorage, sourceControlPreferencesService, telemetry, userRepository, aiBuilderTemporaryWorkflowRepository, errorReporter, ssrfProtectionConfig, ssrfProtectionService, eventService) {
+    get mcpClientManager() {
+        if (!this._mcpClientManager) {
+            this._mcpClientManager = new instance_ai_1.McpClientManager(this._ssrfProtectionConfig.enabled ? this._ssrfProtectionService : undefined);
+        }
+        return this._mcpClientManager;
+    }
+    constructor(logger, globalConfig, instanceSettings, adapterService, eventBus, settingsService, agentMemory, checkpointStore, aiService, push, threadRepo, urlService, dbSnapshotStorage, dbIterationLogStorage, sourceControlPreferencesService, telemetry, userRepository, aiBuilderTemporaryWorkflowRepository, errorReporter, ssrfProtectionConfig, ssrfProtectionService, eventService) {
+        this.instanceSettings = instanceSettings;
         this.adapterService = adapterService;
         this.eventBus = eventBus;
         this.settingsService = settingsService;
-        this.compositeStore = compositeStore;
-        this.compactionService = compactionService;
+        this.agentMemory = agentMemory;
+        this.checkpointStore = checkpointStore;
         this.aiService = aiService;
         this.push = push;
         this.threadRepo = threadRepo;
@@ -266,6 +354,7 @@ let InstanceAiService = class InstanceAiService {
         this.backgroundTasks = new instance_ai_1.BackgroundTaskManager(MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD);
         this.traceContextsByRunId = new Map();
         this.sandboxes = new Map();
+        this.sandboxCreations = new Map();
         this.gatewayRegistry = new filesystem_1.LocalGatewayRegistry();
         this.domainAccessTrackersByThread = new Map();
         this.threadPushRef = new Map();
@@ -274,6 +363,7 @@ let InstanceAiService = class InstanceAiService {
         this.pendingTerminalOutcomes = new Map();
         this.creditedThreads = new Set();
         this.traceReplay = new trace_replay_state_1.TraceReplayState();
+        this.checkpointPruningStopped = true;
         this.logger = logger.scoped('instance-ai');
         this.instanceAiConfig = globalConfig.instanceAi;
         const livenessPolicyConfig = (0, instance_ai_1.createInstanceAiLivenessPolicyConfig)({
@@ -290,26 +380,30 @@ let InstanceAiService = class InstanceAiService {
                 void this.finalizeCancelledSuspendedRun(suspended, reason);
             },
         });
-        this.builderSandboxSessions = new instance_ai_1.BuilderSandboxSessionRegistry(this.instanceAiConfig.builderSandboxTtlMs);
         this.defaultTimeZone = globalConfig.generic.timezone;
         const restEndpoint = globalConfig.endpoints.rest;
         this.oauth2CallbackUrl = `${this.urlService.getInstanceBaseUrl()}/${restEndpoint}/oauth2-credential/callback`;
         this.webhookBaseUrl = `${this.urlService.getWebhookBaseUrl()}${globalConfig.endpoints.webhook}`;
         this.formBaseUrl = `${this.urlService.getWebhookBaseUrl()}${globalConfig.endpoints.form}`;
-        this.mcpClientManager = new instance_ai_1.McpClientManager(ssrfProtectionConfig.enabled ? ssrfProtectionService : undefined);
+        this._ssrfProtectionConfig = ssrfProtectionConfig;
+        this._ssrfProtectionService = ssrfProtectionService;
         this.eventService.on('instance-ai-settings-updated', ({ mcpSettingsChanged }) => {
             if (!mcpSettingsChanged)
                 return;
-            this.mcpClientManager.disconnect().catch((error) => {
+            if (!this._mcpClientManager)
+                return;
+            this._mcpClientManager.disconnect().catch((error) => {
                 this.logger.warn('Failed to disconnect MCP clients after settings change', {
                     error: getErrorMessage(error),
                 });
             });
         });
         this.liveness.start();
+        if (this.instanceSettings.isLeader)
+            this.startCheckpointPruning();
     }
     getSandboxConfigFromEnv() {
-        const { sandboxEnabled, sandboxProvider, daytonaApiUrl, daytonaApiKey, n8nSandboxServiceUrl, n8nSandboxServiceApiKey, sandboxImage, sandboxTimeout, } = this.instanceAiConfig;
+        const { sandboxEnabled, sandboxProvider, daytonaApiUrl, daytonaApiKey, n8nSandboxServiceUrl, n8nSandboxServiceApiKey, sandboxImage, sandboxTimeout, sandboxNamePrefix, daytonaTokenRefreshSkewMs, } = this.instanceAiConfig;
         if (!sandboxEnabled) {
             return {
                 enabled: false,
@@ -330,6 +424,8 @@ let InstanceAiService = class InstanceAiService {
                 image: sandboxImage || undefined,
                 n8nVersion: constants_1.N8N_VERSION || undefined,
                 timeout: sandboxTimeout,
+                namePrefix: sandboxNamePrefix || undefined,
+                refreshSkewMs: daytonaTokenRefreshSkewMs,
             };
         }
         if (sandboxProvider === 'n8n-sandbox') {
@@ -359,6 +455,7 @@ let InstanceAiService = class InstanceAiService {
                     ...base,
                     daytonaApiUrl: client.getSandboxProxyBaseUrl(),
                     image: proxyConfig.image,
+                    logger: this.logger,
                     getAuthToken: async () => {
                         const token = await client.getBuilderApiProxyToken({ id: user.id }, { userMessageId: (0, nanoid_1.nanoid)() });
                         return token.accessToken;
@@ -382,47 +479,152 @@ let InstanceAiService = class InstanceAiService {
         }
         return base;
     }
-    async createBuilderFactory(user) {
-        const config = await this.resolveSandboxConfig(user);
-        if (!config.enabled)
-            return undefined;
-        if (config.provider === 'daytona') {
-            return new instance_ai_1.BuilderSandboxFactory(config, new instance_ai_1.SnapshotManager(config.image, this.logger, config.n8nVersion, this.errorReporter), this.logger, this.errorReporter);
+    async getOrCreateWorkspaceEntry(threadId, user, runId) {
+        const existing = this.sandboxes.get(threadId);
+        if (existing) {
+            if (this.isSandboxEntryExpired(existing) && !this.isSandboxInUse(threadId)) {
+                this.evictSandboxEntry(threadId, existing);
+            }
+            else {
+                this.touchSandboxEntry(threadId, existing);
+                return existing;
+            }
+        }
+        const pending = this.sandboxCreations.get(threadId);
+        if (pending)
+            return await pending;
+        const creation = this.createWorkspaceEntry(threadId, user, runId);
+        this.sandboxCreations.set(threadId, creation);
+        try {
+            return await creation;
+        }
+        finally {
+            this.sandboxCreations.delete(threadId);
         }
-        return new instance_ai_1.BuilderSandboxFactory(config, undefined, this.logger);
     }
-    async getOrCreateWorkspace(threadId, user) {
-        const existing = this.sandboxes.get(threadId);
-        if (existing)
-            return existing;
-        const config = await this.resolveSandboxConfig(user);
+    async getOrCreateWorkspace(threadId, user, context, runId) {
+        const entry = await this.getOrCreateWorkspaceEntry(threadId, user, runId);
+        if (entry)
+            await this.ensureWorkspaceSetup(entry, context);
+        return entry;
+    }
+    async ensureWorkspaceSetup(entry, context) {
+        if (entry.setupComplete)
+            return;
+        entry.setupPromise ??= (0, instance_ai_1.setupSandboxWorkspace)(entry.workspace, context)
+            .then(() => {
+            entry.setupComplete = true;
+        })
+            .finally(() => {
+            entry.setupPromise = undefined;
+        });
+        await entry.setupPromise;
+    }
+    async createWorkspaceEntry(threadId, user, runId) {
+        const config = withThreadScopedSandboxIdentity(await this.resolveSandboxConfig(user), threadId, runId);
         if (!config.enabled)
             return undefined;
-        const sandbox = await (0, instance_ai_1.createSandbox)(config);
+        const sandbox = await (0, instance_ai_1.createSandbox)(config, {
+            logger: this.logger,
+            errorReporter: this.errorReporter,
+            useSnapshotFallback: true,
+        });
         const workspace = (0, instance_ai_1.createWorkspace)(sandbox);
         if (!sandbox || !workspace)
             return undefined;
-        const entry = { sandbox, workspace };
+        try {
+            await workspace.init();
+        }
+        catch (error) {
+            try {
+                await workspace.destroy();
+            }
+            catch {
+            }
+            throw error;
+        }
+        const entry = {
+            sandbox,
+            workspace,
+            setupComplete: false,
+            setupPromise: undefined,
+            expiresAt: this.nextSandboxExpiry(),
+        };
         this.sandboxes.set(threadId, entry);
+        this.scheduleSandboxExpiry(threadId, entry);
         return entry;
     }
-    async destroySandbox(threadId) {
+    evictSandboxEntry(threadId, entry) {
+        if (this.sandboxes.get(threadId) !== entry)
+            return;
+        this.sandboxes.delete(threadId);
+        if (entry.cleanupTimer) {
+            clearTimeout(entry.cleanupTimer);
+            entry.cleanupTimer = undefined;
+        }
+    }
+    async destroySandbox(threadId, reason = 'thread_cleanup') {
         const entry = this.sandboxes.get(threadId);
         if (!entry?.sandbox)
             return;
-        this.sandboxes.delete(threadId);
+        this.evictSandboxEntry(threadId, entry);
         try {
-            if ('destroy' in entry.sandbox && typeof entry.sandbox.destroy === 'function') {
-                await entry.sandbox.destroy();
-            }
+            await entry.workspace?.destroy();
         }
         catch (error) {
             this.logger.warn('Failed to destroy sandbox', {
                 threadId,
+                reason,
                 error: error instanceof Error ? error.message : String(error),
             });
         }
     }
+    get sandboxTtlMs() {
+        return this.instanceAiConfig?.builderSandboxTtlMs ?? DEFAULT_SANDBOX_TTL_MS;
+    }
+    nextSandboxExpiry() {
+        return Date.now() + this.sandboxTtlMs;
+    }
+    isSandboxEntryExpired(entry) {
+        return this.sandboxTtlMs > 0 && entry.expiresAt <= Date.now();
+    }
+    touchSandboxEntry(threadId, entry) {
+        if (this.sandboxTtlMs <= 0)
+            return;
+        entry.expiresAt = this.nextSandboxExpiry();
+        this.scheduleSandboxExpiry(threadId, entry);
+    }
+    isSandboxInUse(threadId) {
+        return Boolean(this.runState.getActiveRunId(threadId) ||
+            this.runState.hasSuspendedRun(threadId) ||
+            this.backgroundTasks.getRunningTasks(threadId).length > 0);
+    }
+    scheduleSandboxExpiry(threadId, entry) {
+        if (this.sandboxTtlMs <= 0)
+            return;
+        if (entry.cleanupTimer)
+            clearTimeout(entry.cleanupTimer);
+        const delay = Math.max(0, entry.expiresAt - Date.now());
+        entry.cleanupTimer = setTimeout(() => {
+            const current = this.sandboxes.get(threadId);
+            if (current !== entry)
+                return;
+            if (this.isSandboxInUse(threadId)) {
+                this.touchSandboxEntry(threadId, entry);
+                return;
+            }
+            this.evictSandboxEntry(threadId, entry);
+        }, delay);
+        entry.cleanupTimer.unref();
+    }
+    stopSandboxExpiryTimers() {
+        for (const entry of this.sandboxes.values()) {
+            if (!entry.cleanupTimer)
+                continue;
+            clearTimeout(entry.cleanupTimer);
+            entry.cleanupTimer = undefined;
+        }
+    }
     async getProxyAuth(user) {
         const client = await this.aiService.getClient();
         const token = await client.getBuilderApiProxyToken({ id: user.id }, { userMessageId: (0, nanoid_1.nanoid)() });
@@ -446,7 +648,7 @@ let InstanceAiService = class InstanceAiService {
         return await this.settingsService.resolveModelConfig(user);
     }
     async resolveProxyModel(user, proxyBaseUrl, tokenManager) {
-        const modelName = await this.settingsService.resolveModelName(user);
+        const modelName = this.settingsService.resolveModelName(user);
         const { createAnthropic } = await Promise.resolve().then(() => __importStar(require('@ai-sdk/anthropic')));
         const provider = createAnthropic({
             baseURL: proxyBaseUrl + '/anthropic/v1',
@@ -561,6 +763,42 @@ let InstanceAiService = class InstanceAiService {
     getTraceContext(runId) {
         return this.traceContextsByRunId.get(runId)?.tracing;
     }
+    getTraceContextForContinuation(threadId, messageGroupId) {
+        const entries = [...this.traceContextsByRunId.values()].reverse();
+        const sameGroup = messageGroupId === undefined
+            ? undefined
+            : entries.find((entry) => entry.threadId === threadId && entry.messageGroupId === messageGroupId)?.tracing;
+        return sameGroup ?? entries.find((entry) => entry.threadId === threadId)?.tracing;
+    }
+    async createOrchestratorResumeTraceContext(options) {
+        const baseTracing = options.baseTracing ??
+            this.getTraceContextForContinuation(options.threadId, options.messageGroupId);
+        if (!baseTracing)
+            return undefined;
+        const tracing = await (0, instance_ai_1.continueInstanceAiTraceContext)(baseTracing, {
+            threadId: options.threadId,
+            messageId: options.messageId,
+            messageGroupId: options.messageGroupId,
+            runId: options.runId,
+            userId: options.userId,
+            modelId: options.modelId,
+            input: options.input,
+            proxyConfig: options.proxyConfig ?? baseTracing?.proxyConfig,
+            metadata: {
+                resume_reason: options.resumeReason,
+                agent_id: ORCHESTRATOR_AGENT_ID,
+                ...options.metadata,
+            },
+            n8nVersion: constants_1.N8N_VERSION,
+            workflowSdkVersion: constants_1.WORKFLOW_SDK_VERSION,
+        });
+        if (tracing) {
+            await this.configureTraceReplayMode(tracing);
+            this.storeTraceContext(options.runId, options.threadId, tracing, options.messageGroupId);
+            this.runState.attachTracing(options.threadId, tracing);
+        }
+        return tracing;
+    }
     async configureTraceReplayMode(tracing) {
         await this.traceReplay.configureReplayMode(tracing);
     }
@@ -645,6 +883,20 @@ let InstanceAiService = class InstanceAiService {
         if (!traceContext)
             return;
         try {
+            if (traceContext.actorRun.id !== traceContext.rootRun.id &&
+                traceContext.actorRun.endTime === undefined) {
+                await traceContext.finishRun(traceContext.actorRun, {
+                    outputs: {
+                        status: options.status,
+                        ...options.outputs,
+                    },
+                    metadata: {
+                        final_status: options.status,
+                        ...options.metadata,
+                    },
+                    ...(options.error ? { error: options.error } : {}),
+                });
+            }
             await traceContext.finishRun(traceContext.rootRun, {
                 outputs: {
                     status: options.status,
@@ -671,7 +923,9 @@ let InstanceAiService = class InstanceAiService {
     async finalizeRunTracing(runId, tracing, options) {
         if (!tracing)
             return;
-        const outputs = {
+        if (tracing.actorRun.endTime)
+            return;
+        const outputs = options.outputs ?? {
             status: options.status,
             runId,
             ...(options.outputText ? { response: options.outputText } : {}),
@@ -680,6 +934,7 @@ let InstanceAiService = class InstanceAiService {
         const metadata = {
             final_status: options.status,
             ...(options.modelId !== undefined ? { model_id: options.modelId } : {}),
+            ...options.metadata,
         };
         try {
             await tracing.finishRun(tracing.actorRun, {
@@ -855,26 +1110,6 @@ let InstanceAiService = class InstanceAiService {
             void this.handlePlannedTaskSettlement(user, task, 'cancelled');
         }
     }
-    async revalidateActiveUser(userId) {
-        try {
-            const user = await this.userRepository.findOne({
-                where: { id: userId },
-                relations: ['role'],
-            });
-            if (!user || user.disabled)
-                return null;
-            if (!(0, permissions_1.hasGlobalScope)(user, 'instanceAi:message'))
-                return null;
-            return user;
-        }
-        catch (error) {
-            this.logger.warn('Failed to revalidate user', {
-                userId,
-                error: getErrorMessage(error),
-            });
-            return null;
-        }
-    }
     cancelAllBackgroundTasks() {
         const cancelled = this.backgroundTasks.cancelAll();
         for (const task of cancelled) {
@@ -920,20 +1155,16 @@ let InstanceAiService = class InstanceAiService {
                 goal: 'Simulate a stuck background task timeout',
             },
         });
-        const memoryStorage = this.compositeStore.stores.memory;
-        await memoryStorage.saveMessages({
+        await this.agentMemory.saveMessages({
+            threadId,
+            resourceId: user.id,
             messages: [
                 {
                     id: messageId,
-                    threadId,
-                    resourceId: user.id,
-                    role: 'assistant',
-                    content: {
-                        format: 2,
-                        parts: [{ type: 'text', text: messageText }],
-                        content: messageText,
-                    },
                     createdAt: new Date(),
+                    type: 'llm',
+                    role: 'assistant',
+                    content: [{ type: 'text', text: messageText }],
                 },
             ],
         });
@@ -1054,6 +1285,7 @@ let InstanceAiService = class InstanceAiService {
         this.gatewayRegistry.clearDisconnectTimer(userId);
     }
     async clearThreadState(threadId) {
+        this.liveness.clearThreadState(threadId);
         const { active, suspended } = this.runState.clearThread(threadId);
         if (active) {
             active.abortController.abort();
@@ -1080,16 +1312,15 @@ let InstanceAiService = class InstanceAiService {
         });
         this.creditedThreads.delete(threadId);
         this.schedulerLocks.delete(threadId);
-        this.liveness.clearThreadState(threadId);
         this.domainAccessTrackersByThread.delete(threadId);
         this.threadPushRef.delete(threadId);
         this.deleteTraceContextsForThread(threadId);
-        await this.builderSandboxSessions.cleanupThread(threadId, 'thread_cleared');
         await this.destroySandbox(threadId);
         await this.reapAiTemporaryForThreadCleanup(threadId);
         this.eventBus.clearThread(threadId);
     }
     async shutdown() {
+        this.stopCheckpointPruning();
         this.liveness.shutdown();
         const { activeRuns, suspendedRuns } = this.runState.shutdown();
         for (const run of activeRuns) {
@@ -1119,38 +1350,68 @@ let InstanceAiService = class InstanceAiService {
             });
         }
         this.gatewayRegistry.disconnectAll();
-        const sandboxCleanups = [...this.sandboxes.keys()].map(async (threadId) => await this.destroySandbox(threadId));
-        await Promise.allSettled([
-            ...sandboxCleanups,
-            this.builderSandboxSessions.cleanupAll('service_shutdown'),
-        ]);
+        this.stopSandboxExpiryTimers();
         this.domainAccessTrackersByThread.clear();
         this.traceContextsByRunId.clear();
         this.eventBus.clear();
-        await this.mcpClientManager.disconnect();
+        await this._mcpClientManager?.disconnect();
         this.logger.debug('Instance AI service shut down');
     }
-    createMemoryConfig() {
+    startCheckpointPruning() {
+        if (this.checkpointPruneTimer || this.instanceAiConfig.snapshotPruneInterval <= 0)
+            return;
+        this.checkpointPruningStopped = false;
+        this.scheduleCheckpointPrune(0);
+    }
+    stopCheckpointPruning() {
+        this.checkpointPruningStopped = true;
+        clearTimeout(this.checkpointPruneTimer);
+        this.checkpointPruneTimer = undefined;
+    }
+    scheduleCheckpointPrune(delayMs = this.instanceAiConfig.snapshotPruneInterval) {
+        if (this.checkpointPruningStopped)
+            return;
+        this.checkpointPruneTimer = setTimeout(() => {
+            void this.pruneStaleCheckpoints();
+        }, delayMs);
+        this.checkpointPruneTimer.unref();
+    }
+    async pruneStaleCheckpoints(now = Date.now()) {
+        const olderThan = new Date(now - this.instanceAiConfig.snapshotRetention);
+        try {
+            const count = await this.checkpointStore.deleteOlderThan(olderThan);
+            if (count > 0) {
+                this.logger.info('Deleted stale Instance AI checkpoints', { count });
+            }
+            else {
+                this.logger.debug('No stale Instance AI checkpoints to delete');
+            }
+            this.scheduleCheckpointPrune();
+        }
+        catch (error) {
+            this.logger.warn('Failed to delete stale Instance AI checkpoints', {
+                error: getErrorMessage(error),
+            });
+            this.scheduleCheckpointPrune(INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS);
+        }
+    }
+    createAgentMemoryOptions() {
         return {
-            storage: this.compositeStore,
-            embedderModel: this.instanceAiConfig.embedderModel || undefined,
             lastMessages: this.instanceAiConfig.lastMessages,
-            semanticRecallTopK: this.instanceAiConfig.semanticRecallTopK,
+            observationalMemory: {
+                observerThresholdTokens: this.instanceAiConfig.observerMessageTokens,
+                reflectorThresholdTokens: this.instanceAiConfig.reflectorObservationTokens,
+            },
         };
     }
     async ensureThreadExists(memory, threadId, resourceId) {
-        const existingThread = await memory.getThreadById({ threadId });
+        const existingThread = await memory.getThread(threadId);
         if (existingThread)
             return;
-        const now = new Date();
         await memory.saveThread({
-            thread: {
-                id: threadId,
-                resourceId,
-                title: '',
-                createdAt: now,
-                updatedAt: now,
-            },
+            id: threadId,
+            resourceId,
+            title: '',
         });
     }
     projectPlannedTaskList(graph) {
@@ -1210,8 +1471,8 @@ let InstanceAiService = class InstanceAiService {
         return `<planned-task-follow-up type="${type}">\n${JSON.stringify(payload, null, 2)}\n</planned-task-follow-up>\n\n${internal_messages_1.AUTO_FOLLOW_UP_MESSAGE}`;
     }
     async createPlannedTaskState() {
-        const memory = (0, instance_ai_1.createMemory)(this.createMemoryConfig());
-        const taskStorage = new instance_ai_1.MastraTaskStorage(memory);
+        const memory = this.agentMemory;
+        const taskStorage = new instance_ai_1.ThreadTaskStorage(memory);
         const plannedTaskStorage = new instance_ai_1.PlannedTaskStorage(memory);
         const plannedTaskService = new instance_ai_1.PlannedTaskCoordinator(plannedTaskStorage);
         return { memory, taskStorage, plannedTaskService };
@@ -1279,7 +1540,7 @@ let InstanceAiService = class InstanceAiService {
         }
     }
     createTerminalOutcomeStorage() {
-        this.terminalOutcomeStorage ??= new instance_ai_1.TerminalOutcomeStorage((0, instance_ai_1.createMemory)(this.createMemoryConfig()));
+        this.terminalOutcomeStorage ??= new instance_ai_1.TerminalOutcomeStorage(this.agentMemory);
         return this.terminalOutcomeStorage;
     }
     async finishInvalidConfirmationRun(args) {
@@ -1536,7 +1797,8 @@ let InstanceAiService = class InstanceAiService {
         }
     }
     async createExecutionEnvironment(user, threadId, runId, abortSignal, messageGroupId, pushRef) {
-        const localGatewayDisabledGlobally = this.settingsService.getAdminSettings().localGatewayDisabled;
+        const adminSettings = this.settingsService.getAdminSettings();
+        const localGatewayDisabledGlobally = adminSettings.localGatewayDisabled;
         const localGatewayDisabledForUser = await this.settingsService.isLocalGatewayDisabledForUser(user.id);
         const userGateway = this.gatewayRegistry.findGateway(user.id);
         let searchProxyConfig;
@@ -1609,9 +1871,9 @@ let InstanceAiService = class InstanceAiService {
         const modelId = proxyBaseUrl && tokenManager
             ? await this.resolveProxyModel(user, proxyBaseUrl, tokenManager)
             : await this.resolveAgentModelConfig(user);
-        const memory = (0, instance_ai_1.createMemory)(this.createMemoryConfig());
+        const memory = this.agentMemory;
         await this.ensureThreadExists(memory, threadId, user.id);
-        const taskStorage = new instance_ai_1.MastraTaskStorage(memory);
+        const taskStorage = new instance_ai_1.ThreadTaskStorage(memory);
         const iterationLog = this.dbIterationLogStorage;
         const snapshotStorage = this.dbSnapshotStorage;
         const workflowLoopStorage = new instance_ai_1.WorkflowLoopStorage(memory);
@@ -1623,7 +1885,35 @@ let InstanceAiService = class InstanceAiService {
             (0, workflow_sdk_1.setSchemaBaseDirs)(nodeDefDirs);
         }
         const domainTools = (0, instance_ai_1.createAllTools)(context);
-        const sandboxEntry = await this.getOrCreateWorkspace(threadId, user);
+        const baseRuntimeSkills = (0, instance_ai_1.loadInstanceAiRuntimeSkillSource)();
+        let runtimeSkills = baseRuntimeSkills;
+        let runtimeWorkspace;
+        if (adminSettings.sandboxEnabled) {
+            let sandboxEntryPromise;
+            const getSandboxEntry = async () => {
+                sandboxEntryPromise ??= this.getOrCreateWorkspaceEntry(threadId, user, runId).catch((error) => {
+                    sandboxEntryPromise = undefined;
+                    throw error;
+                });
+                return await sandboxEntryPromise;
+            };
+            const getSetupSandboxEntry = async () => {
+                return await this.getOrCreateWorkspace(threadId, user, context, runId);
+            };
+            runtimeWorkspace = (0, instance_ai_1.createLazyRuntimeWorkspace)({
+                ensureWorkspace: async () => (await getSetupSandboxEntry())?.workspace,
+            });
+            const runtimeSkillWorkspace = (0, instance_ai_1.createLazyRuntimeWorkspace)({
+                id: 'instance-ai-runtime-skill-workspace',
+                name: 'Instance AI runtime skill workspace',
+                ensureWorkspace: async () => (await getSandboxEntry())?.workspace,
+            });
+            runtimeSkills = (0, instance_ai_1.createLazyWorkspaceRuntimeSkillSource)({
+                source: baseRuntimeSkills,
+                workspace: runtimeSkillWorkspace,
+                logger: this.logger,
+            });
+        }
         const orchestrationContext = {
             threadId,
             runId,
@@ -1631,7 +1921,7 @@ let InstanceAiService = class InstanceAiService {
             userId: user.id,
             orchestratorAgentId: ORCHESTRATOR_AGENT_ID,
             modelId,
-            storage: this.compositeStore,
+            checkpointStore: this.checkpointStore,
             subAgentMaxSteps: this.instanceAiConfig.subAgentMaxSteps,
             eventBus: this.eventBus,
             logger: this.logger,
@@ -1646,6 +1936,8 @@ let InstanceAiService = class InstanceAiService {
                 ? { name: 'chrome-devtools', command: 'npx', args: ['-y', 'chrome-devtools-mcp@latest'] }
                 : undefined,
             localMcpServer: context.localMcpServer,
+            runtimeSkills,
+            runtimeSkillCatalog: baseRuntimeSkills,
             oauth2CallbackUrl: this.oauth2CallbackUrl,
             webhookBaseUrl: this.webhookBaseUrl,
             formBaseUrl: this.formBaseUrl,
@@ -1672,9 +1964,7 @@ let InstanceAiService = class InstanceAiService {
             iterationLog,
             sendCorrectionToTask: (taskId, correction) => this.sendCorrectionToTask(threadId, taskId, correction),
             workflowTaskService: workflowTasks,
-            workspace: sandboxEntry?.workspace,
-            builderSandboxFactory: await this.createBuilderFactory(user),
-            builderSandboxSessionRegistry: this.builderSandboxSessions,
+            workspace: runtimeWorkspace,
             nodeDefinitionDirs: nodeDefDirs.length > 0 ? nodeDefDirs : undefined,
             domainContext: context,
             tracingProxyConfig,
@@ -1690,7 +1980,6 @@ let InstanceAiService = class InstanceAiService {
             plannedTaskService,
             modelId,
             orchestrationContext,
-            sandboxEntry,
         };
     }
     async dispatchPlannedTask(task, context, graph) {
@@ -1706,21 +1995,6 @@ let InstanceAiService = class InstanceAiService {
                     conversationContext,
                 });
                 break;
-            case 'manage-data-tables':
-                started = await (0, instance_ai_1.startDataTableAgentTask)(taskContext, {
-                    task: task.spec,
-                    plannedTaskId: task.id,
-                    conversationContext,
-                });
-                break;
-            case 'research':
-                started = await (0, instance_ai_1.startResearchAgentTask)(taskContext, {
-                    goal: task.title,
-                    constraints: task.spec,
-                    plannedTaskId: task.id,
-                    conversationContext,
-                });
-                break;
             case 'delegate':
                 started = await (0, instance_ai_1.startDetachedDelegateTask)(taskContext, {
                     title: task.title,
@@ -1823,7 +2097,12 @@ let InstanceAiService = class InstanceAiService {
             messageGroupId,
         });
         const timeZone = this.runState.getTimeZone(threadId) ?? this.defaultTimeZone;
-        void this.executeRun(user, threadId, runId, message, abortController, undefined, messageGroupId, timeZone, isReplanFollowUp, checkpoint);
+        const resumeReason = checkpoint
+            ? 'planned_checkpoint'
+            : isReplanFollowUp
+                ? 'replan'
+                : 'background_task_completed';
+        void this.executeRun(user, threadId, runId, message, abortController, undefined, messageGroupId, timeZone, isReplanFollowUp, checkpoint, resumeReason);
         return runId;
     }
     async schedulePlannedTasks(user, threadId) {
@@ -1898,9 +2177,8 @@ let InstanceAiService = class InstanceAiService {
         }
         await this.doSchedulePlannedTasks(activeUser, threadId);
     }
-    async executeRun(user, threadId, runId, message, abortController, attachments, messageGroupId, timeZone, isReplanFollowUp = false, checkpoint) {
+    async executeRun(user, threadId, runId, message, abortController, attachments, messageGroupId, timeZone, isReplanFollowUp = false, checkpoint, resumeReason) {
         const signal = abortController.signal;
-        let mastraRunId = '';
         let tracing;
         let messageTraceFinalization;
         let aiCreatedWorkflowIds;
@@ -1949,7 +2227,7 @@ let InstanceAiService = class InstanceAiService {
             if (attachments && attachments.length > 0) {
                 context.currentUserAttachments = attachments;
             }
-            const memoryConfig = this.createMemoryConfig();
+            const memoryConfig = this.createAgentMemoryOptions();
             const traceInput = {
                 message,
                 ...(attachments?.length
@@ -1962,29 +2240,48 @@ let InstanceAiService = class InstanceAiService {
                     : {}),
                 ...(messageGroupId ? { messageGroupId } : {}),
             };
-            tracing = await (0, instance_ai_1.createInstanceAiTraceContext)({
-                threadId,
-                messageId,
-                messageGroupId,
-                runId,
-                userId: user.id,
-                modelId,
-                input: traceInput,
-                n8nVersion: constants_1.N8N_VERSION,
-                workflowSdkVersion: constants_1.WORKFLOW_SDK_VERSION,
-                proxyConfig: orchestrationContext.tracingProxyConfig,
-            });
+            tracing = resumeReason
+                ? await this.createOrchestratorResumeTraceContext({
+                    threadId,
+                    messageId,
+                    messageGroupId,
+                    runId,
+                    userId: user.id,
+                    modelId,
+                    input: traceInput,
+                    proxyConfig: orchestrationContext.tracingProxyConfig,
+                    resumeReason,
+                    metadata: {
+                        ...(checkpoint?.isCheckpointFollowUp
+                            ? { checkpoint_task_id: checkpoint.checkpointTaskId }
+                            : {}),
+                    },
+                })
+                : await (0, instance_ai_1.createInstanceAiTraceContext)({
+                    threadId,
+                    messageId,
+                    messageGroupId,
+                    runId,
+                    userId: user.id,
+                    modelId,
+                    input: traceInput,
+                    proxyConfig: orchestrationContext.tracingProxyConfig,
+                    n8nVersion: constants_1.N8N_VERSION,
+                    workflowSdkVersion: constants_1.WORKFLOW_SDK_VERSION,
+                });
             if (!tracing && process.env.E2E_TESTS === 'true') {
                 const { createTraceReplayOnlyContext } = await Promise.resolve().then(() => __importStar(require('@n8n/instance-ai')));
                 tracing = createTraceReplayOnlyContext();
             }
             if (tracing) {
-                await this.configureTraceReplayMode(tracing);
                 orchestrationContext.tracing = tracing;
-                this.runState.attachTracing(threadId, tracing);
-                this.storeTraceContext(runId, threadId, tracing, messageGroupId);
+                if (this.getTraceContext(runId) !== tracing) {
+                    await this.configureTraceReplayMode(tracing);
+                    this.runState.attachTracing(threadId, tracing);
+                    this.storeTraceContext(runId, threadId, tracing, messageGroupId);
+                }
             }
-            const thread = await memory.getThreadById({ threadId });
+            const thread = await memory.getThread(threadId);
             if (thread && !thread.title) {
                 await (0, instance_ai_1.patchThread)(memory, {
                     threadId,
@@ -2000,109 +2297,45 @@ let InstanceAiService = class InstanceAiService {
                     payload: { tasks: existingTasks },
                 });
             }
-            const agent = await (0, instance_ai_1.createInstanceAgent)({
-                modelId,
-                context,
-                orchestrationContext,
-                mcpServers,
-                mcpManager: this.mcpClientManager,
-                memoryConfig,
-                memory,
-                disableDeferredTools: true,
-                timeZone: timeZone ?? this.defaultTimeZone,
-            });
             const enrichedMessage = await this.buildMessageWithRunningTasks(threadId, message);
-            let multimodalAttachments = [];
+            let nonStructuredAttachments = [];
             let attachmentManifest = '';
             let hasParseableAttachment = false;
             if (attachments && attachments.length > 0) {
                 const classifiedAttachments = (0, instance_ai_1.classifyAttachments)(attachments);
-                multimodalAttachments = attachments.filter((attachment) => !(0, instance_ai_1.isParseableAttachment)(attachment) && attachment.mimeType.startsWith('image/'));
+                nonStructuredAttachments = attachments.filter((attachment) => !(0, instance_ai_1.isParseableAttachment)(attachment));
                 hasParseableAttachment = classifiedAttachments.some((attachment) => attachment.parseable);
                 attachmentManifest = (0, instance_ai_1.buildAttachmentManifest)(classifiedAttachments);
             }
-            const messageWithoutSummary = !message && hasParseableAttachment
+            const fullMessage = !message && hasParseableAttachment
                 ? `The user attached file(s) without a message. Inspect the first parseable attachment with parse-file and provide a concise summary.\n\n${attachmentManifest}`
                 : attachmentManifest
                     ? `${enrichedMessage}\n\n${attachmentManifest}`
                     : enrichedMessage;
-            const messageWithoutSummaryTokens = estimateTokens(messageWithoutSummary);
-            this.eventBus.publish(threadId, {
-                type: 'status',
-                runId,
-                agentId: ORCHESTRATOR_AGENT_ID,
-                payload: { message: 'Recalling conversation...' },
-            });
-            const contextCompactionRun = tracing
-                ? await tracing.startChildRun(tracing.actorRun, {
-                    name: 'context_compaction',
-                    tags: ['context'],
-                    metadata: { agent_role: 'context_compaction' },
-                    inputs: {
-                        threadId,
-                        lastMessages: this.instanceAiConfig.lastMessages ?? 20,
-                        currentInputTokens: messageWithoutSummaryTokens,
-                    },
-                })
-                : undefined;
-            let conversationSummary;
-            try {
-                conversationSummary = await this.compactionService.prepareCompactedContext(threadId, memory, modelId, this.instanceAiConfig.lastMessages ?? 20, 0.8, {
-                    label: 'orchestrator-current-input',
-                    text: messageWithoutSummary,
-                });
-                if (contextCompactionRun && tracing) {
-                    await tracing.finishRun(contextCompactionRun, {
-                        outputs: {
-                            summarized: Boolean(conversationSummary),
-                            summary: conversationSummary ?? '',
-                            currentInputTokens: messageWithoutSummaryTokens,
-                        },
-                        metadata: { final_status: 'completed' },
-                    });
-                }
-            }
-            catch (error) {
-                if (contextCompactionRun && tracing) {
-                    await tracing.failRun(contextCompactionRun, error, {
-                        final_status: 'error',
-                    });
-                }
-                throw error;
-            }
-            this.eventBus.publish(threadId, {
-                type: 'status',
-                runId,
-                agentId: ORCHESTRATOR_AGENT_ID,
-                payload: { message: '' },
-            });
             const promptBuildRun = tracing
-                ? await tracing.startChildRun(tracing.actorRun, {
-                    name: 'prompt_build',
+                ? await tracing.startChildRun(tracing.messageRun, {
+                    name: 'prepare: prompt',
+                    canonicalName: 'instance-ai.prompt_build',
                     tags: ['prompt'],
                     metadata: { agent_role: 'prompt_build' },
                     inputs: {
                         message,
-                        hasConversationSummary: Boolean(conversationSummary),
                         attachmentCount: attachments?.length ?? 0,
                     },
                 })
                 : undefined;
             let streamInput;
             try {
-                const fullMessage = conversationSummary
-                    ? `${conversationSummary}\n\n${messageWithoutSummary}`
-                    : messageWithoutSummary;
-                if (multimodalAttachments.length > 0) {
+                if (nonStructuredAttachments.length > 0) {
                     streamInput = [
                         {
                             role: 'user',
                             content: [
                                 { type: 'text', text: fullMessage },
-                                ...multimodalAttachments.map((attachment) => ({
+                                ...nonStructuredAttachments.map((attachment) => ({
                                     type: 'file',
                                     data: attachment.data,
-                                    mimeType: attachment.mimeType,
+                                    mediaType: attachment.mimeType,
                                 })),
                             ],
                         },
@@ -2117,7 +2350,7 @@ let InstanceAiService = class InstanceAiService {
                         : {
                             fullMessage,
                             attachmentCount: attachments?.length ?? 0,
-                            multimodalAttachmentCount: multimodalAttachments.length,
+                            nonStructuredAttachmentCount: nonStructuredAttachments.length,
                         };
                     await tracing.finishRun(promptBuildRun, {
                         outputs: traceOutput,
@@ -2133,14 +2366,41 @@ let InstanceAiService = class InstanceAiService {
                 }
                 throw error;
             }
+            if (tracing && tracing.actorRun.id === tracing.rootRun.id) {
+                const actorRun = await tracing.startChildRun(tracing.rootRun, {
+                    name: 'agent: orchestrator',
+                    canonicalName: 'instance-ai.agent.orchestrator',
+                    tags: ['orchestrator'],
+                    metadata: {
+                        agent_role: 'orchestrator',
+                        agent_id: ORCHESTRATOR_AGENT_ID,
+                        execution_mode: 'foreground',
+                        trace_kind: tracing.traceKind,
+                    },
+                    inputs: traceInput,
+                });
+                tracing.actorRun = actorRun;
+                tracing.orchestratorRun = actorRun;
+            }
+            const agent = await (0, instance_ai_1.createInstanceAgent)({
+                modelId,
+                context,
+                orchestrationContext,
+                mcpServers,
+                mcpManager: this.mcpClientManager,
+                memoryConfig,
+                memory,
+                checkpointStore: this.checkpointStore,
+                timeZone: timeZone ?? this.defaultTimeZone,
+            });
             const result = tracing
-                ? await tracing.withRunTree(tracing.actorRun, async () => {
+                ? await tracing.withActiveSpan(tracing.actorRun, async () => {
                     return await (0, instance_ai_1.streamAgentRun)(agent, streamInput, {
-                        maxSteps: instance_ai_1.MAX_STEPS.ORCHESTRATOR,
+                        maxIterations: instance_ai_1.MAX_STEPS.ORCHESTRATOR,
                         abortSignal: signal,
-                        memory: {
-                            resource: user.id,
-                            thread: threadId,
+                        persistence: {
+                            resourceId: user.id,
+                            threadId,
                         },
                         providerOptions: {
                             anthropic: { cacheControl: { type: 'ephemeral' } },
@@ -2156,11 +2416,11 @@ let InstanceAiService = class InstanceAiService {
                     });
                 })
                 : await (0, instance_ai_1.streamAgentRun)(agent, streamInput, {
-                    maxSteps: instance_ai_1.MAX_STEPS.ORCHESTRATOR,
+                    maxIterations: instance_ai_1.MAX_STEPS.ORCHESTRATOR,
                     abortSignal: signal,
-                    memory: {
-                        resource: user.id,
-                        thread: threadId,
+                    persistence: {
+                        resourceId: user.id,
+                        threadId,
                     },
                     providerOptions: {
                         anthropic: { cacheControl: { type: 'ephemeral' } },
@@ -2174,12 +2434,11 @@ let InstanceAiService = class InstanceAiService {
                     logger: this.logger,
                     onActivity: () => this.runState.touchActiveRun(threadId),
                 });
-            mastraRunId = result.mastraRunId;
             if (result.status === 'suspended') {
                 if (result.suspension) {
                     this.runState.suspendRun(threadId, {
                         runId,
-                        mastraRunId: result.mastraRunId,
+                        agentRunId: result.agentRunId,
                         agent,
                         threadId,
                         user,
@@ -2189,6 +2448,7 @@ let InstanceAiService = class InstanceAiService {
                         messageGroupId,
                         createdAt: Date.now(),
                         tracing,
+                        modelId,
                         checkpoint,
                     });
                 }
@@ -2219,6 +2479,43 @@ let InstanceAiService = class InstanceAiService {
                     this.eventBus.publish(threadId, result.confirmationEvent);
                 }
                 await this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage);
+                const suspensionOutputs = {
+                    status: 'suspended',
+                    runId,
+                    ...(result.suspension?.requestId ? { requestId: result.suspension.requestId } : {}),
+                    ...(result.suspension?.toolCallId
+                        ? { pendingToolCallId: result.suspension.toolCallId }
+                        : {}),
+                    ...(result.suspension?.toolName ? { toolName: result.suspension.toolName } : {}),
+                };
+                await this.finalizeRunTracing(runId, tracing, {
+                    status: 'suspended',
+                    outputs: suspensionOutputs,
+                    metadata: {
+                        completion_source: 'orchestrator',
+                        ...(result.suspension?.requestId ? { request_id: result.suspension.requestId } : {}),
+                        ...(result.suspension?.toolCallId
+                            ? { pending_tool_call_id: result.suspension.toolCallId }
+                            : {}),
+                        ...(result.suspension?.toolName
+                            ? { pending_tool_name: result.suspension.toolName }
+                            : {}),
+                    },
+                });
+                messageTraceFinalization = {
+                    status: 'suspended',
+                    outputs: suspensionOutputs,
+                    metadata: {
+                        completion_source: 'orchestrator',
+                        ...(result.suspension?.requestId ? { request_id: result.suspension.requestId } : {}),
+                        ...(result.suspension?.toolCallId
+                            ? { pending_tool_call_id: result.suspension.toolCallId }
+                            : {}),
+                        ...(result.suspension?.toolName
+                            ? { pending_tool_name: result.suspension.toolName }
+                            : {}),
+                    },
+                };
                 return;
             }
             const outputText = await (result.text ?? Promise.resolve(''));
@@ -2334,9 +2631,6 @@ let InstanceAiService = class InstanceAiService {
                     this.liveness.consumeRunTimeout(runId);
                 }
             }
-            if (!this.runState.hasSuspendedRun(threadId) && mastraRunId) {
-                void this.cleanupMastraSnapshots(mastraRunId);
-            }
             if (!this.runState.hasSuspendedRun(threadId)) {
                 if (checkpoint?.isCheckpointFollowUp) {
                     await this.finalizeCheckpointFollowUp(user, threadId, checkpoint.checkpointTaskId);
@@ -2456,20 +2750,20 @@ let InstanceAiService = class InstanceAiService {
     }
     async resolveConfirmation(requestingUserId, requestId, request) {
         const data = toConfirmationData(request);
-        const revalidated = await this.revalidateActiveUser(requestingUserId);
-        if (!revalidated) {
-            this.logger.warn('Rejecting confirmation: user no longer authorized for AI Assistant', {
-                userId: requestingUserId,
-                requestId,
-            });
+        const freshUser = await this.revalidateActiveUser(requestingUserId);
+        if (!freshUser) {
             this.runState.rejectPendingConfirmation(requestId);
             const suspended = this.runState.findSuspendedByRequestId(requestId);
-            if (suspended && suspended.user.id === requestingUserId) {
+            if (suspended?.user.id === requestingUserId) {
                 this.cancelRun(suspended.threadId);
             }
+            this.logger.warn('Rejecting confirmation: user no longer authorized for AI Assistant', {
+                userId: requestingUserId,
+                requestId,
+            });
             return false;
         }
-        if (this.runState.resolvePendingConfirmation(requestingUserId, requestId, data)) {
+        if (this.runState.resolvePendingConfirmation(freshUser.id, requestId, data)) {
             this.logger.debug('Resolved pending confirmation (sub-agent HITL)', {
                 requestId,
                 approved: data.approved,
@@ -2482,6 +2776,25 @@ let InstanceAiService = class InstanceAiService {
         });
         return await this.resumeSuspendedRun(requestingUserId, requestId, data);
     }
+    async revalidateActiveUser(userId) {
+        try {
+            const user = await this.userRepository.findOne({
+                where: { id: userId },
+                relations: ['role'],
+            });
+            if (!user || user.disabled)
+                return null;
+            const hasInstanceAiMessageScope = user.role?.scopes?.some((scope) => scope.slug === 'instanceAi:message') ?? false;
+            return hasInstanceAiMessageScope ? user : null;
+        }
+        catch (error) {
+            this.logger.warn('Failed to revalidate user', {
+                userId,
+                error: getErrorMessage(error),
+            });
+            return null;
+        }
+    }
     async resumeSuspendedRun(requestingUserId, requestId, data) {
         const suspended = this.runState.findSuspendedByRequestId(requestId);
         if (!suspended) {
@@ -2491,11 +2804,11 @@ let InstanceAiService = class InstanceAiService {
             });
             return false;
         }
-        const { agent, runId, mastraRunId, threadId, user, toolCallId, abortController, tracing, checkpoint, } = suspended;
+        const { agent, runId, agentRunId, threadId, user, toolCallId, abortController, tracing, modelId, messageGroupId, checkpoint, } = suspended;
         if (user.id !== requestingUserId)
             return false;
-        const revalidated = await this.revalidateActiveUser(user.id);
-        if (!revalidated) {
+        const activeUser = await this.revalidateActiveUser(user.id);
+        if (!activeUser) {
             this.logger.warn('Cancelling suspended run: user no longer authorized for AI Assistant', {
                 userId: user.id,
                 threadId,
@@ -2517,16 +2830,41 @@ let InstanceAiService = class InstanceAiService {
             ...(data.answers ? { answers: data.answers } : {}),
             ...(data.resourceDecision ? { resourceDecision: data.resourceDecision } : {}),
         };
+        const resumeTracing = await this.createOrchestratorResumeTraceContext({
+            baseTracing: tracing,
+            threadId,
+            messageId: (0, nanoid_1.nanoid)(),
+            messageGroupId,
+            runId,
+            userId: activeUser.id,
+            modelId,
+            input: {
+                requestId,
+                toolCallId,
+                approved: data.approved,
+                resumeFields: Object.keys(resumeData),
+            },
+            resumeReason: 'approval',
+            metadata: {
+                request_id: requestId,
+                pending_tool_call_id: toolCallId,
+                approved: data.approved,
+                ...(checkpoint?.isCheckpointFollowUp
+                    ? { checkpoint_task_id: checkpoint.checkpointTaskId }
+                    : {}),
+            },
+        });
         void this.processResumedStream(agent, resumeData, {
             runId,
-            mastraRunId,
+            agentRunId,
             threadId,
-            user: revalidated,
+            user: activeUser,
             toolCallId,
             signal: abortController.signal,
             abortController,
             snapshotStorage: this.dbSnapshotStorage,
-            tracing,
+            tracing: resumeTracing ?? tracing,
+            modelId,
             checkpoint,
         });
         return true;
@@ -2534,12 +2872,28 @@ let InstanceAiService = class InstanceAiService {
     async processResumedStream(agent, resumeData, opts) {
         let messageTraceFinalization;
         try {
+            if (opts.tracing?.getTelemetry && isTelemetryConfigurableAgent(agent)) {
+                try {
+                    agent.telemetry(opts.tracing.getTelemetry({
+                        agentRole: 'orchestrator',
+                        functionId: 'instance-ai.orchestrator',
+                        executionMode: opts.tracing.traceKind === 'orchestrator_resume' ? 'resume' : 'foreground',
+                    }));
+                }
+                catch (error) {
+                    this.logger.warn('Failed to configure Instance AI resume tracing', {
+                        error: getErrorMessage(error),
+                        threadId: opts.threadId,
+                        runId: opts.runId,
+                    });
+                }
+            }
             const result = opts.tracing
-                ? await opts.tracing.withRunTree(opts.tracing.actorRun, async () => {
+                ? await opts.tracing.withActiveSpan(opts.tracing.actorRun, async () => {
                     return await (0, instance_ai_1.resumeAgentRun)(agent, resumeData, {
-                        runId: opts.mastraRunId,
+                        runId: opts.agentRunId,
                         toolCallId: opts.toolCallId,
-                        memory: { resource: opts.user.id, thread: opts.threadId },
+                        persistence: { resourceId: opts.user.id, threadId: opts.threadId },
                     }, {
                         threadId: opts.threadId,
                         runId: opts.runId,
@@ -2547,14 +2901,14 @@ let InstanceAiService = class InstanceAiService {
                         signal: opts.signal,
                         eventBus: this.eventBus,
                         logger: this.logger,
-                        mastraRunId: opts.mastraRunId,
+                        agentRunId: opts.agentRunId,
                         onActivity: () => this.runState.touchActiveRun(opts.threadId),
                     });
                 })
                 : await (0, instance_ai_1.resumeAgentRun)(agent, resumeData, {
-                    runId: opts.mastraRunId,
+                    runId: opts.agentRunId,
                     toolCallId: opts.toolCallId,
-                    memory: { resource: opts.user.id, thread: opts.threadId },
+                    persistence: { resourceId: opts.user.id, threadId: opts.threadId },
                 }, {
                     threadId: opts.threadId,
                     runId: opts.runId,
@@ -2562,14 +2916,14 @@ let InstanceAiService = class InstanceAiService {
                     signal: opts.signal,
                     eventBus: this.eventBus,
                     logger: this.logger,
-                    mastraRunId: opts.mastraRunId,
+                    agentRunId: opts.agentRunId,
                     onActivity: () => this.runState.touchActiveRun(opts.threadId),
                 });
             if (result.status === 'suspended') {
                 if (result.suspension) {
                     this.runState.suspendRun(opts.threadId, {
                         runId: opts.runId,
-                        mastraRunId: result.mastraRunId,
+                        agentRunId: result.agentRunId,
                         agent,
                         threadId: opts.threadId,
                         user: opts.user,
@@ -2579,6 +2933,7 @@ let InstanceAiService = class InstanceAiService {
                         messageGroupId: this.traceContextsByRunId.get(opts.runId)?.messageGroupId,
                         createdAt: Date.now(),
                         tracing: opts.tracing,
+                        ...(opts.modelId !== undefined ? { modelId: opts.modelId } : {}),
                         checkpoint: opts.checkpoint,
                     });
                 }
@@ -2607,6 +2962,43 @@ let InstanceAiService = class InstanceAiService {
                     this.eventBus.publish(opts.threadId, result.confirmationEvent);
                 }
                 await this.saveAgentTreeSnapshot(opts.threadId, opts.runId, opts.snapshotStorage);
+                const suspensionOutputs = {
+                    status: 'suspended',
+                    runId: opts.runId,
+                    ...(result.suspension?.requestId ? { requestId: result.suspension.requestId } : {}),
+                    ...(result.suspension?.toolCallId
+                        ? { pendingToolCallId: result.suspension.toolCallId }
+                        : {}),
+                    ...(result.suspension?.toolName ? { toolName: result.suspension.toolName } : {}),
+                };
+                await this.finalizeRunTracing(opts.runId, opts.tracing, {
+                    status: 'suspended',
+                    outputs: suspensionOutputs,
+                    metadata: {
+                        completion_source: 'orchestrator',
+                        ...(result.suspension?.requestId ? { request_id: result.suspension.requestId } : {}),
+                        ...(result.suspension?.toolCallId
+                            ? { pending_tool_call_id: result.suspension.toolCallId }
+                            : {}),
+                        ...(result.suspension?.toolName
+                            ? { pending_tool_name: result.suspension.toolName }
+                            : {}),
+                    },
+                });
+                messageTraceFinalization = {
+                    status: 'suspended',
+                    outputs: suspensionOutputs,
+                    metadata: {
+                        completion_source: 'orchestrator',
+                        ...(result.suspension?.requestId ? { request_id: result.suspension.requestId } : {}),
+                        ...(result.suspension?.toolCallId
+                            ? { pending_tool_call_id: result.suspension.toolCallId }
+                            : {}),
+                        ...(result.suspension?.toolName
+                            ? { pending_tool_name: result.suspension.toolName }
+                            : {}),
+                    },
+                };
                 return;
             }
             const outputText = await (result.text ?? Promise.resolve(''));
@@ -2739,6 +3131,7 @@ let InstanceAiService = class InstanceAiService {
             plannedTaskId: opts.plannedTaskId,
             workItemId: opts.workItemId,
             traceContext: opts.traceContext,
+            createTraceContext: opts.createTraceContext,
             dedupeKey: opts.dedupeKey,
             parentCheckpointId: opts.parentCheckpointId,
             run: opts.run,
@@ -2986,9 +3379,6 @@ let InstanceAiService = class InstanceAiService {
         const archivedWorkflowIds = await this.reapAiTemporaryFromRun(suspended.threadId, suspended.user, undefined);
         this.publishRunFinish(suspended.threadId, suspended.runId, 'cancelled', reason, archivedWorkflowIds);
         await this.saveAgentTreeSnapshot(suspended.threadId, suspended.runId, this.dbSnapshotStorage, true);
-        if (suspended.mastraRunId) {
-            void this.cleanupMastraSnapshots(suspended.mastraRunId);
-        }
         await this.maybeFinalizeRunTraceRoot(suspended.runId, {
             status: 'cancelled',
             reason,
@@ -3078,20 +3468,79 @@ let InstanceAiService = class InstanceAiService {
     }
     async refineTitleIfNeeded(threadId, userId, modelId) {
         try {
-            const memory = (0, instance_ai_1.createMemory)(this.createMemoryConfig());
-            const thread = await memory.getThreadById({ threadId });
+            const memory = this.agentMemory;
+            const thread = await memory.getThread(threadId);
             if (!thread?.title)
                 return;
             if (thread.metadata?.titleRefined)
                 return;
-            const result = await memory.recall({ threadId, resourceId: userId, perPage: 5 });
-            const userTexts = result.messages
-                .filter((m) => m.role === 'user')
-                .map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content)));
+            const history = await memory.getMessages(threadId, { limit: 5 });
+            const userTexts = history.flatMap((m) => {
+                if (!('role' in m) || m.role !== 'user')
+                    return [];
+                const text = this.extractStoredMessageText(m.content);
+                return text.length > 0 ? [text] : [];
+            });
             if (userTexts.length === 0)
                 return;
             const userText = userTexts.join('\n');
-            const llmTitle = await (0, instance_ai_1.generateTitleForRun)(modelId, userText);
+            const baseTracing = this.getTraceContextForContinuation(threadId);
+            const titleTracing = await (0, instance_ai_1.createInternalOperationTraceContext)({
+                threadId,
+                conversationId: threadId,
+                messageId: `internal:title:${threadId}`,
+                runId: `title-${(0, nanoid_1.nanoid)()}`,
+                userId,
+                modelId,
+                operationName: 'thread_title',
+                input: {
+                    message_count: userTexts.length,
+                    source: 'thread_title_refinement',
+                },
+                proxyConfig: baseTracing?.proxyConfig,
+                metadata: {
+                    n8n_version: constants_1.N8N_VERSION || undefined,
+                    operation_name: 'thread_title',
+                    trigger: 'run_completed',
+                },
+            });
+            const titleTelemetry = titleTracing?.getTelemetry?.({
+                agentRole: 'thread_title',
+                functionId: 'instance-ai.thread_title',
+                executionMode: 'internal',
+                metadata: {
+                    operation_name: 'thread_title',
+                },
+            });
+            let llmTitle;
+            if (titleTracing) {
+                try {
+                    llmTitle = await titleTracing.withActiveSpan(titleTracing.rootRun, async () => {
+                        const title = await (0, instance_ai_1.generateTitleForRun)(modelId, userText, {
+                            ...(titleTelemetry ? { telemetry: titleTelemetry } : {}),
+                        });
+                        if (title) {
+                            await titleTracing.finishRun(titleTracing.rootRun, {
+                                outputs: { title },
+                                metadata: { final_status: 'completed' },
+                            });
+                        }
+                        else {
+                            await titleTracing.finishRun(titleTracing.rootRun, {
+                                outputs: { title: null },
+                                metadata: { final_status: 'skipped' },
+                            });
+                        }
+                        return title;
+                    });
+                }
+                finally {
+                    (0, instance_ai_1.releaseTraceClient)(titleTracing.rootRun.traceId);
+                }
+            }
+            else {
+                llmTitle = await (0, instance_ai_1.generateTitleForRun)(modelId, userText);
+            }
             if (!llmTitle)
                 return;
             await (0, instance_ai_1.patchThread)(memory, {
@@ -3115,17 +3564,13 @@ let InstanceAiService = class InstanceAiService {
             });
         }
     }
-    async cleanupMastraSnapshots(mastraRunId) {
-        try {
-            const workflowsStorage = this.compositeStore.stores.workflows;
-            await workflowsStorage.deleteAllByRunId(mastraRunId);
-        }
-        catch (error) {
-            this.logger.warn('Failed to clean up Mastra workflow snapshots', {
-                mastraRunId,
-                error: getErrorMessage(error),
-            });
+    extractStoredMessageText(content) {
+        if (typeof content === 'string')
+            return content;
+        if (Array.isArray(content)) {
+            return content.flatMap((part) => (isTextMessagePart(part) ? [part.text] : [])).join('\n');
         }
+        return '';
     }
     async saveAgentTreeSnapshot(threadId, runId, snapshotStorage, isUpdate = false, overrideMessageGroupId) {
         try {
@@ -3156,6 +3601,8 @@ let InstanceAiService = class InstanceAiService {
             const saveOptions = {
                 messageGroupId,
                 runIds: groupRunIds,
+                traceId: tracing?.rootRun.otelTraceId,
+                spanId: tracing?.rootRun.otelSpanId,
                 langsmithRunId: tracing?.rootRun.id,
                 langsmithTraceId: tracing?.rootRun.traceId,
             };
@@ -3184,15 +3631,28 @@ let InstanceAiService = class InstanceAiService {
     }
 };
 exports.InstanceAiService = InstanceAiService;
+__decorate([
+    (0, decorators_1.OnLeaderTakeover)(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], InstanceAiService.prototype, "startCheckpointPruning", null);
+__decorate([
+    (0, decorators_1.OnLeaderStepdown)(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], InstanceAiService.prototype, "stopCheckpointPruning", null);
 exports.InstanceAiService = InstanceAiService = __decorate([
     (0, di_1.Service)(),
     __metadata("design:paramtypes", [backend_common_1.Logger,
         config_1.GlobalConfig,
+        n8n_core_1.InstanceSettings,
         instance_ai_adapter_service_1.InstanceAiAdapterService,
         in_process_event_bus_1.InProcessEventBus,
         instance_ai_settings_service_1.InstanceAiSettingsService,
-        typeorm_composite_store_1.TypeORMCompositeStore,
-        compaction_service_1.InstanceAiCompactionService,
+        typeorm_agent_memory_1.TypeORMAgentMemory,
+        typeorm_agent_checkpoint_store_1.TypeORMAgentCheckpointStore,
         ai_service_1.AiService,
         push_1.Push,
         instance_ai_thread_repository_1.InstanceAiThreadRepository,