n8n 2.45.4 → 2.46.0

package/dist/memory-guard/memory-guard.service.js

@@ -0,0 +1,453 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryGuardService = void 0;
+const backend_common_1 = require("@n8n/backend-common");
+const backend_services_1 = require("@n8n/backend-services");
+const config_1 = require("@n8n/config");
+const db_1 = require("@n8n/db");
+const decorators_1 = require("@n8n/decorators");
+const di_1 = require("@n8n/di");
+const ensure_error_1 = require("@n8n/utils/errors/ensure-error");
+const bytes_1 = require("@n8n/utils/number/bytes");
+const sleep_1 = require("@n8n/utils/sleep");
+const n8n_workflow_1 = require("n8n-workflow");
+const active_executions_1 = require("../active-executions");
+const execution_recovery_service_1 = require("../executions/execution-recovery.service");
+const user_management_mailer_1 = require("../user-management/email/user-management-mailer");
+const utils_1 = require("../utils");
+const workflow_push_notifier_service_1 = require("../workflows/workflow-push-notifier.service");
+const memory_guard_constants_1 = require("./memory-guard.constants");
+const memory_guard_utils_1 = require("./memory-guard.utils");
+const memory_usage_1 = require("./memory-usage");
+const production_hold_1 = require("./production-hold");
+const run_data_tracker_1 = require("./run-data-tracker");
+const memory_guard_journal_1 = require("./memory-guard-journal");
+const memory_stopped_executions_1 = require("./memory-stopped-executions");
+let MemoryGuardService = class MemoryGuardService {
+    constructor(logger, config, activeExecutions, productionHold, tracker, memoryStoppedExecutions, eventService, journal, workflowRepository, executionRecoveryService, mailer, workflowPushNotifier) {
+        this.logger = logger;
+        this.config = config;
+        this.activeExecutions = activeExecutions;
+        this.productionHold = productionHold;
+        this.tracker = tracker;
+        this.memoryStoppedExecutions = memoryStoppedExecutions;
+        this.eventService = eventService;
+        this.journal = journal;
+        this.workflowRepository = workflowRepository;
+        this.executionRecoveryService = executionRecoveryService;
+        this.mailer = mailer;
+        this.workflowPushNotifier = workflowPushNotifier;
+        this.isShutDown = false;
+        this.lastUtilization = 0;
+        this.holdStreak = 0;
+        this.stopStreak = 0;
+        this.fastSamplingUntil = 0;
+        this.admitBatch = 1;
+        this.lastAdmitAt = 0;
+        this.utilizationAtLastAdmit = 0;
+        this.lastStarvationAdmitAt = 0;
+        this.lastStopEndedAt = 0;
+        this.isStandingDown = false;
+        this.isHoldSuspended = false;
+        this.stopsByWorkflow = new Map();
+        this.deactivating = new Set();
+        this.logger = this.logger.scoped('memory-guard');
+    }
+    init() {
+        this.containerLimit = (0, memory_usage_1.getContainerLimit)();
+        this.forceGc = (0, memory_guard_utils_1.exposeGc)();
+        this.memoryStoppedExecutions.enabled = true;
+        this.eventService.emit('memory-guard-crashes-counted', {
+            crashes: { ...this.journal.crashes },
+        });
+        this.scheduleNext(memory_guard_constants_1.SLOW_INTERVAL_MS);
+    }
+    shutdown() {
+        this.isShutDown = true;
+        clearTimeout(this.timer);
+        this.journal.reset();
+    }
+    scheduleNext(delayMs) {
+        if (this.isShutDown)
+            return;
+        this.timer = setTimeout(() => this.check(), delayMs);
+        this.timer.unref();
+    }
+    check() {
+        try {
+            this.memoryStoppedExecutions.pruneFinishing((id) => this.activeExecutions.has(id));
+            this.evaluate((0, memory_usage_1.readMemoryUsage)(this.containerLimit));
+        }
+        catch (error) {
+            this.logger.error('Memory guard check failed', { error: (0, ensure_error_1.ensureError)(error) });
+        }
+        const isUnderPressure = this.lastUtilization >= memory_guard_constants_1.FAST_SAMPLING_LEVEL ||
+            this.isHolding ||
+            this.productionHold.heldCount > 0 ||
+            this.pendingStop !== undefined ||
+            this.isStandingDown ||
+            Date.now() < this.fastSamplingUntil;
+        this.scheduleNext(isUnderPressure ? memory_guard_constants_1.FAST_INTERVAL_MS : memory_guard_constants_1.SLOW_INTERVAL_MS);
+    }
+    get isHolding() {
+        return this.holdStartedAt !== undefined;
+    }
+    evaluate(usage) {
+        const { utilization } = usage;
+        this.lastUtilization = utilization;
+        if (utilization < Math.min(memory_guard_constants_1.FAST_SAMPLING_LEVEL, this.config.resumeLevel))
+            this.journal.reset();
+        this.holdStreak = utilization >= this.config.holdLevel ? this.holdStreak + 1 : 0;
+        this.stopStreak = utilization >= this.config.stopLevel ? this.stopStreak + 1 : 0;
+        const isCritical = utilization >= this.config.criticalLevel;
+        if (this.isStandingDown && utilization < this.config.resumeLevel) {
+            this.isStandingDown = false;
+            this.logger.info(`Memory fell below ${percent(this.config.resumeLevel)}, so the memory guard again stops executions at ${percent(this.config.stopLevel)}`);
+        }
+        if (this.isHoldSuspended && utilization < this.config.resumeLevel)
+            this.isHoldSuspended = false;
+        const shouldHold = this.holdStreak >= memory_guard_constants_1.SAMPLES_TO_ACT || isCritical;
+        if (!this.isHolding && !this.isHoldSuspended && shouldHold) {
+            this.startHold(usage);
+        }
+        else if (this.isHolding && utilization < this.config.resumeLevel) {
+            const heldMs = this.endHold();
+            this.logger.info(`Memory fell below ${percent(this.config.resumeLevel)} after a hold of ${heldMs}ms, so held executions now start in batches`);
+        }
+        if (this.isHolding) {
+            this.admitBatch = 1;
+            if (!this.pendingStop && !this.hasRunningExecution())
+                this.admitOneHeld();
+        }
+        else {
+            this.admitHeldGradually(utilization);
+        }
+        if (this.pendingStop)
+            return;
+        if (this.isStandingDown) {
+            if (isCritical)
+                this.stop('critical');
+            else if (this.hasHoldLimitPassed())
+                this.suspendHold('The last stop freed less memory than expected');
+            return;
+        }
+        const stop = this.getStop(isCritical);
+        if (!stop)
+            return;
+        const isStopped = this.stop(stop.trigger, stop.usageAfterGc);
+        if (!isStopped && this.hasHoldLimitPassed()) {
+            this.suspendHold('No execution can be stopped to end the hold');
+        }
+    }
+    suspendHold(reason) {
+        this.isHoldSuspended = true;
+        this.endHold();
+        this.logger.warn(`${reason}, so the memory guard ends the hold and starts no new hold until memory falls below ${percent(this.config.resumeLevel)}`);
+    }
+    hasRunningExecution() {
+        return this.activeExecutions.getActiveExecutions().some((e) => e.status === 'running');
+    }
+    admitOneHeld() {
+        const now = Date.now();
+        if (now - this.lastStarvationAdmitAt < memory_guard_constants_1.STARVATION_ADMIT_INTERVAL_MS)
+            return;
+        if (this.productionHold.admit(1) > 0)
+            this.lastStarvationAdmitAt = now;
+    }
+    getStop(isCritical) {
+        if (isCritical)
+            return { trigger: 'critical' };
+        if (this.stopStreak >= memory_guard_constants_1.SAMPLES_TO_ACT && this.findStopTarget()) {
+            const usageAfterGc = this.readUsageAfterGc();
+            if (usageAfterGc.utilization >= this.config.stopLevel)
+                return { trigger: 'threshold', usageAfterGc };
+            this.stopStreak = 0;
+        }
+        if (this.hasHoldLimitPassed())
+            return { trigger: 'hold-limit' };
+        return undefined;
+    }
+    readUsageAfterGc() {
+        this.forceGc?.();
+        return (0, memory_usage_1.readMemoryUsage)(this.containerLimit);
+    }
+    hasHoldLimitPassed() {
+        const { oldestHeldAt } = this.productionHold;
+        if (!this.isHolding || oldestHeldAt === undefined)
+            return false;
+        return Date.now() - Math.max(oldestHeldAt, this.lastStopEndedAt) >= memory_guard_constants_1.HOLD_LIMIT_MS;
+    }
+    admitHeldGradually(utilization) {
+        if (this.productionHold.heldCount === 0) {
+            this.admitBatch = 1;
+            return;
+        }
+        const now = Date.now();
+        if (now - this.lastAdmitAt < memory_guard_constants_1.ADMIT_INTERVAL_MS)
+            return;
+        if (utilization >= this.config.resumeLevel || utilization > this.utilizationAtLastAdmit) {
+            this.admitBatch = 1;
+        }
+        this.productionHold.admit(this.admitBatch);
+        this.admitBatch *= 2;
+        this.lastAdmitAt = now;
+        this.utilizationAtLastAdmit = utilization;
+    }
+    startHold(usage) {
+        this.holdStartedAt = Date.now();
+        this.productionHold.start();
+        this.journal.record('hold');
+        this.logger.warn(`Memory at ${(0, bytes_1.formatBytes)(usage.usedBytes)} of ${(0, bytes_1.formatBytes)(usage.limitBytes)}, holding new production executions`);
+        this.eventService.emit('memory-guard-hold-started', {
+            memoryUsedBytes: usage.usedBytes,
+            memoryLimitBytes: usage.limitBytes,
+        });
+    }
+    endHold() {
+        if (this.holdStartedAt === undefined)
+            return;
+        const heldMs = Date.now() - this.holdStartedAt;
+        this.holdStartedAt = undefined;
+        this.fastSamplingUntil = Date.now() + memory_guard_constants_1.FAST_SAMPLING_AFTER_HOLD_MS;
+        this.productionHold.end();
+        this.eventService.emit('memory-guard-hold-ended', { heldMs });
+        return heldMs;
+    }
+    stop(trigger, usageAfterGc) {
+        const target = this.findStopTarget();
+        if (!target)
+            return false;
+        this.journal.record('stop');
+        const usage = usageAfterGc ?? this.readUsageAfterGc();
+        const engines = [];
+        for (const executionId of target.memberIds) {
+            const execution = this.activeExecutions.getExecutionOrFail(executionId);
+            const estimate = this.tracker.getEstimate(executionId);
+            this.memoryStoppedExecutions.markStopped(executionId, {
+                error: (0, memory_guard_utils_1.buildStopError)(execution.executionData.workflowData, estimate, usage),
+                runningNode: estimate?.runningNode,
+            });
+            const forget = () => this.memoryStoppedExecutions.forget(executionId);
+            engines.push(Promise.resolve(execution.workflowExecution).then(forget, forget));
+        }
+        for (const executionId of target.memberIds) {
+            this.activeExecutions.stopExecution(executionId, new n8n_workflow_1.MemoryPressureExecutionCancelledError(executionId));
+        }
+        this.stopStreak = 0;
+        this.pendingStopWorkflowId = target.workflow.id;
+        this.pendingStop = this.finishStop(target, engines, usage, trigger)
+            .catch((error) => {
+            this.logger.error('The memory guard stopped an execution, but could not check how much memory the stop freed', { error: (0, ensure_error_1.ensureError)(error) });
+        })
+            .finally(() => {
+            this.pendingStop = undefined;
+            this.pendingStopWorkflowId = undefined;
+            this.lastStopEndedAt = Date.now();
+        });
+        return true;
+    }
+    async finishStop(target, engines, before, trigger) {
+        await (0, memory_guard_utils_1.settleWithin)(engines, memory_guard_constants_1.RELEASE_TIMEOUT_MS);
+        const { after, freedMemory } = await this.measureAfterStop(before, target.estimatedBytes);
+        const isCounted = freedMemory &&
+            before.utilization >= this.config.stopLevel &&
+            this.pendingStopWorkflowId === target.workflow.id;
+        const workflowDeactivationStarted = isCounted && this.countStop(target);
+        this.logger.warn(`Stopped execution ${target.rootId} of workflow ${target.workflow.id} at ${(0, bytes_1.formatBytes)(before.usedBytes)} of ${(0, bytes_1.formatBytes)(before.limitBytes)} (${trigger}). It held an estimated ${(0, bytes_1.formatBytes)(target.estimatedBytes)} of run data. Memory after the stop: ${(0, bytes_1.formatBytes)(after.usedBytes)}`, { executionId: target.rootId, workflowId: target.workflow.id });
+        if (!freedMemory && after.utilization >= this.config.stopLevel) {
+            this.isStandingDown = true;
+            this.logger.warn(`The stop freed less memory than expected, so the memory guard now stops executions only at ${percent(this.config.criticalLevel)} or more, until memory falls below ${percent(this.config.resumeLevel)}`);
+        }
+        this.eventService.emit('memory-guard-execution-stopped', {
+            executionId: target.rootId,
+            workflowId: target.workflow.id,
+            mode: target.mode,
+            reason: trigger,
+            runningNodeType: target.runningNodeType,
+            largestDataNodeType: target.largestDataNodeType,
+            estimatedBytes: target.estimatedBytes,
+            memoryUsedBytes: before.usedBytes,
+            memoryLimitBytes: before.limitBytes,
+            freedMemory,
+            workflowDeactivationStarted,
+        });
+    }
+    async measureAfterStop(before, estimatedBytes) {
+        const deadline = Date.now() + memory_guard_constants_1.REMEASURE_TIMEOUT_MS;
+        for (;;) {
+            this.forceGc?.();
+            const after = (0, memory_usage_1.readMemoryUsage)(this.containerLimit);
+            const freedMemory = before.usedBytes - after.usedBytes >= estimatedBytes * memory_guard_constants_1.FREED_SHARE;
+            if (freedMemory || Date.now() >= deadline)
+                return { after, freedMemory };
+            await (0, sleep_1.sleep)(memory_guard_constants_1.REMEASURE_INTERVAL_MS);
+        }
+    }
+    findStopTarget() {
+        const trees = new Map();
+        for (const { id, status } of this.activeExecutions.getActiveExecutions()) {
+            if (status !== 'running')
+                continue;
+            if (this.memoryStoppedExecutions.isStopped(id) ||
+                this.memoryStoppedExecutions.isFinishing(id))
+                continue;
+            if (!this.activeExecutions.getExecutionOrFail(id).workflowExecution)
+                continue;
+            const { rootId, depth } = this.findRoot(id);
+            const members = trees.get(rootId) ?? [];
+            members.push({ executionId: id, depth });
+            trees.set(rootId, members);
+        }
+        let target;
+        for (const [rootId, members] of trees) {
+            let estimatedBytes = 0;
+            let largest;
+            let largestNodeData;
+            for (const { executionId } of members) {
+                const estimate = this.tracker.getEstimate(executionId);
+                if (!estimate)
+                    continue;
+                estimatedBytes += estimate.totalBytes;
+                if (!largest || estimate.totalBytes > largest.estimate.totalBytes) {
+                    largest = { executionId, estimate };
+                }
+                const node = (0, memory_guard_utils_1.largestNode)(estimate);
+                if (node && (!largestNodeData || node.bytes > largestNodeData.bytes)) {
+                    largestNodeData = { executionId, ...node };
+                }
+            }
+            if (estimatedBytes === 0 || (target && estimatedBytes <= target.estimatedBytes))
+                continue;
+            const root = this.activeExecutions.getExecutionOrFail(rootId).executionData;
+            target = {
+                rootId,
+                workflow: root.workflowData,
+                mode: root.executionMode,
+                memberIds: members.sort((a, b) => a.depth - b.depth).map((m) => m.executionId),
+                estimatedBytes,
+                runningNodeType: largest && this.nodeTypeIn(largest.executionId, largest.estimate.runningNode),
+                largestDataNodeType: largestNodeData && this.nodeTypeIn(largestNodeData.executionId, largestNodeData.name),
+            };
+        }
+        return target;
+    }
+    nodeTypeIn(executionId, nodeName) {
+        const { workflowData } = this.activeExecutions.getExecutionOrFail(executionId).executionData;
+        return (0, memory_guard_utils_1.nodeType)(workflowData, nodeName);
+    }
+    findRoot(executionId) {
+        let rootId = executionId;
+        let depth = 0;
+        const visited = new Set([executionId]);
+        for (;;) {
+            const { executionData } = this.activeExecutions.getExecutionOrFail(rootId).executionData;
+            const parentId = executionData?.parentExecution?.executionId;
+            if (!parentId || visited.has(parentId) || !this.activeExecutions.has(parentId))
+                break;
+            visited.add(parentId);
+            rootId = parentId;
+            depth++;
+        }
+        return { rootId, depth };
+    }
+    onExecutionEnd({ workflow, mode, runData }) {
+        if (runData.status === 'success' && (0, memory_guard_utils_1.isProductionMode)(mode)) {
+            this.stopsByWorkflow.delete(workflow.id);
+            if (this.pendingStopWorkflowId === workflow.id)
+                this.pendingStopWorkflowId = undefined;
+        }
+    }
+    countStop({ workflow, mode }) {
+        if (this.config.stopsToUnpublish === 0)
+            return false;
+        if (!(0, memory_guard_utils_1.isProductionMode)(mode) || !(0, utils_1.isWorkflowIdValid)(workflow.id))
+            return false;
+        const stops = (this.stopsByWorkflow.get(workflow.id) ?? 0) + 1;
+        this.stopsByWorkflow.set(workflow.id, stops);
+        if (stops < this.config.stopsToUnpublish || this.deactivating.has(workflow.id))
+            return false;
+        this.deactivating.add(workflow.id);
+        void this.deactivate(workflow.id).finally(() => this.deactivating.delete(workflow.id));
+        return true;
+    }
+    async deactivate(workflowId) {
+        try {
+            const workflow = await this.workflowRepository.findOne({ where: { id: workflowId } });
+            if (!workflow || workflow.activeVersionId === null) {
+                this.stopsByWorkflow.delete(workflowId);
+                return;
+            }
+            const { WorkflowService } = await import('../workflows/workflow.service.js');
+            await di_1.Container.get(WorkflowService).deactivateWorkflowAsSystem(workflowId);
+            this.stopsByWorkflow.delete(workflowId);
+            this.logger.warn(`Deactivated workflow ${workflowId}, because n8n stopped ${this.config.stopsToUnpublish} of its executions to keep the instance from running out of memory`, { workflowId });
+            this.eventService.emit('memory-guard-workflow-deactivated', { workflowId });
+            void this.notifyDeactivated(workflow);
+        }
+        catch (error) {
+            this.logger.error(`Failed to deactivate workflow ${workflowId}`, {
+                workflowId,
+                error: (0, ensure_error_1.ensureError)(error),
+            });
+        }
+    }
+    async notifyDeactivated(workflow) {
+        const workflowId = workflow.id;
+        try {
+            await this.workflowPushNotifier.notify(workflowId, {
+                type: 'workflowAutoDeactivated',
+                data: { workflowId },
+            });
+        }
+        catch (error) {
+            this.logger.error(`Failed to push the deactivation of workflow ${workflowId}`, {
+                workflowId,
+                error: (0, ensure_error_1.ensureError)(error),
+            });
+        }
+        try {
+            const recipient = await this.executionRecoveryService.getAutodeactivationRecipient(workflow);
+            await this.mailer.notifyWorkflowAutodeactivated({
+                recipient,
+                workflow,
+                reason: 'memory-pressure',
+            });
+        }
+        catch (error) {
+            this.logger.error(`Failed to email about the deactivation of workflow ${workflowId}`, {
+                workflowId,
+                error: (0, ensure_error_1.ensureError)(error),
+            });
+        }
+    }
+};
+exports.MemoryGuardService = MemoryGuardService;
+__decorate([
+    (0, decorators_1.OnShutdown)(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MemoryGuardService.prototype, "shutdown", null);
+__decorate([
+    (0, decorators_1.OnLifecycleEvent)('workflowExecuteAfter'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", void 0)
+], MemoryGuardService.prototype, "onExecutionEnd", null);
+exports.MemoryGuardService = MemoryGuardService = __decorate([
+    (0, di_1.Service)(),
+    __metadata("design:paramtypes", [backend_common_1.Logger, config_1.MemoryGuardConfig, active_executions_1.ActiveExecutions, production_hold_1.ProductionHold, run_data_tracker_1.RunDataTracker, memory_stopped_executions_1.MemoryStoppedExecutions, backend_services_1.EventService, memory_guard_journal_1.MemoryGuardJournal, db_1.WorkflowRepository, execution_recovery_service_1.ExecutionRecoveryService, user_management_mailer_1.UserManagementMailer, workflow_push_notifier_service_1.WorkflowPushNotifier])
+], MemoryGuardService);
+function percent(level) {
+    return `${Math.round(level * 100)}%`;
+}
+//# sourceMappingURL=memory-guard.service.js.map

package/dist/memory-guard/memory-guard.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-guard.service.js","sourceRoot":"","sources":["../../src/memory-guard/memory-guard.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAA6C;AAC7C,4DAAqD;AACrD,wCAAgD;AAChD,gCAAkE;AAClE,gDAAiG;AACjG,gCAA6C;AAC7C,iEAA6D;AAC7D,mDAAsD;AACtD,4CAAyC;AACzC,+CAAqE;AAErE,2DAAuD;AACvD,wFAAmF;AACnF,2FAAsF;AACtF,mCAA4C;AAC5C,+FAAkF;AAElF,qEAakC;AAElC,6DAO8B;AAC9B,iDAAsF;AACtF,uDAAmD;AACnD,yDAA0E;AAC1E,iEAA4D;AAC5D,2EAAsE;AAW/D,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;IAwD9B,YACkB,MAAc,EACd,MAAyB,EACzB,gBAAkC,EAClC,cAA8B,EAC9B,OAAuB,EACvB,uBAAgD,EAChD,YAA0B,EAC1B,OAA2B,EAC3B,kBAAsC,EACtC,wBAAkD,EAClD,MAA4B,EAC5B,oBAA0C;QAX1C,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAAmB;QACzB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,mBAAc,GAAd,cAAc,CAAgB;QAC9B,YAAO,GAAP,OAAO,CAAgB;QACvB,4BAAuB,GAAvB,uBAAuB,CAAyB;QAChD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,YAAO,GAAP,OAAO,CAAoB;QAC3B,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,WAAM,GAAN,MAAM,CAAsB;QAC5B,yBAAoB,GAApB,oBAAoB,CAAsB;QAjEpD,eAAU,GAAG,KAAK,CAAC;QAMnB,oBAAe,GAAG,CAAC,CAAC;QAEpB,eAAU,GAAG,CAAC,CAAC;QAEf,eAAU,GAAG,CAAC,CAAC;QAIf,sBAAiB,GAAG,CAAC,CAAC;QAGtB,eAAU,GAAG,CAAC,CAAC;QAEf,gBAAW,GAAG,CAAC,CAAC;QAEhB,2BAAsB,GAAG,CAAC,CAAC;QAE3B,0BAAqB,GAAG,CAAC,CAAC;QAE1B,oBAAe,GAAG,CAAC,CAAC;QAcpB,mBAAc,GAAG,KAAK,CAAC;QAMvB,oBAAe,GAAG,KAAK,CAAC;QAGf,oBAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;QAG5C,iBAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QAgBjD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAClD,CAAC;IAED,IAAI;QACH,IAAI,CAAC,cAAc,GAAG,IAAA,gCAAiB,GAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG,IAAA,6BAAQ,GAAE,CAAC;QAC1B,IAAI,CAAC,uBAAuB,CAAC,OAAO,GAAG,IAAI,CAAC;QAC5C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,8BAA8B,EAAE;YACtD,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,CAAC,yCAAgB,CAAC,CAAC;IACrC,CAAC;IAGD,QAAQ;QAEP,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;IAEO,YAAY,CAAC,OAAe;QACnC,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK;QACZ,IAAI,CAAC;YACJ,IAAI,CAAC,uBAAuB,CAAC,cAAc,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YACnF,IAAI,CAAC,QAAQ,CAAC,IAAA,8BAAe,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,IAAA,0BAAW,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,eAAe,GACpB,IAAI,CAAC,eAAe,IAAI,4CAAmB;YAC3C,IAAI,CAAC,SAAS;YACd,IAAI,CAAC,cAAc,CAAC,SAAS,GAAG,CAAC;YACjC,IAAI,CAAC,WAAW,KAAK,SAAS;YAC9B,IAAI,CAAC,cAAc;YACnB,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC;QACrC,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC,CAAC,yCAAgB,CAAC,CAAC,CAAC,yCAAgB,CAAC,CAAC;IAC1E,CAAC;IAED,IAAY,SAAS;QACpB,OAAO,IAAI,CAAC,aAAa,KAAK,SAAS,CAAC;IACzC,CAAC;IAEO,QAAQ,CAAC,KAAkB;QAClC,MAAM,EAAE,WAAW,EAAE,GAAG,KAAK,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC;QAEnC,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,4CAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;YAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/F,IAAI,CAAC,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjF,IAAI,CAAC,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjF,MAAM,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;QAE5D,IAAI,IAAI,CAAC,cAAc,IAAI,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAClE,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,qBAAqB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mDAAmD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CACxI,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,eAAe,IAAI,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW;YAAE,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAEhG,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,uCAAc,IAAI,UAAU,CAAC;QACnE,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,UAAU,EAAE,CAAC;YAC5D,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,IAAI,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YACpE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,qBAAqB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,oBAAoB,MAAM,6CAA6C,CAC5H,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;YAGpB,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;gBAAE,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3E,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,UAAU;gBAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;iBACjC,IAAI,IAAI,CAAC,kBAAkB,EAAE;gBACjC,IAAI,CAAC,WAAW,CAAC,+CAA+C,CAAC,CAAC;YACnE,OAAO;QACR,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC;YAC7C,IAAI,CAAC,WAAW,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;IACF,CAAC;IAEO,WAAW,CAAC,MAAc;QACjC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,GAAG,MAAM,uFAAuF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAClI,CAAC;IACH,CAAC;IAEO,mBAAmB;QAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IACxF,CAAC;IAEO,YAAY;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,IAAI,CAAC,qBAAqB,GAAG,qDAA4B;YAAE,OAAO;QAC5E,IAAI,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,qBAAqB,GAAG,GAAG,CAAC;IACxE,CAAC;IAMO,OAAO,CACd,UAAmB;QAEnB,IAAI,UAAU;YAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;QAE/C,IAAI,IAAI,CAAC,UAAU,IAAI,uCAAc,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;YAChE,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS;gBACpD,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;YAC/C,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,IAAI,CAAC,kBAAkB,EAAE;YAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;QAChE,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,gBAAgB;QACvB,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACjB,OAAO,IAAA,8BAAe,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7C,CAAC;IAEO,kBAAkB;QACzB,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,YAAY,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QAChE,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,sCAAa,CAAC;IACnF,CAAC;IAMO,kBAAkB,CAAC,WAAmB;QAC7C,IAAI,IAAI,CAAC,cAAc,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;YACpB,OAAO;QACR,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,GAAG,0CAAiB;YAAE,OAAO;QAEvD,IAAI,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,WAAW,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACzF,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC;QACvB,IAAI,CAAC,sBAAsB,GAAG,WAAW,CAAC;IAC3C,CAAC;IAEO,SAAS,CAAC,KAAkB;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,aAAa,IAAA,mBAAW,EAAC,KAAK,CAAC,SAAS,CAAC,OAAO,IAAA,mBAAW,EAAC,KAAK,CAAC,UAAU,CAAC,qCAAqC,CAClH,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACnD,eAAe,EAAE,KAAK,CAAC,SAAS;YAChC,gBAAgB,EAAE,KAAK,CAAC,UAAU;SAClC,CAAC,CAAC;IACJ,CAAC;IAEO,OAAO;QACd,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS;YAAE,OAAO;QAE7C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;QAC/B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,oDAA2B,CAAC;QAClE,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9D,OAAO,MAAM,CAAC;IACf,CAAC;IAGO,IAAI,CAAC,OAAoB,EAAE,YAA0B;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAG5B,MAAM,KAAK,GAAG,YAAY,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAEtD,MAAM,OAAO,GAAyB,EAAE,CAAC;QACzC,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;YACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACvD,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,WAAW,EAAE;gBACrD,KAAK,EAAE,IAAA,mCAAc,EAAC,SAAS,CAAC,aAAa,CAAC,YAAY,EAAE,QAAQ,EAAE,KAAK,CAAC;gBAC5E,WAAW,EAAE,QAAQ,EAAE,WAAW;aAClC,CAAC,CAAC;YAKH,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACtE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QACjF,CAAC;QAGD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAClC,WAAW,EACX,IAAI,oDAAqC,CAAC,WAAW,CAAC,CACtD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;aACjE,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAChB,2FAA2F,EAC3F,EAAE,KAAK,EAAE,IAAA,0BAAW,EAAC,KAAK,CAAC,EAAE,CAC7B,CAAC;QACH,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACb,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;YAC7B,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;YACvC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACnC,CAAC,CAAC,CAAC;QAEJ,OAAO,IAAI,CAAC;IACb,CAAC;IAQO,KAAK,CAAC,UAAU,CACvB,MAAkB,EAClB,OAA6B,EAC7B,MAAmB,EACnB,OAAoB;QAEpB,MAAM,IAAA,iCAAY,EAAC,OAAO,EAAE,2CAAkB,CAAC,CAAC;QAChD,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QAE1F,MAAM,SAAS,GACd,WAAW;YACX,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS;YAC3C,IAAI,CAAC,qBAAqB,KAAK,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnD,MAAM,2BAA2B,GAAG,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAExE,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,qBAAqB,MAAM,CAAC,MAAM,gBAAgB,MAAM,CAAC,QAAQ,CAAC,EAAE,OAAO,IAAA,mBAAW,EAAC,MAAM,CAAC,SAAS,CAAC,OAAO,IAAA,mBAAW,EAAC,MAAM,CAAC,UAAU,CAAC,KAAK,OAAO,2BAA2B,IAAA,mBAAW,EAAC,MAAM,CAAC,cAAc,CAAC,wCAAwC,IAAA,mBAAW,EAAC,KAAK,CAAC,SAAS,CAAC,EAAE,EAC5R,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,CAC9D,CAAC;QAEF,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAChE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,8FAA8F,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,sCAAsC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CACxM,CAAC;QACH,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,gCAAgC,EAAE;YACxD,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE;YAC9B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,OAAO;YACf,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;YAC/C,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,eAAe,EAAE,MAAM,CAAC,SAAS;YACjC,gBAAgB,EAAE,MAAM,CAAC,UAAU;YACnC,WAAW;YACX,2BAA2B;SAC3B,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,MAAmB,EAAE,cAAsB;QACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,6CAAoB,CAAC;QACnD,SAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,IAAA,8BAAe,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnD,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,cAAc,GAAG,oCAAW,CAAC;YACvF,IAAI,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ;gBAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;YACzE,MAAM,IAAA,aAAK,EAAC,8CAAqB,CAAC,CAAC;QACpC,CAAC;IACF,CAAC;IAMO,cAAc;QACrB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAyD,CAAC;QAC/E,KAAK,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAC1E,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YAEnC,IACC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1C,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,EAAE,CAAC;gBAE5C,SAAS;YAEV,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,iBAAiB;gBAAE,SAAS;YAE9E,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACzC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,MAA8B,CAAC;QACnC,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;YACvC,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,IAAI,OAAuE,CAAC;YAC5E,IAAI,eAAiF,CAAC;YACtF,KAAK,MAAM,EAAE,WAAW,EAAE,IAAI,OAAO,EAAE,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;gBACvD,IAAI,CAAC,QAAQ;oBAAE,SAAS;gBACxB,cAAc,IAAI,QAAQ,CAAC,UAAU,CAAC;gBACtC,IAAI,CAAC,OAAO,IAAI,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;oBACnE,OAAO,GAAG,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;gBACrC,CAAC;gBACD,MAAM,IAAI,GAAG,IAAA,gCAAW,EAAC,QAAQ,CAAC,CAAC;gBACnC,IAAI,IAAI,IAAI,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtE,eAAe,GAAG,EAAE,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;gBAC5C,CAAC;YACF,CAAC;YACD,IAAI,cAAc,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC;gBAAE,SAAS;YAE1F,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;YAC5E,MAAM,GAAG;gBACR,MAAM;gBACN,QAAQ,EAAE,IAAI,CAAC,YAAY;gBAC3B,IAAI,EAAE,IAAI,CAAC,aAAa;gBACxB,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;gBAC9E,cAAc;gBACd,eAAe,EACd,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAC9E,mBAAmB,EAClB,eAAe,IAAI,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,eAAe,CAAC,IAAI,CAAC;aACtF,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAEO,UAAU,CAAC,WAAmB,EAAE,QAA4B;QACnE,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC,aAAa,CAAC;QAC7F,OAAO,IAAA,6BAAQ,EAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACzC,CAAC;IAGO,QAAQ,CAAC,WAAmB;QACnC,IAAI,MAAM,GAAG,WAAW,CAAC;QACzB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QACvC,SAAS,CAAC;YACT,MAAM,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;YACzF,MAAM,QAAQ,GAAG,aAAa,EAAE,eAAe,EAAE,WAAW,CAAC;YAC7D,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,MAAM;YACtF,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtB,MAAM,GAAG,QAAQ,CAAC;YAClB,KAAK,EAAE,CAAC;QACT,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;IAID,cAAc,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAA+B;QACtE,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,IAAA,qCAAgB,EAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,IAAI,CAAC,qBAAqB,KAAK,QAAQ,CAAC,EAAE;gBAAE,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;QACxF,CAAC;IACF,CAAC;IAOO,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAc;QAC/C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACrD,IAAI,CAAC,IAAA,qCAAgB,EAAC,IAAI,CAAC,IAAI,CAAC,IAAA,yBAAiB,EAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAE7E,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7C,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAE7F,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACnC,KAAK,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC;IACb,CAAC;IAGO,KAAK,CAAC,UAAU,CAAC,UAAkB;QAC1C,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;YACtF,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,eAAe,KAAK,IAAI,EAAE,CAAC;gBACpD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBACxC,OAAO;YACR,CAAC;YAGD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;YAC5E,MAAM,cAAS,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;YAC5E,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,wBAAwB,UAAU,yBAAyB,IAAI,CAAC,MAAM,CAAC,gBAAgB,oEAAoE,EAC3J,EAAE,UAAU,EAAE,CACd,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;YAC5E,KAAK,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,UAAU,EAAE,EAAE;gBAChE,UAAU;gBACV,KAAK,EAAE,IAAA,0BAAW,EAAC,KAAK,CAAC;aACzB,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,QAAwB;QACvD,MAAM,UAAU,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC/B,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,UAAU,EAAE;gBAClD,IAAI,EAAE,yBAAyB;gBAC/B,IAAI,EAAE,EAAE,UAAU,EAAE;aACpB,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+CAA+C,UAAU,EAAE,EAAE;gBAC9E,UAAU;gBACV,KAAK,EAAE,IAAA,0BAAW,EAAC,KAAK,CAAC;aACzB,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,4BAA4B,CAAC,QAAQ,CAAC,CAAC;YAC7F,MAAM,IAAI,CAAC,MAAM,CAAC,6BAA6B,CAAC;gBAC/C,SAAS;gBACT,QAAQ;gBACR,MAAM,EAAE,iBAAiB;aACzB,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sDAAsD,UAAU,EAAE,EAAE;gBACrF,UAAU;gBACV,KAAK,EAAE,IAAA,0BAAW,EAAC,KAAK,CAAC;aACzB,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;CACD,CAAA;;AAzcA;IADC,IAAA,uBAAU,GAAE;;;;kDAOZ;AAmXD;IADC,IAAA,6BAAgB,EAAC,sBAAsB,CAAC;;;;wDAMxC;6BAldW,kBAAkB;IAD9B,IAAA,YAAO,GAAE;qCA0DiB,uBAAM,EACN,0BAAiB,EACP,oCAAgB,EAClB,gCAAc,EACrB,iCAAc,EACE,mDAAuB,EAClC,+BAAY,EACjB,yCAAkB,EACP,uBAAkB,EACZ,qDAAwB,EAC1C,6CAAoB,EACN,qDAAoB;GApEhD,kBAAkB,CA6hB9B;AAED,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC;AACtC,CAAC"}

package/dist/memory-guard/memory-guard.types.d.ts

@@ -0,0 +1,11 @@
+import type { IWorkflowBase, WorkflowExecuteMode } from 'n8n-workflow';
+export type StopTrigger = 'threshold' | 'critical' | 'hold-limit';
+export interface StopTarget {
+    rootId: string;
+    workflow: IWorkflowBase;
+    mode: WorkflowExecuteMode;
+    memberIds: string[];
+    estimatedBytes: number;
+    runningNodeType: string | undefined;
+    largestDataNodeType: string | undefined;
+}

package/dist/{modules/agents/agent-runtime-instrumentation.js → memory-guard/memory-guard.types.js}

@@ -1,3 +1,3 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=agent-runtime-instrumentation.js.map
+//# sourceMappingURL=memory-guard.types.js.map

package/dist/memory-guard/memory-guard.types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-guard.types.js","sourceRoot":"","sources":["../../src/memory-guard/memory-guard.types.ts"],"names":[],"mappings":""}

package/dist/memory-guard/memory-guard.utils.d.ts

@@ -0,0 +1,13 @@
+import type { IWorkflowBase, WorkflowExecuteMode } from 'n8n-workflow';
+import { WorkflowOperationError } from 'n8n-workflow';
+import type { MemoryUsage } from './memory-usage';
+import type { RunDataEstimate } from './run-data-tracker';
+export declare function buildStopError(workflow: IWorkflowBase, estimate: RunDataEstimate | undefined, usage: MemoryUsage): WorkflowOperationError;
+export declare function largestNode(estimate: RunDataEstimate | undefined): {
+    name: string;
+    bytes: number;
+} | undefined;
+export declare function nodeType(workflow: IWorkflowBase, nodeName: string | undefined): string | undefined;
+export declare function settleWithin(promises: Array<Promise<void>>, timeoutMs: number): Promise<void>;
+export declare function exposeGc(): (() => void) | undefined;
+export declare function isProductionMode(mode: WorkflowExecuteMode): mode is "chat" | "trigger" | "webhook";

package/dist/memory-guard/memory-guard.utils.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildStopError = buildStopError;
+exports.largestNode = largestNode;
+exports.nodeType = nodeType;
+exports.settleWithin = settleWithin;
+exports.exposeGc = exposeGc;
+exports.isProductionMode = isProductionMode;
+const bytes_1 = require("@n8n/utils/number/bytes");
+const n8n_workflow_1 = require("n8n-workflow");
+const node_v8_1 = require("node:v8");
+const node_vm_1 = require("node:vm");
+const memory_guard_constants_1 = require("./memory-guard.constants");
+function buildStopError(workflow, estimate, usage) {
+    const sentences = ['Execution stopped because the instance was running out of memory.'];
+    const runningNode = estimate?.runningNode;
+    if (runningNode)
+        sentences.push(`Running node: "${runningNode}".`);
+    const largest = largestNode(estimate);
+    if (largest) {
+        sentences.push(`Node holding the most data: "${largest.name}" (about ${(0, bytes_1.formatBytes)(largest.bytes)}).`);
+    }
+    const description = [
+        `Memory was at ${(0, bytes_1.formatBytes)(usage.usedBytes)} of ${(0, bytes_1.formatBytes)(usage.limitBytes)}.`,
+        'To keep the instance and its other executions running, n8n stopped the execution that held the most data.',
+        'This execution is the likely cause, but not a certain one.',
+        `To use less memory, process data in smaller batches. See ${memory_guard_constants_1.MEMORY_DOCS_URL}`,
+    ].join(' ');
+    const node = workflow.nodes.find((n) => n.name === runningNode);
+    return new n8n_workflow_1.WorkflowOperationError(sentences.join(' '), node, description);
+}
+function largestNode(estimate) {
+    let largest;
+    for (const [name, bytes] of estimate?.bytesByNode ?? []) {
+        if (!largest || bytes > largest.bytes)
+            largest = { name, bytes };
+    }
+    return largest;
+}
+function nodeType(workflow, nodeName) {
+    return workflow.nodes.find((n) => n.name === nodeName)?.type;
+}
+async function settleWithin(promises, timeoutMs) {
+    let timeout;
+    await Promise.race([
+        Promise.allSettled(promises),
+        new Promise((resolve) => {
+            timeout = setTimeout(resolve, timeoutMs);
+        }),
+    ]);
+    clearTimeout(timeout);
+}
+function isGarbageCollect(value) {
+    return typeof value === 'function';
+}
+function exposeGc() {
+    try {
+        (0, node_v8_1.setFlagsFromString)('--expose-gc');
+        const gc = (0, node_vm_1.runInNewContext)('gc');
+        if (!isGarbageCollect(gc))
+            return undefined;
+        return () => gc({ type: 'major', execution: 'sync', flavor: 'last-resort' });
+    }
+    catch {
+        return undefined;
+    }
+    finally {
+        (0, node_v8_1.setFlagsFromString)('--no-expose-gc');
+    }
+}
+function isProductionMode(mode) {
+    return mode === 'webhook' || mode === 'trigger' || mode === 'chat';
+}
+//# sourceMappingURL=memory-guard.utils.js.map

package/dist/memory-guard/memory-guard.utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-guard.utils.js","sourceRoot":"","sources":["../../src/memory-guard/memory-guard.utils.ts"],"names":[],"mappings":";;;;;;;;AAAA,mDAAsD;AAEtD,+CAAsD;AACtD,qCAA6C;AAC7C,qCAA0C;AAE1C,qEAA2D;AAI3D,wBACC,QAAuB,EACvB,QAAqC,EACrC,KAAkB;IAGlB,MAAM,SAAS,GAAG,CAAC,mEAAmE,CAAC,CAAC;IACxF,MAAM,WAAW,GAAG,QAAQ,EAAE,WAAW,CAAC;IAC1C,IAAI,WAAW;QAAE,SAAS,CAAC,IAAI,CAAC,kBAAkB,WAAW,IAAI,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,OAAO,EAAE,CAAC;QACb,SAAS,CAAC,IAAI,CACb,gCAAgC,OAAO,CAAC,IAAI,YAAY,IAAA,mBAAW,EAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CACtF,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG;QACnB,iBAAiB,IAAA,mBAAW,EAAC,KAAK,CAAC,SAAS,CAAC,OAAO,IAAA,mBAAW,EAAC,KAAK,CAAC,UAAU,CAAC,GAAG;QACpF,2GAA2G;QAC3G,4DAA4D;QAC5D,4DAA4D,wCAAe,EAAE;KAC7E,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEZ,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;IAChE,OAAO,IAAI,qCAAsB,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAC3E,CAAC;AAED,qBAA4B,QAAqC;IAChE,IAAI,OAAoD,CAAC;IACzD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,WAAW,IAAI,EAAE,EAAE,CAAC;QACzD,IAAI,CAAC,OAAO,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK;YAAE,OAAO,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAClE,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,kBAAyB,QAAuB,EAAE,QAA4B;IAC7E,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,EAAE,IAAI,CAAC;AAC9D,CAAC;AAEM,KAAK,uBAAuB,QAA8B,EAAE,SAAiB;IACnF,IAAI,OAAmC,CAAC;IACxC,MAAM,OAAO,CAAC,IAAI,CAAC;QAClB,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC5B,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC7B,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1C,CAAC,CAAC;KACF,CAAC,CAAC;IACH,YAAY,CAAC,OAAO,CAAC,CAAC;AACvB,CAAC;AAQD,SAAS,gBAAgB,CAAC,KAAc;IACvC,OAAO,OAAO,KAAK,KAAK,UAAU,CAAC;AACpC,CAAC;AAMD;IACC,IAAI,CAAC;QACJ,IAAA,4BAAkB,EAAC,aAAa,CAAC,CAAC;QAClC,MAAM,EAAE,GAAY,IAAA,yBAAe,EAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAAE,OAAO,SAAS,CAAC;QAC5C,OAAO,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;YAAS,CAAC;QACV,IAAA,4BAAkB,EAAC,gBAAgB,CAAC,CAAC;IACtC,CAAC;AACF,CAAC;AAED,0BAAiC,IAAyB;IACzD,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,CAAC;AACpE,CAAC"}

package/dist/memory-guard/memory-stopped-executions.d.ts

@@ -0,0 +1,18 @@
+import type { IRun, WorkflowOperationError } from 'n8n-workflow';
+interface StoppedExecution {
+    error: WorkflowOperationError;
+    runningNode: string | undefined;
+}
+export declare class MemoryStoppedExecutions {
+    enabled: boolean;
+    private readonly stopped;
+    private readonly finishing;
+    markStopped(executionId: string, stop: StoppedExecution): void;
+    forget(executionId: string): void;
+    isStopped(executionId: string): boolean;
+    isFinishing(executionId: string): boolean;
+    pruneFinishing(isActive: (executionId: string) => boolean): void;
+    onWorkflowExecuteStart(executionId: string): void;
+    onWorkflowExecuteAfter(executionId: string, run: IRun): void;
+}
+export {};