@trigger.dev/core 4.0.2 → 4.0.3

package/dist/esm/v3/runEngineWorker/supervisor/scalingStrategies.test.js

@@ -0,0 +1,233 @@
+import { describe, it, expect } from "vitest";
+import { NoneScalingStrategy, SmoothScalingStrategy, AggressiveScalingStrategy, } from "./scalingStrategies.js";
+import { QueueMetricsProcessor } from "./queueMetricsProcessor.js";
+describe("Scaling Strategies", () => {
+    const baseOptions = {
+        minConsumerCount: 1,
+        maxConsumerCount: 20,
+        targetRatio: 1.0,
+    };
+    function createMetricsProcessor(smoothedValue) {
+        const processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+        // Initialize processor with the target smoothed value
+        processor.addSample(smoothedValue);
+        processor.processBatch();
+        return processor;
+    }
+    describe("NoneScalingStrategy", () => {
+        const strategy = new NoneScalingStrategy(baseOptions);
+        it("should always return current count (static mode)", () => {
+            expect(strategy.calculateTargetCount(5)).toBe(5);
+            expect(strategy.calculateTargetCount(1)).toBe(1);
+            expect(strategy.calculateTargetCount(10)).toBe(10);
+            // Clamping still applies
+            expect(strategy.calculateTargetCount(25)).toBe(20); // Clamped to max
+            expect(strategy.calculateTargetCount(0)).toBe(1); // Clamped to min
+        });
+        it("should have correct name", () => {
+            expect(strategy.name).toBe("none");
+        });
+        it("should handle zero current count", () => {
+            // Should clamp to minConsumerCount
+            const result = strategy.calculateTargetCount(0);
+            expect(result).toBe(1);
+        });
+    });
+    describe("SmoothScalingStrategy", () => {
+        it("should calculate target based on smoothed queue length", () => {
+            const metricsProcessor = createMetricsProcessor(10); // smoothed value = 10
+            const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor });
+            // With targetRatio=1.0, target consumers = ceil(10/1.0) = 10
+            // With dampingFactor=0.7 and currentCount=5:
+            // dampedTarget = 5 + (10 - 5) * 0.7 = 5 + 3.5 = 8.5 → 9
+            const result = strategy.calculateTargetCount(5);
+            expect(result).toBe(9);
+        });
+        it("should apply damping factor correctly", () => {
+            const metricsProcessor = createMetricsProcessor(20); // smoothed value = 20
+            const strategy = new SmoothScalingStrategy({
+                ...baseOptions,
+                metricsProcessor,
+                dampingFactor: 0.5,
+            }); // 50% damping
+            // With targetRatio=1.0, target consumers = ceil(20/1.0) = 20
+            // With dampingFactor=0.5 and currentCount=5:
+            // dampedTarget = 5 + (20 - 5) * 0.5 = 5 + 7.5 = 12.5 → 13
+            const result = strategy.calculateTargetCount(5);
+            expect(result).toBe(13);
+        });
+        it("should handle zero current count", () => {
+            const metricsProcessor = createMetricsProcessor(5);
+            const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor });
+            // With smoothedQueueLength=5, targetRatio=1.0:
+            // targetConsumers = ceil(5/1.0) = 5
+            // dampedTarget = 0 + (5 - 0) * 0.7 = 3.5 → 4
+            const result = strategy.calculateTargetCount(0);
+            expect(result).toBe(4);
+        });
+        it("should validate damping factor", () => {
+            const metricsProcessor = createMetricsProcessor(10);
+            expect(() => new SmoothScalingStrategy({
+                ...baseOptions,
+                metricsProcessor,
+                dampingFactor: -0.1,
+            })).toThrow("dampingFactor must be between 0 and 1");
+            expect(() => new SmoothScalingStrategy({
+                ...baseOptions,
+                metricsProcessor,
+                dampingFactor: 1.1,
+            })).toThrow("dampingFactor must be between 0 and 1");
+            expect(() => new SmoothScalingStrategy({
+                ...baseOptions,
+                metricsProcessor,
+                dampingFactor: 0,
+            })).not.toThrow();
+            expect(() => new SmoothScalingStrategy({
+                ...baseOptions,
+                metricsProcessor,
+                dampingFactor: 1,
+            })).not.toThrow();
+        });
+        it("should handle zero current count", () => {
+            const metricsProcessor = createMetricsProcessor(10);
+            const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor });
+            // With smoothedQueueLength=10, targetRatio=1.0:
+            // targetConsumers = ceil(10/1.0) = 10
+            // dampedTarget = 0 + (10 - 0) * 0.7 = 7
+            const result = strategy.calculateTargetCount(0);
+            expect(result).toBe(7);
+        });
+    });
+    describe("AggressiveScalingStrategy", () => {
+        it("should scale down when under-utilized", () => {
+            // queuePerConsumer = 2/5 = 0.4, scaleDownThreshold = 1.0 * 0.5 = 0.5
+            // Under-utilized since 0.4 < 0.5
+            const metricsProcessor = createMetricsProcessor(2);
+            const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+            const result = strategy.calculateTargetCount(5);
+            expect(result).toBeLessThan(5);
+            expect(result).toBeGreaterThanOrEqual(baseOptions.minConsumerCount);
+        });
+        it("should maintain count when in optimal zone", () => {
+            // queuePerConsumer = 5/5 = 1.0
+            // Optimal zone: 0.5 < 1.0 < 2.0
+            const metricsProcessor = createMetricsProcessor(5);
+            const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+            const result = strategy.calculateTargetCount(5);
+            expect(result).toBe(5);
+        });
+        it("should scale up when over-utilized", () => {
+            // queuePerConsumer = 15/5 = 3.0, scaleUpThreshold = 1.0 * 2.0 = 2.0
+            // Over-utilized since 3.0 > 2.0
+            const metricsProcessor = createMetricsProcessor(15);
+            const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+            const result = strategy.calculateTargetCount(5);
+            expect(result).toBeGreaterThan(5);
+            expect(result).toBeLessThanOrEqual(baseOptions.maxConsumerCount);
+        });
+        it("should scale aggressively for critical load", () => {
+            // queuePerConsumer = 25/5 = 5.0 (critical: 5x target ratio)
+            const metricsProcessor = createMetricsProcessor(25);
+            const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+            const result = strategy.calculateTargetCount(5);
+            // Should apply 50% scale factor: ceil(5 * 1.5) = 8
+            // But capped by 50% max increment: 5 + ceil(5 * 0.5) = 5 + 3 = 8
+            expect(result).toBe(8);
+        });
+        it("should respect max consumer count", () => {
+            const metricsProcessor = createMetricsProcessor(50); // Very high load
+            const strategy = new AggressiveScalingStrategy({
+                ...baseOptions,
+                maxConsumerCount: 6,
+                metricsProcessor,
+            });
+            const result = strategy.calculateTargetCount(5);
+            expect(result).toBeLessThanOrEqual(6);
+        });
+        it("should respect min consumer count", () => {
+            const metricsProcessor = createMetricsProcessor(0.1); // Very low load
+            const strategy = new AggressiveScalingStrategy({
+                ...baseOptions,
+                minConsumerCount: 3,
+                metricsProcessor,
+            });
+            const result = strategy.calculateTargetCount(5);
+            expect(result).toBeGreaterThanOrEqual(3);
+        });
+        it("should return thresholds", () => {
+            const metricsProcessor = createMetricsProcessor(10);
+            const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+            const thresholds = strategy.getThresholds(1.0);
+            expect(thresholds).toEqual({
+                scaleDownThreshold: 0.5,
+                scaleUpThreshold: 2.0,
+                criticalThreshold: 5.0,
+                highThreshold: 3.0,
+            });
+        });
+        it("should handle zero current count without division by zero", () => {
+            const metricsProcessor = createMetricsProcessor(10);
+            const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+            // Should use (currentCount || 1) to prevent division by zero
+            // queuePerConsumer = 10 / 1 = 10 (not 10 / 0)
+            // This is over-utilized (10 > 2.0), should scale up
+            const result = strategy.calculateTargetCount(0);
+            expect(result).toBeGreaterThan(0);
+            expect(result).toBeLessThanOrEqual(baseOptions.maxConsumerCount);
+        });
+        it("should handle zero queue with zero consumers", () => {
+            const metricsProcessor = createMetricsProcessor(0);
+            const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+            // queuePerConsumer = 0 / 1 = 0
+            // This is under-utilized (0 < 0.5), should scale down
+            // But already at 0, so should return minConsumerCount
+            const result = strategy.calculateTargetCount(0);
+            expect(result).toBe(baseOptions.minConsumerCount);
+        });
+    });
+    describe("Integration scenarios", () => {
+        it("should handle gradual load increase with smooth strategy", () => {
+            const metricsProcessor = createMetricsProcessor(2);
+            const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor });
+            let currentCount = 2;
+            // Gradual increase: 2 → 6 → 10 → 15
+            const loads = [2, 6, 10, 15];
+            const results = [];
+            for (const load of loads) {
+                // Update the processor with the new load
+                metricsProcessor.addSample(load);
+                metricsProcessor.processBatch();
+                const target = strategy.calculateTargetCount(currentCount);
+                results.push(target);
+                currentCount = target;
+            }
+            // Should show gradual increase due to damping
+            expect(results[0]).toBeLessThan(results[1]);
+            expect(results[1]).toBeLessThan(results[2]);
+            expect(results[2]).toBeLessThan(results[3]);
+            // But not immediate jumps due to damping
+            expect(results[1] - results[0]).toBeLessThan(loads[1] - loads[0]);
+        });
+        it("should handle load spike with aggressive strategy", () => {
+            let currentCount = 3;
+            // Sudden spike from normal to critical
+            const normalLoad = 3; // queuePerConsumer = 1.0 (optimal)
+            const spikeLoad = 15; // queuePerConsumer = 5.0 (critical)
+            const normalProcessor = createMetricsProcessor(normalLoad);
+            const normalStrategy = new AggressiveScalingStrategy({
+                ...baseOptions,
+                metricsProcessor: normalProcessor,
+            });
+            const normalTarget = normalStrategy.calculateTargetCount(currentCount);
+            expect(normalTarget).toBe(3); // Should maintain
+            const spikeProcessor = createMetricsProcessor(spikeLoad);
+            const spikeStrategy = new AggressiveScalingStrategy({
+                ...baseOptions,
+                metricsProcessor: spikeProcessor,
+            });
+            const spikeTarget = spikeStrategy.calculateTargetCount(currentCount);
+            expect(spikeTarget).toBeGreaterThan(3); // Should scale up aggressively
+        });
+    });
+});
+//# sourceMappingURL=scalingStrategies.test.js.map

package/dist/esm/v3/runEngineWorker/supervisor/scalingStrategies.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scalingStrategies.test.js","sourceRoot":"","sources":["../../../../../src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,yBAAyB,GAE1B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AAEnE,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,MAAM,WAAW,GAA2B;QAC1C,gBAAgB,EAAE,CAAC;QACnB,gBAAgB,EAAE,EAAE;QACpB,WAAW,EAAE,GAAG;KACjB,CAAC;IAEF,SAAS,sBAAsB,CAAC,aAAqB;QACnD,MAAM,SAAS,GAAG,IAAI,qBAAqB,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACrF,sDAAsD;QACtD,SAAS,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACnC,SAAS,CAAC,YAAY,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAEtD,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnD,yBAAyB;YACzB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB;YACrE,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB;QACrE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,mCAAmC;YACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC,CAAC,sBAAsB;YAC3E,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAEjF,6DAA6D;YAC7D,6CAA6C;YAC7C,wDAAwD;YACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC,CAAC,sBAAsB;YAC3E,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC;gBACzC,GAAG,WAAW;gBACd,gBAAgB;gBAChB,aAAa,EAAE,GAAG;aACnB,CAAC,CAAC,CAAC,cAAc;YAElB,6DAA6D;YAC7D,6CAA6C;YAC7C,0DAA0D;YAC1D,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAEjF,+CAA+C;YAC/C,oCAAoC;YACpC,6CAA6C;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;YACpD,MAAM,CACJ,GAAG,EAAE,CACH,IAAI,qBAAqB,CAAC;gBACxB,GAAG,WAAW;gBACd,gBAAgB;gBAChB,aAAa,EAAE,CAAC,GAAG;aACpB,CAAC,CACL,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;YAEnD,MAAM,CACJ,GAAG,EAAE,CACH,IAAI,qBAAqB,CAAC;gBACxB,GAAG,WAAW;gBACd,gBAAgB;gBAChB,aAAa,EAAE,GAAG;aACnB,CAAC,CACL,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;YAEnD,MAAM,CACJ,GAAG,EAAE,CACH,IAAI,qBAAqB,CAAC;gBACxB,GAAG,WAAW;gBACd,gBAAgB;gBAChB,aAAa,EAAE,CAAC;aACjB,CAAC,CACL,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAEhB,MAAM,CACJ,GAAG,EAAE,CACH,IAAI,qBAAqB,CAAC;gBACxB,GAAG,WAAW;gBACd,gBAAgB;gBAChB,aAAa,EAAE,CAAC;aACjB,CAAC,CACL,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAEjF,gDAAgD;YAChD,sCAAsC;YACtC,wCAAwC;YACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,qEAAqE;YACrE,iCAAiC;YACjC,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAErF,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,+BAA+B;YAC/B,gCAAgC;YAChC,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAErF,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,oEAAoE;YACpE,gCAAgC;YAChC,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAErF,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,4DAA4D;YAC5D,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAErF,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,mDAAmD;YACnD,iEAAiE;YACjE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB;YACtE,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC;gBAC7C,GAAG,WAAW;gBACd,gBAAgB,EAAE,CAAC;gBACnB,gBAAgB;aACjB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB;YACtE,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC;gBAC7C,GAAG,WAAW;gBACd,gBAAgB,EAAE,CAAC;gBACnB,gBAAgB;aACjB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YACrF,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/C,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC;gBACzB,kBAAkB,EAAE,GAAG;gBACvB,gBAAgB,EAAE,GAAG;gBACrB,iBAAiB,EAAE,GAAG;gBACtB,aAAa,EAAE,GAAG;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;YACnE,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAErF,6DAA6D;YAC7D,8CAA8C;YAC9C,oDAAoD;YACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAErF,+BAA+B;YAC/B,sDAAsD;YACtD,sDAAsD;YACtD,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;YAClE,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAE,GAAG,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;YACjF,IAAI,YAAY,GAAG,CAAC,CAAC;YAErB,oCAAoC;YACpC,MAAM,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7B,MAAM,OAAO,GAAG,EAAE,CAAC;YAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,yCAAyC;gBACzC,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBACjC,gBAAgB,CAAC,YAAY,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;gBAC3D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,YAAY,GAAG,MAAM,CAAC;YACxB,CAAC;YAED,8CAA8C;YAC9C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAC7C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAC7C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAE7C,yCAAyC;YACzC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAE,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,IAAI,YAAY,GAAG,CAAC,CAAC;YAErB,uCAAuC;YACvC,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,mCAAmC;YACzD,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC,oCAAoC;YAE1D,MAAM,eAAe,GAAG,sBAAsB,CAAC,UAAU,CAAC,CAAC;YAC3D,MAAM,cAAc,GAAG,IAAI,yBAAyB,CAAC;gBACnD,GAAG,WAAW;gBACd,gBAAgB,EAAE,eAAe;aAClC,CAAC,CAAC;YACH,MAAM,YAAY,GAAG,cAAc,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACvE,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB;YAEhD,MAAM,cAAc,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC;YACzD,MAAM,aAAa,GAAG,IAAI,yBAAyB,CAAC;gBAClD,GAAG,WAAW;gBACd,gBAAgB,EAAE,cAAc;aACjC,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,aAAa,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACrE,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,+BAA+B;QACzE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/esm/v3/runEngineWorker/supervisor/session.d.ts

@@ -1,7 +1,9 @@
 import { SupervisorHttpClient } from "./http.js";
 import { PreDequeueFn, PreSkipFn, SupervisorClientCommonOptions } from "./types.js";
+import { ScalingOptions } from "./consumerPool.js";
 import { WorkerEvents } from "./events.js";
 import EventEmitter from "events";
+import type { Registry } from "prom-client";
 type SupervisorSessionOptions = SupervisorClientCommonOptions & {
     queueConsumerEnabled?: boolean;
     runNotificationsEnabled?: boolean;
@@ -11,8 +13,9 @@ type SupervisorSessionOptions = SupervisorClientCommonOptions & {
     preDequeue?: PreDequeueFn;
     preSkip?: PreSkipFn;
     maxRunCount?: number;
-    maxConsumerCount?: number;
     sendRunDebugLogs?: boolean;
+    scaling: ScalingOptions;
+    metricsRegistry?: Registry;
 };
 export declare class SupervisorSession extends EventEmitter<WorkerEvents> {
     private opts;
@@ -21,7 +24,7 @@ export declare class SupervisorSession extends EventEmitter<WorkerEvents> {
     private readonly runNotificationsEnabled;
     private runNotificationsSocket?;
     private readonly queueConsumerEnabled;
-    private readonly queueConsumers;
+    private readonly consumerPool;
     private readonly heartbeat;
     constructor(opts: SupervisorSessionOptions);
     private onDequeue;

package/dist/esm/v3/runEngineWorker/supervisor/session.js

@@ -1,5 +1,5 @@
 import { SupervisorHttpClient } from "./http.js";
-import { RunQueueConsumer } from "./queueConsumer.js";
+import { RunQueueConsumerPool } from "./consumerPool.js";
 import EventEmitter from "events";
 import { VERSION } from "../../../version.js";
 import { io } from "socket.io-client";
@@ -13,7 +13,7 @@ export class SupervisorSession extends EventEmitter {
     runNotificationsEnabled;
     runNotificationsSocket;
     queueConsumerEnabled;
-    queueConsumers;
+    consumerPool;
     heartbeat;
     constructor(opts) {
         super();
@@ -21,8 +21,8 @@ export class SupervisorSession extends EventEmitter {
         this.runNotificationsEnabled = opts.runNotificationsEnabled ?? true;
         this.queueConsumerEnabled = opts.queueConsumerEnabled ?? true;
         this.httpClient = new SupervisorHttpClient(opts);
-        this.queueConsumers = Array.from({ length: opts.maxConsumerCount ?? 1 }, () => {
-            return new RunQueueConsumer({
+        this.consumerPool = new RunQueueConsumerPool({
+            consumer: {
                 client: this.httpClient,
                 preDequeue: opts.preDequeue,
                 preSkip: opts.preSkip,
@@ -30,7 +30,9 @@ export class SupervisorSession extends EventEmitter {
                 intervalMs: opts.dequeueIntervalMs,
                 idleIntervalMs: opts.dequeueIdleIntervalMs,
                 maxRunCount: opts.maxRunCount,
-            });
+            },
+            scaling: opts.scaling,
+            metricsRegistry: opts.metricsRegistry,
         });
         this.heartbeat = new IntervalService({
             onInterval: async () => {
@@ -126,8 +128,12 @@ export class SupervisorSession extends EventEmitter {
             name: workerGroup.name,
         });
         if (this.queueConsumerEnabled) {
-            this.logger.log("Queue consumer enabled");
-            await Promise.allSettled(this.queueConsumers.map(async (q) => q.start()));
+            this.logger.log("Queue consumer enabled", {
+                scalingStrategy: this.consumerPool["scalingStrategy"],
+                minConsumers: this.consumerPool["minConsumerCount"],
+                maxConsumers: this.consumerPool["maxConsumerCount"],
+            });
+            await this.consumerPool.start();
             this.heartbeat.start();
         }
         else {
@@ -142,7 +148,7 @@ export class SupervisorSession extends EventEmitter {
         }
     }
     async stop() {
-        await Promise.allSettled(this.queueConsumers.map(async (q) => q.stop()));
+        await this.consumerPool.stop();
         this.heartbeat.stop();
         this.runNotificationsSocket?.disconnect();
     }

package/dist/esm/v3/runEngineWorker/supervisor/session.js.map

@@ -1 +1 @@
-{"version":3,"file":"session.js","sourceRoot":"","sources":["../../../../../src/v3/runEngineWorker/supervisor/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAGjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,YAAY,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,EAAE,EAAU,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAezE,MAAM,OAAO,iBAAkB,SAAQ,YAA0B;IAa3C;IAZJ,UAAU,CAAuB;IAEhC,MAAM,GAAG,IAAI,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;IAE1D,uBAAuB,CAAU;IAC1C,sBAAsB,CAAkE;IAE/E,oBAAoB,CAAU;IAC9B,cAAc,CAAqB;IAEnC,SAAS,CAAkB;IAE5C,YAAoB,IAA8B;QAChD,KAAK,EAAE,CAAC;QADU,SAAI,GAAJ,IAAI,CAA0B;QAGhD,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC;QACpE,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC;QAE9D,IAAI,CAAC,UAAU,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,gBAAgB,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE;YAC5E,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,MAAM,EAAE,IAAI,CAAC,UAAU;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,iBAAiB;gBAClC,cAAc,EAAE,IAAI,CAAC,qBAAqB;gBAC1C,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAI,eAAe,CAAC;YACnC,UAAU,EAAE,KAAK,IAAI,EAAE;gBACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;gBAEvC,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACrC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;gBAE7D,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;oBACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;YACD,UAAU,EAAE,IAAI,CAAC,wBAAwB,GAAG,IAAI;YAChD,WAAW,EAAE,KAAK;YAClB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,QAAsC;QAC5D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAE7F,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;gBAC3B,IAAI,EAAE,IAAI,IAAI,EAAE;gBAChB,OAAO;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,2BAA2B,CAAC,cAAwB;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAE1E,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC1C,OAAO;QACT,CAAC;QAED,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC;QAEpF,OAAO,CAAC,UAAU,CAChB,cAAc,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CACnC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,aAAa,EAAE;YAC1C,IAAI,EAAE,IAAI,IAAI,EAAE;YAChB,OAAO,EAAE,sCAAsC;SAChD,CAAC,CACH,CACF,CAAC;IACJ,CAAC;IAED,+BAA+B,CAAC,cAAwB;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAE9E,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC1C,OAAO;QACT,CAAC;QAED,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC;QAEtF,OAAO,CAAC,UAAU,CAChB,cAAc,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CACnC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,aAAa,EAAE;YAC1C,IAAI,EAAE,IAAI,IAAI,EAAE;YAChB,OAAO,EAAE,wCAAwC;YACjD,UAAU,EAAE;gBACV,cAAc;aACf;SACF,CAAC,CACH,CACF,CAAC;IACJ,CAAC;IAEO,4BAA4B;QAClC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC;QAE3B,MAAM,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE;YAC5B,UAAU,EAAE,CAAC,WAAW,CAAC;YACzB,YAAY,EAAE,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC;SACjD,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE;YAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAExD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE;gBAC3C,IAAI,EAAE,IAAI,IAAI,EAAE;gBAChB,OAAO,EAAE,mCAAmC;aAC7C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YACxB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;YACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE;YAC9C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iCAAiC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC5C,QAAQ,EAAE;gBACR,aAAa,EAAE,OAAO;aACvB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE;YACvC,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,IAAI,EAAE,WAAW,CAAC,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YAC1C,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC1E,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;YAC7C,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACtB,IAAI,CAAC,sBAAsB,EAAE,UAAU,EAAE,CAAC;IAC5C,CAAC;IAEO,gBAAgB;QACtB,OAAO;YACL,GAAG,EAAE;gBACH,IAAI,EAAE,GAAG;gBACT,SAAS,EAAE,GAAG;aACf;YACD,MAAM,EAAE;gBACN,IAAI,EAAE,GAAG;gBACT,SAAS,EAAE,GAAG;aACf;YACD,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"session.js","sourceRoot":"","sources":["../../../../../src/v3/runEngineWorker/supervisor/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAGjD,OAAO,EAAE,oBAAoB,EAAkB,MAAM,mBAAmB,CAAC;AAEzE,OAAO,YAAY,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,EAAE,EAAU,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAiBzE,MAAM,OAAO,iBAAkB,SAAQ,YAA0B;IAa3C;IAZJ,UAAU,CAAuB;IAEhC,MAAM,GAAG,IAAI,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;IAE1D,uBAAuB,CAAU;IAC1C,sBAAsB,CAAkE;IAE/E,oBAAoB,CAAU;IAC9B,YAAY,CAAuB;IAEnC,SAAS,CAAkB;IAE5C,YAAoB,IAA8B;QAChD,KAAK,EAAE,CAAC;QADU,SAAI,GAAJ,IAAI,CAA0B;QAGhD,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC;QACpE,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC;QAE9D,IAAI,CAAC,UAAU,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,CAAC,YAAY,GAAG,IAAI,oBAAoB,CAAC;YAC3C,QAAQ,EAAE;gBACR,MAAM,EAAE,IAAI,CAAC,UAAU;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,iBAAiB;gBAClC,cAAc,EAAE,IAAI,CAAC,qBAAqB;gBAC1C,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B;YACD,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,eAAe,EAAE,IAAI,CAAC,eAAe;SACtC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAI,eAAe,CAAC;YACnC,UAAU,EAAE,KAAK,IAAI,EAAE;gBACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;gBAEvC,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACrC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;gBAE7D,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;oBACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;YACD,UAAU,EAAE,IAAI,CAAC,wBAAwB,GAAG,IAAI;YAChD,WAAW,EAAE,KAAK;YAClB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,QAAsC;QAC5D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAE7F,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;gBAC3B,IAAI,EAAE,IAAI,IAAI,EAAE;gBAChB,OAAO;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,2BAA2B,CAAC,cAAwB;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAE1E,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC1C,OAAO;QACT,CAAC;QAED,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC;QAEpF,OAAO,CAAC,UAAU,CAChB,cAAc,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CACnC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,aAAa,EAAE;YAC1C,IAAI,EAAE,IAAI,IAAI,EAAE;YAChB,OAAO,EAAE,sCAAsC;SAChD,CAAC,CACH,CACF,CAAC;IACJ,CAAC;IAED,+BAA+B,CAAC,cAAwB;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAE9E,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC1C,OAAO;QACT,CAAC;QAED,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC;QAEtF,OAAO,CAAC,UAAU,CAChB,cAAc,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CACnC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,aAAa,EAAE;YAC1C,IAAI,EAAE,IAAI,IAAI,EAAE;YAChB,OAAO,EAAE,wCAAwC;YACjD,UAAU,EAAE;gBACV,cAAc;aACf;SACF,CAAC,CACH,CACF,CAAC;IACJ,CAAC;IAEO,4BAA4B;QAClC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC;QAE3B,MAAM,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE;YAC5B,UAAU,EAAE,CAAC,WAAW,CAAC;YACzB,YAAY,EAAE,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC;SACjD,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE;YAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAExD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE;gBAC3C,IAAI,EAAE,IAAI,IAAI,EAAE;gBAChB,OAAO,EAAE,mCAAmC;aAC7C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YACxB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;YACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE;YAC9C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iCAAiC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC5C,QAAQ,EAAE;gBACR,aAAa,EAAE,OAAO;aACvB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE;YACvC,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,IAAI,EAAE,WAAW,CAAC,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,EAAE;gBACxC,eAAe,EAAE,IAAI,CAAC,YAAY,CAAC,iBAAiB,CAAC;gBACrD,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;gBACnD,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;aACpD,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;YAC7C,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACtB,IAAI,CAAC,sBAAsB,EAAE,UAAU,EAAE,CAAC;IAC5C,CAAC;IAEO,gBAAgB;QACtB,OAAO;YACL,GAAG,EAAE;gBACH,IAAI,EAAE,GAAG;gBACT,SAAS,EAAE,GAAG;aACf;YACD,MAAM,EAAE;gBACN,IAAI,EAAE,GAAG;gBACT,SAAS,EAAE,GAAG;aACf;YACD,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;CACF"}

package/dist/esm/v3/runEngineWorker/workload/http.d.ts

@@ -10,30 +10,46 @@ export declare class WorkloadHttpClient {
     updateApiUrl(apiUrl: string): void;
     updateRunnerId(runnerId: string): void;
     defaultHeaders(): Record<string, string>;
-    heartbeatRun(runId: string, snapshotId: string, body?: WorkloadHeartbeatRequestBody): Promise<import("../../zodfetch.js").ApiResult<{
-        ok: true;
-    }>>;
+    private isConnectionError;
+    private withConnectionErrorDetection;
+    heartbeatRun(runId: string, snapshotId: string, body?: WorkloadHeartbeatRequestBody): Promise<{
+        success: false;
+        error: string;
+        isConnectionError?: boolean;
+    } | {
+        success: true;
+        data: {
+            ok: true;
+        };
+    }>;
     suspendRun(runId: string, snapshotId: string): Promise<import("../../zodfetch.js").ApiResult<{
         ok: true;
     } | {
         error: string;
         ok: false;
     }>>;
-    continueRunExecution(runId: string, snapshotId: string): Promise<import("../../zodfetch.js").ApiResult<{
-        run: {
-            status: import("../../index.js").TaskRunStatus;
-            id: string;
-            friendlyId: string;
-            attemptNumber?: number | null | undefined;
-        };
-        snapshot: {
-            id: string;
-            createdAt: Date;
-            description: string;
-            friendlyId: string;
-            executionStatus: import("../../index.js").TaskRunExecutionStatus;
+    continueRunExecution(runId: string, snapshotId: string): Promise<{
+        success: false;
+        error: string;
+        isConnectionError?: boolean;
+    } | {
+        success: true;
+        data: {
+            run: {
+                status: import("../../index.js").TaskRunStatus;
+                id: string;
+                friendlyId: string;
+                attemptNumber?: number | null | undefined;
+            };
+            snapshot: {
+                id: string;
+                createdAt: Date;
+                description: string;
+                friendlyId: string;
+                executionStatus: import("../../index.js").TaskRunExecutionStatus;
+            };
         };
-    }>>;
+    }>;
     startRunAttempt(runId: string, snapshotId: string, body: WorkloadRunAttemptStartRequestBody): Promise<import("../../zodfetch.js").ApiResult<{
         run: {
             status: import("../../index.js").TaskRunStatus;
@@ -175,114 +191,67 @@ export declare class WorkloadHttpClient {
             };
         };
     }>>;
-    getRunExecutionData(runId: string): Promise<import("../../zodfetch.js").ApiResult<{
-        execution: {
-            version: "1";
-            run: {
-                status: import("../../index.js").TaskRunStatus;
-                id: string;
-                friendlyId: string;
-                attemptNumber?: number | null | undefined;
-            };
-            snapshot: {
-                id: string;
-                createdAt: Date;
-                description: string;
-                friendlyId: string;
-                executionStatus: import("../../index.js").TaskRunExecutionStatus;
-            };
-            completedWaitpoints: {
-                type: import("../../index.js").WaitpointType;
-                id: string;
-                friendlyId: string;
-                completedAt: Date;
-                outputIsError: boolean;
-                idempotencyKey?: string | undefined;
-                output?: string | undefined;
-                outputType?: string | undefined;
-                index?: number | undefined;
-                completedByTaskRun?: {
+    getSnapshotsSince(runId: string, snapshotId: string): Promise<{
+        success: false;
+        error: string;
+        isConnectionError?: boolean;
+    } | {
+        success: true;
+        data: {
+            snapshots: {
+                version: "1";
+                run: {
+                    status: import("../../index.js").TaskRunStatus;
                     id: string;
                     friendlyId: string;
-                    batch?: {
-                        id: string;
-                        friendlyId: string;
-                    } | undefined;
-                } | undefined;
-                completedAfter?: Date | undefined;
-                completedByBatch?: {
+                    attemptNumber?: number | null | undefined;
+                };
+                snapshot: {
                     id: string;
+                    createdAt: Date;
+                    description: string;
                     friendlyId: string;
-                } | undefined;
-            }[];
-            batch?: {
-                id: string;
-                friendlyId: string;
-            } | undefined;
-            checkpoint?: {
-                type: import("../../index.js").CheckpointTypeEnum;
-                id: string;
-                friendlyId: string;
-                location: string;
-                imageRef?: string | null | undefined;
-                reason?: string | null | undefined;
-            } | undefined;
-        };
-    }>>;
-    getSnapshotsSince(runId: string, snapshotId: string): Promise<import("../../zodfetch.js").ApiResult<{
-        snapshots: {
-            version: "1";
-            run: {
-                status: import("../../index.js").TaskRunStatus;
-                id: string;
-                friendlyId: string;
-                attemptNumber?: number | null | undefined;
-            };
-            snapshot: {
-                id: string;
-                createdAt: Date;
-                description: string;
-                friendlyId: string;
-                executionStatus: import("../../index.js").TaskRunExecutionStatus;
-            };
-            completedWaitpoints: {
-                type: import("../../index.js").WaitpointType;
-                id: string;
-                friendlyId: string;
-                completedAt: Date;
-                outputIsError: boolean;
-                idempotencyKey?: string | undefined;
-                output?: string | undefined;
-                outputType?: string | undefined;
-                index?: number | undefined;
-                completedByTaskRun?: {
+                    executionStatus: import("../../index.js").TaskRunExecutionStatus;
+                };
+                completedWaitpoints: {
+                    type: import("../../index.js").WaitpointType;
                     id: string;
                     friendlyId: string;
-                    batch?: {
+                    completedAt: Date;
+                    outputIsError: boolean;
+                    idempotencyKey?: string | undefined;
+                    output?: string | undefined;
+                    outputType?: string | undefined;
+                    index?: number | undefined;
+                    completedByTaskRun?: {
+                        id: string;
+                        friendlyId: string;
+                        batch?: {
+                            id: string;
+                            friendlyId: string;
+                        } | undefined;
+                    } | undefined;
+                    completedAfter?: Date | undefined;
+                    completedByBatch?: {
                         id: string;
                         friendlyId: string;
                     } | undefined;
+                }[];
+                batch?: {
+                    id: string;
+                    friendlyId: string;
                 } | undefined;
-                completedAfter?: Date | undefined;
-                completedByBatch?: {
+                checkpoint?: {
+                    type: import("../../index.js").CheckpointTypeEnum;
                     id: string;
                     friendlyId: string;
+                    location: string;
+                    imageRef?: string | null | undefined;
+                    reason?: string | null | undefined;
                 } | undefined;
             }[];
-            batch?: {
-                id: string;
-                friendlyId: string;
-            } | undefined;
-            checkpoint?: {
-                type: import("../../index.js").CheckpointTypeEnum;
-                id: string;
-                friendlyId: string;
-                location: string;
-                imageRef?: string | null | undefined;
-                reason?: string | null | undefined;
-            } | undefined;
-        }[];
-    }>>;
+        };
+    }>;
     sendDebugLog(runId: string, body: WorkloadDebugLogRequestBody): Promise<void>;
     /** @deprecated Not currently used */
     dequeue(): Promise<import("../../zodfetch.js").ApiResult<{