@trigger.dev/core 0.0.0-buffer-fix-20231206151546 → 0.0.0-bun-prerelease-20260909132648

package/dist/commonjs/v3/runEngineWorker/supervisor/consumerPool.js

@@ -0,0 +1,331 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunQueueConsumerPool = void 0;
+const structuredLogger_js_1 = require("../../utils/structuredLogger.js");
+const consumerPoolMetrics_js_1 = require("./consumerPoolMetrics.js");
+const queueConsumer_js_1 = require("./queueConsumer.js");
+const queueMetricsProcessor_js_1 = require("./queueMetricsProcessor.js");
+const scalingStrategies_js_1 = require("./scalingStrategies.js");
+class RunQueueConsumerPool {
+    consumerOptions;
+    logger = new structuredLogger_js_1.SimpleStructuredLogger("consumer-pool");
+    promMetrics;
+    minConsumerCount;
+    maxConsumerCount;
+    scalingStrategy;
+    disableJitter;
+    shouldPauseScaling;
+    consumers = new Map();
+    consumerFactory;
+    isEnabled = false;
+    isScaling = false;
+    metrics;
+    metricsProcessor;
+    // Scaling parameters
+    ewmaAlpha;
+    scaleUpCooldownMs;
+    scaleDownCooldownMs;
+    batchWindowMs;
+    constructor(opts) {
+        this.consumerOptions = opts.consumer;
+        // Initialize Prometheus metrics if registry provided
+        if (opts.metricsRegistry) {
+            this.promMetrics = new consumerPoolMetrics_js_1.ConsumerPoolMetrics({
+                register: opts.metricsRegistry,
+            });
+        }
+        this.minConsumerCount = Math.max(1, opts.scaling.minConsumerCount ?? 1);
+        this.maxConsumerCount = Math.max(this.minConsumerCount, opts.scaling.maxConsumerCount ?? 10);
+        this.scaleUpCooldownMs = opts.scaling.scaleUpCooldownMs ?? 10000; // 10 seconds default
+        this.scaleDownCooldownMs = opts.scaling.scaleDownCooldownMs ?? 60000; // 60 seconds default
+        this.disableJitter = opts.scaling.disableJitter ?? false;
+        this.shouldPauseScaling = opts.scaling.shouldPauseScaling;
+        // Configure EWMA parameters from options
+        this.ewmaAlpha = opts.scaling.ewmaAlpha ?? 0.3;
+        this.batchWindowMs = opts.scaling.batchWindowMs ?? 1000;
+        // Validate EWMA parameters
+        if (this.ewmaAlpha < 0 || this.ewmaAlpha > 1) {
+            throw new Error(`ewmaAlpha must be between 0 and 1, got: ${this.ewmaAlpha}`);
+        }
+        if (this.batchWindowMs <= 0) {
+            throw new Error(`batchWindowMs must be positive, got: ${this.batchWindowMs}`);
+        }
+        // Initialize metrics processor
+        this.metricsProcessor = new queueMetricsProcessor_js_1.QueueMetricsProcessor({
+            ewmaAlpha: this.ewmaAlpha,
+            batchWindowMs: this.batchWindowMs,
+        });
+        const targetRatio = opts.scaling.targetRatio ?? 1.0;
+        const dampingFactor = opts.scaling.dampingFactor;
+        // Create scaling strategy with metrics processor injected
+        this.scalingStrategy = scalingStrategies_js_1.ScalingStrategy.create(opts.scaling.strategy ?? "none", {
+            metricsProcessor: this.metricsProcessor,
+            dampingFactor,
+            targetRatio,
+            minConsumerCount: this.minConsumerCount,
+            maxConsumerCount: this.maxConsumerCount,
+        });
+        // Use provided factory or default to RunQueueConsumer
+        this.consumerFactory =
+            opts.consumerFactory || ((consumerOpts) => new queueConsumer_js_1.RunQueueConsumer(consumerOpts));
+        this.metrics = {
+            targetConsumerCount: this.minConsumerCount,
+            queueLength: undefined,
+            smoothedQueueLength: 0,
+            lastScaleTime: new Date(0),
+            lastQueueLengthUpdate: new Date(0),
+        };
+        this.logger.log("Initialized consumer pool", {
+            minConsumerCount: this.minConsumerCount,
+            maxConsumerCount: this.maxConsumerCount,
+            scalingStrategy: this.scalingStrategy.name,
+            mode: this.scalingStrategy.name === "none" ? "static" : "dynamic",
+            ewmaAlpha: this.ewmaAlpha,
+            batchWindowMs: this.batchWindowMs,
+        });
+    }
+    async start() {
+        if (this.isEnabled) {
+            return;
+        }
+        this.isEnabled = true;
+        // For 'none' strategy, start with max consumers (static mode)
+        // For dynamic strategies, start with minimum
+        const initialCount = this.scalingStrategy.name === "none" ? this.maxConsumerCount : this.minConsumerCount;
+        // Set initial metrics
+        this.metrics.targetConsumerCount = initialCount;
+        this.addConsumers(initialCount);
+        this.logger.log("Started dynamic consumer pool", {
+            initialConsumerCount: this.consumers.size,
+        });
+        // Initialize Prometheus metrics with initial state
+        this.promMetrics?.updateState({
+            consumerCount: this.consumers.size,
+            queueLength: this.metrics.queueLength,
+            smoothedQueueLength: this.metrics.smoothedQueueLength,
+            targetConsumerCount: initialCount,
+            strategy: this.scalingStrategy.name,
+        });
+    }
+    async stop() {
+        if (!this.isEnabled) {
+            return;
+        }
+        this.isEnabled = false;
+        // Stop all consumers
+        Array.from(this.consumers.values()).forEach((consumer) => consumer.stop());
+        this.consumers.clear();
+        this.logger.log("Stopped dynamic consumer pool");
+    }
+    /**
+     * Updates the queue length metric and triggers scaling decisions
+     * Uses QueueMetricsProcessor for batching and EWMA smoothing
+     */
+    updateQueueLength(queueLength) {
+        // Track queue length update in metrics
+        this.promMetrics?.recordQueueLengthUpdate();
+        // Skip metrics tracking for static mode
+        if (this.scalingStrategy.name === "none") {
+            return;
+        }
+        // Add sample to metrics processor
+        this.metricsProcessor.addSample(queueLength);
+        // Check if we should process the current batch
+        if (this.metricsProcessor.shouldProcessBatch()) {
+            this.processMetricsBatch();
+        }
+    }
+    processMetricsBatch() {
+        // Process batch using the metrics processor
+        const result = this.metricsProcessor.processBatch();
+        if (!result) {
+            this.logger.debug("No queue length samples in batch window - skipping scaling evaluation");
+            return;
+        }
+        // Update metrics
+        this.metrics.queueLength = result.median;
+        this.metrics.smoothedQueueLength = result.smoothedValue;
+        this.metrics.lastQueueLengthUpdate = new Date();
+        this.logger.verbose("Queue metrics batch processed", {
+            samples: result.sampleCount,
+            median: result.median,
+            smoothed: result.smoothedValue,
+            currentConsumerCount: this.consumers.size,
+        });
+        // Make scaling decision
+        this.evaluateScaling();
+    }
+    evaluateScaling() {
+        if (!this.isEnabled) {
+            return;
+        }
+        // No scaling in static mode
+        if (this.scalingStrategy.name === "none") {
+            return;
+        }
+        // Skip if already scaling
+        if (this.isScaling) {
+            this.logger.debug("Scaling blocked - operation already in progress", {
+                currentCount: this.consumers.size,
+                targetCount: this.metrics.targetConsumerCount,
+                actualCount: this.consumers.size,
+            });
+            return;
+        }
+        const targetCount = this.calculateTargetConsumerCount();
+        if (targetCount === this.consumers.size) {
+            return;
+        }
+        const timeSinceLastScale = Date.now() - this.metrics.lastScaleTime.getTime();
+        // Add random jitter to avoid thundering herd when multiple replicas exist
+        // Works without needing to know replica index or count
+        const jitterMs = this.disableJitter ? 0 : Math.random() * 3000; // 0-3 seconds random jitter
+        // Check cooldown periods with jitter
+        if (targetCount > this.consumers.size) {
+            // Freeze scale-up while backpressure is engaged - don't add consumers to
+            // drain a queue we're deliberately holding. Scale-down stays allowed.
+            if (this.shouldPauseScaling?.()) {
+                this.logger.debug("Scale up frozen by backpressure", {
+                    currentCount: this.consumers.size,
+                    targetCount,
+                });
+                return;
+            }
+            // Scale up
+            const effectiveCooldown = this.scaleUpCooldownMs + jitterMs;
+            if (timeSinceLastScale < effectiveCooldown) {
+                this.logger.debug("Scale up blocked by cooldown", {
+                    timeSinceLastScale,
+                    cooldownMs: effectiveCooldown,
+                    jitterMs,
+                    remainingMs: effectiveCooldown - timeSinceLastScale,
+                });
+                this.promMetrics?.recordCooldownApplied("up");
+                return;
+            }
+        }
+        else if (targetCount < this.consumers.size) {
+            // Scale down
+            const effectiveCooldown = this.scaleDownCooldownMs + jitterMs;
+            if (timeSinceLastScale < effectiveCooldown) {
+                this.logger.debug("Scale down blocked by cooldown", {
+                    timeSinceLastScale,
+                    cooldownMs: effectiveCooldown,
+                    jitterMs,
+                    remainingMs: effectiveCooldown - timeSinceLastScale,
+                });
+                this.promMetrics?.recordCooldownApplied("down");
+                return;
+            }
+        }
+        this.logger.info("Scaling consumer pool", {
+            from: this.consumers.size,
+            to: targetCount,
+            queueLength: this.metrics.queueLength,
+            smoothedQueueLength: this.metrics.smoothedQueueLength,
+            strategy: this.scalingStrategy,
+        });
+        // Set flag before scaling
+        this.isScaling = true;
+        // Update target metric for visibility
+        const previousTarget = this.metrics.targetConsumerCount;
+        this.metrics.targetConsumerCount = targetCount;
+        try {
+            this.scaleToTarget(targetCount);
+        }
+        catch (error) {
+            this.logger.error("Failed to scale consumer pool", { error });
+            // Revert target on failure
+            this.metrics.targetConsumerCount = previousTarget;
+        }
+        finally {
+            this.isScaling = false;
+        }
+    }
+    calculateTargetConsumerCount() {
+        return this.scalingStrategy.calculateTargetCount(this.consumers.size);
+    }
+    scaleToTarget(targetCount) {
+        const actualCurrentCount = this.consumers.size;
+        if (targetCount > actualCurrentCount) {
+            // Scale up
+            const count = targetCount - actualCurrentCount;
+            this.addConsumers(count);
+            this.promMetrics?.recordScalingOperation("up", this.scalingStrategy.name, count);
+        }
+        else if (targetCount < actualCurrentCount) {
+            // Scale down
+            const count = actualCurrentCount - targetCount;
+            this.removeConsumers(count);
+            this.promMetrics?.recordScalingOperation("down", this.scalingStrategy.name, count);
+        }
+        this.metrics.lastScaleTime = new Date();
+        // Update Prometheus state metrics
+        this.promMetrics?.updateState({
+            consumerCount: this.consumers.size,
+            queueLength: this.metrics.queueLength,
+            smoothedQueueLength: this.metrics.smoothedQueueLength,
+            targetConsumerCount: targetCount,
+            strategy: this.scalingStrategy.name,
+        });
+    }
+    addConsumers(count) {
+        const newConsumers = [];
+        for (let i = 0; i < count; i++) {
+            const consumerId = `consumer-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+            const consumer = this.consumerFactory({
+                ...this.consumerOptions,
+                // Share the pool's single metrics instance so every consumer records onto
+                // the same histogram (re-registering the metric name would throw). Fall
+                // back to a caller-supplied instance rather than clobbering it.
+                metrics: this.promMetrics ?? this.consumerOptions.metrics,
+                onDequeue: async (messages, timing) => {
+                    // Always update queue length, default to 0 for empty dequeues or missing value
+                    this.updateQueueLength(messages[0]?.workerQueueLength ?? 0);
+                    // Forward to the original handler
+                    await this.consumerOptions.onDequeue(messages, timing);
+                },
+            });
+            this.consumers.set(consumerId, consumer);
+            newConsumers.push(consumer);
+        }
+        // Start all new consumers
+        newConsumers.forEach((c) => c.start());
+        this.logger.info("Added consumers", {
+            count,
+            totalConsumers: this.consumers.size,
+        });
+    }
+    removeConsumers(count) {
+        const allIds = Array.from(this.consumers.keys());
+        const consumerIds = allIds.slice(-count); // Take from the end
+        const consumersToStop = [];
+        for (const id of consumerIds) {
+            const consumer = this.consumers.get(id);
+            if (consumer) {
+                consumersToStop.push(consumer);
+                this.consumers.delete(id);
+            }
+        }
+        // Stop removed consumers
+        consumersToStop.forEach((c) => c.stop());
+        this.logger.info("Removed consumers", {
+            count: consumersToStop.length,
+            totalConsumers: this.consumers.size,
+        });
+    }
+    /**
+     * Get current pool metrics for monitoring
+     */
+    getMetrics() {
+        return { ...this.metrics };
+    }
+    /**
+     * Get current number of consumers in the pool
+     */
+    get size() {
+        return this.consumers.size;
+    }
+}
+exports.RunQueueConsumerPool = RunQueueConsumerPool;
+//# sourceMappingURL=consumerPool.js.map

package/dist/commonjs/v3/runEngineWorker/supervisor/consumerPool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consumerPool.js","sourceRoot":"","sources":["../../../../../src/v3/runEngineWorker/supervisor/consumerPool.ts"],"names":[],"mappings":";;;AACA,yEAAyE;AACzE,qEAA+D;AAE/D,yDAAsD;AACtD,yEAAmE;AAEnE,iEAAyD;AAsCzD;IACmB,eAAe,CAA0B;IAEzC,MAAM,GAAG,IAAI,4CAAsB,CAAC,eAAe,CAAC,CAAC;IACrD,WAAW,CAAuB;IAElC,gBAAgB,CAAS;IACzB,gBAAgB,CAAS;IACzB,eAAe,CAAkB;IACjC,aAAa,CAAU;IACvB,kBAAkB,CAAiB;IAE5C,SAAS,GAA+B,IAAI,GAAG,EAAE,CAAC;IACzC,eAAe,CAAuB;IAC/C,SAAS,GAAY,KAAK,CAAC;IAC3B,SAAS,GAAY,KAAK,CAAC;IAE3B,OAAO,CAAiB;IACf,gBAAgB,CAAwB;IAEzD,qBAAqB;IACJ,SAAS,CAAS;IAClB,iBAAiB,CAAS;IAC1B,mBAAmB,CAAS;IAC5B,aAAa,CAAS;IAEvC,YAAY,IAAyB;QACnC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC;QAErC,qDAAqD;QACrD,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,IAAI,4CAAmB,CAAC;gBACzC,QAAQ,EAAE,IAAI,CAAC,eAAe;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;QAC7F,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,IAAI,KAAK,CAAC,CAAC,qBAAqB;QACvF,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,IAAI,KAAK,CAAC,CAAC,qBAAqB;QAC3F,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;QACzD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAE1D,yCAAyC;QACzC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;QAExD,2BAA2B;QAC3B,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,gBAAgB,GAAG,IAAI,gDAAqB,CAAC;YAChD,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QACpD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;QAEjD,0DAA0D;QAC1D,IAAI,CAAC,eAAe,GAAG,sCAAe,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE;YAC7E,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,aAAa;YACb,WAAW;YACX,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACxC,CAAC,CAAC;QAEH,sDAAsD;QACtD,IAAI,CAAC,eAAe;YAClB,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,IAAI,mCAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAEjF,IAAI,CAAC,OAAO,GAAG;YACb,mBAAmB,EAAE,IAAI,CAAC,gBAAgB;YAC1C,WAAW,EAAE,SAAS;YACtB,mBAAmB,EAAE,CAAC;YACtB,aAAa,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;YAC1B,qBAAqB,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;SACnC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2BAA2B,EAAE;YAC3C,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI;YAC1C,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YACjE,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,8DAA8D;QAC9D,6CAA6C;QAC7C,MAAM,YAAY,GAChB,IAAI,CAAC,eAAe,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAEvF,sBAAsB;QACtB,IAAI,CAAC,OAAO,CAAC,mBAAmB,GAAG,YAAY,CAAC;QAEhD,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAEhC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,+BAA+B,EAAE;YAC/C,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;SAC1C,CAAC,CAAC;QAEH,mDAAmD;QACnD,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC;YAC5B,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;YAClC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;YACrC,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,mBAAmB;YACrD,mBAAmB,EAAE,YAAY;YACjC,QAAQ,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI;SACpC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAEvB,qBAAqB;QACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAE3E,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAEvB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,iBAAiB,CAAC,WAAmB;QACnC,uCAAuC;QACvC,IAAI,CAAC,WAAW,EAAE,uBAAuB,EAAE,CAAC;QAE5C,wCAAwC;QACxC,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzC,OAAO;QACT,CAAC;QAED,kCAAkC;QAClC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAE7C,+CAA+C;QAC/C,IAAI,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,EAAE,CAAC;YAC/C,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,mBAAmB;QACzB,4CAA4C;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC;QAEpD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;YAC3F,OAAO;QACT,CAAC;QAED,iBAAiB;QACjB,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;QACzC,IAAI,CAAC,OAAO,CAAC,mBAAmB,GAAG,MAAM,CAAC,aAAa,CAAC;QACxD,IAAI,CAAC,OAAO,CAAC,qBAAqB,GAAG,IAAI,IAAI,EAAE,CAAC;QAEhD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,EAAE;YACnD,OAAO,EAAE,MAAM,CAAC,WAAW;YAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,aAAa;YAC9B,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;SAC1C,CAAC,CAAC;QAEH,wBAAwB;QACxB,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAEO,eAAe;QACrB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QAED,4BAA4B;QAC5B,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzC,OAAO;QACT,CAAC;QAED,0BAA0B;QAC1B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE;gBACnE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;gBACjC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,mBAAmB;gBAC7C,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;aACjC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;QAExD,IAAI,WAAW,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YACxC,OAAO;QACT,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QAE7E,0EAA0E;QAC1E,uDAAuD;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,4BAA4B;QAE5F,qCAAqC;QACrC,IAAI,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YACtC,yEAAyE;YACzE,sEAAsE;YACtE,IAAI,IAAI,CAAC,kBAAkB,EAAE,EAAE,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE;oBACnD,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;oBACjC,WAAW;iBACZ,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,WAAW;YACX,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC;YAC5D,IAAI,kBAAkB,GAAG,iBAAiB,EAAE,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;oBAChD,kBAAkB;oBAClB,UAAU,EAAE,iBAAiB;oBAC7B,QAAQ;oBACR,WAAW,EAAE,iBAAiB,GAAG,kBAAkB;iBACpD,CAAC,CAAC;gBACH,IAAI,CAAC,WAAW,EAAE,qBAAqB,CAAC,IAAI,CAAC,CAAC;gBAC9C,OAAO;YACT,CAAC;QACH,CAAC;aAAM,IAAI,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAC7C,aAAa;YACb,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC;YAC9D,IAAI,kBAAkB,GAAG,iBAAiB,EAAE,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;oBAClD,kBAAkB;oBAClB,UAAU,EAAE,iBAAiB;oBAC7B,QAAQ;oBACR,WAAW,EAAE,iBAAiB,GAAG,kBAAkB;iBACpD,CAAC,CAAC;gBACH,IAAI,CAAC,WAAW,EAAE,qBAAqB,CAAC,MAAM,CAAC,CAAC;gBAChD,OAAO;YACT,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE;YACxC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;YACzB,EAAE,EAAE,WAAW;YACf,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;YACrC,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,mBAAmB;YACrD,QAAQ,EAAE,IAAI,CAAC,eAAe;SAC/B,CAAC,CAAC;QAEH,0BAA0B;QAC1B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,sCAAsC;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC;QACxD,IAAI,CAAC,OAAO,CAAC,mBAAmB,GAAG,WAAW,CAAC;QAE/C,IAAI,CAAC;YACH,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9D,2BAA2B;YAC3B,IAAI,CAAC,OAAO,CAAC,mBAAmB,GAAG,cAAc,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,4BAA4B;QAClC,OAAO,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxE,CAAC;IAEO,aAAa,CAAC,WAAmB;QACvC,MAAM,kBAAkB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;QAE/C,IAAI,WAAW,GAAG,kBAAkB,EAAE,CAAC;YACrC,WAAW;YACX,MAAM,KAAK,GAAG,WAAW,GAAG,kBAAkB,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,CAAC,WAAW,EAAE,sBAAsB,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACnF,CAAC;aAAM,IAAI,WAAW,GAAG,kBAAkB,EAAE,CAAC;YAC5C,aAAa;YACb,MAAM,KAAK,GAAG,kBAAkB,GAAG,WAAW,CAAC;YAC/C,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,WAAW,EAAE,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACrF,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC;QAExC,kCAAkC;QAClC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC;YAC5B,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;YAClC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;YACrC,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,mBAAmB;YACrD,mBAAmB,EAAE,WAAW;YAChC,QAAQ,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,YAAY,CAAC,KAAa;QAChC,MAAM,YAAY,GAAoB,EAAE,CAAC;QAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,YAAY,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAE3F,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC;gBACpC,GAAG,IAAI,CAAC,eAAe;gBACvB,0EAA0E;gBAC1E,wEAAwE;gBACxE,gEAAgE;gBAChE,OAAO,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO;gBACzD,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE;oBACpC,+EAA+E;oBAC/E,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAC;oBAE5D,kCAAkC;oBAClC,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACzD,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YACzC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;QAED,0BAA0B;QAC1B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAEvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAClC,KAAK;YACL,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,KAAa;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,oBAAoB;QAC9D,MAAM,eAAe,GAAoB,EAAE,CAAC;QAE5C,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxC,IAAI,QAAQ,EAAE,CAAC;gBACb,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC/B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE;YACpC,KAAK,EAAE,eAAe,CAAC,MAAM;YAC7B,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;SACpC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAC7B,CAAC;CACF"}

package/dist/commonjs/v3/runEngineWorker/supervisor/consumerPoolMetrics.d.ts

@@ -0,0 +1,58 @@
+import { Counter, Gauge, Histogram, Registry } from "prom-client";
+export interface ConsumerPoolMetricsOptions {
+    register?: Registry;
+    prefix?: string;
+}
+/**
+ * Outcome of a single dequeue API round-trip, used as a low-cardinality label
+ * on the dequeue latency histogram.
+ * - `success`: the call returned at least one run
+ * - `empty`: the call succeeded but returned no runs (the common idle case)
+ * - `error`: the call failed (unsuccessful response, network error, or timeout)
+ */
+export type DequeueOutcome = "success" | "empty" | "error";
+export declare class ConsumerPoolMetrics {
+    private readonly register;
+    private readonly prefix;
+    readonly consumerCount: Gauge;
+    readonly queueLength: Gauge;
+    readonly smoothedQueueLength: Gauge;
+    readonly targetConsumerCount: Gauge;
+    readonly scalingStrategy: Gauge;
+    readonly scalingOperationsTotal: Counter;
+    readonly consumersAddedTotal: Counter;
+    readonly consumersRemovedTotal: Counter;
+    readonly scalingCooldownsApplied: Counter;
+    readonly queueLengthUpdatesTotal: Counter;
+    readonly batchesProcessedTotal: Counter;
+    readonly dequeueDurationSeconds: Histogram;
+    constructor(opts?: ConsumerPoolMetricsOptions);
+    /**
+     * Update all gauge metrics with current state
+     */
+    updateState(state: {
+        consumerCount: number;
+        queueLength?: number;
+        smoothedQueueLength: number;
+        targetConsumerCount: number;
+        strategy: string;
+    }): void;
+    /**
+     * Record a scaling operation
+     */
+    recordScalingOperation(direction: "up" | "down" | "none", strategy: string, count: number): void;
+    /**
+     * Record that scaling was prevented by cooldown
+     */
+    recordCooldownApplied(direction: "up" | "down"): void;
+    /**
+     * Record a queue length update
+     */
+    recordQueueLengthUpdate(): void;
+    /**
+     * Record the client-side latency of a single dequeue API round-trip.
+     * @param seconds Wall-clock duration of the dequeue call, in seconds.
+     * @param outcome Whether the call returned runs, was empty, or errored.
+     */
+    observeDequeueLatency(seconds: number, outcome: DequeueOutcome): void;
+}

package/dist/commonjs/v3/runEngineWorker/supervisor/consumerPoolMetrics.js

@@ -0,0 +1,157 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConsumerPoolMetrics = void 0;
+const prom_client_1 = require("prom-client");
+class ConsumerPoolMetrics {
+    register;
+    prefix;
+    // Current state metrics
+    consumerCount;
+    queueLength;
+    smoothedQueueLength;
+    targetConsumerCount;
+    scalingStrategy;
+    // Scaling operation metrics
+    scalingOperationsTotal;
+    consumersAddedTotal;
+    consumersRemovedTotal;
+    scalingCooldownsApplied;
+    // Performance metrics
+    queueLengthUpdatesTotal;
+    batchesProcessedTotal;
+    // Dequeue API latency (client-side, measured around the dequeue HTTP call)
+    dequeueDurationSeconds;
+    constructor(opts = {}) {
+        this.register = opts.register ?? new prom_client_1.Registry();
+        this.prefix = opts.prefix ?? "queue_consumer_pool";
+        // Current state metrics
+        this.consumerCount = new prom_client_1.Gauge({
+            name: `${this.prefix}_consumer_count`,
+            help: "Current number of active queue consumers",
+            labelNames: ["strategy"],
+            registers: [this.register],
+        });
+        this.queueLength = new prom_client_1.Gauge({
+            name: `${this.prefix}_queue_length`,
+            help: "Current queue length (median of recent samples)",
+            registers: [this.register],
+        });
+        this.smoothedQueueLength = new prom_client_1.Gauge({
+            name: `${this.prefix}_smoothed_queue_length`,
+            help: "EWMA smoothed queue length",
+            registers: [this.register],
+        });
+        this.targetConsumerCount = new prom_client_1.Gauge({
+            name: `${this.prefix}_target_consumer_count`,
+            help: "Target number of consumers calculated by scaling strategy",
+            labelNames: ["strategy"],
+            registers: [this.register],
+        });
+        this.scalingStrategy = new prom_client_1.Gauge({
+            name: `${this.prefix}_scaling_strategy_info`,
+            help: "Information about the active scaling strategy (1 = active, 0 = inactive)",
+            labelNames: ["strategy"],
+            registers: [this.register],
+        });
+        // Scaling operation metrics
+        this.scalingOperationsTotal = new prom_client_1.Counter({
+            name: `${this.prefix}_scaling_operations_total`,
+            help: "Total number of scaling operations performed",
+            labelNames: ["direction", "strategy"],
+            registers: [this.register],
+        });
+        this.consumersAddedTotal = new prom_client_1.Counter({
+            name: `${this.prefix}_consumers_added_total`,
+            help: "Total number of consumers added",
+            registers: [this.register],
+        });
+        this.consumersRemovedTotal = new prom_client_1.Counter({
+            name: `${this.prefix}_consumers_removed_total`,
+            help: "Total number of consumers removed",
+            registers: [this.register],
+        });
+        this.scalingCooldownsApplied = new prom_client_1.Counter({
+            name: `${this.prefix}_scaling_cooldowns_applied_total`,
+            help: "Number of times scaling was prevented due to cooldown",
+            labelNames: ["direction"],
+            registers: [this.register],
+        });
+        this.queueLengthUpdatesTotal = new prom_client_1.Counter({
+            name: `${this.prefix}_queue_length_updates_total`,
+            help: "Total number of queue length updates received",
+            registers: [this.register],
+        });
+        this.batchesProcessedTotal = new prom_client_1.Counter({
+            name: `${this.prefix}_batches_processed_total`,
+            help: "Total number of metric batches processed",
+            registers: [this.register],
+        });
+        this.dequeueDurationSeconds = new prom_client_1.Histogram({
+            name: `${this.prefix}_dequeue_duration_seconds`,
+            help: "Client-side duration of the dequeue API call (POST /engine/v1/worker-actions/dequeue), including the HTTP client's internal retries and backoff",
+            labelNames: ["outcome"],
+            // The HTTP client retries internally (up to 5 attempts with 0.5-5s backoff),
+            // so one observation can span multiple requests plus sleeps. A retryable
+            // failure surfaces as `error` only after >=7.5s of backoff - the 10-30s
+            // buckets exist so that mode doesn't collapse into +Inf. The server also
+            // long-polls (RUN_ENGINE_DEQUEUE_BLOCKING_TIMEOUT_SECONDS, default 10s),
+            // parking empty dequeues at ~10s - the 11/12.5/15/20 buckets give the
+            // quantiles resolution just above that boundary, where the mass sits.
+            // 60s brackets the worst-case error envelope (5 attempts that each hit
+            // the ~10s hold, plus backoff); beyond that the connection is hung.
+            buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 11, 12.5, 15, 20, 30, 60],
+            registers: [this.register],
+        });
+    }
+    /**
+     * Update all gauge metrics with current state
+     */
+    updateState(state) {
+        this.consumerCount.set({ strategy: state.strategy }, state.consumerCount);
+        if (state.queueLength !== undefined) {
+            this.queueLength.set(state.queueLength);
+        }
+        this.smoothedQueueLength.set(state.smoothedQueueLength);
+        this.targetConsumerCount.set({ strategy: state.strategy }, state.targetConsumerCount);
+        // Set strategy info (1 for active strategy, 0 for others)
+        ["none", "smooth", "aggressive"].forEach((s) => {
+            this.scalingStrategy.set({ strategy: s }, s === state.strategy ? 1 : 0);
+        });
+    }
+    /**
+     * Record a scaling operation
+     */
+    recordScalingOperation(direction, strategy, count) {
+        if (direction !== "none") {
+            this.scalingOperationsTotal.inc({ direction, strategy });
+            if (direction === "up") {
+                this.consumersAddedTotal.inc(count);
+            }
+            else {
+                this.consumersRemovedTotal.inc(count);
+            }
+        }
+    }
+    /**
+     * Record that scaling was prevented by cooldown
+     */
+    recordCooldownApplied(direction) {
+        this.scalingCooldownsApplied.inc({ direction });
+    }
+    /**
+     * Record a queue length update
+     */
+    recordQueueLengthUpdate() {
+        this.queueLengthUpdatesTotal.inc();
+    }
+    /**
+     * Record the client-side latency of a single dequeue API round-trip.
+     * @param seconds Wall-clock duration of the dequeue call, in seconds.
+     * @param outcome Whether the call returned runs, was empty, or errored.
+     */
+    observeDequeueLatency(seconds, outcome) {
+        this.dequeueDurationSeconds.observe({ outcome }, seconds);
+    }
+}
+exports.ConsumerPoolMetrics = ConsumerPoolMetrics;
+//# sourceMappingURL=consumerPoolMetrics.js.map

package/dist/commonjs/v3/runEngineWorker/supervisor/consumerPoolMetrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consumerPoolMetrics.js","sourceRoot":"","sources":["../../../../../src/v3/runEngineWorker/supervisor/consumerPoolMetrics.ts"],"names":[],"mappings":";;;AAAA,6CAAkE;AAgBlE;IACmB,QAAQ,CAAW;IACnB,MAAM,CAAS;IAEhC,wBAAwB;IACR,aAAa,CAAQ;IACrB,WAAW,CAAQ;IACnB,mBAAmB,CAAQ;IAC3B,mBAAmB,CAAQ;IAC3B,eAAe,CAAQ;IAEvC,4BAA4B;IACZ,sBAAsB,CAAU;IAChC,mBAAmB,CAAU;IAC7B,qBAAqB,CAAU;IAC/B,uBAAuB,CAAU;IAEjD,sBAAsB;IACN,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAE/C,2EAA2E;IAC3D,sBAAsB,CAAY;IAElD,YAAY,IAAI,GAA+B,EAAE;QAC/C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,sBAAQ,EAAE,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,qBAAqB,CAAC;QAEnD,wBAAwB;QACxB,IAAI,CAAC,aAAa,GAAG,IAAI,mBAAK,CAAC;YAC7B,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,iBAAiB;YACrC,IAAI,EAAE,0CAA0C;YAChD,UAAU,EAAE,CAAC,UAAU,CAAC;YACxB,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,IAAI,mBAAK,CAAC;YAC3B,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,eAAe;YACnC,IAAI,EAAE,iDAAiD;YACvD,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAK,CAAC;YACnC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,wBAAwB;YAC5C,IAAI,EAAE,4BAA4B;YAClC,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAK,CAAC;YACnC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,wBAAwB;YAC5C,IAAI,EAAE,2DAA2D;YACjE,UAAU,EAAE,CAAC,UAAU,CAAC;YACxB,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,GAAG,IAAI,mBAAK,CAAC;YAC/B,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,wBAAwB;YAC5C,IAAI,EAAE,0EAA0E;YAChF,UAAU,EAAE,CAAC,UAAU,CAAC;YACxB,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,4BAA4B;QAC5B,IAAI,CAAC,sBAAsB,GAAG,IAAI,qBAAO,CAAC;YACxC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,2BAA2B;YAC/C,IAAI,EAAE,8CAA8C;YACpD,UAAU,EAAE,CAAC,WAAW,EAAE,UAAU,CAAC;YACrC,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,mBAAmB,GAAG,IAAI,qBAAO,CAAC;YACrC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,wBAAwB;YAC5C,IAAI,EAAE,iCAAiC;YACvC,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,qBAAqB,GAAG,IAAI,qBAAO,CAAC;YACvC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,0BAA0B;YAC9C,IAAI,EAAE,mCAAmC;YACzC,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,uBAAuB,GAAG,IAAI,qBAAO,CAAC;YACzC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,kCAAkC;YACtD,IAAI,EAAE,uDAAuD;YAC7D,UAAU,EAAE,CAAC,WAAW,CAAC;YACzB,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,uBAAuB,GAAG,IAAI,qBAAO,CAAC;YACzC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,6BAA6B;YACjD,IAAI,EAAE,+CAA+C;YACrD,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,qBAAqB,GAAG,IAAI,qBAAO,CAAC;YACvC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,0BAA0B;YAC9C,IAAI,EAAE,0CAA0C;YAChD,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,sBAAsB,GAAG,IAAI,uBAAS,CAAC;YAC1C,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,2BAA2B;YAC/C,IAAI,EAAE,iJAAiJ;YACvJ,UAAU,EAAE,CAAC,SAAS,CAAC;YACvB,6EAA6E;YAC7E,yEAAyE;YACzE,wEAAwE;YACxE,yEAAyE;YACzE,yEAAyE;YACzE,sEAAsE;YACtE,sEAAsE;YACtE,uEAAuE;YACvE,oEAAoE;YACpE,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;YAC1F,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SAC3B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,KAMX;QACC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAE1E,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACxD,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAEtF,0DAA0D;QAC1D,CAAC,MAAM,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YAC7C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,SAAiC,EAAE,QAAgB,EAAE,KAAa;QACvF,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;YAEzD,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gBACvB,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,qBAAqB,CAAC,SAAwB;QAC5C,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,uBAAuB;QACrB,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACH,qBAAqB,CAAC,OAAe,EAAE,OAAuB;QAC5D,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,CAAC;IAC5D,CAAC;CACF"}

package/dist/commonjs/v3/runEngineWorker/supervisor/events.d.ts

@@ -0,0 +1,50 @@
+import type { TaskRunExecutionResult } from "../../schemas/common.js";
+import type { DequeuedMessage, StartRunAttemptResult } from "../../schemas/runEngine.js";
+export type WorkerEvents = {
+    runQueueMessage: [
+        {
+            time: Date;
+            message: DequeuedMessage;
+            dequeueResponseMs?: number;
+            pollingIntervalMs?: number;
+        }
+    ];
+    requestRunAttemptStart: [
+        {
+            time: Date;
+            run: {
+                friendlyId: string;
+            };
+            snapshot: {
+                friendlyId: string;
+            };
+        }
+    ];
+    runAttemptStarted: [
+        {
+            time: Date;
+        } & StartRunAttemptResult & {
+            envVars: Record<string, string>;
+        }
+    ];
+    runAttemptCompleted: [
+        {
+            time: Date;
+            run: {
+                friendlyId: string;
+            };
+            snapshot: {
+                friendlyId: string;
+            };
+            completion: TaskRunExecutionResult;
+        }
+    ];
+    runNotification: [
+        {
+            time: Date;
+            run: {
+                friendlyId: string;
+            };
+        }
+    ];
+};

package/dist/commonjs/v3/runEngineWorker/supervisor/events.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=events.js.map

package/dist/commonjs/v3/runEngineWorker/supervisor/events.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"events.js","sourceRoot":"","sources":["../../../../../src/v3/runEngineWorker/supervisor/events.ts"],"names":[],"mappings":""}