@adonis-agora/durable 0.48.0 → 0.49.1

package/dist/src/transports/bullmq/adaptive-concurrency.js

@@ -0,0 +1,352 @@
+import { readFileSync } from 'node:fs';
+import { cpus, totalmem } from 'node:os';
+const DEFAULT_MIN = 1;
+const DEFAULT_MAX = 32;
+const DEFAULT_RAM_CEILING_PCT = 85;
+const DEFAULT_TICK_MS = 2000;
+function clamp(value, min, max) {
+    return Math.min(max, Math.max(min, value));
+}
+/** `value` floored when it is a finite number of at least 1, else `fallback`. */
+function wholeAtLeastOne(value, fallback) {
+    return value !== undefined && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
+}
+/** `value` when it is a finite number above 0, else `fallback`. */
+function finitePositive(value, fallback) {
+    return value !== undefined && Number.isFinite(value) && value > 0 ? value : fallback;
+}
+/**
+ * Normalise a {@link ConcurrencyOption} into a {@link ResolvedConcurrency}: a bare number (or
+ * `undefined` → 1) becomes `{ mode:'fixed', fixed }`; `'adaptive'` / `{ mode:'adaptive', ... }`
+ * become `{ mode:'adaptive', adaptive }` with every default from the spec applied and `start`
+ * clamped into `[min,max]`.
+ *
+ * A non-finite or out-of-range number falls back to its default (a bad `cpuCeilingPct` turns the
+ * CPU ceiling off): BullMQ throws on a `NaN` or sub-1 concurrency, a `NaN` ceiling never brakes, and
+ * a `NaN` `tickMs` makes the control loop fire every millisecond.
+ */
+export function resolveConcurrency(opt) {
+    if (opt === undefined)
+        return { mode: 'fixed', fixed: 1 };
+    if (typeof opt === 'number')
+        return { mode: 'fixed', fixed: wholeAtLeastOne(opt, 1) };
+    const overrides = opt === 'adaptive' ? {} : opt;
+    const min = wholeAtLeastOne(overrides.min, DEFAULT_MIN);
+    const max = Math.max(min, wholeAtLeastOne(overrides.max, DEFAULT_MAX));
+    const start = clamp(wholeAtLeastOne(overrides.start, min), min, max);
+    const cpuCeilingPct = finitePositive(overrides.cpuCeilingPct, undefined);
+    const adaptive = {
+        min,
+        max,
+        start,
+        ramCeilingPct: finitePositive(overrides.ramCeilingPct, DEFAULT_RAM_CEILING_PCT),
+        tickMs: wholeAtLeastOne(overrides.tickMs, DEFAULT_TICK_MS),
+        ...(cpuCeilingPct !== undefined ? { cpuCeilingPct } : {}),
+    };
+    return { mode: 'adaptive', adaptive };
+}
+// The cgroup "unlimited" sentinel: a limit at/above this is no real cap, so fall through to host total.
+const CGROUP_UNLIMITED = 2 ** 62;
+/**
+ * The process memory ceiling in bytes, read once (cgroup limits don't move at runtime): cgroup v2
+ * `memory.max`, then cgroup v1 `memory.limit_in_bytes`, then the host total. A `max` / absurdly-large
+ * ("unlimited") cgroup value falls through to {@link totalmem}. No new deps — just `node:fs`/`node:os`.
+ */
+export function readMemoryLimitBytes() {
+    try {
+        const raw = readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim();
+        if (raw && raw !== 'max') {
+            const n = Number(raw);
+            if (Number.isFinite(n) && n > 0 && n < CGROUP_UNLIMITED)
+                return n;
+        }
+    }
+    catch {
+        /* not cgroup v2 (or unreadable) — try v1 next */
+    }
+    try {
+        const raw = readFileSync('/sys/fs/cgroup/memory/memory.limit_in_bytes', 'utf8').trim();
+        const n = Number(raw);
+        if (Number.isFinite(n) && n > 0 && n < CGROUP_UNLIMITED)
+            return n;
+    }
+    catch {
+        /* not cgroup v1 (or unreadable) — fall through to host total */
+    }
+    return totalmem();
+}
+// How many recent completions the controller reasons over (latency percentiles + throughput + errors).
+const WINDOW_SIZE = 100;
+// EWMA smoothing for the "no-queuing" baseline (rttLong): low weight so a single fast sample doesn't
+// reset the baseline, but it still tracks a genuine floor shift.
+const RTT_LONG_EWMA_ALPHA = 0.2;
+// Consecutive ticks with in-flight work but zero completions before we call the worker stalled.
+const STALL_TICKS = 2;
+// Throughput is a rate over the trailing minute: a window whose newest completion is older than this
+// reports 0, so an idle worker never advertises its last burst as live.
+const THROUGHPUT_WINDOW_MS = 60_000;
+// The shortest span a rate is computed over, so a worker's first completion doesn't extrapolate a
+// sub-millisecond gap into a huge per-minute figure.
+const THROUGHPUT_MIN_SPAN_MS = 1_000;
+function percentile(sortedAsc, p) {
+    if (sortedAsc.length === 0)
+        return 0;
+    const idx = clamp(Math.ceil((p / 100) * sortedAsc.length) - 1, 0, sortedAsc.length - 1);
+    return sortedAsc[idx] ?? 0;
+}
+/**
+ * A single gradient-limit (AIMD) concurrency controller, shared by every per-routing-token task
+ * worker of one {@link import('./bullmq-transport.js').BullMQTransport}. It tracks in-flight work and a rolling window of
+ * completions, and — in `adaptive` mode — re-evaluates the worker's concurrency every `tickMs` from a
+ * latency gradient, an error/stall backpressure signal, a hard RAM brake and an optional CPU ceiling.
+ *
+ * In `fixed` mode no control loop runs and the limit never moves, but the controller still tracks
+ * `inFlight`/RSS/throughput/p95 (and samples CPU when a snapshot is taken) so the observability half
+ * (the heartbeat status payload) works without adaptive. Either way, {@link snapshot} returns the
+ * shared {@link WorkerStatus} contract the heartbeat writer stamps.
+ */
+export class AdaptiveController {
+    resolved;
+    apply;
+    now;
+    readRss;
+    memoryLimitBytes;
+    tickMs;
+    readCpuUsage;
+    cpuCores;
+    currentLimit;
+    inFlightCount = 0;
+    window = [];
+    rttLongMs;
+    cpuPct;
+    lastAdjust;
+    completionsThisTick = 0;
+    stallTicks = 0;
+    lastCpuUsage;
+    lastCpuSampleAt = 0;
+    timer;
+    constructor(options = {}) {
+        this.resolved = resolveConcurrency(options.concurrency);
+        this.apply = options.apply ?? (() => { });
+        this.now = options.now ?? Date.now;
+        this.readRss = options.readRss ?? (() => process.memoryUsage().rss);
+        this.memoryLimitBytes = (options.readMemoryLimit ?? readMemoryLimitBytes)();
+        this.tickMs = this.resolved.adaptive?.tickMs ?? DEFAULT_TICK_MS;
+        this.readCpuUsage = options.readCpuUsage ?? (() => process.cpuUsage());
+        this.cpuCores = wholeAtLeastOne((options.readCpuCount ?? (() => cpus().length))(), 1);
+        this.currentLimit =
+            this.resolved.mode === 'adaptive'
+                ? (this.resolved.adaptive?.start ?? DEFAULT_MIN)
+                : (this.resolved.fixed ?? 1);
+    }
+    /** The concurrency to hand the BullMQ Worker on construction (fixed N, or the adaptive `start`). */
+    get initialLimit() {
+        return this.currentLimit;
+    }
+    /** The live limit in effect right now. */
+    get limit() {
+        return this.currentLimit;
+    }
+    /** `true` when the controller will move the limit on its own (adaptive mode). */
+    get isAdaptive() {
+        return this.resolved.mode === 'adaptive';
+    }
+    /**
+     * Start the control loop (adaptive mode only; fixed mode just takes a CPU baseline). The timer is
+     * `unref`'d so it never keeps the process alive; idempotent.
+     */
+    start() {
+        if (this.timer)
+            return;
+        if (!this.lastCpuUsage)
+            this.resetCpuBaseline();
+        if (!this.isAdaptive)
+            return;
+        this.timer = setInterval(() => this.tick(), this.tickMs);
+        this.timer.unref?.();
+    }
+    /** Stop the control loop (call on worker close). Idempotent. */
+    stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = undefined;
+        }
+    }
+    /** A task started — bump in-flight. Call from the job processor before awaiting the body. */
+    onStart() {
+        this.inFlightCount += 1;
+    }
+    /**
+     * A task settled. ALWAYS drops in-flight and counts toward stall liveness (`completionsThisTick`),
+     * so the saturation gate, the heartbeat `inFlight`, and the "alive-but-busy" signal see every task.
+     *
+     * `kind` gates the MEASUREMENT window: only `'step'` completions push a duration into the rolling
+     * window that feeds the latency gradient / throughput / p95 / error rate. A workflow TURN
+     * (`kind: 'workflow'`) is NOT measured — a 5ms suspend sitting next to a 2s step would corrupt the
+     * gradient — but it still counts as in-flight + forward progress. This is the single concurrency
+     * pool measuring steps only; the transport tags the kind when it routes a job by shape.
+     */
+    onSettle(durationMs, ok, kind) {
+        this.inFlightCount = Math.max(0, this.inFlightCount - 1);
+        // Forward progress for stall detection counts BOTH kinds — a worker churning workflow turns is
+        // not stalled even when no step settles this tick.
+        this.completionsThisTick += 1;
+        if (kind !== 'step')
+            return;
+        this.window.push({ durationMs, ok, at: this.now() });
+        if (this.window.length > WINDOW_SIZE)
+            this.window.shift();
+    }
+    /**
+     * One control-loop iteration: sample CPU, refresh the latency baseline, then decide the next limit
+     * and apply it. A no-op in fixed mode. Exposed (not private) so a test can drive ticks
+     * deterministically.
+     */
+    tick() {
+        if (this.resolved.mode !== 'adaptive' || !this.resolved.adaptive)
+            return;
+        this.sampleCpu(this.tickMs);
+        this.refreshRttLong();
+        // Stall = in-flight work but no completion this tick, sustained over STALL_TICKS.
+        if (this.inFlightCount > 0 && this.completionsThisTick === 0)
+            this.stallTicks += 1;
+        else
+            this.stallTicks = 0;
+        const stalled = this.stallTicks >= STALL_TICKS;
+        this.completionsThisTick = 0;
+        this.decide(this.resolved.adaptive, stalled);
+    }
+    resetCpuBaseline() {
+        this.lastCpuUsage = this.readCpuUsage();
+        this.lastCpuSampleAt = this.now();
+    }
+    /** Set `cpuPct` from the CPU time spent since the previous sample, over `elapsedMs` of wall time. */
+    sampleCpu(elapsedMs) {
+        const previous = this.lastCpuUsage;
+        this.resetCpuBaseline();
+        if (!previous || !this.lastCpuUsage || elapsedMs <= 0)
+            return;
+        // CpuUsage is microseconds of CPU; elapsedMs of wall time is elapsedMs*1000 micros per core.
+        const micros = this.lastCpuUsage.user - previous.user + (this.lastCpuUsage.system - previous.system);
+        const pct = (micros / (elapsedMs * 1000 * this.cpuCores)) * 100;
+        if (Number.isFinite(pct))
+            this.cpuPct = Math.max(0, pct);
+    }
+    /** Fixed mode has no control loop, so a snapshot samples CPU itself, at most once per tick period
+     *  (several heartbeat keys snapshot back to back, and a near-zero span gives a meaningless rate). */
+    sampleCpuForSnapshot() {
+        if (this.isAdaptive)
+            return;
+        if (!this.lastCpuUsage) {
+            this.resetCpuBaseline();
+            return;
+        }
+        const elapsedMs = this.now() - this.lastCpuSampleAt;
+        if (elapsedMs >= this.tickMs)
+            this.sampleCpu(elapsedMs);
+    }
+    refreshRttLong() {
+        if (this.window.length === 0)
+            return;
+        let minDuration = this.window[0]?.durationMs ?? 0;
+        for (const c of this.window)
+            if (c.durationMs < minDuration)
+                minDuration = c.durationMs;
+        this.rttLongMs =
+            this.rttLongMs === undefined
+                ? minDuration
+                : RTT_LONG_EWMA_ALPHA * minDuration + (1 - RTT_LONG_EWMA_ALPHA) * this.rttLongMs;
+    }
+    rssPct() {
+        if (this.memoryLimitBytes <= 0)
+            return 0;
+        return (100 * this.readRss()) / this.memoryLimitBytes;
+    }
+    decide(cfg, stalled) {
+        const current = this.currentLimit;
+        const durations = this.window.map((c) => c.durationMs).sort((a, b) => a - b);
+        const p50 = percentile(durations, 50);
+        const errors = this.window.reduce((n, c) => n + (c.ok ? 0 : 1), 0);
+        const errorRate = this.window.length === 0 ? 0 : errors / this.window.length;
+        const gradient = this.rttLongMs !== undefined && p50 > 0 ? Math.min(1, this.rttLongMs / p50) : 1;
+        const rssPct = this.rssPct();
+        const saturated = this.inFlightCount >= current * 0.8;
+        let next = current;
+        let reason;
+        if (rssPct >= cfg.ramCeilingPct) {
+            next = Math.max(cfg.min, Math.floor(current * 0.8));
+            reason = 'ram_ceiling';
+        }
+        else if (cfg.cpuCeilingPct !== undefined && (this.cpuPct ?? 0) >= cfg.cpuCeilingPct) {
+            // CPU at/over the ceiling: never grow; shed one to relieve it.
+            next = Math.max(cfg.min, current - 1);
+            reason = 'cpu_ceiling';
+        }
+        else if (errorRate > 0.2 || stalled) {
+            next = Math.max(cfg.min, current - 1);
+            reason = 'backpressure';
+        }
+        else if (gradient < 0.7) {
+            next = Math.max(cfg.min, Math.floor(current * gradient));
+            reason = 'shrink';
+        }
+        else if (gradient >= 0.9 && saturated) {
+            next = Math.min(cfg.max, current + 1);
+            reason = 'grow';
+        }
+        next = clamp(next, cfg.min, cfg.max);
+        if (next === current || reason === undefined)
+            return;
+        this.currentLimit = next;
+        this.lastAdjust = { at: this.now(), from: current, to: next, reason };
+        this.apply(next);
+    }
+    /** The shared {@link WorkerStatus} the heartbeat writer stamps. Best-effort fields are omitted when
+     *  unmeasured (no completions yet → no p95/throughput; no CPU sample yet → no `cpuPct`). */
+    snapshot() {
+        this.sampleCpuForSnapshot();
+        const durations = this.window.map((c) => c.durationMs).sort((a, b) => a - b);
+        const rssBytes = this.readRss();
+        const rssLimitBytes = this.memoryLimitBytes;
+        const status = {
+            runtime: 'node',
+            concurrency: {
+                mode: this.resolved.mode,
+                limit: this.currentLimit,
+                ...(this.resolved.mode === 'adaptive' && this.resolved.adaptive
+                    ? { min: this.resolved.adaptive.min, max: this.resolved.adaptive.max }
+                    : {}),
+            },
+            inFlight: this.inFlightCount,
+            rssBytes,
+            rssLimitBytes,
+            rssPct: rssLimitBytes > 0 ? (100 * rssBytes) / rssLimitBytes : 0,
+        };
+        if (this.cpuPct !== undefined)
+            status.cpuPct = this.cpuPct;
+        if (this.window.length > 0) {
+            status.throughputPerMin = this.throughputPerMin();
+            status.p95Ms = percentile(durations, 95);
+        }
+        if (this.lastAdjust)
+            status.lastAdjust = this.lastAdjust;
+        return status;
+    }
+    /**
+     * Completions per minute over the trailing minute: the window's completions from the last minute,
+     * over the span from the oldest of them (or a full minute, when the window reaches further back) to
+     * now, floored at a second. The rate decays while the worker is idle and reads 0 once a minute passes with no completion.
+     */
+    throughputPerMin() {
+        const first = this.window[0];
+        if (!first)
+            return 0;
+        const now = this.now();
+        const recent = this.window.filter((c) => now - c.at <= THROUGHPUT_WINDOW_MS).length;
+        if (recent === 0)
+            return 0;
+        const spanMs = clamp(now - first.at, THROUGHPUT_MIN_SPAN_MS, THROUGHPUT_WINDOW_MS);
+        return (recent / spanMs) * 60_000;
+    }
+}
+//# sourceMappingURL=adaptive-concurrency.js.map

package/dist/src/transports/bullmq/adaptive-concurrency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptive-concurrency.js","sourceRoot":"","sources":["../../../../src/transports/bullmq/adaptive-concurrency.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAmDzC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,SAAS,KAAK,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,iFAAiF;AACjF,SAAS,eAAe,CAAC,KAAyB,EAAE,QAAgB;IAClE,OAAO,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpG,CAAC;AAED,mEAAmE;AACnE,SAAS,cAAc,CAAI,KAAyB,EAAE,QAAW;IAC/D,OAAO,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAuB;IACxD,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC1D,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;IAEtF,MAAM,SAAS,GAA+B,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,GAAG,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACxD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACrE,MAAM,aAAa,GAAG,cAAc,CAAC,SAAS,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAmB;QAC/B,GAAG;QACH,GAAG;QACH,KAAK;QACL,aAAa,EAAE,cAAc,CAAC,SAAS,CAAC,aAAa,EAAE,uBAAuB,CAAC;QAC/E,MAAM,EAAE,eAAe,CAAC,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC;QAC1D,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AACxC,CAAC;AAED,wGAAwG;AACxG,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,CAAC;AAEjC;;;;GAIG;AACH,MAAM,UAAU,oBAAoB;IAClC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,gBAAgB;gBAAE,OAAO,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;IACnD,CAAC;IACD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,6CAA6C,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACvF,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,gBAAgB;YAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,gEAAgE;IAClE,CAAC;IACD,OAAO,QAAQ,EAAE,CAAC;AACpB,CAAC;AASD,uGAAuG;AACvG,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,qGAAqG;AACrG,iEAAiE;AACjE,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,gGAAgG;AAChG,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,qGAAqG;AACrG,wEAAwE;AACxE,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,kGAAkG;AAClG,qDAAqD;AACrD,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,SAAS,UAAU,CAAC,SAAmB,EAAE,CAAS;IAChD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxF,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAwBD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,kBAAkB;IACZ,QAAQ,CAAsB;IAC9B,KAAK,CAA0B;IAC/B,GAAG,CAAe;IAClB,OAAO,CAAe;IACtB,gBAAgB,CAAS;IACzB,MAAM,CAAS;IACf,YAAY,CAAwB;IACpC,QAAQ,CAAS;IAE1B,YAAY,CAAS;IACrB,aAAa,GAAG,CAAC,CAAC;IACT,MAAM,GAAiB,EAAE,CAAC;IACnC,SAAS,CAAU;IACnB,MAAM,CAAU;IAChB,UAAU,CAAgB;IAE1B,mBAAmB,GAAG,CAAC,CAAC;IACxB,UAAU,GAAG,CAAC,CAAC;IACf,YAAY,CAAmB;IAC/B,eAAe,GAAG,CAAC,CAAC;IACpB,KAAK,CAA6C;IAE1D,YAAY,OAAO,GAA8B,EAAE;QACjD,IAAI,CAAC,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;QACpE,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,eAAe,IAAI,oBAAoB,CAAC,EAAE,CAAC;QAC5E,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,eAAe,CAAC;QAChE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,GAAG,eAAe,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,YAAY;YACf,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU;gBAC/B,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,IAAI,WAAW,CAAC;gBAChD,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,oGAAoG;IACpG,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,0CAA0C;IAC1C,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,iFAAiF;IACjF,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACvB,CAAC;IAED,gEAAgE;IAChE,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACzB,CAAC;IACH,CAAC;IAED,6FAA6F;IAC7F,OAAO;QACL,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;OASG;IACH,QAAQ,CAAC,UAAkB,EAAE,EAAW,EAAE,IAAyB;QACjE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;QACzD,+FAA+F;QAC/F,mDAAmD;QACnD,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAC9B,IAAI,IAAI,KAAK,MAAM;YAAE,OAAO;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW;YAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACH,IAAI;QACF,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO;QACzE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,kFAAkF;QAClF,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,IAAI,CAAC,mBAAmB,KAAK,CAAC;YAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;;YAC9E,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,IAAI,WAAW,CAAC;QAC/C,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAE7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,qGAAqG;IAC7F,SAAS,CAAC,SAAiB;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,SAAS,IAAI,CAAC;YAAE,OAAO;QAC9D,6FAA6F;QAC7F,MAAM,MAAM,GACV,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxF,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,SAAS,GAAG,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC;QAChE,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3D,CAAC;IAED;yGACqG;IAC7F,oBAAoB;QAC1B,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QACpD,IAAI,SAAS,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC1D,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACrC,IAAI,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC;QAClD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,CAAC,UAAU,GAAG,WAAW;gBAAE,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC;QACxF,IAAI,CAAC,SAAS;YACZ,IAAI,CAAC,SAAS,KAAK,SAAS;gBAC1B,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,mBAAmB,GAAG,WAAW,GAAG,CAAC,CAAC,GAAG,mBAAmB,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;IACvF,CAAC;IAEO,MAAM;QACZ,IAAI,IAAI,CAAC,gBAAgB,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAEO,MAAM,CAAC,GAAmB,EAAE,OAAgB;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC7E,MAAM,QAAQ,GACZ,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,GAAG,GAAG,CAAC;QAEtD,IAAI,IAAI,GAAG,OAAO,CAAC;QACnB,IAAI,MAA0C,CAAC;QAE/C,IAAI,MAAM,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YAChC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,GAAG,aAAa,CAAC;QACzB,CAAC;aAAM,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YACtF,+DAA+D;YAC/D,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,aAAa,CAAC;QACzB,CAAC;aAAM,IAAI,SAAS,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,cAAc,CAAC;QAC1B,CAAC;aAAM,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YAC1B,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC;YACzD,MAAM,GAAG,QAAQ,CAAC;QACpB,CAAC;aAAM,IAAI,QAAQ,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YACxC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,MAAM,CAAC;QAClB,CAAC;QAED,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,KAAK,OAAO,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED;gGAC4F;IAC5F,QAAQ;QACN,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC5C,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,MAAM;YACf,WAAW,EAAE;gBACX,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;gBACxB,KAAK,EAAE,IAAI,CAAC,YAAY;gBACxB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ;oBAC7D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE;oBACtE,CAAC,CAAC,EAAE,CAAC;aACR;YACD,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,QAAQ;YACR,aAAa;YACb,MAAM,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;SACjE,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3D,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,IAAI,CAAC,UAAU;YAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACzD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACK,gBAAgB;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,oBAAoB,CAAC,CAAC,MAAM,CAAC;QACpF,IAAI,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,EAAE,sBAAsB,EAAE,oBAAoB,CAAC,CAAC;QACnF,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;IACpC,CAAC;CACF"}

package/dist/src/transports/bullmq/bullmq-transport.d.ts

@@ -2,7 +2,8 @@ import type { WorkerDescriptor } from '../../handshake/descriptor.js';
 import type { GroupHealth, Heartbeat, RemoteTask, RunReply, RunRequest, StartRunMessage, StepResult, TenantEvent, Transport, WorkflowDecision, WorkflowStepEvent, WorkflowTask } from '../../interfaces.js';
 import { type StepHandler } from '../../protocol.js';
 import { type WorkflowTurnHandler } from '../../workflow-turn.js';
-import type { BullMQDeps, RedisLike } from './deps.js';
+import { type ConcurrencyOption } from './adaptive-concurrency.js';
+import { type BullMQDeps, type RedisLike } from './deps.js';
 export interface BullMQTransportOptions {
     /**
      * Ceiling (ms) `close()` gives the graceful shutdown before it stops waiting and drops the Redis
@@ -34,6 +35,22 @@ export interface BullMQTransportOptions {
     instanceId?: string | undefined;
     /** Where a background failure (a failed heartbeat refresh, a result-publish error) is reported. */
     onError?: ((err: unknown) => void) | undefined;
+    /**
+     * How many step results this instance settles at once from the results queue. With the default
+     * of 1, a wide fan's results queue up behind one checkpoint write at a time on each pod. The engine
+     * still resumes one run's turns in order on an instance, so a run never gets two concurrent turns
+     * from it. A non-finite or sub-1 value falls back to the default. Default 1.
+     */
+    resultConcurrency?: number | undefined;
+    /**
+     * How many tasks each per-routing-token task worker runs at once. A number pins it (a non-finite or
+     * sub-1 value falls back to 1). `'adaptive'` or `{ mode: 'adaptive', min, max, start, tickMs,
+     * ramCeilingPct, cpuCeilingPct }` lets an {@link AdaptiveController} move the limit at runtime from
+     * step latency, error rate and stalls, braking on RAM (and CPU, when a ceiling is set). The live
+     * limit, in-flight count and last adjustment ride the worker heartbeat in both modes. Unset leaves
+     * each task worker at the deps' default concurrency.
+     */
+    concurrency?: ConcurrencyOption | undefined;
 }
 /**
  * A queue-backed {@link Transport} over BullMQ/Redis, BYTE-COMPATIBLE with the aviary

package/dist/src/transports/bullmq/bullmq-transport.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bullmq-transport.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/bullmq-transport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,UAAU,EACV,QAAQ,EACR,UAAU,EACV,eAAe,EACf,UAAU,EACV,WAAW,EACX,SAAS,EAGT,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,OAAO,EAAkB,KAAK,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,KAAK,EAAE,UAAU,EAAa,SAAS,EAAc,MAAM,WAAW,CAAC;AA4B9E,MAAM,WAAW,sBAAsB;IACrC;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,2FAA2F;IAC3F,IAAI,EAAE,UAAU,CAAC;IACjB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,oEAAoE;IACpE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yFAAyF;IACzF,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,mGAAmG;IACnG,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;CAChD;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,SAAS;;IAgD/C,YAAY,OAAO,EAAE,sBAAsB,EAU1C;IAED,iEAAiE;IACjE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAGpC;IAyBD;;;;;;;OAOG;IACH,qBAAqB,IAAI,SAAS,CAEjC;IAMD;+EAC2E;IACrE,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9C;IAED;2GACuG;IACjG,oBAAoB,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAM5D;IAMD;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,IAAI,CAG1C;IAED;;;;;;OAMG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAG5D;IA0HD;oGACgG;IAC1F,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9C;IA8BD,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAe7D;IAED,UAAU,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKvE;IAED;uEACmE;IAC7D,iBAAiB,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAM/D;IAED,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKtE;IAED,WAAW,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAiB7D;IAyED;;gHAE4G;IACtG,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAgB1C;IAYK,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAmCtE;IAED;uGACmG;IAC7F,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CASrD;IA+CD,sGAAsG;IAChG,gBAAgB,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;IAED,uGAAuG;IACvG,UAAU,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKjE;IAED,qGAAqG;IAC/F,kBAAkB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAED,kGAAkG;IAClG,YAAY,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAK9D;IAED,mGAAmG;IAC7F,eAAe,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,wGAAwG;IACxG,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,IAAI,CAoBnD;IAED,2GAA2G;IACrG,kBAAkB,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAMxD;IAED;;;;;OAKG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAqB7E;IAyBD,uGAAuG;IACjG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA8B3B;CACF"}
+{"version":3,"file":"bullmq-transport.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/bullmq-transport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,UAAU,EACV,QAAQ,EACR,UAAU,EACV,eAAe,EACf,UAAU,EACV,WAAW,EACX,SAAS,EAGT,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,OAAO,EAAkB,KAAK,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAsB,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACvF,OAAO,EACL,KAAK,UAAU,EAEf,KAAK,SAAS,EAGf,MAAM,WAAW,CAAC;AA4BnB,MAAM,WAAW,sBAAsB;IACrC;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,2FAA2F;IAC3F,IAAI,EAAE,UAAU,CAAC;IACjB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,oEAAoE;IACpE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yFAAyF;IACzF,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,mGAAmG;IACnG,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAC/C;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;CAC7C;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,SAAS;;IAgD/C,YAAY,OAAO,EAAE,sBAAsB,EAY1C;IAeD,iEAAiE;IACjE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAGpC;IAyBD;;;;;;;OAOG;IACH,qBAAqB,IAAI,SAAS,CAEjC;IAMD;+EAC2E;IACrE,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9C;IAED;2GACuG;IACjG,oBAAoB,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAM5D;IAMD;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,IAAI,CAG1C;IAED;;;;;;OAMG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAG5D;IA0ID;oGACgG;IAC1F,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9C;IA8BD,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAiB7D;IAED,UAAU,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKvE;IAED;uEACmE;IAC7D,iBAAiB,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAM/D;IAED,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKtE;IAED,WAAW,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAiB7D;IA2CD;;gHAE4G;IACtG,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAgB1C;IAYK,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAmCtE;IAED;uGACmG;IAC7F,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CASrD;IA+CD,sGAAsG;IAChG,gBAAgB,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;IAED,uGAAuG;IACvG,UAAU,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKjE;IAED,qGAAqG;IAC/F,kBAAkB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAED,kGAAkG;IAClG,YAAY,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAK9D;IAED,mGAAmG;IAC7F,eAAe,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,wGAAwG;IACxG,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,IAAI,CAoBnD;IAED,2GAA2G;IACrG,kBAAkB,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAMxD;IAED;;;;;OAKG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAqB7E;IAyBD,uGAAuG;IACjG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA+B3B;CACF"}

package/dist/src/transports/bullmq/bullmq-transport.js

@@ -1,6 +1,8 @@
 import { runStepHandler } from '../../protocol.js';
 import { swallowSubscribe } from '../../safe-subscribe.js';
 import { isWorkflowTask } from '../../workflow-turn.js';
+import { AdaptiveController } from './adaptive-concurrency.js';
+import { workerConcurrency, } from './deps.js';
 import { decisionsName, effectivePrefix, heartbeatChannel, resultsName, routingToken, runReplyChannel, runRequestName, startRunName, stepEventsName, tasksName, tenantEventsChannel, workerDescriptorTokenPrefix, workerHeartbeatKey, workerHeartbeatKeyPrefix, } from './naming.js';
 import { buildInstanceId, heartbeatKeyValue, jobOptions, parseHeartbeatValue, resultJobOptions, taskJobOptions, WORKER_HEARTBEAT_INTERVAL_MS, WORKER_HEARTBEAT_TTL_SECONDS, } from './serialization.js';
 /**
@@ -30,6 +32,11 @@ export class BullMQTransport {
     #hasOnError;
     #closing = false;
     #closeTimeoutMs;
+    #resultConcurrency;
+    #concurrency;
+    // Shared by every task worker: tracks in-flight work and the step-completion window, snapshots the
+    // heartbeat status and, in adaptive mode, sets every task worker's `concurrency` on each move.
+    #controller;
     #handlers = new Map();
     // Worker-side workflow turn consumers, keyed by workflow name — a `workflow`-shaped job routes here.
     #workflowTurns = new Map();
@@ -46,11 +53,6 @@ export class BullMQTransport {
     #workerRedis;
     #workerHeartbeatTimer;
     #heartbeatTokens = new Set();
-    // Live execution telemetry carried on the heartbeat (the console's worker cards): jobs currently
-    // in a handler, and a bounded window of recent completions for throughput / p95.
-    #inFlightJobs = 0;
-    #recentCompletions = [];
-    #lastCpu;
     // P4 store-less protocol: start-run / run-request queues + consumers, run-reply / tenant-events pub/sub.
     #startRunWorker;
     #runRequestWorker;
@@ -72,6 +74,21 @@ export class BullMQTransport {
         this.#onError = options.onError ?? (() => { });
         this.#hasOnError = options.onError !== undefined;
         this.#closeTimeoutMs = options.closeTimeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS;
+        this.#resultConcurrency = workerConcurrency(options.resultConcurrency);
+        this.#concurrency = options.concurrency;
+    }
+    /** The shared concurrency controller, created on first use (a task worker, a job or a heartbeat). */
+    #concurrencyController() {
+        if (!this.#controller) {
+            this.#controller = new AdaptiveController({
+                ...(this.#concurrency !== undefined ? { concurrency: this.#concurrency } : {}),
+                apply: (limit) => {
+                    for (const worker of this.#taskWorkers.values())
+                        worker.concurrency = limit;
+                },
+            });
+        }
+        return this.#controller;
     }
     /** Stable id stamped on this instance's worker-liveness keys. */
     get instanceId() {
@@ -158,7 +175,10 @@ export class BullMQTransport {
     #ensureTaskWorker(token) {
         if (this.#taskWorkers.has(token))
             return;
-        const worker = this.#deps.makeWorker(tasksName(this.#effectivePrefix(), token), (job) => this.#runJob(job.data));
+        const controller = this.#concurrencyController();
+        // A new worker starts at the controller's live limit, so a token registered after an adaptive
+        // move joins at the current limit rather than at `start`.
+        const worker = this.#deps.makeWorker(tasksName(this.#effectivePrefix(), token), (job) => this.#runJob(job.data), this.#concurrency !== undefined ? { concurrency: controller.limit } : undefined);
         // Terminal-failure bridge: a task job reaching a TERMINAL `failed` state is an INFRASTRUCTURE
         // failure (worker crash / stalled-count exhaustion), never a handler business error —
         // `runStepHandler` catches every handler throw and publishes a failed StepResult so the job
@@ -174,23 +194,31 @@ export class BullMQTransport {
             void this.#bridgeTaskFailure(identity, job?.failedReason ?? error?.message ?? 'unknown');
         });
         this.#taskWorkers.set(token, worker);
+        controller.start();
         this.#startWorkerHeartbeat(token);
     }
     /** Route one task job by SHAPE (spec §6.3): a workflow turn runs the replay → decision path, a step
      *  runs the `runStepHandler` → result path. Both kinds share a token's queue but never each other's. */
     async #runJob(data) {
+        const controller = this.#concurrencyController();
         const startedAt = Date.now();
-        this.#inFlightJobs += 1;
-        try {
-            if (isWorkflowTask(data))
+        controller.onStart();
+        // Only step durations feed the latency gradient; a workflow turn still counts as in-flight work
+        // and as forward progress for stall detection.
+        if (isWorkflowTask(data)) {
+            try {
                 return await this.#runWorkflowTask(data);
-            return await this.#runTask(data);
+            }
+            finally {
+                controller.onSettle(Date.now() - startedAt, true, 'workflow');
+            }
+        }
+        let ok = false;
+        try {
+            ok = await this.#runTask(data);
         }
         finally {
-            this.#inFlightJobs -= 1;
-            this.#recentCompletions.push({ at: Date.now(), ms: Date.now() - startedAt });
-            if (this.#recentCompletions.length > 200)
-                this.#recentCompletions.shift();
+            controller.onSettle(Date.now() - startedAt, ok, 'step');
         }
     }
     /** Worker side: replay a workflow turn and publish its {@link WorkflowDecision} on `${P}-decisions`
@@ -235,6 +263,7 @@ export class BullMQTransport {
             throw err;
         }
     }
+    /** Run one step and publish its result. Resolves `true` when the step completed. */
     async #runTask(task) {
         let result;
         try {
@@ -257,6 +286,7 @@ export class BullMQTransport {
         }
         try {
             await this.#queue(resultsName(this.#effectivePrefix())).add('result', result, resultJobOptions());
+            return result.status === 'completed';
         }
         catch (err) {
             // The results-queue publish itself threw (e.g. Redis blipped). We still hold `task`, so
@@ -296,7 +326,7 @@ export class BullMQTransport {
     onResult(handler) {
         if (this.#resultsWorker)
             return;
-        this.#resultsWorker = this.#deps.makeWorker(resultsName(this.#effectivePrefix()), (job) => handler(job.data));
+        this.#resultsWorker = this.#deps.makeWorker(resultsName(this.#effectivePrefix()), (job) => handler(job.data), { concurrency: this.#resultConcurrency });
         this.#resultsWorker.on('failed', (job, err) => {
             if (!job || (job.attemptsMade ?? 0) < (job.opts?.attempts ?? 1))
                 return;
@@ -342,42 +372,12 @@ export class BullMQTransport {
     // worker liveness registry — `${P}-worker-heartbeat:${token}:${instanceId}` (SET … EX 35)
     // ---------------------------------------------------------------------------
     /**
-     * The live execution/resource snapshot each heartbeat carries — what lights up the console's
-     * worker cards (concurrency mode, in-flight, RSS/CPU, throughput, p95). Parity with the NestJS
-     * fleet's heartbeat payload; the AdonisJS heartbeat used to carry only the timestamp, leaving the
-     * cards permanently on "no status". Cheap by construction: process-level reads plus a bounded
-     * completions window.
+     * The live execution/resource snapshot each heartbeat carries, which lights up the console's worker
+     * cards (concurrency mode and live limit, in-flight, RSS/CPU, throughput, p95, last adjustment):
+     * the same {@link WorkerStatus} the NestJS fleet stamps.
      */
     #workerStatus() {
-        const now = Date.now();
-        const windowStart = now - 60_000;
-        const recent = this.#recentCompletions.filter((c) => c.at >= windowStart);
-        const durations = [...this.#recentCompletions].map((c) => c.ms).sort((a, b) => a - b);
-        const p95 = durations.length
-            ? durations[Math.min(durations.length - 1, Math.floor(durations.length * 0.95))]
-            : undefined;
-        const mem = process.memoryUsage();
-        // CPU% since the previous beat (user+system µs over wall-clock µs).
-        let cpuPct;
-        const cpuNow = process.cpuUsage();
-        if (this.#lastCpu) {
-            const elapsedUs = (now - this.#lastCpu.at) * 1000;
-            const usedUs = cpuNow.user - this.#lastCpu.usage.user + (cpuNow.system - this.#lastCpu.usage.system);
-            if (elapsedUs > 0)
-                cpuPct = Math.round((usedUs / elapsedUs) * 100);
-        }
-        this.#lastCpu = { usage: cpuNow, at: now };
-        return {
-            runtime: 'node',
-            // One BullMQ Worker per handled routing token, each at the adapter's default concurrency (1)
-            // — so this instance can run up to one job per token simultaneously.
-            concurrency: { mode: 'fixed', limit: Math.max(1, this.#taskWorkers.size) },
-            inFlight: this.#inFlightJobs,
-            rssBytes: mem.rss,
-            ...(cpuPct !== undefined ? { cpuPct } : {}),
-            throughputPerMin: recent.length,
-            ...(p95 !== undefined ? { p95Ms: p95 } : {}),
-        };
+        return this.#concurrencyController().snapshot();
     }
     /** Refresh EVERY handled routing token's TTL'd liveness key on ONE shared 10s interval until
      *  `close()`. A newly added token beats immediately so a freshly-registered handler is visible
@@ -630,6 +630,7 @@ export class BullMQTransport {
     /** Close all workers/queues and disconnect pub/sub + heartbeat connections so the process can exit. */
     async close() {
         this.#closing = true;
+        this.#controller?.stop();
         if (this.#workerHeartbeatTimer)
             clearInterval(this.#workerHeartbeatTimer);
         this.#workerHeartbeatTimer = undefined;