@adonis-agora/durable 0.49.1 → 0.51.0

package/dist/src/transports/adaptive-concurrency.d.ts

@@ -0,0 +1,168 @@
+import type { WorkerStatus } from '../interfaces.js';
+/**
+ * The adaptive controller's tunables — every field optional, each with a default from the spec.
+ * Spread under `{ mode: 'adaptive' }` to widen a plain `concurrency: number` into a self-tuning knob.
+ */
+export interface AdaptiveConcurrencyOptions {
+    /** Floor the controller won't go below. Default 1. */
+    min?: number;
+    /** Ceiling the controller won't exceed. Default 32. */
+    max?: number;
+    /** Initial limit. Default `min` (clamped into `[min,max]`). */
+    start?: number;
+    /** RSS percent of the memory ceiling at which the hard brake fires. Default 85. */
+    ramCeilingPct?: number;
+    /** CPU percent (0..100×cores normalised to 0..100) ceiling; absent = off. Default undefined. */
+    cpuCeilingPct?: number;
+    /** Control-loop period in ms. Default 2000. */
+    tickMs?: number;
+}
+/**
+ * The `concurrency` option, widened (backward-compatible): a plain `number` is fixed (unchanged), the
+ * string `'adaptive'` is adaptive-with-defaults, and an object `{ mode:'adaptive', ... }` is adaptive
+ * with overrides. `undefined` keeps the historical default of 1 (fixed).
+ */
+export type ConcurrencyOption = number | 'adaptive' | ({
+    mode: 'adaptive';
+} & AdaptiveConcurrencyOptions);
+/** The fully-resolved adaptive tunables (every default applied), carried on a resolved config. */
+export interface AdaptiveConfig {
+    min: number;
+    max: number;
+    start: number;
+    ramCeilingPct: number;
+    cpuCeilingPct?: number;
+    tickMs: number;
+}
+/** A normalised `concurrency` option: a `fixed` number, or an `adaptive` config with every default applied. */
+export interface ResolvedConcurrency {
+    mode: 'fixed' | 'adaptive';
+    /** Present when `mode === 'fixed'` — the never-moving concurrency. */
+    fixed?: number;
+    /** Present when `mode === 'adaptive'` — the controller's tunables. */
+    adaptive?: AdaptiveConfig;
+}
+/**
+ * Normalise a {@link ConcurrencyOption} into a {@link ResolvedConcurrency}: a bare number (or
+ * `undefined` → 1) becomes `{ mode:'fixed', fixed }`; `'adaptive'` / `{ mode:'adaptive', ... }`
+ * become `{ mode:'adaptive', adaptive }` with every default from the spec applied and `start`
+ * clamped into `[min,max]`.
+ *
+ * A non-finite or out-of-range number falls back to its default (a bad `cpuCeilingPct` turns the
+ * CPU ceiling off): BullMQ throws on a `NaN` or sub-1 concurrency, a `NaN` ceiling never brakes, and
+ * a `NaN` `tickMs` makes the control loop fire every millisecond.
+ */
+export declare function resolveConcurrency(opt?: ConcurrencyOption): ResolvedConcurrency;
+/**
+ * The process memory ceiling in bytes, read once (cgroup limits don't move at runtime): cgroup v2
+ * `memory.max`, then cgroup v1 `memory.limit_in_bytes`, then the host total. A `max` / absurdly-large
+ * ("unlimited") cgroup value falls through to {@link totalmem}. No new deps — just `node:fs`/`node:os`.
+ */
+export declare function readMemoryLimitBytes(): number;
+/** Construction surface for {@link AdaptiveController}. */
+export interface AdaptiveControllerOptions {
+    /** The raw `concurrency` option (resolved internally). */
+    concurrency?: ConcurrencyOption;
+    /**
+     * Applied whenever the live limit changes — BullMQ wires it to `worker.concurrency = limit`, the
+     * queue transport to waking its slot loops. Called once with the initial limit is NOT done here; the
+     * caller starts at {@link AdaptiveController.initialLimit} and reads {@link AdaptiveController.limit}.
+     * Subsequent adaptive moves call this.
+     */
+    apply?: (limit: number) => void;
+    /** Injectable clock (tests). Defaults to {@link Date.now}. */
+    now?: () => number;
+    /** Injectable RSS reader (tests). Defaults to `process.memoryUsage().rss`. */
+    readRss?: () => number;
+    /** Injectable memory-ceiling reader (tests). Defaults to {@link readMemoryLimitBytes} (read once). */
+    readMemoryLimit?: () => number;
+    /** Injectable cumulative CPU-time reader (tests). Defaults to `process.cpuUsage()`. */
+    readCpuUsage?: () => NodeJS.CpuUsage;
+    /** Injectable core-count reader (tests), read once. Defaults to `os.cpus().length`. */
+    readCpuCount?: () => number;
+}
+/**
+ * A single gradient-limit (AIMD) concurrency controller, shared by every per-routing-token task
+ * worker of one transport ({@link import('./bullmq/bullmq-transport.js').BullMQTransport} or
+ * {@link import('./queue.js').QueueTransport}). It tracks in-flight work and a rolling window of
+ * completions, and — in `adaptive` mode — re-evaluates the worker's concurrency every `tickMs` from a
+ * latency gradient, an error/stall backpressure signal, a hard RAM brake and an optional CPU ceiling.
+ *
+ * In `fixed` mode no control loop runs and the limit never moves, but the controller still tracks
+ * `inFlight`/RSS/throughput/p95 (and samples CPU when a snapshot is taken) so the observability half
+ * (the heartbeat status payload) works without adaptive. Either way, {@link snapshot} returns the
+ * shared {@link WorkerStatus} contract the heartbeat writer stamps.
+ */
+export declare class AdaptiveController {
+    private readonly resolved;
+    private readonly apply;
+    private readonly now;
+    private readonly readRss;
+    private readonly memoryLimitBytes;
+    private readonly tickMs;
+    private readonly readCpuUsage;
+    private readonly cpuCores;
+    private currentLimit;
+    private inFlightCount;
+    private readonly window;
+    private rttLongMs?;
+    private cpuPct?;
+    private lastAdjust?;
+    private completionsThisTick;
+    private stallTicks;
+    private lastCpuUsage?;
+    private lastCpuSampleAt;
+    private timer;
+    constructor(options?: AdaptiveControllerOptions);
+    /** The concurrency a task consumer starts at (fixed N, or the adaptive `start`). */
+    get initialLimit(): number;
+    /** The live limit in effect right now. */
+    get limit(): number;
+    /** `true` when the controller will move the limit on its own (adaptive mode). */
+    get isAdaptive(): boolean;
+    /**
+     * Start the control loop (adaptive mode only; fixed mode just takes a CPU baseline). The timer is
+     * `unref`'d so it never keeps the process alive; idempotent.
+     */
+    start(): void;
+    /** Stop the control loop (call on worker close). Idempotent. */
+    stop(): void;
+    /** A task started — bump in-flight. Call from the job processor before awaiting the body. */
+    onStart(): void;
+    /**
+     * A task settled. ALWAYS drops in-flight and counts toward stall liveness (`completionsThisTick`),
+     * so the saturation gate, the heartbeat `inFlight`, and the "alive-but-busy" signal see every task.
+     *
+     * `kind` gates the MEASUREMENT window: only `'step'` completions push a duration into the rolling
+     * window that feeds the latency gradient / throughput / p95 / error rate. A workflow TURN
+     * (`kind: 'workflow'`) is NOT measured — a 5ms suspend sitting next to a 2s step would corrupt the
+     * gradient — but it still counts as in-flight + forward progress. This is the single concurrency
+     * pool measuring steps only; the transport tags the kind when it routes a job by shape.
+     */
+    onSettle(durationMs: number, ok: boolean, kind: 'workflow' | 'step'): void;
+    /**
+     * One control-loop iteration: sample CPU, refresh the latency baseline, then decide the next limit
+     * and apply it. A no-op in fixed mode. Exposed (not private) so a test can drive ticks
+     * deterministically.
+     */
+    tick(): void;
+    private resetCpuBaseline;
+    /** Set `cpuPct` from the CPU time spent since the previous sample, over `elapsedMs` of wall time. */
+    private sampleCpu;
+    /** Fixed mode has no control loop, so a snapshot samples CPU itself, at most once per tick period
+     *  (several heartbeat keys snapshot back to back, and a near-zero span gives a meaningless rate). */
+    private sampleCpuForSnapshot;
+    private refreshRttLong;
+    private rssPct;
+    private decide;
+    /** The shared {@link WorkerStatus} the heartbeat writer stamps. Best-effort fields are omitted when
+     *  unmeasured (no completions yet → no p95/throughput; no CPU sample yet → no `cpuPct`). */
+    snapshot(): WorkerStatus;
+    /**
+     * Completions per minute over the trailing minute: the window's completions from the last minute,
+     * over the span from the oldest of them (or a full minute, when the window reaches further back) to
+     * now, floored at a second. The rate decays while the worker is idle and reads 0 once a minute passes with no completion.
+     */
+    private throughputPerMin;
+}
+//# sourceMappingURL=adaptive-concurrency.d.ts.map

package/dist/src/transports/adaptive-concurrency.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptive-concurrency.d.ts","sourceRoot":"","sources":["../../../src/transports/adaptive-concurrency.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAgB,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;GAGG;AACH,MAAM,WAAW,0BAA0B;IACzC,sDAAsD;IACtD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mFAAmF;IACnF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gGAAgG;IAChG,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GACzB,MAAM,GACN,UAAU,GACV,CAAC;IAAE,IAAI,EAAE,UAAU,CAAA;CAAE,GAAG,0BAA0B,CAAC,CAAC;AAExD,kGAAkG;AAClG,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,+GAA+G;AAC/G,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,GAAG,UAAU,CAAC;IAC3B,sEAAsE;IACtE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC3B;AAqBD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,CAAC,EAAE,iBAAiB,GAAG,mBAAmB,CAkB/E;AAKD;;;;GAIG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAkB7C;AA6BD,2DAA2D;AAC3D,MAAM,WAAW,yBAAyB;IACxC,0DAA0D;IAC1D,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC;;;;;OAKG;IACH,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,MAAM,CAAC;IACvB,sGAAsG;IACtG,eAAe,CAAC,EAAE,MAAM,MAAM,CAAC;IAC/B,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,MAAM,CAAC,QAAQ,CAAC;IACrC,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,MAAM,CAAC;CAC7B;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IACvC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAElC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,UAAU,CAAC,CAAe;IAElC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,YAAY,CAAC,CAAkB;IACvC,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,KAAK,CAA6C;IAE1D,YAAY,OAAO,GAAE,yBAA8B,EAalD;IAED,oFAAoF;IACpF,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,0CAA0C;IAC1C,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,iFAAiF;IACjF,IAAI,UAAU,IAAI,OAAO,CAExB;IAED;;;OAGG;IACH,KAAK,IAAI,IAAI,CAMZ;IAED,gEAAgE;IAChE,IAAI,IAAI,IAAI,CAKX;IAED,6FAA6F;IAC7F,OAAO,IAAI,IAAI,CAEd;IAED;;;;;;;;;OASG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAQzE;IAED;;;;OAIG;IACH,IAAI,IAAI,IAAI,CAYX;IAED,OAAO,CAAC,gBAAgB;IAKxB,qGAAqG;IACrG,OAAO,CAAC,SAAS;IAWjB;yGACqG;IACrG,OAAO,CAAC,oBAAoB;IAU5B,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,MAAM;IAKd,OAAO,CAAC,MAAM;IAwCd;gGAC4F;IAC5F,QAAQ,IAAI,YAAY,CA0BvB;IAED;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;CASzB"}

package/dist/src/transports/adaptive-concurrency.js

@@ -0,0 +1,353 @@
+import { readFileSync } from 'node:fs';
+import { cpus, totalmem } from 'node:os';
+const DEFAULT_MIN = 1;
+const DEFAULT_MAX = 32;
+const DEFAULT_RAM_CEILING_PCT = 85;
+const DEFAULT_TICK_MS = 2000;
+function clamp(value, min, max) {
+    return Math.min(max, Math.max(min, value));
+}
+/** `value` floored when it is a finite number of at least 1, else `fallback`. */
+function wholeAtLeastOne(value, fallback) {
+    return value !== undefined && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
+}
+/** `value` when it is a finite number above 0, else `fallback`. */
+function finitePositive(value, fallback) {
+    return value !== undefined && Number.isFinite(value) && value > 0 ? value : fallback;
+}
+/**
+ * Normalise a {@link ConcurrencyOption} into a {@link ResolvedConcurrency}: a bare number (or
+ * `undefined` → 1) becomes `{ mode:'fixed', fixed }`; `'adaptive'` / `{ mode:'adaptive', ... }`
+ * become `{ mode:'adaptive', adaptive }` with every default from the spec applied and `start`
+ * clamped into `[min,max]`.
+ *
+ * A non-finite or out-of-range number falls back to its default (a bad `cpuCeilingPct` turns the
+ * CPU ceiling off): BullMQ throws on a `NaN` or sub-1 concurrency, a `NaN` ceiling never brakes, and
+ * a `NaN` `tickMs` makes the control loop fire every millisecond.
+ */
+export function resolveConcurrency(opt) {
+    if (opt === undefined)
+        return { mode: 'fixed', fixed: 1 };
+    if (typeof opt === 'number')
+        return { mode: 'fixed', fixed: wholeAtLeastOne(opt, 1) };
+    const overrides = opt === 'adaptive' ? {} : opt;
+    const min = wholeAtLeastOne(overrides.min, DEFAULT_MIN);
+    const max = Math.max(min, wholeAtLeastOne(overrides.max, DEFAULT_MAX));
+    const start = clamp(wholeAtLeastOne(overrides.start, min), min, max);
+    const cpuCeilingPct = finitePositive(overrides.cpuCeilingPct, undefined);
+    const adaptive = {
+        min,
+        max,
+        start,
+        ramCeilingPct: finitePositive(overrides.ramCeilingPct, DEFAULT_RAM_CEILING_PCT),
+        tickMs: wholeAtLeastOne(overrides.tickMs, DEFAULT_TICK_MS),
+        ...(cpuCeilingPct !== undefined ? { cpuCeilingPct } : {}),
+    };
+    return { mode: 'adaptive', adaptive };
+}
+// The cgroup "unlimited" sentinel: a limit at/above this is no real cap, so fall through to host total.
+const CGROUP_UNLIMITED = 2 ** 62;
+/**
+ * The process memory ceiling in bytes, read once (cgroup limits don't move at runtime): cgroup v2
+ * `memory.max`, then cgroup v1 `memory.limit_in_bytes`, then the host total. A `max` / absurdly-large
+ * ("unlimited") cgroup value falls through to {@link totalmem}. No new deps — just `node:fs`/`node:os`.
+ */
+export function readMemoryLimitBytes() {
+    try {
+        const raw = readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim();
+        if (raw && raw !== 'max') {
+            const n = Number(raw);
+            if (Number.isFinite(n) && n > 0 && n < CGROUP_UNLIMITED)
+                return n;
+        }
+    }
+    catch {
+        /* not cgroup v2 (or unreadable) — try v1 next */
+    }
+    try {
+        const raw = readFileSync('/sys/fs/cgroup/memory/memory.limit_in_bytes', 'utf8').trim();
+        const n = Number(raw);
+        if (Number.isFinite(n) && n > 0 && n < CGROUP_UNLIMITED)
+            return n;
+    }
+    catch {
+        /* not cgroup v1 (or unreadable) — fall through to host total */
+    }
+    return totalmem();
+}
+// How many recent completions the controller reasons over (latency percentiles + throughput + errors).
+const WINDOW_SIZE = 100;
+// EWMA smoothing for the "no-queuing" baseline (rttLong): low weight so a single fast sample doesn't
+// reset the baseline, but it still tracks a genuine floor shift.
+const RTT_LONG_EWMA_ALPHA = 0.2;
+// Consecutive ticks with in-flight work but zero completions before we call the worker stalled.
+const STALL_TICKS = 2;
+// Throughput is a rate over the trailing minute: a window whose newest completion is older than this
+// reports 0, so an idle worker never advertises its last burst as live.
+const THROUGHPUT_WINDOW_MS = 60_000;
+// The shortest span a rate is computed over, so a worker's first completion doesn't extrapolate a
+// sub-millisecond gap into a huge per-minute figure.
+const THROUGHPUT_MIN_SPAN_MS = 1_000;
+function percentile(sortedAsc, p) {
+    if (sortedAsc.length === 0)
+        return 0;
+    const idx = clamp(Math.ceil((p / 100) * sortedAsc.length) - 1, 0, sortedAsc.length - 1);
+    return sortedAsc[idx] ?? 0;
+}
+/**
+ * A single gradient-limit (AIMD) concurrency controller, shared by every per-routing-token task
+ * worker of one transport ({@link import('./bullmq/bullmq-transport.js').BullMQTransport} or
+ * {@link import('./queue.js').QueueTransport}). It tracks in-flight work and a rolling window of
+ * completions, and — in `adaptive` mode — re-evaluates the worker's concurrency every `tickMs` from a
+ * latency gradient, an error/stall backpressure signal, a hard RAM brake and an optional CPU ceiling.
+ *
+ * In `fixed` mode no control loop runs and the limit never moves, but the controller still tracks
+ * `inFlight`/RSS/throughput/p95 (and samples CPU when a snapshot is taken) so the observability half
+ * (the heartbeat status payload) works without adaptive. Either way, {@link snapshot} returns the
+ * shared {@link WorkerStatus} contract the heartbeat writer stamps.
+ */
+export class AdaptiveController {
+    resolved;
+    apply;
+    now;
+    readRss;
+    memoryLimitBytes;
+    tickMs;
+    readCpuUsage;
+    cpuCores;
+    currentLimit;
+    inFlightCount = 0;
+    window = [];
+    rttLongMs;
+    cpuPct;
+    lastAdjust;
+    completionsThisTick = 0;
+    stallTicks = 0;
+    lastCpuUsage;
+    lastCpuSampleAt = 0;
+    timer;
+    constructor(options = {}) {
+        this.resolved = resolveConcurrency(options.concurrency);
+        this.apply = options.apply ?? (() => { });
+        this.now = options.now ?? Date.now;
+        this.readRss = options.readRss ?? (() => process.memoryUsage().rss);
+        this.memoryLimitBytes = (options.readMemoryLimit ?? readMemoryLimitBytes)();
+        this.tickMs = this.resolved.adaptive?.tickMs ?? DEFAULT_TICK_MS;
+        this.readCpuUsage = options.readCpuUsage ?? (() => process.cpuUsage());
+        this.cpuCores = wholeAtLeastOne((options.readCpuCount ?? (() => cpus().length))(), 1);
+        this.currentLimit =
+            this.resolved.mode === 'adaptive'
+                ? (this.resolved.adaptive?.start ?? DEFAULT_MIN)
+                : (this.resolved.fixed ?? 1);
+    }
+    /** The concurrency a task consumer starts at (fixed N, or the adaptive `start`). */
+    get initialLimit() {
+        return this.currentLimit;
+    }
+    /** The live limit in effect right now. */
+    get limit() {
+        return this.currentLimit;
+    }
+    /** `true` when the controller will move the limit on its own (adaptive mode). */
+    get isAdaptive() {
+        return this.resolved.mode === 'adaptive';
+    }
+    /**
+     * Start the control loop (adaptive mode only; fixed mode just takes a CPU baseline). The timer is
+     * `unref`'d so it never keeps the process alive; idempotent.
+     */
+    start() {
+        if (this.timer)
+            return;
+        if (!this.lastCpuUsage)
+            this.resetCpuBaseline();
+        if (!this.isAdaptive)
+            return;
+        this.timer = setInterval(() => this.tick(), this.tickMs);
+        this.timer.unref?.();
+    }
+    /** Stop the control loop (call on worker close). Idempotent. */
+    stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = undefined;
+        }
+    }
+    /** A task started — bump in-flight. Call from the job processor before awaiting the body. */
+    onStart() {
+        this.inFlightCount += 1;
+    }
+    /**
+     * A task settled. ALWAYS drops in-flight and counts toward stall liveness (`completionsThisTick`),
+     * so the saturation gate, the heartbeat `inFlight`, and the "alive-but-busy" signal see every task.
+     *
+     * `kind` gates the MEASUREMENT window: only `'step'` completions push a duration into the rolling
+     * window that feeds the latency gradient / throughput / p95 / error rate. A workflow TURN
+     * (`kind: 'workflow'`) is NOT measured — a 5ms suspend sitting next to a 2s step would corrupt the
+     * gradient — but it still counts as in-flight + forward progress. This is the single concurrency
+     * pool measuring steps only; the transport tags the kind when it routes a job by shape.
+     */
+    onSettle(durationMs, ok, kind) {
+        this.inFlightCount = Math.max(0, this.inFlightCount - 1);
+        // Forward progress for stall detection counts BOTH kinds — a worker churning workflow turns is
+        // not stalled even when no step settles this tick.
+        this.completionsThisTick += 1;
+        if (kind !== 'step')
+            return;
+        this.window.push({ durationMs, ok, at: this.now() });
+        if (this.window.length > WINDOW_SIZE)
+            this.window.shift();
+    }
+    /**
+     * One control-loop iteration: sample CPU, refresh the latency baseline, then decide the next limit
+     * and apply it. A no-op in fixed mode. Exposed (not private) so a test can drive ticks
+     * deterministically.
+     */
+    tick() {
+        if (this.resolved.mode !== 'adaptive' || !this.resolved.adaptive)
+            return;
+        this.sampleCpu(this.tickMs);
+        this.refreshRttLong();
+        // Stall = in-flight work but no completion this tick, sustained over STALL_TICKS.
+        if (this.inFlightCount > 0 && this.completionsThisTick === 0)
+            this.stallTicks += 1;
+        else
+            this.stallTicks = 0;
+        const stalled = this.stallTicks >= STALL_TICKS;
+        this.completionsThisTick = 0;
+        this.decide(this.resolved.adaptive, stalled);
+    }
+    resetCpuBaseline() {
+        this.lastCpuUsage = this.readCpuUsage();
+        this.lastCpuSampleAt = this.now();
+    }
+    /** Set `cpuPct` from the CPU time spent since the previous sample, over `elapsedMs` of wall time. */
+    sampleCpu(elapsedMs) {
+        const previous = this.lastCpuUsage;
+        this.resetCpuBaseline();
+        if (!previous || !this.lastCpuUsage || elapsedMs <= 0)
+            return;
+        // CpuUsage is microseconds of CPU; elapsedMs of wall time is elapsedMs*1000 micros per core.
+        const micros = this.lastCpuUsage.user - previous.user + (this.lastCpuUsage.system - previous.system);
+        const pct = (micros / (elapsedMs * 1000 * this.cpuCores)) * 100;
+        if (Number.isFinite(pct))
+            this.cpuPct = Math.max(0, pct);
+    }
+    /** Fixed mode has no control loop, so a snapshot samples CPU itself, at most once per tick period
+     *  (several heartbeat keys snapshot back to back, and a near-zero span gives a meaningless rate). */
+    sampleCpuForSnapshot() {
+        if (this.isAdaptive)
+            return;
+        if (!this.lastCpuUsage) {
+            this.resetCpuBaseline();
+            return;
+        }
+        const elapsedMs = this.now() - this.lastCpuSampleAt;
+        if (elapsedMs >= this.tickMs)
+            this.sampleCpu(elapsedMs);
+    }
+    refreshRttLong() {
+        if (this.window.length === 0)
+            return;
+        let minDuration = this.window[0]?.durationMs ?? 0;
+        for (const c of this.window)
+            if (c.durationMs < minDuration)
+                minDuration = c.durationMs;
+        this.rttLongMs =
+            this.rttLongMs === undefined
+                ? minDuration
+                : RTT_LONG_EWMA_ALPHA * minDuration + (1 - RTT_LONG_EWMA_ALPHA) * this.rttLongMs;
+    }
+    rssPct() {
+        if (this.memoryLimitBytes <= 0)
+            return 0;
+        return (100 * this.readRss()) / this.memoryLimitBytes;
+    }
+    decide(cfg, stalled) {
+        const current = this.currentLimit;
+        const durations = this.window.map((c) => c.durationMs).sort((a, b) => a - b);
+        const p50 = percentile(durations, 50);
+        const errors = this.window.reduce((n, c) => n + (c.ok ? 0 : 1), 0);
+        const errorRate = this.window.length === 0 ? 0 : errors / this.window.length;
+        const gradient = this.rttLongMs !== undefined && p50 > 0 ? Math.min(1, this.rttLongMs / p50) : 1;
+        const rssPct = this.rssPct();
+        const saturated = this.inFlightCount >= current * 0.8;
+        let next = current;
+        let reason;
+        if (rssPct >= cfg.ramCeilingPct) {
+            next = Math.max(cfg.min, Math.floor(current * 0.8));
+            reason = 'ram_ceiling';
+        }
+        else if (cfg.cpuCeilingPct !== undefined && (this.cpuPct ?? 0) >= cfg.cpuCeilingPct) {
+            // CPU at/over the ceiling: never grow; shed one to relieve it.
+            next = Math.max(cfg.min, current - 1);
+            reason = 'cpu_ceiling';
+        }
+        else if (errorRate > 0.2 || stalled) {
+            next = Math.max(cfg.min, current - 1);
+            reason = 'backpressure';
+        }
+        else if (gradient < 0.7) {
+            next = Math.max(cfg.min, Math.floor(current * gradient));
+            reason = 'shrink';
+        }
+        else if (gradient >= 0.9 && saturated) {
+            next = Math.min(cfg.max, current + 1);
+            reason = 'grow';
+        }
+        next = clamp(next, cfg.min, cfg.max);
+        if (next === current || reason === undefined)
+            return;
+        this.currentLimit = next;
+        this.lastAdjust = { at: this.now(), from: current, to: next, reason };
+        this.apply(next);
+    }
+    /** The shared {@link WorkerStatus} the heartbeat writer stamps. Best-effort fields are omitted when
+     *  unmeasured (no completions yet → no p95/throughput; no CPU sample yet → no `cpuPct`). */
+    snapshot() {
+        this.sampleCpuForSnapshot();
+        const durations = this.window.map((c) => c.durationMs).sort((a, b) => a - b);
+        const rssBytes = this.readRss();
+        const rssLimitBytes = this.memoryLimitBytes;
+        const status = {
+            runtime: 'node',
+            concurrency: {
+                mode: this.resolved.mode,
+                limit: this.currentLimit,
+                ...(this.resolved.mode === 'adaptive' && this.resolved.adaptive
+                    ? { min: this.resolved.adaptive.min, max: this.resolved.adaptive.max }
+                    : {}),
+            },
+            inFlight: this.inFlightCount,
+            rssBytes,
+            rssLimitBytes,
+            rssPct: rssLimitBytes > 0 ? (100 * rssBytes) / rssLimitBytes : 0,
+        };
+        if (this.cpuPct !== undefined)
+            status.cpuPct = this.cpuPct;
+        if (this.window.length > 0) {
+            status.throughputPerMin = this.throughputPerMin();
+            status.p95Ms = percentile(durations, 95);
+        }
+        if (this.lastAdjust)
+            status.lastAdjust = this.lastAdjust;
+        return status;
+    }
+    /**
+     * Completions per minute over the trailing minute: the window's completions from the last minute,
+     * over the span from the oldest of them (or a full minute, when the window reaches further back) to
+     * now, floored at a second. The rate decays while the worker is idle and reads 0 once a minute passes with no completion.
+     */
+    throughputPerMin() {
+        const first = this.window[0];
+        if (!first)
+            return 0;
+        const now = this.now();
+        const recent = this.window.filter((c) => now - c.at <= THROUGHPUT_WINDOW_MS).length;
+        if (recent === 0)
+            return 0;
+        const spanMs = clamp(now - first.at, THROUGHPUT_MIN_SPAN_MS, THROUGHPUT_WINDOW_MS);
+        return (recent / spanMs) * 60_000;
+    }
+}
+//# sourceMappingURL=adaptive-concurrency.js.map

package/dist/src/transports/adaptive-concurrency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptive-concurrency.js","sourceRoot":"","sources":["../../../src/transports/adaptive-concurrency.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAmDzC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,SAAS,KAAK,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,iFAAiF;AACjF,SAAS,eAAe,CAAC,KAAyB,EAAE,QAAgB;IAClE,OAAO,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpG,CAAC;AAED,mEAAmE;AACnE,SAAS,cAAc,CAAI,KAAyB,EAAE,QAAW;IAC/D,OAAO,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAuB;IACxD,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC1D,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;IAEtF,MAAM,SAAS,GAA+B,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,GAAG,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACxD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACrE,MAAM,aAAa,GAAG,cAAc,CAAC,SAAS,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAmB;QAC/B,GAAG;QACH,GAAG;QACH,KAAK;QACL,aAAa,EAAE,cAAc,CAAC,SAAS,CAAC,aAAa,EAAE,uBAAuB,CAAC;QAC/E,MAAM,EAAE,eAAe,CAAC,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC;QAC1D,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AACxC,CAAC;AAED,wGAAwG;AACxG,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,CAAC;AAEjC;;;;GAIG;AACH,MAAM,UAAU,oBAAoB;IAClC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,gBAAgB;gBAAE,OAAO,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;IACnD,CAAC;IACD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,6CAA6C,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACvF,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,gBAAgB;YAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,gEAAgE;IAClE,CAAC;IACD,OAAO,QAAQ,EAAE,CAAC;AACpB,CAAC;AASD,uGAAuG;AACvG,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,qGAAqG;AACrG,iEAAiE;AACjE,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,gGAAgG;AAChG,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,qGAAqG;AACrG,wEAAwE;AACxE,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,kGAAkG;AAClG,qDAAqD;AACrD,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,SAAS,UAAU,CAAC,SAAmB,EAAE,CAAS;IAChD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxF,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAyBD;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,kBAAkB;IACZ,QAAQ,CAAsB;IAC9B,KAAK,CAA0B;IAC/B,GAAG,CAAe;IAClB,OAAO,CAAe;IACtB,gBAAgB,CAAS;IACzB,MAAM,CAAS;IACf,YAAY,CAAwB;IACpC,QAAQ,CAAS;IAE1B,YAAY,CAAS;IACrB,aAAa,GAAG,CAAC,CAAC;IACT,MAAM,GAAiB,EAAE,CAAC;IACnC,SAAS,CAAU;IACnB,MAAM,CAAU;IAChB,UAAU,CAAgB;IAE1B,mBAAmB,GAAG,CAAC,CAAC;IACxB,UAAU,GAAG,CAAC,CAAC;IACf,YAAY,CAAmB;IAC/B,eAAe,GAAG,CAAC,CAAC;IACpB,KAAK,CAA6C;IAE1D,YAAY,OAAO,GAA8B,EAAE;QACjD,IAAI,CAAC,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;QACpE,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,eAAe,IAAI,oBAAoB,CAAC,EAAE,CAAC;QAC5E,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,eAAe,CAAC;QAChE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,GAAG,eAAe,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,YAAY;YACf,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU;gBAC/B,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,IAAI,WAAW,CAAC;gBAChD,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,oFAAoF;IACpF,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,0CAA0C;IAC1C,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,iFAAiF;IACjF,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACvB,CAAC;IAED,gEAAgE;IAChE,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACzB,CAAC;IACH,CAAC;IAED,6FAA6F;IAC7F,OAAO;QACL,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;OASG;IACH,QAAQ,CAAC,UAAkB,EAAE,EAAW,EAAE,IAAyB;QACjE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;QACzD,+FAA+F;QAC/F,mDAAmD;QACnD,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAC9B,IAAI,IAAI,KAAK,MAAM;YAAE,OAAO;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW;YAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACH,IAAI;QACF,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO;QACzE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,kFAAkF;QAClF,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,IAAI,CAAC,mBAAmB,KAAK,CAAC;YAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;;YAC9E,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,IAAI,WAAW,CAAC;QAC/C,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAE7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,qGAAqG;IAC7F,SAAS,CAAC,SAAiB;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,SAAS,IAAI,CAAC;YAAE,OAAO;QAC9D,6FAA6F;QAC7F,MAAM,MAAM,GACV,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxF,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,SAAS,GAAG,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC;QAChE,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3D,CAAC;IAED;yGACqG;IAC7F,oBAAoB;QAC1B,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QACpD,IAAI,SAAS,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC1D,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACrC,IAAI,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC;QAClD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,CAAC,UAAU,GAAG,WAAW;gBAAE,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC;QACxF,IAAI,CAAC,SAAS;YACZ,IAAI,CAAC,SAAS,KAAK,SAAS;gBAC1B,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,mBAAmB,GAAG,WAAW,GAAG,CAAC,CAAC,GAAG,mBAAmB,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;IACvF,CAAC;IAEO,MAAM;QACZ,IAAI,IAAI,CAAC,gBAAgB,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAEO,MAAM,CAAC,GAAmB,EAAE,OAAgB;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC7E,MAAM,QAAQ,GACZ,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,GAAG,GAAG,CAAC;QAEtD,IAAI,IAAI,GAAG,OAAO,CAAC;QACnB,IAAI,MAA0C,CAAC;QAE/C,IAAI,MAAM,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YAChC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,GAAG,aAAa,CAAC;QACzB,CAAC;aAAM,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YACtF,+DAA+D;YAC/D,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,aAAa,CAAC;QACzB,CAAC;aAAM,IAAI,SAAS,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,cAAc,CAAC;QAC1B,CAAC;aAAM,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YAC1B,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC;YACzD,MAAM,GAAG,QAAQ,CAAC;QACpB,CAAC;aAAM,IAAI,QAAQ,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YACxC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,MAAM,CAAC;QAClB,CAAC;QAED,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,KAAK,OAAO,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED;gGAC4F;IAC5F,QAAQ;QACN,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC5C,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,MAAM;YACf,WAAW,EAAE;gBACX,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;gBACxB,KAAK,EAAE,IAAI,CAAC,YAAY;gBACxB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ;oBAC7D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE;oBACtE,CAAC,CAAC,EAAE,CAAC;aACR;YACD,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,QAAQ;YACR,aAAa;YACb,MAAM,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;SACjE,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3D,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,IAAI,CAAC,UAAU;YAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACzD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACK,gBAAgB;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,oBAAoB,CAAC,CAAC,MAAM,CAAC;QACpF,IAAI,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,EAAE,sBAAsB,EAAE,oBAAoB,CAAC,CAAC;QACnF,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;IACpC,CAAC;CACF"}