@adonis-agora/durable 0.49.1 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +251 -0
- package/README.md +53 -22
- package/dist/assets/spa/assets/index-DupjrK7N.js +16 -0
- package/dist/assets/spa/assets/index-DupjrK7N.js.map +1 -0
- package/dist/assets/spa/index.html +1 -1
- package/dist/commands/durable_worker.d.ts.map +1 -1
- package/dist/commands/durable_worker.js +9 -8
- package/dist/commands/durable_worker.js.map +1 -1
- package/dist/configure.d.ts.map +1 -1
- package/dist/configure.js +1 -0
- package/dist/configure.js.map +1 -1
- package/dist/providers/dashboard_provider.d.ts +2 -0
- package/dist/providers/dashboard_provider.d.ts.map +1 -1
- package/dist/providers/dashboard_provider.js +50 -6
- package/dist/providers/dashboard_provider.js.map +1 -1
- package/dist/providers/durable_provider.d.ts.map +1 -1
- package/dist/providers/durable_provider.js +143 -34
- package/dist/providers/durable_provider.js.map +1 -1
- package/dist/src/admission-redis/redis-admission-backend.d.ts.map +1 -1
- package/dist/src/admission-redis/redis-admission-backend.js +3 -1
- package/dist/src/admission-redis/redis-admission-backend.js.map +1 -1
- package/dist/src/ambient-step.d.ts +24 -0
- package/dist/src/ambient-step.d.ts.map +1 -0
- package/dist/src/ambient-step.js +42 -0
- package/dist/src/ambient-step.js.map +1 -0
- package/dist/src/base-workflow.d.ts +10 -1
- package/dist/src/base-workflow.d.ts.map +1 -1
- package/dist/src/base-workflow.js.map +1 -1
- package/dist/src/codec-state-store.d.ts +28 -2
- package/dist/src/codec-state-store.d.ts.map +1 -1
- package/dist/src/codec-state-store.js +61 -3
- package/dist/src/codec-state-store.js.map +1 -1
- package/dist/src/commands/worker.d.ts +4 -1
- package/dist/src/commands/worker.d.ts.map +1 -1
- package/dist/src/commands/worker.js +27 -3
- package/dist/src/commands/worker.js.map +1 -1
- package/dist/src/config_types.d.ts +6 -0
- package/dist/src/config_types.d.ts.map +1 -1
- package/dist/src/control-plane-redis/redis-control-plane.d.ts.map +1 -1
- package/dist/src/control-plane-redis/redis-control-plane.js +22 -6
- package/dist/src/control-plane-redis/redis-control-plane.js.map +1 -1
- package/dist/src/control-planes/factory.d.ts +6 -0
- package/dist/src/control-planes/factory.d.ts.map +1 -1
- package/dist/src/control-planes/factory.js +1 -0
- package/dist/src/control-planes/factory.js.map +1 -1
- package/dist/src/dashboard/gateway-adapter.d.ts +15 -5
- package/dist/src/dashboard/gateway-adapter.d.ts.map +1 -1
- package/dist/src/dashboard/gateway-adapter.js +71 -7
- package/dist/src/dashboard/gateway-adapter.js.map +1 -1
- package/dist/src/dashboard/handlers.d.ts +53 -1
- package/dist/src/dashboard/handlers.d.ts.map +1 -1
- package/dist/src/dashboard/handlers.js +200 -20
- package/dist/src/dashboard/handlers.js.map +1 -1
- package/dist/src/dashboard/index.d.ts +2 -2
- package/dist/src/dashboard/index.d.ts.map +1 -1
- package/dist/src/dashboard/index.js +2 -2
- package/dist/src/dashboard/index.js.map +1 -1
- package/dist/src/dashboard/openapi.d.ts.map +1 -1
- package/dist/src/dashboard/openapi.js +71 -1
- package/dist/src/dashboard/openapi.js.map +1 -1
- package/dist/src/define_config.d.ts +122 -8
- package/dist/src/define_config.d.ts.map +1 -1
- package/dist/src/define_config.js.map +1 -1
- package/dist/src/diagnostics-bridge.d.ts +16 -3
- package/dist/src/diagnostics-bridge.d.ts.map +1 -1
- package/dist/src/diagnostics-bridge.js +9 -3
- package/dist/src/diagnostics-bridge.js.map +1 -1
- package/dist/src/discovery-roots.d.ts +92 -0
- package/dist/src/discovery-roots.d.ts.map +1 -0
- package/dist/src/discovery-roots.js +150 -0
- package/dist/src/discovery-roots.js.map +1 -0
- package/dist/src/engine.d.ts +418 -20
- package/dist/src/engine.d.ts.map +1 -1
- package/dist/src/engine.js +1523 -159
- package/dist/src/engine.js.map +1 -1
- package/dist/src/entities.d.ts +36 -1
- package/dist/src/entities.d.ts.map +1 -1
- package/dist/src/entities.js +65 -4
- package/dist/src/entities.js.map +1 -1
- package/dist/src/errors.d.ts +36 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +44 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/event-trigger-bridge.d.ts +8 -0
- package/dist/src/event-trigger-bridge.d.ts.map +1 -1
- package/dist/src/event-trigger-bridge.js +24 -3
- package/dist/src/event-trigger-bridge.js.map +1 -1
- package/dist/src/handshake/announced.d.ts +27 -0
- package/dist/src/handshake/announced.d.ts.map +1 -1
- package/dist/src/handshake/announced.js +62 -0
- package/dist/src/handshake/announced.js.map +1 -1
- package/dist/src/handshake/directory.d.ts +92 -0
- package/dist/src/handshake/directory.d.ts.map +1 -0
- package/dist/src/handshake/directory.js +139 -0
- package/dist/src/handshake/directory.js.map +1 -0
- package/dist/src/hooks/barrel-sources.d.ts +28 -0
- package/dist/src/hooks/barrel-sources.d.ts.map +1 -0
- package/dist/src/hooks/barrel-sources.js +92 -0
- package/dist/src/hooks/barrel-sources.js.map +1 -0
- package/dist/src/hooks/steps.d.ts +12 -3
- package/dist/src/hooks/steps.d.ts.map +1 -1
- package/dist/src/hooks/steps.js +7 -11
- package/dist/src/hooks/steps.js.map +1 -1
- package/dist/src/hooks/workflows.d.ts +12 -3
- package/dist/src/hooks/workflows.d.ts.map +1 -1
- package/dist/src/hooks/workflows.js +7 -13
- package/dist/src/hooks/workflows.js.map +1 -1
- package/dist/src/index.d.ts +12 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +12 -3
- package/dist/src/index.js.map +1 -1
- package/dist/src/interfaces.d.ts +309 -21
- package/dist/src/interfaces.d.ts.map +1 -1
- package/dist/src/interfaces.js +10 -0
- package/dist/src/interfaces.js.map +1 -1
- package/dist/src/metrics.d.ts.map +1 -1
- package/dist/src/metrics.js +5 -0
- package/dist/src/metrics.js.map +1 -1
- package/dist/src/namespaces.d.ts +17 -0
- package/dist/src/namespaces.d.ts.map +1 -0
- package/dist/src/namespaces.js +24 -0
- package/dist/src/namespaces.js.map +1 -0
- package/dist/src/otel/durable-metrics.d.ts.map +1 -1
- package/dist/src/otel/durable-metrics.js +5 -0
- package/dist/src/otel/durable-metrics.js.map +1 -1
- package/dist/src/otel/durable-otel.d.ts.map +1 -1
- package/dist/src/otel/durable-otel.js +5 -0
- package/dist/src/otel/durable-otel.js.map +1 -1
- package/dist/src/pollers.d.ts +10 -1
- package/dist/src/pollers.d.ts.map +1 -1
- package/dist/src/pollers.js +15 -5
- package/dist/src/pollers.js.map +1 -1
- package/dist/src/protocol.d.ts +12 -0
- package/dist/src/protocol.d.ts.map +1 -1
- package/dist/src/protocol.js +27 -1
- package/dist/src/protocol.js.map +1 -1
- package/dist/src/remote-event.d.ts +6 -0
- package/dist/src/remote-event.d.ts.map +1 -0
- package/dist/src/remote-event.js +19 -0
- package/dist/src/remote-event.js.map +1 -0
- package/dist/src/remote-workflow-executor.d.ts +34 -12
- package/dist/src/remote-workflow-executor.d.ts.map +1 -1
- package/dist/src/remote-workflow-executor.js +42 -40
- package/dist/src/remote-workflow-executor.js.map +1 -1
- package/dist/src/run-detail-summary.d.ts +90 -3
- package/dist/src/run-detail-summary.d.ts.map +1 -1
- package/dist/src/run-detail-summary.js +360 -2
- package/dist/src/run-detail-summary.js.map +1 -1
- package/dist/src/run-facets.d.ts +48 -0
- package/dist/src/run-facets.d.ts.map +1 -0
- package/dist/src/run-facets.js +100 -0
- package/dist/src/run-facets.js.map +1 -0
- package/dist/src/run-gateway/interface.d.ts +27 -1
- package/dist/src/run-gateway/interface.d.ts.map +1 -1
- package/dist/src/run-gateway/proxy-run-gateway.d.ts +32 -4
- package/dist/src/run-gateway/proxy-run-gateway.d.ts.map +1 -1
- package/dist/src/run-gateway/proxy-run-gateway.js +101 -5
- package/dist/src/run-gateway/proxy-run-gateway.js.map +1 -1
- package/dist/src/run-gateway/run-request-responder.d.ts +18 -5
- package/dist/src/run-gateway/run-request-responder.d.ts.map +1 -1
- package/dist/src/run-gateway/run-request-responder.js +155 -32
- package/dist/src/run-gateway/run-request-responder.js.map +1 -1
- package/dist/src/run-gateway/store-run-gateway.d.ts +23 -1
- package/dist/src/run-gateway/store-run-gateway.d.ts.map +1 -1
- package/dist/src/run-gateway/store-run-gateway.js +38 -0
- package/dist/src/run-gateway/store-run-gateway.js.map +1 -1
- package/dist/src/search-attributes-schema.d.ts +44 -0
- package/dist/src/search-attributes-schema.d.ts.map +1 -0
- package/dist/src/search-attributes-schema.js +51 -0
- package/dist/src/search-attributes-schema.js.map +1 -0
- package/dist/src/singleton-admission.d.ts +75 -0
- package/dist/src/singleton-admission.d.ts.map +1 -0
- package/dist/src/singleton-admission.js +156 -0
- package/dist/src/singleton-admission.js.map +1 -0
- package/dist/src/singleton-gate.d.ts +23 -8
- package/dist/src/singleton-gate.d.ts.map +1 -1
- package/dist/src/singleton-gate.js +59 -12
- package/dist/src/singleton-gate.js.map +1 -1
- package/dist/src/standard-schema.d.ts +83 -0
- package/dist/src/standard-schema.d.ts.map +1 -0
- package/dist/src/standard-schema.js +31 -0
- package/dist/src/standard-schema.js.map +1 -0
- package/dist/src/step-discovery.d.ts +21 -11
- package/dist/src/step-discovery.d.ts.map +1 -1
- package/dist/src/step-discovery.js +86 -45
- package/dist/src/step-discovery.js.map +1 -1
- package/dist/src/step-ref.d.ts +38 -12
- package/dist/src/step-ref.d.ts.map +1 -1
- package/dist/src/step-ref.js +5 -13
- package/dist/src/step-ref.js.map +1 -1
- package/dist/src/stores/factory.d.ts +6 -0
- package/dist/src/stores/factory.d.ts.map +1 -1
- package/dist/src/stores/factory.js +4 -1
- package/dist/src/stores/factory.js.map +1 -1
- package/dist/src/stores/lucid-helpers.d.ts +2 -0
- package/dist/src/stores/lucid-helpers.d.ts.map +1 -1
- package/dist/src/stores/lucid-helpers.js +7 -0
- package/dist/src/stores/lucid-helpers.js.map +1 -1
- package/dist/src/stores/lucid-mappers.d.ts +25 -0
- package/dist/src/stores/lucid-mappers.d.ts.map +1 -1
- package/dist/src/stores/lucid-mappers.js +81 -18
- package/dist/src/stores/lucid-mappers.js.map +1 -1
- package/dist/src/stores/lucid-schema.d.ts +59 -1
- package/dist/src/stores/lucid-schema.d.ts.map +1 -1
- package/dist/src/stores/lucid-schema.js +271 -25
- package/dist/src/stores/lucid-schema.js.map +1 -1
- package/dist/src/stores/lucid.d.ts +61 -2
- package/dist/src/stores/lucid.d.ts.map +1 -1
- package/dist/src/stores/lucid.js +346 -41
- package/dist/src/stores/lucid.js.map +1 -1
- package/dist/src/telescope/dashboard.d.ts.map +1 -1
- package/dist/src/telescope/dashboard.js +30 -1
- package/dist/src/telescope/dashboard.js.map +1 -1
- package/dist/src/telescope/data-providers.d.ts +19 -1
- package/dist/src/telescope/data-providers.d.ts.map +1 -1
- package/dist/src/telescope/data-providers.js +119 -3
- package/dist/src/telescope/data-providers.js.map +1 -1
- package/dist/src/telescope/extension.d.ts.map +1 -1
- package/dist/src/telescope/extension.js +2 -1
- package/dist/src/telescope/extension.js.map +1 -1
- package/dist/src/telescope/index.d.ts +1 -1
- package/dist/src/telescope/index.d.ts.map +1 -1
- package/dist/src/telescope/index.js +1 -1
- package/dist/src/telescope/index.js.map +1 -1
- package/dist/src/testing/in-memory-state-store.d.ts +44 -15
- package/dist/src/testing/in-memory-state-store.d.ts.map +1 -1
- package/dist/src/testing/in-memory-state-store.js +204 -41
- package/dist/src/testing/in-memory-state-store.js.map +1 -1
- package/dist/src/testing-kit/checkpoint-aggregation-contract.d.ts +16 -0
- package/dist/src/testing-kit/checkpoint-aggregation-contract.d.ts.map +1 -0
- package/dist/src/testing-kit/checkpoint-aggregation-contract.js +228 -0
- package/dist/src/testing-kit/checkpoint-aggregation-contract.js.map +1 -0
- package/dist/src/testing-kit/state-store-conformance.d.ts.map +1 -1
- package/dist/src/testing-kit/state-store-conformance.js +355 -3
- package/dist/src/testing-kit/state-store-conformance.js.map +1 -1
- package/dist/src/transport-pool.d.ts +15 -1
- package/dist/src/transport-pool.d.ts.map +1 -1
- package/dist/src/transport-pool.js +50 -4
- package/dist/src/transport-pool.js.map +1 -1
- package/dist/src/transports/adaptive-concurrency.d.ts +168 -0
- package/dist/src/transports/adaptive-concurrency.d.ts.map +1 -0
- package/dist/src/transports/adaptive-concurrency.js +353 -0
- package/dist/src/transports/adaptive-concurrency.js.map +1 -0
- package/dist/src/transports/bullmq/adaptive-concurrency.d.ts +1 -165
- package/dist/src/transports/bullmq/adaptive-concurrency.d.ts.map +1 -1
- package/dist/src/transports/bullmq/adaptive-concurrency.js +3 -351
- package/dist/src/transports/bullmq/adaptive-concurrency.js.map +1 -1
- package/dist/src/transports/bullmq/bullmq-transport.d.ts +49 -1
- package/dist/src/transports/bullmq/bullmq-transport.d.ts.map +1 -1
- package/dist/src/transports/bullmq/bullmq-transport.js +330 -38
- package/dist/src/transports/bullmq/bullmq-transport.js.map +1 -1
- package/dist/src/transports/bullmq/deps.d.ts +18 -2
- package/dist/src/transports/bullmq/deps.d.ts.map +1 -1
- package/dist/src/transports/bullmq/deps.js +44 -2
- package/dist/src/transports/bullmq/deps.js.map +1 -1
- package/dist/src/transports/db-helpers.d.ts +39 -0
- package/dist/src/transports/db-helpers.d.ts.map +1 -1
- package/dist/src/transports/db-helpers.js +73 -0
- package/dist/src/transports/db-helpers.js.map +1 -1
- package/dist/src/transports/db-remote.d.ts +53 -0
- package/dist/src/transports/db-remote.d.ts.map +1 -0
- package/dist/src/transports/db-remote.js +147 -0
- package/dist/src/transports/db-remote.js.map +1 -0
- package/dist/src/transports/db-schema.d.ts +44 -1
- package/dist/src/transports/db-schema.d.ts.map +1 -1
- package/dist/src/transports/db-schema.js +227 -88
- package/dist/src/transports/db-schema.js.map +1 -1
- package/dist/src/transports/db.d.ts +189 -6
- package/dist/src/transports/db.d.ts.map +1 -1
- package/dist/src/transports/db.js +1291 -137
- package/dist/src/transports/db.js.map +1 -1
- package/dist/src/transports/event-emitter.d.ts +8 -0
- package/dist/src/transports/event-emitter.d.ts.map +1 -1
- package/dist/src/transports/event-emitter.js +36 -3
- package/dist/src/transports/event-emitter.js.map +1 -1
- package/dist/src/transports/factory.d.ts +110 -5
- package/dist/src/transports/factory.d.ts.map +1 -1
- package/dist/src/transports/factory.js +56 -9
- package/dist/src/transports/factory.js.map +1 -1
- package/dist/src/transports/queue-mock-adapter.d.ts +7 -2
- package/dist/src/transports/queue-mock-adapter.d.ts.map +1 -1
- package/dist/src/transports/queue-mock-adapter.js +18 -2
- package/dist/src/transports/queue-mock-adapter.js.map +1 -1
- package/dist/src/transports/queue-remote.d.ts +146 -0
- package/dist/src/transports/queue-remote.d.ts.map +1 -0
- package/dist/src/transports/queue-remote.js +398 -0
- package/dist/src/transports/queue-remote.js.map +1 -0
- package/dist/src/transports/queue.d.ts +159 -6
- package/dist/src/transports/queue.d.ts.map +1 -1
- package/dist/src/transports/queue.js +747 -37
- package/dist/src/transports/queue.js.map +1 -1
- package/dist/src/transports/task-failure.d.ts +31 -0
- package/dist/src/transports/task-failure.d.ts.map +1 -0
- package/dist/src/transports/task-failure.js +50 -0
- package/dist/src/transports/task-failure.js.map +1 -0
- package/dist/src/workflow-als.d.ts +1 -1
- package/dist/src/workflow-als.d.ts.map +1 -1
- package/dist/src/workflow-ctx.d.ts +10 -0
- package/dist/src/workflow-ctx.d.ts.map +1 -1
- package/dist/src/workflow-ctx.js +97 -15
- package/dist/src/workflow-ctx.js.map +1 -1
- package/dist/src/workflow-discovery.d.ts +32 -41
- package/dist/src/workflow-discovery.d.ts.map +1 -1
- package/dist/src/workflow-discovery.js +65 -88
- package/dist/src/workflow-discovery.js.map +1 -1
- package/dist/src/workflow-ref.d.ts +24 -0
- package/dist/src/workflow-ref.d.ts.map +1 -1
- package/dist/src/workflow-ref.js.map +1 -1
- package/dist/stubs/config/durable.stub +6 -0
- package/dist/stubs/config/durable_dashboard.stub +7 -3
- package/dist/stubs/database/migrations/create_durable_tables.stub +2 -1
- package/dist/stubs/database/migrations/create_durable_transport_messages.stub +43 -0
- package/dist/stubs/database/migrations/create_durable_transport_tables.stub +12 -9
- package/dist/stubs/database/migrations/create_durable_transport_worker_keys.stub +1 -1
- package/dist/stubs/make/workflow/main.stub +1 -1
- package/package.json +6 -4
- package/skills/durable-cluster/SKILL.md +80 -11
- package/skills/durable-determinism/SKILL.md +11 -5
- package/skills/durable-reliability/SKILL.md +77 -37
- package/skills/durable-setup/SKILL.md +51 -22
- package/skills/durable-transports-stores/SKILL.md +128 -18
- package/skills/durable-workflows/SKILL.md +71 -18
- package/dist/assets/spa/assets/index-DdB9zxuX.js +0 -16
- package/dist/assets/spa/assets/index-DdB9zxuX.js.map +0 -1
|
@@ -1,166 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
/**
|
|
3
|
-
* The adaptive controller's tunables — every field optional, each with a default from the spec.
|
|
4
|
-
* Spread under `{ mode: 'adaptive' }` to widen a plain `concurrency: number` into a self-tuning knob.
|
|
5
|
-
*/
|
|
6
|
-
export interface AdaptiveConcurrencyOptions {
|
|
7
|
-
/** Floor the controller won't go below. Default 1. */
|
|
8
|
-
min?: number;
|
|
9
|
-
/** Ceiling the controller won't exceed. Default 32. */
|
|
10
|
-
max?: number;
|
|
11
|
-
/** Initial limit. Default `min` (clamped into `[min,max]`). */
|
|
12
|
-
start?: number;
|
|
13
|
-
/** RSS percent of the memory ceiling at which the hard brake fires. Default 85. */
|
|
14
|
-
ramCeilingPct?: number;
|
|
15
|
-
/** CPU percent (0..100×cores normalised to 0..100) ceiling; absent = off. Default undefined. */
|
|
16
|
-
cpuCeilingPct?: number;
|
|
17
|
-
/** Control-loop period in ms. Default 2000. */
|
|
18
|
-
tickMs?: number;
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* The `concurrency` option, widened (backward-compatible): a plain `number` is fixed (unchanged), the
|
|
22
|
-
* string `'adaptive'` is adaptive-with-defaults, and an object `{ mode:'adaptive', ... }` is adaptive
|
|
23
|
-
* with overrides. `undefined` keeps the historical default of 1 (fixed).
|
|
24
|
-
*/
|
|
25
|
-
export type ConcurrencyOption = number | 'adaptive' | ({
|
|
26
|
-
mode: 'adaptive';
|
|
27
|
-
} & AdaptiveConcurrencyOptions);
|
|
28
|
-
/** The fully-resolved adaptive tunables (every default applied), carried on a resolved config. */
|
|
29
|
-
export interface AdaptiveConfig {
|
|
30
|
-
min: number;
|
|
31
|
-
max: number;
|
|
32
|
-
start: number;
|
|
33
|
-
ramCeilingPct: number;
|
|
34
|
-
cpuCeilingPct?: number;
|
|
35
|
-
tickMs: number;
|
|
36
|
-
}
|
|
37
|
-
/** A normalised `concurrency` option: a `fixed` number, or an `adaptive` config with every default applied. */
|
|
38
|
-
export interface ResolvedConcurrency {
|
|
39
|
-
mode: 'fixed' | 'adaptive';
|
|
40
|
-
/** Present when `mode === 'fixed'` — the never-moving concurrency. */
|
|
41
|
-
fixed?: number;
|
|
42
|
-
/** Present when `mode === 'adaptive'` — the controller's tunables. */
|
|
43
|
-
adaptive?: AdaptiveConfig;
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* Normalise a {@link ConcurrencyOption} into a {@link ResolvedConcurrency}: a bare number (or
|
|
47
|
-
* `undefined` → 1) becomes `{ mode:'fixed', fixed }`; `'adaptive'` / `{ mode:'adaptive', ... }`
|
|
48
|
-
* become `{ mode:'adaptive', adaptive }` with every default from the spec applied and `start`
|
|
49
|
-
* clamped into `[min,max]`.
|
|
50
|
-
*
|
|
51
|
-
* A non-finite or out-of-range number falls back to its default (a bad `cpuCeilingPct` turns the
|
|
52
|
-
* CPU ceiling off): BullMQ throws on a `NaN` or sub-1 concurrency, a `NaN` ceiling never brakes, and
|
|
53
|
-
* a `NaN` `tickMs` makes the control loop fire every millisecond.
|
|
54
|
-
*/
|
|
55
|
-
export declare function resolveConcurrency(opt?: ConcurrencyOption): ResolvedConcurrency;
|
|
56
|
-
/**
|
|
57
|
-
* The process memory ceiling in bytes, read once (cgroup limits don't move at runtime): cgroup v2
|
|
58
|
-
* `memory.max`, then cgroup v1 `memory.limit_in_bytes`, then the host total. A `max` / absurdly-large
|
|
59
|
-
* ("unlimited") cgroup value falls through to {@link totalmem}. No new deps — just `node:fs`/`node:os`.
|
|
60
|
-
*/
|
|
61
|
-
export declare function readMemoryLimitBytes(): number;
|
|
62
|
-
/** Construction surface for {@link AdaptiveController}. */
|
|
63
|
-
export interface AdaptiveControllerOptions {
|
|
64
|
-
/** The raw `concurrency` option (resolved internally). */
|
|
65
|
-
concurrency?: ConcurrencyOption;
|
|
66
|
-
/**
|
|
67
|
-
* Applied whenever the live limit changes — wire it to `worker.concurrency = limit`. Called once
|
|
68
|
-
* with the initial limit is NOT done here; the caller passes the initial limit to the Worker ctor
|
|
69
|
-
* and reads {@link AdaptiveController.limit}. Subsequent adaptive moves call this.
|
|
70
|
-
*/
|
|
71
|
-
apply?: (limit: number) => void;
|
|
72
|
-
/** Injectable clock (tests). Defaults to {@link Date.now}. */
|
|
73
|
-
now?: () => number;
|
|
74
|
-
/** Injectable RSS reader (tests). Defaults to `process.memoryUsage().rss`. */
|
|
75
|
-
readRss?: () => number;
|
|
76
|
-
/** Injectable memory-ceiling reader (tests). Defaults to {@link readMemoryLimitBytes} (read once). */
|
|
77
|
-
readMemoryLimit?: () => number;
|
|
78
|
-
/** Injectable cumulative CPU-time reader (tests). Defaults to `process.cpuUsage()`. */
|
|
79
|
-
readCpuUsage?: () => NodeJS.CpuUsage;
|
|
80
|
-
/** Injectable core-count reader (tests), read once. Defaults to `os.cpus().length`. */
|
|
81
|
-
readCpuCount?: () => number;
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* A single gradient-limit (AIMD) concurrency controller, shared by every per-routing-token task
|
|
85
|
-
* worker of one {@link import('./bullmq-transport.js').BullMQTransport}. It tracks in-flight work and a rolling window of
|
|
86
|
-
* completions, and — in `adaptive` mode — re-evaluates the worker's concurrency every `tickMs` from a
|
|
87
|
-
* latency gradient, an error/stall backpressure signal, a hard RAM brake and an optional CPU ceiling.
|
|
88
|
-
*
|
|
89
|
-
* In `fixed` mode no control loop runs and the limit never moves, but the controller still tracks
|
|
90
|
-
* `inFlight`/RSS/throughput/p95 (and samples CPU when a snapshot is taken) so the observability half
|
|
91
|
-
* (the heartbeat status payload) works without adaptive. Either way, {@link snapshot} returns the
|
|
92
|
-
* shared {@link WorkerStatus} contract the heartbeat writer stamps.
|
|
93
|
-
*/
|
|
94
|
-
export declare class AdaptiveController {
|
|
95
|
-
private readonly resolved;
|
|
96
|
-
private readonly apply;
|
|
97
|
-
private readonly now;
|
|
98
|
-
private readonly readRss;
|
|
99
|
-
private readonly memoryLimitBytes;
|
|
100
|
-
private readonly tickMs;
|
|
101
|
-
private readonly readCpuUsage;
|
|
102
|
-
private readonly cpuCores;
|
|
103
|
-
private currentLimit;
|
|
104
|
-
private inFlightCount;
|
|
105
|
-
private readonly window;
|
|
106
|
-
private rttLongMs?;
|
|
107
|
-
private cpuPct?;
|
|
108
|
-
private lastAdjust?;
|
|
109
|
-
private completionsThisTick;
|
|
110
|
-
private stallTicks;
|
|
111
|
-
private lastCpuUsage?;
|
|
112
|
-
private lastCpuSampleAt;
|
|
113
|
-
private timer;
|
|
114
|
-
constructor(options?: AdaptiveControllerOptions);
|
|
115
|
-
/** The concurrency to hand the BullMQ Worker on construction (fixed N, or the adaptive `start`). */
|
|
116
|
-
get initialLimit(): number;
|
|
117
|
-
/** The live limit in effect right now. */
|
|
118
|
-
get limit(): number;
|
|
119
|
-
/** `true` when the controller will move the limit on its own (adaptive mode). */
|
|
120
|
-
get isAdaptive(): boolean;
|
|
121
|
-
/**
|
|
122
|
-
* Start the control loop (adaptive mode only; fixed mode just takes a CPU baseline). The timer is
|
|
123
|
-
* `unref`'d so it never keeps the process alive; idempotent.
|
|
124
|
-
*/
|
|
125
|
-
start(): void;
|
|
126
|
-
/** Stop the control loop (call on worker close). Idempotent. */
|
|
127
|
-
stop(): void;
|
|
128
|
-
/** A task started — bump in-flight. Call from the job processor before awaiting the body. */
|
|
129
|
-
onStart(): void;
|
|
130
|
-
/**
|
|
131
|
-
* A task settled. ALWAYS drops in-flight and counts toward stall liveness (`completionsThisTick`),
|
|
132
|
-
* so the saturation gate, the heartbeat `inFlight`, and the "alive-but-busy" signal see every task.
|
|
133
|
-
*
|
|
134
|
-
* `kind` gates the MEASUREMENT window: only `'step'` completions push a duration into the rolling
|
|
135
|
-
* window that feeds the latency gradient / throughput / p95 / error rate. A workflow TURN
|
|
136
|
-
* (`kind: 'workflow'`) is NOT measured — a 5ms suspend sitting next to a 2s step would corrupt the
|
|
137
|
-
* gradient — but it still counts as in-flight + forward progress. This is the single concurrency
|
|
138
|
-
* pool measuring steps only; the transport tags the kind when it routes a job by shape.
|
|
139
|
-
*/
|
|
140
|
-
onSettle(durationMs: number, ok: boolean, kind: 'workflow' | 'step'): void;
|
|
141
|
-
/**
|
|
142
|
-
* One control-loop iteration: sample CPU, refresh the latency baseline, then decide the next limit
|
|
143
|
-
* and apply it. A no-op in fixed mode. Exposed (not private) so a test can drive ticks
|
|
144
|
-
* deterministically.
|
|
145
|
-
*/
|
|
146
|
-
tick(): void;
|
|
147
|
-
private resetCpuBaseline;
|
|
148
|
-
/** Set `cpuPct` from the CPU time spent since the previous sample, over `elapsedMs` of wall time. */
|
|
149
|
-
private sampleCpu;
|
|
150
|
-
/** Fixed mode has no control loop, so a snapshot samples CPU itself, at most once per tick period
|
|
151
|
-
* (several heartbeat keys snapshot back to back, and a near-zero span gives a meaningless rate). */
|
|
152
|
-
private sampleCpuForSnapshot;
|
|
153
|
-
private refreshRttLong;
|
|
154
|
-
private rssPct;
|
|
155
|
-
private decide;
|
|
156
|
-
/** The shared {@link WorkerStatus} the heartbeat writer stamps. Best-effort fields are omitted when
|
|
157
|
-
* unmeasured (no completions yet → no p95/throughput; no CPU sample yet → no `cpuPct`). */
|
|
158
|
-
snapshot(): WorkerStatus;
|
|
159
|
-
/**
|
|
160
|
-
* Completions per minute over the trailing minute: the window's completions from the last minute,
|
|
161
|
-
* over the span from the oldest of them (or a full minute, when the window reaches further back) to
|
|
162
|
-
* now, floored at a second. The rate decays while the worker is idle and reads 0 once a minute passes with no completion.
|
|
163
|
-
*/
|
|
164
|
-
private throughputPerMin;
|
|
165
|
-
}
|
|
1
|
+
export * from '../adaptive-concurrency.js';
|
|
166
2
|
//# sourceMappingURL=adaptive-concurrency.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adaptive-concurrency.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/adaptive-concurrency.ts"],"names":[],"mappings":"AAEA,
|
|
1
|
+
{"version":3,"file":"adaptive-concurrency.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/adaptive-concurrency.ts"],"names":[],"mappings":"AAEA,cAAc,4BAA4B,CAAC"}
|
|
@@ -1,352 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const DEFAULT_MAX = 32;
|
|
5
|
-
const DEFAULT_RAM_CEILING_PCT = 85;
|
|
6
|
-
const DEFAULT_TICK_MS = 2000;
|
|
7
|
-
function clamp(value, min, max) {
|
|
8
|
-
return Math.min(max, Math.max(min, value));
|
|
9
|
-
}
|
|
10
|
-
/** `value` floored when it is a finite number of at least 1, else `fallback`. */
|
|
11
|
-
function wholeAtLeastOne(value, fallback) {
|
|
12
|
-
return value !== undefined && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
13
|
-
}
|
|
14
|
-
/** `value` when it is a finite number above 0, else `fallback`. */
|
|
15
|
-
function finitePositive(value, fallback) {
|
|
16
|
-
return value !== undefined && Number.isFinite(value) && value > 0 ? value : fallback;
|
|
17
|
-
}
|
|
18
|
-
/**
|
|
19
|
-
* Normalise a {@link ConcurrencyOption} into a {@link ResolvedConcurrency}: a bare number (or
|
|
20
|
-
* `undefined` → 1) becomes `{ mode:'fixed', fixed }`; `'adaptive'` / `{ mode:'adaptive', ... }`
|
|
21
|
-
* become `{ mode:'adaptive', adaptive }` with every default from the spec applied and `start`
|
|
22
|
-
* clamped into `[min,max]`.
|
|
23
|
-
*
|
|
24
|
-
* A non-finite or out-of-range number falls back to its default (a bad `cpuCeilingPct` turns the
|
|
25
|
-
* CPU ceiling off): BullMQ throws on a `NaN` or sub-1 concurrency, a `NaN` ceiling never brakes, and
|
|
26
|
-
* a `NaN` `tickMs` makes the control loop fire every millisecond.
|
|
27
|
-
*/
|
|
28
|
-
export function resolveConcurrency(opt) {
|
|
29
|
-
if (opt === undefined)
|
|
30
|
-
return { mode: 'fixed', fixed: 1 };
|
|
31
|
-
if (typeof opt === 'number')
|
|
32
|
-
return { mode: 'fixed', fixed: wholeAtLeastOne(opt, 1) };
|
|
33
|
-
const overrides = opt === 'adaptive' ? {} : opt;
|
|
34
|
-
const min = wholeAtLeastOne(overrides.min, DEFAULT_MIN);
|
|
35
|
-
const max = Math.max(min, wholeAtLeastOne(overrides.max, DEFAULT_MAX));
|
|
36
|
-
const start = clamp(wholeAtLeastOne(overrides.start, min), min, max);
|
|
37
|
-
const cpuCeilingPct = finitePositive(overrides.cpuCeilingPct, undefined);
|
|
38
|
-
const adaptive = {
|
|
39
|
-
min,
|
|
40
|
-
max,
|
|
41
|
-
start,
|
|
42
|
-
ramCeilingPct: finitePositive(overrides.ramCeilingPct, DEFAULT_RAM_CEILING_PCT),
|
|
43
|
-
tickMs: wholeAtLeastOne(overrides.tickMs, DEFAULT_TICK_MS),
|
|
44
|
-
...(cpuCeilingPct !== undefined ? { cpuCeilingPct } : {}),
|
|
45
|
-
};
|
|
46
|
-
return { mode: 'adaptive', adaptive };
|
|
47
|
-
}
|
|
48
|
-
// The cgroup "unlimited" sentinel: a limit at/above this is no real cap, so fall through to host total.
|
|
49
|
-
const CGROUP_UNLIMITED = 2 ** 62;
|
|
50
|
-
/**
|
|
51
|
-
* The process memory ceiling in bytes, read once (cgroup limits don't move at runtime): cgroup v2
|
|
52
|
-
* `memory.max`, then cgroup v1 `memory.limit_in_bytes`, then the host total. A `max` / absurdly-large
|
|
53
|
-
* ("unlimited") cgroup value falls through to {@link totalmem}. No new deps — just `node:fs`/`node:os`.
|
|
54
|
-
*/
|
|
55
|
-
export function readMemoryLimitBytes() {
|
|
56
|
-
try {
|
|
57
|
-
const raw = readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim();
|
|
58
|
-
if (raw && raw !== 'max') {
|
|
59
|
-
const n = Number(raw);
|
|
60
|
-
if (Number.isFinite(n) && n > 0 && n < CGROUP_UNLIMITED)
|
|
61
|
-
return n;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
catch {
|
|
65
|
-
/* not cgroup v2 (or unreadable) — try v1 next */
|
|
66
|
-
}
|
|
67
|
-
try {
|
|
68
|
-
const raw = readFileSync('/sys/fs/cgroup/memory/memory.limit_in_bytes', 'utf8').trim();
|
|
69
|
-
const n = Number(raw);
|
|
70
|
-
if (Number.isFinite(n) && n > 0 && n < CGROUP_UNLIMITED)
|
|
71
|
-
return n;
|
|
72
|
-
}
|
|
73
|
-
catch {
|
|
74
|
-
/* not cgroup v1 (or unreadable) — fall through to host total */
|
|
75
|
-
}
|
|
76
|
-
return totalmem();
|
|
77
|
-
}
|
|
78
|
-
// How many recent completions the controller reasons over (latency percentiles + throughput + errors).
|
|
79
|
-
const WINDOW_SIZE = 100;
|
|
80
|
-
// EWMA smoothing for the "no-queuing" baseline (rttLong): low weight so a single fast sample doesn't
|
|
81
|
-
// reset the baseline, but it still tracks a genuine floor shift.
|
|
82
|
-
const RTT_LONG_EWMA_ALPHA = 0.2;
|
|
83
|
-
// Consecutive ticks with in-flight work but zero completions before we call the worker stalled.
|
|
84
|
-
const STALL_TICKS = 2;
|
|
85
|
-
// Throughput is a rate over the trailing minute: a window whose newest completion is older than this
|
|
86
|
-
// reports 0, so an idle worker never advertises its last burst as live.
|
|
87
|
-
const THROUGHPUT_WINDOW_MS = 60_000;
|
|
88
|
-
// The shortest span a rate is computed over, so a worker's first completion doesn't extrapolate a
|
|
89
|
-
// sub-millisecond gap into a huge per-minute figure.
|
|
90
|
-
const THROUGHPUT_MIN_SPAN_MS = 1_000;
|
|
91
|
-
function percentile(sortedAsc, p) {
|
|
92
|
-
if (sortedAsc.length === 0)
|
|
93
|
-
return 0;
|
|
94
|
-
const idx = clamp(Math.ceil((p / 100) * sortedAsc.length) - 1, 0, sortedAsc.length - 1);
|
|
95
|
-
return sortedAsc[idx] ?? 0;
|
|
96
|
-
}
|
|
97
|
-
/**
|
|
98
|
-
* A single gradient-limit (AIMD) concurrency controller, shared by every per-routing-token task
|
|
99
|
-
* worker of one {@link import('./bullmq-transport.js').BullMQTransport}. It tracks in-flight work and a rolling window of
|
|
100
|
-
* completions, and — in `adaptive` mode — re-evaluates the worker's concurrency every `tickMs` from a
|
|
101
|
-
* latency gradient, an error/stall backpressure signal, a hard RAM brake and an optional CPU ceiling.
|
|
102
|
-
*
|
|
103
|
-
* In `fixed` mode no control loop runs and the limit never moves, but the controller still tracks
|
|
104
|
-
* `inFlight`/RSS/throughput/p95 (and samples CPU when a snapshot is taken) so the observability half
|
|
105
|
-
* (the heartbeat status payload) works without adaptive. Either way, {@link snapshot} returns the
|
|
106
|
-
* shared {@link WorkerStatus} contract the heartbeat writer stamps.
|
|
107
|
-
*/
|
|
108
|
-
export class AdaptiveController {
|
|
109
|
-
resolved;
|
|
110
|
-
apply;
|
|
111
|
-
now;
|
|
112
|
-
readRss;
|
|
113
|
-
memoryLimitBytes;
|
|
114
|
-
tickMs;
|
|
115
|
-
readCpuUsage;
|
|
116
|
-
cpuCores;
|
|
117
|
-
currentLimit;
|
|
118
|
-
inFlightCount = 0;
|
|
119
|
-
window = [];
|
|
120
|
-
rttLongMs;
|
|
121
|
-
cpuPct;
|
|
122
|
-
lastAdjust;
|
|
123
|
-
completionsThisTick = 0;
|
|
124
|
-
stallTicks = 0;
|
|
125
|
-
lastCpuUsage;
|
|
126
|
-
lastCpuSampleAt = 0;
|
|
127
|
-
timer;
|
|
128
|
-
constructor(options = {}) {
|
|
129
|
-
this.resolved = resolveConcurrency(options.concurrency);
|
|
130
|
-
this.apply = options.apply ?? (() => { });
|
|
131
|
-
this.now = options.now ?? Date.now;
|
|
132
|
-
this.readRss = options.readRss ?? (() => process.memoryUsage().rss);
|
|
133
|
-
this.memoryLimitBytes = (options.readMemoryLimit ?? readMemoryLimitBytes)();
|
|
134
|
-
this.tickMs = this.resolved.adaptive?.tickMs ?? DEFAULT_TICK_MS;
|
|
135
|
-
this.readCpuUsage = options.readCpuUsage ?? (() => process.cpuUsage());
|
|
136
|
-
this.cpuCores = wholeAtLeastOne((options.readCpuCount ?? (() => cpus().length))(), 1);
|
|
137
|
-
this.currentLimit =
|
|
138
|
-
this.resolved.mode === 'adaptive'
|
|
139
|
-
? (this.resolved.adaptive?.start ?? DEFAULT_MIN)
|
|
140
|
-
: (this.resolved.fixed ?? 1);
|
|
141
|
-
}
|
|
142
|
-
/** The concurrency to hand the BullMQ Worker on construction (fixed N, or the adaptive `start`). */
|
|
143
|
-
get initialLimit() {
|
|
144
|
-
return this.currentLimit;
|
|
145
|
-
}
|
|
146
|
-
/** The live limit in effect right now. */
|
|
147
|
-
get limit() {
|
|
148
|
-
return this.currentLimit;
|
|
149
|
-
}
|
|
150
|
-
/** `true` when the controller will move the limit on its own (adaptive mode). */
|
|
151
|
-
get isAdaptive() {
|
|
152
|
-
return this.resolved.mode === 'adaptive';
|
|
153
|
-
}
|
|
154
|
-
/**
|
|
155
|
-
* Start the control loop (adaptive mode only; fixed mode just takes a CPU baseline). The timer is
|
|
156
|
-
* `unref`'d so it never keeps the process alive; idempotent.
|
|
157
|
-
*/
|
|
158
|
-
start() {
|
|
159
|
-
if (this.timer)
|
|
160
|
-
return;
|
|
161
|
-
if (!this.lastCpuUsage)
|
|
162
|
-
this.resetCpuBaseline();
|
|
163
|
-
if (!this.isAdaptive)
|
|
164
|
-
return;
|
|
165
|
-
this.timer = setInterval(() => this.tick(), this.tickMs);
|
|
166
|
-
this.timer.unref?.();
|
|
167
|
-
}
|
|
168
|
-
/** Stop the control loop (call on worker close). Idempotent. */
|
|
169
|
-
stop() {
|
|
170
|
-
if (this.timer) {
|
|
171
|
-
clearInterval(this.timer);
|
|
172
|
-
this.timer = undefined;
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
/** A task started — bump in-flight. Call from the job processor before awaiting the body. */
|
|
176
|
-
onStart() {
|
|
177
|
-
this.inFlightCount += 1;
|
|
178
|
-
}
|
|
179
|
-
/**
|
|
180
|
-
* A task settled. ALWAYS drops in-flight and counts toward stall liveness (`completionsThisTick`),
|
|
181
|
-
* so the saturation gate, the heartbeat `inFlight`, and the "alive-but-busy" signal see every task.
|
|
182
|
-
*
|
|
183
|
-
* `kind` gates the MEASUREMENT window: only `'step'` completions push a duration into the rolling
|
|
184
|
-
* window that feeds the latency gradient / throughput / p95 / error rate. A workflow TURN
|
|
185
|
-
* (`kind: 'workflow'`) is NOT measured — a 5ms suspend sitting next to a 2s step would corrupt the
|
|
186
|
-
* gradient — but it still counts as in-flight + forward progress. This is the single concurrency
|
|
187
|
-
* pool measuring steps only; the transport tags the kind when it routes a job by shape.
|
|
188
|
-
*/
|
|
189
|
-
onSettle(durationMs, ok, kind) {
|
|
190
|
-
this.inFlightCount = Math.max(0, this.inFlightCount - 1);
|
|
191
|
-
// Forward progress for stall detection counts BOTH kinds — a worker churning workflow turns is
|
|
192
|
-
// not stalled even when no step settles this tick.
|
|
193
|
-
this.completionsThisTick += 1;
|
|
194
|
-
if (kind !== 'step')
|
|
195
|
-
return;
|
|
196
|
-
this.window.push({ durationMs, ok, at: this.now() });
|
|
197
|
-
if (this.window.length > WINDOW_SIZE)
|
|
198
|
-
this.window.shift();
|
|
199
|
-
}
|
|
200
|
-
/**
|
|
201
|
-
* One control-loop iteration: sample CPU, refresh the latency baseline, then decide the next limit
|
|
202
|
-
* and apply it. A no-op in fixed mode. Exposed (not private) so a test can drive ticks
|
|
203
|
-
* deterministically.
|
|
204
|
-
*/
|
|
205
|
-
tick() {
|
|
206
|
-
if (this.resolved.mode !== 'adaptive' || !this.resolved.adaptive)
|
|
207
|
-
return;
|
|
208
|
-
this.sampleCpu(this.tickMs);
|
|
209
|
-
this.refreshRttLong();
|
|
210
|
-
// Stall = in-flight work but no completion this tick, sustained over STALL_TICKS.
|
|
211
|
-
if (this.inFlightCount > 0 && this.completionsThisTick === 0)
|
|
212
|
-
this.stallTicks += 1;
|
|
213
|
-
else
|
|
214
|
-
this.stallTicks = 0;
|
|
215
|
-
const stalled = this.stallTicks >= STALL_TICKS;
|
|
216
|
-
this.completionsThisTick = 0;
|
|
217
|
-
this.decide(this.resolved.adaptive, stalled);
|
|
218
|
-
}
|
|
219
|
-
resetCpuBaseline() {
|
|
220
|
-
this.lastCpuUsage = this.readCpuUsage();
|
|
221
|
-
this.lastCpuSampleAt = this.now();
|
|
222
|
-
}
|
|
223
|
-
/** Set `cpuPct` from the CPU time spent since the previous sample, over `elapsedMs` of wall time. */
|
|
224
|
-
sampleCpu(elapsedMs) {
|
|
225
|
-
const previous = this.lastCpuUsage;
|
|
226
|
-
this.resetCpuBaseline();
|
|
227
|
-
if (!previous || !this.lastCpuUsage || elapsedMs <= 0)
|
|
228
|
-
return;
|
|
229
|
-
// CpuUsage is microseconds of CPU; elapsedMs of wall time is elapsedMs*1000 micros per core.
|
|
230
|
-
const micros = this.lastCpuUsage.user - previous.user + (this.lastCpuUsage.system - previous.system);
|
|
231
|
-
const pct = (micros / (elapsedMs * 1000 * this.cpuCores)) * 100;
|
|
232
|
-
if (Number.isFinite(pct))
|
|
233
|
-
this.cpuPct = Math.max(0, pct);
|
|
234
|
-
}
|
|
235
|
-
/** Fixed mode has no control loop, so a snapshot samples CPU itself, at most once per tick period
|
|
236
|
-
* (several heartbeat keys snapshot back to back, and a near-zero span gives a meaningless rate). */
|
|
237
|
-
sampleCpuForSnapshot() {
|
|
238
|
-
if (this.isAdaptive)
|
|
239
|
-
return;
|
|
240
|
-
if (!this.lastCpuUsage) {
|
|
241
|
-
this.resetCpuBaseline();
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
const elapsedMs = this.now() - this.lastCpuSampleAt;
|
|
245
|
-
if (elapsedMs >= this.tickMs)
|
|
246
|
-
this.sampleCpu(elapsedMs);
|
|
247
|
-
}
|
|
248
|
-
refreshRttLong() {
|
|
249
|
-
if (this.window.length === 0)
|
|
250
|
-
return;
|
|
251
|
-
let minDuration = this.window[0]?.durationMs ?? 0;
|
|
252
|
-
for (const c of this.window)
|
|
253
|
-
if (c.durationMs < minDuration)
|
|
254
|
-
minDuration = c.durationMs;
|
|
255
|
-
this.rttLongMs =
|
|
256
|
-
this.rttLongMs === undefined
|
|
257
|
-
? minDuration
|
|
258
|
-
: RTT_LONG_EWMA_ALPHA * minDuration + (1 - RTT_LONG_EWMA_ALPHA) * this.rttLongMs;
|
|
259
|
-
}
|
|
260
|
-
rssPct() {
|
|
261
|
-
if (this.memoryLimitBytes <= 0)
|
|
262
|
-
return 0;
|
|
263
|
-
return (100 * this.readRss()) / this.memoryLimitBytes;
|
|
264
|
-
}
|
|
265
|
-
decide(cfg, stalled) {
|
|
266
|
-
const current = this.currentLimit;
|
|
267
|
-
const durations = this.window.map((c) => c.durationMs).sort((a, b) => a - b);
|
|
268
|
-
const p50 = percentile(durations, 50);
|
|
269
|
-
const errors = this.window.reduce((n, c) => n + (c.ok ? 0 : 1), 0);
|
|
270
|
-
const errorRate = this.window.length === 0 ? 0 : errors / this.window.length;
|
|
271
|
-
const gradient = this.rttLongMs !== undefined && p50 > 0 ? Math.min(1, this.rttLongMs / p50) : 1;
|
|
272
|
-
const rssPct = this.rssPct();
|
|
273
|
-
const saturated = this.inFlightCount >= current * 0.8;
|
|
274
|
-
let next = current;
|
|
275
|
-
let reason;
|
|
276
|
-
if (rssPct >= cfg.ramCeilingPct) {
|
|
277
|
-
next = Math.max(cfg.min, Math.floor(current * 0.8));
|
|
278
|
-
reason = 'ram_ceiling';
|
|
279
|
-
}
|
|
280
|
-
else if (cfg.cpuCeilingPct !== undefined && (this.cpuPct ?? 0) >= cfg.cpuCeilingPct) {
|
|
281
|
-
// CPU at/over the ceiling: never grow; shed one to relieve it.
|
|
282
|
-
next = Math.max(cfg.min, current - 1);
|
|
283
|
-
reason = 'cpu_ceiling';
|
|
284
|
-
}
|
|
285
|
-
else if (errorRate > 0.2 || stalled) {
|
|
286
|
-
next = Math.max(cfg.min, current - 1);
|
|
287
|
-
reason = 'backpressure';
|
|
288
|
-
}
|
|
289
|
-
else if (gradient < 0.7) {
|
|
290
|
-
next = Math.max(cfg.min, Math.floor(current * gradient));
|
|
291
|
-
reason = 'shrink';
|
|
292
|
-
}
|
|
293
|
-
else if (gradient >= 0.9 && saturated) {
|
|
294
|
-
next = Math.min(cfg.max, current + 1);
|
|
295
|
-
reason = 'grow';
|
|
296
|
-
}
|
|
297
|
-
next = clamp(next, cfg.min, cfg.max);
|
|
298
|
-
if (next === current || reason === undefined)
|
|
299
|
-
return;
|
|
300
|
-
this.currentLimit = next;
|
|
301
|
-
this.lastAdjust = { at: this.now(), from: current, to: next, reason };
|
|
302
|
-
this.apply(next);
|
|
303
|
-
}
|
|
304
|
-
/** The shared {@link WorkerStatus} the heartbeat writer stamps. Best-effort fields are omitted when
|
|
305
|
-
* unmeasured (no completions yet → no p95/throughput; no CPU sample yet → no `cpuPct`). */
|
|
306
|
-
snapshot() {
|
|
307
|
-
this.sampleCpuForSnapshot();
|
|
308
|
-
const durations = this.window.map((c) => c.durationMs).sort((a, b) => a - b);
|
|
309
|
-
const rssBytes = this.readRss();
|
|
310
|
-
const rssLimitBytes = this.memoryLimitBytes;
|
|
311
|
-
const status = {
|
|
312
|
-
runtime: 'node',
|
|
313
|
-
concurrency: {
|
|
314
|
-
mode: this.resolved.mode,
|
|
315
|
-
limit: this.currentLimit,
|
|
316
|
-
...(this.resolved.mode === 'adaptive' && this.resolved.adaptive
|
|
317
|
-
? { min: this.resolved.adaptive.min, max: this.resolved.adaptive.max }
|
|
318
|
-
: {}),
|
|
319
|
-
},
|
|
320
|
-
inFlight: this.inFlightCount,
|
|
321
|
-
rssBytes,
|
|
322
|
-
rssLimitBytes,
|
|
323
|
-
rssPct: rssLimitBytes > 0 ? (100 * rssBytes) / rssLimitBytes : 0,
|
|
324
|
-
};
|
|
325
|
-
if (this.cpuPct !== undefined)
|
|
326
|
-
status.cpuPct = this.cpuPct;
|
|
327
|
-
if (this.window.length > 0) {
|
|
328
|
-
status.throughputPerMin = this.throughputPerMin();
|
|
329
|
-
status.p95Ms = percentile(durations, 95);
|
|
330
|
-
}
|
|
331
|
-
if (this.lastAdjust)
|
|
332
|
-
status.lastAdjust = this.lastAdjust;
|
|
333
|
-
return status;
|
|
334
|
-
}
|
|
335
|
-
/**
|
|
336
|
-
* Completions per minute over the trailing minute: the window's completions from the last minute,
|
|
337
|
-
* over the span from the oldest of them (or a full minute, when the window reaches further back) to
|
|
338
|
-
* now, floored at a second. The rate decays while the worker is idle and reads 0 once a minute passes with no completion.
|
|
339
|
-
*/
|
|
340
|
-
throughputPerMin() {
|
|
341
|
-
const first = this.window[0];
|
|
342
|
-
if (!first)
|
|
343
|
-
return 0;
|
|
344
|
-
const now = this.now();
|
|
345
|
-
const recent = this.window.filter((c) => now - c.at <= THROUGHPUT_WINDOW_MS).length;
|
|
346
|
-
if (recent === 0)
|
|
347
|
-
return 0;
|
|
348
|
-
const spanMs = clamp(now - first.at, THROUGHPUT_MIN_SPAN_MS, THROUGHPUT_WINDOW_MS);
|
|
349
|
-
return (recent / spanMs) * 60_000;
|
|
350
|
-
}
|
|
351
|
-
}
|
|
1
|
+
// The adaptive controller is transport-neutral (the queue transport drives it too); it lives in
|
|
2
|
+
// `../adaptive-concurrency.ts`. Re-exported here so existing imports keep working.
|
|
3
|
+
export * from '../adaptive-concurrency.js';
|
|
352
4
|
//# sourceMappingURL=adaptive-concurrency.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adaptive-concurrency.js","sourceRoot":"","sources":["../../../../src/transports/bullmq/adaptive-concurrency.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAmDzC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,SAAS,KAAK,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,iFAAiF;AACjF,SAAS,eAAe,CAAC,KAAyB,EAAE,QAAgB;IAClE,OAAO,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpG,CAAC;AAED,mEAAmE;AACnE,SAAS,cAAc,CAAI,KAAyB,EAAE,QAAW;IAC/D,OAAO,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAuB;IACxD,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC1D,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;IAEtF,MAAM,SAAS,GAA+B,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,GAAG,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACxD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACrE,MAAM,aAAa,GAAG,cAAc,CAAC,SAAS,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAmB;QAC/B,GAAG;QACH,GAAG;QACH,KAAK;QACL,aAAa,EAAE,cAAc,CAAC,SAAS,CAAC,aAAa,EAAE,uBAAuB,CAAC;QAC/E,MAAM,EAAE,eAAe,CAAC,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC;QAC1D,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AACxC,CAAC;AAED,wGAAwG;AACxG,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,CAAC;AAEjC;;;;GAIG;AACH,MAAM,UAAU,oBAAoB;IAClC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,gBAAgB;gBAAE,OAAO,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;IACnD,CAAC;IACD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,6CAA6C,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACvF,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,gBAAgB;YAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,gEAAgE;IAClE,CAAC;IACD,OAAO,QAAQ,EAAE,CAAC;AACpB,CAAC;AASD,uGAAuG;AACvG,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,qGAAqG;AACrG,iEAAiE;AACjE,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,gGAAgG;AAChG,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,qGAAqG;AACrG,wEAAwE;AACxE,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,kGAAkG;AAClG,qDAAqD;AACrD,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,SAAS,UAAU,CAAC,SAAmB,EAAE,CAAS;IAChD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxF,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAwBD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,kBAAkB;IACZ,QAAQ,CAAsB;IAC9B,KAAK,CAA0B;IAC/B,GAAG,CAAe;IAClB,OAAO,CAAe;IACtB,gBAAgB,CAAS;IACzB,MAAM,CAAS;IACf,YAAY,CAAwB;IACpC,QAAQ,CAAS;IAE1B,YAAY,CAAS;IACrB,aAAa,GAAG,CAAC,CAAC;IACT,MAAM,GAAiB,EAAE,CAAC;IACnC,SAAS,CAAU;IACnB,MAAM,CAAU;IAChB,UAAU,CAAgB;IAE1B,mBAAmB,GAAG,CAAC,CAAC;IACxB,UAAU,GAAG,CAAC,CAAC;IACf,YAAY,CAAmB;IAC/B,eAAe,GAAG,CAAC,CAAC;IACpB,KAAK,CAA6C;IAE1D,YAAY,OAAO,GAA8B,EAAE;QACjD,IAAI,CAAC,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;QACpE,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,eAAe,IAAI,oBAAoB,CAAC,EAAE,CAAC;QAC5E,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,eAAe,CAAC;QAChE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,GAAG,eAAe,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,YAAY;YACf,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU;gBAC/B,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,IAAI,WAAW,CAAC;gBAChD,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,oGAAoG;IACpG,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,0CAA0C;IAC1C,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,iFAAiF;IACjF,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACvB,CAAC;IAED,gEAAgE;IAChE,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACzB,CAAC;IACH,CAAC;IAED,6FAA6F;IAC7F,OAAO;QACL,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;OASG;IACH,QAAQ,CAAC,UAAkB,EAAE,EAAW,EAAE,IAAyB;QACjE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;QACzD,+FAA+F;QAC/F,mDAAmD;QACnD,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAC9B,IAAI,IAAI,KAAK,MAAM;YAAE,OAAO;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW;YAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACH,IAAI;QACF,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO;QACzE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,kFAAkF;QAClF,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,IAAI,CAAC,mBAAmB,KAAK,CAAC;YAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;;YAC9E,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,IAAI,WAAW,CAAC;QAC/C,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAE7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,qGAAqG;IAC7F,SAAS,CAAC,SAAiB;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,SAAS,IAAI,CAAC;YAAE,OAAO;QAC9D,6FAA6F;QAC7F,MAAM,MAAM,GACV,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxF,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,SAAS,GAAG,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC;QAChE,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3D,CAAC;IAED;yGACqG;IAC7F,oBAAoB;QAC1B,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QACpD,IAAI,SAAS,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC1D,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACrC,IAAI,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC;QAClD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,CAAC,UAAU,GAAG,WAAW;gBAAE,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC;QACxF,IAAI,CAAC,SAAS;YACZ,IAAI,CAAC,SAAS,KAAK,SAAS;gBAC1B,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,mBAAmB,GAAG,WAAW,GAAG,CAAC,CAAC,GAAG,mBAAmB,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;IACvF,CAAC;IAEO,MAAM;QACZ,IAAI,IAAI,CAAC,gBAAgB,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAEO,MAAM,CAAC,GAAmB,EAAE,OAAgB;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC7E,MAAM,QAAQ,GACZ,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,GAAG,GAAG,CAAC;QAEtD,IAAI,IAAI,GAAG,OAAO,CAAC;QACnB,IAAI,MAA0C,CAAC;QAE/C,IAAI,MAAM,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YAChC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,GAAG,aAAa,CAAC;QACzB,CAAC;aAAM,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YACtF,+DAA+D;YAC/D,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,aAAa,CAAC;QACzB,CAAC;aAAM,IAAI,SAAS,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,cAAc,CAAC;QAC1B,CAAC;aAAM,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YAC1B,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC;YACzD,MAAM,GAAG,QAAQ,CAAC;QACpB,CAAC;aAAM,IAAI,QAAQ,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YACxC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,MAAM,CAAC;QAClB,CAAC;QAED,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,KAAK,OAAO,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED;gGAC4F;IAC5F,QAAQ;QACN,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC5C,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,MAAM;YACf,WAAW,EAAE;gBACX,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;gBACxB,KAAK,EAAE,IAAI,CAAC,YAAY;gBACxB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ;oBAC7D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE;oBACtE,CAAC,CAAC,EAAE,CAAC;aACR;YACD,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,QAAQ;YACR,aAAa;YACb,MAAM,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;SACjE,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3D,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,IAAI,CAAC,UAAU;YAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACzD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACK,gBAAgB;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,oBAAoB,CAAC,CAAC,MAAM,CAAC;QACpF,IAAI,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,EAAE,sBAAsB,EAAE,oBAAoB,CAAC,CAAC;QACnF,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;IACpC,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"adaptive-concurrency.js","sourceRoot":"","sources":["../../../../src/transports/bullmq/adaptive-concurrency.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,mFAAmF;AACnF,cAAc,4BAA4B,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { WorkerDescriptor } from '../../handshake/descriptor.js';
|
|
2
|
-
import type { GroupHealth, Heartbeat, RemoteTask, RunReply, RunRequest, StartRunMessage, StepResult, TenantEvent, Transport, WorkflowDecision, WorkflowStepEvent, WorkflowTask } from '../../interfaces.js';
|
|
2
|
+
import type { GroupHealth, Heartbeat, RemoteTask, RunReply, RunRequest, StartRunMessage, StepResult, TenantEvent, Transport, WorkerHeartbeat, WorkflowDecision, WorkflowStepEvent, WorkflowTask } from '../../interfaces.js';
|
|
3
3
|
import { type StepHandler } from '../../protocol.js';
|
|
4
4
|
import { type WorkflowTurnHandler } from '../../workflow-turn.js';
|
|
5
5
|
import { type ConcurrencyOption } from './adaptive-concurrency.js';
|
|
@@ -51,6 +51,28 @@ export interface BullMQTransportOptions {
|
|
|
51
51
|
* each task worker at the deps' default concurrency.
|
|
52
52
|
*/
|
|
53
53
|
concurrency?: ConcurrencyOption | undefined;
|
|
54
|
+
/**
|
|
55
|
+
* How often (ms) the shared subscriber watchdog PINGs every pub/sub subscriber connection this
|
|
56
|
+
* transport owns (the long-step heartbeat, run-reply and tenant-event subscribers) to detect, and
|
|
57
|
+
* recover from, a silent connection loss. A subscriber connection never writes on its own: when a
|
|
58
|
+
* VPN, NAT or idle timeout drops the TCP connection, no write fails and no timeout fires, so the
|
|
59
|
+
* connection sits "subscribed" while the server's `PUBSUB NUMSUB` already reads 0. Heartbeats stop
|
|
60
|
+
* resetting step leases, and a tenant's run-gateway calls time out until the process restarts. A
|
|
61
|
+
* PING that rejects or times out marks the connection dead: it is `disconnect(true)`ed, ioredis
|
|
62
|
+
* reconnects and `autoResubscribe` restores its channels. `0` or `false` disables the watchdog.
|
|
63
|
+
* Default `30_000`.
|
|
64
|
+
*/
|
|
65
|
+
pingIntervalMs?: number | false | undefined;
|
|
66
|
+
/**
|
|
67
|
+
* Wall-clock ceiling (ms) for one step handler on THIS worker. A wedged handler (an await that will
|
|
68
|
+
* never settle: a dead connection swallowed by a wrapper stream, an external call with no timeout)
|
|
69
|
+
* otherwise holds its BullMQ job forever, because the job lock renews on a timer whether or not the
|
|
70
|
+
* handler makes progress, so the job is never reclaimed as stalled. When set, a step still running
|
|
71
|
+
* after `stepTimeoutMs` publishes a RETRYABLE failed StepResult (the engine's durable retry
|
|
72
|
+
* re-dispatches it) and the orphaned handler promise is abandoned. Opt-in: pick a value well above
|
|
73
|
+
* your slowest legitimate step. Unset (the default) = no ceiling.
|
|
74
|
+
*/
|
|
75
|
+
stepTimeoutMs?: number | undefined;
|
|
54
76
|
}
|
|
55
77
|
/**
|
|
56
78
|
* A queue-backed {@link Transport} over BullMQ/Redis, BYTE-COMPATIBLE with the aviary
|
|
@@ -71,6 +93,26 @@ export interface BullMQTransportOptions {
|
|
|
71
93
|
export declare class BullMQTransport implements Transport {
|
|
72
94
|
#private;
|
|
73
95
|
constructor(options: BullMQTransportOptions);
|
|
96
|
+
/**
|
|
97
|
+
* Park every point-to-point consumer registered from now on — the task, results, decisions,
|
|
98
|
+
* step-events, start-run and run-request workers — until {@link startConsumers}.
|
|
99
|
+
*
|
|
100
|
+
* WHY: a BullMQ `Worker` starts fetching the moment it is constructed, and these queues hand each
|
|
101
|
+
* job to ONE consumer. Any process that registers them competes with the real worker fleet: a
|
|
102
|
+
* `node ace` command or REPL claims jobs it then dies holding, and steals results addressed to the
|
|
103
|
+
* long-lived engine (the bug the `queue` transport fixed the same way). Deferring construction makes
|
|
104
|
+
* such a process a pure producer — it still dispatches and publishes; the jobs stay on the queues.
|
|
105
|
+
* Pub/sub subscriptions (heartbeats, run replies, tenant events) are broadcast, take nothing from
|
|
106
|
+
* anyone, and are left alone.
|
|
107
|
+
*
|
|
108
|
+
* Call it before any subscription is made (the provider does, right after building the transport);
|
|
109
|
+
* workers already started are not stopped.
|
|
110
|
+
*/
|
|
111
|
+
deferConsumers(): void;
|
|
112
|
+
/** Construct every worker parked by {@link deferConsumers} and stop deferring — the explicit "this
|
|
113
|
+
* process IS a worker" declaration (`durable:work` makes it via `engine.startConsumers()`).
|
|
114
|
+
* Idempotent; a no-op when consumption was never deferred. */
|
|
115
|
+
startConsumers(): void;
|
|
74
116
|
/** Stable id stamped on this instance's worker-liveness keys. */
|
|
75
117
|
get instanceId(): string;
|
|
76
118
|
/**
|
|
@@ -122,6 +164,12 @@ export declare class BullMQTransport implements Transport {
|
|
|
122
164
|
* neither segment carries a `:`, so the token is the segment between the fixed prefix and the next `:`. */
|
|
123
165
|
listWorkerGroups(): Promise<string[]>;
|
|
124
166
|
listWorkerDescriptors(token: string): Promise<WorkerDescriptor[]>;
|
|
167
|
+
/** Every live descriptor on every token: SCAN `${P}-worker-descriptor:*`, MGET per page. Degrades
|
|
168
|
+
* to `[]` (reported through `onError`) when the keyspace cannot be read. */
|
|
169
|
+
readAllWorkerDescriptors(): Promise<WorkerDescriptor[]>;
|
|
170
|
+
/** Every live worker heartbeat on every token: SCAN `${P}-worker-heartbeat:*`. A key is
|
|
171
|
+
* `<prefix><token>:<instance>` and neither segment carries a `:`. Degrades to `[]`. */
|
|
172
|
+
readAllWorkerHeartbeats(): Promise<WorkerHeartbeat[]>;
|
|
125
173
|
/** Per-group worker-health: queue depth (waiting+active+delayed+prioritized) + live workers (their
|
|
126
174
|
* non-expired heartbeat keys). `group` is a routing token — the same value a dispatch targets. */
|
|
127
175
|
groupHealth(group: string): Promise<GroupHealth>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bullmq-transport.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/bullmq-transport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,UAAU,EACV,QAAQ,EACR,UAAU,EACV,eAAe,EACf,UAAU,EACV,WAAW,EACX,SAAS,
|
|
1
|
+
{"version":3,"file":"bullmq-transport.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/bullmq-transport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,UAAU,EACV,QAAQ,EACR,UAAU,EACV,eAAe,EACf,UAAU,EACV,WAAW,EACX,SAAS,EACT,eAAe,EAEf,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,KAAK,WAAW,EAAqB,MAAM,mBAAmB,CAAC;AAExF,OAAO,EAAkB,KAAK,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAOlF,OAAO,EAAsB,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACvF,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,SAAS,EAGf,MAAM,WAAW,CAAC;AA4BnB,MAAM,WAAW,sBAAsB;IACrC;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,2FAA2F;IAC3F,IAAI,EAAE,UAAU,CAAC;IACjB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,oEAAoE;IACpE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yFAAyF;IACzF,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,mGAAmG;IACnG,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAC/C;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;IAC5C;;;;;;;;;;OAUG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;IAC5C;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAsBD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,SAAS;;IA2D/C,YAAY,OAAO,EAAE,sBAAsB,EAmB1C;IAiGD;;;;;;;;;;;;;;OAcG;IACH,cAAc,IAAI,IAAI,CAErB;IAED;;mEAE+D;IAC/D,cAAc,IAAI,IAAI,CAIrB;IAgDD,iEAAiE;IACjE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAGpC;IAyBD;;;;;;;OAOG;IACH,qBAAqB,IAAI,SAAS,CAEjC;IAMD;+EAC2E;IACrE,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9C;IAED;2GACuG;IACjG,oBAAoB,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAM5D;IAMD;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,IAAI,CAG1C;IAED;;;;;;OAMG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAG5D;IAsLD;oGACgG;IAC1F,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9C;IAqBD,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAiB7D;IAED,UAAU,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAkBvE;IAED;uEACmE;IAC7D,iBAAiB,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAM/D;IAED,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKtE;IAED,WAAW,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAkB7D;IA2CD;;gHAE4G;IACtG,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAgB1C;IAYK,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAmCtE;IAED;iFAC6E;IACvE,wBAAwB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAqB5D;IAED;4FACwF;IAClF,uBAAuB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAwB1D;IAuBD;uGACmG;IAC7F,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CASrD;IA+CD,sGAAsG;IAChG,gBAAgB,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;IAED,uGAAuG;IACvG,UAAU,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKjE;IAED,qGAAqG;IAC/F,kBAAkB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAED,kGAAkG;IAClG,YAAY,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAK9D;IAED,mGAAmG;IAC7F,eAAe,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,wGAAwG;IACxG,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,IAAI,CAoBnD;IAED,2GAA2G;IACrG,kBAAkB,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAMxD;IAED;;;;;OAKG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAqB7E;IAyBD,uGAAuG;IACjG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAiC3B;CACF"}
|