@adonis-agora/durable 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/assets/dashboard.html +116 -3
- package/dist/commands/durable_worker.d.ts +25 -0
- package/dist/commands/durable_worker.d.ts.map +1 -0
- package/dist/commands/durable_worker.js +113 -0
- package/dist/commands/durable_worker.js.map +1 -0
- package/dist/commands/main.d.ts +6 -5
- package/dist/commands/main.d.ts.map +1 -1
- package/dist/commands/main.js +11 -4
- package/dist/commands/main.js.map +1 -1
- package/dist/providers/dashboard_provider.d.ts +37 -0
- package/dist/providers/dashboard_provider.d.ts.map +1 -1
- package/dist/providers/dashboard_provider.js +98 -9
- package/dist/providers/dashboard_provider.js.map +1 -1
- package/dist/providers/durable_provider.d.ts +24 -16
- package/dist/providers/durable_provider.d.ts.map +1 -1
- package/dist/providers/durable_provider.js +195 -20
- package/dist/providers/durable_provider.js.map +1 -1
- package/dist/src/config_types.d.ts +92 -0
- package/dist/src/config_types.d.ts.map +1 -0
- package/dist/src/config_types.js +2 -0
- package/dist/src/config_types.js.map +1 -0
- package/dist/src/dashboard/compat-view.d.ts +13 -0
- package/dist/src/dashboard/compat-view.d.ts.map +1 -0
- package/dist/src/dashboard/compat-view.js +32 -0
- package/dist/src/dashboard/compat-view.js.map +1 -0
- package/dist/src/dashboard/compat.d.ts +76 -0
- package/dist/src/dashboard/compat.d.ts.map +1 -0
- package/dist/src/dashboard/compat.js +130 -0
- package/dist/src/dashboard/compat.js.map +1 -0
- package/dist/src/dashboard/diagnostics-recorder.d.ts +59 -0
- package/dist/src/dashboard/diagnostics-recorder.d.ts.map +1 -0
- package/dist/src/dashboard/diagnostics-recorder.js +75 -0
- package/dist/src/dashboard/diagnostics-recorder.js.map +1 -0
- package/dist/src/dashboard/gateway-adapter.d.ts +37 -0
- package/dist/src/dashboard/gateway-adapter.d.ts.map +1 -0
- package/dist/src/dashboard/gateway-adapter.js +49 -0
- package/dist/src/dashboard/gateway-adapter.js.map +1 -0
- package/dist/src/dashboard/handlers.d.ts +31 -8
- package/dist/src/dashboard/handlers.d.ts.map +1 -1
- package/dist/src/dashboard/handlers.js +2 -1
- package/dist/src/dashboard/handlers.js.map +1 -1
- package/dist/src/dashboard/index.d.ts +10 -3
- package/dist/src/dashboard/index.d.ts.map +1 -1
- package/dist/src/dashboard/index.js +8 -2
- package/dist/src/dashboard/index.js.map +1 -1
- package/dist/src/define_config.d.ts +31 -5
- package/dist/src/define_config.d.ts.map +1 -1
- package/dist/src/define_config.js +4 -3
- package/dist/src/define_config.js.map +1 -1
- package/dist/src/dispatch-routing.d.ts +105 -0
- package/dist/src/dispatch-routing.d.ts.map +1 -0
- package/dist/src/dispatch-routing.js +114 -0
- package/dist/src/dispatch-routing.js.map +1 -0
- package/dist/src/engine.d.ts +65 -2
- package/dist/src/engine.d.ts.map +1 -1
- package/dist/src/engine.js +131 -3
- package/dist/src/engine.js.map +1 -1
- package/dist/src/handshake/descriptor.d.ts +128 -0
- package/dist/src/handshake/descriptor.d.ts.map +1 -0
- package/dist/src/handshake/descriptor.js +133 -0
- package/dist/src/handshake/descriptor.js.map +1 -0
- package/dist/src/handshake/negotiate.d.ts +83 -0
- package/dist/src/handshake/negotiate.d.ts.map +1 -0
- package/dist/src/handshake/negotiate.js +108 -0
- package/dist/src/handshake/negotiate.js.map +1 -0
- package/dist/src/handshake/routing.d.ts +51 -0
- package/dist/src/handshake/routing.d.ts.map +1 -0
- package/dist/src/handshake/routing.js +57 -0
- package/dist/src/handshake/routing.js.map +1 -0
- package/dist/src/index.d.ts +14 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +12 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/interfaces.d.ts +168 -2
- package/dist/src/interfaces.d.ts.map +1 -1
- package/dist/src/remote-workflow-executor.d.ts +25 -1
- package/dist/src/remote-workflow-executor.d.ts.map +1 -1
- package/dist/src/remote-workflow-executor.js +41 -0
- package/dist/src/remote-workflow-executor.js.map +1 -1
- package/dist/src/role_bindings.d.ts +85 -0
- package/dist/src/role_bindings.d.ts.map +1 -0
- package/dist/src/role_bindings.js +73 -0
- package/dist/src/role_bindings.js.map +1 -0
- package/dist/src/run-gateway/interface.d.ts +89 -0
- package/dist/src/run-gateway/interface.d.ts.map +1 -0
- package/dist/src/run-gateway/interface.js +2 -0
- package/dist/src/run-gateway/interface.js.map +1 -0
- package/dist/src/run-gateway/proxy-run-gateway.d.ts +84 -0
- package/dist/src/run-gateway/proxy-run-gateway.d.ts.map +1 -0
- package/dist/src/run-gateway/proxy-run-gateway.js +142 -0
- package/dist/src/run-gateway/proxy-run-gateway.js.map +1 -0
- package/dist/src/run-gateway/run-request-responder.d.ts +67 -0
- package/dist/src/run-gateway/run-request-responder.d.ts.map +1 -0
- package/dist/src/run-gateway/run-request-responder.js +228 -0
- package/dist/src/run-gateway/run-request-responder.js.map +1 -0
- package/dist/src/run-gateway/store-run-gateway.d.ts +90 -0
- package/dist/src/run-gateway/store-run-gateway.d.ts.map +1 -0
- package/dist/src/run-gateway/store-run-gateway.js +80 -0
- package/dist/src/run-gateway/store-run-gateway.js.map +1 -0
- package/dist/src/run-gateway/tenant-auth.d.ts +19 -0
- package/dist/src/run-gateway/tenant-auth.d.ts.map +1 -0
- package/dist/src/run-gateway/tenant-auth.js +71 -0
- package/dist/src/run-gateway/tenant-auth.js.map +1 -0
- package/dist/src/services/main.d.ts +16 -3
- package/dist/src/services/main.d.ts.map +1 -1
- package/dist/src/services/main.js +23 -4
- package/dist/src/services/main.js.map +1 -1
- package/dist/src/step-name-symbol.d.ts +3 -0
- package/dist/src/step-name-symbol.d.ts.map +1 -1
- package/dist/src/step-name-symbol.js.map +1 -1
- package/dist/src/step-ref.d.ts +4 -0
- package/dist/src/step-ref.d.ts.map +1 -1
- package/dist/src/step-ref.js +2 -0
- package/dist/src/step-ref.js.map +1 -1
- package/dist/src/transports/bullmq/bullmq-transport.d.ts +130 -0
- package/dist/src/transports/bullmq/bullmq-transport.d.ts.map +1 -0
- package/dist/src/transports/bullmq/bullmq-transport.js +562 -0
- package/dist/src/transports/bullmq/bullmq-transport.js.map +1 -0
- package/dist/src/transports/bullmq/deps.d.ts +59 -0
- package/dist/src/transports/bullmq/deps.d.ts.map +1 -0
- package/dist/src/transports/bullmq/deps.js +41 -0
- package/dist/src/transports/bullmq/deps.js.map +1 -0
- package/dist/src/transports/bullmq/index.d.ts +5 -0
- package/dist/src/transports/bullmq/index.d.ts.map +1 -0
- package/dist/src/transports/bullmq/index.js +5 -0
- package/dist/src/transports/bullmq/index.js.map +1 -0
- package/dist/src/transports/bullmq/naming.d.ts +71 -0
- package/dist/src/transports/bullmq/naming.d.ts.map +1 -0
- package/dist/src/transports/bullmq/naming.js +101 -0
- package/dist/src/transports/bullmq/naming.js.map +1 -0
- package/dist/src/transports/bullmq/serialization.d.ts +56 -0
- package/dist/src/transports/bullmq/serialization.d.ts.map +1 -0
- package/dist/src/transports/bullmq/serialization.js +101 -0
- package/dist/src/transports/bullmq/serialization.js.map +1 -0
- package/dist/src/transports/factory.d.ts +31 -0
- package/dist/src/transports/factory.d.ts.map +1 -1
- package/dist/src/transports/factory.js +21 -0
- package/dist/src/transports/factory.js.map +1 -1
- package/dist/src/worker-runtime/index.d.ts +22 -0
- package/dist/src/worker-runtime/index.d.ts.map +1 -0
- package/dist/src/worker-runtime/index.js +27 -0
- package/dist/src/worker-runtime/index.js.map +1 -0
- package/dist/src/worker-runtime/naming.d.ts +24 -0
- package/dist/src/worker-runtime/naming.d.ts.map +1 -0
- package/dist/src/worker-runtime/naming.js +28 -0
- package/dist/src/worker-runtime/naming.js.map +1 -0
- package/dist/src/worker-runtime/registry.d.ts +81 -0
- package/dist/src/worker-runtime/registry.d.ts.map +1 -0
- package/dist/src/worker-runtime/registry.js +44 -0
- package/dist/src/worker-runtime/registry.js.map +1 -0
- package/dist/src/worker-runtime/worker-runtime.d.ts +155 -0
- package/dist/src/worker-runtime/worker-runtime.d.ts.map +1 -0
- package/dist/src/worker-runtime/worker-runtime.js +265 -0
- package/dist/src/worker-runtime/worker-runtime.js.map +1 -0
- package/dist/src/workflow-ctx.d.ts.map +1 -1
- package/dist/src/workflow-ctx.js +3 -0
- package/dist/src/workflow-ctx.js.map +1 -1
- package/dist/src/workflow-turn.d.ts +167 -0
- package/dist/src/workflow-turn.d.ts.map +1 -0
- package/dist/src/workflow-turn.js +372 -0
- package/dist/src/workflow-turn.js.map +1 -0
- package/package.json +7 -1
|
@@ -0,0 +1,562 @@
|
|
|
1
|
+
import { runStepHandler } from '../../protocol.js';
|
|
2
|
+
import { isWorkflowTask } from '../../workflow-turn.js';
|
|
3
|
+
import { decisionsName, effectivePrefix, heartbeatChannel, resultsName, routingToken, runReplyChannel, runRequestName, startRunName, stepEventsName, tasksName, tenantEventsChannel, workerDescriptorTokenPrefix, workerHeartbeatKey, workerHeartbeatKeyPrefix, } from './naming.js';
|
|
4
|
+
import { WORKER_HEARTBEAT_INTERVAL_MS, WORKER_HEARTBEAT_TTL_SECONDS, buildInstanceId, heartbeatKeyValue, jobOptions, parseHeartbeatValue, taskJobOptions, } from './serialization.js';
|
|
5
|
+
/**
|
|
6
|
+
* A queue-backed {@link Transport} over BullMQ/Redis, BYTE-COMPATIBLE with the aviary
|
|
7
|
+
* (`nestjs-durable`) BullMQ transport and its Python raw-redis worker — so an Adonis engine can
|
|
8
|
+
* dispatch to a Python worker (and vice versa) on the SAME Redis keys. Steps/workflow tasks go to a
|
|
9
|
+
* per-routing-token queue (`${P}-tasks-${token}`); step results, workflow decisions and streamed step
|
|
10
|
+
* events ride their own queues; run / long-step heartbeats ride a `${P}-heartbeat` pub/sub; and a
|
|
11
|
+
* TTL'd `${P}-worker-heartbeat:${token}:${instance}` key registry advertises live workers.
|
|
12
|
+
*
|
|
13
|
+
* It implements the CURRENT {@link Transport} interface (dispatch/result/heartbeat + the optional
|
|
14
|
+
* workflow-task / decision / step-event / group-health hooks). It deliberately does NOT implement the
|
|
15
|
+
* control plane: the shipped `RedisControlPlane` already owns the byte-compatible `${P}-control`
|
|
16
|
+
* channel, so pass it as the engine's `controlPlane` alongside this transport.
|
|
17
|
+
*
|
|
18
|
+
* The concrete broker is injected as {@link BullMQDeps} so this class stays pure — construct it via
|
|
19
|
+
* `transports.bullmq({...})`, which builds the real bullmq/ioredis deps lazily.
|
|
20
|
+
*/
|
|
21
|
+
export class BullMQTransport {
|
|
22
|
+
#deps;
|
|
23
|
+
#partition;
|
|
24
|
+
#prefix;
|
|
25
|
+
#namespace;
|
|
26
|
+
#explicitNamespace;
|
|
27
|
+
#instanceId;
|
|
28
|
+
#onError;
|
|
29
|
+
#handlers = new Map();
|
|
30
|
+
// Worker-side workflow turn consumers, keyed by workflow name — a `workflow`-shaped job routes here.
|
|
31
|
+
#workflowTurns = new Map();
|
|
32
|
+
// One task queue/worker per registered handler name (its routing token) — never a shared queue.
|
|
33
|
+
#queues = new Map();
|
|
34
|
+
#taskWorkers = new Map();
|
|
35
|
+
#resultsWorker;
|
|
36
|
+
#decisionsWorker;
|
|
37
|
+
#stepEventsWorker;
|
|
38
|
+
// Long-step heartbeat pub/sub (distinct from the worker-liveness key registry below).
|
|
39
|
+
#heartbeatPub;
|
|
40
|
+
#heartbeatSub;
|
|
41
|
+
// Worker liveness: one TTL'd key per handled routing token, refreshed on a single shared interval.
|
|
42
|
+
#workerRedis;
|
|
43
|
+
#workerHeartbeatTimer;
|
|
44
|
+
#heartbeatTokens = new Set();
|
|
45
|
+
// P4 store-less protocol: start-run / run-request queues + consumers, run-reply / tenant-events pub/sub.
|
|
46
|
+
#startRunWorker;
|
|
47
|
+
#runRequestWorker;
|
|
48
|
+
#runReplyPub;
|
|
49
|
+
#runReplySub;
|
|
50
|
+
#tenantEventPub;
|
|
51
|
+
// ONE shared subscriber for every tenant-events channel; a message is routed to the channel's
|
|
52
|
+
// handler set, so `onTenantEvent` can be called for several tenants (or several times per tenant).
|
|
53
|
+
#tenantEventSub;
|
|
54
|
+
#tenantEventChannels = new Set();
|
|
55
|
+
#tenantEventHandlers = new Map();
|
|
56
|
+
constructor(options) {
|
|
57
|
+
this.#deps = options.deps;
|
|
58
|
+
this.#partition = options.partition;
|
|
59
|
+
this.#prefix = options.prefix ?? 'durable';
|
|
60
|
+
this.#namespace = options.namespace;
|
|
61
|
+
this.#explicitNamespace = options.namespace !== undefined;
|
|
62
|
+
this.#instanceId = options.instanceId ?? buildInstanceId();
|
|
63
|
+
this.#onError = options.onError ?? (() => { });
|
|
64
|
+
}
|
|
65
|
+
/** Stable id stamped on this instance's worker-liveness keys. */
|
|
66
|
+
get instanceId() {
|
|
67
|
+
return this.#instanceId;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Adopt `namespace` (the engine's), segmenting every name by it — but ONLY if one wasn't passed
|
|
71
|
+
* explicitly to the constructor (an explicit one always wins). Idempotent. No-op for `"default"`.
|
|
72
|
+
*/
|
|
73
|
+
useNamespace(namespace) {
|
|
74
|
+
if (this.#explicitNamespace)
|
|
75
|
+
return;
|
|
76
|
+
this.#namespace = namespace;
|
|
77
|
+
}
|
|
78
|
+
#effectivePrefix() {
|
|
79
|
+
return effectivePrefix(this.#prefix, this.#namespace);
|
|
80
|
+
}
|
|
81
|
+
#queue(name) {
|
|
82
|
+
let queue = this.#queues.get(name);
|
|
83
|
+
if (!queue) {
|
|
84
|
+
queue = this.#deps.makeQueue(name);
|
|
85
|
+
this.#queues.set(name, queue);
|
|
86
|
+
}
|
|
87
|
+
return queue;
|
|
88
|
+
}
|
|
89
|
+
#workerRedisClient() {
|
|
90
|
+
if (!this.#workerRedis)
|
|
91
|
+
this.#workerRedis = this.#deps.makeRedis();
|
|
92
|
+
return this.#workerRedis;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Capability hook consumed by the `tenant`-role provider: mint a DEDICATED Redis client the
|
|
96
|
+
* store-less {@link WorkerRuntime}'s registry uses to publish/refresh its `-worker-descriptor:`
|
|
97
|
+
* and `-worker-heartbeat:` keys (SET…EX). Dedicated so the registry disconnecting on worker
|
|
98
|
+
* drain never tears down the transport's own broker clients. The returned `RedisLike` structurally
|
|
99
|
+
* satisfies the registry's `DescriptorRedis` port (`set`/`del`/`disconnect`), and is what the
|
|
100
|
+
* control-plane's `listWorkerDescriptors` SCAN later reads back — closing the descriptor loop.
|
|
101
|
+
*/
|
|
102
|
+
createDescriptorRedis() {
|
|
103
|
+
return this.#deps.makeRedis();
|
|
104
|
+
}
|
|
105
|
+
// ---------------------------------------------------------------------------
|
|
106
|
+
// engine → worker
|
|
107
|
+
// ---------------------------------------------------------------------------
|
|
108
|
+
/** `task.group` already carries the FINAL routing token (computed by the engine) — target that
|
|
109
|
+
* handler's dedicated queue. Job name `task`; DTO is the raw job data. */
|
|
110
|
+
async dispatch(task) {
|
|
111
|
+
await this.#queue(tasksName(this.#effectivePrefix(), task.group)).add('task', task, taskJobOptions(task.priority));
|
|
112
|
+
}
|
|
113
|
+
/** engine → workflow worker: a `WorkflowTask` on the workflow's dedicated task queue (same queue a
|
|
114
|
+
* Python workflow worker consumes). Job name `workflow`; the decision returns on `${P}-decisions`. */
|
|
115
|
+
async dispatchWorkflowTask(task) {
|
|
116
|
+
await this.#queue(tasksName(this.#effectivePrefix(), task.group)).add('workflow', task, jobOptions(task.priority));
|
|
117
|
+
}
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
// worker side — register a step handler, run it, push the result back
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
/**
|
|
122
|
+
* Register a step handler (worker side). Starts a DEDICATED task worker for `name`'s routing token on
|
|
123
|
+
* first registration (see {@link #ensureTaskWorker}) and begins stamping this instance's liveness key
|
|
124
|
+
* for that token. Re-registering swaps the handler fn; its worker is untouched.
|
|
125
|
+
*/
|
|
126
|
+
handle(name, fn) {
|
|
127
|
+
this.#handlers.set(name, fn);
|
|
128
|
+
this.#ensureTaskWorker(routingToken(name, this.#partition));
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Register a WORKFLOW turn consumer (worker side, design §4 / spec §6.3). Starts a DEDICATED task
|
|
132
|
+
* worker for the workflow `name`'s routing token — the SAME `${P}-tasks-<token>` queue a Python
|
|
133
|
+
* workflow worker consumes — which discriminates step-vs-workflow BY SHAPE ({@link #runJob}): a
|
|
134
|
+
* `workflow`-shaped job runs `turn(task)` (the replay → decision) and its {@link WorkflowDecision} is
|
|
135
|
+
* published on `${P}-decisions`. Re-registering swaps the turn fn; the worker is untouched.
|
|
136
|
+
*/
|
|
137
|
+
handleWorkflow(name, turn) {
|
|
138
|
+
this.#workflowTurns.set(name, turn);
|
|
139
|
+
this.#ensureTaskWorker(routingToken(name, this.#partition));
|
|
140
|
+
}
|
|
141
|
+
/** Start (once per routing token) the discriminating task worker + its liveness heartbeat. Shared by
|
|
142
|
+
* {@link handle} (steps) and {@link handleWorkflow} (workflow turns) — both ride `${P}-tasks-<token>`. */
|
|
143
|
+
#ensureTaskWorker(token) {
|
|
144
|
+
if (this.#taskWorkers.has(token))
|
|
145
|
+
return;
|
|
146
|
+
const worker = this.#deps.makeWorker(tasksName(this.#effectivePrefix(), token), (job) => this.#runJob(job.data));
|
|
147
|
+
// Terminal-failure bridge: a task job reaching a TERMINAL `failed` state is an INFRASTRUCTURE
|
|
148
|
+
// failure (worker crash / stalled-count exhaustion), never a handler business error —
|
|
149
|
+
// `runStepHandler` catches every handler throw and publishes a failed StepResult so the job
|
|
150
|
+
// SUCCEEDS. `failed` fires only when no StepResult was produced, so without this the run hangs on
|
|
151
|
+
// its `pending` checkpoint forever. Synthesize a RETRYABLE failed StepResult so the engine settles
|
|
152
|
+
// the checkpoint and re-dispatches. A PEER replica's worker fails a crashed worker's stalled job,
|
|
153
|
+
// so this still fires cross-process. (A failed WORKFLOW job carries no step identity, so the bridge
|
|
154
|
+
// no-ops for it — the engine's RemoteWorkflowExecutor re-drives the turn on recovery instead.)
|
|
155
|
+
worker.on('failed', (job, error) => {
|
|
156
|
+
const identity = failedTaskIdentity(job?.data);
|
|
157
|
+
if (!identity)
|
|
158
|
+
return; // payload already GC'd, malformed, or a workflow job — nothing safe to publish
|
|
159
|
+
void this.#bridgeTaskFailure(identity, job?.failedReason ?? error?.message ?? 'unknown');
|
|
160
|
+
});
|
|
161
|
+
this.#taskWorkers.set(token, worker);
|
|
162
|
+
this.#startWorkerHeartbeat(token);
|
|
163
|
+
}
|
|
164
|
+
/** Route one task job by SHAPE (spec §6.3): a workflow turn runs the replay → decision path, a step
|
|
165
|
+
* runs the `runStepHandler` → result path. Both kinds share a token's queue but never each other's. */
|
|
166
|
+
async #runJob(data) {
|
|
167
|
+
if (isWorkflowTask(data))
|
|
168
|
+
return this.#runWorkflowTask(data);
|
|
169
|
+
return this.#runTask(data);
|
|
170
|
+
}
|
|
171
|
+
/** Worker side: replay a workflow turn and publish its {@link WorkflowDecision} on `${P}-decisions`
|
|
172
|
+
* (job `decision`) — the queue the engine's `onDecision` consumes. An unknown workflow or a turn
|
|
173
|
+
* that throws still publishes a `failed` decision so the engine settles the run (never hangs). */
|
|
174
|
+
async #runWorkflowTask(task) {
|
|
175
|
+
let decision;
|
|
176
|
+
try {
|
|
177
|
+
const turn = this.#workflowTurns.get(task.workflow);
|
|
178
|
+
decision = turn
|
|
179
|
+
? await turn(task)
|
|
180
|
+
: {
|
|
181
|
+
taskId: task.taskId,
|
|
182
|
+
runId: task.runId,
|
|
183
|
+
status: 'failed',
|
|
184
|
+
commands: [],
|
|
185
|
+
error: { message: `no workflow registered for '${task.workflow}'`, code: 'no_workflow' },
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
catch (err) {
|
|
189
|
+
// A turn handler is expected to be pure (it returns a `failed` decision rather than throwing), so
|
|
190
|
+
// reaching here is unusual — publish a failed decision anyway so the run settles.
|
|
191
|
+
this.#onError(err);
|
|
192
|
+
decision = {
|
|
193
|
+
taskId: task.taskId,
|
|
194
|
+
runId: task.runId,
|
|
195
|
+
status: 'failed',
|
|
196
|
+
commands: [],
|
|
197
|
+
error: { message: err instanceof Error ? err.message : String(err) },
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
try {
|
|
201
|
+
await this.#queue(decisionsName(this.#effectivePrefix())).add('decision', decision, jobOptions());
|
|
202
|
+
}
|
|
203
|
+
catch (err) {
|
|
204
|
+
// The decisions-queue publish threw (Redis blipped). Rethrow so the job is marked `failed` and the
|
|
205
|
+
// engine re-drives the turn on recovery rather than the run hanging awaiting a decision.
|
|
206
|
+
this.#onError(err);
|
|
207
|
+
throw err;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async #runTask(task) {
|
|
211
|
+
let result;
|
|
212
|
+
try {
|
|
213
|
+
result = await runStepHandler(task, this.#handlers.get(task.name));
|
|
214
|
+
}
|
|
215
|
+
catch (err) {
|
|
216
|
+
// runStepHandler is pure (a handler throw becomes a failed StepResult), so reaching here is a
|
|
217
|
+
// bug — guard anyway so a future refactor can't turn it into an unsettled `pending` checkpoint.
|
|
218
|
+
this.#onError(err);
|
|
219
|
+
result = {
|
|
220
|
+
runId: task.runId,
|
|
221
|
+
seq: task.seq,
|
|
222
|
+
stepId: task.stepId,
|
|
223
|
+
status: 'failed',
|
|
224
|
+
error: {
|
|
225
|
+
message: 'remote step worker failed: handler threw unexpectedly',
|
|
226
|
+
retryable: true,
|
|
227
|
+
},
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
try {
|
|
231
|
+
await this.#queue(resultsName(this.#effectivePrefix())).add('result', result, jobOptions());
|
|
232
|
+
}
|
|
233
|
+
catch (err) {
|
|
234
|
+
// The results-queue publish itself threw (e.g. Redis blipped). We still hold `task`, so
|
|
235
|
+
// rethrow so the BullMQ job is marked `failed` and the terminal-failure bridge + durable retry
|
|
236
|
+
// become the last line of defence rather than the run hanging on `pending`.
|
|
237
|
+
this.#onError(err);
|
|
238
|
+
throw err;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
/** Worker side: publish a liveness heartbeat for an in-flight long step on the `${P}-heartbeat`
|
|
242
|
+
* pub/sub channel (resets the engine's `timeoutMs` window on whichever pod is awaiting it). */
|
|
243
|
+
async heartbeat(beat) {
|
|
244
|
+
if (!this.#heartbeatPub)
|
|
245
|
+
this.#heartbeatPub = this.#deps.makeRedis();
|
|
246
|
+
await this.#heartbeatPub.publish(heartbeatChannel(this.#effectivePrefix()), JSON.stringify(beat));
|
|
247
|
+
}
|
|
248
|
+
/** Publish a synthetic FAILED StepResult for a task that failed at the infrastructure level (see the
|
|
249
|
+
* `handle()` bridge). Correlated purely by runId/seq/stepId (StepResult has no `name`). Best-effort. */
|
|
250
|
+
async #bridgeTaskFailure(identity, reason) {
|
|
251
|
+
const result = {
|
|
252
|
+
runId: identity.runId,
|
|
253
|
+
seq: identity.seq,
|
|
254
|
+
stepId: identity.stepId,
|
|
255
|
+
status: 'failed',
|
|
256
|
+
error: { message: `remote step worker failed: ${reason}`, retryable: true },
|
|
257
|
+
};
|
|
258
|
+
try {
|
|
259
|
+
await this.#queue(resultsName(this.#effectivePrefix())).add('result', result, jobOptions());
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
this.#onError(err); // best-effort — the engine's reconcile path is the last resort
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
// ---------------------------------------------------------------------------
|
|
266
|
+
// worker → engine — the engine consumes results / decisions / step events / heartbeats
|
|
267
|
+
// ---------------------------------------------------------------------------
|
|
268
|
+
onResult(handler) {
|
|
269
|
+
if (this.#resultsWorker)
|
|
270
|
+
return;
|
|
271
|
+
this.#resultsWorker = this.#deps.makeWorker(resultsName(this.#effectivePrefix()), (job) => handler(job.data));
|
|
272
|
+
}
|
|
273
|
+
onDecision(handler) {
|
|
274
|
+
if (this.#decisionsWorker)
|
|
275
|
+
return;
|
|
276
|
+
this.#decisionsWorker = this.#deps.makeWorker(decisionsName(this.#effectivePrefix()), (job) => handler(job.data));
|
|
277
|
+
}
|
|
278
|
+
/** workflow worker → engine: stream a local step's lifecycle on `${P}-step-events` (job `stepEvent`),
|
|
279
|
+
* point-to-point so one engine instance checkpoints each once. */
|
|
280
|
+
async dispatchStepEvent(event) {
|
|
281
|
+
await this.#queue(stepEventsName(this.#effectivePrefix())).add('stepEvent', event, jobOptions());
|
|
282
|
+
}
|
|
283
|
+
onStepEvent(handler) {
|
|
284
|
+
if (this.#stepEventsWorker)
|
|
285
|
+
return;
|
|
286
|
+
this.#stepEventsWorker = this.#deps.makeWorker(stepEventsName(this.#effectivePrefix()), (job) => handler(job.data));
|
|
287
|
+
}
|
|
288
|
+
onHeartbeat(handler) {
|
|
289
|
+
if (this.#heartbeatSub)
|
|
290
|
+
return; // one subscription per transport
|
|
291
|
+
const sub = this.#deps.makeRedis();
|
|
292
|
+
this.#heartbeatSub = sub;
|
|
293
|
+
void sub.subscribe(heartbeatChannel(this.#effectivePrefix()));
|
|
294
|
+
sub.on('message', ((_channel, payload) => {
|
|
295
|
+
try {
|
|
296
|
+
void handler(JSON.parse(payload));
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
/* ignore malformed heartbeats */
|
|
300
|
+
}
|
|
301
|
+
}));
|
|
302
|
+
}
|
|
303
|
+
// ---------------------------------------------------------------------------
|
|
304
|
+
// worker liveness registry — `${P}-worker-heartbeat:${token}:${instanceId}` (SET … EX 35)
|
|
305
|
+
// ---------------------------------------------------------------------------
|
|
306
|
+
/** Refresh EVERY handled routing token's TTL'd liveness key on ONE shared 10s interval until
|
|
307
|
+
* `close()`. A newly added token beats immediately so a freshly-registered handler is visible
|
|
308
|
+
* without waiting a full interval. Best-effort: a failed refresh is swallowed (the key then expires,
|
|
309
|
+
* and that gap is itself the signal). */
|
|
310
|
+
#startWorkerHeartbeat(token) {
|
|
311
|
+
const client = this.#workerRedisClient();
|
|
312
|
+
const beatOne = (t) => {
|
|
313
|
+
const key = workerHeartbeatKey(this.#effectivePrefix(), t, this.#instanceId);
|
|
314
|
+
void client
|
|
315
|
+
.set(key, heartbeatKeyValue(), 'EX', WORKER_HEARTBEAT_TTL_SECONDS)
|
|
316
|
+
.catch((err) => this.#onError(err));
|
|
317
|
+
};
|
|
318
|
+
const isNew = !this.#heartbeatTokens.has(token);
|
|
319
|
+
this.#heartbeatTokens.add(token);
|
|
320
|
+
if (isNew)
|
|
321
|
+
beatOne(token);
|
|
322
|
+
if (this.#workerHeartbeatTimer)
|
|
323
|
+
return;
|
|
324
|
+
this.#workerHeartbeatTimer = setInterval(() => {
|
|
325
|
+
for (const t of this.#heartbeatTokens)
|
|
326
|
+
beatOne(t);
|
|
327
|
+
}, WORKER_HEARTBEAT_INTERVAL_MS);
|
|
328
|
+
// Don't keep the event loop alive just for the heartbeat.
|
|
329
|
+
this.#workerHeartbeatTimer.unref?.();
|
|
330
|
+
}
|
|
331
|
+
/** Distinct routing tokens with a live worker heartbeat, discovered by SCANning the heartbeat
|
|
332
|
+
* keyspace (never KEYS — it blocks Redis). A key is `${P}-worker-heartbeat:<token>:<instance>`, and
|
|
333
|
+
* neither segment carries a `:`, so the token is the segment between the fixed prefix and the next `:`. */
|
|
334
|
+
async listWorkerGroups() {
|
|
335
|
+
const client = this.#workerRedisClient();
|
|
336
|
+
const prefix = workerHeartbeatKeyPrefix(this.#effectivePrefix());
|
|
337
|
+
const groups = new Set();
|
|
338
|
+
let cursor = '0';
|
|
339
|
+
do {
|
|
340
|
+
const [next, keys] = await client.scan(cursor, 'MATCH', `${prefix}*`, 'COUNT', 100);
|
|
341
|
+
cursor = next;
|
|
342
|
+
for (const key of keys) {
|
|
343
|
+
const rest = key.slice(prefix.length);
|
|
344
|
+
const sep = rest.indexOf(':');
|
|
345
|
+
const group = sep === -1 ? rest : rest.slice(0, sep);
|
|
346
|
+
if (group)
|
|
347
|
+
groups.add(group);
|
|
348
|
+
}
|
|
349
|
+
} while (cursor !== '0');
|
|
350
|
+
return [...groups];
|
|
351
|
+
}
|
|
352
|
+
/** The LIVE worker descriptors advertised on `token` — SCAN the `${P}-worker-descriptor:<token>:*`
|
|
353
|
+
* keys a worker runtime publishes (design §7.2), GET + parse each. Degrades gracefully to `[]`: a
|
|
354
|
+
* missing/unreadable/malformed key is skipped (never throws), so a broker without descriptor
|
|
355
|
+
* advertisement (or a scan that races a key's expiry) reads as "no descriptors" and the control-plane
|
|
356
|
+
* falls back to legacy assume-compatible dispatch. Never uses KEYS (it blocks Redis). */
|
|
357
|
+
async listWorkerDescriptors(token) {
|
|
358
|
+
const client = this.#workerRedisClient();
|
|
359
|
+
const prefix = workerDescriptorTokenPrefix(this.#effectivePrefix(), token);
|
|
360
|
+
const descriptors = [];
|
|
361
|
+
try {
|
|
362
|
+
let cursor = '0';
|
|
363
|
+
do {
|
|
364
|
+
const [next, keys] = await client.scan(cursor, 'MATCH', `${prefix}*`, 'COUNT', 100);
|
|
365
|
+
cursor = next;
|
|
366
|
+
for (const key of keys) {
|
|
367
|
+
const raw = await client.get(key);
|
|
368
|
+
if (raw == null)
|
|
369
|
+
continue; // expired between SCAN and GET — just skip it
|
|
370
|
+
try {
|
|
371
|
+
descriptors.push(JSON.parse(raw));
|
|
372
|
+
}
|
|
373
|
+
catch {
|
|
374
|
+
/* a malformed descriptor value never blocks routing — treat it as absent */
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
} while (cursor !== '0');
|
|
378
|
+
}
|
|
379
|
+
catch (err) {
|
|
380
|
+
// A backend that can't SCAN/GET the descriptor keyspace degrades to "no descriptors" — the
|
|
381
|
+
// engine then dispatches legacy assume-compatible rather than parking every run.
|
|
382
|
+
this.#onError(err);
|
|
383
|
+
return [];
|
|
384
|
+
}
|
|
385
|
+
return descriptors;
|
|
386
|
+
}
|
|
387
|
+
/** Per-group worker-health: queue depth (waiting+active+delayed+prioritized) + live workers (their
|
|
388
|
+
* non-expired heartbeat keys). `group` is a routing token — the same value a dispatch targets. */
|
|
389
|
+
async groupHealth(group) {
|
|
390
|
+
const counts = await this.#queue(tasksName(this.#effectivePrefix(), group)).getJobCounts('waiting', 'active', 'delayed', 'prioritized');
|
|
391
|
+
const depth = Object.values(counts).reduce((sum, n) => sum + (n ?? 0), 0);
|
|
392
|
+
return { group, depth, liveWorkers: await this.#listLiveWorkers(group) };
|
|
393
|
+
}
|
|
394
|
+
async #listLiveWorkers(group) {
|
|
395
|
+
const client = this.#workerRedisClient();
|
|
396
|
+
const keyPrefix = workerHeartbeatKey(this.#effectivePrefix(), group, '');
|
|
397
|
+
const match = `${keyPrefix}*`;
|
|
398
|
+
const workers = [];
|
|
399
|
+
let cursor = '0';
|
|
400
|
+
do {
|
|
401
|
+
const [next, keys] = await client.scan(cursor, 'MATCH', match, 'COUNT', 100);
|
|
402
|
+
cursor = next;
|
|
403
|
+
for (const key of keys) {
|
|
404
|
+
const raw = await client.get(key);
|
|
405
|
+
workers.push({
|
|
406
|
+
group,
|
|
407
|
+
instanceId: key.slice(keyPrefix.length),
|
|
408
|
+
lastBeatAt: parseHeartbeatValue(raw).lastBeatAt,
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
} while (cursor !== '0');
|
|
412
|
+
return workers;
|
|
413
|
+
}
|
|
414
|
+
// ---------------------------------------------------------------------------
|
|
415
|
+
// P4 — store-less read/control/start protocol (spec §6.2, §8)
|
|
416
|
+
//
|
|
417
|
+
// start-run & run-request ride BullMQ QUEUES (durable, single consumer on the control plane); the DTO
|
|
418
|
+
// is the raw job data (no envelope), job name `startRun`/`runRequest`, options `jobOptions()`
|
|
419
|
+
// (removeOnComplete/removeOnFail: true) — byte-identical to aviary. run-reply & tenant-events ride
|
|
420
|
+
// Redis PUB/SUB (fan-out) over the transport's own `makeRedis()` clients — the SAME substrate the
|
|
421
|
+
// shipped `RedisControlPlane` uses (spec §6 note: fan-out "rides the shared ControlPlane pub/sub"),
|
|
422
|
+
// kept on the transport so the whole P4 wire surface is one cohesive object and every channel name
|
|
423
|
+
// lives with the rest of the naming contract in `naming.ts`.
|
|
424
|
+
// ---------------------------------------------------------------------------
|
|
425
|
+
/** tenant → control plane: enqueue a {@link StartRunMessage} on `${P}-start-run` (job `startRun`). */
|
|
426
|
+
async dispatchStartRun(msg) {
|
|
427
|
+
await this.#queue(startRunName(this.#effectivePrefix())).add('startRun', msg, jobOptions());
|
|
428
|
+
}
|
|
429
|
+
/** control plane ← tenant: consume {@link StartRunMessage}s. Idempotent — a second call is a no-op. */
|
|
430
|
+
onStartRun(handler) {
|
|
431
|
+
if (this.#startRunWorker)
|
|
432
|
+
return;
|
|
433
|
+
this.#startRunWorker = this.#deps.makeWorker(startRunName(this.#effectivePrefix()), (job) => handler(job.data));
|
|
434
|
+
}
|
|
435
|
+
/** tenant → control plane: enqueue a {@link RunRequest} on `${P}-run-request` (job `runRequest`). */
|
|
436
|
+
async dispatchRunRequest(msg) {
|
|
437
|
+
await this.#queue(runRequestName(this.#effectivePrefix())).add('runRequest', msg, jobOptions());
|
|
438
|
+
}
|
|
439
|
+
/** control plane ← tenant: consume {@link RunRequest}s. Idempotent — a second call is a no-op. */
|
|
440
|
+
onRunRequest(handler) {
|
|
441
|
+
if (this.#runRequestWorker)
|
|
442
|
+
return;
|
|
443
|
+
this.#runRequestWorker = this.#deps.makeWorker(runRequestName(this.#effectivePrefix()), (job) => handler(job.data));
|
|
444
|
+
}
|
|
445
|
+
/** control plane → tenant: publish a correlated {@link RunReply} on `${P}-run-reply` (pub/sub). */
|
|
446
|
+
async publishRunReply(reply) {
|
|
447
|
+
if (!this.#runReplyPub)
|
|
448
|
+
this.#runReplyPub = this.#deps.makeRedis();
|
|
449
|
+
await this.#runReplyPub.publish(runReplyChannel(this.#effectivePrefix()), JSON.stringify(reply));
|
|
450
|
+
}
|
|
451
|
+
/** tenant ← control plane: consume {@link RunReply}s (the proxy filters by `requestId`). Idempotent. */
|
|
452
|
+
onRunReply(handler) {
|
|
453
|
+
if (this.#runReplySub)
|
|
454
|
+
return; // one subscription per transport
|
|
455
|
+
const channel = runReplyChannel(this.#effectivePrefix());
|
|
456
|
+
const sub = this.#deps.makeRedis();
|
|
457
|
+
this.#runReplySub = sub;
|
|
458
|
+
void sub.subscribe(channel);
|
|
459
|
+
sub.on('message', ((msgChannel, payload) => {
|
|
460
|
+
// Filter by channel: a shared pub/sub connection may also carry other channels' messages.
|
|
461
|
+
if (msgChannel !== channel)
|
|
462
|
+
return;
|
|
463
|
+
try {
|
|
464
|
+
handler(JSON.parse(payload));
|
|
465
|
+
}
|
|
466
|
+
catch {
|
|
467
|
+
/* ignore malformed run replies */
|
|
468
|
+
}
|
|
469
|
+
}));
|
|
470
|
+
}
|
|
471
|
+
/** control plane → tenant: re-publish a lifecycle {@link TenantEvent} on `${P}-tenant-events-<tenant>`. */
|
|
472
|
+
async publishTenantEvent(evt) {
|
|
473
|
+
if (!this.#tenantEventPub)
|
|
474
|
+
this.#tenantEventPub = this.#deps.makeRedis();
|
|
475
|
+
await this.#tenantEventPub.publish(tenantEventsChannel(this.#effectivePrefix(), evt.tenant), JSON.stringify(evt));
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* tenant ← control plane: subscribe to THIS tenant's event channel. Returns an unsubscribe fn that
|
|
479
|
+
* detaches only this handler (the shared subscriber connection and channel subscription stay for any
|
|
480
|
+
* other listeners; both are torn down in {@link close}). Several tenants — or several callers for one
|
|
481
|
+
* tenant — may subscribe independently, each routed by channel through the ONE shared subscriber.
|
|
482
|
+
*/
|
|
483
|
+
onTenantEvent(tenant, handler) {
|
|
484
|
+
const channel = tenantEventsChannel(this.#effectivePrefix(), tenant);
|
|
485
|
+
const sub = this.#tenantEventSubscriber();
|
|
486
|
+
if (!this.#tenantEventChannels.has(channel)) {
|
|
487
|
+
this.#tenantEventChannels.add(channel);
|
|
488
|
+
void sub.subscribe(channel);
|
|
489
|
+
}
|
|
490
|
+
let handlers = this.#tenantEventHandlers.get(channel);
|
|
491
|
+
if (!handlers) {
|
|
492
|
+
handlers = new Set();
|
|
493
|
+
this.#tenantEventHandlers.set(channel, handlers);
|
|
494
|
+
}
|
|
495
|
+
handlers.add(handler);
|
|
496
|
+
return () => {
|
|
497
|
+
this.#tenantEventHandlers.get(channel)?.delete(handler);
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
/** The ONE shared tenant-events subscriber, created lazily with a single channel-routing listener. */
|
|
501
|
+
#tenantEventSubscriber() {
|
|
502
|
+
if (this.#tenantEventSub)
|
|
503
|
+
return this.#tenantEventSub;
|
|
504
|
+
const sub = this.#deps.makeRedis();
|
|
505
|
+
this.#tenantEventSub = sub;
|
|
506
|
+
sub.on('message', ((msgChannel, payload) => {
|
|
507
|
+
const handlers = this.#tenantEventHandlers.get(msgChannel);
|
|
508
|
+
if (!handlers || handlers.size === 0)
|
|
509
|
+
return;
|
|
510
|
+
let evt;
|
|
511
|
+
try {
|
|
512
|
+
evt = JSON.parse(payload);
|
|
513
|
+
}
|
|
514
|
+
catch {
|
|
515
|
+
return; // ignore malformed tenant events
|
|
516
|
+
}
|
|
517
|
+
for (const h of handlers)
|
|
518
|
+
h(evt);
|
|
519
|
+
}));
|
|
520
|
+
return sub;
|
|
521
|
+
}
|
|
522
|
+
// ---------------------------------------------------------------------------
|
|
523
|
+
// shutdown
|
|
524
|
+
// ---------------------------------------------------------------------------
|
|
525
|
+
/** Close all workers/queues and disconnect pub/sub + heartbeat connections so the process can exit. */
|
|
526
|
+
async close() {
|
|
527
|
+
if (this.#workerHeartbeatTimer)
|
|
528
|
+
clearInterval(this.#workerHeartbeatTimer);
|
|
529
|
+
this.#workerHeartbeatTimer = undefined;
|
|
530
|
+
await Promise.all([...this.#taskWorkers.values()].map((w) => w.close()));
|
|
531
|
+
await this.#resultsWorker?.close();
|
|
532
|
+
await this.#decisionsWorker?.close();
|
|
533
|
+
await this.#stepEventsWorker?.close();
|
|
534
|
+
await this.#startRunWorker?.close();
|
|
535
|
+
await this.#runRequestWorker?.close();
|
|
536
|
+
await Promise.all([...this.#queues.values()].map((q) => q.close()));
|
|
537
|
+
this.#heartbeatPub?.disconnect();
|
|
538
|
+
this.#heartbeatSub?.disconnect();
|
|
539
|
+
this.#workerRedis?.disconnect();
|
|
540
|
+
this.#runReplyPub?.disconnect();
|
|
541
|
+
this.#runReplySub?.disconnect();
|
|
542
|
+
this.#tenantEventPub?.disconnect();
|
|
543
|
+
this.#tenantEventSub?.disconnect();
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
/**
|
|
547
|
+
* Narrow a FAILED task job's `data` to the {@link RemoteTask} fields the terminal-failure bridge needs
|
|
548
|
+
* (runId/seq/stepId), WITHOUT an unsafe cast. Returns `undefined` for a payload BullMQ has already
|
|
549
|
+
* GC'd or a malformed job — so the bridge safely no-ops instead of publishing a bogus result.
|
|
550
|
+
*/
|
|
551
|
+
function failedTaskIdentity(data) {
|
|
552
|
+
if (typeof data !== 'object' || data === null)
|
|
553
|
+
return undefined;
|
|
554
|
+
if (!('runId' in data) || !('seq' in data) || !('stepId' in data))
|
|
555
|
+
return undefined;
|
|
556
|
+
const { runId, seq, stepId } = data;
|
|
557
|
+
if (typeof runId !== 'string' || typeof seq !== 'number' || typeof stepId !== 'string') {
|
|
558
|
+
return undefined;
|
|
559
|
+
}
|
|
560
|
+
return { runId, seq, stepId };
|
|
561
|
+
}
|
|
562
|
+
//# sourceMappingURL=bullmq-transport.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bullmq-transport.js","sourceRoot":"","sources":["../../../../src/transports/bullmq/bullmq-transport.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAoB,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAA4B,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAElF,OAAO,EACL,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,eAAe,EACf,cAAc,EACd,YAAY,EACZ,cAAc,EACd,SAAS,EACT,mBAAmB,EACnB,2BAA2B,EAC3B,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,4BAA4B,EAC5B,4BAA4B,EAC5B,eAAe,EACf,iBAAiB,EACjB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,MAAM,oBAAoB,CAAC;AAyB5B;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,eAAe;IACjB,KAAK,CAAa;IAClB,UAAU,CAAqB;IAC/B,OAAO,CAAS;IACzB,UAAU,CAAqB;IACtB,kBAAkB,CAAU;IAC5B,WAAW,CAAS;IACpB,QAAQ,CAAyB;IAEjC,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IACpD,qGAAqG;IAC5F,cAAc,GAAG,IAAI,GAAG,EAA+B,CAAC;IACjE,gGAAgG;IACvF,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;IACvC,YAAY,GAAG,IAAI,GAAG,EAAsB,CAAC;IACtD,cAAc,CAAyB;IACvC,gBAAgB,CAAyB;IACzC,iBAAiB,CAAyB;IAE1C,sFAAsF;IACtF,aAAa,CAAwB;IACrC,aAAa,CAAwB;IAErC,mGAAmG;IACnG,YAAY,CAAwB;IACpC,qBAAqB,CAA6C;IACzD,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE9C,yGAAyG;IACzG,eAAe,CAAyB;IACxC,iBAAiB,CAAyB;IAC1C,YAAY,CAAwB;IACpC,YAAY,CAAwB;IACpC,eAAe,CAAwB;IACvC,8FAA8F;IAC9F,mGAAmG;IACnG,eAAe,CAAwB;IAC9B,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,oBAAoB,GAAG,IAAI,GAAG,EAA2C,CAAC;IAEnF,YAAY,OAA+B;QACzC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;QACpC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;QAC3C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;QACpC,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC;QAC1D,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,eAAe,EAAE,CAAC;QAC3D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,iEAAiE;IACjE,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAAiB;QAC5B,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,gBAAgB;QACd,OAAO,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,IAAY;QACjB,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,kBAAkB;QAChB,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACnE,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACH,qBAAqB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;IAChC,CAAC;IAED,8EAA8E;IAC9E,kBAAkB;IAClB,8EAA8E;IAE9E;+EAC2E;IAC3E,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAC7B,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CACnE,MAAM,EACN,IAAI,EACJ,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED;2GACuG;IACvG,KAAK,CAAC,oBAAoB,CAAC,IAAkB;QAC3C,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CACnE,UAAU,EACV,IAAI,EACJ,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,sEAAsE;IACtE,8EAA8E;IAE9E;;;;OAIG;IACH,MAAM,CAAC,IAAY,EAAE,EAAe;QAClC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC7B,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;;OAMG;IACH,cAAc,CAAC,IAAY,EAAE,IAAyB;QACpD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED;+GAC2G;IAC3G,iBAAiB,CAAC,KAAa;QAC7B,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CACtF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CACvB,CAAC;QACF,8FAA8F;QAC9F,sFAAsF;QACtF,4FAA4F;QAC5F,kGAAkG;QAClG,mGAAmG;QACnG,kGAAkG;QAClG,oGAAoG;QACpG,+FAA+F;QAC/F,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACjC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC/C,IAAI,CAAC,QAAQ;gBAAE,OAAO,CAAC,+EAA+E;YACtG,KAAK,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,GAAG,EAAE,YAAY,IAAI,KAAK,EAAE,OAAO,IAAI,SAAS,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED;4GACwG;IACxG,KAAK,CAAC,OAAO,CAAC,IAAa;QACzB,IAAI,cAAc,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAkB,CAAC,CAAC;IAC3C,CAAC;IAED;;uGAEmG;IACnG,KAAK,CAAC,gBAAgB,CAAC,IAAkB;QACvC,IAAI,QAA0B,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpD,QAAQ,GAAG,IAAI;gBACb,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC;gBAClB,CAAC,CAAC;oBACE,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,QAAQ;oBAChB,QAAQ,EAAE,EAAE;oBACZ,KAAK,EAAE,EAAE,OAAO,EAAE,+BAA+B,IAAI,CAAC,QAAQ,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE;iBACzF,CAAC;QACR,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,kGAAkG;YAClG,kFAAkF;YAClF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACnB,QAAQ,GAAG;gBACT,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,QAAQ;gBAChB,QAAQ,EAAE,EAAE;gBACZ,KAAK,EAAE,EAAE,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aACrE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,GAAG,CAC3D,UAAU,EACV,QAAQ,EACR,UAAU,EAAE,CACb,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mGAAmG;YACnG,yFAAyF;YACzF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACnB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAC7B,IAAI,MAAkB,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,8FAA8F;YAC9F,gGAAgG;YAChG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACnB,MAAM,GAAG;gBACP,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,QAAQ;gBAChB,KAAK,EAAE;oBACL,OAAO,EAAE,uDAAuD;oBAChE,SAAS,EAAE,IAAI;iBAChB;aACF,CAAC;QACJ,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC9F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wFAAwF;YACxF,+FAA+F;YAC/F,4EAA4E;YAC5E,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACnB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;oGACgG;IAChG,KAAK,CAAC,SAAS,CAAC,IAAe;QAC7B,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACrE,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAC9B,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EACzC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CACrB,CAAC;IACJ,CAAC;IAED;6GACyG;IACzG,KAAK,CAAC,kBAAkB,CACtB,QAAwD,EACxD,MAAc;QAEd,MAAM,MAAM,GAAe;YACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,EAAE,OAAO,EAAE,8BAA8B,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;SAC5E,CAAC;QACF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC9F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,+DAA+D;QACrF,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,uFAAuF;IACvF,8EAA8E;IAE9E,QAAQ,CAAC,OAA8C;QACrD,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO;QAChC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CACxF,OAAO,CAAC,GAAG,CAAC,IAAkB,CAAC,CAChC,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAsD;QAC/D,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAClC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAC5F,OAAO,CAAC,GAAG,CAAC,IAAwB,CAAC,CACtC,CAAC;IACJ,CAAC;IAED;uEACmE;IACnE,KAAK,CAAC,iBAAiB,CAAC,KAAwB;QAC9C,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,GAAG,CAC5D,WAAW,EACX,KAAK,EACL,UAAU,EAAE,CACb,CAAC;IACJ,CAAC;IAED,WAAW,CAAC,OAAoD;QAC9D,IAAI,IAAI,CAAC,iBAAiB;YAAE,OAAO;QACnC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAC9F,OAAO,CAAC,GAAG,CAAC,IAAyB,CAAC,CACvC,CAAC;IACJ,CAAC;IAED,WAAW,CAAC,OAA2C;QACrD,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO,CAAC,iCAAiC;QACjE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;QACzB,KAAK,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QAC9D,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,QAAgB,EAAE,OAAe,EAAE,EAAE;YACvD,IAAI,CAAC;gBACH,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;QACH,CAAC,CAAU,CAAC,CAAC;IACf,CAAC;IAED,8EAA8E;IAC9E,0FAA0F;IAC1F,8EAA8E;IAE9E;;;8CAG0C;IAC1C,qBAAqB,CAAC,KAAa;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,CAAC,CAAS,EAAQ,EAAE;YAClC,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7E,KAAK,MAAM;iBACR,GAAG,CAAC,GAAG,EAAE,iBAAiB,EAAE,EAAE,IAAI,EAAE,4BAA4B,CAAC;iBACjE,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,CAAC,CAAC;QACF,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,KAAK;YAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,qBAAqB;YAAE,OAAO;QACvC,IAAI,CAAC,qBAAqB,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,gBAAgB;gBAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC,EAAE,4BAA4B,CAAC,CAAC;QACjC,0DAA0D;QAC1D,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,EAAE,CAAC;IACvC,CAAC;IAED;;gHAE4G;IAC5G,KAAK,CAAC,gBAAgB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,wBAAwB,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;QACjC,IAAI,MAAM,GAAG,GAAG,CAAC;QACjB,GAAG,CAAC;YACF,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YACpF,MAAM,GAAG,IAAI,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACtC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBACrD,IAAI,KAAK;oBAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC,QAAQ,MAAM,KAAK,GAAG,EAAE;QACzB,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;IACrB,CAAC;IAED;;;;8FAI0F;IAC1F,KAAK,CAAC,qBAAqB,CAAC,KAAa;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,2BAA2B,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,KAAK,CAAC,CAAC;QAC3E,MAAM,WAAW,GAAuB,EAAE,CAAC;QAC3C,IAAI,CAAC;YACH,IAAI,MAAM,GAAG,GAAG,CAAC;YACjB,GAAG,CAAC;gBACF,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;gBACpF,MAAM,GAAG,IAAI,CAAC;gBACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,GAAG,IAAI,IAAI;wBAAE,SAAS,CAAC,8CAA8C;oBACzE,IAAI,CAAC;wBACH,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAqB,CAAC,CAAC;oBACxD,CAAC;oBAAC,MAAM,CAAC;wBACP,4EAA4E;oBAC9E,CAAC;gBACH,CAAC;YACH,CAAC,QAAQ,MAAM,KAAK,GAAG,EAAE;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,2FAA2F;YAC3F,iFAAiF;YACjF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;uGACmG;IACnG,KAAK,CAAC,WAAW,CAAC,KAAa;QAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,CACtF,SAAS,EACT,QAAQ,EACR,SAAS,EACT,aAAa,CACd,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1E,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,KAAa;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACzE,MAAM,KAAK,GAAG,GAAG,SAAS,GAAG,CAAC;QAC9B,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,IAAI,MAAM,GAAG,GAAG,CAAC;QACjB,GAAG,CAAC;YACF,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YAC7E,MAAM,GAAG,IAAI,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,UAAU,EAAE,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC;oBACvC,UAAU,EAAE,mBAAmB,CAAC,GAAG,CAAC,CAAC,UAAU;iBAChD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,QAAQ,MAAM,KAAK,GAAG,EAAE;QACzB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,8EAA8E;IAC9E,8DAA8D;IAC9D,EAAE;IACF,sGAAsG;IACtG,8FAA8F;IAC9F,mGAAmG;IACnG,kGAAkG;IAClG,oGAAoG;IACpG,mGAAmG;IACnG,6DAA6D;IAC7D,8EAA8E;IAE9E,sGAAsG;IACtG,KAAK,CAAC,gBAAgB,CAAC,GAAoB;QACzC,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,uGAAuG;IACvG,UAAU,CAAC,OAAgD;QACzD,IAAI,IAAI,CAAC,eAAe;YAAE,OAAO;QACjC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAC1F,OAAO,CAAC,GAAG,CAAC,IAAuB,CAAC,CACrC,CAAC;IACJ,CAAC;IAED,qGAAqG;IACrG,KAAK,CAAC,kBAAkB,CAAC,GAAe;QACtC,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IAClG,CAAC;IAED,kGAAkG;IAClG,YAAY,CAAC,OAA2C;QACtD,IAAI,IAAI,CAAC,iBAAiB;YAAE,OAAO;QACnC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAC9F,OAAO,CAAC,GAAG,CAAC,IAAkB,CAAC,CAChC,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,KAAK,CAAC,eAAe,CAAC,KAAe;QACnC,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACnE,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAC7B,eAAe,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EACxC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CACtB,CAAC;IACJ,CAAC;IAED,wGAAwG;IACxG,UAAU,CAAC,OAAkC;QAC3C,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,CAAC,iCAAiC;QAChE,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;QACxB,KAAK,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,UAAkB,EAAE,OAAe,EAAE,EAAE;YACzD,0FAA0F;YAC1F,IAAI,UAAU,KAAK,OAAO;gBAAE,OAAO;YACnC,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAa,CAAC,CAAC;YAC3C,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;QACH,CAAC,CAAU,CAAC,CAAC;IACf,CAAC;IAED,2GAA2G;IAC3G,KAAK,CAAC,kBAAkB,CAAC,GAAgB;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACzE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAChC,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,EACxD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CACpB,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,aAAa,CAAC,MAAc,EAAE,OAAmC;QAC/D,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,MAAM,CAAC,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACvC,KAAK,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACnD,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACtB,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAC1D,CAAC,CAAC;IACJ,CAAC;IAED,sGAAsG;IACtG,sBAAsB;QACpB,IAAI,IAAI,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC,eAAe,CAAC;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;QAC3B,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,UAAkB,EAAE,OAAe,EAAE,EAAE;YACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;gBAAE,OAAO;YAC7C,IAAI,GAAgB,CAAC;YACrB,IAAI,CAAC;gBACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAgB,CAAC;YAC3C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,iCAAiC;YAC3C,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,QAAQ;gBAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC,CAAU,CAAC,CAAC;QACb,OAAO,GAAG,CAAC;IACb,CAAC;IAED,8EAA8E;IAC9E,WAAW;IACX,8EAA8E;IAE9E,uGAAuG;IACvG,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,qBAAqB;YAAE,aAAa,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAC1E,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;QACvC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACzE,MAAM,IAAI,CAAC,cAAc,EAAE,KAAK,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,iBAAiB,EAAE,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QACpC,MAAM,IAAI,CAAC,iBAAiB,EAAE,KAAK,EAAE,CAAC;QACtC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;QACjC,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;QACjC,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,CAAC;QAChC,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,CAAC;QAChC,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,EAAE,UAAU,EAAE,CAAC;QACnC,IAAI,CAAC,eAAe,EAAE,UAAU,EAAE,CAAC;IACrC,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CACzB,IAAa;IAEb,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAChE,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACpF,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,IAA+B,CAAC;IAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QACvF,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AAChC,CAAC"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The tiny broker surface the {@link import('./bullmq-transport.js').BullMQTransport} depends on, and
|
|
3
|
+
* the real-`bullmq` implementation of it. Splitting the concrete broker behind these structural
|
|
4
|
+
* interfaces is what keeps the transport itself PURE and unit-testable with fakes — and lets the
|
|
5
|
+
* package build/test without `bullmq` installed, since the only place that touches the real dependency
|
|
6
|
+
* is {@link createBullMQDeps}, which `import()`s it LAZILY (so a non-bullmq fleet never loads it).
|
|
7
|
+
*/
|
|
8
|
+
/** A BullMQ job as this transport reads it — only the fields it uses. `data` is the raw DTO. */
|
|
9
|
+
export interface JobLike {
|
|
10
|
+
data: unknown;
|
|
11
|
+
failedReason?: string | undefined;
|
|
12
|
+
}
|
|
13
|
+
/** The processor a task/result/decision/step-event worker runs per job. */
|
|
14
|
+
export type ProcessFn = (job: JobLike) => Promise<unknown>;
|
|
15
|
+
/** The subset of a BullMQ `Queue` this transport uses. */
|
|
16
|
+
export interface QueueLike {
|
|
17
|
+
add(name: string, data: unknown, opts?: unknown): Promise<unknown>;
|
|
18
|
+
getJobCounts(...types: string[]): Promise<Record<string, number>>;
|
|
19
|
+
close(): Promise<unknown>;
|
|
20
|
+
}
|
|
21
|
+
/** The subset of a BullMQ `Worker` this transport uses. */
|
|
22
|
+
export interface WorkerLike {
|
|
23
|
+
on(event: 'failed', listener: (job: JobLike | undefined, err: Error) => void): unknown;
|
|
24
|
+
close(): Promise<unknown>;
|
|
25
|
+
}
|
|
26
|
+
/** The subset of an ioredis client this transport uses (pub/sub + the worker-heartbeat keyspace). */
|
|
27
|
+
export interface RedisLike {
|
|
28
|
+
publish(channel: string, message: string): Promise<unknown> | unknown;
|
|
29
|
+
subscribe(channel: string): Promise<unknown> | unknown;
|
|
30
|
+
on(event: string, listener: (...args: never[]) => void): unknown;
|
|
31
|
+
set(key: string, value: string, mode: 'EX', ttl: number): Promise<unknown>;
|
|
32
|
+
scan(cursor: string | number, ...args: (string | number)[]): Promise<[string, string[]]>;
|
|
33
|
+
get(key: string): Promise<string | null>;
|
|
34
|
+
disconnect(): void;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* The broker factory the transport is constructed with. Each `make*` mints a fresh broker object
|
|
38
|
+
* bound to the configured connection. The default (real bullmq/ioredis) implementation comes from
|
|
39
|
+
* {@link createBullMQDeps}; tests pass an in-memory fake to prove naming + job shape + the heartbeat
|
|
40
|
+
* registry with no Redis.
|
|
41
|
+
*/
|
|
42
|
+
export interface BullMQDeps {
|
|
43
|
+
makeQueue(name: string): QueueLike;
|
|
44
|
+
makeWorker(name: string, process: ProcessFn): WorkerLike;
|
|
45
|
+
/** A fresh standalone Redis client (pub/sub needs its own connection; the heartbeat registry reuses one). */
|
|
46
|
+
makeRedis(): RedisLike;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Build the real bullmq/ioredis-backed {@link BullMQDeps} for `connection` (ioredis `ConnectionOptions`
|
|
50
|
+
* or a `Redis` instance) at a `concurrency`. `bullmq` and `ioredis` are imported LAZILY here — via a
|
|
51
|
+
* non-literal specifier so the package type-checks and unit-tests without them installed — and only
|
|
52
|
+
* this call (i.e. only when the transport is actually selected + run) pulls them in.
|
|
53
|
+
*
|
|
54
|
+
* Workers require `maxRetriesPerRequest: null`; a plain-options connection is cloned with it set (a
|
|
55
|
+
* passed-in `Redis` instance is preserved as-is). A standalone client for pub/sub / heartbeat keys is
|
|
56
|
+
* a `duplicate()` of a passed-in instance, or a fresh client with a keepalive floor from the options.
|
|
57
|
+
*/
|
|
58
|
+
export declare function createBullMQDeps(connection: unknown, concurrency?: number): Promise<BullMQDeps>;
|
|
59
|
+
//# sourceMappingURL=deps.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deps.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/deps.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,gGAAgG;AAChG,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,2EAA2E;AAC3E,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAE3D,0DAA0D;AAC1D,MAAM,WAAW,SAAS;IACxB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnE,YAAY,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC3B;AAED,2DAA2D;AAC3D,MAAM,WAAW,UAAU;IACzB,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,EAAE,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC;IACvF,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC3B;AAED,qGAAqG;AACrG,MAAM,WAAW,SAAS;IACxB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IACtE,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IACvD,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC;IACjE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3E,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IACzF,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACzC,UAAU,IAAI,IAAI,CAAC;CACpB;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,GAAG,UAAU,CAAC;IACzD,6GAA6G;IAC7G,SAAS,IAAI,SAAS,CAAC;CACxB;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,OAAO,EAAE,WAAW,SAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAiChG"}
|