@adonis-agora/durable 0.58.3 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/dist/assets/spa/assets/index-70hGzOnT.css +1 -0
- package/dist/assets/spa/assets/index-BI1EANJO.js +16 -0
- package/dist/assets/spa/assets/index-BI1EANJO.js.map +1 -0
- package/dist/assets/spa/index.html +2 -2
- package/dist/commands/lifecycle_republish.d.ts +23 -0
- package/dist/commands/lifecycle_republish.d.ts.map +1 -0
- package/dist/commands/lifecycle_republish.js +64 -0
- package/dist/commands/lifecycle_republish.js.map +1 -0
- package/dist/commands/main.d.ts +4 -3
- package/dist/commands/main.d.ts.map +1 -1
- package/dist/commands/main.js +3 -1
- package/dist/commands/main.js.map +1 -1
- package/dist/providers/dashboard_provider.d.ts +2 -0
- package/dist/providers/dashboard_provider.d.ts.map +1 -1
- package/dist/providers/dashboard_provider.js +9 -1
- package/dist/providers/dashboard_provider.js.map +1 -1
- package/dist/providers/durable_provider.d.ts.map +1 -1
- package/dist/providers/durable_provider.js +3 -0
- package/dist/providers/durable_provider.js.map +1 -1
- package/dist/src/codec-state-store.d.ts +10 -1
- package/dist/src/codec-state-store.d.ts.map +1 -1
- package/dist/src/codec-state-store.js +25 -0
- package/dist/src/codec-state-store.js.map +1 -1
- package/dist/src/commands/index.d.ts +1 -1
- package/dist/src/commands/index.d.ts.map +1 -1
- package/dist/src/commands/index.js +1 -1
- package/dist/src/commands/index.js.map +1 -1
- package/dist/src/commands/runs.d.ts +24 -1
- package/dist/src/commands/runs.d.ts.map +1 -1
- package/dist/src/commands/runs.js +50 -0
- package/dist/src/commands/runs.js.map +1 -1
- package/dist/src/commands/worker.d.ts.map +1 -1
- package/dist/src/commands/worker.js +3 -0
- package/dist/src/commands/worker.js.map +1 -1
- package/dist/src/dashboard/gateway-adapter.d.ts +5 -0
- package/dist/src/dashboard/gateway-adapter.d.ts.map +1 -1
- package/dist/src/dashboard/gateway-adapter.js +20 -0
- package/dist/src/dashboard/gateway-adapter.js.map +1 -1
- package/dist/src/dashboard/handlers.d.ts +29 -1
- package/dist/src/dashboard/handlers.d.ts.map +1 -1
- package/dist/src/dashboard/handlers.js +103 -0
- package/dist/src/dashboard/handlers.js.map +1 -1
- package/dist/src/dashboard/index.d.ts +2 -2
- package/dist/src/dashboard/index.d.ts.map +1 -1
- package/dist/src/dashboard/index.js +2 -2
- package/dist/src/dashboard/index.js.map +1 -1
- package/dist/src/dashboard/openapi.d.ts.map +1 -1
- package/dist/src/dashboard/openapi.js +39 -0
- package/dist/src/dashboard/openapi.js.map +1 -1
- package/dist/src/define_config.d.ts +13 -0
- package/dist/src/define_config.d.ts.map +1 -1
- package/dist/src/define_config.js.map +1 -1
- package/dist/src/diagnostics-bridge.d.ts +5 -1
- package/dist/src/diagnostics-bridge.d.ts.map +1 -1
- package/dist/src/diagnostics-bridge.js +16 -2
- package/dist/src/diagnostics-bridge.js.map +1 -1
- package/dist/src/engine.d.ts +426 -6
- package/dist/src/engine.d.ts.map +1 -1
- package/dist/src/engine.js +1951 -101
- package/dist/src/engine.js.map +1 -1
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +4 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/event-accumulators.d.ts +28 -5
- package/dist/src/event-accumulators.d.ts.map +1 -1
- package/dist/src/event-accumulators.js +102 -21
- package/dist/src/event-accumulators.js.map +1 -1
- package/dist/src/events.d.ts +25 -0
- package/dist/src/events.d.ts.map +1 -1
- package/dist/src/events.js +24 -0
- package/dist/src/events.js.map +1 -1
- package/dist/src/handshake/announced.d.ts +6 -0
- package/dist/src/handshake/announced.d.ts.map +1 -1
- package/dist/src/handshake/announced.js +20 -0
- package/dist/src/handshake/announced.js.map +1 -1
- package/dist/src/handshake/descriptor.d.ts +6 -0
- package/dist/src/handshake/descriptor.d.ts.map +1 -1
- package/dist/src/handshake/descriptor.js +1 -0
- package/dist/src/handshake/descriptor.js.map +1 -1
- package/dist/src/index.d.ts +3 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/interfaces.d.ts +139 -3
- package/dist/src/interfaces.d.ts.map +1 -1
- package/dist/src/interfaces.js +11 -0
- package/dist/src/interfaces.js.map +1 -1
- package/dist/src/lifecycle-events.d.ts +372 -0
- package/dist/src/lifecycle-events.d.ts.map +1 -0
- package/dist/src/lifecycle-events.js +383 -0
- package/dist/src/lifecycle-events.js.map +1 -0
- package/dist/src/metrics.d.ts +15 -0
- package/dist/src/metrics.d.ts.map +1 -1
- package/dist/src/metrics.js +36 -1
- package/dist/src/metrics.js.map +1 -1
- package/dist/src/otel/durable-metrics.d.ts +9 -0
- package/dist/src/otel/durable-metrics.d.ts.map +1 -1
- package/dist/src/otel/durable-metrics.js +17 -0
- package/dist/src/otel/durable-metrics.js.map +1 -1
- package/dist/src/otel/durable-otel.d.ts +7 -0
- package/dist/src/otel/durable-otel.d.ts.map +1 -1
- package/dist/src/otel/durable-otel.js +69 -3
- package/dist/src/otel/durable-otel.js.map +1 -1
- package/dist/src/run-detail-summary.d.ts +48 -1
- package/dist/src/run-detail-summary.d.ts.map +1 -1
- package/dist/src/run-detail-summary.js +30 -2
- package/dist/src/run-detail-summary.js.map +1 -1
- package/dist/src/run-gateway/interface.d.ts +16 -0
- package/dist/src/run-gateway/interface.d.ts.map +1 -1
- package/dist/src/run-gateway/proxy-run-gateway.d.ts +31 -1
- package/dist/src/run-gateway/proxy-run-gateway.d.ts.map +1 -1
- package/dist/src/run-gateway/proxy-run-gateway.js +59 -7
- package/dist/src/run-gateway/proxy-run-gateway.js.map +1 -1
- package/dist/src/run-gateway/run-request-responder.d.ts +4 -29
- package/dist/src/run-gateway/run-request-responder.d.ts.map +1 -1
- package/dist/src/run-gateway/run-request-responder.js +183 -6
- package/dist/src/run-gateway/run-request-responder.js.map +1 -1
- package/dist/src/run-gateway/store-run-gateway.d.ts +9 -0
- package/dist/src/run-gateway/store-run-gateway.d.ts.map +1 -1
- package/dist/src/run-gateway/store-run-gateway.js +8 -0
- package/dist/src/run-gateway/store-run-gateway.js.map +1 -1
- package/dist/src/run-waiting.d.ts +1 -0
- package/dist/src/run-waiting.d.ts.map +1 -1
- package/dist/src/run-waiting.js +6 -1
- package/dist/src/run-waiting.js.map +1 -1
- package/dist/src/sticky-continuation.d.ts +141 -0
- package/dist/src/sticky-continuation.d.ts.map +1 -0
- package/dist/src/sticky-continuation.js +231 -0
- package/dist/src/sticky-continuation.js.map +1 -0
- package/dist/src/stores/factory.d.ts +6 -0
- package/dist/src/stores/factory.d.ts.map +1 -1
- package/dist/src/stores/factory.js +3 -0
- package/dist/src/stores/factory.js.map +1 -1
- package/dist/src/stores/lucid-pg-hot-path.d.ts +30 -0
- package/dist/src/stores/lucid-pg-hot-path.d.ts.map +1 -0
- package/dist/src/stores/lucid-pg-hot-path.js +67 -0
- package/dist/src/stores/lucid-pg-hot-path.js.map +1 -0
- package/dist/src/stores/lucid.d.ts +30 -1
- package/dist/src/stores/lucid.d.ts.map +1 -1
- package/dist/src/stores/lucid.js +460 -17
- package/dist/src/stores/lucid.js.map +1 -1
- package/dist/src/telescope/dashboard.d.ts.map +1 -1
- package/dist/src/telescope/dashboard.js +50 -0
- package/dist/src/telescope/dashboard.js.map +1 -1
- package/dist/src/telescope/data-providers.d.ts +19 -0
- package/dist/src/telescope/data-providers.d.ts.map +1 -1
- package/dist/src/telescope/data-providers.js +128 -2
- package/dist/src/telescope/data-providers.js.map +1 -1
- package/dist/src/telescope/extension.d.ts.map +1 -1
- package/dist/src/telescope/extension.js +4 -1
- package/dist/src/telescope/extension.js.map +1 -1
- package/dist/src/telescope/index.d.ts +3 -2
- package/dist/src/telescope/index.d.ts.map +1 -1
- package/dist/src/telescope/index.js +2 -1
- package/dist/src/telescope/index.js.map +1 -1
- package/dist/src/telescope/telescope-sdk.d.ts +16 -0
- package/dist/src/telescope/telescope-sdk.d.ts.map +1 -1
- package/dist/src/telescope/watcher.d.ts +66 -0
- package/dist/src/telescope/watcher.d.ts.map +1 -0
- package/dist/src/telescope/watcher.js +185 -0
- package/dist/src/telescope/watcher.js.map +1 -0
- package/dist/src/testing/in-memory-state-store.d.ts +12 -1
- package/dist/src/testing/in-memory-state-store.d.ts.map +1 -1
- package/dist/src/testing/in-memory-state-store.js +109 -1
- package/dist/src/testing/in-memory-state-store.js.map +1 -1
- package/dist/src/testing-kit/harness.d.ts +1 -1
- package/dist/src/testing-kit/harness.d.ts.map +1 -1
- package/dist/src/testing-kit/harness.js +3 -1
- package/dist/src/testing-kit/harness.js.map +1 -1
- package/dist/src/testing-kit/state-store-conformance.d.ts.map +1 -1
- package/dist/src/testing-kit/state-store-conformance.js +258 -0
- package/dist/src/testing-kit/state-store-conformance.js.map +1 -1
- package/dist/src/transport-pool.d.ts +10 -0
- package/dist/src/transport-pool.d.ts.map +1 -1
- package/dist/src/transport-pool.js +23 -2
- package/dist/src/transport-pool.js.map +1 -1
- package/dist/src/transports/bullmq/bullmq-transport.d.ts +41 -1
- package/dist/src/transports/bullmq/bullmq-transport.d.ts.map +1 -1
- package/dist/src/transports/bullmq/bullmq-transport.js +283 -5
- package/dist/src/transports/bullmq/bullmq-transport.js.map +1 -1
- package/dist/src/transports/bullmq/deps.d.ts +12 -1
- package/dist/src/transports/bullmq/deps.d.ts.map +1 -1
- package/dist/src/transports/bullmq/deps.js.map +1 -1
- package/dist/src/transports/bullmq/naming.d.ts +10 -0
- package/dist/src/transports/bullmq/naming.d.ts.map +1 -1
- package/dist/src/transports/bullmq/naming.js +16 -0
- package/dist/src/transports/bullmq/naming.js.map +1 -1
- package/dist/src/transports/db.d.ts +3 -1
- package/dist/src/transports/db.d.ts.map +1 -1
- package/dist/src/transports/db.js +4 -4
- package/dist/src/transports/db.js.map +1 -1
- package/dist/src/transports/queue.d.ts +3 -1
- package/dist/src/transports/queue.d.ts.map +1 -1
- package/dist/src/transports/queue.js +2 -2
- package/dist/src/transports/queue.js.map +1 -1
- package/dist/src/transports/worker-keys.d.ts +2 -1
- package/dist/src/transports/worker-keys.d.ts.map +1 -1
- package/dist/src/transports/worker-keys.js +3 -2
- package/dist/src/transports/worker-keys.js.map +1 -1
- package/dist/src/workflow-ctx.d.ts +7 -0
- package/dist/src/workflow-ctx.d.ts.map +1 -1
- package/dist/src/workflow-ctx.js +90 -1
- package/dist/src/workflow-ctx.js.map +1 -1
- package/dist/src/workflow-discovery.d.ts.map +1 -1
- package/dist/src/workflow-discovery.js +1 -0
- package/dist/src/workflow-discovery.js.map +1 -1
- package/dist/src/workflow-events.d.ts +101 -0
- package/dist/src/workflow-events.d.ts.map +1 -1
- package/dist/src/workflow-events.js +79 -1
- package/dist/src/workflow-events.js.map +1 -1
- package/dist/src/workflow-ref.d.ts +17 -2
- package/dist/src/workflow-ref.d.ts.map +1 -1
- package/dist/src/workflow-ref.js.map +1 -1
- package/package.json +5 -5
- package/skills/durable-cluster/SKILL.md +4 -2
- package/dist/assets/spa/assets/index-6nmSTfT7.css +0 -1
- package/dist/assets/spa/assets/index-Dj0X-IAu.js +0 -16
- package/dist/assets/spa/assets/index-Dj0X-IAu.js.map +0 -1
|
@@ -8,7 +8,17 @@ import type { GroupHealth, Heartbeat, NamedTransport, RemoteTask, StepResult, Tr
|
|
|
8
8
|
export declare class TransportPool {
|
|
9
9
|
private readonly transports;
|
|
10
10
|
private warnedBulkRefusal;
|
|
11
|
+
/** Ids of the transports that route replies to this engine instance (see {@link routeReplies}). */
|
|
12
|
+
private readonly replying;
|
|
11
13
|
constructor(transports: NamedTransport[]);
|
|
14
|
+
/**
|
|
15
|
+
* Have every transport that can ({@link Transport.consumeRepliesFor}) send this engine instance
|
|
16
|
+
* the results of the tasks it dispatches with `replyTo`. Returns whether any transport does.
|
|
17
|
+
*/
|
|
18
|
+
routeReplies(instanceId: string): boolean;
|
|
19
|
+
/** `task` as sent on transport `id`: a reply goes back only on a transport that routes it, so
|
|
20
|
+
* elsewhere `replyTo` is dropped (its result rides the shared results). */
|
|
21
|
+
private withReply;
|
|
12
22
|
get size(): number;
|
|
13
23
|
/** Register the engine's result/heartbeat/step-event handlers on EVERY transport — a result can come
|
|
14
24
|
* back on whichever one delivered the task, so failover stays symmetric. `onStepEvent` is wired only
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transport-pool.d.ts","sourceRoot":"","sources":["../../src/transport-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,cAAc,EACd,UAAU,EACV,UAAU,EACV,SAAS,EACT,eAAe,EACf,iBAAiB,EAClB,MAAM,iBAAiB,CAAC;AAKzB;;;;GAIG;AACH,qBAAa,aAAa;
|
|
1
|
+
{"version":3,"file":"transport-pool.d.ts","sourceRoot":"","sources":["../../src/transport-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,cAAc,EACd,UAAU,EACV,UAAU,EACV,SAAS,EACT,eAAe,EACf,iBAAiB,EAClB,MAAM,iBAAiB,CAAC;AAKzB;;;;GAIG;AACH,qBAAa,aAAa;IAKZ,OAAO,CAAC,QAAQ,CAAC,UAAU;IAJvC,OAAO,CAAC,iBAAiB,CAAS;IAClC,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAE9C,YAA6B,UAAU,EAAE,cAAc,EAAE,EAAI;IAE7D;;;OAGG;IACH,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAKxC;IAED;gFAC4E;IAC5E,OAAO,CAAC,SAAS;IAMjB,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;qGAEiG;IACjG,IAAI,CACF,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,EAC/C,WAAW,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,EAC/C,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,GACxD,IAAI,CAQN;IAED;;;;OAIG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAWpF;IAED;;;;OAIG;IACG,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAyB3F;IAED;;oFAEgF;IAC1E,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAajE;IAED,sGAAsG;IAChG,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAO1C;IAED,0FAA0F;IAC1F,IAAI,gBAAgB,IAAI,OAAO,CAI9B;IAED,+EAA+E;IACzE,wBAAwB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAO5D;IAED,8EAA8E;IACxE,uBAAuB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAO1D;IAED;;;;;OAKG;IACG,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAM1E;IAED;;;;;OAKG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAIpC;IAED;uFACmF;IACnF,cAAc,IAAI,IAAI,CAIrB;IAED,+DAA+D;IAC/D,OAAO,CAAC,OAAO;CAkBhB"}
|
|
@@ -8,9 +8,30 @@ const DISPATCH_CHUNK = 200;
|
|
|
8
8
|
export class TransportPool {
|
|
9
9
|
transports;
|
|
10
10
|
warnedBulkRefusal = false;
|
|
11
|
+
/** Ids of the transports that route replies to this engine instance (see {@link routeReplies}). */
|
|
12
|
+
replying = new Set();
|
|
11
13
|
constructor(transports) {
|
|
12
14
|
this.transports = transports;
|
|
13
15
|
}
|
|
16
|
+
/**
|
|
17
|
+
* Have every transport that can ({@link Transport.consumeRepliesFor}) send this engine instance
|
|
18
|
+
* the results of the tasks it dispatches with `replyTo`. Returns whether any transport does.
|
|
19
|
+
*/
|
|
20
|
+
routeReplies(instanceId) {
|
|
21
|
+
for (const { id, transport } of this.transports) {
|
|
22
|
+
if (transport.consumeRepliesFor?.(instanceId))
|
|
23
|
+
this.replying.add(id);
|
|
24
|
+
}
|
|
25
|
+
return this.replying.size > 0;
|
|
26
|
+
}
|
|
27
|
+
/** `task` as sent on transport `id`: a reply goes back only on a transport that routes it, so
|
|
28
|
+
* elsewhere `replyTo` is dropped (its result rides the shared results). */
|
|
29
|
+
withReply(task, id) {
|
|
30
|
+
if (task.replyTo === undefined || this.replying.has(id))
|
|
31
|
+
return task;
|
|
32
|
+
const { replyTo: _replyTo, ...rest } = task;
|
|
33
|
+
return rest;
|
|
34
|
+
}
|
|
14
35
|
get size() {
|
|
15
36
|
return this.transports.length;
|
|
16
37
|
}
|
|
@@ -35,7 +56,7 @@ export class TransportPool {
|
|
|
35
56
|
let lastErr;
|
|
36
57
|
for (const { id, transport } of this.ordered(preferId)) {
|
|
37
58
|
try {
|
|
38
|
-
await transport.dispatch({ ...task, transport: id });
|
|
59
|
+
await transport.dispatch({ ...this.withReply(task, id), transport: id });
|
|
39
60
|
return;
|
|
40
61
|
}
|
|
41
62
|
catch (err) {
|
|
@@ -55,7 +76,7 @@ export class TransportPool {
|
|
|
55
76
|
const first = this.ordered(preferId)[0];
|
|
56
77
|
if (first?.transport.dispatchMany) {
|
|
57
78
|
try {
|
|
58
|
-
await first.transport.dispatchMany(tasks.map((task) => ({ ...task, transport: first.id })));
|
|
79
|
+
await first.transport.dispatchMany(tasks.map((task) => ({ ...this.withReply(task, first.id), transport: first.id })));
|
|
59
80
|
return;
|
|
60
81
|
}
|
|
61
82
|
catch (err) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transport-pool.js","sourceRoot":"","sources":["../../src/transport-pool.ts"],"names":[],"mappings":"AAYA,sGAAsG;AACtG,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;;;GAIG;AACH,MAAM,OAAO,aAAa;
|
|
1
|
+
{"version":3,"file":"transport-pool.js","sourceRoot":"","sources":["../../src/transport-pool.ts"],"names":[],"mappings":"AAYA,sGAAsG;AACtG,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;;;GAIG;AACH,MAAM,OAAO,aAAa;IAKK,UAAU;IAJ/B,iBAAiB,GAAG,KAAK,CAAC;IAClC,mGAAmG;IAClF,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAE9C,YAA6B,UAA4B;0BAA5B,UAAU;IAAqB,CAAC;IAE7D;;;OAGG;IACH,YAAY,CAAC,UAAkB;QAC7B,KAAK,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAChD,IAAI,SAAS,CAAC,iBAAiB,EAAE,CAAC,UAAU,CAAC;gBAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC;IAChC,CAAC;IAED;gFAC4E;IACpE,SAAS,CAA6C,IAAO,EAAE,EAAU;QAC/E,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC;QACrE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;QAC5C,OAAO,IAAS,CAAC;IACnB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;qGAEiG;IACjG,IAAI,CACF,QAA+C,EAC/C,WAA+C,EAC/C,WAAyD;QAEzD,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC7B,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACnC,IAAI,WAAW,IAAI,SAAS,CAAC,WAAW,EAAE,CAAC;gBACzC,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmC,EAAE,QAAiB;QACnE,IAAI,OAAgB,CAAC;QACrB,KAAK,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,MAAM,SAAS,CAAC,QAAQ,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;gBACzE,OAAO;YACT,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,GAAG,GAAG,CAAC;YAChB,CAAC;QACH,CAAC;QACD,MAAM,OAAO,IAAI,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACnE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,KAAsC,EAAE,QAAiB;QAC1E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,IAAI,KAAK,EAAE,SAAS,CAAC,YAAY,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,SAAS,CAAC,YAAY,CAChC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAClF,CAAC;gBACF,OAAO;YACT,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;oBAC9B,OAAO,CAAC,IAAI,CACV,+BAA+B,KAAK,CAAC,EAAE,4EACrC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,IAAI,cAAc,EAAE,CAAC;YACzD,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAClF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;oFAEgF;IAChF,KAAK,CAAC,WAAW,CAAC,KAAa;QAC7B,2FAA2F;QAC3F,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CACxC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAC5D,CACF,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QAC3C,OAAO;YACL,KAAK;YACL,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YACnD,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;SACnD,CAAC;IACJ,CAAC;IAED,sGAAsG;IACtG,KAAK,CAAC,gBAAgB;QACpB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CACxC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CACjE,CACF,CAAC;QACF,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,0FAA0F;IAC1F,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CACzB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,wBAAwB,IAAI,SAAS,CAAC,uBAAuB,CAC3F,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,KAAK,CAAC,wBAAwB;QAC5B,MAAM,GAAG,GAAuB,EAAE,CAAC;QACnC,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,SAAS,CAAC,wBAAwB;gBACpC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,SAAS,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,8EAA8E;IAC9E,KAAK,CAAC,uBAAuB;QAC3B,MAAM,GAAG,GAAsB,EAAE,CAAC;QAClC,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,SAAS,CAAC,uBAAuB;gBACnC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,SAAS,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,sBAAsB,CAAC,KAAa;QACxC,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,CAAC,SAAS,CAAC,gBAAgB;gBAAE,SAAS;YAC1C,IAAI,CAAC,MAAM,SAAS,CAAC,gBAAgB,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;QAC7E,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,SAAiB;QAC5B,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,SAAS,CAAC,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED;uFACmF;IACnF,cAAc;QACZ,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,+DAA+D;IACvD,OAAO,CAAC,QAAiB;QAC/B,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC;QACtC,IAAI,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;QAC5D,gGAAgG;QAChG,0FAA0F;QAC1F,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC;YACzE,MAAM,IAAI,KAAK,CACb,cAAc,QAAQ,mDAAmD,GAAG,KAAK;gBAC/E,8EAA8E;gBAC9E,6EAA6E,CAChF,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;CACF"}
|
|
@@ -73,6 +73,23 @@ export interface BullMQTransportOptions {
|
|
|
73
73
|
* your slowest legitimate step. Unset (the default) = no ceiling.
|
|
74
74
|
*/
|
|
75
75
|
stepTimeoutMs?: number | undefined;
|
|
76
|
+
/**
|
|
77
|
+
* Route a dispatched `ctx.step`'s result back to the engine instance that dispatched it (the task's
|
|
78
|
+
* `replyTo`), on a queue of its own, while that instance is alive: it holds the workflow body parked
|
|
79
|
+
* on the step, so with several engines the result continues the body instead of replaying it on
|
|
80
|
+
* whichever engine takes it off the shared results. An instance announces it is alive with a Redis
|
|
81
|
+
* key it refreshes; once the key lapses (a crash, a scale-down, a shutdown), workers send its
|
|
82
|
+
* results to the shared queue again and every transport moves what is left on its queue there, so
|
|
83
|
+
* no result is lost. Default `true`. `false` neither consumes nor sends routed replies. The keys
|
|
84
|
+
* and queue are the NestJS transport's and the Python worker's, so a mixed fleet routes too.
|
|
85
|
+
*/
|
|
86
|
+
replyRouting?: boolean | undefined;
|
|
87
|
+
/** How long an engine's alive key outlives its last refresh (refreshed every third of it). Default 15 s. */
|
|
88
|
+
replyAliveTtlMs?: number | undefined;
|
|
89
|
+
/** How often a transport looks for dead engines' reply queues to move to the shared results. Default 5 s. */
|
|
90
|
+
replySweepMs?: number | undefined;
|
|
91
|
+
/** How long a dead engine's emptied reply queue is still watched before it is forgotten. Default 90 s. */
|
|
92
|
+
replyDrainIdleMs?: number | undefined;
|
|
76
93
|
}
|
|
77
94
|
/**
|
|
78
95
|
* A queue-backed {@link Transport} over BullMQ/Redis, BYTE-COMPATIBLE with the aviary
|
|
@@ -159,6 +176,22 @@ export declare class BullMQTransport implements Transport {
|
|
|
159
176
|
* pub/sub channel (resets the engine's `timeoutMs` window on whichever pod is awaiting it). */
|
|
160
177
|
heartbeat(beat: Heartbeat): Promise<void>;
|
|
161
178
|
onResult(handler: (result: StepResult) => Promise<void>): void;
|
|
179
|
+
/**
|
|
180
|
+
* Engine side: consume the results routed to engine instance `replyTo`, keep its alive key fresh,
|
|
181
|
+
* and sweep dead instances' reply queues. Once per transport. Resolves whether replies are routed
|
|
182
|
+
* here (`false` with `replyRouting: false`, or a Redis client without the keys it needs). A process
|
|
183
|
+
* whose consumers are deferred announces nothing until they start: until then its tasks' results
|
|
184
|
+
* ride the shared queue.
|
|
185
|
+
*/
|
|
186
|
+
consumeRepliesFor(replyTo: string): boolean;
|
|
187
|
+
/**
|
|
188
|
+
* Move the reply queue of every engine instance whose alive key lapsed onto the shared results: a
|
|
189
|
+
* drain worker forwards each job (waiting, delayed, or stalled under the dead consumer) there. Once
|
|
190
|
+
* a dead instance's queue has stayed empty for `replyDrainIdleMs` (past BullMQ's stalled check, and
|
|
191
|
+
* past any sender's cached view of the instance), it is forgotten. Runs on its own every
|
|
192
|
+
* `replySweepMs` on a transport that routes replies; exposed for tests.
|
|
193
|
+
*/
|
|
194
|
+
sweepReplyQueues(): Promise<void>;
|
|
162
195
|
onDecision(handler: (decision: WorkflowDecision) => Promise<void>): void;
|
|
163
196
|
/** workflow worker → engine: stream a local step's lifecycle on `${P}-step-events` (job `stepEvent`),
|
|
164
197
|
* point-to-point so one engine instance checkpoints each once. */
|
|
@@ -170,7 +203,14 @@ export declare class BullMQTransport implements Transport {
|
|
|
170
203
|
* neither segment carries a `:`, so the token is the segment between the fixed prefix and the next `:`.
|
|
171
204
|
* Reads the shared {@link #liveHeartbeats} snapshot, so it costs no extra walk next to `groupHealth`. */
|
|
172
205
|
listWorkerGroups(): Promise<string[]>;
|
|
173
|
-
|
|
206
|
+
/**
|
|
207
|
+
* `fresh` skips the reuse: the engine re-reads before it parks a run `blocked`, so a capable
|
|
208
|
+
* worker that joined inside the cache window is never missed. A read already in flight is joined
|
|
209
|
+
* either way (it started no earlier).
|
|
210
|
+
*/
|
|
211
|
+
listWorkerDescriptors(token: string, opts?: {
|
|
212
|
+
fresh?: boolean | undefined;
|
|
213
|
+
}): Promise<WorkerDescriptor[]>;
|
|
174
214
|
/** Every live descriptor on every token: SCAN `${P}-worker-descriptor:*`, MGET per page. Degrades
|
|
175
215
|
* to `[]` (reported through `onError`) when the keyspace cannot be read. */
|
|
176
216
|
readAllWorkerDescriptors(): Promise<WorkerDescriptor[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bullmq-transport.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/bullmq-transport.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,UAAU,EACV,WAAW,EACX,QAAQ,EACR,UAAU,EACV,eAAe,EACf,UAAU,EACV,WAAW,EACX,SAAS,EACT,eAAe,EAEf,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,KAAK,WAAW,EAAqB,MAAM,mBAAmB,CAAC;AAExF,OAAO,EAAkB,KAAK,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAOlF,OAAO,EAAsB,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACvF,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,SAAS,EAGf,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"bullmq-transport.d.ts","sourceRoot":"","sources":["../../../../src/transports/bullmq/bullmq-transport.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,UAAU,EACV,WAAW,EACX,QAAQ,EACR,UAAU,EACV,eAAe,EACf,UAAU,EACV,WAAW,EACX,SAAS,EACT,eAAe,EAEf,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,KAAK,WAAW,EAAqB,MAAM,mBAAmB,CAAC;AAExF,OAAO,EAAkB,KAAK,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAOlF,OAAO,EAAsB,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACvF,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,SAAS,EAGf,MAAM,WAAW,CAAC;AAkCnB,MAAM,WAAW,sBAAsB;IACrC;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,2FAA2F;IAC3F,IAAI,EAAE,UAAU,CAAC;IACjB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,oEAAoE;IACpE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yFAAyF;IACzF,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,mGAAmG;IACnG,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAC/C;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;IAC5C;;;;;;;;;;OAUG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;IAC5C;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC;;;;;;;;;OASG;IACH,YAAY,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACnC,4GAA4G;IAC5G,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,6GAA6G;IAC7G,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,0GAA0G;IAC1G,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC;AA2BD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,SAAS;;IA2G/C,YAAY,OAAO,EAAE,sBAAsB,EA6B1C;IAiGD;;;;;;;;;;;;;;OAcG;IACH,cAAc,IAAI,IAAI,CAErB;IAED;;mEAE+D;IAC/D,cAAc,IAAI,IAAI,CAIrB;IAgDD,iEAAiE;IACjE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAGpC;IAmGD;;;;;;;OAOG;IACH,qBAAqB,IAAI,SAAS,CAEjC;IAMD;+EAC2E;IACrE,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9C;IAED;;;;OAIG;IACG,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCrD;IAED;2GACuG;IACjG,oBAAoB,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAM5D;IAMD;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,IAAI,CAG1C;IAED;;;;;;OAMG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAG5D;IAyOD;oGACgG;IAC1F,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9C;IAqBD,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAoC7D;IAqCD;;;;;;OAMG;IACH,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAuC1C;IAkGD;;;;;;OAMG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CA+CtC;IAED,UAAU,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAkBvE;IAED;uEACmE;IAC7D,iBAAiB,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAM/D;IAED,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKtE;IAED,WAAW,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAkB7D;IAiDD;;;8GAG0G;IACpG,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAU1C;IA+ED;;;;OAIG;IACG,qBAAqB,CACzB,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;KAAE,GACrC,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAY7B;IAmCD;iFAC6E;IACvE,wBAAwB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAqB5D;IAED;4FACwF;IAClF,uBAAuB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAuB1D;IAuBD;;mGAE+F;IACzF,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAYrD;IAoCD;;;;;;OAMG;IACG,YAAY,CAChB,IAAI,EAAE,SAAS,WAAW,EAAE,EAC5B,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,GACxC,OAAO,CAAC,IAAI,CAAC,CAgBf;IAOD;;;;OAIG;IACH,eAAe,CACb,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,EAC5C,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,GAC5D,MAAM,OAAO,CAAC,IAAI,CAAC,CAiBrB;IAED,sGAAsG;IAChG,gBAAgB,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;IAED,uGAAuG;IACvG,UAAU,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAKjE;IAED,qGAAqG;IAC/F,kBAAkB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAED,kGAAkG;IAClG,YAAY,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAK9D;IAED,mGAAmG;IAC7F,eAAe,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,wGAAwG;IACxG,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,IAAI,CAoBnD;IAED,2GAA2G;IACrG,kBAAkB,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAMxD;IAED;;;;;OAKG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAqB7E;IAyBD,uGAAuG;IACjG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6D3B;CACF"}
|
|
@@ -5,7 +5,7 @@ import { isWorkflowTask } from '../../workflow-turn.js';
|
|
|
5
5
|
import { failedTaskIdentity, fanOutDecision, infrastructureFailure, } from '../task-failure.js';
|
|
6
6
|
import { AdaptiveController } from './adaptive-concurrency.js';
|
|
7
7
|
import { workerConcurrency, } from './deps.js';
|
|
8
|
-
import { decisionsName, effectivePrefix, heartbeatChannel, resultBatchesName, resultsName, routingToken, runReplyChannel, runRequestName, runStartJobId, runsName, startRunName, stepEventsName, tasksName, tenantEventsChannel, workerDescriptorTokenPrefix, workerHeartbeatKey, workerHeartbeatKeyPrefix, } from './naming.js';
|
|
8
|
+
import { decisionsName, effectivePrefix, heartbeatChannel, replyAliveKey, replyInstancesKey, replyQueueName, resultBatchesName, resultsName, routingToken, runReplyChannel, runRequestName, runStartJobId, runsName, startRunName, stepEventsName, tasksName, tenantEventsChannel, workerDescriptorTokenPrefix, workerHeartbeatKey, workerHeartbeatKeyPrefix, } from './naming.js';
|
|
9
9
|
import { buildInstanceId, heartbeatKeyValue, jobOptions, parseHeartbeatValue, resultJobOptions, taskJobOptions, WORKER_HEARTBEAT_INTERVAL_MS, WORKER_HEARTBEAT_TTL_SECONDS, } from './serialization.js';
|
|
10
10
|
/** Default {@link BullMQTransportOptions.pingIntervalMs}. */
|
|
11
11
|
const DEFAULT_PING_INTERVAL_MS = 30_000;
|
|
@@ -83,6 +83,8 @@ export class BullMQTransport {
|
|
|
83
83
|
#taskWorkers = new Map();
|
|
84
84
|
#resultsWorker;
|
|
85
85
|
#resultBatchesWorker;
|
|
86
|
+
/** The engine's result handlers (one per engine), also fed by the routed-reply queue. */
|
|
87
|
+
#resultHandlers = new Set();
|
|
86
88
|
// Step results waiting for the results-batch publish in flight to finish (see `#publishResult`).
|
|
87
89
|
#resultsToBatch = [];
|
|
88
90
|
#publishingResultBatch = false;
|
|
@@ -117,6 +119,23 @@ export class BullMQTransport {
|
|
|
117
119
|
#pingIntervalMs;
|
|
118
120
|
#stepTimeoutMs;
|
|
119
121
|
#pingWatchdogTimer;
|
|
122
|
+
// --- Reply routing (see `replyRouting`) ---
|
|
123
|
+
#replyRouting;
|
|
124
|
+
#replyAliveTtlMs;
|
|
125
|
+
#replySweepMs;
|
|
126
|
+
#replyDrainIdleMs;
|
|
127
|
+
/** The engine instance this transport consumes routed replies for. */
|
|
128
|
+
#replyId;
|
|
129
|
+
#replyWorker;
|
|
130
|
+
#replyRedisClient;
|
|
131
|
+
#replyBeatTimer;
|
|
132
|
+
#replySweepTimer;
|
|
133
|
+
#replySweeping = false;
|
|
134
|
+
#replyAliveCache = new Map();
|
|
135
|
+
/** Routed results waiting for the batch in flight to their instance's queue, by queue. */
|
|
136
|
+
#replyOutboxes = new Map();
|
|
137
|
+
/** Dead engines' reply queues this transport is moving to the shared results. */
|
|
138
|
+
#replyDrains = new Map();
|
|
120
139
|
constructor(options) {
|
|
121
140
|
this.#deps = options.deps;
|
|
122
141
|
this.#partition = options.partition;
|
|
@@ -132,6 +151,10 @@ export class BullMQTransport {
|
|
|
132
151
|
// results per write. `resultConcurrency` only raises it.
|
|
133
152
|
this.#resultConcurrency = Math.max(RESULT_CONSUMER_MIN, workerConcurrency(options.resultConcurrency));
|
|
134
153
|
this.#concurrency = options.concurrency;
|
|
154
|
+
this.#replyRouting = options.replyRouting !== false;
|
|
155
|
+
this.#replyAliveTtlMs = positiveMs(options.replyAliveTtlMs, 15_000);
|
|
156
|
+
this.#replySweepMs = positiveMs(options.replySweepMs, 5_000);
|
|
157
|
+
this.#replyDrainIdleMs = positiveMs(options.replyDrainIdleMs, 90_000);
|
|
135
158
|
this.#pingIntervalMs = normalizePingInterval(options.pingIntervalMs);
|
|
136
159
|
this.#stepTimeoutMs =
|
|
137
160
|
options.stepTimeoutMs !== undefined &&
|
|
@@ -332,7 +355,9 @@ export class BullMQTransport {
|
|
|
332
355
|
* result as soon as it finishes. A batch that fails to publish goes out one result per job.
|
|
333
356
|
* Otherwise each result is its own job on the results queue.
|
|
334
357
|
*/
|
|
335
|
-
async #publishResult(result) {
|
|
358
|
+
async #publishResult(result, replyTo) {
|
|
359
|
+
if (replyTo !== undefined && (await this.#publishReply(result, replyTo)))
|
|
360
|
+
return;
|
|
336
361
|
if (!(await this.#resultBatchesConsumed())) {
|
|
337
362
|
await this.#queue(resultsName(this.#effectivePrefix())).add('result', result, resultJobOptions());
|
|
338
363
|
return;
|
|
@@ -666,7 +691,7 @@ export class BullMQTransport {
|
|
|
666
691
|
if (this.#handedBack.has(task))
|
|
667
692
|
return false;
|
|
668
693
|
try {
|
|
669
|
-
await this.#publishResult(result);
|
|
694
|
+
await this.#publishResult(result, task.replyTo);
|
|
670
695
|
return result.status === 'completed';
|
|
671
696
|
}
|
|
672
697
|
catch (err) {
|
|
@@ -727,6 +752,7 @@ export class BullMQTransport {
|
|
|
727
752
|
onResult(handler) {
|
|
728
753
|
if (this.#resultsWorker)
|
|
729
754
|
return;
|
|
755
|
+
this.#resultHandlers.add(handler);
|
|
730
756
|
this.#resultBatchesWorker = this.#makeWorker(resultBatchesName(this.#effectivePrefix()), async (job) => {
|
|
731
757
|
const { results } = job.data;
|
|
732
758
|
await Promise.all(results.map((result) => handler(result)));
|
|
@@ -753,6 +779,222 @@ export class BullMQTransport {
|
|
|
753
779
|
console.error(`[adonis-durable] ${giveUp.message}`);
|
|
754
780
|
});
|
|
755
781
|
}
|
|
782
|
+
// --- Reply routing ------------------------------------------------------------------------------
|
|
783
|
+
// A task carrying `replyTo` asks for its result back on `<P>-results-to-<replyTo>`, a queue only
|
|
784
|
+
// that engine instance consumes, while its alive key (`<P>:reply-alive:<id>`, refreshed every third
|
|
785
|
+
// of `replyAliveTtlMs`) exists. Every instance that consumed or was sent a routed reply is
|
|
786
|
+
// remembered in `<P>:reply-instances`; any transport that routes replies sweeps that set and moves
|
|
787
|
+
// the queue of an instance whose key lapsed onto the shared results, stalled jobs included (BullMQ's
|
|
788
|
+
// stalled check hands a dead consumer's job to the next worker on that queue).
|
|
789
|
+
/** The connection reply routing uses, or `undefined` when routing is off or the client lacks it. */
|
|
790
|
+
#replyRedis() {
|
|
791
|
+
if (!this.#replyRouting)
|
|
792
|
+
return undefined;
|
|
793
|
+
this.#replyRedisClient ??= this.#deps.makeRedis();
|
|
794
|
+
const r = this.#replyRedisClient;
|
|
795
|
+
if (!r.exists || !r.del || !r.sadd || !r.srem || !r.smembers)
|
|
796
|
+
return undefined;
|
|
797
|
+
return r;
|
|
798
|
+
}
|
|
799
|
+
/** Run `start` now, or once consumers start (see {@link deferConsumers}). */
|
|
800
|
+
#whenConsuming(start) {
|
|
801
|
+
if (this.#consumersDeferred)
|
|
802
|
+
this.#deferredWorkers.push(start);
|
|
803
|
+
else
|
|
804
|
+
start();
|
|
805
|
+
}
|
|
806
|
+
/** A results-queue job's results: one `result`, or a `results` batch. */
|
|
807
|
+
#resultsOf(job) {
|
|
808
|
+
// A batch is recognized by its shape too: a fake job may carry no name.
|
|
809
|
+
const data = job.data;
|
|
810
|
+
return job.name === 'results' || (job.name === undefined && 'results' in data)
|
|
811
|
+
? (data.results ?? [])
|
|
812
|
+
: [data];
|
|
813
|
+
}
|
|
814
|
+
/**
|
|
815
|
+
* Engine side: consume the results routed to engine instance `replyTo`, keep its alive key fresh,
|
|
816
|
+
* and sweep dead instances' reply queues. Once per transport. Resolves whether replies are routed
|
|
817
|
+
* here (`false` with `replyRouting: false`, or a Redis client without the keys it needs). A process
|
|
818
|
+
* whose consumers are deferred announces nothing until they start: until then its tasks' results
|
|
819
|
+
* ride the shared queue.
|
|
820
|
+
*/
|
|
821
|
+
consumeRepliesFor(replyTo) {
|
|
822
|
+
if (this.#replyId !== undefined)
|
|
823
|
+
return this.#replyId === replyTo;
|
|
824
|
+
const redis = this.#replyRedis();
|
|
825
|
+
if (!redis || this.#resultHandlers.size === 0)
|
|
826
|
+
return false;
|
|
827
|
+
this.#replyId = replyTo;
|
|
828
|
+
const prefix = this.#effectivePrefix();
|
|
829
|
+
this.#replyWorker = this.#makeWorker(replyQueueName(prefix, replyTo), (job) => this.#handleResultJob(job), { concurrency: this.#resultConcurrency });
|
|
830
|
+
this.#replyWorker.on('failed', (job, err) => {
|
|
831
|
+
if (!job || (job.attemptsMade ?? 0) < (job.opts?.attempts ?? 1))
|
|
832
|
+
return;
|
|
833
|
+
const giveUp = new Error(`gave up on a routed step result after ${job.attemptsMade} attempts: ${err.message}`, { cause: err });
|
|
834
|
+
if (this.#hasOnError)
|
|
835
|
+
this.#onError(giveUp);
|
|
836
|
+
else
|
|
837
|
+
console.error(`[adonis-durable] ${giveUp.message}`);
|
|
838
|
+
});
|
|
839
|
+
this.#whenConsuming(() => {
|
|
840
|
+
if (this.#closing)
|
|
841
|
+
return;
|
|
842
|
+
const beat = async () => {
|
|
843
|
+
try {
|
|
844
|
+
await redis.set(replyAliveKey(prefix, replyTo), '1', 'PX', this.#replyAliveTtlMs);
|
|
845
|
+
await redis.sadd(replyInstancesKey(prefix), replyTo);
|
|
846
|
+
}
|
|
847
|
+
catch {
|
|
848
|
+
// The next beat retries; a lapsed key only sends this instance's results to the shared queue.
|
|
849
|
+
}
|
|
850
|
+
};
|
|
851
|
+
void beat();
|
|
852
|
+
this.#replyBeatTimer = setInterval(() => void beat(), Math.max(50, Math.floor(this.#replyAliveTtlMs / 3)));
|
|
853
|
+
this.#replyBeatTimer.unref?.();
|
|
854
|
+
this.#startReplySweeper();
|
|
855
|
+
});
|
|
856
|
+
return true;
|
|
857
|
+
}
|
|
858
|
+
/** Hand a results job to every result handler. */
|
|
859
|
+
async #handleResultJob(job) {
|
|
860
|
+
const results = this.#resultsOf(job);
|
|
861
|
+
await Promise.all(results.flatMap((result) => [...this.#resultHandlers].map((handler) => handler(result))));
|
|
862
|
+
}
|
|
863
|
+
/** Whether engine instance `id` is alive to take its routed replies (read at most every second). */
|
|
864
|
+
async #replyAlive(id) {
|
|
865
|
+
if (id === this.#replyId && this.#replyBeatTimer)
|
|
866
|
+
return true;
|
|
867
|
+
const known = this.#replyAliveCache.get(id);
|
|
868
|
+
if (known && Date.now() - known.at < 1_000)
|
|
869
|
+
return known.alive;
|
|
870
|
+
const redis = this.#replyRedis();
|
|
871
|
+
if (!redis)
|
|
872
|
+
return false;
|
|
873
|
+
const alive = (await redis.exists(replyAliveKey(this.#effectivePrefix(), id))) === 1;
|
|
874
|
+
this.#replyAliveCache.set(id, { alive, at: Date.now() });
|
|
875
|
+
return alive;
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Worker side: send `result` to engine instance `replyTo` while it is alive. Resolves `false` to
|
|
879
|
+
* send it to the shared results instead (routing off, the instance gone, or Redis refusing).
|
|
880
|
+
*/
|
|
881
|
+
async #publishReply(result, replyTo) {
|
|
882
|
+
if (!this.#replyRouting)
|
|
883
|
+
return false;
|
|
884
|
+
try {
|
|
885
|
+
if (!(await this.#replyAlive(replyTo)))
|
|
886
|
+
return false;
|
|
887
|
+
}
|
|
888
|
+
catch {
|
|
889
|
+
return false;
|
|
890
|
+
}
|
|
891
|
+
// Batched like the shared results: the results that finish while one batch to this instance is
|
|
892
|
+
// being published go out together in the next. A batch that cannot be published goes to the
|
|
893
|
+
// shared results, one result per job.
|
|
894
|
+
const queue = replyQueueName(this.#effectivePrefix(), replyTo);
|
|
895
|
+
let outbox = this.#replyOutboxes.get(queue);
|
|
896
|
+
if (!outbox) {
|
|
897
|
+
outbox = { entries: [], publishing: false };
|
|
898
|
+
this.#replyOutboxes.set(queue, outbox);
|
|
899
|
+
}
|
|
900
|
+
const box = outbox;
|
|
901
|
+
const sent = await new Promise((resolve) => {
|
|
902
|
+
box.entries.push({ result, resolve });
|
|
903
|
+
if (!box.publishing)
|
|
904
|
+
void this.#publishReplyBatches(queue, box);
|
|
905
|
+
});
|
|
906
|
+
if (!sent)
|
|
907
|
+
return false;
|
|
908
|
+
// A sender also watches for dead engines, so routed results never wait on an engine coming back.
|
|
909
|
+
this.#startReplySweeper();
|
|
910
|
+
return true;
|
|
911
|
+
}
|
|
912
|
+
async #publishReplyBatches(queue, outbox) {
|
|
913
|
+
outbox.publishing = true;
|
|
914
|
+
try {
|
|
915
|
+
while (outbox.entries.length > 0) {
|
|
916
|
+
const batch = outbox.entries.splice(0, RESULT_BATCH_MAX);
|
|
917
|
+
try {
|
|
918
|
+
await (batch.length === 1
|
|
919
|
+
? this.#queue(queue).add('result', batch[0]?.result, resultJobOptions())
|
|
920
|
+
: this.#queue(queue).add('results', { results: batch.map((entry) => entry.result) }, resultJobOptions()));
|
|
921
|
+
for (const entry of batch)
|
|
922
|
+
entry.resolve(true);
|
|
923
|
+
}
|
|
924
|
+
catch {
|
|
925
|
+
for (const entry of batch)
|
|
926
|
+
entry.resolve(false);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
finally {
|
|
931
|
+
outbox.publishing = false;
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
#startReplySweeper() {
|
|
935
|
+
if (this.#replySweepTimer || this.#closing || this.#consumersDeferred || !this.#replyRedis()) {
|
|
936
|
+
return;
|
|
937
|
+
}
|
|
938
|
+
this.#replySweepTimer = setInterval(() => {
|
|
939
|
+
if (this.#replySweeping)
|
|
940
|
+
return;
|
|
941
|
+
this.#replySweeping = true;
|
|
942
|
+
void this.sweepReplyQueues()
|
|
943
|
+
.catch(() => undefined)
|
|
944
|
+
.finally(() => {
|
|
945
|
+
this.#replySweeping = false;
|
|
946
|
+
});
|
|
947
|
+
}, this.#replySweepMs);
|
|
948
|
+
this.#replySweepTimer.unref?.();
|
|
949
|
+
}
|
|
950
|
+
/**
|
|
951
|
+
* Move the reply queue of every engine instance whose alive key lapsed onto the shared results: a
|
|
952
|
+
* drain worker forwards each job (waiting, delayed, or stalled under the dead consumer) there. Once
|
|
953
|
+
* a dead instance's queue has stayed empty for `replyDrainIdleMs` (past BullMQ's stalled check, and
|
|
954
|
+
* past any sender's cached view of the instance), it is forgotten. Runs on its own every
|
|
955
|
+
* `replySweepMs` on a transport that routes replies; exposed for tests.
|
|
956
|
+
*/
|
|
957
|
+
async sweepReplyQueues() {
|
|
958
|
+
const redis = this.#replyRedis();
|
|
959
|
+
if (!redis || this.#closing)
|
|
960
|
+
return;
|
|
961
|
+
const prefix = this.#effectivePrefix();
|
|
962
|
+
for (const id of await redis.smembers(replyInstancesKey(prefix))) {
|
|
963
|
+
if (id === this.#replyId || this.#closing)
|
|
964
|
+
continue;
|
|
965
|
+
const alive = (await redis.exists(replyAliveKey(prefix, id))) === 1;
|
|
966
|
+
const drain = this.#replyDrains.get(id);
|
|
967
|
+
if (alive) {
|
|
968
|
+
if (drain) {
|
|
969
|
+
this.#replyDrains.delete(id);
|
|
970
|
+
await drain.worker.close().catch(() => undefined);
|
|
971
|
+
}
|
|
972
|
+
continue;
|
|
973
|
+
}
|
|
974
|
+
if (!drain) {
|
|
975
|
+
const worker = this.#deps.makeWorker(replyQueueName(prefix, id), async (job) => {
|
|
976
|
+
await Promise.all(this.#resultsOf(job).map((result) => this.#queue(resultsName(prefix)).add('result', result, resultJobOptions())));
|
|
977
|
+
});
|
|
978
|
+
this.#replyDrains.set(id, { worker });
|
|
979
|
+
continue;
|
|
980
|
+
}
|
|
981
|
+
const counts = await this.#queue(replyQueueName(prefix, id)).getJobCounts('waiting', 'wait', 'active', 'delayed', 'prioritized');
|
|
982
|
+
const left = Object.values(counts).reduce((sum, n) => sum + n, 0);
|
|
983
|
+
if (left > 0) {
|
|
984
|
+
drain.idleSince = undefined;
|
|
985
|
+
continue;
|
|
986
|
+
}
|
|
987
|
+
drain.idleSince ??= Date.now();
|
|
988
|
+
if (Date.now() - drain.idleSince < this.#replyDrainIdleMs)
|
|
989
|
+
continue;
|
|
990
|
+
this.#replyDrains.delete(id);
|
|
991
|
+
await drain.worker.close().catch(() => undefined);
|
|
992
|
+
// Forgotten only while still dead: an instance that came back re-adds itself on its next beat.
|
|
993
|
+
if ((await redis.exists(replyAliveKey(prefix, id))) === 0) {
|
|
994
|
+
await redis.srem(replyInstancesKey(prefix), id);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
}
|
|
756
998
|
onDecision(handler) {
|
|
757
999
|
// EVERY handler gets every decision. An engine subscribes one stable handler (it applies a decision
|
|
758
1000
|
// by run id, dropping any its run no longer awaits); a set keeps a second registration of it, or a
|
|
@@ -917,12 +1159,30 @@ export class BullMQTransport {
|
|
|
917
1159
|
* cadence (tens of seconds), but `ensureRoutable` asks before EVERY dispatch — without this memo
|
|
918
1160
|
* a 500-step/s engine ran 500 SCAN cycles/s against Redis for an answer that was identical. */
|
|
919
1161
|
#descriptorCache = new Map();
|
|
920
|
-
|
|
1162
|
+
/** The descriptor read in flight per token: concurrent callers share it instead of each scanning. */
|
|
1163
|
+
#descriptorReads = new Map();
|
|
1164
|
+
/**
|
|
1165
|
+
* `fresh` skips the reuse: the engine re-reads before it parks a run `blocked`, so a capable
|
|
1166
|
+
* worker that joined inside the cache window is never missed. A read already in flight is joined
|
|
1167
|
+
* either way (it started no earlier).
|
|
1168
|
+
*/
|
|
1169
|
+
async listWorkerDescriptors(token, opts) {
|
|
921
1170
|
const cached = this.#descriptorCache.get(token);
|
|
922
1171
|
// A short TTL (5s): a fraction of the descriptor heartbeat TTL, so a scaled-down fleet is
|
|
923
1172
|
// noticed within one cache window while the per-dispatch SCAN storm disappears.
|
|
924
|
-
if (cached && Date.now() - cached.at < 5_000)
|
|
1173
|
+
if (!opts?.fresh && cached && Date.now() - cached.at < 5_000)
|
|
925
1174
|
return cached.descriptors;
|
|
1175
|
+
const inFlight = this.#descriptorReads.get(token);
|
|
1176
|
+
if (inFlight)
|
|
1177
|
+
return inFlight;
|
|
1178
|
+
const read = this.#readWorkerDescriptors(token).finally(() => {
|
|
1179
|
+
if (this.#descriptorReads.get(token) === read)
|
|
1180
|
+
this.#descriptorReads.delete(token);
|
|
1181
|
+
});
|
|
1182
|
+
this.#descriptorReads.set(token, read);
|
|
1183
|
+
return read;
|
|
1184
|
+
}
|
|
1185
|
+
async #readWorkerDescriptors(token) {
|
|
926
1186
|
const client = this.#workerRedisClient();
|
|
927
1187
|
const prefix = workerDescriptorTokenPrefix(this.#effectivePrefix(), token);
|
|
928
1188
|
const descriptors = [];
|
|
@@ -1234,6 +1494,17 @@ export class BullMQTransport {
|
|
|
1234
1494
|
if (this.#pingWatchdogTimer)
|
|
1235
1495
|
clearInterval(this.#pingWatchdogTimer);
|
|
1236
1496
|
this.#pingWatchdogTimer = undefined;
|
|
1497
|
+
if (this.#replyBeatTimer)
|
|
1498
|
+
clearInterval(this.#replyBeatTimer);
|
|
1499
|
+
if (this.#replySweepTimer)
|
|
1500
|
+
clearInterval(this.#replySweepTimer);
|
|
1501
|
+
// Stop taking routed replies first: with the alive key gone, workers send to the shared results
|
|
1502
|
+
// and other instances move what is left on this instance's queue there.
|
|
1503
|
+
if (this.#replyId !== undefined && this.#replyBeatTimer && this.#replyRedisClient?.del) {
|
|
1504
|
+
await withTimeout(this.#replyRedisClient
|
|
1505
|
+
.del(replyAliveKey(this.#effectivePrefix(), this.#replyId))
|
|
1506
|
+
.catch(() => undefined), this.#closeTimeoutMs);
|
|
1507
|
+
}
|
|
1237
1508
|
// Workers first (concurrently with each other), THEN queues: a worker's close() waits for its
|
|
1238
1509
|
// in-flight job, which publishes its result/decision through a queue, so that queue must still be
|
|
1239
1510
|
// open until every worker has settled. One shared deadline bounds both phases: a graceful
|
|
@@ -1244,6 +1515,8 @@ export class BullMQTransport {
|
|
|
1244
1515
|
...[...this.#taskWorkers.values()].map((w) => w.close()),
|
|
1245
1516
|
this.#resultsWorker?.close(),
|
|
1246
1517
|
this.#resultBatchesWorker?.close(),
|
|
1518
|
+
this.#replyWorker?.close(),
|
|
1519
|
+
...[...this.#replyDrains.values()].map((d) => d.worker.close()),
|
|
1247
1520
|
this.#decisionsWorker?.close(),
|
|
1248
1521
|
this.#stepEventsWorker?.close(),
|
|
1249
1522
|
this.#startRunWorker?.close(),
|
|
@@ -1268,8 +1541,13 @@ export class BullMQTransport {
|
|
|
1268
1541
|
this.#runReplySub?.disconnect();
|
|
1269
1542
|
this.#tenantEventPub?.disconnect();
|
|
1270
1543
|
this.#tenantEventSub?.disconnect();
|
|
1544
|
+
this.#replyRedisClient?.disconnect();
|
|
1271
1545
|
}
|
|
1272
1546
|
}
|
|
1547
|
+
/** A positive finite duration, else `fallback`. */
|
|
1548
|
+
function positiveMs(value, fallback) {
|
|
1549
|
+
return value !== undefined && Number.isFinite(value) && value > 0 ? value : fallback;
|
|
1550
|
+
}
|
|
1273
1551
|
/** Resolve when `work` settles or `ms` elapses (the loser is abandoned). `ms <= 0` skips the wait. */
|
|
1274
1552
|
async function withTimeout(work, ms) {
|
|
1275
1553
|
if (ms <= 0)
|