@dudousxd/nestjs-durable-core 0.84.1 → 0.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +993 -51
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +259 -4
- package/dist/index.d.ts +259 -4
- package/dist/index.js +990 -51
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -47,6 +47,7 @@ __export(index_exports, {
|
|
|
47
47
|
GatherError: () => GatherError,
|
|
48
48
|
HANDOFF_SEQ_BASE: () => HANDOFF_SEQ_BASE,
|
|
49
49
|
InMemoryAdmissionBackend: () => InMemoryAdmissionBackend,
|
|
50
|
+
InMemoryRunQueue: () => InMemoryRunQueue,
|
|
50
51
|
InMemoryStateStore: () => InMemoryStateStore,
|
|
51
52
|
InMemoryTransport: () => InMemoryTransport,
|
|
52
53
|
LEGACY_V1_CAPABILITIES: () => LEGACY_V1_CAPABILITIES,
|
|
@@ -131,6 +132,7 @@ __export(index_exports, {
|
|
|
131
132
|
fanPageEtag: () => fanPageEtag,
|
|
132
133
|
findStalledRuns: () => findStalledRuns,
|
|
133
134
|
foldedCount: () => foldedCount,
|
|
135
|
+
getRunsByIds: () => getRunsByIds,
|
|
134
136
|
handoffHandlerId: () => handoffHandlerId,
|
|
135
137
|
handoffName: () => handoffName,
|
|
136
138
|
hasDurableExecutionWrappers: () => hasDurableExecutionWrappers,
|
|
@@ -168,6 +170,7 @@ __export(index_exports, {
|
|
|
168
170
|
poolNamespace: () => poolNamespace,
|
|
169
171
|
prevCronFireMs: () => prevCronFireMs,
|
|
170
172
|
queryCheckpointsOf: () => queryCheckpointsOf,
|
|
173
|
+
readRunsByIds: () => readRunsByIds,
|
|
171
174
|
readSearchAttributes: () => readSearchAttributes,
|
|
172
175
|
requiredCapabilities: () => requiredCapabilities,
|
|
173
176
|
resolveConcurrency: () => resolveConcurrency,
|
|
@@ -780,6 +783,82 @@ function backoffDelay(attempt, options) {
|
|
|
780
783
|
}
|
|
781
784
|
__name(backoffDelay, "backoffDelay");
|
|
782
785
|
|
|
786
|
+
// src/bounded-runner.ts
|
|
787
|
+
var BoundedRunner = class {
|
|
788
|
+
static {
|
|
789
|
+
__name(this, "BoundedRunner");
|
|
790
|
+
}
|
|
791
|
+
limit;
|
|
792
|
+
active = 0;
|
|
793
|
+
waiting = /* @__PURE__ */ new Map();
|
|
794
|
+
seq = 0;
|
|
795
|
+
constructor(limit) {
|
|
796
|
+
this.limit = limit;
|
|
797
|
+
}
|
|
798
|
+
/** Tasks running now plus tasks waiting for a slot. */
|
|
799
|
+
get size() {
|
|
800
|
+
return this.active + this.waiting.size;
|
|
801
|
+
}
|
|
802
|
+
/** Tasks running now. */
|
|
803
|
+
get running() {
|
|
804
|
+
return this.active;
|
|
805
|
+
}
|
|
806
|
+
submit(key, task) {
|
|
807
|
+
if (this.active < this.limit) {
|
|
808
|
+
this.active += 1;
|
|
809
|
+
queueMicrotask(() => this.execute(task));
|
|
810
|
+
return;
|
|
811
|
+
}
|
|
812
|
+
if (!this.waiting.has(key)) this.waiting.set(key, {
|
|
813
|
+
task
|
|
814
|
+
});
|
|
815
|
+
}
|
|
816
|
+
/**
|
|
817
|
+
* Run `task` under the bound and resolve once it settled, or once it was dropped by
|
|
818
|
+
* {@link clear} before it got a slot. Never deduplicated.
|
|
819
|
+
*/
|
|
820
|
+
run(task) {
|
|
821
|
+
return new Promise((resolve) => {
|
|
822
|
+
const done = /* @__PURE__ */ __name(() => resolve(), "done");
|
|
823
|
+
const wrapped = /* @__PURE__ */ __name(() => task().then(done, done), "wrapped");
|
|
824
|
+
if (this.active < this.limit) {
|
|
825
|
+
this.active += 1;
|
|
826
|
+
queueMicrotask(() => this.execute(wrapped));
|
|
827
|
+
return;
|
|
828
|
+
}
|
|
829
|
+
this.seq += 1;
|
|
830
|
+
this.waiting.set(`\0run:${this.seq}`, {
|
|
831
|
+
task: wrapped,
|
|
832
|
+
dropped: done
|
|
833
|
+
});
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
/** Forget every task still waiting for a slot. */
|
|
837
|
+
clear() {
|
|
838
|
+
for (const { dropped } of this.waiting.values()) dropped?.();
|
|
839
|
+
this.waiting.clear();
|
|
840
|
+
}
|
|
841
|
+
execute(task) {
|
|
842
|
+
let settled;
|
|
843
|
+
try {
|
|
844
|
+
settled = task();
|
|
845
|
+
} catch {
|
|
846
|
+
settled = Promise.resolve();
|
|
847
|
+
}
|
|
848
|
+
void settled.catch(() => void 0).then(() => {
|
|
849
|
+
const next = this.waiting.entries().next();
|
|
850
|
+
if (next.done) {
|
|
851
|
+
this.active -= 1;
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
const [key, waiting] = next.value;
|
|
855
|
+
this.waiting.delete(key);
|
|
856
|
+
this.execute(waiting.task);
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
};
|
|
860
|
+
var LOCAL_RUN_CONCURRENCY = 256;
|
|
861
|
+
|
|
783
862
|
// src/execution-hooks.ts
|
|
784
863
|
var wrappers = [];
|
|
785
864
|
function useDurableExecution(wrapper) {
|
|
@@ -2340,6 +2419,61 @@ var RemoteWorkflowExecutor = class {
|
|
|
2340
2419
|
}
|
|
2341
2420
|
};
|
|
2342
2421
|
|
|
2422
|
+
// src/run-dispatch-batcher.ts
|
|
2423
|
+
var RUN_DISPATCH_BATCH = 500;
|
|
2424
|
+
var RUN_DISPATCH_WINDOW_MS = 5;
|
|
2425
|
+
var RunDispatchBatcher = class {
|
|
2426
|
+
static {
|
|
2427
|
+
__name(this, "RunDispatchBatcher");
|
|
2428
|
+
}
|
|
2429
|
+
send;
|
|
2430
|
+
onError;
|
|
2431
|
+
buffer = /* @__PURE__ */ new Map();
|
|
2432
|
+
timer;
|
|
2433
|
+
sending = /* @__PURE__ */ new Set();
|
|
2434
|
+
constructor(send, onError) {
|
|
2435
|
+
this.send = send;
|
|
2436
|
+
this.onError = onError;
|
|
2437
|
+
}
|
|
2438
|
+
/** Nothing buffered and nothing being sent. */
|
|
2439
|
+
get idle() {
|
|
2440
|
+
return this.buffer.size === 0 && this.sending.size === 0;
|
|
2441
|
+
}
|
|
2442
|
+
/** Buffer `run`. A wake for a run already buffered replaces its start: a wake also runs a `pending` run. */
|
|
2443
|
+
add(run) {
|
|
2444
|
+
const prior = this.buffer.get(run.runId);
|
|
2445
|
+
if (prior && run.reason !== "wake") return;
|
|
2446
|
+
this.buffer.set(run.runId, run);
|
|
2447
|
+
if (this.buffer.size >= RUN_DISPATCH_BATCH) {
|
|
2448
|
+
this.flush();
|
|
2449
|
+
return;
|
|
2450
|
+
}
|
|
2451
|
+
if (!this.timer) {
|
|
2452
|
+
this.timer = setTimeout(() => this.flush(), RUN_DISPATCH_WINDOW_MS);
|
|
2453
|
+
this.timer.unref?.();
|
|
2454
|
+
}
|
|
2455
|
+
}
|
|
2456
|
+
/** Send what is buffered now. */
|
|
2457
|
+
flush() {
|
|
2458
|
+
if (this.timer) clearTimeout(this.timer);
|
|
2459
|
+
this.timer = void 0;
|
|
2460
|
+
if (this.buffer.size === 0) return;
|
|
2461
|
+
const runs = [
|
|
2462
|
+
...this.buffer.values()
|
|
2463
|
+
];
|
|
2464
|
+
this.buffer = /* @__PURE__ */ new Map();
|
|
2465
|
+
const sent = this.send(runs).catch((err) => this.onError(runs, err)).finally(() => this.sending.delete(sent));
|
|
2466
|
+
this.sending.add(sent);
|
|
2467
|
+
}
|
|
2468
|
+
/** Send what is buffered and wait for every send in flight. */
|
|
2469
|
+
async settle() {
|
|
2470
|
+
this.flush();
|
|
2471
|
+
await Promise.allSettled([
|
|
2472
|
+
...this.sending
|
|
2473
|
+
]);
|
|
2474
|
+
}
|
|
2475
|
+
};
|
|
2476
|
+
|
|
2343
2477
|
// src/run-waiting.ts
|
|
2344
2478
|
var WEBHOOK_PREFIX = "wh:";
|
|
2345
2479
|
var CHILD_PREFIX = "child:";
|
|
@@ -2410,6 +2544,51 @@ function indexWaitersByRun(waiters) {
|
|
|
2410
2544
|
return byRun;
|
|
2411
2545
|
}
|
|
2412
2546
|
__name(indexWaitersByRun, "indexWaitersByRun");
|
|
2547
|
+
var GET_RUN_FALLBACK_PARALLEL = 50;
|
|
2548
|
+
async function getRunsByIds(store, runIds) {
|
|
2549
|
+
const { runs, unread, error: error2 } = await readRunsByIds(store, runIds);
|
|
2550
|
+
if (unread.length > 0) throw error2;
|
|
2551
|
+
return runs;
|
|
2552
|
+
}
|
|
2553
|
+
__name(getRunsByIds, "getRunsByIds");
|
|
2554
|
+
async function readRunsByIds(store, runIds) {
|
|
2555
|
+
const unique = [
|
|
2556
|
+
...new Set(runIds)
|
|
2557
|
+
];
|
|
2558
|
+
const runs = /* @__PURE__ */ new Map();
|
|
2559
|
+
const unread = [];
|
|
2560
|
+
let error2;
|
|
2561
|
+
const batch = store.getRuns ? RUN_ID_BATCH : GET_RUN_FALLBACK_PARALLEL;
|
|
2562
|
+
for (let at = 0; at < unique.length; at += batch) {
|
|
2563
|
+
const ids = unique.slice(at, at + batch);
|
|
2564
|
+
if (store.getRuns) {
|
|
2565
|
+
try {
|
|
2566
|
+
for (const run of await store.getRuns(ids)) runs.set(run.id, run);
|
|
2567
|
+
} catch (err) {
|
|
2568
|
+
error2 ??= err;
|
|
2569
|
+
unread.push(...ids);
|
|
2570
|
+
}
|
|
2571
|
+
continue;
|
|
2572
|
+
}
|
|
2573
|
+
const reads = await Promise.allSettled(ids.map((id) => store.getRun(id)));
|
|
2574
|
+
reads.forEach((read, k) => {
|
|
2575
|
+
if (read.status === "fulfilled") {
|
|
2576
|
+
if (read.value) runs.set(read.value.id, read.value);
|
|
2577
|
+
return;
|
|
2578
|
+
}
|
|
2579
|
+
error2 ??= read.reason;
|
|
2580
|
+
unread.push(ids[k]);
|
|
2581
|
+
});
|
|
2582
|
+
}
|
|
2583
|
+
return {
|
|
2584
|
+
runs,
|
|
2585
|
+
unread,
|
|
2586
|
+
...unread.length > 0 ? {
|
|
2587
|
+
error: error2
|
|
2588
|
+
} : {}
|
|
2589
|
+
};
|
|
2590
|
+
}
|
|
2591
|
+
__name(readRunsByIds, "readRunsByIds");
|
|
2413
2592
|
|
|
2414
2593
|
// src/scheduler.ts
|
|
2415
2594
|
var import_node_module = require("module");
|
|
@@ -3302,6 +3481,39 @@ var TransportPool = class {
|
|
|
3302
3481
|
transport.onStartRun?.(handler);
|
|
3303
3482
|
}
|
|
3304
3483
|
}
|
|
3484
|
+
/** Does any transport here carry a run queue ({@link Transport.dispatchRuns} + `onDispatchedRun`)? */
|
|
3485
|
+
get dispatchesRuns() {
|
|
3486
|
+
return this.transports.some(({ transport }) => transport.dispatchRuns && transport.onDispatchedRun);
|
|
3487
|
+
}
|
|
3488
|
+
/**
|
|
3489
|
+
* Queue `runs` on the first transport that carries a run queue, failing over to the next on an
|
|
3490
|
+
* error, like {@link dispatch}. Throws if none accepted them.
|
|
3491
|
+
*/
|
|
3492
|
+
async dispatchRuns(runs, opts) {
|
|
3493
|
+
let lastErr;
|
|
3494
|
+
for (const { transport } of this.transports) {
|
|
3495
|
+
if (!transport.dispatchRuns || !transport.onDispatchedRun) continue;
|
|
3496
|
+
try {
|
|
3497
|
+
await transport.dispatchRuns(runs, opts);
|
|
3498
|
+
return;
|
|
3499
|
+
} catch (err) {
|
|
3500
|
+
lastErr = err;
|
|
3501
|
+
}
|
|
3502
|
+
}
|
|
3503
|
+
throw lastErr ?? new Error("no transport carries a run queue");
|
|
3504
|
+
}
|
|
3505
|
+
/**
|
|
3506
|
+
* Consume the run queue of every transport that carries one (a run may have been queued on any of
|
|
3507
|
+
* them after a failover). Returns one stop function per consumer.
|
|
3508
|
+
*/
|
|
3509
|
+
onDispatchedRun(handler, opts) {
|
|
3510
|
+
const stops = [];
|
|
3511
|
+
for (const { transport } of this.transports) {
|
|
3512
|
+
if (!transport.dispatchRuns || !transport.onDispatchedRun) continue;
|
|
3513
|
+
stops.push(transport.onDispatchedRun(handler, opts));
|
|
3514
|
+
}
|
|
3515
|
+
return stops;
|
|
3516
|
+
}
|
|
3305
3517
|
/**
|
|
3306
3518
|
* The live handshake descriptors advertised for `group` (design §7.2), merged across every transport
|
|
3307
3519
|
* that can report them. Returns `[]` when NO transport implements
|
|
@@ -3834,21 +4046,28 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3834
4046
|
return inputs.map((_, i) => i).slice(from, from + waves.waveSize);
|
|
3835
4047
|
}, "waveItems");
|
|
3836
4048
|
if (mode === "failFast") {
|
|
3837
|
-
|
|
3838
|
-
const cp = existing[
|
|
3839
|
-
const
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
]
|
|
4049
|
+
const failure = /* @__PURE__ */ __name((i2) => {
|
|
4050
|
+
const cp = existing[i2];
|
|
4051
|
+
const c2 = cp?.status === "completed" ? cp.output : null;
|
|
4052
|
+
return c2 && c2.ok === false ? c2 : null;
|
|
4053
|
+
}, "failure");
|
|
4054
|
+
const first = /* @__PURE__ */ __name((cancelled) => inputs.findIndex((_, i2) => {
|
|
4055
|
+
const c2 = failure(i2);
|
|
4056
|
+
return c2 !== null && c2.code === "cancelled" === cancelled;
|
|
4057
|
+
}), "first");
|
|
4058
|
+
const i = first(false) !== -1 ? first(false) : first(true);
|
|
4059
|
+
const c = i === -1 ? null : failure(i);
|
|
4060
|
+
if (c) {
|
|
4061
|
+
for (let j = 0; j < inputs.length; j += 1) {
|
|
4062
|
+
if (j !== i && existing[j]?.status !== "completed") host.cancelChild?.(id(j));
|
|
3851
4063
|
}
|
|
4064
|
+
throw new GatherError([
|
|
4065
|
+
{
|
|
4066
|
+
index: i,
|
|
4067
|
+
id: id(i),
|
|
4068
|
+
error: c.error ?? "unknown"
|
|
4069
|
+
}
|
|
4070
|
+
], inputs.length);
|
|
3852
4071
|
}
|
|
3853
4072
|
}
|
|
3854
4073
|
if (store.compactCheckpoints && inputs.length >= FOLD_MIN_FAN) {
|
|
@@ -3927,10 +4146,12 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3927
4146
|
await registerChildWaits(entries);
|
|
3928
4147
|
for (const i of chunk) host.startChild(name, inputs[i], id(i));
|
|
3929
4148
|
}
|
|
3930
|
-
|
|
3931
|
-
await
|
|
3932
|
-
|
|
3933
|
-
|
|
4149
|
+
if (toEnsure.length > 0) {
|
|
4150
|
+
const { runs: started, unread } = await readRunsByIds(store, toEnsure.map((i) => id(i)));
|
|
4151
|
+
const skip = new Set(unread);
|
|
4152
|
+
for (const i of toEnsure) {
|
|
4153
|
+
if (!started.has(id(i)) && !skip.has(id(i))) host.startChild(name, inputs[i], id(i));
|
|
4154
|
+
}
|
|
3934
4155
|
}
|
|
3935
4156
|
}
|
|
3936
4157
|
if (active.length > 0) {
|
|
@@ -4390,6 +4611,56 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
|
4390
4611
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
4391
4612
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
4392
4613
|
var STEP_FAN_CHUNK = 200;
|
|
4614
|
+
var PARENT_NOTIFY_ATTEMPTS = 4;
|
|
4615
|
+
var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
4616
|
+
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
4617
|
+
var LEASE_RECHECK_SLACK_MS = 100;
|
|
4618
|
+
var LEASE_RECHECK_ATTEMPTS = 30;
|
|
4619
|
+
function deferred() {
|
|
4620
|
+
let resolve;
|
|
4621
|
+
let reject;
|
|
4622
|
+
const promise = new Promise((res, rej) => {
|
|
4623
|
+
resolve = res;
|
|
4624
|
+
reject = rej;
|
|
4625
|
+
});
|
|
4626
|
+
promise.catch(() => void 0);
|
|
4627
|
+
return {
|
|
4628
|
+
promise,
|
|
4629
|
+
resolve,
|
|
4630
|
+
reject
|
|
4631
|
+
};
|
|
4632
|
+
}
|
|
4633
|
+
__name(deferred, "deferred");
|
|
4634
|
+
var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
|
|
4635
|
+
var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
|
|
4636
|
+
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4637
|
+
var DELIVER_CHUNK = 50;
|
|
4638
|
+
function childCompletion(child) {
|
|
4639
|
+
if (child.status === "completed") return {
|
|
4640
|
+
ok: true,
|
|
4641
|
+
value: child.output
|
|
4642
|
+
};
|
|
4643
|
+
if (child.status === "failed") return failedCompletion(child.error ?? {
|
|
4644
|
+
message: "failed"
|
|
4645
|
+
});
|
|
4646
|
+
if (child.status === "dead") return failedCompletion(child.error ?? {
|
|
4647
|
+
message: "dead"
|
|
4648
|
+
});
|
|
4649
|
+
if (child.status === "cancelled") return cancelledCompletion(child.error);
|
|
4650
|
+
return void 0;
|
|
4651
|
+
}
|
|
4652
|
+
__name(childCompletion, "childCompletion");
|
|
4653
|
+
var CHILD_CANCELLED = "cancelled";
|
|
4654
|
+
function cancelledCompletion(error2) {
|
|
4655
|
+
const base = error2 ?? {
|
|
4656
|
+
message: "cancelled"
|
|
4657
|
+
};
|
|
4658
|
+
return failedCompletion({
|
|
4659
|
+
...base,
|
|
4660
|
+
code: base.code ?? CHILD_CANCELLED
|
|
4661
|
+
});
|
|
4662
|
+
}
|
|
4663
|
+
__name(cancelledCompletion, "cancelledCompletion");
|
|
4393
4664
|
var RESULT_BATCH_MAX = 256;
|
|
4394
4665
|
function shuffle(items) {
|
|
4395
4666
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
@@ -4566,6 +4837,35 @@ var WorkflowEngine = class {
|
|
|
4566
4837
|
/** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
|
|
4567
4838
|
inflight = /* @__PURE__ */ new Set();
|
|
4568
4839
|
/**
|
|
4840
|
+
* Work a settled turn leaves running in the background (a parent notification, a post-release
|
|
4841
|
+
* re-drive), so a graceful shutdown waits for it too: dropping one on exit strands the run until
|
|
4842
|
+
* its reconcile timer.
|
|
4843
|
+
*/
|
|
4844
|
+
background = /* @__PURE__ */ new Set();
|
|
4845
|
+
/**
|
|
4846
|
+
* Runs with an implicit wake executing on this instance, and whether another wake arrived for the
|
|
4847
|
+
* run meanwhile (see {@link resumeImplicit}).
|
|
4848
|
+
*/
|
|
4849
|
+
wakeTurns = /* @__PURE__ */ new Map();
|
|
4850
|
+
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
4851
|
+
leaseRetries = /* @__PURE__ */ new Map();
|
|
4852
|
+
/**
|
|
4853
|
+
* Every run turn and child start this instance takes on itself, under one budget of
|
|
4854
|
+
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
4855
|
+
* enqueued pickups and the runs it takes off the transport's run queue.
|
|
4856
|
+
*/
|
|
4857
|
+
localWork;
|
|
4858
|
+
/** The dispatcher the caller supplied, if any; without one the engine picks (see {@link dispatchRun}). */
|
|
4859
|
+
customDispatcher;
|
|
4860
|
+
/** Runs on their way to the transport's run queue (see {@link consumeRuns}). */
|
|
4861
|
+
runBatcher;
|
|
4862
|
+
/** Set while this engine consumes a transport's run queue, and so dispatches its runs to it. */
|
|
4863
|
+
dispatchesOverTransport = false;
|
|
4864
|
+
/** Stops for the run-queue consumers started by {@link consumeRuns}. */
|
|
4865
|
+
runConsumerStops = [];
|
|
4866
|
+
/** The consumers being stopped, so {@link drain} waits for their runs in flight. */
|
|
4867
|
+
runConsumersStopping;
|
|
4868
|
+
/**
|
|
4569
4869
|
* The run lease of each execution in flight on THIS instance, by run id — renewed while it runs
|
|
4570
4870
|
* and checked by every settle (the zombie fence, see {@link ExecutionLease}). Also what `drain()`
|
|
4571
4871
|
* releases when its budget runs out.
|
|
@@ -4621,15 +4921,27 @@ var WorkflowEngine = class {
|
|
|
4621
4921
|
this.rehydrate = deps.rehydrate ?? ((_carrier, fn) => fn());
|
|
4622
4922
|
this.compensationRetries = Math.max(1, deps.compensationRetries ?? 1);
|
|
4623
4923
|
this.trackStepStart = deps.trackStepStart ?? true;
|
|
4624
|
-
this.
|
|
4625
|
-
|
|
4626
|
-
|
|
4924
|
+
this.localWork = new BoundedRunner(LOCAL_RUN_CONCURRENCY);
|
|
4925
|
+
this.customDispatcher = deps.runDispatcher;
|
|
4926
|
+
this.runBatcher = new RunDispatchBatcher((runs) => this.pool.dispatchRuns(runs, {
|
|
4927
|
+
namespace: this.namespace
|
|
4928
|
+
}), (runs, err) => {
|
|
4929
|
+
for (const run of runs) {
|
|
4930
|
+
if (run.reason === "wake" && !this.draining) {
|
|
4931
|
+
this.inBackground(this.resumeImplicit(run.runId));
|
|
4932
|
+
}
|
|
4933
|
+
}
|
|
4934
|
+
console.error(`[nestjs-durable] could not queue ${runs.length} run(s) on the transport; starts stay pending for the next poll, wakes resume here:`, err);
|
|
4935
|
+
});
|
|
4936
|
+
this.runDispatcher = {
|
|
4937
|
+
dispatch: /* @__PURE__ */ __name((runId) => this.dispatchRun(runId), "dispatch")
|
|
4627
4938
|
};
|
|
4628
4939
|
this.singletons = new SingletonGate({
|
|
4629
4940
|
store: this.store,
|
|
4630
4941
|
clock: this.clock,
|
|
4631
4942
|
dispatch: /* @__PURE__ */ __name((runId) => {
|
|
4632
|
-
this.
|
|
4943
|
+
if (this.queuesRunsOverTransport) void this.wake(runId);
|
|
4944
|
+
else this.runDispatcher.dispatch(runId);
|
|
4633
4945
|
}, "dispatch"),
|
|
4634
4946
|
configFor: /* @__PURE__ */ __name((run) => (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton, "configFor")
|
|
4635
4947
|
});
|
|
@@ -4754,13 +5066,224 @@ var WorkflowEngine = class {
|
|
|
4754
5066
|
}
|
|
4755
5067
|
/**
|
|
4756
5068
|
* Be notified when a run is enqueued on ANOTHER instance (via the control plane), so a worker can
|
|
4757
|
-
* pick it up at once — e.g. `engine.onEnqueued((runId) => engine.
|
|
5069
|
+
* pick it up at once — e.g. `engine.onEnqueued((runId) => engine.runEnqueued(runId))`. Returns an
|
|
4758
5070
|
* unsubscribe function. Only wire this on instances that should execute runs (workers).
|
|
4759
5071
|
*/
|
|
4760
5072
|
onEnqueued(listener) {
|
|
4761
5073
|
this.enqueuedListeners.add(listener);
|
|
4762
5074
|
return () => this.enqueuedListeners.delete(listener);
|
|
4763
5075
|
}
|
|
5076
|
+
/**
|
|
5077
|
+
* Pick up a run another instance announced as enqueued: run it here if it is still `pending`.
|
|
5078
|
+
* Bounded by {@link LOCAL_RUN_CONCURRENCY}, and a run another instance already took is skipped
|
|
5079
|
+
* after one read, instead of being leased and replayed here once more.
|
|
5080
|
+
*/
|
|
5081
|
+
runEnqueued(runId) {
|
|
5082
|
+
this.localWork.submit(`run:${runId}`, () => this.runIfPending(runId));
|
|
5083
|
+
}
|
|
5084
|
+
/**
|
|
5085
|
+
* Consume the run queue of every transport that carries one ({@link Transport.dispatchRuns} and
|
|
5086
|
+
* {@link Transport.onDispatchedRun}), and from then on dispatch the runs this engine starts there
|
|
5087
|
+
* rather than executing them in-process: a run, a `ctx.all` fan's children included, executes on
|
|
5088
|
+
* whichever consuming instance takes it, at most {@link LOCAL_RUN_CONCURRENCY} at once per
|
|
5089
|
+
* instance. Call it on every instance that drives runs; the Nest module does on a driving operator.
|
|
5090
|
+
*
|
|
5091
|
+
* Returns a function that stops consuming and resolves once the runs in flight settled, after
|
|
5092
|
+
* which runs are executed in-process again; {@link beginDrain} stops consuming too. Undefined
|
|
5093
|
+
* when no transport carries a run queue: runs keep executing in-process.
|
|
5094
|
+
*/
|
|
5095
|
+
consumeRuns() {
|
|
5096
|
+
if (!this.pool.dispatchesRuns) return void 0;
|
|
5097
|
+
if (this.runConsumerStops.length === 0) {
|
|
5098
|
+
this.runConsumerStops = this.pool.onDispatchedRun((run) => this.localWork.run(() => this.runFromQueue(run)), {
|
|
5099
|
+
concurrency: LOCAL_RUN_CONCURRENCY,
|
|
5100
|
+
namespace: this.namespace
|
|
5101
|
+
});
|
|
5102
|
+
}
|
|
5103
|
+
this.dispatchesOverTransport = true;
|
|
5104
|
+
return async () => {
|
|
5105
|
+
this.dispatchesOverTransport = false;
|
|
5106
|
+
await this.runBatcher.settle();
|
|
5107
|
+
await this.stopRunConsumers();
|
|
5108
|
+
};
|
|
5109
|
+
}
|
|
5110
|
+
/** Stop every run-queue consumer; resolves once their runs in flight settled. */
|
|
5111
|
+
stopRunConsumers() {
|
|
5112
|
+
if (this.runConsumerStops.length > 0) {
|
|
5113
|
+
const stops = this.runConsumerStops;
|
|
5114
|
+
this.runConsumerStops = [];
|
|
5115
|
+
this.runConsumersStopping = Promise.allSettled(stops.map((stop) => stop()));
|
|
5116
|
+
}
|
|
5117
|
+
return this.runConsumersStopping ?? Promise.resolve();
|
|
5118
|
+
}
|
|
5119
|
+
/** Whether {@link dispatchRun} queues runs on the transport (see {@link consumeRuns}). */
|
|
5120
|
+
get queuesRunsOverTransport() {
|
|
5121
|
+
return this.customDispatcher === void 0 && this.dispatchesOverTransport;
|
|
5122
|
+
}
|
|
5123
|
+
/**
|
|
5124
|
+
* Hand a `pending` run to whatever executes it: the caller's {@link WorkflowEngineDeps.runDispatcher};
|
|
5125
|
+
* else the transport's run queue while this engine consumes it, batched; else this instance,
|
|
5126
|
+
* bounded by {@link LOCAL_RUN_CONCURRENCY}. A run whose queueing fails stays `pending` for a
|
|
5127
|
+
* `runPending` poll.
|
|
5128
|
+
*/
|
|
5129
|
+
dispatchRun(runId, priority, key) {
|
|
5130
|
+
if (this.customDispatcher) return this.customDispatcher.dispatch(runId);
|
|
5131
|
+
if (this.dispatchesOverTransport) {
|
|
5132
|
+
this.runBatcher.add({
|
|
5133
|
+
runId,
|
|
5134
|
+
reason: "start",
|
|
5135
|
+
...priority !== void 0 ? {
|
|
5136
|
+
priority
|
|
5137
|
+
} : {},
|
|
5138
|
+
...key !== void 0 ? {
|
|
5139
|
+
key
|
|
5140
|
+
} : {}
|
|
5141
|
+
});
|
|
5142
|
+
return;
|
|
5143
|
+
}
|
|
5144
|
+
this.localWork.submit(`run:${runId}`, () => this.runOne(runId));
|
|
5145
|
+
}
|
|
5146
|
+
/**
|
|
5147
|
+
* Wake a suspended run that has something new to see (a delivered signal or child completion, a
|
|
5148
|
+
* lease that came free, a freed slot): while this engine consumes the transport's run queue, on
|
|
5149
|
+
* whichever instance takes the wake (the result then only says it was handed over); otherwise
|
|
5150
|
+
* here, now.
|
|
5151
|
+
*/
|
|
5152
|
+
wake(runId) {
|
|
5153
|
+
if (this.queuesRunsOverTransport) {
|
|
5154
|
+
this.runBatcher.add({
|
|
5155
|
+
runId,
|
|
5156
|
+
reason: "wake"
|
|
5157
|
+
});
|
|
5158
|
+
return Promise.resolve({
|
|
5159
|
+
runId,
|
|
5160
|
+
status: "suspended"
|
|
5161
|
+
});
|
|
5162
|
+
}
|
|
5163
|
+
return this.resumeImplicit(runId);
|
|
5164
|
+
}
|
|
5165
|
+
/**
|
|
5166
|
+
* Take a turn of a run taken off the transport's run queue. A `start` runs only while the run is
|
|
5167
|
+
* still `pending` (or `cancelling`, a recovered cancel): any other state means another instance
|
|
5168
|
+
* already took it. A `wake` leases and resumes the run unless it is finished. The run lease admits
|
|
5169
|
+
* one executor either way. Never rejects.
|
|
5170
|
+
*/
|
|
5171
|
+
async runFromQueue(dispatch) {
|
|
5172
|
+
if (this.draining) return;
|
|
5173
|
+
const { runId } = dispatch;
|
|
5174
|
+
try {
|
|
5175
|
+
const run = await this.store.getRun(runId);
|
|
5176
|
+
if (!run) return;
|
|
5177
|
+
if (run.status === "pending" || run.status === "cancelling") {
|
|
5178
|
+
await this.runOne(runId);
|
|
5179
|
+
return;
|
|
5180
|
+
}
|
|
5181
|
+
if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
5182
|
+
const nowMs = this.clock();
|
|
5183
|
+
if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
|
|
5184
|
+
const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
5185
|
+
runId
|
|
5186
|
+
]));
|
|
5187
|
+
const resumed = await this.resumeLeased([
|
|
5188
|
+
run
|
|
5189
|
+
], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
|
|
5190
|
+
if (resumed.length === 0) this.recheckAfterLease(runId, run.lockedUntil);
|
|
5191
|
+
return;
|
|
5192
|
+
}
|
|
5193
|
+
await this.resumeImplicit(runId).catch((err) => {
|
|
5194
|
+
if (!(err instanceof NamespaceMismatch)) throw err;
|
|
5195
|
+
});
|
|
5196
|
+
} catch (error2) {
|
|
5197
|
+
console.error(`[nestjs-durable] could not run queued run ${runId}:`, error2);
|
|
5198
|
+
}
|
|
5199
|
+
}
|
|
5200
|
+
/**
|
|
5201
|
+
* Hand due runs (timers, blocked runs due for a fleet re-check) to whichever instance takes them
|
|
5202
|
+
* off the run queue. Each is claimed first, under its lease, by pushing its `wakeAt` one lease
|
|
5203
|
+
* period out, so the next poll (here or elsewhere) does not queue it again while it waits; a wake
|
|
5204
|
+
* that is lost brings it back due after that period. False when the transport refused the wakes,
|
|
5205
|
+
* so the caller resumes them itself.
|
|
5206
|
+
*/
|
|
5207
|
+
async handOffDue(runs, nowMs) {
|
|
5208
|
+
const claimed = [];
|
|
5209
|
+
for (const run of runs) {
|
|
5210
|
+
if (this.draining) break;
|
|
5211
|
+
if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5212
|
+
continue;
|
|
5213
|
+
}
|
|
5214
|
+
try {
|
|
5215
|
+
const patch = {
|
|
5216
|
+
wakeAt: nowMs + this.leaseMs
|
|
5217
|
+
};
|
|
5218
|
+
if (this.store.updateRunIf) {
|
|
5219
|
+
if (await this.store.updateRunIf(run.id, [
|
|
5220
|
+
run.status
|
|
5221
|
+
], patch)) claimed.push(run);
|
|
5222
|
+
} else {
|
|
5223
|
+
await this.store.updateRun(run.id, patch);
|
|
5224
|
+
claimed.push(run);
|
|
5225
|
+
}
|
|
5226
|
+
} finally {
|
|
5227
|
+
await this.store.releaseRunLock(run.id);
|
|
5228
|
+
}
|
|
5229
|
+
}
|
|
5230
|
+
try {
|
|
5231
|
+
for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
|
|
5232
|
+
await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
|
|
5233
|
+
runId: run.id,
|
|
5234
|
+
reason: "wake",
|
|
5235
|
+
...run.priority !== void 0 ? {
|
|
5236
|
+
priority: run.priority
|
|
5237
|
+
} : {}
|
|
5238
|
+
})), {
|
|
5239
|
+
namespace: this.namespace
|
|
5240
|
+
});
|
|
5241
|
+
}
|
|
5242
|
+
return true;
|
|
5243
|
+
} catch (error2) {
|
|
5244
|
+
console.error("[nestjs-durable] could not queue due runs on the transport:", error2);
|
|
5245
|
+
return false;
|
|
5246
|
+
}
|
|
5247
|
+
}
|
|
5248
|
+
/**
|
|
5249
|
+
* Queue `runs` on the transport for whichever consuming instance takes them, in batches. False
|
|
5250
|
+
* when the transport refused them, so the caller executes them itself instead.
|
|
5251
|
+
*/
|
|
5252
|
+
async queueOnTransport(runs) {
|
|
5253
|
+
try {
|
|
5254
|
+
for (let at = 0; at < runs.length; at += RUN_DISPATCH_BATCH) {
|
|
5255
|
+
await this.pool.dispatchRuns(runs.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
|
|
5256
|
+
runId: run.id,
|
|
5257
|
+
reason: "start",
|
|
5258
|
+
key: pendingStateKey(run.updatedAt),
|
|
5259
|
+
...run.priority !== void 0 ? {
|
|
5260
|
+
priority: run.priority
|
|
5261
|
+
} : {}
|
|
5262
|
+
})), {
|
|
5263
|
+
namespace: this.namespace
|
|
5264
|
+
});
|
|
5265
|
+
}
|
|
5266
|
+
return true;
|
|
5267
|
+
} catch (error2) {
|
|
5268
|
+
console.error("[nestjs-durable] could not queue runs on the transport; running them here:", error2);
|
|
5269
|
+
return false;
|
|
5270
|
+
}
|
|
5271
|
+
}
|
|
5272
|
+
/**
|
|
5273
|
+
* Execute a run announced by another instance, if it is still `pending`: a
|
|
5274
|
+
* run another instance already took is skipped after one read instead of being leased and
|
|
5275
|
+
* replayed here once more. Never rejects.
|
|
5276
|
+
*/
|
|
5277
|
+
async runIfPending(runId) {
|
|
5278
|
+
if (this.draining) return;
|
|
5279
|
+
try {
|
|
5280
|
+
const run = await this.store.getRun(runId);
|
|
5281
|
+
if (run?.status !== "pending") return;
|
|
5282
|
+
await this.runOne(runId);
|
|
5283
|
+
} catch (error2) {
|
|
5284
|
+
console.error(`[nestjs-durable] could not run enqueued run ${runId}:`, error2);
|
|
5285
|
+
}
|
|
5286
|
+
}
|
|
4764
5287
|
/** Fire cooperative-cancellation listeners for `runId` (a worker bridge aborts in-flight work). */
|
|
4765
5288
|
notifyCancelled(runId) {
|
|
4766
5289
|
for (const fn of this.cancelListeners) {
|
|
@@ -5156,8 +5679,8 @@ var WorkflowEngine = class {
|
|
|
5156
5679
|
status: "suspended"
|
|
5157
5680
|
};
|
|
5158
5681
|
}
|
|
5159
|
-
await this.
|
|
5160
|
-
if (this.controlPlane) {
|
|
5682
|
+
await this.dispatchRun(runId, run.priority, pendingStateKey(run.updatedAt));
|
|
5683
|
+
if (this.controlPlane && !this.queuesRunsOverTransport) {
|
|
5161
5684
|
void this.controlPlane.publishControl({
|
|
5162
5685
|
kind: "enqueued",
|
|
5163
5686
|
runId,
|
|
@@ -5196,7 +5719,43 @@ var WorkflowEngine = class {
|
|
|
5196
5719
|
* stale waiter or a late result left behind by a failed run silently re-executes its body.
|
|
5197
5720
|
*/
|
|
5198
5721
|
resumeImplicit(runId) {
|
|
5199
|
-
|
|
5722
|
+
const turn = this.wakeTurns.get(runId);
|
|
5723
|
+
if (turn) {
|
|
5724
|
+
turn.again ??= deferred();
|
|
5725
|
+
if (currentWorkflowCtx()?.runId === runId) {
|
|
5726
|
+
return Promise.resolve({
|
|
5727
|
+
runId,
|
|
5728
|
+
status: "running"
|
|
5729
|
+
});
|
|
5730
|
+
}
|
|
5731
|
+
return turn.again.promise;
|
|
5732
|
+
}
|
|
5733
|
+
const mine = {};
|
|
5734
|
+
this.wakeTurns.set(runId, mine);
|
|
5735
|
+
const finish = /* @__PURE__ */ __name(() => {
|
|
5736
|
+
if (this.wakeTurns.get(runId) === mine) this.wakeTurns.delete(runId);
|
|
5737
|
+
const again = mine.again;
|
|
5738
|
+
if (!again) return;
|
|
5739
|
+
if (this.draining) {
|
|
5740
|
+
again.resolve({
|
|
5741
|
+
runId,
|
|
5742
|
+
status: "running"
|
|
5743
|
+
});
|
|
5744
|
+
return;
|
|
5745
|
+
}
|
|
5746
|
+
const next = this.resumeImplicit(runId);
|
|
5747
|
+
next.then(again.resolve, again.reject);
|
|
5748
|
+
this.inBackground(next);
|
|
5749
|
+
}, "finish");
|
|
5750
|
+
const result = this.resumeRun(runId, false);
|
|
5751
|
+
void result.then(finish, finish);
|
|
5752
|
+
return result;
|
|
5753
|
+
}
|
|
5754
|
+
/** Keep `work` in {@link background} until it settles; its failure is swallowed. */
|
|
5755
|
+
inBackground(work) {
|
|
5756
|
+
const tracked = work.catch(() => void 0);
|
|
5757
|
+
this.background.add(tracked);
|
|
5758
|
+
void tracked.then(() => this.background.delete(tracked));
|
|
5200
5759
|
}
|
|
5201
5760
|
/**
|
|
5202
5761
|
* One execution of a run at a time in this process. A resume that lands while the run executes
|
|
@@ -5483,6 +6042,10 @@ var WorkflowEngine = class {
|
|
|
5483
6042
|
*/
|
|
5484
6043
|
beginDrain(spentMs = 0) {
|
|
5485
6044
|
this.draining = true;
|
|
6045
|
+
if (this.runConsumerStops.length > 0) {
|
|
6046
|
+
this.runBatcher.flush();
|
|
6047
|
+
void this.stopRunConsumers();
|
|
6048
|
+
}
|
|
5486
6049
|
if (Number.isFinite(spentMs) && spentMs > 0) {
|
|
5487
6050
|
this.drainSpentMs += spentMs;
|
|
5488
6051
|
for (const rearm of this.drainRearms) rearm();
|
|
@@ -5503,7 +6066,14 @@ var WorkflowEngine = class {
|
|
|
5503
6066
|
*/
|
|
5504
6067
|
async drain(timeoutMs = 1e4) {
|
|
5505
6068
|
this.beginDrain();
|
|
5506
|
-
|
|
6069
|
+
this.localWork.clear();
|
|
6070
|
+
for (const timer2 of this.leaseRetries.values()) clearTimeout(timer2);
|
|
6071
|
+
this.leaseRetries.clear();
|
|
6072
|
+
const consumers = this.runConsumersStopping;
|
|
6073
|
+
if (this.inflight.size === 0 && this.background.size === 0 && !consumers && this.runBatcher.idle) {
|
|
6074
|
+
return;
|
|
6075
|
+
}
|
|
6076
|
+
const queued = this.runBatcher.settle();
|
|
5507
6077
|
const start = Date.now();
|
|
5508
6078
|
let rearm = /* @__PURE__ */ __name(() => {
|
|
5509
6079
|
}, "rearm");
|
|
@@ -5522,7 +6092,10 @@ var WorkflowEngine = class {
|
|
|
5522
6092
|
try {
|
|
5523
6093
|
await Promise.race([
|
|
5524
6094
|
Promise.allSettled([
|
|
5525
|
-
...this.inflight
|
|
6095
|
+
...this.inflight,
|
|
6096
|
+
...this.background,
|
|
6097
|
+
queued,
|
|
6098
|
+
consumers
|
|
5526
6099
|
]).then(() => {
|
|
5527
6100
|
settled = true;
|
|
5528
6101
|
}),
|
|
@@ -5629,6 +6202,7 @@ var WorkflowEngine = class {
|
|
|
5629
6202
|
error: error2
|
|
5630
6203
|
});
|
|
5631
6204
|
await this.cancelChildren(run.id).catch(() => void 0);
|
|
6205
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
5632
6206
|
}
|
|
5633
6207
|
}
|
|
5634
6208
|
}
|
|
@@ -5643,7 +6217,9 @@ var WorkflowEngine = class {
|
|
|
5643
6217
|
if (this.draining) break;
|
|
5644
6218
|
const outcome = this.isOwnLiveLease(listed, nowMs) ? await this.reclaimOwnLease(listed, nowMs) : await this.recoverOrphan(listed, nowMs);
|
|
5645
6219
|
if (!outcome) continue;
|
|
5646
|
-
if (outcome.dispatch)
|
|
6220
|
+
if (outcome.dispatch) {
|
|
6221
|
+
await this.dispatchRun(outcome.result.runId, listed.priority, freshPendingKey());
|
|
6222
|
+
}
|
|
5647
6223
|
results.push(outcome.result);
|
|
5648
6224
|
}
|
|
5649
6225
|
return results;
|
|
@@ -5835,6 +6411,18 @@ var WorkflowEngine = class {
|
|
|
5835
6411
|
const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
|
|
5836
6412
|
const blocked = await this.dueBlockedRuns(nowMs);
|
|
5837
6413
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
6414
|
+
if (this.queuesRunsOverTransport && !this.draining) {
|
|
6415
|
+
const runs = [
|
|
6416
|
+
...due,
|
|
6417
|
+
...blocked
|
|
6418
|
+
];
|
|
6419
|
+
if (await this.handOffDue(runs, nowMs)) {
|
|
6420
|
+
return runs.map((run) => ({
|
|
6421
|
+
runId: run.id,
|
|
6422
|
+
status: run.status
|
|
6423
|
+
}));
|
|
6424
|
+
}
|
|
6425
|
+
}
|
|
5838
6426
|
const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
5839
6427
|
...due,
|
|
5840
6428
|
...blocked
|
|
@@ -5868,7 +6456,7 @@ var WorkflowEngine = class {
|
|
|
5868
6456
|
const buffered = await this.store.takeBufferedSignal(waiter.token);
|
|
5869
6457
|
if (!buffered) return void 0;
|
|
5870
6458
|
await this.store.removeSignalWaiter(waiter);
|
|
5871
|
-
const settled = await this.deliverSignal(waiter, buffered.payload);
|
|
6459
|
+
const settled = await this.deliverSignal(waiter, buffered.payload, true);
|
|
5872
6460
|
return settled ?? {
|
|
5873
6461
|
runId: run.id,
|
|
5874
6462
|
status: run.status
|
|
@@ -5894,7 +6482,7 @@ var WorkflowEngine = class {
|
|
|
5894
6482
|
if (!eventMatches(candidate.payload, match)) continue;
|
|
5895
6483
|
if (!await this.store.removeBufferedEvent(candidate.id)) continue;
|
|
5896
6484
|
await this.store.removeSignalWaiter(waiter);
|
|
5897
|
-
const settled = await this.deliverSignal(waiter, candidate.payload);
|
|
6485
|
+
const settled = await this.deliverSignal(waiter, candidate.payload, true);
|
|
5898
6486
|
return settled ?? {
|
|
5899
6487
|
runId: run.id,
|
|
5900
6488
|
status: run.status
|
|
@@ -6014,7 +6602,7 @@ var WorkflowEngine = class {
|
|
|
6014
6602
|
} : {},
|
|
6015
6603
|
updatedAt: /* @__PURE__ */ new Date()
|
|
6016
6604
|
});
|
|
6017
|
-
await this.
|
|
6605
|
+
await this.dispatchRun(runId, run.priority, freshPendingKey());
|
|
6018
6606
|
return {
|
|
6019
6607
|
runId,
|
|
6020
6608
|
status: "pending"
|
|
@@ -6089,7 +6677,16 @@ var WorkflowEngine = class {
|
|
|
6089
6677
|
* {@link resumeDueTimers}.
|
|
6090
6678
|
*/
|
|
6091
6679
|
async runPending(nowMs = this.clock()) {
|
|
6092
|
-
|
|
6680
|
+
const pending = await this.store.listPendingRuns(100, this.namespace);
|
|
6681
|
+
if (this.queuesRunsOverTransport && !this.draining && pending.length > 0) {
|
|
6682
|
+
if (await this.queueOnTransport(pending)) {
|
|
6683
|
+
return pending.map((run) => ({
|
|
6684
|
+
runId: run.id,
|
|
6685
|
+
status: run.status
|
|
6686
|
+
}));
|
|
6687
|
+
}
|
|
6688
|
+
}
|
|
6689
|
+
return this.resumeLeased(pending, nowMs);
|
|
6093
6690
|
}
|
|
6094
6691
|
/**
|
|
6095
6692
|
* Resolve once `runId` reaches a settled state — terminal (completed/failed/cancelled/dead) or
|
|
@@ -6266,7 +6863,7 @@ var WorkflowEngine = class {
|
|
|
6266
6863
|
* due-timer reconcile sweep ({@link reconcileSignalWaiter}) — all three resolve a waiter the exact
|
|
6267
6864
|
* same way.
|
|
6268
6865
|
*/
|
|
6269
|
-
async deliverSignal(waiter, payload) {
|
|
6866
|
+
async deliverSignal(waiter, payload, resumeHere = false) {
|
|
6270
6867
|
await this.store.saveCheckpoint(instantCheckpoint({
|
|
6271
6868
|
runId: waiter.runId,
|
|
6272
6869
|
seq: waiter.seq,
|
|
@@ -6285,7 +6882,7 @@ var WorkflowEngine = class {
|
|
|
6285
6882
|
status: "suspended"
|
|
6286
6883
|
};
|
|
6287
6884
|
}
|
|
6288
|
-
return this.resumeImplicit(waiter.runId).catch((err) => {
|
|
6885
|
+
return (resumeHere ? this.resumeImplicit(waiter.runId) : this.wake(waiter.runId)).catch((err) => {
|
|
6289
6886
|
if (err instanceof NamespaceMismatch) return null;
|
|
6290
6887
|
throw err;
|
|
6291
6888
|
});
|
|
@@ -6372,6 +6969,101 @@ var WorkflowEngine = class {
|
|
|
6372
6969
|
return;
|
|
6373
6970
|
}
|
|
6374
6971
|
}
|
|
6972
|
+
/**
|
|
6973
|
+
* Deliver the outcome of every awaited child that already finished but whose completion never
|
|
6974
|
+
* reached this run, into `checkpoints` (this turn's snapshot) and the store.
|
|
6975
|
+
*
|
|
6976
|
+
* A child persists its terminal status first and tells its parent after ({@link notifyParent}). A
|
|
6977
|
+
* crash, a scale-down or a store error between the two leaves the parent's `running` placeholder
|
|
6978
|
+
* and `child:<id>` waiter behind for a child that is done, and nothing else would ever settle
|
|
6979
|
+
* them: the parent's reconcile wake replays straight back to the same wait. So each turn looks up
|
|
6980
|
+
* the children of its `running` placeholders (one batched read per 500) and settles the finished
|
|
6981
|
+
* ones exactly as the child's own notification would have: the `signal:child:<id>` checkpoint
|
|
6982
|
+
* with the child's completion, then the waiter removed. A notification arriving late writes the
|
|
6983
|
+
* same checkpoint and finds no waiter. Best effort: a store error leaves the rest to the next turn.
|
|
6984
|
+
*/
|
|
6985
|
+
async deliverFinishedChildren(runId, checkpoints) {
|
|
6986
|
+
const awaited = [];
|
|
6987
|
+
for (const cp of checkpoints) {
|
|
6988
|
+
if (cp.kind !== "signal" || cp.status !== "running") continue;
|
|
6989
|
+
if (!cp.name.startsWith(CHILD_SIGNAL_PREFIX)) continue;
|
|
6990
|
+
awaited.push({
|
|
6991
|
+
childId: cp.name.slice(CHILD_SIGNAL_PREFIX.length),
|
|
6992
|
+
seq: cp.seq,
|
|
6993
|
+
parallelGroup: cp.parallelGroup
|
|
6994
|
+
});
|
|
6995
|
+
}
|
|
6996
|
+
await this.settleFinishedChildren(runId, awaited, checkpoints);
|
|
6997
|
+
}
|
|
6998
|
+
/**
|
|
6999
|
+
* {@link deliverFinishedChildren} for a remote workflow, whose awaited children (`startChild`
|
|
7000
|
+
* commands) leave a `child:<id>` waiter but no placeholder checkpoint: the run's own waiters say
|
|
7001
|
+
* which children it is waiting on. One indexed read per turn of a remote run.
|
|
7002
|
+
*/
|
|
7003
|
+
async deliverFinishedChildWaiters(runId, checkpoints) {
|
|
7004
|
+
let waiters;
|
|
7005
|
+
try {
|
|
7006
|
+
waiters = await listSignalWaitersOfRuns(this.store, [
|
|
7007
|
+
runId
|
|
7008
|
+
]);
|
|
7009
|
+
} catch {
|
|
7010
|
+
return;
|
|
7011
|
+
}
|
|
7012
|
+
const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
|
|
7013
|
+
const awaited = waiters.filter((w) => w.token.startsWith("child:") && !settled.has(w.seq)).map((w) => ({
|
|
7014
|
+
childId: w.token.slice("child:".length),
|
|
7015
|
+
seq: w.seq,
|
|
7016
|
+
parallelGroup: w.parallelGroup
|
|
7017
|
+
}));
|
|
7018
|
+
await this.settleFinishedChildren(runId, awaited, checkpoints);
|
|
7019
|
+
}
|
|
7020
|
+
/**
|
|
7021
|
+
* Settle every child in `awaited` that has finished: its `signal:child:<id>` checkpoint with the
|
|
7022
|
+
* child's completion, then its waiter removed, exactly as the child's own notification does, and
|
|
7023
|
+
* the checkpoint put into `checkpoints`. Children are read in batches of 500.
|
|
7024
|
+
*/
|
|
7025
|
+
async settleFinishedChildren(runId, awaited, checkpoints) {
|
|
7026
|
+
if (awaited.length === 0) return;
|
|
7027
|
+
try {
|
|
7028
|
+
const { runs: children } = await readRunsByIds(this.store, awaited.map((a) => a.childId));
|
|
7029
|
+
const finished = awaited.flatMap((a) => {
|
|
7030
|
+
const child = children.get(a.childId);
|
|
7031
|
+
const completion = child ? childCompletion(child) : void 0;
|
|
7032
|
+
return completion ? [
|
|
7033
|
+
{
|
|
7034
|
+
...a,
|
|
7035
|
+
completion
|
|
7036
|
+
}
|
|
7037
|
+
] : [];
|
|
7038
|
+
});
|
|
7039
|
+
for (let at = 0; at < finished.length; at += DELIVER_CHUNK) {
|
|
7040
|
+
await Promise.all(finished.slice(at, at + DELIVER_CHUNK).map(async (a) => {
|
|
7041
|
+
const settled = instantCheckpoint({
|
|
7042
|
+
runId,
|
|
7043
|
+
seq: a.seq,
|
|
7044
|
+
name: `${CHILD_SIGNAL_PREFIX}${a.childId}`,
|
|
7045
|
+
kind: "signal",
|
|
7046
|
+
output: a.completion,
|
|
7047
|
+
parallelGroup: a.parallelGroup
|
|
7048
|
+
});
|
|
7049
|
+
await this.store.saveCheckpoint(settled);
|
|
7050
|
+
await this.store.removeSignalWaiter({
|
|
7051
|
+
token: `child:${a.childId}`,
|
|
7052
|
+
runId,
|
|
7053
|
+
seq: a.seq,
|
|
7054
|
+
...a.parallelGroup !== void 0 ? {
|
|
7055
|
+
parallelGroup: a.parallelGroup
|
|
7056
|
+
} : {}
|
|
7057
|
+
});
|
|
7058
|
+
const index = checkpoints.findIndex((cp) => cp.seq === a.seq);
|
|
7059
|
+
if (index === -1) checkpoints.push(settled);
|
|
7060
|
+
else checkpoints[index] = settled;
|
|
7061
|
+
}));
|
|
7062
|
+
}
|
|
7063
|
+
checkpoints.sort((a, b) => a.seq - b.seq);
|
|
7064
|
+
} catch {
|
|
7065
|
+
}
|
|
7066
|
+
}
|
|
6375
7067
|
async redriveSettledChildren(runId, fan) {
|
|
6376
7068
|
if (!this.store.hasSignalWaiter) return;
|
|
6377
7069
|
for (const wait of fan.waits) {
|
|
@@ -6470,13 +7162,63 @@ var WorkflowEngine = class {
|
|
|
6470
7162
|
* knowing about the child feature.
|
|
6471
7163
|
*/
|
|
6472
7164
|
notifyParent(runId, completion) {
|
|
6473
|
-
|
|
7165
|
+
this.inBackground(this.deliverToParent(`child:${runId}`, completion));
|
|
6474
7166
|
const at = runId.lastIndexOf("~retry~");
|
|
6475
7167
|
if (at !== -1 && completion.ok) {
|
|
6476
|
-
|
|
7168
|
+
this.inBackground(this.deliverToParent(`child:${runId.slice(0, at)}`, completion));
|
|
6477
7169
|
}
|
|
6478
7170
|
}
|
|
6479
7171
|
/**
|
|
7172
|
+
* {@link signal} a child's completion to its parent, retrying a store error a few times. The child
|
|
7173
|
+
* is already persisted terminal, so a failure here would leave the parent waiting on a child that
|
|
7174
|
+
* finished; the parent's own turns reconcile that ({@link deliverFinishedChildren}), but only on
|
|
7175
|
+
* its next wake. A waiter already taken is delivered to directly on a retry, so the completion is
|
|
7176
|
+
* not buffered for a waiter that no longer exists.
|
|
7177
|
+
*/
|
|
7178
|
+
async deliverToParent(token, completion) {
|
|
7179
|
+
let waiter;
|
|
7180
|
+
const cancelled = !completion.ok && completion.code === CHILD_CANCELLED;
|
|
7181
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
7182
|
+
try {
|
|
7183
|
+
if (waiter === void 0) waiter = await this.store.takeSignalWaiter(token);
|
|
7184
|
+
if (waiter) {
|
|
7185
|
+
const parent = cancelled ? await this.store.getRun(waiter.runId) : void 0;
|
|
7186
|
+
if (parent && TERMINAL_RUN_STATUSES.includes(parent.status)) return;
|
|
7187
|
+
await this.deliverSignal(waiter, completion);
|
|
7188
|
+
return;
|
|
7189
|
+
}
|
|
7190
|
+
if (cancelled || await this.parentHoldsCompletion(token, completion)) return;
|
|
7191
|
+
await this.signal(token, completion);
|
|
7192
|
+
return;
|
|
7193
|
+
} catch (err) {
|
|
7194
|
+
if (err instanceof NamespaceMismatch) return;
|
|
7195
|
+
if (attempt >= PARENT_NOTIFY_ATTEMPTS || this.draining) return;
|
|
7196
|
+
await new Promise((resolve) => {
|
|
7197
|
+
const timer = setTimeout(resolve, PARENT_NOTIFY_BACKOFF_MS * 4 ** (attempt - 1));
|
|
7198
|
+
timer.unref?.();
|
|
7199
|
+
});
|
|
7200
|
+
}
|
|
7201
|
+
}
|
|
7202
|
+
}
|
|
7203
|
+
/**
|
|
7204
|
+
* Whether the parent of the child behind `token` (`child:<id>`) already holds `completion` as that
|
|
7205
|
+
* child's settled `signal:child:<id>` checkpoint: its own turn settled the child from the child's
|
|
7206
|
+
* row, so this notification is a duplicate. A different completion (a fix-and-replay run's success
|
|
7207
|
+
* for a child the parent saw fail) is not, and is buffered for the parent's retry. Only for a child
|
|
7208
|
+
* whose id the engine minted (its parent is derivable from it).
|
|
7209
|
+
*/
|
|
7210
|
+
async parentHoldsCompletion(token, completion) {
|
|
7211
|
+
const childId = token.slice("child:".length);
|
|
7212
|
+
const parentId = parentRunIdOf(childId);
|
|
7213
|
+
if (!parentId) return false;
|
|
7214
|
+
const name = `${CHILD_SIGNAL_PREFIX}${childId}`;
|
|
7215
|
+
const checkpoints = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(parentId, [
|
|
7216
|
+
name
|
|
7217
|
+
]) : await this.store.listCheckpoints(parentId);
|
|
7218
|
+
const expected = JSON.stringify(completion);
|
|
7219
|
+
return checkpoints.some((cp) => cp.name === name && cp.status === "completed" && JSON.stringify(cp.output) === expected);
|
|
7220
|
+
}
|
|
7221
|
+
/**
|
|
6480
7222
|
* Deferred child start shared by the in-process ctx host and the remote `startChild` command.
|
|
6481
7223
|
* Deferred (microtask) so a fast child can't reentrantly resume a still-suspending parent. A start
|
|
6482
7224
|
* that THROWS (unregistered/unroutable workflow, input validation, singleton back-pressure, store
|
|
@@ -6489,13 +7231,16 @@ var WorkflowEngine = class {
|
|
|
6489
7231
|
* correctly observes the failed start.
|
|
6490
7232
|
*/
|
|
6491
7233
|
startChildDeferred(workflow, input, childId, opts) {
|
|
6492
|
-
|
|
6493
|
-
|
|
6494
|
-
this.
|
|
6495
|
-
|
|
6496
|
-
|
|
7234
|
+
this.localWork.submit(`start:${childId}`, async () => {
|
|
7235
|
+
if (this.draining) return;
|
|
7236
|
+
await this.start(workflow, input, childId, opts).catch((err) => {
|
|
7237
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
7238
|
+
this.notifyParent(childId, {
|
|
7239
|
+
ok: false,
|
|
7240
|
+
error: `child workflow "${workflow}" failed to start: ${message}`
|
|
7241
|
+
});
|
|
6497
7242
|
});
|
|
6498
|
-
})
|
|
7243
|
+
});
|
|
6499
7244
|
}
|
|
6500
7245
|
/**
|
|
6501
7246
|
* Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
|
|
@@ -6559,6 +7304,7 @@ var WorkflowEngine = class {
|
|
|
6559
7304
|
error: error2
|
|
6560
7305
|
});
|
|
6561
7306
|
await this.cancelChildren(runId, opts);
|
|
7307
|
+
this.notifyParent(runId, cancelledCompletion(error2));
|
|
6562
7308
|
this.notifyCancelled(runId);
|
|
6563
7309
|
if (this.controlPlane) {
|
|
6564
7310
|
void this.controlPlane.publishControl({
|
|
@@ -7073,6 +7819,7 @@ var WorkflowEngine = class {
|
|
|
7073
7819
|
const lockNow = this.clock();
|
|
7074
7820
|
if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
|
|
7075
7821
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
7822
|
+
this.recheckAfterLease(run.id, run.lockedUntil);
|
|
7076
7823
|
return {
|
|
7077
7824
|
runId: run.id,
|
|
7078
7825
|
status: run.status
|
|
@@ -7091,6 +7838,51 @@ var WorkflowEngine = class {
|
|
|
7091
7838
|
}
|
|
7092
7839
|
}
|
|
7093
7840
|
/**
|
|
7841
|
+
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
7842
|
+
* holder's turn either sees what the wake wrote, or re-checks once it releases the lease
|
|
7843
|
+
* ({@link redriveSettledSteps}, {@link redriveSettledChildren}). Not if the holder goes down first
|
|
7844
|
+
* (a scale-down mid-turn, an OOM): its lease lapses with the run `suspended` on a step that has
|
|
7845
|
+
* finished, and nothing wakes it before its timer.
|
|
7846
|
+
*
|
|
7847
|
+
* So the dropped wake is kept here: once the lease is free (looked at every
|
|
7848
|
+
* {@link LEASE_RECHECK_MAX_MS} at most, for a bounded number of times), a run still `suspended` is
|
|
7849
|
+
* leased and resumed. A holder that already moved the run on makes this one spare replay that
|
|
7850
|
+
* re-parks. Best effort; the run's timer remains the fallback if this instance goes down too.
|
|
7851
|
+
*/
|
|
7852
|
+
recheckAfterLease(runId, lockedUntil, attempt = 1) {
|
|
7853
|
+
if (this.draining || this.leaseRetries.has(runId)) return;
|
|
7854
|
+
const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
|
|
7855
|
+
const timer = setTimeout(() => {
|
|
7856
|
+
this.leaseRetries.delete(runId);
|
|
7857
|
+
this.inBackground(this.retryDroppedWake(runId, attempt));
|
|
7858
|
+
}, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
|
|
7859
|
+
timer.unref?.();
|
|
7860
|
+
this.leaseRetries.set(runId, timer);
|
|
7861
|
+
}
|
|
7862
|
+
async retryDroppedWake(runId, attempt) {
|
|
7863
|
+
if (this.draining) return;
|
|
7864
|
+
const run = await this.store.getRun(runId);
|
|
7865
|
+
if (run?.status !== "suspended") return;
|
|
7866
|
+
const nowMs = this.clock();
|
|
7867
|
+
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
|
|
7868
|
+
if (attempt < LEASE_RECHECK_ATTEMPTS) this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
|
|
7869
|
+
return;
|
|
7870
|
+
}
|
|
7871
|
+
if (this.queuesRunsOverTransport) {
|
|
7872
|
+
await this.wake(runId);
|
|
7873
|
+
return;
|
|
7874
|
+
}
|
|
7875
|
+
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) return;
|
|
7876
|
+
const fresh = await this.store.getRun(runId);
|
|
7877
|
+
if (fresh?.status !== "suspended") {
|
|
7878
|
+
await this.store.releaseRunLock(runId);
|
|
7879
|
+
return;
|
|
7880
|
+
}
|
|
7881
|
+
await this.resumeImplicit(runId).catch((err) => {
|
|
7882
|
+
if (!(err instanceof NamespaceMismatch)) throw err;
|
|
7883
|
+
});
|
|
7884
|
+
}
|
|
7885
|
+
/**
|
|
7094
7886
|
* Renew `runId`'s lease for as long as this execution holds it, and register it in
|
|
7095
7887
|
* {@link executionLeases} so settles can check the zombie fence and `drain()` can release it.
|
|
7096
7888
|
*
|
|
@@ -7454,6 +8246,7 @@ ${run.workflowVersion}`;
|
|
|
7454
8246
|
namespace: run.namespace,
|
|
7455
8247
|
error: error2
|
|
7456
8248
|
});
|
|
8249
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
7457
8250
|
this.notifyCancelled(run.id);
|
|
7458
8251
|
if (this.controlPlane) {
|
|
7459
8252
|
void this.controlPlane.publishControl({
|
|
@@ -7485,6 +8278,8 @@ ${run.workflowVersion}`;
|
|
|
7485
8278
|
const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
|
|
7486
8279
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
7487
8280
|
const checkpoints = await this.store.listCheckpoints(run.id);
|
|
8281
|
+
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
8282
|
+
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
7488
8283
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
7489
8284
|
if (handoff) {
|
|
7490
8285
|
const result = await this.settleRun(run, await this.followHandoff(run, handoff));
|
|
@@ -8059,6 +8854,7 @@ ${run.workflowVersion}`;
|
|
|
8059
8854
|
namespace: run.namespace,
|
|
8060
8855
|
error: error2
|
|
8061
8856
|
});
|
|
8857
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
8062
8858
|
return {
|
|
8063
8859
|
runId: run.id,
|
|
8064
8860
|
status: "cancelled",
|
|
@@ -8105,6 +8901,7 @@ ${run.workflowVersion}`;
|
|
|
8105
8901
|
}
|
|
8106
8902
|
const compensations = [];
|
|
8107
8903
|
const snapshot = await this.store.listCheckpoints(run.id);
|
|
8904
|
+
await this.deliverFinishedChildren(run.id, snapshot);
|
|
8108
8905
|
const replay = /* @__PURE__ */ new Map();
|
|
8109
8906
|
for (const cp of snapshot) replay.set(cp.seq, cp);
|
|
8110
8907
|
let suspendedOnFan;
|
|
@@ -8201,13 +8998,13 @@ ${run.workflowVersion}`;
|
|
|
8201
8998
|
if (this.awaitedStepFans.get(run.id) === awaitedFans) this.awaitedStepFans.delete(run.id);
|
|
8202
8999
|
await this.releaseLeaseBestEffort(run.id);
|
|
8203
9000
|
if (suspendedOnFan) {
|
|
8204
|
-
|
|
9001
|
+
this.inBackground(this.redriveSettledChildren(run.id, suspendedOnFan));
|
|
8205
9002
|
}
|
|
8206
9003
|
if (suspendedOnSteps && awaitedSteps.size > 0) {
|
|
8207
|
-
|
|
9004
|
+
this.inBackground(this.redriveSettledSteps(run.id, awaitedSteps));
|
|
8208
9005
|
}
|
|
8209
9006
|
if (suspendedOnSteps && awaitedFans.length > 0) {
|
|
8210
|
-
|
|
9007
|
+
this.inBackground(this.redriveSettledStepFans(run.id, awaitedFans));
|
|
8211
9008
|
}
|
|
8212
9009
|
}
|
|
8213
9010
|
}
|
|
@@ -8647,9 +9444,11 @@ ${run.workflowVersion}`;
|
|
|
8647
9444
|
})) {
|
|
8648
9445
|
return this.callRemote(runId, seq, step, input, queue, transport, void 0, admission, routing);
|
|
8649
9446
|
}
|
|
8650
|
-
throw new WorkflowSuspended(wakeAt);
|
|
9447
|
+
throw new WorkflowSuspended(this.stepLeaseWakeAt(wakeAt));
|
|
9448
|
+
}
|
|
9449
|
+
if (this.clock() < existing.wakeAt) {
|
|
9450
|
+
throw new WorkflowSuspended(this.stepLeaseWakeAt(existing.wakeAt));
|
|
8651
9451
|
}
|
|
8652
|
-
if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
|
|
8653
9452
|
if (existing.attempts >= this.remoteRedispatchMax) {
|
|
8654
9453
|
throw new RemoteStepError({
|
|
8655
9454
|
message: `remote step "${step.name}" lost \u2014 no result after ${existing.attempts} re-dispatch(es)`,
|
|
@@ -8695,7 +9494,7 @@ ${run.workflowVersion}`;
|
|
|
8695
9494
|
kind: "remote",
|
|
8696
9495
|
redispatched: true
|
|
8697
9496
|
});
|
|
8698
|
-
throw new WorkflowSuspended(nextDeadline);
|
|
9497
|
+
throw new WorkflowSuspended(this.stepLeaseWakeAt(nextDeadline));
|
|
8699
9498
|
}
|
|
8700
9499
|
let attempt = 1;
|
|
8701
9500
|
if (existing?.status === "failed") {
|
|
@@ -9058,6 +9857,7 @@ ${run.workflowVersion}`;
|
|
|
9058
9857
|
});
|
|
9059
9858
|
const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
|
|
9060
9859
|
const pending = [];
|
|
9860
|
+
const resettled = [];
|
|
9061
9861
|
reads.forEach((read, k) => {
|
|
9062
9862
|
const index = live[k];
|
|
9063
9863
|
if (read.status === "rejected") outcomes[index] = {
|
|
@@ -9067,8 +9867,17 @@ ${run.workflowVersion}`;
|
|
|
9067
9867
|
index,
|
|
9068
9868
|
cp: read.value
|
|
9069
9869
|
});
|
|
9870
|
+
else if (read.value?.status === "completed" || read.value?.status === "failed") {
|
|
9871
|
+
resettled.push({
|
|
9872
|
+
index,
|
|
9873
|
+
cp: read.value
|
|
9874
|
+
});
|
|
9875
|
+
}
|
|
9070
9876
|
});
|
|
9071
|
-
if (pending.length === 0)
|
|
9877
|
+
if (pending.length === 0) {
|
|
9878
|
+
if (resettled.length === 0) return outcomes;
|
|
9879
|
+
return this.resumeForRedelivered(runId, resettled, outcomes);
|
|
9880
|
+
}
|
|
9072
9881
|
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
9073
9882
|
const run = await this.store.getRun(runId);
|
|
9074
9883
|
if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
|
|
@@ -9120,6 +9929,29 @@ ${run.workflowVersion}`;
|
|
|
9120
9929
|
return outcomes;
|
|
9121
9930
|
}
|
|
9122
9931
|
/**
|
|
9932
|
+
* The outcomes for a batch of redelivered results (their steps already settled): one of them
|
|
9933
|
+
* resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
|
|
9934
|
+
* resume of a run that had in fact moved on replays to its current wait and re-parks.
|
|
9935
|
+
*/
|
|
9936
|
+
async resumeForRedelivered(runId, resettled, outcomes) {
|
|
9937
|
+
const run = await this.store.getRun(runId);
|
|
9938
|
+
if (run?.status !== "suspended") return outcomes;
|
|
9939
|
+
const barriers = /* @__PURE__ */ new Map();
|
|
9940
|
+
for (const { index, cp } of resettled) {
|
|
9941
|
+
const asResult = {
|
|
9942
|
+
runId,
|
|
9943
|
+
seq: cp.seq,
|
|
9944
|
+
stepId: cp.stepId ?? stepId(runId, cp.seq),
|
|
9945
|
+
status: cp.status === "failed" ? "failed" : "completed"
|
|
9946
|
+
};
|
|
9947
|
+
if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
|
|
9948
|
+
outcomes[index] = true;
|
|
9949
|
+
break;
|
|
9950
|
+
}
|
|
9951
|
+
}
|
|
9952
|
+
return outcomes;
|
|
9953
|
+
}
|
|
9954
|
+
/**
|
|
9123
9955
|
* The join barrier of a step fan: a result for one of its steps resumes the parent only once no
|
|
9124
9956
|
* step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
|
|
9125
9957
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
@@ -9243,7 +10075,7 @@ ${run.workflowVersion}`;
|
|
|
9243
10075
|
...waiters
|
|
9244
10076
|
];
|
|
9245
10077
|
waiters.clear();
|
|
9246
|
-
for (const runId of runIds) void this.
|
|
10078
|
+
for (const runId of runIds) void this.wake(runId).catch(() => void 0);
|
|
9247
10079
|
}
|
|
9248
10080
|
/** Release the flow-control slot a dispatched step held (if any), by its stepId. */
|
|
9249
10081
|
async releaseQueueSlot(id) {
|
|
@@ -9985,6 +10817,8 @@ var CodecStateStore = class {
|
|
|
9985
10817
|
registerChildWaits;
|
|
9986
10818
|
/** The folded checkpoint carries the children's outputs, so it is encoded. */
|
|
9987
10819
|
compactCheckpoints;
|
|
10820
|
+
/** Decodes each run like {@link getRun}. Bound only when the inner store has it. */
|
|
10821
|
+
getRuns;
|
|
9988
10822
|
/** Forwarded verbatim — a waiter carries no payload. */
|
|
9989
10823
|
listSignalWaitersByRun;
|
|
9990
10824
|
/** Forwarded verbatim — a waiter carries no payload. */
|
|
@@ -10028,6 +10862,8 @@ var CodecStateStore = class {
|
|
|
10028
10862
|
})));
|
|
10029
10863
|
const compact = inner.compactCheckpoints;
|
|
10030
10864
|
if (compact) this.compactCheckpoints = (runId, fromSeq, toSeq, into) => compact.call(inner, runId, fromSeq, toSeq, this.encCp(into));
|
|
10865
|
+
const getRuns = inner.getRuns;
|
|
10866
|
+
if (getRuns) this.getRuns = async (runIds) => (await getRuns.call(inner, runIds)).map((r) => this.decRun(r));
|
|
10031
10867
|
const byRun = inner.listSignalWaitersByRun;
|
|
10032
10868
|
if (byRun) this.listSignalWaitersByRun = (runIds) => byRun.call(inner, runIds);
|
|
10033
10869
|
const hasWaiter = inner.hasSignalWaiter;
|
|
@@ -10580,6 +11416,16 @@ var InMemoryStateStore = class {
|
|
|
10580
11416
|
...run
|
|
10581
11417
|
} : null;
|
|
10582
11418
|
}
|
|
11419
|
+
async getRuns(runIds) {
|
|
11420
|
+
const out = [];
|
|
11421
|
+
for (const id of new Set(runIds)) {
|
|
11422
|
+
const run = this.runs.get(id);
|
|
11423
|
+
if (run) out.push({
|
|
11424
|
+
...run
|
|
11425
|
+
});
|
|
11426
|
+
}
|
|
11427
|
+
return out;
|
|
11428
|
+
}
|
|
10583
11429
|
async deleteRun(runId) {
|
|
10584
11430
|
this.runs.delete(runId);
|
|
10585
11431
|
for (const [key, cp] of this.checkpoints) {
|
|
@@ -11031,6 +11877,99 @@ var InMemoryTransport = class {
|
|
|
11031
11877
|
this.controlHandlers.add(handler);
|
|
11032
11878
|
}
|
|
11033
11879
|
};
|
|
11880
|
+
|
|
11881
|
+
// src/testing/in-memory-run-queue.ts
|
|
11882
|
+
var InMemoryRunQueue = class {
|
|
11883
|
+
static {
|
|
11884
|
+
__name(this, "InMemoryRunQueue");
|
|
11885
|
+
}
|
|
11886
|
+
lanes = /* @__PURE__ */ new Map();
|
|
11887
|
+
/** A transport on this queue. */
|
|
11888
|
+
transport() {
|
|
11889
|
+
return {
|
|
11890
|
+
dispatch: /* @__PURE__ */ __name(async (_task) => {
|
|
11891
|
+
throw new Error("InMemoryRunQueue carries no remote steps");
|
|
11892
|
+
}, "dispatch"),
|
|
11893
|
+
onResult: /* @__PURE__ */ __name((_handler) => {
|
|
11894
|
+
}, "onResult"),
|
|
11895
|
+
onHeartbeat: /* @__PURE__ */ __name((_handler) => {
|
|
11896
|
+
}, "onHeartbeat"),
|
|
11897
|
+
dispatchRuns: /* @__PURE__ */ __name(async (runs, opts) => this.add(runs, opts), "dispatchRuns"),
|
|
11898
|
+
onDispatchedRun: /* @__PURE__ */ __name((handler, opts) => this.consume(handler, opts), "onDispatchedRun")
|
|
11899
|
+
};
|
|
11900
|
+
}
|
|
11901
|
+
/** Runs waiting in the queue of `namespace`. */
|
|
11902
|
+
waiting(namespace) {
|
|
11903
|
+
return (this.lanes.get(namespace ?? "")?.waiting ?? []).map((r) => r.runId);
|
|
11904
|
+
}
|
|
11905
|
+
lane(namespace) {
|
|
11906
|
+
const key = namespace ?? "";
|
|
11907
|
+
let lane = this.lanes.get(key);
|
|
11908
|
+
if (!lane) {
|
|
11909
|
+
lane = {
|
|
11910
|
+
waiting: [],
|
|
11911
|
+
queued: /* @__PURE__ */ new Map(),
|
|
11912
|
+
consumers: [],
|
|
11913
|
+
next: 0
|
|
11914
|
+
};
|
|
11915
|
+
this.lanes.set(key, lane);
|
|
11916
|
+
}
|
|
11917
|
+
return lane;
|
|
11918
|
+
}
|
|
11919
|
+
add(runs, opts) {
|
|
11920
|
+
const lane = this.lane(opts?.namespace);
|
|
11921
|
+
for (const run of runs) {
|
|
11922
|
+
const waiting = lane.queued.get(run.runId);
|
|
11923
|
+
if (waiting) {
|
|
11924
|
+
if (run.reason === "wake") waiting.reason = "wake";
|
|
11925
|
+
continue;
|
|
11926
|
+
}
|
|
11927
|
+
const entry = {
|
|
11928
|
+
...run
|
|
11929
|
+
};
|
|
11930
|
+
lane.queued.set(run.runId, entry);
|
|
11931
|
+
lane.waiting.push(entry);
|
|
11932
|
+
}
|
|
11933
|
+
lane.waiting.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
|
|
11934
|
+
setImmediate(() => this.pump(lane));
|
|
11935
|
+
}
|
|
11936
|
+
consume(handler, opts) {
|
|
11937
|
+
const lane = this.lane(opts.namespace);
|
|
11938
|
+
const consumer = {
|
|
11939
|
+
handler,
|
|
11940
|
+
concurrency: Math.max(1, opts.concurrency),
|
|
11941
|
+
active: 0,
|
|
11942
|
+
stopped: false,
|
|
11943
|
+
idle: []
|
|
11944
|
+
};
|
|
11945
|
+
lane.consumers.push(consumer);
|
|
11946
|
+
setImmediate(() => this.pump(lane));
|
|
11947
|
+
return () => {
|
|
11948
|
+
consumer.stopped = true;
|
|
11949
|
+
lane.consumers = lane.consumers.filter((c) => c !== consumer);
|
|
11950
|
+
if (consumer.active === 0) return Promise.resolve();
|
|
11951
|
+
return new Promise((resolve) => consumer.idle.push(resolve));
|
|
11952
|
+
};
|
|
11953
|
+
}
|
|
11954
|
+
pump(lane) {
|
|
11955
|
+
while (lane.waiting.length > 0) {
|
|
11956
|
+
const free = lane.consumers.filter((c) => !c.stopped && c.active < c.concurrency);
|
|
11957
|
+
if (free.length === 0) return;
|
|
11958
|
+
const consumer = free[lane.next % free.length];
|
|
11959
|
+
lane.next += 1;
|
|
11960
|
+
const run = lane.waiting.shift();
|
|
11961
|
+
lane.queued.delete(run.runId);
|
|
11962
|
+
consumer.active += 1;
|
|
11963
|
+
void consumer.handler(run).catch(() => void 0).then(() => {
|
|
11964
|
+
consumer.active -= 1;
|
|
11965
|
+
if (consumer.active === 0 && consumer.stopped) {
|
|
11966
|
+
for (const resolve of consumer.idle.splice(0)) resolve();
|
|
11967
|
+
}
|
|
11968
|
+
this.pump(lane);
|
|
11969
|
+
});
|
|
11970
|
+
}
|
|
11971
|
+
}
|
|
11972
|
+
};
|
|
11034
11973
|
// Annotate the CommonJS export names for ESM import in node:
|
|
11035
11974
|
0 && (module.exports = {
|
|
11036
11975
|
CHILD_EDGE_PREFIXES,
|
|
@@ -11058,6 +11997,7 @@ var InMemoryTransport = class {
|
|
|
11058
11997
|
GatherError,
|
|
11059
11998
|
HANDOFF_SEQ_BASE,
|
|
11060
11999
|
InMemoryAdmissionBackend,
|
|
12000
|
+
InMemoryRunQueue,
|
|
11061
12001
|
InMemoryStateStore,
|
|
11062
12002
|
InMemoryTransport,
|
|
11063
12003
|
LEGACY_V1_CAPABILITIES,
|
|
@@ -11142,6 +12082,7 @@ var InMemoryTransport = class {
|
|
|
11142
12082
|
fanPageEtag,
|
|
11143
12083
|
findStalledRuns,
|
|
11144
12084
|
foldedCount,
|
|
12085
|
+
getRunsByIds,
|
|
11145
12086
|
handoffHandlerId,
|
|
11146
12087
|
handoffName,
|
|
11147
12088
|
hasDurableExecutionWrappers,
|
|
@@ -11179,6 +12120,7 @@ var InMemoryTransport = class {
|
|
|
11179
12120
|
poolNamespace,
|
|
11180
12121
|
prevCronFireMs,
|
|
11181
12122
|
queryCheckpointsOf,
|
|
12123
|
+
readRunsByIds,
|
|
11182
12124
|
readSearchAttributes,
|
|
11183
12125
|
requiredCapabilities,
|
|
11184
12126
|
resolveConcurrency,
|