@dudousxd/nestjs-durable-core 0.72.1 → 0.74.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +588 -138
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +170 -16
- package/dist/index.d.ts +170 -16
- package/dist/index.js +581 -138
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -389,12 +389,15 @@ var WorkflowSuspended = class extends Error {
|
|
|
389
389
|
}
|
|
390
390
|
/** Epoch ms to auto-resume (durable sleep), or undefined when waiting on an external signal. */
|
|
391
391
|
wakeAt;
|
|
392
|
+
/** Set when awaited children are what the run is waiting on — see {@link FanJoin}. */
|
|
393
|
+
fan;
|
|
392
394
|
/** Marks this as a control-flow signal — see {@link isWorkflowControlFlowSignal}. */
|
|
393
395
|
[CONTROL_FLOW_SIGNAL] = true;
|
|
394
|
-
constructor(wakeAt) {
|
|
396
|
+
constructor(wakeAt, fan) {
|
|
395
397
|
super("workflow suspended");
|
|
396
398
|
this.name = "WorkflowSuspended";
|
|
397
399
|
this.wakeAt = wakeAt;
|
|
400
|
+
this.fan = fan;
|
|
398
401
|
}
|
|
399
402
|
};
|
|
400
403
|
var ContinueAsNew = class extends Error {
|
|
@@ -821,6 +824,21 @@ function instantCheckpoint(p) {
|
|
|
821
824
|
}
|
|
822
825
|
__name(instantCheckpoint, "instantCheckpoint");
|
|
823
826
|
|
|
827
|
+
// src/completion.ts
|
|
828
|
+
function isFailedCompletion(payload) {
|
|
829
|
+
const c = payload;
|
|
830
|
+
return !!c && typeof c === "object" && "ok" in c && c.ok === false;
|
|
831
|
+
}
|
|
832
|
+
__name(isFailedCompletion, "isFailedCompletion");
|
|
833
|
+
function unwrapCompletion(payload, label) {
|
|
834
|
+
const c = payload;
|
|
835
|
+
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
836
|
+
throw new FatalError(`${label} failed: ${c.error}`);
|
|
837
|
+
}
|
|
838
|
+
return c?.value;
|
|
839
|
+
}
|
|
840
|
+
__name(unwrapCompletion, "unwrapCompletion");
|
|
841
|
+
|
|
824
842
|
// src/concurrency.ts
|
|
825
843
|
var DEFAULT_CONCURRENCY_STATUSES = [
|
|
826
844
|
"pending",
|
|
@@ -1060,6 +1078,81 @@ function eventMatches(payload, match) {
|
|
|
1060
1078
|
}
|
|
1061
1079
|
__name(eventMatches, "eventMatches");
|
|
1062
1080
|
|
|
1081
|
+
// src/fan-join.ts
|
|
1082
|
+
function allChildId(runId, firstSeq, index, wave) {
|
|
1083
|
+
return `${allBase(runId, firstSeq, wave)}${index}`;
|
|
1084
|
+
}
|
|
1085
|
+
__name(allChildId, "allChildId");
|
|
1086
|
+
function allGroup(firstSeq, mode) {
|
|
1087
|
+
return mode === "failFast" ? `all:${firstSeq}:failFast` : `all:${firstSeq}`;
|
|
1088
|
+
}
|
|
1089
|
+
__name(allGroup, "allGroup");
|
|
1090
|
+
function allWait(runId, firstSeq, wave) {
|
|
1091
|
+
const base = allBase(runId, firstSeq, wave);
|
|
1092
|
+
return {
|
|
1093
|
+
waiterPrefix: `child:${base}`,
|
|
1094
|
+
checkpointPrefix: `signal:child:${base}`
|
|
1095
|
+
};
|
|
1096
|
+
}
|
|
1097
|
+
__name(allWait, "allWait");
|
|
1098
|
+
function fanJoin(waits) {
|
|
1099
|
+
return {
|
|
1100
|
+
waits
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
__name(fanJoin, "fanJoin");
|
|
1104
|
+
var FOLDED_PREFIX = "fan:";
|
|
1105
|
+
function allFoldedName(runId, firstSeq, wave) {
|
|
1106
|
+
return `${FOLDED_PREFIX}${allBase(runId, firstSeq, wave)}`;
|
|
1107
|
+
}
|
|
1108
|
+
__name(allFoldedName, "allFoldedName");
|
|
1109
|
+
function foldedChildIds(checkpoint) {
|
|
1110
|
+
const base = checkpoint.name.slice(FOLDED_PREFIX.length);
|
|
1111
|
+
const { from, values } = checkpoint.output;
|
|
1112
|
+
return values.map((_, j) => `${base}${from + j}`);
|
|
1113
|
+
}
|
|
1114
|
+
__name(foldedChildIds, "foldedChildIds");
|
|
1115
|
+
function allBase(runId, firstSeq, wave) {
|
|
1116
|
+
return wave === void 0 ? `${runId}.all.${firstSeq}.` : `${runId}.all.${firstSeq}.w${wave}.`;
|
|
1117
|
+
}
|
|
1118
|
+
__name(allBase, "allBase");
|
|
1119
|
+
function allWaves(size, concurrency) {
|
|
1120
|
+
if (concurrency === void 0 || concurrency >= size) return void 0;
|
|
1121
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
1122
|
+
throw new Error(`ctx.all: concurrency must be a positive integer, got ${concurrency}`);
|
|
1123
|
+
}
|
|
1124
|
+
if (concurrency === 1) return {
|
|
1125
|
+
waveSize: 1,
|
|
1126
|
+
maxWaves: 1
|
|
1127
|
+
};
|
|
1128
|
+
return {
|
|
1129
|
+
waveSize: Math.floor(concurrency / 2),
|
|
1130
|
+
maxWaves: 2
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
__name(allWaves, "allWaves");
|
|
1134
|
+
var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+)$/;
|
|
1135
|
+
function parentRunIdOf(runId) {
|
|
1136
|
+
return ENGINE_CHILD_ID.exec(runId)?.[1];
|
|
1137
|
+
}
|
|
1138
|
+
__name(parentRunIdOf, "parentRunIdOf");
|
|
1139
|
+
var GROUP = /^all:(\d+)(:failFast)?$/;
|
|
1140
|
+
var REST = /^(?:w(\d+)\.)?\d+$/;
|
|
1141
|
+
function fanOfWaiter(waiter) {
|
|
1142
|
+
const group = waiter.parallelGroup ? GROUP.exec(waiter.parallelGroup) : null;
|
|
1143
|
+
if (!group) return void 0;
|
|
1144
|
+
const base = `child:${waiter.runId}.all.${group[1]}.`;
|
|
1145
|
+
if (!waiter.token.startsWith(base)) return void 0;
|
|
1146
|
+
const rest = REST.exec(waiter.token.slice(base.length));
|
|
1147
|
+
if (!rest) return void 0;
|
|
1148
|
+
const waiterPrefix = rest[1] === void 0 ? base : `${base}w${rest[1]}.`;
|
|
1149
|
+
return {
|
|
1150
|
+
waiterPrefix,
|
|
1151
|
+
failFast: group[2] !== void 0
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
__name(fanOfWaiter, "fanOfWaiter");
|
|
1155
|
+
|
|
1063
1156
|
// src/handshake/descriptor.ts
|
|
1064
1157
|
var CURRENT_PROTOCOL_VERSION = 1;
|
|
1065
1158
|
var LEGACY_V1_PROTOCOL = {
|
|
@@ -1801,6 +1894,23 @@ function resolveRunWaiting(run, waiterByRun) {
|
|
|
1801
1894
|
return waiter ? classifyWaiterToken(waiter.token) : void 0;
|
|
1802
1895
|
}
|
|
1803
1896
|
__name(resolveRunWaiting, "resolveRunWaiting");
|
|
1897
|
+
var RUN_ID_BATCH = 500;
|
|
1898
|
+
async function listSignalWaitersOfRuns(store, runIds) {
|
|
1899
|
+
const unique = [
|
|
1900
|
+
...new Set(runIds)
|
|
1901
|
+
];
|
|
1902
|
+
if (unique.length === 0) return [];
|
|
1903
|
+
if (!store.listSignalWaitersByRun) {
|
|
1904
|
+
const ids = new Set(unique);
|
|
1905
|
+
return (await store.listSignalWaiters("")).filter((w) => ids.has(w.runId));
|
|
1906
|
+
}
|
|
1907
|
+
const out = [];
|
|
1908
|
+
for (let at = 0; at < unique.length; at += RUN_ID_BATCH) {
|
|
1909
|
+
out.push(...await store.listSignalWaitersByRun(unique.slice(at, at + RUN_ID_BATCH)));
|
|
1910
|
+
}
|
|
1911
|
+
return out;
|
|
1912
|
+
}
|
|
1913
|
+
__name(listSignalWaitersOfRuns, "listSignalWaitersOfRuns");
|
|
1804
1914
|
function indexWaitersByRun(waiters) {
|
|
1805
1915
|
const byRun = /* @__PURE__ */ new Map();
|
|
1806
1916
|
for (const w of waiters) if (!byRun.has(w.runId)) byRun.set(w.runId, w);
|
|
@@ -2288,6 +2398,54 @@ function errorMessage(error2) {
|
|
|
2288
2398
|
}
|
|
2289
2399
|
__name(errorMessage, "errorMessage");
|
|
2290
2400
|
|
|
2401
|
+
// src/singleton-admission.ts
|
|
2402
|
+
var SINGLETON_ADMITTED_TAG = "durable:singleton:admitted";
|
|
2403
|
+
var SINGLETON_ACTIVE_STATUSES = [
|
|
2404
|
+
"pending",
|
|
2405
|
+
"running",
|
|
2406
|
+
"suspended",
|
|
2407
|
+
"blocked",
|
|
2408
|
+
"cancelling"
|
|
2409
|
+
];
|
|
2410
|
+
function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
|
|
2411
|
+
if (!Number.isInteger(limit) || limit < 1) throw new Error("singleton limit must be a positive integer");
|
|
2412
|
+
const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && run.tags?.includes(tag));
|
|
2413
|
+
const candidate = active.find((run) => run.id === runId);
|
|
2414
|
+
if (!candidate) return null;
|
|
2415
|
+
if (candidate.tags?.includes(SINGLETON_ADMITTED_TAG)) return candidate.tags;
|
|
2416
|
+
if (candidate.status === "cancelling") return null;
|
|
2417
|
+
const holders = active.filter((run) => run.tags?.includes(SINGLETON_ADMITTED_TAG));
|
|
2418
|
+
const remaining = limit - holders.length;
|
|
2419
|
+
if (remaining <= 0) return null;
|
|
2420
|
+
const waiters = active.filter((run) => !run.tags?.includes(SINGLETON_ADMITTED_TAG)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2421
|
+
if (!waiters.slice(0, remaining).some((run) => run.id === runId)) return null;
|
|
2422
|
+
return [
|
|
2423
|
+
...candidate.tags ?? [],
|
|
2424
|
+
SINGLETON_ADMITTED_TAG
|
|
2425
|
+
];
|
|
2426
|
+
}
|
|
2427
|
+
__name(singletonAdmissionTags, "singletonAdmissionTags");
|
|
2428
|
+
async function clearTerminalSingletonAdmission(runId, patch, getRun) {
|
|
2429
|
+
if (!patch.status || !TERMINAL_RUN_STATUSES.includes(patch.status)) return patch;
|
|
2430
|
+
const tags = "tags" in patch ? patch.tags : (await getRun(runId))?.tags;
|
|
2431
|
+
return {
|
|
2432
|
+
...patch,
|
|
2433
|
+
tags: tags?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
|
|
2434
|
+
};
|
|
2435
|
+
}
|
|
2436
|
+
__name(clearTerminalSingletonAdmission, "clearTerminalSingletonAdmission");
|
|
2437
|
+
function singletonAdmissionDenialPatch(runs, runId, tag, workflow, retryWakeAt) {
|
|
2438
|
+
if (retryWakeAt === void 0) return void 0;
|
|
2439
|
+
const candidate = runs.find((run) => run.id === runId && run.workflow === workflow && run.tags?.includes(tag) && SINGLETON_ACTIVE_STATUSES.includes(run.status));
|
|
2440
|
+
if (!candidate || candidate.status === "cancelling") return void 0;
|
|
2441
|
+
return {
|
|
2442
|
+
status: "suspended",
|
|
2443
|
+
wakeAt: retryWakeAt,
|
|
2444
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2445
|
+
};
|
|
2446
|
+
}
|
|
2447
|
+
__name(singletonAdmissionDenialPatch, "singletonAdmissionDenialPatch");
|
|
2448
|
+
|
|
2291
2449
|
// src/singleton-gate.ts
|
|
2292
2450
|
var SINGLETON_RETRY_MS = 1e3;
|
|
2293
2451
|
var SINGLETON_RETRY_JITTER_MS = 250;
|
|
@@ -2301,6 +2459,9 @@ var SingletonGate = class {
|
|
|
2301
2459
|
}
|
|
2302
2460
|
/** The tag a singleton run carries, so the gate can find others sharing its key. */
|
|
2303
2461
|
tag(cfg, input) {
|
|
2462
|
+
if (!this.deps.store.tryAdmitSingleton) {
|
|
2463
|
+
throw new Error("Singleton workflows require StateStore.tryAdmitSingleton for atomic durable admission");
|
|
2464
|
+
}
|
|
2304
2465
|
return `singleton:${cfg.key(input)}`;
|
|
2305
2466
|
}
|
|
2306
2467
|
/** Next wake time for a gated run: the base retry delay jittered to avoid a wakeup stampede. */
|
|
@@ -2309,7 +2470,7 @@ var SingletonGate = class {
|
|
|
2309
2470
|
}
|
|
2310
2471
|
/**
|
|
2311
2472
|
* Reject a start that would grow the same-key backlog past `limit + maxQueueDepth` (counting
|
|
2312
|
-
* `pending`/`running`/`suspended` runs sharing the key in one scan). No-op when no `maxQueueDepth`
|
|
2473
|
+
* `pending`/`running`/`suspended`/`blocked`/`cancelling` runs sharing the key in one scan). No-op when no `maxQueueDepth`
|
|
2313
2474
|
* is configured.
|
|
2314
2475
|
*/
|
|
2315
2476
|
async assertCapacity(workflow, cfg, input) {
|
|
@@ -2318,34 +2479,25 @@ var SingletonGate = class {
|
|
|
2318
2479
|
const queued = await this.deps.store.listRuns({
|
|
2319
2480
|
tag: this.tag(cfg, input),
|
|
2320
2481
|
workflow,
|
|
2321
|
-
statuses:
|
|
2322
|
-
"pending",
|
|
2323
|
-
"running",
|
|
2324
|
-
"suspended",
|
|
2325
|
-
"cancelling"
|
|
2326
|
-
]
|
|
2482
|
+
statuses: SINGLETON_ACTIVE_STATUSES
|
|
2327
2483
|
});
|
|
2328
2484
|
if (queued.length >= cap) {
|
|
2329
2485
|
throw new SingletonQueueFullError(workflow, cfg.key(input), cfg.maxQueueDepth);
|
|
2330
2486
|
}
|
|
2331
2487
|
}
|
|
2332
|
-
/**
|
|
2333
|
-
* Whether `run` may run now under its key: it's among the `limit` oldest in-flight (running or
|
|
2334
|
-
* suspended) runs sharing the key, by `(createdAt, id)` order. A consistent store gives every
|
|
2335
|
-
* instance the same ordering, so admission is race-free + FIFO.
|
|
2336
|
-
*/
|
|
2488
|
+
/** Preserve durable holders and atomically claim one of the remaining slots. */
|
|
2337
2489
|
async admit(run, cfg) {
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
return
|
|
2490
|
+
if (!this.deps.store.tryAdmitSingleton) {
|
|
2491
|
+
throw new Error("Singleton workflows require StateStore.tryAdmitSingleton for atomic durable admission");
|
|
2492
|
+
}
|
|
2493
|
+
const admitted = await this.deps.store.tryAdmitSingleton(run.id, this.tag(cfg, run.input), run.workflow, cfg.limit ?? 1, this.retryWakeAt());
|
|
2494
|
+
if (admitted && !run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
|
|
2495
|
+
run.tags = [
|
|
2496
|
+
...run.tags ?? [],
|
|
2497
|
+
SINGLETON_ADMITTED_TAG
|
|
2498
|
+
];
|
|
2499
|
+
}
|
|
2500
|
+
return admitted;
|
|
2349
2501
|
}
|
|
2350
2502
|
/**
|
|
2351
2503
|
* Notify-on-release: a singleton run settled (freeing a slot), so dispatch the oldest gated
|
|
@@ -2569,16 +2721,6 @@ var TransportPool = class {
|
|
|
2569
2721
|
}
|
|
2570
2722
|
};
|
|
2571
2723
|
|
|
2572
|
-
// src/completion.ts
|
|
2573
|
-
function unwrapCompletion(payload, label) {
|
|
2574
|
-
const c = payload;
|
|
2575
|
-
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
2576
|
-
throw new FatalError(`${label} failed: ${c.error}`);
|
|
2577
|
-
}
|
|
2578
|
-
return c?.value;
|
|
2579
|
-
}
|
|
2580
|
-
__name(unwrapCompletion, "unwrapCompletion");
|
|
2581
|
-
|
|
2582
2724
|
// src/step-name-symbol.ts
|
|
2583
2725
|
var DURABLE_STEP_NAME = Symbol.for("durable.step.name");
|
|
2584
2726
|
var DURABLE_STEP_CONFIG = Symbol.for("durable.step.config");
|
|
@@ -2600,6 +2742,8 @@ function stepConfigOf(ref) {
|
|
|
2600
2742
|
__name(stepConfigOf, "stepConfigOf");
|
|
2601
2743
|
|
|
2602
2744
|
// src/workflow-ctx.ts
|
|
2745
|
+
var FAN_CHUNK = 500;
|
|
2746
|
+
var FOLD_MIN_FAN = 100;
|
|
2603
2747
|
function normalizeChildOptions(options) {
|
|
2604
2748
|
return typeof options === "string" ? {
|
|
2605
2749
|
childId: options
|
|
@@ -2942,7 +3086,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
2942
3086
|
if (existing && existing.status === "completed") {
|
|
2943
3087
|
return unwrapCompletion(existing.output, `child "${id}"`);
|
|
2944
3088
|
}
|
|
2945
|
-
await store.putSignalWaiter({
|
|
3089
|
+
if (!existing) await store.putSignalWaiter({
|
|
2946
3090
|
token: `child:${id}`,
|
|
2947
3091
|
runId,
|
|
2948
3092
|
seq: current
|
|
@@ -2973,16 +3117,33 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
2973
3117
|
status: "running"
|
|
2974
3118
|
}));
|
|
2975
3119
|
}
|
|
2976
|
-
throw new WorkflowSuspended(
|
|
3120
|
+
throw new WorkflowSuspended(void 0, fanJoin([
|
|
3121
|
+
{
|
|
3122
|
+
waiterPrefix: `child:${id}`,
|
|
3123
|
+
checkpointPrefix: `signal:child:${id}`
|
|
3124
|
+
}
|
|
3125
|
+
]));
|
|
2977
3126
|
}, "child");
|
|
2978
3127
|
const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
|
|
2979
3128
|
if (inputs.length === 0) return [];
|
|
2980
3129
|
const mode = opts?.mode ?? "waitAll";
|
|
2981
3130
|
const name = workflowName(workflow2);
|
|
3131
|
+
const waves = allWaves(inputs.length, opts?.concurrency);
|
|
2982
3132
|
const positions = inputs.map(() => pos.next());
|
|
2983
|
-
const
|
|
2984
|
-
const
|
|
2985
|
-
const
|
|
3133
|
+
const firstSeq = positions[0];
|
|
3134
|
+
const group = allGroup(firstSeq, mode);
|
|
3135
|
+
const waveOf = /* @__PURE__ */ __name((i) => waves ? Math.floor(i / waves.waveSize) : void 0, "waveOf");
|
|
3136
|
+
const id = /* @__PURE__ */ __name((i) => allChildId(runId, firstSeq, i, waveOf(i)), "id");
|
|
3137
|
+
const { checkpoints: existing, folded } = await readFan(positions, [
|
|
3138
|
+
allWait(runId, firstSeq).checkpointPrefix,
|
|
3139
|
+
allFoldedName(runId, firstSeq)
|
|
3140
|
+
]);
|
|
3141
|
+
const waveCount = waves ? Math.ceil(inputs.length / waves.waveSize) : 1;
|
|
3142
|
+
const waveItems = /* @__PURE__ */ __name((k) => {
|
|
3143
|
+
if (!waves) return inputs.map((_, i) => i);
|
|
3144
|
+
const from = k * waves.waveSize;
|
|
3145
|
+
return inputs.map((_, i) => i).slice(from, from + waves.waveSize);
|
|
3146
|
+
}, "waveItems");
|
|
2986
3147
|
if (mode === "failFast") {
|
|
2987
3148
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
2988
3149
|
const cp = existing[i];
|
|
@@ -3001,32 +3162,91 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3001
3162
|
}
|
|
3002
3163
|
}
|
|
3003
3164
|
}
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
if (!cp) {
|
|
3019
|
-
await writeCheckpoint(instantCheckpoint({
|
|
3165
|
+
if (store.compactCheckpoints && inputs.length >= FOLD_MIN_FAN) {
|
|
3166
|
+
for (let k = 0; k < waveCount; k += 1) {
|
|
3167
|
+
const items = waveItems(k);
|
|
3168
|
+
const first = items[0];
|
|
3169
|
+
const last = items[items.length - 1];
|
|
3170
|
+
if (folded[first]) continue;
|
|
3171
|
+
const values = [];
|
|
3172
|
+
for (const i of items) {
|
|
3173
|
+
const c = existing[i]?.status === "completed" ? existing[i]?.output : void 0;
|
|
3174
|
+
if (!c || typeof c !== "object" || c.ok !== true) break;
|
|
3175
|
+
values.push(c.value);
|
|
3176
|
+
}
|
|
3177
|
+
if (values.length < items.length) continue;
|
|
3178
|
+
const into = instantCheckpoint({
|
|
3020
3179
|
runId,
|
|
3021
|
-
seq,
|
|
3022
|
-
name:
|
|
3180
|
+
seq: positions[first],
|
|
3181
|
+
name: allFoldedName(runId, firstSeq, waves ? k : void 0),
|
|
3023
3182
|
kind: "signal",
|
|
3024
|
-
|
|
3183
|
+
output: {
|
|
3184
|
+
from: first,
|
|
3185
|
+
values
|
|
3186
|
+
},
|
|
3025
3187
|
parallelGroup: group
|
|
3188
|
+
});
|
|
3189
|
+
await store.compactCheckpoints(runId, positions[first], positions[last], into);
|
|
3190
|
+
for (const i of items) replay?.delete(positions[i]);
|
|
3191
|
+
replay?.set(into.seq, into);
|
|
3192
|
+
}
|
|
3193
|
+
}
|
|
3194
|
+
const sent = /* @__PURE__ */ __name((k) => waveItems(k).some((i) => existing[i]), "sent");
|
|
3195
|
+
const settled = /* @__PURE__ */ __name((k) => waveItems(k).every((i) => existing[i]?.status === "completed"), "settled");
|
|
3196
|
+
const active = [];
|
|
3197
|
+
let unsettled = 0;
|
|
3198
|
+
for (let k = 0; k < waveCount; k += 1) {
|
|
3199
|
+
if (settled(k)) continue;
|
|
3200
|
+
if (sent(k) || unsettled < (waves?.maxWaves ?? 1)) {
|
|
3201
|
+
active.push(k);
|
|
3202
|
+
unsettled += 1;
|
|
3203
|
+
}
|
|
3204
|
+
}
|
|
3205
|
+
if (active.length > 0) {
|
|
3206
|
+
const toRegister = [];
|
|
3207
|
+
const toEnsure = [];
|
|
3208
|
+
for (const k of active) {
|
|
3209
|
+
for (const i of waveItems(k)) {
|
|
3210
|
+
const cp = existing[i];
|
|
3211
|
+
if (cp?.status === "completed") continue;
|
|
3212
|
+
(cp ? toEnsure : toRegister).push(i);
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
for (let at = 0; at < toRegister.length; at += FAN_CHUNK) {
|
|
3216
|
+
const chunk = toRegister.slice(at, at + FAN_CHUNK);
|
|
3217
|
+
const entries = chunk.map((i) => ({
|
|
3218
|
+
// Carry the fan `group` onto the waiter too: the child's terminal `signal:child:`
|
|
3219
|
+
// checkpoint is (re)written by engine.signal when the child notifies the parent,
|
|
3220
|
+
// OVERWRITING the running placeholder at the same seq — so without this the resolved
|
|
3221
|
+
// checkpoint would lose the group and the dashboard would render the resolved fan as a
|
|
3222
|
+
// sequential chain.
|
|
3223
|
+
waiter: {
|
|
3224
|
+
token: `child:${id(i)}`,
|
|
3225
|
+
runId,
|
|
3226
|
+
seq: positions[i],
|
|
3227
|
+
parallelGroup: group
|
|
3228
|
+
},
|
|
3229
|
+
checkpoint: instantCheckpoint({
|
|
3230
|
+
runId,
|
|
3231
|
+
seq: positions[i],
|
|
3232
|
+
name: `signal:child:${id(i)}`,
|
|
3233
|
+
kind: "signal",
|
|
3234
|
+
status: "running",
|
|
3235
|
+
parallelGroup: group
|
|
3236
|
+
})
|
|
3237
|
+
}));
|
|
3238
|
+
await registerChildWaits(entries);
|
|
3239
|
+
for (const i of chunk) host.startChild(name, inputs[i], id(i));
|
|
3240
|
+
}
|
|
3241
|
+
for (let at = 0; at < toEnsure.length; at += FAN_CHUNK) {
|
|
3242
|
+
await Promise.all(toEnsure.slice(at, at + FAN_CHUNK).map(async (i) => {
|
|
3243
|
+
if (!await store.getRun(id(i))) host.startChild(name, inputs[i], id(i));
|
|
3026
3244
|
}));
|
|
3027
3245
|
}
|
|
3028
3246
|
}
|
|
3029
|
-
if (
|
|
3247
|
+
if (active.length > 0) {
|
|
3248
|
+
throw new WorkflowSuspended(void 0, fanJoin(active.map((k) => allWait(runId, firstSeq, waves ? k : void 0))));
|
|
3249
|
+
}
|
|
3030
3250
|
const outputs = [];
|
|
3031
3251
|
const failures = [];
|
|
3032
3252
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
@@ -3045,6 +3265,67 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3045
3265
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
3046
3266
|
return outputs;
|
|
3047
3267
|
}, "all");
|
|
3268
|
+
const readFan = /* @__PURE__ */ __name(async (positions, prefixes) => {
|
|
3269
|
+
const checkpoints = positions.map((seq) => replay?.get(seq) ?? null);
|
|
3270
|
+
const folded = positions.map(() => false);
|
|
3271
|
+
const expand = /* @__PURE__ */ __name(() => {
|
|
3272
|
+
for (let i = 0; i < checkpoints.length; i += 1) {
|
|
3273
|
+
const cp = checkpoints[i];
|
|
3274
|
+
if (!cp?.name.startsWith(FOLDED_PREFIX) || folded[i]) continue;
|
|
3275
|
+
const fold = cp.output;
|
|
3276
|
+
if (!fold || typeof fold.from !== "number" || !Array.isArray(fold.values)) {
|
|
3277
|
+
throw new Error(`ctx.all: folded checkpoint "${cp.name}" at seq ${cp.seq} is malformed`);
|
|
3278
|
+
}
|
|
3279
|
+
const { from, values } = fold;
|
|
3280
|
+
values.forEach((value, j) => {
|
|
3281
|
+
checkpoints[from + j] = {
|
|
3282
|
+
...cp,
|
|
3283
|
+
seq: positions[from + j],
|
|
3284
|
+
output: {
|
|
3285
|
+
ok: true,
|
|
3286
|
+
value
|
|
3287
|
+
}
|
|
3288
|
+
};
|
|
3289
|
+
folded[from + j] = true;
|
|
3290
|
+
});
|
|
3291
|
+
}
|
|
3292
|
+
}, "expand");
|
|
3293
|
+
expand();
|
|
3294
|
+
const missing = positions.filter((_, i) => !checkpoints[i]);
|
|
3295
|
+
if (missing.length === 0) return {
|
|
3296
|
+
checkpoints,
|
|
3297
|
+
folded
|
|
3298
|
+
};
|
|
3299
|
+
if (missing.length > 1 && store.listCheckpointsByNamePrefix) {
|
|
3300
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
3301
|
+
for (const cp of await store.listCheckpointsByNamePrefix(runId, prefixes)) {
|
|
3302
|
+
bySeq.set(cp.seq, cp);
|
|
3303
|
+
}
|
|
3304
|
+
positions.forEach((seq, i) => {
|
|
3305
|
+
checkpoints[i] ??= bySeq.get(seq) ?? null;
|
|
3306
|
+
});
|
|
3307
|
+
} else {
|
|
3308
|
+
await Promise.all(positions.map(async (seq, i) => {
|
|
3309
|
+
checkpoints[i] ??= await readCheckpoint(seq);
|
|
3310
|
+
}));
|
|
3311
|
+
}
|
|
3312
|
+
expand();
|
|
3313
|
+
return {
|
|
3314
|
+
checkpoints,
|
|
3315
|
+
folded
|
|
3316
|
+
};
|
|
3317
|
+
}, "readFan");
|
|
3318
|
+
const registerChildWaits = /* @__PURE__ */ __name(async (entries) => {
|
|
3319
|
+
if (store.registerChildWaits) {
|
|
3320
|
+
await store.registerChildWaits(entries);
|
|
3321
|
+
} else {
|
|
3322
|
+
await Promise.all(entries.map(async (e) => {
|
|
3323
|
+
await store.putSignalWaiter(e.waiter);
|
|
3324
|
+
await store.saveCheckpoint(e.checkpoint);
|
|
3325
|
+
}));
|
|
3326
|
+
}
|
|
3327
|
+
for (const e of entries) replay?.set(e.checkpoint.seq, e.checkpoint);
|
|
3328
|
+
}, "registerChildWaits");
|
|
3048
3329
|
const startChild = /* @__PURE__ */ __name(async (workflow2, input, options) => {
|
|
3049
3330
|
const { childId, priority, version } = normalizeChildOptions(options);
|
|
3050
3331
|
const current = pos.next();
|
|
@@ -3862,7 +4143,7 @@ var WorkflowEngine = class {
|
|
|
3862
4143
|
VERSION_UNDECLARED_TAG
|
|
3863
4144
|
] : []
|
|
3864
4145
|
];
|
|
3865
|
-
const tags = mergeTags(registered.tags, stampedTags);
|
|
4146
|
+
const tags = mergeTags(registered.tags, stampedTags)?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG);
|
|
3866
4147
|
if (registered.singleton) {
|
|
3867
4148
|
await this.singletons.assertCapacity(name, registered.singleton, input);
|
|
3868
4149
|
}
|
|
@@ -3937,6 +4218,12 @@ var WorkflowEngine = class {
|
|
|
3937
4218
|
await this.store.releaseRunLock(runId);
|
|
3938
4219
|
throw new NamespaceMismatch();
|
|
3939
4220
|
}
|
|
4221
|
+
if (run.status === "failed" && run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
|
|
4222
|
+
run.tags = run.tags.filter((tag) => tag !== SINGLETON_ADMITTED_TAG);
|
|
4223
|
+
await this.store.updateRun(run.id, {
|
|
4224
|
+
tags: run.tags
|
|
4225
|
+
});
|
|
4226
|
+
}
|
|
3940
4227
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
3941
4228
|
if (!registered) {
|
|
3942
4229
|
throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
|
|
@@ -4078,15 +4365,16 @@ var WorkflowEngine = class {
|
|
|
4078
4365
|
*/
|
|
4079
4366
|
async findParentRunId(childRunId) {
|
|
4080
4367
|
const token = `child:${childRunId}`;
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
}
|
|
4084
|
-
return void 0;
|
|
4368
|
+
const waiters = await this.store.listSignalWaiters(token);
|
|
4369
|
+
return waiters.find((waiter) => waiter.token === token)?.runId;
|
|
4085
4370
|
}
|
|
4086
4371
|
/** Track an in-flight execution so {@link drain} can wait for it. */
|
|
4087
4372
|
track(p) {
|
|
4088
4373
|
this.inflight.add(p);
|
|
4089
|
-
|
|
4374
|
+
const done = /* @__PURE__ */ __name(() => {
|
|
4375
|
+
this.inflight.delete(p);
|
|
4376
|
+
}, "done");
|
|
4377
|
+
void p.then(done, done);
|
|
4090
4378
|
return p;
|
|
4091
4379
|
}
|
|
4092
4380
|
/**
|
|
@@ -4299,7 +4587,10 @@ var WorkflowEngine = class {
|
|
|
4299
4587
|
const due = await this.store.listDueTimers(nowMs, this.namespace);
|
|
4300
4588
|
const blocked = await this.dueBlockedRuns(nowMs);
|
|
4301
4589
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
4302
|
-
const waiterByRun = indexWaitersByRun(await this.store
|
|
4590
|
+
const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
4591
|
+
...due,
|
|
4592
|
+
...blocked
|
|
4593
|
+
].map((run) => run.id)));
|
|
4303
4594
|
return this.resumeLeased([
|
|
4304
4595
|
...due,
|
|
4305
4596
|
...blocked
|
|
@@ -4409,11 +4700,6 @@ var WorkflowEngine = class {
|
|
|
4409
4700
|
} : {}
|
|
4410
4701
|
});
|
|
4411
4702
|
} else if (cp.kind === "signal" && cp.name.startsWith("signal:child:") && cp.status === "completed" && isFailureCompletion(cp.output)) {
|
|
4412
|
-
await this.store.saveCheckpoint({
|
|
4413
|
-
...cp,
|
|
4414
|
-
status: "running",
|
|
4415
|
-
output: void 0
|
|
4416
|
-
});
|
|
4417
4703
|
const childId = cp.name.slice("signal:child:".length);
|
|
4418
4704
|
const parked = [];
|
|
4419
4705
|
for (; ; ) {
|
|
@@ -4421,8 +4707,28 @@ var WorkflowEngine = class {
|
|
|
4421
4707
|
if (!buffered) break;
|
|
4422
4708
|
parked.push(buffered.payload);
|
|
4423
4709
|
}
|
|
4710
|
+
const successAt = parked.findIndex((payload) => typeof payload === "object" && payload !== null && payload.ok === true);
|
|
4711
|
+
const success = successAt === -1 ? void 0 : parked.splice(successAt, 1)[0];
|
|
4424
4712
|
for (const payload of parked) await this.store.bufferSignal(`child:${childId}`, payload);
|
|
4425
|
-
const bufferedSuccess =
|
|
4713
|
+
const bufferedSuccess = success !== void 0;
|
|
4714
|
+
if (bufferedSuccess) {
|
|
4715
|
+
await this.store.saveCheckpoint({
|
|
4716
|
+
...cp,
|
|
4717
|
+
output: success
|
|
4718
|
+
});
|
|
4719
|
+
} else {
|
|
4720
|
+
await this.store.putSignalWaiter({
|
|
4721
|
+
token: `child:${childId}`,
|
|
4722
|
+
runId,
|
|
4723
|
+
seq: cp.seq,
|
|
4724
|
+
parallelGroup: cp.parallelGroup
|
|
4725
|
+
});
|
|
4726
|
+
await this.store.saveCheckpoint({
|
|
4727
|
+
...cp,
|
|
4728
|
+
status: "running",
|
|
4729
|
+
output: void 0
|
|
4730
|
+
});
|
|
4731
|
+
}
|
|
4426
4732
|
if (!bufferedSuccess) {
|
|
4427
4733
|
const child = await this.store.getRun(childId);
|
|
4428
4734
|
if (child && (child.status === "failed" || child.status === "dead")) {
|
|
@@ -4437,7 +4743,8 @@ var WorkflowEngine = class {
|
|
|
4437
4743
|
status: "pending",
|
|
4438
4744
|
error: void 0,
|
|
4439
4745
|
...resurrecting ? {
|
|
4440
|
-
recoveryAttempts: 0
|
|
4746
|
+
recoveryAttempts: 0,
|
|
4747
|
+
tags: run.tags?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
|
|
4441
4748
|
} : {},
|
|
4442
4749
|
updatedAt: /* @__PURE__ */ new Date()
|
|
4443
4750
|
});
|
|
@@ -4556,10 +4863,19 @@ var WorkflowEngine = class {
|
|
|
4556
4863
|
off();
|
|
4557
4864
|
resolve(toResult(run));
|
|
4558
4865
|
}, "finish");
|
|
4866
|
+
const fail = /* @__PURE__ */ __name((error2) => {
|
|
4867
|
+
if (done) return;
|
|
4868
|
+
done = true;
|
|
4869
|
+
if (timer) clearTimeout(timer);
|
|
4870
|
+
if (poller) clearInterval(poller);
|
|
4871
|
+
off();
|
|
4872
|
+
reject(error2);
|
|
4873
|
+
}, "fail");
|
|
4559
4874
|
const check = /* @__PURE__ */ __name(() => {
|
|
4875
|
+
if (done) return;
|
|
4560
4876
|
void this.store.getRun(runId).then((run) => {
|
|
4561
4877
|
if (run && isSettled(run)) finish(run);
|
|
4562
|
-
});
|
|
4878
|
+
}, fail);
|
|
4563
4879
|
}, "check");
|
|
4564
4880
|
const poller = opts?.until === "terminal" ? (() => {
|
|
4565
4881
|
const interval = setInterval(check, 250);
|
|
@@ -4692,12 +5008,62 @@ var WorkflowEngine = class {
|
|
|
4692
5008
|
// for an ordinary (non-fan) signal/child await.
|
|
4693
5009
|
parallelGroup: waiter.parallelGroup
|
|
4694
5010
|
}));
|
|
5011
|
+
if (await this.fanStillWaiting(waiter, payload)) {
|
|
5012
|
+
return {
|
|
5013
|
+
runId: waiter.runId,
|
|
5014
|
+
status: "suspended"
|
|
5015
|
+
};
|
|
5016
|
+
}
|
|
4695
5017
|
return this.resumeImplicit(waiter.runId).catch((err) => {
|
|
4696
5018
|
if (err instanceof NamespaceMismatch) return null;
|
|
4697
5019
|
throw err;
|
|
4698
5020
|
});
|
|
4699
5021
|
}
|
|
4700
5022
|
/**
|
|
5023
|
+
* The `ctx.all` join barrier: true when `waiter` was one child of a fan (or of one wave of it)
|
|
5024
|
+
* whose siblings are still out, so waking the parent now would only replay the fan and re-suspend.
|
|
5025
|
+
* Skipping that wake is what keeps a fan of N children at O(N) work instead of N replays of N items.
|
|
5026
|
+
*
|
|
5027
|
+
* Safe because the LAST sibling to settle always wakes the parent: each child consumes its own
|
|
5028
|
+
* waiter before it asks, so whichever asks last sees none left. A failed child of a failFast fan
|
|
5029
|
+
* always wakes it. Anything that is not a `ctx.all` waiter, or a store that cannot answer the
|
|
5030
|
+
* question cheaply, wakes the parent as before.
|
|
5031
|
+
*/
|
|
5032
|
+
async fanStillWaiting(waiter, payload) {
|
|
5033
|
+
if (!this.store.hasSignalWaiter) return false;
|
|
5034
|
+
const fan = fanOfWaiter(waiter);
|
|
5035
|
+
if (!fan) return false;
|
|
5036
|
+
if (fan.failFast && isFailedCompletion(payload)) return false;
|
|
5037
|
+
return this.store.hasSignalWaiter(waiter.runId, fan.waiterPrefix);
|
|
5038
|
+
}
|
|
5039
|
+
/**
|
|
5040
|
+
* Re-drive a run that just suspended on awaited children (a `ctx.all` fan or wave, a `ctx.child`)
|
|
5041
|
+
* which, by now, have settled.
|
|
5042
|
+
*
|
|
5043
|
+
* {@link execute} drops a wake that finds the run's lease held. So if a child settles while the
|
|
5044
|
+
* parent is still executing (the turn that dispatched it, or one woken by an earlier child), that
|
|
5045
|
+
* wake is spent, and the turn suspends on a snapshot that predates the child. With the `ctx.all`
|
|
5046
|
+
* barrier that wake was the only one the fan would get. Nothing would wake the run until the
|
|
5047
|
+
* `reconcileMs` sweep, minutes later.
|
|
5048
|
+
*
|
|
5049
|
+
* Called after the turn released its lease, so any child that settles from here on wakes the run
|
|
5050
|
+
* itself. Only re-drives for a group with no waiter left whose checkpoints are all settled, which
|
|
5051
|
+
* the next turn is then guaranteed to see — so it cannot loop.
|
|
5052
|
+
*/
|
|
5053
|
+
async redriveSettledChildren(runId, fan) {
|
|
5054
|
+
if (!this.store.hasSignalWaiter) return;
|
|
5055
|
+
for (const wait of fan.waits) {
|
|
5056
|
+
if (await this.store.hasSignalWaiter(runId, wait.waiterPrefix)) continue;
|
|
5057
|
+
const checkpoints = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(runId, [
|
|
5058
|
+
wait.checkpointPrefix
|
|
5059
|
+
]) : (await this.store.listCheckpoints(runId)).filter((cp) => cp.name.startsWith(wait.checkpointPrefix));
|
|
5060
|
+
if (checkpoints.length > 0 && checkpoints.every((cp) => cp.status === "completed")) {
|
|
5061
|
+
await this.resumeImplicit(runId);
|
|
5062
|
+
return;
|
|
5063
|
+
}
|
|
5064
|
+
}
|
|
5065
|
+
}
|
|
5066
|
+
/**
|
|
4701
5067
|
* Deliver an external signal to the run waiting on `token`, resume it with `payload`, and return
|
|
4702
5068
|
* the run result — or null if no run is (or ends up) waiting for it.
|
|
4703
5069
|
*
|
|
@@ -4990,23 +5356,11 @@ var WorkflowEngine = class {
|
|
|
4990
5356
|
}
|
|
4991
5357
|
/** The given runs plus every descendant, children before parents. One waiter scan for the batch. */
|
|
4992
5358
|
async collectSubtrees(rootIds) {
|
|
4993
|
-
const
|
|
4994
|
-
for (const w of await this.store.listSignalWaiters("child:")) {
|
|
4995
|
-
const list = waitersByParent.get(w.runId) ?? [];
|
|
4996
|
-
list.push(w.token.slice("child:".length));
|
|
4997
|
-
waitersByParent.set(w.runId, list);
|
|
4998
|
-
}
|
|
4999
|
-
const prefixes = [
|
|
5000
|
-
"signal:child:",
|
|
5001
|
-
"spawn:"
|
|
5002
|
-
];
|
|
5359
|
+
const prefixes = CHILD_EDGE_PREFIXES;
|
|
5003
5360
|
const childrenOf = /* @__PURE__ */ __name(async (parent) => {
|
|
5004
|
-
const ids = new Set(
|
|
5361
|
+
const ids = new Set(await this.awaitedChildIds(parent));
|
|
5005
5362
|
const cps = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(parent, prefixes) : (await this.store.listCheckpoints(parent)).filter((cp) => prefixes.some((p) => cp.name.startsWith(p)));
|
|
5006
|
-
for (const
|
|
5007
|
-
if (cp.name.startsWith("signal:child:")) ids.add(cp.name.slice("signal:child:".length));
|
|
5008
|
-
if (cp.name.startsWith("spawn:") && typeof cp.output === "string") ids.add(cp.output);
|
|
5009
|
-
}
|
|
5363
|
+
for (const id of childIdsOf(cps)) ids.add(id);
|
|
5010
5364
|
return [
|
|
5011
5365
|
...ids
|
|
5012
5366
|
];
|
|
@@ -5056,22 +5410,20 @@ var WorkflowEngine = class {
|
|
|
5056
5410
|
* so a completed parent (or a completed awaited child) would otherwise drop out of the tree. The
|
|
5057
5411
|
* `signal:child:<id>` checkpoint (the placeholder written when the parent first awaits the child,
|
|
5058
5412
|
* overwritten as completed/failed when it settles) persists across completion, so we read both and
|
|
5059
|
-
* dedupe — the edge stays stable for a finished run, not just a live one.
|
|
5413
|
+
* dedupe — the edge stays stable for a finished run, not just a live one. A settled `ctx.all`
|
|
5414
|
+
* group folded into one `fan:` checkpoint stands for every child in it.
|
|
5060
5415
|
*/
|
|
5416
|
+
/** The children `parentRunId` is awaiting right now, read off its own live `child:` waiters. */
|
|
5417
|
+
async awaitedChildIds(parentRunId) {
|
|
5418
|
+
return (await listSignalWaitersOfRuns(this.store, [
|
|
5419
|
+
parentRunId
|
|
5420
|
+
])).filter((w) => w.token.startsWith("child:")).map((w) => w.token.slice("child:".length));
|
|
5421
|
+
}
|
|
5061
5422
|
async getRunChildren(parentRunId) {
|
|
5062
|
-
const childIds =
|
|
5063
|
-
|
|
5064
|
-
if (w.runId === parentRunId) childIds.add(w.token.slice("child:".length));
|
|
5065
|
-
}
|
|
5066
|
-
const prefixes = [
|
|
5067
|
-
"signal:child:",
|
|
5068
|
-
"spawn:"
|
|
5069
|
-
];
|
|
5423
|
+
const childIds = new Set(await this.awaitedChildIds(parentRunId));
|
|
5424
|
+
const prefixes = CHILD_EDGE_PREFIXES;
|
|
5070
5425
|
const childCheckpoints = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(parentRunId, prefixes) : (await this.store.listCheckpoints(parentRunId)).filter((cp) => prefixes.some((p) => cp.name.startsWith(p)));
|
|
5071
|
-
for (const
|
|
5072
|
-
if (cp.name.startsWith("signal:child:")) childIds.add(cp.name.slice("signal:child:".length));
|
|
5073
|
-
if (cp.name.startsWith("spawn:") && typeof cp.output === "string") childIds.add(cp.output);
|
|
5074
|
-
}
|
|
5426
|
+
for (const id of childIdsOf(childCheckpoints)) childIds.add(id);
|
|
5075
5427
|
return [
|
|
5076
5428
|
...childIds
|
|
5077
5429
|
];
|
|
@@ -5342,6 +5694,26 @@ var WorkflowEngine = class {
|
|
|
5342
5694
|
durationMs: Date.now() - step.startedAt.getTime()
|
|
5343
5695
|
});
|
|
5344
5696
|
}
|
|
5697
|
+
/** Admission persists the wait atomically; never overwrite a cancellation after it returns. */
|
|
5698
|
+
async singletonWaitResult(run) {
|
|
5699
|
+
const current = await this.store.getRun(run.id);
|
|
5700
|
+
await this.store.releaseRunLock(run.id);
|
|
5701
|
+
if (!current) throw new Error(`run ${run.id} was deleted during singleton admission`);
|
|
5702
|
+
if (current.status === "suspended") {
|
|
5703
|
+
this.emit({
|
|
5704
|
+
type: "run.suspended",
|
|
5705
|
+
runId: current.id,
|
|
5706
|
+
workflow: current.workflow,
|
|
5707
|
+
namespace: current.namespace
|
|
5708
|
+
});
|
|
5709
|
+
}
|
|
5710
|
+
return {
|
|
5711
|
+
runId: current.id,
|
|
5712
|
+
status: current.status,
|
|
5713
|
+
output: current.output,
|
|
5714
|
+
error: current.error
|
|
5715
|
+
};
|
|
5716
|
+
}
|
|
5345
5717
|
async execute(run, fn) {
|
|
5346
5718
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
5347
5719
|
const lockNow = this.clock();
|
|
@@ -5511,23 +5883,7 @@ var WorkflowEngine = class {
|
|
|
5511
5883
|
});
|
|
5512
5884
|
}
|
|
5513
5885
|
if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
5514
|
-
|
|
5515
|
-
await this.store.updateRun(run.id, {
|
|
5516
|
-
status: "suspended",
|
|
5517
|
-
wakeAt,
|
|
5518
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
5519
|
-
});
|
|
5520
|
-
this.emit({
|
|
5521
|
-
type: "run.suspended",
|
|
5522
|
-
runId: run.id,
|
|
5523
|
-
workflow: run.workflow,
|
|
5524
|
-
namespace: run.namespace
|
|
5525
|
-
});
|
|
5526
|
-
await this.store.releaseRunLock(run.id);
|
|
5527
|
-
return {
|
|
5528
|
-
runId: run.id,
|
|
5529
|
-
status: "suspended"
|
|
5530
|
-
};
|
|
5886
|
+
return this.singletonWaitResult(run);
|
|
5531
5887
|
}
|
|
5532
5888
|
const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
|
|
5533
5889
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
@@ -6141,28 +6497,13 @@ var WorkflowEngine = class {
|
|
|
6141
6497
|
});
|
|
6142
6498
|
}
|
|
6143
6499
|
if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
6144
|
-
|
|
6145
|
-
await this.store.updateRun(run.id, {
|
|
6146
|
-
status: "suspended",
|
|
6147
|
-
wakeAt,
|
|
6148
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
6149
|
-
});
|
|
6150
|
-
this.emit({
|
|
6151
|
-
type: "run.suspended",
|
|
6152
|
-
runId: run.id,
|
|
6153
|
-
workflow: run.workflow,
|
|
6154
|
-
namespace: run.namespace
|
|
6155
|
-
});
|
|
6156
|
-
await this.store.releaseRunLock(run.id);
|
|
6157
|
-
return {
|
|
6158
|
-
runId: run.id,
|
|
6159
|
-
status: "suspended"
|
|
6160
|
-
};
|
|
6500
|
+
return this.singletonWaitResult(run);
|
|
6161
6501
|
}
|
|
6162
6502
|
const compensations = [];
|
|
6163
6503
|
const snapshot = await this.store.listCheckpoints(run.id);
|
|
6164
6504
|
const replay = /* @__PURE__ */ new Map();
|
|
6165
6505
|
for (const cp of snapshot) replay.set(cp.seq, cp);
|
|
6506
|
+
let suspendedOnFan;
|
|
6166
6507
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
|
|
6167
6508
|
try {
|
|
6168
6509
|
const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
|
|
@@ -6214,10 +6555,12 @@ var WorkflowEngine = class {
|
|
|
6214
6555
|
if (this.cancelRequested.has(run.id)) {
|
|
6215
6556
|
return await this.compensateAndCancel(run, compensations, replay);
|
|
6216
6557
|
}
|
|
6217
|
-
|
|
6558
|
+
const suspended = await this.settleRun(run, {
|
|
6218
6559
|
kind: "suspended",
|
|
6219
6560
|
wakeAt: err.wakeAt
|
|
6220
6561
|
});
|
|
6562
|
+
suspendedOnFan = err.fan;
|
|
6563
|
+
return suspended;
|
|
6221
6564
|
}
|
|
6222
6565
|
if (this.cancelRequested.has(run.id)) {
|
|
6223
6566
|
return await this.compensateAndCancel(run, compensations, replay);
|
|
@@ -6237,6 +6580,9 @@ var WorkflowEngine = class {
|
|
|
6237
6580
|
});
|
|
6238
6581
|
} finally {
|
|
6239
6582
|
await this.store.releaseRunLock(run.id);
|
|
6583
|
+
if (suspendedOnFan) {
|
|
6584
|
+
void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
|
|
6585
|
+
}
|
|
6240
6586
|
}
|
|
6241
6587
|
}
|
|
6242
6588
|
/**
|
|
@@ -7051,6 +7397,21 @@ var RemoteStepError = class extends Error {
|
|
|
7051
7397
|
this.stepError = stepError;
|
|
7052
7398
|
}
|
|
7053
7399
|
};
|
|
7400
|
+
var CHILD_EDGE_PREFIXES = [
|
|
7401
|
+
"signal:child:",
|
|
7402
|
+
FOLDED_PREFIX,
|
|
7403
|
+
"spawn:"
|
|
7404
|
+
];
|
|
7405
|
+
function childIdsOf(checkpoints) {
|
|
7406
|
+
const ids = [];
|
|
7407
|
+
for (const cp of checkpoints) {
|
|
7408
|
+
if (cp.name.startsWith("signal:child:")) ids.push(cp.name.slice("signal:child:".length));
|
|
7409
|
+
if (cp.name.startsWith(FOLDED_PREFIX)) ids.push(...foldedChildIds(cp));
|
|
7410
|
+
if (cp.name.startsWith("spawn:") && typeof cp.output === "string") ids.push(cp.output);
|
|
7411
|
+
}
|
|
7412
|
+
return ids;
|
|
7413
|
+
}
|
|
7414
|
+
__name(childIdsOf, "childIdsOf");
|
|
7054
7415
|
|
|
7055
7416
|
// src/run-facets.ts
|
|
7056
7417
|
function facetOrigin(origin) {
|
|
@@ -7176,6 +7537,8 @@ var CodecStateStore = class {
|
|
|
7176
7537
|
}
|
|
7177
7538
|
inner;
|
|
7178
7539
|
codec;
|
|
7540
|
+
/** Atomic claims touch only searchable metadata and bypass payload encoding. */
|
|
7541
|
+
tryAdmitSingleton;
|
|
7179
7542
|
/**
|
|
7180
7543
|
* Forwarded verbatim — a facet cell counts `status`/`origin`, neither of which the codec touches
|
|
7181
7544
|
* (it encodes payloads: input/output/error). Bound as a PROPERTY, and only when the inner store
|
|
@@ -7198,6 +7561,14 @@ var CodecStateStore = class {
|
|
|
7198
7561
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store counts, so the
|
|
7199
7562
|
* engine's facet/listing fallback still applies through the wrapper. */
|
|
7200
7563
|
countRuns;
|
|
7564
|
+
/** The placeholder checkpoints are encoded like any other checkpoint. */
|
|
7565
|
+
registerChildWaits;
|
|
7566
|
+
/** The folded checkpoint carries the children's outputs, so it is encoded. */
|
|
7567
|
+
compactCheckpoints;
|
|
7568
|
+
/** Forwarded verbatim — a waiter carries no payload. */
|
|
7569
|
+
listSignalWaitersByRun;
|
|
7570
|
+
/** Forwarded verbatim — a waiter carries no payload. */
|
|
7571
|
+
hasSignalWaiter;
|
|
7201
7572
|
/** Schedules persist their run `input` inside `spec` — encoded like any other payload. Bound only
|
|
7202
7573
|
* when the inner store persists schedules, so `engine.schedules` reports absence faithfully. */
|
|
7203
7574
|
saveSchedule;
|
|
@@ -7208,12 +7579,24 @@ var CodecStateStore = class {
|
|
|
7208
7579
|
constructor(inner, codec) {
|
|
7209
7580
|
this.inner = inner;
|
|
7210
7581
|
this.codec = codec;
|
|
7582
|
+
if (inner.tryAdmitSingleton) this.tryAdmitSingleton = inner.tryAdmitSingleton.bind(inner);
|
|
7211
7583
|
const deleteRuns = inner.deleteRuns;
|
|
7212
7584
|
if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
|
|
7213
7585
|
const prune = inner.pruneTerminalRuns;
|
|
7214
7586
|
if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
|
|
7215
7587
|
const count = inner.countRuns;
|
|
7216
7588
|
if (count) this.countRuns = (query) => count.call(inner, query);
|
|
7589
|
+
const registerChildWaits = inner.registerChildWaits;
|
|
7590
|
+
if (registerChildWaits) this.registerChildWaits = (entries) => registerChildWaits.call(inner, entries.map((e) => ({
|
|
7591
|
+
waiter: e.waiter,
|
|
7592
|
+
checkpoint: this.encCp(e.checkpoint)
|
|
7593
|
+
})));
|
|
7594
|
+
const compact = inner.compactCheckpoints;
|
|
7595
|
+
if (compact) this.compactCheckpoints = (runId, fromSeq, toSeq, into) => compact.call(inner, runId, fromSeq, toSeq, this.encCp(into));
|
|
7596
|
+
const byRun = inner.listSignalWaitersByRun;
|
|
7597
|
+
if (byRun) this.listSignalWaitersByRun = (runIds) => byRun.call(inner, runIds);
|
|
7598
|
+
const hasWaiter = inner.hasSignalWaiter;
|
|
7599
|
+
if (hasWaiter) this.hasSignalWaiter = (runId, tokenPrefix) => hasWaiter.call(inner, runId, tokenPrefix);
|
|
7217
7600
|
const { saveSchedule, getSchedule, updateSchedule, deleteSchedule, listSchedules } = inner;
|
|
7218
7601
|
if (saveSchedule && getSchedule && updateSchedule && deleteSchedule && listSchedules) {
|
|
7219
7602
|
this.saveSchedule = (r) => saveSchedule.call(inner, this.encSchedule(r));
|
|
@@ -7660,6 +8043,28 @@ var InMemoryStateStore = class {
|
|
|
7660
8043
|
byRun.set(runId, row);
|
|
7661
8044
|
}
|
|
7662
8045
|
}
|
|
8046
|
+
async tryAdmitSingleton(runId, tag, workflow, limit, retryWakeAt) {
|
|
8047
|
+
const runs = [
|
|
8048
|
+
...this.runs.values()
|
|
8049
|
+
];
|
|
8050
|
+
const tags = singletonAdmissionTags(runs, runId, tag, workflow, limit);
|
|
8051
|
+
if (!tags) {
|
|
8052
|
+
const patch = singletonAdmissionDenialPatch(runs, runId, tag, workflow, retryWakeAt);
|
|
8053
|
+
const candidate = this.runs.get(runId);
|
|
8054
|
+
if (patch && candidate) this.runs.set(runId, {
|
|
8055
|
+
...candidate,
|
|
8056
|
+
...patch
|
|
8057
|
+
});
|
|
8058
|
+
return false;
|
|
8059
|
+
}
|
|
8060
|
+
const run = this.runs.get(runId);
|
|
8061
|
+
if (!run) return false;
|
|
8062
|
+
this.runs.set(runId, {
|
|
8063
|
+
...run,
|
|
8064
|
+
tags
|
|
8065
|
+
});
|
|
8066
|
+
return true;
|
|
8067
|
+
}
|
|
7663
8068
|
async createRun(run) {
|
|
7664
8069
|
this.runs.set(run.id, {
|
|
7665
8070
|
...run,
|
|
@@ -7667,7 +8072,11 @@ var InMemoryStateStore = class {
|
|
|
7667
8072
|
});
|
|
7668
8073
|
this.reindexAttributes(run.id, run.searchAttributes);
|
|
7669
8074
|
}
|
|
7670
|
-
async updateRun(runId,
|
|
8075
|
+
async updateRun(runId, inputPatch) {
|
|
8076
|
+
const patch = inputPatch.status && TERMINAL_RUN_STATUSES.includes(inputPatch.status) ? {
|
|
8077
|
+
...inputPatch,
|
|
8078
|
+
tags: ("tags" in inputPatch ? inputPatch.tags : this.runs.get(runId)?.tags)?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
|
|
8079
|
+
} : inputPatch;
|
|
7671
8080
|
const existing = this.runs.get(runId);
|
|
7672
8081
|
if (!existing) throw new Error(`run ${runId} not found`);
|
|
7673
8082
|
const next = {
|
|
@@ -7798,6 +8207,33 @@ var InMemoryStateStore = class {
|
|
|
7798
8207
|
...w
|
|
7799
8208
|
}));
|
|
7800
8209
|
}
|
|
8210
|
+
async registerChildWaits(entries) {
|
|
8211
|
+
for (const { waiter } of entries) this.signalWaiters.set(waiter.token, {
|
|
8212
|
+
...waiter
|
|
8213
|
+
});
|
|
8214
|
+
for (const { checkpoint } of entries) await this.saveCheckpoint(checkpoint);
|
|
8215
|
+
}
|
|
8216
|
+
async compactCheckpoints(runId, fromSeq, toSeq, into) {
|
|
8217
|
+
for (let seq = fromSeq; seq <= toSeq; seq += 1) this.checkpoints.delete(this.key(runId, seq));
|
|
8218
|
+
this.checkpoints.set(this.key(runId, into.seq), {
|
|
8219
|
+
...into,
|
|
8220
|
+
runId
|
|
8221
|
+
});
|
|
8222
|
+
}
|
|
8223
|
+
async listSignalWaitersByRun(runIds) {
|
|
8224
|
+
const ids = new Set(runIds);
|
|
8225
|
+
return [
|
|
8226
|
+
...this.signalWaiters.values()
|
|
8227
|
+
].filter((w) => ids.has(w.runId)).map((w) => ({
|
|
8228
|
+
...w
|
|
8229
|
+
}));
|
|
8230
|
+
}
|
|
8231
|
+
async hasSignalWaiter(runId, tokenPrefix) {
|
|
8232
|
+
for (const w of this.signalWaiters.values()) {
|
|
8233
|
+
if (w.runId === runId && w.token.startsWith(tokenPrefix)) return true;
|
|
8234
|
+
}
|
|
8235
|
+
return false;
|
|
8236
|
+
}
|
|
7801
8237
|
async removeSignalWaiter(waiter) {
|
|
7802
8238
|
const current = this.signalWaiters.get(waiter.token);
|
|
7803
8239
|
if (current && current.runId === waiter.runId && current.seq === waiter.seq) {
|
|
@@ -8091,6 +8527,8 @@ export {
|
|
|
8091
8527
|
RemoteWorkflowExecutor,
|
|
8092
8528
|
RemoteWorkflowTimeout,
|
|
8093
8529
|
RunGateway,
|
|
8530
|
+
SINGLETON_ACTIVE_STATUSES,
|
|
8531
|
+
SINGLETON_ADMITTED_TAG,
|
|
8094
8532
|
STATE_STORE,
|
|
8095
8533
|
STATE_STORE_CANONICAL,
|
|
8096
8534
|
ScheduleAlreadyExistsError,
|
|
@@ -8121,6 +8559,7 @@ export {
|
|
|
8121
8559
|
capableWorkers,
|
|
8122
8560
|
classifyWaiterToken,
|
|
8123
8561
|
clearDurableExecutionWrappers,
|
|
8562
|
+
clearTerminalSingletonAdmission,
|
|
8124
8563
|
collectMetrics,
|
|
8125
8564
|
concurrencyTag,
|
|
8126
8565
|
controlPlaneDescriptor,
|
|
@@ -8149,6 +8588,7 @@ export {
|
|
|
8149
8588
|
isLegacyDescriptor,
|
|
8150
8589
|
isNonEmptyRunScope,
|
|
8151
8590
|
isWorkflowControlFlowSignal,
|
|
8591
|
+
listSignalWaitersOfRuns,
|
|
8152
8592
|
log,
|
|
8153
8593
|
matchesAttributes,
|
|
8154
8594
|
mergeRunFacetRows,
|
|
@@ -8159,6 +8599,7 @@ export {
|
|
|
8159
8599
|
normalizeAttributeRows,
|
|
8160
8600
|
normalizeDescriptor,
|
|
8161
8601
|
observedAnnouncements,
|
|
8602
|
+
parentRunIdOf,
|
|
8162
8603
|
parseDuration,
|
|
8163
8604
|
partitionOfToken,
|
|
8164
8605
|
planDispatch,
|
|
@@ -8178,6 +8619,8 @@ export {
|
|
|
8178
8619
|
runValueFacetsFromRuns,
|
|
8179
8620
|
sanitizeQueueToken,
|
|
8180
8621
|
scheduledRunId,
|
|
8622
|
+
singletonAdmissionDenialPatch,
|
|
8623
|
+
singletonAdmissionTags,
|
|
8181
8624
|
sqlComparator,
|
|
8182
8625
|
stepConfigOf,
|
|
8183
8626
|
stepId,
|