@dudousxd/nestjs-durable-core 0.29.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +272 -212
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +31 -15
- package/dist/index.d.ts +31 -15
- package/dist/index.js +272 -212
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -275,6 +275,31 @@ async function runStepHandler(task, handler) {
|
|
|
275
275
|
__name(runStepHandler, "runStepHandler");
|
|
276
276
|
|
|
277
277
|
// src/checkpoints.ts
|
|
278
|
+
function nonEmptyEvents(events) {
|
|
279
|
+
return events && events.length > 0 ? events : void 0;
|
|
280
|
+
}
|
|
281
|
+
__name(nonEmptyEvents, "nonEmptyEvents");
|
|
282
|
+
function stepCheckpoint(p) {
|
|
283
|
+
return {
|
|
284
|
+
runId: p.runId,
|
|
285
|
+
seq: p.seq,
|
|
286
|
+
name: p.name,
|
|
287
|
+
kind: p.kind,
|
|
288
|
+
stepId: stepId(p.runId, p.seq),
|
|
289
|
+
status: p.status,
|
|
290
|
+
input: p.input,
|
|
291
|
+
output: p.output,
|
|
292
|
+
error: p.error,
|
|
293
|
+
events: nonEmptyEvents(p.events),
|
|
294
|
+
attempts: p.attempts,
|
|
295
|
+
workerGroup: p.workerGroup,
|
|
296
|
+
wakeAt: p.wakeAt,
|
|
297
|
+
enqueuedAt: p.enqueuedAt,
|
|
298
|
+
startedAt: p.startedAt,
|
|
299
|
+
finishedAt: p.finishedAt
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
__name(stepCheckpoint, "stepCheckpoint");
|
|
278
303
|
function instantCheckpoint(p) {
|
|
279
304
|
const at = /* @__PURE__ */ new Date();
|
|
280
305
|
return {
|
|
@@ -683,6 +708,93 @@ var QueueController = class {
|
|
|
683
708
|
}
|
|
684
709
|
};
|
|
685
710
|
|
|
711
|
+
// src/singleton-gate.ts
|
|
712
|
+
var SINGLETON_RETRY_MS = 1e3;
|
|
713
|
+
var SINGLETON_RETRY_JITTER_MS = 250;
|
|
714
|
+
var SingletonGate = class {
|
|
715
|
+
static {
|
|
716
|
+
__name(this, "SingletonGate");
|
|
717
|
+
}
|
|
718
|
+
deps;
|
|
719
|
+
constructor(deps) {
|
|
720
|
+
this.deps = deps;
|
|
721
|
+
}
|
|
722
|
+
/** The tag a singleton run carries, so the gate can find others sharing its key. */
|
|
723
|
+
tag(cfg, input) {
|
|
724
|
+
return `singleton:${cfg.key(input)}`;
|
|
725
|
+
}
|
|
726
|
+
/** Next wake time for a gated run: the base retry delay jittered to avoid a wakeup stampede. */
|
|
727
|
+
retryWakeAt() {
|
|
728
|
+
return this.deps.clock() + SINGLETON_RETRY_MS + Math.floor((Math.random() * 2 - 1) * SINGLETON_RETRY_JITTER_MS);
|
|
729
|
+
}
|
|
730
|
+
/**
|
|
731
|
+
* Reject a start that would grow the same-key backlog past `limit + maxQueueDepth` (counting
|
|
732
|
+
* `pending`/`running`/`suspended` runs sharing the key in one scan). No-op when no `maxQueueDepth`
|
|
733
|
+
* is configured.
|
|
734
|
+
*/
|
|
735
|
+
async assertCapacity(workflow, cfg, input) {
|
|
736
|
+
if (cfg.maxQueueDepth == null) return;
|
|
737
|
+
const cap = (cfg.limit ?? 1) + cfg.maxQueueDepth;
|
|
738
|
+
const queued = await this.deps.store.listRuns({
|
|
739
|
+
tag: this.tag(cfg, input),
|
|
740
|
+
workflow,
|
|
741
|
+
statuses: [
|
|
742
|
+
"pending",
|
|
743
|
+
"running",
|
|
744
|
+
"suspended"
|
|
745
|
+
]
|
|
746
|
+
});
|
|
747
|
+
if (queued.length >= cap) {
|
|
748
|
+
throw new SingletonQueueFullError(workflow, cfg.key(input), cfg.maxQueueDepth);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* Whether `run` may run now under its key: it's among the `limit` oldest in-flight (running or
|
|
753
|
+
* suspended) runs sharing the key, by `(createdAt, id)` order. A consistent store gives every
|
|
754
|
+
* instance the same ordering, so admission is race-free + FIFO.
|
|
755
|
+
*/
|
|
756
|
+
async admit(run, cfg) {
|
|
757
|
+
const inflight = (await this.deps.store.listRuns({
|
|
758
|
+
tag: this.tag(cfg, run.input),
|
|
759
|
+
workflow: run.workflow,
|
|
760
|
+
statuses: [
|
|
761
|
+
"running",
|
|
762
|
+
"suspended"
|
|
763
|
+
]
|
|
764
|
+
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
765
|
+
const idx = inflight.findIndex((r) => r.id === run.id);
|
|
766
|
+
return idx >= 0 && idx < (cfg.limit ?? 1);
|
|
767
|
+
}
|
|
768
|
+
/**
|
|
769
|
+
* Notify-on-release: a singleton run settled (freeing a slot), so dispatch the oldest gated
|
|
770
|
+
* (`suspended` + same tag) waiters now instead of waiting for their ~1s retry timer. Each re-checks
|
|
771
|
+
* admission in the executor and runs only if it actually wins a slot, so FIFO/race-free guarantees
|
|
772
|
+
* hold; the durable timer remains the cross-instance/crash fallback.
|
|
773
|
+
*/
|
|
774
|
+
async wakeNext(settled) {
|
|
775
|
+
const cfg = this.deps.configFor(settled);
|
|
776
|
+
if (!cfg) return;
|
|
777
|
+
const tag = settled.tags?.find((t) => t.startsWith("singleton:"));
|
|
778
|
+
if (!tag) return;
|
|
779
|
+
const gated = (await this.deps.store.listRuns({
|
|
780
|
+
tag,
|
|
781
|
+
workflow: settled.workflow,
|
|
782
|
+
statuses: [
|
|
783
|
+
"suspended"
|
|
784
|
+
]
|
|
785
|
+
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
786
|
+
for (const next of gated.slice(0, cfg.limit ?? 1)) {
|
|
787
|
+
if (next.wakeAt != null) {
|
|
788
|
+
await this.deps.store.updateRun(next.id, {
|
|
789
|
+
wakeAt: void 0,
|
|
790
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
791
|
+
}).catch(() => void 0);
|
|
792
|
+
}
|
|
793
|
+
this.deps.dispatch(next.id);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
};
|
|
797
|
+
|
|
686
798
|
// src/transport-pool.ts
|
|
687
799
|
var TransportPool = class {
|
|
688
800
|
static {
|
|
@@ -1213,10 +1325,6 @@ function createWorkflowCtx(host, runId, compensations, workflow = "") {
|
|
|
1213
1325
|
__name(createWorkflowCtx, "createWorkflowCtx");
|
|
1214
1326
|
|
|
1215
1327
|
// src/engine.ts
|
|
1216
|
-
var SINGLETON_RETRY_MS = 1e3;
|
|
1217
|
-
var SINGLETON_RETRY_JITTER_MS = 250;
|
|
1218
|
-
var singletonRetryWakeAt = /* @__PURE__ */ __name((nowMs) => nowMs + SINGLETON_RETRY_MS + Math.floor((Math.random() * 2 - 1) * SINGLETON_RETRY_JITTER_MS), "singletonRetryWakeAt");
|
|
1219
|
-
var singletonTag = /* @__PURE__ */ __name((cfg, input) => `singleton:${cfg.key(input)}`, "singletonTag");
|
|
1220
1328
|
function mergeTags(staticTags, runTags) {
|
|
1221
1329
|
if (!staticTags?.length && !runTags?.length) return void 0;
|
|
1222
1330
|
return [
|
|
@@ -1266,6 +1374,8 @@ var WorkflowEngine = class {
|
|
|
1266
1374
|
trackStepStart;
|
|
1267
1375
|
/** Where a freshly-started run executes — in-process by default (see {@link RunDispatcher}). */
|
|
1268
1376
|
runDispatcher;
|
|
1377
|
+
/** Per-key serialization for singleton workflows (admission, back-pressure, notify-on-release). */
|
|
1378
|
+
singletons;
|
|
1269
1379
|
/** Every registered workflow, keyed by `name@version` — so old versions stay runnable. */
|
|
1270
1380
|
workflows = /* @__PURE__ */ new Map();
|
|
1271
1381
|
/** The newest registered version per workflow name — used to `start` new runs. */
|
|
@@ -1323,6 +1433,14 @@ var WorkflowEngine = class {
|
|
|
1323
1433
|
dispatch: /* @__PURE__ */ __name((runId) => queueMicrotask(() => void this.runOne(runId).catch(() => {
|
|
1324
1434
|
})), "dispatch")
|
|
1325
1435
|
};
|
|
1436
|
+
this.singletons = new SingletonGate({
|
|
1437
|
+
store: this.store,
|
|
1438
|
+
clock: this.clock,
|
|
1439
|
+
dispatch: /* @__PURE__ */ __name((runId) => {
|
|
1440
|
+
this.runDispatcher.dispatch(runId);
|
|
1441
|
+
}, "dispatch"),
|
|
1442
|
+
configFor: /* @__PURE__ */ __name((run) => (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton, "configFor")
|
|
1443
|
+
});
|
|
1326
1444
|
this.pool.bind(
|
|
1327
1445
|
async (result) => {
|
|
1328
1446
|
const waiter = this.pending.get(result.stepId);
|
|
@@ -1573,25 +1691,10 @@ var WorkflowEngine = class {
|
|
|
1573
1691
|
const now = /* @__PURE__ */ new Date();
|
|
1574
1692
|
const tags = mergeTags(registered.tags, registered.singleton ? [
|
|
1575
1693
|
...opts?.tags ?? [],
|
|
1576
|
-
|
|
1694
|
+
this.singletons.tag(registered.singleton, input)
|
|
1577
1695
|
] : opts?.tags);
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
const cfg = registered.singleton;
|
|
1581
|
-
const key = cfg.key(input);
|
|
1582
|
-
const cap = (cfg.limit ?? 1) + maxQueueDepth;
|
|
1583
|
-
const queued = await this.store.listRuns({
|
|
1584
|
-
tag: singletonTag(cfg, input),
|
|
1585
|
-
workflow: name,
|
|
1586
|
-
statuses: [
|
|
1587
|
-
"pending",
|
|
1588
|
-
"running",
|
|
1589
|
-
"suspended"
|
|
1590
|
-
]
|
|
1591
|
-
});
|
|
1592
|
-
if (queued.length >= cap) {
|
|
1593
|
-
throw new SingletonQueueFullError(name, key, maxQueueDepth);
|
|
1594
|
-
}
|
|
1696
|
+
if (registered.singleton) {
|
|
1697
|
+
await this.singletons.assertCapacity(name, registered.singleton, input);
|
|
1595
1698
|
}
|
|
1596
1699
|
const run = {
|
|
1597
1700
|
id: runId,
|
|
@@ -2050,7 +2153,7 @@ var WorkflowEngine = class {
|
|
|
2050
2153
|
from: this.instanceId
|
|
2051
2154
|
}).catch(() => void 0);
|
|
2052
2155
|
}
|
|
2053
|
-
void this.
|
|
2156
|
+
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
2054
2157
|
return {
|
|
2055
2158
|
runId,
|
|
2056
2159
|
status: "cancelled",
|
|
@@ -2076,6 +2179,26 @@ var WorkflowEngine = class {
|
|
|
2076
2179
|
return results;
|
|
2077
2180
|
}
|
|
2078
2181
|
/**
|
|
2182
|
+
* Hard-delete a run and its entire subtree. Unlike {@link cancel} (which marks a run `cancelled`
|
|
2183
|
+
* but keeps it as history), delete REMOVES the run and all its rows (checkpoints, signal waiters,
|
|
2184
|
+
* search-attribute rows) — it vanishes from {@link getRun} and {@link listRuns}. Use it to purge a
|
|
2185
|
+
* finished run whose data is being deleted (e.g. a pipeline run whose result rows are cleared).
|
|
2186
|
+
*
|
|
2187
|
+
* Cascades depth-first via {@link getRunChildren} so children are gone before the parent (no
|
|
2188
|
+
* orphaned child runs). Prefer {@link cancel} first for a live run — deleting one mid-flight orphans
|
|
2189
|
+
* its worker (its next checkpoint write fails). Returns the number of runs deleted (0 if absent).
|
|
2190
|
+
*/
|
|
2191
|
+
async deleteRun(runId) {
|
|
2192
|
+
const run = await this.store.getRun(runId);
|
|
2193
|
+
if (!run) return 0;
|
|
2194
|
+
let deleted = 0;
|
|
2195
|
+
for (const childId of await this.getRunChildren(runId)) {
|
|
2196
|
+
deleted += await this.deleteRun(childId);
|
|
2197
|
+
}
|
|
2198
|
+
await this.store.deleteRun(runId);
|
|
2199
|
+
return deleted + 1;
|
|
2200
|
+
}
|
|
2201
|
+
/**
|
|
2079
2202
|
* Cascade cancellation to a run's children — both awaited (`ctx.child`, found via its live
|
|
2080
2203
|
* `child:<id>` waiter) and fire-and-forget (`ctx.startChild`, found via its `spawn:<id>`
|
|
2081
2204
|
* checkpoint). Recursive, so a whole subtree is cancelled; the terminal guard in `cancel` stops it
|
|
@@ -2157,21 +2280,19 @@ var WorkflowEngine = class {
|
|
|
2157
2280
|
*/
|
|
2158
2281
|
async persistStepEvent(event) {
|
|
2159
2282
|
const startedAt = new Date(event.startedAt);
|
|
2160
|
-
const events = event.events && event.events.length > 0 ? event.events : void 0;
|
|
2161
2283
|
if (event.phase === "running") {
|
|
2162
|
-
await this.store.saveCheckpoint({
|
|
2284
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2163
2285
|
runId: event.runId,
|
|
2164
2286
|
seq: event.seq,
|
|
2165
2287
|
name: event.name,
|
|
2166
2288
|
kind: "local",
|
|
2167
|
-
stepId: stepId(event.runId, event.seq),
|
|
2168
2289
|
status: "running",
|
|
2169
|
-
events,
|
|
2290
|
+
events: event.events,
|
|
2170
2291
|
attempts: 1,
|
|
2171
2292
|
enqueuedAt: startedAt,
|
|
2172
2293
|
startedAt,
|
|
2173
2294
|
finishedAt: startedAt
|
|
2174
|
-
});
|
|
2295
|
+
}));
|
|
2175
2296
|
this.emit({
|
|
2176
2297
|
type: "step.started",
|
|
2177
2298
|
runId: event.runId,
|
|
@@ -2182,21 +2303,20 @@ var WorkflowEngine = class {
|
|
|
2182
2303
|
return;
|
|
2183
2304
|
}
|
|
2184
2305
|
const failed = event.phase === "failed";
|
|
2185
|
-
await this.store.saveCheckpoint({
|
|
2306
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2186
2307
|
runId: event.runId,
|
|
2187
2308
|
seq: event.seq,
|
|
2188
2309
|
name: event.name,
|
|
2189
2310
|
kind: "local",
|
|
2190
|
-
stepId: stepId(event.runId, event.seq),
|
|
2191
2311
|
status: failed ? "failed" : "completed",
|
|
2192
2312
|
output: failed ? void 0 : event.output,
|
|
2193
2313
|
error: failed ? event.error : void 0,
|
|
2194
|
-
events,
|
|
2314
|
+
events: event.events,
|
|
2195
2315
|
attempts: 1,
|
|
2196
2316
|
enqueuedAt: startedAt,
|
|
2197
2317
|
startedAt,
|
|
2198
2318
|
finishedAt: event.finishedAt != null ? new Date(event.finishedAt) : /* @__PURE__ */ new Date()
|
|
2199
|
-
});
|
|
2319
|
+
}));
|
|
2200
2320
|
this.emit({
|
|
2201
2321
|
type: failed ? "step.failed" : "step.completed",
|
|
2202
2322
|
runId: event.runId,
|
|
@@ -2216,21 +2336,20 @@ var WorkflowEngine = class {
|
|
|
2216
2336
|
*/
|
|
2217
2337
|
async startStep(step) {
|
|
2218
2338
|
if (this.trackStepStart) {
|
|
2219
|
-
await this.store.saveCheckpoint({
|
|
2339
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2220
2340
|
runId: step.runId,
|
|
2221
2341
|
seq: step.seq,
|
|
2222
2342
|
name: step.name,
|
|
2223
2343
|
kind: step.kind,
|
|
2224
|
-
stepId: stepId(step.runId, step.seq),
|
|
2225
2344
|
status: "running",
|
|
2226
2345
|
input: step.input,
|
|
2227
|
-
events: step.events
|
|
2346
|
+
events: step.events,
|
|
2228
2347
|
attempts: step.attempts,
|
|
2229
2348
|
workerGroup: step.workerGroup,
|
|
2230
2349
|
enqueuedAt: step.enqueuedAt,
|
|
2231
2350
|
startedAt: step.startedAt,
|
|
2232
2351
|
finishedAt: step.startedAt
|
|
2233
|
-
});
|
|
2352
|
+
}));
|
|
2234
2353
|
}
|
|
2235
2354
|
this.emit({
|
|
2236
2355
|
type: "step.started",
|
|
@@ -2242,22 +2361,21 @@ var WorkflowEngine = class {
|
|
|
2242
2361
|
}
|
|
2243
2362
|
/** Checkpoint a finished step and announce it — the two things that must always happen together. */
|
|
2244
2363
|
async completeStep(step) {
|
|
2245
|
-
await this.store.saveCheckpoint({
|
|
2364
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2246
2365
|
runId: step.runId,
|
|
2247
2366
|
seq: step.seq,
|
|
2248
2367
|
name: step.name,
|
|
2249
2368
|
kind: step.kind,
|
|
2250
|
-
stepId: stepId(step.runId, step.seq),
|
|
2251
2369
|
status: "completed",
|
|
2252
2370
|
input: step.input,
|
|
2253
2371
|
output: step.output,
|
|
2254
|
-
events: step.events
|
|
2372
|
+
events: step.events,
|
|
2255
2373
|
attempts: step.attempts,
|
|
2256
2374
|
workerGroup: step.workerGroup,
|
|
2257
2375
|
enqueuedAt: step.enqueuedAt,
|
|
2258
2376
|
startedAt: step.startedAt,
|
|
2259
2377
|
finishedAt: /* @__PURE__ */ new Date()
|
|
2260
|
-
});
|
|
2378
|
+
}));
|
|
2261
2379
|
this.emit({
|
|
2262
2380
|
type: "step.completed",
|
|
2263
2381
|
runId: step.runId,
|
|
@@ -2271,22 +2389,21 @@ var WorkflowEngine = class {
|
|
|
2271
2389
|
}
|
|
2272
2390
|
/** Checkpoint a step that failed terminally, so the failure point is visible (not just the run). */
|
|
2273
2391
|
async failStep(step) {
|
|
2274
|
-
await this.store.saveCheckpoint({
|
|
2392
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2275
2393
|
runId: step.runId,
|
|
2276
2394
|
seq: step.seq,
|
|
2277
2395
|
name: step.name,
|
|
2278
2396
|
kind: step.kind,
|
|
2279
|
-
stepId: stepId(step.runId, step.seq),
|
|
2280
2397
|
status: "failed",
|
|
2281
2398
|
input: step.input,
|
|
2282
2399
|
error: step.error,
|
|
2283
|
-
events: step.events
|
|
2400
|
+
events: step.events,
|
|
2284
2401
|
attempts: step.attempts,
|
|
2285
2402
|
workerGroup: step.workerGroup,
|
|
2286
2403
|
enqueuedAt: step.enqueuedAt,
|
|
2287
2404
|
startedAt: step.startedAt,
|
|
2288
2405
|
finishedAt: /* @__PURE__ */ new Date()
|
|
2289
|
-
});
|
|
2406
|
+
}));
|
|
2290
2407
|
this.emit({
|
|
2291
2408
|
type: "step.failed",
|
|
2292
2409
|
runId: step.runId,
|
|
@@ -2298,55 +2415,6 @@ var WorkflowEngine = class {
|
|
|
2298
2415
|
durationMs: Date.now() - step.startedAt.getTime()
|
|
2299
2416
|
});
|
|
2300
2417
|
}
|
|
2301
|
-
/**
|
|
2302
|
-
* Whether `run` may run now under its singleton key: it's among the `limit` oldest in-flight runs
|
|
2303
|
-
* (running or suspended) sharing the key, by `(createdAt, id)` order. A consistent store gives every
|
|
2304
|
-
* instance the same ordering, so admission is race-free + FIFO.
|
|
2305
|
-
*/
|
|
2306
|
-
async admitSingleton(run, cfg) {
|
|
2307
|
-
const tag = singletonTag(cfg, run.input);
|
|
2308
|
-
const inflight = (await this.store.listRuns({
|
|
2309
|
-
tag,
|
|
2310
|
-
workflow: run.workflow,
|
|
2311
|
-
statuses: [
|
|
2312
|
-
"running",
|
|
2313
|
-
"suspended"
|
|
2314
|
-
]
|
|
2315
|
-
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2316
|
-
const idx = inflight.findIndex((r) => r.id === run.id);
|
|
2317
|
-
return idx >= 0 && idx < (cfg.limit ?? 1);
|
|
2318
|
-
}
|
|
2319
|
-
/**
|
|
2320
|
-
* Notify-on-release: a singleton run just settled (freeing a slot), so wake the gated waiters that
|
|
2321
|
-
* can now be admitted — immediately, instead of letting them sleep until their ~1s retry timer.
|
|
2322
|
-
* Dispatches up to `limit` of the oldest gated (`suspended` + same singleton tag) waiters by
|
|
2323
|
-
* `(createdAt, id)`; each re-checks admission in `runExecution` and runs only if it actually wins a
|
|
2324
|
-
* slot, so this never breaks the FIFO + race-free guarantees (the durable timer remains the
|
|
2325
|
-
* cross-instance / crash fallback — this just removes the poll latency on the happy path).
|
|
2326
|
-
*/
|
|
2327
|
-
async wakeNextSingletons(settled) {
|
|
2328
|
-
const registered = this.workflows.get(versionKey(settled.workflow, settled.workflowVersion)) ?? this.latest.get(settled.workflow);
|
|
2329
|
-
const cfg = registered?.singleton;
|
|
2330
|
-
if (!cfg) return;
|
|
2331
|
-
const tag = settled.tags?.find((t) => t.startsWith("singleton:"));
|
|
2332
|
-
if (!tag) return;
|
|
2333
|
-
const gated = (await this.store.listRuns({
|
|
2334
|
-
tag,
|
|
2335
|
-
workflow: settled.workflow,
|
|
2336
|
-
statuses: [
|
|
2337
|
-
"suspended"
|
|
2338
|
-
]
|
|
2339
|
-
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2340
|
-
for (const next of gated.slice(0, cfg.limit ?? 1)) {
|
|
2341
|
-
if (next.wakeAt != null) {
|
|
2342
|
-
await this.store.updateRun(next.id, {
|
|
2343
|
-
wakeAt: void 0,
|
|
2344
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2345
|
-
}).catch(() => void 0);
|
|
2346
|
-
}
|
|
2347
|
-
this.runDispatcher.dispatch(next.id);
|
|
2348
|
-
}
|
|
2349
|
-
}
|
|
2350
2418
|
async execute(run, fn) {
|
|
2351
2419
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
|
|
2352
2420
|
const lockNow = this.clock();
|
|
@@ -2365,7 +2433,7 @@ var WorkflowEngine = class {
|
|
|
2365
2433
|
try {
|
|
2366
2434
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
2367
2435
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
2368
|
-
void this.
|
|
2436
|
+
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
2369
2437
|
}
|
|
2370
2438
|
return result;
|
|
2371
2439
|
} finally {
|
|
@@ -2378,6 +2446,75 @@ var WorkflowEngine = class {
|
|
|
2378
2446
|
* {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
|
|
2379
2447
|
* us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
|
|
2380
2448
|
*/
|
|
2449
|
+
/**
|
|
2450
|
+
* Apply a terminal/suspended transition once: persist the new run status, emit the matching
|
|
2451
|
+
* lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
|
|
2452
|
+
* executors funnel their completed/failed/suspended outcomes through here so a status update can
|
|
2453
|
+
* never drift from its event or `notifyParent` call. Does NOT touch the run lock — each executor
|
|
2454
|
+
* keeps its own lease handling (the TS executor's `finally`, the remote path's caller).
|
|
2455
|
+
*/
|
|
2456
|
+
async settleRun(run, outcome) {
|
|
2457
|
+
const updatedAt = /* @__PURE__ */ new Date();
|
|
2458
|
+
if (outcome.kind === "completed") {
|
|
2459
|
+
await this.store.updateRun(run.id, {
|
|
2460
|
+
status: "completed",
|
|
2461
|
+
output: outcome.output,
|
|
2462
|
+
error: void 0,
|
|
2463
|
+
updatedAt
|
|
2464
|
+
});
|
|
2465
|
+
this.emit({
|
|
2466
|
+
type: "run.completed",
|
|
2467
|
+
runId: run.id,
|
|
2468
|
+
workflow: run.workflow,
|
|
2469
|
+
output: outcome.output
|
|
2470
|
+
});
|
|
2471
|
+
void this.notifyParent(run.id, {
|
|
2472
|
+
ok: true,
|
|
2473
|
+
value: outcome.output
|
|
2474
|
+
});
|
|
2475
|
+
return {
|
|
2476
|
+
runId: run.id,
|
|
2477
|
+
status: "completed",
|
|
2478
|
+
output: outcome.output
|
|
2479
|
+
};
|
|
2480
|
+
}
|
|
2481
|
+
if (outcome.kind === "failed") {
|
|
2482
|
+
await this.store.updateRun(run.id, {
|
|
2483
|
+
status: "failed",
|
|
2484
|
+
error: outcome.error,
|
|
2485
|
+
updatedAt
|
|
2486
|
+
});
|
|
2487
|
+
this.emit({
|
|
2488
|
+
type: "run.failed",
|
|
2489
|
+
runId: run.id,
|
|
2490
|
+
workflow: run.workflow,
|
|
2491
|
+
error: outcome.error
|
|
2492
|
+
});
|
|
2493
|
+
void this.notifyParent(run.id, {
|
|
2494
|
+
ok: false,
|
|
2495
|
+
error: outcome.error.message
|
|
2496
|
+
});
|
|
2497
|
+
return {
|
|
2498
|
+
runId: run.id,
|
|
2499
|
+
status: "failed",
|
|
2500
|
+
error: outcome.error
|
|
2501
|
+
};
|
|
2502
|
+
}
|
|
2503
|
+
await this.store.updateRun(run.id, {
|
|
2504
|
+
status: "suspended",
|
|
2505
|
+
wakeAt: outcome.wakeAt,
|
|
2506
|
+
updatedAt
|
|
2507
|
+
});
|
|
2508
|
+
this.emit({
|
|
2509
|
+
type: "run.suspended",
|
|
2510
|
+
runId: run.id,
|
|
2511
|
+
workflow: run.workflow
|
|
2512
|
+
});
|
|
2513
|
+
return {
|
|
2514
|
+
runId: run.id,
|
|
2515
|
+
status: "suspended"
|
|
2516
|
+
};
|
|
2517
|
+
}
|
|
2381
2518
|
async runRemoteExecution(run, registered) {
|
|
2382
2519
|
const remote = registered.remote;
|
|
2383
2520
|
if (run.status === "pending") {
|
|
@@ -2392,8 +2529,8 @@ var WorkflowEngine = class {
|
|
|
2392
2529
|
workflow: run.workflow
|
|
2393
2530
|
});
|
|
2394
2531
|
}
|
|
2395
|
-
if (registered.singleton && !await this.
|
|
2396
|
-
const wakeAt2 =
|
|
2532
|
+
if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
2533
|
+
const wakeAt2 = this.singletons.retryWakeAt();
|
|
2397
2534
|
await this.store.updateRun(run.id, {
|
|
2398
2535
|
status: "suspended",
|
|
2399
2536
|
wakeAt: wakeAt2,
|
|
@@ -2438,53 +2575,19 @@ var WorkflowEngine = class {
|
|
|
2438
2575
|
}
|
|
2439
2576
|
if (decision.status === "completed") {
|
|
2440
2577
|
await this.applyCommands(run, decision.commands);
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
output: decision.output,
|
|
2444
|
-
error: void 0,
|
|
2445
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2446
|
-
});
|
|
2447
|
-
this.emit({
|
|
2448
|
-
type: "run.completed",
|
|
2449
|
-
runId: run.id,
|
|
2450
|
-
workflow: run.workflow,
|
|
2578
|
+
return this.settleRun(run, {
|
|
2579
|
+
kind: "completed",
|
|
2451
2580
|
output: decision.output
|
|
2452
2581
|
});
|
|
2453
|
-
void this.notifyParent(run.id, {
|
|
2454
|
-
ok: true,
|
|
2455
|
-
value: decision.output
|
|
2456
|
-
});
|
|
2457
|
-
return {
|
|
2458
|
-
runId: run.id,
|
|
2459
|
-
status: "completed",
|
|
2460
|
-
output: decision.output
|
|
2461
|
-
};
|
|
2462
2582
|
}
|
|
2463
2583
|
if (decision.status === "failed") {
|
|
2464
2584
|
await this.applyCommands(run, decision.commands);
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
error,
|
|
2471
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2472
|
-
});
|
|
2473
|
-
this.emit({
|
|
2474
|
-
type: "run.failed",
|
|
2475
|
-
runId: run.id,
|
|
2476
|
-
workflow: run.workflow,
|
|
2477
|
-
error
|
|
2478
|
-
});
|
|
2479
|
-
void this.notifyParent(run.id, {
|
|
2480
|
-
ok: false,
|
|
2481
|
-
error: error.message
|
|
2585
|
+
return this.settleRun(run, {
|
|
2586
|
+
kind: "failed",
|
|
2587
|
+
error: decision.error ?? {
|
|
2588
|
+
message: "workflow failed"
|
|
2589
|
+
}
|
|
2482
2590
|
});
|
|
2483
|
-
return {
|
|
2484
|
-
runId: run.id,
|
|
2485
|
-
status: "failed",
|
|
2486
|
-
error
|
|
2487
|
-
};
|
|
2488
2591
|
}
|
|
2489
2592
|
if (decision.status === "cancelled") {
|
|
2490
2593
|
await this.applyCommands(run, decision.commands);
|
|
@@ -2498,20 +2601,10 @@ var WorkflowEngine = class {
|
|
|
2498
2601
|
};
|
|
2499
2602
|
}
|
|
2500
2603
|
const wakeAt = await this.applyCommands(run, decision.commands);
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
wakeAt
|
|
2504
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2604
|
+
return this.settleRun(run, {
|
|
2605
|
+
kind: "suspended",
|
|
2606
|
+
wakeAt
|
|
2505
2607
|
});
|
|
2506
|
-
this.emit({
|
|
2507
|
-
type: "run.suspended",
|
|
2508
|
-
runId: run.id,
|
|
2509
|
-
workflow: run.workflow
|
|
2510
|
-
});
|
|
2511
|
-
return {
|
|
2512
|
-
runId: run.id,
|
|
2513
|
-
status: "suspended"
|
|
2514
|
-
};
|
|
2515
2608
|
}
|
|
2516
2609
|
/** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
|
|
2517
2610
|
async remoteHistory(runId) {
|
|
@@ -2566,21 +2659,20 @@ var WorkflowEngine = class {
|
|
|
2566
2659
|
if (cmd.kind === "recordStep") {
|
|
2567
2660
|
const startedAt = cmd.startedAt != null ? new Date(cmd.startedAt) : at;
|
|
2568
2661
|
const finishedAt = cmd.finishedAt != null ? new Date(cmd.finishedAt) : at;
|
|
2569
|
-
await this.store.saveCheckpoint({
|
|
2662
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2570
2663
|
runId: run.id,
|
|
2571
2664
|
seq: cmd.seq,
|
|
2572
2665
|
name: cmd.name,
|
|
2573
2666
|
kind: "local",
|
|
2574
|
-
stepId: id,
|
|
2575
2667
|
status: cmd.error ? "failed" : "completed",
|
|
2576
2668
|
output: cmd.output,
|
|
2577
2669
|
error: cmd.error,
|
|
2578
|
-
events: cmd.events
|
|
2670
|
+
events: cmd.events,
|
|
2579
2671
|
attempts: 1,
|
|
2580
2672
|
enqueuedAt: startedAt,
|
|
2581
2673
|
startedAt,
|
|
2582
2674
|
finishedAt
|
|
2583
|
-
});
|
|
2675
|
+
}));
|
|
2584
2676
|
this.emit({
|
|
2585
2677
|
type: cmd.error ? "step.failed" : "step.completed",
|
|
2586
2678
|
runId: run.id,
|
|
@@ -2591,12 +2683,11 @@ var WorkflowEngine = class {
|
|
|
2591
2683
|
error: cmd.error
|
|
2592
2684
|
});
|
|
2593
2685
|
} else if (cmd.kind === "call") {
|
|
2594
|
-
await this.store.saveCheckpoint({
|
|
2686
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2595
2687
|
runId: run.id,
|
|
2596
2688
|
seq: cmd.seq,
|
|
2597
2689
|
name: cmd.name,
|
|
2598
2690
|
kind: "remote",
|
|
2599
|
-
stepId: id,
|
|
2600
2691
|
status: "pending",
|
|
2601
2692
|
input: cmd.input,
|
|
2602
2693
|
attempts: 1,
|
|
@@ -2604,7 +2695,7 @@ var WorkflowEngine = class {
|
|
|
2604
2695
|
enqueuedAt: at,
|
|
2605
2696
|
startedAt: at,
|
|
2606
2697
|
finishedAt: at
|
|
2607
|
-
});
|
|
2698
|
+
}));
|
|
2608
2699
|
await this.pool.dispatch({
|
|
2609
2700
|
runId: run.id,
|
|
2610
2701
|
seq: cmd.seq,
|
|
@@ -2625,19 +2716,18 @@ var WorkflowEngine = class {
|
|
|
2625
2716
|
});
|
|
2626
2717
|
} else if (cmd.kind === "sleep") {
|
|
2627
2718
|
const deadline = this.clock() + cmd.ms;
|
|
2628
|
-
await this.store.saveCheckpoint({
|
|
2719
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2629
2720
|
runId: run.id,
|
|
2630
2721
|
seq: cmd.seq,
|
|
2631
2722
|
name: `sleep:${cmd.seq}`,
|
|
2632
2723
|
kind: "sleep",
|
|
2633
|
-
stepId: id,
|
|
2634
2724
|
status: "pending",
|
|
2635
2725
|
attempts: 1,
|
|
2636
2726
|
wakeAt: deadline,
|
|
2637
2727
|
enqueuedAt: at,
|
|
2638
2728
|
startedAt: at,
|
|
2639
2729
|
finishedAt: at
|
|
2640
|
-
});
|
|
2730
|
+
}));
|
|
2641
2731
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
2642
2732
|
} else if (cmd.kind === "waitSignal") {
|
|
2643
2733
|
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
@@ -2687,8 +2777,8 @@ var WorkflowEngine = class {
|
|
|
2687
2777
|
workflow: run.workflow
|
|
2688
2778
|
});
|
|
2689
2779
|
}
|
|
2690
|
-
if (registered?.singleton && !await this.
|
|
2691
|
-
const wakeAt =
|
|
2780
|
+
if (registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
2781
|
+
const wakeAt = this.singletons.retryWakeAt();
|
|
2692
2782
|
await this.store.updateRun(run.id, {
|
|
2693
2783
|
status: "suspended",
|
|
2694
2784
|
wakeAt,
|
|
@@ -2712,27 +2802,10 @@ var WorkflowEngine = class {
|
|
|
2712
2802
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay), run.id, compensations, run.workflow);
|
|
2713
2803
|
try {
|
|
2714
2804
|
const output = await fn(ctx, run.input);
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
output,
|
|
2718
|
-
error: void 0,
|
|
2719
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2720
|
-
});
|
|
2721
|
-
this.emit({
|
|
2722
|
-
type: "run.completed",
|
|
2723
|
-
runId: run.id,
|
|
2724
|
-
workflow: run.workflow,
|
|
2805
|
+
return this.settleRun(run, {
|
|
2806
|
+
kind: "completed",
|
|
2725
2807
|
output
|
|
2726
2808
|
});
|
|
2727
|
-
void this.notifyParent(run.id, {
|
|
2728
|
-
ok: true,
|
|
2729
|
-
value: output
|
|
2730
|
-
});
|
|
2731
|
-
return {
|
|
2732
|
-
runId: run.id,
|
|
2733
|
-
status: "completed",
|
|
2734
|
-
output
|
|
2735
|
-
};
|
|
2736
2809
|
} catch (err) {
|
|
2737
2810
|
if (err instanceof ContinueAsNew) {
|
|
2738
2811
|
await this.store.updateRun(run.id, {
|
|
@@ -2784,20 +2857,10 @@ var WorkflowEngine = class {
|
|
|
2784
2857
|
error: error2
|
|
2785
2858
|
};
|
|
2786
2859
|
}
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
wakeAt: err.wakeAt
|
|
2790
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2791
|
-
});
|
|
2792
|
-
this.emit({
|
|
2793
|
-
type: "run.suspended",
|
|
2794
|
-
runId: run.id,
|
|
2795
|
-
workflow: run.workflow
|
|
2860
|
+
return this.settleRun(run, {
|
|
2861
|
+
kind: "suspended",
|
|
2862
|
+
wakeAt: err.wakeAt
|
|
2796
2863
|
});
|
|
2797
|
-
return {
|
|
2798
|
-
runId: run.id,
|
|
2799
|
-
status: "suspended"
|
|
2800
|
-
};
|
|
2801
2864
|
}
|
|
2802
2865
|
const error = {
|
|
2803
2866
|
message: err instanceof Error ? err.message : String(err),
|
|
@@ -2808,26 +2871,10 @@ var WorkflowEngine = class {
|
|
|
2808
2871
|
if (!comp) continue;
|
|
2809
2872
|
await this.runCompensation(run, comp);
|
|
2810
2873
|
}
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
error,
|
|
2814
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2815
|
-
});
|
|
2816
|
-
this.emit({
|
|
2817
|
-
type: "run.failed",
|
|
2818
|
-
runId: run.id,
|
|
2819
|
-
workflow: run.workflow,
|
|
2874
|
+
return this.settleRun(run, {
|
|
2875
|
+
kind: "failed",
|
|
2820
2876
|
error
|
|
2821
2877
|
});
|
|
2822
|
-
void this.notifyParent(run.id, {
|
|
2823
|
-
ok: false,
|
|
2824
|
-
error: error.message
|
|
2825
|
-
});
|
|
2826
|
-
return {
|
|
2827
|
-
runId: run.id,
|
|
2828
|
-
status: "failed",
|
|
2829
|
-
error
|
|
2830
|
-
};
|
|
2831
2878
|
} finally {
|
|
2832
2879
|
await this.store.releaseRunLock(run.id);
|
|
2833
2880
|
}
|
|
@@ -3311,6 +3358,9 @@ var CodecStateStore = class {
|
|
|
3311
3358
|
async getRun(runId) {
|
|
3312
3359
|
return this.decRun(await this.inner.getRun(runId));
|
|
3313
3360
|
}
|
|
3361
|
+
deleteRun(runId) {
|
|
3362
|
+
return this.inner.deleteRun(runId);
|
|
3363
|
+
}
|
|
3314
3364
|
async getCheckpoint(runId, seq) {
|
|
3315
3365
|
return this.decCp(await this.inner.getCheckpoint(runId, seq));
|
|
3316
3366
|
}
|
|
@@ -3705,6 +3755,16 @@ var InMemoryStateStore = class {
|
|
|
3705
3755
|
...run
|
|
3706
3756
|
} : null;
|
|
3707
3757
|
}
|
|
3758
|
+
async deleteRun(runId) {
|
|
3759
|
+
this.runs.delete(runId);
|
|
3760
|
+
for (const [key, cp] of this.checkpoints) {
|
|
3761
|
+
if (cp.runId === runId) this.checkpoints.delete(key);
|
|
3762
|
+
}
|
|
3763
|
+
for (const [token, waiter] of this.signalWaiters) {
|
|
3764
|
+
if (waiter.runId === runId) this.signalWaiters.delete(token);
|
|
3765
|
+
}
|
|
3766
|
+
this.reindexAttributes(runId, void 0);
|
|
3767
|
+
}
|
|
3708
3768
|
async getCheckpoint(runId, seq) {
|
|
3709
3769
|
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
3710
3770
|
return cp ? {
|