@dudousxd/nestjs-durable-core 0.29.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +272 -212
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +31 -15
- package/dist/index.d.ts +31 -15
- package/dist/index.js +272 -212
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -211,6 +211,31 @@ async function runStepHandler(task, handler) {
|
|
|
211
211
|
__name(runStepHandler, "runStepHandler");
|
|
212
212
|
|
|
213
213
|
// src/checkpoints.ts
|
|
214
|
+
function nonEmptyEvents(events) {
|
|
215
|
+
return events && events.length > 0 ? events : void 0;
|
|
216
|
+
}
|
|
217
|
+
__name(nonEmptyEvents, "nonEmptyEvents");
|
|
218
|
+
function stepCheckpoint(p) {
|
|
219
|
+
return {
|
|
220
|
+
runId: p.runId,
|
|
221
|
+
seq: p.seq,
|
|
222
|
+
name: p.name,
|
|
223
|
+
kind: p.kind,
|
|
224
|
+
stepId: stepId(p.runId, p.seq),
|
|
225
|
+
status: p.status,
|
|
226
|
+
input: p.input,
|
|
227
|
+
output: p.output,
|
|
228
|
+
error: p.error,
|
|
229
|
+
events: nonEmptyEvents(p.events),
|
|
230
|
+
attempts: p.attempts,
|
|
231
|
+
workerGroup: p.workerGroup,
|
|
232
|
+
wakeAt: p.wakeAt,
|
|
233
|
+
enqueuedAt: p.enqueuedAt,
|
|
234
|
+
startedAt: p.startedAt,
|
|
235
|
+
finishedAt: p.finishedAt
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
__name(stepCheckpoint, "stepCheckpoint");
|
|
214
239
|
function instantCheckpoint(p) {
|
|
215
240
|
const at = /* @__PURE__ */ new Date();
|
|
216
241
|
return {
|
|
@@ -619,6 +644,93 @@ var QueueController = class {
|
|
|
619
644
|
}
|
|
620
645
|
};
|
|
621
646
|
|
|
647
|
+
// src/singleton-gate.ts
|
|
648
|
+
var SINGLETON_RETRY_MS = 1e3;
|
|
649
|
+
var SINGLETON_RETRY_JITTER_MS = 250;
|
|
650
|
+
var SingletonGate = class {
|
|
651
|
+
static {
|
|
652
|
+
__name(this, "SingletonGate");
|
|
653
|
+
}
|
|
654
|
+
deps;
|
|
655
|
+
constructor(deps) {
|
|
656
|
+
this.deps = deps;
|
|
657
|
+
}
|
|
658
|
+
/** The tag a singleton run carries, so the gate can find others sharing its key. */
|
|
659
|
+
tag(cfg, input) {
|
|
660
|
+
return `singleton:${cfg.key(input)}`;
|
|
661
|
+
}
|
|
662
|
+
/** Next wake time for a gated run: the base retry delay jittered to avoid a wakeup stampede. */
|
|
663
|
+
retryWakeAt() {
|
|
664
|
+
return this.deps.clock() + SINGLETON_RETRY_MS + Math.floor((Math.random() * 2 - 1) * SINGLETON_RETRY_JITTER_MS);
|
|
665
|
+
}
|
|
666
|
+
/**
|
|
667
|
+
* Reject a start that would grow the same-key backlog past `limit + maxQueueDepth` (counting
|
|
668
|
+
* `pending`/`running`/`suspended` runs sharing the key in one scan). No-op when no `maxQueueDepth`
|
|
669
|
+
* is configured.
|
|
670
|
+
*/
|
|
671
|
+
async assertCapacity(workflow, cfg, input) {
|
|
672
|
+
if (cfg.maxQueueDepth == null) return;
|
|
673
|
+
const cap = (cfg.limit ?? 1) + cfg.maxQueueDepth;
|
|
674
|
+
const queued = await this.deps.store.listRuns({
|
|
675
|
+
tag: this.tag(cfg, input),
|
|
676
|
+
workflow,
|
|
677
|
+
statuses: [
|
|
678
|
+
"pending",
|
|
679
|
+
"running",
|
|
680
|
+
"suspended"
|
|
681
|
+
]
|
|
682
|
+
});
|
|
683
|
+
if (queued.length >= cap) {
|
|
684
|
+
throw new SingletonQueueFullError(workflow, cfg.key(input), cfg.maxQueueDepth);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* Whether `run` may run now under its key: it's among the `limit` oldest in-flight (running or
|
|
689
|
+
* suspended) runs sharing the key, by `(createdAt, id)` order. A consistent store gives every
|
|
690
|
+
* instance the same ordering, so admission is race-free + FIFO.
|
|
691
|
+
*/
|
|
692
|
+
async admit(run, cfg) {
|
|
693
|
+
const inflight = (await this.deps.store.listRuns({
|
|
694
|
+
tag: this.tag(cfg, run.input),
|
|
695
|
+
workflow: run.workflow,
|
|
696
|
+
statuses: [
|
|
697
|
+
"running",
|
|
698
|
+
"suspended"
|
|
699
|
+
]
|
|
700
|
+
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
701
|
+
const idx = inflight.findIndex((r) => r.id === run.id);
|
|
702
|
+
return idx >= 0 && idx < (cfg.limit ?? 1);
|
|
703
|
+
}
|
|
704
|
+
/**
|
|
705
|
+
* Notify-on-release: a singleton run settled (freeing a slot), so dispatch the oldest gated
|
|
706
|
+
* (`suspended` + same tag) waiters now instead of waiting for their ~1s retry timer. Each re-checks
|
|
707
|
+
* admission in the executor and runs only if it actually wins a slot, so FIFO/race-free guarantees
|
|
708
|
+
* hold; the durable timer remains the cross-instance/crash fallback.
|
|
709
|
+
*/
|
|
710
|
+
async wakeNext(settled) {
|
|
711
|
+
const cfg = this.deps.configFor(settled);
|
|
712
|
+
if (!cfg) return;
|
|
713
|
+
const tag = settled.tags?.find((t) => t.startsWith("singleton:"));
|
|
714
|
+
if (!tag) return;
|
|
715
|
+
const gated = (await this.deps.store.listRuns({
|
|
716
|
+
tag,
|
|
717
|
+
workflow: settled.workflow,
|
|
718
|
+
statuses: [
|
|
719
|
+
"suspended"
|
|
720
|
+
]
|
|
721
|
+
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
722
|
+
for (const next of gated.slice(0, cfg.limit ?? 1)) {
|
|
723
|
+
if (next.wakeAt != null) {
|
|
724
|
+
await this.deps.store.updateRun(next.id, {
|
|
725
|
+
wakeAt: void 0,
|
|
726
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
727
|
+
}).catch(() => void 0);
|
|
728
|
+
}
|
|
729
|
+
this.deps.dispatch(next.id);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
};
|
|
733
|
+
|
|
622
734
|
// src/transport-pool.ts
|
|
623
735
|
var TransportPool = class {
|
|
624
736
|
static {
|
|
@@ -1149,10 +1261,6 @@ function createWorkflowCtx(host, runId, compensations, workflow = "") {
|
|
|
1149
1261
|
__name(createWorkflowCtx, "createWorkflowCtx");
|
|
1150
1262
|
|
|
1151
1263
|
// src/engine.ts
|
|
1152
|
-
var SINGLETON_RETRY_MS = 1e3;
|
|
1153
|
-
var SINGLETON_RETRY_JITTER_MS = 250;
|
|
1154
|
-
var singletonRetryWakeAt = /* @__PURE__ */ __name((nowMs) => nowMs + SINGLETON_RETRY_MS + Math.floor((Math.random() * 2 - 1) * SINGLETON_RETRY_JITTER_MS), "singletonRetryWakeAt");
|
|
1155
|
-
var singletonTag = /* @__PURE__ */ __name((cfg, input) => `singleton:${cfg.key(input)}`, "singletonTag");
|
|
1156
1264
|
function mergeTags(staticTags, runTags) {
|
|
1157
1265
|
if (!staticTags?.length && !runTags?.length) return void 0;
|
|
1158
1266
|
return [
|
|
@@ -1202,6 +1310,8 @@ var WorkflowEngine = class {
|
|
|
1202
1310
|
trackStepStart;
|
|
1203
1311
|
/** Where a freshly-started run executes — in-process by default (see {@link RunDispatcher}). */
|
|
1204
1312
|
runDispatcher;
|
|
1313
|
+
/** Per-key serialization for singleton workflows (admission, back-pressure, notify-on-release). */
|
|
1314
|
+
singletons;
|
|
1205
1315
|
/** Every registered workflow, keyed by `name@version` — so old versions stay runnable. */
|
|
1206
1316
|
workflows = /* @__PURE__ */ new Map();
|
|
1207
1317
|
/** The newest registered version per workflow name — used to `start` new runs. */
|
|
@@ -1259,6 +1369,14 @@ var WorkflowEngine = class {
|
|
|
1259
1369
|
dispatch: /* @__PURE__ */ __name((runId) => queueMicrotask(() => void this.runOne(runId).catch(() => {
|
|
1260
1370
|
})), "dispatch")
|
|
1261
1371
|
};
|
|
1372
|
+
this.singletons = new SingletonGate({
|
|
1373
|
+
store: this.store,
|
|
1374
|
+
clock: this.clock,
|
|
1375
|
+
dispatch: /* @__PURE__ */ __name((runId) => {
|
|
1376
|
+
this.runDispatcher.dispatch(runId);
|
|
1377
|
+
}, "dispatch"),
|
|
1378
|
+
configFor: /* @__PURE__ */ __name((run) => (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton, "configFor")
|
|
1379
|
+
});
|
|
1262
1380
|
this.pool.bind(
|
|
1263
1381
|
async (result) => {
|
|
1264
1382
|
const waiter = this.pending.get(result.stepId);
|
|
@@ -1509,25 +1627,10 @@ var WorkflowEngine = class {
|
|
|
1509
1627
|
const now = /* @__PURE__ */ new Date();
|
|
1510
1628
|
const tags = mergeTags(registered.tags, registered.singleton ? [
|
|
1511
1629
|
...opts?.tags ?? [],
|
|
1512
|
-
|
|
1630
|
+
this.singletons.tag(registered.singleton, input)
|
|
1513
1631
|
] : opts?.tags);
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
const cfg = registered.singleton;
|
|
1517
|
-
const key = cfg.key(input);
|
|
1518
|
-
const cap = (cfg.limit ?? 1) + maxQueueDepth;
|
|
1519
|
-
const queued = await this.store.listRuns({
|
|
1520
|
-
tag: singletonTag(cfg, input),
|
|
1521
|
-
workflow: name,
|
|
1522
|
-
statuses: [
|
|
1523
|
-
"pending",
|
|
1524
|
-
"running",
|
|
1525
|
-
"suspended"
|
|
1526
|
-
]
|
|
1527
|
-
});
|
|
1528
|
-
if (queued.length >= cap) {
|
|
1529
|
-
throw new SingletonQueueFullError(name, key, maxQueueDepth);
|
|
1530
|
-
}
|
|
1632
|
+
if (registered.singleton) {
|
|
1633
|
+
await this.singletons.assertCapacity(name, registered.singleton, input);
|
|
1531
1634
|
}
|
|
1532
1635
|
const run = {
|
|
1533
1636
|
id: runId,
|
|
@@ -1986,7 +2089,7 @@ var WorkflowEngine = class {
|
|
|
1986
2089
|
from: this.instanceId
|
|
1987
2090
|
}).catch(() => void 0);
|
|
1988
2091
|
}
|
|
1989
|
-
void this.
|
|
2092
|
+
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
1990
2093
|
return {
|
|
1991
2094
|
runId,
|
|
1992
2095
|
status: "cancelled",
|
|
@@ -2012,6 +2115,26 @@ var WorkflowEngine = class {
|
|
|
2012
2115
|
return results;
|
|
2013
2116
|
}
|
|
2014
2117
|
/**
|
|
2118
|
+
* Hard-delete a run and its entire subtree. Unlike {@link cancel} (which marks a run `cancelled`
|
|
2119
|
+
* but keeps it as history), delete REMOVES the run and all its rows (checkpoints, signal waiters,
|
|
2120
|
+
* search-attribute rows) — it vanishes from {@link getRun} and {@link listRuns}. Use it to purge a
|
|
2121
|
+
* finished run whose data is being deleted (e.g. a pipeline run whose result rows are cleared).
|
|
2122
|
+
*
|
|
2123
|
+
* Cascades depth-first via {@link getRunChildren} so children are gone before the parent (no
|
|
2124
|
+
* orphaned child runs). Prefer {@link cancel} first for a live run — deleting one mid-flight orphans
|
|
2125
|
+
* its worker (its next checkpoint write fails). Returns the number of runs deleted (0 if absent).
|
|
2126
|
+
*/
|
|
2127
|
+
async deleteRun(runId) {
|
|
2128
|
+
const run = await this.store.getRun(runId);
|
|
2129
|
+
if (!run) return 0;
|
|
2130
|
+
let deleted = 0;
|
|
2131
|
+
for (const childId of await this.getRunChildren(runId)) {
|
|
2132
|
+
deleted += await this.deleteRun(childId);
|
|
2133
|
+
}
|
|
2134
|
+
await this.store.deleteRun(runId);
|
|
2135
|
+
return deleted + 1;
|
|
2136
|
+
}
|
|
2137
|
+
/**
|
|
2015
2138
|
* Cascade cancellation to a run's children — both awaited (`ctx.child`, found via its live
|
|
2016
2139
|
* `child:<id>` waiter) and fire-and-forget (`ctx.startChild`, found via its `spawn:<id>`
|
|
2017
2140
|
* checkpoint). Recursive, so a whole subtree is cancelled; the terminal guard in `cancel` stops it
|
|
@@ -2093,21 +2216,19 @@ var WorkflowEngine = class {
|
|
|
2093
2216
|
*/
|
|
2094
2217
|
async persistStepEvent(event) {
|
|
2095
2218
|
const startedAt = new Date(event.startedAt);
|
|
2096
|
-
const events = event.events && event.events.length > 0 ? event.events : void 0;
|
|
2097
2219
|
if (event.phase === "running") {
|
|
2098
|
-
await this.store.saveCheckpoint({
|
|
2220
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2099
2221
|
runId: event.runId,
|
|
2100
2222
|
seq: event.seq,
|
|
2101
2223
|
name: event.name,
|
|
2102
2224
|
kind: "local",
|
|
2103
|
-
stepId: stepId(event.runId, event.seq),
|
|
2104
2225
|
status: "running",
|
|
2105
|
-
events,
|
|
2226
|
+
events: event.events,
|
|
2106
2227
|
attempts: 1,
|
|
2107
2228
|
enqueuedAt: startedAt,
|
|
2108
2229
|
startedAt,
|
|
2109
2230
|
finishedAt: startedAt
|
|
2110
|
-
});
|
|
2231
|
+
}));
|
|
2111
2232
|
this.emit({
|
|
2112
2233
|
type: "step.started",
|
|
2113
2234
|
runId: event.runId,
|
|
@@ -2118,21 +2239,20 @@ var WorkflowEngine = class {
|
|
|
2118
2239
|
return;
|
|
2119
2240
|
}
|
|
2120
2241
|
const failed = event.phase === "failed";
|
|
2121
|
-
await this.store.saveCheckpoint({
|
|
2242
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2122
2243
|
runId: event.runId,
|
|
2123
2244
|
seq: event.seq,
|
|
2124
2245
|
name: event.name,
|
|
2125
2246
|
kind: "local",
|
|
2126
|
-
stepId: stepId(event.runId, event.seq),
|
|
2127
2247
|
status: failed ? "failed" : "completed",
|
|
2128
2248
|
output: failed ? void 0 : event.output,
|
|
2129
2249
|
error: failed ? event.error : void 0,
|
|
2130
|
-
events,
|
|
2250
|
+
events: event.events,
|
|
2131
2251
|
attempts: 1,
|
|
2132
2252
|
enqueuedAt: startedAt,
|
|
2133
2253
|
startedAt,
|
|
2134
2254
|
finishedAt: event.finishedAt != null ? new Date(event.finishedAt) : /* @__PURE__ */ new Date()
|
|
2135
|
-
});
|
|
2255
|
+
}));
|
|
2136
2256
|
this.emit({
|
|
2137
2257
|
type: failed ? "step.failed" : "step.completed",
|
|
2138
2258
|
runId: event.runId,
|
|
@@ -2152,21 +2272,20 @@ var WorkflowEngine = class {
|
|
|
2152
2272
|
*/
|
|
2153
2273
|
async startStep(step) {
|
|
2154
2274
|
if (this.trackStepStart) {
|
|
2155
|
-
await this.store.saveCheckpoint({
|
|
2275
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2156
2276
|
runId: step.runId,
|
|
2157
2277
|
seq: step.seq,
|
|
2158
2278
|
name: step.name,
|
|
2159
2279
|
kind: step.kind,
|
|
2160
|
-
stepId: stepId(step.runId, step.seq),
|
|
2161
2280
|
status: "running",
|
|
2162
2281
|
input: step.input,
|
|
2163
|
-
events: step.events
|
|
2282
|
+
events: step.events,
|
|
2164
2283
|
attempts: step.attempts,
|
|
2165
2284
|
workerGroup: step.workerGroup,
|
|
2166
2285
|
enqueuedAt: step.enqueuedAt,
|
|
2167
2286
|
startedAt: step.startedAt,
|
|
2168
2287
|
finishedAt: step.startedAt
|
|
2169
|
-
});
|
|
2288
|
+
}));
|
|
2170
2289
|
}
|
|
2171
2290
|
this.emit({
|
|
2172
2291
|
type: "step.started",
|
|
@@ -2178,22 +2297,21 @@ var WorkflowEngine = class {
|
|
|
2178
2297
|
}
|
|
2179
2298
|
/** Checkpoint a finished step and announce it — the two things that must always happen together. */
|
|
2180
2299
|
async completeStep(step) {
|
|
2181
|
-
await this.store.saveCheckpoint({
|
|
2300
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2182
2301
|
runId: step.runId,
|
|
2183
2302
|
seq: step.seq,
|
|
2184
2303
|
name: step.name,
|
|
2185
2304
|
kind: step.kind,
|
|
2186
|
-
stepId: stepId(step.runId, step.seq),
|
|
2187
2305
|
status: "completed",
|
|
2188
2306
|
input: step.input,
|
|
2189
2307
|
output: step.output,
|
|
2190
|
-
events: step.events
|
|
2308
|
+
events: step.events,
|
|
2191
2309
|
attempts: step.attempts,
|
|
2192
2310
|
workerGroup: step.workerGroup,
|
|
2193
2311
|
enqueuedAt: step.enqueuedAt,
|
|
2194
2312
|
startedAt: step.startedAt,
|
|
2195
2313
|
finishedAt: /* @__PURE__ */ new Date()
|
|
2196
|
-
});
|
|
2314
|
+
}));
|
|
2197
2315
|
this.emit({
|
|
2198
2316
|
type: "step.completed",
|
|
2199
2317
|
runId: step.runId,
|
|
@@ -2207,22 +2325,21 @@ var WorkflowEngine = class {
|
|
|
2207
2325
|
}
|
|
2208
2326
|
/** Checkpoint a step that failed terminally, so the failure point is visible (not just the run). */
|
|
2209
2327
|
async failStep(step) {
|
|
2210
|
-
await this.store.saveCheckpoint({
|
|
2328
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2211
2329
|
runId: step.runId,
|
|
2212
2330
|
seq: step.seq,
|
|
2213
2331
|
name: step.name,
|
|
2214
2332
|
kind: step.kind,
|
|
2215
|
-
stepId: stepId(step.runId, step.seq),
|
|
2216
2333
|
status: "failed",
|
|
2217
2334
|
input: step.input,
|
|
2218
2335
|
error: step.error,
|
|
2219
|
-
events: step.events
|
|
2336
|
+
events: step.events,
|
|
2220
2337
|
attempts: step.attempts,
|
|
2221
2338
|
workerGroup: step.workerGroup,
|
|
2222
2339
|
enqueuedAt: step.enqueuedAt,
|
|
2223
2340
|
startedAt: step.startedAt,
|
|
2224
2341
|
finishedAt: /* @__PURE__ */ new Date()
|
|
2225
|
-
});
|
|
2342
|
+
}));
|
|
2226
2343
|
this.emit({
|
|
2227
2344
|
type: "step.failed",
|
|
2228
2345
|
runId: step.runId,
|
|
@@ -2234,55 +2351,6 @@ var WorkflowEngine = class {
|
|
|
2234
2351
|
durationMs: Date.now() - step.startedAt.getTime()
|
|
2235
2352
|
});
|
|
2236
2353
|
}
|
|
2237
|
-
/**
|
|
2238
|
-
* Whether `run` may run now under its singleton key: it's among the `limit` oldest in-flight runs
|
|
2239
|
-
* (running or suspended) sharing the key, by `(createdAt, id)` order. A consistent store gives every
|
|
2240
|
-
* instance the same ordering, so admission is race-free + FIFO.
|
|
2241
|
-
*/
|
|
2242
|
-
async admitSingleton(run, cfg) {
|
|
2243
|
-
const tag = singletonTag(cfg, run.input);
|
|
2244
|
-
const inflight = (await this.store.listRuns({
|
|
2245
|
-
tag,
|
|
2246
|
-
workflow: run.workflow,
|
|
2247
|
-
statuses: [
|
|
2248
|
-
"running",
|
|
2249
|
-
"suspended"
|
|
2250
|
-
]
|
|
2251
|
-
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2252
|
-
const idx = inflight.findIndex((r) => r.id === run.id);
|
|
2253
|
-
return idx >= 0 && idx < (cfg.limit ?? 1);
|
|
2254
|
-
}
|
|
2255
|
-
/**
|
|
2256
|
-
* Notify-on-release: a singleton run just settled (freeing a slot), so wake the gated waiters that
|
|
2257
|
-
* can now be admitted — immediately, instead of letting them sleep until their ~1s retry timer.
|
|
2258
|
-
* Dispatches up to `limit` of the oldest gated (`suspended` + same singleton tag) waiters by
|
|
2259
|
-
* `(createdAt, id)`; each re-checks admission in `runExecution` and runs only if it actually wins a
|
|
2260
|
-
* slot, so this never breaks the FIFO + race-free guarantees (the durable timer remains the
|
|
2261
|
-
* cross-instance / crash fallback — this just removes the poll latency on the happy path).
|
|
2262
|
-
*/
|
|
2263
|
-
async wakeNextSingletons(settled) {
|
|
2264
|
-
const registered = this.workflows.get(versionKey(settled.workflow, settled.workflowVersion)) ?? this.latest.get(settled.workflow);
|
|
2265
|
-
const cfg = registered?.singleton;
|
|
2266
|
-
if (!cfg) return;
|
|
2267
|
-
const tag = settled.tags?.find((t) => t.startsWith("singleton:"));
|
|
2268
|
-
if (!tag) return;
|
|
2269
|
-
const gated = (await this.store.listRuns({
|
|
2270
|
-
tag,
|
|
2271
|
-
workflow: settled.workflow,
|
|
2272
|
-
statuses: [
|
|
2273
|
-
"suspended"
|
|
2274
|
-
]
|
|
2275
|
-
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2276
|
-
for (const next of gated.slice(0, cfg.limit ?? 1)) {
|
|
2277
|
-
if (next.wakeAt != null) {
|
|
2278
|
-
await this.store.updateRun(next.id, {
|
|
2279
|
-
wakeAt: void 0,
|
|
2280
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2281
|
-
}).catch(() => void 0);
|
|
2282
|
-
}
|
|
2283
|
-
this.runDispatcher.dispatch(next.id);
|
|
2284
|
-
}
|
|
2285
|
-
}
|
|
2286
2354
|
async execute(run, fn) {
|
|
2287
2355
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
|
|
2288
2356
|
const lockNow = this.clock();
|
|
@@ -2301,7 +2369,7 @@ var WorkflowEngine = class {
|
|
|
2301
2369
|
try {
|
|
2302
2370
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
2303
2371
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
2304
|
-
void this.
|
|
2372
|
+
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
2305
2373
|
}
|
|
2306
2374
|
return result;
|
|
2307
2375
|
} finally {
|
|
@@ -2314,6 +2382,75 @@ var WorkflowEngine = class {
|
|
|
2314
2382
|
* {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
|
|
2315
2383
|
* us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
|
|
2316
2384
|
*/
|
|
2385
|
+
/**
|
|
2386
|
+
* Apply a terminal/suspended transition once: persist the new run status, emit the matching
|
|
2387
|
+
* lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
|
|
2388
|
+
* executors funnel their completed/failed/suspended outcomes through here so a status update can
|
|
2389
|
+
* never drift from its event or `notifyParent` call. Does NOT touch the run lock — each executor
|
|
2390
|
+
* keeps its own lease handling (the TS executor's `finally`, the remote path's caller).
|
|
2391
|
+
*/
|
|
2392
|
+
async settleRun(run, outcome) {
|
|
2393
|
+
const updatedAt = /* @__PURE__ */ new Date();
|
|
2394
|
+
if (outcome.kind === "completed") {
|
|
2395
|
+
await this.store.updateRun(run.id, {
|
|
2396
|
+
status: "completed",
|
|
2397
|
+
output: outcome.output,
|
|
2398
|
+
error: void 0,
|
|
2399
|
+
updatedAt
|
|
2400
|
+
});
|
|
2401
|
+
this.emit({
|
|
2402
|
+
type: "run.completed",
|
|
2403
|
+
runId: run.id,
|
|
2404
|
+
workflow: run.workflow,
|
|
2405
|
+
output: outcome.output
|
|
2406
|
+
});
|
|
2407
|
+
void this.notifyParent(run.id, {
|
|
2408
|
+
ok: true,
|
|
2409
|
+
value: outcome.output
|
|
2410
|
+
});
|
|
2411
|
+
return {
|
|
2412
|
+
runId: run.id,
|
|
2413
|
+
status: "completed",
|
|
2414
|
+
output: outcome.output
|
|
2415
|
+
};
|
|
2416
|
+
}
|
|
2417
|
+
if (outcome.kind === "failed") {
|
|
2418
|
+
await this.store.updateRun(run.id, {
|
|
2419
|
+
status: "failed",
|
|
2420
|
+
error: outcome.error,
|
|
2421
|
+
updatedAt
|
|
2422
|
+
});
|
|
2423
|
+
this.emit({
|
|
2424
|
+
type: "run.failed",
|
|
2425
|
+
runId: run.id,
|
|
2426
|
+
workflow: run.workflow,
|
|
2427
|
+
error: outcome.error
|
|
2428
|
+
});
|
|
2429
|
+
void this.notifyParent(run.id, {
|
|
2430
|
+
ok: false,
|
|
2431
|
+
error: outcome.error.message
|
|
2432
|
+
});
|
|
2433
|
+
return {
|
|
2434
|
+
runId: run.id,
|
|
2435
|
+
status: "failed",
|
|
2436
|
+
error: outcome.error
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2439
|
+
await this.store.updateRun(run.id, {
|
|
2440
|
+
status: "suspended",
|
|
2441
|
+
wakeAt: outcome.wakeAt,
|
|
2442
|
+
updatedAt
|
|
2443
|
+
});
|
|
2444
|
+
this.emit({
|
|
2445
|
+
type: "run.suspended",
|
|
2446
|
+
runId: run.id,
|
|
2447
|
+
workflow: run.workflow
|
|
2448
|
+
});
|
|
2449
|
+
return {
|
|
2450
|
+
runId: run.id,
|
|
2451
|
+
status: "suspended"
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2317
2454
|
async runRemoteExecution(run, registered) {
|
|
2318
2455
|
const remote = registered.remote;
|
|
2319
2456
|
if (run.status === "pending") {
|
|
@@ -2328,8 +2465,8 @@ var WorkflowEngine = class {
|
|
|
2328
2465
|
workflow: run.workflow
|
|
2329
2466
|
});
|
|
2330
2467
|
}
|
|
2331
|
-
if (registered.singleton && !await this.
|
|
2332
|
-
const wakeAt2 =
|
|
2468
|
+
if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
2469
|
+
const wakeAt2 = this.singletons.retryWakeAt();
|
|
2333
2470
|
await this.store.updateRun(run.id, {
|
|
2334
2471
|
status: "suspended",
|
|
2335
2472
|
wakeAt: wakeAt2,
|
|
@@ -2374,53 +2511,19 @@ var WorkflowEngine = class {
|
|
|
2374
2511
|
}
|
|
2375
2512
|
if (decision.status === "completed") {
|
|
2376
2513
|
await this.applyCommands(run, decision.commands);
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
output: decision.output,
|
|
2380
|
-
error: void 0,
|
|
2381
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2382
|
-
});
|
|
2383
|
-
this.emit({
|
|
2384
|
-
type: "run.completed",
|
|
2385
|
-
runId: run.id,
|
|
2386
|
-
workflow: run.workflow,
|
|
2514
|
+
return this.settleRun(run, {
|
|
2515
|
+
kind: "completed",
|
|
2387
2516
|
output: decision.output
|
|
2388
2517
|
});
|
|
2389
|
-
void this.notifyParent(run.id, {
|
|
2390
|
-
ok: true,
|
|
2391
|
-
value: decision.output
|
|
2392
|
-
});
|
|
2393
|
-
return {
|
|
2394
|
-
runId: run.id,
|
|
2395
|
-
status: "completed",
|
|
2396
|
-
output: decision.output
|
|
2397
|
-
};
|
|
2398
2518
|
}
|
|
2399
2519
|
if (decision.status === "failed") {
|
|
2400
2520
|
await this.applyCommands(run, decision.commands);
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
error,
|
|
2407
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2408
|
-
});
|
|
2409
|
-
this.emit({
|
|
2410
|
-
type: "run.failed",
|
|
2411
|
-
runId: run.id,
|
|
2412
|
-
workflow: run.workflow,
|
|
2413
|
-
error
|
|
2414
|
-
});
|
|
2415
|
-
void this.notifyParent(run.id, {
|
|
2416
|
-
ok: false,
|
|
2417
|
-
error: error.message
|
|
2521
|
+
return this.settleRun(run, {
|
|
2522
|
+
kind: "failed",
|
|
2523
|
+
error: decision.error ?? {
|
|
2524
|
+
message: "workflow failed"
|
|
2525
|
+
}
|
|
2418
2526
|
});
|
|
2419
|
-
return {
|
|
2420
|
-
runId: run.id,
|
|
2421
|
-
status: "failed",
|
|
2422
|
-
error
|
|
2423
|
-
};
|
|
2424
2527
|
}
|
|
2425
2528
|
if (decision.status === "cancelled") {
|
|
2426
2529
|
await this.applyCommands(run, decision.commands);
|
|
@@ -2434,20 +2537,10 @@ var WorkflowEngine = class {
|
|
|
2434
2537
|
};
|
|
2435
2538
|
}
|
|
2436
2539
|
const wakeAt = await this.applyCommands(run, decision.commands);
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
wakeAt
|
|
2440
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2540
|
+
return this.settleRun(run, {
|
|
2541
|
+
kind: "suspended",
|
|
2542
|
+
wakeAt
|
|
2441
2543
|
});
|
|
2442
|
-
this.emit({
|
|
2443
|
-
type: "run.suspended",
|
|
2444
|
-
runId: run.id,
|
|
2445
|
-
workflow: run.workflow
|
|
2446
|
-
});
|
|
2447
|
-
return {
|
|
2448
|
-
runId: run.id,
|
|
2449
|
-
status: "suspended"
|
|
2450
|
-
};
|
|
2451
2544
|
}
|
|
2452
2545
|
/** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
|
|
2453
2546
|
async remoteHistory(runId) {
|
|
@@ -2502,21 +2595,20 @@ var WorkflowEngine = class {
|
|
|
2502
2595
|
if (cmd.kind === "recordStep") {
|
|
2503
2596
|
const startedAt = cmd.startedAt != null ? new Date(cmd.startedAt) : at;
|
|
2504
2597
|
const finishedAt = cmd.finishedAt != null ? new Date(cmd.finishedAt) : at;
|
|
2505
|
-
await this.store.saveCheckpoint({
|
|
2598
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2506
2599
|
runId: run.id,
|
|
2507
2600
|
seq: cmd.seq,
|
|
2508
2601
|
name: cmd.name,
|
|
2509
2602
|
kind: "local",
|
|
2510
|
-
stepId: id,
|
|
2511
2603
|
status: cmd.error ? "failed" : "completed",
|
|
2512
2604
|
output: cmd.output,
|
|
2513
2605
|
error: cmd.error,
|
|
2514
|
-
events: cmd.events
|
|
2606
|
+
events: cmd.events,
|
|
2515
2607
|
attempts: 1,
|
|
2516
2608
|
enqueuedAt: startedAt,
|
|
2517
2609
|
startedAt,
|
|
2518
2610
|
finishedAt
|
|
2519
|
-
});
|
|
2611
|
+
}));
|
|
2520
2612
|
this.emit({
|
|
2521
2613
|
type: cmd.error ? "step.failed" : "step.completed",
|
|
2522
2614
|
runId: run.id,
|
|
@@ -2527,12 +2619,11 @@ var WorkflowEngine = class {
|
|
|
2527
2619
|
error: cmd.error
|
|
2528
2620
|
});
|
|
2529
2621
|
} else if (cmd.kind === "call") {
|
|
2530
|
-
await this.store.saveCheckpoint({
|
|
2622
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2531
2623
|
runId: run.id,
|
|
2532
2624
|
seq: cmd.seq,
|
|
2533
2625
|
name: cmd.name,
|
|
2534
2626
|
kind: "remote",
|
|
2535
|
-
stepId: id,
|
|
2536
2627
|
status: "pending",
|
|
2537
2628
|
input: cmd.input,
|
|
2538
2629
|
attempts: 1,
|
|
@@ -2540,7 +2631,7 @@ var WorkflowEngine = class {
|
|
|
2540
2631
|
enqueuedAt: at,
|
|
2541
2632
|
startedAt: at,
|
|
2542
2633
|
finishedAt: at
|
|
2543
|
-
});
|
|
2634
|
+
}));
|
|
2544
2635
|
await this.pool.dispatch({
|
|
2545
2636
|
runId: run.id,
|
|
2546
2637
|
seq: cmd.seq,
|
|
@@ -2561,19 +2652,18 @@ var WorkflowEngine = class {
|
|
|
2561
2652
|
});
|
|
2562
2653
|
} else if (cmd.kind === "sleep") {
|
|
2563
2654
|
const deadline = this.clock() + cmd.ms;
|
|
2564
|
-
await this.store.saveCheckpoint({
|
|
2655
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2565
2656
|
runId: run.id,
|
|
2566
2657
|
seq: cmd.seq,
|
|
2567
2658
|
name: `sleep:${cmd.seq}`,
|
|
2568
2659
|
kind: "sleep",
|
|
2569
|
-
stepId: id,
|
|
2570
2660
|
status: "pending",
|
|
2571
2661
|
attempts: 1,
|
|
2572
2662
|
wakeAt: deadline,
|
|
2573
2663
|
enqueuedAt: at,
|
|
2574
2664
|
startedAt: at,
|
|
2575
2665
|
finishedAt: at
|
|
2576
|
-
});
|
|
2666
|
+
}));
|
|
2577
2667
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
2578
2668
|
} else if (cmd.kind === "waitSignal") {
|
|
2579
2669
|
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
@@ -2623,8 +2713,8 @@ var WorkflowEngine = class {
|
|
|
2623
2713
|
workflow: run.workflow
|
|
2624
2714
|
});
|
|
2625
2715
|
}
|
|
2626
|
-
if (registered?.singleton && !await this.
|
|
2627
|
-
const wakeAt =
|
|
2716
|
+
if (registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
2717
|
+
const wakeAt = this.singletons.retryWakeAt();
|
|
2628
2718
|
await this.store.updateRun(run.id, {
|
|
2629
2719
|
status: "suspended",
|
|
2630
2720
|
wakeAt,
|
|
@@ -2648,27 +2738,10 @@ var WorkflowEngine = class {
|
|
|
2648
2738
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay), run.id, compensations, run.workflow);
|
|
2649
2739
|
try {
|
|
2650
2740
|
const output = await fn(ctx, run.input);
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
output,
|
|
2654
|
-
error: void 0,
|
|
2655
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2656
|
-
});
|
|
2657
|
-
this.emit({
|
|
2658
|
-
type: "run.completed",
|
|
2659
|
-
runId: run.id,
|
|
2660
|
-
workflow: run.workflow,
|
|
2741
|
+
return this.settleRun(run, {
|
|
2742
|
+
kind: "completed",
|
|
2661
2743
|
output
|
|
2662
2744
|
});
|
|
2663
|
-
void this.notifyParent(run.id, {
|
|
2664
|
-
ok: true,
|
|
2665
|
-
value: output
|
|
2666
|
-
});
|
|
2667
|
-
return {
|
|
2668
|
-
runId: run.id,
|
|
2669
|
-
status: "completed",
|
|
2670
|
-
output
|
|
2671
|
-
};
|
|
2672
2745
|
} catch (err) {
|
|
2673
2746
|
if (err instanceof ContinueAsNew) {
|
|
2674
2747
|
await this.store.updateRun(run.id, {
|
|
@@ -2720,20 +2793,10 @@ var WorkflowEngine = class {
|
|
|
2720
2793
|
error: error2
|
|
2721
2794
|
};
|
|
2722
2795
|
}
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
wakeAt: err.wakeAt
|
|
2726
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2727
|
-
});
|
|
2728
|
-
this.emit({
|
|
2729
|
-
type: "run.suspended",
|
|
2730
|
-
runId: run.id,
|
|
2731
|
-
workflow: run.workflow
|
|
2796
|
+
return this.settleRun(run, {
|
|
2797
|
+
kind: "suspended",
|
|
2798
|
+
wakeAt: err.wakeAt
|
|
2732
2799
|
});
|
|
2733
|
-
return {
|
|
2734
|
-
runId: run.id,
|
|
2735
|
-
status: "suspended"
|
|
2736
|
-
};
|
|
2737
2800
|
}
|
|
2738
2801
|
const error = {
|
|
2739
2802
|
message: err instanceof Error ? err.message : String(err),
|
|
@@ -2744,26 +2807,10 @@ var WorkflowEngine = class {
|
|
|
2744
2807
|
if (!comp) continue;
|
|
2745
2808
|
await this.runCompensation(run, comp);
|
|
2746
2809
|
}
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
error,
|
|
2750
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2751
|
-
});
|
|
2752
|
-
this.emit({
|
|
2753
|
-
type: "run.failed",
|
|
2754
|
-
runId: run.id,
|
|
2755
|
-
workflow: run.workflow,
|
|
2810
|
+
return this.settleRun(run, {
|
|
2811
|
+
kind: "failed",
|
|
2756
2812
|
error
|
|
2757
2813
|
});
|
|
2758
|
-
void this.notifyParent(run.id, {
|
|
2759
|
-
ok: false,
|
|
2760
|
-
error: error.message
|
|
2761
|
-
});
|
|
2762
|
-
return {
|
|
2763
|
-
runId: run.id,
|
|
2764
|
-
status: "failed",
|
|
2765
|
-
error
|
|
2766
|
-
};
|
|
2767
2814
|
} finally {
|
|
2768
2815
|
await this.store.releaseRunLock(run.id);
|
|
2769
2816
|
}
|
|
@@ -3247,6 +3294,9 @@ var CodecStateStore = class {
|
|
|
3247
3294
|
async getRun(runId) {
|
|
3248
3295
|
return this.decRun(await this.inner.getRun(runId));
|
|
3249
3296
|
}
|
|
3297
|
+
deleteRun(runId) {
|
|
3298
|
+
return this.inner.deleteRun(runId);
|
|
3299
|
+
}
|
|
3250
3300
|
async getCheckpoint(runId, seq) {
|
|
3251
3301
|
return this.decCp(await this.inner.getCheckpoint(runId, seq));
|
|
3252
3302
|
}
|
|
@@ -3641,6 +3691,16 @@ var InMemoryStateStore = class {
|
|
|
3641
3691
|
...run
|
|
3642
3692
|
} : null;
|
|
3643
3693
|
}
|
|
3694
|
+
async deleteRun(runId) {
|
|
3695
|
+
this.runs.delete(runId);
|
|
3696
|
+
for (const [key, cp] of this.checkpoints) {
|
|
3697
|
+
if (cp.runId === runId) this.checkpoints.delete(key);
|
|
3698
|
+
}
|
|
3699
|
+
for (const [token, waiter] of this.signalWaiters) {
|
|
3700
|
+
if (waiter.runId === runId) this.signalWaiters.delete(token);
|
|
3701
|
+
}
|
|
3702
|
+
this.reindexAttributes(runId, void 0);
|
|
3703
|
+
}
|
|
3644
3704
|
async getCheckpoint(runId, seq) {
|
|
3645
3705
|
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
3646
3706
|
return cp ? {
|