@dudousxd/nestjs-durable-core 0.28.2 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +245 -207
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +13 -17
- package/dist/index.d.ts +13 -17
- package/dist/index.js +245 -207
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -211,6 +211,31 @@ async function runStepHandler(task, handler) {
|
|
|
211
211
|
__name(runStepHandler, "runStepHandler");
|
|
212
212
|
|
|
213
213
|
// src/checkpoints.ts
|
|
214
|
+
function nonEmptyEvents(events) {
|
|
215
|
+
return events && events.length > 0 ? events : void 0;
|
|
216
|
+
}
|
|
217
|
+
__name(nonEmptyEvents, "nonEmptyEvents");
|
|
218
|
+
function stepCheckpoint(p) {
|
|
219
|
+
return {
|
|
220
|
+
runId: p.runId,
|
|
221
|
+
seq: p.seq,
|
|
222
|
+
name: p.name,
|
|
223
|
+
kind: p.kind,
|
|
224
|
+
stepId: stepId(p.runId, p.seq),
|
|
225
|
+
status: p.status,
|
|
226
|
+
input: p.input,
|
|
227
|
+
output: p.output,
|
|
228
|
+
error: p.error,
|
|
229
|
+
events: nonEmptyEvents(p.events),
|
|
230
|
+
attempts: p.attempts,
|
|
231
|
+
workerGroup: p.workerGroup,
|
|
232
|
+
wakeAt: p.wakeAt,
|
|
233
|
+
enqueuedAt: p.enqueuedAt,
|
|
234
|
+
startedAt: p.startedAt,
|
|
235
|
+
finishedAt: p.finishedAt
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
__name(stepCheckpoint, "stepCheckpoint");
|
|
214
239
|
function instantCheckpoint(p) {
|
|
215
240
|
const at = /* @__PURE__ */ new Date();
|
|
216
241
|
return {
|
|
@@ -619,6 +644,93 @@ var QueueController = class {
|
|
|
619
644
|
}
|
|
620
645
|
};
|
|
621
646
|
|
|
647
|
+
// src/singleton-gate.ts
|
|
648
|
+
var SINGLETON_RETRY_MS = 1e3;
|
|
649
|
+
var SINGLETON_RETRY_JITTER_MS = 250;
|
|
650
|
+
var SingletonGate = class {
|
|
651
|
+
static {
|
|
652
|
+
__name(this, "SingletonGate");
|
|
653
|
+
}
|
|
654
|
+
deps;
|
|
655
|
+
constructor(deps) {
|
|
656
|
+
this.deps = deps;
|
|
657
|
+
}
|
|
658
|
+
/** The tag a singleton run carries, so the gate can find others sharing its key. */
|
|
659
|
+
tag(cfg, input) {
|
|
660
|
+
return `singleton:${cfg.key(input)}`;
|
|
661
|
+
}
|
|
662
|
+
/** Next wake time for a gated run: the base retry delay jittered to avoid a wakeup stampede. */
|
|
663
|
+
retryWakeAt() {
|
|
664
|
+
return this.deps.clock() + SINGLETON_RETRY_MS + Math.floor((Math.random() * 2 - 1) * SINGLETON_RETRY_JITTER_MS);
|
|
665
|
+
}
|
|
666
|
+
/**
|
|
667
|
+
* Reject a start that would grow the same-key backlog past `limit + maxQueueDepth` (counting
|
|
668
|
+
* `pending`/`running`/`suspended` runs sharing the key in one scan). No-op when no `maxQueueDepth`
|
|
669
|
+
* is configured.
|
|
670
|
+
*/
|
|
671
|
+
async assertCapacity(workflow, cfg, input) {
|
|
672
|
+
if (cfg.maxQueueDepth == null) return;
|
|
673
|
+
const cap = (cfg.limit ?? 1) + cfg.maxQueueDepth;
|
|
674
|
+
const queued = await this.deps.store.listRuns({
|
|
675
|
+
tag: this.tag(cfg, input),
|
|
676
|
+
workflow,
|
|
677
|
+
statuses: [
|
|
678
|
+
"pending",
|
|
679
|
+
"running",
|
|
680
|
+
"suspended"
|
|
681
|
+
]
|
|
682
|
+
});
|
|
683
|
+
if (queued.length >= cap) {
|
|
684
|
+
throw new SingletonQueueFullError(workflow, cfg.key(input), cfg.maxQueueDepth);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* Whether `run` may run now under its key: it's among the `limit` oldest in-flight (running or
|
|
689
|
+
* suspended) runs sharing the key, by `(createdAt, id)` order. A consistent store gives every
|
|
690
|
+
* instance the same ordering, so admission is race-free + FIFO.
|
|
691
|
+
*/
|
|
692
|
+
async admit(run, cfg) {
|
|
693
|
+
const inflight = (await this.deps.store.listRuns({
|
|
694
|
+
tag: this.tag(cfg, run.input),
|
|
695
|
+
workflow: run.workflow,
|
|
696
|
+
statuses: [
|
|
697
|
+
"running",
|
|
698
|
+
"suspended"
|
|
699
|
+
]
|
|
700
|
+
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
701
|
+
const idx = inflight.findIndex((r) => r.id === run.id);
|
|
702
|
+
return idx >= 0 && idx < (cfg.limit ?? 1);
|
|
703
|
+
}
|
|
704
|
+
/**
|
|
705
|
+
* Notify-on-release: a singleton run settled (freeing a slot), so dispatch the oldest gated
|
|
706
|
+
* (`suspended` + same tag) waiters now instead of waiting for their ~1s retry timer. Each re-checks
|
|
707
|
+
* admission in the executor and runs only if it actually wins a slot, so FIFO/race-free guarantees
|
|
708
|
+
* hold; the durable timer remains the cross-instance/crash fallback.
|
|
709
|
+
*/
|
|
710
|
+
async wakeNext(settled) {
|
|
711
|
+
const cfg = this.deps.configFor(settled);
|
|
712
|
+
if (!cfg) return;
|
|
713
|
+
const tag = settled.tags?.find((t) => t.startsWith("singleton:"));
|
|
714
|
+
if (!tag) return;
|
|
715
|
+
const gated = (await this.deps.store.listRuns({
|
|
716
|
+
tag,
|
|
717
|
+
workflow: settled.workflow,
|
|
718
|
+
statuses: [
|
|
719
|
+
"suspended"
|
|
720
|
+
]
|
|
721
|
+
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
722
|
+
for (const next of gated.slice(0, cfg.limit ?? 1)) {
|
|
723
|
+
if (next.wakeAt != null) {
|
|
724
|
+
await this.deps.store.updateRun(next.id, {
|
|
725
|
+
wakeAt: void 0,
|
|
726
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
727
|
+
}).catch(() => void 0);
|
|
728
|
+
}
|
|
729
|
+
this.deps.dispatch(next.id);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
};
|
|
733
|
+
|
|
622
734
|
// src/transport-pool.ts
|
|
623
735
|
var TransportPool = class {
|
|
624
736
|
static {
|
|
@@ -1149,10 +1261,6 @@ function createWorkflowCtx(host, runId, compensations, workflow = "") {
|
|
|
1149
1261
|
__name(createWorkflowCtx, "createWorkflowCtx");
|
|
1150
1262
|
|
|
1151
1263
|
// src/engine.ts
|
|
1152
|
-
var SINGLETON_RETRY_MS = 1e3;
|
|
1153
|
-
var SINGLETON_RETRY_JITTER_MS = 250;
|
|
1154
|
-
var singletonRetryWakeAt = /* @__PURE__ */ __name((nowMs) => nowMs + SINGLETON_RETRY_MS + Math.floor((Math.random() * 2 - 1) * SINGLETON_RETRY_JITTER_MS), "singletonRetryWakeAt");
|
|
1155
|
-
var singletonTag = /* @__PURE__ */ __name((cfg, input) => `singleton:${cfg.key(input)}`, "singletonTag");
|
|
1156
1264
|
function mergeTags(staticTags, runTags) {
|
|
1157
1265
|
if (!staticTags?.length && !runTags?.length) return void 0;
|
|
1158
1266
|
return [
|
|
@@ -1202,6 +1310,8 @@ var WorkflowEngine = class {
|
|
|
1202
1310
|
trackStepStart;
|
|
1203
1311
|
/** Where a freshly-started run executes — in-process by default (see {@link RunDispatcher}). */
|
|
1204
1312
|
runDispatcher;
|
|
1313
|
+
/** Per-key serialization for singleton workflows (admission, back-pressure, notify-on-release). */
|
|
1314
|
+
singletons;
|
|
1205
1315
|
/** Every registered workflow, keyed by `name@version` — so old versions stay runnable. */
|
|
1206
1316
|
workflows = /* @__PURE__ */ new Map();
|
|
1207
1317
|
/** The newest registered version per workflow name — used to `start` new runs. */
|
|
@@ -1259,6 +1369,14 @@ var WorkflowEngine = class {
|
|
|
1259
1369
|
dispatch: /* @__PURE__ */ __name((runId) => queueMicrotask(() => void this.runOne(runId).catch(() => {
|
|
1260
1370
|
})), "dispatch")
|
|
1261
1371
|
};
|
|
1372
|
+
this.singletons = new SingletonGate({
|
|
1373
|
+
store: this.store,
|
|
1374
|
+
clock: this.clock,
|
|
1375
|
+
dispatch: /* @__PURE__ */ __name((runId) => {
|
|
1376
|
+
this.runDispatcher.dispatch(runId);
|
|
1377
|
+
}, "dispatch"),
|
|
1378
|
+
configFor: /* @__PURE__ */ __name((run) => (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton, "configFor")
|
|
1379
|
+
});
|
|
1262
1380
|
this.pool.bind(
|
|
1263
1381
|
async (result) => {
|
|
1264
1382
|
const waiter = this.pending.get(result.stepId);
|
|
@@ -1509,25 +1627,10 @@ var WorkflowEngine = class {
|
|
|
1509
1627
|
const now = /* @__PURE__ */ new Date();
|
|
1510
1628
|
const tags = mergeTags(registered.tags, registered.singleton ? [
|
|
1511
1629
|
...opts?.tags ?? [],
|
|
1512
|
-
|
|
1630
|
+
this.singletons.tag(registered.singleton, input)
|
|
1513
1631
|
] : opts?.tags);
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
const cfg = registered.singleton;
|
|
1517
|
-
const key = cfg.key(input);
|
|
1518
|
-
const cap = (cfg.limit ?? 1) + maxQueueDepth;
|
|
1519
|
-
const queued = await this.store.listRuns({
|
|
1520
|
-
tag: singletonTag(cfg, input),
|
|
1521
|
-
workflow: name,
|
|
1522
|
-
statuses: [
|
|
1523
|
-
"pending",
|
|
1524
|
-
"running",
|
|
1525
|
-
"suspended"
|
|
1526
|
-
]
|
|
1527
|
-
});
|
|
1528
|
-
if (queued.length >= cap) {
|
|
1529
|
-
throw new SingletonQueueFullError(name, key, maxQueueDepth);
|
|
1530
|
-
}
|
|
1632
|
+
if (registered.singleton) {
|
|
1633
|
+
await this.singletons.assertCapacity(name, registered.singleton, input);
|
|
1531
1634
|
}
|
|
1532
1635
|
const run = {
|
|
1533
1636
|
id: runId,
|
|
@@ -1986,7 +2089,7 @@ var WorkflowEngine = class {
|
|
|
1986
2089
|
from: this.instanceId
|
|
1987
2090
|
}).catch(() => void 0);
|
|
1988
2091
|
}
|
|
1989
|
-
void this.
|
|
2092
|
+
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
1990
2093
|
return {
|
|
1991
2094
|
runId,
|
|
1992
2095
|
status: "cancelled",
|
|
@@ -2093,21 +2196,19 @@ var WorkflowEngine = class {
|
|
|
2093
2196
|
*/
|
|
2094
2197
|
async persistStepEvent(event) {
|
|
2095
2198
|
const startedAt = new Date(event.startedAt);
|
|
2096
|
-
const events = event.events && event.events.length > 0 ? event.events : void 0;
|
|
2097
2199
|
if (event.phase === "running") {
|
|
2098
|
-
await this.store.saveCheckpoint({
|
|
2200
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2099
2201
|
runId: event.runId,
|
|
2100
2202
|
seq: event.seq,
|
|
2101
2203
|
name: event.name,
|
|
2102
2204
|
kind: "local",
|
|
2103
|
-
stepId: stepId(event.runId, event.seq),
|
|
2104
2205
|
status: "running",
|
|
2105
|
-
events,
|
|
2206
|
+
events: event.events,
|
|
2106
2207
|
attempts: 1,
|
|
2107
2208
|
enqueuedAt: startedAt,
|
|
2108
2209
|
startedAt,
|
|
2109
2210
|
finishedAt: startedAt
|
|
2110
|
-
});
|
|
2211
|
+
}));
|
|
2111
2212
|
this.emit({
|
|
2112
2213
|
type: "step.started",
|
|
2113
2214
|
runId: event.runId,
|
|
@@ -2118,21 +2219,20 @@ var WorkflowEngine = class {
|
|
|
2118
2219
|
return;
|
|
2119
2220
|
}
|
|
2120
2221
|
const failed = event.phase === "failed";
|
|
2121
|
-
await this.store.saveCheckpoint({
|
|
2222
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2122
2223
|
runId: event.runId,
|
|
2123
2224
|
seq: event.seq,
|
|
2124
2225
|
name: event.name,
|
|
2125
2226
|
kind: "local",
|
|
2126
|
-
stepId: stepId(event.runId, event.seq),
|
|
2127
2227
|
status: failed ? "failed" : "completed",
|
|
2128
2228
|
output: failed ? void 0 : event.output,
|
|
2129
2229
|
error: failed ? event.error : void 0,
|
|
2130
|
-
events,
|
|
2230
|
+
events: event.events,
|
|
2131
2231
|
attempts: 1,
|
|
2132
2232
|
enqueuedAt: startedAt,
|
|
2133
2233
|
startedAt,
|
|
2134
2234
|
finishedAt: event.finishedAt != null ? new Date(event.finishedAt) : /* @__PURE__ */ new Date()
|
|
2135
|
-
});
|
|
2235
|
+
}));
|
|
2136
2236
|
this.emit({
|
|
2137
2237
|
type: failed ? "step.failed" : "step.completed",
|
|
2138
2238
|
runId: event.runId,
|
|
@@ -2152,21 +2252,20 @@ var WorkflowEngine = class {
|
|
|
2152
2252
|
*/
|
|
2153
2253
|
async startStep(step) {
|
|
2154
2254
|
if (this.trackStepStart) {
|
|
2155
|
-
await this.store.saveCheckpoint({
|
|
2255
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2156
2256
|
runId: step.runId,
|
|
2157
2257
|
seq: step.seq,
|
|
2158
2258
|
name: step.name,
|
|
2159
2259
|
kind: step.kind,
|
|
2160
|
-
stepId: stepId(step.runId, step.seq),
|
|
2161
2260
|
status: "running",
|
|
2162
2261
|
input: step.input,
|
|
2163
|
-
events: step.events
|
|
2262
|
+
events: step.events,
|
|
2164
2263
|
attempts: step.attempts,
|
|
2165
2264
|
workerGroup: step.workerGroup,
|
|
2166
2265
|
enqueuedAt: step.enqueuedAt,
|
|
2167
2266
|
startedAt: step.startedAt,
|
|
2168
2267
|
finishedAt: step.startedAt
|
|
2169
|
-
});
|
|
2268
|
+
}));
|
|
2170
2269
|
}
|
|
2171
2270
|
this.emit({
|
|
2172
2271
|
type: "step.started",
|
|
@@ -2178,22 +2277,21 @@ var WorkflowEngine = class {
|
|
|
2178
2277
|
}
|
|
2179
2278
|
/** Checkpoint a finished step and announce it — the two things that must always happen together. */
|
|
2180
2279
|
async completeStep(step) {
|
|
2181
|
-
await this.store.saveCheckpoint({
|
|
2280
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2182
2281
|
runId: step.runId,
|
|
2183
2282
|
seq: step.seq,
|
|
2184
2283
|
name: step.name,
|
|
2185
2284
|
kind: step.kind,
|
|
2186
|
-
stepId: stepId(step.runId, step.seq),
|
|
2187
2285
|
status: "completed",
|
|
2188
2286
|
input: step.input,
|
|
2189
2287
|
output: step.output,
|
|
2190
|
-
events: step.events
|
|
2288
|
+
events: step.events,
|
|
2191
2289
|
attempts: step.attempts,
|
|
2192
2290
|
workerGroup: step.workerGroup,
|
|
2193
2291
|
enqueuedAt: step.enqueuedAt,
|
|
2194
2292
|
startedAt: step.startedAt,
|
|
2195
2293
|
finishedAt: /* @__PURE__ */ new Date()
|
|
2196
|
-
});
|
|
2294
|
+
}));
|
|
2197
2295
|
this.emit({
|
|
2198
2296
|
type: "step.completed",
|
|
2199
2297
|
runId: step.runId,
|
|
@@ -2207,22 +2305,21 @@ var WorkflowEngine = class {
|
|
|
2207
2305
|
}
|
|
2208
2306
|
/** Checkpoint a step that failed terminally, so the failure point is visible (not just the run). */
|
|
2209
2307
|
async failStep(step) {
|
|
2210
|
-
await this.store.saveCheckpoint({
|
|
2308
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2211
2309
|
runId: step.runId,
|
|
2212
2310
|
seq: step.seq,
|
|
2213
2311
|
name: step.name,
|
|
2214
2312
|
kind: step.kind,
|
|
2215
|
-
stepId: stepId(step.runId, step.seq),
|
|
2216
2313
|
status: "failed",
|
|
2217
2314
|
input: step.input,
|
|
2218
2315
|
error: step.error,
|
|
2219
|
-
events: step.events
|
|
2316
|
+
events: step.events,
|
|
2220
2317
|
attempts: step.attempts,
|
|
2221
2318
|
workerGroup: step.workerGroup,
|
|
2222
2319
|
enqueuedAt: step.enqueuedAt,
|
|
2223
2320
|
startedAt: step.startedAt,
|
|
2224
2321
|
finishedAt: /* @__PURE__ */ new Date()
|
|
2225
|
-
});
|
|
2322
|
+
}));
|
|
2226
2323
|
this.emit({
|
|
2227
2324
|
type: "step.failed",
|
|
2228
2325
|
runId: step.runId,
|
|
@@ -2234,55 +2331,6 @@ var WorkflowEngine = class {
|
|
|
2234
2331
|
durationMs: Date.now() - step.startedAt.getTime()
|
|
2235
2332
|
});
|
|
2236
2333
|
}
|
|
2237
|
-
/**
|
|
2238
|
-
* Whether `run` may run now under its singleton key: it's among the `limit` oldest in-flight runs
|
|
2239
|
-
* (running or suspended) sharing the key, by `(createdAt, id)` order. A consistent store gives every
|
|
2240
|
-
* instance the same ordering, so admission is race-free + FIFO.
|
|
2241
|
-
*/
|
|
2242
|
-
async admitSingleton(run, cfg) {
|
|
2243
|
-
const tag = singletonTag(cfg, run.input);
|
|
2244
|
-
const inflight = (await this.store.listRuns({
|
|
2245
|
-
tag,
|
|
2246
|
-
workflow: run.workflow,
|
|
2247
|
-
statuses: [
|
|
2248
|
-
"running",
|
|
2249
|
-
"suspended"
|
|
2250
|
-
]
|
|
2251
|
-
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2252
|
-
const idx = inflight.findIndex((r) => r.id === run.id);
|
|
2253
|
-
return idx >= 0 && idx < (cfg.limit ?? 1);
|
|
2254
|
-
}
|
|
2255
|
-
/**
|
|
2256
|
-
* Notify-on-release: a singleton run just settled (freeing a slot), so wake the gated waiters that
|
|
2257
|
-
* can now be admitted — immediately, instead of letting them sleep until their ~1s retry timer.
|
|
2258
|
-
* Dispatches up to `limit` of the oldest gated (`suspended` + same singleton tag) waiters by
|
|
2259
|
-
* `(createdAt, id)`; each re-checks admission in `runExecution` and runs only if it actually wins a
|
|
2260
|
-
* slot, so this never breaks the FIFO + race-free guarantees (the durable timer remains the
|
|
2261
|
-
* cross-instance / crash fallback — this just removes the poll latency on the happy path).
|
|
2262
|
-
*/
|
|
2263
|
-
async wakeNextSingletons(settled) {
|
|
2264
|
-
const registered = this.workflows.get(versionKey(settled.workflow, settled.workflowVersion)) ?? this.latest.get(settled.workflow);
|
|
2265
|
-
const cfg = registered?.singleton;
|
|
2266
|
-
if (!cfg) return;
|
|
2267
|
-
const tag = settled.tags?.find((t) => t.startsWith("singleton:"));
|
|
2268
|
-
if (!tag) return;
|
|
2269
|
-
const gated = (await this.store.listRuns({
|
|
2270
|
-
tag,
|
|
2271
|
-
workflow: settled.workflow,
|
|
2272
|
-
statuses: [
|
|
2273
|
-
"suspended"
|
|
2274
|
-
]
|
|
2275
|
-
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2276
|
-
for (const next of gated.slice(0, cfg.limit ?? 1)) {
|
|
2277
|
-
if (next.wakeAt != null) {
|
|
2278
|
-
await this.store.updateRun(next.id, {
|
|
2279
|
-
wakeAt: void 0,
|
|
2280
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2281
|
-
}).catch(() => void 0);
|
|
2282
|
-
}
|
|
2283
|
-
this.runDispatcher.dispatch(next.id);
|
|
2284
|
-
}
|
|
2285
|
-
}
|
|
2286
2334
|
async execute(run, fn) {
|
|
2287
2335
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
|
|
2288
2336
|
const lockNow = this.clock();
|
|
@@ -2301,7 +2349,7 @@ var WorkflowEngine = class {
|
|
|
2301
2349
|
try {
|
|
2302
2350
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
2303
2351
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
2304
|
-
void this.
|
|
2352
|
+
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
2305
2353
|
}
|
|
2306
2354
|
return result;
|
|
2307
2355
|
} finally {
|
|
@@ -2314,6 +2362,75 @@ var WorkflowEngine = class {
|
|
|
2314
2362
|
* {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
|
|
2315
2363
|
* us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
|
|
2316
2364
|
*/
|
|
2365
|
+
/**
|
|
2366
|
+
* Apply a terminal/suspended transition once: persist the new run status, emit the matching
|
|
2367
|
+
* lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
|
|
2368
|
+
* executors funnel their completed/failed/suspended outcomes through here so a status update can
|
|
2369
|
+
* never drift from its event or `notifyParent` call. Does NOT touch the run lock — each executor
|
|
2370
|
+
* keeps its own lease handling (the TS executor's `finally`, the remote path's caller).
|
|
2371
|
+
*/
|
|
2372
|
+
async settleRun(run, outcome) {
|
|
2373
|
+
const updatedAt = /* @__PURE__ */ new Date();
|
|
2374
|
+
if (outcome.kind === "completed") {
|
|
2375
|
+
await this.store.updateRun(run.id, {
|
|
2376
|
+
status: "completed",
|
|
2377
|
+
output: outcome.output,
|
|
2378
|
+
error: void 0,
|
|
2379
|
+
updatedAt
|
|
2380
|
+
});
|
|
2381
|
+
this.emit({
|
|
2382
|
+
type: "run.completed",
|
|
2383
|
+
runId: run.id,
|
|
2384
|
+
workflow: run.workflow,
|
|
2385
|
+
output: outcome.output
|
|
2386
|
+
});
|
|
2387
|
+
void this.notifyParent(run.id, {
|
|
2388
|
+
ok: true,
|
|
2389
|
+
value: outcome.output
|
|
2390
|
+
});
|
|
2391
|
+
return {
|
|
2392
|
+
runId: run.id,
|
|
2393
|
+
status: "completed",
|
|
2394
|
+
output: outcome.output
|
|
2395
|
+
};
|
|
2396
|
+
}
|
|
2397
|
+
if (outcome.kind === "failed") {
|
|
2398
|
+
await this.store.updateRun(run.id, {
|
|
2399
|
+
status: "failed",
|
|
2400
|
+
error: outcome.error,
|
|
2401
|
+
updatedAt
|
|
2402
|
+
});
|
|
2403
|
+
this.emit({
|
|
2404
|
+
type: "run.failed",
|
|
2405
|
+
runId: run.id,
|
|
2406
|
+
workflow: run.workflow,
|
|
2407
|
+
error: outcome.error
|
|
2408
|
+
});
|
|
2409
|
+
void this.notifyParent(run.id, {
|
|
2410
|
+
ok: false,
|
|
2411
|
+
error: outcome.error.message
|
|
2412
|
+
});
|
|
2413
|
+
return {
|
|
2414
|
+
runId: run.id,
|
|
2415
|
+
status: "failed",
|
|
2416
|
+
error: outcome.error
|
|
2417
|
+
};
|
|
2418
|
+
}
|
|
2419
|
+
await this.store.updateRun(run.id, {
|
|
2420
|
+
status: "suspended",
|
|
2421
|
+
wakeAt: outcome.wakeAt,
|
|
2422
|
+
updatedAt
|
|
2423
|
+
});
|
|
2424
|
+
this.emit({
|
|
2425
|
+
type: "run.suspended",
|
|
2426
|
+
runId: run.id,
|
|
2427
|
+
workflow: run.workflow
|
|
2428
|
+
});
|
|
2429
|
+
return {
|
|
2430
|
+
runId: run.id,
|
|
2431
|
+
status: "suspended"
|
|
2432
|
+
};
|
|
2433
|
+
}
|
|
2317
2434
|
async runRemoteExecution(run, registered) {
|
|
2318
2435
|
const remote = registered.remote;
|
|
2319
2436
|
if (run.status === "pending") {
|
|
@@ -2328,8 +2445,8 @@ var WorkflowEngine = class {
|
|
|
2328
2445
|
workflow: run.workflow
|
|
2329
2446
|
});
|
|
2330
2447
|
}
|
|
2331
|
-
if (registered.singleton && !await this.
|
|
2332
|
-
const wakeAt2 =
|
|
2448
|
+
if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
2449
|
+
const wakeAt2 = this.singletons.retryWakeAt();
|
|
2333
2450
|
await this.store.updateRun(run.id, {
|
|
2334
2451
|
status: "suspended",
|
|
2335
2452
|
wakeAt: wakeAt2,
|
|
@@ -2374,69 +2491,36 @@ var WorkflowEngine = class {
|
|
|
2374
2491
|
}
|
|
2375
2492
|
if (decision.status === "completed") {
|
|
2376
2493
|
await this.applyCommands(run, decision.commands);
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
output: decision.output,
|
|
2380
|
-
error: void 0,
|
|
2381
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2382
|
-
});
|
|
2383
|
-
this.emit({
|
|
2384
|
-
type: "run.completed",
|
|
2385
|
-
runId: run.id,
|
|
2386
|
-
workflow: run.workflow,
|
|
2494
|
+
return this.settleRun(run, {
|
|
2495
|
+
kind: "completed",
|
|
2387
2496
|
output: decision.output
|
|
2388
2497
|
});
|
|
2389
|
-
void this.notifyParent(run.id, {
|
|
2390
|
-
ok: true,
|
|
2391
|
-
value: decision.output
|
|
2392
|
-
});
|
|
2393
|
-
return {
|
|
2394
|
-
runId: run.id,
|
|
2395
|
-
status: "completed",
|
|
2396
|
-
output: decision.output
|
|
2397
|
-
};
|
|
2398
2498
|
}
|
|
2399
2499
|
if (decision.status === "failed") {
|
|
2400
2500
|
await this.applyCommands(run, decision.commands);
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2501
|
+
return this.settleRun(run, {
|
|
2502
|
+
kind: "failed",
|
|
2503
|
+
error: decision.error ?? {
|
|
2504
|
+
message: "workflow failed"
|
|
2505
|
+
}
|
|
2506
|
+
});
|
|
2507
|
+
}
|
|
2508
|
+
if (decision.status === "cancelled") {
|
|
2509
|
+
await this.applyCommands(run, decision.commands);
|
|
2404
2510
|
await this.store.updateRun(run.id, {
|
|
2405
|
-
status: "
|
|
2406
|
-
error,
|
|
2511
|
+
status: "cancelled",
|
|
2407
2512
|
updatedAt: /* @__PURE__ */ new Date()
|
|
2408
2513
|
});
|
|
2409
|
-
this.emit({
|
|
2410
|
-
type: "run.failed",
|
|
2411
|
-
runId: run.id,
|
|
2412
|
-
workflow: run.workflow,
|
|
2413
|
-
error
|
|
2414
|
-
});
|
|
2415
|
-
void this.notifyParent(run.id, {
|
|
2416
|
-
ok: false,
|
|
2417
|
-
error: error.message
|
|
2418
|
-
});
|
|
2419
2514
|
return {
|
|
2420
2515
|
runId: run.id,
|
|
2421
|
-
status: "
|
|
2422
|
-
error
|
|
2516
|
+
status: "cancelled"
|
|
2423
2517
|
};
|
|
2424
2518
|
}
|
|
2425
2519
|
const wakeAt = await this.applyCommands(run, decision.commands);
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
wakeAt
|
|
2429
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2520
|
+
return this.settleRun(run, {
|
|
2521
|
+
kind: "suspended",
|
|
2522
|
+
wakeAt
|
|
2430
2523
|
});
|
|
2431
|
-
this.emit({
|
|
2432
|
-
type: "run.suspended",
|
|
2433
|
-
runId: run.id,
|
|
2434
|
-
workflow: run.workflow
|
|
2435
|
-
});
|
|
2436
|
-
return {
|
|
2437
|
-
runId: run.id,
|
|
2438
|
-
status: "suspended"
|
|
2439
|
-
};
|
|
2440
2524
|
}
|
|
2441
2525
|
/** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
|
|
2442
2526
|
async remoteHistory(runId) {
|
|
@@ -2491,21 +2575,20 @@ var WorkflowEngine = class {
|
|
|
2491
2575
|
if (cmd.kind === "recordStep") {
|
|
2492
2576
|
const startedAt = cmd.startedAt != null ? new Date(cmd.startedAt) : at;
|
|
2493
2577
|
const finishedAt = cmd.finishedAt != null ? new Date(cmd.finishedAt) : at;
|
|
2494
|
-
await this.store.saveCheckpoint({
|
|
2578
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2495
2579
|
runId: run.id,
|
|
2496
2580
|
seq: cmd.seq,
|
|
2497
2581
|
name: cmd.name,
|
|
2498
2582
|
kind: "local",
|
|
2499
|
-
stepId: id,
|
|
2500
2583
|
status: cmd.error ? "failed" : "completed",
|
|
2501
2584
|
output: cmd.output,
|
|
2502
2585
|
error: cmd.error,
|
|
2503
|
-
events: cmd.events
|
|
2586
|
+
events: cmd.events,
|
|
2504
2587
|
attempts: 1,
|
|
2505
2588
|
enqueuedAt: startedAt,
|
|
2506
2589
|
startedAt,
|
|
2507
2590
|
finishedAt
|
|
2508
|
-
});
|
|
2591
|
+
}));
|
|
2509
2592
|
this.emit({
|
|
2510
2593
|
type: cmd.error ? "step.failed" : "step.completed",
|
|
2511
2594
|
runId: run.id,
|
|
@@ -2516,12 +2599,11 @@ var WorkflowEngine = class {
|
|
|
2516
2599
|
error: cmd.error
|
|
2517
2600
|
});
|
|
2518
2601
|
} else if (cmd.kind === "call") {
|
|
2519
|
-
await this.store.saveCheckpoint({
|
|
2602
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2520
2603
|
runId: run.id,
|
|
2521
2604
|
seq: cmd.seq,
|
|
2522
2605
|
name: cmd.name,
|
|
2523
2606
|
kind: "remote",
|
|
2524
|
-
stepId: id,
|
|
2525
2607
|
status: "pending",
|
|
2526
2608
|
input: cmd.input,
|
|
2527
2609
|
attempts: 1,
|
|
@@ -2529,7 +2611,7 @@ var WorkflowEngine = class {
|
|
|
2529
2611
|
enqueuedAt: at,
|
|
2530
2612
|
startedAt: at,
|
|
2531
2613
|
finishedAt: at
|
|
2532
|
-
});
|
|
2614
|
+
}));
|
|
2533
2615
|
await this.pool.dispatch({
|
|
2534
2616
|
runId: run.id,
|
|
2535
2617
|
seq: cmd.seq,
|
|
@@ -2550,19 +2632,18 @@ var WorkflowEngine = class {
|
|
|
2550
2632
|
});
|
|
2551
2633
|
} else if (cmd.kind === "sleep") {
|
|
2552
2634
|
const deadline = this.clock() + cmd.ms;
|
|
2553
|
-
await this.store.saveCheckpoint({
|
|
2635
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
2554
2636
|
runId: run.id,
|
|
2555
2637
|
seq: cmd.seq,
|
|
2556
2638
|
name: `sleep:${cmd.seq}`,
|
|
2557
2639
|
kind: "sleep",
|
|
2558
|
-
stepId: id,
|
|
2559
2640
|
status: "pending",
|
|
2560
2641
|
attempts: 1,
|
|
2561
2642
|
wakeAt: deadline,
|
|
2562
2643
|
enqueuedAt: at,
|
|
2563
2644
|
startedAt: at,
|
|
2564
2645
|
finishedAt: at
|
|
2565
|
-
});
|
|
2646
|
+
}));
|
|
2566
2647
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
2567
2648
|
} else if (cmd.kind === "waitSignal") {
|
|
2568
2649
|
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
@@ -2612,8 +2693,8 @@ var WorkflowEngine = class {
|
|
|
2612
2693
|
workflow: run.workflow
|
|
2613
2694
|
});
|
|
2614
2695
|
}
|
|
2615
|
-
if (registered?.singleton && !await this.
|
|
2616
|
-
const wakeAt =
|
|
2696
|
+
if (registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
2697
|
+
const wakeAt = this.singletons.retryWakeAt();
|
|
2617
2698
|
await this.store.updateRun(run.id, {
|
|
2618
2699
|
status: "suspended",
|
|
2619
2700
|
wakeAt,
|
|
@@ -2637,27 +2718,10 @@ var WorkflowEngine = class {
|
|
|
2637
2718
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay), run.id, compensations, run.workflow);
|
|
2638
2719
|
try {
|
|
2639
2720
|
const output = await fn(ctx, run.input);
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
output,
|
|
2643
|
-
error: void 0,
|
|
2644
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2645
|
-
});
|
|
2646
|
-
this.emit({
|
|
2647
|
-
type: "run.completed",
|
|
2648
|
-
runId: run.id,
|
|
2649
|
-
workflow: run.workflow,
|
|
2721
|
+
return this.settleRun(run, {
|
|
2722
|
+
kind: "completed",
|
|
2650
2723
|
output
|
|
2651
2724
|
});
|
|
2652
|
-
void this.notifyParent(run.id, {
|
|
2653
|
-
ok: true,
|
|
2654
|
-
value: output
|
|
2655
|
-
});
|
|
2656
|
-
return {
|
|
2657
|
-
runId: run.id,
|
|
2658
|
-
status: "completed",
|
|
2659
|
-
output
|
|
2660
|
-
};
|
|
2661
2725
|
} catch (err) {
|
|
2662
2726
|
if (err instanceof ContinueAsNew) {
|
|
2663
2727
|
await this.store.updateRun(run.id, {
|
|
@@ -2709,20 +2773,10 @@ var WorkflowEngine = class {
|
|
|
2709
2773
|
error: error2
|
|
2710
2774
|
};
|
|
2711
2775
|
}
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
wakeAt: err.wakeAt
|
|
2715
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2776
|
+
return this.settleRun(run, {
|
|
2777
|
+
kind: "suspended",
|
|
2778
|
+
wakeAt: err.wakeAt
|
|
2716
2779
|
});
|
|
2717
|
-
this.emit({
|
|
2718
|
-
type: "run.suspended",
|
|
2719
|
-
runId: run.id,
|
|
2720
|
-
workflow: run.workflow
|
|
2721
|
-
});
|
|
2722
|
-
return {
|
|
2723
|
-
runId: run.id,
|
|
2724
|
-
status: "suspended"
|
|
2725
|
-
};
|
|
2726
2780
|
}
|
|
2727
2781
|
const error = {
|
|
2728
2782
|
message: err instanceof Error ? err.message : String(err),
|
|
@@ -2733,26 +2787,10 @@ var WorkflowEngine = class {
|
|
|
2733
2787
|
if (!comp) continue;
|
|
2734
2788
|
await this.runCompensation(run, comp);
|
|
2735
2789
|
}
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
error,
|
|
2739
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2740
|
-
});
|
|
2741
|
-
this.emit({
|
|
2742
|
-
type: "run.failed",
|
|
2743
|
-
runId: run.id,
|
|
2744
|
-
workflow: run.workflow,
|
|
2790
|
+
return this.settleRun(run, {
|
|
2791
|
+
kind: "failed",
|
|
2745
2792
|
error
|
|
2746
2793
|
});
|
|
2747
|
-
void this.notifyParent(run.id, {
|
|
2748
|
-
ok: false,
|
|
2749
|
-
error: error.message
|
|
2750
|
-
});
|
|
2751
|
-
return {
|
|
2752
|
-
runId: run.id,
|
|
2753
|
-
status: "failed",
|
|
2754
|
-
error
|
|
2755
|
-
};
|
|
2756
2794
|
} finally {
|
|
2757
2795
|
await this.store.releaseRunLock(run.id);
|
|
2758
2796
|
}
|