@dudousxd/nestjs-durable-core 0.104.0 → 0.106.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +263 -71
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +102 -2
- package/dist/index.d.ts +102 -2
- package/dist/index.js +262 -71
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -246,6 +246,7 @@ __export(index_exports, {
|
|
|
246
246
|
resolveWireTenant: () => resolveWireTenant,
|
|
247
247
|
retentionKeeps: () => retentionKeeps,
|
|
248
248
|
retiredHandoffName: () => retiredHandoffName,
|
|
249
|
+
retryLocalStep: () => retryLocalStep,
|
|
249
250
|
routeFor: () => routeFor,
|
|
250
251
|
rowTenant: () => rowTenant,
|
|
251
252
|
runCompletedEvent: () => runCompletedEvent,
|
|
@@ -981,6 +982,27 @@ function backoffDelay(attempt, options) {
|
|
|
981
982
|
return options?.jitter ? Math.round(capped * (0.5 + Math.random() * 0.5)) : capped;
|
|
982
983
|
}
|
|
983
984
|
__name(backoffDelay, "backoffDelay");
|
|
985
|
+
async function retryLocalStep(options, attempt) {
|
|
986
|
+
const maxAttempts = Math.max(1, options?.retries ?? 1);
|
|
987
|
+
for (let n = 1; ; n += 1) {
|
|
988
|
+
try {
|
|
989
|
+
return {
|
|
990
|
+
ok: true,
|
|
991
|
+
value: await attempt(n),
|
|
992
|
+
attempts: n
|
|
993
|
+
};
|
|
994
|
+
} catch (error2) {
|
|
995
|
+
if (error2 instanceof FatalError || n >= maxAttempts) return {
|
|
996
|
+
ok: false,
|
|
997
|
+
error: error2,
|
|
998
|
+
attempts: n
|
|
999
|
+
};
|
|
1000
|
+
const wait = backoffDelay(n, options);
|
|
1001
|
+
if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
__name(retryLocalStep, "retryLocalStep");
|
|
984
1006
|
|
|
985
1007
|
// src/bounded-runner.ts
|
|
986
1008
|
var BoundedRunner = class {
|
|
@@ -3474,6 +3496,11 @@ var RemoteWorkflowExecutor = class {
|
|
|
3474
3496
|
};
|
|
3475
3497
|
await this.transport.dispatchWorkflowTask(task);
|
|
3476
3498
|
}
|
|
3499
|
+
/** Where the broker has the task dispatched under `taskId`, when the transport can tell. */
|
|
3500
|
+
async taskState(_run, taskId) {
|
|
3501
|
+
if (!this.transport.workflowTaskState) return "unknown";
|
|
3502
|
+
return this.transport.workflowTaskState(tenantGroup(sanitizeQueueToken(this.name), this.partition), taskId);
|
|
3503
|
+
}
|
|
3477
3504
|
};
|
|
3478
3505
|
|
|
3479
3506
|
// src/run-dispatch-batcher.ts
|
|
@@ -6210,7 +6237,6 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
6210
6237
|
});
|
|
6211
6238
|
return existing.output;
|
|
6212
6239
|
}
|
|
6213
|
-
const maxAttempts = Math.max(1, options?.retries ?? 1);
|
|
6214
6240
|
const startedAt = /* @__PURE__ */ new Date();
|
|
6215
6241
|
await host.startStep({
|
|
6216
6242
|
runId,
|
|
@@ -6221,60 +6247,59 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
6221
6247
|
enqueuedAt: startedAt,
|
|
6222
6248
|
startedAt
|
|
6223
6249
|
});
|
|
6224
|
-
|
|
6225
|
-
|
|
6226
|
-
|
|
6227
|
-
|
|
6228
|
-
|
|
6229
|
-
|
|
6230
|
-
|
|
6231
|
-
|
|
6232
|
-
|
|
6233
|
-
|
|
6234
|
-
|
|
6235
|
-
|
|
6236
|
-
|
|
6237
|
-
|
|
6238
|
-
|
|
6239
|
-
|
|
6240
|
-
|
|
6241
|
-
|
|
6242
|
-
|
|
6243
|
-
|
|
6244
|
-
|
|
6245
|
-
|
|
6246
|
-
|
|
6247
|
-
|
|
6248
|
-
|
|
6249
|
-
|
|
6250
|
-
|
|
6251
|
-
|
|
6252
|
-
|
|
6253
|
-
|
|
6254
|
-
|
|
6255
|
-
|
|
6256
|
-
|
|
6257
|
-
|
|
6258
|
-
|
|
6259
|
-
seq: current,
|
|
6260
|
-
name,
|
|
6261
|
-
kind: "local",
|
|
6262
|
-
error: {
|
|
6263
|
-
message: err instanceof Error ? err.message : String(err),
|
|
6264
|
-
stack: err instanceof Error ? err.stack : void 0,
|
|
6265
|
-
...codeAndDetails(err)
|
|
6266
|
-
},
|
|
6267
|
-
events,
|
|
6268
|
-
attempts: attempt,
|
|
6269
|
-
enqueuedAt: startedAt,
|
|
6270
|
-
startedAt
|
|
6271
|
-
});
|
|
6272
|
-
throw err;
|
|
6273
|
-
}
|
|
6274
|
-
const wait = backoffDelay(attempt, options);
|
|
6275
|
-
if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
|
|
6276
|
-
}
|
|
6250
|
+
let events = [];
|
|
6251
|
+
const outcome = await retryLocalStep(options, async (attempt) => {
|
|
6252
|
+
const attemptEvents = [];
|
|
6253
|
+
events = attemptEvents;
|
|
6254
|
+
const invocation = {
|
|
6255
|
+
runId,
|
|
6256
|
+
workflow,
|
|
6257
|
+
stepName: name,
|
|
6258
|
+
seq: current,
|
|
6259
|
+
attempt
|
|
6260
|
+
};
|
|
6261
|
+
const body = /* @__PURE__ */ __name(() => {
|
|
6262
|
+
const logger = createStepLogger(attemptEvents, host.clock);
|
|
6263
|
+
return runInStepScope(scopeOwner, name, () => runInStepLogger(logger, () => fn(logger)));
|
|
6264
|
+
}, "body");
|
|
6265
|
+
const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
|
|
6266
|
+
await host.completeStep({
|
|
6267
|
+
runId,
|
|
6268
|
+
seq: current,
|
|
6269
|
+
name,
|
|
6270
|
+
kind: "local",
|
|
6271
|
+
output,
|
|
6272
|
+
events: attemptEvents,
|
|
6273
|
+
attempts: attempt,
|
|
6274
|
+
enqueuedAt: startedAt,
|
|
6275
|
+
startedAt
|
|
6276
|
+
});
|
|
6277
|
+
return output;
|
|
6278
|
+
});
|
|
6279
|
+
if (outcome.ok) {
|
|
6280
|
+
if (options?.compensate) compensations.push({
|
|
6281
|
+
name,
|
|
6282
|
+
fn: options.compensate
|
|
6283
|
+
});
|
|
6284
|
+
return outcome.value;
|
|
6277
6285
|
}
|
|
6286
|
+
const err = outcome.error;
|
|
6287
|
+
await host.failStep({
|
|
6288
|
+
runId,
|
|
6289
|
+
seq: current,
|
|
6290
|
+
name,
|
|
6291
|
+
kind: "local",
|
|
6292
|
+
error: {
|
|
6293
|
+
message: err instanceof Error ? err.message : String(err),
|
|
6294
|
+
stack: err instanceof Error ? err.stack : void 0,
|
|
6295
|
+
...codeAndDetails(err)
|
|
6296
|
+
},
|
|
6297
|
+
events,
|
|
6298
|
+
attempts: outcome.attempts,
|
|
6299
|
+
enqueuedAt: startedAt,
|
|
6300
|
+
startedAt
|
|
6301
|
+
});
|
|
6302
|
+
throw err;
|
|
6278
6303
|
}, "localStep");
|
|
6279
6304
|
const transaction = /* @__PURE__ */ __name(async (name, fn) => {
|
|
6280
6305
|
if (!store.transaction) {
|
|
@@ -7296,6 +7321,20 @@ function mergeTags(staticTags, runTags) {
|
|
|
7296
7321
|
}
|
|
7297
7322
|
__name(mergeTags, "mergeTags");
|
|
7298
7323
|
var BREAKPOINT = "breakpoint";
|
|
7324
|
+
var LOST_TURN_SUFFIX = ":lost";
|
|
7325
|
+
var GRACED_TURN_SUFFIX = ":grace";
|
|
7326
|
+
function awaitsDecision(run, decision) {
|
|
7327
|
+
const marker = run.awaitingDecisionTaskId;
|
|
7328
|
+
return marker !== void 0 && awaitedTaskId(marker) === decision.taskId;
|
|
7329
|
+
}
|
|
7330
|
+
__name(awaitsDecision, "awaitsDecision");
|
|
7331
|
+
function awaitedTaskId(marker) {
|
|
7332
|
+
let id = marker;
|
|
7333
|
+
if (id.endsWith(LOST_TURN_SUFFIX)) id = id.slice(0, -LOST_TURN_SUFFIX.length);
|
|
7334
|
+
if (id.endsWith(GRACED_TURN_SUFFIX)) id = id.slice(0, -GRACED_TURN_SUFFIX.length);
|
|
7335
|
+
return id;
|
|
7336
|
+
}
|
|
7337
|
+
__name(awaitedTaskId, "awaitedTaskId");
|
|
7299
7338
|
var PENDING_POLL_BATCH = 100;
|
|
7300
7339
|
var DEFAULT_STICKY_RUNS = 1e3;
|
|
7301
7340
|
var DEFAULT_STICKY_IDLE_MS = 3e5;
|
|
@@ -7565,6 +7604,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7565
7604
|
* `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
|
|
7566
7605
|
*/
|
|
7567
7606
|
ownLeaseCutoffMs;
|
|
7607
|
+
/** When this engine was built (clock ms): {@link noteLapsedTurn} counts no lost turn within one
|
|
7608
|
+
* turn window of it, while live workers' beats catch up with deadlines that went stale. */
|
|
7609
|
+
bootedAtMs;
|
|
7568
7610
|
maxRecoveryAttempts;
|
|
7569
7611
|
unregisteredWorkflowGraceMs;
|
|
7570
7612
|
remoteAdvanceSilenceMs;
|
|
@@ -7794,6 +7836,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7794
7836
|
this.namespace = poolNamespace(deps.namespace);
|
|
7795
7837
|
this.leaseMs = deps.leaseMs ?? 3e4;
|
|
7796
7838
|
this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
|
|
7839
|
+
this.bootedAtMs = this.clock();
|
|
7797
7840
|
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
7798
7841
|
const grace = deps.unregisteredWorkflowGraceMs;
|
|
7799
7842
|
this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
|
|
@@ -8171,6 +8214,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8171
8214
|
return;
|
|
8172
8215
|
}
|
|
8173
8216
|
if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
8217
|
+
if (dispatch.awaiting !== void 0 && (run.awaitingDecisionTaskId === void 0 || awaitedTaskId(run.awaitingDecisionTaskId) !== dispatch.awaiting)) {
|
|
8218
|
+
return;
|
|
8219
|
+
}
|
|
8174
8220
|
const nowMs = this.clock();
|
|
8175
8221
|
if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
|
|
8176
8222
|
const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
@@ -8222,6 +8268,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8222
8268
|
*/
|
|
8223
8269
|
async handOffDue(runs, nowMs, settled) {
|
|
8224
8270
|
const claimed = [];
|
|
8271
|
+
const awaiting = /* @__PURE__ */ new Map();
|
|
8225
8272
|
for (const run of runs) {
|
|
8226
8273
|
if (this.draining) break;
|
|
8227
8274
|
this.markClaim(run.id);
|
|
@@ -8236,16 +8283,26 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8236
8283
|
continue;
|
|
8237
8284
|
}
|
|
8238
8285
|
try {
|
|
8286
|
+
const seen = {};
|
|
8287
|
+
const handled = await this.noteLapsedTurn(run.id, nowMs, seen);
|
|
8288
|
+
if (handled) {
|
|
8289
|
+
if (handled.status === "dead") settled?.set(run.id, handled);
|
|
8290
|
+
continue;
|
|
8291
|
+
}
|
|
8239
8292
|
const patch = {
|
|
8240
8293
|
wakeAt: nowMs + this.leaseMs
|
|
8241
8294
|
};
|
|
8295
|
+
let won = true;
|
|
8242
8296
|
if (this.store.updateRunIf) {
|
|
8243
|
-
|
|
8297
|
+
won = await this.store.updateRunIf(run.id, [
|
|
8244
8298
|
run.status
|
|
8245
|
-
], patch)
|
|
8299
|
+
], patch);
|
|
8246
8300
|
} else {
|
|
8247
8301
|
await this.store.updateRun(run.id, patch);
|
|
8302
|
+
}
|
|
8303
|
+
if (won) {
|
|
8248
8304
|
claimed.push(run);
|
|
8305
|
+
if (seen.marker !== void 0) awaiting.set(run.id, awaitedTaskId(seen.marker));
|
|
8249
8306
|
}
|
|
8250
8307
|
} finally {
|
|
8251
8308
|
await this.store.releaseRunLock(run.id, this.instanceId);
|
|
@@ -8256,13 +8313,19 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8256
8313
|
}
|
|
8257
8314
|
try {
|
|
8258
8315
|
for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
|
|
8259
|
-
await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) =>
|
|
8260
|
-
|
|
8261
|
-
|
|
8262
|
-
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
|
|
8316
|
+
await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => {
|
|
8317
|
+
const turn = awaiting.get(run.id);
|
|
8318
|
+
return {
|
|
8319
|
+
runId: run.id,
|
|
8320
|
+
reason: "wake",
|
|
8321
|
+
...run.priority !== void 0 ? {
|
|
8322
|
+
priority: run.priority
|
|
8323
|
+
} : {},
|
|
8324
|
+
...turn !== void 0 ? {
|
|
8325
|
+
awaiting: turn
|
|
8326
|
+
} : {}
|
|
8327
|
+
};
|
|
8328
|
+
}), {
|
|
8266
8329
|
namespace: this.namespace
|
|
8267
8330
|
});
|
|
8268
8331
|
}
|
|
@@ -11336,28 +11399,39 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
11336
11399
|
* `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
|
|
11337
11400
|
* result to skip the resume, or `undefined` to proceed.
|
|
11338
11401
|
*/
|
|
11339
|
-
async countRecovery(run) {
|
|
11402
|
+
async countRecovery(run, extra = {}) {
|
|
11340
11403
|
const attempts = (run.recoveryAttempts ?? 0) + 1;
|
|
11404
|
+
const updateIf = /* @__PURE__ */ __name((patch) => (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
11405
|
+
run.status
|
|
11406
|
+
], patch), "updateIf");
|
|
11407
|
+
const moved = /* @__PURE__ */ __name(async () => {
|
|
11408
|
+
await this.store.releaseRunLock(run.id);
|
|
11409
|
+
return this.echoCurrentStatus(run.id);
|
|
11410
|
+
}, "moved");
|
|
11341
11411
|
if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
|
|
11342
11412
|
const error2 = {
|
|
11343
11413
|
message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
|
|
11344
11414
|
code: "max_recovery_attempts"
|
|
11345
11415
|
};
|
|
11346
|
-
await
|
|
11416
|
+
const written = await updateIf(await this.withLifecycleMarker(run, {
|
|
11347
11417
|
status: "dead",
|
|
11348
11418
|
error: error2,
|
|
11349
11419
|
updatedAt: /* @__PURE__ */ new Date()
|
|
11350
11420
|
}));
|
|
11421
|
+
if (!written) return moved();
|
|
11351
11422
|
await this.store.releaseRunLock(run.id);
|
|
11352
11423
|
return this.announceDead({
|
|
11353
11424
|
...run,
|
|
11354
11425
|
recoveryAttempts: attempts
|
|
11355
11426
|
}, error2);
|
|
11356
11427
|
}
|
|
11357
|
-
await
|
|
11428
|
+
if (!await updateIf({
|
|
11429
|
+
...extra,
|
|
11358
11430
|
recoveryAttempts: attempts,
|
|
11359
11431
|
updatedAt: /* @__PURE__ */ new Date()
|
|
11360
|
-
})
|
|
11432
|
+
})) {
|
|
11433
|
+
return moved();
|
|
11434
|
+
}
|
|
11361
11435
|
return void 0;
|
|
11362
11436
|
}
|
|
11363
11437
|
/** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
|
|
@@ -11504,7 +11578,87 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
11504
11578
|
return this.resumeLeased([
|
|
11505
11579
|
...due,
|
|
11506
11580
|
...blocked
|
|
11507
|
-
], nowMs, (run) => this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
|
|
11581
|
+
], nowMs, async (run) => await this.noteLapsedTurn(run.id, nowMs) ?? this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
|
|
11582
|
+
}
|
|
11583
|
+
/**
|
|
11584
|
+
* Called under the run's lease by the due-timer pass, for a run that THIS instance would re-drive.
|
|
11585
|
+
* A run suspended awaiting a DISPATCHED workflow turn's decision (`awaitingDecisionTaskId`) whose
|
|
11586
|
+
* re-drive deadline (`wakeAt`) has passed lost that turn: no decision came back and the worker
|
|
11587
|
+
* stopped beating (every beat pushes `wakeAt` forward, see {@link rearmDecisionDeadline}), so the
|
|
11588
|
+
* worker running it died, or the task was lost with it. That is the dispatched form of a crashed
|
|
11589
|
+
* inline turn, so it spends the same {@link maxRecoveryAttempts} budget through
|
|
11590
|
+
* {@link countRecovery}: a turn that keeps killing its worker ends `dead` (with the same
|
|
11591
|
+
* `run.failed` announcement) instead of being re-dispatched, and redelivered, for ever.
|
|
11592
|
+
*
|
|
11593
|
+
* Counted at most once per awaited turn: the count rewrites the marker to its `:lost` form, which
|
|
11594
|
+
* no later pass counts again however long the re-drive waits on the run queue, and the re-drive's
|
|
11595
|
+
* dispatch replaces it with the next turn's id. Not counted, and the re-drive left for later:
|
|
11596
|
+
* - while the broker still holds the task unstarted (a backed-up queue, not a dead worker), or
|
|
11597
|
+
* - within one turn window of this engine's boot, when beats from live workers have not yet had
|
|
11598
|
+
* the chance to push a deadline that went stale while the engine was down. Once per turn (the
|
|
11599
|
+
* `:grace` marker): a lapse after that takes the normal path, whichever instance judges it.
|
|
11600
|
+
* A late decision of a counted turn is still applied while the re-drive has not dispatched yet
|
|
11601
|
+
* (see `awaitsDecision`), and gives the attempt back.
|
|
11602
|
+
* Never counted by an instance that cannot resolve the workflow (it re-drives nothing), nor for a
|
|
11603
|
+
* decision that arrives, a worker that keeps beating, or any write a healthy turn makes on its way.
|
|
11604
|
+
*
|
|
11605
|
+
* Re-reads under the lease: the listing may predate a decision applied since. Returns a result when
|
|
11606
|
+
* the run was dead-lettered or left waiting (the lease released either way), else `undefined` for
|
|
11607
|
+
* the caller to re-drive as before. A decision that lands while this holds the lease is dropped (it
|
|
11608
|
+
* cannot take the lease), whichever way this pass goes: with a run it dead-letters, or with one it
|
|
11609
|
+
* graces or leaves waiting on a `queued` task, whose next lapse then counts the turn and runs it
|
|
11610
|
+
* again. A narrow race, accepted.
|
|
11611
|
+
*
|
|
11612
|
+
* The marker grows by up to 11 characters (`:grace:lost`) beyond the dispatched id, which is
|
|
11613
|
+
* `<runId>:wf:<8 hex>`: `runId` plus 23 in all. A store that bounds the column (TypeORM's
|
|
11614
|
+
* `varchar(191)`) therefore dispatches a run whose id is 169 to 179 characters long, then refuses
|
|
11615
|
+
* the grace or lost write for it.
|
|
11616
|
+
*/
|
|
11617
|
+
async noteLapsedTurn(runId, nowMs, seen) {
|
|
11618
|
+
const run = await this.store.getRun(runId);
|
|
11619
|
+
const awaited = run?.awaitingDecisionTaskId;
|
|
11620
|
+
if (seen) seen.marker = awaited;
|
|
11621
|
+
if (!run || run.status !== "suspended" || awaited === void 0 || awaited.endsWith(LOST_TURN_SUFFIX) || run.wakeAt == null || run.wakeAt > nowMs) {
|
|
11622
|
+
return void 0;
|
|
11623
|
+
}
|
|
11624
|
+
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) return void 0;
|
|
11625
|
+
const windowMs = this.dispatchedTurnSilenceMs;
|
|
11626
|
+
if (windowMs != null && nowMs < this.bootedAtMs + windowMs && !awaited.endsWith(GRACED_TURN_SUFFIX)) {
|
|
11627
|
+
return this.leaveTurnWaiting(run, this.bootedAtMs + windowMs, {
|
|
11628
|
+
awaitingDecisionTaskId: `${awaited}${GRACED_TURN_SUFFIX}`
|
|
11629
|
+
});
|
|
11630
|
+
}
|
|
11631
|
+
const state = await this.dispatchedTurnState(run, awaitedTaskId(awaited));
|
|
11632
|
+
if (state === "queued") return this.leaveTurnWaiting(run, nowMs + (windowMs ?? this.leaseMs));
|
|
11633
|
+
if (state === "error") return void 0;
|
|
11634
|
+
return this.countRecovery(run, {
|
|
11635
|
+
awaitingDecisionTaskId: `${awaited}${LOST_TURN_SUFFIX}`
|
|
11636
|
+
});
|
|
11637
|
+
}
|
|
11638
|
+
/** Where the broker has the task of a dispatched turn: see {@link WorkflowExecutor.taskState}. */
|
|
11639
|
+
async dispatchedTurnState(run, taskId) {
|
|
11640
|
+
const executor = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.remote?.executor;
|
|
11641
|
+
if (!executor?.taskState) return "unknown";
|
|
11642
|
+
try {
|
|
11643
|
+
return await executor.taskState(run, taskId);
|
|
11644
|
+
} catch {
|
|
11645
|
+
return "error";
|
|
11646
|
+
}
|
|
11647
|
+
}
|
|
11648
|
+
/** Push a lapsed turn's deadline to `until` without re-driving it, and give the lease back. */
|
|
11649
|
+
async leaveTurnWaiting(run, until, extra = {}) {
|
|
11650
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
11651
|
+
"suspended"
|
|
11652
|
+
], {
|
|
11653
|
+
...extra,
|
|
11654
|
+
wakeAt: until,
|
|
11655
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
11656
|
+
});
|
|
11657
|
+
await this.store.releaseRunLock(run.id);
|
|
11658
|
+
return {
|
|
11659
|
+
runId: run.id,
|
|
11660
|
+
status: "suspended"
|
|
11661
|
+
};
|
|
11508
11662
|
}
|
|
11509
11663
|
/**
|
|
11510
11664
|
* The reconcile safety net's per-run check (see {@link resumeDueTimers}): if this suspended run has
|
|
@@ -13806,6 +13960,12 @@ ${run.workflowVersion}`;
|
|
|
13806
13960
|
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
|
|
13807
13961
|
status: "suspended",
|
|
13808
13962
|
wakeAt: this.reconcileWakeAt(outcome.wakeAt),
|
|
13963
|
+
// A turn that ran here suspended the run, so no dispatched turn is awaited any more: a marker
|
|
13964
|
+
// left by an earlier dispatch (a re-drive that followed an onError handoff, say) would make
|
|
13965
|
+
// this suspend read as a lost turn at its next reconcile wake.
|
|
13966
|
+
...run.awaitingDecisionTaskId !== void 0 ? {
|
|
13967
|
+
awaitingDecisionTaskId: void 0
|
|
13968
|
+
} : {},
|
|
13809
13969
|
updatedAt: stamp
|
|
13810
13970
|
}));
|
|
13811
13971
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
@@ -14403,7 +14563,7 @@ ${run.workflowVersion}`;
|
|
|
14403
14563
|
output: cmd.output,
|
|
14404
14564
|
error: cmd.error,
|
|
14405
14565
|
events: cmd.events,
|
|
14406
|
-
attempts: 1,
|
|
14566
|
+
attempts: cmd.attempts ?? 1,
|
|
14407
14567
|
enqueuedAt: startedAt,
|
|
14408
14568
|
startedAt,
|
|
14409
14569
|
finishedAt,
|
|
@@ -16628,7 +16788,7 @@ ${run.workflowVersion}`;
|
|
|
16628
16788
|
*/
|
|
16629
16789
|
async completeRemoteDecision(decision) {
|
|
16630
16790
|
const run = await this.store.getRun(decision.runId);
|
|
16631
|
-
if (!run || run
|
|
16791
|
+
if (!run || !awaitsDecision(run, decision)) return;
|
|
16632
16792
|
if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
|
|
16633
16793
|
if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
|
|
16634
16794
|
return;
|
|
@@ -16640,16 +16800,20 @@ ${run.workflowVersion}`;
|
|
|
16640
16800
|
const held = this.holdExecutionLease(decision.runId);
|
|
16641
16801
|
try {
|
|
16642
16802
|
const fresh = await this.store.getRun(decision.runId);
|
|
16643
|
-
if (!fresh || fresh
|
|
16803
|
+
if (!fresh || !awaitsDecision(fresh, decision)) return;
|
|
16644
16804
|
if (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead") {
|
|
16645
16805
|
return;
|
|
16646
16806
|
}
|
|
16647
16807
|
await this.store.updateRun(decision.runId, {
|
|
16648
16808
|
awaitingDecisionTaskId: void 0,
|
|
16649
16809
|
wakeAt: void 0,
|
|
16810
|
+
...fresh.recoveryAttempts ? {
|
|
16811
|
+
recoveryAttempts: 0
|
|
16812
|
+
} : {},
|
|
16650
16813
|
updatedAt: /* @__PURE__ */ new Date()
|
|
16651
16814
|
});
|
|
16652
16815
|
fresh.awaitingDecisionTaskId = void 0;
|
|
16816
|
+
if (fresh.recoveryAttempts) fresh.recoveryAttempts = 0;
|
|
16653
16817
|
await this.applyDecision(fresh, decision);
|
|
16654
16818
|
} finally {
|
|
16655
16819
|
const fenced = held.lease.lost;
|
|
@@ -16685,6 +16849,9 @@ ${run.workflowVersion}`;
|
|
|
16685
16849
|
this.decisionDeadlineRearms.delete(runId);
|
|
16686
16850
|
return;
|
|
16687
16851
|
}
|
|
16852
|
+
if (run.awaitingDecisionTaskId.endsWith(GRACED_TURN_SUFFIX) && !await this.clearTurnGrace(runId, run.awaitingDecisionTaskId, now)) {
|
|
16853
|
+
return;
|
|
16854
|
+
}
|
|
16688
16855
|
const halfWindow = Math.ceil(silenceMs / 2);
|
|
16689
16856
|
if (run.wakeAt != null && run.wakeAt > now + halfWindow) {
|
|
16690
16857
|
this.decisionDeadlineRearms.set(runId, run.wakeAt - halfWindow);
|
|
@@ -16701,6 +16868,29 @@ ${run.workflowVersion}`;
|
|
|
16701
16868
|
}
|
|
16702
16869
|
}
|
|
16703
16870
|
/**
|
|
16871
|
+
* Write a graced turn's marker back to its bare task id (see {@link GRACED_TURN_SUFFIX}), under the
|
|
16872
|
+
* run's lease and only while the marker is still `marker`: a decision and a re-dispatch both write
|
|
16873
|
+
* the marker under that lease, so this never puts back the id of a turn they already replaced.
|
|
16874
|
+
* False when the lease was busy or the write failed; the caller's next beat tries again.
|
|
16875
|
+
*/
|
|
16876
|
+
async clearTurnGrace(runId, marker, nowMs) {
|
|
16877
|
+
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
16878
|
+
return false;
|
|
16879
|
+
}
|
|
16880
|
+
try {
|
|
16881
|
+
const fresh = await this.store.getRun(runId);
|
|
16882
|
+
if (fresh?.awaitingDecisionTaskId === marker) {
|
|
16883
|
+
await this.store.updateRun(runId, {
|
|
16884
|
+
awaitingDecisionTaskId: marker.slice(0, -GRACED_TURN_SUFFIX.length),
|
|
16885
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
16886
|
+
});
|
|
16887
|
+
}
|
|
16888
|
+
return true;
|
|
16889
|
+
} finally {
|
|
16890
|
+
await this.store.releaseRunLock(runId);
|
|
16891
|
+
}
|
|
16892
|
+
}
|
|
16893
|
+
/**
|
|
16704
16894
|
* A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
|
|
16705
16895
|
* re-contend now instead of at their retry tick. Snapshot-and-clear — a run still blocked after the
|
|
16706
16896
|
* retry re-registers itself, and one that's gone (cancelled/admitted) is dropped. Best-effort.
|
|
@@ -19096,6 +19286,7 @@ var InMemoryRunQueue = class {
|
|
|
19096
19286
|
const waiting = lane.queued.get(run.runId);
|
|
19097
19287
|
if (waiting) {
|
|
19098
19288
|
if (run.reason === "wake") waiting.reason = "wake";
|
|
19289
|
+
if (waiting.awaiting !== run.awaiting) waiting.awaiting = void 0;
|
|
19099
19290
|
continue;
|
|
19100
19291
|
}
|
|
19101
19292
|
const entry = {
|
|
@@ -19434,6 +19625,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
19434
19625
|
resolveWireTenant,
|
|
19435
19626
|
retentionKeeps,
|
|
19436
19627
|
retiredHandoffName,
|
|
19628
|
+
retryLocalStep,
|
|
19437
19629
|
routeFor,
|
|
19438
19630
|
rowTenant,
|
|
19439
19631
|
runCompletedEvent,
|