@dudousxd/nestjs-durable-core 0.104.0 → 0.106.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +263 -71
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +102 -2
- package/dist/index.d.ts +102 -2
- package/dist/index.js +262 -71
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -679,6 +679,27 @@ function backoffDelay(attempt, options) {
|
|
|
679
679
|
return options?.jitter ? Math.round(capped * (0.5 + Math.random() * 0.5)) : capped;
|
|
680
680
|
}
|
|
681
681
|
__name(backoffDelay, "backoffDelay");
|
|
682
|
+
async function retryLocalStep(options, attempt) {
|
|
683
|
+
const maxAttempts = Math.max(1, options?.retries ?? 1);
|
|
684
|
+
for (let n = 1; ; n += 1) {
|
|
685
|
+
try {
|
|
686
|
+
return {
|
|
687
|
+
ok: true,
|
|
688
|
+
value: await attempt(n),
|
|
689
|
+
attempts: n
|
|
690
|
+
};
|
|
691
|
+
} catch (error2) {
|
|
692
|
+
if (error2 instanceof FatalError || n >= maxAttempts) return {
|
|
693
|
+
ok: false,
|
|
694
|
+
error: error2,
|
|
695
|
+
attempts: n
|
|
696
|
+
};
|
|
697
|
+
const wait = backoffDelay(n, options);
|
|
698
|
+
if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
__name(retryLocalStep, "retryLocalStep");
|
|
682
703
|
|
|
683
704
|
// src/bounded-runner.ts
|
|
684
705
|
var BoundedRunner = class {
|
|
@@ -3172,6 +3193,11 @@ var RemoteWorkflowExecutor = class {
|
|
|
3172
3193
|
};
|
|
3173
3194
|
await this.transport.dispatchWorkflowTask(task);
|
|
3174
3195
|
}
|
|
3196
|
+
/** Where the broker has the task dispatched under `taskId`, when the transport can tell. */
|
|
3197
|
+
async taskState(_run, taskId) {
|
|
3198
|
+
if (!this.transport.workflowTaskState) return "unknown";
|
|
3199
|
+
return this.transport.workflowTaskState(tenantGroup(sanitizeQueueToken(this.name), this.partition), taskId);
|
|
3200
|
+
}
|
|
3175
3201
|
};
|
|
3176
3202
|
|
|
3177
3203
|
// src/run-dispatch-batcher.ts
|
|
@@ -5908,7 +5934,6 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
5908
5934
|
});
|
|
5909
5935
|
return existing.output;
|
|
5910
5936
|
}
|
|
5911
|
-
const maxAttempts = Math.max(1, options?.retries ?? 1);
|
|
5912
5937
|
const startedAt = /* @__PURE__ */ new Date();
|
|
5913
5938
|
await host.startStep({
|
|
5914
5939
|
runId,
|
|
@@ -5919,60 +5944,59 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
5919
5944
|
enqueuedAt: startedAt,
|
|
5920
5945
|
startedAt
|
|
5921
5946
|
});
|
|
5922
|
-
|
|
5923
|
-
|
|
5924
|
-
|
|
5925
|
-
|
|
5926
|
-
|
|
5927
|
-
|
|
5928
|
-
|
|
5929
|
-
|
|
5930
|
-
|
|
5931
|
-
|
|
5932
|
-
|
|
5933
|
-
|
|
5934
|
-
|
|
5935
|
-
|
|
5936
|
-
|
|
5937
|
-
|
|
5938
|
-
|
|
5939
|
-
|
|
5940
|
-
|
|
5941
|
-
|
|
5942
|
-
|
|
5943
|
-
|
|
5944
|
-
|
|
5945
|
-
|
|
5946
|
-
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
seq: current,
|
|
5958
|
-
name,
|
|
5959
|
-
kind: "local",
|
|
5960
|
-
error: {
|
|
5961
|
-
message: err instanceof Error ? err.message : String(err),
|
|
5962
|
-
stack: err instanceof Error ? err.stack : void 0,
|
|
5963
|
-
...codeAndDetails(err)
|
|
5964
|
-
},
|
|
5965
|
-
events,
|
|
5966
|
-
attempts: attempt,
|
|
5967
|
-
enqueuedAt: startedAt,
|
|
5968
|
-
startedAt
|
|
5969
|
-
});
|
|
5970
|
-
throw err;
|
|
5971
|
-
}
|
|
5972
|
-
const wait = backoffDelay(attempt, options);
|
|
5973
|
-
if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
|
|
5974
|
-
}
|
|
5947
|
+
let events = [];
|
|
5948
|
+
const outcome = await retryLocalStep(options, async (attempt) => {
|
|
5949
|
+
const attemptEvents = [];
|
|
5950
|
+
events = attemptEvents;
|
|
5951
|
+
const invocation = {
|
|
5952
|
+
runId,
|
|
5953
|
+
workflow,
|
|
5954
|
+
stepName: name,
|
|
5955
|
+
seq: current,
|
|
5956
|
+
attempt
|
|
5957
|
+
};
|
|
5958
|
+
const body = /* @__PURE__ */ __name(() => {
|
|
5959
|
+
const logger = createStepLogger(attemptEvents, host.clock);
|
|
5960
|
+
return runInStepScope(scopeOwner, name, () => runInStepLogger(logger, () => fn(logger)));
|
|
5961
|
+
}, "body");
|
|
5962
|
+
const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
|
|
5963
|
+
await host.completeStep({
|
|
5964
|
+
runId,
|
|
5965
|
+
seq: current,
|
|
5966
|
+
name,
|
|
5967
|
+
kind: "local",
|
|
5968
|
+
output,
|
|
5969
|
+
events: attemptEvents,
|
|
5970
|
+
attempts: attempt,
|
|
5971
|
+
enqueuedAt: startedAt,
|
|
5972
|
+
startedAt
|
|
5973
|
+
});
|
|
5974
|
+
return output;
|
|
5975
|
+
});
|
|
5976
|
+
if (outcome.ok) {
|
|
5977
|
+
if (options?.compensate) compensations.push({
|
|
5978
|
+
name,
|
|
5979
|
+
fn: options.compensate
|
|
5980
|
+
});
|
|
5981
|
+
return outcome.value;
|
|
5975
5982
|
}
|
|
5983
|
+
const err = outcome.error;
|
|
5984
|
+
await host.failStep({
|
|
5985
|
+
runId,
|
|
5986
|
+
seq: current,
|
|
5987
|
+
name,
|
|
5988
|
+
kind: "local",
|
|
5989
|
+
error: {
|
|
5990
|
+
message: err instanceof Error ? err.message : String(err),
|
|
5991
|
+
stack: err instanceof Error ? err.stack : void 0,
|
|
5992
|
+
...codeAndDetails(err)
|
|
5993
|
+
},
|
|
5994
|
+
events,
|
|
5995
|
+
attempts: outcome.attempts,
|
|
5996
|
+
enqueuedAt: startedAt,
|
|
5997
|
+
startedAt
|
|
5998
|
+
});
|
|
5999
|
+
throw err;
|
|
5976
6000
|
}, "localStep");
|
|
5977
6001
|
const transaction = /* @__PURE__ */ __name(async (name, fn) => {
|
|
5978
6002
|
if (!store.transaction) {
|
|
@@ -6994,6 +7018,20 @@ function mergeTags(staticTags, runTags) {
|
|
|
6994
7018
|
}
|
|
6995
7019
|
__name(mergeTags, "mergeTags");
|
|
6996
7020
|
var BREAKPOINT = "breakpoint";
|
|
7021
|
+
var LOST_TURN_SUFFIX = ":lost";
|
|
7022
|
+
var GRACED_TURN_SUFFIX = ":grace";
|
|
7023
|
+
function awaitsDecision(run, decision) {
|
|
7024
|
+
const marker = run.awaitingDecisionTaskId;
|
|
7025
|
+
return marker !== void 0 && awaitedTaskId(marker) === decision.taskId;
|
|
7026
|
+
}
|
|
7027
|
+
__name(awaitsDecision, "awaitsDecision");
|
|
7028
|
+
function awaitedTaskId(marker) {
|
|
7029
|
+
let id = marker;
|
|
7030
|
+
if (id.endsWith(LOST_TURN_SUFFIX)) id = id.slice(0, -LOST_TURN_SUFFIX.length);
|
|
7031
|
+
if (id.endsWith(GRACED_TURN_SUFFIX)) id = id.slice(0, -GRACED_TURN_SUFFIX.length);
|
|
7032
|
+
return id;
|
|
7033
|
+
}
|
|
7034
|
+
__name(awaitedTaskId, "awaitedTaskId");
|
|
6997
7035
|
var PENDING_POLL_BATCH = 100;
|
|
6998
7036
|
var DEFAULT_STICKY_RUNS = 1e3;
|
|
6999
7037
|
var DEFAULT_STICKY_IDLE_MS = 3e5;
|
|
@@ -7263,6 +7301,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7263
7301
|
* `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
|
|
7264
7302
|
*/
|
|
7265
7303
|
ownLeaseCutoffMs;
|
|
7304
|
+
/** When this engine was built (clock ms): {@link noteLapsedTurn} counts no lost turn within one
|
|
7305
|
+
* turn window of it, while live workers' beats catch up with deadlines that went stale. */
|
|
7306
|
+
bootedAtMs;
|
|
7266
7307
|
maxRecoveryAttempts;
|
|
7267
7308
|
unregisteredWorkflowGraceMs;
|
|
7268
7309
|
remoteAdvanceSilenceMs;
|
|
@@ -7492,6 +7533,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7492
7533
|
this.namespace = poolNamespace(deps.namespace);
|
|
7493
7534
|
this.leaseMs = deps.leaseMs ?? 3e4;
|
|
7494
7535
|
this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
|
|
7536
|
+
this.bootedAtMs = this.clock();
|
|
7495
7537
|
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
7496
7538
|
const grace = deps.unregisteredWorkflowGraceMs;
|
|
7497
7539
|
this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
|
|
@@ -7869,6 +7911,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7869
7911
|
return;
|
|
7870
7912
|
}
|
|
7871
7913
|
if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
7914
|
+
if (dispatch.awaiting !== void 0 && (run.awaitingDecisionTaskId === void 0 || awaitedTaskId(run.awaitingDecisionTaskId) !== dispatch.awaiting)) {
|
|
7915
|
+
return;
|
|
7916
|
+
}
|
|
7872
7917
|
const nowMs = this.clock();
|
|
7873
7918
|
if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
|
|
7874
7919
|
const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
@@ -7920,6 +7965,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7920
7965
|
*/
|
|
7921
7966
|
async handOffDue(runs, nowMs, settled) {
|
|
7922
7967
|
const claimed = [];
|
|
7968
|
+
const awaiting = /* @__PURE__ */ new Map();
|
|
7923
7969
|
for (const run of runs) {
|
|
7924
7970
|
if (this.draining) break;
|
|
7925
7971
|
this.markClaim(run.id);
|
|
@@ -7934,16 +7980,26 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7934
7980
|
continue;
|
|
7935
7981
|
}
|
|
7936
7982
|
try {
|
|
7983
|
+
const seen = {};
|
|
7984
|
+
const handled = await this.noteLapsedTurn(run.id, nowMs, seen);
|
|
7985
|
+
if (handled) {
|
|
7986
|
+
if (handled.status === "dead") settled?.set(run.id, handled);
|
|
7987
|
+
continue;
|
|
7988
|
+
}
|
|
7937
7989
|
const patch = {
|
|
7938
7990
|
wakeAt: nowMs + this.leaseMs
|
|
7939
7991
|
};
|
|
7992
|
+
let won = true;
|
|
7940
7993
|
if (this.store.updateRunIf) {
|
|
7941
|
-
|
|
7994
|
+
won = await this.store.updateRunIf(run.id, [
|
|
7942
7995
|
run.status
|
|
7943
|
-
], patch)
|
|
7996
|
+
], patch);
|
|
7944
7997
|
} else {
|
|
7945
7998
|
await this.store.updateRun(run.id, patch);
|
|
7999
|
+
}
|
|
8000
|
+
if (won) {
|
|
7946
8001
|
claimed.push(run);
|
|
8002
|
+
if (seen.marker !== void 0) awaiting.set(run.id, awaitedTaskId(seen.marker));
|
|
7947
8003
|
}
|
|
7948
8004
|
} finally {
|
|
7949
8005
|
await this.store.releaseRunLock(run.id, this.instanceId);
|
|
@@ -7954,13 +8010,19 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7954
8010
|
}
|
|
7955
8011
|
try {
|
|
7956
8012
|
for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
|
|
7957
|
-
await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) =>
|
|
7958
|
-
|
|
7959
|
-
|
|
7960
|
-
|
|
7961
|
-
|
|
7962
|
-
|
|
7963
|
-
|
|
8013
|
+
await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => {
|
|
8014
|
+
const turn = awaiting.get(run.id);
|
|
8015
|
+
return {
|
|
8016
|
+
runId: run.id,
|
|
8017
|
+
reason: "wake",
|
|
8018
|
+
...run.priority !== void 0 ? {
|
|
8019
|
+
priority: run.priority
|
|
8020
|
+
} : {},
|
|
8021
|
+
...turn !== void 0 ? {
|
|
8022
|
+
awaiting: turn
|
|
8023
|
+
} : {}
|
|
8024
|
+
};
|
|
8025
|
+
}), {
|
|
7964
8026
|
namespace: this.namespace
|
|
7965
8027
|
});
|
|
7966
8028
|
}
|
|
@@ -11034,28 +11096,39 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
11034
11096
|
* `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
|
|
11035
11097
|
* result to skip the resume, or `undefined` to proceed.
|
|
11036
11098
|
*/
|
|
11037
|
-
async countRecovery(run) {
|
|
11099
|
+
async countRecovery(run, extra = {}) {
|
|
11038
11100
|
const attempts = (run.recoveryAttempts ?? 0) + 1;
|
|
11101
|
+
const updateIf = /* @__PURE__ */ __name((patch) => (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
11102
|
+
run.status
|
|
11103
|
+
], patch), "updateIf");
|
|
11104
|
+
const moved = /* @__PURE__ */ __name(async () => {
|
|
11105
|
+
await this.store.releaseRunLock(run.id);
|
|
11106
|
+
return this.echoCurrentStatus(run.id);
|
|
11107
|
+
}, "moved");
|
|
11039
11108
|
if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
|
|
11040
11109
|
const error2 = {
|
|
11041
11110
|
message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
|
|
11042
11111
|
code: "max_recovery_attempts"
|
|
11043
11112
|
};
|
|
11044
|
-
await
|
|
11113
|
+
const written = await updateIf(await this.withLifecycleMarker(run, {
|
|
11045
11114
|
status: "dead",
|
|
11046
11115
|
error: error2,
|
|
11047
11116
|
updatedAt: /* @__PURE__ */ new Date()
|
|
11048
11117
|
}));
|
|
11118
|
+
if (!written) return moved();
|
|
11049
11119
|
await this.store.releaseRunLock(run.id);
|
|
11050
11120
|
return this.announceDead({
|
|
11051
11121
|
...run,
|
|
11052
11122
|
recoveryAttempts: attempts
|
|
11053
11123
|
}, error2);
|
|
11054
11124
|
}
|
|
11055
|
-
await
|
|
11125
|
+
if (!await updateIf({
|
|
11126
|
+
...extra,
|
|
11056
11127
|
recoveryAttempts: attempts,
|
|
11057
11128
|
updatedAt: /* @__PURE__ */ new Date()
|
|
11058
|
-
})
|
|
11129
|
+
})) {
|
|
11130
|
+
return moved();
|
|
11131
|
+
}
|
|
11059
11132
|
return void 0;
|
|
11060
11133
|
}
|
|
11061
11134
|
/** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
|
|
@@ -11202,7 +11275,87 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
11202
11275
|
return this.resumeLeased([
|
|
11203
11276
|
...due,
|
|
11204
11277
|
...blocked
|
|
11205
|
-
], nowMs, (run) => this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
|
|
11278
|
+
], nowMs, async (run) => await this.noteLapsedTurn(run.id, nowMs) ?? this.reconcileSignalWaiter(run, waiterByRun.get(run.id)));
|
|
11279
|
+
}
|
|
11280
|
+
/**
|
|
11281
|
+
* Called under the run's lease by the due-timer pass, for a run that THIS instance would re-drive.
|
|
11282
|
+
* A run suspended awaiting a DISPATCHED workflow turn's decision (`awaitingDecisionTaskId`) whose
|
|
11283
|
+
* re-drive deadline (`wakeAt`) has passed lost that turn: no decision came back and the worker
|
|
11284
|
+
* stopped beating (every beat pushes `wakeAt` forward, see {@link rearmDecisionDeadline}), so the
|
|
11285
|
+
* worker running it died, or the task was lost with it. That is the dispatched form of a crashed
|
|
11286
|
+
* inline turn, so it spends the same {@link maxRecoveryAttempts} budget through
|
|
11287
|
+
* {@link countRecovery}: a turn that keeps killing its worker ends `dead` (with the same
|
|
11288
|
+
* `run.failed` announcement) instead of being re-dispatched, and redelivered, for ever.
|
|
11289
|
+
*
|
|
11290
|
+
* Counted at most once per awaited turn: the count rewrites the marker to its `:lost` form, which
|
|
11291
|
+
* no later pass counts again however long the re-drive waits on the run queue, and the re-drive's
|
|
11292
|
+
* dispatch replaces it with the next turn's id. Not counted, and the re-drive left for later:
|
|
11293
|
+
* - while the broker still holds the task unstarted (a backed-up queue, not a dead worker), or
|
|
11294
|
+
* - within one turn window of this engine's boot, when beats from live workers have not yet had
|
|
11295
|
+
* the chance to push a deadline that went stale while the engine was down. Once per turn (the
|
|
11296
|
+
* `:grace` marker): a lapse after that takes the normal path, whichever instance judges it.
|
|
11297
|
+
* A late decision of a counted turn is still applied while the re-drive has not dispatched yet
|
|
11298
|
+
* (see `awaitsDecision`), and gives the attempt back.
|
|
11299
|
+
* Never counted by an instance that cannot resolve the workflow (it re-drives nothing), nor for a
|
|
11300
|
+
* decision that arrives, a worker that keeps beating, or any write a healthy turn makes on its way.
|
|
11301
|
+
*
|
|
11302
|
+
* Re-reads under the lease: the listing may predate a decision applied since. Returns a result when
|
|
11303
|
+
* the run was dead-lettered or left waiting (the lease released either way), else `undefined` for
|
|
11304
|
+
* the caller to re-drive as before. A decision that lands while this holds the lease is dropped (it
|
|
11305
|
+
* cannot take the lease), whichever way this pass goes: with a run it dead-letters, or with one it
|
|
11306
|
+
* graces or leaves waiting on a `queued` task, whose next lapse then counts the turn and runs it
|
|
11307
|
+
* again. A narrow race, accepted.
|
|
11308
|
+
*
|
|
11309
|
+
* The marker grows by up to 11 characters (`:grace:lost`) beyond the dispatched id, which is
|
|
11310
|
+
* `<runId>:wf:<8 hex>`: `runId` plus 23 in all. A store that bounds the column (TypeORM's
|
|
11311
|
+
* `varchar(191)`) therefore dispatches a run whose id is 169 to 179 characters long, then refuses
|
|
11312
|
+
* the grace or lost write for it.
|
|
11313
|
+
*/
|
|
11314
|
+
async noteLapsedTurn(runId, nowMs, seen) {
|
|
11315
|
+
const run = await this.store.getRun(runId);
|
|
11316
|
+
const awaited = run?.awaitingDecisionTaskId;
|
|
11317
|
+
if (seen) seen.marker = awaited;
|
|
11318
|
+
if (!run || run.status !== "suspended" || awaited === void 0 || awaited.endsWith(LOST_TURN_SUFFIX) || run.wakeAt == null || run.wakeAt > nowMs) {
|
|
11319
|
+
return void 0;
|
|
11320
|
+
}
|
|
11321
|
+
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) return void 0;
|
|
11322
|
+
const windowMs = this.dispatchedTurnSilenceMs;
|
|
11323
|
+
if (windowMs != null && nowMs < this.bootedAtMs + windowMs && !awaited.endsWith(GRACED_TURN_SUFFIX)) {
|
|
11324
|
+
return this.leaveTurnWaiting(run, this.bootedAtMs + windowMs, {
|
|
11325
|
+
awaitingDecisionTaskId: `${awaited}${GRACED_TURN_SUFFIX}`
|
|
11326
|
+
});
|
|
11327
|
+
}
|
|
11328
|
+
const state = await this.dispatchedTurnState(run, awaitedTaskId(awaited));
|
|
11329
|
+
if (state === "queued") return this.leaveTurnWaiting(run, nowMs + (windowMs ?? this.leaseMs));
|
|
11330
|
+
if (state === "error") return void 0;
|
|
11331
|
+
return this.countRecovery(run, {
|
|
11332
|
+
awaitingDecisionTaskId: `${awaited}${LOST_TURN_SUFFIX}`
|
|
11333
|
+
});
|
|
11334
|
+
}
|
|
11335
|
+
/** Where the broker has the task of a dispatched turn: see {@link WorkflowExecutor.taskState}. */
|
|
11336
|
+
async dispatchedTurnState(run, taskId) {
|
|
11337
|
+
const executor = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.remote?.executor;
|
|
11338
|
+
if (!executor?.taskState) return "unknown";
|
|
11339
|
+
try {
|
|
11340
|
+
return await executor.taskState(run, taskId);
|
|
11341
|
+
} catch {
|
|
11342
|
+
return "error";
|
|
11343
|
+
}
|
|
11344
|
+
}
|
|
11345
|
+
/** Push a lapsed turn's deadline to `until` without re-driving it, and give the lease back. */
|
|
11346
|
+
async leaveTurnWaiting(run, until, extra = {}) {
|
|
11347
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
11348
|
+
"suspended"
|
|
11349
|
+
], {
|
|
11350
|
+
...extra,
|
|
11351
|
+
wakeAt: until,
|
|
11352
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
11353
|
+
});
|
|
11354
|
+
await this.store.releaseRunLock(run.id);
|
|
11355
|
+
return {
|
|
11356
|
+
runId: run.id,
|
|
11357
|
+
status: "suspended"
|
|
11358
|
+
};
|
|
11206
11359
|
}
|
|
11207
11360
|
/**
|
|
11208
11361
|
* The reconcile safety net's per-run check (see {@link resumeDueTimers}): if this suspended run has
|
|
@@ -13504,6 +13657,12 @@ ${run.workflowVersion}`;
|
|
|
13504
13657
|
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
|
|
13505
13658
|
status: "suspended",
|
|
13506
13659
|
wakeAt: this.reconcileWakeAt(outcome.wakeAt),
|
|
13660
|
+
// A turn that ran here suspended the run, so no dispatched turn is awaited any more: a marker
|
|
13661
|
+
// left by an earlier dispatch (a re-drive that followed an onError handoff, say) would make
|
|
13662
|
+
// this suspend read as a lost turn at its next reconcile wake.
|
|
13663
|
+
...run.awaitingDecisionTaskId !== void 0 ? {
|
|
13664
|
+
awaitingDecisionTaskId: void 0
|
|
13665
|
+
} : {},
|
|
13507
13666
|
updatedAt: stamp
|
|
13508
13667
|
}));
|
|
13509
13668
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
@@ -14101,7 +14260,7 @@ ${run.workflowVersion}`;
|
|
|
14101
14260
|
output: cmd.output,
|
|
14102
14261
|
error: cmd.error,
|
|
14103
14262
|
events: cmd.events,
|
|
14104
|
-
attempts: 1,
|
|
14263
|
+
attempts: cmd.attempts ?? 1,
|
|
14105
14264
|
enqueuedAt: startedAt,
|
|
14106
14265
|
startedAt,
|
|
14107
14266
|
finishedAt,
|
|
@@ -16326,7 +16485,7 @@ ${run.workflowVersion}`;
|
|
|
16326
16485
|
*/
|
|
16327
16486
|
async completeRemoteDecision(decision) {
|
|
16328
16487
|
const run = await this.store.getRun(decision.runId);
|
|
16329
|
-
if (!run || run
|
|
16488
|
+
if (!run || !awaitsDecision(run, decision)) return;
|
|
16330
16489
|
if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
|
|
16331
16490
|
if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
|
|
16332
16491
|
return;
|
|
@@ -16338,16 +16497,20 @@ ${run.workflowVersion}`;
|
|
|
16338
16497
|
const held = this.holdExecutionLease(decision.runId);
|
|
16339
16498
|
try {
|
|
16340
16499
|
const fresh = await this.store.getRun(decision.runId);
|
|
16341
|
-
if (!fresh || fresh
|
|
16500
|
+
if (!fresh || !awaitsDecision(fresh, decision)) return;
|
|
16342
16501
|
if (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead") {
|
|
16343
16502
|
return;
|
|
16344
16503
|
}
|
|
16345
16504
|
await this.store.updateRun(decision.runId, {
|
|
16346
16505
|
awaitingDecisionTaskId: void 0,
|
|
16347
16506
|
wakeAt: void 0,
|
|
16507
|
+
...fresh.recoveryAttempts ? {
|
|
16508
|
+
recoveryAttempts: 0
|
|
16509
|
+
} : {},
|
|
16348
16510
|
updatedAt: /* @__PURE__ */ new Date()
|
|
16349
16511
|
});
|
|
16350
16512
|
fresh.awaitingDecisionTaskId = void 0;
|
|
16513
|
+
if (fresh.recoveryAttempts) fresh.recoveryAttempts = 0;
|
|
16351
16514
|
await this.applyDecision(fresh, decision);
|
|
16352
16515
|
} finally {
|
|
16353
16516
|
const fenced = held.lease.lost;
|
|
@@ -16383,6 +16546,9 @@ ${run.workflowVersion}`;
|
|
|
16383
16546
|
this.decisionDeadlineRearms.delete(runId);
|
|
16384
16547
|
return;
|
|
16385
16548
|
}
|
|
16549
|
+
if (run.awaitingDecisionTaskId.endsWith(GRACED_TURN_SUFFIX) && !await this.clearTurnGrace(runId, run.awaitingDecisionTaskId, now)) {
|
|
16550
|
+
return;
|
|
16551
|
+
}
|
|
16386
16552
|
const halfWindow = Math.ceil(silenceMs / 2);
|
|
16387
16553
|
if (run.wakeAt != null && run.wakeAt > now + halfWindow) {
|
|
16388
16554
|
this.decisionDeadlineRearms.set(runId, run.wakeAt - halfWindow);
|
|
@@ -16399,6 +16565,29 @@ ${run.workflowVersion}`;
|
|
|
16399
16565
|
}
|
|
16400
16566
|
}
|
|
16401
16567
|
/**
|
|
16568
|
+
* Write a graced turn's marker back to its bare task id (see {@link GRACED_TURN_SUFFIX}), under the
|
|
16569
|
+
* run's lease and only while the marker is still `marker`: a decision and a re-dispatch both write
|
|
16570
|
+
* the marker under that lease, so this never puts back the id of a turn they already replaced.
|
|
16571
|
+
* False when the lease was busy or the write failed; the caller's next beat tries again.
|
|
16572
|
+
*/
|
|
16573
|
+
async clearTurnGrace(runId, marker, nowMs) {
|
|
16574
|
+
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
16575
|
+
return false;
|
|
16576
|
+
}
|
|
16577
|
+
try {
|
|
16578
|
+
const fresh = await this.store.getRun(runId);
|
|
16579
|
+
if (fresh?.awaitingDecisionTaskId === marker) {
|
|
16580
|
+
await this.store.updateRun(runId, {
|
|
16581
|
+
awaitingDecisionTaskId: marker.slice(0, -GRACED_TURN_SUFFIX.length),
|
|
16582
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
16583
|
+
});
|
|
16584
|
+
}
|
|
16585
|
+
return true;
|
|
16586
|
+
} finally {
|
|
16587
|
+
await this.store.releaseRunLock(runId);
|
|
16588
|
+
}
|
|
16589
|
+
}
|
|
16590
|
+
/**
|
|
16402
16591
|
* A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
|
|
16403
16592
|
* re-contend now instead of at their retry tick. Snapshot-and-clear — a run still blocked after the
|
|
16404
16593
|
* retry re-registers itself, and one that's gone (cancelled/admitted) is dropped. Best-effort.
|
|
@@ -18794,6 +18983,7 @@ var InMemoryRunQueue = class {
|
|
|
18794
18983
|
const waiting = lane.queued.get(run.runId);
|
|
18795
18984
|
if (waiting) {
|
|
18796
18985
|
if (run.reason === "wake") waiting.reason = "wake";
|
|
18986
|
+
if (waiting.awaiting !== run.awaiting) waiting.awaiting = void 0;
|
|
18797
18987
|
continue;
|
|
18798
18988
|
}
|
|
18799
18989
|
const entry = {
|
|
@@ -19131,6 +19321,7 @@ export {
|
|
|
19131
19321
|
resolveWireTenant,
|
|
19132
19322
|
retentionKeeps,
|
|
19133
19323
|
retiredHandoffName,
|
|
19324
|
+
retryLocalStep,
|
|
19134
19325
|
routeFor,
|
|
19135
19326
|
rowTenant,
|
|
19136
19327
|
runCompletedEvent,
|