@dudousxd/nestjs-durable-core 0.102.0 → 0.103.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +459 -36
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +168 -1
- package/dist/index.d.ts +168 -1
- package/dist/index.js +455 -36
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -881,6 +881,14 @@ function isNonEmptyRunScope(scope) {
|
|
|
881
881
|
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
882
882
|
}
|
|
883
883
|
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
884
|
+
var REMOTE_WORKFLOW_FEATURES = [
|
|
885
|
+
"callPolicy",
|
|
886
|
+
"sleepUntil",
|
|
887
|
+
"waitSignalTimeout",
|
|
888
|
+
"waitEvent",
|
|
889
|
+
"spawnChild",
|
|
890
|
+
"continueAsNew"
|
|
891
|
+
];
|
|
884
892
|
var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
|
|
885
893
|
|
|
886
894
|
// src/events.ts
|
|
@@ -3069,6 +3077,35 @@ function codeAndDetails(err) {
|
|
|
3069
3077
|
}
|
|
3070
3078
|
__name(codeAndDetails, "codeAndDetails");
|
|
3071
3079
|
|
|
3080
|
+
// src/remote-call-policy.ts
|
|
3081
|
+
function remoteCallPolicyOf(config, opts) {
|
|
3082
|
+
const policy = {};
|
|
3083
|
+
const retries = opts?.retries ?? config?.retries;
|
|
3084
|
+
if (retries !== void 0) policy.retries = retries;
|
|
3085
|
+
const backoff = opts?.backoff ?? config?.backoff;
|
|
3086
|
+
if (backoff !== void 0) policy.backoff = backoff;
|
|
3087
|
+
const backoffMs = opts?.backoffMs ?? config?.backoffMs;
|
|
3088
|
+
if (backoffMs !== void 0) policy.backoffMs = backoffMs;
|
|
3089
|
+
const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
|
|
3090
|
+
if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
|
|
3091
|
+
const jitter = opts?.jitter ?? config?.jitter;
|
|
3092
|
+
if (jitter !== void 0) policy.jitter = jitter;
|
|
3093
|
+
const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
|
|
3094
|
+
if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
|
|
3095
|
+
return Object.keys(policy).length > 0 ? policy : void 0;
|
|
3096
|
+
}
|
|
3097
|
+
__name(remoteCallPolicyOf, "remoteCallPolicyOf");
|
|
3098
|
+
function remoteCallMaxAttempts(policy) {
|
|
3099
|
+
return Math.max(1, policy?.retries ?? 1);
|
|
3100
|
+
}
|
|
3101
|
+
__name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
|
|
3102
|
+
function remoteCallRetries(policy, attempts, error2) {
|
|
3103
|
+
if (policy === void 0 || policy.timeoutMs) return false;
|
|
3104
|
+
if (error2?.retryable === false) return false;
|
|
3105
|
+
return attempts < remoteCallMaxAttempts(policy);
|
|
3106
|
+
}
|
|
3107
|
+
__name(remoteCallRetries, "remoteCallRetries");
|
|
3108
|
+
|
|
3072
3109
|
// src/remote-event.ts
|
|
3073
3110
|
var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
|
|
3074
3111
|
function markRemoteEvent(event) {
|
|
@@ -3126,7 +3163,12 @@ var RemoteWorkflowExecutor = class {
|
|
|
3126
3163
|
} : {},
|
|
3127
3164
|
group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
|
|
3128
3165
|
priority: run.priority,
|
|
3129
|
-
attempt: 1
|
|
3166
|
+
attempt: 1,
|
|
3167
|
+
// What this control plane applies beyond the original command kinds, so a worker fails a turn
|
|
3168
|
+
// that needs more instead of emitting a command this engine would not understand.
|
|
3169
|
+
features: [
|
|
3170
|
+
...REMOTE_WORKFLOW_FEATURES
|
|
3171
|
+
]
|
|
3130
3172
|
};
|
|
3131
3173
|
await this.transport.dispatchWorkflowTask(task);
|
|
3132
3174
|
}
|
|
@@ -13453,7 +13495,7 @@ ${run.workflowVersion}`;
|
|
|
13453
13495
|
let wfPlan = await this.planRoute(wfToken, registered.requires);
|
|
13454
13496
|
if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
|
|
13455
13497
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
13456
|
-
const checkpoints = await this.replayCheckpoints(run.id);
|
|
13498
|
+
const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
|
|
13457
13499
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
13458
13500
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
13459
13501
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -13539,6 +13581,10 @@ ${run.workflowVersion}`;
|
|
|
13539
13581
|
*/
|
|
13540
13582
|
async applyDecision(run, decision) {
|
|
13541
13583
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
13584
|
+
const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
|
|
13585
|
+
if (handoff !== void 0 && decision.status === "completed") {
|
|
13586
|
+
return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
|
|
13587
|
+
}
|
|
13542
13588
|
if (decision.status === "completed") {
|
|
13543
13589
|
await this.applyCommands(run, decision.commands);
|
|
13544
13590
|
return this.settleRun(run, {
|
|
@@ -13585,6 +13631,63 @@ ${run.workflowVersion}`;
|
|
|
13585
13631
|
return suspended;
|
|
13586
13632
|
}
|
|
13587
13633
|
/**
|
|
13634
|
+
* A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
|
|
13635
|
+
* run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
|
|
13636
|
+
* engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
|
|
13637
|
+
* turn replayed stays cancelled and starts nothing.
|
|
13638
|
+
*/
|
|
13639
|
+
async applyRemoteContinueAsNew(run, commands, input) {
|
|
13640
|
+
await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
|
|
13641
|
+
const fresh = await this.store.getRun(run.id);
|
|
13642
|
+
if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
|
|
13643
|
+
return {
|
|
13644
|
+
runId: run.id,
|
|
13645
|
+
status: fresh.status,
|
|
13646
|
+
output: fresh.output,
|
|
13647
|
+
error: fresh.error
|
|
13648
|
+
};
|
|
13649
|
+
}
|
|
13650
|
+
const nextId = nextContinuationId(run.id);
|
|
13651
|
+
try {
|
|
13652
|
+
await this.start(run.workflow, input, nextId, {
|
|
13653
|
+
namespace: run.namespace,
|
|
13654
|
+
...inheritTenant(run)
|
|
13655
|
+
});
|
|
13656
|
+
} catch (err) {
|
|
13657
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13658
|
+
return this.settleRun(run, {
|
|
13659
|
+
kind: "failed",
|
|
13660
|
+
error: {
|
|
13661
|
+
message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
|
|
13662
|
+
code: "continue_as_new_failed"
|
|
13663
|
+
}
|
|
13664
|
+
});
|
|
13665
|
+
}
|
|
13666
|
+
await this.store.updateRun(run.id, {
|
|
13667
|
+
status: "completed",
|
|
13668
|
+
output: void 0,
|
|
13669
|
+
error: void 0,
|
|
13670
|
+
wakeAt: void 0,
|
|
13671
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
13672
|
+
});
|
|
13673
|
+
this.emit({
|
|
13674
|
+
type: "run.completed",
|
|
13675
|
+
runId: run.id,
|
|
13676
|
+
workflow: run.workflow,
|
|
13677
|
+
namespace: run.namespace
|
|
13678
|
+
}, {
|
|
13679
|
+
lifecycle: false
|
|
13680
|
+
});
|
|
13681
|
+
void this.notifyParent(run.id, {
|
|
13682
|
+
ok: true,
|
|
13683
|
+
value: void 0
|
|
13684
|
+
});
|
|
13685
|
+
return {
|
|
13686
|
+
runId: run.id,
|
|
13687
|
+
status: "completed"
|
|
13688
|
+
};
|
|
13689
|
+
}
|
|
13690
|
+
/**
|
|
13588
13691
|
* Re-drive a run whose turn was computed from a history OLDER than the store's.
|
|
13589
13692
|
*
|
|
13590
13693
|
* A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
|
|
@@ -13602,11 +13705,12 @@ ${run.workflowVersion}`;
|
|
|
13602
13705
|
* Scheduled off the current stack so it runs after the caller released the run lease.
|
|
13603
13706
|
*/
|
|
13604
13707
|
async redriveIfTurnSawStaleHistory(runId, commands) {
|
|
13605
|
-
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
|
|
13708
|
+
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
|
|
13606
13709
|
if (blocking.length === 0) return;
|
|
13607
13710
|
for (const cmd of blocking) {
|
|
13608
13711
|
const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
|
|
13609
|
-
|
|
13712
|
+
const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
|
|
13713
|
+
if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
|
|
13610
13714
|
setTimeout(() => {
|
|
13611
13715
|
void this.resumeImplicit(runId).catch(() => void 0);
|
|
13612
13716
|
}, 0).unref?.();
|
|
@@ -13855,7 +13959,12 @@ ${run.workflowVersion}`;
|
|
|
13855
13959
|
// name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
|
|
13856
13960
|
name: cp.kind === "signal" ? void 0 : cp.name,
|
|
13857
13961
|
output: cp.status === "completed" ? cp.output : void 0,
|
|
13858
|
-
error: cp.status === "failed" ? cp.error : void 0
|
|
13962
|
+
error: cp.status === "failed" ? cp.error : void 0,
|
|
13963
|
+
// How many times a dispatched step ran: a worker applying a `call` policy decides a retry
|
|
13964
|
+
// from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
|
|
13965
|
+
...cp.kind === "remote" ? {
|
|
13966
|
+
attempts: cp.attempts
|
|
13967
|
+
} : {}
|
|
13859
13968
|
});
|
|
13860
13969
|
} else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
|
|
13861
13970
|
events.push({
|
|
@@ -13912,11 +14021,12 @@ ${run.workflowVersion}`;
|
|
|
13912
14021
|
} else if (cmd.kind === "call") {
|
|
13913
14022
|
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
13914
14023
|
if (existing) {
|
|
13915
|
-
const
|
|
14024
|
+
const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
|
|
14025
|
+
const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
|
|
13916
14026
|
if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
|
|
13917
14027
|
continue;
|
|
13918
14028
|
}
|
|
13919
|
-
const lease = this.
|
|
14029
|
+
const lease = this.callLease(cmd.policy);
|
|
13920
14030
|
await this.store.saveCheckpoint(stepCheckpoint({
|
|
13921
14031
|
runId: run.id,
|
|
13922
14032
|
seq: cmd.seq,
|
|
@@ -13976,35 +14086,7 @@ ${run.workflowVersion}`;
|
|
|
13976
14086
|
}));
|
|
13977
14087
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
13978
14088
|
} else if (cmd.kind === "waitSignal") {
|
|
13979
|
-
|
|
13980
|
-
await this.store.saveCheckpoint(instantCheckpoint({
|
|
13981
|
-
runId: run.id,
|
|
13982
|
-
seq: cmd.seq,
|
|
13983
|
-
name: `signal:${cmd.signal}`,
|
|
13984
|
-
kind: "signal",
|
|
13985
|
-
output: payload
|
|
13986
|
-
}));
|
|
13987
|
-
setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
|
|
13988
|
-
}, "deliverBuffered");
|
|
13989
|
-
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
13990
|
-
if (buffered) {
|
|
13991
|
-
await deliverBuffered(buffered.payload);
|
|
13992
|
-
} else {
|
|
13993
|
-
await this.store.putSignalWaiter({
|
|
13994
|
-
token: cmd.signal,
|
|
13995
|
-
runId: run.id,
|
|
13996
|
-
seq: cmd.seq
|
|
13997
|
-
});
|
|
13998
|
-
const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
13999
|
-
if (lateBuffered) {
|
|
14000
|
-
await this.store.removeSignalWaiter({
|
|
14001
|
-
token: cmd.signal,
|
|
14002
|
-
runId: run.id,
|
|
14003
|
-
seq: cmd.seq
|
|
14004
|
-
});
|
|
14005
|
-
await deliverBuffered(lateBuffered.payload);
|
|
14006
|
-
}
|
|
14007
|
-
}
|
|
14089
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14008
14090
|
} else if (cmd.kind === "waitAny") {
|
|
14009
14091
|
const state = await enterWaitAny({
|
|
14010
14092
|
store: this.store,
|
|
@@ -14036,6 +14118,100 @@ ${run.workflowVersion}`;
|
|
|
14036
14118
|
} else {
|
|
14037
14119
|
this.redeliverFinishedChild(childId, existingChild);
|
|
14038
14120
|
}
|
|
14121
|
+
} else if (cmd.kind === "sleepUntil") {
|
|
14122
|
+
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
14123
|
+
const deadline = existing?.wakeAt ?? cmd.until;
|
|
14124
|
+
if (!existing) {
|
|
14125
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14126
|
+
runId: run.id,
|
|
14127
|
+
seq: cmd.seq,
|
|
14128
|
+
name: `sleep:${cmd.seq}`,
|
|
14129
|
+
kind: "sleep",
|
|
14130
|
+
status: "pending",
|
|
14131
|
+
attempts: 1,
|
|
14132
|
+
wakeAt: deadline,
|
|
14133
|
+
enqueuedAt: at,
|
|
14134
|
+
startedAt: at,
|
|
14135
|
+
finishedAt: at
|
|
14136
|
+
}));
|
|
14137
|
+
}
|
|
14138
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14139
|
+
} else if (cmd.kind === "waitSignalTimeout") {
|
|
14140
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14141
|
+
const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
|
|
14142
|
+
waitSeq: cmd.seq,
|
|
14143
|
+
token: cmd.signal
|
|
14144
|
+
});
|
|
14145
|
+
if (deadline <= this.clock()) {
|
|
14146
|
+
this.resumeSoon(run.id);
|
|
14147
|
+
continue;
|
|
14148
|
+
}
|
|
14149
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14150
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14151
|
+
} else if (cmd.kind === "waitEvent") {
|
|
14152
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14153
|
+
const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
|
|
14154
|
+
let deadline;
|
|
14155
|
+
if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
|
|
14156
|
+
deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
|
|
14157
|
+
waitSeq: cmd.seq,
|
|
14158
|
+
token
|
|
14159
|
+
});
|
|
14160
|
+
if (deadline <= this.clock()) {
|
|
14161
|
+
this.resumeSoon(run.id);
|
|
14162
|
+
continue;
|
|
14163
|
+
}
|
|
14164
|
+
}
|
|
14165
|
+
await this.store.putSignalWaiter({
|
|
14166
|
+
token,
|
|
14167
|
+
runId: run.id,
|
|
14168
|
+
seq: cmd.seq
|
|
14169
|
+
});
|
|
14170
|
+
const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
|
|
14171
|
+
if (hit) {
|
|
14172
|
+
await this.store.removeSignalWaiter({
|
|
14173
|
+
token,
|
|
14174
|
+
runId: run.id,
|
|
14175
|
+
seq: cmd.seq
|
|
14176
|
+
});
|
|
14177
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14178
|
+
runId: run.id,
|
|
14179
|
+
seq: cmd.seq,
|
|
14180
|
+
name: `signal:${token}`,
|
|
14181
|
+
kind: "signal",
|
|
14182
|
+
output: hit.payload
|
|
14183
|
+
}));
|
|
14184
|
+
this.resumeSoon(run.id);
|
|
14185
|
+
} else if (deadline != null) {
|
|
14186
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14187
|
+
}
|
|
14188
|
+
} else if (cmd.kind === "spawnChild") {
|
|
14189
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14190
|
+
if (!await this.store.getRun(cmd.childId)) {
|
|
14191
|
+
try {
|
|
14192
|
+
await this.start(cmd.workflow, cmd.input, cmd.childId, {
|
|
14193
|
+
namespace: run.namespace,
|
|
14194
|
+
...inheritTenant(run),
|
|
14195
|
+
...cmd.priority !== void 0 ? {
|
|
14196
|
+
priority: cmd.priority
|
|
14197
|
+
} : {}
|
|
14198
|
+
});
|
|
14199
|
+
} catch (err) {
|
|
14200
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14201
|
+
this.notifyParent(cmd.childId, {
|
|
14202
|
+
ok: false,
|
|
14203
|
+
error: `child workflow "${cmd.workflow}" failed to start: ${message}`
|
|
14204
|
+
});
|
|
14205
|
+
}
|
|
14206
|
+
}
|
|
14207
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14208
|
+
runId: run.id,
|
|
14209
|
+
seq: cmd.seq,
|
|
14210
|
+
name: `spawn:${cmd.childId}`,
|
|
14211
|
+
kind: "local",
|
|
14212
|
+
output: cmd.childId
|
|
14213
|
+
}));
|
|
14214
|
+
} else if (cmd.kind === "continueAsNew") {
|
|
14039
14215
|
} else {
|
|
14040
14216
|
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
14041
14217
|
}
|
|
@@ -14043,6 +14219,245 @@ ${run.workflowVersion}`;
|
|
|
14043
14219
|
return wakeAt;
|
|
14044
14220
|
}
|
|
14045
14221
|
/**
|
|
14222
|
+
* Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
|
|
14223
|
+
* the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
|
|
14224
|
+
* (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
|
|
14225
|
+
* re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
|
|
14226
|
+
* bails).
|
|
14227
|
+
*/
|
|
14228
|
+
async parkRemoteOnSignal(run, seq, token) {
|
|
14229
|
+
const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
|
|
14230
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14231
|
+
runId: run.id,
|
|
14232
|
+
seq,
|
|
14233
|
+
name: `signal:${token}`,
|
|
14234
|
+
kind: "signal",
|
|
14235
|
+
output: payload
|
|
14236
|
+
}));
|
|
14237
|
+
this.resumeSoon(run.id);
|
|
14238
|
+
}, "deliverBuffered");
|
|
14239
|
+
const buffered = await this.store.takeBufferedSignal(token);
|
|
14240
|
+
if (buffered) {
|
|
14241
|
+
await deliverBuffered(buffered.payload);
|
|
14242
|
+
} else {
|
|
14243
|
+
await this.store.putSignalWaiter({
|
|
14244
|
+
token,
|
|
14245
|
+
runId: run.id,
|
|
14246
|
+
seq
|
|
14247
|
+
});
|
|
14248
|
+
const lateBuffered = await this.store.takeBufferedSignal(token);
|
|
14249
|
+
if (lateBuffered) {
|
|
14250
|
+
await this.store.removeSignalWaiter({
|
|
14251
|
+
token,
|
|
14252
|
+
runId: run.id,
|
|
14253
|
+
seq
|
|
14254
|
+
});
|
|
14255
|
+
await deliverBuffered(lateBuffered.payload);
|
|
14256
|
+
}
|
|
14257
|
+
}
|
|
14258
|
+
}
|
|
14259
|
+
/** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
|
|
14260
|
+
resumeSoon(runId) {
|
|
14261
|
+
setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
|
|
14262
|
+
}
|
|
14263
|
+
/**
|
|
14264
|
+
* Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
|
|
14265
|
+
* `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
|
|
14266
|
+
* back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
|
|
14267
|
+
* deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
|
|
14268
|
+
*/
|
|
14269
|
+
async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
|
|
14270
|
+
const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
|
|
14271
|
+
if (recorded?.wakeAt != null) return recorded.wakeAt;
|
|
14272
|
+
const deadline = this.clock() + timeoutMs;
|
|
14273
|
+
const at = /* @__PURE__ */ new Date();
|
|
14274
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14275
|
+
runId,
|
|
14276
|
+
seq: deadlineSeq,
|
|
14277
|
+
name: `timeout:${label}`,
|
|
14278
|
+
kind: "sleep",
|
|
14279
|
+
status: "pending",
|
|
14280
|
+
attempts: 1,
|
|
14281
|
+
input: wait,
|
|
14282
|
+
wakeAt: deadline,
|
|
14283
|
+
enqueuedAt: at,
|
|
14284
|
+
startedAt: at,
|
|
14285
|
+
finishedAt: at
|
|
14286
|
+
}));
|
|
14287
|
+
return deadline;
|
|
14288
|
+
}
|
|
14289
|
+
/**
|
|
14290
|
+
* Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
|
|
14291
|
+
* (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
|
|
14292
|
+
* time out — the in-process wait removes its waiter the same way when it finds the deadline past.
|
|
14293
|
+
* A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
|
|
14294
|
+
* worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
|
|
14295
|
+
*/
|
|
14296
|
+
async expireRemoteDeadlines(runId, checkpoints) {
|
|
14297
|
+
const now = this.clock();
|
|
14298
|
+
const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
|
|
14299
|
+
let out = checkpoints;
|
|
14300
|
+
for (const cp of checkpoints) {
|
|
14301
|
+
if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
|
|
14302
|
+
if (cp.wakeAt == null || cp.wakeAt > now) continue;
|
|
14303
|
+
const wait = cp.input;
|
|
14304
|
+
if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
|
|
14305
|
+
if (settled.has(wait.waitSeq)) continue;
|
|
14306
|
+
await this.store.removeSignalWaiter({
|
|
14307
|
+
token: wait.token,
|
|
14308
|
+
runId,
|
|
14309
|
+
seq: wait.waitSeq
|
|
14310
|
+
});
|
|
14311
|
+
const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
|
|
14312
|
+
if (landed && landed.status === "completed") {
|
|
14313
|
+
out = [
|
|
14314
|
+
...out.filter((c) => c.seq !== landed.seq),
|
|
14315
|
+
landed
|
|
14316
|
+
].sort((a, b) => a.seq - b.seq);
|
|
14317
|
+
}
|
|
14318
|
+
}
|
|
14319
|
+
return out;
|
|
14320
|
+
}
|
|
14321
|
+
/**
|
|
14322
|
+
* Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
|
|
14323
|
+
* see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
|
|
14324
|
+
* candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
|
|
14325
|
+
* {@link eventBufferTtlMs} is set, like the reconcile sweep.
|
|
14326
|
+
*/
|
|
14327
|
+
async claimBufferedEvent(run, name, match) {
|
|
14328
|
+
const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
|
|
14329
|
+
const nowMs = this.clock();
|
|
14330
|
+
for (const candidate of candidates) {
|
|
14331
|
+
if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
|
|
14332
|
+
await this.store.removeBufferedEvent(candidate.id);
|
|
14333
|
+
continue;
|
|
14334
|
+
}
|
|
14335
|
+
const { payload, tenantIds } = readBufferedEvent(candidate.payload);
|
|
14336
|
+
if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
|
|
14337
|
+
if (!eventMatches(payload, match)) continue;
|
|
14338
|
+
if (!await this.store.removeBufferedEvent(candidate.id)) continue;
|
|
14339
|
+
return {
|
|
14340
|
+
payload
|
|
14341
|
+
};
|
|
14342
|
+
}
|
|
14343
|
+
return void 0;
|
|
14344
|
+
}
|
|
14345
|
+
/** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
|
|
14346
|
+
* sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
|
|
14347
|
+
callLease(policy) {
|
|
14348
|
+
if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
|
|
14349
|
+
return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
14350
|
+
}
|
|
14351
|
+
/**
|
|
14352
|
+
* A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
|
|
14353
|
+
* checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
|
|
14354
|
+
* callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
|
|
14355
|
+
* behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
|
|
14356
|
+
* timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
|
|
14357
|
+
* and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
|
|
14358
|
+
*
|
|
14359
|
+
* - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
|
|
14360
|
+
* the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
|
|
14361
|
+
* dispatched again under the same `stepId` with `attempts + 1`.
|
|
14362
|
+
* - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
|
|
14363
|
+
* re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
|
|
14364
|
+
* workflow can catch, as `RemoteStepTimeout` would surface in-process.
|
|
14365
|
+
*/
|
|
14366
|
+
async applyCallPolicy(run, cmd, cp) {
|
|
14367
|
+
const policy = cmd.policy;
|
|
14368
|
+
if (cp.kind !== "remote") return null;
|
|
14369
|
+
if (cp.status === "failed") {
|
|
14370
|
+
if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
|
|
14371
|
+
let due = cp.wakeAt;
|
|
14372
|
+
if (due == null) {
|
|
14373
|
+
due = this.clock() + backoffDelay(cp.attempts, policy);
|
|
14374
|
+
if (due > this.clock()) {
|
|
14375
|
+
await this.store.saveCheckpoint({
|
|
14376
|
+
...cp,
|
|
14377
|
+
wakeAt: due
|
|
14378
|
+
});
|
|
14379
|
+
return due;
|
|
14380
|
+
}
|
|
14381
|
+
}
|
|
14382
|
+
if (this.clock() < due) return due;
|
|
14383
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
|
|
14384
|
+
}
|
|
14385
|
+
if (cp.status !== "pending" || !policy.timeoutMs) return null;
|
|
14386
|
+
if (cp.wakeAt == null) {
|
|
14387
|
+
const lease = this.clock() + policy.timeoutMs;
|
|
14388
|
+
await this.store.saveCheckpoint({
|
|
14389
|
+
...cp,
|
|
14390
|
+
wakeAt: lease
|
|
14391
|
+
});
|
|
14392
|
+
return this.stepLeaseWakeAt(lease);
|
|
14393
|
+
}
|
|
14394
|
+
if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
|
|
14395
|
+
if (cp.attempts < remoteCallMaxAttempts(policy)) {
|
|
14396
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
|
|
14397
|
+
}
|
|
14398
|
+
const id = cp.stepId ?? stepId(run.id, cp.seq);
|
|
14399
|
+
const error2 = {
|
|
14400
|
+
message: new RemoteStepTimeout(id, policy.timeoutMs).message,
|
|
14401
|
+
code: "remote_step_timeout"
|
|
14402
|
+
};
|
|
14403
|
+
await this.store.saveCheckpoint({
|
|
14404
|
+
...cp,
|
|
14405
|
+
status: "failed",
|
|
14406
|
+
error: error2,
|
|
14407
|
+
wakeAt: void 0,
|
|
14408
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
14409
|
+
});
|
|
14410
|
+
this.emit({
|
|
14411
|
+
type: "step.failed",
|
|
14412
|
+
runId: run.id,
|
|
14413
|
+
seq: cp.seq,
|
|
14414
|
+
name: cp.name,
|
|
14415
|
+
kind: "remote",
|
|
14416
|
+
error: error2
|
|
14417
|
+
});
|
|
14418
|
+
return void 0;
|
|
14419
|
+
}
|
|
14420
|
+
/** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
|
|
14421
|
+
async redispatchCall(run, cmd, cp, attempt, policy, why) {
|
|
14422
|
+
const lease = this.callLease(policy);
|
|
14423
|
+
const at = /* @__PURE__ */ new Date();
|
|
14424
|
+
await this.store.saveCheckpoint({
|
|
14425
|
+
...cp,
|
|
14426
|
+
status: "pending",
|
|
14427
|
+
output: void 0,
|
|
14428
|
+
error: void 0,
|
|
14429
|
+
attempts: attempt,
|
|
14430
|
+
wakeAt: lease,
|
|
14431
|
+
enqueuedAt: at,
|
|
14432
|
+
startedAt: at,
|
|
14433
|
+
finishedAt: at,
|
|
14434
|
+
events: why === "timeout" ? [
|
|
14435
|
+
...cp.events ?? [],
|
|
14436
|
+
lostStepEvent(this.clock(), attempt, "redispatched")
|
|
14437
|
+
] : cp.events
|
|
14438
|
+
});
|
|
14439
|
+
await this.dispatchRemoteTask({
|
|
14440
|
+
runId: run.id,
|
|
14441
|
+
seq: cp.seq,
|
|
14442
|
+
name: cmd.name,
|
|
14443
|
+
stepId: cp.stepId ?? stepId(run.id, cp.seq),
|
|
14444
|
+
group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
|
|
14445
|
+
input: cp.input,
|
|
14446
|
+
attempt
|
|
14447
|
+
});
|
|
14448
|
+
this.emit({
|
|
14449
|
+
type: "step.started",
|
|
14450
|
+
runId: run.id,
|
|
14451
|
+
seq: cp.seq,
|
|
14452
|
+
name: cmd.name,
|
|
14453
|
+
kind: "remote",
|
|
14454
|
+
...why === "timeout" ? {
|
|
14455
|
+
redispatched: true
|
|
14456
|
+
} : {}
|
|
14457
|
+
});
|
|
14458
|
+
return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
|
|
14459
|
+
}
|
|
14460
|
+
/**
|
|
14046
14461
|
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
14047
14462
|
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
14048
14463
|
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
@@ -18443,6 +18858,7 @@ export {
|
|
|
18443
18858
|
PARENT_CLOSE_TAG_PREFIX,
|
|
18444
18859
|
QueueController,
|
|
18445
18860
|
REMOTE_EVENT,
|
|
18861
|
+
REMOTE_WORKFLOW_FEATURES,
|
|
18446
18862
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
18447
18863
|
RUN_VALUE_FACET_LIMIT,
|
|
18448
18864
|
RUN_VALUE_FACET_SCAN,
|
|
@@ -18600,6 +19016,9 @@ export {
|
|
|
18600
19016
|
readBufferedEvent,
|
|
18601
19017
|
readRunsByIds,
|
|
18602
19018
|
readSearchAttributes,
|
|
19019
|
+
remoteCallMaxAttempts,
|
|
19020
|
+
remoteCallPolicyOf,
|
|
19021
|
+
remoteCallRetries,
|
|
18603
19022
|
requiredCapabilities,
|
|
18604
19023
|
resolveConcurrency,
|
|
18605
19024
|
resolveRouting,
|