@dudousxd/nestjs-durable-core 0.102.0 → 0.103.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +459 -36
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +168 -1
- package/dist/index.d.ts +168 -1
- package/dist/index.js +455 -36
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -77,6 +77,7 @@ __export(index_exports, {
|
|
|
77
77
|
PARENT_CLOSE_TAG_PREFIX: () => PARENT_CLOSE_TAG_PREFIX,
|
|
78
78
|
QueueController: () => QueueController,
|
|
79
79
|
REMOTE_EVENT: () => REMOTE_EVENT,
|
|
80
|
+
REMOTE_WORKFLOW_FEATURES: () => REMOTE_WORKFLOW_FEATURES,
|
|
80
81
|
RUN_HISTORY_FIXTURE_VERSION: () => RUN_HISTORY_FIXTURE_VERSION,
|
|
81
82
|
RUN_VALUE_FACET_LIMIT: () => RUN_VALUE_FACET_LIMIT,
|
|
82
83
|
RUN_VALUE_FACET_SCAN: () => RUN_VALUE_FACET_SCAN,
|
|
@@ -234,6 +235,9 @@ __export(index_exports, {
|
|
|
234
235
|
readBufferedEvent: () => readBufferedEvent,
|
|
235
236
|
readRunsByIds: () => readRunsByIds,
|
|
236
237
|
readSearchAttributes: () => readSearchAttributes,
|
|
238
|
+
remoteCallMaxAttempts: () => remoteCallMaxAttempts,
|
|
239
|
+
remoteCallPolicyOf: () => remoteCallPolicyOf,
|
|
240
|
+
remoteCallRetries: () => remoteCallRetries,
|
|
237
241
|
requiredCapabilities: () => requiredCapabilities,
|
|
238
242
|
resolveConcurrency: () => resolveConcurrency,
|
|
239
243
|
resolveRouting: () => resolveRouting,
|
|
@@ -1178,6 +1182,14 @@ function isNonEmptyRunScope(scope) {
|
|
|
1178
1182
|
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
1179
1183
|
}
|
|
1180
1184
|
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
1185
|
+
var REMOTE_WORKFLOW_FEATURES = [
|
|
1186
|
+
"callPolicy",
|
|
1187
|
+
"sleepUntil",
|
|
1188
|
+
"waitSignalTimeout",
|
|
1189
|
+
"waitEvent",
|
|
1190
|
+
"spawnChild",
|
|
1191
|
+
"continueAsNew"
|
|
1192
|
+
];
|
|
1181
1193
|
var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
|
|
1182
1194
|
|
|
1183
1195
|
// src/events.ts
|
|
@@ -3366,6 +3378,35 @@ function codeAndDetails(err) {
|
|
|
3366
3378
|
}
|
|
3367
3379
|
__name(codeAndDetails, "codeAndDetails");
|
|
3368
3380
|
|
|
3381
|
+
// src/remote-call-policy.ts
|
|
3382
|
+
function remoteCallPolicyOf(config, opts) {
|
|
3383
|
+
const policy = {};
|
|
3384
|
+
const retries = opts?.retries ?? config?.retries;
|
|
3385
|
+
if (retries !== void 0) policy.retries = retries;
|
|
3386
|
+
const backoff = opts?.backoff ?? config?.backoff;
|
|
3387
|
+
if (backoff !== void 0) policy.backoff = backoff;
|
|
3388
|
+
const backoffMs = opts?.backoffMs ?? config?.backoffMs;
|
|
3389
|
+
if (backoffMs !== void 0) policy.backoffMs = backoffMs;
|
|
3390
|
+
const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
|
|
3391
|
+
if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
|
|
3392
|
+
const jitter = opts?.jitter ?? config?.jitter;
|
|
3393
|
+
if (jitter !== void 0) policy.jitter = jitter;
|
|
3394
|
+
const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
|
|
3395
|
+
if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
|
|
3396
|
+
return Object.keys(policy).length > 0 ? policy : void 0;
|
|
3397
|
+
}
|
|
3398
|
+
__name(remoteCallPolicyOf, "remoteCallPolicyOf");
|
|
3399
|
+
function remoteCallMaxAttempts(policy) {
|
|
3400
|
+
return Math.max(1, policy?.retries ?? 1);
|
|
3401
|
+
}
|
|
3402
|
+
__name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
|
|
3403
|
+
function remoteCallRetries(policy, attempts, error2) {
|
|
3404
|
+
if (policy === void 0 || policy.timeoutMs) return false;
|
|
3405
|
+
if (error2?.retryable === false) return false;
|
|
3406
|
+
return attempts < remoteCallMaxAttempts(policy);
|
|
3407
|
+
}
|
|
3408
|
+
__name(remoteCallRetries, "remoteCallRetries");
|
|
3409
|
+
|
|
3369
3410
|
// src/remote-event.ts
|
|
3370
3411
|
var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
|
|
3371
3412
|
function markRemoteEvent(event) {
|
|
@@ -3423,7 +3464,12 @@ var RemoteWorkflowExecutor = class {
|
|
|
3423
3464
|
} : {},
|
|
3424
3465
|
group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
|
|
3425
3466
|
priority: run.priority,
|
|
3426
|
-
attempt: 1
|
|
3467
|
+
attempt: 1,
|
|
3468
|
+
// What this control plane applies beyond the original command kinds, so a worker fails a turn
|
|
3469
|
+
// that needs more instead of emitting a command this engine would not understand.
|
|
3470
|
+
features: [
|
|
3471
|
+
...REMOTE_WORKFLOW_FEATURES
|
|
3472
|
+
]
|
|
3427
3473
|
};
|
|
3428
3474
|
await this.transport.dispatchWorkflowTask(task);
|
|
3429
3475
|
}
|
|
@@ -13750,7 +13796,7 @@ ${run.workflowVersion}`;
|
|
|
13750
13796
|
let wfPlan = await this.planRoute(wfToken, registered.requires);
|
|
13751
13797
|
if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
|
|
13752
13798
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
13753
|
-
const checkpoints = await this.replayCheckpoints(run.id);
|
|
13799
|
+
const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
|
|
13754
13800
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
13755
13801
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
13756
13802
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -13836,6 +13882,10 @@ ${run.workflowVersion}`;
|
|
|
13836
13882
|
*/
|
|
13837
13883
|
async applyDecision(run, decision) {
|
|
13838
13884
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
13885
|
+
const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
|
|
13886
|
+
if (handoff !== void 0 && decision.status === "completed") {
|
|
13887
|
+
return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
|
|
13888
|
+
}
|
|
13839
13889
|
if (decision.status === "completed") {
|
|
13840
13890
|
await this.applyCommands(run, decision.commands);
|
|
13841
13891
|
return this.settleRun(run, {
|
|
@@ -13882,6 +13932,63 @@ ${run.workflowVersion}`;
|
|
|
13882
13932
|
return suspended;
|
|
13883
13933
|
}
|
|
13884
13934
|
/**
|
|
13935
|
+
* A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
|
|
13936
|
+
* run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
|
|
13937
|
+
* engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
|
|
13938
|
+
* turn replayed stays cancelled and starts nothing.
|
|
13939
|
+
*/
|
|
13940
|
+
async applyRemoteContinueAsNew(run, commands, input) {
|
|
13941
|
+
await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
|
|
13942
|
+
const fresh = await this.store.getRun(run.id);
|
|
13943
|
+
if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
|
|
13944
|
+
return {
|
|
13945
|
+
runId: run.id,
|
|
13946
|
+
status: fresh.status,
|
|
13947
|
+
output: fresh.output,
|
|
13948
|
+
error: fresh.error
|
|
13949
|
+
};
|
|
13950
|
+
}
|
|
13951
|
+
const nextId = nextContinuationId(run.id);
|
|
13952
|
+
try {
|
|
13953
|
+
await this.start(run.workflow, input, nextId, {
|
|
13954
|
+
namespace: run.namespace,
|
|
13955
|
+
...inheritTenant(run)
|
|
13956
|
+
});
|
|
13957
|
+
} catch (err) {
|
|
13958
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13959
|
+
return this.settleRun(run, {
|
|
13960
|
+
kind: "failed",
|
|
13961
|
+
error: {
|
|
13962
|
+
message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
|
|
13963
|
+
code: "continue_as_new_failed"
|
|
13964
|
+
}
|
|
13965
|
+
});
|
|
13966
|
+
}
|
|
13967
|
+
await this.store.updateRun(run.id, {
|
|
13968
|
+
status: "completed",
|
|
13969
|
+
output: void 0,
|
|
13970
|
+
error: void 0,
|
|
13971
|
+
wakeAt: void 0,
|
|
13972
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
13973
|
+
});
|
|
13974
|
+
this.emit({
|
|
13975
|
+
type: "run.completed",
|
|
13976
|
+
runId: run.id,
|
|
13977
|
+
workflow: run.workflow,
|
|
13978
|
+
namespace: run.namespace
|
|
13979
|
+
}, {
|
|
13980
|
+
lifecycle: false
|
|
13981
|
+
});
|
|
13982
|
+
void this.notifyParent(run.id, {
|
|
13983
|
+
ok: true,
|
|
13984
|
+
value: void 0
|
|
13985
|
+
});
|
|
13986
|
+
return {
|
|
13987
|
+
runId: run.id,
|
|
13988
|
+
status: "completed"
|
|
13989
|
+
};
|
|
13990
|
+
}
|
|
13991
|
+
/**
|
|
13885
13992
|
* Re-drive a run whose turn was computed from a history OLDER than the store's.
|
|
13886
13993
|
*
|
|
13887
13994
|
* A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
|
|
@@ -13899,11 +14006,12 @@ ${run.workflowVersion}`;
|
|
|
13899
14006
|
* Scheduled off the current stack so it runs after the caller released the run lease.
|
|
13900
14007
|
*/
|
|
13901
14008
|
async redriveIfTurnSawStaleHistory(runId, commands) {
|
|
13902
|
-
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
|
|
14009
|
+
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
|
|
13903
14010
|
if (blocking.length === 0) return;
|
|
13904
14011
|
for (const cmd of blocking) {
|
|
13905
14012
|
const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
|
|
13906
|
-
|
|
14013
|
+
const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
|
|
14014
|
+
if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
|
|
13907
14015
|
setTimeout(() => {
|
|
13908
14016
|
void this.resumeImplicit(runId).catch(() => void 0);
|
|
13909
14017
|
}, 0).unref?.();
|
|
@@ -14152,7 +14260,12 @@ ${run.workflowVersion}`;
|
|
|
14152
14260
|
// name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
|
|
14153
14261
|
name: cp.kind === "signal" ? void 0 : cp.name,
|
|
14154
14262
|
output: cp.status === "completed" ? cp.output : void 0,
|
|
14155
|
-
error: cp.status === "failed" ? cp.error : void 0
|
|
14263
|
+
error: cp.status === "failed" ? cp.error : void 0,
|
|
14264
|
+
// How many times a dispatched step ran: a worker applying a `call` policy decides a retry
|
|
14265
|
+
// from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
|
|
14266
|
+
...cp.kind === "remote" ? {
|
|
14267
|
+
attempts: cp.attempts
|
|
14268
|
+
} : {}
|
|
14156
14269
|
});
|
|
14157
14270
|
} else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
|
|
14158
14271
|
events.push({
|
|
@@ -14209,11 +14322,12 @@ ${run.workflowVersion}`;
|
|
|
14209
14322
|
} else if (cmd.kind === "call") {
|
|
14210
14323
|
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
14211
14324
|
if (existing) {
|
|
14212
|
-
const
|
|
14325
|
+
const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
|
|
14326
|
+
const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
|
|
14213
14327
|
if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
|
|
14214
14328
|
continue;
|
|
14215
14329
|
}
|
|
14216
|
-
const lease = this.
|
|
14330
|
+
const lease = this.callLease(cmd.policy);
|
|
14217
14331
|
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14218
14332
|
runId: run.id,
|
|
14219
14333
|
seq: cmd.seq,
|
|
@@ -14273,35 +14387,7 @@ ${run.workflowVersion}`;
|
|
|
14273
14387
|
}));
|
|
14274
14388
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14275
14389
|
} else if (cmd.kind === "waitSignal") {
|
|
14276
|
-
|
|
14277
|
-
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14278
|
-
runId: run.id,
|
|
14279
|
-
seq: cmd.seq,
|
|
14280
|
-
name: `signal:${cmd.signal}`,
|
|
14281
|
-
kind: "signal",
|
|
14282
|
-
output: payload
|
|
14283
|
-
}));
|
|
14284
|
-
setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
|
|
14285
|
-
}, "deliverBuffered");
|
|
14286
|
-
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
14287
|
-
if (buffered) {
|
|
14288
|
-
await deliverBuffered(buffered.payload);
|
|
14289
|
-
} else {
|
|
14290
|
-
await this.store.putSignalWaiter({
|
|
14291
|
-
token: cmd.signal,
|
|
14292
|
-
runId: run.id,
|
|
14293
|
-
seq: cmd.seq
|
|
14294
|
-
});
|
|
14295
|
-
const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
14296
|
-
if (lateBuffered) {
|
|
14297
|
-
await this.store.removeSignalWaiter({
|
|
14298
|
-
token: cmd.signal,
|
|
14299
|
-
runId: run.id,
|
|
14300
|
-
seq: cmd.seq
|
|
14301
|
-
});
|
|
14302
|
-
await deliverBuffered(lateBuffered.payload);
|
|
14303
|
-
}
|
|
14304
|
-
}
|
|
14390
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14305
14391
|
} else if (cmd.kind === "waitAny") {
|
|
14306
14392
|
const state = await enterWaitAny({
|
|
14307
14393
|
store: this.store,
|
|
@@ -14333,6 +14419,100 @@ ${run.workflowVersion}`;
|
|
|
14333
14419
|
} else {
|
|
14334
14420
|
this.redeliverFinishedChild(childId, existingChild);
|
|
14335
14421
|
}
|
|
14422
|
+
} else if (cmd.kind === "sleepUntil") {
|
|
14423
|
+
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
14424
|
+
const deadline = existing?.wakeAt ?? cmd.until;
|
|
14425
|
+
if (!existing) {
|
|
14426
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14427
|
+
runId: run.id,
|
|
14428
|
+
seq: cmd.seq,
|
|
14429
|
+
name: `sleep:${cmd.seq}`,
|
|
14430
|
+
kind: "sleep",
|
|
14431
|
+
status: "pending",
|
|
14432
|
+
attempts: 1,
|
|
14433
|
+
wakeAt: deadline,
|
|
14434
|
+
enqueuedAt: at,
|
|
14435
|
+
startedAt: at,
|
|
14436
|
+
finishedAt: at
|
|
14437
|
+
}));
|
|
14438
|
+
}
|
|
14439
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14440
|
+
} else if (cmd.kind === "waitSignalTimeout") {
|
|
14441
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14442
|
+
const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
|
|
14443
|
+
waitSeq: cmd.seq,
|
|
14444
|
+
token: cmd.signal
|
|
14445
|
+
});
|
|
14446
|
+
if (deadline <= this.clock()) {
|
|
14447
|
+
this.resumeSoon(run.id);
|
|
14448
|
+
continue;
|
|
14449
|
+
}
|
|
14450
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14451
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14452
|
+
} else if (cmd.kind === "waitEvent") {
|
|
14453
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14454
|
+
const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
|
|
14455
|
+
let deadline;
|
|
14456
|
+
if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
|
|
14457
|
+
deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
|
|
14458
|
+
waitSeq: cmd.seq,
|
|
14459
|
+
token
|
|
14460
|
+
});
|
|
14461
|
+
if (deadline <= this.clock()) {
|
|
14462
|
+
this.resumeSoon(run.id);
|
|
14463
|
+
continue;
|
|
14464
|
+
}
|
|
14465
|
+
}
|
|
14466
|
+
await this.store.putSignalWaiter({
|
|
14467
|
+
token,
|
|
14468
|
+
runId: run.id,
|
|
14469
|
+
seq: cmd.seq
|
|
14470
|
+
});
|
|
14471
|
+
const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
|
|
14472
|
+
if (hit) {
|
|
14473
|
+
await this.store.removeSignalWaiter({
|
|
14474
|
+
token,
|
|
14475
|
+
runId: run.id,
|
|
14476
|
+
seq: cmd.seq
|
|
14477
|
+
});
|
|
14478
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14479
|
+
runId: run.id,
|
|
14480
|
+
seq: cmd.seq,
|
|
14481
|
+
name: `signal:${token}`,
|
|
14482
|
+
kind: "signal",
|
|
14483
|
+
output: hit.payload
|
|
14484
|
+
}));
|
|
14485
|
+
this.resumeSoon(run.id);
|
|
14486
|
+
} else if (deadline != null) {
|
|
14487
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14488
|
+
}
|
|
14489
|
+
} else if (cmd.kind === "spawnChild") {
|
|
14490
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14491
|
+
if (!await this.store.getRun(cmd.childId)) {
|
|
14492
|
+
try {
|
|
14493
|
+
await this.start(cmd.workflow, cmd.input, cmd.childId, {
|
|
14494
|
+
namespace: run.namespace,
|
|
14495
|
+
...inheritTenant(run),
|
|
14496
|
+
...cmd.priority !== void 0 ? {
|
|
14497
|
+
priority: cmd.priority
|
|
14498
|
+
} : {}
|
|
14499
|
+
});
|
|
14500
|
+
} catch (err) {
|
|
14501
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14502
|
+
this.notifyParent(cmd.childId, {
|
|
14503
|
+
ok: false,
|
|
14504
|
+
error: `child workflow "${cmd.workflow}" failed to start: ${message}`
|
|
14505
|
+
});
|
|
14506
|
+
}
|
|
14507
|
+
}
|
|
14508
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14509
|
+
runId: run.id,
|
|
14510
|
+
seq: cmd.seq,
|
|
14511
|
+
name: `spawn:${cmd.childId}`,
|
|
14512
|
+
kind: "local",
|
|
14513
|
+
output: cmd.childId
|
|
14514
|
+
}));
|
|
14515
|
+
} else if (cmd.kind === "continueAsNew") {
|
|
14336
14516
|
} else {
|
|
14337
14517
|
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
14338
14518
|
}
|
|
@@ -14340,6 +14520,245 @@ ${run.workflowVersion}`;
|
|
|
14340
14520
|
return wakeAt;
|
|
14341
14521
|
}
|
|
14342
14522
|
/**
|
|
14523
|
+
* Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
|
|
14524
|
+
* the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
|
|
14525
|
+
* (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
|
|
14526
|
+
* re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
|
|
14527
|
+
* bails).
|
|
14528
|
+
*/
|
|
14529
|
+
async parkRemoteOnSignal(run, seq, token) {
|
|
14530
|
+
const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
|
|
14531
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14532
|
+
runId: run.id,
|
|
14533
|
+
seq,
|
|
14534
|
+
name: `signal:${token}`,
|
|
14535
|
+
kind: "signal",
|
|
14536
|
+
output: payload
|
|
14537
|
+
}));
|
|
14538
|
+
this.resumeSoon(run.id);
|
|
14539
|
+
}, "deliverBuffered");
|
|
14540
|
+
const buffered = await this.store.takeBufferedSignal(token);
|
|
14541
|
+
if (buffered) {
|
|
14542
|
+
await deliverBuffered(buffered.payload);
|
|
14543
|
+
} else {
|
|
14544
|
+
await this.store.putSignalWaiter({
|
|
14545
|
+
token,
|
|
14546
|
+
runId: run.id,
|
|
14547
|
+
seq
|
|
14548
|
+
});
|
|
14549
|
+
const lateBuffered = await this.store.takeBufferedSignal(token);
|
|
14550
|
+
if (lateBuffered) {
|
|
14551
|
+
await this.store.removeSignalWaiter({
|
|
14552
|
+
token,
|
|
14553
|
+
runId: run.id,
|
|
14554
|
+
seq
|
|
14555
|
+
});
|
|
14556
|
+
await deliverBuffered(lateBuffered.payload);
|
|
14557
|
+
}
|
|
14558
|
+
}
|
|
14559
|
+
}
|
|
14560
|
+
/** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
|
|
14561
|
+
resumeSoon(runId) {
|
|
14562
|
+
setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
|
|
14563
|
+
}
|
|
14564
|
+
/**
|
|
14565
|
+
* Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
|
|
14566
|
+
* `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
|
|
14567
|
+
* back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
|
|
14568
|
+
* deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
|
|
14569
|
+
*/
|
|
14570
|
+
async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
|
|
14571
|
+
const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
|
|
14572
|
+
if (recorded?.wakeAt != null) return recorded.wakeAt;
|
|
14573
|
+
const deadline = this.clock() + timeoutMs;
|
|
14574
|
+
const at = /* @__PURE__ */ new Date();
|
|
14575
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14576
|
+
runId,
|
|
14577
|
+
seq: deadlineSeq,
|
|
14578
|
+
name: `timeout:${label}`,
|
|
14579
|
+
kind: "sleep",
|
|
14580
|
+
status: "pending",
|
|
14581
|
+
attempts: 1,
|
|
14582
|
+
input: wait,
|
|
14583
|
+
wakeAt: deadline,
|
|
14584
|
+
enqueuedAt: at,
|
|
14585
|
+
startedAt: at,
|
|
14586
|
+
finishedAt: at
|
|
14587
|
+
}));
|
|
14588
|
+
return deadline;
|
|
14589
|
+
}
|
|
14590
|
+
/**
|
|
14591
|
+
* Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
|
|
14592
|
+
* (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
|
|
14593
|
+
* time out — the in-process wait removes its waiter the same way when it finds the deadline past.
|
|
14594
|
+
* A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
|
|
14595
|
+
* worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
|
|
14596
|
+
*/
|
|
14597
|
+
async expireRemoteDeadlines(runId, checkpoints) {
|
|
14598
|
+
const now = this.clock();
|
|
14599
|
+
const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
|
|
14600
|
+
let out = checkpoints;
|
|
14601
|
+
for (const cp of checkpoints) {
|
|
14602
|
+
if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
|
|
14603
|
+
if (cp.wakeAt == null || cp.wakeAt > now) continue;
|
|
14604
|
+
const wait = cp.input;
|
|
14605
|
+
if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
|
|
14606
|
+
if (settled.has(wait.waitSeq)) continue;
|
|
14607
|
+
await this.store.removeSignalWaiter({
|
|
14608
|
+
token: wait.token,
|
|
14609
|
+
runId,
|
|
14610
|
+
seq: wait.waitSeq
|
|
14611
|
+
});
|
|
14612
|
+
const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
|
|
14613
|
+
if (landed && landed.status === "completed") {
|
|
14614
|
+
out = [
|
|
14615
|
+
...out.filter((c) => c.seq !== landed.seq),
|
|
14616
|
+
landed
|
|
14617
|
+
].sort((a, b) => a.seq - b.seq);
|
|
14618
|
+
}
|
|
14619
|
+
}
|
|
14620
|
+
return out;
|
|
14621
|
+
}
|
|
14622
|
+
/**
|
|
14623
|
+
* Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
|
|
14624
|
+
* see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
|
|
14625
|
+
* candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
|
|
14626
|
+
* {@link eventBufferTtlMs} is set, like the reconcile sweep.
|
|
14627
|
+
*/
|
|
14628
|
+
async claimBufferedEvent(run, name, match) {
|
|
14629
|
+
const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
|
|
14630
|
+
const nowMs = this.clock();
|
|
14631
|
+
for (const candidate of candidates) {
|
|
14632
|
+
if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
|
|
14633
|
+
await this.store.removeBufferedEvent(candidate.id);
|
|
14634
|
+
continue;
|
|
14635
|
+
}
|
|
14636
|
+
const { payload, tenantIds } = readBufferedEvent(candidate.payload);
|
|
14637
|
+
if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
|
|
14638
|
+
if (!eventMatches(payload, match)) continue;
|
|
14639
|
+
if (!await this.store.removeBufferedEvent(candidate.id)) continue;
|
|
14640
|
+
return {
|
|
14641
|
+
payload
|
|
14642
|
+
};
|
|
14643
|
+
}
|
|
14644
|
+
return void 0;
|
|
14645
|
+
}
|
|
14646
|
+
/** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
|
|
14647
|
+
* sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
|
|
14648
|
+
callLease(policy) {
|
|
14649
|
+
if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
|
|
14650
|
+
return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
14651
|
+
}
|
|
14652
|
+
/**
|
|
14653
|
+
* A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
|
|
14654
|
+
* checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
|
|
14655
|
+
* callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
|
|
14656
|
+
* behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
|
|
14657
|
+
* timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
|
|
14658
|
+
* and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
|
|
14659
|
+
*
|
|
14660
|
+
* - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
|
|
14661
|
+
* the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
|
|
14662
|
+
* dispatched again under the same `stepId` with `attempts + 1`.
|
|
14663
|
+
* - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
|
|
14664
|
+
* re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
|
|
14665
|
+
* workflow can catch, as `RemoteStepTimeout` would surface in-process.
|
|
14666
|
+
*/
|
|
14667
|
+
async applyCallPolicy(run, cmd, cp) {
|
|
14668
|
+
const policy = cmd.policy;
|
|
14669
|
+
if (cp.kind !== "remote") return null;
|
|
14670
|
+
if (cp.status === "failed") {
|
|
14671
|
+
if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
|
|
14672
|
+
let due = cp.wakeAt;
|
|
14673
|
+
if (due == null) {
|
|
14674
|
+
due = this.clock() + backoffDelay(cp.attempts, policy);
|
|
14675
|
+
if (due > this.clock()) {
|
|
14676
|
+
await this.store.saveCheckpoint({
|
|
14677
|
+
...cp,
|
|
14678
|
+
wakeAt: due
|
|
14679
|
+
});
|
|
14680
|
+
return due;
|
|
14681
|
+
}
|
|
14682
|
+
}
|
|
14683
|
+
if (this.clock() < due) return due;
|
|
14684
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
|
|
14685
|
+
}
|
|
14686
|
+
if (cp.status !== "pending" || !policy.timeoutMs) return null;
|
|
14687
|
+
if (cp.wakeAt == null) {
|
|
14688
|
+
const lease = this.clock() + policy.timeoutMs;
|
|
14689
|
+
await this.store.saveCheckpoint({
|
|
14690
|
+
...cp,
|
|
14691
|
+
wakeAt: lease
|
|
14692
|
+
});
|
|
14693
|
+
return this.stepLeaseWakeAt(lease);
|
|
14694
|
+
}
|
|
14695
|
+
if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
|
|
14696
|
+
if (cp.attempts < remoteCallMaxAttempts(policy)) {
|
|
14697
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
|
|
14698
|
+
}
|
|
14699
|
+
const id = cp.stepId ?? stepId(run.id, cp.seq);
|
|
14700
|
+
const error2 = {
|
|
14701
|
+
message: new RemoteStepTimeout(id, policy.timeoutMs).message,
|
|
14702
|
+
code: "remote_step_timeout"
|
|
14703
|
+
};
|
|
14704
|
+
await this.store.saveCheckpoint({
|
|
14705
|
+
...cp,
|
|
14706
|
+
status: "failed",
|
|
14707
|
+
error: error2,
|
|
14708
|
+
wakeAt: void 0,
|
|
14709
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
14710
|
+
});
|
|
14711
|
+
this.emit({
|
|
14712
|
+
type: "step.failed",
|
|
14713
|
+
runId: run.id,
|
|
14714
|
+
seq: cp.seq,
|
|
14715
|
+
name: cp.name,
|
|
14716
|
+
kind: "remote",
|
|
14717
|
+
error: error2
|
|
14718
|
+
});
|
|
14719
|
+
return void 0;
|
|
14720
|
+
}
|
|
14721
|
+
/** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
|
|
14722
|
+
async redispatchCall(run, cmd, cp, attempt, policy, why) {
|
|
14723
|
+
const lease = this.callLease(policy);
|
|
14724
|
+
const at = /* @__PURE__ */ new Date();
|
|
14725
|
+
await this.store.saveCheckpoint({
|
|
14726
|
+
...cp,
|
|
14727
|
+
status: "pending",
|
|
14728
|
+
output: void 0,
|
|
14729
|
+
error: void 0,
|
|
14730
|
+
attempts: attempt,
|
|
14731
|
+
wakeAt: lease,
|
|
14732
|
+
enqueuedAt: at,
|
|
14733
|
+
startedAt: at,
|
|
14734
|
+
finishedAt: at,
|
|
14735
|
+
events: why === "timeout" ? [
|
|
14736
|
+
...cp.events ?? [],
|
|
14737
|
+
lostStepEvent(this.clock(), attempt, "redispatched")
|
|
14738
|
+
] : cp.events
|
|
14739
|
+
});
|
|
14740
|
+
await this.dispatchRemoteTask({
|
|
14741
|
+
runId: run.id,
|
|
14742
|
+
seq: cp.seq,
|
|
14743
|
+
name: cmd.name,
|
|
14744
|
+
stepId: cp.stepId ?? stepId(run.id, cp.seq),
|
|
14745
|
+
group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
|
|
14746
|
+
input: cp.input,
|
|
14747
|
+
attempt
|
|
14748
|
+
});
|
|
14749
|
+
this.emit({
|
|
14750
|
+
type: "step.started",
|
|
14751
|
+
runId: run.id,
|
|
14752
|
+
seq: cp.seq,
|
|
14753
|
+
name: cmd.name,
|
|
14754
|
+
kind: "remote",
|
|
14755
|
+
...why === "timeout" ? {
|
|
14756
|
+
redispatched: true
|
|
14757
|
+
} : {}
|
|
14758
|
+
});
|
|
14759
|
+
return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
|
|
14760
|
+
}
|
|
14761
|
+
/**
|
|
14343
14762
|
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
14344
14763
|
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
14345
14764
|
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
@@ -18741,6 +19160,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18741
19160
|
PARENT_CLOSE_TAG_PREFIX,
|
|
18742
19161
|
QueueController,
|
|
18743
19162
|
REMOTE_EVENT,
|
|
19163
|
+
REMOTE_WORKFLOW_FEATURES,
|
|
18744
19164
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
18745
19165
|
RUN_VALUE_FACET_LIMIT,
|
|
18746
19166
|
RUN_VALUE_FACET_SCAN,
|
|
@@ -18898,6 +19318,9 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18898
19318
|
readBufferedEvent,
|
|
18899
19319
|
readRunsByIds,
|
|
18900
19320
|
readSearchAttributes,
|
|
19321
|
+
remoteCallMaxAttempts,
|
|
19322
|
+
remoteCallPolicyOf,
|
|
19323
|
+
remoteCallRetries,
|
|
18901
19324
|
requiredCapabilities,
|
|
18902
19325
|
resolveConcurrency,
|
|
18903
19326
|
resolveRouting,
|