@dudousxd/nestjs-durable-core 0.102.0 → 0.104.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +590 -61
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +200 -7
- package/dist/index.d.ts +200 -7
- package/dist/index.js +585 -61
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -77,6 +77,7 @@ __export(index_exports, {
|
|
|
77
77
|
PARENT_CLOSE_TAG_PREFIX: () => PARENT_CLOSE_TAG_PREFIX,
|
|
78
78
|
QueueController: () => QueueController,
|
|
79
79
|
REMOTE_EVENT: () => REMOTE_EVENT,
|
|
80
|
+
REMOTE_WORKFLOW_FEATURES: () => REMOTE_WORKFLOW_FEATURES,
|
|
80
81
|
RUN_HISTORY_FIXTURE_VERSION: () => RUN_HISTORY_FIXTURE_VERSION,
|
|
81
82
|
RUN_VALUE_FACET_LIMIT: () => RUN_VALUE_FACET_LIMIT,
|
|
82
83
|
RUN_VALUE_FACET_SCAN: () => RUN_VALUE_FACET_SCAN,
|
|
@@ -234,6 +235,9 @@ __export(index_exports, {
|
|
|
234
235
|
readBufferedEvent: () => readBufferedEvent,
|
|
235
236
|
readRunsByIds: () => readRunsByIds,
|
|
236
237
|
readSearchAttributes: () => readSearchAttributes,
|
|
238
|
+
remoteCallMaxAttempts: () => remoteCallMaxAttempts,
|
|
239
|
+
remoteCallPolicyOf: () => remoteCallPolicyOf,
|
|
240
|
+
remoteCallRetries: () => remoteCallRetries,
|
|
237
241
|
requiredCapabilities: () => requiredCapabilities,
|
|
238
242
|
resolveConcurrency: () => resolveConcurrency,
|
|
239
243
|
resolveRouting: () => resolveRouting,
|
|
@@ -270,6 +274,7 @@ __export(index_exports, {
|
|
|
270
274
|
singletonAdmissionDenialPatch: () => singletonAdmissionDenialPatch,
|
|
271
275
|
singletonAdmissionTags: () => singletonAdmissionTags,
|
|
272
276
|
singletonClaimantIds: () => singletonClaimantIds,
|
|
277
|
+
singletonKeyTag: () => singletonKeyTag,
|
|
273
278
|
sqlComparator: () => sqlComparator,
|
|
274
279
|
stepConfigOf: () => stepConfigOf,
|
|
275
280
|
stepId: () => stepId,
|
|
@@ -1178,6 +1183,14 @@ function isNonEmptyRunScope(scope) {
|
|
|
1178
1183
|
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
1179
1184
|
}
|
|
1180
1185
|
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
1186
|
+
var REMOTE_WORKFLOW_FEATURES = [
|
|
1187
|
+
"callPolicy",
|
|
1188
|
+
"sleepUntil",
|
|
1189
|
+
"waitSignalTimeout",
|
|
1190
|
+
"waitEvent",
|
|
1191
|
+
"spawnChild",
|
|
1192
|
+
"continueAsNew"
|
|
1193
|
+
];
|
|
1181
1194
|
var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
|
|
1182
1195
|
|
|
1183
1196
|
// src/events.ts
|
|
@@ -3366,6 +3379,35 @@ function codeAndDetails(err) {
|
|
|
3366
3379
|
}
|
|
3367
3380
|
__name(codeAndDetails, "codeAndDetails");
|
|
3368
3381
|
|
|
3382
|
+
// src/remote-call-policy.ts
|
|
3383
|
+
function remoteCallPolicyOf(config, opts) {
|
|
3384
|
+
const policy = {};
|
|
3385
|
+
const retries = opts?.retries ?? config?.retries;
|
|
3386
|
+
if (retries !== void 0) policy.retries = retries;
|
|
3387
|
+
const backoff = opts?.backoff ?? config?.backoff;
|
|
3388
|
+
if (backoff !== void 0) policy.backoff = backoff;
|
|
3389
|
+
const backoffMs = opts?.backoffMs ?? config?.backoffMs;
|
|
3390
|
+
if (backoffMs !== void 0) policy.backoffMs = backoffMs;
|
|
3391
|
+
const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
|
|
3392
|
+
if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
|
|
3393
|
+
const jitter = opts?.jitter ?? config?.jitter;
|
|
3394
|
+
if (jitter !== void 0) policy.jitter = jitter;
|
|
3395
|
+
const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
|
|
3396
|
+
if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
|
|
3397
|
+
return Object.keys(policy).length > 0 ? policy : void 0;
|
|
3398
|
+
}
|
|
3399
|
+
__name(remoteCallPolicyOf, "remoteCallPolicyOf");
|
|
3400
|
+
function remoteCallMaxAttempts(policy) {
|
|
3401
|
+
return Math.max(1, policy?.retries ?? 1);
|
|
3402
|
+
}
|
|
3403
|
+
__name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
|
|
3404
|
+
function remoteCallRetries(policy, attempts, error2) {
|
|
3405
|
+
if (policy === void 0 || policy.timeoutMs) return false;
|
|
3406
|
+
if (error2?.retryable === false) return false;
|
|
3407
|
+
return attempts < remoteCallMaxAttempts(policy);
|
|
3408
|
+
}
|
|
3409
|
+
__name(remoteCallRetries, "remoteCallRetries");
|
|
3410
|
+
|
|
3369
3411
|
// src/remote-event.ts
|
|
3370
3412
|
var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
|
|
3371
3413
|
function markRemoteEvent(event) {
|
|
@@ -3423,7 +3465,12 @@ var RemoteWorkflowExecutor = class {
|
|
|
3423
3465
|
} : {},
|
|
3424
3466
|
group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
|
|
3425
3467
|
priority: run.priority,
|
|
3426
|
-
attempt: 1
|
|
3468
|
+
attempt: 1,
|
|
3469
|
+
// What this control plane applies beyond the original command kinds, so a worker fails a turn
|
|
3470
|
+
// that needs more instead of emitting a command this engine would not understand.
|
|
3471
|
+
features: [
|
|
3472
|
+
...REMOTE_WORKFLOW_FEATURES
|
|
3473
|
+
]
|
|
3427
3474
|
};
|
|
3428
3475
|
await this.transport.dispatchWorkflowTask(task);
|
|
3429
3476
|
}
|
|
@@ -4863,6 +4910,14 @@ var LEGACY_SCAN_STATUSES = [
|
|
|
4863
4910
|
"cancelling"
|
|
4864
4911
|
];
|
|
4865
4912
|
var byAdmissionOrder = /* @__PURE__ */ __name((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id), "byAdmissionOrder");
|
|
4913
|
+
function singletonKeyTag(tags) {
|
|
4914
|
+
if (!tags) return void 0;
|
|
4915
|
+
for (let i = tags.length - 1; i >= 0; i -= 1) {
|
|
4916
|
+
if (tags[i].startsWith("singleton:")) return tags[i];
|
|
4917
|
+
}
|
|
4918
|
+
return void 0;
|
|
4919
|
+
}
|
|
4920
|
+
__name(singletonKeyTag, "singletonKeyTag");
|
|
4866
4921
|
function withoutSingletonMarkers(tags) {
|
|
4867
4922
|
return tags?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
4868
4923
|
}
|
|
@@ -4891,7 +4946,7 @@ function singletonClaimantIds(rows, tag) {
|
|
|
4891
4946
|
__name(singletonClaimantIds, "singletonClaimantIds");
|
|
4892
4947
|
function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
|
|
4893
4948
|
if (!Number.isInteger(limit) || limit < 1) throw new Error("singleton limit must be a positive integer");
|
|
4894
|
-
const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && run.tags
|
|
4949
|
+
const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && singletonKeyTag(run.tags) === tag);
|
|
4895
4950
|
const candidate = active.find((run) => run.id === runId);
|
|
4896
4951
|
if (!candidate) return null;
|
|
4897
4952
|
if (candidate.tags?.includes(SINGLETON_ADMITTED_TAG)) return candidate.tags;
|
|
@@ -4919,7 +4974,7 @@ async function clearTerminalSingletonAdmission(runId, patch, getRun) {
|
|
|
4919
4974
|
__name(clearTerminalSingletonAdmission, "clearTerminalSingletonAdmission");
|
|
4920
4975
|
function singletonAdmissionDenialPatch(runs, runId, tag, workflow, retryWakeAt) {
|
|
4921
4976
|
if (retryWakeAt === void 0) return void 0;
|
|
4922
|
-
const candidate = runs.find((run) => run.id === runId && run.workflow === workflow && run.tags
|
|
4977
|
+
const candidate = runs.find((run) => run.id === runId && run.workflow === workflow && singletonKeyTag(run.tags) === tag && SINGLETON_ACTIVE_STATUSES.includes(run.status));
|
|
4923
4978
|
if (!candidate || candidate.status === "cancelling") return void 0;
|
|
4924
4979
|
return {
|
|
4925
4980
|
status: "suspended",
|
|
@@ -4968,6 +5023,28 @@ function isLockConflict(error2) {
|
|
|
4968
5023
|
return false;
|
|
4969
5024
|
}
|
|
4970
5025
|
__name(isLockConflict, "isLockConflict");
|
|
5026
|
+
var UnparseableStoredInput = class {
|
|
5027
|
+
static {
|
|
5028
|
+
__name(this, "UnparseableStoredInput");
|
|
5029
|
+
}
|
|
5030
|
+
error;
|
|
5031
|
+
constructor(error2) {
|
|
5032
|
+
this.error = error2;
|
|
5033
|
+
}
|
|
5034
|
+
};
|
|
5035
|
+
async function admissionInput(run, parseInput) {
|
|
5036
|
+
if (!parseInput) return run.input;
|
|
5037
|
+
try {
|
|
5038
|
+
return await parseInput(run.input);
|
|
5039
|
+
} catch (err) {
|
|
5040
|
+
return new UnparseableStoredInput(err instanceof SchemaValidationError && err.code === "invalid_input" ? err : new SchemaValidationError(`workflow "${run.workflow}" input`, [
|
|
5041
|
+
{
|
|
5042
|
+
message: err instanceof Error ? err.message : String(err)
|
|
5043
|
+
}
|
|
5044
|
+
], "invalid_input"));
|
|
5045
|
+
}
|
|
5046
|
+
}
|
|
5047
|
+
__name(admissionInput, "admissionInput");
|
|
4971
5048
|
var SingletonGate = class {
|
|
4972
5049
|
static {
|
|
4973
5050
|
__name(this, "SingletonGate");
|
|
@@ -5004,12 +5081,19 @@ var SingletonGate = class {
|
|
|
5004
5081
|
throw new SingletonQueueFullError(workflow, cfg.key(input), cfg.maxQueueDepth);
|
|
5005
5082
|
}
|
|
5006
5083
|
}
|
|
5007
|
-
/**
|
|
5008
|
-
|
|
5084
|
+
/**
|
|
5085
|
+
* Preserve durable holders and atomically claim one of the remaining slots. With `parseInput` (the
|
|
5086
|
+
* workflow's input schema), the key reads the parsed input, the value `run(ctx, input)` gets; a
|
|
5087
|
+
* stored input that no longer parses answers {@link UnparseableStoredInput}, which the caller
|
|
5088
|
+
* must fail the turn with (never run it).
|
|
5089
|
+
*/
|
|
5090
|
+
async admit(run, cfg, parseInput) {
|
|
5009
5091
|
if (!this.deps.store.tryAdmitSingleton) {
|
|
5010
5092
|
throw new Error("Singleton workflows require StateStore.tryAdmitSingleton for atomic durable admission");
|
|
5011
5093
|
}
|
|
5012
|
-
const
|
|
5094
|
+
const keyInput = await admissionInput(run, parseInput);
|
|
5095
|
+
if (keyInput instanceof UnparseableStoredInput) return keyInput;
|
|
5096
|
+
const admitted = await this.claim(run, cfg, this.admissionTag(run, cfg, keyInput, parseInput));
|
|
5013
5097
|
if (admitted && !run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
|
|
5014
5098
|
run.tags = [
|
|
5015
5099
|
...(run.tags ?? []).filter((tag) => tag !== SINGLETON_TRACKED_TAG),
|
|
@@ -5019,17 +5103,43 @@ var SingletonGate = class {
|
|
|
5019
5103
|
return admitted;
|
|
5020
5104
|
}
|
|
5021
5105
|
/**
|
|
5106
|
+
* The exact tag a turn claims under: the key of the parsed input, the tag `start` stamps. A run
|
|
5107
|
+
* queued by an engine that predates `parseInput` was stamped with the key of its RAW input
|
|
5108
|
+
* instead; when the run carries that tag and not the parsed one, it claims under it, so an
|
|
5109
|
+
* upgrade does not strand it. Both are computed from this run's own input, never read off a tag.
|
|
5110
|
+
* Only that case calls the key with the raw input (what the older engine always did), and only
|
|
5111
|
+
* for a run that carries a `singleton:` tag at all; a key that throws on it gets the parsed tag.
|
|
5112
|
+
*/
|
|
5113
|
+
admissionTag(run, cfg, keyInput, parseInput) {
|
|
5114
|
+
const parsed = this.tag(cfg, keyInput);
|
|
5115
|
+
const stamped = singletonKeyTag(run.tags);
|
|
5116
|
+
if (!parseInput || stamped === void 0 || stamped === parsed) return parsed;
|
|
5117
|
+
let raw;
|
|
5118
|
+
try {
|
|
5119
|
+
raw = this.tag(cfg, run.input);
|
|
5120
|
+
} catch {
|
|
5121
|
+
return parsed;
|
|
5122
|
+
}
|
|
5123
|
+
return stamped === raw ? raw : parsed;
|
|
5124
|
+
}
|
|
5125
|
+
/**
|
|
5022
5126
|
* One claim, retried when the database aborts it as a lock conflict. A conflict that outlasts the
|
|
5023
5127
|
* retries answers "not admitted", so the engine releases this turn's lease, instead of stranding
|
|
5024
5128
|
* the run until the lease lapses. It writes nothing to the run: this turn's lease may have lapsed
|
|
5025
5129
|
* during the claim, and another instance may now hold the run, its slot and newer tags. A later
|
|
5026
5130
|
* sweep re-drives the run: recovery once it is unleased, or its timer if it is parked.
|
|
5131
|
+
*
|
|
5132
|
+
* The claim is under the tag the key computes NOW, the one `start` stamped from the same input
|
|
5133
|
+
* (the schema is deterministic). The store admits a run only when it carries that exact tag, so
|
|
5134
|
+
* a run whose stored tags lack it (a caller tag from before callers' `singleton:` tags were
|
|
5135
|
+
* dropped, a run started before the workflow became a singleton) is never admitted under some
|
|
5136
|
+
* other key; it waits instead.
|
|
5027
5137
|
*/
|
|
5028
|
-
async claim(run, cfg) {
|
|
5138
|
+
async claim(run, cfg, tag) {
|
|
5029
5139
|
const admit = this.deps.store.tryAdmitSingleton;
|
|
5030
5140
|
for (let attempt = 1; ; attempt += 1) {
|
|
5031
5141
|
try {
|
|
5032
|
-
return await admit.call(this.deps.store, run.id,
|
|
5142
|
+
return await admit.call(this.deps.store, run.id, tag, run.workflow, cfg.limit ?? 1, this.retryWakeAt());
|
|
5033
5143
|
} catch (error2) {
|
|
5034
5144
|
if (!isLockConflict(error2)) throw error2;
|
|
5035
5145
|
if (attempt >= ADMISSION_ATTEMPTS) return false;
|
|
@@ -5047,7 +5157,7 @@ var SingletonGate = class {
|
|
|
5047
5157
|
async wakeNext(settled) {
|
|
5048
5158
|
const cfg = this.deps.configFor(settled);
|
|
5049
5159
|
if (!cfg) return;
|
|
5050
|
-
const tag = settled.tags
|
|
5160
|
+
const tag = singletonKeyTag(settled.tags);
|
|
5051
5161
|
if (!tag) return;
|
|
5052
5162
|
const gated = (await this.deps.store.listRuns({
|
|
5053
5163
|
tag,
|
|
@@ -7196,9 +7306,30 @@ function tenantScopedKey(namespace, key) {
|
|
|
7196
7306
|
return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
|
|
7197
7307
|
}
|
|
7198
7308
|
__name(tenantScopedKey, "tenantScopedKey");
|
|
7309
|
+
function failingBody(error2) {
|
|
7310
|
+
return async () => {
|
|
7311
|
+
throw error2;
|
|
7312
|
+
};
|
|
7313
|
+
}
|
|
7314
|
+
__name(failingBody, "failingBody");
|
|
7315
|
+
function storedInputError(invalid) {
|
|
7316
|
+
return {
|
|
7317
|
+
message: invalid.error.message,
|
|
7318
|
+
code: "invalid_input",
|
|
7319
|
+
retryable: false,
|
|
7320
|
+
details: {
|
|
7321
|
+
issues: invalid.error.issues
|
|
7322
|
+
}
|
|
7323
|
+
};
|
|
7324
|
+
}
|
|
7325
|
+
__name(storedInputError, "storedInputError");
|
|
7326
|
+
function callerTags(tags) {
|
|
7327
|
+
return tags?.filter((tag) => !tag.startsWith(SINGLETON_TAG_PREFIX));
|
|
7328
|
+
}
|
|
7329
|
+
__name(callerTags, "callerTags");
|
|
7330
|
+
var SINGLETON_TAG_PREFIX = "singleton:";
|
|
7199
7331
|
function singletonKeyOf(run) {
|
|
7200
|
-
|
|
7201
|
-
return tag?.slice("singleton:".length);
|
|
7332
|
+
return singletonKeyTag(run.tags)?.slice("singleton:".length);
|
|
7202
7333
|
}
|
|
7203
7334
|
__name(singletonKeyOf, "singletonKeyOf");
|
|
7204
7335
|
function conventionSingleton(run, announced, wire) {
|
|
@@ -8218,6 +8349,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8218
8349
|
concurrency: opts?.concurrency,
|
|
8219
8350
|
executionTimeoutMs: opts?.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
|
|
8220
8351
|
validateInput: opts?.validateInput,
|
|
8352
|
+
parseInput: opts?.parseInput,
|
|
8221
8353
|
searchAttributesSchema: opts?.searchAttributesSchema,
|
|
8222
8354
|
onEvent: opts?.onEvent,
|
|
8223
8355
|
eventBatch: opts?.eventBatch,
|
|
@@ -9919,24 +10051,31 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9919
10051
|
if (!registered) {
|
|
9920
10052
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
9921
10053
|
}
|
|
9922
|
-
|
|
10054
|
+
const check = registered;
|
|
10055
|
+
const parseAndValidate = /* @__PURE__ */ __name(async () => {
|
|
10056
|
+
const parsed = check.parseInput ? await check.parseInput(input) : input;
|
|
10057
|
+
await check.validateInput?.(parsed);
|
|
10058
|
+
return parsed;
|
|
10059
|
+
}, "parseAndValidate");
|
|
10060
|
+
let keyInput;
|
|
10061
|
+
if (parkUnroutable && (registered.validateInput || registered.parseInput)) {
|
|
9923
10062
|
try {
|
|
9924
|
-
await
|
|
10063
|
+
keyInput = await parseAndValidate();
|
|
9925
10064
|
} catch (err) {
|
|
9926
10065
|
throw new TenantStartInvalid(err);
|
|
9927
10066
|
}
|
|
9928
10067
|
} else {
|
|
9929
|
-
await
|
|
10068
|
+
keyInput = await parseAndValidate();
|
|
9930
10069
|
}
|
|
9931
10070
|
const prior = await this.store.getRun(runId);
|
|
9932
10071
|
if (prior) return this.existingStartResult(prior);
|
|
9933
|
-
const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency,
|
|
10072
|
+
const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, keyInput);
|
|
9934
10073
|
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9935
10074
|
const now = /* @__PURE__ */ new Date();
|
|
9936
10075
|
const stampedTags = [
|
|
9937
|
-
...opts?.tags ?? [],
|
|
10076
|
+
...callerTags(opts?.tags) ?? [],
|
|
9938
10077
|
...registered.singleton ? [
|
|
9939
|
-
this.singletons.tag(registered.singleton,
|
|
10078
|
+
this.singletons.tag(registered.singleton, keyInput)
|
|
9940
10079
|
] : [],
|
|
9941
10080
|
...quota ? [
|
|
9942
10081
|
concurrencyTag(quota.key)
|
|
@@ -9945,9 +10084,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9945
10084
|
VERSION_UNDECLARED_TAG
|
|
9946
10085
|
] : []
|
|
9947
10086
|
];
|
|
9948
|
-
const tags = mergeTags(registered.tags, stampedTags)?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
10087
|
+
const tags = mergeTags(callerTags(registered.tags), stampedTags)?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
9949
10088
|
if (registered.singleton) {
|
|
9950
|
-
await this.singletons.assertCapacity(name, registered.singleton,
|
|
10089
|
+
await this.singletons.assertCapacity(name, registered.singleton, keyInput);
|
|
9951
10090
|
}
|
|
9952
10091
|
const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
|
|
9953
10092
|
const delayed = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > this.clock();
|
|
@@ -10036,7 +10175,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10036
10175
|
};
|
|
10037
10176
|
const now = /* @__PURE__ */ new Date();
|
|
10038
10177
|
const tags = mergeTags(void 0, [
|
|
10039
|
-
...opts?.tags ?? [],
|
|
10178
|
+
...callerTags(opts?.tags) ?? [],
|
|
10040
10179
|
...singleton ? [
|
|
10041
10180
|
this.singletons.tag(singleton, input)
|
|
10042
10181
|
] : [],
|
|
@@ -10216,7 +10355,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10216
10355
|
namespace: poolNamespace(opts.namespace) ?? this.namespace,
|
|
10217
10356
|
input,
|
|
10218
10357
|
// The same derived tags an admitted or parked start carries, so quota-key queries find it.
|
|
10219
|
-
tags: mergeTags(opts.tags, [
|
|
10358
|
+
tags: mergeTags(callerTags(opts.tags), [
|
|
10220
10359
|
...opts.concurrency ? [
|
|
10221
10360
|
concurrencyTag(opts.concurrency.key)
|
|
10222
10361
|
] : [],
|
|
@@ -13743,14 +13882,23 @@ ${run.workflowVersion}`;
|
|
|
13743
13882
|
...await this.sourceTraceOf(run)
|
|
13744
13883
|
});
|
|
13745
13884
|
}
|
|
13746
|
-
if (registered.singleton
|
|
13747
|
-
|
|
13885
|
+
if (registered.singleton) {
|
|
13886
|
+
const admission = await this.singletons.admit(run, registered.singleton, registered.parseInput);
|
|
13887
|
+
if (admission instanceof UnparseableStoredInput) {
|
|
13888
|
+
const result = await this.settleRun(run, {
|
|
13889
|
+
kind: "failed",
|
|
13890
|
+
error: storedInputError(admission)
|
|
13891
|
+
});
|
|
13892
|
+
await this.releaseLeaseBestEffort(run.id);
|
|
13893
|
+
return result;
|
|
13894
|
+
}
|
|
13895
|
+
if (!admission) return this.singletonWaitResult(run);
|
|
13748
13896
|
}
|
|
13749
13897
|
const wfToken = tenantGroup(sanitizeQueueToken(remote.group), void 0);
|
|
13750
13898
|
let wfPlan = await this.planRoute(wfToken, registered.requires);
|
|
13751
13899
|
if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
|
|
13752
13900
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
13753
|
-
const checkpoints = await this.replayCheckpoints(run.id);
|
|
13901
|
+
const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
|
|
13754
13902
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
13755
13903
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
13756
13904
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -13836,6 +13984,10 @@ ${run.workflowVersion}`;
|
|
|
13836
13984
|
*/
|
|
13837
13985
|
async applyDecision(run, decision) {
|
|
13838
13986
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
13987
|
+
const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
|
|
13988
|
+
if (handoff !== void 0 && decision.status === "completed") {
|
|
13989
|
+
return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
|
|
13990
|
+
}
|
|
13839
13991
|
if (decision.status === "completed") {
|
|
13840
13992
|
await this.applyCommands(run, decision.commands);
|
|
13841
13993
|
return this.settleRun(run, {
|
|
@@ -13882,6 +14034,63 @@ ${run.workflowVersion}`;
|
|
|
13882
14034
|
return suspended;
|
|
13883
14035
|
}
|
|
13884
14036
|
/**
|
|
14037
|
+
* A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
|
|
14038
|
+
* run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
|
|
14039
|
+
* engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
|
|
14040
|
+
* turn replayed stays cancelled and starts nothing.
|
|
14041
|
+
*/
|
|
14042
|
+
async applyRemoteContinueAsNew(run, commands, input) {
|
|
14043
|
+
await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
|
|
14044
|
+
const fresh = await this.store.getRun(run.id);
|
|
14045
|
+
if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
|
|
14046
|
+
return {
|
|
14047
|
+
runId: run.id,
|
|
14048
|
+
status: fresh.status,
|
|
14049
|
+
output: fresh.output,
|
|
14050
|
+
error: fresh.error
|
|
14051
|
+
};
|
|
14052
|
+
}
|
|
14053
|
+
const nextId = nextContinuationId(run.id);
|
|
14054
|
+
try {
|
|
14055
|
+
await this.start(run.workflow, input, nextId, {
|
|
14056
|
+
namespace: run.namespace,
|
|
14057
|
+
...inheritTenant(run)
|
|
14058
|
+
});
|
|
14059
|
+
} catch (err) {
|
|
14060
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14061
|
+
return this.settleRun(run, {
|
|
14062
|
+
kind: "failed",
|
|
14063
|
+
error: {
|
|
14064
|
+
message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
|
|
14065
|
+
code: "continue_as_new_failed"
|
|
14066
|
+
}
|
|
14067
|
+
});
|
|
14068
|
+
}
|
|
14069
|
+
await this.store.updateRun(run.id, {
|
|
14070
|
+
status: "completed",
|
|
14071
|
+
output: void 0,
|
|
14072
|
+
error: void 0,
|
|
14073
|
+
wakeAt: void 0,
|
|
14074
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
14075
|
+
});
|
|
14076
|
+
this.emit({
|
|
14077
|
+
type: "run.completed",
|
|
14078
|
+
runId: run.id,
|
|
14079
|
+
workflow: run.workflow,
|
|
14080
|
+
namespace: run.namespace
|
|
14081
|
+
}, {
|
|
14082
|
+
lifecycle: false
|
|
14083
|
+
});
|
|
14084
|
+
void this.notifyParent(run.id, {
|
|
14085
|
+
ok: true,
|
|
14086
|
+
value: void 0
|
|
14087
|
+
});
|
|
14088
|
+
return {
|
|
14089
|
+
runId: run.id,
|
|
14090
|
+
status: "completed"
|
|
14091
|
+
};
|
|
14092
|
+
}
|
|
14093
|
+
/**
|
|
13885
14094
|
* Re-drive a run whose turn was computed from a history OLDER than the store's.
|
|
13886
14095
|
*
|
|
13887
14096
|
* A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
|
|
@@ -13899,11 +14108,12 @@ ${run.workflowVersion}`;
|
|
|
13899
14108
|
* Scheduled off the current stack so it runs after the caller released the run lease.
|
|
13900
14109
|
*/
|
|
13901
14110
|
async redriveIfTurnSawStaleHistory(runId, commands) {
|
|
13902
|
-
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
|
|
14111
|
+
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
|
|
13903
14112
|
if (blocking.length === 0) return;
|
|
13904
14113
|
for (const cmd of blocking) {
|
|
13905
14114
|
const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
|
|
13906
|
-
|
|
14115
|
+
const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
|
|
14116
|
+
if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
|
|
13907
14117
|
setTimeout(() => {
|
|
13908
14118
|
void this.resumeImplicit(runId).catch(() => void 0);
|
|
13909
14119
|
}, 0).unref?.();
|
|
@@ -14152,7 +14362,12 @@ ${run.workflowVersion}`;
|
|
|
14152
14362
|
// name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
|
|
14153
14363
|
name: cp.kind === "signal" ? void 0 : cp.name,
|
|
14154
14364
|
output: cp.status === "completed" ? cp.output : void 0,
|
|
14155
|
-
error: cp.status === "failed" ? cp.error : void 0
|
|
14365
|
+
error: cp.status === "failed" ? cp.error : void 0,
|
|
14366
|
+
// How many times a dispatched step ran: a worker applying a `call` policy decides a retry
|
|
14367
|
+
// from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
|
|
14368
|
+
...cp.kind === "remote" ? {
|
|
14369
|
+
attempts: cp.attempts
|
|
14370
|
+
} : {}
|
|
14156
14371
|
});
|
|
14157
14372
|
} else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
|
|
14158
14373
|
events.push({
|
|
@@ -14209,11 +14424,12 @@ ${run.workflowVersion}`;
|
|
|
14209
14424
|
} else if (cmd.kind === "call") {
|
|
14210
14425
|
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
14211
14426
|
if (existing) {
|
|
14212
|
-
const
|
|
14427
|
+
const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
|
|
14428
|
+
const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
|
|
14213
14429
|
if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
|
|
14214
14430
|
continue;
|
|
14215
14431
|
}
|
|
14216
|
-
const lease = this.
|
|
14432
|
+
const lease = this.callLease(cmd.policy);
|
|
14217
14433
|
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14218
14434
|
runId: run.id,
|
|
14219
14435
|
seq: cmd.seq,
|
|
@@ -14273,35 +14489,7 @@ ${run.workflowVersion}`;
|
|
|
14273
14489
|
}));
|
|
14274
14490
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14275
14491
|
} else if (cmd.kind === "waitSignal") {
|
|
14276
|
-
|
|
14277
|
-
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14278
|
-
runId: run.id,
|
|
14279
|
-
seq: cmd.seq,
|
|
14280
|
-
name: `signal:${cmd.signal}`,
|
|
14281
|
-
kind: "signal",
|
|
14282
|
-
output: payload
|
|
14283
|
-
}));
|
|
14284
|
-
setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
|
|
14285
|
-
}, "deliverBuffered");
|
|
14286
|
-
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
14287
|
-
if (buffered) {
|
|
14288
|
-
await deliverBuffered(buffered.payload);
|
|
14289
|
-
} else {
|
|
14290
|
-
await this.store.putSignalWaiter({
|
|
14291
|
-
token: cmd.signal,
|
|
14292
|
-
runId: run.id,
|
|
14293
|
-
seq: cmd.seq
|
|
14294
|
-
});
|
|
14295
|
-
const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
14296
|
-
if (lateBuffered) {
|
|
14297
|
-
await this.store.removeSignalWaiter({
|
|
14298
|
-
token: cmd.signal,
|
|
14299
|
-
runId: run.id,
|
|
14300
|
-
seq: cmd.seq
|
|
14301
|
-
});
|
|
14302
|
-
await deliverBuffered(lateBuffered.payload);
|
|
14303
|
-
}
|
|
14304
|
-
}
|
|
14492
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14305
14493
|
} else if (cmd.kind === "waitAny") {
|
|
14306
14494
|
const state = await enterWaitAny({
|
|
14307
14495
|
store: this.store,
|
|
@@ -14333,6 +14521,100 @@ ${run.workflowVersion}`;
|
|
|
14333
14521
|
} else {
|
|
14334
14522
|
this.redeliverFinishedChild(childId, existingChild);
|
|
14335
14523
|
}
|
|
14524
|
+
} else if (cmd.kind === "sleepUntil") {
|
|
14525
|
+
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
14526
|
+
const deadline = existing?.wakeAt ?? cmd.until;
|
|
14527
|
+
if (!existing) {
|
|
14528
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14529
|
+
runId: run.id,
|
|
14530
|
+
seq: cmd.seq,
|
|
14531
|
+
name: `sleep:${cmd.seq}`,
|
|
14532
|
+
kind: "sleep",
|
|
14533
|
+
status: "pending",
|
|
14534
|
+
attempts: 1,
|
|
14535
|
+
wakeAt: deadline,
|
|
14536
|
+
enqueuedAt: at,
|
|
14537
|
+
startedAt: at,
|
|
14538
|
+
finishedAt: at
|
|
14539
|
+
}));
|
|
14540
|
+
}
|
|
14541
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14542
|
+
} else if (cmd.kind === "waitSignalTimeout") {
|
|
14543
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14544
|
+
const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
|
|
14545
|
+
waitSeq: cmd.seq,
|
|
14546
|
+
token: cmd.signal
|
|
14547
|
+
});
|
|
14548
|
+
if (deadline <= this.clock()) {
|
|
14549
|
+
this.resumeSoon(run.id);
|
|
14550
|
+
continue;
|
|
14551
|
+
}
|
|
14552
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14553
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14554
|
+
} else if (cmd.kind === "waitEvent") {
|
|
14555
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14556
|
+
const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
|
|
14557
|
+
let deadline;
|
|
14558
|
+
if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
|
|
14559
|
+
deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
|
|
14560
|
+
waitSeq: cmd.seq,
|
|
14561
|
+
token
|
|
14562
|
+
});
|
|
14563
|
+
if (deadline <= this.clock()) {
|
|
14564
|
+
this.resumeSoon(run.id);
|
|
14565
|
+
continue;
|
|
14566
|
+
}
|
|
14567
|
+
}
|
|
14568
|
+
await this.store.putSignalWaiter({
|
|
14569
|
+
token,
|
|
14570
|
+
runId: run.id,
|
|
14571
|
+
seq: cmd.seq
|
|
14572
|
+
});
|
|
14573
|
+
const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
|
|
14574
|
+
if (hit) {
|
|
14575
|
+
await this.store.removeSignalWaiter({
|
|
14576
|
+
token,
|
|
14577
|
+
runId: run.id,
|
|
14578
|
+
seq: cmd.seq
|
|
14579
|
+
});
|
|
14580
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14581
|
+
runId: run.id,
|
|
14582
|
+
seq: cmd.seq,
|
|
14583
|
+
name: `signal:${token}`,
|
|
14584
|
+
kind: "signal",
|
|
14585
|
+
output: hit.payload
|
|
14586
|
+
}));
|
|
14587
|
+
this.resumeSoon(run.id);
|
|
14588
|
+
} else if (deadline != null) {
|
|
14589
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14590
|
+
}
|
|
14591
|
+
} else if (cmd.kind === "spawnChild") {
|
|
14592
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14593
|
+
if (!await this.store.getRun(cmd.childId)) {
|
|
14594
|
+
try {
|
|
14595
|
+
await this.start(cmd.workflow, cmd.input, cmd.childId, {
|
|
14596
|
+
namespace: run.namespace,
|
|
14597
|
+
...inheritTenant(run),
|
|
14598
|
+
...cmd.priority !== void 0 ? {
|
|
14599
|
+
priority: cmd.priority
|
|
14600
|
+
} : {}
|
|
14601
|
+
});
|
|
14602
|
+
} catch (err) {
|
|
14603
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14604
|
+
this.notifyParent(cmd.childId, {
|
|
14605
|
+
ok: false,
|
|
14606
|
+
error: `child workflow "${cmd.workflow}" failed to start: ${message}`
|
|
14607
|
+
});
|
|
14608
|
+
}
|
|
14609
|
+
}
|
|
14610
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14611
|
+
runId: run.id,
|
|
14612
|
+
seq: cmd.seq,
|
|
14613
|
+
name: `spawn:${cmd.childId}`,
|
|
14614
|
+
kind: "local",
|
|
14615
|
+
output: cmd.childId
|
|
14616
|
+
}));
|
|
14617
|
+
} else if (cmd.kind === "continueAsNew") {
|
|
14336
14618
|
} else {
|
|
14337
14619
|
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
14338
14620
|
}
|
|
@@ -14340,6 +14622,245 @@ ${run.workflowVersion}`;
|
|
|
14340
14622
|
return wakeAt;
|
|
14341
14623
|
}
|
|
14342
14624
|
/**
|
|
14625
|
+
* Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
|
|
14626
|
+
* the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
|
|
14627
|
+
* (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
|
|
14628
|
+
* re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
|
|
14629
|
+
* bails).
|
|
14630
|
+
*/
|
|
14631
|
+
async parkRemoteOnSignal(run, seq, token) {
|
|
14632
|
+
const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
|
|
14633
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14634
|
+
runId: run.id,
|
|
14635
|
+
seq,
|
|
14636
|
+
name: `signal:${token}`,
|
|
14637
|
+
kind: "signal",
|
|
14638
|
+
output: payload
|
|
14639
|
+
}));
|
|
14640
|
+
this.resumeSoon(run.id);
|
|
14641
|
+
}, "deliverBuffered");
|
|
14642
|
+
const buffered = await this.store.takeBufferedSignal(token);
|
|
14643
|
+
if (buffered) {
|
|
14644
|
+
await deliverBuffered(buffered.payload);
|
|
14645
|
+
} else {
|
|
14646
|
+
await this.store.putSignalWaiter({
|
|
14647
|
+
token,
|
|
14648
|
+
runId: run.id,
|
|
14649
|
+
seq
|
|
14650
|
+
});
|
|
14651
|
+
const lateBuffered = await this.store.takeBufferedSignal(token);
|
|
14652
|
+
if (lateBuffered) {
|
|
14653
|
+
await this.store.removeSignalWaiter({
|
|
14654
|
+
token,
|
|
14655
|
+
runId: run.id,
|
|
14656
|
+
seq
|
|
14657
|
+
});
|
|
14658
|
+
await deliverBuffered(lateBuffered.payload);
|
|
14659
|
+
}
|
|
14660
|
+
}
|
|
14661
|
+
}
|
|
14662
|
+
/** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
|
|
14663
|
+
resumeSoon(runId) {
|
|
14664
|
+
setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
|
|
14665
|
+
}
|
|
14666
|
+
/**
|
|
14667
|
+
* Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
|
|
14668
|
+
* `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
|
|
14669
|
+
* back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
|
|
14670
|
+
* deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
|
|
14671
|
+
*/
|
|
14672
|
+
async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
|
|
14673
|
+
const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
|
|
14674
|
+
if (recorded?.wakeAt != null) return recorded.wakeAt;
|
|
14675
|
+
const deadline = this.clock() + timeoutMs;
|
|
14676
|
+
const at = /* @__PURE__ */ new Date();
|
|
14677
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14678
|
+
runId,
|
|
14679
|
+
seq: deadlineSeq,
|
|
14680
|
+
name: `timeout:${label}`,
|
|
14681
|
+
kind: "sleep",
|
|
14682
|
+
status: "pending",
|
|
14683
|
+
attempts: 1,
|
|
14684
|
+
input: wait,
|
|
14685
|
+
wakeAt: deadline,
|
|
14686
|
+
enqueuedAt: at,
|
|
14687
|
+
startedAt: at,
|
|
14688
|
+
finishedAt: at
|
|
14689
|
+
}));
|
|
14690
|
+
return deadline;
|
|
14691
|
+
}
|
|
14692
|
+
/**
|
|
14693
|
+
* Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
|
|
14694
|
+
* (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
|
|
14695
|
+
* time out — the in-process wait removes its waiter the same way when it finds the deadline past.
|
|
14696
|
+
* A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
|
|
14697
|
+
* worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
|
|
14698
|
+
*/
|
|
14699
|
+
async expireRemoteDeadlines(runId, checkpoints) {
|
|
14700
|
+
const now = this.clock();
|
|
14701
|
+
const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
|
|
14702
|
+
let out = checkpoints;
|
|
14703
|
+
for (const cp of checkpoints) {
|
|
14704
|
+
if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
|
|
14705
|
+
if (cp.wakeAt == null || cp.wakeAt > now) continue;
|
|
14706
|
+
const wait = cp.input;
|
|
14707
|
+
if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
|
|
14708
|
+
if (settled.has(wait.waitSeq)) continue;
|
|
14709
|
+
await this.store.removeSignalWaiter({
|
|
14710
|
+
token: wait.token,
|
|
14711
|
+
runId,
|
|
14712
|
+
seq: wait.waitSeq
|
|
14713
|
+
});
|
|
14714
|
+
const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
|
|
14715
|
+
if (landed && landed.status === "completed") {
|
|
14716
|
+
out = [
|
|
14717
|
+
...out.filter((c) => c.seq !== landed.seq),
|
|
14718
|
+
landed
|
|
14719
|
+
].sort((a, b) => a.seq - b.seq);
|
|
14720
|
+
}
|
|
14721
|
+
}
|
|
14722
|
+
return out;
|
|
14723
|
+
}
|
|
14724
|
+
/**
|
|
14725
|
+
* Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
|
|
14726
|
+
* see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
|
|
14727
|
+
* candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
|
|
14728
|
+
* {@link eventBufferTtlMs} is set, like the reconcile sweep.
|
|
14729
|
+
*/
|
|
14730
|
+
async claimBufferedEvent(run, name, match) {
|
|
14731
|
+
const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
|
|
14732
|
+
const nowMs = this.clock();
|
|
14733
|
+
for (const candidate of candidates) {
|
|
14734
|
+
if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
|
|
14735
|
+
await this.store.removeBufferedEvent(candidate.id);
|
|
14736
|
+
continue;
|
|
14737
|
+
}
|
|
14738
|
+
const { payload, tenantIds } = readBufferedEvent(candidate.payload);
|
|
14739
|
+
if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
|
|
14740
|
+
if (!eventMatches(payload, match)) continue;
|
|
14741
|
+
if (!await this.store.removeBufferedEvent(candidate.id)) continue;
|
|
14742
|
+
return {
|
|
14743
|
+
payload
|
|
14744
|
+
};
|
|
14745
|
+
}
|
|
14746
|
+
return void 0;
|
|
14747
|
+
}
|
|
14748
|
+
/** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
|
|
14749
|
+
* sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
|
|
14750
|
+
callLease(policy) {
|
|
14751
|
+
if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
|
|
14752
|
+
return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
14753
|
+
}
|
|
14754
|
+
/**
|
|
14755
|
+
* A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
|
|
14756
|
+
* checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
|
|
14757
|
+
* callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
|
|
14758
|
+
* behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
|
|
14759
|
+
* timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
|
|
14760
|
+
* and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
|
|
14761
|
+
*
|
|
14762
|
+
* - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
|
|
14763
|
+
* the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
|
|
14764
|
+
* dispatched again under the same `stepId` with `attempts + 1`.
|
|
14765
|
+
* - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
|
|
14766
|
+
* re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
|
|
14767
|
+
* workflow can catch, as `RemoteStepTimeout` would surface in-process.
|
|
14768
|
+
*/
|
|
14769
|
+
async applyCallPolicy(run, cmd, cp) {
|
|
14770
|
+
const policy = cmd.policy;
|
|
14771
|
+
if (cp.kind !== "remote") return null;
|
|
14772
|
+
if (cp.status === "failed") {
|
|
14773
|
+
if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
|
|
14774
|
+
let due = cp.wakeAt;
|
|
14775
|
+
if (due == null) {
|
|
14776
|
+
due = this.clock() + backoffDelay(cp.attempts, policy);
|
|
14777
|
+
if (due > this.clock()) {
|
|
14778
|
+
await this.store.saveCheckpoint({
|
|
14779
|
+
...cp,
|
|
14780
|
+
wakeAt: due
|
|
14781
|
+
});
|
|
14782
|
+
return due;
|
|
14783
|
+
}
|
|
14784
|
+
}
|
|
14785
|
+
if (this.clock() < due) return due;
|
|
14786
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
|
|
14787
|
+
}
|
|
14788
|
+
if (cp.status !== "pending" || !policy.timeoutMs) return null;
|
|
14789
|
+
if (cp.wakeAt == null) {
|
|
14790
|
+
const lease = this.clock() + policy.timeoutMs;
|
|
14791
|
+
await this.store.saveCheckpoint({
|
|
14792
|
+
...cp,
|
|
14793
|
+
wakeAt: lease
|
|
14794
|
+
});
|
|
14795
|
+
return this.stepLeaseWakeAt(lease);
|
|
14796
|
+
}
|
|
14797
|
+
if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
|
|
14798
|
+
if (cp.attempts < remoteCallMaxAttempts(policy)) {
|
|
14799
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
|
|
14800
|
+
}
|
|
14801
|
+
const id = cp.stepId ?? stepId(run.id, cp.seq);
|
|
14802
|
+
const error2 = {
|
|
14803
|
+
message: new RemoteStepTimeout(id, policy.timeoutMs).message,
|
|
14804
|
+
code: "remote_step_timeout"
|
|
14805
|
+
};
|
|
14806
|
+
await this.store.saveCheckpoint({
|
|
14807
|
+
...cp,
|
|
14808
|
+
status: "failed",
|
|
14809
|
+
error: error2,
|
|
14810
|
+
wakeAt: void 0,
|
|
14811
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
14812
|
+
});
|
|
14813
|
+
this.emit({
|
|
14814
|
+
type: "step.failed",
|
|
14815
|
+
runId: run.id,
|
|
14816
|
+
seq: cp.seq,
|
|
14817
|
+
name: cp.name,
|
|
14818
|
+
kind: "remote",
|
|
14819
|
+
error: error2
|
|
14820
|
+
});
|
|
14821
|
+
return void 0;
|
|
14822
|
+
}
|
|
14823
|
+
/** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
|
|
14824
|
+
async redispatchCall(run, cmd, cp, attempt, policy, why) {
|
|
14825
|
+
const lease = this.callLease(policy);
|
|
14826
|
+
const at = /* @__PURE__ */ new Date();
|
|
14827
|
+
await this.store.saveCheckpoint({
|
|
14828
|
+
...cp,
|
|
14829
|
+
status: "pending",
|
|
14830
|
+
output: void 0,
|
|
14831
|
+
error: void 0,
|
|
14832
|
+
attempts: attempt,
|
|
14833
|
+
wakeAt: lease,
|
|
14834
|
+
enqueuedAt: at,
|
|
14835
|
+
startedAt: at,
|
|
14836
|
+
finishedAt: at,
|
|
14837
|
+
events: why === "timeout" ? [
|
|
14838
|
+
...cp.events ?? [],
|
|
14839
|
+
lostStepEvent(this.clock(), attempt, "redispatched")
|
|
14840
|
+
] : cp.events
|
|
14841
|
+
});
|
|
14842
|
+
await this.dispatchRemoteTask({
|
|
14843
|
+
runId: run.id,
|
|
14844
|
+
seq: cp.seq,
|
|
14845
|
+
name: cmd.name,
|
|
14846
|
+
stepId: cp.stepId ?? stepId(run.id, cp.seq),
|
|
14847
|
+
group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
|
|
14848
|
+
input: cp.input,
|
|
14849
|
+
attempt
|
|
14850
|
+
});
|
|
14851
|
+
this.emit({
|
|
14852
|
+
type: "step.started",
|
|
14853
|
+
runId: run.id,
|
|
14854
|
+
seq: cp.seq,
|
|
14855
|
+
name: cmd.name,
|
|
14856
|
+
kind: "remote",
|
|
14857
|
+
...why === "timeout" ? {
|
|
14858
|
+
redispatched: true
|
|
14859
|
+
} : {}
|
|
14860
|
+
});
|
|
14861
|
+
return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
|
|
14862
|
+
}
|
|
14863
|
+
/**
|
|
14343
14864
|
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
14344
14865
|
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
14345
14866
|
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
@@ -14408,7 +14929,8 @@ ${run.workflowVersion}`;
|
|
|
14408
14929
|
attempt: void 0
|
|
14409
14930
|
}, () => this.runExecutionTurn(run, fn, registered));
|
|
14410
14931
|
}
|
|
14411
|
-
async runExecutionTurn(run,
|
|
14932
|
+
async runExecutionTurn(run, registeredFn, registered) {
|
|
14933
|
+
let fn = registeredFn;
|
|
14412
14934
|
if (run.status === "cancelling") this.cancelRequested.add(run.id);
|
|
14413
14935
|
if (run.status === "pending") {
|
|
14414
14936
|
if (!await this.markStarted(run, registered?.singleton !== void 0)) {
|
|
@@ -14424,8 +14946,10 @@ ${run.workflowVersion}`;
|
|
|
14424
14946
|
});
|
|
14425
14947
|
}
|
|
14426
14948
|
await this.noteServedWakes(run.id);
|
|
14427
|
-
if (run.status !== "cancelling" && registered?.singleton
|
|
14428
|
-
|
|
14949
|
+
if (run.status !== "cancelling" && registered?.singleton) {
|
|
14950
|
+
const admission = await this.singletons.admit(run, registered.singleton, registered.parseInput);
|
|
14951
|
+
if (admission instanceof UnparseableStoredInput) fn = failingBody(admission.error);
|
|
14952
|
+
else if (!admission) return this.singletonWaitResult(run);
|
|
14429
14953
|
}
|
|
14430
14954
|
const stickyOk = this.stickyEligible(run);
|
|
14431
14955
|
let cont = stickyOk ? this.takeContinuation(run, fn) : void 0;
|
|
@@ -18741,6 +19265,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18741
19265
|
PARENT_CLOSE_TAG_PREFIX,
|
|
18742
19266
|
QueueController,
|
|
18743
19267
|
REMOTE_EVENT,
|
|
19268
|
+
REMOTE_WORKFLOW_FEATURES,
|
|
18744
19269
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
18745
19270
|
RUN_VALUE_FACET_LIMIT,
|
|
18746
19271
|
RUN_VALUE_FACET_SCAN,
|
|
@@ -18898,6 +19423,9 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18898
19423
|
readBufferedEvent,
|
|
18899
19424
|
readRunsByIds,
|
|
18900
19425
|
readSearchAttributes,
|
|
19426
|
+
remoteCallMaxAttempts,
|
|
19427
|
+
remoteCallPolicyOf,
|
|
19428
|
+
remoteCallRetries,
|
|
18901
19429
|
requiredCapabilities,
|
|
18902
19430
|
resolveConcurrency,
|
|
18903
19431
|
resolveRouting,
|
|
@@ -18934,6 +19462,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18934
19462
|
singletonAdmissionDenialPatch,
|
|
18935
19463
|
singletonAdmissionTags,
|
|
18936
19464
|
singletonClaimantIds,
|
|
19465
|
+
singletonKeyTag,
|
|
18937
19466
|
sqlComparator,
|
|
18938
19467
|
stepConfigOf,
|
|
18939
19468
|
stepId,
|