@dudousxd/nestjs-durable-core 0.102.0 → 0.104.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +590 -61
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +200 -7
- package/dist/index.d.ts +200 -7
- package/dist/index.js +585 -61
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -881,6 +881,14 @@ function isNonEmptyRunScope(scope) {
|
|
|
881
881
|
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
882
882
|
}
|
|
883
883
|
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
884
|
+
var REMOTE_WORKFLOW_FEATURES = [
|
|
885
|
+
"callPolicy",
|
|
886
|
+
"sleepUntil",
|
|
887
|
+
"waitSignalTimeout",
|
|
888
|
+
"waitEvent",
|
|
889
|
+
"spawnChild",
|
|
890
|
+
"continueAsNew"
|
|
891
|
+
];
|
|
884
892
|
var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
|
|
885
893
|
|
|
886
894
|
// src/events.ts
|
|
@@ -3069,6 +3077,35 @@ function codeAndDetails(err) {
|
|
|
3069
3077
|
}
|
|
3070
3078
|
__name(codeAndDetails, "codeAndDetails");
|
|
3071
3079
|
|
|
3080
|
+
// src/remote-call-policy.ts
|
|
3081
|
+
function remoteCallPolicyOf(config, opts) {
|
|
3082
|
+
const policy = {};
|
|
3083
|
+
const retries = opts?.retries ?? config?.retries;
|
|
3084
|
+
if (retries !== void 0) policy.retries = retries;
|
|
3085
|
+
const backoff = opts?.backoff ?? config?.backoff;
|
|
3086
|
+
if (backoff !== void 0) policy.backoff = backoff;
|
|
3087
|
+
const backoffMs = opts?.backoffMs ?? config?.backoffMs;
|
|
3088
|
+
if (backoffMs !== void 0) policy.backoffMs = backoffMs;
|
|
3089
|
+
const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
|
|
3090
|
+
if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
|
|
3091
|
+
const jitter = opts?.jitter ?? config?.jitter;
|
|
3092
|
+
if (jitter !== void 0) policy.jitter = jitter;
|
|
3093
|
+
const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
|
|
3094
|
+
if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
|
|
3095
|
+
return Object.keys(policy).length > 0 ? policy : void 0;
|
|
3096
|
+
}
|
|
3097
|
+
__name(remoteCallPolicyOf, "remoteCallPolicyOf");
|
|
3098
|
+
function remoteCallMaxAttempts(policy) {
|
|
3099
|
+
return Math.max(1, policy?.retries ?? 1);
|
|
3100
|
+
}
|
|
3101
|
+
__name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
|
|
3102
|
+
function remoteCallRetries(policy, attempts, error2) {
|
|
3103
|
+
if (policy === void 0 || policy.timeoutMs) return false;
|
|
3104
|
+
if (error2?.retryable === false) return false;
|
|
3105
|
+
return attempts < remoteCallMaxAttempts(policy);
|
|
3106
|
+
}
|
|
3107
|
+
__name(remoteCallRetries, "remoteCallRetries");
|
|
3108
|
+
|
|
3072
3109
|
// src/remote-event.ts
|
|
3073
3110
|
var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
|
|
3074
3111
|
function markRemoteEvent(event) {
|
|
@@ -3126,7 +3163,12 @@ var RemoteWorkflowExecutor = class {
|
|
|
3126
3163
|
} : {},
|
|
3127
3164
|
group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
|
|
3128
3165
|
priority: run.priority,
|
|
3129
|
-
attempt: 1
|
|
3166
|
+
attempt: 1,
|
|
3167
|
+
// What this control plane applies beyond the original command kinds, so a worker fails a turn
|
|
3168
|
+
// that needs more instead of emitting a command this engine would not understand.
|
|
3169
|
+
features: [
|
|
3170
|
+
...REMOTE_WORKFLOW_FEATURES
|
|
3171
|
+
]
|
|
3130
3172
|
};
|
|
3131
3173
|
await this.transport.dispatchWorkflowTask(task);
|
|
3132
3174
|
}
|
|
@@ -4566,6 +4608,14 @@ var LEGACY_SCAN_STATUSES = [
|
|
|
4566
4608
|
"cancelling"
|
|
4567
4609
|
];
|
|
4568
4610
|
var byAdmissionOrder = /* @__PURE__ */ __name((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id), "byAdmissionOrder");
|
|
4611
|
+
function singletonKeyTag(tags) {
|
|
4612
|
+
if (!tags) return void 0;
|
|
4613
|
+
for (let i = tags.length - 1; i >= 0; i -= 1) {
|
|
4614
|
+
if (tags[i].startsWith("singleton:")) return tags[i];
|
|
4615
|
+
}
|
|
4616
|
+
return void 0;
|
|
4617
|
+
}
|
|
4618
|
+
__name(singletonKeyTag, "singletonKeyTag");
|
|
4569
4619
|
function withoutSingletonMarkers(tags) {
|
|
4570
4620
|
return tags?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
4571
4621
|
}
|
|
@@ -4594,7 +4644,7 @@ function singletonClaimantIds(rows, tag) {
|
|
|
4594
4644
|
__name(singletonClaimantIds, "singletonClaimantIds");
|
|
4595
4645
|
function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
|
|
4596
4646
|
if (!Number.isInteger(limit) || limit < 1) throw new Error("singleton limit must be a positive integer");
|
|
4597
|
-
const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && run.tags
|
|
4647
|
+
const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && singletonKeyTag(run.tags) === tag);
|
|
4598
4648
|
const candidate = active.find((run) => run.id === runId);
|
|
4599
4649
|
if (!candidate) return null;
|
|
4600
4650
|
if (candidate.tags?.includes(SINGLETON_ADMITTED_TAG)) return candidate.tags;
|
|
@@ -4622,7 +4672,7 @@ async function clearTerminalSingletonAdmission(runId, patch, getRun) {
|
|
|
4622
4672
|
__name(clearTerminalSingletonAdmission, "clearTerminalSingletonAdmission");
|
|
4623
4673
|
function singletonAdmissionDenialPatch(runs, runId, tag, workflow, retryWakeAt) {
|
|
4624
4674
|
if (retryWakeAt === void 0) return void 0;
|
|
4625
|
-
const candidate = runs.find((run) => run.id === runId && run.workflow === workflow && run.tags
|
|
4675
|
+
const candidate = runs.find((run) => run.id === runId && run.workflow === workflow && singletonKeyTag(run.tags) === tag && SINGLETON_ACTIVE_STATUSES.includes(run.status));
|
|
4626
4676
|
if (!candidate || candidate.status === "cancelling") return void 0;
|
|
4627
4677
|
return {
|
|
4628
4678
|
status: "suspended",
|
|
@@ -4671,6 +4721,28 @@ function isLockConflict(error2) {
|
|
|
4671
4721
|
return false;
|
|
4672
4722
|
}
|
|
4673
4723
|
__name(isLockConflict, "isLockConflict");
|
|
4724
|
+
var UnparseableStoredInput = class {
|
|
4725
|
+
static {
|
|
4726
|
+
__name(this, "UnparseableStoredInput");
|
|
4727
|
+
}
|
|
4728
|
+
error;
|
|
4729
|
+
constructor(error2) {
|
|
4730
|
+
this.error = error2;
|
|
4731
|
+
}
|
|
4732
|
+
};
|
|
4733
|
+
async function admissionInput(run, parseInput) {
|
|
4734
|
+
if (!parseInput) return run.input;
|
|
4735
|
+
try {
|
|
4736
|
+
return await parseInput(run.input);
|
|
4737
|
+
} catch (err) {
|
|
4738
|
+
return new UnparseableStoredInput(err instanceof SchemaValidationError && err.code === "invalid_input" ? err : new SchemaValidationError(`workflow "${run.workflow}" input`, [
|
|
4739
|
+
{
|
|
4740
|
+
message: err instanceof Error ? err.message : String(err)
|
|
4741
|
+
}
|
|
4742
|
+
], "invalid_input"));
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
4745
|
+
__name(admissionInput, "admissionInput");
|
|
4674
4746
|
var SingletonGate = class {
|
|
4675
4747
|
static {
|
|
4676
4748
|
__name(this, "SingletonGate");
|
|
@@ -4707,12 +4779,19 @@ var SingletonGate = class {
|
|
|
4707
4779
|
throw new SingletonQueueFullError(workflow, cfg.key(input), cfg.maxQueueDepth);
|
|
4708
4780
|
}
|
|
4709
4781
|
}
|
|
4710
|
-
/**
|
|
4711
|
-
|
|
4782
|
+
/**
|
|
4783
|
+
* Preserve durable holders and atomically claim one of the remaining slots. With `parseInput` (the
|
|
4784
|
+
* workflow's input schema), the key reads the parsed input, the value `run(ctx, input)` gets; a
|
|
4785
|
+
* stored input that no longer parses answers {@link UnparseableStoredInput}, which the caller
|
|
4786
|
+
* must fail the turn with (never run it).
|
|
4787
|
+
*/
|
|
4788
|
+
async admit(run, cfg, parseInput) {
|
|
4712
4789
|
if (!this.deps.store.tryAdmitSingleton) {
|
|
4713
4790
|
throw new Error("Singleton workflows require StateStore.tryAdmitSingleton for atomic durable admission");
|
|
4714
4791
|
}
|
|
4715
|
-
const
|
|
4792
|
+
const keyInput = await admissionInput(run, parseInput);
|
|
4793
|
+
if (keyInput instanceof UnparseableStoredInput) return keyInput;
|
|
4794
|
+
const admitted = await this.claim(run, cfg, this.admissionTag(run, cfg, keyInput, parseInput));
|
|
4716
4795
|
if (admitted && !run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
|
|
4717
4796
|
run.tags = [
|
|
4718
4797
|
...(run.tags ?? []).filter((tag) => tag !== SINGLETON_TRACKED_TAG),
|
|
@@ -4722,17 +4801,43 @@ var SingletonGate = class {
|
|
|
4722
4801
|
return admitted;
|
|
4723
4802
|
}
|
|
4724
4803
|
/**
|
|
4804
|
+
* The exact tag a turn claims under: the key of the parsed input, the tag `start` stamps. A run
|
|
4805
|
+
* queued by an engine that predates `parseInput` was stamped with the key of its RAW input
|
|
4806
|
+
* instead; when the run carries that tag and not the parsed one, it claims under it, so an
|
|
4807
|
+
* upgrade does not strand it. Both are computed from this run's own input, never read off a tag.
|
|
4808
|
+
* Only that case calls the key with the raw input (what the older engine always did), and only
|
|
4809
|
+
* for a run that carries a `singleton:` tag at all; a key that throws on it gets the parsed tag.
|
|
4810
|
+
*/
|
|
4811
|
+
admissionTag(run, cfg, keyInput, parseInput) {
|
|
4812
|
+
const parsed = this.tag(cfg, keyInput);
|
|
4813
|
+
const stamped = singletonKeyTag(run.tags);
|
|
4814
|
+
if (!parseInput || stamped === void 0 || stamped === parsed) return parsed;
|
|
4815
|
+
let raw;
|
|
4816
|
+
try {
|
|
4817
|
+
raw = this.tag(cfg, run.input);
|
|
4818
|
+
} catch {
|
|
4819
|
+
return parsed;
|
|
4820
|
+
}
|
|
4821
|
+
return stamped === raw ? raw : parsed;
|
|
4822
|
+
}
|
|
4823
|
+
/**
|
|
4725
4824
|
* One claim, retried when the database aborts it as a lock conflict. A conflict that outlasts the
|
|
4726
4825
|
* retries answers "not admitted", so the engine releases this turn's lease, instead of stranding
|
|
4727
4826
|
* the run until the lease lapses. It writes nothing to the run: this turn's lease may have lapsed
|
|
4728
4827
|
* during the claim, and another instance may now hold the run, its slot and newer tags. A later
|
|
4729
4828
|
* sweep re-drives the run: recovery once it is unleased, or its timer if it is parked.
|
|
4829
|
+
*
|
|
4830
|
+
* The claim is under the tag the key computes NOW, the one `start` stamped from the same input
|
|
4831
|
+
* (the schema is deterministic). The store admits a run only when it carries that exact tag, so
|
|
4832
|
+
* a run whose stored tags lack it (a caller tag from before callers' `singleton:` tags were
|
|
4833
|
+
* dropped, a run started before the workflow became a singleton) is never admitted under some
|
|
4834
|
+
* other key; it waits instead.
|
|
4730
4835
|
*/
|
|
4731
|
-
async claim(run, cfg) {
|
|
4836
|
+
async claim(run, cfg, tag) {
|
|
4732
4837
|
const admit = this.deps.store.tryAdmitSingleton;
|
|
4733
4838
|
for (let attempt = 1; ; attempt += 1) {
|
|
4734
4839
|
try {
|
|
4735
|
-
return await admit.call(this.deps.store, run.id,
|
|
4840
|
+
return await admit.call(this.deps.store, run.id, tag, run.workflow, cfg.limit ?? 1, this.retryWakeAt());
|
|
4736
4841
|
} catch (error2) {
|
|
4737
4842
|
if (!isLockConflict(error2)) throw error2;
|
|
4738
4843
|
if (attempt >= ADMISSION_ATTEMPTS) return false;
|
|
@@ -4750,7 +4855,7 @@ var SingletonGate = class {
|
|
|
4750
4855
|
async wakeNext(settled) {
|
|
4751
4856
|
const cfg = this.deps.configFor(settled);
|
|
4752
4857
|
if (!cfg) return;
|
|
4753
|
-
const tag = settled.tags
|
|
4858
|
+
const tag = singletonKeyTag(settled.tags);
|
|
4754
4859
|
if (!tag) return;
|
|
4755
4860
|
const gated = (await this.deps.store.listRuns({
|
|
4756
4861
|
tag,
|
|
@@ -6899,9 +7004,30 @@ function tenantScopedKey(namespace, key) {
|
|
|
6899
7004
|
return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
|
|
6900
7005
|
}
|
|
6901
7006
|
__name(tenantScopedKey, "tenantScopedKey");
|
|
7007
|
+
function failingBody(error2) {
|
|
7008
|
+
return async () => {
|
|
7009
|
+
throw error2;
|
|
7010
|
+
};
|
|
7011
|
+
}
|
|
7012
|
+
__name(failingBody, "failingBody");
|
|
7013
|
+
function storedInputError(invalid) {
|
|
7014
|
+
return {
|
|
7015
|
+
message: invalid.error.message,
|
|
7016
|
+
code: "invalid_input",
|
|
7017
|
+
retryable: false,
|
|
7018
|
+
details: {
|
|
7019
|
+
issues: invalid.error.issues
|
|
7020
|
+
}
|
|
7021
|
+
};
|
|
7022
|
+
}
|
|
7023
|
+
__name(storedInputError, "storedInputError");
|
|
7024
|
+
function callerTags(tags) {
|
|
7025
|
+
return tags?.filter((tag) => !tag.startsWith(SINGLETON_TAG_PREFIX));
|
|
7026
|
+
}
|
|
7027
|
+
__name(callerTags, "callerTags");
|
|
7028
|
+
var SINGLETON_TAG_PREFIX = "singleton:";
|
|
6902
7029
|
function singletonKeyOf(run) {
|
|
6903
|
-
|
|
6904
|
-
return tag?.slice("singleton:".length);
|
|
7030
|
+
return singletonKeyTag(run.tags)?.slice("singleton:".length);
|
|
6905
7031
|
}
|
|
6906
7032
|
__name(singletonKeyOf, "singletonKeyOf");
|
|
6907
7033
|
function conventionSingleton(run, announced, wire) {
|
|
@@ -7921,6 +8047,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7921
8047
|
concurrency: opts?.concurrency,
|
|
7922
8048
|
executionTimeoutMs: opts?.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
|
|
7923
8049
|
validateInput: opts?.validateInput,
|
|
8050
|
+
parseInput: opts?.parseInput,
|
|
7924
8051
|
searchAttributesSchema: opts?.searchAttributesSchema,
|
|
7925
8052
|
onEvent: opts?.onEvent,
|
|
7926
8053
|
eventBatch: opts?.eventBatch,
|
|
@@ -9622,24 +9749,31 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9622
9749
|
if (!registered) {
|
|
9623
9750
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
9624
9751
|
}
|
|
9625
|
-
|
|
9752
|
+
const check = registered;
|
|
9753
|
+
const parseAndValidate = /* @__PURE__ */ __name(async () => {
|
|
9754
|
+
const parsed = check.parseInput ? await check.parseInput(input) : input;
|
|
9755
|
+
await check.validateInput?.(parsed);
|
|
9756
|
+
return parsed;
|
|
9757
|
+
}, "parseAndValidate");
|
|
9758
|
+
let keyInput;
|
|
9759
|
+
if (parkUnroutable && (registered.validateInput || registered.parseInput)) {
|
|
9626
9760
|
try {
|
|
9627
|
-
await
|
|
9761
|
+
keyInput = await parseAndValidate();
|
|
9628
9762
|
} catch (err) {
|
|
9629
9763
|
throw new TenantStartInvalid(err);
|
|
9630
9764
|
}
|
|
9631
9765
|
} else {
|
|
9632
|
-
await
|
|
9766
|
+
keyInput = await parseAndValidate();
|
|
9633
9767
|
}
|
|
9634
9768
|
const prior = await this.store.getRun(runId);
|
|
9635
9769
|
if (prior) return this.existingStartResult(prior);
|
|
9636
|
-
const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency,
|
|
9770
|
+
const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, keyInput);
|
|
9637
9771
|
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9638
9772
|
const now = /* @__PURE__ */ new Date();
|
|
9639
9773
|
const stampedTags = [
|
|
9640
|
-
...opts?.tags ?? [],
|
|
9774
|
+
...callerTags(opts?.tags) ?? [],
|
|
9641
9775
|
...registered.singleton ? [
|
|
9642
|
-
this.singletons.tag(registered.singleton,
|
|
9776
|
+
this.singletons.tag(registered.singleton, keyInput)
|
|
9643
9777
|
] : [],
|
|
9644
9778
|
...quota ? [
|
|
9645
9779
|
concurrencyTag(quota.key)
|
|
@@ -9648,9 +9782,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9648
9782
|
VERSION_UNDECLARED_TAG
|
|
9649
9783
|
] : []
|
|
9650
9784
|
];
|
|
9651
|
-
const tags = mergeTags(registered.tags, stampedTags)?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
9785
|
+
const tags = mergeTags(callerTags(registered.tags), stampedTags)?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
9652
9786
|
if (registered.singleton) {
|
|
9653
|
-
await this.singletons.assertCapacity(name, registered.singleton,
|
|
9787
|
+
await this.singletons.assertCapacity(name, registered.singleton, keyInput);
|
|
9654
9788
|
}
|
|
9655
9789
|
const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
|
|
9656
9790
|
const delayed = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > this.clock();
|
|
@@ -9739,7 +9873,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9739
9873
|
};
|
|
9740
9874
|
const now = /* @__PURE__ */ new Date();
|
|
9741
9875
|
const tags = mergeTags(void 0, [
|
|
9742
|
-
...opts?.tags ?? [],
|
|
9876
|
+
...callerTags(opts?.tags) ?? [],
|
|
9743
9877
|
...singleton ? [
|
|
9744
9878
|
this.singletons.tag(singleton, input)
|
|
9745
9879
|
] : [],
|
|
@@ -9919,7 +10053,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9919
10053
|
namespace: poolNamespace(opts.namespace) ?? this.namespace,
|
|
9920
10054
|
input,
|
|
9921
10055
|
// The same derived tags an admitted or parked start carries, so quota-key queries find it.
|
|
9922
|
-
tags: mergeTags(opts.tags, [
|
|
10056
|
+
tags: mergeTags(callerTags(opts.tags), [
|
|
9923
10057
|
...opts.concurrency ? [
|
|
9924
10058
|
concurrencyTag(opts.concurrency.key)
|
|
9925
10059
|
] : [],
|
|
@@ -13446,14 +13580,23 @@ ${run.workflowVersion}`;
|
|
|
13446
13580
|
...await this.sourceTraceOf(run)
|
|
13447
13581
|
});
|
|
13448
13582
|
}
|
|
13449
|
-
if (registered.singleton
|
|
13450
|
-
|
|
13583
|
+
if (registered.singleton) {
|
|
13584
|
+
const admission = await this.singletons.admit(run, registered.singleton, registered.parseInput);
|
|
13585
|
+
if (admission instanceof UnparseableStoredInput) {
|
|
13586
|
+
const result = await this.settleRun(run, {
|
|
13587
|
+
kind: "failed",
|
|
13588
|
+
error: storedInputError(admission)
|
|
13589
|
+
});
|
|
13590
|
+
await this.releaseLeaseBestEffort(run.id);
|
|
13591
|
+
return result;
|
|
13592
|
+
}
|
|
13593
|
+
if (!admission) return this.singletonWaitResult(run);
|
|
13451
13594
|
}
|
|
13452
13595
|
const wfToken = tenantGroup(sanitizeQueueToken(remote.group), void 0);
|
|
13453
13596
|
let wfPlan = await this.planRoute(wfToken, registered.requires);
|
|
13454
13597
|
if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
|
|
13455
13598
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
13456
|
-
const checkpoints = await this.replayCheckpoints(run.id);
|
|
13599
|
+
const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
|
|
13457
13600
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
13458
13601
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
13459
13602
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -13539,6 +13682,10 @@ ${run.workflowVersion}`;
|
|
|
13539
13682
|
*/
|
|
13540
13683
|
async applyDecision(run, decision) {
|
|
13541
13684
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
13685
|
+
const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
|
|
13686
|
+
if (handoff !== void 0 && decision.status === "completed") {
|
|
13687
|
+
return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
|
|
13688
|
+
}
|
|
13542
13689
|
if (decision.status === "completed") {
|
|
13543
13690
|
await this.applyCommands(run, decision.commands);
|
|
13544
13691
|
return this.settleRun(run, {
|
|
@@ -13585,6 +13732,63 @@ ${run.workflowVersion}`;
|
|
|
13585
13732
|
return suspended;
|
|
13586
13733
|
}
|
|
13587
13734
|
/**
|
|
13735
|
+
* A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
|
|
13736
|
+
* run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
|
|
13737
|
+
* engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
|
|
13738
|
+
* turn replayed stays cancelled and starts nothing.
|
|
13739
|
+
*/
|
|
13740
|
+
async applyRemoteContinueAsNew(run, commands, input) {
|
|
13741
|
+
await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
|
|
13742
|
+
const fresh = await this.store.getRun(run.id);
|
|
13743
|
+
if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
|
|
13744
|
+
return {
|
|
13745
|
+
runId: run.id,
|
|
13746
|
+
status: fresh.status,
|
|
13747
|
+
output: fresh.output,
|
|
13748
|
+
error: fresh.error
|
|
13749
|
+
};
|
|
13750
|
+
}
|
|
13751
|
+
const nextId = nextContinuationId(run.id);
|
|
13752
|
+
try {
|
|
13753
|
+
await this.start(run.workflow, input, nextId, {
|
|
13754
|
+
namespace: run.namespace,
|
|
13755
|
+
...inheritTenant(run)
|
|
13756
|
+
});
|
|
13757
|
+
} catch (err) {
|
|
13758
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13759
|
+
return this.settleRun(run, {
|
|
13760
|
+
kind: "failed",
|
|
13761
|
+
error: {
|
|
13762
|
+
message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
|
|
13763
|
+
code: "continue_as_new_failed"
|
|
13764
|
+
}
|
|
13765
|
+
});
|
|
13766
|
+
}
|
|
13767
|
+
await this.store.updateRun(run.id, {
|
|
13768
|
+
status: "completed",
|
|
13769
|
+
output: void 0,
|
|
13770
|
+
error: void 0,
|
|
13771
|
+
wakeAt: void 0,
|
|
13772
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
13773
|
+
});
|
|
13774
|
+
this.emit({
|
|
13775
|
+
type: "run.completed",
|
|
13776
|
+
runId: run.id,
|
|
13777
|
+
workflow: run.workflow,
|
|
13778
|
+
namespace: run.namespace
|
|
13779
|
+
}, {
|
|
13780
|
+
lifecycle: false
|
|
13781
|
+
});
|
|
13782
|
+
void this.notifyParent(run.id, {
|
|
13783
|
+
ok: true,
|
|
13784
|
+
value: void 0
|
|
13785
|
+
});
|
|
13786
|
+
return {
|
|
13787
|
+
runId: run.id,
|
|
13788
|
+
status: "completed"
|
|
13789
|
+
};
|
|
13790
|
+
}
|
|
13791
|
+
/**
|
|
13588
13792
|
* Re-drive a run whose turn was computed from a history OLDER than the store's.
|
|
13589
13793
|
*
|
|
13590
13794
|
* A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
|
|
@@ -13602,11 +13806,12 @@ ${run.workflowVersion}`;
|
|
|
13602
13806
|
* Scheduled off the current stack so it runs after the caller released the run lease.
|
|
13603
13807
|
*/
|
|
13604
13808
|
async redriveIfTurnSawStaleHistory(runId, commands) {
|
|
13605
|
-
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
|
|
13809
|
+
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
|
|
13606
13810
|
if (blocking.length === 0) return;
|
|
13607
13811
|
for (const cmd of blocking) {
|
|
13608
13812
|
const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
|
|
13609
|
-
|
|
13813
|
+
const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
|
|
13814
|
+
if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
|
|
13610
13815
|
setTimeout(() => {
|
|
13611
13816
|
void this.resumeImplicit(runId).catch(() => void 0);
|
|
13612
13817
|
}, 0).unref?.();
|
|
@@ -13855,7 +14060,12 @@ ${run.workflowVersion}`;
|
|
|
13855
14060
|
// name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
|
|
13856
14061
|
name: cp.kind === "signal" ? void 0 : cp.name,
|
|
13857
14062
|
output: cp.status === "completed" ? cp.output : void 0,
|
|
13858
|
-
error: cp.status === "failed" ? cp.error : void 0
|
|
14063
|
+
error: cp.status === "failed" ? cp.error : void 0,
|
|
14064
|
+
// How many times a dispatched step ran: a worker applying a `call` policy decides a retry
|
|
14065
|
+
// from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
|
|
14066
|
+
...cp.kind === "remote" ? {
|
|
14067
|
+
attempts: cp.attempts
|
|
14068
|
+
} : {}
|
|
13859
14069
|
});
|
|
13860
14070
|
} else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
|
|
13861
14071
|
events.push({
|
|
@@ -13912,11 +14122,12 @@ ${run.workflowVersion}`;
|
|
|
13912
14122
|
} else if (cmd.kind === "call") {
|
|
13913
14123
|
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
13914
14124
|
if (existing) {
|
|
13915
|
-
const
|
|
14125
|
+
const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
|
|
14126
|
+
const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
|
|
13916
14127
|
if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
|
|
13917
14128
|
continue;
|
|
13918
14129
|
}
|
|
13919
|
-
const lease = this.
|
|
14130
|
+
const lease = this.callLease(cmd.policy);
|
|
13920
14131
|
await this.store.saveCheckpoint(stepCheckpoint({
|
|
13921
14132
|
runId: run.id,
|
|
13922
14133
|
seq: cmd.seq,
|
|
@@ -13976,35 +14187,7 @@ ${run.workflowVersion}`;
|
|
|
13976
14187
|
}));
|
|
13977
14188
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
13978
14189
|
} else if (cmd.kind === "waitSignal") {
|
|
13979
|
-
|
|
13980
|
-
await this.store.saveCheckpoint(instantCheckpoint({
|
|
13981
|
-
runId: run.id,
|
|
13982
|
-
seq: cmd.seq,
|
|
13983
|
-
name: `signal:${cmd.signal}`,
|
|
13984
|
-
kind: "signal",
|
|
13985
|
-
output: payload
|
|
13986
|
-
}));
|
|
13987
|
-
setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
|
|
13988
|
-
}, "deliverBuffered");
|
|
13989
|
-
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
13990
|
-
if (buffered) {
|
|
13991
|
-
await deliverBuffered(buffered.payload);
|
|
13992
|
-
} else {
|
|
13993
|
-
await this.store.putSignalWaiter({
|
|
13994
|
-
token: cmd.signal,
|
|
13995
|
-
runId: run.id,
|
|
13996
|
-
seq: cmd.seq
|
|
13997
|
-
});
|
|
13998
|
-
const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
13999
|
-
if (lateBuffered) {
|
|
14000
|
-
await this.store.removeSignalWaiter({
|
|
14001
|
-
token: cmd.signal,
|
|
14002
|
-
runId: run.id,
|
|
14003
|
-
seq: cmd.seq
|
|
14004
|
-
});
|
|
14005
|
-
await deliverBuffered(lateBuffered.payload);
|
|
14006
|
-
}
|
|
14007
|
-
}
|
|
14190
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14008
14191
|
} else if (cmd.kind === "waitAny") {
|
|
14009
14192
|
const state = await enterWaitAny({
|
|
14010
14193
|
store: this.store,
|
|
@@ -14036,6 +14219,100 @@ ${run.workflowVersion}`;
|
|
|
14036
14219
|
} else {
|
|
14037
14220
|
this.redeliverFinishedChild(childId, existingChild);
|
|
14038
14221
|
}
|
|
14222
|
+
} else if (cmd.kind === "sleepUntil") {
|
|
14223
|
+
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
14224
|
+
const deadline = existing?.wakeAt ?? cmd.until;
|
|
14225
|
+
if (!existing) {
|
|
14226
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14227
|
+
runId: run.id,
|
|
14228
|
+
seq: cmd.seq,
|
|
14229
|
+
name: `sleep:${cmd.seq}`,
|
|
14230
|
+
kind: "sleep",
|
|
14231
|
+
status: "pending",
|
|
14232
|
+
attempts: 1,
|
|
14233
|
+
wakeAt: deadline,
|
|
14234
|
+
enqueuedAt: at,
|
|
14235
|
+
startedAt: at,
|
|
14236
|
+
finishedAt: at
|
|
14237
|
+
}));
|
|
14238
|
+
}
|
|
14239
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14240
|
+
} else if (cmd.kind === "waitSignalTimeout") {
|
|
14241
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14242
|
+
const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
|
|
14243
|
+
waitSeq: cmd.seq,
|
|
14244
|
+
token: cmd.signal
|
|
14245
|
+
});
|
|
14246
|
+
if (deadline <= this.clock()) {
|
|
14247
|
+
this.resumeSoon(run.id);
|
|
14248
|
+
continue;
|
|
14249
|
+
}
|
|
14250
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14251
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14252
|
+
} else if (cmd.kind === "waitEvent") {
|
|
14253
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14254
|
+
const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
|
|
14255
|
+
let deadline;
|
|
14256
|
+
if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
|
|
14257
|
+
deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
|
|
14258
|
+
waitSeq: cmd.seq,
|
|
14259
|
+
token
|
|
14260
|
+
});
|
|
14261
|
+
if (deadline <= this.clock()) {
|
|
14262
|
+
this.resumeSoon(run.id);
|
|
14263
|
+
continue;
|
|
14264
|
+
}
|
|
14265
|
+
}
|
|
14266
|
+
await this.store.putSignalWaiter({
|
|
14267
|
+
token,
|
|
14268
|
+
runId: run.id,
|
|
14269
|
+
seq: cmd.seq
|
|
14270
|
+
});
|
|
14271
|
+
const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
|
|
14272
|
+
if (hit) {
|
|
14273
|
+
await this.store.removeSignalWaiter({
|
|
14274
|
+
token,
|
|
14275
|
+
runId: run.id,
|
|
14276
|
+
seq: cmd.seq
|
|
14277
|
+
});
|
|
14278
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14279
|
+
runId: run.id,
|
|
14280
|
+
seq: cmd.seq,
|
|
14281
|
+
name: `signal:${token}`,
|
|
14282
|
+
kind: "signal",
|
|
14283
|
+
output: hit.payload
|
|
14284
|
+
}));
|
|
14285
|
+
this.resumeSoon(run.id);
|
|
14286
|
+
} else if (deadline != null) {
|
|
14287
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14288
|
+
}
|
|
14289
|
+
} else if (cmd.kind === "spawnChild") {
|
|
14290
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14291
|
+
if (!await this.store.getRun(cmd.childId)) {
|
|
14292
|
+
try {
|
|
14293
|
+
await this.start(cmd.workflow, cmd.input, cmd.childId, {
|
|
14294
|
+
namespace: run.namespace,
|
|
14295
|
+
...inheritTenant(run),
|
|
14296
|
+
...cmd.priority !== void 0 ? {
|
|
14297
|
+
priority: cmd.priority
|
|
14298
|
+
} : {}
|
|
14299
|
+
});
|
|
14300
|
+
} catch (err) {
|
|
14301
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14302
|
+
this.notifyParent(cmd.childId, {
|
|
14303
|
+
ok: false,
|
|
14304
|
+
error: `child workflow "${cmd.workflow}" failed to start: ${message}`
|
|
14305
|
+
});
|
|
14306
|
+
}
|
|
14307
|
+
}
|
|
14308
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14309
|
+
runId: run.id,
|
|
14310
|
+
seq: cmd.seq,
|
|
14311
|
+
name: `spawn:${cmd.childId}`,
|
|
14312
|
+
kind: "local",
|
|
14313
|
+
output: cmd.childId
|
|
14314
|
+
}));
|
|
14315
|
+
} else if (cmd.kind === "continueAsNew") {
|
|
14039
14316
|
} else {
|
|
14040
14317
|
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
14041
14318
|
}
|
|
@@ -14043,6 +14320,245 @@ ${run.workflowVersion}`;
|
|
|
14043
14320
|
return wakeAt;
|
|
14044
14321
|
}
|
|
14045
14322
|
/**
|
|
14323
|
+
* Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
|
|
14324
|
+
* the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
|
|
14325
|
+
* (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
|
|
14326
|
+
* re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
|
|
14327
|
+
* bails).
|
|
14328
|
+
*/
|
|
14329
|
+
async parkRemoteOnSignal(run, seq, token) {
|
|
14330
|
+
const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
|
|
14331
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14332
|
+
runId: run.id,
|
|
14333
|
+
seq,
|
|
14334
|
+
name: `signal:${token}`,
|
|
14335
|
+
kind: "signal",
|
|
14336
|
+
output: payload
|
|
14337
|
+
}));
|
|
14338
|
+
this.resumeSoon(run.id);
|
|
14339
|
+
}, "deliverBuffered");
|
|
14340
|
+
const buffered = await this.store.takeBufferedSignal(token);
|
|
14341
|
+
if (buffered) {
|
|
14342
|
+
await deliverBuffered(buffered.payload);
|
|
14343
|
+
} else {
|
|
14344
|
+
await this.store.putSignalWaiter({
|
|
14345
|
+
token,
|
|
14346
|
+
runId: run.id,
|
|
14347
|
+
seq
|
|
14348
|
+
});
|
|
14349
|
+
const lateBuffered = await this.store.takeBufferedSignal(token);
|
|
14350
|
+
if (lateBuffered) {
|
|
14351
|
+
await this.store.removeSignalWaiter({
|
|
14352
|
+
token,
|
|
14353
|
+
runId: run.id,
|
|
14354
|
+
seq
|
|
14355
|
+
});
|
|
14356
|
+
await deliverBuffered(lateBuffered.payload);
|
|
14357
|
+
}
|
|
14358
|
+
}
|
|
14359
|
+
}
|
|
14360
|
+
/** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
|
|
14361
|
+
resumeSoon(runId) {
|
|
14362
|
+
setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
|
|
14363
|
+
}
|
|
14364
|
+
/**
|
|
14365
|
+
* Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
|
|
14366
|
+
* `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
|
|
14367
|
+
* back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
|
|
14368
|
+
* deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
|
|
14369
|
+
*/
|
|
14370
|
+
async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
|
|
14371
|
+
const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
|
|
14372
|
+
if (recorded?.wakeAt != null) return recorded.wakeAt;
|
|
14373
|
+
const deadline = this.clock() + timeoutMs;
|
|
14374
|
+
const at = /* @__PURE__ */ new Date();
|
|
14375
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14376
|
+
runId,
|
|
14377
|
+
seq: deadlineSeq,
|
|
14378
|
+
name: `timeout:${label}`,
|
|
14379
|
+
kind: "sleep",
|
|
14380
|
+
status: "pending",
|
|
14381
|
+
attempts: 1,
|
|
14382
|
+
input: wait,
|
|
14383
|
+
wakeAt: deadline,
|
|
14384
|
+
enqueuedAt: at,
|
|
14385
|
+
startedAt: at,
|
|
14386
|
+
finishedAt: at
|
|
14387
|
+
}));
|
|
14388
|
+
return deadline;
|
|
14389
|
+
}
|
|
14390
|
+
/**
|
|
14391
|
+
* Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
|
|
14392
|
+
* (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
|
|
14393
|
+
* time out — the in-process wait removes its waiter the same way when it finds the deadline past.
|
|
14394
|
+
* A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
|
|
14395
|
+
* worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
|
|
14396
|
+
*/
|
|
14397
|
+
async expireRemoteDeadlines(runId, checkpoints) {
|
|
14398
|
+
const now = this.clock();
|
|
14399
|
+
const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
|
|
14400
|
+
let out = checkpoints;
|
|
14401
|
+
for (const cp of checkpoints) {
|
|
14402
|
+
if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
|
|
14403
|
+
if (cp.wakeAt == null || cp.wakeAt > now) continue;
|
|
14404
|
+
const wait = cp.input;
|
|
14405
|
+
if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
|
|
14406
|
+
if (settled.has(wait.waitSeq)) continue;
|
|
14407
|
+
await this.store.removeSignalWaiter({
|
|
14408
|
+
token: wait.token,
|
|
14409
|
+
runId,
|
|
14410
|
+
seq: wait.waitSeq
|
|
14411
|
+
});
|
|
14412
|
+
const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
|
|
14413
|
+
if (landed && landed.status === "completed") {
|
|
14414
|
+
out = [
|
|
14415
|
+
...out.filter((c) => c.seq !== landed.seq),
|
|
14416
|
+
landed
|
|
14417
|
+
].sort((a, b) => a.seq - b.seq);
|
|
14418
|
+
}
|
|
14419
|
+
}
|
|
14420
|
+
return out;
|
|
14421
|
+
}
|
|
14422
|
+
/**
|
|
14423
|
+
* Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
|
|
14424
|
+
* see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
|
|
14425
|
+
* candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
|
|
14426
|
+
* {@link eventBufferTtlMs} is set, like the reconcile sweep.
|
|
14427
|
+
*/
|
|
14428
|
+
async claimBufferedEvent(run, name, match) {
|
|
14429
|
+
const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
|
|
14430
|
+
const nowMs = this.clock();
|
|
14431
|
+
for (const candidate of candidates) {
|
|
14432
|
+
if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
|
|
14433
|
+
await this.store.removeBufferedEvent(candidate.id);
|
|
14434
|
+
continue;
|
|
14435
|
+
}
|
|
14436
|
+
const { payload, tenantIds } = readBufferedEvent(candidate.payload);
|
|
14437
|
+
if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
|
|
14438
|
+
if (!eventMatches(payload, match)) continue;
|
|
14439
|
+
if (!await this.store.removeBufferedEvent(candidate.id)) continue;
|
|
14440
|
+
return {
|
|
14441
|
+
payload
|
|
14442
|
+
};
|
|
14443
|
+
}
|
|
14444
|
+
return void 0;
|
|
14445
|
+
}
|
|
14446
|
+
/** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
|
|
14447
|
+
* sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
|
|
14448
|
+
callLease(policy) {
|
|
14449
|
+
if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
|
|
14450
|
+
return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
14451
|
+
}
|
|
14452
|
+
/**
|
|
14453
|
+
* A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
|
|
14454
|
+
* checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
|
|
14455
|
+
* callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
|
|
14456
|
+
* behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
|
|
14457
|
+
* timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
|
|
14458
|
+
* and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
|
|
14459
|
+
*
|
|
14460
|
+
* - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
|
|
14461
|
+
* the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
|
|
14462
|
+
* dispatched again under the same `stepId` with `attempts + 1`.
|
|
14463
|
+
* - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
|
|
14464
|
+
* re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
|
|
14465
|
+
* workflow can catch, as `RemoteStepTimeout` would surface in-process.
|
|
14466
|
+
*/
|
|
14467
|
+
async applyCallPolicy(run, cmd, cp) {
|
|
14468
|
+
const policy = cmd.policy;
|
|
14469
|
+
if (cp.kind !== "remote") return null;
|
|
14470
|
+
if (cp.status === "failed") {
|
|
14471
|
+
if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
|
|
14472
|
+
let due = cp.wakeAt;
|
|
14473
|
+
if (due == null) {
|
|
14474
|
+
due = this.clock() + backoffDelay(cp.attempts, policy);
|
|
14475
|
+
if (due > this.clock()) {
|
|
14476
|
+
await this.store.saveCheckpoint({
|
|
14477
|
+
...cp,
|
|
14478
|
+
wakeAt: due
|
|
14479
|
+
});
|
|
14480
|
+
return due;
|
|
14481
|
+
}
|
|
14482
|
+
}
|
|
14483
|
+
if (this.clock() < due) return due;
|
|
14484
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
|
|
14485
|
+
}
|
|
14486
|
+
if (cp.status !== "pending" || !policy.timeoutMs) return null;
|
|
14487
|
+
if (cp.wakeAt == null) {
|
|
14488
|
+
const lease = this.clock() + policy.timeoutMs;
|
|
14489
|
+
await this.store.saveCheckpoint({
|
|
14490
|
+
...cp,
|
|
14491
|
+
wakeAt: lease
|
|
14492
|
+
});
|
|
14493
|
+
return this.stepLeaseWakeAt(lease);
|
|
14494
|
+
}
|
|
14495
|
+
if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
|
|
14496
|
+
if (cp.attempts < remoteCallMaxAttempts(policy)) {
|
|
14497
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
|
|
14498
|
+
}
|
|
14499
|
+
const id = cp.stepId ?? stepId(run.id, cp.seq);
|
|
14500
|
+
const error2 = {
|
|
14501
|
+
message: new RemoteStepTimeout(id, policy.timeoutMs).message,
|
|
14502
|
+
code: "remote_step_timeout"
|
|
14503
|
+
};
|
|
14504
|
+
await this.store.saveCheckpoint({
|
|
14505
|
+
...cp,
|
|
14506
|
+
status: "failed",
|
|
14507
|
+
error: error2,
|
|
14508
|
+
wakeAt: void 0,
|
|
14509
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
14510
|
+
});
|
|
14511
|
+
this.emit({
|
|
14512
|
+
type: "step.failed",
|
|
14513
|
+
runId: run.id,
|
|
14514
|
+
seq: cp.seq,
|
|
14515
|
+
name: cp.name,
|
|
14516
|
+
kind: "remote",
|
|
14517
|
+
error: error2
|
|
14518
|
+
});
|
|
14519
|
+
return void 0;
|
|
14520
|
+
}
|
|
14521
|
+
/** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
|
|
14522
|
+
async redispatchCall(run, cmd, cp, attempt, policy, why) {
|
|
14523
|
+
const lease = this.callLease(policy);
|
|
14524
|
+
const at = /* @__PURE__ */ new Date();
|
|
14525
|
+
await this.store.saveCheckpoint({
|
|
14526
|
+
...cp,
|
|
14527
|
+
status: "pending",
|
|
14528
|
+
output: void 0,
|
|
14529
|
+
error: void 0,
|
|
14530
|
+
attempts: attempt,
|
|
14531
|
+
wakeAt: lease,
|
|
14532
|
+
enqueuedAt: at,
|
|
14533
|
+
startedAt: at,
|
|
14534
|
+
finishedAt: at,
|
|
14535
|
+
events: why === "timeout" ? [
|
|
14536
|
+
...cp.events ?? [],
|
|
14537
|
+
lostStepEvent(this.clock(), attempt, "redispatched")
|
|
14538
|
+
] : cp.events
|
|
14539
|
+
});
|
|
14540
|
+
await this.dispatchRemoteTask({
|
|
14541
|
+
runId: run.id,
|
|
14542
|
+
seq: cp.seq,
|
|
14543
|
+
name: cmd.name,
|
|
14544
|
+
stepId: cp.stepId ?? stepId(run.id, cp.seq),
|
|
14545
|
+
group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
|
|
14546
|
+
input: cp.input,
|
|
14547
|
+
attempt
|
|
14548
|
+
});
|
|
14549
|
+
this.emit({
|
|
14550
|
+
type: "step.started",
|
|
14551
|
+
runId: run.id,
|
|
14552
|
+
seq: cp.seq,
|
|
14553
|
+
name: cmd.name,
|
|
14554
|
+
kind: "remote",
|
|
14555
|
+
...why === "timeout" ? {
|
|
14556
|
+
redispatched: true
|
|
14557
|
+
} : {}
|
|
14558
|
+
});
|
|
14559
|
+
return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
|
|
14560
|
+
}
|
|
14561
|
+
/**
|
|
14046
14562
|
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
14047
14563
|
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
14048
14564
|
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
@@ -14111,7 +14627,8 @@ ${run.workflowVersion}`;
|
|
|
14111
14627
|
attempt: void 0
|
|
14112
14628
|
}, () => this.runExecutionTurn(run, fn, registered));
|
|
14113
14629
|
}
|
|
14114
|
-
async runExecutionTurn(run,
|
|
14630
|
+
async runExecutionTurn(run, registeredFn, registered) {
|
|
14631
|
+
let fn = registeredFn;
|
|
14115
14632
|
if (run.status === "cancelling") this.cancelRequested.add(run.id);
|
|
14116
14633
|
if (run.status === "pending") {
|
|
14117
14634
|
if (!await this.markStarted(run, registered?.singleton !== void 0)) {
|
|
@@ -14127,8 +14644,10 @@ ${run.workflowVersion}`;
|
|
|
14127
14644
|
});
|
|
14128
14645
|
}
|
|
14129
14646
|
await this.noteServedWakes(run.id);
|
|
14130
|
-
if (run.status !== "cancelling" && registered?.singleton
|
|
14131
|
-
|
|
14647
|
+
if (run.status !== "cancelling" && registered?.singleton) {
|
|
14648
|
+
const admission = await this.singletons.admit(run, registered.singleton, registered.parseInput);
|
|
14649
|
+
if (admission instanceof UnparseableStoredInput) fn = failingBody(admission.error);
|
|
14650
|
+
else if (!admission) return this.singletonWaitResult(run);
|
|
14132
14651
|
}
|
|
14133
14652
|
const stickyOk = this.stickyEligible(run);
|
|
14134
14653
|
let cont = stickyOk ? this.takeContinuation(run, fn) : void 0;
|
|
@@ -18443,6 +18962,7 @@ export {
|
|
|
18443
18962
|
PARENT_CLOSE_TAG_PREFIX,
|
|
18444
18963
|
QueueController,
|
|
18445
18964
|
REMOTE_EVENT,
|
|
18965
|
+
REMOTE_WORKFLOW_FEATURES,
|
|
18446
18966
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
18447
18967
|
RUN_VALUE_FACET_LIMIT,
|
|
18448
18968
|
RUN_VALUE_FACET_SCAN,
|
|
@@ -18600,6 +19120,9 @@ export {
|
|
|
18600
19120
|
readBufferedEvent,
|
|
18601
19121
|
readRunsByIds,
|
|
18602
19122
|
readSearchAttributes,
|
|
19123
|
+
remoteCallMaxAttempts,
|
|
19124
|
+
remoteCallPolicyOf,
|
|
19125
|
+
remoteCallRetries,
|
|
18603
19126
|
requiredCapabilities,
|
|
18604
19127
|
resolveConcurrency,
|
|
18605
19128
|
resolveRouting,
|
|
@@ -18636,6 +19159,7 @@ export {
|
|
|
18636
19159
|
singletonAdmissionDenialPatch,
|
|
18637
19160
|
singletonAdmissionTags,
|
|
18638
19161
|
singletonClaimantIds,
|
|
19162
|
+
singletonKeyTag,
|
|
18639
19163
|
sqlComparator,
|
|
18640
19164
|
stepConfigOf,
|
|
18641
19165
|
stepId,
|