@dudousxd/nestjs-durable-core 0.101.0 → 0.103.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +769 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1155 -824
- package/dist/index.d.ts +1155 -824
- package/dist/index.js +765 -74
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -77,6 +77,7 @@ __export(index_exports, {
|
|
|
77
77
|
PARENT_CLOSE_TAG_PREFIX: () => PARENT_CLOSE_TAG_PREFIX,
|
|
78
78
|
QueueController: () => QueueController,
|
|
79
79
|
REMOTE_EVENT: () => REMOTE_EVENT,
|
|
80
|
+
REMOTE_WORKFLOW_FEATURES: () => REMOTE_WORKFLOW_FEATURES,
|
|
80
81
|
RUN_HISTORY_FIXTURE_VERSION: () => RUN_HISTORY_FIXTURE_VERSION,
|
|
81
82
|
RUN_VALUE_FACET_LIMIT: () => RUN_VALUE_FACET_LIMIT,
|
|
82
83
|
RUN_VALUE_FACET_SCAN: () => RUN_VALUE_FACET_SCAN,
|
|
@@ -234,6 +235,9 @@ __export(index_exports, {
|
|
|
234
235
|
readBufferedEvent: () => readBufferedEvent,
|
|
235
236
|
readRunsByIds: () => readRunsByIds,
|
|
236
237
|
readSearchAttributes: () => readSearchAttributes,
|
|
238
|
+
remoteCallMaxAttempts: () => remoteCallMaxAttempts,
|
|
239
|
+
remoteCallPolicyOf: () => remoteCallPolicyOf,
|
|
240
|
+
remoteCallRetries: () => remoteCallRetries,
|
|
237
241
|
requiredCapabilities: () => requiredCapabilities,
|
|
238
242
|
resolveConcurrency: () => resolveConcurrency,
|
|
239
243
|
resolveRouting: () => resolveRouting,
|
|
@@ -1178,6 +1182,14 @@ function isNonEmptyRunScope(scope) {
|
|
|
1178
1182
|
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
1179
1183
|
}
|
|
1180
1184
|
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
1185
|
+
var REMOTE_WORKFLOW_FEATURES = [
|
|
1186
|
+
"callPolicy",
|
|
1187
|
+
"sleepUntil",
|
|
1188
|
+
"waitSignalTimeout",
|
|
1189
|
+
"waitEvent",
|
|
1190
|
+
"spawnChild",
|
|
1191
|
+
"continueAsNew"
|
|
1192
|
+
];
|
|
1181
1193
|
var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
|
|
1182
1194
|
|
|
1183
1195
|
// src/events.ts
|
|
@@ -2639,6 +2651,13 @@ function canonicalizeRegistrations(regs) {
|
|
|
2639
2651
|
subscribes: [
|
|
2640
2652
|
...new Set(r.subscribes)
|
|
2641
2653
|
].sort()
|
|
2654
|
+
} : {},
|
|
2655
|
+
// Conditional too: a registration without them keeps its hash; a changed option changes it.
|
|
2656
|
+
...r.options !== void 0 ? {
|
|
2657
|
+
options: stableJson(r.options)
|
|
2658
|
+
} : {},
|
|
2659
|
+
...r.deadLetter !== void 0 ? {
|
|
2660
|
+
deadLetter: r.deadLetter
|
|
2642
2661
|
} : {}
|
|
2643
2662
|
}));
|
|
2644
2663
|
const byJson = /* @__PURE__ */ new Map();
|
|
@@ -3359,6 +3378,35 @@ function codeAndDetails(err) {
|
|
|
3359
3378
|
}
|
|
3360
3379
|
__name(codeAndDetails, "codeAndDetails");
|
|
3361
3380
|
|
|
3381
|
+
// src/remote-call-policy.ts
|
|
3382
|
+
function remoteCallPolicyOf(config, opts) {
|
|
3383
|
+
const policy = {};
|
|
3384
|
+
const retries = opts?.retries ?? config?.retries;
|
|
3385
|
+
if (retries !== void 0) policy.retries = retries;
|
|
3386
|
+
const backoff = opts?.backoff ?? config?.backoff;
|
|
3387
|
+
if (backoff !== void 0) policy.backoff = backoff;
|
|
3388
|
+
const backoffMs = opts?.backoffMs ?? config?.backoffMs;
|
|
3389
|
+
if (backoffMs !== void 0) policy.backoffMs = backoffMs;
|
|
3390
|
+
const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
|
|
3391
|
+
if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
|
|
3392
|
+
const jitter = opts?.jitter ?? config?.jitter;
|
|
3393
|
+
if (jitter !== void 0) policy.jitter = jitter;
|
|
3394
|
+
const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
|
|
3395
|
+
if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
|
|
3396
|
+
return Object.keys(policy).length > 0 ? policy : void 0;
|
|
3397
|
+
}
|
|
3398
|
+
__name(remoteCallPolicyOf, "remoteCallPolicyOf");
|
|
3399
|
+
function remoteCallMaxAttempts(policy) {
|
|
3400
|
+
return Math.max(1, policy?.retries ?? 1);
|
|
3401
|
+
}
|
|
3402
|
+
__name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
|
|
3403
|
+
function remoteCallRetries(policy, attempts, error2) {
|
|
3404
|
+
if (policy === void 0 || policy.timeoutMs) return false;
|
|
3405
|
+
if (error2?.retryable === false) return false;
|
|
3406
|
+
return attempts < remoteCallMaxAttempts(policy);
|
|
3407
|
+
}
|
|
3408
|
+
__name(remoteCallRetries, "remoteCallRetries");
|
|
3409
|
+
|
|
3362
3410
|
// src/remote-event.ts
|
|
3363
3411
|
var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
|
|
3364
3412
|
function markRemoteEvent(event) {
|
|
@@ -3416,7 +3464,12 @@ var RemoteWorkflowExecutor = class {
|
|
|
3416
3464
|
} : {},
|
|
3417
3465
|
group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
|
|
3418
3466
|
priority: run.priority,
|
|
3419
|
-
attempt: 1
|
|
3467
|
+
attempt: 1,
|
|
3468
|
+
// What this control plane applies beyond the original command kinds, so a worker fails a turn
|
|
3469
|
+
// that needs more instead of emitting a command this engine would not understand.
|
|
3470
|
+
features: [
|
|
3471
|
+
...REMOTE_WORKFLOW_FEATURES
|
|
3472
|
+
]
|
|
3420
3473
|
};
|
|
3421
3474
|
await this.transport.dispatchWorkflowTask(task);
|
|
3422
3475
|
}
|
|
@@ -4349,7 +4402,8 @@ var ScheduleClient = class {
|
|
|
4349
4402
|
const input = specOf(record).input;
|
|
4350
4403
|
const opts = startOptions(record);
|
|
4351
4404
|
if (isAnnounced(record) && this.host.startParked) {
|
|
4352
|
-
|
|
4405
|
+
const singleton = record.state.announce?.singleton;
|
|
4406
|
+
return singleton ? this.host.startParked(record.workflow, input, runId, opts, singleton) : this.host.startParked(record.workflow, input, runId, opts);
|
|
4353
4407
|
}
|
|
4354
4408
|
return this.host.start(record.workflow, input, runId, opts);
|
|
4355
4409
|
}
|
|
@@ -4545,7 +4599,8 @@ var ScheduleClient = class {
|
|
|
4545
4599
|
announce: {
|
|
4546
4600
|
key: doc.key,
|
|
4547
4601
|
fingerprint,
|
|
4548
|
-
declaredPaused: doc.declaredPaused
|
|
4602
|
+
declaredPaused: doc.declaredPaused,
|
|
4603
|
+
singleton: doc.singleton
|
|
4549
4604
|
}
|
|
4550
4605
|
}),
|
|
4551
4606
|
updatedAt: new Date(now)
|
|
@@ -4582,7 +4637,10 @@ var ScheduleClient = class {
|
|
|
4582
4637
|
announce: {
|
|
4583
4638
|
key: item.key,
|
|
4584
4639
|
fingerprint,
|
|
4585
|
-
declaredPaused
|
|
4640
|
+
declaredPaused,
|
|
4641
|
+
...validSingleton(item.singleton) ? {
|
|
4642
|
+
singleton: item.singleton
|
|
4643
|
+
} : {}
|
|
4586
4644
|
}
|
|
4587
4645
|
});
|
|
4588
4646
|
out.adopted.push(id);
|
|
@@ -4732,6 +4790,10 @@ function specOf(record) {
|
|
|
4732
4790
|
return record.spec;
|
|
4733
4791
|
}
|
|
4734
4792
|
__name(specOf, "specOf");
|
|
4793
|
+
function validSingleton(value) {
|
|
4794
|
+
return value !== null && typeof value === "object" && typeof value.key === "string" && value.key !== "";
|
|
4795
|
+
}
|
|
4796
|
+
__name(validSingleton, "validSingleton");
|
|
4735
4797
|
function isAnnounced(record) {
|
|
4736
4798
|
return record.state?.announce !== void 0;
|
|
4737
4799
|
}
|
|
@@ -7175,6 +7237,43 @@ var DEFAULT_STICKY_RUNS = 1e3;
|
|
|
7175
7237
|
var DEFAULT_STICKY_IDLE_MS = 3e5;
|
|
7176
7238
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
7177
7239
|
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
7240
|
+
var ANNOUNCED_TIMEOUTS_TTL_MS = 15e3;
|
|
7241
|
+
function tenantScopedKey(namespace, key) {
|
|
7242
|
+
return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
|
|
7243
|
+
}
|
|
7244
|
+
__name(tenantScopedKey, "tenantScopedKey");
|
|
7245
|
+
function singletonKeyOf(run) {
|
|
7246
|
+
const tag = run.tags?.find((t) => t.startsWith("singleton:"));
|
|
7247
|
+
return tag?.slice("singleton:".length);
|
|
7248
|
+
}
|
|
7249
|
+
__name(singletonKeyOf, "singletonKeyOf");
|
|
7250
|
+
function conventionSingleton(run, announced, wire) {
|
|
7251
|
+
if (wire !== void 0) {
|
|
7252
|
+
const key2 = wire.key;
|
|
7253
|
+
return {
|
|
7254
|
+
key: /* @__PURE__ */ __name(() => key2, "key"),
|
|
7255
|
+
...typeof wire.limit === "number" ? {
|
|
7256
|
+
limit: wire.limit
|
|
7257
|
+
} : {},
|
|
7258
|
+
...typeof wire.maxQueueDepth === "number" ? {
|
|
7259
|
+
maxQueueDepth: wire.maxQueueDepth
|
|
7260
|
+
} : {}
|
|
7261
|
+
};
|
|
7262
|
+
}
|
|
7263
|
+
const key = singletonKeyOf(run);
|
|
7264
|
+
if (key === void 0 || announced === void 0) return void 0;
|
|
7265
|
+
if (announced === null) return {
|
|
7266
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7267
|
+
limit: 1
|
|
7268
|
+
};
|
|
7269
|
+
const declared = announced.options?.singleton;
|
|
7270
|
+
if (declared === void 0) return void 0;
|
|
7271
|
+
return {
|
|
7272
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7273
|
+
limit: typeof declared.limit === "number" ? declared.limit : 1
|
|
7274
|
+
};
|
|
7275
|
+
}
|
|
7276
|
+
__name(conventionSingleton, "conventionSingleton");
|
|
7178
7277
|
function servedNamespace(namespace, group) {
|
|
7179
7278
|
if (namespace) return namespace;
|
|
7180
7279
|
const at = group?.lastIndexOf("@") ?? -1;
|
|
@@ -7664,13 +7763,28 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7664
7763
|
if (this.queuesRunsOverTransport) void this.wake(runId);
|
|
7665
7764
|
else this.runDispatcher.dispatch(runId);
|
|
7666
7765
|
}, "dispatch"),
|
|
7667
|
-
configFor: /* @__PURE__ */ __name((run) =>
|
|
7766
|
+
configFor: /* @__PURE__ */ __name((run) => {
|
|
7767
|
+
const local = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton;
|
|
7768
|
+
if (local) return local;
|
|
7769
|
+
const key = this.latest.has(run.workflow) ? void 0 : singletonKeyOf(run);
|
|
7770
|
+
if (key === void 0) return void 0;
|
|
7771
|
+
const limit = this.conventionSingletonLimits.get(`${runNamespace(run.namespace)}\0${run.workflow}`);
|
|
7772
|
+
return {
|
|
7773
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7774
|
+
...limit !== void 0 ? {
|
|
7775
|
+
limit
|
|
7776
|
+
} : {}
|
|
7777
|
+
};
|
|
7778
|
+
}, "configFor")
|
|
7668
7779
|
});
|
|
7669
7780
|
this.schedules = new ScheduleClient(this.store, {
|
|
7670
7781
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7671
7782
|
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7672
7783
|
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7673
|
-
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true
|
|
7784
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts, singleton) => this.startWith(workflow, input, runId, opts, true, singleton ? {
|
|
7785
|
+
...singleton,
|
|
7786
|
+
key: tenantScopedKey(opts?.namespace, singleton.key)
|
|
7787
|
+
} : void 0), "startParked"),
|
|
7674
7788
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7675
7789
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7676
7790
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -8480,6 +8594,39 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8480
8594
|
} catch {
|
|
8481
8595
|
}
|
|
8482
8596
|
}
|
|
8597
|
+
if (!this.latest.has(run.workflow)) this.inBackground(this.routeAnnouncedDeadLetter(run));
|
|
8598
|
+
}
|
|
8599
|
+
/**
|
|
8600
|
+
* The dead-letter workflow the live workers serving `run`'s workflow announce (a tenant's inline
|
|
8601
|
+
* `@DeadLetter()` handler, its `deadLetterWorkflow`, or its module-level default), or `undefined`
|
|
8602
|
+
* when the workflow is registered on this engine (its own routing applies), they announce none, or
|
|
8603
|
+
* they cannot be read. A dead-letter listener with a catch-all default (`DurableModule`'s
|
|
8604
|
+
* `deadLetterWorkflow`) checks it first, so the app's own route wins over the default.
|
|
8605
|
+
*/
|
|
8606
|
+
async announcedDeadLetter(run) {
|
|
8607
|
+
if (this.latest.has(run.workflow) || this.isRegisteredHere(run)) return void 0;
|
|
8608
|
+
const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
|
|
8609
|
+
const reg = await this.announcedRegistration(group, run.workflow, run.workflowVersion);
|
|
8610
|
+
const target = reg?.deadLetter;
|
|
8611
|
+
return typeof target === "string" && target !== "" ? target : void 0;
|
|
8612
|
+
}
|
|
8613
|
+
/**
|
|
8614
|
+
* Start the announced dead-letter workflow of a dead run (see {@link announcedDeadLetter}) in the
|
|
8615
|
+
* run's namespace, as `dlq:<runId>` with the `DeadLetter` payload, through the tenant start path:
|
|
8616
|
+
* with no live worker of the app it waits `blocked` instead of being lost. Idempotent by run id.
|
|
8617
|
+
*/
|
|
8618
|
+
async routeAnnouncedDeadLetter(run) {
|
|
8619
|
+
const target = await this.announcedDeadLetter(run);
|
|
8620
|
+
if (!target) return;
|
|
8621
|
+
await this.startWith(target, {
|
|
8622
|
+
deadRunId: run.id,
|
|
8623
|
+
workflow: run.workflow,
|
|
8624
|
+
input: run.input,
|
|
8625
|
+
error: run.error
|
|
8626
|
+
}, `dlq:${run.id}`, {
|
|
8627
|
+
namespace: run.namespace,
|
|
8628
|
+
...inheritTenant(run)
|
|
8629
|
+
}, true);
|
|
8483
8630
|
}
|
|
8484
8631
|
/**
|
|
8485
8632
|
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
@@ -9780,7 +9927,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9780
9927
|
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9781
9928
|
* a start whose caller has already returned is never lost.
|
|
9782
9929
|
*/
|
|
9783
|
-
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9930
|
+
async startWith(workflow, input, runId, opts, parkUnroutable, wireSingleton) {
|
|
9784
9931
|
const name = workflowName(workflow);
|
|
9785
9932
|
const pinned = opts?.version;
|
|
9786
9933
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9805,12 +9952,15 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9805
9952
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9806
9953
|
};
|
|
9807
9954
|
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9808
|
-
askFleetForVersion: true
|
|
9955
|
+
askFleetForVersion: true,
|
|
9956
|
+
...wireSingleton !== void 0 ? {
|
|
9957
|
+
singleton: wireSingleton
|
|
9958
|
+
} : {}
|
|
9809
9959
|
});
|
|
9810
9960
|
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9811
9961
|
}
|
|
9812
9962
|
if (!registered && parkUnroutable) {
|
|
9813
|
-
return this.parkStartForWorker(name, input, runId, opts);
|
|
9963
|
+
return this.parkStartForWorker(name, input, runId, opts, wireSingleton);
|
|
9814
9964
|
}
|
|
9815
9965
|
if (!registered) {
|
|
9816
9966
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
@@ -9912,11 +10062,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9912
10062
|
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9913
10063
|
* start is checked first at its `startAt`.
|
|
9914
10064
|
*/
|
|
9915
|
-
async parkStartForWorker(name, input, runId, opts) {
|
|
10065
|
+
async parkStartForWorker(name, input, runId, opts, wireSingleton) {
|
|
9916
10066
|
const prior = await this.store.getRun(runId);
|
|
9917
10067
|
if (prior) return this.existingStartResult(prior);
|
|
9918
10068
|
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9919
10069
|
if (quota) await assertConcurrency(this.store, name, quota);
|
|
10070
|
+
const singleton = conventionSingleton({}, void 0, wireSingleton);
|
|
10071
|
+
if (singleton) await this.singletons.assertCapacity(name, singleton, input);
|
|
9920
10072
|
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9921
10073
|
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9922
10074
|
const pinned = opts?.version;
|
|
@@ -9931,6 +10083,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9931
10083
|
const now = /* @__PURE__ */ new Date();
|
|
9932
10084
|
const tags = mergeTags(void 0, [
|
|
9933
10085
|
...opts?.tags ?? [],
|
|
10086
|
+
...singleton ? [
|
|
10087
|
+
this.singletons.tag(singleton, input)
|
|
10088
|
+
] : [],
|
|
9934
10089
|
...quota ? [
|
|
9935
10090
|
concurrencyTag(quota.key)
|
|
9936
10091
|
] : [],
|
|
@@ -10019,8 +10174,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10019
10174
|
concurrency: message.concurrency
|
|
10020
10175
|
} : {}
|
|
10021
10176
|
};
|
|
10177
|
+
const singleton = message.singleton !== void 0 && typeof message.singleton?.key === "string" ? {
|
|
10178
|
+
...message.singleton,
|
|
10179
|
+
key: tenantScopedKey(namespace, message.singleton.key)
|
|
10180
|
+
} : void 0;
|
|
10022
10181
|
try {
|
|
10023
|
-
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
10182
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true, singleton);
|
|
10024
10183
|
} catch (err) {
|
|
10025
10184
|
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
10026
10185
|
throw err;
|
|
@@ -10028,6 +10187,62 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10028
10187
|
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
10029
10188
|
}
|
|
10030
10189
|
}
|
|
10190
|
+
/**
|
|
10191
|
+
* The control plane's half of a tenant's `signalWithStart` (a `signalWithStart` run request):
|
|
10192
|
+
* start `start` like {@link startForTenant} unless its run exists, then signal `signal.token`.
|
|
10193
|
+
* Authorization is the caller's (the run-request responder): the run must belong to `namespace`,
|
|
10194
|
+
* and so must a run already waiting on the token. Delivered at once to a waiting run; with none
|
|
10195
|
+
* waiting yet it is buffered only when the token contains the run id (a run-scoped token), so a
|
|
10196
|
+
* tenant can never plant a payload for a wait another namespace opens later. `signalled: false`
|
|
10197
|
+
* when it was neither delivered nor buffered.
|
|
10198
|
+
*/
|
|
10199
|
+
async signalWithStartForTenant(start, namespace, tenant, signal) {
|
|
10200
|
+
const started = await this.startForTenant(start, namespace, tenant);
|
|
10201
|
+
if (TERMINAL_RUN_STATUSES.includes(started.status)) {
|
|
10202
|
+
return {
|
|
10203
|
+
runId: start.runId,
|
|
10204
|
+
signalled: false
|
|
10205
|
+
};
|
|
10206
|
+
}
|
|
10207
|
+
const waiter = (await this.store.listSignalWaiters(signal.token)).find((w) => w.token === signal.token);
|
|
10208
|
+
if (waiter) {
|
|
10209
|
+
const run = await this.store.getRun(waiter.runId);
|
|
10210
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) {
|
|
10211
|
+
return {
|
|
10212
|
+
runId: start.runId,
|
|
10213
|
+
signalled: false
|
|
10214
|
+
};
|
|
10215
|
+
}
|
|
10216
|
+
} else if (!signal.token.includes(start.runId)) {
|
|
10217
|
+
return {
|
|
10218
|
+
runId: start.runId,
|
|
10219
|
+
signalled: false
|
|
10220
|
+
};
|
|
10221
|
+
}
|
|
10222
|
+
await this.signalUnchecked(signal.token, signal.payload);
|
|
10223
|
+
return {
|
|
10224
|
+
runId: start.runId,
|
|
10225
|
+
signalled: true
|
|
10226
|
+
};
|
|
10227
|
+
}
|
|
10228
|
+
/**
|
|
10229
|
+
* The control plane's half of a tenant's `publishEvent` (a `publishEvent` run request): resume the
|
|
10230
|
+
* runs of `namespace` waiting on event `name` whose match the payload satisfies. Starting the
|
|
10231
|
+
* tenant's own subscribers is the tenant's (it holds their `where` and `input`); nothing is
|
|
10232
|
+
* buffered, since a buffered event could reach another namespace's later wait. Returns how many
|
|
10233
|
+
* runs it resumed.
|
|
10234
|
+
*/
|
|
10235
|
+
async publishEventForTenant(name, payload, namespace) {
|
|
10236
|
+
let touched = 0;
|
|
10237
|
+
for (const w of await this.store.listSignalWaiters(eventPrefix(name))) {
|
|
10238
|
+
if (!eventMatches(payload, eventMatchOf(w.token))) continue;
|
|
10239
|
+
const run = await this.store.getRun(w.runId);
|
|
10240
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) continue;
|
|
10241
|
+
await this.signalUnchecked(w.token, payload);
|
|
10242
|
+
touched += 1;
|
|
10243
|
+
}
|
|
10244
|
+
return touched;
|
|
10245
|
+
}
|
|
10031
10246
|
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
10032
10247
|
async recordRejectedStart(name, input, runId, opts, err) {
|
|
10033
10248
|
const prior = await this.store.getRun(runId);
|
|
@@ -10499,9 +10714,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10499
10714
|
if (!liveTransport) return void 0;
|
|
10500
10715
|
const executor = new RemoteWorkflowExecutor(liveTransport, run.workflow, run.namespace);
|
|
10501
10716
|
const declared = opts.askFleetForVersion ? await this.declaredVersion(liveTransport, group, run.workflow) : void 0;
|
|
10717
|
+
const version = declared ?? run.workflowVersion;
|
|
10718
|
+
const announced = await this.announcedRegistration(group, run.workflow, version);
|
|
10719
|
+
const singleton = conventionSingleton(run, announced, opts.singleton);
|
|
10720
|
+
if (singleton?.limit !== void 0) {
|
|
10721
|
+
this.conventionSingletonLimits.set(`${runNamespace(run.namespace)}\0${run.workflow}`, singleton.limit);
|
|
10722
|
+
}
|
|
10723
|
+
const tags = announced?.options?.tags;
|
|
10502
10724
|
return {
|
|
10503
10725
|
name: run.workflow,
|
|
10504
|
-
version
|
|
10726
|
+
version,
|
|
10505
10727
|
fn: /* @__PURE__ */ __name(() => {
|
|
10506
10728
|
throw new Error(`remote workflow ${run.workflow} has no local body`);
|
|
10507
10729
|
}, "fn"),
|
|
@@ -10510,12 +10732,47 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10510
10732
|
group,
|
|
10511
10733
|
executor
|
|
10512
10734
|
},
|
|
10735
|
+
...Array.isArray(tags) && tags.length > 0 ? {
|
|
10736
|
+
tags
|
|
10737
|
+
} : {},
|
|
10738
|
+
...singleton ? {
|
|
10739
|
+
singleton
|
|
10740
|
+
} : {},
|
|
10513
10741
|
...opts.askFleetForVersion ? {
|
|
10514
10742
|
versionDeclared: declared !== void 0
|
|
10515
10743
|
} : {}
|
|
10516
10744
|
};
|
|
10517
10745
|
}
|
|
10518
10746
|
/**
|
|
10747
|
+
* The registration the live workers of `group` announce for `workflow` (at `version`, when one of
|
|
10748
|
+
* them states it): where they disagree (a rolling deploy), the one from the worker that started
|
|
10749
|
+
* last, as for announced schedules. `undefined` when none announces it, `null` when the read
|
|
10750
|
+
* failed (what the workers state is then unknown, not "nothing"). Reads
|
|
10751
|
+
* through the descriptor cache ({@link workerDescriptorCacheMs}), so a turn costs no extra round
|
|
10752
|
+
* trip in steady state.
|
|
10753
|
+
*/
|
|
10754
|
+
async announcedRegistration(group, workflow, version) {
|
|
10755
|
+
let descriptors;
|
|
10756
|
+
try {
|
|
10757
|
+
descriptors = await this.liveWorkerDescriptors(group);
|
|
10758
|
+
} catch {
|
|
10759
|
+
return null;
|
|
10760
|
+
}
|
|
10761
|
+
let best;
|
|
10762
|
+
for (const descriptor of descriptors) {
|
|
10763
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10764
|
+
if (reg?.name !== workflow) continue;
|
|
10765
|
+
if (version !== void 0 && reg.version !== void 0 && reg.version !== version) continue;
|
|
10766
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10767
|
+
if (!best || at > best.at) best = {
|
|
10768
|
+
at,
|
|
10769
|
+
reg
|
|
10770
|
+
};
|
|
10771
|
+
}
|
|
10772
|
+
}
|
|
10773
|
+
return best?.reg;
|
|
10774
|
+
}
|
|
10775
|
+
/**
|
|
10519
10776
|
* The version the LIVE fleet declares for `workflow` on `group`, or `undefined` when it declares
|
|
10520
10777
|
* none — which is a fact about the fleet, not a failure to read it.
|
|
10521
10778
|
*
|
|
@@ -10739,34 +10996,95 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10739
10996
|
for (const reg of new Set(this.latest.values())) {
|
|
10740
10997
|
if (this.draining) return;
|
|
10741
10998
|
if (reg.executionTimeoutMs == null) continue;
|
|
10742
|
-
|
|
10743
|
-
|
|
10744
|
-
|
|
10745
|
-
|
|
10746
|
-
|
|
10747
|
-
|
|
10748
|
-
|
|
10749
|
-
|
|
10750
|
-
|
|
10751
|
-
|
|
10752
|
-
|
|
10753
|
-
|
|
10754
|
-
|
|
10755
|
-
|
|
10756
|
-
|
|
10757
|
-
|
|
10758
|
-
|
|
10759
|
-
|
|
10760
|
-
|
|
10761
|
-
|
|
10762
|
-
|
|
10763
|
-
|
|
10764
|
-
|
|
10765
|
-
|
|
10766
|
-
|
|
10767
|
-
|
|
10768
|
-
|
|
10999
|
+
await this.cancelTimedOut(reg.name, this.namespace, now - reg.executionTimeoutMs);
|
|
11000
|
+
}
|
|
11001
|
+
for (const { workflow, namespace, timeoutMs } of await this.announcedTimeouts(now)) {
|
|
11002
|
+
if (this.draining) return;
|
|
11003
|
+
await this.cancelTimedOut(workflow, namespace, now - timeoutMs);
|
|
11004
|
+
}
|
|
11005
|
+
}
|
|
11006
|
+
/**
|
|
11007
|
+
* The singleton limit last resolved for a workflow served by convention, by `<namespace>\0<name>`
|
|
11008
|
+
* (see {@link resolveRemoteByConvention}): what the singleton gate's release wake-up reads, since
|
|
11009
|
+
* it cannot wait on the fleet. One entry per convention singleton the fleet serves.
|
|
11010
|
+
*/
|
|
11011
|
+
conventionSingletonLimits = /* @__PURE__ */ new Map();
|
|
11012
|
+
/** The last read of the announced execution timeouts (see {@link announcedTimeouts}). */
|
|
11013
|
+
announcedTimeoutsRead;
|
|
11014
|
+
/**
|
|
11015
|
+
* The `executionTimeoutMs` live workers announce (`@Workflow({ executionTimeout })` on a tenant),
|
|
11016
|
+
* per workflow and namespace, for workflows this engine does not register (a registration here
|
|
11017
|
+
* applies its own). Where workers disagree, the one that started last wins. Read at most once per
|
|
11018
|
+
* {@link ANNOUNCED_TIMEOUTS_TTL_MS}; empty without a transport that reads the fleet, or when the
|
|
11019
|
+
* read fails.
|
|
11020
|
+
*/
|
|
11021
|
+
async announcedTimeouts(now) {
|
|
11022
|
+
if (!this.pool.introspectsFleet) return [];
|
|
11023
|
+
const cached = this.announcedTimeoutsRead;
|
|
11024
|
+
if (cached && now - cached.at < ANNOUNCED_TIMEOUTS_TTL_MS) return cached.list;
|
|
11025
|
+
const newest = /* @__PURE__ */ new Map();
|
|
11026
|
+
try {
|
|
11027
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
11028
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
11029
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
11030
|
+
if (!reg || typeof reg.name !== "string" || this.latest.has(reg.name)) continue;
|
|
11031
|
+
const namespace = servedNamespace(descriptor.namespace, reg.group);
|
|
11032
|
+
if (this.namespace !== void 0 && runNamespace(this.namespace) !== namespace) continue;
|
|
11033
|
+
const key = `${namespace}\0${reg.name}`;
|
|
11034
|
+
const prior = newest.get(key);
|
|
11035
|
+
if (prior && prior.at >= at) continue;
|
|
11036
|
+
newest.set(key, {
|
|
11037
|
+
at,
|
|
11038
|
+
workflow: reg.name,
|
|
11039
|
+
namespace,
|
|
11040
|
+
ms: Number(reg.options?.executionTimeoutMs)
|
|
11041
|
+
});
|
|
11042
|
+
}
|
|
10769
11043
|
}
|
|
11044
|
+
} catch {
|
|
11045
|
+
return cached?.list ?? [];
|
|
11046
|
+
}
|
|
11047
|
+
const list2 = [
|
|
11048
|
+
...newest.values()
|
|
11049
|
+
].filter((e) => Number.isFinite(e.ms) && e.ms > 0).map((e) => ({
|
|
11050
|
+
workflow: e.workflow,
|
|
11051
|
+
namespace: e.namespace,
|
|
11052
|
+
timeoutMs: e.ms
|
|
11053
|
+
}));
|
|
11054
|
+
this.announcedTimeoutsRead = {
|
|
11055
|
+
at: now,
|
|
11056
|
+
list: list2
|
|
11057
|
+
};
|
|
11058
|
+
return list2;
|
|
11059
|
+
}
|
|
11060
|
+
/** Cancel (`execution_timeout`) the in-flight runs of `workflow` in `namespace` created before `deadline`. */
|
|
11061
|
+
async cancelTimedOut(workflow, namespace, deadline) {
|
|
11062
|
+
const inflight = await this.store.listRuns({
|
|
11063
|
+
workflow,
|
|
11064
|
+
statuses: TIMEOUT_SWEPT_STATUSES,
|
|
11065
|
+
namespace
|
|
11066
|
+
});
|
|
11067
|
+
for (const run of inflight) {
|
|
11068
|
+
if (this.draining) return;
|
|
11069
|
+
if (run.createdAt.getTime() > deadline) continue;
|
|
11070
|
+
const error2 = {
|
|
11071
|
+
message: "execution timeout",
|
|
11072
|
+
code: "execution_timeout"
|
|
11073
|
+
};
|
|
11074
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
11075
|
+
status: "cancelled",
|
|
11076
|
+
error: error2,
|
|
11077
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
11078
|
+
}));
|
|
11079
|
+
this.emit({
|
|
11080
|
+
type: "run.failed",
|
|
11081
|
+
runId: run.id,
|
|
11082
|
+
workflow: run.workflow,
|
|
11083
|
+
namespace: run.namespace,
|
|
11084
|
+
error: error2
|
|
11085
|
+
});
|
|
11086
|
+
await this.cancelChildren(run.id).catch(() => void 0);
|
|
11087
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
10770
11088
|
}
|
|
10771
11089
|
}
|
|
10772
11090
|
/**
|
|
@@ -13478,7 +13796,7 @@ ${run.workflowVersion}`;
|
|
|
13478
13796
|
let wfPlan = await this.planRoute(wfToken, registered.requires);
|
|
13479
13797
|
if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
|
|
13480
13798
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
13481
|
-
const checkpoints = await this.replayCheckpoints(run.id);
|
|
13799
|
+
const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
|
|
13482
13800
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
13483
13801
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
13484
13802
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -13564,6 +13882,10 @@ ${run.workflowVersion}`;
|
|
|
13564
13882
|
*/
|
|
13565
13883
|
async applyDecision(run, decision) {
|
|
13566
13884
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
13885
|
+
const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
|
|
13886
|
+
if (handoff !== void 0 && decision.status === "completed") {
|
|
13887
|
+
return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
|
|
13888
|
+
}
|
|
13567
13889
|
if (decision.status === "completed") {
|
|
13568
13890
|
await this.applyCommands(run, decision.commands);
|
|
13569
13891
|
return this.settleRun(run, {
|
|
@@ -13610,6 +13932,63 @@ ${run.workflowVersion}`;
|
|
|
13610
13932
|
return suspended;
|
|
13611
13933
|
}
|
|
13612
13934
|
/**
|
|
13935
|
+
* A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
|
|
13936
|
+
* run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
|
|
13937
|
+
* engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
|
|
13938
|
+
* turn replayed stays cancelled and starts nothing.
|
|
13939
|
+
*/
|
|
13940
|
+
async applyRemoteContinueAsNew(run, commands, input) {
|
|
13941
|
+
await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
|
|
13942
|
+
const fresh = await this.store.getRun(run.id);
|
|
13943
|
+
if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
|
|
13944
|
+
return {
|
|
13945
|
+
runId: run.id,
|
|
13946
|
+
status: fresh.status,
|
|
13947
|
+
output: fresh.output,
|
|
13948
|
+
error: fresh.error
|
|
13949
|
+
};
|
|
13950
|
+
}
|
|
13951
|
+
const nextId = nextContinuationId(run.id);
|
|
13952
|
+
try {
|
|
13953
|
+
await this.start(run.workflow, input, nextId, {
|
|
13954
|
+
namespace: run.namespace,
|
|
13955
|
+
...inheritTenant(run)
|
|
13956
|
+
});
|
|
13957
|
+
} catch (err) {
|
|
13958
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13959
|
+
return this.settleRun(run, {
|
|
13960
|
+
kind: "failed",
|
|
13961
|
+
error: {
|
|
13962
|
+
message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
|
|
13963
|
+
code: "continue_as_new_failed"
|
|
13964
|
+
}
|
|
13965
|
+
});
|
|
13966
|
+
}
|
|
13967
|
+
await this.store.updateRun(run.id, {
|
|
13968
|
+
status: "completed",
|
|
13969
|
+
output: void 0,
|
|
13970
|
+
error: void 0,
|
|
13971
|
+
wakeAt: void 0,
|
|
13972
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
13973
|
+
});
|
|
13974
|
+
this.emit({
|
|
13975
|
+
type: "run.completed",
|
|
13976
|
+
runId: run.id,
|
|
13977
|
+
workflow: run.workflow,
|
|
13978
|
+
namespace: run.namespace
|
|
13979
|
+
}, {
|
|
13980
|
+
lifecycle: false
|
|
13981
|
+
});
|
|
13982
|
+
void this.notifyParent(run.id, {
|
|
13983
|
+
ok: true,
|
|
13984
|
+
value: void 0
|
|
13985
|
+
});
|
|
13986
|
+
return {
|
|
13987
|
+
runId: run.id,
|
|
13988
|
+
status: "completed"
|
|
13989
|
+
};
|
|
13990
|
+
}
|
|
13991
|
+
/**
|
|
13613
13992
|
* Re-drive a run whose turn was computed from a history OLDER than the store's.
|
|
13614
13993
|
*
|
|
13615
13994
|
* A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
|
|
@@ -13627,11 +14006,12 @@ ${run.workflowVersion}`;
|
|
|
13627
14006
|
* Scheduled off the current stack so it runs after the caller released the run lease.
|
|
13628
14007
|
*/
|
|
13629
14008
|
async redriveIfTurnSawStaleHistory(runId, commands) {
|
|
13630
|
-
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
|
|
14009
|
+
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
|
|
13631
14010
|
if (blocking.length === 0) return;
|
|
13632
14011
|
for (const cmd of blocking) {
|
|
13633
14012
|
const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
|
|
13634
|
-
|
|
14013
|
+
const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
|
|
14014
|
+
if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
|
|
13635
14015
|
setTimeout(() => {
|
|
13636
14016
|
void this.resumeImplicit(runId).catch(() => void 0);
|
|
13637
14017
|
}, 0).unref?.();
|
|
@@ -13880,7 +14260,12 @@ ${run.workflowVersion}`;
|
|
|
13880
14260
|
// name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
|
|
13881
14261
|
name: cp.kind === "signal" ? void 0 : cp.name,
|
|
13882
14262
|
output: cp.status === "completed" ? cp.output : void 0,
|
|
13883
|
-
error: cp.status === "failed" ? cp.error : void 0
|
|
14263
|
+
error: cp.status === "failed" ? cp.error : void 0,
|
|
14264
|
+
// How many times a dispatched step ran: a worker applying a `call` policy decides a retry
|
|
14265
|
+
// from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
|
|
14266
|
+
...cp.kind === "remote" ? {
|
|
14267
|
+
attempts: cp.attempts
|
|
14268
|
+
} : {}
|
|
13884
14269
|
});
|
|
13885
14270
|
} else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
|
|
13886
14271
|
events.push({
|
|
@@ -13937,11 +14322,12 @@ ${run.workflowVersion}`;
|
|
|
13937
14322
|
} else if (cmd.kind === "call") {
|
|
13938
14323
|
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
13939
14324
|
if (existing) {
|
|
13940
|
-
const
|
|
14325
|
+
const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
|
|
14326
|
+
const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
|
|
13941
14327
|
if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
|
|
13942
14328
|
continue;
|
|
13943
14329
|
}
|
|
13944
|
-
const lease = this.
|
|
14330
|
+
const lease = this.callLease(cmd.policy);
|
|
13945
14331
|
await this.store.saveCheckpoint(stepCheckpoint({
|
|
13946
14332
|
runId: run.id,
|
|
13947
14333
|
seq: cmd.seq,
|
|
@@ -14001,35 +14387,7 @@ ${run.workflowVersion}`;
|
|
|
14001
14387
|
}));
|
|
14002
14388
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14003
14389
|
} else if (cmd.kind === "waitSignal") {
|
|
14004
|
-
|
|
14005
|
-
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14006
|
-
runId: run.id,
|
|
14007
|
-
seq: cmd.seq,
|
|
14008
|
-
name: `signal:${cmd.signal}`,
|
|
14009
|
-
kind: "signal",
|
|
14010
|
-
output: payload
|
|
14011
|
-
}));
|
|
14012
|
-
setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
|
|
14013
|
-
}, "deliverBuffered");
|
|
14014
|
-
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
14015
|
-
if (buffered) {
|
|
14016
|
-
await deliverBuffered(buffered.payload);
|
|
14017
|
-
} else {
|
|
14018
|
-
await this.store.putSignalWaiter({
|
|
14019
|
-
token: cmd.signal,
|
|
14020
|
-
runId: run.id,
|
|
14021
|
-
seq: cmd.seq
|
|
14022
|
-
});
|
|
14023
|
-
const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
14024
|
-
if (lateBuffered) {
|
|
14025
|
-
await this.store.removeSignalWaiter({
|
|
14026
|
-
token: cmd.signal,
|
|
14027
|
-
runId: run.id,
|
|
14028
|
-
seq: cmd.seq
|
|
14029
|
-
});
|
|
14030
|
-
await deliverBuffered(lateBuffered.payload);
|
|
14031
|
-
}
|
|
14032
|
-
}
|
|
14390
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14033
14391
|
} else if (cmd.kind === "waitAny") {
|
|
14034
14392
|
const state = await enterWaitAny({
|
|
14035
14393
|
store: this.store,
|
|
@@ -14061,6 +14419,100 @@ ${run.workflowVersion}`;
|
|
|
14061
14419
|
} else {
|
|
14062
14420
|
this.redeliverFinishedChild(childId, existingChild);
|
|
14063
14421
|
}
|
|
14422
|
+
} else if (cmd.kind === "sleepUntil") {
|
|
14423
|
+
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
14424
|
+
const deadline = existing?.wakeAt ?? cmd.until;
|
|
14425
|
+
if (!existing) {
|
|
14426
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14427
|
+
runId: run.id,
|
|
14428
|
+
seq: cmd.seq,
|
|
14429
|
+
name: `sleep:${cmd.seq}`,
|
|
14430
|
+
kind: "sleep",
|
|
14431
|
+
status: "pending",
|
|
14432
|
+
attempts: 1,
|
|
14433
|
+
wakeAt: deadline,
|
|
14434
|
+
enqueuedAt: at,
|
|
14435
|
+
startedAt: at,
|
|
14436
|
+
finishedAt: at
|
|
14437
|
+
}));
|
|
14438
|
+
}
|
|
14439
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14440
|
+
} else if (cmd.kind === "waitSignalTimeout") {
|
|
14441
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14442
|
+
const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
|
|
14443
|
+
waitSeq: cmd.seq,
|
|
14444
|
+
token: cmd.signal
|
|
14445
|
+
});
|
|
14446
|
+
if (deadline <= this.clock()) {
|
|
14447
|
+
this.resumeSoon(run.id);
|
|
14448
|
+
continue;
|
|
14449
|
+
}
|
|
14450
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14451
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14452
|
+
} else if (cmd.kind === "waitEvent") {
|
|
14453
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14454
|
+
const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
|
|
14455
|
+
let deadline;
|
|
14456
|
+
if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
|
|
14457
|
+
deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
|
|
14458
|
+
waitSeq: cmd.seq,
|
|
14459
|
+
token
|
|
14460
|
+
});
|
|
14461
|
+
if (deadline <= this.clock()) {
|
|
14462
|
+
this.resumeSoon(run.id);
|
|
14463
|
+
continue;
|
|
14464
|
+
}
|
|
14465
|
+
}
|
|
14466
|
+
await this.store.putSignalWaiter({
|
|
14467
|
+
token,
|
|
14468
|
+
runId: run.id,
|
|
14469
|
+
seq: cmd.seq
|
|
14470
|
+
});
|
|
14471
|
+
const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
|
|
14472
|
+
if (hit) {
|
|
14473
|
+
await this.store.removeSignalWaiter({
|
|
14474
|
+
token,
|
|
14475
|
+
runId: run.id,
|
|
14476
|
+
seq: cmd.seq
|
|
14477
|
+
});
|
|
14478
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14479
|
+
runId: run.id,
|
|
14480
|
+
seq: cmd.seq,
|
|
14481
|
+
name: `signal:${token}`,
|
|
14482
|
+
kind: "signal",
|
|
14483
|
+
output: hit.payload
|
|
14484
|
+
}));
|
|
14485
|
+
this.resumeSoon(run.id);
|
|
14486
|
+
} else if (deadline != null) {
|
|
14487
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14488
|
+
}
|
|
14489
|
+
} else if (cmd.kind === "spawnChild") {
|
|
14490
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14491
|
+
if (!await this.store.getRun(cmd.childId)) {
|
|
14492
|
+
try {
|
|
14493
|
+
await this.start(cmd.workflow, cmd.input, cmd.childId, {
|
|
14494
|
+
namespace: run.namespace,
|
|
14495
|
+
...inheritTenant(run),
|
|
14496
|
+
...cmd.priority !== void 0 ? {
|
|
14497
|
+
priority: cmd.priority
|
|
14498
|
+
} : {}
|
|
14499
|
+
});
|
|
14500
|
+
} catch (err) {
|
|
14501
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14502
|
+
this.notifyParent(cmd.childId, {
|
|
14503
|
+
ok: false,
|
|
14504
|
+
error: `child workflow "${cmd.workflow}" failed to start: ${message}`
|
|
14505
|
+
});
|
|
14506
|
+
}
|
|
14507
|
+
}
|
|
14508
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14509
|
+
runId: run.id,
|
|
14510
|
+
seq: cmd.seq,
|
|
14511
|
+
name: `spawn:${cmd.childId}`,
|
|
14512
|
+
kind: "local",
|
|
14513
|
+
output: cmd.childId
|
|
14514
|
+
}));
|
|
14515
|
+
} else if (cmd.kind === "continueAsNew") {
|
|
14064
14516
|
} else {
|
|
14065
14517
|
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
14066
14518
|
}
|
|
@@ -14068,6 +14520,245 @@ ${run.workflowVersion}`;
|
|
|
14068
14520
|
return wakeAt;
|
|
14069
14521
|
}
|
|
14070
14522
|
/**
|
|
14523
|
+
* Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
|
|
14524
|
+
* the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
|
|
14525
|
+
* (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
|
|
14526
|
+
* re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
|
|
14527
|
+
* bails).
|
|
14528
|
+
*/
|
|
14529
|
+
async parkRemoteOnSignal(run, seq, token) {
|
|
14530
|
+
const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
|
|
14531
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14532
|
+
runId: run.id,
|
|
14533
|
+
seq,
|
|
14534
|
+
name: `signal:${token}`,
|
|
14535
|
+
kind: "signal",
|
|
14536
|
+
output: payload
|
|
14537
|
+
}));
|
|
14538
|
+
this.resumeSoon(run.id);
|
|
14539
|
+
}, "deliverBuffered");
|
|
14540
|
+
const buffered = await this.store.takeBufferedSignal(token);
|
|
14541
|
+
if (buffered) {
|
|
14542
|
+
await deliverBuffered(buffered.payload);
|
|
14543
|
+
} else {
|
|
14544
|
+
await this.store.putSignalWaiter({
|
|
14545
|
+
token,
|
|
14546
|
+
runId: run.id,
|
|
14547
|
+
seq
|
|
14548
|
+
});
|
|
14549
|
+
const lateBuffered = await this.store.takeBufferedSignal(token);
|
|
14550
|
+
if (lateBuffered) {
|
|
14551
|
+
await this.store.removeSignalWaiter({
|
|
14552
|
+
token,
|
|
14553
|
+
runId: run.id,
|
|
14554
|
+
seq
|
|
14555
|
+
});
|
|
14556
|
+
await deliverBuffered(lateBuffered.payload);
|
|
14557
|
+
}
|
|
14558
|
+
}
|
|
14559
|
+
}
|
|
14560
|
+
/** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
|
|
14561
|
+
resumeSoon(runId) {
|
|
14562
|
+
setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
|
|
14563
|
+
}
|
|
14564
|
+
/**
|
|
14565
|
+
* Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
|
|
14566
|
+
* `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
|
|
14567
|
+
* back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
|
|
14568
|
+
* deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
|
|
14569
|
+
*/
|
|
14570
|
+
async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
|
|
14571
|
+
const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
|
|
14572
|
+
if (recorded?.wakeAt != null) return recorded.wakeAt;
|
|
14573
|
+
const deadline = this.clock() + timeoutMs;
|
|
14574
|
+
const at = /* @__PURE__ */ new Date();
|
|
14575
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14576
|
+
runId,
|
|
14577
|
+
seq: deadlineSeq,
|
|
14578
|
+
name: `timeout:${label}`,
|
|
14579
|
+
kind: "sleep",
|
|
14580
|
+
status: "pending",
|
|
14581
|
+
attempts: 1,
|
|
14582
|
+
input: wait,
|
|
14583
|
+
wakeAt: deadline,
|
|
14584
|
+
enqueuedAt: at,
|
|
14585
|
+
startedAt: at,
|
|
14586
|
+
finishedAt: at
|
|
14587
|
+
}));
|
|
14588
|
+
return deadline;
|
|
14589
|
+
}
|
|
14590
|
+
/**
|
|
14591
|
+
* Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
|
|
14592
|
+
* (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
|
|
14593
|
+
* time out — the in-process wait removes its waiter the same way when it finds the deadline past.
|
|
14594
|
+
* A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
|
|
14595
|
+
* worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
|
|
14596
|
+
*/
|
|
14597
|
+
async expireRemoteDeadlines(runId, checkpoints) {
|
|
14598
|
+
const now = this.clock();
|
|
14599
|
+
const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
|
|
14600
|
+
let out = checkpoints;
|
|
14601
|
+
for (const cp of checkpoints) {
|
|
14602
|
+
if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
|
|
14603
|
+
if (cp.wakeAt == null || cp.wakeAt > now) continue;
|
|
14604
|
+
const wait = cp.input;
|
|
14605
|
+
if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
|
|
14606
|
+
if (settled.has(wait.waitSeq)) continue;
|
|
14607
|
+
await this.store.removeSignalWaiter({
|
|
14608
|
+
token: wait.token,
|
|
14609
|
+
runId,
|
|
14610
|
+
seq: wait.waitSeq
|
|
14611
|
+
});
|
|
14612
|
+
const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
|
|
14613
|
+
if (landed && landed.status === "completed") {
|
|
14614
|
+
out = [
|
|
14615
|
+
...out.filter((c) => c.seq !== landed.seq),
|
|
14616
|
+
landed
|
|
14617
|
+
].sort((a, b) => a.seq - b.seq);
|
|
14618
|
+
}
|
|
14619
|
+
}
|
|
14620
|
+
return out;
|
|
14621
|
+
}
|
|
14622
|
+
/**
|
|
14623
|
+
* Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
|
|
14624
|
+
* see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
|
|
14625
|
+
* candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
|
|
14626
|
+
* {@link eventBufferTtlMs} is set, like the reconcile sweep.
|
|
14627
|
+
*/
|
|
14628
|
+
async claimBufferedEvent(run, name, match) {
|
|
14629
|
+
const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
|
|
14630
|
+
const nowMs = this.clock();
|
|
14631
|
+
for (const candidate of candidates) {
|
|
14632
|
+
if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
|
|
14633
|
+
await this.store.removeBufferedEvent(candidate.id);
|
|
14634
|
+
continue;
|
|
14635
|
+
}
|
|
14636
|
+
const { payload, tenantIds } = readBufferedEvent(candidate.payload);
|
|
14637
|
+
if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
|
|
14638
|
+
if (!eventMatches(payload, match)) continue;
|
|
14639
|
+
if (!await this.store.removeBufferedEvent(candidate.id)) continue;
|
|
14640
|
+
return {
|
|
14641
|
+
payload
|
|
14642
|
+
};
|
|
14643
|
+
}
|
|
14644
|
+
return void 0;
|
|
14645
|
+
}
|
|
14646
|
+
/** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
|
|
14647
|
+
* sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
|
|
14648
|
+
callLease(policy) {
|
|
14649
|
+
if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
|
|
14650
|
+
return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
14651
|
+
}
|
|
14652
|
+
/**
|
|
14653
|
+
* A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
|
|
14654
|
+
* checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
|
|
14655
|
+
* callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
|
|
14656
|
+
* behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
|
|
14657
|
+
* timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
|
|
14658
|
+
* and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
|
|
14659
|
+
*
|
|
14660
|
+
* - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
|
|
14661
|
+
* the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
|
|
14662
|
+
* dispatched again under the same `stepId` with `attempts + 1`.
|
|
14663
|
+
* - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
|
|
14664
|
+
* re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
|
|
14665
|
+
* workflow can catch, as `RemoteStepTimeout` would surface in-process.
|
|
14666
|
+
*/
|
|
14667
|
+
async applyCallPolicy(run, cmd, cp) {
|
|
14668
|
+
const policy = cmd.policy;
|
|
14669
|
+
if (cp.kind !== "remote") return null;
|
|
14670
|
+
if (cp.status === "failed") {
|
|
14671
|
+
if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
|
|
14672
|
+
let due = cp.wakeAt;
|
|
14673
|
+
if (due == null) {
|
|
14674
|
+
due = this.clock() + backoffDelay(cp.attempts, policy);
|
|
14675
|
+
if (due > this.clock()) {
|
|
14676
|
+
await this.store.saveCheckpoint({
|
|
14677
|
+
...cp,
|
|
14678
|
+
wakeAt: due
|
|
14679
|
+
});
|
|
14680
|
+
return due;
|
|
14681
|
+
}
|
|
14682
|
+
}
|
|
14683
|
+
if (this.clock() < due) return due;
|
|
14684
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
|
|
14685
|
+
}
|
|
14686
|
+
if (cp.status !== "pending" || !policy.timeoutMs) return null;
|
|
14687
|
+
if (cp.wakeAt == null) {
|
|
14688
|
+
const lease = this.clock() + policy.timeoutMs;
|
|
14689
|
+
await this.store.saveCheckpoint({
|
|
14690
|
+
...cp,
|
|
14691
|
+
wakeAt: lease
|
|
14692
|
+
});
|
|
14693
|
+
return this.stepLeaseWakeAt(lease);
|
|
14694
|
+
}
|
|
14695
|
+
if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
|
|
14696
|
+
if (cp.attempts < remoteCallMaxAttempts(policy)) {
|
|
14697
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
|
|
14698
|
+
}
|
|
14699
|
+
const id = cp.stepId ?? stepId(run.id, cp.seq);
|
|
14700
|
+
const error2 = {
|
|
14701
|
+
message: new RemoteStepTimeout(id, policy.timeoutMs).message,
|
|
14702
|
+
code: "remote_step_timeout"
|
|
14703
|
+
};
|
|
14704
|
+
await this.store.saveCheckpoint({
|
|
14705
|
+
...cp,
|
|
14706
|
+
status: "failed",
|
|
14707
|
+
error: error2,
|
|
14708
|
+
wakeAt: void 0,
|
|
14709
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
14710
|
+
});
|
|
14711
|
+
this.emit({
|
|
14712
|
+
type: "step.failed",
|
|
14713
|
+
runId: run.id,
|
|
14714
|
+
seq: cp.seq,
|
|
14715
|
+
name: cp.name,
|
|
14716
|
+
kind: "remote",
|
|
14717
|
+
error: error2
|
|
14718
|
+
});
|
|
14719
|
+
return void 0;
|
|
14720
|
+
}
|
|
14721
|
+
/** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
|
|
14722
|
+
async redispatchCall(run, cmd, cp, attempt, policy, why) {
|
|
14723
|
+
const lease = this.callLease(policy);
|
|
14724
|
+
const at = /* @__PURE__ */ new Date();
|
|
14725
|
+
await this.store.saveCheckpoint({
|
|
14726
|
+
...cp,
|
|
14727
|
+
status: "pending",
|
|
14728
|
+
output: void 0,
|
|
14729
|
+
error: void 0,
|
|
14730
|
+
attempts: attempt,
|
|
14731
|
+
wakeAt: lease,
|
|
14732
|
+
enqueuedAt: at,
|
|
14733
|
+
startedAt: at,
|
|
14734
|
+
finishedAt: at,
|
|
14735
|
+
events: why === "timeout" ? [
|
|
14736
|
+
...cp.events ?? [],
|
|
14737
|
+
lostStepEvent(this.clock(), attempt, "redispatched")
|
|
14738
|
+
] : cp.events
|
|
14739
|
+
});
|
|
14740
|
+
await this.dispatchRemoteTask({
|
|
14741
|
+
runId: run.id,
|
|
14742
|
+
seq: cp.seq,
|
|
14743
|
+
name: cmd.name,
|
|
14744
|
+
stepId: cp.stepId ?? stepId(run.id, cp.seq),
|
|
14745
|
+
group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
|
|
14746
|
+
input: cp.input,
|
|
14747
|
+
attempt
|
|
14748
|
+
});
|
|
14749
|
+
this.emit({
|
|
14750
|
+
type: "step.started",
|
|
14751
|
+
runId: run.id,
|
|
14752
|
+
seq: cp.seq,
|
|
14753
|
+
name: cmd.name,
|
|
14754
|
+
kind: "remote",
|
|
14755
|
+
...why === "timeout" ? {
|
|
14756
|
+
redispatched: true
|
|
14757
|
+
} : {}
|
|
14758
|
+
});
|
|
14759
|
+
return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
|
|
14760
|
+
}
|
|
14761
|
+
/**
|
|
14071
14762
|
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
14072
14763
|
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
14073
14764
|
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
@@ -18469,6 +19160,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18469
19160
|
PARENT_CLOSE_TAG_PREFIX,
|
|
18470
19161
|
QueueController,
|
|
18471
19162
|
REMOTE_EVENT,
|
|
19163
|
+
REMOTE_WORKFLOW_FEATURES,
|
|
18472
19164
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
18473
19165
|
RUN_VALUE_FACET_LIMIT,
|
|
18474
19166
|
RUN_VALUE_FACET_SCAN,
|
|
@@ -18626,6 +19318,9 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18626
19318
|
readBufferedEvent,
|
|
18627
19319
|
readRunsByIds,
|
|
18628
19320
|
readSearchAttributes,
|
|
19321
|
+
remoteCallMaxAttempts,
|
|
19322
|
+
remoteCallPolicyOf,
|
|
19323
|
+
remoteCallRetries,
|
|
18629
19324
|
requiredCapabilities,
|
|
18630
19325
|
resolveConcurrency,
|
|
18631
19326
|
resolveRouting,
|