@dudousxd/nestjs-durable-core 0.101.0 → 0.103.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +769 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1155 -824
- package/dist/index.d.ts +1155 -824
- package/dist/index.js +765 -74
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -881,6 +881,14 @@ function isNonEmptyRunScope(scope) {
|
|
|
881
881
|
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.tenantId !== void 0 || scope.tenantIds !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
882
882
|
}
|
|
883
883
|
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
884
|
+
var REMOTE_WORKFLOW_FEATURES = [
|
|
885
|
+
"callPolicy",
|
|
886
|
+
"sleepUntil",
|
|
887
|
+
"waitSignalTimeout",
|
|
888
|
+
"waitEvent",
|
|
889
|
+
"spawnChild",
|
|
890
|
+
"continueAsNew"
|
|
891
|
+
];
|
|
884
892
|
var PARENT_CLOSE_TAG_PREFIX = "parent-close:";
|
|
885
893
|
|
|
886
894
|
// src/events.ts
|
|
@@ -2342,6 +2350,13 @@ function canonicalizeRegistrations(regs) {
|
|
|
2342
2350
|
subscribes: [
|
|
2343
2351
|
...new Set(r.subscribes)
|
|
2344
2352
|
].sort()
|
|
2353
|
+
} : {},
|
|
2354
|
+
// Conditional too: a registration without them keeps its hash; a changed option changes it.
|
|
2355
|
+
...r.options !== void 0 ? {
|
|
2356
|
+
options: stableJson(r.options)
|
|
2357
|
+
} : {},
|
|
2358
|
+
...r.deadLetter !== void 0 ? {
|
|
2359
|
+
deadLetter: r.deadLetter
|
|
2345
2360
|
} : {}
|
|
2346
2361
|
}));
|
|
2347
2362
|
const byJson = /* @__PURE__ */ new Map();
|
|
@@ -3062,6 +3077,35 @@ function codeAndDetails(err) {
|
|
|
3062
3077
|
}
|
|
3063
3078
|
__name(codeAndDetails, "codeAndDetails");
|
|
3064
3079
|
|
|
3080
|
+
// src/remote-call-policy.ts
|
|
3081
|
+
function remoteCallPolicyOf(config, opts) {
|
|
3082
|
+
const policy = {};
|
|
3083
|
+
const retries = opts?.retries ?? config?.retries;
|
|
3084
|
+
if (retries !== void 0) policy.retries = retries;
|
|
3085
|
+
const backoff = opts?.backoff ?? config?.backoff;
|
|
3086
|
+
if (backoff !== void 0) policy.backoff = backoff;
|
|
3087
|
+
const backoffMs = opts?.backoffMs ?? config?.backoffMs;
|
|
3088
|
+
if (backoffMs !== void 0) policy.backoffMs = backoffMs;
|
|
3089
|
+
const backoffMaxMs = opts?.backoffMaxMs ?? config?.backoffMaxMs;
|
|
3090
|
+
if (backoffMaxMs !== void 0) policy.backoffMaxMs = backoffMaxMs;
|
|
3091
|
+
const jitter = opts?.jitter ?? config?.jitter;
|
|
3092
|
+
if (jitter !== void 0) policy.jitter = jitter;
|
|
3093
|
+
const timeoutMs = opts?.timeoutMs ?? config?.timeoutMs;
|
|
3094
|
+
if (timeoutMs !== void 0) policy.timeoutMs = timeoutMs;
|
|
3095
|
+
return Object.keys(policy).length > 0 ? policy : void 0;
|
|
3096
|
+
}
|
|
3097
|
+
__name(remoteCallPolicyOf, "remoteCallPolicyOf");
|
|
3098
|
+
function remoteCallMaxAttempts(policy) {
|
|
3099
|
+
return Math.max(1, policy?.retries ?? 1);
|
|
3100
|
+
}
|
|
3101
|
+
__name(remoteCallMaxAttempts, "remoteCallMaxAttempts");
|
|
3102
|
+
function remoteCallRetries(policy, attempts, error2) {
|
|
3103
|
+
if (policy === void 0 || policy.timeoutMs) return false;
|
|
3104
|
+
if (error2?.retryable === false) return false;
|
|
3105
|
+
return attempts < remoteCallMaxAttempts(policy);
|
|
3106
|
+
}
|
|
3107
|
+
__name(remoteCallRetries, "remoteCallRetries");
|
|
3108
|
+
|
|
3065
3109
|
// src/remote-event.ts
|
|
3066
3110
|
var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
|
|
3067
3111
|
function markRemoteEvent(event) {
|
|
@@ -3119,7 +3163,12 @@ var RemoteWorkflowExecutor = class {
|
|
|
3119
3163
|
} : {},
|
|
3120
3164
|
group: tenantGroup(sanitizeQueueToken(this.name), this.partition),
|
|
3121
3165
|
priority: run.priority,
|
|
3122
|
-
attempt: 1
|
|
3166
|
+
attempt: 1,
|
|
3167
|
+
// What this control plane applies beyond the original command kinds, so a worker fails a turn
|
|
3168
|
+
// that needs more instead of emitting a command this engine would not understand.
|
|
3169
|
+
features: [
|
|
3170
|
+
...REMOTE_WORKFLOW_FEATURES
|
|
3171
|
+
]
|
|
3123
3172
|
};
|
|
3124
3173
|
await this.transport.dispatchWorkflowTask(task);
|
|
3125
3174
|
}
|
|
@@ -4052,7 +4101,8 @@ var ScheduleClient = class {
|
|
|
4052
4101
|
const input = specOf(record).input;
|
|
4053
4102
|
const opts = startOptions(record);
|
|
4054
4103
|
if (isAnnounced(record) && this.host.startParked) {
|
|
4055
|
-
|
|
4104
|
+
const singleton = record.state.announce?.singleton;
|
|
4105
|
+
return singleton ? this.host.startParked(record.workflow, input, runId, opts, singleton) : this.host.startParked(record.workflow, input, runId, opts);
|
|
4056
4106
|
}
|
|
4057
4107
|
return this.host.start(record.workflow, input, runId, opts);
|
|
4058
4108
|
}
|
|
@@ -4248,7 +4298,8 @@ var ScheduleClient = class {
|
|
|
4248
4298
|
announce: {
|
|
4249
4299
|
key: doc.key,
|
|
4250
4300
|
fingerprint,
|
|
4251
|
-
declaredPaused: doc.declaredPaused
|
|
4301
|
+
declaredPaused: doc.declaredPaused,
|
|
4302
|
+
singleton: doc.singleton
|
|
4252
4303
|
}
|
|
4253
4304
|
}),
|
|
4254
4305
|
updatedAt: new Date(now)
|
|
@@ -4285,7 +4336,10 @@ var ScheduleClient = class {
|
|
|
4285
4336
|
announce: {
|
|
4286
4337
|
key: item.key,
|
|
4287
4338
|
fingerprint,
|
|
4288
|
-
declaredPaused
|
|
4339
|
+
declaredPaused,
|
|
4340
|
+
...validSingleton(item.singleton) ? {
|
|
4341
|
+
singleton: item.singleton
|
|
4342
|
+
} : {}
|
|
4289
4343
|
}
|
|
4290
4344
|
});
|
|
4291
4345
|
out.adopted.push(id);
|
|
@@ -4435,6 +4489,10 @@ function specOf(record) {
|
|
|
4435
4489
|
return record.spec;
|
|
4436
4490
|
}
|
|
4437
4491
|
__name(specOf, "specOf");
|
|
4492
|
+
function validSingleton(value) {
|
|
4493
|
+
return value !== null && typeof value === "object" && typeof value.key === "string" && value.key !== "";
|
|
4494
|
+
}
|
|
4495
|
+
__name(validSingleton, "validSingleton");
|
|
4438
4496
|
function isAnnounced(record) {
|
|
4439
4497
|
return record.state?.announce !== void 0;
|
|
4440
4498
|
}
|
|
@@ -6878,6 +6936,43 @@ var DEFAULT_STICKY_RUNS = 1e3;
|
|
|
6878
6936
|
var DEFAULT_STICKY_IDLE_MS = 3e5;
|
|
6879
6937
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
6880
6938
|
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
6939
|
+
var ANNOUNCED_TIMEOUTS_TTL_MS = 15e3;
|
|
6940
|
+
function tenantScopedKey(namespace, key) {
|
|
6941
|
+
return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
|
|
6942
|
+
}
|
|
6943
|
+
__name(tenantScopedKey, "tenantScopedKey");
|
|
6944
|
+
function singletonKeyOf(run) {
|
|
6945
|
+
const tag = run.tags?.find((t) => t.startsWith("singleton:"));
|
|
6946
|
+
return tag?.slice("singleton:".length);
|
|
6947
|
+
}
|
|
6948
|
+
__name(singletonKeyOf, "singletonKeyOf");
|
|
6949
|
+
function conventionSingleton(run, announced, wire) {
|
|
6950
|
+
if (wire !== void 0) {
|
|
6951
|
+
const key2 = wire.key;
|
|
6952
|
+
return {
|
|
6953
|
+
key: /* @__PURE__ */ __name(() => key2, "key"),
|
|
6954
|
+
...typeof wire.limit === "number" ? {
|
|
6955
|
+
limit: wire.limit
|
|
6956
|
+
} : {},
|
|
6957
|
+
...typeof wire.maxQueueDepth === "number" ? {
|
|
6958
|
+
maxQueueDepth: wire.maxQueueDepth
|
|
6959
|
+
} : {}
|
|
6960
|
+
};
|
|
6961
|
+
}
|
|
6962
|
+
const key = singletonKeyOf(run);
|
|
6963
|
+
if (key === void 0 || announced === void 0) return void 0;
|
|
6964
|
+
if (announced === null) return {
|
|
6965
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
6966
|
+
limit: 1
|
|
6967
|
+
};
|
|
6968
|
+
const declared = announced.options?.singleton;
|
|
6969
|
+
if (declared === void 0) return void 0;
|
|
6970
|
+
return {
|
|
6971
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
6972
|
+
limit: typeof declared.limit === "number" ? declared.limit : 1
|
|
6973
|
+
};
|
|
6974
|
+
}
|
|
6975
|
+
__name(conventionSingleton, "conventionSingleton");
|
|
6881
6976
|
function servedNamespace(namespace, group) {
|
|
6882
6977
|
if (namespace) return namespace;
|
|
6883
6978
|
const at = group?.lastIndexOf("@") ?? -1;
|
|
@@ -7367,13 +7462,28 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7367
7462
|
if (this.queuesRunsOverTransport) void this.wake(runId);
|
|
7368
7463
|
else this.runDispatcher.dispatch(runId);
|
|
7369
7464
|
}, "dispatch"),
|
|
7370
|
-
configFor: /* @__PURE__ */ __name((run) =>
|
|
7465
|
+
configFor: /* @__PURE__ */ __name((run) => {
|
|
7466
|
+
const local = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton;
|
|
7467
|
+
if (local) return local;
|
|
7468
|
+
const key = this.latest.has(run.workflow) ? void 0 : singletonKeyOf(run);
|
|
7469
|
+
if (key === void 0) return void 0;
|
|
7470
|
+
const limit = this.conventionSingletonLimits.get(`${runNamespace(run.namespace)}\0${run.workflow}`);
|
|
7471
|
+
return {
|
|
7472
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7473
|
+
...limit !== void 0 ? {
|
|
7474
|
+
limit
|
|
7475
|
+
} : {}
|
|
7476
|
+
};
|
|
7477
|
+
}, "configFor")
|
|
7371
7478
|
});
|
|
7372
7479
|
this.schedules = new ScheduleClient(this.store, {
|
|
7373
7480
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7374
7481
|
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7375
7482
|
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7376
|
-
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true
|
|
7483
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts, singleton) => this.startWith(workflow, input, runId, opts, true, singleton ? {
|
|
7484
|
+
...singleton,
|
|
7485
|
+
key: tenantScopedKey(opts?.namespace, singleton.key)
|
|
7486
|
+
} : void 0), "startParked"),
|
|
7377
7487
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7378
7488
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7379
7489
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -8183,6 +8293,39 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8183
8293
|
} catch {
|
|
8184
8294
|
}
|
|
8185
8295
|
}
|
|
8296
|
+
if (!this.latest.has(run.workflow)) this.inBackground(this.routeAnnouncedDeadLetter(run));
|
|
8297
|
+
}
|
|
8298
|
+
/**
|
|
8299
|
+
* The dead-letter workflow the live workers serving `run`'s workflow announce (a tenant's inline
|
|
8300
|
+
* `@DeadLetter()` handler, its `deadLetterWorkflow`, or its module-level default), or `undefined`
|
|
8301
|
+
* when the workflow is registered on this engine (its own routing applies), they announce none, or
|
|
8302
|
+
* they cannot be read. A dead-letter listener with a catch-all default (`DurableModule`'s
|
|
8303
|
+
* `deadLetterWorkflow`) checks it first, so the app's own route wins over the default.
|
|
8304
|
+
*/
|
|
8305
|
+
async announcedDeadLetter(run) {
|
|
8306
|
+
if (this.latest.has(run.workflow) || this.isRegisteredHere(run)) return void 0;
|
|
8307
|
+
const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
|
|
8308
|
+
const reg = await this.announcedRegistration(group, run.workflow, run.workflowVersion);
|
|
8309
|
+
const target = reg?.deadLetter;
|
|
8310
|
+
return typeof target === "string" && target !== "" ? target : void 0;
|
|
8311
|
+
}
|
|
8312
|
+
/**
|
|
8313
|
+
* Start the announced dead-letter workflow of a dead run (see {@link announcedDeadLetter}) in the
|
|
8314
|
+
* run's namespace, as `dlq:<runId>` with the `DeadLetter` payload, through the tenant start path:
|
|
8315
|
+
* with no live worker of the app it waits `blocked` instead of being lost. Idempotent by run id.
|
|
8316
|
+
*/
|
|
8317
|
+
async routeAnnouncedDeadLetter(run) {
|
|
8318
|
+
const target = await this.announcedDeadLetter(run);
|
|
8319
|
+
if (!target) return;
|
|
8320
|
+
await this.startWith(target, {
|
|
8321
|
+
deadRunId: run.id,
|
|
8322
|
+
workflow: run.workflow,
|
|
8323
|
+
input: run.input,
|
|
8324
|
+
error: run.error
|
|
8325
|
+
}, `dlq:${run.id}`, {
|
|
8326
|
+
namespace: run.namespace,
|
|
8327
|
+
...inheritTenant(run)
|
|
8328
|
+
}, true);
|
|
8186
8329
|
}
|
|
8187
8330
|
/**
|
|
8188
8331
|
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
@@ -9483,7 +9626,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9483
9626
|
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9484
9627
|
* a start whose caller has already returned is never lost.
|
|
9485
9628
|
*/
|
|
9486
|
-
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9629
|
+
async startWith(workflow, input, runId, opts, parkUnroutable, wireSingleton) {
|
|
9487
9630
|
const name = workflowName(workflow);
|
|
9488
9631
|
const pinned = opts?.version;
|
|
9489
9632
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9508,12 +9651,15 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9508
9651
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9509
9652
|
};
|
|
9510
9653
|
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9511
|
-
askFleetForVersion: true
|
|
9654
|
+
askFleetForVersion: true,
|
|
9655
|
+
...wireSingleton !== void 0 ? {
|
|
9656
|
+
singleton: wireSingleton
|
|
9657
|
+
} : {}
|
|
9512
9658
|
});
|
|
9513
9659
|
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9514
9660
|
}
|
|
9515
9661
|
if (!registered && parkUnroutable) {
|
|
9516
|
-
return this.parkStartForWorker(name, input, runId, opts);
|
|
9662
|
+
return this.parkStartForWorker(name, input, runId, opts, wireSingleton);
|
|
9517
9663
|
}
|
|
9518
9664
|
if (!registered) {
|
|
9519
9665
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
@@ -9615,11 +9761,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9615
9761
|
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9616
9762
|
* start is checked first at its `startAt`.
|
|
9617
9763
|
*/
|
|
9618
|
-
async parkStartForWorker(name, input, runId, opts) {
|
|
9764
|
+
async parkStartForWorker(name, input, runId, opts, wireSingleton) {
|
|
9619
9765
|
const prior = await this.store.getRun(runId);
|
|
9620
9766
|
if (prior) return this.existingStartResult(prior);
|
|
9621
9767
|
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9622
9768
|
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9769
|
+
const singleton = conventionSingleton({}, void 0, wireSingleton);
|
|
9770
|
+
if (singleton) await this.singletons.assertCapacity(name, singleton, input);
|
|
9623
9771
|
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9624
9772
|
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9625
9773
|
const pinned = opts?.version;
|
|
@@ -9634,6 +9782,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9634
9782
|
const now = /* @__PURE__ */ new Date();
|
|
9635
9783
|
const tags = mergeTags(void 0, [
|
|
9636
9784
|
...opts?.tags ?? [],
|
|
9785
|
+
...singleton ? [
|
|
9786
|
+
this.singletons.tag(singleton, input)
|
|
9787
|
+
] : [],
|
|
9637
9788
|
...quota ? [
|
|
9638
9789
|
concurrencyTag(quota.key)
|
|
9639
9790
|
] : [],
|
|
@@ -9722,8 +9873,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9722
9873
|
concurrency: message.concurrency
|
|
9723
9874
|
} : {}
|
|
9724
9875
|
};
|
|
9876
|
+
const singleton = message.singleton !== void 0 && typeof message.singleton?.key === "string" ? {
|
|
9877
|
+
...message.singleton,
|
|
9878
|
+
key: tenantScopedKey(namespace, message.singleton.key)
|
|
9879
|
+
} : void 0;
|
|
9725
9880
|
try {
|
|
9726
|
-
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
9881
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true, singleton);
|
|
9727
9882
|
} catch (err) {
|
|
9728
9883
|
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
9729
9884
|
throw err;
|
|
@@ -9731,6 +9886,62 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9731
9886
|
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
9732
9887
|
}
|
|
9733
9888
|
}
|
|
9889
|
+
/**
|
|
9890
|
+
* The control plane's half of a tenant's `signalWithStart` (a `signalWithStart` run request):
|
|
9891
|
+
* start `start` like {@link startForTenant} unless its run exists, then signal `signal.token`.
|
|
9892
|
+
* Authorization is the caller's (the run-request responder): the run must belong to `namespace`,
|
|
9893
|
+
* and so must a run already waiting on the token. Delivered at once to a waiting run; with none
|
|
9894
|
+
* waiting yet it is buffered only when the token contains the run id (a run-scoped token), so a
|
|
9895
|
+
* tenant can never plant a payload for a wait another namespace opens later. `signalled: false`
|
|
9896
|
+
* when it was neither delivered nor buffered.
|
|
9897
|
+
*/
|
|
9898
|
+
async signalWithStartForTenant(start, namespace, tenant, signal) {
|
|
9899
|
+
const started = await this.startForTenant(start, namespace, tenant);
|
|
9900
|
+
if (TERMINAL_RUN_STATUSES.includes(started.status)) {
|
|
9901
|
+
return {
|
|
9902
|
+
runId: start.runId,
|
|
9903
|
+
signalled: false
|
|
9904
|
+
};
|
|
9905
|
+
}
|
|
9906
|
+
const waiter = (await this.store.listSignalWaiters(signal.token)).find((w) => w.token === signal.token);
|
|
9907
|
+
if (waiter) {
|
|
9908
|
+
const run = await this.store.getRun(waiter.runId);
|
|
9909
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) {
|
|
9910
|
+
return {
|
|
9911
|
+
runId: start.runId,
|
|
9912
|
+
signalled: false
|
|
9913
|
+
};
|
|
9914
|
+
}
|
|
9915
|
+
} else if (!signal.token.includes(start.runId)) {
|
|
9916
|
+
return {
|
|
9917
|
+
runId: start.runId,
|
|
9918
|
+
signalled: false
|
|
9919
|
+
};
|
|
9920
|
+
}
|
|
9921
|
+
await this.signalUnchecked(signal.token, signal.payload);
|
|
9922
|
+
return {
|
|
9923
|
+
runId: start.runId,
|
|
9924
|
+
signalled: true
|
|
9925
|
+
};
|
|
9926
|
+
}
|
|
9927
|
+
/**
|
|
9928
|
+
* The control plane's half of a tenant's `publishEvent` (a `publishEvent` run request): resume the
|
|
9929
|
+
* runs of `namespace` waiting on event `name` whose match the payload satisfies. Starting the
|
|
9930
|
+
* tenant's own subscribers is the tenant's (it holds their `where` and `input`); nothing is
|
|
9931
|
+
* buffered, since a buffered event could reach another namespace's later wait. Returns how many
|
|
9932
|
+
* runs it resumed.
|
|
9933
|
+
*/
|
|
9934
|
+
async publishEventForTenant(name, payload, namespace) {
|
|
9935
|
+
let touched = 0;
|
|
9936
|
+
for (const w of await this.store.listSignalWaiters(eventPrefix(name))) {
|
|
9937
|
+
if (!eventMatches(payload, eventMatchOf(w.token))) continue;
|
|
9938
|
+
const run = await this.store.getRun(w.runId);
|
|
9939
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) continue;
|
|
9940
|
+
await this.signalUnchecked(w.token, payload);
|
|
9941
|
+
touched += 1;
|
|
9942
|
+
}
|
|
9943
|
+
return touched;
|
|
9944
|
+
}
|
|
9734
9945
|
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
9735
9946
|
async recordRejectedStart(name, input, runId, opts, err) {
|
|
9736
9947
|
const prior = await this.store.getRun(runId);
|
|
@@ -10202,9 +10413,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10202
10413
|
if (!liveTransport) return void 0;
|
|
10203
10414
|
const executor = new RemoteWorkflowExecutor(liveTransport, run.workflow, run.namespace);
|
|
10204
10415
|
const declared = opts.askFleetForVersion ? await this.declaredVersion(liveTransport, group, run.workflow) : void 0;
|
|
10416
|
+
const version = declared ?? run.workflowVersion;
|
|
10417
|
+
const announced = await this.announcedRegistration(group, run.workflow, version);
|
|
10418
|
+
const singleton = conventionSingleton(run, announced, opts.singleton);
|
|
10419
|
+
if (singleton?.limit !== void 0) {
|
|
10420
|
+
this.conventionSingletonLimits.set(`${runNamespace(run.namespace)}\0${run.workflow}`, singleton.limit);
|
|
10421
|
+
}
|
|
10422
|
+
const tags = announced?.options?.tags;
|
|
10205
10423
|
return {
|
|
10206
10424
|
name: run.workflow,
|
|
10207
|
-
version
|
|
10425
|
+
version,
|
|
10208
10426
|
fn: /* @__PURE__ */ __name(() => {
|
|
10209
10427
|
throw new Error(`remote workflow ${run.workflow} has no local body`);
|
|
10210
10428
|
}, "fn"),
|
|
@@ -10213,12 +10431,47 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10213
10431
|
group,
|
|
10214
10432
|
executor
|
|
10215
10433
|
},
|
|
10434
|
+
...Array.isArray(tags) && tags.length > 0 ? {
|
|
10435
|
+
tags
|
|
10436
|
+
} : {},
|
|
10437
|
+
...singleton ? {
|
|
10438
|
+
singleton
|
|
10439
|
+
} : {},
|
|
10216
10440
|
...opts.askFleetForVersion ? {
|
|
10217
10441
|
versionDeclared: declared !== void 0
|
|
10218
10442
|
} : {}
|
|
10219
10443
|
};
|
|
10220
10444
|
}
|
|
10221
10445
|
/**
|
|
10446
|
+
* The registration the live workers of `group` announce for `workflow` (at `version`, when one of
|
|
10447
|
+
* them states it): where they disagree (a rolling deploy), the one from the worker that started
|
|
10448
|
+
* last, as for announced schedules. `undefined` when none announces it, `null` when the read
|
|
10449
|
+
* failed (what the workers state is then unknown, not "nothing"). Reads
|
|
10450
|
+
* through the descriptor cache ({@link workerDescriptorCacheMs}), so a turn costs no extra round
|
|
10451
|
+
* trip in steady state.
|
|
10452
|
+
*/
|
|
10453
|
+
async announcedRegistration(group, workflow, version) {
|
|
10454
|
+
let descriptors;
|
|
10455
|
+
try {
|
|
10456
|
+
descriptors = await this.liveWorkerDescriptors(group);
|
|
10457
|
+
} catch {
|
|
10458
|
+
return null;
|
|
10459
|
+
}
|
|
10460
|
+
let best;
|
|
10461
|
+
for (const descriptor of descriptors) {
|
|
10462
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10463
|
+
if (reg?.name !== workflow) continue;
|
|
10464
|
+
if (version !== void 0 && reg.version !== void 0 && reg.version !== version) continue;
|
|
10465
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10466
|
+
if (!best || at > best.at) best = {
|
|
10467
|
+
at,
|
|
10468
|
+
reg
|
|
10469
|
+
};
|
|
10470
|
+
}
|
|
10471
|
+
}
|
|
10472
|
+
return best?.reg;
|
|
10473
|
+
}
|
|
10474
|
+
/**
|
|
10222
10475
|
* The version the LIVE fleet declares for `workflow` on `group`, or `undefined` when it declares
|
|
10223
10476
|
* none — which is a fact about the fleet, not a failure to read it.
|
|
10224
10477
|
*
|
|
@@ -10442,34 +10695,95 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10442
10695
|
for (const reg of new Set(this.latest.values())) {
|
|
10443
10696
|
if (this.draining) return;
|
|
10444
10697
|
if (reg.executionTimeoutMs == null) continue;
|
|
10445
|
-
|
|
10446
|
-
|
|
10447
|
-
|
|
10448
|
-
|
|
10449
|
-
|
|
10450
|
-
|
|
10451
|
-
|
|
10452
|
-
|
|
10453
|
-
|
|
10454
|
-
|
|
10455
|
-
|
|
10456
|
-
|
|
10457
|
-
|
|
10458
|
-
|
|
10459
|
-
|
|
10460
|
-
|
|
10461
|
-
|
|
10462
|
-
|
|
10463
|
-
|
|
10464
|
-
|
|
10465
|
-
|
|
10466
|
-
|
|
10467
|
-
|
|
10468
|
-
|
|
10469
|
-
|
|
10470
|
-
|
|
10471
|
-
|
|
10698
|
+
await this.cancelTimedOut(reg.name, this.namespace, now - reg.executionTimeoutMs);
|
|
10699
|
+
}
|
|
10700
|
+
for (const { workflow, namespace, timeoutMs } of await this.announcedTimeouts(now)) {
|
|
10701
|
+
if (this.draining) return;
|
|
10702
|
+
await this.cancelTimedOut(workflow, namespace, now - timeoutMs);
|
|
10703
|
+
}
|
|
10704
|
+
}
|
|
10705
|
+
/**
|
|
10706
|
+
* The singleton limit last resolved for a workflow served by convention, by `<namespace>\0<name>`
|
|
10707
|
+
* (see {@link resolveRemoteByConvention}): what the singleton gate's release wake-up reads, since
|
|
10708
|
+
* it cannot wait on the fleet. One entry per convention singleton the fleet serves.
|
|
10709
|
+
*/
|
|
10710
|
+
conventionSingletonLimits = /* @__PURE__ */ new Map();
|
|
10711
|
+
/** The last read of the announced execution timeouts (see {@link announcedTimeouts}). */
|
|
10712
|
+
announcedTimeoutsRead;
|
|
10713
|
+
/**
|
|
10714
|
+
* The `executionTimeoutMs` live workers announce (`@Workflow({ executionTimeout })` on a tenant),
|
|
10715
|
+
* per workflow and namespace, for workflows this engine does not register (a registration here
|
|
10716
|
+
* applies its own). Where workers disagree, the one that started last wins. Read at most once per
|
|
10717
|
+
* {@link ANNOUNCED_TIMEOUTS_TTL_MS}; empty without a transport that reads the fleet, or when the
|
|
10718
|
+
* read fails.
|
|
10719
|
+
*/
|
|
10720
|
+
async announcedTimeouts(now) {
|
|
10721
|
+
if (!this.pool.introspectsFleet) return [];
|
|
10722
|
+
const cached = this.announcedTimeoutsRead;
|
|
10723
|
+
if (cached && now - cached.at < ANNOUNCED_TIMEOUTS_TTL_MS) return cached.list;
|
|
10724
|
+
const newest = /* @__PURE__ */ new Map();
|
|
10725
|
+
try {
|
|
10726
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
10727
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10728
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10729
|
+
if (!reg || typeof reg.name !== "string" || this.latest.has(reg.name)) continue;
|
|
10730
|
+
const namespace = servedNamespace(descriptor.namespace, reg.group);
|
|
10731
|
+
if (this.namespace !== void 0 && runNamespace(this.namespace) !== namespace) continue;
|
|
10732
|
+
const key = `${namespace}\0${reg.name}`;
|
|
10733
|
+
const prior = newest.get(key);
|
|
10734
|
+
if (prior && prior.at >= at) continue;
|
|
10735
|
+
newest.set(key, {
|
|
10736
|
+
at,
|
|
10737
|
+
workflow: reg.name,
|
|
10738
|
+
namespace,
|
|
10739
|
+
ms: Number(reg.options?.executionTimeoutMs)
|
|
10740
|
+
});
|
|
10741
|
+
}
|
|
10472
10742
|
}
|
|
10743
|
+
} catch {
|
|
10744
|
+
return cached?.list ?? [];
|
|
10745
|
+
}
|
|
10746
|
+
const list2 = [
|
|
10747
|
+
...newest.values()
|
|
10748
|
+
].filter((e) => Number.isFinite(e.ms) && e.ms > 0).map((e) => ({
|
|
10749
|
+
workflow: e.workflow,
|
|
10750
|
+
namespace: e.namespace,
|
|
10751
|
+
timeoutMs: e.ms
|
|
10752
|
+
}));
|
|
10753
|
+
this.announcedTimeoutsRead = {
|
|
10754
|
+
at: now,
|
|
10755
|
+
list: list2
|
|
10756
|
+
};
|
|
10757
|
+
return list2;
|
|
10758
|
+
}
|
|
10759
|
+
/** Cancel (`execution_timeout`) the in-flight runs of `workflow` in `namespace` created before `deadline`. */
|
|
10760
|
+
async cancelTimedOut(workflow, namespace, deadline) {
|
|
10761
|
+
const inflight = await this.store.listRuns({
|
|
10762
|
+
workflow,
|
|
10763
|
+
statuses: TIMEOUT_SWEPT_STATUSES,
|
|
10764
|
+
namespace
|
|
10765
|
+
});
|
|
10766
|
+
for (const run of inflight) {
|
|
10767
|
+
if (this.draining) return;
|
|
10768
|
+
if (run.createdAt.getTime() > deadline) continue;
|
|
10769
|
+
const error2 = {
|
|
10770
|
+
message: "execution timeout",
|
|
10771
|
+
code: "execution_timeout"
|
|
10772
|
+
};
|
|
10773
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
10774
|
+
status: "cancelled",
|
|
10775
|
+
error: error2,
|
|
10776
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
10777
|
+
}));
|
|
10778
|
+
this.emit({
|
|
10779
|
+
type: "run.failed",
|
|
10780
|
+
runId: run.id,
|
|
10781
|
+
workflow: run.workflow,
|
|
10782
|
+
namespace: run.namespace,
|
|
10783
|
+
error: error2
|
|
10784
|
+
});
|
|
10785
|
+
await this.cancelChildren(run.id).catch(() => void 0);
|
|
10786
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
10473
10787
|
}
|
|
10474
10788
|
}
|
|
10475
10789
|
/**
|
|
@@ -13181,7 +13495,7 @@ ${run.workflowVersion}`;
|
|
|
13181
13495
|
let wfPlan = await this.planRoute(wfToken, registered.requires);
|
|
13182
13496
|
if (wfPlan?.status === "blocked") wfPlan = await this.planRoute(wfToken, registered.requires, true);
|
|
13183
13497
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
13184
|
-
const checkpoints = await this.replayCheckpoints(run.id);
|
|
13498
|
+
const checkpoints = await this.expireRemoteDeadlines(run.id, await this.replayCheckpoints(run.id));
|
|
13185
13499
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
13186
13500
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
13187
13501
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -13267,6 +13581,10 @@ ${run.workflowVersion}`;
|
|
|
13267
13581
|
*/
|
|
13268
13582
|
async applyDecision(run, decision) {
|
|
13269
13583
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
13584
|
+
const handoff = decision.commands.find((cmd) => cmd.kind === "continueAsNew");
|
|
13585
|
+
if (handoff !== void 0 && decision.status === "completed") {
|
|
13586
|
+
return this.applyRemoteContinueAsNew(run, decision.commands, handoff.input);
|
|
13587
|
+
}
|
|
13270
13588
|
if (decision.status === "completed") {
|
|
13271
13589
|
await this.applyCommands(run, decision.commands);
|
|
13272
13590
|
return this.settleRun(run, {
|
|
@@ -13313,6 +13631,63 @@ ${run.workflowVersion}`;
|
|
|
13313
13631
|
return suspended;
|
|
13314
13632
|
}
|
|
13315
13633
|
/**
|
|
13634
|
+
* A remote turn that ended in `ctx.continueAsNew(input)`: record what the turn ran, complete this
|
|
13635
|
+
* run and start its continuation (`<runId>~N`) with `input` on a clean history — the in-process
|
|
13636
|
+
* engine's handoff, with the run's partition and tenant carried over. A run cancelled while the
|
|
13637
|
+
* turn replayed stays cancelled and starts nothing.
|
|
13638
|
+
*/
|
|
13639
|
+
async applyRemoteContinueAsNew(run, commands, input) {
|
|
13640
|
+
await this.applyCommands(run, commands.filter((cmd) => cmd.kind !== "continueAsNew"));
|
|
13641
|
+
const fresh = await this.store.getRun(run.id);
|
|
13642
|
+
if (fresh && (fresh.status === "cancelled" || fresh.status === "cancelling" || fresh.status === "completed" || fresh.status === "dead")) {
|
|
13643
|
+
return {
|
|
13644
|
+
runId: run.id,
|
|
13645
|
+
status: fresh.status,
|
|
13646
|
+
output: fresh.output,
|
|
13647
|
+
error: fresh.error
|
|
13648
|
+
};
|
|
13649
|
+
}
|
|
13650
|
+
const nextId = nextContinuationId(run.id);
|
|
13651
|
+
try {
|
|
13652
|
+
await this.start(run.workflow, input, nextId, {
|
|
13653
|
+
namespace: run.namespace,
|
|
13654
|
+
...inheritTenant(run)
|
|
13655
|
+
});
|
|
13656
|
+
} catch (err) {
|
|
13657
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13658
|
+
return this.settleRun(run, {
|
|
13659
|
+
kind: "failed",
|
|
13660
|
+
error: {
|
|
13661
|
+
message: `continueAsNew: the continuation ${nextId} failed to start: ${message}`,
|
|
13662
|
+
code: "continue_as_new_failed"
|
|
13663
|
+
}
|
|
13664
|
+
});
|
|
13665
|
+
}
|
|
13666
|
+
await this.store.updateRun(run.id, {
|
|
13667
|
+
status: "completed",
|
|
13668
|
+
output: void 0,
|
|
13669
|
+
error: void 0,
|
|
13670
|
+
wakeAt: void 0,
|
|
13671
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
13672
|
+
});
|
|
13673
|
+
this.emit({
|
|
13674
|
+
type: "run.completed",
|
|
13675
|
+
runId: run.id,
|
|
13676
|
+
workflow: run.workflow,
|
|
13677
|
+
namespace: run.namespace
|
|
13678
|
+
}, {
|
|
13679
|
+
lifecycle: false
|
|
13680
|
+
});
|
|
13681
|
+
void this.notifyParent(run.id, {
|
|
13682
|
+
ok: true,
|
|
13683
|
+
value: void 0
|
|
13684
|
+
});
|
|
13685
|
+
return {
|
|
13686
|
+
runId: run.id,
|
|
13687
|
+
status: "completed"
|
|
13688
|
+
};
|
|
13689
|
+
}
|
|
13690
|
+
/**
|
|
13316
13691
|
* Re-drive a run whose turn was computed from a history OLDER than the store's.
|
|
13317
13692
|
*
|
|
13318
13693
|
* A turn replays against a SNAPSHOT, and an op can settle while its decision is in flight. The
|
|
@@ -13330,11 +13705,12 @@ ${run.workflowVersion}`;
|
|
|
13330
13705
|
* Scheduled off the current stack so it runs after the caller released the run lease.
|
|
13331
13706
|
*/
|
|
13332
13707
|
async redriveIfTurnSawStaleHistory(runId, commands) {
|
|
13333
|
-
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal");
|
|
13708
|
+
const blocking = commands.filter((cmd) => cmd.kind === "call" || cmd.kind === "startChild" || cmd.kind === "waitSignal" || cmd.kind === "waitSignalTimeout" || cmd.kind === "waitEvent");
|
|
13334
13709
|
if (blocking.length === 0) return;
|
|
13335
13710
|
for (const cmd of blocking) {
|
|
13336
13711
|
const checkpoint = await this.store.getCheckpoint(runId, cmd.seq);
|
|
13337
|
-
|
|
13712
|
+
const backingOff = cmd.kind === "call" && checkpoint?.status === "failed" && checkpoint.wakeAt != null;
|
|
13713
|
+
if (checkpoint && checkpoint.status !== "pending" && !backingOff) {
|
|
13338
13714
|
setTimeout(() => {
|
|
13339
13715
|
void this.resumeImplicit(runId).catch(() => void 0);
|
|
13340
13716
|
}, 0).unref?.();
|
|
@@ -13583,7 +13959,12 @@ ${run.workflowVersion}`;
|
|
|
13583
13959
|
// name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
|
|
13584
13960
|
name: cp.kind === "signal" ? void 0 : cp.name,
|
|
13585
13961
|
output: cp.status === "completed" ? cp.output : void 0,
|
|
13586
|
-
error: cp.status === "failed" ? cp.error : void 0
|
|
13962
|
+
error: cp.status === "failed" ? cp.error : void 0,
|
|
13963
|
+
// How many times a dispatched step ran: a worker applying a `call` policy decides a retry
|
|
13964
|
+
// from it (see `remoteCallRetries`), exactly as the in-process path reads the checkpoint.
|
|
13965
|
+
...cp.kind === "remote" ? {
|
|
13966
|
+
attempts: cp.attempts
|
|
13967
|
+
} : {}
|
|
13587
13968
|
});
|
|
13588
13969
|
} else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
|
|
13589
13970
|
events.push({
|
|
@@ -13640,11 +14021,12 @@ ${run.workflowVersion}`;
|
|
|
13640
14021
|
} else if (cmd.kind === "call") {
|
|
13641
14022
|
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
13642
14023
|
if (existing) {
|
|
13643
|
-
const
|
|
14024
|
+
const policed = redriveLost && cmd.policy !== void 0 ? await this.applyCallPolicy(run, cmd, existing) : null;
|
|
14025
|
+
const leaseAt = policed !== null ? policed : redriveLost ? await this.redriveLostCall(run, existing) : void 0;
|
|
13644
14026
|
if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
|
|
13645
14027
|
continue;
|
|
13646
14028
|
}
|
|
13647
|
-
const lease = this.
|
|
14029
|
+
const lease = this.callLease(cmd.policy);
|
|
13648
14030
|
await this.store.saveCheckpoint(stepCheckpoint({
|
|
13649
14031
|
runId: run.id,
|
|
13650
14032
|
seq: cmd.seq,
|
|
@@ -13704,35 +14086,7 @@ ${run.workflowVersion}`;
|
|
|
13704
14086
|
}));
|
|
13705
14087
|
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
13706
14088
|
} else if (cmd.kind === "waitSignal") {
|
|
13707
|
-
|
|
13708
|
-
await this.store.saveCheckpoint(instantCheckpoint({
|
|
13709
|
-
runId: run.id,
|
|
13710
|
-
seq: cmd.seq,
|
|
13711
|
-
name: `signal:${cmd.signal}`,
|
|
13712
|
-
kind: "signal",
|
|
13713
|
-
output: payload
|
|
13714
|
-
}));
|
|
13715
|
-
setTimeout(() => void this.resumeImplicit(run.id).catch(() => void 0), 0);
|
|
13716
|
-
}, "deliverBuffered");
|
|
13717
|
-
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
13718
|
-
if (buffered) {
|
|
13719
|
-
await deliverBuffered(buffered.payload);
|
|
13720
|
-
} else {
|
|
13721
|
-
await this.store.putSignalWaiter({
|
|
13722
|
-
token: cmd.signal,
|
|
13723
|
-
runId: run.id,
|
|
13724
|
-
seq: cmd.seq
|
|
13725
|
-
});
|
|
13726
|
-
const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
13727
|
-
if (lateBuffered) {
|
|
13728
|
-
await this.store.removeSignalWaiter({
|
|
13729
|
-
token: cmd.signal,
|
|
13730
|
-
runId: run.id,
|
|
13731
|
-
seq: cmd.seq
|
|
13732
|
-
});
|
|
13733
|
-
await deliverBuffered(lateBuffered.payload);
|
|
13734
|
-
}
|
|
13735
|
-
}
|
|
14089
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
13736
14090
|
} else if (cmd.kind === "waitAny") {
|
|
13737
14091
|
const state = await enterWaitAny({
|
|
13738
14092
|
store: this.store,
|
|
@@ -13764,6 +14118,100 @@ ${run.workflowVersion}`;
|
|
|
13764
14118
|
} else {
|
|
13765
14119
|
this.redeliverFinishedChild(childId, existingChild);
|
|
13766
14120
|
}
|
|
14121
|
+
} else if (cmd.kind === "sleepUntil") {
|
|
14122
|
+
const existing = await this.store.getCheckpoint(run.id, cmd.seq);
|
|
14123
|
+
const deadline = existing?.wakeAt ?? cmd.until;
|
|
14124
|
+
if (!existing) {
|
|
14125
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14126
|
+
runId: run.id,
|
|
14127
|
+
seq: cmd.seq,
|
|
14128
|
+
name: `sleep:${cmd.seq}`,
|
|
14129
|
+
kind: "sleep",
|
|
14130
|
+
status: "pending",
|
|
14131
|
+
attempts: 1,
|
|
14132
|
+
wakeAt: deadline,
|
|
14133
|
+
enqueuedAt: at,
|
|
14134
|
+
startedAt: at,
|
|
14135
|
+
finishedAt: at
|
|
14136
|
+
}));
|
|
14137
|
+
}
|
|
14138
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14139
|
+
} else if (cmd.kind === "waitSignalTimeout") {
|
|
14140
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14141
|
+
const deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, cmd.signal, cmd.timeoutMs, {
|
|
14142
|
+
waitSeq: cmd.seq,
|
|
14143
|
+
token: cmd.signal
|
|
14144
|
+
});
|
|
14145
|
+
if (deadline <= this.clock()) {
|
|
14146
|
+
this.resumeSoon(run.id);
|
|
14147
|
+
continue;
|
|
14148
|
+
}
|
|
14149
|
+
await this.parkRemoteOnSignal(run, cmd.seq, cmd.signal);
|
|
14150
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14151
|
+
} else if (cmd.kind === "waitEvent") {
|
|
14152
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14153
|
+
const token = eventToken(cmd.name, cmd.match, run.id, cmd.seq);
|
|
14154
|
+
let deadline;
|
|
14155
|
+
if (cmd.timeoutMs != null && cmd.deadlineSeq != null) {
|
|
14156
|
+
deadline = await this.stampRemoteDeadline(run.id, cmd.deadlineSeq, `event:${cmd.name}`, cmd.timeoutMs, {
|
|
14157
|
+
waitSeq: cmd.seq,
|
|
14158
|
+
token
|
|
14159
|
+
});
|
|
14160
|
+
if (deadline <= this.clock()) {
|
|
14161
|
+
this.resumeSoon(run.id);
|
|
14162
|
+
continue;
|
|
14163
|
+
}
|
|
14164
|
+
}
|
|
14165
|
+
await this.store.putSignalWaiter({
|
|
14166
|
+
token,
|
|
14167
|
+
runId: run.id,
|
|
14168
|
+
seq: cmd.seq
|
|
14169
|
+
});
|
|
14170
|
+
const hit = await this.claimBufferedEvent(run, cmd.name, cmd.match ?? {});
|
|
14171
|
+
if (hit) {
|
|
14172
|
+
await this.store.removeSignalWaiter({
|
|
14173
|
+
token,
|
|
14174
|
+
runId: run.id,
|
|
14175
|
+
seq: cmd.seq
|
|
14176
|
+
});
|
|
14177
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14178
|
+
runId: run.id,
|
|
14179
|
+
seq: cmd.seq,
|
|
14180
|
+
name: `signal:${token}`,
|
|
14181
|
+
kind: "signal",
|
|
14182
|
+
output: hit.payload
|
|
14183
|
+
}));
|
|
14184
|
+
this.resumeSoon(run.id);
|
|
14185
|
+
} else if (deadline != null) {
|
|
14186
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
14187
|
+
}
|
|
14188
|
+
} else if (cmd.kind === "spawnChild") {
|
|
14189
|
+
if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
|
|
14190
|
+
if (!await this.store.getRun(cmd.childId)) {
|
|
14191
|
+
try {
|
|
14192
|
+
await this.start(cmd.workflow, cmd.input, cmd.childId, {
|
|
14193
|
+
namespace: run.namespace,
|
|
14194
|
+
...inheritTenant(run),
|
|
14195
|
+
...cmd.priority !== void 0 ? {
|
|
14196
|
+
priority: cmd.priority
|
|
14197
|
+
} : {}
|
|
14198
|
+
});
|
|
14199
|
+
} catch (err) {
|
|
14200
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14201
|
+
this.notifyParent(cmd.childId, {
|
|
14202
|
+
ok: false,
|
|
14203
|
+
error: `child workflow "${cmd.workflow}" failed to start: ${message}`
|
|
14204
|
+
});
|
|
14205
|
+
}
|
|
14206
|
+
}
|
|
14207
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14208
|
+
runId: run.id,
|
|
14209
|
+
seq: cmd.seq,
|
|
14210
|
+
name: `spawn:${cmd.childId}`,
|
|
14211
|
+
kind: "local",
|
|
14212
|
+
output: cmd.childId
|
|
14213
|
+
}));
|
|
14214
|
+
} else if (cmd.kind === "continueAsNew") {
|
|
13767
14215
|
} else {
|
|
13768
14216
|
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
13769
14217
|
}
|
|
@@ -13771,6 +14219,245 @@ ${run.workflowVersion}`;
|
|
|
13771
14219
|
return wakeAt;
|
|
13772
14220
|
}
|
|
13773
14221
|
/**
|
|
14222
|
+
* Park a remote turn on signal `token` at `seq`: register a waiter so `engine.signal(token)` lands
|
|
14223
|
+
* the resolving `signal` checkpoint here and resumes the run. If the signal was already delivered
|
|
14224
|
+
* (buffered before the workflow reached this point — e.g. signalWithStart), resolve it now and
|
|
14225
|
+
* re-drive on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume
|
|
14226
|
+
* bails).
|
|
14227
|
+
*/
|
|
14228
|
+
async parkRemoteOnSignal(run, seq, token) {
|
|
14229
|
+
const deliverBuffered = /* @__PURE__ */ __name(async (payload) => {
|
|
14230
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
14231
|
+
runId: run.id,
|
|
14232
|
+
seq,
|
|
14233
|
+
name: `signal:${token}`,
|
|
14234
|
+
kind: "signal",
|
|
14235
|
+
output: payload
|
|
14236
|
+
}));
|
|
14237
|
+
this.resumeSoon(run.id);
|
|
14238
|
+
}, "deliverBuffered");
|
|
14239
|
+
const buffered = await this.store.takeBufferedSignal(token);
|
|
14240
|
+
if (buffered) {
|
|
14241
|
+
await deliverBuffered(buffered.payload);
|
|
14242
|
+
} else {
|
|
14243
|
+
await this.store.putSignalWaiter({
|
|
14244
|
+
token,
|
|
14245
|
+
runId: run.id,
|
|
14246
|
+
seq
|
|
14247
|
+
});
|
|
14248
|
+
const lateBuffered = await this.store.takeBufferedSignal(token);
|
|
14249
|
+
if (lateBuffered) {
|
|
14250
|
+
await this.store.removeSignalWaiter({
|
|
14251
|
+
token,
|
|
14252
|
+
runId: run.id,
|
|
14253
|
+
seq
|
|
14254
|
+
});
|
|
14255
|
+
await deliverBuffered(lateBuffered.payload);
|
|
14256
|
+
}
|
|
14257
|
+
}
|
|
14258
|
+
}
|
|
14259
|
+
/** Re-drive `runId` on a macrotask, after the current turn has suspended and freed its lease. */
|
|
14260
|
+
resumeSoon(runId) {
|
|
14261
|
+
setTimeout(() => void this.resumeImplicit(runId).catch(() => void 0), 0);
|
|
14262
|
+
}
|
|
14263
|
+
/**
|
|
14264
|
+
* Stamp the deadline of a BOUNDED remote wait at `deadlineSeq`, once: the first apply records
|
|
14265
|
+
* `now + timeoutMs` as a `timeout:<label>` timer, and every later one reads the recorded deadline
|
|
14266
|
+
* back (the in-process `stampDeadline`). The checkpoint's `input` names the wait it bounds, so the
|
|
14267
|
+
* deadline can drop that wait's waiter when it lapses — see {@link expireRemoteDeadlines}.
|
|
14268
|
+
*/
|
|
14269
|
+
async stampRemoteDeadline(runId, deadlineSeq, label, timeoutMs, wait) {
|
|
14270
|
+
const recorded = await this.store.getCheckpoint(runId, deadlineSeq);
|
|
14271
|
+
if (recorded?.wakeAt != null) return recorded.wakeAt;
|
|
14272
|
+
const deadline = this.clock() + timeoutMs;
|
|
14273
|
+
const at = /* @__PURE__ */ new Date();
|
|
14274
|
+
await this.store.saveCheckpoint(stepCheckpoint({
|
|
14275
|
+
runId,
|
|
14276
|
+
seq: deadlineSeq,
|
|
14277
|
+
name: `timeout:${label}`,
|
|
14278
|
+
kind: "sleep",
|
|
14279
|
+
status: "pending",
|
|
14280
|
+
attempts: 1,
|
|
14281
|
+
input: wait,
|
|
14282
|
+
wakeAt: deadline,
|
|
14283
|
+
enqueuedAt: at,
|
|
14284
|
+
startedAt: at,
|
|
14285
|
+
finishedAt: at
|
|
14286
|
+
}));
|
|
14287
|
+
return deadline;
|
|
14288
|
+
}
|
|
14289
|
+
/**
|
|
14290
|
+
* Before a remote turn reads its history: a bounded wait whose deadline has lapsed drops its waiter
|
|
14291
|
+
* (exact row only), so a signal or event arriving later cannot resolve a wait the turn already saw
|
|
14292
|
+
* time out — the in-process wait removes its waiter the same way when it finds the deadline past.
|
|
14293
|
+
* A delivery that took the waiter first wins: its checkpoint is read back into `checkpoints`, and the
|
|
14294
|
+
* worker's replay checks the wait before the deadline. Returns the (possibly extended) list.
|
|
14295
|
+
*/
|
|
14296
|
+
async expireRemoteDeadlines(runId, checkpoints) {
|
|
14297
|
+
const now = this.clock();
|
|
14298
|
+
const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
|
|
14299
|
+
let out = checkpoints;
|
|
14300
|
+
for (const cp of checkpoints) {
|
|
14301
|
+
if (cp.kind !== "sleep" || cp.status !== "pending" || !cp.name.startsWith("timeout:")) continue;
|
|
14302
|
+
if (cp.wakeAt == null || cp.wakeAt > now) continue;
|
|
14303
|
+
const wait = cp.input;
|
|
14304
|
+
if (typeof wait?.waitSeq !== "number" || typeof wait.token !== "string") continue;
|
|
14305
|
+
if (settled.has(wait.waitSeq)) continue;
|
|
14306
|
+
await this.store.removeSignalWaiter({
|
|
14307
|
+
token: wait.token,
|
|
14308
|
+
runId,
|
|
14309
|
+
seq: wait.waitSeq
|
|
14310
|
+
});
|
|
14311
|
+
const landed = await this.store.getCheckpoint(runId, wait.waitSeq);
|
|
14312
|
+
if (landed && landed.status === "completed") {
|
|
14313
|
+
out = [
|
|
14314
|
+
...out.filter((c) => c.seq !== landed.seq),
|
|
14315
|
+
landed
|
|
14316
|
+
].sort((a, b) => a.seq - b.seq);
|
|
14317
|
+
}
|
|
14318
|
+
}
|
|
14319
|
+
return out;
|
|
14320
|
+
}
|
|
14321
|
+
/**
|
|
14322
|
+
* Claim one buffered event `name` whose payload satisfies `match` and that this run's tenant may
|
|
14323
|
+
* see — the remote twin of the in-process `waitForEvent`'s buffered consumption (oldest first; a
|
|
14324
|
+
* candidate claimed elsewhere first is skipped, not waited on). Expired candidates are pruned when
|
|
14325
|
+
* {@link eventBufferTtlMs} is set, like the reconcile sweep.
|
|
14326
|
+
*/
|
|
14327
|
+
async claimBufferedEvent(run, name, match) {
|
|
14328
|
+
const candidates = await this.store.listBufferedEvents(name, EVENT_BUFFER_SCAN_LIMIT);
|
|
14329
|
+
const nowMs = this.clock();
|
|
14330
|
+
for (const candidate of candidates) {
|
|
14331
|
+
if (this.eventBufferTtlMs != null && nowMs - candidate.publishedAt > this.eventBufferTtlMs) {
|
|
14332
|
+
await this.store.removeBufferedEvent(candidate.id);
|
|
14333
|
+
continue;
|
|
14334
|
+
}
|
|
14335
|
+
const { payload, tenantIds } = readBufferedEvent(candidate.payload);
|
|
14336
|
+
if (!bufferedEventReaches(tenantIds, run.tenantId)) continue;
|
|
14337
|
+
if (!eventMatches(payload, match)) continue;
|
|
14338
|
+
if (!await this.store.removeBufferedEvent(candidate.id)) continue;
|
|
14339
|
+
return {
|
|
14340
|
+
payload
|
|
14341
|
+
};
|
|
14342
|
+
}
|
|
14343
|
+
return void 0;
|
|
14344
|
+
}
|
|
14345
|
+
/** The lease a freshly dispatched remote `call` parks on: its liveness `timeoutMs` when its policy
|
|
14346
|
+
* sets one, else the lost-dispatch window (`remoteRedispatchMs`), else none. */
|
|
14347
|
+
callLease(policy) {
|
|
14348
|
+
if (policy?.timeoutMs) return this.clock() + policy.timeoutMs;
|
|
14349
|
+
return this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
14350
|
+
}
|
|
14351
|
+
/**
|
|
14352
|
+
* A re-emitted remote `call` that carries its step's policy: apply the policy to the step's
|
|
14353
|
+
* checkpoint exactly as the in-process `ctx.step` path ({@link callRemote} / {@link
|
|
14354
|
+
* callRemoteInMemory}) applies it, so a step declared with retries, backoff or a timeout on a tenant
|
|
14355
|
+
* behaves as it does on a standalone engine. Returns the time to park the run on (`undefined`: no
|
|
14356
|
+
* timer, the step's result or the next turn moves it), or `null` when the policy has nothing to say
|
|
14357
|
+
* and the lost-dispatch self-heal ({@link redriveLostCall}) decides.
|
|
14358
|
+
*
|
|
14359
|
+
* - FAILED and retried (see `remoteCallRetries`): the first sight stamps the backoff deadline on
|
|
14360
|
+
* the failed checkpoint (`wakeAt`, durable and stable across replays); once due, the step is
|
|
14361
|
+
* dispatched again under the same `stepId` with `attempts + 1`.
|
|
14362
|
+
* - PENDING with a `timeoutMs`: the step's lease is `timeoutMs` from its dispatch. A lapsed lease
|
|
14363
|
+
* re-dispatches while attempts remain, then fails the step with a `remote_step_timeout` error the
|
|
14364
|
+
* workflow can catch, as `RemoteStepTimeout` would surface in-process.
|
|
14365
|
+
*/
|
|
14366
|
+
async applyCallPolicy(run, cmd, cp) {
|
|
14367
|
+
const policy = cmd.policy;
|
|
14368
|
+
if (cp.kind !== "remote") return null;
|
|
14369
|
+
if (cp.status === "failed") {
|
|
14370
|
+
if (!remoteCallRetries(policy, cp.attempts, cp.error)) return void 0;
|
|
14371
|
+
let due = cp.wakeAt;
|
|
14372
|
+
if (due == null) {
|
|
14373
|
+
due = this.clock() + backoffDelay(cp.attempts, policy);
|
|
14374
|
+
if (due > this.clock()) {
|
|
14375
|
+
await this.store.saveCheckpoint({
|
|
14376
|
+
...cp,
|
|
14377
|
+
wakeAt: due
|
|
14378
|
+
});
|
|
14379
|
+
return due;
|
|
14380
|
+
}
|
|
14381
|
+
}
|
|
14382
|
+
if (this.clock() < due) return due;
|
|
14383
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "retry");
|
|
14384
|
+
}
|
|
14385
|
+
if (cp.status !== "pending" || !policy.timeoutMs) return null;
|
|
14386
|
+
if (cp.wakeAt == null) {
|
|
14387
|
+
const lease = this.clock() + policy.timeoutMs;
|
|
14388
|
+
await this.store.saveCheckpoint({
|
|
14389
|
+
...cp,
|
|
14390
|
+
wakeAt: lease
|
|
14391
|
+
});
|
|
14392
|
+
return this.stepLeaseWakeAt(lease);
|
|
14393
|
+
}
|
|
14394
|
+
if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
|
|
14395
|
+
if (cp.attempts < remoteCallMaxAttempts(policy)) {
|
|
14396
|
+
return this.redispatchCall(run, cmd, cp, cp.attempts + 1, policy, "timeout");
|
|
14397
|
+
}
|
|
14398
|
+
const id = cp.stepId ?? stepId(run.id, cp.seq);
|
|
14399
|
+
const error2 = {
|
|
14400
|
+
message: new RemoteStepTimeout(id, policy.timeoutMs).message,
|
|
14401
|
+
code: "remote_step_timeout"
|
|
14402
|
+
};
|
|
14403
|
+
await this.store.saveCheckpoint({
|
|
14404
|
+
...cp,
|
|
14405
|
+
status: "failed",
|
|
14406
|
+
error: error2,
|
|
14407
|
+
wakeAt: void 0,
|
|
14408
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
14409
|
+
});
|
|
14410
|
+
this.emit({
|
|
14411
|
+
type: "step.failed",
|
|
14412
|
+
runId: run.id,
|
|
14413
|
+
seq: cp.seq,
|
|
14414
|
+
name: cp.name,
|
|
14415
|
+
kind: "remote",
|
|
14416
|
+
error: error2
|
|
14417
|
+
});
|
|
14418
|
+
return void 0;
|
|
14419
|
+
}
|
|
14420
|
+
/** Dispatch a policed remote `call` again as attempt `attempt` (a retry, or a timed-out attempt). */
|
|
14421
|
+
async redispatchCall(run, cmd, cp, attempt, policy, why) {
|
|
14422
|
+
const lease = this.callLease(policy);
|
|
14423
|
+
const at = /* @__PURE__ */ new Date();
|
|
14424
|
+
await this.store.saveCheckpoint({
|
|
14425
|
+
...cp,
|
|
14426
|
+
status: "pending",
|
|
14427
|
+
output: void 0,
|
|
14428
|
+
error: void 0,
|
|
14429
|
+
attempts: attempt,
|
|
14430
|
+
wakeAt: lease,
|
|
14431
|
+
enqueuedAt: at,
|
|
14432
|
+
startedAt: at,
|
|
14433
|
+
finishedAt: at,
|
|
14434
|
+
events: why === "timeout" ? [
|
|
14435
|
+
...cp.events ?? [],
|
|
14436
|
+
lostStepEvent(this.clock(), attempt, "redispatched")
|
|
14437
|
+
] : cp.events
|
|
14438
|
+
});
|
|
14439
|
+
await this.dispatchRemoteTask({
|
|
14440
|
+
runId: run.id,
|
|
14441
|
+
seq: cp.seq,
|
|
14442
|
+
name: cmd.name,
|
|
14443
|
+
stepId: cp.stepId ?? stepId(run.id, cp.seq),
|
|
14444
|
+
group: cp.workerGroup ?? tenantGroup(sanitizeQueueToken(cmd.name), cmd.group),
|
|
14445
|
+
input: cp.input,
|
|
14446
|
+
attempt
|
|
14447
|
+
});
|
|
14448
|
+
this.emit({
|
|
14449
|
+
type: "step.started",
|
|
14450
|
+
runId: run.id,
|
|
14451
|
+
seq: cp.seq,
|
|
14452
|
+
name: cmd.name,
|
|
14453
|
+
kind: "remote",
|
|
14454
|
+
...why === "timeout" ? {
|
|
14455
|
+
redispatched: true
|
|
14456
|
+
} : {}
|
|
14457
|
+
});
|
|
14458
|
+
return lease == null ? void 0 : this.stepLeaseWakeAt(lease);
|
|
14459
|
+
}
|
|
14460
|
+
/**
|
|
13774
14461
|
* Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
|
|
13775
14462
|
* exit path of a run with a compensating cancel in flight — whether the resumed replay returned,
|
|
13776
14463
|
* suspended, hit continue-as-new, or threw — so a cancel is never lost just because the body didn't
|
|
@@ -18171,6 +18858,7 @@ export {
|
|
|
18171
18858
|
PARENT_CLOSE_TAG_PREFIX,
|
|
18172
18859
|
QueueController,
|
|
18173
18860
|
REMOTE_EVENT,
|
|
18861
|
+
REMOTE_WORKFLOW_FEATURES,
|
|
18174
18862
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
18175
18863
|
RUN_VALUE_FACET_LIMIT,
|
|
18176
18864
|
RUN_VALUE_FACET_SCAN,
|
|
@@ -18328,6 +19016,9 @@ export {
|
|
|
18328
19016
|
readBufferedEvent,
|
|
18329
19017
|
readRunsByIds,
|
|
18330
19018
|
readSearchAttributes,
|
|
19019
|
+
remoteCallMaxAttempts,
|
|
19020
|
+
remoteCallPolicyOf,
|
|
19021
|
+
remoteCallRetries,
|
|
18331
19022
|
requiredCapabilities,
|
|
18332
19023
|
resolveConcurrency,
|
|
18333
19024
|
resolveRouting,
|