@dudousxd/nestjs-durable-core 0.101.0 → 0.102.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +310 -38
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1031 -867
- package/dist/index.d.ts +1031 -867
- package/dist/index.js +310 -38
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -2639,6 +2639,13 @@ function canonicalizeRegistrations(regs) {
|
|
|
2639
2639
|
subscribes: [
|
|
2640
2640
|
...new Set(r.subscribes)
|
|
2641
2641
|
].sort()
|
|
2642
|
+
} : {},
|
|
2643
|
+
// Conditional too: a registration without them keeps its hash; a changed option changes it.
|
|
2644
|
+
...r.options !== void 0 ? {
|
|
2645
|
+
options: stableJson(r.options)
|
|
2646
|
+
} : {},
|
|
2647
|
+
...r.deadLetter !== void 0 ? {
|
|
2648
|
+
deadLetter: r.deadLetter
|
|
2642
2649
|
} : {}
|
|
2643
2650
|
}));
|
|
2644
2651
|
const byJson = /* @__PURE__ */ new Map();
|
|
@@ -4349,7 +4356,8 @@ var ScheduleClient = class {
|
|
|
4349
4356
|
const input = specOf(record).input;
|
|
4350
4357
|
const opts = startOptions(record);
|
|
4351
4358
|
if (isAnnounced(record) && this.host.startParked) {
|
|
4352
|
-
|
|
4359
|
+
const singleton = record.state.announce?.singleton;
|
|
4360
|
+
return singleton ? this.host.startParked(record.workflow, input, runId, opts, singleton) : this.host.startParked(record.workflow, input, runId, opts);
|
|
4353
4361
|
}
|
|
4354
4362
|
return this.host.start(record.workflow, input, runId, opts);
|
|
4355
4363
|
}
|
|
@@ -4545,7 +4553,8 @@ var ScheduleClient = class {
|
|
|
4545
4553
|
announce: {
|
|
4546
4554
|
key: doc.key,
|
|
4547
4555
|
fingerprint,
|
|
4548
|
-
declaredPaused: doc.declaredPaused
|
|
4556
|
+
declaredPaused: doc.declaredPaused,
|
|
4557
|
+
singleton: doc.singleton
|
|
4549
4558
|
}
|
|
4550
4559
|
}),
|
|
4551
4560
|
updatedAt: new Date(now)
|
|
@@ -4582,7 +4591,10 @@ var ScheduleClient = class {
|
|
|
4582
4591
|
announce: {
|
|
4583
4592
|
key: item.key,
|
|
4584
4593
|
fingerprint,
|
|
4585
|
-
declaredPaused
|
|
4594
|
+
declaredPaused,
|
|
4595
|
+
...validSingleton(item.singleton) ? {
|
|
4596
|
+
singleton: item.singleton
|
|
4597
|
+
} : {}
|
|
4586
4598
|
}
|
|
4587
4599
|
});
|
|
4588
4600
|
out.adopted.push(id);
|
|
@@ -4732,6 +4744,10 @@ function specOf(record) {
|
|
|
4732
4744
|
return record.spec;
|
|
4733
4745
|
}
|
|
4734
4746
|
__name(specOf, "specOf");
|
|
4747
|
+
function validSingleton(value) {
|
|
4748
|
+
return value !== null && typeof value === "object" && typeof value.key === "string" && value.key !== "";
|
|
4749
|
+
}
|
|
4750
|
+
__name(validSingleton, "validSingleton");
|
|
4735
4751
|
function isAnnounced(record) {
|
|
4736
4752
|
return record.state?.announce !== void 0;
|
|
4737
4753
|
}
|
|
@@ -7175,6 +7191,43 @@ var DEFAULT_STICKY_RUNS = 1e3;
|
|
|
7175
7191
|
var DEFAULT_STICKY_IDLE_MS = 3e5;
|
|
7176
7192
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
7177
7193
|
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
7194
|
+
var ANNOUNCED_TIMEOUTS_TTL_MS = 15e3;
|
|
7195
|
+
function tenantScopedKey(namespace, key) {
|
|
7196
|
+
return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
|
|
7197
|
+
}
|
|
7198
|
+
__name(tenantScopedKey, "tenantScopedKey");
|
|
7199
|
+
function singletonKeyOf(run) {
|
|
7200
|
+
const tag = run.tags?.find((t) => t.startsWith("singleton:"));
|
|
7201
|
+
return tag?.slice("singleton:".length);
|
|
7202
|
+
}
|
|
7203
|
+
__name(singletonKeyOf, "singletonKeyOf");
|
|
7204
|
+
function conventionSingleton(run, announced, wire) {
|
|
7205
|
+
if (wire !== void 0) {
|
|
7206
|
+
const key2 = wire.key;
|
|
7207
|
+
return {
|
|
7208
|
+
key: /* @__PURE__ */ __name(() => key2, "key"),
|
|
7209
|
+
...typeof wire.limit === "number" ? {
|
|
7210
|
+
limit: wire.limit
|
|
7211
|
+
} : {},
|
|
7212
|
+
...typeof wire.maxQueueDepth === "number" ? {
|
|
7213
|
+
maxQueueDepth: wire.maxQueueDepth
|
|
7214
|
+
} : {}
|
|
7215
|
+
};
|
|
7216
|
+
}
|
|
7217
|
+
const key = singletonKeyOf(run);
|
|
7218
|
+
if (key === void 0 || announced === void 0) return void 0;
|
|
7219
|
+
if (announced === null) return {
|
|
7220
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7221
|
+
limit: 1
|
|
7222
|
+
};
|
|
7223
|
+
const declared = announced.options?.singleton;
|
|
7224
|
+
if (declared === void 0) return void 0;
|
|
7225
|
+
return {
|
|
7226
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7227
|
+
limit: typeof declared.limit === "number" ? declared.limit : 1
|
|
7228
|
+
};
|
|
7229
|
+
}
|
|
7230
|
+
__name(conventionSingleton, "conventionSingleton");
|
|
7178
7231
|
function servedNamespace(namespace, group) {
|
|
7179
7232
|
if (namespace) return namespace;
|
|
7180
7233
|
const at = group?.lastIndexOf("@") ?? -1;
|
|
@@ -7664,13 +7717,28 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7664
7717
|
if (this.queuesRunsOverTransport) void this.wake(runId);
|
|
7665
7718
|
else this.runDispatcher.dispatch(runId);
|
|
7666
7719
|
}, "dispatch"),
|
|
7667
|
-
configFor: /* @__PURE__ */ __name((run) =>
|
|
7720
|
+
configFor: /* @__PURE__ */ __name((run) => {
|
|
7721
|
+
const local = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton;
|
|
7722
|
+
if (local) return local;
|
|
7723
|
+
const key = this.latest.has(run.workflow) ? void 0 : singletonKeyOf(run);
|
|
7724
|
+
if (key === void 0) return void 0;
|
|
7725
|
+
const limit = this.conventionSingletonLimits.get(`${runNamespace(run.namespace)}\0${run.workflow}`);
|
|
7726
|
+
return {
|
|
7727
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7728
|
+
...limit !== void 0 ? {
|
|
7729
|
+
limit
|
|
7730
|
+
} : {}
|
|
7731
|
+
};
|
|
7732
|
+
}, "configFor")
|
|
7668
7733
|
});
|
|
7669
7734
|
this.schedules = new ScheduleClient(this.store, {
|
|
7670
7735
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7671
7736
|
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7672
7737
|
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7673
|
-
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true
|
|
7738
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts, singleton) => this.startWith(workflow, input, runId, opts, true, singleton ? {
|
|
7739
|
+
...singleton,
|
|
7740
|
+
key: tenantScopedKey(opts?.namespace, singleton.key)
|
|
7741
|
+
} : void 0), "startParked"),
|
|
7674
7742
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7675
7743
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7676
7744
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -8480,6 +8548,39 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8480
8548
|
} catch {
|
|
8481
8549
|
}
|
|
8482
8550
|
}
|
|
8551
|
+
if (!this.latest.has(run.workflow)) this.inBackground(this.routeAnnouncedDeadLetter(run));
|
|
8552
|
+
}
|
|
8553
|
+
/**
|
|
8554
|
+
* The dead-letter workflow the live workers serving `run`'s workflow announce (a tenant's inline
|
|
8555
|
+
* `@DeadLetter()` handler, its `deadLetterWorkflow`, or its module-level default), or `undefined`
|
|
8556
|
+
* when the workflow is registered on this engine (its own routing applies), they announce none, or
|
|
8557
|
+
* they cannot be read. A dead-letter listener with a catch-all default (`DurableModule`'s
|
|
8558
|
+
* `deadLetterWorkflow`) checks it first, so the app's own route wins over the default.
|
|
8559
|
+
*/
|
|
8560
|
+
async announcedDeadLetter(run) {
|
|
8561
|
+
if (this.latest.has(run.workflow) || this.isRegisteredHere(run)) return void 0;
|
|
8562
|
+
const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
|
|
8563
|
+
const reg = await this.announcedRegistration(group, run.workflow, run.workflowVersion);
|
|
8564
|
+
const target = reg?.deadLetter;
|
|
8565
|
+
return typeof target === "string" && target !== "" ? target : void 0;
|
|
8566
|
+
}
|
|
8567
|
+
/**
|
|
8568
|
+
* Start the announced dead-letter workflow of a dead run (see {@link announcedDeadLetter}) in the
|
|
8569
|
+
* run's namespace, as `dlq:<runId>` with the `DeadLetter` payload, through the tenant start path:
|
|
8570
|
+
* with no live worker of the app it waits `blocked` instead of being lost. Idempotent by run id.
|
|
8571
|
+
*/
|
|
8572
|
+
async routeAnnouncedDeadLetter(run) {
|
|
8573
|
+
const target = await this.announcedDeadLetter(run);
|
|
8574
|
+
if (!target) return;
|
|
8575
|
+
await this.startWith(target, {
|
|
8576
|
+
deadRunId: run.id,
|
|
8577
|
+
workflow: run.workflow,
|
|
8578
|
+
input: run.input,
|
|
8579
|
+
error: run.error
|
|
8580
|
+
}, `dlq:${run.id}`, {
|
|
8581
|
+
namespace: run.namespace,
|
|
8582
|
+
...inheritTenant(run)
|
|
8583
|
+
}, true);
|
|
8483
8584
|
}
|
|
8484
8585
|
/**
|
|
8485
8586
|
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
@@ -9780,7 +9881,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9780
9881
|
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9781
9882
|
* a start whose caller has already returned is never lost.
|
|
9782
9883
|
*/
|
|
9783
|
-
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9884
|
+
async startWith(workflow, input, runId, opts, parkUnroutable, wireSingleton) {
|
|
9784
9885
|
const name = workflowName(workflow);
|
|
9785
9886
|
const pinned = opts?.version;
|
|
9786
9887
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9805,12 +9906,15 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9805
9906
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9806
9907
|
};
|
|
9807
9908
|
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9808
|
-
askFleetForVersion: true
|
|
9909
|
+
askFleetForVersion: true,
|
|
9910
|
+
...wireSingleton !== void 0 ? {
|
|
9911
|
+
singleton: wireSingleton
|
|
9912
|
+
} : {}
|
|
9809
9913
|
});
|
|
9810
9914
|
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9811
9915
|
}
|
|
9812
9916
|
if (!registered && parkUnroutable) {
|
|
9813
|
-
return this.parkStartForWorker(name, input, runId, opts);
|
|
9917
|
+
return this.parkStartForWorker(name, input, runId, opts, wireSingleton);
|
|
9814
9918
|
}
|
|
9815
9919
|
if (!registered) {
|
|
9816
9920
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
@@ -9912,11 +10016,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9912
10016
|
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9913
10017
|
* start is checked first at its `startAt`.
|
|
9914
10018
|
*/
|
|
9915
|
-
async parkStartForWorker(name, input, runId, opts) {
|
|
10019
|
+
async parkStartForWorker(name, input, runId, opts, wireSingleton) {
|
|
9916
10020
|
const prior = await this.store.getRun(runId);
|
|
9917
10021
|
if (prior) return this.existingStartResult(prior);
|
|
9918
10022
|
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9919
10023
|
if (quota) await assertConcurrency(this.store, name, quota);
|
|
10024
|
+
const singleton = conventionSingleton({}, void 0, wireSingleton);
|
|
10025
|
+
if (singleton) await this.singletons.assertCapacity(name, singleton, input);
|
|
9920
10026
|
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9921
10027
|
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9922
10028
|
const pinned = opts?.version;
|
|
@@ -9931,6 +10037,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9931
10037
|
const now = /* @__PURE__ */ new Date();
|
|
9932
10038
|
const tags = mergeTags(void 0, [
|
|
9933
10039
|
...opts?.tags ?? [],
|
|
10040
|
+
...singleton ? [
|
|
10041
|
+
this.singletons.tag(singleton, input)
|
|
10042
|
+
] : [],
|
|
9934
10043
|
...quota ? [
|
|
9935
10044
|
concurrencyTag(quota.key)
|
|
9936
10045
|
] : [],
|
|
@@ -10019,8 +10128,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10019
10128
|
concurrency: message.concurrency
|
|
10020
10129
|
} : {}
|
|
10021
10130
|
};
|
|
10131
|
+
const singleton = message.singleton !== void 0 && typeof message.singleton?.key === "string" ? {
|
|
10132
|
+
...message.singleton,
|
|
10133
|
+
key: tenantScopedKey(namespace, message.singleton.key)
|
|
10134
|
+
} : void 0;
|
|
10022
10135
|
try {
|
|
10023
|
-
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
10136
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true, singleton);
|
|
10024
10137
|
} catch (err) {
|
|
10025
10138
|
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
10026
10139
|
throw err;
|
|
@@ -10028,6 +10141,62 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10028
10141
|
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
10029
10142
|
}
|
|
10030
10143
|
}
|
|
10144
|
+
/**
|
|
10145
|
+
* The control plane's half of a tenant's `signalWithStart` (a `signalWithStart` run request):
|
|
10146
|
+
* start `start` like {@link startForTenant} unless its run exists, then signal `signal.token`.
|
|
10147
|
+
* Authorization is the caller's (the run-request responder): the run must belong to `namespace`,
|
|
10148
|
+
* and so must a run already waiting on the token. Delivered at once to a waiting run; with none
|
|
10149
|
+
* waiting yet it is buffered only when the token contains the run id (a run-scoped token), so a
|
|
10150
|
+
* tenant can never plant a payload for a wait another namespace opens later. `signalled: false`
|
|
10151
|
+
* when it was neither delivered nor buffered.
|
|
10152
|
+
*/
|
|
10153
|
+
async signalWithStartForTenant(start, namespace, tenant, signal) {
|
|
10154
|
+
const started = await this.startForTenant(start, namespace, tenant);
|
|
10155
|
+
if (TERMINAL_RUN_STATUSES.includes(started.status)) {
|
|
10156
|
+
return {
|
|
10157
|
+
runId: start.runId,
|
|
10158
|
+
signalled: false
|
|
10159
|
+
};
|
|
10160
|
+
}
|
|
10161
|
+
const waiter = (await this.store.listSignalWaiters(signal.token)).find((w) => w.token === signal.token);
|
|
10162
|
+
if (waiter) {
|
|
10163
|
+
const run = await this.store.getRun(waiter.runId);
|
|
10164
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) {
|
|
10165
|
+
return {
|
|
10166
|
+
runId: start.runId,
|
|
10167
|
+
signalled: false
|
|
10168
|
+
};
|
|
10169
|
+
}
|
|
10170
|
+
} else if (!signal.token.includes(start.runId)) {
|
|
10171
|
+
return {
|
|
10172
|
+
runId: start.runId,
|
|
10173
|
+
signalled: false
|
|
10174
|
+
};
|
|
10175
|
+
}
|
|
10176
|
+
await this.signalUnchecked(signal.token, signal.payload);
|
|
10177
|
+
return {
|
|
10178
|
+
runId: start.runId,
|
|
10179
|
+
signalled: true
|
|
10180
|
+
};
|
|
10181
|
+
}
|
|
10182
|
+
/**
|
|
10183
|
+
* The control plane's half of a tenant's `publishEvent` (a `publishEvent` run request): resume the
|
|
10184
|
+
* runs of `namespace` waiting on event `name` whose match the payload satisfies. Starting the
|
|
10185
|
+
* tenant's own subscribers is the tenant's (it holds their `where` and `input`); nothing is
|
|
10186
|
+
* buffered, since a buffered event could reach another namespace's later wait. Returns how many
|
|
10187
|
+
* runs it resumed.
|
|
10188
|
+
*/
|
|
10189
|
+
async publishEventForTenant(name, payload, namespace) {
|
|
10190
|
+
let touched = 0;
|
|
10191
|
+
for (const w of await this.store.listSignalWaiters(eventPrefix(name))) {
|
|
10192
|
+
if (!eventMatches(payload, eventMatchOf(w.token))) continue;
|
|
10193
|
+
const run = await this.store.getRun(w.runId);
|
|
10194
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) continue;
|
|
10195
|
+
await this.signalUnchecked(w.token, payload);
|
|
10196
|
+
touched += 1;
|
|
10197
|
+
}
|
|
10198
|
+
return touched;
|
|
10199
|
+
}
|
|
10031
10200
|
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
10032
10201
|
async recordRejectedStart(name, input, runId, opts, err) {
|
|
10033
10202
|
const prior = await this.store.getRun(runId);
|
|
@@ -10499,9 +10668,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10499
10668
|
if (!liveTransport) return void 0;
|
|
10500
10669
|
const executor = new RemoteWorkflowExecutor(liveTransport, run.workflow, run.namespace);
|
|
10501
10670
|
const declared = opts.askFleetForVersion ? await this.declaredVersion(liveTransport, group, run.workflow) : void 0;
|
|
10671
|
+
const version = declared ?? run.workflowVersion;
|
|
10672
|
+
const announced = await this.announcedRegistration(group, run.workflow, version);
|
|
10673
|
+
const singleton = conventionSingleton(run, announced, opts.singleton);
|
|
10674
|
+
if (singleton?.limit !== void 0) {
|
|
10675
|
+
this.conventionSingletonLimits.set(`${runNamespace(run.namespace)}\0${run.workflow}`, singleton.limit);
|
|
10676
|
+
}
|
|
10677
|
+
const tags = announced?.options?.tags;
|
|
10502
10678
|
return {
|
|
10503
10679
|
name: run.workflow,
|
|
10504
|
-
version
|
|
10680
|
+
version,
|
|
10505
10681
|
fn: /* @__PURE__ */ __name(() => {
|
|
10506
10682
|
throw new Error(`remote workflow ${run.workflow} has no local body`);
|
|
10507
10683
|
}, "fn"),
|
|
@@ -10510,12 +10686,47 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10510
10686
|
group,
|
|
10511
10687
|
executor
|
|
10512
10688
|
},
|
|
10689
|
+
...Array.isArray(tags) && tags.length > 0 ? {
|
|
10690
|
+
tags
|
|
10691
|
+
} : {},
|
|
10692
|
+
...singleton ? {
|
|
10693
|
+
singleton
|
|
10694
|
+
} : {},
|
|
10513
10695
|
...opts.askFleetForVersion ? {
|
|
10514
10696
|
versionDeclared: declared !== void 0
|
|
10515
10697
|
} : {}
|
|
10516
10698
|
};
|
|
10517
10699
|
}
|
|
10518
10700
|
/**
|
|
10701
|
+
* The registration the live workers of `group` announce for `workflow` (at `version`, when one of
|
|
10702
|
+
* them states it): where they disagree (a rolling deploy), the one from the worker that started
|
|
10703
|
+
* last, as for announced schedules. `undefined` when none announces it, `null` when the read
|
|
10704
|
+
* failed (what the workers state is then unknown, not "nothing"). Reads
|
|
10705
|
+
* through the descriptor cache ({@link workerDescriptorCacheMs}), so a turn costs no extra round
|
|
10706
|
+
* trip in steady state.
|
|
10707
|
+
*/
|
|
10708
|
+
async announcedRegistration(group, workflow, version) {
|
|
10709
|
+
let descriptors;
|
|
10710
|
+
try {
|
|
10711
|
+
descriptors = await this.liveWorkerDescriptors(group);
|
|
10712
|
+
} catch {
|
|
10713
|
+
return null;
|
|
10714
|
+
}
|
|
10715
|
+
let best;
|
|
10716
|
+
for (const descriptor of descriptors) {
|
|
10717
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10718
|
+
if (reg?.name !== workflow) continue;
|
|
10719
|
+
if (version !== void 0 && reg.version !== void 0 && reg.version !== version) continue;
|
|
10720
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10721
|
+
if (!best || at > best.at) best = {
|
|
10722
|
+
at,
|
|
10723
|
+
reg
|
|
10724
|
+
};
|
|
10725
|
+
}
|
|
10726
|
+
}
|
|
10727
|
+
return best?.reg;
|
|
10728
|
+
}
|
|
10729
|
+
/**
|
|
10519
10730
|
* The version the LIVE fleet declares for `workflow` on `group`, or `undefined` when it declares
|
|
10520
10731
|
* none — which is a fact about the fleet, not a failure to read it.
|
|
10521
10732
|
*
|
|
@@ -10739,34 +10950,95 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10739
10950
|
for (const reg of new Set(this.latest.values())) {
|
|
10740
10951
|
if (this.draining) return;
|
|
10741
10952
|
if (reg.executionTimeoutMs == null) continue;
|
|
10742
|
-
|
|
10743
|
-
|
|
10744
|
-
|
|
10745
|
-
|
|
10746
|
-
|
|
10747
|
-
|
|
10748
|
-
|
|
10749
|
-
|
|
10750
|
-
|
|
10751
|
-
|
|
10752
|
-
|
|
10753
|
-
|
|
10754
|
-
|
|
10755
|
-
|
|
10756
|
-
|
|
10757
|
-
|
|
10758
|
-
|
|
10759
|
-
|
|
10760
|
-
|
|
10761
|
-
|
|
10762
|
-
|
|
10763
|
-
|
|
10764
|
-
|
|
10765
|
-
|
|
10766
|
-
|
|
10767
|
-
|
|
10768
|
-
|
|
10953
|
+
await this.cancelTimedOut(reg.name, this.namespace, now - reg.executionTimeoutMs);
|
|
10954
|
+
}
|
|
10955
|
+
for (const { workflow, namespace, timeoutMs } of await this.announcedTimeouts(now)) {
|
|
10956
|
+
if (this.draining) return;
|
|
10957
|
+
await this.cancelTimedOut(workflow, namespace, now - timeoutMs);
|
|
10958
|
+
}
|
|
10959
|
+
}
|
|
10960
|
+
/**
|
|
10961
|
+
* The singleton limit last resolved for a workflow served by convention, by `<namespace>\0<name>`
|
|
10962
|
+
* (see {@link resolveRemoteByConvention}): what the singleton gate's release wake-up reads, since
|
|
10963
|
+
* it cannot wait on the fleet. One entry per convention singleton the fleet serves.
|
|
10964
|
+
*/
|
|
10965
|
+
conventionSingletonLimits = /* @__PURE__ */ new Map();
|
|
10966
|
+
/** The last read of the announced execution timeouts (see {@link announcedTimeouts}). */
|
|
10967
|
+
announcedTimeoutsRead;
|
|
10968
|
+
/**
|
|
10969
|
+
* The `executionTimeoutMs` live workers announce (`@Workflow({ executionTimeout })` on a tenant),
|
|
10970
|
+
* per workflow and namespace, for workflows this engine does not register (a registration here
|
|
10971
|
+
* applies its own). Where workers disagree, the one that started last wins. Read at most once per
|
|
10972
|
+
* {@link ANNOUNCED_TIMEOUTS_TTL_MS}; empty without a transport that reads the fleet, or when the
|
|
10973
|
+
* read fails.
|
|
10974
|
+
*/
|
|
10975
|
+
async announcedTimeouts(now) {
|
|
10976
|
+
if (!this.pool.introspectsFleet) return [];
|
|
10977
|
+
const cached = this.announcedTimeoutsRead;
|
|
10978
|
+
if (cached && now - cached.at < ANNOUNCED_TIMEOUTS_TTL_MS) return cached.list;
|
|
10979
|
+
const newest = /* @__PURE__ */ new Map();
|
|
10980
|
+
try {
|
|
10981
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
10982
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10983
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10984
|
+
if (!reg || typeof reg.name !== "string" || this.latest.has(reg.name)) continue;
|
|
10985
|
+
const namespace = servedNamespace(descriptor.namespace, reg.group);
|
|
10986
|
+
if (this.namespace !== void 0 && runNamespace(this.namespace) !== namespace) continue;
|
|
10987
|
+
const key = `${namespace}\0${reg.name}`;
|
|
10988
|
+
const prior = newest.get(key);
|
|
10989
|
+
if (prior && prior.at >= at) continue;
|
|
10990
|
+
newest.set(key, {
|
|
10991
|
+
at,
|
|
10992
|
+
workflow: reg.name,
|
|
10993
|
+
namespace,
|
|
10994
|
+
ms: Number(reg.options?.executionTimeoutMs)
|
|
10995
|
+
});
|
|
10996
|
+
}
|
|
10769
10997
|
}
|
|
10998
|
+
} catch {
|
|
10999
|
+
return cached?.list ?? [];
|
|
11000
|
+
}
|
|
11001
|
+
const list2 = [
|
|
11002
|
+
...newest.values()
|
|
11003
|
+
].filter((e) => Number.isFinite(e.ms) && e.ms > 0).map((e) => ({
|
|
11004
|
+
workflow: e.workflow,
|
|
11005
|
+
namespace: e.namespace,
|
|
11006
|
+
timeoutMs: e.ms
|
|
11007
|
+
}));
|
|
11008
|
+
this.announcedTimeoutsRead = {
|
|
11009
|
+
at: now,
|
|
11010
|
+
list: list2
|
|
11011
|
+
};
|
|
11012
|
+
return list2;
|
|
11013
|
+
}
|
|
11014
|
+
/** Cancel (`execution_timeout`) the in-flight runs of `workflow` in `namespace` created before `deadline`. */
|
|
11015
|
+
async cancelTimedOut(workflow, namespace, deadline) {
|
|
11016
|
+
const inflight = await this.store.listRuns({
|
|
11017
|
+
workflow,
|
|
11018
|
+
statuses: TIMEOUT_SWEPT_STATUSES,
|
|
11019
|
+
namespace
|
|
11020
|
+
});
|
|
11021
|
+
for (const run of inflight) {
|
|
11022
|
+
if (this.draining) return;
|
|
11023
|
+
if (run.createdAt.getTime() > deadline) continue;
|
|
11024
|
+
const error2 = {
|
|
11025
|
+
message: "execution timeout",
|
|
11026
|
+
code: "execution_timeout"
|
|
11027
|
+
};
|
|
11028
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
11029
|
+
status: "cancelled",
|
|
11030
|
+
error: error2,
|
|
11031
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
11032
|
+
}));
|
|
11033
|
+
this.emit({
|
|
11034
|
+
type: "run.failed",
|
|
11035
|
+
runId: run.id,
|
|
11036
|
+
workflow: run.workflow,
|
|
11037
|
+
namespace: run.namespace,
|
|
11038
|
+
error: error2
|
|
11039
|
+
});
|
|
11040
|
+
await this.cancelChildren(run.id).catch(() => void 0);
|
|
11041
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
10770
11042
|
}
|
|
10771
11043
|
}
|
|
10772
11044
|
/**
|