@dudousxd/nestjs-durable-core 0.101.0 → 0.102.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +310 -38
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1031 -867
- package/dist/index.d.ts +1031 -867
- package/dist/index.js +310 -38
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -2342,6 +2342,13 @@ function canonicalizeRegistrations(regs) {
|
|
|
2342
2342
|
subscribes: [
|
|
2343
2343
|
...new Set(r.subscribes)
|
|
2344
2344
|
].sort()
|
|
2345
|
+
} : {},
|
|
2346
|
+
// Conditional too: a registration without them keeps its hash; a changed option changes it.
|
|
2347
|
+
...r.options !== void 0 ? {
|
|
2348
|
+
options: stableJson(r.options)
|
|
2349
|
+
} : {},
|
|
2350
|
+
...r.deadLetter !== void 0 ? {
|
|
2351
|
+
deadLetter: r.deadLetter
|
|
2345
2352
|
} : {}
|
|
2346
2353
|
}));
|
|
2347
2354
|
const byJson = /* @__PURE__ */ new Map();
|
|
@@ -4052,7 +4059,8 @@ var ScheduleClient = class {
|
|
|
4052
4059
|
const input = specOf(record).input;
|
|
4053
4060
|
const opts = startOptions(record);
|
|
4054
4061
|
if (isAnnounced(record) && this.host.startParked) {
|
|
4055
|
-
|
|
4062
|
+
const singleton = record.state.announce?.singleton;
|
|
4063
|
+
return singleton ? this.host.startParked(record.workflow, input, runId, opts, singleton) : this.host.startParked(record.workflow, input, runId, opts);
|
|
4056
4064
|
}
|
|
4057
4065
|
return this.host.start(record.workflow, input, runId, opts);
|
|
4058
4066
|
}
|
|
@@ -4248,7 +4256,8 @@ var ScheduleClient = class {
|
|
|
4248
4256
|
announce: {
|
|
4249
4257
|
key: doc.key,
|
|
4250
4258
|
fingerprint,
|
|
4251
|
-
declaredPaused: doc.declaredPaused
|
|
4259
|
+
declaredPaused: doc.declaredPaused,
|
|
4260
|
+
singleton: doc.singleton
|
|
4252
4261
|
}
|
|
4253
4262
|
}),
|
|
4254
4263
|
updatedAt: new Date(now)
|
|
@@ -4285,7 +4294,10 @@ var ScheduleClient = class {
|
|
|
4285
4294
|
announce: {
|
|
4286
4295
|
key: item.key,
|
|
4287
4296
|
fingerprint,
|
|
4288
|
-
declaredPaused
|
|
4297
|
+
declaredPaused,
|
|
4298
|
+
...validSingleton(item.singleton) ? {
|
|
4299
|
+
singleton: item.singleton
|
|
4300
|
+
} : {}
|
|
4289
4301
|
}
|
|
4290
4302
|
});
|
|
4291
4303
|
out.adopted.push(id);
|
|
@@ -4435,6 +4447,10 @@ function specOf(record) {
|
|
|
4435
4447
|
return record.spec;
|
|
4436
4448
|
}
|
|
4437
4449
|
__name(specOf, "specOf");
|
|
4450
|
+
function validSingleton(value) {
|
|
4451
|
+
return value !== null && typeof value === "object" && typeof value.key === "string" && value.key !== "";
|
|
4452
|
+
}
|
|
4453
|
+
__name(validSingleton, "validSingleton");
|
|
4438
4454
|
function isAnnounced(record) {
|
|
4439
4455
|
return record.state?.announce !== void 0;
|
|
4440
4456
|
}
|
|
@@ -6878,6 +6894,43 @@ var DEFAULT_STICKY_RUNS = 1e3;
|
|
|
6878
6894
|
var DEFAULT_STICKY_IDLE_MS = 3e5;
|
|
6879
6895
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
6880
6896
|
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
6897
|
+
var ANNOUNCED_TIMEOUTS_TTL_MS = 15e3;
|
|
6898
|
+
function tenantScopedKey(namespace, key) {
|
|
6899
|
+
return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
|
|
6900
|
+
}
|
|
6901
|
+
__name(tenantScopedKey, "tenantScopedKey");
|
|
6902
|
+
function singletonKeyOf(run) {
|
|
6903
|
+
const tag = run.tags?.find((t) => t.startsWith("singleton:"));
|
|
6904
|
+
return tag?.slice("singleton:".length);
|
|
6905
|
+
}
|
|
6906
|
+
__name(singletonKeyOf, "singletonKeyOf");
|
|
6907
|
+
function conventionSingleton(run, announced, wire) {
|
|
6908
|
+
if (wire !== void 0) {
|
|
6909
|
+
const key2 = wire.key;
|
|
6910
|
+
return {
|
|
6911
|
+
key: /* @__PURE__ */ __name(() => key2, "key"),
|
|
6912
|
+
...typeof wire.limit === "number" ? {
|
|
6913
|
+
limit: wire.limit
|
|
6914
|
+
} : {},
|
|
6915
|
+
...typeof wire.maxQueueDepth === "number" ? {
|
|
6916
|
+
maxQueueDepth: wire.maxQueueDepth
|
|
6917
|
+
} : {}
|
|
6918
|
+
};
|
|
6919
|
+
}
|
|
6920
|
+
const key = singletonKeyOf(run);
|
|
6921
|
+
if (key === void 0 || announced === void 0) return void 0;
|
|
6922
|
+
if (announced === null) return {
|
|
6923
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
6924
|
+
limit: 1
|
|
6925
|
+
};
|
|
6926
|
+
const declared = announced.options?.singleton;
|
|
6927
|
+
if (declared === void 0) return void 0;
|
|
6928
|
+
return {
|
|
6929
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
6930
|
+
limit: typeof declared.limit === "number" ? declared.limit : 1
|
|
6931
|
+
};
|
|
6932
|
+
}
|
|
6933
|
+
__name(conventionSingleton, "conventionSingleton");
|
|
6881
6934
|
function servedNamespace(namespace, group) {
|
|
6882
6935
|
if (namespace) return namespace;
|
|
6883
6936
|
const at = group?.lastIndexOf("@") ?? -1;
|
|
@@ -7367,13 +7420,28 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7367
7420
|
if (this.queuesRunsOverTransport) void this.wake(runId);
|
|
7368
7421
|
else this.runDispatcher.dispatch(runId);
|
|
7369
7422
|
}, "dispatch"),
|
|
7370
|
-
configFor: /* @__PURE__ */ __name((run) =>
|
|
7423
|
+
configFor: /* @__PURE__ */ __name((run) => {
|
|
7424
|
+
const local = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton;
|
|
7425
|
+
if (local) return local;
|
|
7426
|
+
const key = this.latest.has(run.workflow) ? void 0 : singletonKeyOf(run);
|
|
7427
|
+
if (key === void 0) return void 0;
|
|
7428
|
+
const limit = this.conventionSingletonLimits.get(`${runNamespace(run.namespace)}\0${run.workflow}`);
|
|
7429
|
+
return {
|
|
7430
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7431
|
+
...limit !== void 0 ? {
|
|
7432
|
+
limit
|
|
7433
|
+
} : {}
|
|
7434
|
+
};
|
|
7435
|
+
}, "configFor")
|
|
7371
7436
|
});
|
|
7372
7437
|
this.schedules = new ScheduleClient(this.store, {
|
|
7373
7438
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7374
7439
|
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7375
7440
|
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7376
|
-
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true
|
|
7441
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts, singleton) => this.startWith(workflow, input, runId, opts, true, singleton ? {
|
|
7442
|
+
...singleton,
|
|
7443
|
+
key: tenantScopedKey(opts?.namespace, singleton.key)
|
|
7444
|
+
} : void 0), "startParked"),
|
|
7377
7445
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7378
7446
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7379
7447
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -8183,6 +8251,39 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8183
8251
|
} catch {
|
|
8184
8252
|
}
|
|
8185
8253
|
}
|
|
8254
|
+
if (!this.latest.has(run.workflow)) this.inBackground(this.routeAnnouncedDeadLetter(run));
|
|
8255
|
+
}
|
|
8256
|
+
/**
|
|
8257
|
+
* The dead-letter workflow the live workers serving `run`'s workflow announce (a tenant's inline
|
|
8258
|
+
* `@DeadLetter()` handler, its `deadLetterWorkflow`, or its module-level default), or `undefined`
|
|
8259
|
+
* when the workflow is registered on this engine (its own routing applies), they announce none, or
|
|
8260
|
+
* they cannot be read. A dead-letter listener with a catch-all default (`DurableModule`'s
|
|
8261
|
+
* `deadLetterWorkflow`) checks it first, so the app's own route wins over the default.
|
|
8262
|
+
*/
|
|
8263
|
+
async announcedDeadLetter(run) {
|
|
8264
|
+
if (this.latest.has(run.workflow) || this.isRegisteredHere(run)) return void 0;
|
|
8265
|
+
const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
|
|
8266
|
+
const reg = await this.announcedRegistration(group, run.workflow, run.workflowVersion);
|
|
8267
|
+
const target = reg?.deadLetter;
|
|
8268
|
+
return typeof target === "string" && target !== "" ? target : void 0;
|
|
8269
|
+
}
|
|
8270
|
+
/**
|
|
8271
|
+
* Start the announced dead-letter workflow of a dead run (see {@link announcedDeadLetter}) in the
|
|
8272
|
+
* run's namespace, as `dlq:<runId>` with the `DeadLetter` payload, through the tenant start path:
|
|
8273
|
+
* with no live worker of the app it waits `blocked` instead of being lost. Idempotent by run id.
|
|
8274
|
+
*/
|
|
8275
|
+
async routeAnnouncedDeadLetter(run) {
|
|
8276
|
+
const target = await this.announcedDeadLetter(run);
|
|
8277
|
+
if (!target) return;
|
|
8278
|
+
await this.startWith(target, {
|
|
8279
|
+
deadRunId: run.id,
|
|
8280
|
+
workflow: run.workflow,
|
|
8281
|
+
input: run.input,
|
|
8282
|
+
error: run.error
|
|
8283
|
+
}, `dlq:${run.id}`, {
|
|
8284
|
+
namespace: run.namespace,
|
|
8285
|
+
...inheritTenant(run)
|
|
8286
|
+
}, true);
|
|
8186
8287
|
}
|
|
8187
8288
|
/**
|
|
8188
8289
|
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
@@ -9483,7 +9584,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9483
9584
|
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9484
9585
|
* a start whose caller has already returned is never lost.
|
|
9485
9586
|
*/
|
|
9486
|
-
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9587
|
+
async startWith(workflow, input, runId, opts, parkUnroutable, wireSingleton) {
|
|
9487
9588
|
const name = workflowName(workflow);
|
|
9488
9589
|
const pinned = opts?.version;
|
|
9489
9590
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9508,12 +9609,15 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9508
9609
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9509
9610
|
};
|
|
9510
9611
|
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9511
|
-
askFleetForVersion: true
|
|
9612
|
+
askFleetForVersion: true,
|
|
9613
|
+
...wireSingleton !== void 0 ? {
|
|
9614
|
+
singleton: wireSingleton
|
|
9615
|
+
} : {}
|
|
9512
9616
|
});
|
|
9513
9617
|
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9514
9618
|
}
|
|
9515
9619
|
if (!registered && parkUnroutable) {
|
|
9516
|
-
return this.parkStartForWorker(name, input, runId, opts);
|
|
9620
|
+
return this.parkStartForWorker(name, input, runId, opts, wireSingleton);
|
|
9517
9621
|
}
|
|
9518
9622
|
if (!registered) {
|
|
9519
9623
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
@@ -9615,11 +9719,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9615
9719
|
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9616
9720
|
* start is checked first at its `startAt`.
|
|
9617
9721
|
*/
|
|
9618
|
-
async parkStartForWorker(name, input, runId, opts) {
|
|
9722
|
+
async parkStartForWorker(name, input, runId, opts, wireSingleton) {
|
|
9619
9723
|
const prior = await this.store.getRun(runId);
|
|
9620
9724
|
if (prior) return this.existingStartResult(prior);
|
|
9621
9725
|
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9622
9726
|
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9727
|
+
const singleton = conventionSingleton({}, void 0, wireSingleton);
|
|
9728
|
+
if (singleton) await this.singletons.assertCapacity(name, singleton, input);
|
|
9623
9729
|
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9624
9730
|
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9625
9731
|
const pinned = opts?.version;
|
|
@@ -9634,6 +9740,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9634
9740
|
const now = /* @__PURE__ */ new Date();
|
|
9635
9741
|
const tags = mergeTags(void 0, [
|
|
9636
9742
|
...opts?.tags ?? [],
|
|
9743
|
+
...singleton ? [
|
|
9744
|
+
this.singletons.tag(singleton, input)
|
|
9745
|
+
] : [],
|
|
9637
9746
|
...quota ? [
|
|
9638
9747
|
concurrencyTag(quota.key)
|
|
9639
9748
|
] : [],
|
|
@@ -9722,8 +9831,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9722
9831
|
concurrency: message.concurrency
|
|
9723
9832
|
} : {}
|
|
9724
9833
|
};
|
|
9834
|
+
const singleton = message.singleton !== void 0 && typeof message.singleton?.key === "string" ? {
|
|
9835
|
+
...message.singleton,
|
|
9836
|
+
key: tenantScopedKey(namespace, message.singleton.key)
|
|
9837
|
+
} : void 0;
|
|
9725
9838
|
try {
|
|
9726
|
-
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
9839
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true, singleton);
|
|
9727
9840
|
} catch (err) {
|
|
9728
9841
|
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
9729
9842
|
throw err;
|
|
@@ -9731,6 +9844,62 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9731
9844
|
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
9732
9845
|
}
|
|
9733
9846
|
}
|
|
9847
|
+
/**
|
|
9848
|
+
* The control plane's half of a tenant's `signalWithStart` (a `signalWithStart` run request):
|
|
9849
|
+
* start `start` like {@link startForTenant} unless its run exists, then signal `signal.token`.
|
|
9850
|
+
* Authorization is the caller's (the run-request responder): the run must belong to `namespace`,
|
|
9851
|
+
* and so must a run already waiting on the token. Delivered at once to a waiting run; with none
|
|
9852
|
+
* waiting yet it is buffered only when the token contains the run id (a run-scoped token), so a
|
|
9853
|
+
* tenant can never plant a payload for a wait another namespace opens later. `signalled: false`
|
|
9854
|
+
* when it was neither delivered nor buffered.
|
|
9855
|
+
*/
|
|
9856
|
+
async signalWithStartForTenant(start, namespace, tenant, signal) {
|
|
9857
|
+
const started = await this.startForTenant(start, namespace, tenant);
|
|
9858
|
+
if (TERMINAL_RUN_STATUSES.includes(started.status)) {
|
|
9859
|
+
return {
|
|
9860
|
+
runId: start.runId,
|
|
9861
|
+
signalled: false
|
|
9862
|
+
};
|
|
9863
|
+
}
|
|
9864
|
+
const waiter = (await this.store.listSignalWaiters(signal.token)).find((w) => w.token === signal.token);
|
|
9865
|
+
if (waiter) {
|
|
9866
|
+
const run = await this.store.getRun(waiter.runId);
|
|
9867
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) {
|
|
9868
|
+
return {
|
|
9869
|
+
runId: start.runId,
|
|
9870
|
+
signalled: false
|
|
9871
|
+
};
|
|
9872
|
+
}
|
|
9873
|
+
} else if (!signal.token.includes(start.runId)) {
|
|
9874
|
+
return {
|
|
9875
|
+
runId: start.runId,
|
|
9876
|
+
signalled: false
|
|
9877
|
+
};
|
|
9878
|
+
}
|
|
9879
|
+
await this.signalUnchecked(signal.token, signal.payload);
|
|
9880
|
+
return {
|
|
9881
|
+
runId: start.runId,
|
|
9882
|
+
signalled: true
|
|
9883
|
+
};
|
|
9884
|
+
}
|
|
9885
|
+
/**
|
|
9886
|
+
* The control plane's half of a tenant's `publishEvent` (a `publishEvent` run request): resume the
|
|
9887
|
+
* runs of `namespace` waiting on event `name` whose match the payload satisfies. Starting the
|
|
9888
|
+
* tenant's own subscribers is the tenant's (it holds their `where` and `input`); nothing is
|
|
9889
|
+
* buffered, since a buffered event could reach another namespace's later wait. Returns how many
|
|
9890
|
+
* runs it resumed.
|
|
9891
|
+
*/
|
|
9892
|
+
async publishEventForTenant(name, payload, namespace) {
|
|
9893
|
+
let touched = 0;
|
|
9894
|
+
for (const w of await this.store.listSignalWaiters(eventPrefix(name))) {
|
|
9895
|
+
if (!eventMatches(payload, eventMatchOf(w.token))) continue;
|
|
9896
|
+
const run = await this.store.getRun(w.runId);
|
|
9897
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) continue;
|
|
9898
|
+
await this.signalUnchecked(w.token, payload);
|
|
9899
|
+
touched += 1;
|
|
9900
|
+
}
|
|
9901
|
+
return touched;
|
|
9902
|
+
}
|
|
9734
9903
|
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
9735
9904
|
async recordRejectedStart(name, input, runId, opts, err) {
|
|
9736
9905
|
const prior = await this.store.getRun(runId);
|
|
@@ -10202,9 +10371,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10202
10371
|
if (!liveTransport) return void 0;
|
|
10203
10372
|
const executor = new RemoteWorkflowExecutor(liveTransport, run.workflow, run.namespace);
|
|
10204
10373
|
const declared = opts.askFleetForVersion ? await this.declaredVersion(liveTransport, group, run.workflow) : void 0;
|
|
10374
|
+
const version = declared ?? run.workflowVersion;
|
|
10375
|
+
const announced = await this.announcedRegistration(group, run.workflow, version);
|
|
10376
|
+
const singleton = conventionSingleton(run, announced, opts.singleton);
|
|
10377
|
+
if (singleton?.limit !== void 0) {
|
|
10378
|
+
this.conventionSingletonLimits.set(`${runNamespace(run.namespace)}\0${run.workflow}`, singleton.limit);
|
|
10379
|
+
}
|
|
10380
|
+
const tags = announced?.options?.tags;
|
|
10205
10381
|
return {
|
|
10206
10382
|
name: run.workflow,
|
|
10207
|
-
version
|
|
10383
|
+
version,
|
|
10208
10384
|
fn: /* @__PURE__ */ __name(() => {
|
|
10209
10385
|
throw new Error(`remote workflow ${run.workflow} has no local body`);
|
|
10210
10386
|
}, "fn"),
|
|
@@ -10213,12 +10389,47 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10213
10389
|
group,
|
|
10214
10390
|
executor
|
|
10215
10391
|
},
|
|
10392
|
+
...Array.isArray(tags) && tags.length > 0 ? {
|
|
10393
|
+
tags
|
|
10394
|
+
} : {},
|
|
10395
|
+
...singleton ? {
|
|
10396
|
+
singleton
|
|
10397
|
+
} : {},
|
|
10216
10398
|
...opts.askFleetForVersion ? {
|
|
10217
10399
|
versionDeclared: declared !== void 0
|
|
10218
10400
|
} : {}
|
|
10219
10401
|
};
|
|
10220
10402
|
}
|
|
10221
10403
|
/**
|
|
10404
|
+
* The registration the live workers of `group` announce for `workflow` (at `version`, when one of
|
|
10405
|
+
* them states it): where they disagree (a rolling deploy), the one from the worker that started
|
|
10406
|
+
* last, as for announced schedules. `undefined` when none announces it, `null` when the read
|
|
10407
|
+
* failed (what the workers state is then unknown, not "nothing"). Reads
|
|
10408
|
+
* through the descriptor cache ({@link workerDescriptorCacheMs}), so a turn costs no extra round
|
|
10409
|
+
* trip in steady state.
|
|
10410
|
+
*/
|
|
10411
|
+
async announcedRegistration(group, workflow, version) {
|
|
10412
|
+
let descriptors;
|
|
10413
|
+
try {
|
|
10414
|
+
descriptors = await this.liveWorkerDescriptors(group);
|
|
10415
|
+
} catch {
|
|
10416
|
+
return null;
|
|
10417
|
+
}
|
|
10418
|
+
let best;
|
|
10419
|
+
for (const descriptor of descriptors) {
|
|
10420
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10421
|
+
if (reg?.name !== workflow) continue;
|
|
10422
|
+
if (version !== void 0 && reg.version !== void 0 && reg.version !== version) continue;
|
|
10423
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10424
|
+
if (!best || at > best.at) best = {
|
|
10425
|
+
at,
|
|
10426
|
+
reg
|
|
10427
|
+
};
|
|
10428
|
+
}
|
|
10429
|
+
}
|
|
10430
|
+
return best?.reg;
|
|
10431
|
+
}
|
|
10432
|
+
/**
|
|
10222
10433
|
* The version the LIVE fleet declares for `workflow` on `group`, or `undefined` when it declares
|
|
10223
10434
|
* none — which is a fact about the fleet, not a failure to read it.
|
|
10224
10435
|
*
|
|
@@ -10442,34 +10653,95 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10442
10653
|
for (const reg of new Set(this.latest.values())) {
|
|
10443
10654
|
if (this.draining) return;
|
|
10444
10655
|
if (reg.executionTimeoutMs == null) continue;
|
|
10445
|
-
|
|
10446
|
-
|
|
10447
|
-
|
|
10448
|
-
|
|
10449
|
-
|
|
10450
|
-
|
|
10451
|
-
|
|
10452
|
-
|
|
10453
|
-
|
|
10454
|
-
|
|
10455
|
-
|
|
10456
|
-
|
|
10457
|
-
|
|
10458
|
-
|
|
10459
|
-
|
|
10460
|
-
|
|
10461
|
-
|
|
10462
|
-
|
|
10463
|
-
|
|
10464
|
-
|
|
10465
|
-
|
|
10466
|
-
|
|
10467
|
-
|
|
10468
|
-
|
|
10469
|
-
|
|
10470
|
-
|
|
10471
|
-
|
|
10656
|
+
await this.cancelTimedOut(reg.name, this.namespace, now - reg.executionTimeoutMs);
|
|
10657
|
+
}
|
|
10658
|
+
for (const { workflow, namespace, timeoutMs } of await this.announcedTimeouts(now)) {
|
|
10659
|
+
if (this.draining) return;
|
|
10660
|
+
await this.cancelTimedOut(workflow, namespace, now - timeoutMs);
|
|
10661
|
+
}
|
|
10662
|
+
}
|
|
10663
|
+
/**
|
|
10664
|
+
* The singleton limit last resolved for a workflow served by convention, by `<namespace>\0<name>`
|
|
10665
|
+
* (see {@link resolveRemoteByConvention}): what the singleton gate's release wake-up reads, since
|
|
10666
|
+
* it cannot wait on the fleet. One entry per convention singleton the fleet serves.
|
|
10667
|
+
*/
|
|
10668
|
+
conventionSingletonLimits = /* @__PURE__ */ new Map();
|
|
10669
|
+
/** The last read of the announced execution timeouts (see {@link announcedTimeouts}). */
|
|
10670
|
+
announcedTimeoutsRead;
|
|
10671
|
+
/**
|
|
10672
|
+
* The `executionTimeoutMs` live workers announce (`@Workflow({ executionTimeout })` on a tenant),
|
|
10673
|
+
* per workflow and namespace, for workflows this engine does not register (a registration here
|
|
10674
|
+
* applies its own). Where workers disagree, the one that started last wins. Read at most once per
|
|
10675
|
+
* {@link ANNOUNCED_TIMEOUTS_TTL_MS}; empty without a transport that reads the fleet, or when the
|
|
10676
|
+
* read fails.
|
|
10677
|
+
*/
|
|
10678
|
+
async announcedTimeouts(now) {
|
|
10679
|
+
if (!this.pool.introspectsFleet) return [];
|
|
10680
|
+
const cached = this.announcedTimeoutsRead;
|
|
10681
|
+
if (cached && now - cached.at < ANNOUNCED_TIMEOUTS_TTL_MS) return cached.list;
|
|
10682
|
+
const newest = /* @__PURE__ */ new Map();
|
|
10683
|
+
try {
|
|
10684
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
10685
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10686
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10687
|
+
if (!reg || typeof reg.name !== "string" || this.latest.has(reg.name)) continue;
|
|
10688
|
+
const namespace = servedNamespace(descriptor.namespace, reg.group);
|
|
10689
|
+
if (this.namespace !== void 0 && runNamespace(this.namespace) !== namespace) continue;
|
|
10690
|
+
const key = `${namespace}\0${reg.name}`;
|
|
10691
|
+
const prior = newest.get(key);
|
|
10692
|
+
if (prior && prior.at >= at) continue;
|
|
10693
|
+
newest.set(key, {
|
|
10694
|
+
at,
|
|
10695
|
+
workflow: reg.name,
|
|
10696
|
+
namespace,
|
|
10697
|
+
ms: Number(reg.options?.executionTimeoutMs)
|
|
10698
|
+
});
|
|
10699
|
+
}
|
|
10472
10700
|
}
|
|
10701
|
+
} catch {
|
|
10702
|
+
return cached?.list ?? [];
|
|
10703
|
+
}
|
|
10704
|
+
const list2 = [
|
|
10705
|
+
...newest.values()
|
|
10706
|
+
].filter((e) => Number.isFinite(e.ms) && e.ms > 0).map((e) => ({
|
|
10707
|
+
workflow: e.workflow,
|
|
10708
|
+
namespace: e.namespace,
|
|
10709
|
+
timeoutMs: e.ms
|
|
10710
|
+
}));
|
|
10711
|
+
this.announcedTimeoutsRead = {
|
|
10712
|
+
at: now,
|
|
10713
|
+
list: list2
|
|
10714
|
+
};
|
|
10715
|
+
return list2;
|
|
10716
|
+
}
|
|
10717
|
+
/** Cancel (`execution_timeout`) the in-flight runs of `workflow` in `namespace` created before `deadline`. */
|
|
10718
|
+
async cancelTimedOut(workflow, namespace, deadline) {
|
|
10719
|
+
const inflight = await this.store.listRuns({
|
|
10720
|
+
workflow,
|
|
10721
|
+
statuses: TIMEOUT_SWEPT_STATUSES,
|
|
10722
|
+
namespace
|
|
10723
|
+
});
|
|
10724
|
+
for (const run of inflight) {
|
|
10725
|
+
if (this.draining) return;
|
|
10726
|
+
if (run.createdAt.getTime() > deadline) continue;
|
|
10727
|
+
const error2 = {
|
|
10728
|
+
message: "execution timeout",
|
|
10729
|
+
code: "execution_timeout"
|
|
10730
|
+
};
|
|
10731
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
10732
|
+
status: "cancelled",
|
|
10733
|
+
error: error2,
|
|
10734
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
10735
|
+
}));
|
|
10736
|
+
this.emit({
|
|
10737
|
+
type: "run.failed",
|
|
10738
|
+
runId: run.id,
|
|
10739
|
+
workflow: run.workflow,
|
|
10740
|
+
namespace: run.namespace,
|
|
10741
|
+
error: error2
|
|
10742
|
+
});
|
|
10743
|
+
await this.cancelChildren(run.id).catch(() => void 0);
|
|
10744
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
10473
10745
|
}
|
|
10474
10746
|
}
|
|
10475
10747
|
/**
|