@dudousxd/nestjs-durable-core 0.100.0 → 0.102.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +643 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +600 -290
- package/dist/index.d.ts +600 -290
- package/dist/index.js +640 -45
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -23,6 +23,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
23
23
|
var index_exports = {};
|
|
24
24
|
__export(index_exports, {
|
|
25
25
|
ACCUMULATOR_SEEN_MAX: () => ACCUMULATOR_SEEN_MAX,
|
|
26
|
+
ANNOUNCED_SCHEDULE_TAG: () => ANNOUNCED_SCHEDULE_TAG,
|
|
26
27
|
ANY_WAIT_GROUP: () => ANY_WAIT_GROUP,
|
|
27
28
|
CHILD_EDGE_PREFIXES: () => CHILD_EDGE_PREFIXES,
|
|
28
29
|
CONTROL_FLOW_SIGNAL: () => CONTROL_FLOW_SIGNAL,
|
|
@@ -31,6 +32,7 @@ __export(index_exports, {
|
|
|
31
32
|
CodecStateStore: () => CodecStateStore,
|
|
32
33
|
ConcurrencyLimitError: () => ConcurrencyLimitError,
|
|
33
34
|
ContinueAsNew: () => ContinueAsNew,
|
|
35
|
+
DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS: () => DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
34
36
|
DEFAULT_CONCURRENCY_STATUSES: () => DEFAULT_CONCURRENCY_STATUSES,
|
|
35
37
|
DEFAULT_PARTITION: () => DEFAULT_PARTITION,
|
|
36
38
|
DEFAULT_STALLED_AFTER_MS: () => DEFAULT_STALLED_AFTER_MS,
|
|
@@ -112,6 +114,7 @@ __export(index_exports, {
|
|
|
112
114
|
accumulatorRunId: () => accumulatorRunId,
|
|
113
115
|
aggregateAnnouncements: () => aggregateAnnouncements,
|
|
114
116
|
aggregatesCheckpoints: () => aggregatesCheckpoints,
|
|
117
|
+
announcedScheduleId: () => announcedScheduleId,
|
|
115
118
|
announcementsOf: () => announcementsOf,
|
|
116
119
|
anyCheckpointName: () => anyCheckpointName,
|
|
117
120
|
applyAttributeQuery: () => applyAttributeQuery,
|
|
@@ -2606,6 +2609,9 @@ function normalizeDescriptor(raw) {
|
|
|
2606
2609
|
...raw.registrations !== void 0 ? {
|
|
2607
2610
|
registrations: raw.registrations
|
|
2608
2611
|
} : {},
|
|
2612
|
+
...raw.schedules !== void 0 ? {
|
|
2613
|
+
schedules: raw.schedules
|
|
2614
|
+
} : {},
|
|
2609
2615
|
...raw.partition !== void 0 ? {
|
|
2610
2616
|
partition: raw.partition
|
|
2611
2617
|
} : {},
|
|
@@ -2633,6 +2639,13 @@ function canonicalizeRegistrations(regs) {
|
|
|
2633
2639
|
subscribes: [
|
|
2634
2640
|
...new Set(r.subscribes)
|
|
2635
2641
|
].sort()
|
|
2642
|
+
} : {},
|
|
2643
|
+
// Conditional too: a registration without them keeps its hash; a changed option changes it.
|
|
2644
|
+
...r.options !== void 0 ? {
|
|
2645
|
+
options: stableJson(r.options)
|
|
2646
|
+
} : {},
|
|
2647
|
+
...r.deadLetter !== void 0 ? {
|
|
2648
|
+
deadLetter: r.deadLetter
|
|
2636
2649
|
} : {}
|
|
2637
2650
|
}));
|
|
2638
2651
|
const byJson = /* @__PURE__ */ new Map();
|
|
@@ -2680,12 +2693,30 @@ function canonicalizeForHash(d) {
|
|
|
2680
2693
|
...d.registrations !== void 0 ? {
|
|
2681
2694
|
registrations: canonicalizeRegistrations(d.registrations)
|
|
2682
2695
|
} : {},
|
|
2696
|
+
// Conditional like `registrations`: a descriptor without schedules keeps its published hash.
|
|
2697
|
+
...d.schedules !== void 0 ? {
|
|
2698
|
+
schedules: canonicalSchedules(d.schedules)
|
|
2699
|
+
} : {},
|
|
2683
2700
|
partition: d.partition ?? null,
|
|
2684
2701
|
namespace: d.namespace ?? null,
|
|
2685
2702
|
startedAt: d.startedAt
|
|
2686
2703
|
};
|
|
2687
2704
|
}
|
|
2688
2705
|
__name(canonicalizeForHash, "canonicalizeForHash");
|
|
2706
|
+
function canonicalSchedules(a) {
|
|
2707
|
+
const items = [
|
|
2708
|
+
...a.items ?? []
|
|
2709
|
+
].sort((x, y) => x.key.localeCompare(y.key));
|
|
2710
|
+
return {
|
|
2711
|
+
tenant: a.tenant,
|
|
2712
|
+
items: items.map((i) => stableJson(i))
|
|
2713
|
+
};
|
|
2714
|
+
}
|
|
2715
|
+
__name(canonicalSchedules, "canonicalSchedules");
|
|
2716
|
+
function stableJson(value) {
|
|
2717
|
+
return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
|
|
2718
|
+
}
|
|
2719
|
+
__name(stableJson, "stableJson");
|
|
2689
2720
|
function fnv1a64Hex(input) {
|
|
2690
2721
|
const OFFSET = 0xcbf29ce484222325n;
|
|
2691
2722
|
const PRIME = 0x100000001b3n;
|
|
@@ -2743,6 +2774,14 @@ function describeWorker(opts) {
|
|
|
2743
2774
|
...registrations.length > 0 ? {
|
|
2744
2775
|
registrations
|
|
2745
2776
|
} : {},
|
|
2777
|
+
...opts.schedules !== void 0 ? {
|
|
2778
|
+
schedules: {
|
|
2779
|
+
tenant: opts.schedules.tenant,
|
|
2780
|
+
items: [
|
|
2781
|
+
...opts.schedules.items
|
|
2782
|
+
].sort((a, b) => a.key.localeCompare(b.key))
|
|
2783
|
+
}
|
|
2784
|
+
} : {},
|
|
2746
2785
|
...opts.partition ? {
|
|
2747
2786
|
partition: opts.partition
|
|
2748
2787
|
} : {},
|
|
@@ -4161,6 +4200,12 @@ function matchesTenantPredicates(row, query) {
|
|
|
4161
4200
|
__name(matchesTenantPredicates, "matchesTenantPredicates");
|
|
4162
4201
|
|
|
4163
4202
|
// src/schedules.ts
|
|
4203
|
+
var ANNOUNCED_SCHEDULE_TAG = "schedule:announced";
|
|
4204
|
+
function announcedScheduleId(namespace, key) {
|
|
4205
|
+
return `app:${namespace}:${key}`;
|
|
4206
|
+
}
|
|
4207
|
+
__name(announcedScheduleId, "announcedScheduleId");
|
|
4208
|
+
var PARKED_FOR_WORKER = "no_live_worker";
|
|
4164
4209
|
var ScheduleAlreadyExistsError = class extends Error {
|
|
4165
4210
|
static {
|
|
4166
4211
|
__name(this, "ScheduleAlreadyExistsError");
|
|
@@ -4301,15 +4346,29 @@ var ScheduleClient = class {
|
|
|
4301
4346
|
async trigger(id) {
|
|
4302
4347
|
const record = await this.mustGet(id);
|
|
4303
4348
|
const runId = `sched:${id}:manual:${this.now()}`;
|
|
4304
|
-
return this.
|
|
4349
|
+
return this.startFor(record, runId);
|
|
4305
4350
|
}
|
|
4306
4351
|
/**
|
|
4307
|
-
*
|
|
4352
|
+
* Start one run of `record`: through the tenant start path for an announced schedule (its run waits
|
|
4353
|
+
* `blocked` for a live worker of its app instead of failing), else a plain `start`.
|
|
4354
|
+
*/
|
|
4355
|
+
startFor(record, runId) {
|
|
4356
|
+
const input = specOf(record).input;
|
|
4357
|
+
const opts = startOptions(record);
|
|
4358
|
+
if (isAnnounced(record) && this.host.startParked) {
|
|
4359
|
+
const singleton = record.state.announce?.singleton;
|
|
4360
|
+
return singleton ? this.host.startParked(record.workflow, input, runId, opts, singleton) : this.host.startParked(record.workflow, input, runId, opts);
|
|
4361
|
+
}
|
|
4362
|
+
return this.host.start(record.workflow, input, runId, opts);
|
|
4363
|
+
}
|
|
4364
|
+
/**
|
|
4365
|
+
* Fire every due schedule (in this engine's namespace; all namespaces for an operator engine; only
|
|
4366
|
+
* those carrying `opts.tag` when given):
|
|
4308
4367
|
* start its window's run and advance it to the next window. Called by the NestJS timer poller on
|
|
4309
4368
|
* each tick when `persistedSchedules` is on; call it yourself from any other driver. Returns the
|
|
4310
4369
|
* run ids started (or re-found, for an idempotent re-fire).
|
|
4311
4370
|
*/
|
|
4312
|
-
async tick(nowMs = this.now()) {
|
|
4371
|
+
async tick(nowMs = this.now(), opts = {}) {
|
|
4313
4372
|
const store = this.requireStore();
|
|
4314
4373
|
const started = [];
|
|
4315
4374
|
for (let round = 0; round < 50; round++) {
|
|
@@ -4317,6 +4376,9 @@ var ScheduleClient = class {
|
|
|
4317
4376
|
const due = await store.listSchedules({
|
|
4318
4377
|
dueBy: nowMs,
|
|
4319
4378
|
namespace: this.options.namespace,
|
|
4379
|
+
...opts.tag !== void 0 ? {
|
|
4380
|
+
tag: opts.tag
|
|
4381
|
+
} : {},
|
|
4320
4382
|
limit: TICK_BATCH
|
|
4321
4383
|
});
|
|
4322
4384
|
for (const record of due) {
|
|
@@ -4330,15 +4392,19 @@ var ScheduleClient = class {
|
|
|
4330
4392
|
}
|
|
4331
4393
|
/**
|
|
4332
4394
|
* When {@link tick} next has a schedule to fire, if one is due by `byMs`: the earliest `nextFireAt`
|
|
4333
|
-
* of the unpaused schedules (of this engine's namespace
|
|
4395
|
+
* of the unpaused schedules (of this engine's namespace, carrying `opts.tag` when given) due by
|
|
4396
|
+
* then, else `undefined`. One indexed
|
|
4334
4397
|
* read (the due-schedule scan `tick` makes, with a later bound and a limit of one). A store that
|
|
4335
4398
|
* does not persist schedules has none.
|
|
4336
4399
|
*/
|
|
4337
|
-
async nextFireAt(byMs) {
|
|
4400
|
+
async nextFireAt(byMs, opts = {}) {
|
|
4338
4401
|
if (!this.supported) return void 0;
|
|
4339
4402
|
const [next] = await this.requireStore().listSchedules({
|
|
4340
4403
|
dueBy: byMs,
|
|
4341
4404
|
namespace: this.options.namespace,
|
|
4405
|
+
...opts.tag !== void 0 ? {
|
|
4406
|
+
tag: opts.tag
|
|
4407
|
+
} : {},
|
|
4342
4408
|
limit: 1
|
|
4343
4409
|
});
|
|
4344
4410
|
return next?.nextFireAt ?? void 0;
|
|
@@ -4373,10 +4439,10 @@ var ScheduleClient = class {
|
|
|
4373
4439
|
let fired = false;
|
|
4374
4440
|
const skipMissed = spec.catchup === "skip" && !onTime;
|
|
4375
4441
|
const beforeWindow = latest < window;
|
|
4376
|
-
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state)) {
|
|
4442
|
+
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state) && !await this.waitingForWorker(record, state)) {
|
|
4377
4443
|
runId = `sched:${record.id}:${latest}`;
|
|
4378
4444
|
try {
|
|
4379
|
-
await this.
|
|
4445
|
+
await this.startFor(record, runId);
|
|
4380
4446
|
fired = true;
|
|
4381
4447
|
} catch (error2) {
|
|
4382
4448
|
if (await this.host.getRun(runId)) fired = true;
|
|
@@ -4400,6 +4466,186 @@ var ScheduleClient = class {
|
|
|
4400
4466
|
}, record.nextFireAt);
|
|
4401
4467
|
return fired ? runId : void 0;
|
|
4402
4468
|
}
|
|
4469
|
+
/**
|
|
4470
|
+
* An announced schedule whose previous run is still parked waiting for a live worker of its app
|
|
4471
|
+
* (the app is down): this window is folded into that run instead of parking one more. However long
|
|
4472
|
+
* the app is down, it finds at most one run per schedule waiting when it comes back, the same
|
|
4473
|
+
* bound `catchup: 'latest'` puts on windows missed while no scheduler polled.
|
|
4474
|
+
*/
|
|
4475
|
+
async waitingForWorker(record, state) {
|
|
4476
|
+
if (!isAnnounced(record) || !state.lastRunId) return false;
|
|
4477
|
+
const prev = await this.host.getRun(state.lastRunId);
|
|
4478
|
+
return prev?.status === "blocked" && prev.error?.code === PARKED_FOR_WORKER;
|
|
4479
|
+
}
|
|
4480
|
+
/**
|
|
4481
|
+
* Reconcile the schedules adopted from one app (`namespace`) with what its LIVE workers announce
|
|
4482
|
+
* now (`items`, one per key; the caller resolved disagreeing workers). The rule:
|
|
4483
|
+
* - an announced key with no schedule yet is created (id {@link announcedScheduleId}, tagged
|
|
4484
|
+
* {@link ANNOUNCED_SCHEDULE_TAG}, `catchup: 'latest'`); one whose definition changed is
|
|
4485
|
+
* re-applied like `upsert` (its history kept, its next fire recomputed from now). An equal
|
|
4486
|
+
* definition writes nothing, so any number of replicas announcing the same set is a no-op;
|
|
4487
|
+
* - a pause or resume made from the console survives: a re-applied definition keeps the current
|
|
4488
|
+
* paused state, unless the app's own declared `paused` changed;
|
|
4489
|
+
* - an adopted schedule the app no longer announces is marked missing, and paused ("withdrawn")
|
|
4490
|
+
* once it has stayed missing for `graceMs`. It is never deleted: its history stays, and an app
|
|
4491
|
+
* that announces it again (a rollback) gets it back, resumed unless an operator had paused it.
|
|
4492
|
+
*
|
|
4493
|
+
* Call it only for an app that has at least one live worker stating its schedules: with none (the
|
|
4494
|
+
* app is down, or runs an SDK that does not announce schedules), nothing is withdrawn, and its
|
|
4495
|
+
* schedules keep firing into runs that wait for a worker.
|
|
4496
|
+
*/
|
|
4497
|
+
async syncAnnounced(namespace, items, opts) {
|
|
4498
|
+
const store = this.requireStore();
|
|
4499
|
+
const now = opts.nowMs ?? this.now();
|
|
4500
|
+
const out = {
|
|
4501
|
+
adopted: [],
|
|
4502
|
+
restored: [],
|
|
4503
|
+
missing: [],
|
|
4504
|
+
withdrawn: [],
|
|
4505
|
+
invalid: []
|
|
4506
|
+
};
|
|
4507
|
+
const adopted = /* @__PURE__ */ new Map();
|
|
4508
|
+
for (let offset = 0; ; offset += TICK_BATCH) {
|
|
4509
|
+
const page = await store.listSchedules({
|
|
4510
|
+
namespace,
|
|
4511
|
+
tag: ANNOUNCED_SCHEDULE_TAG,
|
|
4512
|
+
limit: TICK_BATCH,
|
|
4513
|
+
offset
|
|
4514
|
+
});
|
|
4515
|
+
for (const record of page) if (isAnnounced(record)) adopted.set(record.id, record);
|
|
4516
|
+
if (page.length < TICK_BATCH) break;
|
|
4517
|
+
}
|
|
4518
|
+
for (const item of items) {
|
|
4519
|
+
const id = announcedScheduleId(namespace, item.key);
|
|
4520
|
+
const existing = adopted.get(id) ?? await store.getSchedule(id);
|
|
4521
|
+
adopted.delete(id);
|
|
4522
|
+
const doc = existing ? existing.state.announce : void 0;
|
|
4523
|
+
if (existing && !doc) {
|
|
4524
|
+
out.invalid.push({
|
|
4525
|
+
key: item.key,
|
|
4526
|
+
error: `schedule id "${id}" is taken`
|
|
4527
|
+
});
|
|
4528
|
+
continue;
|
|
4529
|
+
}
|
|
4530
|
+
const fingerprint = stableJson(item);
|
|
4531
|
+
if (existing && doc && doc.fingerprint === fingerprint) {
|
|
4532
|
+
if (doc.missingSince === void 0 && doc.withdrawnAt === void 0) continue;
|
|
4533
|
+
const resume = doc.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal && existing.paused;
|
|
4534
|
+
const spec = specOf(existing);
|
|
4535
|
+
const window = resume ? nextWindowAfter(spec, now) : void 0;
|
|
4536
|
+
await store.updateSchedule(id, {
|
|
4537
|
+
// The withdrawal note goes; the note it replaced comes back.
|
|
4538
|
+
...doc.withdrawnAt !== void 0 ? {
|
|
4539
|
+
spec: stripUndefined({
|
|
4540
|
+
...spec,
|
|
4541
|
+
note: doc.noteBeforeWithdrawal
|
|
4542
|
+
})
|
|
4543
|
+
} : {},
|
|
4544
|
+
...resume ? {
|
|
4545
|
+
paused: false,
|
|
4546
|
+
nextFireAt: window == null ? null : window + jitterFor(id, window, spec)
|
|
4547
|
+
} : {},
|
|
4548
|
+
state: stripUndefined({
|
|
4549
|
+
...existing.state,
|
|
4550
|
+
...resume ? {
|
|
4551
|
+
window
|
|
4552
|
+
} : {},
|
|
4553
|
+
announce: {
|
|
4554
|
+
key: doc.key,
|
|
4555
|
+
fingerprint,
|
|
4556
|
+
declaredPaused: doc.declaredPaused,
|
|
4557
|
+
singleton: doc.singleton
|
|
4558
|
+
}
|
|
4559
|
+
}),
|
|
4560
|
+
updatedAt: new Date(now)
|
|
4561
|
+
});
|
|
4562
|
+
out.restored.push(id);
|
|
4563
|
+
continue;
|
|
4564
|
+
}
|
|
4565
|
+
const declaredPaused = item.paused === true;
|
|
4566
|
+
const paused = existing === null || existing === void 0 ? declaredPaused : declaredPaused !== (doc?.declaredPaused === true) ? declaredPaused : doc?.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal ? declaredPaused : void 0;
|
|
4567
|
+
try {
|
|
4568
|
+
await this.write({
|
|
4569
|
+
id,
|
|
4570
|
+
workflow: item.workflow,
|
|
4571
|
+
input: item.input,
|
|
4572
|
+
cron: item.cron,
|
|
4573
|
+
every: item.everyMs,
|
|
4574
|
+
timezone: item.timezone,
|
|
4575
|
+
jitter: item.jitterMs,
|
|
4576
|
+
overlap: item.overlap,
|
|
4577
|
+
catchup: "latest",
|
|
4578
|
+
namespace,
|
|
4579
|
+
tags: [
|
|
4580
|
+
ANNOUNCED_SCHEDULE_TAG
|
|
4581
|
+
],
|
|
4582
|
+
paused,
|
|
4583
|
+
// A note (an operator's pause reason) is kept; a withdrawal note is replaced by the one
|
|
4584
|
+
// it had replaced.
|
|
4585
|
+
note: doc?.withdrawnAt !== void 0 ? doc.noteBeforeWithdrawal : existing ? specOf(existing).note : void 0,
|
|
4586
|
+
// Fires belong to the app, never to whichever request context reached the sync.
|
|
4587
|
+
...existing ? {} : {
|
|
4588
|
+
tenantId: null
|
|
4589
|
+
}
|
|
4590
|
+
}, existing ?? null, {
|
|
4591
|
+
announce: {
|
|
4592
|
+
key: item.key,
|
|
4593
|
+
fingerprint,
|
|
4594
|
+
declaredPaused,
|
|
4595
|
+
...validSingleton(item.singleton) ? {
|
|
4596
|
+
singleton: item.singleton
|
|
4597
|
+
} : {}
|
|
4598
|
+
}
|
|
4599
|
+
});
|
|
4600
|
+
out.adopted.push(id);
|
|
4601
|
+
} catch (error2) {
|
|
4602
|
+
out.invalid.push({
|
|
4603
|
+
key: item.key,
|
|
4604
|
+
error: errorMessage(error2)
|
|
4605
|
+
});
|
|
4606
|
+
}
|
|
4607
|
+
}
|
|
4608
|
+
for (const record of adopted.values()) {
|
|
4609
|
+
const state = record.state;
|
|
4610
|
+
const doc = state.announce;
|
|
4611
|
+
if (doc.withdrawnAt !== void 0) continue;
|
|
4612
|
+
if (doc.missingSince === void 0) {
|
|
4613
|
+
await store.updateSchedule(record.id, {
|
|
4614
|
+
state: {
|
|
4615
|
+
...state,
|
|
4616
|
+
announce: {
|
|
4617
|
+
...doc,
|
|
4618
|
+
missingSince: now
|
|
4619
|
+
}
|
|
4620
|
+
},
|
|
4621
|
+
updatedAt: new Date(now)
|
|
4622
|
+
});
|
|
4623
|
+
out.missing.push(record.id);
|
|
4624
|
+
continue;
|
|
4625
|
+
}
|
|
4626
|
+
if (now - doc.missingSince < opts.graceMs) continue;
|
|
4627
|
+
const spec = {
|
|
4628
|
+
...specOf(record)
|
|
4629
|
+
};
|
|
4630
|
+
spec.note = `withdrawn: app "${namespace}" no longer announces schedule "${doc.key}"`;
|
|
4631
|
+
await store.updateSchedule(record.id, {
|
|
4632
|
+
paused: true,
|
|
4633
|
+
spec,
|
|
4634
|
+
state: {
|
|
4635
|
+
...state,
|
|
4636
|
+
announce: stripUndefined({
|
|
4637
|
+
...doc,
|
|
4638
|
+
withdrawnAt: now,
|
|
4639
|
+
pausedBeforeWithdrawal: record.paused,
|
|
4640
|
+
noteBeforeWithdrawal: specOf(record).note
|
|
4641
|
+
})
|
|
4642
|
+
},
|
|
4643
|
+
updatedAt: new Date(now)
|
|
4644
|
+
});
|
|
4645
|
+
out.withdrawn.push(record.id);
|
|
4646
|
+
}
|
|
4647
|
+
return out;
|
|
4648
|
+
}
|
|
4403
4649
|
/** `overlap: 'skip'` and the previous run of this schedule is still in flight. */
|
|
4404
4650
|
async overlapping(spec, state) {
|
|
4405
4651
|
if (spec.overlap !== "skip" || !state.lastRunId) return false;
|
|
@@ -4435,7 +4681,7 @@ var ScheduleClient = class {
|
|
|
4435
4681
|
tenantId: tenant.id
|
|
4436
4682
|
};
|
|
4437
4683
|
}
|
|
4438
|
-
async write(opts, existing) {
|
|
4684
|
+
async write(opts, existing, extraState = {}) {
|
|
4439
4685
|
const store = this.requireStore();
|
|
4440
4686
|
const spec = toSpec(opts);
|
|
4441
4687
|
const now = this.now();
|
|
@@ -4451,7 +4697,8 @@ var ScheduleClient = class {
|
|
|
4451
4697
|
spec,
|
|
4452
4698
|
state: stripUndefined({
|
|
4453
4699
|
...existing?.state ?? {},
|
|
4454
|
-
window
|
|
4700
|
+
window,
|
|
4701
|
+
...extraState
|
|
4455
4702
|
}),
|
|
4456
4703
|
...this.scheduleTenant(opts, existing),
|
|
4457
4704
|
createdAt: existing?.createdAt ?? new Date(now),
|
|
@@ -4497,6 +4744,14 @@ function specOf(record) {
|
|
|
4497
4744
|
return record.spec;
|
|
4498
4745
|
}
|
|
4499
4746
|
__name(specOf, "specOf");
|
|
4747
|
+
function validSingleton(value) {
|
|
4748
|
+
return value !== null && typeof value === "object" && typeof value.key === "string" && value.key !== "";
|
|
4749
|
+
}
|
|
4750
|
+
__name(validSingleton, "validSingleton");
|
|
4751
|
+
function isAnnounced(record) {
|
|
4752
|
+
return record.state?.announce !== void 0;
|
|
4753
|
+
}
|
|
4754
|
+
__name(isAnnounced, "isAnnounced");
|
|
4500
4755
|
function latestWindowAtOrBefore(spec, atMs) {
|
|
4501
4756
|
if (spec.cron != null) return prevCronFireMs(spec.cron, atMs, spec.timezone);
|
|
4502
4757
|
if (spec.rrule != null) {
|
|
@@ -4571,6 +4826,9 @@ function describe(record) {
|
|
|
4571
4826
|
lastError: state.lastError,
|
|
4572
4827
|
tenantId: record.tenantId ?? void 0,
|
|
4573
4828
|
tenantKind: record.tenantKind ?? void 0,
|
|
4829
|
+
announcedKey: state.announce?.key,
|
|
4830
|
+
unannouncedSince: state.announce?.missingSince != null ? new Date(state.announce.missingSince) : void 0,
|
|
4831
|
+
withdrawnAt: state.announce?.withdrawnAt != null ? new Date(state.announce.withdrawnAt) : void 0,
|
|
4574
4832
|
createdAt: record.createdAt,
|
|
4575
4833
|
updatedAt: record.updatedAt
|
|
4576
4834
|
});
|
|
@@ -6933,6 +7191,43 @@ var DEFAULT_STICKY_RUNS = 1e3;
|
|
|
6933
7191
|
var DEFAULT_STICKY_IDLE_MS = 3e5;
|
|
6934
7192
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
6935
7193
|
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
7194
|
+
var ANNOUNCED_TIMEOUTS_TTL_MS = 15e3;
|
|
7195
|
+
function tenantScopedKey(namespace, key) {
|
|
7196
|
+
return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
|
|
7197
|
+
}
|
|
7198
|
+
__name(tenantScopedKey, "tenantScopedKey");
|
|
7199
|
+
function singletonKeyOf(run) {
|
|
7200
|
+
const tag = run.tags?.find((t) => t.startsWith("singleton:"));
|
|
7201
|
+
return tag?.slice("singleton:".length);
|
|
7202
|
+
}
|
|
7203
|
+
__name(singletonKeyOf, "singletonKeyOf");
|
|
7204
|
+
function conventionSingleton(run, announced, wire) {
|
|
7205
|
+
if (wire !== void 0) {
|
|
7206
|
+
const key2 = wire.key;
|
|
7207
|
+
return {
|
|
7208
|
+
key: /* @__PURE__ */ __name(() => key2, "key"),
|
|
7209
|
+
...typeof wire.limit === "number" ? {
|
|
7210
|
+
limit: wire.limit
|
|
7211
|
+
} : {},
|
|
7212
|
+
...typeof wire.maxQueueDepth === "number" ? {
|
|
7213
|
+
maxQueueDepth: wire.maxQueueDepth
|
|
7214
|
+
} : {}
|
|
7215
|
+
};
|
|
7216
|
+
}
|
|
7217
|
+
const key = singletonKeyOf(run);
|
|
7218
|
+
if (key === void 0 || announced === void 0) return void 0;
|
|
7219
|
+
if (announced === null) return {
|
|
7220
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7221
|
+
limit: 1
|
|
7222
|
+
};
|
|
7223
|
+
const declared = announced.options?.singleton;
|
|
7224
|
+
if (declared === void 0) return void 0;
|
|
7225
|
+
return {
|
|
7226
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7227
|
+
limit: typeof declared.limit === "number" ? declared.limit : 1
|
|
7228
|
+
};
|
|
7229
|
+
}
|
|
7230
|
+
__name(conventionSingleton, "conventionSingleton");
|
|
6936
7231
|
function servedNamespace(namespace, group) {
|
|
6937
7232
|
if (namespace) return namespace;
|
|
6938
7233
|
const at = group?.lastIndexOf("@") ?? -1;
|
|
@@ -6957,6 +7252,7 @@ function runNamespace(namespace) {
|
|
|
6957
7252
|
__name(runNamespace, "runNamespace");
|
|
6958
7253
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
6959
7254
|
var NO_LIVE_WORKER = "no_live_worker";
|
|
7255
|
+
var DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS = 10 * 6e4;
|
|
6960
7256
|
var TenantStartInvalid = class TenantStartInvalid2 extends Error {
|
|
6961
7257
|
static {
|
|
6962
7258
|
__name(this, "TenantStartInvalid");
|
|
@@ -7170,6 +7466,10 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7170
7466
|
remoteRedispatchMax;
|
|
7171
7467
|
/** How long a `blocked` run waits before the recovery poll re-checks the live fleet (design §7.5). */
|
|
7172
7468
|
blockedPollMs;
|
|
7469
|
+
/** Verifies the wire tenant of a start-run, and of a worker's schedule announcement. */
|
|
7470
|
+
verifyTenant;
|
|
7471
|
+
/** Wire tenants whose schedule announcement failed verification, warned about once. */
|
|
7472
|
+
rejectedScheduleTenants = /* @__PURE__ */ new Set();
|
|
7173
7473
|
workerDescriptorCacheMs;
|
|
7174
7474
|
descriptorCache = /* @__PURE__ */ new Map();
|
|
7175
7475
|
/** This engine's control-plane handshake descriptor — negotiated against each worker (design §7.3). */
|
|
@@ -7380,6 +7680,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7380
7680
|
this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
|
|
7381
7681
|
this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
|
|
7382
7682
|
this.blockedPollMs = deps.blockedPollMs ?? 5e3;
|
|
7683
|
+
this.verifyTenant = deps.verifyTenant;
|
|
7383
7684
|
this.workerDescriptorCacheMs = deps.workerDescriptorCacheMs ?? 2e3;
|
|
7384
7685
|
this.cpDescriptor = controlPlaneDescriptor({
|
|
7385
7686
|
instanceId: `cp-${this.instanceId}`,
|
|
@@ -7416,10 +7717,28 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7416
7717
|
if (this.queuesRunsOverTransport) void this.wake(runId);
|
|
7417
7718
|
else this.runDispatcher.dispatch(runId);
|
|
7418
7719
|
}, "dispatch"),
|
|
7419
|
-
configFor: /* @__PURE__ */ __name((run) =>
|
|
7720
|
+
configFor: /* @__PURE__ */ __name((run) => {
|
|
7721
|
+
const local = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton;
|
|
7722
|
+
if (local) return local;
|
|
7723
|
+
const key = this.latest.has(run.workflow) ? void 0 : singletonKeyOf(run);
|
|
7724
|
+
if (key === void 0) return void 0;
|
|
7725
|
+
const limit = this.conventionSingletonLimits.get(`${runNamespace(run.namespace)}\0${run.workflow}`);
|
|
7726
|
+
return {
|
|
7727
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7728
|
+
...limit !== void 0 ? {
|
|
7729
|
+
limit
|
|
7730
|
+
} : {}
|
|
7731
|
+
};
|
|
7732
|
+
}, "configFor")
|
|
7420
7733
|
});
|
|
7421
7734
|
this.schedules = new ScheduleClient(this.store, {
|
|
7422
7735
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7736
|
+
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7737
|
+
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7738
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts, singleton) => this.startWith(workflow, input, runId, opts, true, singleton ? {
|
|
7739
|
+
...singleton,
|
|
7740
|
+
key: tenantScopedKey(opts?.namespace, singleton.key)
|
|
7741
|
+
} : void 0), "startParked"),
|
|
7423
7742
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7424
7743
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7425
7744
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -7593,14 +7912,17 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7593
7912
|
* The earliest known time (epoch ms) the poll has work, if it is at or before `byMs`: this
|
|
7594
7913
|
* namespace's next due timer (`listDueTimers` up to `byMs`, limit one: it returns the most overdue
|
|
7595
7914
|
* first), the next `blocked` run's recovery poll, and with `persistedSchedules` the next persisted
|
|
7596
|
-
* schedule to fire (see {@link ScheduleClient.nextFireAt}
|
|
7915
|
+
* schedule to fire (see {@link ScheduleClient.nextFireAt}; with `announcedSchedules` alone, the next
|
|
7916
|
+
* schedule adopted from a tenant). `undefined` when nothing is due by
|
|
7597
7917
|
* then. A poller reads it before waiting longer than its tick, so the wait never outlasts a timer.
|
|
7598
7918
|
*/
|
|
7599
7919
|
async nextDueAt(byMs, opts = {}) {
|
|
7600
7920
|
const [timers, blocked, schedule] = await Promise.all([
|
|
7601
7921
|
this.store.listDueTimers(byMs, this.namespace, 1),
|
|
7602
7922
|
this.dueBlockedRuns(byMs),
|
|
7603
|
-
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) :
|
|
7923
|
+
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) : opts.announcedSchedules && this.schedules.supported ? this.schedules.nextFireAt(byMs, {
|
|
7924
|
+
tag: ANNOUNCED_SCHEDULE_TAG
|
|
7925
|
+
}) : void 0
|
|
7604
7926
|
]);
|
|
7605
7927
|
let next;
|
|
7606
7928
|
for (const at of [
|
|
@@ -8226,6 +8548,39 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
8226
8548
|
} catch {
|
|
8227
8549
|
}
|
|
8228
8550
|
}
|
|
8551
|
+
if (!this.latest.has(run.workflow)) this.inBackground(this.routeAnnouncedDeadLetter(run));
|
|
8552
|
+
}
|
|
8553
|
+
/**
|
|
8554
|
+
* The dead-letter workflow the live workers serving `run`'s workflow announce (a tenant's inline
|
|
8555
|
+
* `@DeadLetter()` handler, its `deadLetterWorkflow`, or its module-level default), or `undefined`
|
|
8556
|
+
* when the workflow is registered on this engine (its own routing applies), they announce none, or
|
|
8557
|
+
* they cannot be read. A dead-letter listener with a catch-all default (`DurableModule`'s
|
|
8558
|
+
* `deadLetterWorkflow`) checks it first, so the app's own route wins over the default.
|
|
8559
|
+
*/
|
|
8560
|
+
async announcedDeadLetter(run) {
|
|
8561
|
+
if (this.latest.has(run.workflow) || this.isRegisteredHere(run)) return void 0;
|
|
8562
|
+
const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
|
|
8563
|
+
const reg = await this.announcedRegistration(group, run.workflow, run.workflowVersion);
|
|
8564
|
+
const target = reg?.deadLetter;
|
|
8565
|
+
return typeof target === "string" && target !== "" ? target : void 0;
|
|
8566
|
+
}
|
|
8567
|
+
/**
|
|
8568
|
+
* Start the announced dead-letter workflow of a dead run (see {@link announcedDeadLetter}) in the
|
|
8569
|
+
* run's namespace, as `dlq:<runId>` with the `DeadLetter` payload, through the tenant start path:
|
|
8570
|
+
* with no live worker of the app it waits `blocked` instead of being lost. Idempotent by run id.
|
|
8571
|
+
*/
|
|
8572
|
+
async routeAnnouncedDeadLetter(run) {
|
|
8573
|
+
const target = await this.announcedDeadLetter(run);
|
|
8574
|
+
if (!target) return;
|
|
8575
|
+
await this.startWith(target, {
|
|
8576
|
+
deadRunId: run.id,
|
|
8577
|
+
workflow: run.workflow,
|
|
8578
|
+
input: run.input,
|
|
8579
|
+
error: run.error
|
|
8580
|
+
}, `dlq:${run.id}`, {
|
|
8581
|
+
namespace: run.namespace,
|
|
8582
|
+
...inheritTenant(run)
|
|
8583
|
+
}, true);
|
|
8229
8584
|
}
|
|
8230
8585
|
/**
|
|
8231
8586
|
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
@@ -9526,7 +9881,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9526
9881
|
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9527
9882
|
* a start whose caller has already returned is never lost.
|
|
9528
9883
|
*/
|
|
9529
|
-
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9884
|
+
async startWith(workflow, input, runId, opts, parkUnroutable, wireSingleton) {
|
|
9530
9885
|
const name = workflowName(workflow);
|
|
9531
9886
|
const pinned = opts?.version;
|
|
9532
9887
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9551,12 +9906,15 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9551
9906
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9552
9907
|
};
|
|
9553
9908
|
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9554
|
-
askFleetForVersion: true
|
|
9909
|
+
askFleetForVersion: true,
|
|
9910
|
+
...wireSingleton !== void 0 ? {
|
|
9911
|
+
singleton: wireSingleton
|
|
9912
|
+
} : {}
|
|
9555
9913
|
});
|
|
9556
9914
|
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9557
9915
|
}
|
|
9558
9916
|
if (!registered && parkUnroutable) {
|
|
9559
|
-
return this.parkStartForWorker(name, input, runId, opts);
|
|
9917
|
+
return this.parkStartForWorker(name, input, runId, opts, wireSingleton);
|
|
9560
9918
|
}
|
|
9561
9919
|
if (!registered) {
|
|
9562
9920
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
@@ -9658,11 +10016,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9658
10016
|
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9659
10017
|
* start is checked first at its `startAt`.
|
|
9660
10018
|
*/
|
|
9661
|
-
async parkStartForWorker(name, input, runId, opts) {
|
|
10019
|
+
async parkStartForWorker(name, input, runId, opts, wireSingleton) {
|
|
9662
10020
|
const prior = await this.store.getRun(runId);
|
|
9663
10021
|
if (prior) return this.existingStartResult(prior);
|
|
9664
10022
|
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9665
10023
|
if (quota) await assertConcurrency(this.store, name, quota);
|
|
10024
|
+
const singleton = conventionSingleton({}, void 0, wireSingleton);
|
|
10025
|
+
if (singleton) await this.singletons.assertCapacity(name, singleton, input);
|
|
9666
10026
|
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9667
10027
|
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9668
10028
|
const pinned = opts?.version;
|
|
@@ -9677,6 +10037,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9677
10037
|
const now = /* @__PURE__ */ new Date();
|
|
9678
10038
|
const tags = mergeTags(void 0, [
|
|
9679
10039
|
...opts?.tags ?? [],
|
|
10040
|
+
...singleton ? [
|
|
10041
|
+
this.singletons.tag(singleton, input)
|
|
10042
|
+
] : [],
|
|
9680
10043
|
...quota ? [
|
|
9681
10044
|
concurrencyTag(quota.key)
|
|
9682
10045
|
] : [],
|
|
@@ -9765,8 +10128,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9765
10128
|
concurrency: message.concurrency
|
|
9766
10129
|
} : {}
|
|
9767
10130
|
};
|
|
10131
|
+
const singleton = message.singleton !== void 0 && typeof message.singleton?.key === "string" ? {
|
|
10132
|
+
...message.singleton,
|
|
10133
|
+
key: tenantScopedKey(namespace, message.singleton.key)
|
|
10134
|
+
} : void 0;
|
|
9768
10135
|
try {
|
|
9769
|
-
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
10136
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true, singleton);
|
|
9770
10137
|
} catch (err) {
|
|
9771
10138
|
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
9772
10139
|
throw err;
|
|
@@ -9774,6 +10141,62 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9774
10141
|
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
9775
10142
|
}
|
|
9776
10143
|
}
|
|
10144
|
+
/**
|
|
10145
|
+
* The control plane's half of a tenant's `signalWithStart` (a `signalWithStart` run request):
|
|
10146
|
+
* start `start` like {@link startForTenant} unless its run exists, then signal `signal.token`.
|
|
10147
|
+
* Authorization is the caller's (the run-request responder): the run must belong to `namespace`,
|
|
10148
|
+
* and so must a run already waiting on the token. Delivered at once to a waiting run; with none
|
|
10149
|
+
* waiting yet it is buffered only when the token contains the run id (a run-scoped token), so a
|
|
10150
|
+
* tenant can never plant a payload for a wait another namespace opens later. `signalled: false`
|
|
10151
|
+
* when it was neither delivered nor buffered.
|
|
10152
|
+
*/
|
|
10153
|
+
async signalWithStartForTenant(start, namespace, tenant, signal) {
|
|
10154
|
+
const started = await this.startForTenant(start, namespace, tenant);
|
|
10155
|
+
if (TERMINAL_RUN_STATUSES.includes(started.status)) {
|
|
10156
|
+
return {
|
|
10157
|
+
runId: start.runId,
|
|
10158
|
+
signalled: false
|
|
10159
|
+
};
|
|
10160
|
+
}
|
|
10161
|
+
const waiter = (await this.store.listSignalWaiters(signal.token)).find((w) => w.token === signal.token);
|
|
10162
|
+
if (waiter) {
|
|
10163
|
+
const run = await this.store.getRun(waiter.runId);
|
|
10164
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) {
|
|
10165
|
+
return {
|
|
10166
|
+
runId: start.runId,
|
|
10167
|
+
signalled: false
|
|
10168
|
+
};
|
|
10169
|
+
}
|
|
10170
|
+
} else if (!signal.token.includes(start.runId)) {
|
|
10171
|
+
return {
|
|
10172
|
+
runId: start.runId,
|
|
10173
|
+
signalled: false
|
|
10174
|
+
};
|
|
10175
|
+
}
|
|
10176
|
+
await this.signalUnchecked(signal.token, signal.payload);
|
|
10177
|
+
return {
|
|
10178
|
+
runId: start.runId,
|
|
10179
|
+
signalled: true
|
|
10180
|
+
};
|
|
10181
|
+
}
|
|
10182
|
+
/**
|
|
10183
|
+
* The control plane's half of a tenant's `publishEvent` (a `publishEvent` run request): resume the
|
|
10184
|
+
* runs of `namespace` waiting on event `name` whose match the payload satisfies. Starting the
|
|
10185
|
+
* tenant's own subscribers is the tenant's (it holds their `where` and `input`); nothing is
|
|
10186
|
+
* buffered, since a buffered event could reach another namespace's later wait. Returns how many
|
|
10187
|
+
* runs it resumed.
|
|
10188
|
+
*/
|
|
10189
|
+
async publishEventForTenant(name, payload, namespace) {
|
|
10190
|
+
let touched = 0;
|
|
10191
|
+
for (const w of await this.store.listSignalWaiters(eventPrefix(name))) {
|
|
10192
|
+
if (!eventMatches(payload, eventMatchOf(w.token))) continue;
|
|
10193
|
+
const run = await this.store.getRun(w.runId);
|
|
10194
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) continue;
|
|
10195
|
+
await this.signalUnchecked(w.token, payload);
|
|
10196
|
+
touched += 1;
|
|
10197
|
+
}
|
|
10198
|
+
return touched;
|
|
10199
|
+
}
|
|
9777
10200
|
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
9778
10201
|
async recordRejectedStart(name, input, runId, opts, err) {
|
|
9779
10202
|
const prior = await this.store.getRun(runId);
|
|
@@ -10245,9 +10668,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10245
10668
|
if (!liveTransport) return void 0;
|
|
10246
10669
|
const executor = new RemoteWorkflowExecutor(liveTransport, run.workflow, run.namespace);
|
|
10247
10670
|
const declared = opts.askFleetForVersion ? await this.declaredVersion(liveTransport, group, run.workflow) : void 0;
|
|
10671
|
+
const version = declared ?? run.workflowVersion;
|
|
10672
|
+
const announced = await this.announcedRegistration(group, run.workflow, version);
|
|
10673
|
+
const singleton = conventionSingleton(run, announced, opts.singleton);
|
|
10674
|
+
if (singleton?.limit !== void 0) {
|
|
10675
|
+
this.conventionSingletonLimits.set(`${runNamespace(run.namespace)}\0${run.workflow}`, singleton.limit);
|
|
10676
|
+
}
|
|
10677
|
+
const tags = announced?.options?.tags;
|
|
10248
10678
|
return {
|
|
10249
10679
|
name: run.workflow,
|
|
10250
|
-
version
|
|
10680
|
+
version,
|
|
10251
10681
|
fn: /* @__PURE__ */ __name(() => {
|
|
10252
10682
|
throw new Error(`remote workflow ${run.workflow} has no local body`);
|
|
10253
10683
|
}, "fn"),
|
|
@@ -10256,12 +10686,47 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10256
10686
|
group,
|
|
10257
10687
|
executor
|
|
10258
10688
|
},
|
|
10689
|
+
...Array.isArray(tags) && tags.length > 0 ? {
|
|
10690
|
+
tags
|
|
10691
|
+
} : {},
|
|
10692
|
+
...singleton ? {
|
|
10693
|
+
singleton
|
|
10694
|
+
} : {},
|
|
10259
10695
|
...opts.askFleetForVersion ? {
|
|
10260
10696
|
versionDeclared: declared !== void 0
|
|
10261
10697
|
} : {}
|
|
10262
10698
|
};
|
|
10263
10699
|
}
|
|
10264
10700
|
/**
|
|
10701
|
+
* The registration the live workers of `group` announce for `workflow` (at `version`, when one of
|
|
10702
|
+
* them states it): where they disagree (a rolling deploy), the one from the worker that started
|
|
10703
|
+
* last, as for announced schedules. `undefined` when none announces it, `null` when the read
|
|
10704
|
+
* failed (what the workers state is then unknown, not "nothing"). Reads
|
|
10705
|
+
* through the descriptor cache ({@link workerDescriptorCacheMs}), so a turn costs no extra round
|
|
10706
|
+
* trip in steady state.
|
|
10707
|
+
*/
|
|
10708
|
+
async announcedRegistration(group, workflow, version) {
|
|
10709
|
+
let descriptors;
|
|
10710
|
+
try {
|
|
10711
|
+
descriptors = await this.liveWorkerDescriptors(group);
|
|
10712
|
+
} catch {
|
|
10713
|
+
return null;
|
|
10714
|
+
}
|
|
10715
|
+
let best;
|
|
10716
|
+
for (const descriptor of descriptors) {
|
|
10717
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10718
|
+
if (reg?.name !== workflow) continue;
|
|
10719
|
+
if (version !== void 0 && reg.version !== void 0 && reg.version !== version) continue;
|
|
10720
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10721
|
+
if (!best || at > best.at) best = {
|
|
10722
|
+
at,
|
|
10723
|
+
reg
|
|
10724
|
+
};
|
|
10725
|
+
}
|
|
10726
|
+
}
|
|
10727
|
+
return best?.reg;
|
|
10728
|
+
}
|
|
10729
|
+
/**
|
|
10265
10730
|
* The version the LIVE fleet declares for `workflow` on `group`, or `undefined` when it declares
|
|
10266
10731
|
* none — which is a fact about the fleet, not a failure to read it.
|
|
10267
10732
|
*
|
|
@@ -10485,34 +10950,95 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10485
10950
|
for (const reg of new Set(this.latest.values())) {
|
|
10486
10951
|
if (this.draining) return;
|
|
10487
10952
|
if (reg.executionTimeoutMs == null) continue;
|
|
10488
|
-
|
|
10489
|
-
|
|
10490
|
-
|
|
10491
|
-
|
|
10492
|
-
|
|
10493
|
-
|
|
10494
|
-
|
|
10495
|
-
|
|
10496
|
-
|
|
10497
|
-
|
|
10498
|
-
|
|
10499
|
-
|
|
10500
|
-
|
|
10501
|
-
|
|
10502
|
-
|
|
10503
|
-
|
|
10504
|
-
|
|
10505
|
-
|
|
10506
|
-
|
|
10507
|
-
|
|
10508
|
-
|
|
10509
|
-
|
|
10510
|
-
|
|
10511
|
-
|
|
10512
|
-
|
|
10513
|
-
|
|
10514
|
-
|
|
10953
|
+
await this.cancelTimedOut(reg.name, this.namespace, now - reg.executionTimeoutMs);
|
|
10954
|
+
}
|
|
10955
|
+
for (const { workflow, namespace, timeoutMs } of await this.announcedTimeouts(now)) {
|
|
10956
|
+
if (this.draining) return;
|
|
10957
|
+
await this.cancelTimedOut(workflow, namespace, now - timeoutMs);
|
|
10958
|
+
}
|
|
10959
|
+
}
|
|
10960
|
+
/**
|
|
10961
|
+
* The singleton limit last resolved for a workflow served by convention, by `<namespace>\0<name>`
|
|
10962
|
+
* (see {@link resolveRemoteByConvention}): what the singleton gate's release wake-up reads, since
|
|
10963
|
+
* it cannot wait on the fleet. One entry per convention singleton the fleet serves.
|
|
10964
|
+
*/
|
|
10965
|
+
conventionSingletonLimits = /* @__PURE__ */ new Map();
|
|
10966
|
+
/** The last read of the announced execution timeouts (see {@link announcedTimeouts}). */
|
|
10967
|
+
announcedTimeoutsRead;
|
|
10968
|
+
/**
|
|
10969
|
+
* The `executionTimeoutMs` live workers announce (`@Workflow({ executionTimeout })` on a tenant),
|
|
10970
|
+
* per workflow and namespace, for workflows this engine does not register (a registration here
|
|
10971
|
+
* applies its own). Where workers disagree, the one that started last wins. Read at most once per
|
|
10972
|
+
* {@link ANNOUNCED_TIMEOUTS_TTL_MS}; empty without a transport that reads the fleet, or when the
|
|
10973
|
+
* read fails.
|
|
10974
|
+
*/
|
|
10975
|
+
async announcedTimeouts(now) {
|
|
10976
|
+
if (!this.pool.introspectsFleet) return [];
|
|
10977
|
+
const cached = this.announcedTimeoutsRead;
|
|
10978
|
+
if (cached && now - cached.at < ANNOUNCED_TIMEOUTS_TTL_MS) return cached.list;
|
|
10979
|
+
const newest = /* @__PURE__ */ new Map();
|
|
10980
|
+
try {
|
|
10981
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
10982
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10983
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10984
|
+
if (!reg || typeof reg.name !== "string" || this.latest.has(reg.name)) continue;
|
|
10985
|
+
const namespace = servedNamespace(descriptor.namespace, reg.group);
|
|
10986
|
+
if (this.namespace !== void 0 && runNamespace(this.namespace) !== namespace) continue;
|
|
10987
|
+
const key = `${namespace}\0${reg.name}`;
|
|
10988
|
+
const prior = newest.get(key);
|
|
10989
|
+
if (prior && prior.at >= at) continue;
|
|
10990
|
+
newest.set(key, {
|
|
10991
|
+
at,
|
|
10992
|
+
workflow: reg.name,
|
|
10993
|
+
namespace,
|
|
10994
|
+
ms: Number(reg.options?.executionTimeoutMs)
|
|
10995
|
+
});
|
|
10996
|
+
}
|
|
10515
10997
|
}
|
|
10998
|
+
} catch {
|
|
10999
|
+
return cached?.list ?? [];
|
|
11000
|
+
}
|
|
11001
|
+
const list2 = [
|
|
11002
|
+
...newest.values()
|
|
11003
|
+
].filter((e) => Number.isFinite(e.ms) && e.ms > 0).map((e) => ({
|
|
11004
|
+
workflow: e.workflow,
|
|
11005
|
+
namespace: e.namespace,
|
|
11006
|
+
timeoutMs: e.ms
|
|
11007
|
+
}));
|
|
11008
|
+
this.announcedTimeoutsRead = {
|
|
11009
|
+
at: now,
|
|
11010
|
+
list: list2
|
|
11011
|
+
};
|
|
11012
|
+
return list2;
|
|
11013
|
+
}
|
|
11014
|
+
/** Cancel (`execution_timeout`) the in-flight runs of `workflow` in `namespace` created before `deadline`. */
|
|
11015
|
+
async cancelTimedOut(workflow, namespace, deadline) {
|
|
11016
|
+
const inflight = await this.store.listRuns({
|
|
11017
|
+
workflow,
|
|
11018
|
+
statuses: TIMEOUT_SWEPT_STATUSES,
|
|
11019
|
+
namespace
|
|
11020
|
+
});
|
|
11021
|
+
for (const run of inflight) {
|
|
11022
|
+
if (this.draining) return;
|
|
11023
|
+
if (run.createdAt.getTime() > deadline) continue;
|
|
11024
|
+
const error2 = {
|
|
11025
|
+
message: "execution timeout",
|
|
11026
|
+
code: "execution_timeout"
|
|
11027
|
+
};
|
|
11028
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
11029
|
+
status: "cancelled",
|
|
11030
|
+
error: error2,
|
|
11031
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
11032
|
+
}));
|
|
11033
|
+
this.emit({
|
|
11034
|
+
type: "run.failed",
|
|
11035
|
+
runId: run.id,
|
|
11036
|
+
workflow: run.workflow,
|
|
11037
|
+
namespace: run.namespace,
|
|
11038
|
+
error: error2
|
|
11039
|
+
});
|
|
11040
|
+
await this.cancelChildren(run.id).catch(() => void 0);
|
|
11041
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
10516
11042
|
}
|
|
10517
11043
|
}
|
|
10518
11044
|
/**
|
|
@@ -12229,6 +12755,75 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
12229
12755
|
*
|
|
12230
12756
|
* Workflows only, and deliberately: see the "steps are out of scope" note in `handshake/announced`.
|
|
12231
12757
|
*/
|
|
12758
|
+
/**
|
|
12759
|
+
* Adopt the schedules tenant apps announce (their `schedules` module option, carried on their live
|
|
12760
|
+
* workers' descriptors) as persisted schedules, and withdraw the ones an app stopped announcing.
|
|
12761
|
+
* The control plane's timer poll calls it; {@link ScheduleClient.tick} fires them.
|
|
12762
|
+
*
|
|
12763
|
+
* Per app (the namespace its announcement's verified `tenant` maps to, as for a start-run): the
|
|
12764
|
+
* schedules announced by its live workers, where two workers disagree on a key the one that
|
|
12765
|
+
* started last (its deploy is the newer one) wins. An app with no live worker that states its
|
|
12766
|
+
* schedules is left alone, so its schedules keep firing into runs that wait for a worker. See
|
|
12767
|
+
* {@link ScheduleClient.syncAnnounced} for the reconciliation rule and `graceMs` (default 10
|
|
12768
|
+
* minutes). A namespaced engine adopts only its own namespace's. Nothing without a store that
|
|
12769
|
+
* persists schedules or a transport that reads worker descriptors.
|
|
12770
|
+
*/
|
|
12771
|
+
async syncAnnouncedSchedules(opts = {}) {
|
|
12772
|
+
const out = /* @__PURE__ */ new Map();
|
|
12773
|
+
if (!this.schedules.supported) return out;
|
|
12774
|
+
const byNamespace = /* @__PURE__ */ new Map();
|
|
12775
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
12776
|
+
const announced = descriptor.schedules;
|
|
12777
|
+
if (!announced || typeof announced.tenant !== "string" || !Array.isArray(announced.items)) {
|
|
12778
|
+
continue;
|
|
12779
|
+
}
|
|
12780
|
+
let verified;
|
|
12781
|
+
try {
|
|
12782
|
+
verified = await verifyWireTenant(announced.tenant, this.verifyTenant);
|
|
12783
|
+
} catch {
|
|
12784
|
+
verified = null;
|
|
12785
|
+
}
|
|
12786
|
+
const tenant = verified?.tenant ?? null;
|
|
12787
|
+
if (tenant === null) {
|
|
12788
|
+
if (!this.rejectedScheduleTenants.has(descriptor.instanceId)) {
|
|
12789
|
+
this.rejectedScheduleTenants.add(descriptor.instanceId);
|
|
12790
|
+
console.warn(`[nestjs-durable] ignored the schedules worker ${descriptor.instanceId} announces: invalid or unauthorized tenant token`);
|
|
12791
|
+
}
|
|
12792
|
+
continue;
|
|
12793
|
+
}
|
|
12794
|
+
const namespace = poolNamespace(tenant) ?? "default";
|
|
12795
|
+
if (this.namespace !== void 0 && namespace !== this.namespace) continue;
|
|
12796
|
+
const list2 = byNamespace.get(namespace) ?? [];
|
|
12797
|
+
list2.push({
|
|
12798
|
+
startedAt: Number(descriptor.startedAt) || 0,
|
|
12799
|
+
items: announced.items
|
|
12800
|
+
});
|
|
12801
|
+
byNamespace.set(namespace, list2);
|
|
12802
|
+
}
|
|
12803
|
+
for (const [namespace, announcers] of byNamespace) {
|
|
12804
|
+
const items = /* @__PURE__ */ new Map();
|
|
12805
|
+
for (const { items: list2 } of announcers.sort((a, b) => b.startedAt - a.startedAt)) {
|
|
12806
|
+
for (const item of list2) {
|
|
12807
|
+
if (item && typeof item.key === "string" && !items.has(item.key)) {
|
|
12808
|
+
items.set(item.key, item);
|
|
12809
|
+
}
|
|
12810
|
+
}
|
|
12811
|
+
}
|
|
12812
|
+
const result = await this.schedules.syncAnnounced(namespace, [
|
|
12813
|
+
...items.values()
|
|
12814
|
+
], {
|
|
12815
|
+
graceMs: opts.graceMs ?? DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
12816
|
+
...opts.nowMs !== void 0 ? {
|
|
12817
|
+
nowMs: opts.nowMs
|
|
12818
|
+
} : {}
|
|
12819
|
+
});
|
|
12820
|
+
for (const { key, error: error2 } of result.invalid) {
|
|
12821
|
+
console.warn(`[nestjs-durable] app "${namespace}" announces schedule "${key}", which cannot be adopted: ${error2}`);
|
|
12822
|
+
}
|
|
12823
|
+
out.set(namespace, result);
|
|
12824
|
+
}
|
|
12825
|
+
return out;
|
|
12826
|
+
}
|
|
12232
12827
|
async announcedWorkflows() {
|
|
12233
12828
|
return (await this.workflowDirectory()).workflows;
|
|
12234
12829
|
}
|
|
@@ -18092,6 +18687,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18092
18687
|
// Annotate the CommonJS export names for ESM import in node:
|
|
18093
18688
|
0 && (module.exports = {
|
|
18094
18689
|
ACCUMULATOR_SEEN_MAX,
|
|
18690
|
+
ANNOUNCED_SCHEDULE_TAG,
|
|
18095
18691
|
ANY_WAIT_GROUP,
|
|
18096
18692
|
CHILD_EDGE_PREFIXES,
|
|
18097
18693
|
CONTROL_FLOW_SIGNAL,
|
|
@@ -18100,6 +18696,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18100
18696
|
CodecStateStore,
|
|
18101
18697
|
ConcurrencyLimitError,
|
|
18102
18698
|
ContinueAsNew,
|
|
18699
|
+
DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
18103
18700
|
DEFAULT_CONCURRENCY_STATUSES,
|
|
18104
18701
|
DEFAULT_PARTITION,
|
|
18105
18702
|
DEFAULT_STALLED_AFTER_MS,
|
|
@@ -18181,6 +18778,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18181
18778
|
accumulatorRunId,
|
|
18182
18779
|
aggregateAnnouncements,
|
|
18183
18780
|
aggregatesCheckpoints,
|
|
18781
|
+
announcedScheduleId,
|
|
18184
18782
|
announcementsOf,
|
|
18185
18783
|
anyCheckpointName,
|
|
18186
18784
|
applyAttributeQuery,
|