@dudousxd/nestjs-durable-core 0.100.0 → 0.101.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +337 -11
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +152 -6
- package/dist/index.d.ts +152 -6
- package/dist/index.js +334 -11
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -23,6 +23,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
23
23
|
var index_exports = {};
|
|
24
24
|
__export(index_exports, {
|
|
25
25
|
ACCUMULATOR_SEEN_MAX: () => ACCUMULATOR_SEEN_MAX,
|
|
26
|
+
ANNOUNCED_SCHEDULE_TAG: () => ANNOUNCED_SCHEDULE_TAG,
|
|
26
27
|
ANY_WAIT_GROUP: () => ANY_WAIT_GROUP,
|
|
27
28
|
CHILD_EDGE_PREFIXES: () => CHILD_EDGE_PREFIXES,
|
|
28
29
|
CONTROL_FLOW_SIGNAL: () => CONTROL_FLOW_SIGNAL,
|
|
@@ -31,6 +32,7 @@ __export(index_exports, {
|
|
|
31
32
|
CodecStateStore: () => CodecStateStore,
|
|
32
33
|
ConcurrencyLimitError: () => ConcurrencyLimitError,
|
|
33
34
|
ContinueAsNew: () => ContinueAsNew,
|
|
35
|
+
DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS: () => DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
34
36
|
DEFAULT_CONCURRENCY_STATUSES: () => DEFAULT_CONCURRENCY_STATUSES,
|
|
35
37
|
DEFAULT_PARTITION: () => DEFAULT_PARTITION,
|
|
36
38
|
DEFAULT_STALLED_AFTER_MS: () => DEFAULT_STALLED_AFTER_MS,
|
|
@@ -112,6 +114,7 @@ __export(index_exports, {
|
|
|
112
114
|
accumulatorRunId: () => accumulatorRunId,
|
|
113
115
|
aggregateAnnouncements: () => aggregateAnnouncements,
|
|
114
116
|
aggregatesCheckpoints: () => aggregatesCheckpoints,
|
|
117
|
+
announcedScheduleId: () => announcedScheduleId,
|
|
115
118
|
announcementsOf: () => announcementsOf,
|
|
116
119
|
anyCheckpointName: () => anyCheckpointName,
|
|
117
120
|
applyAttributeQuery: () => applyAttributeQuery,
|
|
@@ -2606,6 +2609,9 @@ function normalizeDescriptor(raw) {
|
|
|
2606
2609
|
...raw.registrations !== void 0 ? {
|
|
2607
2610
|
registrations: raw.registrations
|
|
2608
2611
|
} : {},
|
|
2612
|
+
...raw.schedules !== void 0 ? {
|
|
2613
|
+
schedules: raw.schedules
|
|
2614
|
+
} : {},
|
|
2609
2615
|
...raw.partition !== void 0 ? {
|
|
2610
2616
|
partition: raw.partition
|
|
2611
2617
|
} : {},
|
|
@@ -2680,12 +2686,30 @@ function canonicalizeForHash(d) {
|
|
|
2680
2686
|
...d.registrations !== void 0 ? {
|
|
2681
2687
|
registrations: canonicalizeRegistrations(d.registrations)
|
|
2682
2688
|
} : {},
|
|
2689
|
+
// Conditional like `registrations`: a descriptor without schedules keeps its published hash.
|
|
2690
|
+
...d.schedules !== void 0 ? {
|
|
2691
|
+
schedules: canonicalSchedules(d.schedules)
|
|
2692
|
+
} : {},
|
|
2683
2693
|
partition: d.partition ?? null,
|
|
2684
2694
|
namespace: d.namespace ?? null,
|
|
2685
2695
|
startedAt: d.startedAt
|
|
2686
2696
|
};
|
|
2687
2697
|
}
|
|
2688
2698
|
__name(canonicalizeForHash, "canonicalizeForHash");
|
|
2699
|
+
function canonicalSchedules(a) {
|
|
2700
|
+
const items = [
|
|
2701
|
+
...a.items ?? []
|
|
2702
|
+
].sort((x, y) => x.key.localeCompare(y.key));
|
|
2703
|
+
return {
|
|
2704
|
+
tenant: a.tenant,
|
|
2705
|
+
items: items.map((i) => stableJson(i))
|
|
2706
|
+
};
|
|
2707
|
+
}
|
|
2708
|
+
__name(canonicalSchedules, "canonicalSchedules");
|
|
2709
|
+
function stableJson(value) {
|
|
2710
|
+
return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
|
|
2711
|
+
}
|
|
2712
|
+
__name(stableJson, "stableJson");
|
|
2689
2713
|
function fnv1a64Hex(input) {
|
|
2690
2714
|
const OFFSET = 0xcbf29ce484222325n;
|
|
2691
2715
|
const PRIME = 0x100000001b3n;
|
|
@@ -2743,6 +2767,14 @@ function describeWorker(opts) {
|
|
|
2743
2767
|
...registrations.length > 0 ? {
|
|
2744
2768
|
registrations
|
|
2745
2769
|
} : {},
|
|
2770
|
+
...opts.schedules !== void 0 ? {
|
|
2771
|
+
schedules: {
|
|
2772
|
+
tenant: opts.schedules.tenant,
|
|
2773
|
+
items: [
|
|
2774
|
+
...opts.schedules.items
|
|
2775
|
+
].sort((a, b) => a.key.localeCompare(b.key))
|
|
2776
|
+
}
|
|
2777
|
+
} : {},
|
|
2746
2778
|
...opts.partition ? {
|
|
2747
2779
|
partition: opts.partition
|
|
2748
2780
|
} : {},
|
|
@@ -4161,6 +4193,12 @@ function matchesTenantPredicates(row, query) {
|
|
|
4161
4193
|
__name(matchesTenantPredicates, "matchesTenantPredicates");
|
|
4162
4194
|
|
|
4163
4195
|
// src/schedules.ts
|
|
4196
|
+
var ANNOUNCED_SCHEDULE_TAG = "schedule:announced";
|
|
4197
|
+
function announcedScheduleId(namespace, key) {
|
|
4198
|
+
return `app:${namespace}:${key}`;
|
|
4199
|
+
}
|
|
4200
|
+
__name(announcedScheduleId, "announcedScheduleId");
|
|
4201
|
+
var PARKED_FOR_WORKER = "no_live_worker";
|
|
4164
4202
|
var ScheduleAlreadyExistsError = class extends Error {
|
|
4165
4203
|
static {
|
|
4166
4204
|
__name(this, "ScheduleAlreadyExistsError");
|
|
@@ -4301,15 +4339,28 @@ var ScheduleClient = class {
|
|
|
4301
4339
|
async trigger(id) {
|
|
4302
4340
|
const record = await this.mustGet(id);
|
|
4303
4341
|
const runId = `sched:${id}:manual:${this.now()}`;
|
|
4304
|
-
return this.
|
|
4342
|
+
return this.startFor(record, runId);
|
|
4305
4343
|
}
|
|
4306
4344
|
/**
|
|
4307
|
-
*
|
|
4345
|
+
* Start one run of `record`: through the tenant start path for an announced schedule (its run waits
|
|
4346
|
+
* `blocked` for a live worker of its app instead of failing), else a plain `start`.
|
|
4347
|
+
*/
|
|
4348
|
+
startFor(record, runId) {
|
|
4349
|
+
const input = specOf(record).input;
|
|
4350
|
+
const opts = startOptions(record);
|
|
4351
|
+
if (isAnnounced(record) && this.host.startParked) {
|
|
4352
|
+
return this.host.startParked(record.workflow, input, runId, opts);
|
|
4353
|
+
}
|
|
4354
|
+
return this.host.start(record.workflow, input, runId, opts);
|
|
4355
|
+
}
|
|
4356
|
+
/**
|
|
4357
|
+
* Fire every due schedule (in this engine's namespace; all namespaces for an operator engine; only
|
|
4358
|
+
* those carrying `opts.tag` when given):
|
|
4308
4359
|
* start its window's run and advance it to the next window. Called by the NestJS timer poller on
|
|
4309
4360
|
* each tick when `persistedSchedules` is on; call it yourself from any other driver. Returns the
|
|
4310
4361
|
* run ids started (or re-found, for an idempotent re-fire).
|
|
4311
4362
|
*/
|
|
4312
|
-
async tick(nowMs = this.now()) {
|
|
4363
|
+
async tick(nowMs = this.now(), opts = {}) {
|
|
4313
4364
|
const store = this.requireStore();
|
|
4314
4365
|
const started = [];
|
|
4315
4366
|
for (let round = 0; round < 50; round++) {
|
|
@@ -4317,6 +4368,9 @@ var ScheduleClient = class {
|
|
|
4317
4368
|
const due = await store.listSchedules({
|
|
4318
4369
|
dueBy: nowMs,
|
|
4319
4370
|
namespace: this.options.namespace,
|
|
4371
|
+
...opts.tag !== void 0 ? {
|
|
4372
|
+
tag: opts.tag
|
|
4373
|
+
} : {},
|
|
4320
4374
|
limit: TICK_BATCH
|
|
4321
4375
|
});
|
|
4322
4376
|
for (const record of due) {
|
|
@@ -4330,15 +4384,19 @@ var ScheduleClient = class {
|
|
|
4330
4384
|
}
|
|
4331
4385
|
/**
|
|
4332
4386
|
* When {@link tick} next has a schedule to fire, if one is due by `byMs`: the earliest `nextFireAt`
|
|
4333
|
-
* of the unpaused schedules (of this engine's namespace
|
|
4387
|
+
* of the unpaused schedules (of this engine's namespace, carrying `opts.tag` when given) due by
|
|
4388
|
+
* then, else `undefined`. One indexed
|
|
4334
4389
|
* read (the due-schedule scan `tick` makes, with a later bound and a limit of one). A store that
|
|
4335
4390
|
* does not persist schedules has none.
|
|
4336
4391
|
*/
|
|
4337
|
-
async nextFireAt(byMs) {
|
|
4392
|
+
async nextFireAt(byMs, opts = {}) {
|
|
4338
4393
|
if (!this.supported) return void 0;
|
|
4339
4394
|
const [next] = await this.requireStore().listSchedules({
|
|
4340
4395
|
dueBy: byMs,
|
|
4341
4396
|
namespace: this.options.namespace,
|
|
4397
|
+
...opts.tag !== void 0 ? {
|
|
4398
|
+
tag: opts.tag
|
|
4399
|
+
} : {},
|
|
4342
4400
|
limit: 1
|
|
4343
4401
|
});
|
|
4344
4402
|
return next?.nextFireAt ?? void 0;
|
|
@@ -4373,10 +4431,10 @@ var ScheduleClient = class {
|
|
|
4373
4431
|
let fired = false;
|
|
4374
4432
|
const skipMissed = spec.catchup === "skip" && !onTime;
|
|
4375
4433
|
const beforeWindow = latest < window;
|
|
4376
|
-
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state)) {
|
|
4434
|
+
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state) && !await this.waitingForWorker(record, state)) {
|
|
4377
4435
|
runId = `sched:${record.id}:${latest}`;
|
|
4378
4436
|
try {
|
|
4379
|
-
await this.
|
|
4437
|
+
await this.startFor(record, runId);
|
|
4380
4438
|
fired = true;
|
|
4381
4439
|
} catch (error2) {
|
|
4382
4440
|
if (await this.host.getRun(runId)) fired = true;
|
|
@@ -4400,6 +4458,182 @@ var ScheduleClient = class {
|
|
|
4400
4458
|
}, record.nextFireAt);
|
|
4401
4459
|
return fired ? runId : void 0;
|
|
4402
4460
|
}
|
|
4461
|
+
/**
|
|
4462
|
+
* An announced schedule whose previous run is still parked waiting for a live worker of its app
|
|
4463
|
+
* (the app is down): this window is folded into that run instead of parking one more. However long
|
|
4464
|
+
* the app is down, it finds at most one run per schedule waiting when it comes back, the same
|
|
4465
|
+
* bound `catchup: 'latest'` puts on windows missed while no scheduler polled.
|
|
4466
|
+
*/
|
|
4467
|
+
async waitingForWorker(record, state) {
|
|
4468
|
+
if (!isAnnounced(record) || !state.lastRunId) return false;
|
|
4469
|
+
const prev = await this.host.getRun(state.lastRunId);
|
|
4470
|
+
return prev?.status === "blocked" && prev.error?.code === PARKED_FOR_WORKER;
|
|
4471
|
+
}
|
|
4472
|
+
/**
|
|
4473
|
+
* Reconcile the schedules adopted from one app (`namespace`) with what its LIVE workers announce
|
|
4474
|
+
* now (`items`, one per key; the caller resolved disagreeing workers). The rule:
|
|
4475
|
+
* - an announced key with no schedule yet is created (id {@link announcedScheduleId}, tagged
|
|
4476
|
+
* {@link ANNOUNCED_SCHEDULE_TAG}, `catchup: 'latest'`); one whose definition changed is
|
|
4477
|
+
* re-applied like `upsert` (its history kept, its next fire recomputed from now). An equal
|
|
4478
|
+
* definition writes nothing, so any number of replicas announcing the same set is a no-op;
|
|
4479
|
+
* - a pause or resume made from the console survives: a re-applied definition keeps the current
|
|
4480
|
+
* paused state, unless the app's own declared `paused` changed;
|
|
4481
|
+
* - an adopted schedule the app no longer announces is marked missing, and paused ("withdrawn")
|
|
4482
|
+
* once it has stayed missing for `graceMs`. It is never deleted: its history stays, and an app
|
|
4483
|
+
* that announces it again (a rollback) gets it back, resumed unless an operator had paused it.
|
|
4484
|
+
*
|
|
4485
|
+
* Call it only for an app that has at least one live worker stating its schedules: with none (the
|
|
4486
|
+
* app is down, or runs an SDK that does not announce schedules), nothing is withdrawn, and its
|
|
4487
|
+
* schedules keep firing into runs that wait for a worker.
|
|
4488
|
+
*/
|
|
4489
|
+
async syncAnnounced(namespace, items, opts) {
|
|
4490
|
+
const store = this.requireStore();
|
|
4491
|
+
const now = opts.nowMs ?? this.now();
|
|
4492
|
+
const out = {
|
|
4493
|
+
adopted: [],
|
|
4494
|
+
restored: [],
|
|
4495
|
+
missing: [],
|
|
4496
|
+
withdrawn: [],
|
|
4497
|
+
invalid: []
|
|
4498
|
+
};
|
|
4499
|
+
const adopted = /* @__PURE__ */ new Map();
|
|
4500
|
+
for (let offset = 0; ; offset += TICK_BATCH) {
|
|
4501
|
+
const page = await store.listSchedules({
|
|
4502
|
+
namespace,
|
|
4503
|
+
tag: ANNOUNCED_SCHEDULE_TAG,
|
|
4504
|
+
limit: TICK_BATCH,
|
|
4505
|
+
offset
|
|
4506
|
+
});
|
|
4507
|
+
for (const record of page) if (isAnnounced(record)) adopted.set(record.id, record);
|
|
4508
|
+
if (page.length < TICK_BATCH) break;
|
|
4509
|
+
}
|
|
4510
|
+
for (const item of items) {
|
|
4511
|
+
const id = announcedScheduleId(namespace, item.key);
|
|
4512
|
+
const existing = adopted.get(id) ?? await store.getSchedule(id);
|
|
4513
|
+
adopted.delete(id);
|
|
4514
|
+
const doc = existing ? existing.state.announce : void 0;
|
|
4515
|
+
if (existing && !doc) {
|
|
4516
|
+
out.invalid.push({
|
|
4517
|
+
key: item.key,
|
|
4518
|
+
error: `schedule id "${id}" is taken`
|
|
4519
|
+
});
|
|
4520
|
+
continue;
|
|
4521
|
+
}
|
|
4522
|
+
const fingerprint = stableJson(item);
|
|
4523
|
+
if (existing && doc && doc.fingerprint === fingerprint) {
|
|
4524
|
+
if (doc.missingSince === void 0 && doc.withdrawnAt === void 0) continue;
|
|
4525
|
+
const resume = doc.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal && existing.paused;
|
|
4526
|
+
const spec = specOf(existing);
|
|
4527
|
+
const window = resume ? nextWindowAfter(spec, now) : void 0;
|
|
4528
|
+
await store.updateSchedule(id, {
|
|
4529
|
+
// The withdrawal note goes; the note it replaced comes back.
|
|
4530
|
+
...doc.withdrawnAt !== void 0 ? {
|
|
4531
|
+
spec: stripUndefined({
|
|
4532
|
+
...spec,
|
|
4533
|
+
note: doc.noteBeforeWithdrawal
|
|
4534
|
+
})
|
|
4535
|
+
} : {},
|
|
4536
|
+
...resume ? {
|
|
4537
|
+
paused: false,
|
|
4538
|
+
nextFireAt: window == null ? null : window + jitterFor(id, window, spec)
|
|
4539
|
+
} : {},
|
|
4540
|
+
state: stripUndefined({
|
|
4541
|
+
...existing.state,
|
|
4542
|
+
...resume ? {
|
|
4543
|
+
window
|
|
4544
|
+
} : {},
|
|
4545
|
+
announce: {
|
|
4546
|
+
key: doc.key,
|
|
4547
|
+
fingerprint,
|
|
4548
|
+
declaredPaused: doc.declaredPaused
|
|
4549
|
+
}
|
|
4550
|
+
}),
|
|
4551
|
+
updatedAt: new Date(now)
|
|
4552
|
+
});
|
|
4553
|
+
out.restored.push(id);
|
|
4554
|
+
continue;
|
|
4555
|
+
}
|
|
4556
|
+
const declaredPaused = item.paused === true;
|
|
4557
|
+
const paused = existing === null || existing === void 0 ? declaredPaused : declaredPaused !== (doc?.declaredPaused === true) ? declaredPaused : doc?.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal ? declaredPaused : void 0;
|
|
4558
|
+
try {
|
|
4559
|
+
await this.write({
|
|
4560
|
+
id,
|
|
4561
|
+
workflow: item.workflow,
|
|
4562
|
+
input: item.input,
|
|
4563
|
+
cron: item.cron,
|
|
4564
|
+
every: item.everyMs,
|
|
4565
|
+
timezone: item.timezone,
|
|
4566
|
+
jitter: item.jitterMs,
|
|
4567
|
+
overlap: item.overlap,
|
|
4568
|
+
catchup: "latest",
|
|
4569
|
+
namespace,
|
|
4570
|
+
tags: [
|
|
4571
|
+
ANNOUNCED_SCHEDULE_TAG
|
|
4572
|
+
],
|
|
4573
|
+
paused,
|
|
4574
|
+
// A note (an operator's pause reason) is kept; a withdrawal note is replaced by the one
|
|
4575
|
+
// it had replaced.
|
|
4576
|
+
note: doc?.withdrawnAt !== void 0 ? doc.noteBeforeWithdrawal : existing ? specOf(existing).note : void 0,
|
|
4577
|
+
// Fires belong to the app, never to whichever request context reached the sync.
|
|
4578
|
+
...existing ? {} : {
|
|
4579
|
+
tenantId: null
|
|
4580
|
+
}
|
|
4581
|
+
}, existing ?? null, {
|
|
4582
|
+
announce: {
|
|
4583
|
+
key: item.key,
|
|
4584
|
+
fingerprint,
|
|
4585
|
+
declaredPaused
|
|
4586
|
+
}
|
|
4587
|
+
});
|
|
4588
|
+
out.adopted.push(id);
|
|
4589
|
+
} catch (error2) {
|
|
4590
|
+
out.invalid.push({
|
|
4591
|
+
key: item.key,
|
|
4592
|
+
error: errorMessage(error2)
|
|
4593
|
+
});
|
|
4594
|
+
}
|
|
4595
|
+
}
|
|
4596
|
+
for (const record of adopted.values()) {
|
|
4597
|
+
const state = record.state;
|
|
4598
|
+
const doc = state.announce;
|
|
4599
|
+
if (doc.withdrawnAt !== void 0) continue;
|
|
4600
|
+
if (doc.missingSince === void 0) {
|
|
4601
|
+
await store.updateSchedule(record.id, {
|
|
4602
|
+
state: {
|
|
4603
|
+
...state,
|
|
4604
|
+
announce: {
|
|
4605
|
+
...doc,
|
|
4606
|
+
missingSince: now
|
|
4607
|
+
}
|
|
4608
|
+
},
|
|
4609
|
+
updatedAt: new Date(now)
|
|
4610
|
+
});
|
|
4611
|
+
out.missing.push(record.id);
|
|
4612
|
+
continue;
|
|
4613
|
+
}
|
|
4614
|
+
if (now - doc.missingSince < opts.graceMs) continue;
|
|
4615
|
+
const spec = {
|
|
4616
|
+
...specOf(record)
|
|
4617
|
+
};
|
|
4618
|
+
spec.note = `withdrawn: app "${namespace}" no longer announces schedule "${doc.key}"`;
|
|
4619
|
+
await store.updateSchedule(record.id, {
|
|
4620
|
+
paused: true,
|
|
4621
|
+
spec,
|
|
4622
|
+
state: {
|
|
4623
|
+
...state,
|
|
4624
|
+
announce: stripUndefined({
|
|
4625
|
+
...doc,
|
|
4626
|
+
withdrawnAt: now,
|
|
4627
|
+
pausedBeforeWithdrawal: record.paused,
|
|
4628
|
+
noteBeforeWithdrawal: specOf(record).note
|
|
4629
|
+
})
|
|
4630
|
+
},
|
|
4631
|
+
updatedAt: new Date(now)
|
|
4632
|
+
});
|
|
4633
|
+
out.withdrawn.push(record.id);
|
|
4634
|
+
}
|
|
4635
|
+
return out;
|
|
4636
|
+
}
|
|
4403
4637
|
/** `overlap: 'skip'` and the previous run of this schedule is still in flight. */
|
|
4404
4638
|
async overlapping(spec, state) {
|
|
4405
4639
|
if (spec.overlap !== "skip" || !state.lastRunId) return false;
|
|
@@ -4435,7 +4669,7 @@ var ScheduleClient = class {
|
|
|
4435
4669
|
tenantId: tenant.id
|
|
4436
4670
|
};
|
|
4437
4671
|
}
|
|
4438
|
-
async write(opts, existing) {
|
|
4672
|
+
async write(opts, existing, extraState = {}) {
|
|
4439
4673
|
const store = this.requireStore();
|
|
4440
4674
|
const spec = toSpec(opts);
|
|
4441
4675
|
const now = this.now();
|
|
@@ -4451,7 +4685,8 @@ var ScheduleClient = class {
|
|
|
4451
4685
|
spec,
|
|
4452
4686
|
state: stripUndefined({
|
|
4453
4687
|
...existing?.state ?? {},
|
|
4454
|
-
window
|
|
4688
|
+
window,
|
|
4689
|
+
...extraState
|
|
4455
4690
|
}),
|
|
4456
4691
|
...this.scheduleTenant(opts, existing),
|
|
4457
4692
|
createdAt: existing?.createdAt ?? new Date(now),
|
|
@@ -4497,6 +4732,10 @@ function specOf(record) {
|
|
|
4497
4732
|
return record.spec;
|
|
4498
4733
|
}
|
|
4499
4734
|
__name(specOf, "specOf");
|
|
4735
|
+
function isAnnounced(record) {
|
|
4736
|
+
return record.state?.announce !== void 0;
|
|
4737
|
+
}
|
|
4738
|
+
__name(isAnnounced, "isAnnounced");
|
|
4500
4739
|
function latestWindowAtOrBefore(spec, atMs) {
|
|
4501
4740
|
if (spec.cron != null) return prevCronFireMs(spec.cron, atMs, spec.timezone);
|
|
4502
4741
|
if (spec.rrule != null) {
|
|
@@ -4571,6 +4810,9 @@ function describe(record) {
|
|
|
4571
4810
|
lastError: state.lastError,
|
|
4572
4811
|
tenantId: record.tenantId ?? void 0,
|
|
4573
4812
|
tenantKind: record.tenantKind ?? void 0,
|
|
4813
|
+
announcedKey: state.announce?.key,
|
|
4814
|
+
unannouncedSince: state.announce?.missingSince != null ? new Date(state.announce.missingSince) : void 0,
|
|
4815
|
+
withdrawnAt: state.announce?.withdrawnAt != null ? new Date(state.announce.withdrawnAt) : void 0,
|
|
4574
4816
|
createdAt: record.createdAt,
|
|
4575
4817
|
updatedAt: record.updatedAt
|
|
4576
4818
|
});
|
|
@@ -6957,6 +7199,7 @@ function runNamespace(namespace) {
|
|
|
6957
7199
|
__name(runNamespace, "runNamespace");
|
|
6958
7200
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
6959
7201
|
var NO_LIVE_WORKER = "no_live_worker";
|
|
7202
|
+
var DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS = 10 * 6e4;
|
|
6960
7203
|
var TenantStartInvalid = class TenantStartInvalid2 extends Error {
|
|
6961
7204
|
static {
|
|
6962
7205
|
__name(this, "TenantStartInvalid");
|
|
@@ -7170,6 +7413,10 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7170
7413
|
remoteRedispatchMax;
|
|
7171
7414
|
/** How long a `blocked` run waits before the recovery poll re-checks the live fleet (design §7.5). */
|
|
7172
7415
|
blockedPollMs;
|
|
7416
|
+
/** Verifies the wire tenant of a start-run, and of a worker's schedule announcement. */
|
|
7417
|
+
verifyTenant;
|
|
7418
|
+
/** Wire tenants whose schedule announcement failed verification, warned about once. */
|
|
7419
|
+
rejectedScheduleTenants = /* @__PURE__ */ new Set();
|
|
7173
7420
|
workerDescriptorCacheMs;
|
|
7174
7421
|
descriptorCache = /* @__PURE__ */ new Map();
|
|
7175
7422
|
/** This engine's control-plane handshake descriptor — negotiated against each worker (design §7.3). */
|
|
@@ -7380,6 +7627,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7380
7627
|
this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
|
|
7381
7628
|
this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
|
|
7382
7629
|
this.blockedPollMs = deps.blockedPollMs ?? 5e3;
|
|
7630
|
+
this.verifyTenant = deps.verifyTenant;
|
|
7383
7631
|
this.workerDescriptorCacheMs = deps.workerDescriptorCacheMs ?? 2e3;
|
|
7384
7632
|
this.cpDescriptor = controlPlaneDescriptor({
|
|
7385
7633
|
instanceId: `cp-${this.instanceId}`,
|
|
@@ -7420,6 +7668,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7420
7668
|
});
|
|
7421
7669
|
this.schedules = new ScheduleClient(this.store, {
|
|
7422
7670
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7671
|
+
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7672
|
+
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7673
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true), "startParked"),
|
|
7423
7674
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7424
7675
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7425
7676
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -7593,14 +7844,17 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7593
7844
|
* The earliest known time (epoch ms) the poll has work, if it is at or before `byMs`: this
|
|
7594
7845
|
* namespace's next due timer (`listDueTimers` up to `byMs`, limit one: it returns the most overdue
|
|
7595
7846
|
* first), the next `blocked` run's recovery poll, and with `persistedSchedules` the next persisted
|
|
7596
|
-
* schedule to fire (see {@link ScheduleClient.nextFireAt}
|
|
7847
|
+
* schedule to fire (see {@link ScheduleClient.nextFireAt}; with `announcedSchedules` alone, the next
|
|
7848
|
+
* schedule adopted from a tenant). `undefined` when nothing is due by
|
|
7597
7849
|
* then. A poller reads it before waiting longer than its tick, so the wait never outlasts a timer.
|
|
7598
7850
|
*/
|
|
7599
7851
|
async nextDueAt(byMs, opts = {}) {
|
|
7600
7852
|
const [timers, blocked, schedule] = await Promise.all([
|
|
7601
7853
|
this.store.listDueTimers(byMs, this.namespace, 1),
|
|
7602
7854
|
this.dueBlockedRuns(byMs),
|
|
7603
|
-
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) :
|
|
7855
|
+
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) : opts.announcedSchedules && this.schedules.supported ? this.schedules.nextFireAt(byMs, {
|
|
7856
|
+
tag: ANNOUNCED_SCHEDULE_TAG
|
|
7857
|
+
}) : void 0
|
|
7604
7858
|
]);
|
|
7605
7859
|
let next;
|
|
7606
7860
|
for (const at of [
|
|
@@ -12229,6 +12483,75 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
12229
12483
|
*
|
|
12230
12484
|
* Workflows only, and deliberately: see the "steps are out of scope" note in `handshake/announced`.
|
|
12231
12485
|
*/
|
|
12486
|
+
/**
|
|
12487
|
+
* Adopt the schedules tenant apps announce (their `schedules` module option, carried on their live
|
|
12488
|
+
* workers' descriptors) as persisted schedules, and withdraw the ones an app stopped announcing.
|
|
12489
|
+
* The control plane's timer poll calls it; {@link ScheduleClient.tick} fires them.
|
|
12490
|
+
*
|
|
12491
|
+
* Per app (the namespace its announcement's verified `tenant` maps to, as for a start-run): the
|
|
12492
|
+
* schedules announced by its live workers, where two workers disagree on a key the one that
|
|
12493
|
+
* started last (its deploy is the newer one) wins. An app with no live worker that states its
|
|
12494
|
+
* schedules is left alone, so its schedules keep firing into runs that wait for a worker. See
|
|
12495
|
+
* {@link ScheduleClient.syncAnnounced} for the reconciliation rule and `graceMs` (default 10
|
|
12496
|
+
* minutes). A namespaced engine adopts only its own namespace's. Nothing without a store that
|
|
12497
|
+
* persists schedules or a transport that reads worker descriptors.
|
|
12498
|
+
*/
|
|
12499
|
+
async syncAnnouncedSchedules(opts = {}) {
|
|
12500
|
+
const out = /* @__PURE__ */ new Map();
|
|
12501
|
+
if (!this.schedules.supported) return out;
|
|
12502
|
+
const byNamespace = /* @__PURE__ */ new Map();
|
|
12503
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
12504
|
+
const announced = descriptor.schedules;
|
|
12505
|
+
if (!announced || typeof announced.tenant !== "string" || !Array.isArray(announced.items)) {
|
|
12506
|
+
continue;
|
|
12507
|
+
}
|
|
12508
|
+
let verified;
|
|
12509
|
+
try {
|
|
12510
|
+
verified = await verifyWireTenant(announced.tenant, this.verifyTenant);
|
|
12511
|
+
} catch {
|
|
12512
|
+
verified = null;
|
|
12513
|
+
}
|
|
12514
|
+
const tenant = verified?.tenant ?? null;
|
|
12515
|
+
if (tenant === null) {
|
|
12516
|
+
if (!this.rejectedScheduleTenants.has(descriptor.instanceId)) {
|
|
12517
|
+
this.rejectedScheduleTenants.add(descriptor.instanceId);
|
|
12518
|
+
console.warn(`[nestjs-durable] ignored the schedules worker ${descriptor.instanceId} announces: invalid or unauthorized tenant token`);
|
|
12519
|
+
}
|
|
12520
|
+
continue;
|
|
12521
|
+
}
|
|
12522
|
+
const namespace = poolNamespace(tenant) ?? "default";
|
|
12523
|
+
if (this.namespace !== void 0 && namespace !== this.namespace) continue;
|
|
12524
|
+
const list2 = byNamespace.get(namespace) ?? [];
|
|
12525
|
+
list2.push({
|
|
12526
|
+
startedAt: Number(descriptor.startedAt) || 0,
|
|
12527
|
+
items: announced.items
|
|
12528
|
+
});
|
|
12529
|
+
byNamespace.set(namespace, list2);
|
|
12530
|
+
}
|
|
12531
|
+
for (const [namespace, announcers] of byNamespace) {
|
|
12532
|
+
const items = /* @__PURE__ */ new Map();
|
|
12533
|
+
for (const { items: list2 } of announcers.sort((a, b) => b.startedAt - a.startedAt)) {
|
|
12534
|
+
for (const item of list2) {
|
|
12535
|
+
if (item && typeof item.key === "string" && !items.has(item.key)) {
|
|
12536
|
+
items.set(item.key, item);
|
|
12537
|
+
}
|
|
12538
|
+
}
|
|
12539
|
+
}
|
|
12540
|
+
const result = await this.schedules.syncAnnounced(namespace, [
|
|
12541
|
+
...items.values()
|
|
12542
|
+
], {
|
|
12543
|
+
graceMs: opts.graceMs ?? DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
12544
|
+
...opts.nowMs !== void 0 ? {
|
|
12545
|
+
nowMs: opts.nowMs
|
|
12546
|
+
} : {}
|
|
12547
|
+
});
|
|
12548
|
+
for (const { key, error: error2 } of result.invalid) {
|
|
12549
|
+
console.warn(`[nestjs-durable] app "${namespace}" announces schedule "${key}", which cannot be adopted: ${error2}`);
|
|
12550
|
+
}
|
|
12551
|
+
out.set(namespace, result);
|
|
12552
|
+
}
|
|
12553
|
+
return out;
|
|
12554
|
+
}
|
|
12232
12555
|
async announcedWorkflows() {
|
|
12233
12556
|
return (await this.workflowDirectory()).workflows;
|
|
12234
12557
|
}
|
|
@@ -18092,6 +18415,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18092
18415
|
// Annotate the CommonJS export names for ESM import in node:
|
|
18093
18416
|
0 && (module.exports = {
|
|
18094
18417
|
ACCUMULATOR_SEEN_MAX,
|
|
18418
|
+
ANNOUNCED_SCHEDULE_TAG,
|
|
18095
18419
|
ANY_WAIT_GROUP,
|
|
18096
18420
|
CHILD_EDGE_PREFIXES,
|
|
18097
18421
|
CONTROL_FLOW_SIGNAL,
|
|
@@ -18100,6 +18424,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18100
18424
|
CodecStateStore,
|
|
18101
18425
|
ConcurrencyLimitError,
|
|
18102
18426
|
ContinueAsNew,
|
|
18427
|
+
DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
18103
18428
|
DEFAULT_CONCURRENCY_STATUSES,
|
|
18104
18429
|
DEFAULT_PARTITION,
|
|
18105
18430
|
DEFAULT_STALLED_AFTER_MS,
|
|
@@ -18181,6 +18506,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
18181
18506
|
accumulatorRunId,
|
|
18182
18507
|
aggregateAnnouncements,
|
|
18183
18508
|
aggregatesCheckpoints,
|
|
18509
|
+
announcedScheduleId,
|
|
18184
18510
|
announcementsOf,
|
|
18185
18511
|
anyCheckpointName,
|
|
18186
18512
|
applyAttributeQuery,
|