@dudousxd/nestjs-durable-core 0.99.0 → 0.101.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +672 -26
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +317 -68
- package/dist/index.d.ts +317 -68
- package/dist/index.js +668 -26
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -23,6 +23,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
23
23
|
var index_exports = {};
|
|
24
24
|
__export(index_exports, {
|
|
25
25
|
ACCUMULATOR_SEEN_MAX: () => ACCUMULATOR_SEEN_MAX,
|
|
26
|
+
ANNOUNCED_SCHEDULE_TAG: () => ANNOUNCED_SCHEDULE_TAG,
|
|
26
27
|
ANY_WAIT_GROUP: () => ANY_WAIT_GROUP,
|
|
27
28
|
CHILD_EDGE_PREFIXES: () => CHILD_EDGE_PREFIXES,
|
|
28
29
|
CONTROL_FLOW_SIGNAL: () => CONTROL_FLOW_SIGNAL,
|
|
@@ -31,6 +32,7 @@ __export(index_exports, {
|
|
|
31
32
|
CodecStateStore: () => CodecStateStore,
|
|
32
33
|
ConcurrencyLimitError: () => ConcurrencyLimitError,
|
|
33
34
|
ContinueAsNew: () => ContinueAsNew,
|
|
35
|
+
DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS: () => DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
34
36
|
DEFAULT_CONCURRENCY_STATUSES: () => DEFAULT_CONCURRENCY_STATUSES,
|
|
35
37
|
DEFAULT_PARTITION: () => DEFAULT_PARTITION,
|
|
36
38
|
DEFAULT_STALLED_AFTER_MS: () => DEFAULT_STALLED_AFTER_MS,
|
|
@@ -69,6 +71,7 @@ __export(index_exports, {
|
|
|
69
71
|
LIFECYCLE_TRACE_TAG_PREFIX: () => LIFECYCLE_TRACE_TAG_PREFIX,
|
|
70
72
|
LIFECYCLE_TRIGGERED_TAG_PREFIX: () => LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
71
73
|
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX: () => LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
74
|
+
NO_LIVE_WORKER: () => NO_LIVE_WORKER,
|
|
72
75
|
NestedWorkflowCallError: () => NestedWorkflowCallError,
|
|
73
76
|
NonDeterminismError: () => NonDeterminismError,
|
|
74
77
|
PARENT_CLOSE_TAG_PREFIX: () => PARENT_CLOSE_TAG_PREFIX,
|
|
@@ -111,6 +114,7 @@ __export(index_exports, {
|
|
|
111
114
|
accumulatorRunId: () => accumulatorRunId,
|
|
112
115
|
aggregateAnnouncements: () => aggregateAnnouncements,
|
|
113
116
|
aggregatesCheckpoints: () => aggregatesCheckpoints,
|
|
117
|
+
announcedScheduleId: () => announcedScheduleId,
|
|
114
118
|
announcementsOf: () => announcementsOf,
|
|
115
119
|
anyCheckpointName: () => anyCheckpointName,
|
|
116
120
|
applyAttributeQuery: () => applyAttributeQuery,
|
|
@@ -768,7 +772,7 @@ __name(bindWorkflowClass, "bindWorkflowClass");
|
|
|
768
772
|
function clientOf(ctor) {
|
|
769
773
|
const client = bindings.get(ctor);
|
|
770
774
|
if (!client) {
|
|
771
|
-
throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow
|
|
775
|
+
throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow (an operator, or a tenant's DurableModule) has booted.`);
|
|
772
776
|
}
|
|
773
777
|
return client;
|
|
774
778
|
}
|
|
@@ -2605,6 +2609,9 @@ function normalizeDescriptor(raw) {
|
|
|
2605
2609
|
...raw.registrations !== void 0 ? {
|
|
2606
2610
|
registrations: raw.registrations
|
|
2607
2611
|
} : {},
|
|
2612
|
+
...raw.schedules !== void 0 ? {
|
|
2613
|
+
schedules: raw.schedules
|
|
2614
|
+
} : {},
|
|
2608
2615
|
...raw.partition !== void 0 ? {
|
|
2609
2616
|
partition: raw.partition
|
|
2610
2617
|
} : {},
|
|
@@ -2679,12 +2686,30 @@ function canonicalizeForHash(d) {
|
|
|
2679
2686
|
...d.registrations !== void 0 ? {
|
|
2680
2687
|
registrations: canonicalizeRegistrations(d.registrations)
|
|
2681
2688
|
} : {},
|
|
2689
|
+
// Conditional like `registrations`: a descriptor without schedules keeps its published hash.
|
|
2690
|
+
...d.schedules !== void 0 ? {
|
|
2691
|
+
schedules: canonicalSchedules(d.schedules)
|
|
2692
|
+
} : {},
|
|
2682
2693
|
partition: d.partition ?? null,
|
|
2683
2694
|
namespace: d.namespace ?? null,
|
|
2684
2695
|
startedAt: d.startedAt
|
|
2685
2696
|
};
|
|
2686
2697
|
}
|
|
2687
2698
|
__name(canonicalizeForHash, "canonicalizeForHash");
|
|
2699
|
+
function canonicalSchedules(a) {
|
|
2700
|
+
const items = [
|
|
2701
|
+
...a.items ?? []
|
|
2702
|
+
].sort((x, y) => x.key.localeCompare(y.key));
|
|
2703
|
+
return {
|
|
2704
|
+
tenant: a.tenant,
|
|
2705
|
+
items: items.map((i) => stableJson(i))
|
|
2706
|
+
};
|
|
2707
|
+
}
|
|
2708
|
+
__name(canonicalSchedules, "canonicalSchedules");
|
|
2709
|
+
function stableJson(value) {
|
|
2710
|
+
return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
|
|
2711
|
+
}
|
|
2712
|
+
__name(stableJson, "stableJson");
|
|
2688
2713
|
function fnv1a64Hex(input) {
|
|
2689
2714
|
const OFFSET = 0xcbf29ce484222325n;
|
|
2690
2715
|
const PRIME = 0x100000001b3n;
|
|
@@ -2742,6 +2767,14 @@ function describeWorker(opts) {
|
|
|
2742
2767
|
...registrations.length > 0 ? {
|
|
2743
2768
|
registrations
|
|
2744
2769
|
} : {},
|
|
2770
|
+
...opts.schedules !== void 0 ? {
|
|
2771
|
+
schedules: {
|
|
2772
|
+
tenant: opts.schedules.tenant,
|
|
2773
|
+
items: [
|
|
2774
|
+
...opts.schedules.items
|
|
2775
|
+
].sort((a, b) => a.key.localeCompare(b.key))
|
|
2776
|
+
}
|
|
2777
|
+
} : {},
|
|
2745
2778
|
...opts.partition ? {
|
|
2746
2779
|
partition: opts.partition
|
|
2747
2780
|
} : {},
|
|
@@ -4160,6 +4193,12 @@ function matchesTenantPredicates(row, query) {
|
|
|
4160
4193
|
__name(matchesTenantPredicates, "matchesTenantPredicates");
|
|
4161
4194
|
|
|
4162
4195
|
// src/schedules.ts
|
|
4196
|
+
var ANNOUNCED_SCHEDULE_TAG = "schedule:announced";
|
|
4197
|
+
function announcedScheduleId(namespace, key) {
|
|
4198
|
+
return `app:${namespace}:${key}`;
|
|
4199
|
+
}
|
|
4200
|
+
__name(announcedScheduleId, "announcedScheduleId");
|
|
4201
|
+
var PARKED_FOR_WORKER = "no_live_worker";
|
|
4163
4202
|
var ScheduleAlreadyExistsError = class extends Error {
|
|
4164
4203
|
static {
|
|
4165
4204
|
__name(this, "ScheduleAlreadyExistsError");
|
|
@@ -4300,15 +4339,28 @@ var ScheduleClient = class {
|
|
|
4300
4339
|
async trigger(id) {
|
|
4301
4340
|
const record = await this.mustGet(id);
|
|
4302
4341
|
const runId = `sched:${id}:manual:${this.now()}`;
|
|
4303
|
-
return this.
|
|
4342
|
+
return this.startFor(record, runId);
|
|
4343
|
+
}
|
|
4344
|
+
/**
|
|
4345
|
+
* Start one run of `record`: through the tenant start path for an announced schedule (its run waits
|
|
4346
|
+
* `blocked` for a live worker of its app instead of failing), else a plain `start`.
|
|
4347
|
+
*/
|
|
4348
|
+
startFor(record, runId) {
|
|
4349
|
+
const input = specOf(record).input;
|
|
4350
|
+
const opts = startOptions(record);
|
|
4351
|
+
if (isAnnounced(record) && this.host.startParked) {
|
|
4352
|
+
return this.host.startParked(record.workflow, input, runId, opts);
|
|
4353
|
+
}
|
|
4354
|
+
return this.host.start(record.workflow, input, runId, opts);
|
|
4304
4355
|
}
|
|
4305
4356
|
/**
|
|
4306
|
-
* Fire every due schedule (in this engine's namespace; all namespaces for an operator engine
|
|
4357
|
+
* Fire every due schedule (in this engine's namespace; all namespaces for an operator engine; only
|
|
4358
|
+
* those carrying `opts.tag` when given):
|
|
4307
4359
|
* start its window's run and advance it to the next window. Called by the NestJS timer poller on
|
|
4308
4360
|
* each tick when `persistedSchedules` is on; call it yourself from any other driver. Returns the
|
|
4309
4361
|
* run ids started (or re-found, for an idempotent re-fire).
|
|
4310
4362
|
*/
|
|
4311
|
-
async tick(nowMs = this.now()) {
|
|
4363
|
+
async tick(nowMs = this.now(), opts = {}) {
|
|
4312
4364
|
const store = this.requireStore();
|
|
4313
4365
|
const started = [];
|
|
4314
4366
|
for (let round = 0; round < 50; round++) {
|
|
@@ -4316,6 +4368,9 @@ var ScheduleClient = class {
|
|
|
4316
4368
|
const due = await store.listSchedules({
|
|
4317
4369
|
dueBy: nowMs,
|
|
4318
4370
|
namespace: this.options.namespace,
|
|
4371
|
+
...opts.tag !== void 0 ? {
|
|
4372
|
+
tag: opts.tag
|
|
4373
|
+
} : {},
|
|
4319
4374
|
limit: TICK_BATCH
|
|
4320
4375
|
});
|
|
4321
4376
|
for (const record of due) {
|
|
@@ -4329,15 +4384,19 @@ var ScheduleClient = class {
|
|
|
4329
4384
|
}
|
|
4330
4385
|
/**
|
|
4331
4386
|
* When {@link tick} next has a schedule to fire, if one is due by `byMs`: the earliest `nextFireAt`
|
|
4332
|
-
* of the unpaused schedules (of this engine's namespace
|
|
4387
|
+
* of the unpaused schedules (of this engine's namespace, carrying `opts.tag` when given) due by
|
|
4388
|
+
* then, else `undefined`. One indexed
|
|
4333
4389
|
* read (the due-schedule scan `tick` makes, with a later bound and a limit of one). A store that
|
|
4334
4390
|
* does not persist schedules has none.
|
|
4335
4391
|
*/
|
|
4336
|
-
async nextFireAt(byMs) {
|
|
4392
|
+
async nextFireAt(byMs, opts = {}) {
|
|
4337
4393
|
if (!this.supported) return void 0;
|
|
4338
4394
|
const [next] = await this.requireStore().listSchedules({
|
|
4339
4395
|
dueBy: byMs,
|
|
4340
4396
|
namespace: this.options.namespace,
|
|
4397
|
+
...opts.tag !== void 0 ? {
|
|
4398
|
+
tag: opts.tag
|
|
4399
|
+
} : {},
|
|
4341
4400
|
limit: 1
|
|
4342
4401
|
});
|
|
4343
4402
|
return next?.nextFireAt ?? void 0;
|
|
@@ -4372,10 +4431,10 @@ var ScheduleClient = class {
|
|
|
4372
4431
|
let fired = false;
|
|
4373
4432
|
const skipMissed = spec.catchup === "skip" && !onTime;
|
|
4374
4433
|
const beforeWindow = latest < window;
|
|
4375
|
-
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state)) {
|
|
4434
|
+
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state) && !await this.waitingForWorker(record, state)) {
|
|
4376
4435
|
runId = `sched:${record.id}:${latest}`;
|
|
4377
4436
|
try {
|
|
4378
|
-
await this.
|
|
4437
|
+
await this.startFor(record, runId);
|
|
4379
4438
|
fired = true;
|
|
4380
4439
|
} catch (error2) {
|
|
4381
4440
|
if (await this.host.getRun(runId)) fired = true;
|
|
@@ -4399,6 +4458,182 @@ var ScheduleClient = class {
|
|
|
4399
4458
|
}, record.nextFireAt);
|
|
4400
4459
|
return fired ? runId : void 0;
|
|
4401
4460
|
}
|
|
4461
|
+
/**
|
|
4462
|
+
* An announced schedule whose previous run is still parked waiting for a live worker of its app
|
|
4463
|
+
* (the app is down): this window is folded into that run instead of parking one more. However long
|
|
4464
|
+
* the app is down, it finds at most one run per schedule waiting when it comes back, the same
|
|
4465
|
+
* bound `catchup: 'latest'` puts on windows missed while no scheduler polled.
|
|
4466
|
+
*/
|
|
4467
|
+
async waitingForWorker(record, state) {
|
|
4468
|
+
if (!isAnnounced(record) || !state.lastRunId) return false;
|
|
4469
|
+
const prev = await this.host.getRun(state.lastRunId);
|
|
4470
|
+
return prev?.status === "blocked" && prev.error?.code === PARKED_FOR_WORKER;
|
|
4471
|
+
}
|
|
4472
|
+
/**
|
|
4473
|
+
* Reconcile the schedules adopted from one app (`namespace`) with what its LIVE workers announce
|
|
4474
|
+
* now (`items`, one per key; the caller resolved disagreeing workers). The rule:
|
|
4475
|
+
* - an announced key with no schedule yet is created (id {@link announcedScheduleId}, tagged
|
|
4476
|
+
* {@link ANNOUNCED_SCHEDULE_TAG}, `catchup: 'latest'`); one whose definition changed is
|
|
4477
|
+
* re-applied like `upsert` (its history kept, its next fire recomputed from now). An equal
|
|
4478
|
+
* definition writes nothing, so any number of replicas announcing the same set is a no-op;
|
|
4479
|
+
* - a pause or resume made from the console survives: a re-applied definition keeps the current
|
|
4480
|
+
* paused state, unless the app's own declared `paused` changed;
|
|
4481
|
+
* - an adopted schedule the app no longer announces is marked missing, and paused ("withdrawn")
|
|
4482
|
+
* once it has stayed missing for `graceMs`. It is never deleted: its history stays, and an app
|
|
4483
|
+
* that announces it again (a rollback) gets it back, resumed unless an operator had paused it.
|
|
4484
|
+
*
|
|
4485
|
+
* Call it only for an app that has at least one live worker stating its schedules: with none (the
|
|
4486
|
+
* app is down, or runs an SDK that does not announce schedules), nothing is withdrawn, and its
|
|
4487
|
+
* schedules keep firing into runs that wait for a worker.
|
|
4488
|
+
*/
|
|
4489
|
+
async syncAnnounced(namespace, items, opts) {
|
|
4490
|
+
const store = this.requireStore();
|
|
4491
|
+
const now = opts.nowMs ?? this.now();
|
|
4492
|
+
const out = {
|
|
4493
|
+
adopted: [],
|
|
4494
|
+
restored: [],
|
|
4495
|
+
missing: [],
|
|
4496
|
+
withdrawn: [],
|
|
4497
|
+
invalid: []
|
|
4498
|
+
};
|
|
4499
|
+
const adopted = /* @__PURE__ */ new Map();
|
|
4500
|
+
for (let offset = 0; ; offset += TICK_BATCH) {
|
|
4501
|
+
const page = await store.listSchedules({
|
|
4502
|
+
namespace,
|
|
4503
|
+
tag: ANNOUNCED_SCHEDULE_TAG,
|
|
4504
|
+
limit: TICK_BATCH,
|
|
4505
|
+
offset
|
|
4506
|
+
});
|
|
4507
|
+
for (const record of page) if (isAnnounced(record)) adopted.set(record.id, record);
|
|
4508
|
+
if (page.length < TICK_BATCH) break;
|
|
4509
|
+
}
|
|
4510
|
+
for (const item of items) {
|
|
4511
|
+
const id = announcedScheduleId(namespace, item.key);
|
|
4512
|
+
const existing = adopted.get(id) ?? await store.getSchedule(id);
|
|
4513
|
+
adopted.delete(id);
|
|
4514
|
+
const doc = existing ? existing.state.announce : void 0;
|
|
4515
|
+
if (existing && !doc) {
|
|
4516
|
+
out.invalid.push({
|
|
4517
|
+
key: item.key,
|
|
4518
|
+
error: `schedule id "${id}" is taken`
|
|
4519
|
+
});
|
|
4520
|
+
continue;
|
|
4521
|
+
}
|
|
4522
|
+
const fingerprint = stableJson(item);
|
|
4523
|
+
if (existing && doc && doc.fingerprint === fingerprint) {
|
|
4524
|
+
if (doc.missingSince === void 0 && doc.withdrawnAt === void 0) continue;
|
|
4525
|
+
const resume = doc.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal && existing.paused;
|
|
4526
|
+
const spec = specOf(existing);
|
|
4527
|
+
const window = resume ? nextWindowAfter(spec, now) : void 0;
|
|
4528
|
+
await store.updateSchedule(id, {
|
|
4529
|
+
// The withdrawal note goes; the note it replaced comes back.
|
|
4530
|
+
...doc.withdrawnAt !== void 0 ? {
|
|
4531
|
+
spec: stripUndefined({
|
|
4532
|
+
...spec,
|
|
4533
|
+
note: doc.noteBeforeWithdrawal
|
|
4534
|
+
})
|
|
4535
|
+
} : {},
|
|
4536
|
+
...resume ? {
|
|
4537
|
+
paused: false,
|
|
4538
|
+
nextFireAt: window == null ? null : window + jitterFor(id, window, spec)
|
|
4539
|
+
} : {},
|
|
4540
|
+
state: stripUndefined({
|
|
4541
|
+
...existing.state,
|
|
4542
|
+
...resume ? {
|
|
4543
|
+
window
|
|
4544
|
+
} : {},
|
|
4545
|
+
announce: {
|
|
4546
|
+
key: doc.key,
|
|
4547
|
+
fingerprint,
|
|
4548
|
+
declaredPaused: doc.declaredPaused
|
|
4549
|
+
}
|
|
4550
|
+
}),
|
|
4551
|
+
updatedAt: new Date(now)
|
|
4552
|
+
});
|
|
4553
|
+
out.restored.push(id);
|
|
4554
|
+
continue;
|
|
4555
|
+
}
|
|
4556
|
+
const declaredPaused = item.paused === true;
|
|
4557
|
+
const paused = existing === null || existing === void 0 ? declaredPaused : declaredPaused !== (doc?.declaredPaused === true) ? declaredPaused : doc?.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal ? declaredPaused : void 0;
|
|
4558
|
+
try {
|
|
4559
|
+
await this.write({
|
|
4560
|
+
id,
|
|
4561
|
+
workflow: item.workflow,
|
|
4562
|
+
input: item.input,
|
|
4563
|
+
cron: item.cron,
|
|
4564
|
+
every: item.everyMs,
|
|
4565
|
+
timezone: item.timezone,
|
|
4566
|
+
jitter: item.jitterMs,
|
|
4567
|
+
overlap: item.overlap,
|
|
4568
|
+
catchup: "latest",
|
|
4569
|
+
namespace,
|
|
4570
|
+
tags: [
|
|
4571
|
+
ANNOUNCED_SCHEDULE_TAG
|
|
4572
|
+
],
|
|
4573
|
+
paused,
|
|
4574
|
+
// A note (an operator's pause reason) is kept; a withdrawal note is replaced by the one
|
|
4575
|
+
// it had replaced.
|
|
4576
|
+
note: doc?.withdrawnAt !== void 0 ? doc.noteBeforeWithdrawal : existing ? specOf(existing).note : void 0,
|
|
4577
|
+
// Fires belong to the app, never to whichever request context reached the sync.
|
|
4578
|
+
...existing ? {} : {
|
|
4579
|
+
tenantId: null
|
|
4580
|
+
}
|
|
4581
|
+
}, existing ?? null, {
|
|
4582
|
+
announce: {
|
|
4583
|
+
key: item.key,
|
|
4584
|
+
fingerprint,
|
|
4585
|
+
declaredPaused
|
|
4586
|
+
}
|
|
4587
|
+
});
|
|
4588
|
+
out.adopted.push(id);
|
|
4589
|
+
} catch (error2) {
|
|
4590
|
+
out.invalid.push({
|
|
4591
|
+
key: item.key,
|
|
4592
|
+
error: errorMessage(error2)
|
|
4593
|
+
});
|
|
4594
|
+
}
|
|
4595
|
+
}
|
|
4596
|
+
for (const record of adopted.values()) {
|
|
4597
|
+
const state = record.state;
|
|
4598
|
+
const doc = state.announce;
|
|
4599
|
+
if (doc.withdrawnAt !== void 0) continue;
|
|
4600
|
+
if (doc.missingSince === void 0) {
|
|
4601
|
+
await store.updateSchedule(record.id, {
|
|
4602
|
+
state: {
|
|
4603
|
+
...state,
|
|
4604
|
+
announce: {
|
|
4605
|
+
...doc,
|
|
4606
|
+
missingSince: now
|
|
4607
|
+
}
|
|
4608
|
+
},
|
|
4609
|
+
updatedAt: new Date(now)
|
|
4610
|
+
});
|
|
4611
|
+
out.missing.push(record.id);
|
|
4612
|
+
continue;
|
|
4613
|
+
}
|
|
4614
|
+
if (now - doc.missingSince < opts.graceMs) continue;
|
|
4615
|
+
const spec = {
|
|
4616
|
+
...specOf(record)
|
|
4617
|
+
};
|
|
4618
|
+
spec.note = `withdrawn: app "${namespace}" no longer announces schedule "${doc.key}"`;
|
|
4619
|
+
await store.updateSchedule(record.id, {
|
|
4620
|
+
paused: true,
|
|
4621
|
+
spec,
|
|
4622
|
+
state: {
|
|
4623
|
+
...state,
|
|
4624
|
+
announce: stripUndefined({
|
|
4625
|
+
...doc,
|
|
4626
|
+
withdrawnAt: now,
|
|
4627
|
+
pausedBeforeWithdrawal: record.paused,
|
|
4628
|
+
noteBeforeWithdrawal: specOf(record).note
|
|
4629
|
+
})
|
|
4630
|
+
},
|
|
4631
|
+
updatedAt: new Date(now)
|
|
4632
|
+
});
|
|
4633
|
+
out.withdrawn.push(record.id);
|
|
4634
|
+
}
|
|
4635
|
+
return out;
|
|
4636
|
+
}
|
|
4402
4637
|
/** `overlap: 'skip'` and the previous run of this schedule is still in flight. */
|
|
4403
4638
|
async overlapping(spec, state) {
|
|
4404
4639
|
if (spec.overlap !== "skip" || !state.lastRunId) return false;
|
|
@@ -4434,7 +4669,7 @@ var ScheduleClient = class {
|
|
|
4434
4669
|
tenantId: tenant.id
|
|
4435
4670
|
};
|
|
4436
4671
|
}
|
|
4437
|
-
async write(opts, existing) {
|
|
4672
|
+
async write(opts, existing, extraState = {}) {
|
|
4438
4673
|
const store = this.requireStore();
|
|
4439
4674
|
const spec = toSpec(opts);
|
|
4440
4675
|
const now = this.now();
|
|
@@ -4450,7 +4685,8 @@ var ScheduleClient = class {
|
|
|
4450
4685
|
spec,
|
|
4451
4686
|
state: stripUndefined({
|
|
4452
4687
|
...existing?.state ?? {},
|
|
4453
|
-
window
|
|
4688
|
+
window,
|
|
4689
|
+
...extraState
|
|
4454
4690
|
}),
|
|
4455
4691
|
...this.scheduleTenant(opts, existing),
|
|
4456
4692
|
createdAt: existing?.createdAt ?? new Date(now),
|
|
@@ -4496,6 +4732,10 @@ function specOf(record) {
|
|
|
4496
4732
|
return record.spec;
|
|
4497
4733
|
}
|
|
4498
4734
|
__name(specOf, "specOf");
|
|
4735
|
+
function isAnnounced(record) {
|
|
4736
|
+
return record.state?.announce !== void 0;
|
|
4737
|
+
}
|
|
4738
|
+
__name(isAnnounced, "isAnnounced");
|
|
4499
4739
|
function latestWindowAtOrBefore(spec, atMs) {
|
|
4500
4740
|
if (spec.cron != null) return prevCronFireMs(spec.cron, atMs, spec.timezone);
|
|
4501
4741
|
if (spec.rrule != null) {
|
|
@@ -4570,6 +4810,9 @@ function describe(record) {
|
|
|
4570
4810
|
lastError: state.lastError,
|
|
4571
4811
|
tenantId: record.tenantId ?? void 0,
|
|
4572
4812
|
tenantKind: record.tenantKind ?? void 0,
|
|
4813
|
+
announcedKey: state.announce?.key,
|
|
4814
|
+
unannouncedSince: state.announce?.missingSince != null ? new Date(state.announce.missingSince) : void 0,
|
|
4815
|
+
withdrawnAt: state.announce?.withdrawnAt != null ? new Date(state.announce.withdrawnAt) : void 0,
|
|
4573
4816
|
createdAt: record.createdAt,
|
|
4574
4817
|
updatedAt: record.updatedAt
|
|
4575
4818
|
});
|
|
@@ -6955,6 +7198,20 @@ function runNamespace(namespace) {
|
|
|
6955
7198
|
}
|
|
6956
7199
|
__name(runNamespace, "runNamespace");
|
|
6957
7200
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
7201
|
+
var NO_LIVE_WORKER = "no_live_worker";
|
|
7202
|
+
var DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS = 10 * 6e4;
|
|
7203
|
+
var TenantStartInvalid = class TenantStartInvalid2 extends Error {
|
|
7204
|
+
static {
|
|
7205
|
+
__name(this, "TenantStartInvalid");
|
|
7206
|
+
}
|
|
7207
|
+
constructor(cause) {
|
|
7208
|
+
super(cause instanceof Error ? cause.message : String(cause), {
|
|
7209
|
+
cause
|
|
7210
|
+
});
|
|
7211
|
+
this.name = "TenantStartInvalid";
|
|
7212
|
+
}
|
|
7213
|
+
};
|
|
7214
|
+
var NO_LIVE_WORKER_MAX_POLL_MS = 6e4;
|
|
6958
7215
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
6959
7216
|
var STEP_FAN_CHUNK = 200;
|
|
6960
7217
|
var STEP_EVENTS_PER_BROADCAST = 500;
|
|
@@ -7127,6 +7384,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7127
7384
|
maxRecoveryAttempts;
|
|
7128
7385
|
unregisteredWorkflowGraceMs;
|
|
7129
7386
|
remoteAdvanceSilenceMs;
|
|
7387
|
+
/**
|
|
7388
|
+
* The re-drive window of a DISPATCHED workflow turn (a remote worker's `advance` sent over the
|
|
7389
|
+
* broker): {@link WorkflowEngineDeps.remoteAdvanceSilenceMs} when set, else `reconcileMs`. Without
|
|
7390
|
+
* it a run suspended on a turn whose worker died (and whose job was lost with it) had no `wakeAt`
|
|
7391
|
+
* and nothing re-drove it, ever. Heartbeats rearm it like the explicit window.
|
|
7392
|
+
*/
|
|
7393
|
+
dispatchedTurnSilenceMs;
|
|
7130
7394
|
reconcileMs;
|
|
7131
7395
|
/** Parked workflow bodies kept between turns (see {@link WorkflowEngineDeps.stickyRuns}). */
|
|
7132
7396
|
sticky;
|
|
@@ -7149,6 +7413,10 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7149
7413
|
remoteRedispatchMax;
|
|
7150
7414
|
/** How long a `blocked` run waits before the recovery poll re-checks the live fleet (design §7.5). */
|
|
7151
7415
|
blockedPollMs;
|
|
7416
|
+
/** Verifies the wire tenant of a start-run, and of a worker's schedule announcement. */
|
|
7417
|
+
verifyTenant;
|
|
7418
|
+
/** Wire tenants whose schedule announcement failed verification, warned about once. */
|
|
7419
|
+
rejectedScheduleTenants = /* @__PURE__ */ new Set();
|
|
7152
7420
|
workerDescriptorCacheMs;
|
|
7153
7421
|
descriptorCache = /* @__PURE__ */ new Map();
|
|
7154
7422
|
/** This engine's control-plane handshake descriptor — negotiated against each worker (design §7.3). */
|
|
@@ -7347,6 +7615,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7347
7615
|
this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
|
|
7348
7616
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
7349
7617
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
7618
|
+
this.dispatchedTurnSilenceMs = this.remoteAdvanceSilenceMs ?? this.reconcileMs;
|
|
7350
7619
|
this.sticky = new StickyCache(Math.max(0, deps.stickyRuns ?? DEFAULT_STICKY_RUNS), deps.stickyIdleMs ?? DEFAULT_STICKY_IDLE_MS);
|
|
7351
7620
|
this.routeReplies = deps.routeReplies ?? true;
|
|
7352
7621
|
const stalledAfter = deps.stalledAfterMs;
|
|
@@ -7358,6 +7627,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7358
7627
|
this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
|
|
7359
7628
|
this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
|
|
7360
7629
|
this.blockedPollMs = deps.blockedPollMs ?? 5e3;
|
|
7630
|
+
this.verifyTenant = deps.verifyTenant;
|
|
7361
7631
|
this.workerDescriptorCacheMs = deps.workerDescriptorCacheMs ?? 2e3;
|
|
7362
7632
|
this.cpDescriptor = controlPlaneDescriptor({
|
|
7363
7633
|
instanceId: `cp-${this.instanceId}`,
|
|
@@ -7398,6 +7668,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7398
7668
|
});
|
|
7399
7669
|
this.schedules = new ScheduleClient(this.store, {
|
|
7400
7670
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7671
|
+
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7672
|
+
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7673
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true), "startParked"),
|
|
7401
7674
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7402
7675
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7403
7676
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -7475,13 +7748,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7475
7748
|
console.warn(`[nestjs-durable] dropped a start-run for "${message.workflow}": invalid or unauthorized tenant token`);
|
|
7476
7749
|
return;
|
|
7477
7750
|
}
|
|
7478
|
-
await this.
|
|
7479
|
-
|
|
7480
|
-
|
|
7481
|
-
searchAttributes: message.searchAttributes,
|
|
7751
|
+
await this.startForTenant(
|
|
7752
|
+
message,
|
|
7753
|
+
tenant,
|
|
7482
7754
|
// The customer a tenant process claims for the run (A4): kept only when its verified
|
|
7483
7755
|
// token grants `tenancy:*` or `tenancy:<id>`, else the run gets no customer tenant.
|
|
7484
|
-
|
|
7756
|
+
message.tenantId !== void 0 ? grantsClientTenant(verified, message.tenantId) ? message.tenantKind !== void 0 ? {
|
|
7485
7757
|
tenantId: message.tenantId,
|
|
7486
7758
|
tenantKind: message.tenantKind
|
|
7487
7759
|
} : {
|
|
@@ -7489,7 +7761,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7489
7761
|
} : {
|
|
7490
7762
|
tenantId: null
|
|
7491
7763
|
} : {}
|
|
7492
|
-
|
|
7764
|
+
);
|
|
7493
7765
|
});
|
|
7494
7766
|
this.controlPlane?.onControl((msg) => {
|
|
7495
7767
|
if (msg.from === this.instanceId) return;
|
|
@@ -7572,14 +7844,17 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7572
7844
|
* The earliest known time (epoch ms) the poll has work, if it is at or before `byMs`: this
|
|
7573
7845
|
* namespace's next due timer (`listDueTimers` up to `byMs`, limit one: it returns the most overdue
|
|
7574
7846
|
* first), the next `blocked` run's recovery poll, and with `persistedSchedules` the next persisted
|
|
7575
|
-
* schedule to fire (see {@link ScheduleClient.nextFireAt}
|
|
7847
|
+
* schedule to fire (see {@link ScheduleClient.nextFireAt}; with `announcedSchedules` alone, the next
|
|
7848
|
+
* schedule adopted from a tenant). `undefined` when nothing is due by
|
|
7576
7849
|
* then. A poller reads it before waiting longer than its tick, so the wait never outlasts a timer.
|
|
7577
7850
|
*/
|
|
7578
7851
|
async nextDueAt(byMs, opts = {}) {
|
|
7579
7852
|
const [timers, blocked, schedule] = await Promise.all([
|
|
7580
7853
|
this.store.listDueTimers(byMs, this.namespace, 1),
|
|
7581
7854
|
this.dueBlockedRuns(byMs),
|
|
7582
|
-
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) :
|
|
7855
|
+
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) : opts.announcedSchedules && this.schedules.supported ? this.schedules.nextFireAt(byMs, {
|
|
7856
|
+
tag: ANNOUNCED_SCHEDULE_TAG
|
|
7857
|
+
}) : void 0
|
|
7583
7858
|
]);
|
|
7584
7859
|
let next;
|
|
7585
7860
|
for (const at of [
|
|
@@ -9496,7 +9771,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9496
9771
|
}
|
|
9497
9772
|
}
|
|
9498
9773
|
}
|
|
9499
|
-
|
|
9774
|
+
start(workflow, input, runId, opts) {
|
|
9775
|
+
return this.startWith(workflow, input, runId, opts, false);
|
|
9776
|
+
}
|
|
9777
|
+
/**
|
|
9778
|
+
* {@link start}'s body. `parkUnroutable` is the tenant start path ({@link startForTenant}): a
|
|
9779
|
+
* workflow that nothing here can route yet (no registration, no live worker group serving it at
|
|
9780
|
+
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9781
|
+
* a start whose caller has already returned is never lost.
|
|
9782
|
+
*/
|
|
9783
|
+
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9500
9784
|
const name = workflowName(workflow);
|
|
9501
9785
|
const pinned = opts?.version;
|
|
9502
9786
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9505,7 +9789,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9505
9789
|
const ancestor = await this.findRemoteAncestor(runId);
|
|
9506
9790
|
if (ancestor) registered = this.synthesizeRemoteChild(name, ancestor.version, ancestor);
|
|
9507
9791
|
}
|
|
9508
|
-
if (!registered
|
|
9792
|
+
if (!registered) {
|
|
9509
9793
|
const conventionRun = {
|
|
9510
9794
|
id: runId,
|
|
9511
9795
|
workflow: name,
|
|
@@ -9520,14 +9804,26 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9520
9804
|
createdAt: /* @__PURE__ */ new Date(),
|
|
9521
9805
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9522
9806
|
};
|
|
9523
|
-
|
|
9807
|
+
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9524
9808
|
askFleetForVersion: true
|
|
9525
9809
|
});
|
|
9810
|
+
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9811
|
+
}
|
|
9812
|
+
if (!registered && parkUnroutable) {
|
|
9813
|
+
return this.parkStartForWorker(name, input, runId, opts);
|
|
9526
9814
|
}
|
|
9527
9815
|
if (!registered) {
|
|
9528
9816
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
9529
9817
|
}
|
|
9530
|
-
|
|
9818
|
+
if (parkUnroutable && registered.validateInput) {
|
|
9819
|
+
try {
|
|
9820
|
+
await registered.validateInput(input);
|
|
9821
|
+
} catch (err) {
|
|
9822
|
+
throw new TenantStartInvalid(err);
|
|
9823
|
+
}
|
|
9824
|
+
} else {
|
|
9825
|
+
await registered.validateInput?.(input);
|
|
9826
|
+
}
|
|
9531
9827
|
const prior = await this.store.getRun(runId);
|
|
9532
9828
|
if (prior) return this.existingStartResult(prior);
|
|
9533
9829
|
const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, input);
|
|
@@ -9609,6 +9905,187 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9609
9905
|
};
|
|
9610
9906
|
}
|
|
9611
9907
|
/**
|
|
9908
|
+
* Create the run of a tenant start that nothing can route yet, parked `blocked` with code
|
|
9909
|
+
* {@link NO_LIVE_WORKER} (see {@link startWith}). Idempotent by run id like {@link start}, counts
|
|
9910
|
+
* against a concurrency quota like it, and keeps the start's options on the row: its version (the
|
|
9911
|
+
* pin, else the `'1'` placeholder tagged {@link VERSION_UNDECLARED_TAG} that the first live worker's
|
|
9912
|
+
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9913
|
+
* start is checked first at its `startAt`.
|
|
9914
|
+
*/
|
|
9915
|
+
async parkStartForWorker(name, input, runId, opts) {
|
|
9916
|
+
const prior = await this.store.getRun(runId);
|
|
9917
|
+
if (prior) return this.existingStartResult(prior);
|
|
9918
|
+
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9919
|
+
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9920
|
+
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9921
|
+
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9922
|
+
const pinned = opts?.version;
|
|
9923
|
+
const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
|
|
9924
|
+
const firstCheck = this.clock() + this.blockedPollMs;
|
|
9925
|
+
const wakeAt = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > firstCheck ? startAtMs : firstCheck;
|
|
9926
|
+
const error2 = {
|
|
9927
|
+
message: `no live worker serves ${group}${pinned !== void 0 ? ` at version ${pinned}` : ""} yet; the run starts when one does`,
|
|
9928
|
+
code: NO_LIVE_WORKER,
|
|
9929
|
+
retryable: true
|
|
9930
|
+
};
|
|
9931
|
+
const now = /* @__PURE__ */ new Date();
|
|
9932
|
+
const tags = mergeTags(void 0, [
|
|
9933
|
+
...opts?.tags ?? [],
|
|
9934
|
+
...quota ? [
|
|
9935
|
+
concurrencyTag(quota.key)
|
|
9936
|
+
] : [],
|
|
9937
|
+
...pinned === void 0 ? [
|
|
9938
|
+
VERSION_UNDECLARED_TAG
|
|
9939
|
+
] : []
|
|
9940
|
+
])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
9941
|
+
const run = {
|
|
9942
|
+
id: runId,
|
|
9943
|
+
workflow: name,
|
|
9944
|
+
workflowVersion: pinned ?? "1",
|
|
9945
|
+
status: "blocked",
|
|
9946
|
+
error: error2,
|
|
9947
|
+
wakeAt,
|
|
9948
|
+
namespace,
|
|
9949
|
+
input,
|
|
9950
|
+
tags,
|
|
9951
|
+
searchAttributes: opts?.searchAttributes,
|
|
9952
|
+
priority: opts?.priority,
|
|
9953
|
+
...this.startTenant(opts),
|
|
9954
|
+
createdAt: now,
|
|
9955
|
+
updatedAt: now
|
|
9956
|
+
};
|
|
9957
|
+
try {
|
|
9958
|
+
await this.store.createRun(run);
|
|
9959
|
+
} catch (err) {
|
|
9960
|
+
const winner = await this.store.getRun(runId).catch(() => null);
|
|
9961
|
+
if (!winner) throw err;
|
|
9962
|
+
return this.existingStartResult(winner);
|
|
9963
|
+
}
|
|
9964
|
+
this.emit({
|
|
9965
|
+
type: "run.blocked",
|
|
9966
|
+
runId,
|
|
9967
|
+
workflow: name,
|
|
9968
|
+
namespace,
|
|
9969
|
+
error: error2
|
|
9970
|
+
});
|
|
9971
|
+
return {
|
|
9972
|
+
runId,
|
|
9973
|
+
status: "blocked",
|
|
9974
|
+
error: error2
|
|
9975
|
+
};
|
|
9976
|
+
}
|
|
9977
|
+
/**
|
|
9978
|
+
* The control plane's half of a tenant start ({@link StartRunMessage}): start the run with every
|
|
9979
|
+
* option the message carries, under the tenant's namespace. The tenant caller has already returned
|
|
9980
|
+
* `pending`, so nothing here may lose the start:
|
|
9981
|
+
* - a workflow no live worker serves yet is parked `blocked` ({@link NO_LIVE_WORKER}) and started
|
|
9982
|
+
* by the blocked-recovery poll once one is live;
|
|
9983
|
+
* - a start the quota or singleton back-pressure rejects, or whose input a registration here
|
|
9984
|
+
* rejects, is recorded as a `failed` run carrying the reason, so the caller's run id resolves
|
|
9985
|
+
* to an answer instead of nothing (and a deterministic failure is never redelivered);
|
|
9986
|
+
* - a message with no `workflow` is logged and dropped: there is nothing to start;
|
|
9987
|
+
* - any other error (a store blip) is rethrown for the transport to redeliver the message, and a
|
|
9988
|
+
* redelivery is a no-op once the run exists (idempotent by run id).
|
|
9989
|
+
*/
|
|
9990
|
+
async startForTenant(message, namespace, tenant) {
|
|
9991
|
+
const runId = message.runId ?? globalThis.crypto.randomUUID();
|
|
9992
|
+
if (typeof message.workflow !== "string" || message.workflow === "") {
|
|
9993
|
+
console.error(`[nestjs-durable] dropped a malformed start-run ${runId}: no workflow name`);
|
|
9994
|
+
return {
|
|
9995
|
+
runId,
|
|
9996
|
+
status: "failed",
|
|
9997
|
+
error: {
|
|
9998
|
+
message: "malformed start-run: no workflow"
|
|
9999
|
+
}
|
|
10000
|
+
};
|
|
10001
|
+
}
|
|
10002
|
+
const opts = {
|
|
10003
|
+
namespace,
|
|
10004
|
+
tags: message.tags,
|
|
10005
|
+
searchAttributes: message.searchAttributes,
|
|
10006
|
+
...tenant,
|
|
10007
|
+
// Each start option only when the message carries it: a message from an older tenant has none
|
|
10008
|
+
// of them, and starts exactly as it did before.
|
|
10009
|
+
...typeof message.version === "string" ? {
|
|
10010
|
+
version: message.version
|
|
10011
|
+
} : {},
|
|
10012
|
+
...typeof message.priority === "number" ? {
|
|
10013
|
+
priority: message.priority
|
|
10014
|
+
} : {},
|
|
10015
|
+
...typeof message.startAt === "number" ? {
|
|
10016
|
+
startAt: message.startAt
|
|
10017
|
+
} : {},
|
|
10018
|
+
...message.concurrency !== void 0 ? {
|
|
10019
|
+
concurrency: message.concurrency
|
|
10020
|
+
} : {}
|
|
10021
|
+
};
|
|
10022
|
+
try {
|
|
10023
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
10024
|
+
} catch (err) {
|
|
10025
|
+
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
10026
|
+
throw err;
|
|
10027
|
+
}
|
|
10028
|
+
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
10029
|
+
}
|
|
10030
|
+
}
|
|
10031
|
+
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
10032
|
+
async recordRejectedStart(name, input, runId, opts, err) {
|
|
10033
|
+
const prior = await this.store.getRun(runId);
|
|
10034
|
+
if (prior) return this.existingStartResult(prior);
|
|
10035
|
+
const error2 = {
|
|
10036
|
+
message: err.message,
|
|
10037
|
+
code: err instanceof ConcurrencyLimitError ? "concurrency_limit" : err instanceof SingletonQueueFullError ? "singleton_queue_full" : "invalid_input",
|
|
10038
|
+
retryable: false
|
|
10039
|
+
};
|
|
10040
|
+
const now = /* @__PURE__ */ new Date();
|
|
10041
|
+
const run = {
|
|
10042
|
+
id: runId,
|
|
10043
|
+
workflow: name,
|
|
10044
|
+
workflowVersion: opts.version ?? "1",
|
|
10045
|
+
status: "failed",
|
|
10046
|
+
error: error2,
|
|
10047
|
+
namespace: poolNamespace(opts.namespace) ?? this.namespace,
|
|
10048
|
+
input,
|
|
10049
|
+
// The same derived tags an admitted or parked start carries, so quota-key queries find it.
|
|
10050
|
+
tags: mergeTags(opts.tags, [
|
|
10051
|
+
...opts.concurrency ? [
|
|
10052
|
+
concurrencyTag(opts.concurrency.key)
|
|
10053
|
+
] : [],
|
|
10054
|
+
...opts.version === void 0 ? [
|
|
10055
|
+
VERSION_UNDECLARED_TAG
|
|
10056
|
+
] : []
|
|
10057
|
+
])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag)),
|
|
10058
|
+
searchAttributes: opts.searchAttributes,
|
|
10059
|
+
priority: opts.priority,
|
|
10060
|
+
...this.startTenant(opts),
|
|
10061
|
+
createdAt: now,
|
|
10062
|
+
updatedAt: now
|
|
10063
|
+
};
|
|
10064
|
+
try {
|
|
10065
|
+
await this.store.createRun(run);
|
|
10066
|
+
} catch (createErr) {
|
|
10067
|
+
const winner = await this.store.getRun(runId).catch(() => null);
|
|
10068
|
+
if (!winner) throw createErr;
|
|
10069
|
+
return this.existingStartResult(winner);
|
|
10070
|
+
}
|
|
10071
|
+
console.warn(`[nestjs-durable] tenant start ${runId} of "${name}" rejected: ${err.message}`);
|
|
10072
|
+
this.emit({
|
|
10073
|
+
type: "run.failed",
|
|
10074
|
+
runId,
|
|
10075
|
+
workflow: name,
|
|
10076
|
+
namespace: run.namespace,
|
|
10077
|
+
error: error2
|
|
10078
|
+
}, {
|
|
10079
|
+
lifecycle: false
|
|
10080
|
+
});
|
|
10081
|
+
void this.deliverRunWaits(runId).catch(() => void 0);
|
|
10082
|
+
return {
|
|
10083
|
+
runId,
|
|
10084
|
+
status: "failed",
|
|
10085
|
+
error: error2
|
|
10086
|
+
};
|
|
10087
|
+
}
|
|
10088
|
+
/**
|
|
9612
10089
|
* The tenant columns a new run gets: an explicit `opts.tenantId` (`null` = none) wins, else the
|
|
9613
10090
|
* context tenant when tenancy is on, else none (no keys at all, so a run row is unchanged when
|
|
9614
10091
|
* tenancy is off).
|
|
@@ -10547,6 +11024,32 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10547
11024
|
console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
|
|
10548
11025
|
return this.announceDead(run, error2);
|
|
10549
11026
|
}
|
|
11027
|
+
/**
|
|
11028
|
+
* Due runs whose workflow nothing could resolve on the last try (`runId` → clock ms of the next
|
|
11029
|
+
* try): a tenant run whose worker is down, say. The timer poll ticks every second and the run stays
|
|
11030
|
+
* due, so without this each tick would lease it, fail to resolve it and log it again. The next try
|
|
11031
|
+
* backs off with the time it has been due ({@link noteUnroutable}). In memory on purpose: the run
|
|
11032
|
+
* row is not touched, so its age toward `unregisteredWorkflowGraceMs` is unchanged, and another
|
|
11033
|
+
* instance (or a restart) simply tries it sooner.
|
|
11034
|
+
*/
|
|
11035
|
+
unroutableUntil = /* @__PURE__ */ new Map();
|
|
11036
|
+
/** Whether `runId` is inside its {@link unroutableUntil} back-off at `nowMs`. */
|
|
11037
|
+
unroutableBackoff(runId, nowMs) {
|
|
11038
|
+
const until = this.unroutableUntil.get(runId);
|
|
11039
|
+
if (until === void 0) return false;
|
|
11040
|
+
if (nowMs >= until) {
|
|
11041
|
+
this.unroutableUntil.delete(runId);
|
|
11042
|
+
return false;
|
|
11043
|
+
}
|
|
11044
|
+
return true;
|
|
11045
|
+
}
|
|
11046
|
+
/** Back off the next timer-poll try of `run`, left for later as unresolvable (see {@link unroutableUntil}). */
|
|
11047
|
+
noteUnroutable(run, nowMs) {
|
|
11048
|
+
const dueSince = Math.max(new Date(run.updatedAt).getTime(), run.wakeAt ?? 0);
|
|
11049
|
+
const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor((nowMs - dueSince) / 10)));
|
|
11050
|
+
if (this.unroutableUntil.size > 4096) this.unroutableUntil.clear();
|
|
11051
|
+
this.unroutableUntil.set(run.id, nowMs + delay);
|
|
11052
|
+
}
|
|
10550
11053
|
/** Log a run skipped for an unregistered workflow — still there, still not progressing. */
|
|
10551
11054
|
logUnregisteredSkip(run) {
|
|
10552
11055
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
|
|
@@ -10567,8 +11070,8 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10567
11070
|
* scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
|
|
10568
11071
|
*/
|
|
10569
11072
|
async resumeDueTimers(nowMs = this.clock()) {
|
|
10570
|
-
const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
|
|
10571
|
-
const blocked = await this.dueBlockedRuns(nowMs);
|
|
11073
|
+
const due = shuffle((await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize)).filter((run) => !this.unroutableBackoff(run.id, nowMs)));
|
|
11074
|
+
const blocked = await this.routableBlocked(await this.dueBlockedRuns(nowMs), nowMs);
|
|
10572
11075
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
10573
11076
|
if (this.queuesRunsOverTransport && !this.draining) {
|
|
10574
11077
|
const runs = [
|
|
@@ -10670,6 +11173,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10670
11173
|
const run = await this.store.getRun(runId);
|
|
10671
11174
|
const settled = run ? await this.settleUnregistered(run, nowMs) : null;
|
|
10672
11175
|
if (settled !== void 0) return settled;
|
|
11176
|
+
if (run) this.noteUnroutable(run, nowMs);
|
|
10673
11177
|
if (run && this.isStartOnlyHere(run)) return null;
|
|
10674
11178
|
}
|
|
10675
11179
|
throw err;
|
|
@@ -10967,6 +11471,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10967
11471
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
10968
11472
|
const settled = await this.settleUnregistered(run, this.clock()).catch(() => void 0);
|
|
10969
11473
|
if (settled) return settled;
|
|
11474
|
+
if (settled === void 0) this.noteUnroutable(run, this.clock());
|
|
10970
11475
|
if (this.isStartOnlyHere(run)) return void 0;
|
|
10971
11476
|
}
|
|
10972
11477
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
@@ -10997,6 +11502,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10997
11502
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
10998
11503
|
const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
|
|
10999
11504
|
if (settled) results.push(settled);
|
|
11505
|
+
if (settled === void 0) this.noteUnroutable(run, nowMs);
|
|
11000
11506
|
if (settled !== void 0 || this.isStartOnlyHere(run)) continue;
|
|
11001
11507
|
}
|
|
11002
11508
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
@@ -11977,6 +12483,75 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
11977
12483
|
*
|
|
11978
12484
|
* Workflows only, and deliberately: see the "steps are out of scope" note in `handshake/announced`.
|
|
11979
12485
|
*/
|
|
12486
|
+
/**
|
|
12487
|
+
* Adopt the schedules tenant apps announce (their `schedules` module option, carried on their live
|
|
12488
|
+
* workers' descriptors) as persisted schedules, and withdraw the ones an app stopped announcing.
|
|
12489
|
+
* The control plane's timer poll calls it; {@link ScheduleClient.tick} fires them.
|
|
12490
|
+
*
|
|
12491
|
+
* Per app (the namespace its announcement's verified `tenant` maps to, as for a start-run): the
|
|
12492
|
+
* schedules announced by its live workers, where two workers disagree on a key the one that
|
|
12493
|
+
* started last (its deploy is the newer one) wins. An app with no live worker that states its
|
|
12494
|
+
* schedules is left alone, so its schedules keep firing into runs that wait for a worker. See
|
|
12495
|
+
* {@link ScheduleClient.syncAnnounced} for the reconciliation rule and `graceMs` (default 10
|
|
12496
|
+
* minutes). A namespaced engine adopts only its own namespace's. Nothing without a store that
|
|
12497
|
+
* persists schedules or a transport that reads worker descriptors.
|
|
12498
|
+
*/
|
|
12499
|
+
async syncAnnouncedSchedules(opts = {}) {
|
|
12500
|
+
const out = /* @__PURE__ */ new Map();
|
|
12501
|
+
if (!this.schedules.supported) return out;
|
|
12502
|
+
const byNamespace = /* @__PURE__ */ new Map();
|
|
12503
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
12504
|
+
const announced = descriptor.schedules;
|
|
12505
|
+
if (!announced || typeof announced.tenant !== "string" || !Array.isArray(announced.items)) {
|
|
12506
|
+
continue;
|
|
12507
|
+
}
|
|
12508
|
+
let verified;
|
|
12509
|
+
try {
|
|
12510
|
+
verified = await verifyWireTenant(announced.tenant, this.verifyTenant);
|
|
12511
|
+
} catch {
|
|
12512
|
+
verified = null;
|
|
12513
|
+
}
|
|
12514
|
+
const tenant = verified?.tenant ?? null;
|
|
12515
|
+
if (tenant === null) {
|
|
12516
|
+
if (!this.rejectedScheduleTenants.has(descriptor.instanceId)) {
|
|
12517
|
+
this.rejectedScheduleTenants.add(descriptor.instanceId);
|
|
12518
|
+
console.warn(`[nestjs-durable] ignored the schedules worker ${descriptor.instanceId} announces: invalid or unauthorized tenant token`);
|
|
12519
|
+
}
|
|
12520
|
+
continue;
|
|
12521
|
+
}
|
|
12522
|
+
const namespace = poolNamespace(tenant) ?? "default";
|
|
12523
|
+
if (this.namespace !== void 0 && namespace !== this.namespace) continue;
|
|
12524
|
+
const list2 = byNamespace.get(namespace) ?? [];
|
|
12525
|
+
list2.push({
|
|
12526
|
+
startedAt: Number(descriptor.startedAt) || 0,
|
|
12527
|
+
items: announced.items
|
|
12528
|
+
});
|
|
12529
|
+
byNamespace.set(namespace, list2);
|
|
12530
|
+
}
|
|
12531
|
+
for (const [namespace, announcers] of byNamespace) {
|
|
12532
|
+
const items = /* @__PURE__ */ new Map();
|
|
12533
|
+
for (const { items: list2 } of announcers.sort((a, b) => b.startedAt - a.startedAt)) {
|
|
12534
|
+
for (const item of list2) {
|
|
12535
|
+
if (item && typeof item.key === "string" && !items.has(item.key)) {
|
|
12536
|
+
items.set(item.key, item);
|
|
12537
|
+
}
|
|
12538
|
+
}
|
|
12539
|
+
}
|
|
12540
|
+
const result = await this.schedules.syncAnnounced(namespace, [
|
|
12541
|
+
...items.values()
|
|
12542
|
+
], {
|
|
12543
|
+
graceMs: opts.graceMs ?? DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
12544
|
+
...opts.nowMs !== void 0 ? {
|
|
12545
|
+
nowMs: opts.nowMs
|
|
12546
|
+
} : {}
|
|
12547
|
+
});
|
|
12548
|
+
for (const { key, error: error2 } of result.invalid) {
|
|
12549
|
+
console.warn(`[nestjs-durable] app "${namespace}" announces schedule "${key}", which cannot be adopted: ${error2}`);
|
|
12550
|
+
}
|
|
12551
|
+
out.set(namespace, result);
|
|
12552
|
+
}
|
|
12553
|
+
return out;
|
|
12554
|
+
}
|
|
11980
12555
|
async announcedWorkflows() {
|
|
11981
12556
|
return (await this.workflowDirectory()).workflows;
|
|
11982
12557
|
}
|
|
@@ -12916,7 +13491,7 @@ ${run.workflowVersion}`;
|
|
|
12916
13491
|
if (remote.executor.dispatch) {
|
|
12917
13492
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
12918
13493
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
12919
|
-
const silenceMs = this.
|
|
13494
|
+
const silenceMs = this.dispatchedTurnSilenceMs;
|
|
12920
13495
|
const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
|
|
12921
13496
|
await this.store.updateRun(run.id, await this.withRunTrace(run, {
|
|
12922
13497
|
status: "suspended",
|
|
@@ -14186,6 +14761,73 @@ ${run.workflowVersion}`;
|
|
|
14186
14761
|
error: error2
|
|
14187
14762
|
};
|
|
14188
14763
|
}
|
|
14764
|
+
/**
|
|
14765
|
+
* Of the due `blocked` runs, the ones to re-drive now. A run parked {@link NO_LIVE_WORKER} (a tenant
|
|
14766
|
+
* start nothing could route) is re-driven only once a live worker serves its group, at its pinned
|
|
14767
|
+
* version when it has one; until then it is re-parked with a longer wait each time, up to
|
|
14768
|
+
* {@link NO_LIVE_WORKER_MAX_POLL_MS}, and never reaches `resume` (which would treat it as an
|
|
14769
|
+
* unregistered workflow and dead-letter it after `unregisteredWorkflowGraceMs`). Every other
|
|
14770
|
+
* blocked run is re-driven as before.
|
|
14771
|
+
*/
|
|
14772
|
+
async routableBlocked(blocked, nowMs) {
|
|
14773
|
+
const ready = [];
|
|
14774
|
+
for (const run of blocked) {
|
|
14775
|
+
if (run.error?.code !== NO_LIVE_WORKER) {
|
|
14776
|
+
ready.push(run);
|
|
14777
|
+
continue;
|
|
14778
|
+
}
|
|
14779
|
+
try {
|
|
14780
|
+
const routed = await this.routeParkedStart(run, nowMs);
|
|
14781
|
+
if (routed) ready.push(routed);
|
|
14782
|
+
} catch {
|
|
14783
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
14784
|
+
"blocked"
|
|
14785
|
+
], {
|
|
14786
|
+
wakeAt: nowMs + this.blockedPollMs
|
|
14787
|
+
}).catch(() => void 0);
|
|
14788
|
+
}
|
|
14789
|
+
}
|
|
14790
|
+
return ready;
|
|
14791
|
+
}
|
|
14792
|
+
/**
|
|
14793
|
+
* One {@link NO_LIVE_WORKER} run: the run to re-drive (its version taken from the live worker when
|
|
14794
|
+
* the start did not pin one and the worker declares it), or `undefined` after re-parking it.
|
|
14795
|
+
*/
|
|
14796
|
+
async routeParkedStart(run, nowMs) {
|
|
14797
|
+
if (this.isRegisteredHere(run)) return run;
|
|
14798
|
+
const undeclared = run.tags?.includes(VERSION_UNDECLARED_TAG) === true;
|
|
14799
|
+
const route = await this.resolveRemoteByConvention(run, {
|
|
14800
|
+
askFleetForVersion: true
|
|
14801
|
+
});
|
|
14802
|
+
const routable = route !== void 0 && (undeclared || route.versionDeclared === true && route.version === run.workflowVersion);
|
|
14803
|
+
const updateIf = this.store.updateRunIf ?? this.updateRunIfFallback;
|
|
14804
|
+
if (!routable) {
|
|
14805
|
+
const waited = Math.max(0, nowMs - new Date(run.createdAt).getTime());
|
|
14806
|
+
const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor(waited / 10)));
|
|
14807
|
+
await updateIf.call(this.store, run.id, [
|
|
14808
|
+
"blocked"
|
|
14809
|
+
], {
|
|
14810
|
+
wakeAt: nowMs + delay
|
|
14811
|
+
});
|
|
14812
|
+
return void 0;
|
|
14813
|
+
}
|
|
14814
|
+
if (undeclared && route.versionDeclared === true) {
|
|
14815
|
+
const tags = run.tags?.filter((tag) => tag !== VERSION_UNDECLARED_TAG);
|
|
14816
|
+
const applied = await updateIf.call(this.store, run.id, [
|
|
14817
|
+
"blocked"
|
|
14818
|
+
], {
|
|
14819
|
+
workflowVersion: route.version,
|
|
14820
|
+
tags
|
|
14821
|
+
});
|
|
14822
|
+
if (!applied) return void 0;
|
|
14823
|
+
return {
|
|
14824
|
+
...run,
|
|
14825
|
+
workflowVersion: route.version,
|
|
14826
|
+
tags
|
|
14827
|
+
};
|
|
14828
|
+
}
|
|
14829
|
+
return run;
|
|
14830
|
+
}
|
|
14189
14831
|
/** Runs parked `blocked` whose recovery poll is due (`wakeAt <= nowMs`), namespace-scoped like
|
|
14190
14832
|
* {@link resumeDueTimers}. Re-driving one re-checks the live fleet: it dispatches if a
|
|
14191
14833
|
* capable+compatible worker has appeared, else re-parks with a fresh `wakeAt` (design §7.5). */
|
|
@@ -15236,7 +15878,7 @@ ${run.workflowVersion}`;
|
|
|
15236
15878
|
* cold (or different) instance re-derives the same answer from the run's own `wakeAt`.
|
|
15237
15879
|
*/
|
|
15238
15880
|
async rearmDecisionDeadline(runId) {
|
|
15239
|
-
const silenceMs = this.
|
|
15881
|
+
const silenceMs = this.dispatchedTurnSilenceMs;
|
|
15240
15882
|
if (silenceMs == null) return;
|
|
15241
15883
|
const now = this.clock();
|
|
15242
15884
|
const lookAgainAt = this.decisionDeadlineRearms.get(runId);
|
|
@@ -17773,6 +18415,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
17773
18415
|
// Annotate the CommonJS export names for ESM import in node:
|
|
17774
18416
|
0 && (module.exports = {
|
|
17775
18417
|
ACCUMULATOR_SEEN_MAX,
|
|
18418
|
+
ANNOUNCED_SCHEDULE_TAG,
|
|
17776
18419
|
ANY_WAIT_GROUP,
|
|
17777
18420
|
CHILD_EDGE_PREFIXES,
|
|
17778
18421
|
CONTROL_FLOW_SIGNAL,
|
|
@@ -17781,6 +18424,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
17781
18424
|
CodecStateStore,
|
|
17782
18425
|
ConcurrencyLimitError,
|
|
17783
18426
|
ContinueAsNew,
|
|
18427
|
+
DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
17784
18428
|
DEFAULT_CONCURRENCY_STATUSES,
|
|
17785
18429
|
DEFAULT_PARTITION,
|
|
17786
18430
|
DEFAULT_STALLED_AFTER_MS,
|
|
@@ -17819,6 +18463,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
17819
18463
|
LIFECYCLE_TRACE_TAG_PREFIX,
|
|
17820
18464
|
LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
17821
18465
|
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
18466
|
+
NO_LIVE_WORKER,
|
|
17822
18467
|
NestedWorkflowCallError,
|
|
17823
18468
|
NonDeterminismError,
|
|
17824
18469
|
PARENT_CLOSE_TAG_PREFIX,
|
|
@@ -17861,6 +18506,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
17861
18506
|
accumulatorRunId,
|
|
17862
18507
|
aggregateAnnouncements,
|
|
17863
18508
|
aggregatesCheckpoints,
|
|
18509
|
+
announcedScheduleId,
|
|
17864
18510
|
announcementsOf,
|
|
17865
18511
|
anyCheckpointName,
|
|
17866
18512
|
applyAttributeQuery,
|