@dudousxd/nestjs-durable-core 0.99.0 → 0.101.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +672 -26
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +317 -68
- package/dist/index.d.ts +317 -68
- package/dist/index.js +668 -26
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -475,7 +475,7 @@ __name(bindWorkflowClass, "bindWorkflowClass");
|
|
|
475
475
|
function clientOf(ctor) {
|
|
476
476
|
const client = bindings.get(ctor);
|
|
477
477
|
if (!client) {
|
|
478
|
-
throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow
|
|
478
|
+
throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow (an operator, or a tenant's DurableModule) has booted.`);
|
|
479
479
|
}
|
|
480
480
|
return client;
|
|
481
481
|
}
|
|
@@ -2312,6 +2312,9 @@ function normalizeDescriptor(raw) {
|
|
|
2312
2312
|
...raw.registrations !== void 0 ? {
|
|
2313
2313
|
registrations: raw.registrations
|
|
2314
2314
|
} : {},
|
|
2315
|
+
...raw.schedules !== void 0 ? {
|
|
2316
|
+
schedules: raw.schedules
|
|
2317
|
+
} : {},
|
|
2315
2318
|
...raw.partition !== void 0 ? {
|
|
2316
2319
|
partition: raw.partition
|
|
2317
2320
|
} : {},
|
|
@@ -2386,12 +2389,30 @@ function canonicalizeForHash(d) {
|
|
|
2386
2389
|
...d.registrations !== void 0 ? {
|
|
2387
2390
|
registrations: canonicalizeRegistrations(d.registrations)
|
|
2388
2391
|
} : {},
|
|
2392
|
+
// Conditional like `registrations`: a descriptor without schedules keeps its published hash.
|
|
2393
|
+
...d.schedules !== void 0 ? {
|
|
2394
|
+
schedules: canonicalSchedules(d.schedules)
|
|
2395
|
+
} : {},
|
|
2389
2396
|
partition: d.partition ?? null,
|
|
2390
2397
|
namespace: d.namespace ?? null,
|
|
2391
2398
|
startedAt: d.startedAt
|
|
2392
2399
|
};
|
|
2393
2400
|
}
|
|
2394
2401
|
__name(canonicalizeForHash, "canonicalizeForHash");
|
|
2402
|
+
function canonicalSchedules(a) {
|
|
2403
|
+
const items = [
|
|
2404
|
+
...a.items ?? []
|
|
2405
|
+
].sort((x, y) => x.key.localeCompare(y.key));
|
|
2406
|
+
return {
|
|
2407
|
+
tenant: a.tenant,
|
|
2408
|
+
items: items.map((i) => stableJson(i))
|
|
2409
|
+
};
|
|
2410
|
+
}
|
|
2411
|
+
__name(canonicalSchedules, "canonicalSchedules");
|
|
2412
|
+
function stableJson(value) {
|
|
2413
|
+
return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
|
|
2414
|
+
}
|
|
2415
|
+
__name(stableJson, "stableJson");
|
|
2395
2416
|
function fnv1a64Hex(input) {
|
|
2396
2417
|
const OFFSET = 0xcbf29ce484222325n;
|
|
2397
2418
|
const PRIME = 0x100000001b3n;
|
|
@@ -2449,6 +2470,14 @@ function describeWorker(opts) {
|
|
|
2449
2470
|
...registrations.length > 0 ? {
|
|
2450
2471
|
registrations
|
|
2451
2472
|
} : {},
|
|
2473
|
+
...opts.schedules !== void 0 ? {
|
|
2474
|
+
schedules: {
|
|
2475
|
+
tenant: opts.schedules.tenant,
|
|
2476
|
+
items: [
|
|
2477
|
+
...opts.schedules.items
|
|
2478
|
+
].sort((a, b) => a.key.localeCompare(b.key))
|
|
2479
|
+
}
|
|
2480
|
+
} : {},
|
|
2452
2481
|
...opts.partition ? {
|
|
2453
2482
|
partition: opts.partition
|
|
2454
2483
|
} : {},
|
|
@@ -3867,6 +3896,12 @@ function matchesTenantPredicates(row, query) {
|
|
|
3867
3896
|
__name(matchesTenantPredicates, "matchesTenantPredicates");
|
|
3868
3897
|
|
|
3869
3898
|
// src/schedules.ts
|
|
3899
|
+
var ANNOUNCED_SCHEDULE_TAG = "schedule:announced";
|
|
3900
|
+
function announcedScheduleId(namespace, key) {
|
|
3901
|
+
return `app:${namespace}:${key}`;
|
|
3902
|
+
}
|
|
3903
|
+
__name(announcedScheduleId, "announcedScheduleId");
|
|
3904
|
+
var PARKED_FOR_WORKER = "no_live_worker";
|
|
3870
3905
|
var ScheduleAlreadyExistsError = class extends Error {
|
|
3871
3906
|
static {
|
|
3872
3907
|
__name(this, "ScheduleAlreadyExistsError");
|
|
@@ -4007,15 +4042,28 @@ var ScheduleClient = class {
|
|
|
4007
4042
|
async trigger(id) {
|
|
4008
4043
|
const record = await this.mustGet(id);
|
|
4009
4044
|
const runId = `sched:${id}:manual:${this.now()}`;
|
|
4010
|
-
return this.
|
|
4045
|
+
return this.startFor(record, runId);
|
|
4046
|
+
}
|
|
4047
|
+
/**
|
|
4048
|
+
* Start one run of `record`: through the tenant start path for an announced schedule (its run waits
|
|
4049
|
+
* `blocked` for a live worker of its app instead of failing), else a plain `start`.
|
|
4050
|
+
*/
|
|
4051
|
+
startFor(record, runId) {
|
|
4052
|
+
const input = specOf(record).input;
|
|
4053
|
+
const opts = startOptions(record);
|
|
4054
|
+
if (isAnnounced(record) && this.host.startParked) {
|
|
4055
|
+
return this.host.startParked(record.workflow, input, runId, opts);
|
|
4056
|
+
}
|
|
4057
|
+
return this.host.start(record.workflow, input, runId, opts);
|
|
4011
4058
|
}
|
|
4012
4059
|
/**
|
|
4013
|
-
* Fire every due schedule (in this engine's namespace; all namespaces for an operator engine
|
|
4060
|
+
* Fire every due schedule (in this engine's namespace; all namespaces for an operator engine; only
|
|
4061
|
+
* those carrying `opts.tag` when given):
|
|
4014
4062
|
* start its window's run and advance it to the next window. Called by the NestJS timer poller on
|
|
4015
4063
|
* each tick when `persistedSchedules` is on; call it yourself from any other driver. Returns the
|
|
4016
4064
|
* run ids started (or re-found, for an idempotent re-fire).
|
|
4017
4065
|
*/
|
|
4018
|
-
async tick(nowMs = this.now()) {
|
|
4066
|
+
async tick(nowMs = this.now(), opts = {}) {
|
|
4019
4067
|
const store = this.requireStore();
|
|
4020
4068
|
const started = [];
|
|
4021
4069
|
for (let round = 0; round < 50; round++) {
|
|
@@ -4023,6 +4071,9 @@ var ScheduleClient = class {
|
|
|
4023
4071
|
const due = await store.listSchedules({
|
|
4024
4072
|
dueBy: nowMs,
|
|
4025
4073
|
namespace: this.options.namespace,
|
|
4074
|
+
...opts.tag !== void 0 ? {
|
|
4075
|
+
tag: opts.tag
|
|
4076
|
+
} : {},
|
|
4026
4077
|
limit: TICK_BATCH
|
|
4027
4078
|
});
|
|
4028
4079
|
for (const record of due) {
|
|
@@ -4036,15 +4087,19 @@ var ScheduleClient = class {
|
|
|
4036
4087
|
}
|
|
4037
4088
|
/**
|
|
4038
4089
|
* When {@link tick} next has a schedule to fire, if one is due by `byMs`: the earliest `nextFireAt`
|
|
4039
|
-
* of the unpaused schedules (of this engine's namespace
|
|
4090
|
+
* of the unpaused schedules (of this engine's namespace, carrying `opts.tag` when given) due by
|
|
4091
|
+
* then, else `undefined`. One indexed
|
|
4040
4092
|
* read (the due-schedule scan `tick` makes, with a later bound and a limit of one). A store that
|
|
4041
4093
|
* does not persist schedules has none.
|
|
4042
4094
|
*/
|
|
4043
|
-
async nextFireAt(byMs) {
|
|
4095
|
+
async nextFireAt(byMs, opts = {}) {
|
|
4044
4096
|
if (!this.supported) return void 0;
|
|
4045
4097
|
const [next] = await this.requireStore().listSchedules({
|
|
4046
4098
|
dueBy: byMs,
|
|
4047
4099
|
namespace: this.options.namespace,
|
|
4100
|
+
...opts.tag !== void 0 ? {
|
|
4101
|
+
tag: opts.tag
|
|
4102
|
+
} : {},
|
|
4048
4103
|
limit: 1
|
|
4049
4104
|
});
|
|
4050
4105
|
return next?.nextFireAt ?? void 0;
|
|
@@ -4079,10 +4134,10 @@ var ScheduleClient = class {
|
|
|
4079
4134
|
let fired = false;
|
|
4080
4135
|
const skipMissed = spec.catchup === "skip" && !onTime;
|
|
4081
4136
|
const beforeWindow = latest < window;
|
|
4082
|
-
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state)) {
|
|
4137
|
+
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state) && !await this.waitingForWorker(record, state)) {
|
|
4083
4138
|
runId = `sched:${record.id}:${latest}`;
|
|
4084
4139
|
try {
|
|
4085
|
-
await this.
|
|
4140
|
+
await this.startFor(record, runId);
|
|
4086
4141
|
fired = true;
|
|
4087
4142
|
} catch (error2) {
|
|
4088
4143
|
if (await this.host.getRun(runId)) fired = true;
|
|
@@ -4106,6 +4161,182 @@ var ScheduleClient = class {
|
|
|
4106
4161
|
}, record.nextFireAt);
|
|
4107
4162
|
return fired ? runId : void 0;
|
|
4108
4163
|
}
|
|
4164
|
+
/**
|
|
4165
|
+
* An announced schedule whose previous run is still parked waiting for a live worker of its app
|
|
4166
|
+
* (the app is down): this window is folded into that run instead of parking one more. However long
|
|
4167
|
+
* the app is down, it finds at most one run per schedule waiting when it comes back, the same
|
|
4168
|
+
* bound `catchup: 'latest'` puts on windows missed while no scheduler polled.
|
|
4169
|
+
*/
|
|
4170
|
+
async waitingForWorker(record, state) {
|
|
4171
|
+
if (!isAnnounced(record) || !state.lastRunId) return false;
|
|
4172
|
+
const prev = await this.host.getRun(state.lastRunId);
|
|
4173
|
+
return prev?.status === "blocked" && prev.error?.code === PARKED_FOR_WORKER;
|
|
4174
|
+
}
|
|
4175
|
+
/**
|
|
4176
|
+
* Reconcile the schedules adopted from one app (`namespace`) with what its LIVE workers announce
|
|
4177
|
+
* now (`items`, one per key; the caller resolved disagreeing workers). The rule:
|
|
4178
|
+
* - an announced key with no schedule yet is created (id {@link announcedScheduleId}, tagged
|
|
4179
|
+
* {@link ANNOUNCED_SCHEDULE_TAG}, `catchup: 'latest'`); one whose definition changed is
|
|
4180
|
+
* re-applied like `upsert` (its history kept, its next fire recomputed from now). An equal
|
|
4181
|
+
* definition writes nothing, so any number of replicas announcing the same set is a no-op;
|
|
4182
|
+
* - a pause or resume made from the console survives: a re-applied definition keeps the current
|
|
4183
|
+
* paused state, unless the app's own declared `paused` changed;
|
|
4184
|
+
* - an adopted schedule the app no longer announces is marked missing, and paused ("withdrawn")
|
|
4185
|
+
* once it has stayed missing for `graceMs`. It is never deleted: its history stays, and an app
|
|
4186
|
+
* that announces it again (a rollback) gets it back, resumed unless an operator had paused it.
|
|
4187
|
+
*
|
|
4188
|
+
* Call it only for an app that has at least one live worker stating its schedules: with none (the
|
|
4189
|
+
* app is down, or runs an SDK that does not announce schedules), nothing is withdrawn, and its
|
|
4190
|
+
* schedules keep firing into runs that wait for a worker.
|
|
4191
|
+
*/
|
|
4192
|
+
async syncAnnounced(namespace, items, opts) {
|
|
4193
|
+
const store = this.requireStore();
|
|
4194
|
+
const now = opts.nowMs ?? this.now();
|
|
4195
|
+
const out = {
|
|
4196
|
+
adopted: [],
|
|
4197
|
+
restored: [],
|
|
4198
|
+
missing: [],
|
|
4199
|
+
withdrawn: [],
|
|
4200
|
+
invalid: []
|
|
4201
|
+
};
|
|
4202
|
+
const adopted = /* @__PURE__ */ new Map();
|
|
4203
|
+
for (let offset = 0; ; offset += TICK_BATCH) {
|
|
4204
|
+
const page = await store.listSchedules({
|
|
4205
|
+
namespace,
|
|
4206
|
+
tag: ANNOUNCED_SCHEDULE_TAG,
|
|
4207
|
+
limit: TICK_BATCH,
|
|
4208
|
+
offset
|
|
4209
|
+
});
|
|
4210
|
+
for (const record of page) if (isAnnounced(record)) adopted.set(record.id, record);
|
|
4211
|
+
if (page.length < TICK_BATCH) break;
|
|
4212
|
+
}
|
|
4213
|
+
for (const item of items) {
|
|
4214
|
+
const id = announcedScheduleId(namespace, item.key);
|
|
4215
|
+
const existing = adopted.get(id) ?? await store.getSchedule(id);
|
|
4216
|
+
adopted.delete(id);
|
|
4217
|
+
const doc = existing ? existing.state.announce : void 0;
|
|
4218
|
+
if (existing && !doc) {
|
|
4219
|
+
out.invalid.push({
|
|
4220
|
+
key: item.key,
|
|
4221
|
+
error: `schedule id "${id}" is taken`
|
|
4222
|
+
});
|
|
4223
|
+
continue;
|
|
4224
|
+
}
|
|
4225
|
+
const fingerprint = stableJson(item);
|
|
4226
|
+
if (existing && doc && doc.fingerprint === fingerprint) {
|
|
4227
|
+
if (doc.missingSince === void 0 && doc.withdrawnAt === void 0) continue;
|
|
4228
|
+
const resume = doc.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal && existing.paused;
|
|
4229
|
+
const spec = specOf(existing);
|
|
4230
|
+
const window = resume ? nextWindowAfter(spec, now) : void 0;
|
|
4231
|
+
await store.updateSchedule(id, {
|
|
4232
|
+
// The withdrawal note goes; the note it replaced comes back.
|
|
4233
|
+
...doc.withdrawnAt !== void 0 ? {
|
|
4234
|
+
spec: stripUndefined({
|
|
4235
|
+
...spec,
|
|
4236
|
+
note: doc.noteBeforeWithdrawal
|
|
4237
|
+
})
|
|
4238
|
+
} : {},
|
|
4239
|
+
...resume ? {
|
|
4240
|
+
paused: false,
|
|
4241
|
+
nextFireAt: window == null ? null : window + jitterFor(id, window, spec)
|
|
4242
|
+
} : {},
|
|
4243
|
+
state: stripUndefined({
|
|
4244
|
+
...existing.state,
|
|
4245
|
+
...resume ? {
|
|
4246
|
+
window
|
|
4247
|
+
} : {},
|
|
4248
|
+
announce: {
|
|
4249
|
+
key: doc.key,
|
|
4250
|
+
fingerprint,
|
|
4251
|
+
declaredPaused: doc.declaredPaused
|
|
4252
|
+
}
|
|
4253
|
+
}),
|
|
4254
|
+
updatedAt: new Date(now)
|
|
4255
|
+
});
|
|
4256
|
+
out.restored.push(id);
|
|
4257
|
+
continue;
|
|
4258
|
+
}
|
|
4259
|
+
const declaredPaused = item.paused === true;
|
|
4260
|
+
const paused = existing === null || existing === void 0 ? declaredPaused : declaredPaused !== (doc?.declaredPaused === true) ? declaredPaused : doc?.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal ? declaredPaused : void 0;
|
|
4261
|
+
try {
|
|
4262
|
+
await this.write({
|
|
4263
|
+
id,
|
|
4264
|
+
workflow: item.workflow,
|
|
4265
|
+
input: item.input,
|
|
4266
|
+
cron: item.cron,
|
|
4267
|
+
every: item.everyMs,
|
|
4268
|
+
timezone: item.timezone,
|
|
4269
|
+
jitter: item.jitterMs,
|
|
4270
|
+
overlap: item.overlap,
|
|
4271
|
+
catchup: "latest",
|
|
4272
|
+
namespace,
|
|
4273
|
+
tags: [
|
|
4274
|
+
ANNOUNCED_SCHEDULE_TAG
|
|
4275
|
+
],
|
|
4276
|
+
paused,
|
|
4277
|
+
// A note (an operator's pause reason) is kept; a withdrawal note is replaced by the one
|
|
4278
|
+
// it had replaced.
|
|
4279
|
+
note: doc?.withdrawnAt !== void 0 ? doc.noteBeforeWithdrawal : existing ? specOf(existing).note : void 0,
|
|
4280
|
+
// Fires belong to the app, never to whichever request context reached the sync.
|
|
4281
|
+
...existing ? {} : {
|
|
4282
|
+
tenantId: null
|
|
4283
|
+
}
|
|
4284
|
+
}, existing ?? null, {
|
|
4285
|
+
announce: {
|
|
4286
|
+
key: item.key,
|
|
4287
|
+
fingerprint,
|
|
4288
|
+
declaredPaused
|
|
4289
|
+
}
|
|
4290
|
+
});
|
|
4291
|
+
out.adopted.push(id);
|
|
4292
|
+
} catch (error2) {
|
|
4293
|
+
out.invalid.push({
|
|
4294
|
+
key: item.key,
|
|
4295
|
+
error: errorMessage(error2)
|
|
4296
|
+
});
|
|
4297
|
+
}
|
|
4298
|
+
}
|
|
4299
|
+
for (const record of adopted.values()) {
|
|
4300
|
+
const state = record.state;
|
|
4301
|
+
const doc = state.announce;
|
|
4302
|
+
if (doc.withdrawnAt !== void 0) continue;
|
|
4303
|
+
if (doc.missingSince === void 0) {
|
|
4304
|
+
await store.updateSchedule(record.id, {
|
|
4305
|
+
state: {
|
|
4306
|
+
...state,
|
|
4307
|
+
announce: {
|
|
4308
|
+
...doc,
|
|
4309
|
+
missingSince: now
|
|
4310
|
+
}
|
|
4311
|
+
},
|
|
4312
|
+
updatedAt: new Date(now)
|
|
4313
|
+
});
|
|
4314
|
+
out.missing.push(record.id);
|
|
4315
|
+
continue;
|
|
4316
|
+
}
|
|
4317
|
+
if (now - doc.missingSince < opts.graceMs) continue;
|
|
4318
|
+
const spec = {
|
|
4319
|
+
...specOf(record)
|
|
4320
|
+
};
|
|
4321
|
+
spec.note = `withdrawn: app "${namespace}" no longer announces schedule "${doc.key}"`;
|
|
4322
|
+
await store.updateSchedule(record.id, {
|
|
4323
|
+
paused: true,
|
|
4324
|
+
spec,
|
|
4325
|
+
state: {
|
|
4326
|
+
...state,
|
|
4327
|
+
announce: stripUndefined({
|
|
4328
|
+
...doc,
|
|
4329
|
+
withdrawnAt: now,
|
|
4330
|
+
pausedBeforeWithdrawal: record.paused,
|
|
4331
|
+
noteBeforeWithdrawal: specOf(record).note
|
|
4332
|
+
})
|
|
4333
|
+
},
|
|
4334
|
+
updatedAt: new Date(now)
|
|
4335
|
+
});
|
|
4336
|
+
out.withdrawn.push(record.id);
|
|
4337
|
+
}
|
|
4338
|
+
return out;
|
|
4339
|
+
}
|
|
4109
4340
|
/** `overlap: 'skip'` and the previous run of this schedule is still in flight. */
|
|
4110
4341
|
async overlapping(spec, state) {
|
|
4111
4342
|
if (spec.overlap !== "skip" || !state.lastRunId) return false;
|
|
@@ -4141,7 +4372,7 @@ var ScheduleClient = class {
|
|
|
4141
4372
|
tenantId: tenant.id
|
|
4142
4373
|
};
|
|
4143
4374
|
}
|
|
4144
|
-
async write(opts, existing) {
|
|
4375
|
+
async write(opts, existing, extraState = {}) {
|
|
4145
4376
|
const store = this.requireStore();
|
|
4146
4377
|
const spec = toSpec(opts);
|
|
4147
4378
|
const now = this.now();
|
|
@@ -4157,7 +4388,8 @@ var ScheduleClient = class {
|
|
|
4157
4388
|
spec,
|
|
4158
4389
|
state: stripUndefined({
|
|
4159
4390
|
...existing?.state ?? {},
|
|
4160
|
-
window
|
|
4391
|
+
window,
|
|
4392
|
+
...extraState
|
|
4161
4393
|
}),
|
|
4162
4394
|
...this.scheduleTenant(opts, existing),
|
|
4163
4395
|
createdAt: existing?.createdAt ?? new Date(now),
|
|
@@ -4203,6 +4435,10 @@ function specOf(record) {
|
|
|
4203
4435
|
return record.spec;
|
|
4204
4436
|
}
|
|
4205
4437
|
__name(specOf, "specOf");
|
|
4438
|
+
function isAnnounced(record) {
|
|
4439
|
+
return record.state?.announce !== void 0;
|
|
4440
|
+
}
|
|
4441
|
+
__name(isAnnounced, "isAnnounced");
|
|
4206
4442
|
function latestWindowAtOrBefore(spec, atMs) {
|
|
4207
4443
|
if (spec.cron != null) return prevCronFireMs(spec.cron, atMs, spec.timezone);
|
|
4208
4444
|
if (spec.rrule != null) {
|
|
@@ -4277,6 +4513,9 @@ function describe(record) {
|
|
|
4277
4513
|
lastError: state.lastError,
|
|
4278
4514
|
tenantId: record.tenantId ?? void 0,
|
|
4279
4515
|
tenantKind: record.tenantKind ?? void 0,
|
|
4516
|
+
announcedKey: state.announce?.key,
|
|
4517
|
+
unannouncedSince: state.announce?.missingSince != null ? new Date(state.announce.missingSince) : void 0,
|
|
4518
|
+
withdrawnAt: state.announce?.withdrawnAt != null ? new Date(state.announce.withdrawnAt) : void 0,
|
|
4280
4519
|
createdAt: record.createdAt,
|
|
4281
4520
|
updatedAt: record.updatedAt
|
|
4282
4521
|
});
|
|
@@ -6662,6 +6901,20 @@ function runNamespace(namespace) {
|
|
|
6662
6901
|
}
|
|
6663
6902
|
__name(runNamespace, "runNamespace");
|
|
6664
6903
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
6904
|
+
var NO_LIVE_WORKER = "no_live_worker";
|
|
6905
|
+
var DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS = 10 * 6e4;
|
|
6906
|
+
var TenantStartInvalid = class TenantStartInvalid2 extends Error {
|
|
6907
|
+
static {
|
|
6908
|
+
__name(this, "TenantStartInvalid");
|
|
6909
|
+
}
|
|
6910
|
+
constructor(cause) {
|
|
6911
|
+
super(cause instanceof Error ? cause.message : String(cause), {
|
|
6912
|
+
cause
|
|
6913
|
+
});
|
|
6914
|
+
this.name = "TenantStartInvalid";
|
|
6915
|
+
}
|
|
6916
|
+
};
|
|
6917
|
+
var NO_LIVE_WORKER_MAX_POLL_MS = 6e4;
|
|
6665
6918
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
6666
6919
|
var STEP_FAN_CHUNK = 200;
|
|
6667
6920
|
var STEP_EVENTS_PER_BROADCAST = 500;
|
|
@@ -6834,6 +7087,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
6834
7087
|
maxRecoveryAttempts;
|
|
6835
7088
|
unregisteredWorkflowGraceMs;
|
|
6836
7089
|
remoteAdvanceSilenceMs;
|
|
7090
|
+
/**
|
|
7091
|
+
* The re-drive window of a DISPATCHED workflow turn (a remote worker's `advance` sent over the
|
|
7092
|
+
* broker): {@link WorkflowEngineDeps.remoteAdvanceSilenceMs} when set, else `reconcileMs`. Without
|
|
7093
|
+
* it a run suspended on a turn whose worker died (and whose job was lost with it) had no `wakeAt`
|
|
7094
|
+
* and nothing re-drove it, ever. Heartbeats rearm it like the explicit window.
|
|
7095
|
+
*/
|
|
7096
|
+
dispatchedTurnSilenceMs;
|
|
6837
7097
|
reconcileMs;
|
|
6838
7098
|
/** Parked workflow bodies kept between turns (see {@link WorkflowEngineDeps.stickyRuns}). */
|
|
6839
7099
|
sticky;
|
|
@@ -6856,6 +7116,10 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
6856
7116
|
remoteRedispatchMax;
|
|
6857
7117
|
/** How long a `blocked` run waits before the recovery poll re-checks the live fleet (design §7.5). */
|
|
6858
7118
|
blockedPollMs;
|
|
7119
|
+
/** Verifies the wire tenant of a start-run, and of a worker's schedule announcement. */
|
|
7120
|
+
verifyTenant;
|
|
7121
|
+
/** Wire tenants whose schedule announcement failed verification, warned about once. */
|
|
7122
|
+
rejectedScheduleTenants = /* @__PURE__ */ new Set();
|
|
6859
7123
|
workerDescriptorCacheMs;
|
|
6860
7124
|
descriptorCache = /* @__PURE__ */ new Map();
|
|
6861
7125
|
/** This engine's control-plane handshake descriptor — negotiated against each worker (design §7.3). */
|
|
@@ -7054,6 +7318,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7054
7318
|
this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
|
|
7055
7319
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
7056
7320
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
7321
|
+
this.dispatchedTurnSilenceMs = this.remoteAdvanceSilenceMs ?? this.reconcileMs;
|
|
7057
7322
|
this.sticky = new StickyCache(Math.max(0, deps.stickyRuns ?? DEFAULT_STICKY_RUNS), deps.stickyIdleMs ?? DEFAULT_STICKY_IDLE_MS);
|
|
7058
7323
|
this.routeReplies = deps.routeReplies ?? true;
|
|
7059
7324
|
const stalledAfter = deps.stalledAfterMs;
|
|
@@ -7065,6 +7330,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7065
7330
|
this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
|
|
7066
7331
|
this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
|
|
7067
7332
|
this.blockedPollMs = deps.blockedPollMs ?? 5e3;
|
|
7333
|
+
this.verifyTenant = deps.verifyTenant;
|
|
7068
7334
|
this.workerDescriptorCacheMs = deps.workerDescriptorCacheMs ?? 2e3;
|
|
7069
7335
|
this.cpDescriptor = controlPlaneDescriptor({
|
|
7070
7336
|
instanceId: `cp-${this.instanceId}`,
|
|
@@ -7105,6 +7371,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7105
7371
|
});
|
|
7106
7372
|
this.schedules = new ScheduleClient(this.store, {
|
|
7107
7373
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7374
|
+
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7375
|
+
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7376
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.startWith(workflow, input, runId, opts, true), "startParked"),
|
|
7108
7377
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7109
7378
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7110
7379
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -7182,13 +7451,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7182
7451
|
console.warn(`[nestjs-durable] dropped a start-run for "${message.workflow}": invalid or unauthorized tenant token`);
|
|
7183
7452
|
return;
|
|
7184
7453
|
}
|
|
7185
|
-
await this.
|
|
7186
|
-
|
|
7187
|
-
|
|
7188
|
-
searchAttributes: message.searchAttributes,
|
|
7454
|
+
await this.startForTenant(
|
|
7455
|
+
message,
|
|
7456
|
+
tenant,
|
|
7189
7457
|
// The customer a tenant process claims for the run (A4): kept only when its verified
|
|
7190
7458
|
// token grants `tenancy:*` or `tenancy:<id>`, else the run gets no customer tenant.
|
|
7191
|
-
|
|
7459
|
+
message.tenantId !== void 0 ? grantsClientTenant(verified, message.tenantId) ? message.tenantKind !== void 0 ? {
|
|
7192
7460
|
tenantId: message.tenantId,
|
|
7193
7461
|
tenantKind: message.tenantKind
|
|
7194
7462
|
} : {
|
|
@@ -7196,7 +7464,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7196
7464
|
} : {
|
|
7197
7465
|
tenantId: null
|
|
7198
7466
|
} : {}
|
|
7199
|
-
|
|
7467
|
+
);
|
|
7200
7468
|
});
|
|
7201
7469
|
this.controlPlane?.onControl((msg) => {
|
|
7202
7470
|
if (msg.from === this.instanceId) return;
|
|
@@ -7279,14 +7547,17 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7279
7547
|
* The earliest known time (epoch ms) the poll has work, if it is at or before `byMs`: this
|
|
7280
7548
|
* namespace's next due timer (`listDueTimers` up to `byMs`, limit one: it returns the most overdue
|
|
7281
7549
|
* first), the next `blocked` run's recovery poll, and with `persistedSchedules` the next persisted
|
|
7282
|
-
* schedule to fire (see {@link ScheduleClient.nextFireAt}
|
|
7550
|
+
* schedule to fire (see {@link ScheduleClient.nextFireAt}; with `announcedSchedules` alone, the next
|
|
7551
|
+
* schedule adopted from a tenant). `undefined` when nothing is due by
|
|
7283
7552
|
* then. A poller reads it before waiting longer than its tick, so the wait never outlasts a timer.
|
|
7284
7553
|
*/
|
|
7285
7554
|
async nextDueAt(byMs, opts = {}) {
|
|
7286
7555
|
const [timers, blocked, schedule] = await Promise.all([
|
|
7287
7556
|
this.store.listDueTimers(byMs, this.namespace, 1),
|
|
7288
7557
|
this.dueBlockedRuns(byMs),
|
|
7289
|
-
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) :
|
|
7558
|
+
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) : opts.announcedSchedules && this.schedules.supported ? this.schedules.nextFireAt(byMs, {
|
|
7559
|
+
tag: ANNOUNCED_SCHEDULE_TAG
|
|
7560
|
+
}) : void 0
|
|
7290
7561
|
]);
|
|
7291
7562
|
let next;
|
|
7292
7563
|
for (const at of [
|
|
@@ -9203,7 +9474,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9203
9474
|
}
|
|
9204
9475
|
}
|
|
9205
9476
|
}
|
|
9206
|
-
|
|
9477
|
+
start(workflow, input, runId, opts) {
|
|
9478
|
+
return this.startWith(workflow, input, runId, opts, false);
|
|
9479
|
+
}
|
|
9480
|
+
/**
|
|
9481
|
+
* {@link start}'s body. `parkUnroutable` is the tenant start path ({@link startForTenant}): a
|
|
9482
|
+
* workflow that nothing here can route yet (no registration, no live worker group serving it at
|
|
9483
|
+
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9484
|
+
* a start whose caller has already returned is never lost.
|
|
9485
|
+
*/
|
|
9486
|
+
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9207
9487
|
const name = workflowName(workflow);
|
|
9208
9488
|
const pinned = opts?.version;
|
|
9209
9489
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9212,7 +9492,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9212
9492
|
const ancestor = await this.findRemoteAncestor(runId);
|
|
9213
9493
|
if (ancestor) registered = this.synthesizeRemoteChild(name, ancestor.version, ancestor);
|
|
9214
9494
|
}
|
|
9215
|
-
if (!registered
|
|
9495
|
+
if (!registered) {
|
|
9216
9496
|
const conventionRun = {
|
|
9217
9497
|
id: runId,
|
|
9218
9498
|
workflow: name,
|
|
@@ -9227,14 +9507,26 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9227
9507
|
createdAt: /* @__PURE__ */ new Date(),
|
|
9228
9508
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9229
9509
|
};
|
|
9230
|
-
|
|
9510
|
+
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9231
9511
|
askFleetForVersion: true
|
|
9232
9512
|
});
|
|
9513
|
+
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9514
|
+
}
|
|
9515
|
+
if (!registered && parkUnroutable) {
|
|
9516
|
+
return this.parkStartForWorker(name, input, runId, opts);
|
|
9233
9517
|
}
|
|
9234
9518
|
if (!registered) {
|
|
9235
9519
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
9236
9520
|
}
|
|
9237
|
-
|
|
9521
|
+
if (parkUnroutable && registered.validateInput) {
|
|
9522
|
+
try {
|
|
9523
|
+
await registered.validateInput(input);
|
|
9524
|
+
} catch (err) {
|
|
9525
|
+
throw new TenantStartInvalid(err);
|
|
9526
|
+
}
|
|
9527
|
+
} else {
|
|
9528
|
+
await registered.validateInput?.(input);
|
|
9529
|
+
}
|
|
9238
9530
|
const prior = await this.store.getRun(runId);
|
|
9239
9531
|
if (prior) return this.existingStartResult(prior);
|
|
9240
9532
|
const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, input);
|
|
@@ -9316,6 +9608,187 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9316
9608
|
};
|
|
9317
9609
|
}
|
|
9318
9610
|
/**
|
|
9611
|
+
* Create the run of a tenant start that nothing can route yet, parked `blocked` with code
|
|
9612
|
+
* {@link NO_LIVE_WORKER} (see {@link startWith}). Idempotent by run id like {@link start}, counts
|
|
9613
|
+
* against a concurrency quota like it, and keeps the start's options on the row: its version (the
|
|
9614
|
+
* pin, else the `'1'` placeholder tagged {@link VERSION_UNDECLARED_TAG} that the first live worker's
|
|
9615
|
+
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9616
|
+
* start is checked first at its `startAt`.
|
|
9617
|
+
*/
|
|
9618
|
+
async parkStartForWorker(name, input, runId, opts) {
|
|
9619
|
+
const prior = await this.store.getRun(runId);
|
|
9620
|
+
if (prior) return this.existingStartResult(prior);
|
|
9621
|
+
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9622
|
+
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9623
|
+
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9624
|
+
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9625
|
+
const pinned = opts?.version;
|
|
9626
|
+
const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
|
|
9627
|
+
const firstCheck = this.clock() + this.blockedPollMs;
|
|
9628
|
+
const wakeAt = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > firstCheck ? startAtMs : firstCheck;
|
|
9629
|
+
const error2 = {
|
|
9630
|
+
message: `no live worker serves ${group}${pinned !== void 0 ? ` at version ${pinned}` : ""} yet; the run starts when one does`,
|
|
9631
|
+
code: NO_LIVE_WORKER,
|
|
9632
|
+
retryable: true
|
|
9633
|
+
};
|
|
9634
|
+
const now = /* @__PURE__ */ new Date();
|
|
9635
|
+
const tags = mergeTags(void 0, [
|
|
9636
|
+
...opts?.tags ?? [],
|
|
9637
|
+
...quota ? [
|
|
9638
|
+
concurrencyTag(quota.key)
|
|
9639
|
+
] : [],
|
|
9640
|
+
...pinned === void 0 ? [
|
|
9641
|
+
VERSION_UNDECLARED_TAG
|
|
9642
|
+
] : []
|
|
9643
|
+
])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
9644
|
+
const run = {
|
|
9645
|
+
id: runId,
|
|
9646
|
+
workflow: name,
|
|
9647
|
+
workflowVersion: pinned ?? "1",
|
|
9648
|
+
status: "blocked",
|
|
9649
|
+
error: error2,
|
|
9650
|
+
wakeAt,
|
|
9651
|
+
namespace,
|
|
9652
|
+
input,
|
|
9653
|
+
tags,
|
|
9654
|
+
searchAttributes: opts?.searchAttributes,
|
|
9655
|
+
priority: opts?.priority,
|
|
9656
|
+
...this.startTenant(opts),
|
|
9657
|
+
createdAt: now,
|
|
9658
|
+
updatedAt: now
|
|
9659
|
+
};
|
|
9660
|
+
try {
|
|
9661
|
+
await this.store.createRun(run);
|
|
9662
|
+
} catch (err) {
|
|
9663
|
+
const winner = await this.store.getRun(runId).catch(() => null);
|
|
9664
|
+
if (!winner) throw err;
|
|
9665
|
+
return this.existingStartResult(winner);
|
|
9666
|
+
}
|
|
9667
|
+
this.emit({
|
|
9668
|
+
type: "run.blocked",
|
|
9669
|
+
runId,
|
|
9670
|
+
workflow: name,
|
|
9671
|
+
namespace,
|
|
9672
|
+
error: error2
|
|
9673
|
+
});
|
|
9674
|
+
return {
|
|
9675
|
+
runId,
|
|
9676
|
+
status: "blocked",
|
|
9677
|
+
error: error2
|
|
9678
|
+
};
|
|
9679
|
+
}
|
|
9680
|
+
/**
|
|
9681
|
+
* The control plane's half of a tenant start ({@link StartRunMessage}): start the run with every
|
|
9682
|
+
* option the message carries, under the tenant's namespace. The tenant caller has already returned
|
|
9683
|
+
* `pending`, so nothing here may lose the start:
|
|
9684
|
+
* - a workflow no live worker serves yet is parked `blocked` ({@link NO_LIVE_WORKER}) and started
|
|
9685
|
+
* by the blocked-recovery poll once one is live;
|
|
9686
|
+
* - a start the quota or singleton back-pressure rejects, or whose input a registration here
|
|
9687
|
+
* rejects, is recorded as a `failed` run carrying the reason, so the caller's run id resolves
|
|
9688
|
+
* to an answer instead of nothing (and a deterministic failure is never redelivered);
|
|
9689
|
+
* - a message with no `workflow` is logged and dropped: there is nothing to start;
|
|
9690
|
+
* - any other error (a store blip) is rethrown for the transport to redeliver the message, and a
|
|
9691
|
+
* redelivery is a no-op once the run exists (idempotent by run id).
|
|
9692
|
+
*/
|
|
9693
|
+
async startForTenant(message, namespace, tenant) {
|
|
9694
|
+
const runId = message.runId ?? globalThis.crypto.randomUUID();
|
|
9695
|
+
if (typeof message.workflow !== "string" || message.workflow === "") {
|
|
9696
|
+
console.error(`[nestjs-durable] dropped a malformed start-run ${runId}: no workflow name`);
|
|
9697
|
+
return {
|
|
9698
|
+
runId,
|
|
9699
|
+
status: "failed",
|
|
9700
|
+
error: {
|
|
9701
|
+
message: "malformed start-run: no workflow"
|
|
9702
|
+
}
|
|
9703
|
+
};
|
|
9704
|
+
}
|
|
9705
|
+
const opts = {
|
|
9706
|
+
namespace,
|
|
9707
|
+
tags: message.tags,
|
|
9708
|
+
searchAttributes: message.searchAttributes,
|
|
9709
|
+
...tenant,
|
|
9710
|
+
// Each start option only when the message carries it: a message from an older tenant has none
|
|
9711
|
+
// of them, and starts exactly as it did before.
|
|
9712
|
+
...typeof message.version === "string" ? {
|
|
9713
|
+
version: message.version
|
|
9714
|
+
} : {},
|
|
9715
|
+
...typeof message.priority === "number" ? {
|
|
9716
|
+
priority: message.priority
|
|
9717
|
+
} : {},
|
|
9718
|
+
...typeof message.startAt === "number" ? {
|
|
9719
|
+
startAt: message.startAt
|
|
9720
|
+
} : {},
|
|
9721
|
+
...message.concurrency !== void 0 ? {
|
|
9722
|
+
concurrency: message.concurrency
|
|
9723
|
+
} : {}
|
|
9724
|
+
};
|
|
9725
|
+
try {
|
|
9726
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
9727
|
+
} catch (err) {
|
|
9728
|
+
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
9729
|
+
throw err;
|
|
9730
|
+
}
|
|
9731
|
+
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
9732
|
+
}
|
|
9733
|
+
}
|
|
9734
|
+
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
9735
|
+
async recordRejectedStart(name, input, runId, opts, err) {
|
|
9736
|
+
const prior = await this.store.getRun(runId);
|
|
9737
|
+
if (prior) return this.existingStartResult(prior);
|
|
9738
|
+
const error2 = {
|
|
9739
|
+
message: err.message,
|
|
9740
|
+
code: err instanceof ConcurrencyLimitError ? "concurrency_limit" : err instanceof SingletonQueueFullError ? "singleton_queue_full" : "invalid_input",
|
|
9741
|
+
retryable: false
|
|
9742
|
+
};
|
|
9743
|
+
const now = /* @__PURE__ */ new Date();
|
|
9744
|
+
const run = {
|
|
9745
|
+
id: runId,
|
|
9746
|
+
workflow: name,
|
|
9747
|
+
workflowVersion: opts.version ?? "1",
|
|
9748
|
+
status: "failed",
|
|
9749
|
+
error: error2,
|
|
9750
|
+
namespace: poolNamespace(opts.namespace) ?? this.namespace,
|
|
9751
|
+
input,
|
|
9752
|
+
// The same derived tags an admitted or parked start carries, so quota-key queries find it.
|
|
9753
|
+
tags: mergeTags(opts.tags, [
|
|
9754
|
+
...opts.concurrency ? [
|
|
9755
|
+
concurrencyTag(opts.concurrency.key)
|
|
9756
|
+
] : [],
|
|
9757
|
+
...opts.version === void 0 ? [
|
|
9758
|
+
VERSION_UNDECLARED_TAG
|
|
9759
|
+
] : []
|
|
9760
|
+
])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag)),
|
|
9761
|
+
searchAttributes: opts.searchAttributes,
|
|
9762
|
+
priority: opts.priority,
|
|
9763
|
+
...this.startTenant(opts),
|
|
9764
|
+
createdAt: now,
|
|
9765
|
+
updatedAt: now
|
|
9766
|
+
};
|
|
9767
|
+
try {
|
|
9768
|
+
await this.store.createRun(run);
|
|
9769
|
+
} catch (createErr) {
|
|
9770
|
+
const winner = await this.store.getRun(runId).catch(() => null);
|
|
9771
|
+
if (!winner) throw createErr;
|
|
9772
|
+
return this.existingStartResult(winner);
|
|
9773
|
+
}
|
|
9774
|
+
console.warn(`[nestjs-durable] tenant start ${runId} of "${name}" rejected: ${err.message}`);
|
|
9775
|
+
this.emit({
|
|
9776
|
+
type: "run.failed",
|
|
9777
|
+
runId,
|
|
9778
|
+
workflow: name,
|
|
9779
|
+
namespace: run.namespace,
|
|
9780
|
+
error: error2
|
|
9781
|
+
}, {
|
|
9782
|
+
lifecycle: false
|
|
9783
|
+
});
|
|
9784
|
+
void this.deliverRunWaits(runId).catch(() => void 0);
|
|
9785
|
+
return {
|
|
9786
|
+
runId,
|
|
9787
|
+
status: "failed",
|
|
9788
|
+
error: error2
|
|
9789
|
+
};
|
|
9790
|
+
}
|
|
9791
|
+
/**
|
|
9319
9792
|
* The tenant columns a new run gets: an explicit `opts.tenantId` (`null` = none) wins, else the
|
|
9320
9793
|
* context tenant when tenancy is on, else none (no keys at all, so a run row is unchanged when
|
|
9321
9794
|
* tenancy is off).
|
|
@@ -10254,6 +10727,32 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10254
10727
|
console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
|
|
10255
10728
|
return this.announceDead(run, error2);
|
|
10256
10729
|
}
|
|
10730
|
+
/**
|
|
10731
|
+
* Due runs whose workflow nothing could resolve on the last try (`runId` → clock ms of the next
|
|
10732
|
+
* try): a tenant run whose worker is down, say. The timer poll ticks every second and the run stays
|
|
10733
|
+
* due, so without this each tick would lease it, fail to resolve it and log it again. The next try
|
|
10734
|
+
* backs off with the time it has been due ({@link noteUnroutable}). In memory on purpose: the run
|
|
10735
|
+
* row is not touched, so its age toward `unregisteredWorkflowGraceMs` is unchanged, and another
|
|
10736
|
+
* instance (or a restart) simply tries it sooner.
|
|
10737
|
+
*/
|
|
10738
|
+
unroutableUntil = /* @__PURE__ */ new Map();
|
|
10739
|
+
/** Whether `runId` is inside its {@link unroutableUntil} back-off at `nowMs`. */
|
|
10740
|
+
unroutableBackoff(runId, nowMs) {
|
|
10741
|
+
const until = this.unroutableUntil.get(runId);
|
|
10742
|
+
if (until === void 0) return false;
|
|
10743
|
+
if (nowMs >= until) {
|
|
10744
|
+
this.unroutableUntil.delete(runId);
|
|
10745
|
+
return false;
|
|
10746
|
+
}
|
|
10747
|
+
return true;
|
|
10748
|
+
}
|
|
10749
|
+
/** Back off the next timer-poll try of `run`, left for later as unresolvable (see {@link unroutableUntil}). */
|
|
10750
|
+
noteUnroutable(run, nowMs) {
|
|
10751
|
+
const dueSince = Math.max(new Date(run.updatedAt).getTime(), run.wakeAt ?? 0);
|
|
10752
|
+
const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor((nowMs - dueSince) / 10)));
|
|
10753
|
+
if (this.unroutableUntil.size > 4096) this.unroutableUntil.clear();
|
|
10754
|
+
this.unroutableUntil.set(run.id, nowMs + delay);
|
|
10755
|
+
}
|
|
10257
10756
|
/** Log a run skipped for an unregistered workflow — still there, still not progressing. */
|
|
10258
10757
|
logUnregisteredSkip(run) {
|
|
10259
10758
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
|
|
@@ -10274,8 +10773,8 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10274
10773
|
* scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
|
|
10275
10774
|
*/
|
|
10276
10775
|
async resumeDueTimers(nowMs = this.clock()) {
|
|
10277
|
-
const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
|
|
10278
|
-
const blocked = await this.dueBlockedRuns(nowMs);
|
|
10776
|
+
const due = shuffle((await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize)).filter((run) => !this.unroutableBackoff(run.id, nowMs)));
|
|
10777
|
+
const blocked = await this.routableBlocked(await this.dueBlockedRuns(nowMs), nowMs);
|
|
10279
10778
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
10280
10779
|
if (this.queuesRunsOverTransport && !this.draining) {
|
|
10281
10780
|
const runs = [
|
|
@@ -10377,6 +10876,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10377
10876
|
const run = await this.store.getRun(runId);
|
|
10378
10877
|
const settled = run ? await this.settleUnregistered(run, nowMs) : null;
|
|
10379
10878
|
if (settled !== void 0) return settled;
|
|
10879
|
+
if (run) this.noteUnroutable(run, nowMs);
|
|
10380
10880
|
if (run && this.isStartOnlyHere(run)) return null;
|
|
10381
10881
|
}
|
|
10382
10882
|
throw err;
|
|
@@ -10674,6 +11174,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10674
11174
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
10675
11175
|
const settled = await this.settleUnregistered(run, this.clock()).catch(() => void 0);
|
|
10676
11176
|
if (settled) return settled;
|
|
11177
|
+
if (settled === void 0) this.noteUnroutable(run, this.clock());
|
|
10677
11178
|
if (this.isStartOnlyHere(run)) return void 0;
|
|
10678
11179
|
}
|
|
10679
11180
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
@@ -10704,6 +11205,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10704
11205
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
10705
11206
|
const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
|
|
10706
11207
|
if (settled) results.push(settled);
|
|
11208
|
+
if (settled === void 0) this.noteUnroutable(run, nowMs);
|
|
10707
11209
|
if (settled !== void 0 || this.isStartOnlyHere(run)) continue;
|
|
10708
11210
|
}
|
|
10709
11211
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
@@ -11684,6 +12186,75 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
11684
12186
|
*
|
|
11685
12187
|
* Workflows only, and deliberately: see the "steps are out of scope" note in `handshake/announced`.
|
|
11686
12188
|
*/
|
|
12189
|
+
/**
|
|
12190
|
+
* Adopt the schedules tenant apps announce (their `schedules` module option, carried on their live
|
|
12191
|
+
* workers' descriptors) as persisted schedules, and withdraw the ones an app stopped announcing.
|
|
12192
|
+
* The control plane's timer poll calls it; {@link ScheduleClient.tick} fires them.
|
|
12193
|
+
*
|
|
12194
|
+
* Per app (the namespace its announcement's verified `tenant` maps to, as for a start-run): the
|
|
12195
|
+
* schedules announced by its live workers, where two workers disagree on a key the one that
|
|
12196
|
+
* started last (its deploy is the newer one) wins. An app with no live worker that states its
|
|
12197
|
+
* schedules is left alone, so its schedules keep firing into runs that wait for a worker. See
|
|
12198
|
+
* {@link ScheduleClient.syncAnnounced} for the reconciliation rule and `graceMs` (default 10
|
|
12199
|
+
* minutes). A namespaced engine adopts only its own namespace's. Nothing without a store that
|
|
12200
|
+
* persists schedules or a transport that reads worker descriptors.
|
|
12201
|
+
*/
|
|
12202
|
+
async syncAnnouncedSchedules(opts = {}) {
|
|
12203
|
+
const out = /* @__PURE__ */ new Map();
|
|
12204
|
+
if (!this.schedules.supported) return out;
|
|
12205
|
+
const byNamespace = /* @__PURE__ */ new Map();
|
|
12206
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
12207
|
+
const announced = descriptor.schedules;
|
|
12208
|
+
if (!announced || typeof announced.tenant !== "string" || !Array.isArray(announced.items)) {
|
|
12209
|
+
continue;
|
|
12210
|
+
}
|
|
12211
|
+
let verified;
|
|
12212
|
+
try {
|
|
12213
|
+
verified = await verifyWireTenant(announced.tenant, this.verifyTenant);
|
|
12214
|
+
} catch {
|
|
12215
|
+
verified = null;
|
|
12216
|
+
}
|
|
12217
|
+
const tenant = verified?.tenant ?? null;
|
|
12218
|
+
if (tenant === null) {
|
|
12219
|
+
if (!this.rejectedScheduleTenants.has(descriptor.instanceId)) {
|
|
12220
|
+
this.rejectedScheduleTenants.add(descriptor.instanceId);
|
|
12221
|
+
console.warn(`[nestjs-durable] ignored the schedules worker ${descriptor.instanceId} announces: invalid or unauthorized tenant token`);
|
|
12222
|
+
}
|
|
12223
|
+
continue;
|
|
12224
|
+
}
|
|
12225
|
+
const namespace = poolNamespace(tenant) ?? "default";
|
|
12226
|
+
if (this.namespace !== void 0 && namespace !== this.namespace) continue;
|
|
12227
|
+
const list2 = byNamespace.get(namespace) ?? [];
|
|
12228
|
+
list2.push({
|
|
12229
|
+
startedAt: Number(descriptor.startedAt) || 0,
|
|
12230
|
+
items: announced.items
|
|
12231
|
+
});
|
|
12232
|
+
byNamespace.set(namespace, list2);
|
|
12233
|
+
}
|
|
12234
|
+
for (const [namespace, announcers] of byNamespace) {
|
|
12235
|
+
const items = /* @__PURE__ */ new Map();
|
|
12236
|
+
for (const { items: list2 } of announcers.sort((a, b) => b.startedAt - a.startedAt)) {
|
|
12237
|
+
for (const item of list2) {
|
|
12238
|
+
if (item && typeof item.key === "string" && !items.has(item.key)) {
|
|
12239
|
+
items.set(item.key, item);
|
|
12240
|
+
}
|
|
12241
|
+
}
|
|
12242
|
+
}
|
|
12243
|
+
const result = await this.schedules.syncAnnounced(namespace, [
|
|
12244
|
+
...items.values()
|
|
12245
|
+
], {
|
|
12246
|
+
graceMs: opts.graceMs ?? DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
12247
|
+
...opts.nowMs !== void 0 ? {
|
|
12248
|
+
nowMs: opts.nowMs
|
|
12249
|
+
} : {}
|
|
12250
|
+
});
|
|
12251
|
+
for (const { key, error: error2 } of result.invalid) {
|
|
12252
|
+
console.warn(`[nestjs-durable] app "${namespace}" announces schedule "${key}", which cannot be adopted: ${error2}`);
|
|
12253
|
+
}
|
|
12254
|
+
out.set(namespace, result);
|
|
12255
|
+
}
|
|
12256
|
+
return out;
|
|
12257
|
+
}
|
|
11687
12258
|
async announcedWorkflows() {
|
|
11688
12259
|
return (await this.workflowDirectory()).workflows;
|
|
11689
12260
|
}
|
|
@@ -12623,7 +13194,7 @@ ${run.workflowVersion}`;
|
|
|
12623
13194
|
if (remote.executor.dispatch) {
|
|
12624
13195
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
12625
13196
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
12626
|
-
const silenceMs = this.
|
|
13197
|
+
const silenceMs = this.dispatchedTurnSilenceMs;
|
|
12627
13198
|
const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
|
|
12628
13199
|
await this.store.updateRun(run.id, await this.withRunTrace(run, {
|
|
12629
13200
|
status: "suspended",
|
|
@@ -13893,6 +14464,73 @@ ${run.workflowVersion}`;
|
|
|
13893
14464
|
error: error2
|
|
13894
14465
|
};
|
|
13895
14466
|
}
|
|
14467
|
+
/**
|
|
14468
|
+
* Of the due `blocked` runs, the ones to re-drive now. A run parked {@link NO_LIVE_WORKER} (a tenant
|
|
14469
|
+
* start nothing could route) is re-driven only once a live worker serves its group, at its pinned
|
|
14470
|
+
* version when it has one; until then it is re-parked with a longer wait each time, up to
|
|
14471
|
+
* {@link NO_LIVE_WORKER_MAX_POLL_MS}, and never reaches `resume` (which would treat it as an
|
|
14472
|
+
* unregistered workflow and dead-letter it after `unregisteredWorkflowGraceMs`). Every other
|
|
14473
|
+
* blocked run is re-driven as before.
|
|
14474
|
+
*/
|
|
14475
|
+
async routableBlocked(blocked, nowMs) {
|
|
14476
|
+
const ready = [];
|
|
14477
|
+
for (const run of blocked) {
|
|
14478
|
+
if (run.error?.code !== NO_LIVE_WORKER) {
|
|
14479
|
+
ready.push(run);
|
|
14480
|
+
continue;
|
|
14481
|
+
}
|
|
14482
|
+
try {
|
|
14483
|
+
const routed = await this.routeParkedStart(run, nowMs);
|
|
14484
|
+
if (routed) ready.push(routed);
|
|
14485
|
+
} catch {
|
|
14486
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
14487
|
+
"blocked"
|
|
14488
|
+
], {
|
|
14489
|
+
wakeAt: nowMs + this.blockedPollMs
|
|
14490
|
+
}).catch(() => void 0);
|
|
14491
|
+
}
|
|
14492
|
+
}
|
|
14493
|
+
return ready;
|
|
14494
|
+
}
|
|
14495
|
+
/**
|
|
14496
|
+
* One {@link NO_LIVE_WORKER} run: the run to re-drive (its version taken from the live worker when
|
|
14497
|
+
* the start did not pin one and the worker declares it), or `undefined` after re-parking it.
|
|
14498
|
+
*/
|
|
14499
|
+
async routeParkedStart(run, nowMs) {
|
|
14500
|
+
if (this.isRegisteredHere(run)) return run;
|
|
14501
|
+
const undeclared = run.tags?.includes(VERSION_UNDECLARED_TAG) === true;
|
|
14502
|
+
const route = await this.resolveRemoteByConvention(run, {
|
|
14503
|
+
askFleetForVersion: true
|
|
14504
|
+
});
|
|
14505
|
+
const routable = route !== void 0 && (undeclared || route.versionDeclared === true && route.version === run.workflowVersion);
|
|
14506
|
+
const updateIf = this.store.updateRunIf ?? this.updateRunIfFallback;
|
|
14507
|
+
if (!routable) {
|
|
14508
|
+
const waited = Math.max(0, nowMs - new Date(run.createdAt).getTime());
|
|
14509
|
+
const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor(waited / 10)));
|
|
14510
|
+
await updateIf.call(this.store, run.id, [
|
|
14511
|
+
"blocked"
|
|
14512
|
+
], {
|
|
14513
|
+
wakeAt: nowMs + delay
|
|
14514
|
+
});
|
|
14515
|
+
return void 0;
|
|
14516
|
+
}
|
|
14517
|
+
if (undeclared && route.versionDeclared === true) {
|
|
14518
|
+
const tags = run.tags?.filter((tag) => tag !== VERSION_UNDECLARED_TAG);
|
|
14519
|
+
const applied = await updateIf.call(this.store, run.id, [
|
|
14520
|
+
"blocked"
|
|
14521
|
+
], {
|
|
14522
|
+
workflowVersion: route.version,
|
|
14523
|
+
tags
|
|
14524
|
+
});
|
|
14525
|
+
if (!applied) return void 0;
|
|
14526
|
+
return {
|
|
14527
|
+
...run,
|
|
14528
|
+
workflowVersion: route.version,
|
|
14529
|
+
tags
|
|
14530
|
+
};
|
|
14531
|
+
}
|
|
14532
|
+
return run;
|
|
14533
|
+
}
|
|
13896
14534
|
/** Runs parked `blocked` whose recovery poll is due (`wakeAt <= nowMs`), namespace-scoped like
|
|
13897
14535
|
* {@link resumeDueTimers}. Re-driving one re-checks the live fleet: it dispatches if a
|
|
13898
14536
|
* capable+compatible worker has appeared, else re-parks with a fresh `wakeAt` (design §7.5). */
|
|
@@ -14943,7 +15581,7 @@ ${run.workflowVersion}`;
|
|
|
14943
15581
|
* cold (or different) instance re-derives the same answer from the run's own `wakeAt`.
|
|
14944
15582
|
*/
|
|
14945
15583
|
async rearmDecisionDeadline(runId) {
|
|
14946
|
-
const silenceMs = this.
|
|
15584
|
+
const silenceMs = this.dispatchedTurnSilenceMs;
|
|
14947
15585
|
if (silenceMs == null) return;
|
|
14948
15586
|
const now = this.clock();
|
|
14949
15587
|
const lookAgainAt = this.decisionDeadlineRearms.get(runId);
|
|
@@ -17479,6 +18117,7 @@ async function settleResultsWithPlainCalls(store, results) {
|
|
|
17479
18117
|
__name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
17480
18118
|
export {
|
|
17481
18119
|
ACCUMULATOR_SEEN_MAX,
|
|
18120
|
+
ANNOUNCED_SCHEDULE_TAG,
|
|
17482
18121
|
ANY_WAIT_GROUP,
|
|
17483
18122
|
CHILD_EDGE_PREFIXES,
|
|
17484
18123
|
CONTROL_FLOW_SIGNAL,
|
|
@@ -17487,6 +18126,7 @@ export {
|
|
|
17487
18126
|
CodecStateStore,
|
|
17488
18127
|
ConcurrencyLimitError,
|
|
17489
18128
|
ContinueAsNew,
|
|
18129
|
+
DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
17490
18130
|
DEFAULT_CONCURRENCY_STATUSES,
|
|
17491
18131
|
DEFAULT_PARTITION,
|
|
17492
18132
|
DEFAULT_STALLED_AFTER_MS,
|
|
@@ -17525,6 +18165,7 @@ export {
|
|
|
17525
18165
|
LIFECYCLE_TRACE_TAG_PREFIX,
|
|
17526
18166
|
LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
17527
18167
|
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
18168
|
+
NO_LIVE_WORKER,
|
|
17528
18169
|
NestedWorkflowCallError,
|
|
17529
18170
|
NonDeterminismError,
|
|
17530
18171
|
PARENT_CLOSE_TAG_PREFIX,
|
|
@@ -17567,6 +18208,7 @@ export {
|
|
|
17567
18208
|
accumulatorRunId,
|
|
17568
18209
|
aggregateAnnouncements,
|
|
17569
18210
|
aggregatesCheckpoints,
|
|
18211
|
+
announcedScheduleId,
|
|
17570
18212
|
announcementsOf,
|
|
17571
18213
|
anyCheckpointName,
|
|
17572
18214
|
applyAttributeQuery,
|