@dudousxd/nestjs-durable-core 0.100.0 → 0.102.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +643 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +600 -290
- package/dist/index.d.ts +600 -290
- package/dist/index.js +640 -45
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -2312,6 +2312,9 @@ function normalizeDescriptor(raw) {
|
|
|
2312
2312
|
...raw.registrations !== void 0 ? {
|
|
2313
2313
|
registrations: raw.registrations
|
|
2314
2314
|
} : {},
|
|
2315
|
+
...raw.schedules !== void 0 ? {
|
|
2316
|
+
schedules: raw.schedules
|
|
2317
|
+
} : {},
|
|
2315
2318
|
...raw.partition !== void 0 ? {
|
|
2316
2319
|
partition: raw.partition
|
|
2317
2320
|
} : {},
|
|
@@ -2339,6 +2342,13 @@ function canonicalizeRegistrations(regs) {
|
|
|
2339
2342
|
subscribes: [
|
|
2340
2343
|
...new Set(r.subscribes)
|
|
2341
2344
|
].sort()
|
|
2345
|
+
} : {},
|
|
2346
|
+
// Conditional too: a registration without them keeps its hash; a changed option changes it.
|
|
2347
|
+
...r.options !== void 0 ? {
|
|
2348
|
+
options: stableJson(r.options)
|
|
2349
|
+
} : {},
|
|
2350
|
+
...r.deadLetter !== void 0 ? {
|
|
2351
|
+
deadLetter: r.deadLetter
|
|
2342
2352
|
} : {}
|
|
2343
2353
|
}));
|
|
2344
2354
|
const byJson = /* @__PURE__ */ new Map();
|
|
@@ -2386,12 +2396,30 @@ function canonicalizeForHash(d) {
|
|
|
2386
2396
|
...d.registrations !== void 0 ? {
|
|
2387
2397
|
registrations: canonicalizeRegistrations(d.registrations)
|
|
2388
2398
|
} : {},
|
|
2399
|
+
// Conditional like `registrations`: a descriptor without schedules keeps its published hash.
|
|
2400
|
+
...d.schedules !== void 0 ? {
|
|
2401
|
+
schedules: canonicalSchedules(d.schedules)
|
|
2402
|
+
} : {},
|
|
2389
2403
|
partition: d.partition ?? null,
|
|
2390
2404
|
namespace: d.namespace ?? null,
|
|
2391
2405
|
startedAt: d.startedAt
|
|
2392
2406
|
};
|
|
2393
2407
|
}
|
|
2394
2408
|
__name(canonicalizeForHash, "canonicalizeForHash");
|
|
2409
|
+
function canonicalSchedules(a) {
|
|
2410
|
+
const items = [
|
|
2411
|
+
...a.items ?? []
|
|
2412
|
+
].sort((x, y) => x.key.localeCompare(y.key));
|
|
2413
|
+
return {
|
|
2414
|
+
tenant: a.tenant,
|
|
2415
|
+
items: items.map((i) => stableJson(i))
|
|
2416
|
+
};
|
|
2417
|
+
}
|
|
2418
|
+
__name(canonicalSchedules, "canonicalSchedules");
|
|
2419
|
+
function stableJson(value) {
|
|
2420
|
+
return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
|
|
2421
|
+
}
|
|
2422
|
+
__name(stableJson, "stableJson");
|
|
2395
2423
|
function fnv1a64Hex(input) {
|
|
2396
2424
|
const OFFSET = 0xcbf29ce484222325n;
|
|
2397
2425
|
const PRIME = 0x100000001b3n;
|
|
@@ -2449,6 +2477,14 @@ function describeWorker(opts) {
|
|
|
2449
2477
|
...registrations.length > 0 ? {
|
|
2450
2478
|
registrations
|
|
2451
2479
|
} : {},
|
|
2480
|
+
...opts.schedules !== void 0 ? {
|
|
2481
|
+
schedules: {
|
|
2482
|
+
tenant: opts.schedules.tenant,
|
|
2483
|
+
items: [
|
|
2484
|
+
...opts.schedules.items
|
|
2485
|
+
].sort((a, b) => a.key.localeCompare(b.key))
|
|
2486
|
+
}
|
|
2487
|
+
} : {},
|
|
2452
2488
|
...opts.partition ? {
|
|
2453
2489
|
partition: opts.partition
|
|
2454
2490
|
} : {},
|
|
@@ -3867,6 +3903,12 @@ function matchesTenantPredicates(row, query) {
|
|
|
3867
3903
|
__name(matchesTenantPredicates, "matchesTenantPredicates");
|
|
3868
3904
|
|
|
3869
3905
|
// src/schedules.ts
|
|
3906
|
+
var ANNOUNCED_SCHEDULE_TAG = "schedule:announced";
|
|
3907
|
+
function announcedScheduleId(namespace, key) {
|
|
3908
|
+
return `app:${namespace}:${key}`;
|
|
3909
|
+
}
|
|
3910
|
+
__name(announcedScheduleId, "announcedScheduleId");
|
|
3911
|
+
var PARKED_FOR_WORKER = "no_live_worker";
|
|
3870
3912
|
var ScheduleAlreadyExistsError = class extends Error {
|
|
3871
3913
|
static {
|
|
3872
3914
|
__name(this, "ScheduleAlreadyExistsError");
|
|
@@ -4007,15 +4049,29 @@ var ScheduleClient = class {
|
|
|
4007
4049
|
async trigger(id) {
|
|
4008
4050
|
const record = await this.mustGet(id);
|
|
4009
4051
|
const runId = `sched:${id}:manual:${this.now()}`;
|
|
4010
|
-
return this.
|
|
4052
|
+
return this.startFor(record, runId);
|
|
4011
4053
|
}
|
|
4012
4054
|
/**
|
|
4013
|
-
*
|
|
4055
|
+
* Start one run of `record`: through the tenant start path for an announced schedule (its run waits
|
|
4056
|
+
* `blocked` for a live worker of its app instead of failing), else a plain `start`.
|
|
4057
|
+
*/
|
|
4058
|
+
startFor(record, runId) {
|
|
4059
|
+
const input = specOf(record).input;
|
|
4060
|
+
const opts = startOptions(record);
|
|
4061
|
+
if (isAnnounced(record) && this.host.startParked) {
|
|
4062
|
+
const singleton = record.state.announce?.singleton;
|
|
4063
|
+
return singleton ? this.host.startParked(record.workflow, input, runId, opts, singleton) : this.host.startParked(record.workflow, input, runId, opts);
|
|
4064
|
+
}
|
|
4065
|
+
return this.host.start(record.workflow, input, runId, opts);
|
|
4066
|
+
}
|
|
4067
|
+
/**
|
|
4068
|
+
* Fire every due schedule (in this engine's namespace; all namespaces for an operator engine; only
|
|
4069
|
+
* those carrying `opts.tag` when given):
|
|
4014
4070
|
* start its window's run and advance it to the next window. Called by the NestJS timer poller on
|
|
4015
4071
|
* each tick when `persistedSchedules` is on; call it yourself from any other driver. Returns the
|
|
4016
4072
|
* run ids started (or re-found, for an idempotent re-fire).
|
|
4017
4073
|
*/
|
|
4018
|
-
async tick(nowMs = this.now()) {
|
|
4074
|
+
async tick(nowMs = this.now(), opts = {}) {
|
|
4019
4075
|
const store = this.requireStore();
|
|
4020
4076
|
const started = [];
|
|
4021
4077
|
for (let round = 0; round < 50; round++) {
|
|
@@ -4023,6 +4079,9 @@ var ScheduleClient = class {
|
|
|
4023
4079
|
const due = await store.listSchedules({
|
|
4024
4080
|
dueBy: nowMs,
|
|
4025
4081
|
namespace: this.options.namespace,
|
|
4082
|
+
...opts.tag !== void 0 ? {
|
|
4083
|
+
tag: opts.tag
|
|
4084
|
+
} : {},
|
|
4026
4085
|
limit: TICK_BATCH
|
|
4027
4086
|
});
|
|
4028
4087
|
for (const record of due) {
|
|
@@ -4036,15 +4095,19 @@ var ScheduleClient = class {
|
|
|
4036
4095
|
}
|
|
4037
4096
|
/**
|
|
4038
4097
|
* When {@link tick} next has a schedule to fire, if one is due by `byMs`: the earliest `nextFireAt`
|
|
4039
|
-
* of the unpaused schedules (of this engine's namespace
|
|
4098
|
+
* of the unpaused schedules (of this engine's namespace, carrying `opts.tag` when given) due by
|
|
4099
|
+
* then, else `undefined`. One indexed
|
|
4040
4100
|
* read (the due-schedule scan `tick` makes, with a later bound and a limit of one). A store that
|
|
4041
4101
|
* does not persist schedules has none.
|
|
4042
4102
|
*/
|
|
4043
|
-
async nextFireAt(byMs) {
|
|
4103
|
+
async nextFireAt(byMs, opts = {}) {
|
|
4044
4104
|
if (!this.supported) return void 0;
|
|
4045
4105
|
const [next] = await this.requireStore().listSchedules({
|
|
4046
4106
|
dueBy: byMs,
|
|
4047
4107
|
namespace: this.options.namespace,
|
|
4108
|
+
...opts.tag !== void 0 ? {
|
|
4109
|
+
tag: opts.tag
|
|
4110
|
+
} : {},
|
|
4048
4111
|
limit: 1
|
|
4049
4112
|
});
|
|
4050
4113
|
return next?.nextFireAt ?? void 0;
|
|
@@ -4079,10 +4142,10 @@ var ScheduleClient = class {
|
|
|
4079
4142
|
let fired = false;
|
|
4080
4143
|
const skipMissed = spec.catchup === "skip" && !onTime;
|
|
4081
4144
|
const beforeWindow = latest < window;
|
|
4082
|
-
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state)) {
|
|
4145
|
+
if (!skipMissed && !beforeWindow && !await this.overlapping(spec, state) && !await this.waitingForWorker(record, state)) {
|
|
4083
4146
|
runId = `sched:${record.id}:${latest}`;
|
|
4084
4147
|
try {
|
|
4085
|
-
await this.
|
|
4148
|
+
await this.startFor(record, runId);
|
|
4086
4149
|
fired = true;
|
|
4087
4150
|
} catch (error2) {
|
|
4088
4151
|
if (await this.host.getRun(runId)) fired = true;
|
|
@@ -4106,6 +4169,186 @@ var ScheduleClient = class {
|
|
|
4106
4169
|
}, record.nextFireAt);
|
|
4107
4170
|
return fired ? runId : void 0;
|
|
4108
4171
|
}
|
|
4172
|
+
/**
|
|
4173
|
+
* An announced schedule whose previous run is still parked waiting for a live worker of its app
|
|
4174
|
+
* (the app is down): this window is folded into that run instead of parking one more. However long
|
|
4175
|
+
* the app is down, it finds at most one run per schedule waiting when it comes back, the same
|
|
4176
|
+
* bound `catchup: 'latest'` puts on windows missed while no scheduler polled.
|
|
4177
|
+
*/
|
|
4178
|
+
async waitingForWorker(record, state) {
|
|
4179
|
+
if (!isAnnounced(record) || !state.lastRunId) return false;
|
|
4180
|
+
const prev = await this.host.getRun(state.lastRunId);
|
|
4181
|
+
return prev?.status === "blocked" && prev.error?.code === PARKED_FOR_WORKER;
|
|
4182
|
+
}
|
|
4183
|
+
/**
|
|
4184
|
+
* Reconcile the schedules adopted from one app (`namespace`) with what its LIVE workers announce
|
|
4185
|
+
* now (`items`, one per key; the caller resolved disagreeing workers). The rule:
|
|
4186
|
+
* - an announced key with no schedule yet is created (id {@link announcedScheduleId}, tagged
|
|
4187
|
+
* {@link ANNOUNCED_SCHEDULE_TAG}, `catchup: 'latest'`); one whose definition changed is
|
|
4188
|
+
* re-applied like `upsert` (its history kept, its next fire recomputed from now). An equal
|
|
4189
|
+
* definition writes nothing, so any number of replicas announcing the same set is a no-op;
|
|
4190
|
+
* - a pause or resume made from the console survives: a re-applied definition keeps the current
|
|
4191
|
+
* paused state, unless the app's own declared `paused` changed;
|
|
4192
|
+
* - an adopted schedule the app no longer announces is marked missing, and paused ("withdrawn")
|
|
4193
|
+
* once it has stayed missing for `graceMs`. It is never deleted: its history stays, and an app
|
|
4194
|
+
* that announces it again (a rollback) gets it back, resumed unless an operator had paused it.
|
|
4195
|
+
*
|
|
4196
|
+
* Call it only for an app that has at least one live worker stating its schedules: with none (the
|
|
4197
|
+
* app is down, or runs an SDK that does not announce schedules), nothing is withdrawn, and its
|
|
4198
|
+
* schedules keep firing into runs that wait for a worker.
|
|
4199
|
+
*/
|
|
4200
|
+
async syncAnnounced(namespace, items, opts) {
|
|
4201
|
+
const store = this.requireStore();
|
|
4202
|
+
const now = opts.nowMs ?? this.now();
|
|
4203
|
+
const out = {
|
|
4204
|
+
adopted: [],
|
|
4205
|
+
restored: [],
|
|
4206
|
+
missing: [],
|
|
4207
|
+
withdrawn: [],
|
|
4208
|
+
invalid: []
|
|
4209
|
+
};
|
|
4210
|
+
const adopted = /* @__PURE__ */ new Map();
|
|
4211
|
+
for (let offset = 0; ; offset += TICK_BATCH) {
|
|
4212
|
+
const page = await store.listSchedules({
|
|
4213
|
+
namespace,
|
|
4214
|
+
tag: ANNOUNCED_SCHEDULE_TAG,
|
|
4215
|
+
limit: TICK_BATCH,
|
|
4216
|
+
offset
|
|
4217
|
+
});
|
|
4218
|
+
for (const record of page) if (isAnnounced(record)) adopted.set(record.id, record);
|
|
4219
|
+
if (page.length < TICK_BATCH) break;
|
|
4220
|
+
}
|
|
4221
|
+
for (const item of items) {
|
|
4222
|
+
const id = announcedScheduleId(namespace, item.key);
|
|
4223
|
+
const existing = adopted.get(id) ?? await store.getSchedule(id);
|
|
4224
|
+
adopted.delete(id);
|
|
4225
|
+
const doc = existing ? existing.state.announce : void 0;
|
|
4226
|
+
if (existing && !doc) {
|
|
4227
|
+
out.invalid.push({
|
|
4228
|
+
key: item.key,
|
|
4229
|
+
error: `schedule id "${id}" is taken`
|
|
4230
|
+
});
|
|
4231
|
+
continue;
|
|
4232
|
+
}
|
|
4233
|
+
const fingerprint = stableJson(item);
|
|
4234
|
+
if (existing && doc && doc.fingerprint === fingerprint) {
|
|
4235
|
+
if (doc.missingSince === void 0 && doc.withdrawnAt === void 0) continue;
|
|
4236
|
+
const resume = doc.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal && existing.paused;
|
|
4237
|
+
const spec = specOf(existing);
|
|
4238
|
+
const window = resume ? nextWindowAfter(spec, now) : void 0;
|
|
4239
|
+
await store.updateSchedule(id, {
|
|
4240
|
+
// The withdrawal note goes; the note it replaced comes back.
|
|
4241
|
+
...doc.withdrawnAt !== void 0 ? {
|
|
4242
|
+
spec: stripUndefined({
|
|
4243
|
+
...spec,
|
|
4244
|
+
note: doc.noteBeforeWithdrawal
|
|
4245
|
+
})
|
|
4246
|
+
} : {},
|
|
4247
|
+
...resume ? {
|
|
4248
|
+
paused: false,
|
|
4249
|
+
nextFireAt: window == null ? null : window + jitterFor(id, window, spec)
|
|
4250
|
+
} : {},
|
|
4251
|
+
state: stripUndefined({
|
|
4252
|
+
...existing.state,
|
|
4253
|
+
...resume ? {
|
|
4254
|
+
window
|
|
4255
|
+
} : {},
|
|
4256
|
+
announce: {
|
|
4257
|
+
key: doc.key,
|
|
4258
|
+
fingerprint,
|
|
4259
|
+
declaredPaused: doc.declaredPaused,
|
|
4260
|
+
singleton: doc.singleton
|
|
4261
|
+
}
|
|
4262
|
+
}),
|
|
4263
|
+
updatedAt: new Date(now)
|
|
4264
|
+
});
|
|
4265
|
+
out.restored.push(id);
|
|
4266
|
+
continue;
|
|
4267
|
+
}
|
|
4268
|
+
const declaredPaused = item.paused === true;
|
|
4269
|
+
const paused = existing === null || existing === void 0 ? declaredPaused : declaredPaused !== (doc?.declaredPaused === true) ? declaredPaused : doc?.withdrawnAt !== void 0 && !doc.pausedBeforeWithdrawal ? declaredPaused : void 0;
|
|
4270
|
+
try {
|
|
4271
|
+
await this.write({
|
|
4272
|
+
id,
|
|
4273
|
+
workflow: item.workflow,
|
|
4274
|
+
input: item.input,
|
|
4275
|
+
cron: item.cron,
|
|
4276
|
+
every: item.everyMs,
|
|
4277
|
+
timezone: item.timezone,
|
|
4278
|
+
jitter: item.jitterMs,
|
|
4279
|
+
overlap: item.overlap,
|
|
4280
|
+
catchup: "latest",
|
|
4281
|
+
namespace,
|
|
4282
|
+
tags: [
|
|
4283
|
+
ANNOUNCED_SCHEDULE_TAG
|
|
4284
|
+
],
|
|
4285
|
+
paused,
|
|
4286
|
+
// A note (an operator's pause reason) is kept; a withdrawal note is replaced by the one
|
|
4287
|
+
// it had replaced.
|
|
4288
|
+
note: doc?.withdrawnAt !== void 0 ? doc.noteBeforeWithdrawal : existing ? specOf(existing).note : void 0,
|
|
4289
|
+
// Fires belong to the app, never to whichever request context reached the sync.
|
|
4290
|
+
...existing ? {} : {
|
|
4291
|
+
tenantId: null
|
|
4292
|
+
}
|
|
4293
|
+
}, existing ?? null, {
|
|
4294
|
+
announce: {
|
|
4295
|
+
key: item.key,
|
|
4296
|
+
fingerprint,
|
|
4297
|
+
declaredPaused,
|
|
4298
|
+
...validSingleton(item.singleton) ? {
|
|
4299
|
+
singleton: item.singleton
|
|
4300
|
+
} : {}
|
|
4301
|
+
}
|
|
4302
|
+
});
|
|
4303
|
+
out.adopted.push(id);
|
|
4304
|
+
} catch (error2) {
|
|
4305
|
+
out.invalid.push({
|
|
4306
|
+
key: item.key,
|
|
4307
|
+
error: errorMessage(error2)
|
|
4308
|
+
});
|
|
4309
|
+
}
|
|
4310
|
+
}
|
|
4311
|
+
for (const record of adopted.values()) {
|
|
4312
|
+
const state = record.state;
|
|
4313
|
+
const doc = state.announce;
|
|
4314
|
+
if (doc.withdrawnAt !== void 0) continue;
|
|
4315
|
+
if (doc.missingSince === void 0) {
|
|
4316
|
+
await store.updateSchedule(record.id, {
|
|
4317
|
+
state: {
|
|
4318
|
+
...state,
|
|
4319
|
+
announce: {
|
|
4320
|
+
...doc,
|
|
4321
|
+
missingSince: now
|
|
4322
|
+
}
|
|
4323
|
+
},
|
|
4324
|
+
updatedAt: new Date(now)
|
|
4325
|
+
});
|
|
4326
|
+
out.missing.push(record.id);
|
|
4327
|
+
continue;
|
|
4328
|
+
}
|
|
4329
|
+
if (now - doc.missingSince < opts.graceMs) continue;
|
|
4330
|
+
const spec = {
|
|
4331
|
+
...specOf(record)
|
|
4332
|
+
};
|
|
4333
|
+
spec.note = `withdrawn: app "${namespace}" no longer announces schedule "${doc.key}"`;
|
|
4334
|
+
await store.updateSchedule(record.id, {
|
|
4335
|
+
paused: true,
|
|
4336
|
+
spec,
|
|
4337
|
+
state: {
|
|
4338
|
+
...state,
|
|
4339
|
+
announce: stripUndefined({
|
|
4340
|
+
...doc,
|
|
4341
|
+
withdrawnAt: now,
|
|
4342
|
+
pausedBeforeWithdrawal: record.paused,
|
|
4343
|
+
noteBeforeWithdrawal: specOf(record).note
|
|
4344
|
+
})
|
|
4345
|
+
},
|
|
4346
|
+
updatedAt: new Date(now)
|
|
4347
|
+
});
|
|
4348
|
+
out.withdrawn.push(record.id);
|
|
4349
|
+
}
|
|
4350
|
+
return out;
|
|
4351
|
+
}
|
|
4109
4352
|
/** `overlap: 'skip'` and the previous run of this schedule is still in flight. */
|
|
4110
4353
|
async overlapping(spec, state) {
|
|
4111
4354
|
if (spec.overlap !== "skip" || !state.lastRunId) return false;
|
|
@@ -4141,7 +4384,7 @@ var ScheduleClient = class {
|
|
|
4141
4384
|
tenantId: tenant.id
|
|
4142
4385
|
};
|
|
4143
4386
|
}
|
|
4144
|
-
async write(opts, existing) {
|
|
4387
|
+
async write(opts, existing, extraState = {}) {
|
|
4145
4388
|
const store = this.requireStore();
|
|
4146
4389
|
const spec = toSpec(opts);
|
|
4147
4390
|
const now = this.now();
|
|
@@ -4157,7 +4400,8 @@ var ScheduleClient = class {
|
|
|
4157
4400
|
spec,
|
|
4158
4401
|
state: stripUndefined({
|
|
4159
4402
|
...existing?.state ?? {},
|
|
4160
|
-
window
|
|
4403
|
+
window,
|
|
4404
|
+
...extraState
|
|
4161
4405
|
}),
|
|
4162
4406
|
...this.scheduleTenant(opts, existing),
|
|
4163
4407
|
createdAt: existing?.createdAt ?? new Date(now),
|
|
@@ -4203,6 +4447,14 @@ function specOf(record) {
|
|
|
4203
4447
|
return record.spec;
|
|
4204
4448
|
}
|
|
4205
4449
|
__name(specOf, "specOf");
|
|
4450
|
+
function validSingleton(value) {
|
|
4451
|
+
return value !== null && typeof value === "object" && typeof value.key === "string" && value.key !== "";
|
|
4452
|
+
}
|
|
4453
|
+
__name(validSingleton, "validSingleton");
|
|
4454
|
+
function isAnnounced(record) {
|
|
4455
|
+
return record.state?.announce !== void 0;
|
|
4456
|
+
}
|
|
4457
|
+
__name(isAnnounced, "isAnnounced");
|
|
4206
4458
|
function latestWindowAtOrBefore(spec, atMs) {
|
|
4207
4459
|
if (spec.cron != null) return prevCronFireMs(spec.cron, atMs, spec.timezone);
|
|
4208
4460
|
if (spec.rrule != null) {
|
|
@@ -4277,6 +4529,9 @@ function describe(record) {
|
|
|
4277
4529
|
lastError: state.lastError,
|
|
4278
4530
|
tenantId: record.tenantId ?? void 0,
|
|
4279
4531
|
tenantKind: record.tenantKind ?? void 0,
|
|
4532
|
+
announcedKey: state.announce?.key,
|
|
4533
|
+
unannouncedSince: state.announce?.missingSince != null ? new Date(state.announce.missingSince) : void 0,
|
|
4534
|
+
withdrawnAt: state.announce?.withdrawnAt != null ? new Date(state.announce.withdrawnAt) : void 0,
|
|
4280
4535
|
createdAt: record.createdAt,
|
|
4281
4536
|
updatedAt: record.updatedAt
|
|
4282
4537
|
});
|
|
@@ -6639,6 +6894,43 @@ var DEFAULT_STICKY_RUNS = 1e3;
|
|
|
6639
6894
|
var DEFAULT_STICKY_IDLE_MS = 3e5;
|
|
6640
6895
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
6641
6896
|
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
6897
|
+
var ANNOUNCED_TIMEOUTS_TTL_MS = 15e3;
|
|
6898
|
+
function tenantScopedKey(namespace, key) {
|
|
6899
|
+
return namespace && namespace !== "default" ? `${namespace}/${key}` : key;
|
|
6900
|
+
}
|
|
6901
|
+
__name(tenantScopedKey, "tenantScopedKey");
|
|
6902
|
+
function singletonKeyOf(run) {
|
|
6903
|
+
const tag = run.tags?.find((t) => t.startsWith("singleton:"));
|
|
6904
|
+
return tag?.slice("singleton:".length);
|
|
6905
|
+
}
|
|
6906
|
+
__name(singletonKeyOf, "singletonKeyOf");
|
|
6907
|
+
function conventionSingleton(run, announced, wire) {
|
|
6908
|
+
if (wire !== void 0) {
|
|
6909
|
+
const key2 = wire.key;
|
|
6910
|
+
return {
|
|
6911
|
+
key: /* @__PURE__ */ __name(() => key2, "key"),
|
|
6912
|
+
...typeof wire.limit === "number" ? {
|
|
6913
|
+
limit: wire.limit
|
|
6914
|
+
} : {},
|
|
6915
|
+
...typeof wire.maxQueueDepth === "number" ? {
|
|
6916
|
+
maxQueueDepth: wire.maxQueueDepth
|
|
6917
|
+
} : {}
|
|
6918
|
+
};
|
|
6919
|
+
}
|
|
6920
|
+
const key = singletonKeyOf(run);
|
|
6921
|
+
if (key === void 0 || announced === void 0) return void 0;
|
|
6922
|
+
if (announced === null) return {
|
|
6923
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
6924
|
+
limit: 1
|
|
6925
|
+
};
|
|
6926
|
+
const declared = announced.options?.singleton;
|
|
6927
|
+
if (declared === void 0) return void 0;
|
|
6928
|
+
return {
|
|
6929
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
6930
|
+
limit: typeof declared.limit === "number" ? declared.limit : 1
|
|
6931
|
+
};
|
|
6932
|
+
}
|
|
6933
|
+
__name(conventionSingleton, "conventionSingleton");
|
|
6642
6934
|
function servedNamespace(namespace, group) {
|
|
6643
6935
|
if (namespace) return namespace;
|
|
6644
6936
|
const at = group?.lastIndexOf("@") ?? -1;
|
|
@@ -6663,6 +6955,7 @@ function runNamespace(namespace) {
|
|
|
6663
6955
|
__name(runNamespace, "runNamespace");
|
|
6664
6956
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
6665
6957
|
var NO_LIVE_WORKER = "no_live_worker";
|
|
6958
|
+
var DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS = 10 * 6e4;
|
|
6666
6959
|
var TenantStartInvalid = class TenantStartInvalid2 extends Error {
|
|
6667
6960
|
static {
|
|
6668
6961
|
__name(this, "TenantStartInvalid");
|
|
@@ -6876,6 +7169,10 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
6876
7169
|
remoteRedispatchMax;
|
|
6877
7170
|
/** How long a `blocked` run waits before the recovery poll re-checks the live fleet (design §7.5). */
|
|
6878
7171
|
blockedPollMs;
|
|
7172
|
+
/** Verifies the wire tenant of a start-run, and of a worker's schedule announcement. */
|
|
7173
|
+
verifyTenant;
|
|
7174
|
+
/** Wire tenants whose schedule announcement failed verification, warned about once. */
|
|
7175
|
+
rejectedScheduleTenants = /* @__PURE__ */ new Set();
|
|
6879
7176
|
workerDescriptorCacheMs;
|
|
6880
7177
|
descriptorCache = /* @__PURE__ */ new Map();
|
|
6881
7178
|
/** This engine's control-plane handshake descriptor — negotiated against each worker (design §7.3). */
|
|
@@ -7086,6 +7383,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7086
7383
|
this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
|
|
7087
7384
|
this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
|
|
7088
7385
|
this.blockedPollMs = deps.blockedPollMs ?? 5e3;
|
|
7386
|
+
this.verifyTenant = deps.verifyTenant;
|
|
7089
7387
|
this.workerDescriptorCacheMs = deps.workerDescriptorCacheMs ?? 2e3;
|
|
7090
7388
|
this.cpDescriptor = controlPlaneDescriptor({
|
|
7091
7389
|
instanceId: `cp-${this.instanceId}`,
|
|
@@ -7122,10 +7420,28 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7122
7420
|
if (this.queuesRunsOverTransport) void this.wake(runId);
|
|
7123
7421
|
else this.runDispatcher.dispatch(runId);
|
|
7124
7422
|
}, "dispatch"),
|
|
7125
|
-
configFor: /* @__PURE__ */ __name((run) =>
|
|
7423
|
+
configFor: /* @__PURE__ */ __name((run) => {
|
|
7424
|
+
const local = (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton;
|
|
7425
|
+
if (local) return local;
|
|
7426
|
+
const key = this.latest.has(run.workflow) ? void 0 : singletonKeyOf(run);
|
|
7427
|
+
if (key === void 0) return void 0;
|
|
7428
|
+
const limit = this.conventionSingletonLimits.get(`${runNamespace(run.namespace)}\0${run.workflow}`);
|
|
7429
|
+
return {
|
|
7430
|
+
key: /* @__PURE__ */ __name(() => key, "key"),
|
|
7431
|
+
...limit !== void 0 ? {
|
|
7432
|
+
limit
|
|
7433
|
+
} : {}
|
|
7434
|
+
};
|
|
7435
|
+
}, "configFor")
|
|
7126
7436
|
});
|
|
7127
7437
|
this.schedules = new ScheduleClient(this.store, {
|
|
7128
7438
|
start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
|
|
7439
|
+
// An announced (tenant) schedule fires through the tenant start path: with no live worker
|
|
7440
|
+
// of its app, the run waits `blocked` instead of the window being lost.
|
|
7441
|
+
startParked: /* @__PURE__ */ __name((workflow, input, runId, opts, singleton) => this.startWith(workflow, input, runId, opts, true, singleton ? {
|
|
7442
|
+
...singleton,
|
|
7443
|
+
key: tenantScopedKey(opts?.namespace, singleton.key)
|
|
7444
|
+
} : void 0), "startParked"),
|
|
7129
7445
|
getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
|
|
7130
7446
|
isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining"),
|
|
7131
7447
|
tenant: /* @__PURE__ */ __name(() => this.tenancy?.current(), "tenant"),
|
|
@@ -7299,14 +7615,17 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7299
7615
|
* The earliest known time (epoch ms) the poll has work, if it is at or before `byMs`: this
|
|
7300
7616
|
* namespace's next due timer (`listDueTimers` up to `byMs`, limit one: it returns the most overdue
|
|
7301
7617
|
* first), the next `blocked` run's recovery poll, and with `persistedSchedules` the next persisted
|
|
7302
|
-
* schedule to fire (see {@link ScheduleClient.nextFireAt}
|
|
7618
|
+
* schedule to fire (see {@link ScheduleClient.nextFireAt}; with `announcedSchedules` alone, the next
|
|
7619
|
+
* schedule adopted from a tenant). `undefined` when nothing is due by
|
|
7303
7620
|
* then. A poller reads it before waiting longer than its tick, so the wait never outlasts a timer.
|
|
7304
7621
|
*/
|
|
7305
7622
|
async nextDueAt(byMs, opts = {}) {
|
|
7306
7623
|
const [timers, blocked, schedule] = await Promise.all([
|
|
7307
7624
|
this.store.listDueTimers(byMs, this.namespace, 1),
|
|
7308
7625
|
this.dueBlockedRuns(byMs),
|
|
7309
|
-
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) :
|
|
7626
|
+
opts.persistedSchedules ? this.schedules.nextFireAt(byMs) : opts.announcedSchedules && this.schedules.supported ? this.schedules.nextFireAt(byMs, {
|
|
7627
|
+
tag: ANNOUNCED_SCHEDULE_TAG
|
|
7628
|
+
}) : void 0
|
|
7310
7629
|
]);
|
|
7311
7630
|
let next;
|
|
7312
7631
|
for (const at of [
|
|
@@ -7932,6 +8251,39 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7932
8251
|
} catch {
|
|
7933
8252
|
}
|
|
7934
8253
|
}
|
|
8254
|
+
if (!this.latest.has(run.workflow)) this.inBackground(this.routeAnnouncedDeadLetter(run));
|
|
8255
|
+
}
|
|
8256
|
+
/**
|
|
8257
|
+
* The dead-letter workflow the live workers serving `run`'s workflow announce (a tenant's inline
|
|
8258
|
+
* `@DeadLetter()` handler, its `deadLetterWorkflow`, or its module-level default), or `undefined`
|
|
8259
|
+
* when the workflow is registered on this engine (its own routing applies), they announce none, or
|
|
8260
|
+
* they cannot be read. A dead-letter listener with a catch-all default (`DurableModule`'s
|
|
8261
|
+
* `deadLetterWorkflow`) checks it first, so the app's own route wins over the default.
|
|
8262
|
+
*/
|
|
8263
|
+
async announcedDeadLetter(run) {
|
|
8264
|
+
if (this.latest.has(run.workflow) || this.isRegisteredHere(run)) return void 0;
|
|
8265
|
+
const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
|
|
8266
|
+
const reg = await this.announcedRegistration(group, run.workflow, run.workflowVersion);
|
|
8267
|
+
const target = reg?.deadLetter;
|
|
8268
|
+
return typeof target === "string" && target !== "" ? target : void 0;
|
|
8269
|
+
}
|
|
8270
|
+
/**
|
|
8271
|
+
* Start the announced dead-letter workflow of a dead run (see {@link announcedDeadLetter}) in the
|
|
8272
|
+
* run's namespace, as `dlq:<runId>` with the `DeadLetter` payload, through the tenant start path:
|
|
8273
|
+
* with no live worker of the app it waits `blocked` instead of being lost. Idempotent by run id.
|
|
8274
|
+
*/
|
|
8275
|
+
async routeAnnouncedDeadLetter(run) {
|
|
8276
|
+
const target = await this.announcedDeadLetter(run);
|
|
8277
|
+
if (!target) return;
|
|
8278
|
+
await this.startWith(target, {
|
|
8279
|
+
deadRunId: run.id,
|
|
8280
|
+
workflow: run.workflow,
|
|
8281
|
+
input: run.input,
|
|
8282
|
+
error: run.error
|
|
8283
|
+
}, `dlq:${run.id}`, {
|
|
8284
|
+
namespace: run.namespace,
|
|
8285
|
+
...inheritTenant(run)
|
|
8286
|
+
}, true);
|
|
7935
8287
|
}
|
|
7936
8288
|
/**
|
|
7937
8289
|
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
@@ -9232,7 +9584,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9232
9584
|
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9233
9585
|
* a start whose caller has already returned is never lost.
|
|
9234
9586
|
*/
|
|
9235
|
-
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9587
|
+
async startWith(workflow, input, runId, opts, parkUnroutable, wireSingleton) {
|
|
9236
9588
|
const name = workflowName(workflow);
|
|
9237
9589
|
const pinned = opts?.version;
|
|
9238
9590
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9257,12 +9609,15 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9257
9609
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9258
9610
|
};
|
|
9259
9611
|
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9260
|
-
askFleetForVersion: true
|
|
9612
|
+
askFleetForVersion: true,
|
|
9613
|
+
...wireSingleton !== void 0 ? {
|
|
9614
|
+
singleton: wireSingleton
|
|
9615
|
+
} : {}
|
|
9261
9616
|
});
|
|
9262
9617
|
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9263
9618
|
}
|
|
9264
9619
|
if (!registered && parkUnroutable) {
|
|
9265
|
-
return this.parkStartForWorker(name, input, runId, opts);
|
|
9620
|
+
return this.parkStartForWorker(name, input, runId, opts, wireSingleton);
|
|
9266
9621
|
}
|
|
9267
9622
|
if (!registered) {
|
|
9268
9623
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
@@ -9364,11 +9719,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9364
9719
|
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9365
9720
|
* start is checked first at its `startAt`.
|
|
9366
9721
|
*/
|
|
9367
|
-
async parkStartForWorker(name, input, runId, opts) {
|
|
9722
|
+
async parkStartForWorker(name, input, runId, opts, wireSingleton) {
|
|
9368
9723
|
const prior = await this.store.getRun(runId);
|
|
9369
9724
|
if (prior) return this.existingStartResult(prior);
|
|
9370
9725
|
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9371
9726
|
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9727
|
+
const singleton = conventionSingleton({}, void 0, wireSingleton);
|
|
9728
|
+
if (singleton) await this.singletons.assertCapacity(name, singleton, input);
|
|
9372
9729
|
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9373
9730
|
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9374
9731
|
const pinned = opts?.version;
|
|
@@ -9383,6 +9740,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9383
9740
|
const now = /* @__PURE__ */ new Date();
|
|
9384
9741
|
const tags = mergeTags(void 0, [
|
|
9385
9742
|
...opts?.tags ?? [],
|
|
9743
|
+
...singleton ? [
|
|
9744
|
+
this.singletons.tag(singleton, input)
|
|
9745
|
+
] : [],
|
|
9386
9746
|
...quota ? [
|
|
9387
9747
|
concurrencyTag(quota.key)
|
|
9388
9748
|
] : [],
|
|
@@ -9471,8 +9831,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9471
9831
|
concurrency: message.concurrency
|
|
9472
9832
|
} : {}
|
|
9473
9833
|
};
|
|
9834
|
+
const singleton = message.singleton !== void 0 && typeof message.singleton?.key === "string" ? {
|
|
9835
|
+
...message.singleton,
|
|
9836
|
+
key: tenantScopedKey(namespace, message.singleton.key)
|
|
9837
|
+
} : void 0;
|
|
9474
9838
|
try {
|
|
9475
|
-
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
9839
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true, singleton);
|
|
9476
9840
|
} catch (err) {
|
|
9477
9841
|
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
9478
9842
|
throw err;
|
|
@@ -9480,6 +9844,62 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9480
9844
|
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
9481
9845
|
}
|
|
9482
9846
|
}
|
|
9847
|
+
/**
|
|
9848
|
+
* The control plane's half of a tenant's `signalWithStart` (a `signalWithStart` run request):
|
|
9849
|
+
* start `start` like {@link startForTenant} unless its run exists, then signal `signal.token`.
|
|
9850
|
+
* Authorization is the caller's (the run-request responder): the run must belong to `namespace`,
|
|
9851
|
+
* and so must a run already waiting on the token. Delivered at once to a waiting run; with none
|
|
9852
|
+
* waiting yet it is buffered only when the token contains the run id (a run-scoped token), so a
|
|
9853
|
+
* tenant can never plant a payload for a wait another namespace opens later. `signalled: false`
|
|
9854
|
+
* when it was neither delivered nor buffered.
|
|
9855
|
+
*/
|
|
9856
|
+
async signalWithStartForTenant(start, namespace, tenant, signal) {
|
|
9857
|
+
const started = await this.startForTenant(start, namespace, tenant);
|
|
9858
|
+
if (TERMINAL_RUN_STATUSES.includes(started.status)) {
|
|
9859
|
+
return {
|
|
9860
|
+
runId: start.runId,
|
|
9861
|
+
signalled: false
|
|
9862
|
+
};
|
|
9863
|
+
}
|
|
9864
|
+
const waiter = (await this.store.listSignalWaiters(signal.token)).find((w) => w.token === signal.token);
|
|
9865
|
+
if (waiter) {
|
|
9866
|
+
const run = await this.store.getRun(waiter.runId);
|
|
9867
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) {
|
|
9868
|
+
return {
|
|
9869
|
+
runId: start.runId,
|
|
9870
|
+
signalled: false
|
|
9871
|
+
};
|
|
9872
|
+
}
|
|
9873
|
+
} else if (!signal.token.includes(start.runId)) {
|
|
9874
|
+
return {
|
|
9875
|
+
runId: start.runId,
|
|
9876
|
+
signalled: false
|
|
9877
|
+
};
|
|
9878
|
+
}
|
|
9879
|
+
await this.signalUnchecked(signal.token, signal.payload);
|
|
9880
|
+
return {
|
|
9881
|
+
runId: start.runId,
|
|
9882
|
+
signalled: true
|
|
9883
|
+
};
|
|
9884
|
+
}
|
|
9885
|
+
/**
|
|
9886
|
+
* The control plane's half of a tenant's `publishEvent` (a `publishEvent` run request): resume the
|
|
9887
|
+
* runs of `namespace` waiting on event `name` whose match the payload satisfies. Starting the
|
|
9888
|
+
* tenant's own subscribers is the tenant's (it holds their `where` and `input`); nothing is
|
|
9889
|
+
* buffered, since a buffered event could reach another namespace's later wait. Returns how many
|
|
9890
|
+
* runs it resumed.
|
|
9891
|
+
*/
|
|
9892
|
+
async publishEventForTenant(name, payload, namespace) {
|
|
9893
|
+
let touched = 0;
|
|
9894
|
+
for (const w of await this.store.listSignalWaiters(eventPrefix(name))) {
|
|
9895
|
+
if (!eventMatches(payload, eventMatchOf(w.token))) continue;
|
|
9896
|
+
const run = await this.store.getRun(w.runId);
|
|
9897
|
+
if (!run || runNamespace(run.namespace) !== runNamespace(namespace)) continue;
|
|
9898
|
+
await this.signalUnchecked(w.token, payload);
|
|
9899
|
+
touched += 1;
|
|
9900
|
+
}
|
|
9901
|
+
return touched;
|
|
9902
|
+
}
|
|
9483
9903
|
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
9484
9904
|
async recordRejectedStart(name, input, runId, opts, err) {
|
|
9485
9905
|
const prior = await this.store.getRun(runId);
|
|
@@ -9951,9 +10371,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9951
10371
|
if (!liveTransport) return void 0;
|
|
9952
10372
|
const executor = new RemoteWorkflowExecutor(liveTransport, run.workflow, run.namespace);
|
|
9953
10373
|
const declared = opts.askFleetForVersion ? await this.declaredVersion(liveTransport, group, run.workflow) : void 0;
|
|
10374
|
+
const version = declared ?? run.workflowVersion;
|
|
10375
|
+
const announced = await this.announcedRegistration(group, run.workflow, version);
|
|
10376
|
+
const singleton = conventionSingleton(run, announced, opts.singleton);
|
|
10377
|
+
if (singleton?.limit !== void 0) {
|
|
10378
|
+
this.conventionSingletonLimits.set(`${runNamespace(run.namespace)}\0${run.workflow}`, singleton.limit);
|
|
10379
|
+
}
|
|
10380
|
+
const tags = announced?.options?.tags;
|
|
9954
10381
|
return {
|
|
9955
10382
|
name: run.workflow,
|
|
9956
|
-
version
|
|
10383
|
+
version,
|
|
9957
10384
|
fn: /* @__PURE__ */ __name(() => {
|
|
9958
10385
|
throw new Error(`remote workflow ${run.workflow} has no local body`);
|
|
9959
10386
|
}, "fn"),
|
|
@@ -9962,12 +10389,47 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9962
10389
|
group,
|
|
9963
10390
|
executor
|
|
9964
10391
|
},
|
|
10392
|
+
...Array.isArray(tags) && tags.length > 0 ? {
|
|
10393
|
+
tags
|
|
10394
|
+
} : {},
|
|
10395
|
+
...singleton ? {
|
|
10396
|
+
singleton
|
|
10397
|
+
} : {},
|
|
9965
10398
|
...opts.askFleetForVersion ? {
|
|
9966
10399
|
versionDeclared: declared !== void 0
|
|
9967
10400
|
} : {}
|
|
9968
10401
|
};
|
|
9969
10402
|
}
|
|
9970
10403
|
/**
|
|
10404
|
+
* The registration the live workers of `group` announce for `workflow` (at `version`, when one of
|
|
10405
|
+
* them states it): where they disagree (a rolling deploy), the one from the worker that started
|
|
10406
|
+
* last, as for announced schedules. `undefined` when none announces it, `null` when the read
|
|
10407
|
+
* failed (what the workers state is then unknown, not "nothing"). Reads
|
|
10408
|
+
* through the descriptor cache ({@link workerDescriptorCacheMs}), so a turn costs no extra round
|
|
10409
|
+
* trip in steady state.
|
|
10410
|
+
*/
|
|
10411
|
+
async announcedRegistration(group, workflow, version) {
|
|
10412
|
+
let descriptors;
|
|
10413
|
+
try {
|
|
10414
|
+
descriptors = await this.liveWorkerDescriptors(group);
|
|
10415
|
+
} catch {
|
|
10416
|
+
return null;
|
|
10417
|
+
}
|
|
10418
|
+
let best;
|
|
10419
|
+
for (const descriptor of descriptors) {
|
|
10420
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10421
|
+
if (reg?.name !== workflow) continue;
|
|
10422
|
+
if (version !== void 0 && reg.version !== void 0 && reg.version !== version) continue;
|
|
10423
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10424
|
+
if (!best || at > best.at) best = {
|
|
10425
|
+
at,
|
|
10426
|
+
reg
|
|
10427
|
+
};
|
|
10428
|
+
}
|
|
10429
|
+
}
|
|
10430
|
+
return best?.reg;
|
|
10431
|
+
}
|
|
10432
|
+
/**
|
|
9971
10433
|
* The version the LIVE fleet declares for `workflow` on `group`, or `undefined` when it declares
|
|
9972
10434
|
* none — which is a fact about the fleet, not a failure to read it.
|
|
9973
10435
|
*
|
|
@@ -10191,34 +10653,95 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10191
10653
|
for (const reg of new Set(this.latest.values())) {
|
|
10192
10654
|
if (this.draining) return;
|
|
10193
10655
|
if (reg.executionTimeoutMs == null) continue;
|
|
10194
|
-
|
|
10195
|
-
|
|
10196
|
-
|
|
10197
|
-
|
|
10198
|
-
|
|
10199
|
-
|
|
10200
|
-
|
|
10201
|
-
|
|
10202
|
-
|
|
10203
|
-
|
|
10204
|
-
|
|
10205
|
-
|
|
10206
|
-
|
|
10207
|
-
|
|
10208
|
-
|
|
10209
|
-
|
|
10210
|
-
|
|
10211
|
-
|
|
10212
|
-
|
|
10213
|
-
|
|
10214
|
-
|
|
10215
|
-
|
|
10216
|
-
|
|
10217
|
-
|
|
10218
|
-
|
|
10219
|
-
|
|
10220
|
-
|
|
10656
|
+
await this.cancelTimedOut(reg.name, this.namespace, now - reg.executionTimeoutMs);
|
|
10657
|
+
}
|
|
10658
|
+
for (const { workflow, namespace, timeoutMs } of await this.announcedTimeouts(now)) {
|
|
10659
|
+
if (this.draining) return;
|
|
10660
|
+
await this.cancelTimedOut(workflow, namespace, now - timeoutMs);
|
|
10661
|
+
}
|
|
10662
|
+
}
|
|
10663
|
+
/**
|
|
10664
|
+
* The singleton limit last resolved for a workflow served by convention, by `<namespace>\0<name>`
|
|
10665
|
+
* (see {@link resolveRemoteByConvention}): what the singleton gate's release wake-up reads, since
|
|
10666
|
+
* it cannot wait on the fleet. One entry per convention singleton the fleet serves.
|
|
10667
|
+
*/
|
|
10668
|
+
conventionSingletonLimits = /* @__PURE__ */ new Map();
|
|
10669
|
+
/** The last read of the announced execution timeouts (see {@link announcedTimeouts}). */
|
|
10670
|
+
announcedTimeoutsRead;
|
|
10671
|
+
/**
|
|
10672
|
+
* The `executionTimeoutMs` live workers announce (`@Workflow({ executionTimeout })` on a tenant),
|
|
10673
|
+
* per workflow and namespace, for workflows this engine does not register (a registration here
|
|
10674
|
+
* applies its own). Where workers disagree, the one that started last wins. Read at most once per
|
|
10675
|
+
* {@link ANNOUNCED_TIMEOUTS_TTL_MS}; empty without a transport that reads the fleet, or when the
|
|
10676
|
+
* read fails.
|
|
10677
|
+
*/
|
|
10678
|
+
async announcedTimeouts(now) {
|
|
10679
|
+
if (!this.pool.introspectsFleet) return [];
|
|
10680
|
+
const cached = this.announcedTimeoutsRead;
|
|
10681
|
+
if (cached && now - cached.at < ANNOUNCED_TIMEOUTS_TTL_MS) return cached.list;
|
|
10682
|
+
const newest = /* @__PURE__ */ new Map();
|
|
10683
|
+
try {
|
|
10684
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
10685
|
+
const at = Number(descriptor.startedAt) || 0;
|
|
10686
|
+
for (const reg of descriptor.registrations ?? []) {
|
|
10687
|
+
if (!reg || typeof reg.name !== "string" || this.latest.has(reg.name)) continue;
|
|
10688
|
+
const namespace = servedNamespace(descriptor.namespace, reg.group);
|
|
10689
|
+
if (this.namespace !== void 0 && runNamespace(this.namespace) !== namespace) continue;
|
|
10690
|
+
const key = `${namespace}\0${reg.name}`;
|
|
10691
|
+
const prior = newest.get(key);
|
|
10692
|
+
if (prior && prior.at >= at) continue;
|
|
10693
|
+
newest.set(key, {
|
|
10694
|
+
at,
|
|
10695
|
+
workflow: reg.name,
|
|
10696
|
+
namespace,
|
|
10697
|
+
ms: Number(reg.options?.executionTimeoutMs)
|
|
10698
|
+
});
|
|
10699
|
+
}
|
|
10221
10700
|
}
|
|
10701
|
+
} catch {
|
|
10702
|
+
return cached?.list ?? [];
|
|
10703
|
+
}
|
|
10704
|
+
const list2 = [
|
|
10705
|
+
...newest.values()
|
|
10706
|
+
].filter((e) => Number.isFinite(e.ms) && e.ms > 0).map((e) => ({
|
|
10707
|
+
workflow: e.workflow,
|
|
10708
|
+
namespace: e.namespace,
|
|
10709
|
+
timeoutMs: e.ms
|
|
10710
|
+
}));
|
|
10711
|
+
this.announcedTimeoutsRead = {
|
|
10712
|
+
at: now,
|
|
10713
|
+
list: list2
|
|
10714
|
+
};
|
|
10715
|
+
return list2;
|
|
10716
|
+
}
|
|
10717
|
+
/** Cancel (`execution_timeout`) the in-flight runs of `workflow` in `namespace` created before `deadline`. */
|
|
10718
|
+
async cancelTimedOut(workflow, namespace, deadline) {
|
|
10719
|
+
const inflight = await this.store.listRuns({
|
|
10720
|
+
workflow,
|
|
10721
|
+
statuses: TIMEOUT_SWEPT_STATUSES,
|
|
10722
|
+
namespace
|
|
10723
|
+
});
|
|
10724
|
+
for (const run of inflight) {
|
|
10725
|
+
if (this.draining) return;
|
|
10726
|
+
if (run.createdAt.getTime() > deadline) continue;
|
|
10727
|
+
const error2 = {
|
|
10728
|
+
message: "execution timeout",
|
|
10729
|
+
code: "execution_timeout"
|
|
10730
|
+
};
|
|
10731
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
10732
|
+
status: "cancelled",
|
|
10733
|
+
error: error2,
|
|
10734
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
10735
|
+
}));
|
|
10736
|
+
this.emit({
|
|
10737
|
+
type: "run.failed",
|
|
10738
|
+
runId: run.id,
|
|
10739
|
+
workflow: run.workflow,
|
|
10740
|
+
namespace: run.namespace,
|
|
10741
|
+
error: error2
|
|
10742
|
+
});
|
|
10743
|
+
await this.cancelChildren(run.id).catch(() => void 0);
|
|
10744
|
+
this.notifyParent(run.id, cancelledCompletion(error2));
|
|
10222
10745
|
}
|
|
10223
10746
|
}
|
|
10224
10747
|
/**
|
|
@@ -11935,6 +12458,75 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
11935
12458
|
*
|
|
11936
12459
|
* Workflows only, and deliberately: see the "steps are out of scope" note in `handshake/announced`.
|
|
11937
12460
|
*/
|
|
12461
|
+
/**
|
|
12462
|
+
* Adopt the schedules tenant apps announce (their `schedules` module option, carried on their live
|
|
12463
|
+
* workers' descriptors) as persisted schedules, and withdraw the ones an app stopped announcing.
|
|
12464
|
+
* The control plane's timer poll calls it; {@link ScheduleClient.tick} fires them.
|
|
12465
|
+
*
|
|
12466
|
+
* Per app (the namespace its announcement's verified `tenant` maps to, as for a start-run): the
|
|
12467
|
+
* schedules announced by its live workers, where two workers disagree on a key the one that
|
|
12468
|
+
* started last (its deploy is the newer one) wins. An app with no live worker that states its
|
|
12469
|
+
* schedules is left alone, so its schedules keep firing into runs that wait for a worker. See
|
|
12470
|
+
* {@link ScheduleClient.syncAnnounced} for the reconciliation rule and `graceMs` (default 10
|
|
12471
|
+
* minutes). A namespaced engine adopts only its own namespace's. Nothing without a store that
|
|
12472
|
+
* persists schedules or a transport that reads worker descriptors.
|
|
12473
|
+
*/
|
|
12474
|
+
async syncAnnouncedSchedules(opts = {}) {
|
|
12475
|
+
const out = /* @__PURE__ */ new Map();
|
|
12476
|
+
if (!this.schedules.supported) return out;
|
|
12477
|
+
const byNamespace = /* @__PURE__ */ new Map();
|
|
12478
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
12479
|
+
const announced = descriptor.schedules;
|
|
12480
|
+
if (!announced || typeof announced.tenant !== "string" || !Array.isArray(announced.items)) {
|
|
12481
|
+
continue;
|
|
12482
|
+
}
|
|
12483
|
+
let verified;
|
|
12484
|
+
try {
|
|
12485
|
+
verified = await verifyWireTenant(announced.tenant, this.verifyTenant);
|
|
12486
|
+
} catch {
|
|
12487
|
+
verified = null;
|
|
12488
|
+
}
|
|
12489
|
+
const tenant = verified?.tenant ?? null;
|
|
12490
|
+
if (tenant === null) {
|
|
12491
|
+
if (!this.rejectedScheduleTenants.has(descriptor.instanceId)) {
|
|
12492
|
+
this.rejectedScheduleTenants.add(descriptor.instanceId);
|
|
12493
|
+
console.warn(`[nestjs-durable] ignored the schedules worker ${descriptor.instanceId} announces: invalid or unauthorized tenant token`);
|
|
12494
|
+
}
|
|
12495
|
+
continue;
|
|
12496
|
+
}
|
|
12497
|
+
const namespace = poolNamespace(tenant) ?? "default";
|
|
12498
|
+
if (this.namespace !== void 0 && namespace !== this.namespace) continue;
|
|
12499
|
+
const list2 = byNamespace.get(namespace) ?? [];
|
|
12500
|
+
list2.push({
|
|
12501
|
+
startedAt: Number(descriptor.startedAt) || 0,
|
|
12502
|
+
items: announced.items
|
|
12503
|
+
});
|
|
12504
|
+
byNamespace.set(namespace, list2);
|
|
12505
|
+
}
|
|
12506
|
+
for (const [namespace, announcers] of byNamespace) {
|
|
12507
|
+
const items = /* @__PURE__ */ new Map();
|
|
12508
|
+
for (const { items: list2 } of announcers.sort((a, b) => b.startedAt - a.startedAt)) {
|
|
12509
|
+
for (const item of list2) {
|
|
12510
|
+
if (item && typeof item.key === "string" && !items.has(item.key)) {
|
|
12511
|
+
items.set(item.key, item);
|
|
12512
|
+
}
|
|
12513
|
+
}
|
|
12514
|
+
}
|
|
12515
|
+
const result = await this.schedules.syncAnnounced(namespace, [
|
|
12516
|
+
...items.values()
|
|
12517
|
+
], {
|
|
12518
|
+
graceMs: opts.graceMs ?? DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
12519
|
+
...opts.nowMs !== void 0 ? {
|
|
12520
|
+
nowMs: opts.nowMs
|
|
12521
|
+
} : {}
|
|
12522
|
+
});
|
|
12523
|
+
for (const { key, error: error2 } of result.invalid) {
|
|
12524
|
+
console.warn(`[nestjs-durable] app "${namespace}" announces schedule "${key}", which cannot be adopted: ${error2}`);
|
|
12525
|
+
}
|
|
12526
|
+
out.set(namespace, result);
|
|
12527
|
+
}
|
|
12528
|
+
return out;
|
|
12529
|
+
}
|
|
11938
12530
|
async announcedWorkflows() {
|
|
11939
12531
|
return (await this.workflowDirectory()).workflows;
|
|
11940
12532
|
}
|
|
@@ -17797,6 +18389,7 @@ async function settleResultsWithPlainCalls(store, results) {
|
|
|
17797
18389
|
__name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
17798
18390
|
export {
|
|
17799
18391
|
ACCUMULATOR_SEEN_MAX,
|
|
18392
|
+
ANNOUNCED_SCHEDULE_TAG,
|
|
17800
18393
|
ANY_WAIT_GROUP,
|
|
17801
18394
|
CHILD_EDGE_PREFIXES,
|
|
17802
18395
|
CONTROL_FLOW_SIGNAL,
|
|
@@ -17805,6 +18398,7 @@ export {
|
|
|
17805
18398
|
CodecStateStore,
|
|
17806
18399
|
ConcurrencyLimitError,
|
|
17807
18400
|
ContinueAsNew,
|
|
18401
|
+
DEFAULT_ANNOUNCED_SCHEDULE_GRACE_MS,
|
|
17808
18402
|
DEFAULT_CONCURRENCY_STATUSES,
|
|
17809
18403
|
DEFAULT_PARTITION,
|
|
17810
18404
|
DEFAULT_STALLED_AFTER_MS,
|
|
@@ -17886,6 +18480,7 @@ export {
|
|
|
17886
18480
|
accumulatorRunId,
|
|
17887
18481
|
aggregateAnnouncements,
|
|
17888
18482
|
aggregatesCheckpoints,
|
|
18483
|
+
announcedScheduleId,
|
|
17889
18484
|
announcementsOf,
|
|
17890
18485
|
anyCheckpointName,
|
|
17891
18486
|
applyAttributeQuery,
|