@dudousxd/nestjs-durable-core 0.99.0 → 0.100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +335 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +166 -63
- package/dist/index.d.ts +166 -63
- package/dist/index.js +334 -15
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -475,7 +475,7 @@ __name(bindWorkflowClass, "bindWorkflowClass");
|
|
|
475
475
|
function clientOf(ctor) {
|
|
476
476
|
const client = bindings.get(ctor);
|
|
477
477
|
if (!client) {
|
|
478
|
-
throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow
|
|
478
|
+
throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow (an operator, or a tenant's DurableModule) has booted.`);
|
|
479
479
|
}
|
|
480
480
|
return client;
|
|
481
481
|
}
|
|
@@ -6662,6 +6662,19 @@ function runNamespace(namespace) {
|
|
|
6662
6662
|
}
|
|
6663
6663
|
__name(runNamespace, "runNamespace");
|
|
6664
6664
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
6665
|
+
var NO_LIVE_WORKER = "no_live_worker";
|
|
6666
|
+
var TenantStartInvalid = class TenantStartInvalid2 extends Error {
|
|
6667
|
+
static {
|
|
6668
|
+
__name(this, "TenantStartInvalid");
|
|
6669
|
+
}
|
|
6670
|
+
constructor(cause) {
|
|
6671
|
+
super(cause instanceof Error ? cause.message : String(cause), {
|
|
6672
|
+
cause
|
|
6673
|
+
});
|
|
6674
|
+
this.name = "TenantStartInvalid";
|
|
6675
|
+
}
|
|
6676
|
+
};
|
|
6677
|
+
var NO_LIVE_WORKER_MAX_POLL_MS = 6e4;
|
|
6665
6678
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
6666
6679
|
var STEP_FAN_CHUNK = 200;
|
|
6667
6680
|
var STEP_EVENTS_PER_BROADCAST = 500;
|
|
@@ -6834,6 +6847,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
6834
6847
|
maxRecoveryAttempts;
|
|
6835
6848
|
unregisteredWorkflowGraceMs;
|
|
6836
6849
|
remoteAdvanceSilenceMs;
|
|
6850
|
+
/**
|
|
6851
|
+
* The re-drive window of a DISPATCHED workflow turn (a remote worker's `advance` sent over the
|
|
6852
|
+
* broker): {@link WorkflowEngineDeps.remoteAdvanceSilenceMs} when set, else `reconcileMs`. Without
|
|
6853
|
+
* it a run suspended on a turn whose worker died (and whose job was lost with it) had no `wakeAt`
|
|
6854
|
+
* and nothing re-drove it, ever. Heartbeats rearm it like the explicit window.
|
|
6855
|
+
*/
|
|
6856
|
+
dispatchedTurnSilenceMs;
|
|
6837
6857
|
reconcileMs;
|
|
6838
6858
|
/** Parked workflow bodies kept between turns (see {@link WorkflowEngineDeps.stickyRuns}). */
|
|
6839
6859
|
sticky;
|
|
@@ -7054,6 +7074,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7054
7074
|
this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
|
|
7055
7075
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
7056
7076
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
7077
|
+
this.dispatchedTurnSilenceMs = this.remoteAdvanceSilenceMs ?? this.reconcileMs;
|
|
7057
7078
|
this.sticky = new StickyCache(Math.max(0, deps.stickyRuns ?? DEFAULT_STICKY_RUNS), deps.stickyIdleMs ?? DEFAULT_STICKY_IDLE_MS);
|
|
7058
7079
|
this.routeReplies = deps.routeReplies ?? true;
|
|
7059
7080
|
const stalledAfter = deps.stalledAfterMs;
|
|
@@ -7182,13 +7203,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7182
7203
|
console.warn(`[nestjs-durable] dropped a start-run for "${message.workflow}": invalid or unauthorized tenant token`);
|
|
7183
7204
|
return;
|
|
7184
7205
|
}
|
|
7185
|
-
await this.
|
|
7186
|
-
|
|
7187
|
-
|
|
7188
|
-
searchAttributes: message.searchAttributes,
|
|
7206
|
+
await this.startForTenant(
|
|
7207
|
+
message,
|
|
7208
|
+
tenant,
|
|
7189
7209
|
// The customer a tenant process claims for the run (A4): kept only when its verified
|
|
7190
7210
|
// token grants `tenancy:*` or `tenancy:<id>`, else the run gets no customer tenant.
|
|
7191
|
-
|
|
7211
|
+
message.tenantId !== void 0 ? grantsClientTenant(verified, message.tenantId) ? message.tenantKind !== void 0 ? {
|
|
7192
7212
|
tenantId: message.tenantId,
|
|
7193
7213
|
tenantKind: message.tenantKind
|
|
7194
7214
|
} : {
|
|
@@ -7196,7 +7216,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7196
7216
|
} : {
|
|
7197
7217
|
tenantId: null
|
|
7198
7218
|
} : {}
|
|
7199
|
-
|
|
7219
|
+
);
|
|
7200
7220
|
});
|
|
7201
7221
|
this.controlPlane?.onControl((msg) => {
|
|
7202
7222
|
if (msg.from === this.instanceId) return;
|
|
@@ -9203,7 +9223,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9203
9223
|
}
|
|
9204
9224
|
}
|
|
9205
9225
|
}
|
|
9206
|
-
|
|
9226
|
+
start(workflow, input, runId, opts) {
|
|
9227
|
+
return this.startWith(workflow, input, runId, opts, false);
|
|
9228
|
+
}
|
|
9229
|
+
/**
|
|
9230
|
+
* {@link start}'s body. `parkUnroutable` is the tenant start path ({@link startForTenant}): a
|
|
9231
|
+
* workflow that nothing here can route yet (no registration, no live worker group serving it at
|
|
9232
|
+
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9233
|
+
* a start whose caller has already returned is never lost.
|
|
9234
|
+
*/
|
|
9235
|
+
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9207
9236
|
const name = workflowName(workflow);
|
|
9208
9237
|
const pinned = opts?.version;
|
|
9209
9238
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9212,7 +9241,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9212
9241
|
const ancestor = await this.findRemoteAncestor(runId);
|
|
9213
9242
|
if (ancestor) registered = this.synthesizeRemoteChild(name, ancestor.version, ancestor);
|
|
9214
9243
|
}
|
|
9215
|
-
if (!registered
|
|
9244
|
+
if (!registered) {
|
|
9216
9245
|
const conventionRun = {
|
|
9217
9246
|
id: runId,
|
|
9218
9247
|
workflow: name,
|
|
@@ -9227,14 +9256,26 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9227
9256
|
createdAt: /* @__PURE__ */ new Date(),
|
|
9228
9257
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9229
9258
|
};
|
|
9230
|
-
|
|
9259
|
+
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9231
9260
|
askFleetForVersion: true
|
|
9232
9261
|
});
|
|
9262
|
+
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9263
|
+
}
|
|
9264
|
+
if (!registered && parkUnroutable) {
|
|
9265
|
+
return this.parkStartForWorker(name, input, runId, opts);
|
|
9233
9266
|
}
|
|
9234
9267
|
if (!registered) {
|
|
9235
9268
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
9236
9269
|
}
|
|
9237
|
-
|
|
9270
|
+
if (parkUnroutable && registered.validateInput) {
|
|
9271
|
+
try {
|
|
9272
|
+
await registered.validateInput(input);
|
|
9273
|
+
} catch (err) {
|
|
9274
|
+
throw new TenantStartInvalid(err);
|
|
9275
|
+
}
|
|
9276
|
+
} else {
|
|
9277
|
+
await registered.validateInput?.(input);
|
|
9278
|
+
}
|
|
9238
9279
|
const prior = await this.store.getRun(runId);
|
|
9239
9280
|
if (prior) return this.existingStartResult(prior);
|
|
9240
9281
|
const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, input);
|
|
@@ -9316,6 +9357,187 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9316
9357
|
};
|
|
9317
9358
|
}
|
|
9318
9359
|
/**
|
|
9360
|
+
* Create the run of a tenant start that nothing can route yet, parked `blocked` with code
|
|
9361
|
+
* {@link NO_LIVE_WORKER} (see {@link startWith}). Idempotent by run id like {@link start}, counts
|
|
9362
|
+
* against a concurrency quota like it, and keeps the start's options on the row: its version (the
|
|
9363
|
+
* pin, else the `'1'` placeholder tagged {@link VERSION_UNDECLARED_TAG} that the first live worker's
|
|
9364
|
+
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9365
|
+
* start is checked first at its `startAt`.
|
|
9366
|
+
*/
|
|
9367
|
+
async parkStartForWorker(name, input, runId, opts) {
|
|
9368
|
+
const prior = await this.store.getRun(runId);
|
|
9369
|
+
if (prior) return this.existingStartResult(prior);
|
|
9370
|
+
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9371
|
+
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9372
|
+
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9373
|
+
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9374
|
+
const pinned = opts?.version;
|
|
9375
|
+
const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
|
|
9376
|
+
const firstCheck = this.clock() + this.blockedPollMs;
|
|
9377
|
+
const wakeAt = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > firstCheck ? startAtMs : firstCheck;
|
|
9378
|
+
const error2 = {
|
|
9379
|
+
message: `no live worker serves ${group}${pinned !== void 0 ? ` at version ${pinned}` : ""} yet; the run starts when one does`,
|
|
9380
|
+
code: NO_LIVE_WORKER,
|
|
9381
|
+
retryable: true
|
|
9382
|
+
};
|
|
9383
|
+
const now = /* @__PURE__ */ new Date();
|
|
9384
|
+
const tags = mergeTags(void 0, [
|
|
9385
|
+
...opts?.tags ?? [],
|
|
9386
|
+
...quota ? [
|
|
9387
|
+
concurrencyTag(quota.key)
|
|
9388
|
+
] : [],
|
|
9389
|
+
...pinned === void 0 ? [
|
|
9390
|
+
VERSION_UNDECLARED_TAG
|
|
9391
|
+
] : []
|
|
9392
|
+
])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
9393
|
+
const run = {
|
|
9394
|
+
id: runId,
|
|
9395
|
+
workflow: name,
|
|
9396
|
+
workflowVersion: pinned ?? "1",
|
|
9397
|
+
status: "blocked",
|
|
9398
|
+
error: error2,
|
|
9399
|
+
wakeAt,
|
|
9400
|
+
namespace,
|
|
9401
|
+
input,
|
|
9402
|
+
tags,
|
|
9403
|
+
searchAttributes: opts?.searchAttributes,
|
|
9404
|
+
priority: opts?.priority,
|
|
9405
|
+
...this.startTenant(opts),
|
|
9406
|
+
createdAt: now,
|
|
9407
|
+
updatedAt: now
|
|
9408
|
+
};
|
|
9409
|
+
try {
|
|
9410
|
+
await this.store.createRun(run);
|
|
9411
|
+
} catch (err) {
|
|
9412
|
+
const winner = await this.store.getRun(runId).catch(() => null);
|
|
9413
|
+
if (!winner) throw err;
|
|
9414
|
+
return this.existingStartResult(winner);
|
|
9415
|
+
}
|
|
9416
|
+
this.emit({
|
|
9417
|
+
type: "run.blocked",
|
|
9418
|
+
runId,
|
|
9419
|
+
workflow: name,
|
|
9420
|
+
namespace,
|
|
9421
|
+
error: error2
|
|
9422
|
+
});
|
|
9423
|
+
return {
|
|
9424
|
+
runId,
|
|
9425
|
+
status: "blocked",
|
|
9426
|
+
error: error2
|
|
9427
|
+
};
|
|
9428
|
+
}
|
|
9429
|
+
/**
|
|
9430
|
+
* The control plane's half of a tenant start ({@link StartRunMessage}): start the run with every
|
|
9431
|
+
* option the message carries, under the tenant's namespace. The tenant caller has already returned
|
|
9432
|
+
* `pending`, so nothing here may lose the start:
|
|
9433
|
+
* - a workflow no live worker serves yet is parked `blocked` ({@link NO_LIVE_WORKER}) and started
|
|
9434
|
+
* by the blocked-recovery poll once one is live;
|
|
9435
|
+
* - a start the quota or singleton back-pressure rejects, or whose input a registration here
|
|
9436
|
+
* rejects, is recorded as a `failed` run carrying the reason, so the caller's run id resolves
|
|
9437
|
+
* to an answer instead of nothing (and a deterministic failure is never redelivered);
|
|
9438
|
+
* - a message with no `workflow` is logged and dropped: there is nothing to start;
|
|
9439
|
+
* - any other error (a store blip) is rethrown for the transport to redeliver the message, and a
|
|
9440
|
+
* redelivery is a no-op once the run exists (idempotent by run id).
|
|
9441
|
+
*/
|
|
9442
|
+
async startForTenant(message, namespace, tenant) {
|
|
9443
|
+
const runId = message.runId ?? globalThis.crypto.randomUUID();
|
|
9444
|
+
if (typeof message.workflow !== "string" || message.workflow === "") {
|
|
9445
|
+
console.error(`[nestjs-durable] dropped a malformed start-run ${runId}: no workflow name`);
|
|
9446
|
+
return {
|
|
9447
|
+
runId,
|
|
9448
|
+
status: "failed",
|
|
9449
|
+
error: {
|
|
9450
|
+
message: "malformed start-run: no workflow"
|
|
9451
|
+
}
|
|
9452
|
+
};
|
|
9453
|
+
}
|
|
9454
|
+
const opts = {
|
|
9455
|
+
namespace,
|
|
9456
|
+
tags: message.tags,
|
|
9457
|
+
searchAttributes: message.searchAttributes,
|
|
9458
|
+
...tenant,
|
|
9459
|
+
// Each start option only when the message carries it: a message from an older tenant has none
|
|
9460
|
+
// of them, and starts exactly as it did before.
|
|
9461
|
+
...typeof message.version === "string" ? {
|
|
9462
|
+
version: message.version
|
|
9463
|
+
} : {},
|
|
9464
|
+
...typeof message.priority === "number" ? {
|
|
9465
|
+
priority: message.priority
|
|
9466
|
+
} : {},
|
|
9467
|
+
...typeof message.startAt === "number" ? {
|
|
9468
|
+
startAt: message.startAt
|
|
9469
|
+
} : {},
|
|
9470
|
+
...message.concurrency !== void 0 ? {
|
|
9471
|
+
concurrency: message.concurrency
|
|
9472
|
+
} : {}
|
|
9473
|
+
};
|
|
9474
|
+
try {
|
|
9475
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
9476
|
+
} catch (err) {
|
|
9477
|
+
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
9478
|
+
throw err;
|
|
9479
|
+
}
|
|
9480
|
+
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
9481
|
+
}
|
|
9482
|
+
}
|
|
9483
|
+
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
9484
|
+
async recordRejectedStart(name, input, runId, opts, err) {
|
|
9485
|
+
const prior = await this.store.getRun(runId);
|
|
9486
|
+
if (prior) return this.existingStartResult(prior);
|
|
9487
|
+
const error2 = {
|
|
9488
|
+
message: err.message,
|
|
9489
|
+
code: err instanceof ConcurrencyLimitError ? "concurrency_limit" : err instanceof SingletonQueueFullError ? "singleton_queue_full" : "invalid_input",
|
|
9490
|
+
retryable: false
|
|
9491
|
+
};
|
|
9492
|
+
const now = /* @__PURE__ */ new Date();
|
|
9493
|
+
const run = {
|
|
9494
|
+
id: runId,
|
|
9495
|
+
workflow: name,
|
|
9496
|
+
workflowVersion: opts.version ?? "1",
|
|
9497
|
+
status: "failed",
|
|
9498
|
+
error: error2,
|
|
9499
|
+
namespace: poolNamespace(opts.namespace) ?? this.namespace,
|
|
9500
|
+
input,
|
|
9501
|
+
// The same derived tags an admitted or parked start carries, so quota-key queries find it.
|
|
9502
|
+
tags: mergeTags(opts.tags, [
|
|
9503
|
+
...opts.concurrency ? [
|
|
9504
|
+
concurrencyTag(opts.concurrency.key)
|
|
9505
|
+
] : [],
|
|
9506
|
+
...opts.version === void 0 ? [
|
|
9507
|
+
VERSION_UNDECLARED_TAG
|
|
9508
|
+
] : []
|
|
9509
|
+
])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag)),
|
|
9510
|
+
searchAttributes: opts.searchAttributes,
|
|
9511
|
+
priority: opts.priority,
|
|
9512
|
+
...this.startTenant(opts),
|
|
9513
|
+
createdAt: now,
|
|
9514
|
+
updatedAt: now
|
|
9515
|
+
};
|
|
9516
|
+
try {
|
|
9517
|
+
await this.store.createRun(run);
|
|
9518
|
+
} catch (createErr) {
|
|
9519
|
+
const winner = await this.store.getRun(runId).catch(() => null);
|
|
9520
|
+
if (!winner) throw createErr;
|
|
9521
|
+
return this.existingStartResult(winner);
|
|
9522
|
+
}
|
|
9523
|
+
console.warn(`[nestjs-durable] tenant start ${runId} of "${name}" rejected: ${err.message}`);
|
|
9524
|
+
this.emit({
|
|
9525
|
+
type: "run.failed",
|
|
9526
|
+
runId,
|
|
9527
|
+
workflow: name,
|
|
9528
|
+
namespace: run.namespace,
|
|
9529
|
+
error: error2
|
|
9530
|
+
}, {
|
|
9531
|
+
lifecycle: false
|
|
9532
|
+
});
|
|
9533
|
+
void this.deliverRunWaits(runId).catch(() => void 0);
|
|
9534
|
+
return {
|
|
9535
|
+
runId,
|
|
9536
|
+
status: "failed",
|
|
9537
|
+
error: error2
|
|
9538
|
+
};
|
|
9539
|
+
}
|
|
9540
|
+
/**
|
|
9319
9541
|
* The tenant columns a new run gets: an explicit `opts.tenantId` (`null` = none) wins, else the
|
|
9320
9542
|
* context tenant when tenancy is on, else none (no keys at all, so a run row is unchanged when
|
|
9321
9543
|
* tenancy is off).
|
|
@@ -10254,6 +10476,32 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10254
10476
|
console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
|
|
10255
10477
|
return this.announceDead(run, error2);
|
|
10256
10478
|
}
|
|
10479
|
+
/**
|
|
10480
|
+
* Due runs whose workflow nothing could resolve on the last try (`runId` → clock ms of the next
|
|
10481
|
+
* try): a tenant run whose worker is down, say. The timer poll ticks every second and the run stays
|
|
10482
|
+
* due, so without this each tick would lease it, fail to resolve it and log it again. The next try
|
|
10483
|
+
* backs off with the time it has been due ({@link noteUnroutable}). In memory on purpose: the run
|
|
10484
|
+
* row is not touched, so its age toward `unregisteredWorkflowGraceMs` is unchanged, and another
|
|
10485
|
+
* instance (or a restart) simply tries it sooner.
|
|
10486
|
+
*/
|
|
10487
|
+
unroutableUntil = /* @__PURE__ */ new Map();
|
|
10488
|
+
/** Whether `runId` is inside its {@link unroutableUntil} back-off at `nowMs`. */
|
|
10489
|
+
unroutableBackoff(runId, nowMs) {
|
|
10490
|
+
const until = this.unroutableUntil.get(runId);
|
|
10491
|
+
if (until === void 0) return false;
|
|
10492
|
+
if (nowMs >= until) {
|
|
10493
|
+
this.unroutableUntil.delete(runId);
|
|
10494
|
+
return false;
|
|
10495
|
+
}
|
|
10496
|
+
return true;
|
|
10497
|
+
}
|
|
10498
|
+
/** Back off the next timer-poll try of `run`, left for later as unresolvable (see {@link unroutableUntil}). */
|
|
10499
|
+
noteUnroutable(run, nowMs) {
|
|
10500
|
+
const dueSince = Math.max(new Date(run.updatedAt).getTime(), run.wakeAt ?? 0);
|
|
10501
|
+
const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor((nowMs - dueSince) / 10)));
|
|
10502
|
+
if (this.unroutableUntil.size > 4096) this.unroutableUntil.clear();
|
|
10503
|
+
this.unroutableUntil.set(run.id, nowMs + delay);
|
|
10504
|
+
}
|
|
10257
10505
|
/** Log a run skipped for an unregistered workflow — still there, still not progressing. */
|
|
10258
10506
|
logUnregisteredSkip(run) {
|
|
10259
10507
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
|
|
@@ -10274,8 +10522,8 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10274
10522
|
* scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
|
|
10275
10523
|
*/
|
|
10276
10524
|
async resumeDueTimers(nowMs = this.clock()) {
|
|
10277
|
-
const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
|
|
10278
|
-
const blocked = await this.dueBlockedRuns(nowMs);
|
|
10525
|
+
const due = shuffle((await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize)).filter((run) => !this.unroutableBackoff(run.id, nowMs)));
|
|
10526
|
+
const blocked = await this.routableBlocked(await this.dueBlockedRuns(nowMs), nowMs);
|
|
10279
10527
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
10280
10528
|
if (this.queuesRunsOverTransport && !this.draining) {
|
|
10281
10529
|
const runs = [
|
|
@@ -10377,6 +10625,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10377
10625
|
const run = await this.store.getRun(runId);
|
|
10378
10626
|
const settled = run ? await this.settleUnregistered(run, nowMs) : null;
|
|
10379
10627
|
if (settled !== void 0) return settled;
|
|
10628
|
+
if (run) this.noteUnroutable(run, nowMs);
|
|
10380
10629
|
if (run && this.isStartOnlyHere(run)) return null;
|
|
10381
10630
|
}
|
|
10382
10631
|
throw err;
|
|
@@ -10674,6 +10923,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10674
10923
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
10675
10924
|
const settled = await this.settleUnregistered(run, this.clock()).catch(() => void 0);
|
|
10676
10925
|
if (settled) return settled;
|
|
10926
|
+
if (settled === void 0) this.noteUnroutable(run, this.clock());
|
|
10677
10927
|
if (this.isStartOnlyHere(run)) return void 0;
|
|
10678
10928
|
}
|
|
10679
10929
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
@@ -10704,6 +10954,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10704
10954
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
10705
10955
|
const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
|
|
10706
10956
|
if (settled) results.push(settled);
|
|
10957
|
+
if (settled === void 0) this.noteUnroutable(run, nowMs);
|
|
10707
10958
|
if (settled !== void 0 || this.isStartOnlyHere(run)) continue;
|
|
10708
10959
|
}
|
|
10709
10960
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
@@ -12623,7 +12874,7 @@ ${run.workflowVersion}`;
|
|
|
12623
12874
|
if (remote.executor.dispatch) {
|
|
12624
12875
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
12625
12876
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
12626
|
-
const silenceMs = this.
|
|
12877
|
+
const silenceMs = this.dispatchedTurnSilenceMs;
|
|
12627
12878
|
const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
|
|
12628
12879
|
await this.store.updateRun(run.id, await this.withRunTrace(run, {
|
|
12629
12880
|
status: "suspended",
|
|
@@ -13893,6 +14144,73 @@ ${run.workflowVersion}`;
|
|
|
13893
14144
|
error: error2
|
|
13894
14145
|
};
|
|
13895
14146
|
}
|
|
14147
|
+
/**
|
|
14148
|
+
* Of the due `blocked` runs, the ones to re-drive now. A run parked {@link NO_LIVE_WORKER} (a tenant
|
|
14149
|
+
* start nothing could route) is re-driven only once a live worker serves its group, at its pinned
|
|
14150
|
+
* version when it has one; until then it is re-parked with a longer wait each time, up to
|
|
14151
|
+
* {@link NO_LIVE_WORKER_MAX_POLL_MS}, and never reaches `resume` (which would treat it as an
|
|
14152
|
+
* unregistered workflow and dead-letter it after `unregisteredWorkflowGraceMs`). Every other
|
|
14153
|
+
* blocked run is re-driven as before.
|
|
14154
|
+
*/
|
|
14155
|
+
async routableBlocked(blocked, nowMs) {
|
|
14156
|
+
const ready = [];
|
|
14157
|
+
for (const run of blocked) {
|
|
14158
|
+
if (run.error?.code !== NO_LIVE_WORKER) {
|
|
14159
|
+
ready.push(run);
|
|
14160
|
+
continue;
|
|
14161
|
+
}
|
|
14162
|
+
try {
|
|
14163
|
+
const routed = await this.routeParkedStart(run, nowMs);
|
|
14164
|
+
if (routed) ready.push(routed);
|
|
14165
|
+
} catch {
|
|
14166
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
14167
|
+
"blocked"
|
|
14168
|
+
], {
|
|
14169
|
+
wakeAt: nowMs + this.blockedPollMs
|
|
14170
|
+
}).catch(() => void 0);
|
|
14171
|
+
}
|
|
14172
|
+
}
|
|
14173
|
+
return ready;
|
|
14174
|
+
}
|
|
14175
|
+
/**
|
|
14176
|
+
* One {@link NO_LIVE_WORKER} run: the run to re-drive (its version taken from the live worker when
|
|
14177
|
+
* the start did not pin one and the worker declares it), or `undefined` after re-parking it.
|
|
14178
|
+
*/
|
|
14179
|
+
async routeParkedStart(run, nowMs) {
|
|
14180
|
+
if (this.isRegisteredHere(run)) return run;
|
|
14181
|
+
const undeclared = run.tags?.includes(VERSION_UNDECLARED_TAG) === true;
|
|
14182
|
+
const route = await this.resolveRemoteByConvention(run, {
|
|
14183
|
+
askFleetForVersion: true
|
|
14184
|
+
});
|
|
14185
|
+
const routable = route !== void 0 && (undeclared || route.versionDeclared === true && route.version === run.workflowVersion);
|
|
14186
|
+
const updateIf = this.store.updateRunIf ?? this.updateRunIfFallback;
|
|
14187
|
+
if (!routable) {
|
|
14188
|
+
const waited = Math.max(0, nowMs - new Date(run.createdAt).getTime());
|
|
14189
|
+
const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor(waited / 10)));
|
|
14190
|
+
await updateIf.call(this.store, run.id, [
|
|
14191
|
+
"blocked"
|
|
14192
|
+
], {
|
|
14193
|
+
wakeAt: nowMs + delay
|
|
14194
|
+
});
|
|
14195
|
+
return void 0;
|
|
14196
|
+
}
|
|
14197
|
+
if (undeclared && route.versionDeclared === true) {
|
|
14198
|
+
const tags = run.tags?.filter((tag) => tag !== VERSION_UNDECLARED_TAG);
|
|
14199
|
+
const applied = await updateIf.call(this.store, run.id, [
|
|
14200
|
+
"blocked"
|
|
14201
|
+
], {
|
|
14202
|
+
workflowVersion: route.version,
|
|
14203
|
+
tags
|
|
14204
|
+
});
|
|
14205
|
+
if (!applied) return void 0;
|
|
14206
|
+
return {
|
|
14207
|
+
...run,
|
|
14208
|
+
workflowVersion: route.version,
|
|
14209
|
+
tags
|
|
14210
|
+
};
|
|
14211
|
+
}
|
|
14212
|
+
return run;
|
|
14213
|
+
}
|
|
13896
14214
|
/** Runs parked `blocked` whose recovery poll is due (`wakeAt <= nowMs`), namespace-scoped like
|
|
13897
14215
|
* {@link resumeDueTimers}. Re-driving one re-checks the live fleet: it dispatches if a
|
|
13898
14216
|
* capable+compatible worker has appeared, else re-parks with a fresh `wakeAt` (design §7.5). */
|
|
@@ -14943,7 +15261,7 @@ ${run.workflowVersion}`;
|
|
|
14943
15261
|
* cold (or different) instance re-derives the same answer from the run's own `wakeAt`.
|
|
14944
15262
|
*/
|
|
14945
15263
|
async rearmDecisionDeadline(runId) {
|
|
14946
|
-
const silenceMs = this.
|
|
15264
|
+
const silenceMs = this.dispatchedTurnSilenceMs;
|
|
14947
15265
|
if (silenceMs == null) return;
|
|
14948
15266
|
const now = this.clock();
|
|
14949
15267
|
const lookAgainAt = this.decisionDeadlineRearms.get(runId);
|
|
@@ -17525,6 +17843,7 @@ export {
|
|
|
17525
17843
|
LIFECYCLE_TRACE_TAG_PREFIX,
|
|
17526
17844
|
LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
17527
17845
|
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
17846
|
+
NO_LIVE_WORKER,
|
|
17528
17847
|
NestedWorkflowCallError,
|
|
17529
17848
|
NonDeterminismError,
|
|
17530
17849
|
PARENT_CLOSE_TAG_PREFIX,
|