@dudousxd/nestjs-durable-core 0.85.11 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +192 -33
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +64 -3
- package/dist/index.d.ts +64 -3
- package/dist/index.js +191 -33
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -85,6 +85,7 @@ __export(index_exports, {
|
|
|
85
85
|
WORKFLOW_NAME_KEY: () => WORKFLOW_NAME_KEY,
|
|
86
86
|
WorkflowBlocked: () => WorkflowBlocked,
|
|
87
87
|
WorkflowEngine: () => WorkflowEngine,
|
|
88
|
+
WorkflowNotRegistered: () => WorkflowNotRegistered,
|
|
88
89
|
WorkflowSuspended: () => WorkflowSuspended,
|
|
89
90
|
aggregateAnnouncements: () => aggregateAnnouncements,
|
|
90
91
|
aggregatesCheckpoints: () => aggregatesCheckpoints,
|
|
@@ -5031,6 +5032,8 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
|
5031
5032
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
5032
5033
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
5033
5034
|
var STEP_FAN_CHUNK = 200;
|
|
5035
|
+
var STEP_EVENTS_PER_BROADCAST = 500;
|
|
5036
|
+
var isStepEvent = /* @__PURE__ */ __name((event) => event.type.startsWith("step."), "isStepEvent");
|
|
5034
5037
|
var FAN_REREAD_CHUNK = 1e3;
|
|
5035
5038
|
var STEP_FAN_SEND_ROUND = 2e3;
|
|
5036
5039
|
var PARENT_NOTIFY_ATTEMPTS = 4;
|
|
@@ -5164,6 +5167,22 @@ var NamespaceMismatch = class NamespaceMismatch2 extends Error {
|
|
|
5164
5167
|
super("namespace-mismatch");
|
|
5165
5168
|
}
|
|
5166
5169
|
};
|
|
5170
|
+
var WorkflowNotRegistered = class extends Error {
|
|
5171
|
+
static {
|
|
5172
|
+
__name(this, "WorkflowNotRegistered");
|
|
5173
|
+
}
|
|
5174
|
+
constructor(workflow, version) {
|
|
5175
|
+
super(`workflow ${workflow}@${version} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
|
|
5176
|
+
this.name = "WorkflowNotRegistered";
|
|
5177
|
+
}
|
|
5178
|
+
};
|
|
5179
|
+
var UNREGISTERED_DEAD_LETTER_FROM = [
|
|
5180
|
+
"pending",
|
|
5181
|
+
"running",
|
|
5182
|
+
"suspended",
|
|
5183
|
+
"cancelling",
|
|
5184
|
+
"blocked"
|
|
5185
|
+
];
|
|
5167
5186
|
var WorkflowEngine = class {
|
|
5168
5187
|
static {
|
|
5169
5188
|
__name(this, "WorkflowEngine");
|
|
@@ -5183,6 +5202,7 @@ var WorkflowEngine = class {
|
|
|
5183
5202
|
*/
|
|
5184
5203
|
ownLeaseCutoffMs;
|
|
5185
5204
|
maxRecoveryAttempts;
|
|
5205
|
+
unregisteredWorkflowGraceMs;
|
|
5186
5206
|
remoteAdvanceSilenceMs;
|
|
5187
5207
|
reconcileMs;
|
|
5188
5208
|
timerBatchSize;
|
|
@@ -5369,6 +5389,8 @@ var WorkflowEngine = class {
|
|
|
5369
5389
|
this.leaseMs = deps.leaseMs ?? 3e4;
|
|
5370
5390
|
this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
|
|
5371
5391
|
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
5392
|
+
const grace = deps.unregisteredWorkflowGraceMs;
|
|
5393
|
+
this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
|
|
5372
5394
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
5373
5395
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
5374
5396
|
const stalledAfter = deps.stalledAfterMs;
|
|
@@ -5501,6 +5523,13 @@ var WorkflowEngine = class {
|
|
|
5501
5523
|
...msg.event,
|
|
5502
5524
|
at: new Date(msg.event.at)
|
|
5503
5525
|
}));
|
|
5526
|
+
} else if (msg.kind === "events") {
|
|
5527
|
+
for (const event of msg.events) {
|
|
5528
|
+
this.deliver(markRemoteEvent({
|
|
5529
|
+
...event,
|
|
5530
|
+
at: new Date(event.at)
|
|
5531
|
+
}));
|
|
5532
|
+
}
|
|
5504
5533
|
} else if (msg.kind === "cancel") {
|
|
5505
5534
|
this.notifyCancelled(msg.runId);
|
|
5506
5535
|
} else if (msg.kind === "enqueued") {
|
|
@@ -5678,13 +5707,19 @@ var WorkflowEngine = class {
|
|
|
5678
5707
|
* that is lost brings it back due after that period. False when the transport refused the wakes,
|
|
5679
5708
|
* so the caller resumes them itself.
|
|
5680
5709
|
*/
|
|
5681
|
-
async handOffDue(runs, nowMs) {
|
|
5710
|
+
async handOffDue(runs, nowMs, settled) {
|
|
5682
5711
|
const claimed = [];
|
|
5683
5712
|
for (const run of runs) {
|
|
5684
5713
|
if (this.draining) break;
|
|
5685
5714
|
if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5686
5715
|
continue;
|
|
5687
5716
|
}
|
|
5717
|
+
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
|
|
5718
|
+
const result = await this.settleUnregistered(run, nowMs);
|
|
5719
|
+
if (result) settled?.set(run.id, result);
|
|
5720
|
+
else if (result === void 0) this.logUnregisteredSkip(run);
|
|
5721
|
+
continue;
|
|
5722
|
+
}
|
|
5688
5723
|
try {
|
|
5689
5724
|
const patch = {
|
|
5690
5725
|
wakeAt: nowMs + this.leaseMs
|
|
@@ -6082,13 +6117,53 @@ var WorkflowEngine = class {
|
|
|
6082
6117
|
at: /* @__PURE__ */ new Date()
|
|
6083
6118
|
};
|
|
6084
6119
|
this.deliver(full);
|
|
6085
|
-
if (this.controlPlane)
|
|
6086
|
-
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6120
|
+
if (!this.controlPlane) return;
|
|
6121
|
+
this.eventsToBroadcast.push(full);
|
|
6122
|
+
if (!this.broadcastingEvents) void this.broadcastEvents();
|
|
6123
|
+
}
|
|
6124
|
+
/** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
|
|
6125
|
+
eventsToBroadcast = [];
|
|
6126
|
+
broadcastingEvents = false;
|
|
6127
|
+
/**
|
|
6128
|
+
* Publish the queued events in the order they were emitted. A run's own events go out one per
|
|
6129
|
+
* message, as every instance expects them. Step events (a wide fan emits two per step) that queued
|
|
6130
|
+
* up while a publish was in flight go out together in one `events` message, so a fan of 40,000
|
|
6131
|
+
* steps costs the control plane a few hundred messages to every instance instead of 80,000.
|
|
6132
|
+
*/
|
|
6133
|
+
async broadcastEvents() {
|
|
6134
|
+
this.broadcastingEvents = true;
|
|
6135
|
+
try {
|
|
6136
|
+
while (this.eventsToBroadcast.length > 0 && this.controlPlane) {
|
|
6137
|
+
const first = this.eventsToBroadcast[0];
|
|
6138
|
+
let msg;
|
|
6139
|
+
if (isStepEvent(first)) {
|
|
6140
|
+
let n = 1;
|
|
6141
|
+
while (n < STEP_EVENTS_PER_BROADCAST && n < this.eventsToBroadcast.length && isStepEvent(this.eventsToBroadcast[n])) {
|
|
6142
|
+
n += 1;
|
|
6143
|
+
}
|
|
6144
|
+
const events = this.eventsToBroadcast.splice(0, n);
|
|
6145
|
+
msg = events.length === 1 ? {
|
|
6146
|
+
kind: "event",
|
|
6147
|
+
event: first,
|
|
6148
|
+
from: this.instanceId
|
|
6149
|
+
} : {
|
|
6150
|
+
kind: "events",
|
|
6151
|
+
events,
|
|
6152
|
+
from: this.instanceId
|
|
6153
|
+
};
|
|
6154
|
+
} else {
|
|
6155
|
+
this.eventsToBroadcast.shift();
|
|
6156
|
+
msg = {
|
|
6157
|
+
kind: "event",
|
|
6158
|
+
event: first,
|
|
6159
|
+
from: this.instanceId
|
|
6160
|
+
};
|
|
6161
|
+
}
|
|
6162
|
+
await this.controlPlane.publishControl(msg).catch(() => {
|
|
6163
|
+
});
|
|
6164
|
+
}
|
|
6165
|
+
} finally {
|
|
6166
|
+
this.broadcastingEvents = false;
|
|
6092
6167
|
}
|
|
6093
6168
|
}
|
|
6094
6169
|
/** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
|
|
@@ -6418,9 +6493,7 @@ var WorkflowEngine = class {
|
|
|
6418
6493
|
});
|
|
6419
6494
|
}
|
|
6420
6495
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
6421
|
-
if (!registered)
|
|
6422
|
-
throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
|
|
6423
|
-
}
|
|
6496
|
+
if (!registered) throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
|
|
6424
6497
|
return this.track(this.execute(run, registered.fn));
|
|
6425
6498
|
}
|
|
6426
6499
|
/**
|
|
@@ -6835,6 +6908,15 @@ var WorkflowEngine = class {
|
|
|
6835
6908
|
};
|
|
6836
6909
|
}
|
|
6837
6910
|
}
|
|
6911
|
+
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
|
|
6912
|
+
const settled2 = await this.settleUnregistered(run, nowMs);
|
|
6913
|
+
if (settled2) return {
|
|
6914
|
+
result: settled2,
|
|
6915
|
+
dispatch: false
|
|
6916
|
+
};
|
|
6917
|
+
if (settled2 === void 0) this.logUnregisteredSkip(run);
|
|
6918
|
+
return void 0;
|
|
6919
|
+
}
|
|
6838
6920
|
const settled = await this.countRecovery(run);
|
|
6839
6921
|
if (settled) return {
|
|
6840
6922
|
result: settled,
|
|
@@ -6906,25 +6988,10 @@ var WorkflowEngine = class {
|
|
|
6906
6988
|
updatedAt: /* @__PURE__ */ new Date()
|
|
6907
6989
|
});
|
|
6908
6990
|
await this.store.releaseRunLock(run.id);
|
|
6909
|
-
this.
|
|
6910
|
-
type: "run.failed",
|
|
6911
|
-
runId: run.id,
|
|
6912
|
-
workflow: run.workflow,
|
|
6913
|
-
namespace: run.namespace,
|
|
6914
|
-
error: error2
|
|
6915
|
-
});
|
|
6916
|
-
this.notifyDead({
|
|
6991
|
+
return this.announceDead({
|
|
6917
6992
|
...run,
|
|
6918
|
-
status: "dead",
|
|
6919
|
-
error: error2,
|
|
6920
6993
|
recoveryAttempts: attempts
|
|
6921
|
-
});
|
|
6922
|
-
this.notifyParent(run.id, failedCompletion(error2));
|
|
6923
|
-
return {
|
|
6924
|
-
runId: run.id,
|
|
6925
|
-
status: "dead",
|
|
6926
|
-
error: error2
|
|
6927
|
-
};
|
|
6994
|
+
}, error2);
|
|
6928
6995
|
}
|
|
6929
6996
|
await this.store.updateRun(run.id, {
|
|
6930
6997
|
recoveryAttempts: attempts,
|
|
@@ -6932,6 +6999,85 @@ var WorkflowEngine = class {
|
|
|
6932
6999
|
});
|
|
6933
7000
|
return void 0;
|
|
6934
7001
|
}
|
|
7002
|
+
/** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
|
|
7003
|
+
announceDead(run, error2) {
|
|
7004
|
+
this.emit({
|
|
7005
|
+
type: "run.failed",
|
|
7006
|
+
runId: run.id,
|
|
7007
|
+
workflow: run.workflow,
|
|
7008
|
+
namespace: run.namespace,
|
|
7009
|
+
error: error2
|
|
7010
|
+
});
|
|
7011
|
+
this.notifyDead({
|
|
7012
|
+
...run,
|
|
7013
|
+
status: "dead",
|
|
7014
|
+
error: error2
|
|
7015
|
+
});
|
|
7016
|
+
this.notifyParent(run.id, failedCompletion(error2));
|
|
7017
|
+
return {
|
|
7018
|
+
runId: run.id,
|
|
7019
|
+
status: "dead",
|
|
7020
|
+
error: error2
|
|
7021
|
+
};
|
|
7022
|
+
}
|
|
7023
|
+
/** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
|
|
7024
|
+
* pays no await (same rule as {@link resume}). */
|
|
7025
|
+
isRegisteredHere(run) {
|
|
7026
|
+
return this.workflows.has(versionKey(run.workflow, run.workflowVersion));
|
|
7027
|
+
}
|
|
7028
|
+
/**
|
|
7029
|
+
* Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
|
|
7030
|
+
* remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
|
|
7031
|
+
*/
|
|
7032
|
+
async canResolveWorkflow(run) {
|
|
7033
|
+
if (this.isRegisteredHere(run)) return true;
|
|
7034
|
+
return await this.findInheritedRegistration(run) !== void 0 || await this.resolveRemoteByConvention(run) !== void 0;
|
|
7035
|
+
}
|
|
7036
|
+
/**
|
|
7037
|
+
* A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
|
|
7038
|
+
* {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
|
|
7039
|
+
* rolling deploy's skew: the lease is released at once so the pod that still has the version takes
|
|
7040
|
+
* the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
|
|
7041
|
+
* capable pod would have taken the run long ago): dead-letter it.
|
|
7042
|
+
*
|
|
7043
|
+
* The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
|
|
7044
|
+
* count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
|
|
7045
|
+
* orphan never ages.
|
|
7046
|
+
*
|
|
7047
|
+
* Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
|
|
7048
|
+
* `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
|
|
7049
|
+
*/
|
|
7050
|
+
async settleUnregistered(listed, nowMs) {
|
|
7051
|
+
const run = await this.store.getRun(listed.id);
|
|
7052
|
+
if (!run || !UNREGISTERED_DEAD_LETTER_FROM.includes(run.status)) {
|
|
7053
|
+
await this.store.releaseRunLock(listed.id);
|
|
7054
|
+
return null;
|
|
7055
|
+
}
|
|
7056
|
+
const dueSince = Math.max(run.updatedAt.getTime(), run.wakeAt ?? 0);
|
|
7057
|
+
if (this.unregisteredWorkflowGraceMs === 0 || nowMs - dueSince < this.unregisteredWorkflowGraceMs) {
|
|
7058
|
+
await this.store.releaseRunLock(run.id);
|
|
7059
|
+
return void 0;
|
|
7060
|
+
}
|
|
7061
|
+
const error2 = {
|
|
7062
|
+
message: `workflow ${run.workflow}@${run.workflowVersion} is not registered on any live instance (unresumable for over ${this.unregisteredWorkflowGraceMs}ms) \u2014 moved to dead-letter`,
|
|
7063
|
+
code: "workflow_not_registered"
|
|
7064
|
+
};
|
|
7065
|
+
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
7066
|
+
run.status
|
|
7067
|
+
], {
|
|
7068
|
+
status: "dead",
|
|
7069
|
+
error: error2,
|
|
7070
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
7071
|
+
});
|
|
7072
|
+
await this.store.releaseRunLock(run.id);
|
|
7073
|
+
if (!applied) return null;
|
|
7074
|
+
console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
|
|
7075
|
+
return this.announceDead(run, error2);
|
|
7076
|
+
}
|
|
7077
|
+
/** Log a run skipped for an unregistered workflow — still there, still not progressing. */
|
|
7078
|
+
logUnregisteredSkip(run) {
|
|
7079
|
+
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
|
|
7080
|
+
}
|
|
6935
7081
|
/**
|
|
6936
7082
|
* Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
|
|
6937
7083
|
* boot. A run still not due re-suspends cheaply without running new work.
|
|
@@ -6956,11 +7102,12 @@ var WorkflowEngine = class {
|
|
|
6956
7102
|
...due,
|
|
6957
7103
|
...blocked
|
|
6958
7104
|
];
|
|
6959
|
-
|
|
6960
|
-
|
|
7105
|
+
const settled = /* @__PURE__ */ new Map();
|
|
7106
|
+
if (await this.handOffDue(runs, nowMs, settled)) {
|
|
7107
|
+
return runs.map((run) => settled.get(run.id) ?? {
|
|
6961
7108
|
runId: run.id,
|
|
6962
7109
|
status: run.status
|
|
6963
|
-
})
|
|
7110
|
+
});
|
|
6964
7111
|
}
|
|
6965
7112
|
}
|
|
6966
7113
|
const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
@@ -7041,8 +7188,13 @@ var WorkflowEngine = class {
|
|
|
7041
7188
|
const nowMs = this.clock();
|
|
7042
7189
|
const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
7043
7190
|
if (!acquired) return null;
|
|
7044
|
-
return this.resumeImplicit(runId).catch((err) => {
|
|
7191
|
+
return this.resumeImplicit(runId).catch(async (err) => {
|
|
7045
7192
|
if (err instanceof NamespaceMismatch) return null;
|
|
7193
|
+
if (err instanceof WorkflowNotRegistered) {
|
|
7194
|
+
const run = await this.store.getRun(runId);
|
|
7195
|
+
const settled = run ? await this.settleUnregistered(run, nowMs) : null;
|
|
7196
|
+
if (settled !== void 0) return settled;
|
|
7197
|
+
}
|
|
7046
7198
|
throw err;
|
|
7047
7199
|
});
|
|
7048
7200
|
}
|
|
@@ -7297,7 +7449,8 @@ var WorkflowEngine = class {
|
|
|
7297
7449
|
* Resume each run only if this instance can acquire its recovery lease — so when several
|
|
7298
7450
|
* replicas recover or poll at once, each run is picked up by exactly one of them.
|
|
7299
7451
|
*
|
|
7300
|
-
* A run this instance cannot resume is SKIPPED, not allowed to abort the batch
|
|
7452
|
+
* A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
|
|
7453
|
+
* stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
|
|
7301
7454
|
* case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
|
|
7302
7455
|
* and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
|
|
7303
7456
|
* and every caller here is a recovery sweep. Letting the first such run end the loop means one
|
|
@@ -7321,6 +7474,11 @@ var WorkflowEngine = class {
|
|
|
7321
7474
|
const settled = onLocked ? await onLocked(run) : void 0;
|
|
7322
7475
|
results.push(settled ?? await this.resumeImplicit(run.id));
|
|
7323
7476
|
} catch (error2) {
|
|
7477
|
+
if (error2 instanceof WorkflowNotRegistered) {
|
|
7478
|
+
const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
|
|
7479
|
+
if (settled) results.push(settled);
|
|
7480
|
+
if (settled !== void 0) continue;
|
|
7481
|
+
}
|
|
7324
7482
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
7325
7483
|
}
|
|
7326
7484
|
}
|
|
@@ -13082,6 +13240,7 @@ var InMemoryRunQueue = class {
|
|
|
13082
13240
|
WORKFLOW_NAME_KEY,
|
|
13083
13241
|
WorkflowBlocked,
|
|
13084
13242
|
WorkflowEngine,
|
|
13243
|
+
WorkflowNotRegistered,
|
|
13085
13244
|
WorkflowSuspended,
|
|
13086
13245
|
aggregateAnnouncements,
|
|
13087
13246
|
aggregatesCheckpoints,
|