@dudousxd/nestjs-durable-core 0.98.0 → 0.100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +335 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +6714 -6563
- package/dist/index.d.ts +6714 -6563
- package/dist/index.js +334 -15
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -69,6 +69,7 @@ __export(index_exports, {
|
|
|
69
69
|
LIFECYCLE_TRACE_TAG_PREFIX: () => LIFECYCLE_TRACE_TAG_PREFIX,
|
|
70
70
|
LIFECYCLE_TRIGGERED_TAG_PREFIX: () => LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
71
71
|
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX: () => LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
72
|
+
NO_LIVE_WORKER: () => NO_LIVE_WORKER,
|
|
72
73
|
NestedWorkflowCallError: () => NestedWorkflowCallError,
|
|
73
74
|
NonDeterminismError: () => NonDeterminismError,
|
|
74
75
|
PARENT_CLOSE_TAG_PREFIX: () => PARENT_CLOSE_TAG_PREFIX,
|
|
@@ -768,7 +769,7 @@ __name(bindWorkflowClass, "bindWorkflowClass");
|
|
|
768
769
|
function clientOf(ctor) {
|
|
769
770
|
const client = bindings.get(ctor);
|
|
770
771
|
if (!client) {
|
|
771
|
-
throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow
|
|
772
|
+
throw new Error(`${workflowName(ctor)} is not bound to a durable engine \u2014 the class-first statics only work after the module that registers this workflow (an operator, or a tenant's DurableModule) has booted.`);
|
|
772
773
|
}
|
|
773
774
|
return client;
|
|
774
775
|
}
|
|
@@ -6955,6 +6956,19 @@ function runNamespace(namespace) {
|
|
|
6955
6956
|
}
|
|
6956
6957
|
__name(runNamespace, "runNamespace");
|
|
6957
6958
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
6959
|
+
var NO_LIVE_WORKER = "no_live_worker";
|
|
6960
|
+
var TenantStartInvalid = class TenantStartInvalid2 extends Error {
|
|
6961
|
+
static {
|
|
6962
|
+
__name(this, "TenantStartInvalid");
|
|
6963
|
+
}
|
|
6964
|
+
constructor(cause) {
|
|
6965
|
+
super(cause instanceof Error ? cause.message : String(cause), {
|
|
6966
|
+
cause
|
|
6967
|
+
});
|
|
6968
|
+
this.name = "TenantStartInvalid";
|
|
6969
|
+
}
|
|
6970
|
+
};
|
|
6971
|
+
var NO_LIVE_WORKER_MAX_POLL_MS = 6e4;
|
|
6958
6972
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
6959
6973
|
var STEP_FAN_CHUNK = 200;
|
|
6960
6974
|
var STEP_EVENTS_PER_BROADCAST = 500;
|
|
@@ -7127,6 +7141,13 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7127
7141
|
maxRecoveryAttempts;
|
|
7128
7142
|
unregisteredWorkflowGraceMs;
|
|
7129
7143
|
remoteAdvanceSilenceMs;
|
|
7144
|
+
/**
|
|
7145
|
+
* The re-drive window of a DISPATCHED workflow turn (a remote worker's `advance` sent over the
|
|
7146
|
+
* broker): {@link WorkflowEngineDeps.remoteAdvanceSilenceMs} when set, else `reconcileMs`. Without
|
|
7147
|
+
* it a run suspended on a turn whose worker died (and whose job was lost with it) had no `wakeAt`
|
|
7148
|
+
* and nothing re-drove it, ever. Heartbeats rearm it like the explicit window.
|
|
7149
|
+
*/
|
|
7150
|
+
dispatchedTurnSilenceMs;
|
|
7130
7151
|
reconcileMs;
|
|
7131
7152
|
/** Parked workflow bodies kept between turns (see {@link WorkflowEngineDeps.stickyRuns}). */
|
|
7132
7153
|
sticky;
|
|
@@ -7347,6 +7368,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7347
7368
|
this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
|
|
7348
7369
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
7349
7370
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
7371
|
+
this.dispatchedTurnSilenceMs = this.remoteAdvanceSilenceMs ?? this.reconcileMs;
|
|
7350
7372
|
this.sticky = new StickyCache(Math.max(0, deps.stickyRuns ?? DEFAULT_STICKY_RUNS), deps.stickyIdleMs ?? DEFAULT_STICKY_IDLE_MS);
|
|
7351
7373
|
this.routeReplies = deps.routeReplies ?? true;
|
|
7352
7374
|
const stalledAfter = deps.stalledAfterMs;
|
|
@@ -7475,13 +7497,12 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7475
7497
|
console.warn(`[nestjs-durable] dropped a start-run for "${message.workflow}": invalid or unauthorized tenant token`);
|
|
7476
7498
|
return;
|
|
7477
7499
|
}
|
|
7478
|
-
await this.
|
|
7479
|
-
|
|
7480
|
-
|
|
7481
|
-
searchAttributes: message.searchAttributes,
|
|
7500
|
+
await this.startForTenant(
|
|
7501
|
+
message,
|
|
7502
|
+
tenant,
|
|
7482
7503
|
// The customer a tenant process claims for the run (A4): kept only when its verified
|
|
7483
7504
|
// token grants `tenancy:*` or `tenancy:<id>`, else the run gets no customer tenant.
|
|
7484
|
-
|
|
7505
|
+
message.tenantId !== void 0 ? grantsClientTenant(verified, message.tenantId) ? message.tenantKind !== void 0 ? {
|
|
7485
7506
|
tenantId: message.tenantId,
|
|
7486
7507
|
tenantKind: message.tenantKind
|
|
7487
7508
|
} : {
|
|
@@ -7489,7 +7510,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
7489
7510
|
} : {
|
|
7490
7511
|
tenantId: null
|
|
7491
7512
|
} : {}
|
|
7492
|
-
|
|
7513
|
+
);
|
|
7493
7514
|
});
|
|
7494
7515
|
this.controlPlane?.onControl((msg) => {
|
|
7495
7516
|
if (msg.from === this.instanceId) return;
|
|
@@ -9496,7 +9517,16 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9496
9517
|
}
|
|
9497
9518
|
}
|
|
9498
9519
|
}
|
|
9499
|
-
|
|
9520
|
+
start(workflow, input, runId, opts) {
|
|
9521
|
+
return this.startWith(workflow, input, runId, opts, false);
|
|
9522
|
+
}
|
|
9523
|
+
/**
|
|
9524
|
+
* {@link start}'s body. `parkUnroutable` is the tenant start path ({@link startForTenant}): a
|
|
9525
|
+
* workflow that nothing here can route yet (no registration, no live worker group serving it at
|
|
9526
|
+
* the asked version) is created `blocked` with code {@link NO_LIVE_WORKER} instead of throwing, so
|
|
9527
|
+
* a start whose caller has already returned is never lost.
|
|
9528
|
+
*/
|
|
9529
|
+
async startWith(workflow, input, runId, opts, parkUnroutable) {
|
|
9500
9530
|
const name = workflowName(workflow);
|
|
9501
9531
|
const pinned = opts?.version;
|
|
9502
9532
|
const startOnly = pinned === void 0 ? this.startOnlyLatest.get(name) : this.startOnly.get(versionKey(name, pinned));
|
|
@@ -9505,7 +9535,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9505
9535
|
const ancestor = await this.findRemoteAncestor(runId);
|
|
9506
9536
|
if (ancestor) registered = this.synthesizeRemoteChild(name, ancestor.version, ancestor);
|
|
9507
9537
|
}
|
|
9508
|
-
if (!registered
|
|
9538
|
+
if (!registered) {
|
|
9509
9539
|
const conventionRun = {
|
|
9510
9540
|
id: runId,
|
|
9511
9541
|
workflow: name,
|
|
@@ -9520,14 +9550,26 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9520
9550
|
createdAt: /* @__PURE__ */ new Date(),
|
|
9521
9551
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9522
9552
|
};
|
|
9523
|
-
|
|
9553
|
+
const convention = await this.resolveRemoteByConvention(conventionRun, {
|
|
9524
9554
|
askFleetForVersion: true
|
|
9525
9555
|
});
|
|
9556
|
+
registered = pinned === void 0 || convention?.versionDeclared === true && convention.version === pinned ? convention : void 0;
|
|
9557
|
+
}
|
|
9558
|
+
if (!registered && parkUnroutable) {
|
|
9559
|
+
return this.parkStartForWorker(name, input, runId, opts);
|
|
9526
9560
|
}
|
|
9527
9561
|
if (!registered) {
|
|
9528
9562
|
throw new Error(pinned === void 0 ? `workflow ${name} is not registered` : `workflow ${versionKey(name, pinned)} is not registered \u2014 a pinned start never falls back to the newest version (registered: ${this.registeredVersions(name)})`);
|
|
9529
9563
|
}
|
|
9530
|
-
|
|
9564
|
+
if (parkUnroutable && registered.validateInput) {
|
|
9565
|
+
try {
|
|
9566
|
+
await registered.validateInput(input);
|
|
9567
|
+
} catch (err) {
|
|
9568
|
+
throw new TenantStartInvalid(err);
|
|
9569
|
+
}
|
|
9570
|
+
} else {
|
|
9571
|
+
await registered.validateInput?.(input);
|
|
9572
|
+
}
|
|
9531
9573
|
const prior = await this.store.getRun(runId);
|
|
9532
9574
|
if (prior) return this.existingStartResult(prior);
|
|
9533
9575
|
const quota = await resolveConcurrency(opts?.concurrency, registered.concurrency, input);
|
|
@@ -9609,6 +9651,187 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
9609
9651
|
};
|
|
9610
9652
|
}
|
|
9611
9653
|
/**
|
|
9654
|
+
* Create the run of a tenant start that nothing can route yet, parked `blocked` with code
|
|
9655
|
+
* {@link NO_LIVE_WORKER} (see {@link startWith}). Idempotent by run id like {@link start}, counts
|
|
9656
|
+
* against a concurrency quota like it, and keeps the start's options on the row: its version (the
|
|
9657
|
+
* pin, else the `'1'` placeholder tagged {@link VERSION_UNDECLARED_TAG} that the first live worker's
|
|
9658
|
+
* declared version replaces), priority, tags, search attributes and customer tenant. A delayed
|
|
9659
|
+
* start is checked first at its `startAt`.
|
|
9660
|
+
*/
|
|
9661
|
+
async parkStartForWorker(name, input, runId, opts) {
|
|
9662
|
+
const prior = await this.store.getRun(runId);
|
|
9663
|
+
if (prior) return this.existingStartResult(prior);
|
|
9664
|
+
const quota = await resolveConcurrency(opts?.concurrency, void 0, input);
|
|
9665
|
+
if (quota) await assertConcurrency(this.store, name, quota);
|
|
9666
|
+
const namespace = poolNamespace(opts?.namespace) ?? this.namespace;
|
|
9667
|
+
const group = tenantGroup(sanitizeQueueToken(name), namespace);
|
|
9668
|
+
const pinned = opts?.version;
|
|
9669
|
+
const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
|
|
9670
|
+
const firstCheck = this.clock() + this.blockedPollMs;
|
|
9671
|
+
const wakeAt = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > firstCheck ? startAtMs : firstCheck;
|
|
9672
|
+
const error2 = {
|
|
9673
|
+
message: `no live worker serves ${group}${pinned !== void 0 ? ` at version ${pinned}` : ""} yet; the run starts when one does`,
|
|
9674
|
+
code: NO_LIVE_WORKER,
|
|
9675
|
+
retryable: true
|
|
9676
|
+
};
|
|
9677
|
+
const now = /* @__PURE__ */ new Date();
|
|
9678
|
+
const tags = mergeTags(void 0, [
|
|
9679
|
+
...opts?.tags ?? [],
|
|
9680
|
+
...quota ? [
|
|
9681
|
+
concurrencyTag(quota.key)
|
|
9682
|
+
] : [],
|
|
9683
|
+
...pinned === void 0 ? [
|
|
9684
|
+
VERSION_UNDECLARED_TAG
|
|
9685
|
+
] : []
|
|
9686
|
+
])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
|
|
9687
|
+
const run = {
|
|
9688
|
+
id: runId,
|
|
9689
|
+
workflow: name,
|
|
9690
|
+
workflowVersion: pinned ?? "1",
|
|
9691
|
+
status: "blocked",
|
|
9692
|
+
error: error2,
|
|
9693
|
+
wakeAt,
|
|
9694
|
+
namespace,
|
|
9695
|
+
input,
|
|
9696
|
+
tags,
|
|
9697
|
+
searchAttributes: opts?.searchAttributes,
|
|
9698
|
+
priority: opts?.priority,
|
|
9699
|
+
...this.startTenant(opts),
|
|
9700
|
+
createdAt: now,
|
|
9701
|
+
updatedAt: now
|
|
9702
|
+
};
|
|
9703
|
+
try {
|
|
9704
|
+
await this.store.createRun(run);
|
|
9705
|
+
} catch (err) {
|
|
9706
|
+
const winner = await this.store.getRun(runId).catch(() => null);
|
|
9707
|
+
if (!winner) throw err;
|
|
9708
|
+
return this.existingStartResult(winner);
|
|
9709
|
+
}
|
|
9710
|
+
this.emit({
|
|
9711
|
+
type: "run.blocked",
|
|
9712
|
+
runId,
|
|
9713
|
+
workflow: name,
|
|
9714
|
+
namespace,
|
|
9715
|
+
error: error2
|
|
9716
|
+
});
|
|
9717
|
+
return {
|
|
9718
|
+
runId,
|
|
9719
|
+
status: "blocked",
|
|
9720
|
+
error: error2
|
|
9721
|
+
};
|
|
9722
|
+
}
|
|
9723
|
+
/**
|
|
9724
|
+
* The control plane's half of a tenant start ({@link StartRunMessage}): start the run with every
|
|
9725
|
+
* option the message carries, under the tenant's namespace. The tenant caller has already returned
|
|
9726
|
+
* `pending`, so nothing here may lose the start:
|
|
9727
|
+
* - a workflow no live worker serves yet is parked `blocked` ({@link NO_LIVE_WORKER}) and started
|
|
9728
|
+
* by the blocked-recovery poll once one is live;
|
|
9729
|
+
* - a start the quota or singleton back-pressure rejects, or whose input a registration here
|
|
9730
|
+
* rejects, is recorded as a `failed` run carrying the reason, so the caller's run id resolves
|
|
9731
|
+
* to an answer instead of nothing (and a deterministic failure is never redelivered);
|
|
9732
|
+
* - a message with no `workflow` is logged and dropped: there is nothing to start;
|
|
9733
|
+
* - any other error (a store blip) is rethrown for the transport to redeliver the message, and a
|
|
9734
|
+
* redelivery is a no-op once the run exists (idempotent by run id).
|
|
9735
|
+
*/
|
|
9736
|
+
async startForTenant(message, namespace, tenant) {
|
|
9737
|
+
const runId = message.runId ?? globalThis.crypto.randomUUID();
|
|
9738
|
+
if (typeof message.workflow !== "string" || message.workflow === "") {
|
|
9739
|
+
console.error(`[nestjs-durable] dropped a malformed start-run ${runId}: no workflow name`);
|
|
9740
|
+
return {
|
|
9741
|
+
runId,
|
|
9742
|
+
status: "failed",
|
|
9743
|
+
error: {
|
|
9744
|
+
message: "malformed start-run: no workflow"
|
|
9745
|
+
}
|
|
9746
|
+
};
|
|
9747
|
+
}
|
|
9748
|
+
const opts = {
|
|
9749
|
+
namespace,
|
|
9750
|
+
tags: message.tags,
|
|
9751
|
+
searchAttributes: message.searchAttributes,
|
|
9752
|
+
...tenant,
|
|
9753
|
+
// Each start option only when the message carries it: a message from an older tenant has none
|
|
9754
|
+
// of them, and starts exactly as it did before.
|
|
9755
|
+
...typeof message.version === "string" ? {
|
|
9756
|
+
version: message.version
|
|
9757
|
+
} : {},
|
|
9758
|
+
...typeof message.priority === "number" ? {
|
|
9759
|
+
priority: message.priority
|
|
9760
|
+
} : {},
|
|
9761
|
+
...typeof message.startAt === "number" ? {
|
|
9762
|
+
startAt: message.startAt
|
|
9763
|
+
} : {},
|
|
9764
|
+
...message.concurrency !== void 0 ? {
|
|
9765
|
+
concurrency: message.concurrency
|
|
9766
|
+
} : {}
|
|
9767
|
+
};
|
|
9768
|
+
try {
|
|
9769
|
+
return await this.startWith(message.workflow, message.input, runId, opts, true);
|
|
9770
|
+
} catch (err) {
|
|
9771
|
+
if (!(err instanceof ConcurrencyLimitError || err instanceof SingletonQueueFullError || err instanceof TenantStartInvalid)) {
|
|
9772
|
+
throw err;
|
|
9773
|
+
}
|
|
9774
|
+
return this.recordRejectedStart(message.workflow, message.input, runId, opts, err);
|
|
9775
|
+
}
|
|
9776
|
+
}
|
|
9777
|
+
/** A tenant start {@link startForTenant} could not admit, kept as a `failed` run with the reason. */
|
|
9778
|
+
async recordRejectedStart(name, input, runId, opts, err) {
|
|
9779
|
+
const prior = await this.store.getRun(runId);
|
|
9780
|
+
if (prior) return this.existingStartResult(prior);
|
|
9781
|
+
const error2 = {
|
|
9782
|
+
message: err.message,
|
|
9783
|
+
code: err instanceof ConcurrencyLimitError ? "concurrency_limit" : err instanceof SingletonQueueFullError ? "singleton_queue_full" : "invalid_input",
|
|
9784
|
+
retryable: false
|
|
9785
|
+
};
|
|
9786
|
+
const now = /* @__PURE__ */ new Date();
|
|
9787
|
+
const run = {
|
|
9788
|
+
id: runId,
|
|
9789
|
+
workflow: name,
|
|
9790
|
+
workflowVersion: opts.version ?? "1",
|
|
9791
|
+
status: "failed",
|
|
9792
|
+
error: error2,
|
|
9793
|
+
namespace: poolNamespace(opts.namespace) ?? this.namespace,
|
|
9794
|
+
input,
|
|
9795
|
+
// The same derived tags an admitted or parked start carries, so quota-key queries find it.
|
|
9796
|
+
tags: mergeTags(opts.tags, [
|
|
9797
|
+
...opts.concurrency ? [
|
|
9798
|
+
concurrencyTag(opts.concurrency.key)
|
|
9799
|
+
] : [],
|
|
9800
|
+
...opts.version === void 0 ? [
|
|
9801
|
+
VERSION_UNDECLARED_TAG
|
|
9802
|
+
] : []
|
|
9803
|
+
])?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag)),
|
|
9804
|
+
searchAttributes: opts.searchAttributes,
|
|
9805
|
+
priority: opts.priority,
|
|
9806
|
+
...this.startTenant(opts),
|
|
9807
|
+
createdAt: now,
|
|
9808
|
+
updatedAt: now
|
|
9809
|
+
};
|
|
9810
|
+
try {
|
|
9811
|
+
await this.store.createRun(run);
|
|
9812
|
+
} catch (createErr) {
|
|
9813
|
+
const winner = await this.store.getRun(runId).catch(() => null);
|
|
9814
|
+
if (!winner) throw createErr;
|
|
9815
|
+
return this.existingStartResult(winner);
|
|
9816
|
+
}
|
|
9817
|
+
console.warn(`[nestjs-durable] tenant start ${runId} of "${name}" rejected: ${err.message}`);
|
|
9818
|
+
this.emit({
|
|
9819
|
+
type: "run.failed",
|
|
9820
|
+
runId,
|
|
9821
|
+
workflow: name,
|
|
9822
|
+
namespace: run.namespace,
|
|
9823
|
+
error: error2
|
|
9824
|
+
}, {
|
|
9825
|
+
lifecycle: false
|
|
9826
|
+
});
|
|
9827
|
+
void this.deliverRunWaits(runId).catch(() => void 0);
|
|
9828
|
+
return {
|
|
9829
|
+
runId,
|
|
9830
|
+
status: "failed",
|
|
9831
|
+
error: error2
|
|
9832
|
+
};
|
|
9833
|
+
}
|
|
9834
|
+
/**
|
|
9612
9835
|
* The tenant columns a new run gets: an explicit `opts.tenantId` (`null` = none) wins, else the
|
|
9613
9836
|
* context tenant when tenancy is on, else none (no keys at all, so a run row is unchanged when
|
|
9614
9837
|
* tenancy is off).
|
|
@@ -10547,6 +10770,32 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10547
10770
|
console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
|
|
10548
10771
|
return this.announceDead(run, error2);
|
|
10549
10772
|
}
|
|
10773
|
+
/**
|
|
10774
|
+
* Due runs whose workflow nothing could resolve on the last try (`runId` → clock ms of the next
|
|
10775
|
+
* try): a tenant run whose worker is down, say. The timer poll ticks every second and the run stays
|
|
10776
|
+
* due, so without this each tick would lease it, fail to resolve it and log it again. The next try
|
|
10777
|
+
* backs off with the time it has been due ({@link noteUnroutable}). In memory on purpose: the run
|
|
10778
|
+
* row is not touched, so its age toward `unregisteredWorkflowGraceMs` is unchanged, and another
|
|
10779
|
+
* instance (or a restart) simply tries it sooner.
|
|
10780
|
+
*/
|
|
10781
|
+
unroutableUntil = /* @__PURE__ */ new Map();
|
|
10782
|
+
/** Whether `runId` is inside its {@link unroutableUntil} back-off at `nowMs`. */
|
|
10783
|
+
unroutableBackoff(runId, nowMs) {
|
|
10784
|
+
const until = this.unroutableUntil.get(runId);
|
|
10785
|
+
if (until === void 0) return false;
|
|
10786
|
+
if (nowMs >= until) {
|
|
10787
|
+
this.unroutableUntil.delete(runId);
|
|
10788
|
+
return false;
|
|
10789
|
+
}
|
|
10790
|
+
return true;
|
|
10791
|
+
}
|
|
10792
|
+
/** Back off the next timer-poll try of `run`, left for later as unresolvable (see {@link unroutableUntil}). */
|
|
10793
|
+
noteUnroutable(run, nowMs) {
|
|
10794
|
+
const dueSince = Math.max(new Date(run.updatedAt).getTime(), run.wakeAt ?? 0);
|
|
10795
|
+
const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor((nowMs - dueSince) / 10)));
|
|
10796
|
+
if (this.unroutableUntil.size > 4096) this.unroutableUntil.clear();
|
|
10797
|
+
this.unroutableUntil.set(run.id, nowMs + delay);
|
|
10798
|
+
}
|
|
10550
10799
|
/** Log a run skipped for an unregistered workflow — still there, still not progressing. */
|
|
10551
10800
|
logUnregisteredSkip(run) {
|
|
10552
10801
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
|
|
@@ -10567,8 +10816,8 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10567
10816
|
* scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
|
|
10568
10817
|
*/
|
|
10569
10818
|
async resumeDueTimers(nowMs = this.clock()) {
|
|
10570
|
-
const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
|
|
10571
|
-
const blocked = await this.dueBlockedRuns(nowMs);
|
|
10819
|
+
const due = shuffle((await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize)).filter((run) => !this.unroutableBackoff(run.id, nowMs)));
|
|
10820
|
+
const blocked = await this.routableBlocked(await this.dueBlockedRuns(nowMs), nowMs);
|
|
10572
10821
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
10573
10822
|
if (this.queuesRunsOverTransport && !this.draining) {
|
|
10574
10823
|
const runs = [
|
|
@@ -10670,6 +10919,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10670
10919
|
const run = await this.store.getRun(runId);
|
|
10671
10920
|
const settled = run ? await this.settleUnregistered(run, nowMs) : null;
|
|
10672
10921
|
if (settled !== void 0) return settled;
|
|
10922
|
+
if (run) this.noteUnroutable(run, nowMs);
|
|
10673
10923
|
if (run && this.isStartOnlyHere(run)) return null;
|
|
10674
10924
|
}
|
|
10675
10925
|
throw err;
|
|
@@ -10967,6 +11217,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10967
11217
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
10968
11218
|
const settled = await this.settleUnregistered(run, this.clock()).catch(() => void 0);
|
|
10969
11219
|
if (settled) return settled;
|
|
11220
|
+
if (settled === void 0) this.noteUnroutable(run, this.clock());
|
|
10970
11221
|
if (this.isStartOnlyHere(run)) return void 0;
|
|
10971
11222
|
}
|
|
10972
11223
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
@@ -10997,6 +11248,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
10997
11248
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
10998
11249
|
const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
|
|
10999
11250
|
if (settled) results.push(settled);
|
|
11251
|
+
if (settled === void 0) this.noteUnroutable(run, nowMs);
|
|
11000
11252
|
if (settled !== void 0 || this.isStartOnlyHere(run)) continue;
|
|
11001
11253
|
}
|
|
11002
11254
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
@@ -12916,7 +13168,7 @@ ${run.workflowVersion}`;
|
|
|
12916
13168
|
if (remote.executor.dispatch) {
|
|
12917
13169
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
12918
13170
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
12919
|
-
const silenceMs = this.
|
|
13171
|
+
const silenceMs = this.dispatchedTurnSilenceMs;
|
|
12920
13172
|
const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
|
|
12921
13173
|
await this.store.updateRun(run.id, await this.withRunTrace(run, {
|
|
12922
13174
|
status: "suspended",
|
|
@@ -14186,6 +14438,73 @@ ${run.workflowVersion}`;
|
|
|
14186
14438
|
error: error2
|
|
14187
14439
|
};
|
|
14188
14440
|
}
|
|
14441
|
+
/**
|
|
14442
|
+
* Of the due `blocked` runs, the ones to re-drive now. A run parked {@link NO_LIVE_WORKER} (a tenant
|
|
14443
|
+
* start nothing could route) is re-driven only once a live worker serves its group, at its pinned
|
|
14444
|
+
* version when it has one; until then it is re-parked with a longer wait each time, up to
|
|
14445
|
+
* {@link NO_LIVE_WORKER_MAX_POLL_MS}, and never reaches `resume` (which would treat it as an
|
|
14446
|
+
* unregistered workflow and dead-letter it after `unregisteredWorkflowGraceMs`). Every other
|
|
14447
|
+
* blocked run is re-driven as before.
|
|
14448
|
+
*/
|
|
14449
|
+
async routableBlocked(blocked, nowMs) {
|
|
14450
|
+
const ready = [];
|
|
14451
|
+
for (const run of blocked) {
|
|
14452
|
+
if (run.error?.code !== NO_LIVE_WORKER) {
|
|
14453
|
+
ready.push(run);
|
|
14454
|
+
continue;
|
|
14455
|
+
}
|
|
14456
|
+
try {
|
|
14457
|
+
const routed = await this.routeParkedStart(run, nowMs);
|
|
14458
|
+
if (routed) ready.push(routed);
|
|
14459
|
+
} catch {
|
|
14460
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
14461
|
+
"blocked"
|
|
14462
|
+
], {
|
|
14463
|
+
wakeAt: nowMs + this.blockedPollMs
|
|
14464
|
+
}).catch(() => void 0);
|
|
14465
|
+
}
|
|
14466
|
+
}
|
|
14467
|
+
return ready;
|
|
14468
|
+
}
|
|
14469
|
+
/**
|
|
14470
|
+
* One {@link NO_LIVE_WORKER} run: the run to re-drive (its version taken from the live worker when
|
|
14471
|
+
* the start did not pin one and the worker declares it), or `undefined` after re-parking it.
|
|
14472
|
+
*/
|
|
14473
|
+
async routeParkedStart(run, nowMs) {
|
|
14474
|
+
if (this.isRegisteredHere(run)) return run;
|
|
14475
|
+
const undeclared = run.tags?.includes(VERSION_UNDECLARED_TAG) === true;
|
|
14476
|
+
const route = await this.resolveRemoteByConvention(run, {
|
|
14477
|
+
askFleetForVersion: true
|
|
14478
|
+
});
|
|
14479
|
+
const routable = route !== void 0 && (undeclared || route.versionDeclared === true && route.version === run.workflowVersion);
|
|
14480
|
+
const updateIf = this.store.updateRunIf ?? this.updateRunIfFallback;
|
|
14481
|
+
if (!routable) {
|
|
14482
|
+
const waited = Math.max(0, nowMs - new Date(run.createdAt).getTime());
|
|
14483
|
+
const delay = Math.min(NO_LIVE_WORKER_MAX_POLL_MS, Math.max(this.blockedPollMs, Math.floor(waited / 10)));
|
|
14484
|
+
await updateIf.call(this.store, run.id, [
|
|
14485
|
+
"blocked"
|
|
14486
|
+
], {
|
|
14487
|
+
wakeAt: nowMs + delay
|
|
14488
|
+
});
|
|
14489
|
+
return void 0;
|
|
14490
|
+
}
|
|
14491
|
+
if (undeclared && route.versionDeclared === true) {
|
|
14492
|
+
const tags = run.tags?.filter((tag) => tag !== VERSION_UNDECLARED_TAG);
|
|
14493
|
+
const applied = await updateIf.call(this.store, run.id, [
|
|
14494
|
+
"blocked"
|
|
14495
|
+
], {
|
|
14496
|
+
workflowVersion: route.version,
|
|
14497
|
+
tags
|
|
14498
|
+
});
|
|
14499
|
+
if (!applied) return void 0;
|
|
14500
|
+
return {
|
|
14501
|
+
...run,
|
|
14502
|
+
workflowVersion: route.version,
|
|
14503
|
+
tags
|
|
14504
|
+
};
|
|
14505
|
+
}
|
|
14506
|
+
return run;
|
|
14507
|
+
}
|
|
14189
14508
|
/** Runs parked `blocked` whose recovery poll is due (`wakeAt <= nowMs`), namespace-scoped like
|
|
14190
14509
|
* {@link resumeDueTimers}. Re-driving one re-checks the live fleet: it dispatches if a
|
|
14191
14510
|
* capable+compatible worker has appeared, else re-parks with a fresh `wakeAt` (design §7.5). */
|
|
@@ -15236,7 +15555,7 @@ ${run.workflowVersion}`;
|
|
|
15236
15555
|
* cold (or different) instance re-derives the same answer from the run's own `wakeAt`.
|
|
15237
15556
|
*/
|
|
15238
15557
|
async rearmDecisionDeadline(runId) {
|
|
15239
|
-
const silenceMs = this.
|
|
15558
|
+
const silenceMs = this.dispatchedTurnSilenceMs;
|
|
15240
15559
|
if (silenceMs == null) return;
|
|
15241
15560
|
const now = this.clock();
|
|
15242
15561
|
const lookAgainAt = this.decisionDeadlineRearms.get(runId);
|
|
@@ -17819,6 +18138,7 @@ __name(settleResultsWithPlainCalls, "settleResultsWithPlainCalls");
|
|
|
17819
18138
|
LIFECYCLE_TRACE_TAG_PREFIX,
|
|
17820
18139
|
LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
17821
18140
|
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
18141
|
+
NO_LIVE_WORKER,
|
|
17822
18142
|
NestedWorkflowCallError,
|
|
17823
18143
|
NonDeterminismError,
|
|
17824
18144
|
PARENT_CLOSE_TAG_PREFIX,
|