@dudousxd/nestjs-durable-core 0.90.0 → 0.90.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +77 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +19 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +77 -15
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -7395,6 +7395,25 @@ declare class WorkflowEngine {
|
|
|
7395
7395
|
private markStarted;
|
|
7396
7396
|
private singletonWaitResult;
|
|
7397
7397
|
private execute;
|
|
7398
|
+
/**
|
|
7399
|
+
* A wake that found `run`'s lease held by another execution: keep it for that execution to honor
|
|
7400
|
+
* once it releases the lease, and answer with the run's status as read.
|
|
7401
|
+
*/
|
|
7402
|
+
private keepWakeForHeldLease;
|
|
7403
|
+
/**
|
|
7404
|
+
* Take `runId`'s lease for an explicit retry of a `failed` run that found it held: the turn that
|
|
7405
|
+
* failed the run has not released it yet, or another instance holds it for a moment. Polls the
|
|
7406
|
+
* lease with the usual fenced take ({@link StateStore.tryLockRun}, which also takes over a lease
|
|
7407
|
+
* that expired; the old holder's renewal then fences it) until it is free. Returns the run, read
|
|
7408
|
+
* under the lease, while it is still `failed`; or the run's state when something else moved it on
|
|
7409
|
+
* meanwhile (nothing to retry). Throws when the lease stays held by a live execution for longer
|
|
7410
|
+
* than one lease period, or the engine starts draining: the retry is refused, never dropped.
|
|
7411
|
+
*/
|
|
7412
|
+
private takeLeaseForRetry;
|
|
7413
|
+
/** `run` as a {@link RunResult}. */
|
|
7414
|
+
private resultOf;
|
|
7415
|
+
/** The body of {@link execute} once this instance holds `run`'s lease. */
|
|
7416
|
+
private executeLeased;
|
|
7398
7417
|
/**
|
|
7399
7418
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
7400
7419
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
package/dist/index.d.ts
CHANGED
|
@@ -7395,6 +7395,25 @@ declare class WorkflowEngine {
|
|
|
7395
7395
|
private markStarted;
|
|
7396
7396
|
private singletonWaitResult;
|
|
7397
7397
|
private execute;
|
|
7398
|
+
/**
|
|
7399
|
+
* A wake that found `run`'s lease held by another execution: keep it for that execution to honor
|
|
7400
|
+
* once it releases the lease, and answer with the run's status as read.
|
|
7401
|
+
*/
|
|
7402
|
+
private keepWakeForHeldLease;
|
|
7403
|
+
/**
|
|
7404
|
+
* Take `runId`'s lease for an explicit retry of a `failed` run that found it held: the turn that
|
|
7405
|
+
* failed the run has not released it yet, or another instance holds it for a moment. Polls the
|
|
7406
|
+
* lease with the usual fenced take ({@link StateStore.tryLockRun}, which also takes over a lease
|
|
7407
|
+
* that expired; the old holder's renewal then fences it) until it is free. Returns the run, read
|
|
7408
|
+
* under the lease, while it is still `failed`; or the run's state when something else moved it on
|
|
7409
|
+
* meanwhile (nothing to retry). Throws when the lease stays held by a live execution for longer
|
|
7410
|
+
* than one lease period, or the engine starts draining: the retry is refused, never dropped.
|
|
7411
|
+
*/
|
|
7412
|
+
private takeLeaseForRetry;
|
|
7413
|
+
/** `run` as a {@link RunResult}. */
|
|
7414
|
+
private resultOf;
|
|
7415
|
+
/** The body of {@link execute} once this instance holds `run`'s lease. */
|
|
7416
|
+
private executeLeased;
|
|
7398
7417
|
/**
|
|
7399
7418
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
7400
7419
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
package/dist/index.js
CHANGED
|
@@ -6413,6 +6413,7 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
|
6413
6413
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
6414
6414
|
var LEASE_RECHECK_SLACK_MS = 100;
|
|
6415
6415
|
var LEASE_RECHECK_ATTEMPTS = 30;
|
|
6416
|
+
var RETRY_LEASE_POLL_MS = 100;
|
|
6416
6417
|
function wakeSeen(request, settledFans, served) {
|
|
6417
6418
|
if (request.id !== void 0 && served.has(request.id)) return true;
|
|
6418
6419
|
return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
|
|
@@ -11579,26 +11580,87 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
11579
11580
|
const reentered = run.lockedBy === this.instanceId && (run.lockedUntil ?? 0) > lockNow && (lockedForTurn || await this.proveOwnLease(run.id, lockNow));
|
|
11580
11581
|
if (!reentered) {
|
|
11581
11582
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
11582
|
-
if (
|
|
11583
|
-
|
|
11584
|
-
|
|
11585
|
-
|
|
11586
|
-
}
|
|
11587
|
-
} else {
|
|
11588
|
-
const fans = {
|
|
11589
|
-
fans: [
|
|
11590
|
-
...reason.fans
|
|
11591
|
-
]
|
|
11592
|
-
};
|
|
11593
|
-
this.inBackground(this.requestWakeAfterTurn(run.id, fans));
|
|
11583
|
+
if (run.status === "failed") {
|
|
11584
|
+
const taken = await this.takeLeaseForRetry(run.id);
|
|
11585
|
+
if ("result" in taken) return taken.result;
|
|
11586
|
+
return this.executeLeased(taken.run, fn, registered);
|
|
11594
11587
|
}
|
|
11595
|
-
this.
|
|
11588
|
+
return this.keepWakeForHeldLease(run, reason);
|
|
11589
|
+
}
|
|
11590
|
+
}
|
|
11591
|
+
return this.executeLeased(run, fn, registered);
|
|
11592
|
+
}
|
|
11593
|
+
/**
|
|
11594
|
+
* A wake that found `run`'s lease held by another execution: keep it for that execution to honor
|
|
11595
|
+
* once it releases the lease, and answer with the run's status as read.
|
|
11596
|
+
*/
|
|
11597
|
+
keepWakeForHeldLease(run, reason) {
|
|
11598
|
+
if (reason.generic) {
|
|
11599
|
+
if (!this.genericWakeRequested.has(run.id)) {
|
|
11600
|
+
this.genericWakeRequested.add(run.id);
|
|
11601
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, null));
|
|
11602
|
+
}
|
|
11603
|
+
} else {
|
|
11604
|
+
const fans = {
|
|
11605
|
+
fans: [
|
|
11606
|
+
...reason.fans
|
|
11607
|
+
]
|
|
11608
|
+
};
|
|
11609
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, fans));
|
|
11610
|
+
}
|
|
11611
|
+
this.recheckAfterLease(run.id, run.lockedUntil);
|
|
11612
|
+
return {
|
|
11613
|
+
runId: run.id,
|
|
11614
|
+
status: run.status
|
|
11615
|
+
};
|
|
11616
|
+
}
|
|
11617
|
+
/**
|
|
11618
|
+
* Take `runId`'s lease for an explicit retry of a `failed` run that found it held: the turn that
|
|
11619
|
+
* failed the run has not released it yet, or another instance holds it for a moment. Polls the
|
|
11620
|
+
* lease with the usual fenced take ({@link StateStore.tryLockRun}, which also takes over a lease
|
|
11621
|
+
* that expired; the old holder's renewal then fences it) until it is free. Returns the run, read
|
|
11622
|
+
* under the lease, while it is still `failed`; or the run's state when something else moved it on
|
|
11623
|
+
* meanwhile (nothing to retry). Throws when the lease stays held by a live execution for longer
|
|
11624
|
+
* than one lease period, or the engine starts draining: the retry is refused, never dropped.
|
|
11625
|
+
*/
|
|
11626
|
+
async takeLeaseForRetry(runId) {
|
|
11627
|
+
const giveUpAt = Date.now() + this.leaseMs + LEASE_RECHECK_MAX_MS;
|
|
11628
|
+
for (; ; ) {
|
|
11629
|
+
const current = await this.store.getRun(runId);
|
|
11630
|
+
if (!current) throw new Error(`run ${runId} not found`);
|
|
11631
|
+
if (current.status !== "failed") return {
|
|
11632
|
+
result: this.resultOf(current)
|
|
11633
|
+
};
|
|
11634
|
+
const nowMs = this.clock();
|
|
11635
|
+
if (await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
11636
|
+
const fresh = await this.store.getRun(runId);
|
|
11637
|
+
if (fresh?.status === "failed") return {
|
|
11638
|
+
run: fresh
|
|
11639
|
+
};
|
|
11640
|
+
await this.releaseLeaseBestEffort(runId);
|
|
11596
11641
|
return {
|
|
11597
|
-
|
|
11598
|
-
status: run.status
|
|
11642
|
+
result: fresh ? this.resultOf(fresh) : await this.echoCurrentStatus(runId)
|
|
11599
11643
|
};
|
|
11600
11644
|
}
|
|
11645
|
+
if (this.draining || Date.now() >= giveUpAt) {
|
|
11646
|
+
throw new Error(`run ${runId} could not be retried: its lease is held by ${current.lockedBy ?? "another execution"}${this.draining ? " and this engine is draining" : ""}; resume it again once that execution settles`);
|
|
11647
|
+
}
|
|
11648
|
+
const leaseLeft = (current.lockedUntil ?? nowMs) - nowMs;
|
|
11649
|
+
const wait = Math.min(RETRY_LEASE_POLL_MS, Math.max(10, leaseLeft + LEASE_RECHECK_SLACK_MS));
|
|
11650
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
11601
11651
|
}
|
|
11652
|
+
}
|
|
11653
|
+
/** `run` as a {@link RunResult}. */
|
|
11654
|
+
resultOf(run) {
|
|
11655
|
+
return {
|
|
11656
|
+
runId: run.id,
|
|
11657
|
+
status: run.status,
|
|
11658
|
+
output: run.output,
|
|
11659
|
+
error: run.error
|
|
11660
|
+
};
|
|
11661
|
+
}
|
|
11662
|
+
/** The body of {@link execute} once this instance holds `run`'s lease. */
|
|
11663
|
+
async executeLeased(run, fn, registered) {
|
|
11602
11664
|
const settledFans = /* @__PURE__ */ new Map();
|
|
11603
11665
|
this.turnSettledFans.set(run.id, settledFans);
|
|
11604
11666
|
const held = this.holdExecutionLease(run.id);
|