@smthrs/run-store 0.0.0-stage → 1.0.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -0
- package/LICENSE +21 -0
- package/README.md +195 -2
- package/dist/cjs/AttemptStore.d.ts +359 -0
- package/dist/cjs/AttemptStore.d.ts.map +1 -0
- package/dist/cjs/AttemptStore.js +731 -0
- package/dist/cjs/AttemptStore.js.map +7 -0
- package/dist/cjs/Heartbeat.d.ts +46 -0
- package/dist/cjs/Heartbeat.d.ts.map +1 -0
- package/dist/cjs/Heartbeat.js +28 -0
- package/dist/cjs/Heartbeat.js.map +7 -0
- package/dist/cjs/Migrations.d.ts +42 -0
- package/dist/cjs/Migrations.d.ts.map +1 -0
- package/dist/cjs/Migrations.js +55 -0
- package/dist/cjs/Migrations.js.map +7 -0
- package/dist/cjs/Ownership.d.ts +275 -0
- package/dist/cjs/Ownership.d.ts.map +1 -0
- package/dist/cjs/Ownership.js +129 -0
- package/dist/cjs/Ownership.js.map +7 -0
- package/dist/cjs/RunStore.d.ts +571 -0
- package/dist/cjs/RunStore.d.ts.map +1 -0
- package/dist/cjs/RunStore.js +1079 -0
- package/dist/cjs/RunStore.js.map +7 -0
- package/dist/cjs/RunStoreMetrics.d.ts +151 -0
- package/dist/cjs/RunStoreMetrics.d.ts.map +1 -0
- package/dist/cjs/RunStoreMetrics.js +91 -0
- package/dist/cjs/RunStoreMetrics.js.map +7 -0
- package/dist/cjs/index.d.ts +47 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +43 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/Boundary.d.ts +80 -0
- package/dist/cjs/internal/Boundary.d.ts.map +1 -0
- package/dist/cjs/internal/Boundary.js +92 -0
- package/dist/cjs/internal/Boundary.js.map +7 -0
- package/dist/cjs/internal/Companion.d.ts +44 -0
- package/dist/cjs/internal/Companion.d.ts.map +1 -0
- package/dist/cjs/internal/Companion.js +50 -0
- package/dist/cjs/internal/Companion.js.map +7 -0
- package/dist/cjs/internal/SpanOutcome.d.ts +37 -0
- package/dist/cjs/internal/SpanOutcome.d.ts.map +1 -0
- package/dist/cjs/internal/SpanOutcome.js +62 -0
- package/dist/cjs/internal/SpanOutcome.js.map +7 -0
- package/dist/cjs/migrations/0001_initial.d.ts +15 -0
- package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
- package/dist/cjs/migrations/0001_initial.js +110 -0
- package/dist/cjs/migrations/0001_initial.js.map +7 -0
- package/dist/cjs/migrations/0002_lineage.d.ts +27 -0
- package/dist/cjs/migrations/0002_lineage.d.ts.map +1 -0
- package/dist/cjs/migrations/0002_lineage.js +43 -0
- package/dist/cjs/migrations/0002_lineage.js.map +7 -0
- package/dist/cjs/migrations/0003_execution_revisions.d.ts +15 -0
- package/dist/cjs/migrations/0003_execution_revisions.d.ts.map +1 -0
- package/dist/cjs/migrations/0003_execution_revisions.js +77 -0
- package/dist/cjs/migrations/0003_execution_revisions.js.map +7 -0
- package/dist/cjs/migrations/0004_waiting_request.d.ts +32 -0
- package/dist/cjs/migrations/0004_waiting_request.d.ts.map +1 -0
- package/dist/cjs/migrations/0004_waiting_request.js +42 -0
- package/dist/cjs/migrations/0004_waiting_request.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/test/TestRunStore.d.ts +17 -0
- package/dist/cjs/test/TestRunStore.d.ts.map +1 -0
- package/dist/cjs/test/TestRunStore.js +42 -0
- package/dist/cjs/test/TestRunStore.js.map +7 -0
- package/dist/esm/AttemptStore.d.ts +359 -0
- package/dist/esm/AttemptStore.d.ts.map +1 -0
- package/dist/esm/AttemptStore.js +835 -0
- package/dist/esm/AttemptStore.js.map +1 -0
- package/dist/esm/Heartbeat.d.ts +46 -0
- package/dist/esm/Heartbeat.d.ts.map +1 -0
- package/dist/esm/Heartbeat.js +46 -0
- package/dist/esm/Heartbeat.js.map +1 -0
- package/dist/esm/Migrations.d.ts +42 -0
- package/dist/esm/Migrations.d.ts.map +1 -0
- package/dist/esm/Migrations.js +56 -0
- package/dist/esm/Migrations.js.map +1 -0
- package/dist/esm/Ownership.d.ts +275 -0
- package/dist/esm/Ownership.d.ts.map +1 -0
- package/dist/esm/Ownership.js +299 -0
- package/dist/esm/Ownership.js.map +1 -0
- package/dist/esm/RunStore.d.ts +571 -0
- package/dist/esm/RunStore.d.ts.map +1 -0
- package/dist/esm/RunStore.js +1287 -0
- package/dist/esm/RunStore.js.map +1 -0
- package/dist/esm/RunStoreMetrics.d.ts +151 -0
- package/dist/esm/RunStoreMetrics.d.ts.map +1 -0
- package/dist/esm/RunStoreMetrics.js +120 -0
- package/dist/esm/RunStoreMetrics.js.map +1 -0
- package/dist/esm/index.d.ts +47 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +47 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/Boundary.d.ts +80 -0
- package/dist/esm/internal/Boundary.d.ts.map +1 -0
- package/dist/esm/internal/Boundary.js +109 -0
- package/dist/esm/internal/Boundary.js.map +1 -0
- package/dist/esm/internal/Companion.d.ts +44 -0
- package/dist/esm/internal/Companion.d.ts.map +1 -0
- package/dist/esm/internal/Companion.js +53 -0
- package/dist/esm/internal/Companion.js.map +1 -0
- package/dist/esm/internal/SpanOutcome.d.ts +37 -0
- package/dist/esm/internal/SpanOutcome.d.ts.map +1 -0
- package/dist/esm/internal/SpanOutcome.js +48 -0
- package/dist/esm/internal/SpanOutcome.js.map +1 -0
- package/dist/esm/migrations/0001_initial.d.ts +15 -0
- package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
- package/dist/esm/migrations/0001_initial.js +88 -0
- package/dist/esm/migrations/0001_initial.js.map +1 -0
- package/dist/esm/migrations/0002_lineage.d.ts +27 -0
- package/dist/esm/migrations/0002_lineage.d.ts.map +1 -0
- package/dist/esm/migrations/0002_lineage.js +33 -0
- package/dist/esm/migrations/0002_lineage.js.map +1 -0
- package/dist/esm/migrations/0003_execution_revisions.d.ts +15 -0
- package/dist/esm/migrations/0003_execution_revisions.d.ts.map +1 -0
- package/dist/esm/migrations/0003_execution_revisions.js +58 -0
- package/dist/esm/migrations/0003_execution_revisions.js.map +1 -0
- package/dist/esm/migrations/0004_waiting_request.d.ts +32 -0
- package/dist/esm/migrations/0004_waiting_request.d.ts.map +1 -0
- package/dist/esm/migrations/0004_waiting_request.js +37 -0
- package/dist/esm/migrations/0004_waiting_request.js.map +1 -0
- package/dist/esm/test/TestRunStore.d.ts +17 -0
- package/dist/esm/test/TestRunStore.d.ts.map +1 -0
- package/dist/esm/test/TestRunStore.js +19 -0
- package/dist/esm/test/TestRunStore.js.map +1 -0
- package/docs/README.md +142 -0
- package/docs/api.md +901 -0
- package/docs/concepts/durable-values.md +140 -0
- package/docs/concepts/execution-revisions.md +53 -0
- package/docs/concepts/fencing.md +146 -0
- package/docs/concepts/leases.md +154 -0
- package/docs/concepts/liveness-evidence.md +142 -0
- package/docs/guides/cancel-a-run.md +156 -0
- package/docs/guides/claim-and-finish-a-run.md +175 -0
- package/docs/guides/compose-the-stores.md +142 -0
- package/docs/guides/observe-outcomes.md +182 -0
- package/docs/guides/record-step-attempts.md +170 -0
- package/docs/guides/recover-a-stalled-run.md +173 -0
- package/docs/guides/testing.md +141 -0
- package/docs/installation.md +109 -0
- package/docs/quickstart.md +201 -0
- package/docs/troubleshooting.md +255 -0
- package/package.json +183 -3
- package/src/AttemptStore.ts +1195 -0
- package/src/Heartbeat.ts +47 -0
- package/src/Migrations.ts +59 -0
- package/src/Ownership.ts +407 -0
- package/src/RunStore.ts +2063 -0
- package/src/RunStoreMetrics.ts +165 -0
- package/src/index.ts +51 -0
- package/src/internal/Boundary.ts +130 -0
- package/src/internal/Companion.ts +64 -0
- package/src/internal/SpanOutcome.ts +71 -0
- package/src/migrations/0001_initial.ts +114 -0
- package/src/migrations/0002_lineage.ts +38 -0
- package/src/migrations/0003_execution_revisions.ts +69 -0
- package/src/migrations/0004_waiting_request.ts +39 -0
- package/src/test/TestRunStore.ts +22 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run ownership arbitration: liveness evidence, probes, and heartbeat
|
|
3
|
+
* supervision.
|
|
4
|
+
*
|
|
5
|
+
* The identity being arbitrated — {@link OwnerId} — is defined by
|
|
6
|
+
* `@smthrs/journal`, because it is the fencing token the journal accepts on
|
|
7
|
+
* durable appends. It is re-exported here so ownership callers keep reading it
|
|
8
|
+
* as one vocabulary.
|
|
9
|
+
*
|
|
10
|
+
* @since 0.1.0
|
|
11
|
+
*/
|
|
12
|
+
import { LivenessEvidence } from "@smthrs/journal/Consensus";
|
|
13
|
+
import { OwnerId } from "@smthrs/journal/OwnerId";
|
|
14
|
+
import { Duration, Effect } from "effect";
|
|
15
|
+
import { RunStore } from "./RunStore.ts";
|
|
16
|
+
export {
|
|
17
|
+
/**
|
|
18
|
+
* A process identity scoped to a host and a unique ownership nonce, defined
|
|
19
|
+
* by `@smthrs/journal` as the fence on durable appends.
|
|
20
|
+
*
|
|
21
|
+
* @since 0.1.0
|
|
22
|
+
* @category models
|
|
23
|
+
*/
|
|
24
|
+
OwnerId };
|
|
25
|
+
export {
|
|
26
|
+
/**
|
|
27
|
+
* Evidence that the owner in an exact run snapshot is no longer live.
|
|
28
|
+
*
|
|
29
|
+
* Defined by `@smthrs/journal`'s `Consensus`, because R5 — steal requires
|
|
30
|
+
* staleness plus liveness evidence — is a consensus rule every strategy
|
|
31
|
+
* validates. Two of the three kinds are collected outside the store:
|
|
32
|
+
* `same-host-pid-dead` is a local process probe, and
|
|
33
|
+
* `cross-host-unreachable-stale` is a reachability judgement the deployment
|
|
34
|
+
* makes. `lease-expired` is different — it asserts only that the persisted
|
|
35
|
+
* heartbeat is older than the staleness cutoff, which is the one claim the
|
|
36
|
+
* strategy can check for itself, and `steal` checks it: the takeover
|
|
37
|
+
* refuses any lease whose heartbeat is still inside the window. It is
|
|
38
|
+
* therefore accepted from a claimant on any host, while the other two stay
|
|
39
|
+
* bound to the host relation that makes them meaningful. The observation
|
|
40
|
+
* must carry the exact instant supplied to the consuming operation:
|
|
41
|
+
* `checkedAtMs` must equal that operation's `nowMs`. A caller that probes
|
|
42
|
+
* at T and calls at T+1 is refused and must build fresh evidence.
|
|
43
|
+
*
|
|
44
|
+
* @since 0.1.0
|
|
45
|
+
* @category models
|
|
46
|
+
*/
|
|
47
|
+
LivenessEvidence };
|
|
48
|
+
/**
|
|
49
|
+
* Evidence-factory signature a composition uses for ownership arbitration.
|
|
50
|
+
*
|
|
51
|
+
* A `LivenessProbe` returns `LivenessEvidence` or `undefined` for `steal`,
|
|
52
|
+
* `claimAndOwn`, and `recoverClaim` to verify. {@link LivenessCheck} is the
|
|
53
|
+
* separate alive-or-not question the engine asks before it decides to steal.
|
|
54
|
+
* `RunStore` validates supplied evidence and never probes a process or network
|
|
55
|
+
* itself.
|
|
56
|
+
*
|
|
57
|
+
* @since 0.1.0
|
|
58
|
+
* @category models
|
|
59
|
+
*/
|
|
60
|
+
export type LivenessProbe<E = never, R = never> = (expectedOwner: OwnerId, claimant: OwnerId, checkedAtMs: number) => Effect.Effect<LivenessEvidence | undefined, E, R>;
|
|
61
|
+
/**
|
|
62
|
+
* What a liveness check knows about the run it is asked about, beyond the
|
|
63
|
+
* owner recorded on it.
|
|
64
|
+
*
|
|
65
|
+
* The lease is here because it is the only liveness signal every deployment
|
|
66
|
+
* has: `heartbeatAtMs` is the last heartbeat the owner persisted, `nowMs` is
|
|
67
|
+
* the reading the arbitration is made against, and `claimant` is the identity
|
|
68
|
+
* that would take the run over. A check that wants to probe a pid compares
|
|
69
|
+
* hosts with {@link sameHostIncarnation} first; a check that has nothing but
|
|
70
|
+
* the lease uses {@link leaseLiveness}.
|
|
71
|
+
*
|
|
72
|
+
* @since 0.1.0
|
|
73
|
+
* @category models
|
|
74
|
+
*/
|
|
75
|
+
export interface LivenessContext {
|
|
76
|
+
readonly claimant: OwnerId;
|
|
77
|
+
readonly heartbeatAtMs: number | null;
|
|
78
|
+
readonly nowMs: number;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The question ownership arbitration asks before it steals a run: is the
|
|
82
|
+
* recorded owner still working?
|
|
83
|
+
*
|
|
84
|
+
* Answering `true` refuses the takeover. The engine consults this only for a
|
|
85
|
+
* run whose lease has already expired, so a check that has no better evidence
|
|
86
|
+
* than the lease answers `false` there — which is exactly what
|
|
87
|
+
* {@link leaseLiveness} does.
|
|
88
|
+
*
|
|
89
|
+
* @since 0.1.0
|
|
90
|
+
* @category models
|
|
91
|
+
*/
|
|
92
|
+
export type LivenessCheck = (expectedOwner: OwnerId, context: LivenessContext) => Effect.Effect<boolean>;
|
|
93
|
+
/**
|
|
94
|
+
* Whether two owner identities are incarnations on the same host.
|
|
95
|
+
*
|
|
96
|
+
* The predicate a probe applies before it inspects a pid: `owner.pid` names a
|
|
97
|
+
* process in the claimant's own process namespace only when the hosts match,
|
|
98
|
+
* so a cross-host probe that read it would be answering about an unrelated
|
|
99
|
+
* process. Exposed because the check that needs it — a
|
|
100
|
+
* `process.kill(pid, 0)` probe — belongs to a platform package rather than
|
|
101
|
+
* here.
|
|
102
|
+
*
|
|
103
|
+
* @since 0.1.0
|
|
104
|
+
* @category ownership
|
|
105
|
+
*/
|
|
106
|
+
export declare const sameHostIncarnation: (expectedOwner: OwnerId, claimant: OwnerId) => boolean;
|
|
107
|
+
/**
|
|
108
|
+
* The default liveness check: the lease, and nothing else.
|
|
109
|
+
*
|
|
110
|
+
* An owner is treated as alive for as long as its persisted heartbeat is
|
|
111
|
+
* younger than `staleAfter`, and as gone once it is not. That is the weakest
|
|
112
|
+
* honest answer, and it is the one every host can give: a fresh process with
|
|
113
|
+
* no application code at all can reclaim a hard-killed owner's runs once the
|
|
114
|
+
* lease it stopped renewing has expired. A deployment that can say more — a
|
|
115
|
+
* pid probe on the owner's host, an orchestrator that reports pod liveness —
|
|
116
|
+
* supplies its own {@link LivenessCheck} and refuses the takeover for longer.
|
|
117
|
+
*
|
|
118
|
+
* An owner with no recorded heartbeat holds no lease and is reported gone; the
|
|
119
|
+
* steal it enables is still gated by the store's own snapshot compare-and-swap.
|
|
120
|
+
*
|
|
121
|
+
* @since 0.1.0
|
|
122
|
+
* @category ownership
|
|
123
|
+
*/
|
|
124
|
+
export declare const leaseLiveness: (staleAfter?: Duration.Input) => LivenessCheck;
|
|
125
|
+
/**
|
|
126
|
+
* The Node hosts' liveness check: does the owner's process still exist?
|
|
127
|
+
*
|
|
128
|
+
* {@link leaseLiveness} is the honest floor — every host can read a persisted
|
|
129
|
+
* heartbeat — but it is only a timeout, so two engine processes over one
|
|
130
|
+
* database steal each other's running rows `heartbeatStaleAfter` after any
|
|
131
|
+
* heartbeat stall: a stop-the-world pause, a swapped-out process, a disk that
|
|
132
|
+
* blocked longer than the window. This answers the question the lease is
|
|
133
|
+
* standing in for, by asking the operating system whether the recorded pid is
|
|
134
|
+
* still there.
|
|
135
|
+
*
|
|
136
|
+
* `process.kill(pid, 0)` sends no signal; it performs only the delivery
|
|
137
|
+
* checks. Three answers matter:
|
|
138
|
+
*
|
|
139
|
+
* - It returns: the process exists and is signalable. The owner is alive.
|
|
140
|
+
* - It throws `ESRCH`: no such process exists. This is the sole death answer,
|
|
141
|
+
* so the owner is gone.
|
|
142
|
+
* - It throws anything else, including `EPERM`, `EINVAL`, or an unrecognized
|
|
143
|
+
* value: the owner is treated as alive because an unknown answer is not
|
|
144
|
+
* death.
|
|
145
|
+
*
|
|
146
|
+
* This `ESRCH`-only death rule deliberately aligns with
|
|
147
|
+
* `@smthrs/platform-node`'s `HostLiveness`.
|
|
148
|
+
*
|
|
149
|
+
* A pid is only meaningful inside one process namespace, so a recorded owner
|
|
150
|
+
* on another host is never probed: the answer is `false` and the arbitration
|
|
151
|
+
* falls back to the evidence that does cross hosts — the expired lease, which
|
|
152
|
+
* `RunStore.steal` verifies for itself. The engine consults this check only
|
|
153
|
+
* for a run whose lease has ALREADY expired, so answering `false` here does
|
|
154
|
+
* not weaken anything; it declines to add evidence.
|
|
155
|
+
*
|
|
156
|
+
* Node hosts only. It is not part of the browser promise: this entry point
|
|
157
|
+
* bundles for the browser because it never imports a `node:` built-in, and a
|
|
158
|
+
* browser composition has no process table to ask, so it keeps
|
|
159
|
+
* {@link leaseLiveness}.
|
|
160
|
+
*
|
|
161
|
+
* `@smthrs/platform-node`'s `HostLiveness.isAlive` asks the same question of
|
|
162
|
+
* the same process table and differs in one deliberate place: it answers
|
|
163
|
+
* `true` for an owner on another host, which refuses the steal outright, while
|
|
164
|
+
* this check answers `false` and lets the expired lease decide. The difference
|
|
165
|
+
* matters after a host dies for good — under the refusing answer its runs are
|
|
166
|
+
* never reclaimed by anyone, because no other machine can ever produce
|
|
167
|
+
* evidence about its pids.
|
|
168
|
+
*
|
|
169
|
+
* Two limits are inherent to asking a pid, and they bound what reclaim can
|
|
170
|
+
* promise. Both are shared with `HostLiveness.isAlive`.
|
|
171
|
+
*
|
|
172
|
+
* - An owner recorded with the CLAIMANT'S OWN pid is always alive. A previous
|
|
173
|
+
* incarnation of this process, or a second engine composed inside it, differs
|
|
174
|
+
* from the claimant only by `nonce`, and the process it names is this one.
|
|
175
|
+
* Such a row is never stolen while the process lives, so an embedded host
|
|
176
|
+
* that re-creates its engine in place should keep {@link leaseLiveness},
|
|
177
|
+
* whose timeout does expire. Reading same-pid-different-nonce as death is
|
|
178
|
+
* not the alternative: it would let two engines in one process — the exact
|
|
179
|
+
* shape this check exists to arbitrate — steal each other's live runs.
|
|
180
|
+
* - A pid the operating system has REUSED reports the unrelated process that
|
|
181
|
+
* now holds it. The dead owner's row stays refused for as long as that
|
|
182
|
+
* process lives, which delays reclaim rather than breaking it: the row is
|
|
183
|
+
* still `running` under an expired lease, and the next probe after the pid
|
|
184
|
+
* is free reclaims it.
|
|
185
|
+
*
|
|
186
|
+
* @since 0.1.0
|
|
187
|
+
* @category ownership
|
|
188
|
+
*/
|
|
189
|
+
export declare const sameHostPidProbe: LivenessCheck;
|
|
190
|
+
export {
|
|
191
|
+
/**
|
|
192
|
+
* How often the supervision loop pulses to renew the owner's lease.
|
|
193
|
+
*
|
|
194
|
+
* @since 0.1.0
|
|
195
|
+
* @category constants
|
|
196
|
+
*/
|
|
197
|
+
heartbeatInterval,
|
|
198
|
+
/**
|
|
199
|
+
* How far the owner's wall clock may run behind a peer's before the lease
|
|
200
|
+
* reasoning stops holding.
|
|
201
|
+
*
|
|
202
|
+
* @since 0.1.0
|
|
203
|
+
* @category constants
|
|
204
|
+
*/
|
|
205
|
+
heartbeatSkewAllowance,
|
|
206
|
+
/**
|
|
207
|
+
* How old a persisted heartbeat must be before a peer may steal the run.
|
|
208
|
+
*
|
|
209
|
+
* @since 0.1.0
|
|
210
|
+
* @category constants
|
|
211
|
+
*/
|
|
212
|
+
heartbeatStaleAfter,
|
|
213
|
+
/**
|
|
214
|
+
* How long the owner may keep working through failing or stalled heartbeat writes.
|
|
215
|
+
*
|
|
216
|
+
* @since 0.1.0
|
|
217
|
+
* @category constants
|
|
218
|
+
*/
|
|
219
|
+
heartbeatWriteTolerance } from "./Heartbeat.ts";
|
|
220
|
+
/**
|
|
221
|
+
* Hooks a {@link heartbeatLoop} caller may supply.
|
|
222
|
+
*
|
|
223
|
+
* @since 1.0.0
|
|
224
|
+
* @category models
|
|
225
|
+
*/
|
|
226
|
+
export interface HeartbeatLoopOptions {
|
|
227
|
+
/** Called after the same unclaimed owner reconfirms an expired heartbeat budget. */
|
|
228
|
+
readonly onReconfirm?: ((unconfirmedMs: number) => Effect.Effect<void>) | undefined;
|
|
229
|
+
/** Called with the unconfirmed milliseconds when the lease lapses, before the loop interrupts itself. */
|
|
230
|
+
readonly onLapse?: ((unconfirmedMs: number) => Effect.Effect<void>) | undefined;
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Runs heartbeats until the persisted ownership fence is lost, then interrupts
|
|
234
|
+
* itself. Race this effect with owned work so structured concurrency
|
|
235
|
+
* interrupts the work when ownership disappears.
|
|
236
|
+
*
|
|
237
|
+
* Each pulse drives the injected `Consensus` strategy's `heartbeat` through
|
|
238
|
+
* `RunStore.heartbeat`, which renews the strategy's lease and mirrors the
|
|
239
|
+
* recorded stamp onto the run row in the same transaction. Heartbeats are
|
|
240
|
+
* lease evidence, never journal events.
|
|
241
|
+
*
|
|
242
|
+
* Pulses are delayed by `heartbeatInterval` and read the Effect `Clock`, so the
|
|
243
|
+
* loop is fully driveable with `TestClock`.
|
|
244
|
+
*
|
|
245
|
+
* A lost fence — any outcome other than `Updated` — is durable evidence and
|
|
246
|
+
* interrupts immediately. A failed heartbeat *write* is not: the persisted
|
|
247
|
+
* heartbeat is still there and no other process may steal the run until it is
|
|
248
|
+
* `heartbeatStaleAfter` old, so transient write errors are tolerated for
|
|
249
|
+
* `heartbeatWriteTolerance` — deliberately shorter than the steal cutoff by a
|
|
250
|
+
* pulse plus `heartbeatSkewAllowance`. The reserved pulse bounds a final
|
|
251
|
+
* reconfirmation; a pending claim or changed owner fails that check. Past that allowance
|
|
252
|
+
* the fence still protects durable writes but non-durable side effects may
|
|
253
|
+
* overlap; see {@link heartbeatWriteTolerance}.
|
|
254
|
+
*
|
|
255
|
+
* An independent deadline races the pulse loop. At expiry it reconfirms the
|
|
256
|
+
* same owner with no pending claim, bounded by one heartbeat interval. A
|
|
257
|
+
* renewed lease continues supervision; loss, error, or timeout interrupts
|
|
258
|
+
* even when an ordinary heartbeat write never returns. Each successful
|
|
259
|
+
* pulse re-arms the deadline from the timestamp sent to the store, not from
|
|
260
|
+
* its completion time. The deadline re-reads the clock after waking, so delayed
|
|
261
|
+
* writes cannot hide expiry behind a stale clock reading.
|
|
262
|
+
*
|
|
263
|
+
* `options.onReconfirm` records a successful reconfirmation after a gap. Its
|
|
264
|
+
* deadline is the remaining renewed lease budget, measured from the timestamp
|
|
265
|
+
* sent to the store. A blocked or failed receipt lapses the lease instead of
|
|
266
|
+
* stranding supervision; receipt completion never grants another lease budget.
|
|
267
|
+
* `options.onLapse` runs once, before the self-interrupt, when reconfirmation
|
|
268
|
+
* fails: it receives how long the lease went unconfirmed, so the caller can
|
|
269
|
+
* record why its work stopped. A lost fence does not call it.
|
|
270
|
+
*
|
|
271
|
+
* @since 0.1.0
|
|
272
|
+
* @category supervision
|
|
273
|
+
*/
|
|
274
|
+
export declare const heartbeatLoop: (runId: string, owner: OwnerId, options?: HeartbeatLoopOptions) => Effect.Effect<never, never, RunStore>;
|
|
275
|
+
//# sourceMappingURL=Ownership.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Ownership.d.ts","sourceRoot":"","sources":["../../src/Ownership.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAgB,QAAQ,EAAE,MAAM,EAAa,MAAM,QAAQ,CAAA;AAElE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAExC,OAAO;AACL;;;;;;GAMG;AACH,OAAO,EACR,CAAA;AAED,OAAO;AACL;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,gBAAgB,EACjB,CAAA;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,aAAa,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,KAAK,IAAI,CAChD,aAAa,EAAE,OAAO,EACtB,QAAQ,EAAE,OAAO,EACjB,WAAW,EAAE,MAAM,KAChB,MAAM,CAAC,MAAM,CAAC,gBAAgB,GAAG,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;AAEtD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAA;IAC1B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,aAAa,GAAG,CAC1B,aAAa,EAAE,OAAO,EACtB,OAAO,EAAE,eAAe,KACrB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AAE3B;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,mBAAmB,GAC9B,eAAe,OAAO,EACtB,UAAU,OAAO,KAChB,OAAmD,CAAA;AAEtD;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,aAAa,GACxB,aAAY,QAAQ,CAAC,KAA2B,KAC/C,aAOF,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,eAAO,MAAM,gBAAgB,EAAE,aAU3B,CAAA;AAEJ,OAAO;AACL;;;;;GAKG;AACH,iBAAiB;AACjB;;;;;;GAMG;AACH,sBAAsB;AACtB;;;;;GAKG;AACH,mBAAmB;AACnB;;;;;GAKG;AACH,uBAAuB,EACxB,MAAM,gBAAgB,CAAA;AAEvB;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACnC,oFAAoF;IACpF,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,aAAa,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAA;IACnF,yGAAyG;IACzG,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,aAAa,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAA;CAChF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,eAAO,MAAM,aAAa,GACxB,OAAO,MAAM,EACb,OAAO,OAAO,EACd,UAAS,oBAAyB,KACjC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAoFnC,CAAA"}
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run ownership arbitration: liveness evidence, probes, and heartbeat
|
|
3
|
+
* supervision.
|
|
4
|
+
*
|
|
5
|
+
* The identity being arbitrated — {@link OwnerId} — is defined by
|
|
6
|
+
* `@smthrs/journal`, because it is the fencing token the journal accepts on
|
|
7
|
+
* durable appends. It is re-exported here so ownership callers keep reading it
|
|
8
|
+
* as one vocabulary.
|
|
9
|
+
*
|
|
10
|
+
* @since 0.1.0
|
|
11
|
+
*/
|
|
12
|
+
import { LivenessEvidence } from "@smthrs/journal/Consensus";
|
|
13
|
+
import { OwnerId } from "@smthrs/journal/OwnerId";
|
|
14
|
+
import { Cause, Clock, Duration, Effect, Semaphore } from "effect";
|
|
15
|
+
import { heartbeatInterval, heartbeatStaleAfter, heartbeatWriteTolerance } from "./Heartbeat.js";
|
|
16
|
+
import { RunStore } from "./RunStore.js";
|
|
17
|
+
export {
|
|
18
|
+
/**
|
|
19
|
+
* A process identity scoped to a host and a unique ownership nonce, defined
|
|
20
|
+
* by `@smthrs/journal` as the fence on durable appends.
|
|
21
|
+
*
|
|
22
|
+
* @since 0.1.0
|
|
23
|
+
* @category models
|
|
24
|
+
*/
|
|
25
|
+
OwnerId };
|
|
26
|
+
export {
|
|
27
|
+
/**
|
|
28
|
+
* Evidence that the owner in an exact run snapshot is no longer live.
|
|
29
|
+
*
|
|
30
|
+
* Defined by `@smthrs/journal`'s `Consensus`, because R5 — steal requires
|
|
31
|
+
* staleness plus liveness evidence — is a consensus rule every strategy
|
|
32
|
+
* validates. Two of the three kinds are collected outside the store:
|
|
33
|
+
* `same-host-pid-dead` is a local process probe, and
|
|
34
|
+
* `cross-host-unreachable-stale` is a reachability judgement the deployment
|
|
35
|
+
* makes. `lease-expired` is different — it asserts only that the persisted
|
|
36
|
+
* heartbeat is older than the staleness cutoff, which is the one claim the
|
|
37
|
+
* strategy can check for itself, and `steal` checks it: the takeover
|
|
38
|
+
* refuses any lease whose heartbeat is still inside the window. It is
|
|
39
|
+
* therefore accepted from a claimant on any host, while the other two stay
|
|
40
|
+
* bound to the host relation that makes them meaningful. The observation
|
|
41
|
+
* must carry the exact instant supplied to the consuming operation:
|
|
42
|
+
* `checkedAtMs` must equal that operation's `nowMs`. A caller that probes
|
|
43
|
+
* at T and calls at T+1 is refused and must build fresh evidence.
|
|
44
|
+
*
|
|
45
|
+
* @since 0.1.0
|
|
46
|
+
* @category models
|
|
47
|
+
*/
|
|
48
|
+
LivenessEvidence };
|
|
49
|
+
/**
|
|
50
|
+
* Whether two owner identities are incarnations on the same host.
|
|
51
|
+
*
|
|
52
|
+
* The predicate a probe applies before it inspects a pid: `owner.pid` names a
|
|
53
|
+
* process in the claimant's own process namespace only when the hosts match,
|
|
54
|
+
* so a cross-host probe that read it would be answering about an unrelated
|
|
55
|
+
* process. Exposed because the check that needs it — a
|
|
56
|
+
* `process.kill(pid, 0)` probe — belongs to a platform package rather than
|
|
57
|
+
* here.
|
|
58
|
+
*
|
|
59
|
+
* @since 0.1.0
|
|
60
|
+
* @category ownership
|
|
61
|
+
*/
|
|
62
|
+
export const sameHostIncarnation = (expectedOwner, claimant) => expectedOwner.hostId === claimant.hostId;
|
|
63
|
+
/**
|
|
64
|
+
* The default liveness check: the lease, and nothing else.
|
|
65
|
+
*
|
|
66
|
+
* An owner is treated as alive for as long as its persisted heartbeat is
|
|
67
|
+
* younger than `staleAfter`, and as gone once it is not. That is the weakest
|
|
68
|
+
* honest answer, and it is the one every host can give: a fresh process with
|
|
69
|
+
* no application code at all can reclaim a hard-killed owner's runs once the
|
|
70
|
+
* lease it stopped renewing has expired. A deployment that can say more — a
|
|
71
|
+
* pid probe on the owner's host, an orchestrator that reports pod liveness —
|
|
72
|
+
* supplies its own {@link LivenessCheck} and refuses the takeover for longer.
|
|
73
|
+
*
|
|
74
|
+
* An owner with no recorded heartbeat holds no lease and is reported gone; the
|
|
75
|
+
* steal it enables is still gated by the store's own snapshot compare-and-swap.
|
|
76
|
+
*
|
|
77
|
+
* @since 0.1.0
|
|
78
|
+
* @category ownership
|
|
79
|
+
*/
|
|
80
|
+
export const leaseLiveness = (staleAfter = heartbeatStaleAfter) => {
|
|
81
|
+
const staleAfterMs = Duration.toMillis(staleAfter);
|
|
82
|
+
return (_expectedOwner, context) => Effect.succeed(context.heartbeatAtMs !== null &&
|
|
83
|
+
context.heartbeatAtMs >= context.nowMs - staleAfterMs);
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* The Node hosts' liveness check: does the owner's process still exist?
|
|
87
|
+
*
|
|
88
|
+
* {@link leaseLiveness} is the honest floor — every host can read a persisted
|
|
89
|
+
* heartbeat — but it is only a timeout, so two engine processes over one
|
|
90
|
+
* database steal each other's running rows `heartbeatStaleAfter` after any
|
|
91
|
+
* heartbeat stall: a stop-the-world pause, a swapped-out process, a disk that
|
|
92
|
+
* blocked longer than the window. This answers the question the lease is
|
|
93
|
+
* standing in for, by asking the operating system whether the recorded pid is
|
|
94
|
+
* still there.
|
|
95
|
+
*
|
|
96
|
+
* `process.kill(pid, 0)` sends no signal; it performs only the delivery
|
|
97
|
+
* checks. Three answers matter:
|
|
98
|
+
*
|
|
99
|
+
* - It returns: the process exists and is signalable. The owner is alive.
|
|
100
|
+
* - It throws `ESRCH`: no such process exists. This is the sole death answer,
|
|
101
|
+
* so the owner is gone.
|
|
102
|
+
* - It throws anything else, including `EPERM`, `EINVAL`, or an unrecognized
|
|
103
|
+
* value: the owner is treated as alive because an unknown answer is not
|
|
104
|
+
* death.
|
|
105
|
+
*
|
|
106
|
+
* This `ESRCH`-only death rule deliberately aligns with
|
|
107
|
+
* `@smthrs/platform-node`'s `HostLiveness`.
|
|
108
|
+
*
|
|
109
|
+
* A pid is only meaningful inside one process namespace, so a recorded owner
|
|
110
|
+
* on another host is never probed: the answer is `false` and the arbitration
|
|
111
|
+
* falls back to the evidence that does cross hosts — the expired lease, which
|
|
112
|
+
* `RunStore.steal` verifies for itself. The engine consults this check only
|
|
113
|
+
* for a run whose lease has ALREADY expired, so answering `false` here does
|
|
114
|
+
* not weaken anything; it declines to add evidence.
|
|
115
|
+
*
|
|
116
|
+
* Node hosts only. It is not part of the browser promise: this entry point
|
|
117
|
+
* bundles for the browser because it never imports a `node:` built-in, and a
|
|
118
|
+
* browser composition has no process table to ask, so it keeps
|
|
119
|
+
* {@link leaseLiveness}.
|
|
120
|
+
*
|
|
121
|
+
* `@smthrs/platform-node`'s `HostLiveness.isAlive` asks the same question of
|
|
122
|
+
* the same process table and differs in one deliberate place: it answers
|
|
123
|
+
* `true` for an owner on another host, which refuses the steal outright, while
|
|
124
|
+
* this check answers `false` and lets the expired lease decide. The difference
|
|
125
|
+
* matters after a host dies for good — under the refusing answer its runs are
|
|
126
|
+
* never reclaimed by anyone, because no other machine can ever produce
|
|
127
|
+
* evidence about its pids.
|
|
128
|
+
*
|
|
129
|
+
* Two limits are inherent to asking a pid, and they bound what reclaim can
|
|
130
|
+
* promise. Both are shared with `HostLiveness.isAlive`.
|
|
131
|
+
*
|
|
132
|
+
* - An owner recorded with the CLAIMANT'S OWN pid is always alive. A previous
|
|
133
|
+
* incarnation of this process, or a second engine composed inside it, differs
|
|
134
|
+
* from the claimant only by `nonce`, and the process it names is this one.
|
|
135
|
+
* Such a row is never stolen while the process lives, so an embedded host
|
|
136
|
+
* that re-creates its engine in place should keep {@link leaseLiveness},
|
|
137
|
+
* whose timeout does expire. Reading same-pid-different-nonce as death is
|
|
138
|
+
* not the alternative: it would let two engines in one process — the exact
|
|
139
|
+
* shape this check exists to arbitrate — steal each other's live runs.
|
|
140
|
+
* - A pid the operating system has REUSED reports the unrelated process that
|
|
141
|
+
* now holds it. The dead owner's row stays refused for as long as that
|
|
142
|
+
* process lives, which delays reclaim rather than breaking it: the row is
|
|
143
|
+
* still `running` under an expired lease, and the next probe after the pid
|
|
144
|
+
* is free reclaims it.
|
|
145
|
+
*
|
|
146
|
+
* @since 0.1.0
|
|
147
|
+
* @category ownership
|
|
148
|
+
*/
|
|
149
|
+
export const sameHostPidProbe = (expectedOwner, context) => Effect.sync(() => {
|
|
150
|
+
if (!sameHostIncarnation(expectedOwner, context.claimant))
|
|
151
|
+
return false;
|
|
152
|
+
if (!Number.isSafeInteger(expectedOwner.pid) || expectedOwner.pid <= 0)
|
|
153
|
+
return true;
|
|
154
|
+
try {
|
|
155
|
+
process.kill(expectedOwner.pid, 0);
|
|
156
|
+
return true;
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
return error?.code !== "ESRCH";
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
export {
|
|
163
|
+
/**
|
|
164
|
+
* How often the supervision loop pulses to renew the owner's lease.
|
|
165
|
+
*
|
|
166
|
+
* @since 0.1.0
|
|
167
|
+
* @category constants
|
|
168
|
+
*/
|
|
169
|
+
heartbeatInterval,
|
|
170
|
+
/**
|
|
171
|
+
* How far the owner's wall clock may run behind a peer's before the lease
|
|
172
|
+
* reasoning stops holding.
|
|
173
|
+
*
|
|
174
|
+
* @since 0.1.0
|
|
175
|
+
* @category constants
|
|
176
|
+
*/
|
|
177
|
+
heartbeatSkewAllowance,
|
|
178
|
+
/**
|
|
179
|
+
* How old a persisted heartbeat must be before a peer may steal the run.
|
|
180
|
+
*
|
|
181
|
+
* @since 0.1.0
|
|
182
|
+
* @category constants
|
|
183
|
+
*/
|
|
184
|
+
heartbeatStaleAfter,
|
|
185
|
+
/**
|
|
186
|
+
* How long the owner may keep working through failing or stalled heartbeat writes.
|
|
187
|
+
*
|
|
188
|
+
* @since 0.1.0
|
|
189
|
+
* @category constants
|
|
190
|
+
*/
|
|
191
|
+
heartbeatWriteTolerance } from "./Heartbeat.js";
|
|
192
|
+
/**
|
|
193
|
+
* Runs heartbeats until the persisted ownership fence is lost, then interrupts
|
|
194
|
+
* itself. Race this effect with owned work so structured concurrency
|
|
195
|
+
* interrupts the work when ownership disappears.
|
|
196
|
+
*
|
|
197
|
+
* Each pulse drives the injected `Consensus` strategy's `heartbeat` through
|
|
198
|
+
* `RunStore.heartbeat`, which renews the strategy's lease and mirrors the
|
|
199
|
+
* recorded stamp onto the run row in the same transaction. Heartbeats are
|
|
200
|
+
* lease evidence, never journal events.
|
|
201
|
+
*
|
|
202
|
+
* Pulses are delayed by `heartbeatInterval` and read the Effect `Clock`, so the
|
|
203
|
+
* loop is fully driveable with `TestClock`.
|
|
204
|
+
*
|
|
205
|
+
* A lost fence — any outcome other than `Updated` — is durable evidence and
|
|
206
|
+
* interrupts immediately. A failed heartbeat *write* is not: the persisted
|
|
207
|
+
* heartbeat is still there and no other process may steal the run until it is
|
|
208
|
+
* `heartbeatStaleAfter` old, so transient write errors are tolerated for
|
|
209
|
+
* `heartbeatWriteTolerance` — deliberately shorter than the steal cutoff by a
|
|
210
|
+
* pulse plus `heartbeatSkewAllowance`. The reserved pulse bounds a final
|
|
211
|
+
* reconfirmation; a pending claim or changed owner fails that check. Past that allowance
|
|
212
|
+
* the fence still protects durable writes but non-durable side effects may
|
|
213
|
+
* overlap; see {@link heartbeatWriteTolerance}.
|
|
214
|
+
*
|
|
215
|
+
* An independent deadline races the pulse loop. At expiry it reconfirms the
|
|
216
|
+
* same owner with no pending claim, bounded by one heartbeat interval. A
|
|
217
|
+
* renewed lease continues supervision; loss, error, or timeout interrupts
|
|
218
|
+
* even when an ordinary heartbeat write never returns. Each successful
|
|
219
|
+
* pulse re-arms the deadline from the timestamp sent to the store, not from
|
|
220
|
+
* its completion time. The deadline re-reads the clock after waking, so delayed
|
|
221
|
+
* writes cannot hide expiry behind a stale clock reading.
|
|
222
|
+
*
|
|
223
|
+
* `options.onReconfirm` records a successful reconfirmation after a gap. Its
|
|
224
|
+
* deadline is the remaining renewed lease budget, measured from the timestamp
|
|
225
|
+
* sent to the store. A blocked or failed receipt lapses the lease instead of
|
|
226
|
+
* stranding supervision; receipt completion never grants another lease budget.
|
|
227
|
+
* `options.onLapse` runs once, before the self-interrupt, when reconfirmation
|
|
228
|
+
* fails: it receives how long the lease went unconfirmed, so the caller can
|
|
229
|
+
* record why its work stopped. A lost fence does not call it.
|
|
230
|
+
*
|
|
231
|
+
* @since 0.1.0
|
|
232
|
+
* @category supervision
|
|
233
|
+
*/
|
|
234
|
+
export const heartbeatLoop = (runId, owner, options = {}) => Effect.gen(function* () {
|
|
235
|
+
const runStore = yield* RunStore;
|
|
236
|
+
const toleranceMs = Duration.toMillis(heartbeatWriteTolerance);
|
|
237
|
+
const intervalMs = Duration.toMillis(heartbeatInterval);
|
|
238
|
+
let lastConfirmedPulseMs = yield* Clock.currentTimeMillis;
|
|
239
|
+
let failing = false;
|
|
240
|
+
// Both overdue timers may wake after a host stall. Serialize expiry and
|
|
241
|
+
// recheck the budget so they cannot reconfirm or journal the same gap twice.
|
|
242
|
+
const expiry = yield* Semaphore.make(1);
|
|
243
|
+
const expire = () => expiry.withPermit(Effect.gen(function* () {
|
|
244
|
+
// A competing expiry may have held the permit across a write. Take
|
|
245
|
+
// fresh evidence after acquisition, never the waiting timer's reading.
|
|
246
|
+
const nowMs = yield* Clock.currentTimeMillis;
|
|
247
|
+
if (nowMs - lastConfirmedPulseMs < toleranceMs)
|
|
248
|
+
return;
|
|
249
|
+
const unconfirmedMs = nowMs - lastConfirmedPulseMs;
|
|
250
|
+
const renewed = yield* runStore.reconfirm(runId, owner, Math.floor(nowMs)).pipe(
|
|
251
|
+
// Only the ownership check consumes the old lease's reserved pulse.
|
|
252
|
+
Effect.timeout(heartbeatInterval), Effect.flatMap((outcome) => Effect.gen(function* () {
|
|
253
|
+
if (outcome._tag !== "Updated")
|
|
254
|
+
return false;
|
|
255
|
+
lastConfirmedPulseMs = Math.max(lastConfirmedPulseMs, Math.floor(nowMs));
|
|
256
|
+
failing = false;
|
|
257
|
+
const confirmedAtMs = yield* Clock.currentTimeMillis;
|
|
258
|
+
const remainingMs = lastConfirmedPulseMs + toleranceMs - confirmedAtMs;
|
|
259
|
+
if (remainingMs <= 0)
|
|
260
|
+
return false;
|
|
261
|
+
// The receipt is still required and bounded, but ownership was
|
|
262
|
+
// already renewed. Spending only its old reserved pulse would
|
|
263
|
+
// interrupt live work despite a fresh, fenced persisted lease.
|
|
264
|
+
yield* (options.onReconfirm?.(unconfirmedMs) ?? Effect.void).pipe(Effect.timeout(remainingMs));
|
|
265
|
+
return true;
|
|
266
|
+
})), Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause) : Effect.succeed(false)));
|
|
267
|
+
if (renewed)
|
|
268
|
+
return;
|
|
269
|
+
yield* Effect.logWarning("run lease lapsed; interrupting owned work").pipe(Effect.annotateLogs({ runId, unconfirmedMs }), Effect.andThen(options.onLapse?.(unconfirmedMs) ?? Effect.void), Effect.andThen(Effect.interrupt));
|
|
270
|
+
}));
|
|
271
|
+
const deadline = Effect.gen(function* () {
|
|
272
|
+
while (true) {
|
|
273
|
+
const nowMs = yield* Clock.currentTimeMillis;
|
|
274
|
+
const remainingMs = lastConfirmedPulseMs + toleranceMs - nowMs;
|
|
275
|
+
if (remainingMs <= 0) {
|
|
276
|
+
yield* expire();
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
// Check alongside pulse intervals, with a shorter final wait when needed.
|
|
280
|
+
yield* Effect.sleep(Math.min(remainingMs, intervalMs));
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
const pulses = Effect.sleep(heartbeatInterval).pipe(Effect.andThen(Clock.currentTimeMillis.pipe(Effect.map(Math.floor))), Effect.flatMap((nowMs) => nowMs - lastConfirmedPulseMs >= toleranceMs ? expire() : runStore.heartbeat(runId, owner, nowMs).pipe(Effect.flatMap((outcome) => outcome._tag === "Updated"
|
|
284
|
+
? Effect.sync(() => {
|
|
285
|
+
lastConfirmedPulseMs = Math.max(lastConfirmedPulseMs, nowMs);
|
|
286
|
+
failing = false;
|
|
287
|
+
})
|
|
288
|
+
: Effect.interrupt),
|
|
289
|
+
// A failed write keeps the confirmed lease; the independent deadline
|
|
290
|
+
// supervises its expiry. Warn on the first failure of an outage.
|
|
291
|
+
Effect.catch((error) => {
|
|
292
|
+
if (failing)
|
|
293
|
+
return Effect.void;
|
|
294
|
+
failing = true;
|
|
295
|
+
return Effect.logWarning("run heartbeat write failed").pipe(Effect.annotateLogs({ runId, code: error.code }));
|
|
296
|
+
}))), Effect.forever);
|
|
297
|
+
return yield* Effect.raceFirst(pulses, deadline);
|
|
298
|
+
});
|
|
299
|
+
//# sourceMappingURL=Ownership.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Ownership.js","sourceRoot":"","sources":["../../src/Ownership.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAClE,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AAChG,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAExC,OAAO;AACL;;;;;;GAMG;AACH,OAAO,EACR,CAAA;AAED,OAAO;AACL;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,gBAAgB,EACjB,CAAA;AAyDD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,aAAsB,EACtB,QAAiB,EACR,EAAE,CAAC,aAAa,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAA;AAEtD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,aAA6B,mBAAmB,EACjC,EAAE;IACjB,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IAClD,OAAO,CAAC,cAAc,EAAE,OAAO,EAAE,EAAE,CACjC,MAAM,CAAC,OAAO,CACZ,OAAO,CAAC,aAAa,KAAK,IAAI;QAC5B,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,KAAK,GAAG,YAAY,CACxD,CAAA;AACL,CAAC,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAkB,CAAC,aAAa,EAAE,OAAO,EAAE,EAAE,CACxE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;IACf,IAAI,CAAC,mBAAmB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAA;IACvE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,GAAG,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IACnF,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QAClC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAQ,KAAuD,EAAE,IAAI,KAAK,OAAO,CAAA;IACnF,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;AACL;;;;;GAKG;AACH,iBAAiB;AACjB;;;;;;GAMG;AACH,sBAAsB;AACtB;;;;;GAKG;AACH,mBAAmB;AACnB;;;;;GAKG;AACH,uBAAuB,EACxB,MAAM,gBAAgB,CAAA;AAevB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,KAAa,EACb,KAAc,EACd,UAAgC,EAAE,EACK,EAAE,CACzC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAA;IAChC,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAA;IAC9D,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAA;IACvD,IAAI,oBAAoB,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;IACzD,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,wEAAwE;IACxE,6EAA6E;IAC7E,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,GAAG,EAAE,CAClB,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACpC,mEAAmE;QACnE,uEAAuE;QACvE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;QAC5C,IAAI,KAAK,GAAG,oBAAoB,GAAG,WAAW;YAAE,OAAM;QACtD,MAAM,aAAa,GAAG,KAAK,GAAG,oBAAoB,CAAA;QAClD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QAC7E,oEAAoE;QACpE,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EACjC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CACzB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAA;YAC5C,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;YACxE,OAAO,GAAG,KAAK,CAAA;YACf,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;YACpD,MAAM,WAAW,GAAG,oBAAoB,GAAG,WAAW,GAAG,aAAa,CAAA;YACtE,IAAI,WAAW,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAA;YAClC,+DAA+D;YAC/D,8DAA8D;YAC9D,+DAA+D;YAC/D,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,aAAa,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;YAC9F,OAAO,IAAI,CAAA;QACb,CAAC,CAAC,CACH,EACD,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAA2B,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CACvG,CACF,CAAA;QACD,IAAI,OAAO;YAAE,OAAM;QACnB,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,2CAA2C,CAAC,CAAC,IAAI,CACxE,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,EAC7C,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,EAC/D,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CACjC,CAAA;IACH,CAAC,CAAC,CAAC,CAAA;IACL,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACnC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;YAC5C,MAAM,WAAW,GAAG,oBAAoB,GAAG,WAAW,GAAG,KAAK,CAAA;YAC9D,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;gBACrB,KAAK,CAAC,CAAC,MAAM,EAAE,CAAA;gBACf,SAAQ;YACV,CAAC;YACD,0EAA0E;YAC1E,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,IAAI,CACjD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EACpE,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACvB,KAAK,GAAG,oBAAoB,IAAI,WAAW,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,IAAI,CACnG,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CACzB,OAAO,CAAC,IAAI,KAAK,SAAS;QACxB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACjB,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAA;YAC5D,OAAO,GAAG,KAAK,CAAA;QACjB,CAAC,CAAC;QACF,CAAC,CAAC,MAAM,CAAC,SAAS,CACrB;IACD,qEAAqE;IACrE,iEAAiE;IACjE,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACrB,IAAI,OAAO;YAAE,OAAO,MAAM,CAAC,IAAI,CAAA;QAC/B,OAAO,GAAG,IAAI,CAAA;QACd,OAAO,MAAM,CAAC,UAAU,CAAC,4BAA4B,CAAC,CAAC,IAAI,CACzD,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CACjD,CAAA;IACH,CAAC,CAAC,CACH,CACF,EACD,MAAM,CAAC,OAAO,CACf,CAAA;IACD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;AAClD,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Run ownership arbitration: liveness evidence, probes, and heartbeat\n * supervision.\n *\n * The identity being arbitrated — {@link OwnerId} — is defined by\n * `@smthrs/journal`, because it is the fencing token the journal accepts on\n * durable appends. It is re-exported here so ownership callers keep reading it\n * as one vocabulary.\n *\n * @since 0.1.0\n */\n\nimport { LivenessEvidence } from \"@smthrs/journal/Consensus\"\nimport { OwnerId } from \"@smthrs/journal/OwnerId\"\nimport { Cause, Clock, Duration, Effect, Semaphore } from \"effect\"\nimport { heartbeatInterval, heartbeatStaleAfter, heartbeatWriteTolerance } from \"./Heartbeat.ts\"\nimport { RunStore } from \"./RunStore.ts\"\n\nexport {\n /**\n * A process identity scoped to a host and a unique ownership nonce, defined\n * by `@smthrs/journal` as the fence on durable appends.\n *\n * @since 0.1.0\n * @category models\n */\n OwnerId\n}\n\nexport {\n /**\n * Evidence that the owner in an exact run snapshot is no longer live.\n *\n * Defined by `@smthrs/journal`'s `Consensus`, because R5 — steal requires\n * staleness plus liveness evidence — is a consensus rule every strategy\n * validates. Two of the three kinds are collected outside the store:\n * `same-host-pid-dead` is a local process probe, and\n * `cross-host-unreachable-stale` is a reachability judgement the deployment\n * makes. `lease-expired` is different — it asserts only that the persisted\n * heartbeat is older than the staleness cutoff, which is the one claim the\n * strategy can check for itself, and `steal` checks it: the takeover\n * refuses any lease whose heartbeat is still inside the window. It is\n * therefore accepted from a claimant on any host, while the other two stay\n * bound to the host relation that makes them meaningful. The observation\n * must carry the exact instant supplied to the consuming operation:\n * `checkedAtMs` must equal that operation's `nowMs`. A caller that probes\n * at T and calls at T+1 is refused and must build fresh evidence.\n *\n * @since 0.1.0\n * @category models\n */\n LivenessEvidence\n}\n\n/**\n * Evidence-factory signature a composition uses for ownership arbitration.\n *\n * A `LivenessProbe` returns `LivenessEvidence` or `undefined` for `steal`,\n * `claimAndOwn`, and `recoverClaim` to verify. {@link LivenessCheck} is the\n * separate alive-or-not question the engine asks before it decides to steal.\n * `RunStore` validates supplied evidence and never probes a process or network\n * itself.\n *\n * @since 0.1.0\n * @category models\n */\nexport type LivenessProbe<E = never, R = never> = (\n expectedOwner: OwnerId,\n claimant: OwnerId,\n checkedAtMs: number\n) => Effect.Effect<LivenessEvidence | undefined, E, R>\n\n/**\n * What a liveness check knows about the run it is asked about, beyond the\n * owner recorded on it.\n *\n * The lease is here because it is the only liveness signal every deployment\n * has: `heartbeatAtMs` is the last heartbeat the owner persisted, `nowMs` is\n * the reading the arbitration is made against, and `claimant` is the identity\n * that would take the run over. A check that wants to probe a pid compares\n * hosts with {@link sameHostIncarnation} first; a check that has nothing but\n * the lease uses {@link leaseLiveness}.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface LivenessContext {\n readonly claimant: OwnerId\n readonly heartbeatAtMs: number | null\n readonly nowMs: number\n}\n\n/**\n * The question ownership arbitration asks before it steals a run: is the\n * recorded owner still working?\n *\n * Answering `true` refuses the takeover. The engine consults this only for a\n * run whose lease has already expired, so a check that has no better evidence\n * than the lease answers `false` there — which is exactly what\n * {@link leaseLiveness} does.\n *\n * @since 0.1.0\n * @category models\n */\nexport type LivenessCheck = (\n expectedOwner: OwnerId,\n context: LivenessContext\n) => Effect.Effect<boolean>\n\n/**\n * Whether two owner identities are incarnations on the same host.\n *\n * The predicate a probe applies before it inspects a pid: `owner.pid` names a\n * process in the claimant's own process namespace only when the hosts match,\n * so a cross-host probe that read it would be answering about an unrelated\n * process. Exposed because the check that needs it — a\n * `process.kill(pid, 0)` probe — belongs to a platform package rather than\n * here.\n *\n * @since 0.1.0\n * @category ownership\n */\nexport const sameHostIncarnation = (\n expectedOwner: OwnerId,\n claimant: OwnerId\n): boolean => expectedOwner.hostId === claimant.hostId\n\n/**\n * The default liveness check: the lease, and nothing else.\n *\n * An owner is treated as alive for as long as its persisted heartbeat is\n * younger than `staleAfter`, and as gone once it is not. That is the weakest\n * honest answer, and it is the one every host can give: a fresh process with\n * no application code at all can reclaim a hard-killed owner's runs once the\n * lease it stopped renewing has expired. A deployment that can say more — a\n * pid probe on the owner's host, an orchestrator that reports pod liveness —\n * supplies its own {@link LivenessCheck} and refuses the takeover for longer.\n *\n * An owner with no recorded heartbeat holds no lease and is reported gone; the\n * steal it enables is still gated by the store's own snapshot compare-and-swap.\n *\n * @since 0.1.0\n * @category ownership\n */\nexport const leaseLiveness = (\n staleAfter: Duration.Input = heartbeatStaleAfter\n): LivenessCheck => {\n const staleAfterMs = Duration.toMillis(staleAfter)\n return (_expectedOwner, context) =>\n Effect.succeed(\n context.heartbeatAtMs !== null &&\n context.heartbeatAtMs >= context.nowMs - staleAfterMs\n )\n}\n\n/**\n * The Node hosts' liveness check: does the owner's process still exist?\n *\n * {@link leaseLiveness} is the honest floor — every host can read a persisted\n * heartbeat — but it is only a timeout, so two engine processes over one\n * database steal each other's running rows `heartbeatStaleAfter` after any\n * heartbeat stall: a stop-the-world pause, a swapped-out process, a disk that\n * blocked longer than the window. This answers the question the lease is\n * standing in for, by asking the operating system whether the recorded pid is\n * still there.\n *\n * `process.kill(pid, 0)` sends no signal; it performs only the delivery\n * checks. Three answers matter:\n *\n * - It returns: the process exists and is signalable. The owner is alive.\n * - It throws `ESRCH`: no such process exists. This is the sole death answer,\n * so the owner is gone.\n * - It throws anything else, including `EPERM`, `EINVAL`, or an unrecognized\n * value: the owner is treated as alive because an unknown answer is not\n * death.\n *\n * This `ESRCH`-only death rule deliberately aligns with\n * `@smthrs/platform-node`'s `HostLiveness`.\n *\n * A pid is only meaningful inside one process namespace, so a recorded owner\n * on another host is never probed: the answer is `false` and the arbitration\n * falls back to the evidence that does cross hosts — the expired lease, which\n * `RunStore.steal` verifies for itself. The engine consults this check only\n * for a run whose lease has ALREADY expired, so answering `false` here does\n * not weaken anything; it declines to add evidence.\n *\n * Node hosts only. It is not part of the browser promise: this entry point\n * bundles for the browser because it never imports a `node:` built-in, and a\n * browser composition has no process table to ask, so it keeps\n * {@link leaseLiveness}.\n *\n * `@smthrs/platform-node`'s `HostLiveness.isAlive` asks the same question of\n * the same process table and differs in one deliberate place: it answers\n * `true` for an owner on another host, which refuses the steal outright, while\n * this check answers `false` and lets the expired lease decide. The difference\n * matters after a host dies for good — under the refusing answer its runs are\n * never reclaimed by anyone, because no other machine can ever produce\n * evidence about its pids.\n *\n * Two limits are inherent to asking a pid, and they bound what reclaim can\n * promise. Both are shared with `HostLiveness.isAlive`.\n *\n * - An owner recorded with the CLAIMANT'S OWN pid is always alive. A previous\n * incarnation of this process, or a second engine composed inside it, differs\n * from the claimant only by `nonce`, and the process it names is this one.\n * Such a row is never stolen while the process lives, so an embedded host\n * that re-creates its engine in place should keep {@link leaseLiveness},\n * whose timeout does expire. Reading same-pid-different-nonce as death is\n * not the alternative: it would let two engines in one process — the exact\n * shape this check exists to arbitrate — steal each other's live runs.\n * - A pid the operating system has REUSED reports the unrelated process that\n * now holds it. The dead owner's row stays refused for as long as that\n * process lives, which delays reclaim rather than breaking it: the row is\n * still `running` under an expired lease, and the next probe after the pid\n * is free reclaims it.\n *\n * @since 0.1.0\n * @category ownership\n */\nexport const sameHostPidProbe: LivenessCheck = (expectedOwner, context) =>\n Effect.sync(() => {\n if (!sameHostIncarnation(expectedOwner, context.claimant)) return false\n if (!Number.isSafeInteger(expectedOwner.pid) || expectedOwner.pid <= 0) return true\n try {\n process.kill(expectedOwner.pid, 0)\n return true\n } catch (error) {\n return (error as { readonly code?: string | undefined } | null)?.code !== \"ESRCH\"\n }\n })\n\nexport {\n /**\n * How often the supervision loop pulses to renew the owner's lease.\n *\n * @since 0.1.0\n * @category constants\n */\n heartbeatInterval,\n /**\n * How far the owner's wall clock may run behind a peer's before the lease\n * reasoning stops holding.\n *\n * @since 0.1.0\n * @category constants\n */\n heartbeatSkewAllowance,\n /**\n * How old a persisted heartbeat must be before a peer may steal the run.\n *\n * @since 0.1.0\n * @category constants\n */\n heartbeatStaleAfter,\n /**\n * How long the owner may keep working through failing or stalled heartbeat writes.\n *\n * @since 0.1.0\n * @category constants\n */\n heartbeatWriteTolerance\n} from \"./Heartbeat.ts\"\n\n/**\n * Hooks a {@link heartbeatLoop} caller may supply.\n *\n * @since 1.0.0\n * @category models\n */\nexport interface HeartbeatLoopOptions {\n /** Called after the same unclaimed owner reconfirms an expired heartbeat budget. */\n readonly onReconfirm?: ((unconfirmedMs: number) => Effect.Effect<void>) | undefined\n /** Called with the unconfirmed milliseconds when the lease lapses, before the loop interrupts itself. */\n readonly onLapse?: ((unconfirmedMs: number) => Effect.Effect<void>) | undefined\n}\n\n/**\n * Runs heartbeats until the persisted ownership fence is lost, then interrupts\n * itself. Race this effect with owned work so structured concurrency\n * interrupts the work when ownership disappears.\n *\n * Each pulse drives the injected `Consensus` strategy's `heartbeat` through\n * `RunStore.heartbeat`, which renews the strategy's lease and mirrors the\n * recorded stamp onto the run row in the same transaction. Heartbeats are\n * lease evidence, never journal events.\n *\n * Pulses are delayed by `heartbeatInterval` and read the Effect `Clock`, so the\n * loop is fully driveable with `TestClock`.\n *\n * A lost fence — any outcome other than `Updated` — is durable evidence and\n * interrupts immediately. A failed heartbeat *write* is not: the persisted\n * heartbeat is still there and no other process may steal the run until it is\n * `heartbeatStaleAfter` old, so transient write errors are tolerated for\n * `heartbeatWriteTolerance` — deliberately shorter than the steal cutoff by a\n * pulse plus `heartbeatSkewAllowance`. The reserved pulse bounds a final\n * reconfirmation; a pending claim or changed owner fails that check. Past that allowance\n * the fence still protects durable writes but non-durable side effects may\n * overlap; see {@link heartbeatWriteTolerance}.\n *\n * An independent deadline races the pulse loop. At expiry it reconfirms the\n * same owner with no pending claim, bounded by one heartbeat interval. A\n * renewed lease continues supervision; loss, error, or timeout interrupts\n * even when an ordinary heartbeat write never returns. Each successful\n * pulse re-arms the deadline from the timestamp sent to the store, not from\n * its completion time. The deadline re-reads the clock after waking, so delayed\n * writes cannot hide expiry behind a stale clock reading.\n *\n * `options.onReconfirm` records a successful reconfirmation after a gap. Its\n * deadline is the remaining renewed lease budget, measured from the timestamp\n * sent to the store. A blocked or failed receipt lapses the lease instead of\n * stranding supervision; receipt completion never grants another lease budget.\n * `options.onLapse` runs once, before the self-interrupt, when reconfirmation\n * fails: it receives how long the lease went unconfirmed, so the caller can\n * record why its work stopped. A lost fence does not call it.\n *\n * @since 0.1.0\n * @category supervision\n */\nexport const heartbeatLoop = (\n runId: string,\n owner: OwnerId,\n options: HeartbeatLoopOptions = {}\n): Effect.Effect<never, never, RunStore> =>\n Effect.gen(function*() {\n const runStore = yield* RunStore\n const toleranceMs = Duration.toMillis(heartbeatWriteTolerance)\n const intervalMs = Duration.toMillis(heartbeatInterval)\n let lastConfirmedPulseMs = yield* Clock.currentTimeMillis\n let failing = false\n // Both overdue timers may wake after a host stall. Serialize expiry and\n // recheck the budget so they cannot reconfirm or journal the same gap twice.\n const expiry = yield* Semaphore.make(1)\n const expire = () =>\n expiry.withPermit(Effect.gen(function*() {\n // A competing expiry may have held the permit across a write. Take\n // fresh evidence after acquisition, never the waiting timer's reading.\n const nowMs = yield* Clock.currentTimeMillis\n if (nowMs - lastConfirmedPulseMs < toleranceMs) return\n const unconfirmedMs = nowMs - lastConfirmedPulseMs\n const renewed = yield* runStore.reconfirm(runId, owner, Math.floor(nowMs)).pipe(\n // Only the ownership check consumes the old lease's reserved pulse.\n Effect.timeout(heartbeatInterval),\n Effect.flatMap((outcome) =>\n Effect.gen(function*() {\n if (outcome._tag !== \"Updated\") return false\n lastConfirmedPulseMs = Math.max(lastConfirmedPulseMs, Math.floor(nowMs))\n failing = false\n const confirmedAtMs = yield* Clock.currentTimeMillis\n const remainingMs = lastConfirmedPulseMs + toleranceMs - confirmedAtMs\n if (remainingMs <= 0) return false\n // The receipt is still required and bounded, but ownership was\n // already renewed. Spending only its old reserved pulse would\n // interrupt live work despite a fresh, fenced persisted lease.\n yield* (options.onReconfirm?.(unconfirmedMs) ?? Effect.void).pipe(Effect.timeout(remainingMs))\n return true\n })\n ),\n Effect.catchCause((cause) =>\n Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause as Cause.Cause<never>) : Effect.succeed(false)\n )\n )\n if (renewed) return\n yield* Effect.logWarning(\"run lease lapsed; interrupting owned work\").pipe(\n Effect.annotateLogs({ runId, unconfirmedMs }),\n Effect.andThen(options.onLapse?.(unconfirmedMs) ?? Effect.void),\n Effect.andThen(Effect.interrupt)\n )\n }))\n const deadline = Effect.gen(function*() {\n while (true) {\n const nowMs = yield* Clock.currentTimeMillis\n const remainingMs = lastConfirmedPulseMs + toleranceMs - nowMs\n if (remainingMs <= 0) {\n yield* expire()\n continue\n }\n // Check alongside pulse intervals, with a shorter final wait when needed.\n yield* Effect.sleep(Math.min(remainingMs, intervalMs))\n }\n })\n const pulses = Effect.sleep(heartbeatInterval).pipe(\n Effect.andThen(Clock.currentTimeMillis.pipe(Effect.map(Math.floor))),\n Effect.flatMap((nowMs) =>\n nowMs - lastConfirmedPulseMs >= toleranceMs ? expire() : runStore.heartbeat(runId, owner, nowMs).pipe(\n Effect.flatMap((outcome) =>\n outcome._tag === \"Updated\"\n ? Effect.sync(() => {\n lastConfirmedPulseMs = Math.max(lastConfirmedPulseMs, nowMs)\n failing = false\n })\n : Effect.interrupt\n ),\n // A failed write keeps the confirmed lease; the independent deadline\n // supervises its expiry. Warn on the first failure of an outage.\n Effect.catch((error) => {\n if (failing) return Effect.void\n failing = true\n return Effect.logWarning(\"run heartbeat write failed\").pipe(\n Effect.annotateLogs({ runId, code: error.code })\n )\n })\n )\n ),\n Effect.forever\n )\n return yield* Effect.raceFirst(pulses, deadline)\n })\n"]}
|