@smthrs/run-store 0.0.0-stage → 1.0.0-rc.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/CHANGELOG.md +112 -0
  2. package/LICENSE +21 -0
  3. package/README.md +195 -2
  4. package/dist/cjs/AttemptStore.d.ts +359 -0
  5. package/dist/cjs/AttemptStore.d.ts.map +1 -0
  6. package/dist/cjs/AttemptStore.js +731 -0
  7. package/dist/cjs/AttemptStore.js.map +7 -0
  8. package/dist/cjs/Heartbeat.d.ts +46 -0
  9. package/dist/cjs/Heartbeat.d.ts.map +1 -0
  10. package/dist/cjs/Heartbeat.js +28 -0
  11. package/dist/cjs/Heartbeat.js.map +7 -0
  12. package/dist/cjs/Migrations.d.ts +42 -0
  13. package/dist/cjs/Migrations.d.ts.map +1 -0
  14. package/dist/cjs/Migrations.js +55 -0
  15. package/dist/cjs/Migrations.js.map +7 -0
  16. package/dist/cjs/Ownership.d.ts +275 -0
  17. package/dist/cjs/Ownership.d.ts.map +1 -0
  18. package/dist/cjs/Ownership.js +129 -0
  19. package/dist/cjs/Ownership.js.map +7 -0
  20. package/dist/cjs/RunStore.d.ts +571 -0
  21. package/dist/cjs/RunStore.d.ts.map +1 -0
  22. package/dist/cjs/RunStore.js +1079 -0
  23. package/dist/cjs/RunStore.js.map +7 -0
  24. package/dist/cjs/RunStoreMetrics.d.ts +151 -0
  25. package/dist/cjs/RunStoreMetrics.d.ts.map +1 -0
  26. package/dist/cjs/RunStoreMetrics.js +91 -0
  27. package/dist/cjs/RunStoreMetrics.js.map +7 -0
  28. package/dist/cjs/index.d.ts +47 -0
  29. package/dist/cjs/index.d.ts.map +1 -0
  30. package/dist/cjs/index.js +43 -0
  31. package/dist/cjs/index.js.map +7 -0
  32. package/dist/cjs/internal/Boundary.d.ts +80 -0
  33. package/dist/cjs/internal/Boundary.d.ts.map +1 -0
  34. package/dist/cjs/internal/Boundary.js +92 -0
  35. package/dist/cjs/internal/Boundary.js.map +7 -0
  36. package/dist/cjs/internal/Companion.d.ts +44 -0
  37. package/dist/cjs/internal/Companion.d.ts.map +1 -0
  38. package/dist/cjs/internal/Companion.js +50 -0
  39. package/dist/cjs/internal/Companion.js.map +7 -0
  40. package/dist/cjs/internal/SpanOutcome.d.ts +37 -0
  41. package/dist/cjs/internal/SpanOutcome.d.ts.map +1 -0
  42. package/dist/cjs/internal/SpanOutcome.js +62 -0
  43. package/dist/cjs/internal/SpanOutcome.js.map +7 -0
  44. package/dist/cjs/migrations/0001_initial.d.ts +15 -0
  45. package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
  46. package/dist/cjs/migrations/0001_initial.js +110 -0
  47. package/dist/cjs/migrations/0001_initial.js.map +7 -0
  48. package/dist/cjs/migrations/0002_lineage.d.ts +27 -0
  49. package/dist/cjs/migrations/0002_lineage.d.ts.map +1 -0
  50. package/dist/cjs/migrations/0002_lineage.js +43 -0
  51. package/dist/cjs/migrations/0002_lineage.js.map +7 -0
  52. package/dist/cjs/migrations/0003_execution_revisions.d.ts +15 -0
  53. package/dist/cjs/migrations/0003_execution_revisions.d.ts.map +1 -0
  54. package/dist/cjs/migrations/0003_execution_revisions.js +77 -0
  55. package/dist/cjs/migrations/0003_execution_revisions.js.map +7 -0
  56. package/dist/cjs/migrations/0004_waiting_request.d.ts +32 -0
  57. package/dist/cjs/migrations/0004_waiting_request.d.ts.map +1 -0
  58. package/dist/cjs/migrations/0004_waiting_request.js +42 -0
  59. package/dist/cjs/migrations/0004_waiting_request.js.map +7 -0
  60. package/dist/cjs/package.json +1 -0
  61. package/dist/cjs/test/TestRunStore.d.ts +17 -0
  62. package/dist/cjs/test/TestRunStore.d.ts.map +1 -0
  63. package/dist/cjs/test/TestRunStore.js +42 -0
  64. package/dist/cjs/test/TestRunStore.js.map +7 -0
  65. package/dist/esm/AttemptStore.d.ts +359 -0
  66. package/dist/esm/AttemptStore.d.ts.map +1 -0
  67. package/dist/esm/AttemptStore.js +835 -0
  68. package/dist/esm/AttemptStore.js.map +1 -0
  69. package/dist/esm/Heartbeat.d.ts +46 -0
  70. package/dist/esm/Heartbeat.d.ts.map +1 -0
  71. package/dist/esm/Heartbeat.js +46 -0
  72. package/dist/esm/Heartbeat.js.map +1 -0
  73. package/dist/esm/Migrations.d.ts +42 -0
  74. package/dist/esm/Migrations.d.ts.map +1 -0
  75. package/dist/esm/Migrations.js +56 -0
  76. package/dist/esm/Migrations.js.map +1 -0
  77. package/dist/esm/Ownership.d.ts +275 -0
  78. package/dist/esm/Ownership.d.ts.map +1 -0
  79. package/dist/esm/Ownership.js +299 -0
  80. package/dist/esm/Ownership.js.map +1 -0
  81. package/dist/esm/RunStore.d.ts +571 -0
  82. package/dist/esm/RunStore.d.ts.map +1 -0
  83. package/dist/esm/RunStore.js +1287 -0
  84. package/dist/esm/RunStore.js.map +1 -0
  85. package/dist/esm/RunStoreMetrics.d.ts +151 -0
  86. package/dist/esm/RunStoreMetrics.d.ts.map +1 -0
  87. package/dist/esm/RunStoreMetrics.js +120 -0
  88. package/dist/esm/RunStoreMetrics.js.map +1 -0
  89. package/dist/esm/index.d.ts +47 -0
  90. package/dist/esm/index.d.ts.map +1 -0
  91. package/dist/esm/index.js +47 -0
  92. package/dist/esm/index.js.map +1 -0
  93. package/dist/esm/internal/Boundary.d.ts +80 -0
  94. package/dist/esm/internal/Boundary.d.ts.map +1 -0
  95. package/dist/esm/internal/Boundary.js +109 -0
  96. package/dist/esm/internal/Boundary.js.map +1 -0
  97. package/dist/esm/internal/Companion.d.ts +44 -0
  98. package/dist/esm/internal/Companion.d.ts.map +1 -0
  99. package/dist/esm/internal/Companion.js +53 -0
  100. package/dist/esm/internal/Companion.js.map +1 -0
  101. package/dist/esm/internal/SpanOutcome.d.ts +37 -0
  102. package/dist/esm/internal/SpanOutcome.d.ts.map +1 -0
  103. package/dist/esm/internal/SpanOutcome.js +48 -0
  104. package/dist/esm/internal/SpanOutcome.js.map +1 -0
  105. package/dist/esm/migrations/0001_initial.d.ts +15 -0
  106. package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
  107. package/dist/esm/migrations/0001_initial.js +88 -0
  108. package/dist/esm/migrations/0001_initial.js.map +1 -0
  109. package/dist/esm/migrations/0002_lineage.d.ts +27 -0
  110. package/dist/esm/migrations/0002_lineage.d.ts.map +1 -0
  111. package/dist/esm/migrations/0002_lineage.js +33 -0
  112. package/dist/esm/migrations/0002_lineage.js.map +1 -0
  113. package/dist/esm/migrations/0003_execution_revisions.d.ts +15 -0
  114. package/dist/esm/migrations/0003_execution_revisions.d.ts.map +1 -0
  115. package/dist/esm/migrations/0003_execution_revisions.js +58 -0
  116. package/dist/esm/migrations/0003_execution_revisions.js.map +1 -0
  117. package/dist/esm/migrations/0004_waiting_request.d.ts +32 -0
  118. package/dist/esm/migrations/0004_waiting_request.d.ts.map +1 -0
  119. package/dist/esm/migrations/0004_waiting_request.js +37 -0
  120. package/dist/esm/migrations/0004_waiting_request.js.map +1 -0
  121. package/dist/esm/test/TestRunStore.d.ts +17 -0
  122. package/dist/esm/test/TestRunStore.d.ts.map +1 -0
  123. package/dist/esm/test/TestRunStore.js +19 -0
  124. package/dist/esm/test/TestRunStore.js.map +1 -0
  125. package/docs/README.md +142 -0
  126. package/docs/api.md +901 -0
  127. package/docs/concepts/durable-values.md +140 -0
  128. package/docs/concepts/execution-revisions.md +53 -0
  129. package/docs/concepts/fencing.md +146 -0
  130. package/docs/concepts/leases.md +154 -0
  131. package/docs/concepts/liveness-evidence.md +142 -0
  132. package/docs/guides/cancel-a-run.md +156 -0
  133. package/docs/guides/claim-and-finish-a-run.md +175 -0
  134. package/docs/guides/compose-the-stores.md +142 -0
  135. package/docs/guides/observe-outcomes.md +182 -0
  136. package/docs/guides/record-step-attempts.md +170 -0
  137. package/docs/guides/recover-a-stalled-run.md +173 -0
  138. package/docs/guides/testing.md +141 -0
  139. package/docs/installation.md +109 -0
  140. package/docs/quickstart.md +201 -0
  141. package/docs/troubleshooting.md +255 -0
  142. package/package.json +183 -3
  143. package/src/AttemptStore.ts +1195 -0
  144. package/src/Heartbeat.ts +47 -0
  145. package/src/Migrations.ts +59 -0
  146. package/src/Ownership.ts +407 -0
  147. package/src/RunStore.ts +2063 -0
  148. package/src/RunStoreMetrics.ts +165 -0
  149. package/src/index.ts +51 -0
  150. package/src/internal/Boundary.ts +130 -0
  151. package/src/internal/Companion.ts +64 -0
  152. package/src/internal/SpanOutcome.ts +71 -0
  153. package/src/migrations/0001_initial.ts +114 -0
  154. package/src/migrations/0002_lineage.ts +38 -0
  155. package/src/migrations/0003_execution_revisions.ts +69 -0
  156. package/src/migrations/0004_waiting_request.ts +39 -0
  157. package/src/test/TestRunStore.ts +22 -0
@@ -0,0 +1,275 @@
1
+ /**
2
+ * Run ownership arbitration: liveness evidence, probes, and heartbeat
3
+ * supervision.
4
+ *
5
+ * The identity being arbitrated — {@link OwnerId} — is defined by
6
+ * `@smthrs/journal`, because it is the fencing token the journal accepts on
7
+ * durable appends. It is re-exported here so ownership callers keep reading it
8
+ * as one vocabulary.
9
+ *
10
+ * @since 0.1.0
11
+ */
12
+ import { LivenessEvidence } from "@smthrs/journal/Consensus";
13
+ import { OwnerId } from "@smthrs/journal/OwnerId";
14
+ import { Duration, Effect } from "effect";
15
+ import { RunStore } from "./RunStore.ts";
16
+ export {
17
+ /**
18
+ * A process identity scoped to a host and a unique ownership nonce, defined
19
+ * by `@smthrs/journal` as the fence on durable appends.
20
+ *
21
+ * @since 0.1.0
22
+ * @category models
23
+ */
24
+ OwnerId };
25
+ export {
26
+ /**
27
+ * Evidence that the owner in an exact run snapshot is no longer live.
28
+ *
29
+ * Defined by `@smthrs/journal`'s `Consensus`, because R5 — steal requires
30
+ * staleness plus liveness evidence — is a consensus rule every strategy
31
+ * validates. Two of the three kinds are collected outside the store:
32
+ * `same-host-pid-dead` is a local process probe, and
33
+ * `cross-host-unreachable-stale` is a reachability judgement the deployment
34
+ * makes. `lease-expired` is different — it asserts only that the persisted
35
+ * heartbeat is older than the staleness cutoff, which is the one claim the
36
+ * strategy can check for itself, and `steal` checks it: the takeover
37
+ * refuses any lease whose heartbeat is still inside the window. It is
38
+ * therefore accepted from a claimant on any host, while the other two stay
39
+ * bound to the host relation that makes them meaningful. The observation
40
+ * must carry the exact instant supplied to the consuming operation:
41
+ * `checkedAtMs` must equal that operation's `nowMs`. A caller that probes
42
+ * at T and calls at T+1 is refused and must build fresh evidence.
43
+ *
44
+ * @since 0.1.0
45
+ * @category models
46
+ */
47
+ LivenessEvidence };
48
+ /**
49
+ * Evidence-factory signature a composition uses for ownership arbitration.
50
+ *
51
+ * A `LivenessProbe` returns `LivenessEvidence` or `undefined` for `steal`,
52
+ * `claimAndOwn`, and `recoverClaim` to verify. {@link LivenessCheck} is the
53
+ * separate alive-or-not question the engine asks before it decides to steal.
54
+ * `RunStore` validates supplied evidence and never probes a process or network
55
+ * itself.
56
+ *
57
+ * @since 0.1.0
58
+ * @category models
59
+ */
60
+ export type LivenessProbe<E = never, R = never> = (expectedOwner: OwnerId, claimant: OwnerId, checkedAtMs: number) => Effect.Effect<LivenessEvidence | undefined, E, R>;
61
+ /**
62
+ * What a liveness check knows about the run it is asked about, beyond the
63
+ * owner recorded on it.
64
+ *
65
+ * The lease is here because it is the only liveness signal every deployment
66
+ * has: `heartbeatAtMs` is the last heartbeat the owner persisted, `nowMs` is
67
+ * the reading the arbitration is made against, and `claimant` is the identity
68
+ * that would take the run over. A check that wants to probe a pid compares
69
+ * hosts with {@link sameHostIncarnation} first; a check that has nothing but
70
+ * the lease uses {@link leaseLiveness}.
71
+ *
72
+ * @since 0.1.0
73
+ * @category models
74
+ */
75
+ export interface LivenessContext {
76
+ readonly claimant: OwnerId;
77
+ readonly heartbeatAtMs: number | null;
78
+ readonly nowMs: number;
79
+ }
80
+ /**
81
+ * The question ownership arbitration asks before it steals a run: is the
82
+ * recorded owner still working?
83
+ *
84
+ * Answering `true` refuses the takeover. The engine consults this only for a
85
+ * run whose lease has already expired, so a check that has no better evidence
86
+ * than the lease answers `false` there — which is exactly what
87
+ * {@link leaseLiveness} does.
88
+ *
89
+ * @since 0.1.0
90
+ * @category models
91
+ */
92
+ export type LivenessCheck = (expectedOwner: OwnerId, context: LivenessContext) => Effect.Effect<boolean>;
93
+ /**
94
+ * Whether two owner identities are incarnations on the same host.
95
+ *
96
+ * The predicate a probe applies before it inspects a pid: `owner.pid` names a
97
+ * process in the claimant's own process namespace only when the hosts match,
98
+ * so a cross-host probe that read it would be answering about an unrelated
99
+ * process. Exposed because the check that needs it — a
100
+ * `process.kill(pid, 0)` probe — belongs to a platform package rather than
101
+ * here.
102
+ *
103
+ * @since 0.1.0
104
+ * @category ownership
105
+ */
106
+ export declare const sameHostIncarnation: (expectedOwner: OwnerId, claimant: OwnerId) => boolean;
107
+ /**
108
+ * The default liveness check: the lease, and nothing else.
109
+ *
110
+ * An owner is treated as alive for as long as its persisted heartbeat is
111
+ * younger than `staleAfter`, and as gone once it is not. That is the weakest
112
+ * honest answer, and it is the one every host can give: a fresh process with
113
+ * no application code at all can reclaim a hard-killed owner's runs once the
114
+ * lease it stopped renewing has expired. A deployment that can say more — a
115
+ * pid probe on the owner's host, an orchestrator that reports pod liveness —
116
+ * supplies its own {@link LivenessCheck} and refuses the takeover for longer.
117
+ *
118
+ * An owner with no recorded heartbeat holds no lease and is reported gone; the
119
+ * steal it enables is still gated by the store's own snapshot compare-and-swap.
120
+ *
121
+ * @since 0.1.0
122
+ * @category ownership
123
+ */
124
+ export declare const leaseLiveness: (staleAfter?: Duration.Input) => LivenessCheck;
125
+ /**
126
+ * The Node hosts' liveness check: does the owner's process still exist?
127
+ *
128
+ * {@link leaseLiveness} is the honest floor — every host can read a persisted
129
+ * heartbeat — but it is only a timeout, so two engine processes over one
130
+ * database steal each other's running rows `heartbeatStaleAfter` after any
131
+ * heartbeat stall: a stop-the-world pause, a swapped-out process, a disk that
132
+ * blocked longer than the window. This answers the question the lease is
133
+ * standing in for, by asking the operating system whether the recorded pid is
134
+ * still there.
135
+ *
136
+ * `process.kill(pid, 0)` sends no signal; it performs only the delivery
137
+ * checks. Three answers matter:
138
+ *
139
+ * - It returns: the process exists and is signalable. The owner is alive.
140
+ * - It throws `ESRCH`: no such process exists. This is the sole death answer,
141
+ * so the owner is gone.
142
+ * - It throws anything else, including `EPERM`, `EINVAL`, or an unrecognized
143
+ * value: the owner is treated as alive because an unknown answer is not
144
+ * death.
145
+ *
146
+ * This `ESRCH`-only death rule deliberately aligns with
147
+ * `@smthrs/platform-node`'s `HostLiveness`.
148
+ *
149
+ * A pid is only meaningful inside one process namespace, so a recorded owner
150
+ * on another host is never probed: the answer is `false` and the arbitration
151
+ * falls back to the evidence that does cross hosts — the expired lease, which
152
+ * `RunStore.steal` verifies for itself. The engine consults this check only
153
+ * for a run whose lease has ALREADY expired, so answering `false` here does
154
+ * not weaken anything; it declines to add evidence.
155
+ *
156
+ * Node hosts only. It is not part of the browser promise: this entry point
157
+ * bundles for the browser because it never imports a `node:` built-in, and a
158
+ * browser composition has no process table to ask, so it keeps
159
+ * {@link leaseLiveness}.
160
+ *
161
+ * `@smthrs/platform-node`'s `HostLiveness.isAlive` asks the same question of
162
+ * the same process table and differs in one deliberate place: it answers
163
+ * `true` for an owner on another host, which refuses the steal outright, while
164
+ * this check answers `false` and lets the expired lease decide. The difference
165
+ * matters after a host dies for good — under the refusing answer its runs are
166
+ * never reclaimed by anyone, because no other machine can ever produce
167
+ * evidence about its pids.
168
+ *
169
+ * Two limits are inherent to asking a pid, and they bound what reclaim can
170
+ * promise. Both are shared with `HostLiveness.isAlive`.
171
+ *
172
+ * - An owner recorded with the CLAIMANT'S OWN pid is always alive. A previous
173
+ * incarnation of this process, or a second engine composed inside it, differs
174
+ * from the claimant only by `nonce`, and the process it names is this one.
175
+ * Such a row is never stolen while the process lives, so an embedded host
176
+ * that re-creates its engine in place should keep {@link leaseLiveness},
177
+ * whose timeout does expire. Reading same-pid-different-nonce as death is
178
+ * not the alternative: it would let two engines in one process — the exact
179
+ * shape this check exists to arbitrate — steal each other's live runs.
180
+ * - A pid the operating system has REUSED reports the unrelated process that
181
+ * now holds it. The dead owner's row stays refused for as long as that
182
+ * process lives, which delays reclaim rather than breaking it: the row is
183
+ * still `running` under an expired lease, and the next probe after the pid
184
+ * is free reclaims it.
185
+ *
186
+ * @since 0.1.0
187
+ * @category ownership
188
+ */
189
+ export declare const sameHostPidProbe: LivenessCheck;
190
+ export {
191
+ /**
192
+ * How often the supervision loop pulses to renew the owner's lease.
193
+ *
194
+ * @since 0.1.0
195
+ * @category constants
196
+ */
197
+ heartbeatInterval,
198
+ /**
199
+ * How far the owner's wall clock may run behind a peer's before the lease
200
+ * reasoning stops holding.
201
+ *
202
+ * @since 0.1.0
203
+ * @category constants
204
+ */
205
+ heartbeatSkewAllowance,
206
+ /**
207
+ * How old a persisted heartbeat must be before a peer may steal the run.
208
+ *
209
+ * @since 0.1.0
210
+ * @category constants
211
+ */
212
+ heartbeatStaleAfter,
213
+ /**
214
+ * How long the owner may keep working through failing or stalled heartbeat writes.
215
+ *
216
+ * @since 0.1.0
217
+ * @category constants
218
+ */
219
+ heartbeatWriteTolerance } from "./Heartbeat.ts";
220
+ /**
221
+ * Hooks a {@link heartbeatLoop} caller may supply.
222
+ *
223
+ * @since 1.0.0
224
+ * @category models
225
+ */
226
+ export interface HeartbeatLoopOptions {
227
+ /** Called after the same unclaimed owner reconfirms an expired heartbeat budget. */
228
+ readonly onReconfirm?: ((unconfirmedMs: number) => Effect.Effect<void>) | undefined;
229
+ /** Called with the unconfirmed milliseconds when the lease lapses, before the loop interrupts itself. */
230
+ readonly onLapse?: ((unconfirmedMs: number) => Effect.Effect<void>) | undefined;
231
+ }
232
+ /**
233
+ * Runs heartbeats until the persisted ownership fence is lost, then interrupts
234
+ * itself. Race this effect with owned work so structured concurrency
235
+ * interrupts the work when ownership disappears.
236
+ *
237
+ * Each pulse drives the injected `Consensus` strategy's `heartbeat` through
238
+ * `RunStore.heartbeat`, which renews the strategy's lease and mirrors the
239
+ * recorded stamp onto the run row in the same transaction. Heartbeats are
240
+ * lease evidence, never journal events.
241
+ *
242
+ * Pulses are delayed by `heartbeatInterval` and read the Effect `Clock`, so the
243
+ * loop is fully driveable with `TestClock`.
244
+ *
245
+ * A lost fence — any outcome other than `Updated` — is durable evidence and
246
+ * interrupts immediately. A failed heartbeat *write* is not: the persisted
247
+ * heartbeat is still there and no other process may steal the run until it is
248
+ * `heartbeatStaleAfter` old, so transient write errors are tolerated for
249
+ * `heartbeatWriteTolerance` — deliberately shorter than the steal cutoff by a
250
+ * pulse plus `heartbeatSkewAllowance`. The reserved pulse bounds a final
251
+ * reconfirmation; a pending claim or changed owner fails that check. Past that allowance
252
+ * the fence still protects durable writes but non-durable side effects may
253
+ * overlap; see {@link heartbeatWriteTolerance}.
254
+ *
255
+ * An independent deadline races the pulse loop. At expiry it reconfirms the
256
+ * same owner with no pending claim, bounded by one heartbeat interval. A
257
+ * renewed lease continues supervision; loss, error, or timeout interrupts
258
+ * even when an ordinary heartbeat write never returns. Each successful
259
+ * pulse re-arms the deadline from the timestamp sent to the store, not from
260
+ * its completion time. The deadline re-reads the clock after waking, so delayed
261
+ * writes cannot hide expiry behind a stale clock reading.
262
+ *
263
+ * `options.onReconfirm` records a successful reconfirmation after a gap. Its
264
+ * deadline is the remaining renewed lease budget, measured from the timestamp
265
+ * sent to the store. A blocked or failed receipt lapses the lease instead of
266
+ * stranding supervision; receipt completion never grants another lease budget.
267
+ * `options.onLapse` runs once, before the self-interrupt, when reconfirmation
268
+ * fails: it receives how long the lease went unconfirmed, so the caller can
269
+ * record why its work stopped. A lost fence does not call it.
270
+ *
271
+ * @since 0.1.0
272
+ * @category supervision
273
+ */
274
+ export declare const heartbeatLoop: (runId: string, owner: OwnerId, options?: HeartbeatLoopOptions) => Effect.Effect<never, never, RunStore>;
275
+ //# sourceMappingURL=Ownership.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Ownership.d.ts","sourceRoot":"","sources":["../../src/Ownership.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAgB,QAAQ,EAAE,MAAM,EAAa,MAAM,QAAQ,CAAA;AAElE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAExC,OAAO;AACL;;;;;;GAMG;AACH,OAAO,EACR,CAAA;AAED,OAAO;AACL;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,gBAAgB,EACjB,CAAA;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,aAAa,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,KAAK,IAAI,CAChD,aAAa,EAAE,OAAO,EACtB,QAAQ,EAAE,OAAO,EACjB,WAAW,EAAE,MAAM,KAChB,MAAM,CAAC,MAAM,CAAC,gBAAgB,GAAG,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;AAEtD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAA;IAC1B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,aAAa,GAAG,CAC1B,aAAa,EAAE,OAAO,EACtB,OAAO,EAAE,eAAe,KACrB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AAE3B;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,mBAAmB,GAC9B,eAAe,OAAO,EACtB,UAAU,OAAO,KAChB,OAAmD,CAAA;AAEtD;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,aAAa,GACxB,aAAY,QAAQ,CAAC,KAA2B,KAC/C,aAOF,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,eAAO,MAAM,gBAAgB,EAAE,aAU3B,CAAA;AAEJ,OAAO;AACL;;;;;GAKG;AACH,iBAAiB;AACjB;;;;;;GAMG;AACH,sBAAsB;AACtB;;;;;GAKG;AACH,mBAAmB;AACnB;;;;;GAKG;AACH,uBAAuB,EACxB,MAAM,gBAAgB,CAAA;AAEvB;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACnC,oFAAoF;IACpF,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,aAAa,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAA;IACnF,yGAAyG;IACzG,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,aAAa,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAA;CAChF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,eAAO,MAAM,aAAa,GACxB,OAAO,MAAM,EACb,OAAO,OAAO,EACd,UAAS,oBAAyB,KACjC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAoFnC,CAAA"}
@@ -0,0 +1,299 @@
1
+ /**
2
+ * Run ownership arbitration: liveness evidence, probes, and heartbeat
3
+ * supervision.
4
+ *
5
+ * The identity being arbitrated — {@link OwnerId} — is defined by
6
+ * `@smthrs/journal`, because it is the fencing token the journal accepts on
7
+ * durable appends. It is re-exported here so ownership callers keep reading it
8
+ * as one vocabulary.
9
+ *
10
+ * @since 0.1.0
11
+ */
12
+ import { LivenessEvidence } from "@smthrs/journal/Consensus";
13
+ import { OwnerId } from "@smthrs/journal/OwnerId";
14
+ import { Cause, Clock, Duration, Effect, Semaphore } from "effect";
15
+ import { heartbeatInterval, heartbeatStaleAfter, heartbeatWriteTolerance } from "./Heartbeat.js";
16
+ import { RunStore } from "./RunStore.js";
17
+ export {
18
+ /**
19
+ * A process identity scoped to a host and a unique ownership nonce, defined
20
+ * by `@smthrs/journal` as the fence on durable appends.
21
+ *
22
+ * @since 0.1.0
23
+ * @category models
24
+ */
25
+ OwnerId };
26
+ export {
27
+ /**
28
+ * Evidence that the owner in an exact run snapshot is no longer live.
29
+ *
30
+ * Defined by `@smthrs/journal`'s `Consensus`, because R5 — steal requires
31
+ * staleness plus liveness evidence — is a consensus rule every strategy
32
+ * validates. Two of the three kinds are collected outside the store:
33
+ * `same-host-pid-dead` is a local process probe, and
34
+ * `cross-host-unreachable-stale` is a reachability judgement the deployment
35
+ * makes. `lease-expired` is different — it asserts only that the persisted
36
+ * heartbeat is older than the staleness cutoff, which is the one claim the
37
+ * strategy can check for itself, and `steal` checks it: the takeover
38
+ * refuses any lease whose heartbeat is still inside the window. It is
39
+ * therefore accepted from a claimant on any host, while the other two stay
40
+ * bound to the host relation that makes them meaningful. The observation
41
+ * must carry the exact instant supplied to the consuming operation:
42
+ * `checkedAtMs` must equal that operation's `nowMs`. A caller that probes
43
+ * at T and calls at T+1 is refused and must build fresh evidence.
44
+ *
45
+ * @since 0.1.0
46
+ * @category models
47
+ */
48
+ LivenessEvidence };
49
+ /**
50
+ * Whether two owner identities are incarnations on the same host.
51
+ *
52
+ * The predicate a probe applies before it inspects a pid: `owner.pid` names a
53
+ * process in the claimant's own process namespace only when the hosts match,
54
+ * so a cross-host probe that read it would be answering about an unrelated
55
+ * process. Exposed because the check that needs it — a
56
+ * `process.kill(pid, 0)` probe — belongs to a platform package rather than
57
+ * here.
58
+ *
59
+ * @since 0.1.0
60
+ * @category ownership
61
+ */
62
+ export const sameHostIncarnation = (expectedOwner, claimant) => expectedOwner.hostId === claimant.hostId;
63
+ /**
64
+ * The default liveness check: the lease, and nothing else.
65
+ *
66
+ * An owner is treated as alive for as long as its persisted heartbeat is
67
+ * younger than `staleAfter`, and as gone once it is not. That is the weakest
68
+ * honest answer, and it is the one every host can give: a fresh process with
69
+ * no application code at all can reclaim a hard-killed owner's runs once the
70
+ * lease it stopped renewing has expired. A deployment that can say more — a
71
+ * pid probe on the owner's host, an orchestrator that reports pod liveness —
72
+ * supplies its own {@link LivenessCheck} and refuses the takeover for longer.
73
+ *
74
+ * An owner with no recorded heartbeat holds no lease and is reported gone; the
75
+ * steal it enables is still gated by the store's own snapshot compare-and-swap.
76
+ *
77
+ * @since 0.1.0
78
+ * @category ownership
79
+ */
80
+ export const leaseLiveness = (staleAfter = heartbeatStaleAfter) => {
81
+ const staleAfterMs = Duration.toMillis(staleAfter);
82
+ return (_expectedOwner, context) => Effect.succeed(context.heartbeatAtMs !== null &&
83
+ context.heartbeatAtMs >= context.nowMs - staleAfterMs);
84
+ };
85
+ /**
86
+ * The Node hosts' liveness check: does the owner's process still exist?
87
+ *
88
+ * {@link leaseLiveness} is the honest floor — every host can read a persisted
89
+ * heartbeat — but it is only a timeout, so two engine processes over one
90
+ * database steal each other's running rows `heartbeatStaleAfter` after any
91
+ * heartbeat stall: a stop-the-world pause, a swapped-out process, a disk that
92
+ * blocked longer than the window. This answers the question the lease is
93
+ * standing in for, by asking the operating system whether the recorded pid is
94
+ * still there.
95
+ *
96
+ * `process.kill(pid, 0)` sends no signal; it performs only the delivery
97
+ * checks. Three answers matter:
98
+ *
99
+ * - It returns: the process exists and is signalable. The owner is alive.
100
+ * - It throws `ESRCH`: no such process exists. This is the sole death answer,
101
+ * so the owner is gone.
102
+ * - It throws anything else, including `EPERM`, `EINVAL`, or an unrecognized
103
+ * value: the owner is treated as alive because an unknown answer is not
104
+ * death.
105
+ *
106
+ * This `ESRCH`-only death rule deliberately aligns with
107
+ * `@smthrs/platform-node`'s `HostLiveness`.
108
+ *
109
+ * A pid is only meaningful inside one process namespace, so a recorded owner
110
+ * on another host is never probed: the answer is `false` and the arbitration
111
+ * falls back to the evidence that does cross hosts — the expired lease, which
112
+ * `RunStore.steal` verifies for itself. The engine consults this check only
113
+ * for a run whose lease has ALREADY expired, so answering `false` here does
114
+ * not weaken anything; it declines to add evidence.
115
+ *
116
+ * Node hosts only. It is not part of the browser promise: this entry point
117
+ * bundles for the browser because it never imports a `node:` built-in, and a
118
+ * browser composition has no process table to ask, so it keeps
119
+ * {@link leaseLiveness}.
120
+ *
121
+ * `@smthrs/platform-node`'s `HostLiveness.isAlive` asks the same question of
122
+ * the same process table and differs in one deliberate place: it answers
123
+ * `true` for an owner on another host, which refuses the steal outright, while
124
+ * this check answers `false` and lets the expired lease decide. The difference
125
+ * matters after a host dies for good — under the refusing answer its runs are
126
+ * never reclaimed by anyone, because no other machine can ever produce
127
+ * evidence about its pids.
128
+ *
129
+ * Two limits are inherent to asking a pid, and they bound what reclaim can
130
+ * promise. Both are shared with `HostLiveness.isAlive`.
131
+ *
132
+ * - An owner recorded with the CLAIMANT'S OWN pid is always alive. A previous
133
+ * incarnation of this process, or a second engine composed inside it, differs
134
+ * from the claimant only by `nonce`, and the process it names is this one.
135
+ * Such a row is never stolen while the process lives, so an embedded host
136
+ * that re-creates its engine in place should keep {@link leaseLiveness},
137
+ * whose timeout does expire. Reading same-pid-different-nonce as death is
138
+ * not the alternative: it would let two engines in one process — the exact
139
+ * shape this check exists to arbitrate — steal each other's live runs.
140
+ * - A pid the operating system has REUSED reports the unrelated process that
141
+ * now holds it. The dead owner's row stays refused for as long as that
142
+ * process lives, which delays reclaim rather than breaking it: the row is
143
+ * still `running` under an expired lease, and the next probe after the pid
144
+ * is free reclaims it.
145
+ *
146
+ * @since 0.1.0
147
+ * @category ownership
148
+ */
149
+ export const sameHostPidProbe = (expectedOwner, context) => Effect.sync(() => {
150
+ if (!sameHostIncarnation(expectedOwner, context.claimant))
151
+ return false;
152
+ if (!Number.isSafeInteger(expectedOwner.pid) || expectedOwner.pid <= 0)
153
+ return true;
154
+ try {
155
+ process.kill(expectedOwner.pid, 0);
156
+ return true;
157
+ }
158
+ catch (error) {
159
+ return error?.code !== "ESRCH";
160
+ }
161
+ });
162
+ export {
163
+ /**
164
+ * How often the supervision loop pulses to renew the owner's lease.
165
+ *
166
+ * @since 0.1.0
167
+ * @category constants
168
+ */
169
+ heartbeatInterval,
170
+ /**
171
+ * How far the owner's wall clock may run behind a peer's before the lease
172
+ * reasoning stops holding.
173
+ *
174
+ * @since 0.1.0
175
+ * @category constants
176
+ */
177
+ heartbeatSkewAllowance,
178
+ /**
179
+ * How old a persisted heartbeat must be before a peer may steal the run.
180
+ *
181
+ * @since 0.1.0
182
+ * @category constants
183
+ */
184
+ heartbeatStaleAfter,
185
+ /**
186
+ * How long the owner may keep working through failing or stalled heartbeat writes.
187
+ *
188
+ * @since 0.1.0
189
+ * @category constants
190
+ */
191
+ heartbeatWriteTolerance } from "./Heartbeat.js";
192
+ /**
193
+ * Runs heartbeats until the persisted ownership fence is lost, then interrupts
194
+ * itself. Race this effect with owned work so structured concurrency
195
+ * interrupts the work when ownership disappears.
196
+ *
197
+ * Each pulse drives the injected `Consensus` strategy's `heartbeat` through
198
+ * `RunStore.heartbeat`, which renews the strategy's lease and mirrors the
199
+ * recorded stamp onto the run row in the same transaction. Heartbeats are
200
+ * lease evidence, never journal events.
201
+ *
202
+ * Pulses are delayed by `heartbeatInterval` and read the Effect `Clock`, so the
203
+ * loop is fully driveable with `TestClock`.
204
+ *
205
+ * A lost fence — any outcome other than `Updated` — is durable evidence and
206
+ * interrupts immediately. A failed heartbeat *write* is not: the persisted
207
+ * heartbeat is still there and no other process may steal the run until it is
208
+ * `heartbeatStaleAfter` old, so transient write errors are tolerated for
209
+ * `heartbeatWriteTolerance` — deliberately shorter than the steal cutoff by a
210
+ * pulse plus `heartbeatSkewAllowance`. The reserved pulse bounds a final
211
+ * reconfirmation; a pending claim or changed owner fails that check. Past that allowance
212
+ * the fence still protects durable writes but non-durable side effects may
213
+ * overlap; see {@link heartbeatWriteTolerance}.
214
+ *
215
+ * An independent deadline races the pulse loop. At expiry it reconfirms the
216
+ * same owner with no pending claim, bounded by one heartbeat interval. A
217
+ * renewed lease continues supervision; loss, error, or timeout interrupts
218
+ * even when an ordinary heartbeat write never returns. Each successful
219
+ * pulse re-arms the deadline from the timestamp sent to the store, not from
220
+ * its completion time. The deadline re-reads the clock after waking, so delayed
221
+ * writes cannot hide expiry behind a stale clock reading.
222
+ *
223
+ * `options.onReconfirm` records a successful reconfirmation after a gap. Its
224
+ * deadline is the remaining renewed lease budget, measured from the timestamp
225
+ * sent to the store. A blocked or failed receipt lapses the lease instead of
226
+ * stranding supervision; receipt completion never grants another lease budget.
227
+ * `options.onLapse` runs once, before the self-interrupt, when reconfirmation
228
+ * fails: it receives how long the lease went unconfirmed, so the caller can
229
+ * record why its work stopped. A lost fence does not call it.
230
+ *
231
+ * @since 0.1.0
232
+ * @category supervision
233
+ */
234
+ export const heartbeatLoop = (runId, owner, options = {}) => Effect.gen(function* () {
235
+ const runStore = yield* RunStore;
236
+ const toleranceMs = Duration.toMillis(heartbeatWriteTolerance);
237
+ const intervalMs = Duration.toMillis(heartbeatInterval);
238
+ let lastConfirmedPulseMs = yield* Clock.currentTimeMillis;
239
+ let failing = false;
240
+ // Both overdue timers may wake after a host stall. Serialize expiry and
241
+ // recheck the budget so they cannot reconfirm or journal the same gap twice.
242
+ const expiry = yield* Semaphore.make(1);
243
+ const expire = () => expiry.withPermit(Effect.gen(function* () {
244
+ // A competing expiry may have held the permit across a write. Take
245
+ // fresh evidence after acquisition, never the waiting timer's reading.
246
+ const nowMs = yield* Clock.currentTimeMillis;
247
+ if (nowMs - lastConfirmedPulseMs < toleranceMs)
248
+ return;
249
+ const unconfirmedMs = nowMs - lastConfirmedPulseMs;
250
+ const renewed = yield* runStore.reconfirm(runId, owner, Math.floor(nowMs)).pipe(
251
+ // Only the ownership check consumes the old lease's reserved pulse.
252
+ Effect.timeout(heartbeatInterval), Effect.flatMap((outcome) => Effect.gen(function* () {
253
+ if (outcome._tag !== "Updated")
254
+ return false;
255
+ lastConfirmedPulseMs = Math.max(lastConfirmedPulseMs, Math.floor(nowMs));
256
+ failing = false;
257
+ const confirmedAtMs = yield* Clock.currentTimeMillis;
258
+ const remainingMs = lastConfirmedPulseMs + toleranceMs - confirmedAtMs;
259
+ if (remainingMs <= 0)
260
+ return false;
261
+ // The receipt is still required and bounded, but ownership was
262
+ // already renewed. Spending only its old reserved pulse would
263
+ // interrupt live work despite a fresh, fenced persisted lease.
264
+ yield* (options.onReconfirm?.(unconfirmedMs) ?? Effect.void).pipe(Effect.timeout(remainingMs));
265
+ return true;
266
+ })), Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause) : Effect.succeed(false)));
267
+ if (renewed)
268
+ return;
269
+ yield* Effect.logWarning("run lease lapsed; interrupting owned work").pipe(Effect.annotateLogs({ runId, unconfirmedMs }), Effect.andThen(options.onLapse?.(unconfirmedMs) ?? Effect.void), Effect.andThen(Effect.interrupt));
270
+ }));
271
+ const deadline = Effect.gen(function* () {
272
+ while (true) {
273
+ const nowMs = yield* Clock.currentTimeMillis;
274
+ const remainingMs = lastConfirmedPulseMs + toleranceMs - nowMs;
275
+ if (remainingMs <= 0) {
276
+ yield* expire();
277
+ continue;
278
+ }
279
+ // Check alongside pulse intervals, with a shorter final wait when needed.
280
+ yield* Effect.sleep(Math.min(remainingMs, intervalMs));
281
+ }
282
+ });
283
+ const pulses = Effect.sleep(heartbeatInterval).pipe(Effect.andThen(Clock.currentTimeMillis.pipe(Effect.map(Math.floor))), Effect.flatMap((nowMs) => nowMs - lastConfirmedPulseMs >= toleranceMs ? expire() : runStore.heartbeat(runId, owner, nowMs).pipe(Effect.flatMap((outcome) => outcome._tag === "Updated"
284
+ ? Effect.sync(() => {
285
+ lastConfirmedPulseMs = Math.max(lastConfirmedPulseMs, nowMs);
286
+ failing = false;
287
+ })
288
+ : Effect.interrupt),
289
+ // A failed write keeps the confirmed lease; the independent deadline
290
+ // supervises its expiry. Warn on the first failure of an outage.
291
+ Effect.catch((error) => {
292
+ if (failing)
293
+ return Effect.void;
294
+ failing = true;
295
+ return Effect.logWarning("run heartbeat write failed").pipe(Effect.annotateLogs({ runId, code: error.code }));
296
+ }))), Effect.forever);
297
+ return yield* Effect.raceFirst(pulses, deadline);
298
+ });
299
+ //# sourceMappingURL=Ownership.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Ownership.js","sourceRoot":"","sources":["../../src/Ownership.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAClE,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AAChG,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAExC,OAAO;AACL;;;;;;GAMG;AACH,OAAO,EACR,CAAA;AAED,OAAO;AACL;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,gBAAgB,EACjB,CAAA;AAyDD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,aAAsB,EACtB,QAAiB,EACR,EAAE,CAAC,aAAa,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAA;AAEtD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,aAA6B,mBAAmB,EACjC,EAAE;IACjB,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IAClD,OAAO,CAAC,cAAc,EAAE,OAAO,EAAE,EAAE,CACjC,MAAM,CAAC,OAAO,CACZ,OAAO,CAAC,aAAa,KAAK,IAAI;QAC5B,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,KAAK,GAAG,YAAY,CACxD,CAAA;AACL,CAAC,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAkB,CAAC,aAAa,EAAE,OAAO,EAAE,EAAE,CACxE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;IACf,IAAI,CAAC,mBAAmB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAA;IACvE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,GAAG,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IACnF,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QAClC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAQ,KAAuD,EAAE,IAAI,KAAK,OAAO,CAAA;IACnF,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;AACL;;;;;GAKG;AACH,iBAAiB;AACjB;;;;;;GAMG;AACH,sBAAsB;AACtB;;;;;GAKG;AACH,mBAAmB;AACnB;;;;;GAKG;AACH,uBAAuB,EACxB,MAAM,gBAAgB,CAAA;AAevB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,KAAa,EACb,KAAc,EACd,UAAgC,EAAE,EACK,EAAE,CACzC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAA;IAChC,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAA;IAC9D,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAA;IACvD,IAAI,oBAAoB,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;IACzD,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,wEAAwE;IACxE,6EAA6E;IAC7E,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,GAAG,EAAE,CAClB,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACpC,mEAAmE;QACnE,uEAAuE;QACvE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;QAC5C,IAAI,KAAK,GAAG,oBAAoB,GAAG,WAAW;YAAE,OAAM;QACtD,MAAM,aAAa,GAAG,KAAK,GAAG,oBAAoB,CAAA;QAClD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QAC7E,oEAAoE;QACpE,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EACjC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CACzB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAA;YAC5C,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;YACxE,OAAO,GAAG,KAAK,CAAA;YACf,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;YACpD,MAAM,WAAW,GAAG,oBAAoB,GAAG,WAAW,GAAG,aAAa,CAAA;YACtE,IAAI,WAAW,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAA;YAClC,+DAA+D;YAC/D,8DAA8D;YAC9D,+DAA+D;YAC/D,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,aAAa,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;YAC9F,OAAO,IAAI,CAAA;QACb,CAAC,CAAC,CACH,EACD,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAA2B,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CACvG,CACF,CAAA;QACD,IAAI,OAAO;YAAE,OAAM;QACnB,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,2CAA2C,CAAC,CAAC,IAAI,CACxE,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,EAC7C,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,EAC/D,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CACjC,CAAA;IACH,CAAC,CAAC,CAAC,CAAA;IACL,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACnC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;YAC5C,MAAM,WAAW,GAAG,oBAAoB,GAAG,WAAW,GAAG,KAAK,CAAA;YAC9D,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;gBACrB,KAAK,CAAC,CAAC,MAAM,EAAE,CAAA;gBACf,SAAQ;YACV,CAAC;YACD,0EAA0E;YAC1E,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,IAAI,CACjD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EACpE,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACvB,KAAK,GAAG,oBAAoB,IAAI,WAAW,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,IAAI,CACnG,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CACzB,OAAO,CAAC,IAAI,KAAK,SAAS;QACxB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACjB,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAA;YAC5D,OAAO,GAAG,KAAK,CAAA;QACjB,CAAC,CAAC;QACF,CAAC,CAAC,MAAM,CAAC,SAAS,CACrB;IACD,qEAAqE;IACrE,iEAAiE;IACjE,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACrB,IAAI,OAAO;YAAE,OAAO,MAAM,CAAC,IAAI,CAAA;QAC/B,OAAO,GAAG,IAAI,CAAA;QACd,OAAO,MAAM,CAAC,UAAU,CAAC,4BAA4B,CAAC,CAAC,IAAI,CACzD,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CACjD,CAAA;IACH,CAAC,CAAC,CACH,CACF,EACD,MAAM,CAAC,OAAO,CACf,CAAA;IACD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;AAClD,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Run ownership arbitration: liveness evidence, probes, and heartbeat\n * supervision.\n *\n * The identity being arbitrated — {@link OwnerId} — is defined by\n * `@smthrs/journal`, because it is the fencing token the journal accepts on\n * durable appends. It is re-exported here so ownership callers keep reading it\n * as one vocabulary.\n *\n * @since 0.1.0\n */\n\nimport { LivenessEvidence } from \"@smthrs/journal/Consensus\"\nimport { OwnerId } from \"@smthrs/journal/OwnerId\"\nimport { Cause, Clock, Duration, Effect, Semaphore } from \"effect\"\nimport { heartbeatInterval, heartbeatStaleAfter, heartbeatWriteTolerance } from \"./Heartbeat.ts\"\nimport { RunStore } from \"./RunStore.ts\"\n\nexport {\n /**\n * A process identity scoped to a host and a unique ownership nonce, defined\n * by `@smthrs/journal` as the fence on durable appends.\n *\n * @since 0.1.0\n * @category models\n */\n OwnerId\n}\n\nexport {\n /**\n * Evidence that the owner in an exact run snapshot is no longer live.\n *\n * Defined by `@smthrs/journal`'s `Consensus`, because R5 — steal requires\n * staleness plus liveness evidence — is a consensus rule every strategy\n * validates. Two of the three kinds are collected outside the store:\n * `same-host-pid-dead` is a local process probe, and\n * `cross-host-unreachable-stale` is a reachability judgement the deployment\n * makes. `lease-expired` is different — it asserts only that the persisted\n * heartbeat is older than the staleness cutoff, which is the one claim the\n * strategy can check for itself, and `steal` checks it: the takeover\n * refuses any lease whose heartbeat is still inside the window. It is\n * therefore accepted from a claimant on any host, while the other two stay\n * bound to the host relation that makes them meaningful. The observation\n * must carry the exact instant supplied to the consuming operation:\n * `checkedAtMs` must equal that operation's `nowMs`. A caller that probes\n * at T and calls at T+1 is refused and must build fresh evidence.\n *\n * @since 0.1.0\n * @category models\n */\n LivenessEvidence\n}\n\n/**\n * Evidence-factory signature a composition uses for ownership arbitration.\n *\n * A `LivenessProbe` returns `LivenessEvidence` or `undefined` for `steal`,\n * `claimAndOwn`, and `recoverClaim` to verify. {@link LivenessCheck} is the\n * separate alive-or-not question the engine asks before it decides to steal.\n * `RunStore` validates supplied evidence and never probes a process or network\n * itself.\n *\n * @since 0.1.0\n * @category models\n */\nexport type LivenessProbe<E = never, R = never> = (\n expectedOwner: OwnerId,\n claimant: OwnerId,\n checkedAtMs: number\n) => Effect.Effect<LivenessEvidence | undefined, E, R>\n\n/**\n * What a liveness check knows about the run it is asked about, beyond the\n * owner recorded on it.\n *\n * The lease is here because it is the only liveness signal every deployment\n * has: `heartbeatAtMs` is the last heartbeat the owner persisted, `nowMs` is\n * the reading the arbitration is made against, and `claimant` is the identity\n * that would take the run over. A check that wants to probe a pid compares\n * hosts with {@link sameHostIncarnation} first; a check that has nothing but\n * the lease uses {@link leaseLiveness}.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface LivenessContext {\n readonly claimant: OwnerId\n readonly heartbeatAtMs: number | null\n readonly nowMs: number\n}\n\n/**\n * The question ownership arbitration asks before it steals a run: is the\n * recorded owner still working?\n *\n * Answering `true` refuses the takeover. The engine consults this only for a\n * run whose lease has already expired, so a check that has no better evidence\n * than the lease answers `false` there — which is exactly what\n * {@link leaseLiveness} does.\n *\n * @since 0.1.0\n * @category models\n */\nexport type LivenessCheck = (\n expectedOwner: OwnerId,\n context: LivenessContext\n) => Effect.Effect<boolean>\n\n/**\n * Whether two owner identities are incarnations on the same host.\n *\n * The predicate a probe applies before it inspects a pid: `owner.pid` names a\n * process in the claimant's own process namespace only when the hosts match,\n * so a cross-host probe that read it would be answering about an unrelated\n * process. Exposed because the check that needs it — a\n * `process.kill(pid, 0)` probe — belongs to a platform package rather than\n * here.\n *\n * @since 0.1.0\n * @category ownership\n */\nexport const sameHostIncarnation = (\n expectedOwner: OwnerId,\n claimant: OwnerId\n): boolean => expectedOwner.hostId === claimant.hostId\n\n/**\n * The default liveness check: the lease, and nothing else.\n *\n * An owner is treated as alive for as long as its persisted heartbeat is\n * younger than `staleAfter`, and as gone once it is not. That is the weakest\n * honest answer, and it is the one every host can give: a fresh process with\n * no application code at all can reclaim a hard-killed owner's runs once the\n * lease it stopped renewing has expired. A deployment that can say more — a\n * pid probe on the owner's host, an orchestrator that reports pod liveness —\n * supplies its own {@link LivenessCheck} and refuses the takeover for longer.\n *\n * An owner with no recorded heartbeat holds no lease and is reported gone; the\n * steal it enables is still gated by the store's own snapshot compare-and-swap.\n *\n * @since 0.1.0\n * @category ownership\n */\nexport const leaseLiveness = (\n staleAfter: Duration.Input = heartbeatStaleAfter\n): LivenessCheck => {\n const staleAfterMs = Duration.toMillis(staleAfter)\n return (_expectedOwner, context) =>\n Effect.succeed(\n context.heartbeatAtMs !== null &&\n context.heartbeatAtMs >= context.nowMs - staleAfterMs\n )\n}\n\n/**\n * The Node hosts' liveness check: does the owner's process still exist?\n *\n * {@link leaseLiveness} is the honest floor — every host can read a persisted\n * heartbeat — but it is only a timeout, so two engine processes over one\n * database steal each other's running rows `heartbeatStaleAfter` after any\n * heartbeat stall: a stop-the-world pause, a swapped-out process, a disk that\n * blocked longer than the window. This answers the question the lease is\n * standing in for, by asking the operating system whether the recorded pid is\n * still there.\n *\n * `process.kill(pid, 0)` sends no signal; it performs only the delivery\n * checks. Three answers matter:\n *\n * - It returns: the process exists and is signalable. The owner is alive.\n * - It throws `ESRCH`: no such process exists. This is the sole death answer,\n * so the owner is gone.\n * - It throws anything else, including `EPERM`, `EINVAL`, or an unrecognized\n * value: the owner is treated as alive because an unknown answer is not\n * death.\n *\n * This `ESRCH`-only death rule deliberately aligns with\n * `@smthrs/platform-node`'s `HostLiveness`.\n *\n * A pid is only meaningful inside one process namespace, so a recorded owner\n * on another host is never probed: the answer is `false` and the arbitration\n * falls back to the evidence that does cross hosts — the expired lease, which\n * `RunStore.steal` verifies for itself. The engine consults this check only\n * for a run whose lease has ALREADY expired, so answering `false` here does\n * not weaken anything; it declines to add evidence.\n *\n * Node hosts only. It is not part of the browser promise: this entry point\n * bundles for the browser because it never imports a `node:` built-in, and a\n * browser composition has no process table to ask, so it keeps\n * {@link leaseLiveness}.\n *\n * `@smthrs/platform-node`'s `HostLiveness.isAlive` asks the same question of\n * the same process table and differs in one deliberate place: it answers\n * `true` for an owner on another host, which refuses the steal outright, while\n * this check answers `false` and lets the expired lease decide. The difference\n * matters after a host dies for good — under the refusing answer its runs are\n * never reclaimed by anyone, because no other machine can ever produce\n * evidence about its pids.\n *\n * Two limits are inherent to asking a pid, and they bound what reclaim can\n * promise. Both are shared with `HostLiveness.isAlive`.\n *\n * - An owner recorded with the CLAIMANT'S OWN pid is always alive. A previous\n * incarnation of this process, or a second engine composed inside it, differs\n * from the claimant only by `nonce`, and the process it names is this one.\n * Such a row is never stolen while the process lives, so an embedded host\n * that re-creates its engine in place should keep {@link leaseLiveness},\n * whose timeout does expire. Reading same-pid-different-nonce as death is\n * not the alternative: it would let two engines in one process — the exact\n * shape this check exists to arbitrate — steal each other's live runs.\n * - A pid the operating system has REUSED reports the unrelated process that\n * now holds it. The dead owner's row stays refused for as long as that\n * process lives, which delays reclaim rather than breaking it: the row is\n * still `running` under an expired lease, and the next probe after the pid\n * is free reclaims it.\n *\n * @since 0.1.0\n * @category ownership\n */\nexport const sameHostPidProbe: LivenessCheck = (expectedOwner, context) =>\n Effect.sync(() => {\n if (!sameHostIncarnation(expectedOwner, context.claimant)) return false\n if (!Number.isSafeInteger(expectedOwner.pid) || expectedOwner.pid <= 0) return true\n try {\n process.kill(expectedOwner.pid, 0)\n return true\n } catch (error) {\n return (error as { readonly code?: string | undefined } | null)?.code !== \"ESRCH\"\n }\n })\n\nexport {\n /**\n * How often the supervision loop pulses to renew the owner's lease.\n *\n * @since 0.1.0\n * @category constants\n */\n heartbeatInterval,\n /**\n * How far the owner's wall clock may run behind a peer's before the lease\n * reasoning stops holding.\n *\n * @since 0.1.0\n * @category constants\n */\n heartbeatSkewAllowance,\n /**\n * How old a persisted heartbeat must be before a peer may steal the run.\n *\n * @since 0.1.0\n * @category constants\n */\n heartbeatStaleAfter,\n /**\n * How long the owner may keep working through failing or stalled heartbeat writes.\n *\n * @since 0.1.0\n * @category constants\n */\n heartbeatWriteTolerance\n} from \"./Heartbeat.ts\"\n\n/**\n * Hooks a {@link heartbeatLoop} caller may supply.\n *\n * @since 1.0.0\n * @category models\n */\nexport interface HeartbeatLoopOptions {\n /** Called after the same unclaimed owner reconfirms an expired heartbeat budget. */\n readonly onReconfirm?: ((unconfirmedMs: number) => Effect.Effect<void>) | undefined\n /** Called with the unconfirmed milliseconds when the lease lapses, before the loop interrupts itself. */\n readonly onLapse?: ((unconfirmedMs: number) => Effect.Effect<void>) | undefined\n}\n\n/**\n * Runs heartbeats until the persisted ownership fence is lost, then interrupts\n * itself. Race this effect with owned work so structured concurrency\n * interrupts the work when ownership disappears.\n *\n * Each pulse drives the injected `Consensus` strategy's `heartbeat` through\n * `RunStore.heartbeat`, which renews the strategy's lease and mirrors the\n * recorded stamp onto the run row in the same transaction. Heartbeats are\n * lease evidence, never journal events.\n *\n * Pulses are delayed by `heartbeatInterval` and read the Effect `Clock`, so the\n * loop is fully driveable with `TestClock`.\n *\n * A lost fence — any outcome other than `Updated` — is durable evidence and\n * interrupts immediately. A failed heartbeat *write* is not: the persisted\n * heartbeat is still there and no other process may steal the run until it is\n * `heartbeatStaleAfter` old, so transient write errors are tolerated for\n * `heartbeatWriteTolerance` — deliberately shorter than the steal cutoff by a\n * pulse plus `heartbeatSkewAllowance`. The reserved pulse bounds a final\n * reconfirmation; a pending claim or changed owner fails that check. Past that allowance\n * the fence still protects durable writes but non-durable side effects may\n * overlap; see {@link heartbeatWriteTolerance}.\n *\n * An independent deadline races the pulse loop. At expiry it reconfirms the\n * same owner with no pending claim, bounded by one heartbeat interval. A\n * renewed lease continues supervision; loss, error, or timeout interrupts\n * even when an ordinary heartbeat write never returns. Each successful\n * pulse re-arms the deadline from the timestamp sent to the store, not from\n * its completion time. The deadline re-reads the clock after waking, so delayed\n * writes cannot hide expiry behind a stale clock reading.\n *\n * `options.onReconfirm` records a successful reconfirmation after a gap. Its\n * deadline is the remaining renewed lease budget, measured from the timestamp\n * sent to the store. A blocked or failed receipt lapses the lease instead of\n * stranding supervision; receipt completion never grants another lease budget.\n * `options.onLapse` runs once, before the self-interrupt, when reconfirmation\n * fails: it receives how long the lease went unconfirmed, so the caller can\n * record why its work stopped. A lost fence does not call it.\n *\n * @since 0.1.0\n * @category supervision\n */\nexport const heartbeatLoop = (\n runId: string,\n owner: OwnerId,\n options: HeartbeatLoopOptions = {}\n): Effect.Effect<never, never, RunStore> =>\n Effect.gen(function*() {\n const runStore = yield* RunStore\n const toleranceMs = Duration.toMillis(heartbeatWriteTolerance)\n const intervalMs = Duration.toMillis(heartbeatInterval)\n let lastConfirmedPulseMs = yield* Clock.currentTimeMillis\n let failing = false\n // Both overdue timers may wake after a host stall. Serialize expiry and\n // recheck the budget so they cannot reconfirm or journal the same gap twice.\n const expiry = yield* Semaphore.make(1)\n const expire = () =>\n expiry.withPermit(Effect.gen(function*() {\n // A competing expiry may have held the permit across a write. Take\n // fresh evidence after acquisition, never the waiting timer's reading.\n const nowMs = yield* Clock.currentTimeMillis\n if (nowMs - lastConfirmedPulseMs < toleranceMs) return\n const unconfirmedMs = nowMs - lastConfirmedPulseMs\n const renewed = yield* runStore.reconfirm(runId, owner, Math.floor(nowMs)).pipe(\n // Only the ownership check consumes the old lease's reserved pulse.\n Effect.timeout(heartbeatInterval),\n Effect.flatMap((outcome) =>\n Effect.gen(function*() {\n if (outcome._tag !== \"Updated\") return false\n lastConfirmedPulseMs = Math.max(lastConfirmedPulseMs, Math.floor(nowMs))\n failing = false\n const confirmedAtMs = yield* Clock.currentTimeMillis\n const remainingMs = lastConfirmedPulseMs + toleranceMs - confirmedAtMs\n if (remainingMs <= 0) return false\n // The receipt is still required and bounded, but ownership was\n // already renewed. Spending only its old reserved pulse would\n // interrupt live work despite a fresh, fenced persisted lease.\n yield* (options.onReconfirm?.(unconfirmedMs) ?? Effect.void).pipe(Effect.timeout(remainingMs))\n return true\n })\n ),\n Effect.catchCause((cause) =>\n Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause as Cause.Cause<never>) : Effect.succeed(false)\n )\n )\n if (renewed) return\n yield* Effect.logWarning(\"run lease lapsed; interrupting owned work\").pipe(\n Effect.annotateLogs({ runId, unconfirmedMs }),\n Effect.andThen(options.onLapse?.(unconfirmedMs) ?? Effect.void),\n Effect.andThen(Effect.interrupt)\n )\n }))\n const deadline = Effect.gen(function*() {\n while (true) {\n const nowMs = yield* Clock.currentTimeMillis\n const remainingMs = lastConfirmedPulseMs + toleranceMs - nowMs\n if (remainingMs <= 0) {\n yield* expire()\n continue\n }\n // Check alongside pulse intervals, with a shorter final wait when needed.\n yield* Effect.sleep(Math.min(remainingMs, intervalMs))\n }\n })\n const pulses = Effect.sleep(heartbeatInterval).pipe(\n Effect.andThen(Clock.currentTimeMillis.pipe(Effect.map(Math.floor))),\n Effect.flatMap((nowMs) =>\n nowMs - lastConfirmedPulseMs >= toleranceMs ? expire() : runStore.heartbeat(runId, owner, nowMs).pipe(\n Effect.flatMap((outcome) =>\n outcome._tag === \"Updated\"\n ? Effect.sync(() => {\n lastConfirmedPulseMs = Math.max(lastConfirmedPulseMs, nowMs)\n failing = false\n })\n : Effect.interrupt\n ),\n // A failed write keeps the confirmed lease; the independent deadline\n // supervises its expiry. Warn on the first failure of an outage.\n Effect.catch((error) => {\n if (failing) return Effect.void\n failing = true\n return Effect.logWarning(\"run heartbeat write failed\").pipe(\n Effect.annotateLogs({ runId, code: error.code })\n )\n })\n )\n ),\n Effect.forever\n )\n return yield* Effect.raceFirst(pulses, deadline)\n })\n"]}