@smthrs/run-store 0.0.0-stage → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/CHANGELOG.md +112 -0
  2. package/LICENSE +21 -0
  3. package/README.md +195 -2
  4. package/dist/cjs/AttemptStore.d.ts +359 -0
  5. package/dist/cjs/AttemptStore.d.ts.map +1 -0
  6. package/dist/cjs/AttemptStore.js +731 -0
  7. package/dist/cjs/AttemptStore.js.map +7 -0
  8. package/dist/cjs/Heartbeat.d.ts +46 -0
  9. package/dist/cjs/Heartbeat.d.ts.map +1 -0
  10. package/dist/cjs/Heartbeat.js +28 -0
  11. package/dist/cjs/Heartbeat.js.map +7 -0
  12. package/dist/cjs/Migrations.d.ts +42 -0
  13. package/dist/cjs/Migrations.d.ts.map +1 -0
  14. package/dist/cjs/Migrations.js +55 -0
  15. package/dist/cjs/Migrations.js.map +7 -0
  16. package/dist/cjs/Ownership.d.ts +275 -0
  17. package/dist/cjs/Ownership.d.ts.map +1 -0
  18. package/dist/cjs/Ownership.js +129 -0
  19. package/dist/cjs/Ownership.js.map +7 -0
  20. package/dist/cjs/RunStore.d.ts +571 -0
  21. package/dist/cjs/RunStore.d.ts.map +1 -0
  22. package/dist/cjs/RunStore.js +1079 -0
  23. package/dist/cjs/RunStore.js.map +7 -0
  24. package/dist/cjs/RunStoreMetrics.d.ts +151 -0
  25. package/dist/cjs/RunStoreMetrics.d.ts.map +1 -0
  26. package/dist/cjs/RunStoreMetrics.js +91 -0
  27. package/dist/cjs/RunStoreMetrics.js.map +7 -0
  28. package/dist/cjs/index.d.ts +47 -0
  29. package/dist/cjs/index.d.ts.map +1 -0
  30. package/dist/cjs/index.js +43 -0
  31. package/dist/cjs/index.js.map +7 -0
  32. package/dist/cjs/internal/Boundary.d.ts +80 -0
  33. package/dist/cjs/internal/Boundary.d.ts.map +1 -0
  34. package/dist/cjs/internal/Boundary.js +92 -0
  35. package/dist/cjs/internal/Boundary.js.map +7 -0
  36. package/dist/cjs/internal/Companion.d.ts +44 -0
  37. package/dist/cjs/internal/Companion.d.ts.map +1 -0
  38. package/dist/cjs/internal/Companion.js +50 -0
  39. package/dist/cjs/internal/Companion.js.map +7 -0
  40. package/dist/cjs/internal/SpanOutcome.d.ts +37 -0
  41. package/dist/cjs/internal/SpanOutcome.d.ts.map +1 -0
  42. package/dist/cjs/internal/SpanOutcome.js +62 -0
  43. package/dist/cjs/internal/SpanOutcome.js.map +7 -0
  44. package/dist/cjs/migrations/0001_initial.d.ts +15 -0
  45. package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
  46. package/dist/cjs/migrations/0001_initial.js +110 -0
  47. package/dist/cjs/migrations/0001_initial.js.map +7 -0
  48. package/dist/cjs/migrations/0002_lineage.d.ts +27 -0
  49. package/dist/cjs/migrations/0002_lineage.d.ts.map +1 -0
  50. package/dist/cjs/migrations/0002_lineage.js +43 -0
  51. package/dist/cjs/migrations/0002_lineage.js.map +7 -0
  52. package/dist/cjs/migrations/0003_execution_revisions.d.ts +15 -0
  53. package/dist/cjs/migrations/0003_execution_revisions.d.ts.map +1 -0
  54. package/dist/cjs/migrations/0003_execution_revisions.js +77 -0
  55. package/dist/cjs/migrations/0003_execution_revisions.js.map +7 -0
  56. package/dist/cjs/migrations/0004_waiting_request.d.ts +32 -0
  57. package/dist/cjs/migrations/0004_waiting_request.d.ts.map +1 -0
  58. package/dist/cjs/migrations/0004_waiting_request.js +42 -0
  59. package/dist/cjs/migrations/0004_waiting_request.js.map +7 -0
  60. package/dist/cjs/package.json +1 -0
  61. package/dist/cjs/test/TestRunStore.d.ts +17 -0
  62. package/dist/cjs/test/TestRunStore.d.ts.map +1 -0
  63. package/dist/cjs/test/TestRunStore.js +42 -0
  64. package/dist/cjs/test/TestRunStore.js.map +7 -0
  65. package/dist/esm/AttemptStore.d.ts +359 -0
  66. package/dist/esm/AttemptStore.d.ts.map +1 -0
  67. package/dist/esm/AttemptStore.js +835 -0
  68. package/dist/esm/AttemptStore.js.map +1 -0
  69. package/dist/esm/Heartbeat.d.ts +46 -0
  70. package/dist/esm/Heartbeat.d.ts.map +1 -0
  71. package/dist/esm/Heartbeat.js +46 -0
  72. package/dist/esm/Heartbeat.js.map +1 -0
  73. package/dist/esm/Migrations.d.ts +42 -0
  74. package/dist/esm/Migrations.d.ts.map +1 -0
  75. package/dist/esm/Migrations.js +56 -0
  76. package/dist/esm/Migrations.js.map +1 -0
  77. package/dist/esm/Ownership.d.ts +275 -0
  78. package/dist/esm/Ownership.d.ts.map +1 -0
  79. package/dist/esm/Ownership.js +299 -0
  80. package/dist/esm/Ownership.js.map +1 -0
  81. package/dist/esm/RunStore.d.ts +571 -0
  82. package/dist/esm/RunStore.d.ts.map +1 -0
  83. package/dist/esm/RunStore.js +1287 -0
  84. package/dist/esm/RunStore.js.map +1 -0
  85. package/dist/esm/RunStoreMetrics.d.ts +151 -0
  86. package/dist/esm/RunStoreMetrics.d.ts.map +1 -0
  87. package/dist/esm/RunStoreMetrics.js +120 -0
  88. package/dist/esm/RunStoreMetrics.js.map +1 -0
  89. package/dist/esm/index.d.ts +47 -0
  90. package/dist/esm/index.d.ts.map +1 -0
  91. package/dist/esm/index.js +47 -0
  92. package/dist/esm/index.js.map +1 -0
  93. package/dist/esm/internal/Boundary.d.ts +80 -0
  94. package/dist/esm/internal/Boundary.d.ts.map +1 -0
  95. package/dist/esm/internal/Boundary.js +109 -0
  96. package/dist/esm/internal/Boundary.js.map +1 -0
  97. package/dist/esm/internal/Companion.d.ts +44 -0
  98. package/dist/esm/internal/Companion.d.ts.map +1 -0
  99. package/dist/esm/internal/Companion.js +53 -0
  100. package/dist/esm/internal/Companion.js.map +1 -0
  101. package/dist/esm/internal/SpanOutcome.d.ts +37 -0
  102. package/dist/esm/internal/SpanOutcome.d.ts.map +1 -0
  103. package/dist/esm/internal/SpanOutcome.js +48 -0
  104. package/dist/esm/internal/SpanOutcome.js.map +1 -0
  105. package/dist/esm/migrations/0001_initial.d.ts +15 -0
  106. package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
  107. package/dist/esm/migrations/0001_initial.js +88 -0
  108. package/dist/esm/migrations/0001_initial.js.map +1 -0
  109. package/dist/esm/migrations/0002_lineage.d.ts +27 -0
  110. package/dist/esm/migrations/0002_lineage.d.ts.map +1 -0
  111. package/dist/esm/migrations/0002_lineage.js +33 -0
  112. package/dist/esm/migrations/0002_lineage.js.map +1 -0
  113. package/dist/esm/migrations/0003_execution_revisions.d.ts +15 -0
  114. package/dist/esm/migrations/0003_execution_revisions.d.ts.map +1 -0
  115. package/dist/esm/migrations/0003_execution_revisions.js +58 -0
  116. package/dist/esm/migrations/0003_execution_revisions.js.map +1 -0
  117. package/dist/esm/migrations/0004_waiting_request.d.ts +32 -0
  118. package/dist/esm/migrations/0004_waiting_request.d.ts.map +1 -0
  119. package/dist/esm/migrations/0004_waiting_request.js +37 -0
  120. package/dist/esm/migrations/0004_waiting_request.js.map +1 -0
  121. package/dist/esm/test/TestRunStore.d.ts +17 -0
  122. package/dist/esm/test/TestRunStore.d.ts.map +1 -0
  123. package/dist/esm/test/TestRunStore.js +19 -0
  124. package/dist/esm/test/TestRunStore.js.map +1 -0
  125. package/docs/README.md +142 -0
  126. package/docs/api.md +901 -0
  127. package/docs/concepts/durable-values.md +140 -0
  128. package/docs/concepts/execution-revisions.md +53 -0
  129. package/docs/concepts/fencing.md +146 -0
  130. package/docs/concepts/leases.md +154 -0
  131. package/docs/concepts/liveness-evidence.md +142 -0
  132. package/docs/guides/cancel-a-run.md +156 -0
  133. package/docs/guides/claim-and-finish-a-run.md +175 -0
  134. package/docs/guides/compose-the-stores.md +142 -0
  135. package/docs/guides/observe-outcomes.md +182 -0
  136. package/docs/guides/record-step-attempts.md +170 -0
  137. package/docs/guides/recover-a-stalled-run.md +173 -0
  138. package/docs/guides/testing.md +141 -0
  139. package/docs/installation.md +109 -0
  140. package/docs/quickstart.md +201 -0
  141. package/docs/troubleshooting.md +255 -0
  142. package/package.json +183 -3
  143. package/src/AttemptStore.ts +1195 -0
  144. package/src/Heartbeat.ts +47 -0
  145. package/src/Migrations.ts +59 -0
  146. package/src/Ownership.ts +407 -0
  147. package/src/RunStore.ts +2063 -0
  148. package/src/RunStoreMetrics.ts +165 -0
  149. package/src/index.ts +51 -0
  150. package/src/internal/Boundary.ts +130 -0
  151. package/src/internal/Companion.ts +64 -0
  152. package/src/internal/SpanOutcome.ts +71 -0
  153. package/src/migrations/0001_initial.ts +114 -0
  154. package/src/migrations/0002_lineage.ts +38 -0
  155. package/src/migrations/0003_execution_revisions.ts +69 -0
  156. package/src/migrations/0004_waiting_request.ts +39 -0
  157. package/src/test/TestRunStore.ts +22 -0
@@ -0,0 +1,170 @@
1
+ ---
2
+ title: "Record a step attempt"
3
+ description: "Start, checkpoint, finish, and annotate one execution of a step: the attempt identity, the run fence every write inherits, first-writer-wins versus upsert, and what patch is for."
4
+ sidebar:
5
+ order: 3
6
+ ---
7
+
8
+ An attempt is one execution of one content-addressed step inside a run. The
9
+ store holds four operations that move it and one that reads it, and every write
10
+ is fenced on the run you own.
11
+
12
+ ## The identity
13
+
14
+ An attempt is addressed by three fields:
15
+
16
+ ```ts
17
+ import type { AttemptId } from "@smthrs/run-store/AttemptStore"
18
+
19
+ const id: AttemptId = { runId: "run-1", stepKeyDigest: "sha256-9f2c", attempt: 0 }
20
+ ```
21
+
22
+ `stepKeyDigest` is the step's content address, so the same step in two runs
23
+ shares no row, and a retry of the same step increments `attempt` rather than
24
+ overwriting the previous try. All three are the table's primary key.
25
+
26
+ ## Start it
27
+
28
+ `put` inserts the row, and the insert only lands while the run is `running` and
29
+ owned by the caller:
30
+
31
+ ```ts
32
+ import { AttemptStore } from "@smthrs/run-store"
33
+ import type { OwnerId } from "@smthrs/run-store/Ownership"
34
+ import * as Clock from "effect/Clock"
35
+ import * as Effect from "effect/Effect"
36
+
37
+ const start = (id: AttemptId, owner: OwnerId) =>
38
+ Effect.gen(function*() {
39
+ const attempts = yield* AttemptStore.AttemptStore
40
+ return yield* attempts.put({
41
+ ...id,
42
+ state: "running",
43
+ startedAtMs: yield* Clock.currentTimeMillis,
44
+ meta: { step: "fetch", cached: false }
45
+ }, owner)
46
+ })
47
+ ```
48
+
49
+ `meta` is required and opaque: its shape belongs to the step executor, and the
50
+ store carries it unchanged across every later state change. `state` is a name
51
+ you choose, and the store only cares whether it is in `inProgressStates`.
52
+
53
+ Five outcomes are possible, and only one is a defect:
54
+
55
+ | Outcome | Meaning |
56
+ | -------------- | ----------------------------------------------------------------------------------------------------------------------- |
57
+ | `Inserted` | The row is new. |
58
+ | `Upserted` | `putMode: "upsert"` replaced a row that was still in progress. |
59
+ | `ExistingSame` | A row already exists and is byte-equivalent, so the write was a no-op. Object key order is ignored; array order is not. |
60
+ | `Conflict` | A row exists with different content, and the store is first-writer-wins. |
61
+ | `RunNotFound` | The run row does not exist. |
62
+ | `FenceLost` | The run exists but is not `running` under your owner. |
63
+
64
+ `ExistingSame` is what makes a replayed `put` safe. `Conflict` means two
65
+ writers disagree about what this attempt is, which is a bug in the caller.
66
+
67
+ ## Checkpoint it
68
+
69
+ `heartbeat` proves the attempt is still moving and optionally saves the value it
70
+ would resume from:
71
+
72
+ ```ts
73
+ const beat = (id: AttemptId, owner: OwnerId, checkpoint: unknown) =>
74
+ Effect.gen(function*() {
75
+ const attempts = yield* AttemptStore.AttemptStore
76
+ return yield* attempts.heartbeat(
77
+ id.runId,
78
+ id.stepKeyDigest,
79
+ id.attempt,
80
+ owner,
81
+ yield* Clock.currentTimeMillis,
82
+ checkpoint as AttemptStore.JsonValue
83
+ )
84
+ })
85
+ ```
86
+
87
+ The checkpoint argument is optional, and omitting it leaves the stored one
88
+ alone: the column is written with `COALESCE`, never cleared. The heartbeat
89
+ stamp is monotonic, so a delayed pulse cannot move it backwards.
90
+
91
+ `heartbeat` fences on two things: the run must be `running` under your owner,
92
+ and the attempt's `state` must still be one of the store's `inProgressStates`.
93
+ Those two are reported separately. `FenceLost` means the run moved on;
94
+ `StateChanged` means you still own the run but the attempt is no longer in
95
+ progress, so something already finished it.
96
+
97
+ ## Finish it
98
+
99
+ `finish` moves the attempt to a terminal state and records how it ended:
100
+
101
+ ```ts
102
+ const succeed = (id: AttemptId, owner: OwnerId, outcome: AttemptStore.JsonValue) =>
103
+ Effect.gen(function*() {
104
+ const attempts = yield* AttemptStore.AttemptStore
105
+ return yield* attempts.finish({
106
+ ...id,
107
+ state: "completed",
108
+ finishedAtMs: yield* Clock.currentTimeMillis,
109
+ outcome
110
+ }, owner)
111
+ })
112
+ ```
113
+
114
+ The target `state` must not be one of `inProgressStates`; passing one fails
115
+ with `invalid_attempt`. `error`, `outcome`, and `meta` are all optional, and an
116
+ omitted field is left as recorded rather than cleared, so a terminal transition
117
+ never erases a value written mid-flight. Supplying one replaces it atomically
118
+ with the terminal state, which is how an executor durably records what it
119
+ learned while handling a failure.
120
+
121
+ The outcomes match `heartbeat`: `Finished`, `NotFound`, `FenceLost`, or
122
+ `StateChanged`. A second `finish` on an already-terminal attempt reports
123
+ `StateChanged`, so repeated terminal transitions never overwrite the winning
124
+ row.
125
+
126
+ ## Annotate it without touching its lifecycle
127
+
128
+ `patch` rewrites the opaque fields and nothing else. It never moves `state`,
129
+ `startedAtMs`, or `finishedAtMs`, so it works on running and terminal rows
130
+ alike:
131
+
132
+ ```ts
133
+ const annotate = (id: AttemptId, owner: OwnerId) =>
134
+ Effect.gen(function*() {
135
+ const attempts = yield* AttemptStore.AttemptStore
136
+ return yield* attempts.patch(id, { meta: { worktree: "/tmp/wt-1" } }, owner)
137
+ })
138
+ ```
139
+
140
+ Use it for response text, worktree pointers, cache flags, and quarantined
141
+ evidence: the things an executor discovers about an attempt after the fact.
142
+ Omitted fields are left as recorded.
143
+
144
+ `patch` is still fenced on run ownership, and that is the one surprising part.
145
+ There is no unfenced write surface, so a delayed patch from an owner that lost
146
+ the run reports `FenceLost` instead of rewriting the winning row. After the run
147
+ reaches any terminal status its owner columns are cleared, so every patch on it
148
+ reports `FenceLost` from then on. Patch before you settle the run.
149
+
150
+ ## Read it back
151
+
152
+ ```ts
153
+ const read = (id: AttemptId) =>
154
+ Effect.gen(function*() {
155
+ const attempts = yield* AttemptStore.AttemptStore
156
+ return yield* attempts.get(id)
157
+ })
158
+ ```
159
+
160
+ `get` is the one unfenced operation and returns `Option<Attempt>`. Absent
161
+ optional columns come back as absent keys rather than nulls, and every stored
162
+ value is re-validated on the way out, so a row corrupted outside these stores
163
+ fails with `decode_failed` instead of being handed back as state.
164
+
165
+ ## Next steps
166
+
167
+ - [Durable values](../concepts/durable-values.md): what the admission boundary
168
+ accepts, and the limits on checkpoints.
169
+ - [Compose the stores into a host](./compose-the-stores.md): setting
170
+ `inProgressStates`, `putMode`, and the checkpoint ceiling.
@@ -0,0 +1,173 @@
1
+ ---
2
+ title: "Take over a stalled run"
3
+ description: "Reclaim a run whose owner stopped: build liveness evidence at the instant you use it, steal the stale row, activate it, and recover a claim nobody ever activated."
4
+ sidebar:
5
+ order: 5
6
+ ---
7
+
8
+ A run stalls in two different places, and each has its own operation:
9
+
10
+ - A **running owner stopped heartbeating.** The row is `running`, its
11
+ `heartbeat_at_ms` is older than `heartbeatStaleAfter`, and no other process
12
+ can write to it. `steal` takes it.
13
+ - A **claim was never activated.** The row is still `pending` or `suspended`,
14
+ its claim columns are held by a claimant that died between `claim` and
15
+ `activate`, and `claim` reports `AlreadyClaimed` forever. `recoverClaim`
16
+ releases it.
17
+
18
+ Both require two independent things: the SQL predicate that the row really is
19
+ stale, and matching `LivenessEvidence` about the process that left it that way.
20
+ Neither alone is enough.
21
+
22
+ ## Build evidence at the instant you spend it
23
+
24
+ `evidence.checkedAtMs` must equal the `nowMs` of the call that consumes it,
25
+ exactly, so evidence cannot be probed once and replayed later. Build it from a
26
+ [`LivenessCheck`](../concepts/liveness-evidence.md) inside the same effect that
27
+ uses it:
28
+
29
+ ```ts
30
+ import { Ownership } from "@smthrs/run-store"
31
+ import * as Effect from "effect/Effect"
32
+
33
+ const probeFrom = (
34
+ check: Ownership.LivenessCheck,
35
+ heartbeatAtMs: number | null
36
+ ): Ownership.LivenessProbe =>
37
+ (expectedOwner, claimant, checkedAtMs) =>
38
+ Effect.map(
39
+ check(expectedOwner, { claimant, heartbeatAtMs, nowMs: checkedAtMs }),
40
+ (alive) =>
41
+ alive ? undefined : {
42
+ expectedOwner,
43
+ checkedAtMs,
44
+ kind: Ownership.sameHostIncarnation(expectedOwner, claimant)
45
+ ? "same-host-pid-dead" as const
46
+ : "cross-host-unreachable-stale" as const
47
+ }
48
+ )
49
+ ```
50
+
51
+ The `kind` has to match the host relation or the store refuses it: a pid probe
52
+ means nothing across hosts, and an unreachability judgment means nothing on the
53
+ owner's own machine. `Ownership.sameHostPidProbe` is the check to pass on a Node
54
+ host; `Ownership.leaseLiveness()` is the floor every host can give.
55
+
56
+ ## Steal a stale running row, then activate it
57
+
58
+ ```ts
59
+ import { RunStore } from "@smthrs/run-store"
60
+ import type { OwnerId } from "@smthrs/run-store/Ownership"
61
+ import * as Clock from "effect/Clock"
62
+
63
+ const takeOver = (runId: string, claimant: OwnerId, check: Ownership.LivenessCheck) =>
64
+ Effect.gen(function*() {
65
+ const runs = yield* RunStore.RunStore
66
+ const row = yield* runs.get(runId)
67
+ if (row.status !== "running" || row.owner === null) return { _tag: "NotRunning" } as const
68
+
69
+ const expected = { status: row.status, owner: row.owner, heartbeatAtMs: row.heartbeatAtMs }
70
+ const nowMs = yield* Clock.currentTimeMillis
71
+ const evidence = yield* probeFrom(check, row.heartbeatAtMs)(row.owner, claimant, nowMs)
72
+ if (evidence === undefined) return { _tag: "OwnerAlive" } as const
73
+
74
+ const stolen = yield* runs.steal(runId, expected, claimant, nowMs, evidence)
75
+ if (stolen._tag !== "Claimed") return stolen
76
+ return yield* runs.activate(runId, claimant, stolen.claimedAtMs, expected)
77
+ })
78
+ ```
79
+
80
+ A successful steal writes the claim columns and nothing else, so the row's
81
+ status, owner, and heartbeat are still the ones you read. That is why the same
82
+ `expected` snapshot satisfies the activation, and why the takeover path and the
83
+ ordinary claim path converge on `activate`.
84
+
85
+ | Steal outcome | What it means |
86
+ | --------------------- | ----------------------------------------------------------------------------------------- |
87
+ | `Claimed` | The claim columns are yours. `claimedAtMs` is your activation token. |
88
+ | `LivenessUnconfirmed` | The evidence did not match the owner, the host relation, or `nowMs`. No comparison ran. |
89
+ | `HeartbeatFresh` | The persisted heartbeat is still inside the staleness window. The owner is not stale yet. |
90
+ | `SnapshotChanged` | The evidence matched, and the row moved between your read and your write. |
91
+ | `AlreadyClaimed` | Another process already holds the claim columns. |
92
+ | `NotFound` | No such run row. |
93
+
94
+ `LivenessUnconfirmed` and `SnapshotChanged` are deliberately separate: the first
95
+ says you were not allowed to try, the second says you tried and lost.
96
+
97
+ ## Recover a claim nobody activated
98
+
99
+ `recoverClaim` clears an exact stale claim so the row becomes claimable again.
100
+ It takes the claimant it is displacing, that claimant's `claimedAtMs` as a fence
101
+ token, your own identity as the observer, and evidence about the dead claimant:
102
+
103
+ ```ts
104
+ const releaseStaleClaim = (runId: string, observer: OwnerId, check: Ownership.LivenessCheck) =>
105
+ Effect.gen(function*() {
106
+ const runs = yield* RunStore.RunStore
107
+ const row = yield* runs.get(runId)
108
+ if (row.claim === null || row.claimedAtMs === null) return { _tag: "NoClaim" } as const
109
+
110
+ const nowMs = yield* Clock.currentTimeMillis
111
+ const evidence = yield* probeFrom(check, row.heartbeatAtMs)(row.claim, observer, nowMs)
112
+ if (evidence === undefined) return { _tag: "ClaimantAlive" } as const
113
+
114
+ return yield* runs.recoverClaim(runId, row.claim, row.claimedAtMs, observer, nowMs, evidence)
115
+ })
116
+ ```
117
+
118
+ `Recovered` means the claim columns are clear. Claim the run normally after
119
+ that; recovery does not hand you the run, it only unblocks it.
120
+
121
+ | Recover outcome | What it means |
122
+ | --------------------- | ---------------------------------------------------------------------------------- |
123
+ | `Recovered` | The stale claim is cleared. Any process may now claim the run. |
124
+ | `ClaimFresh` | The claim is younger than `heartbeatStaleAfter`. Wait, or leave it alone. |
125
+ | `ClaimChanged` | The row no longer carries the claim you named, so someone else already handled it. |
126
+ | `LivenessUnconfirmed` | The evidence did not match the claimant, the host relation, or `nowMs`. |
127
+ | `NotFound` | No such run row. |
128
+
129
+ An owner that loses its own activation does not need any of this:
130
+ `abandonClaim` releases a claim cooperatively, and it is what
131
+ [Claim a run and finish it](./claim-and-finish-a-run.md) does when an activation
132
+ loses. `recoverClaim` is for the claimant that will never come back.
133
+
134
+ ## The two cutoffs are the same number
135
+
136
+ `steal` refuses any row whose `heartbeat_at_ms` is within `heartbeatStaleAfter`
137
+ of your `nowMs`, and `recoverClaim` refuses any claim whose `claimed_at_ms` is
138
+ within the same window. Both compare against the `nowMs` you passed, and both
139
+ refuse a `nowMs` that runs more than `heartbeatSkewAllowance` ahead of the
140
+ store's own clock with `invalid_run`, so no caller can shorten either window by
141
+ reporting a later time.
142
+
143
+ ## Let the engine do it
144
+
145
+ If you are running the Smithers engine, this sweep already exists. Its
146
+ `NodeRuntime`, from [`@smthrs/flows`](/api/flows) rather than this package,
147
+ requires an `isAlive` check precisely so it can run the sweep, and it takes no
148
+ default: a check that answers "gone" without asking steals runs out of live
149
+ processes.
150
+
151
+ ```ts
152
+ import type * as NodeRuntime from "@smthrs/flows/NodeRuntime"
153
+ import { Ownership } from "@smthrs/run-store"
154
+
155
+ const options: NodeRuntime.Options = {
156
+ filename: ".smithers/smithers.db",
157
+ workspaceRoot: "/srv/workspace",
158
+ owner: { hostId: "host-a" },
159
+ isAlive: Ownership.sameHostPidProbe
160
+ }
161
+ ```
162
+
163
+ Write the sweep yourself only for a host, an adapter, or an operator tool that
164
+ has to agree with the engine about who owns what.
165
+
166
+ ## Next steps
167
+
168
+ - [Liveness evidence](../concepts/liveness-evidence.md): the three kinds, and
169
+ why every unknown probe answer is read as life.
170
+ - [The heartbeat lease](../concepts/leases.md): what the staleness window can
171
+ and cannot promise.
172
+ - [Observe store outcomes](./observe-outcomes.md): the counters that make a
173
+ takeover visible without reading logs.
@@ -0,0 +1,141 @@
1
+ ---
2
+ title: "Test against the real stores"
3
+ description: "Test ownership and attempt code deterministically: the in-memory layer that runs the production stores, TestClock for every lease rule, the noop layers, and when a real file is the only honest test."
4
+ sidebar:
5
+ order: 7
6
+ ---
7
+
8
+ Nothing in these stores needs a mock. Every nondeterministic input is a service:
9
+ the database is a `SqlClient`, the lease reasoning reads the Effect `Clock`, and
10
+ the liveness answer is a `LivenessCheck` you pass in. A test swaps those and runs
11
+ the production code.
12
+
13
+ ## Run the production stores in memory
14
+
15
+ `@smthrs/run-store/test/TestRunStore` provides `RunStore` and `AttemptStore` over
16
+ a fresh in-memory SQLite database with the migrations already applied. The stores
17
+ are the real ones; only the file is gone.
18
+
19
+ ```ts
20
+ import { AttemptStore, RunStore } from "@smthrs/run-store"
21
+ import * as TestRunStore from "@smthrs/run-store/test/TestRunStore"
22
+ import * as Effect from "effect/Effect"
23
+
24
+ const withStores = <A, E>(
25
+ effect: Effect.Effect<A, E, RunStore.RunStore | AttemptStore.AttemptStore>
26
+ ) => effect.pipe(Effect.provide(TestRunStore.layer), Effect.scoped)
27
+ ```
28
+
29
+ The layer is Node only, because it binds `node:sqlite`. Nothing else in the
30
+ package is: the root entry point names no driver and bundles for the browser.
31
+
32
+ Each provision gets its own database, so tests do not share rows. `Effect.scoped`
33
+ is required: the database is a scoped resource and closes with the scope.
34
+
35
+ ## Drive every lease rule with TestClock
36
+
37
+ The lease is the part most worth testing and the part you must never wait for in
38
+ wall time. Both the store's own stamps and `Ownership.heartbeatLoop` read the
39
+ injected `Clock`, so `TestClock` controls all of it:
40
+
41
+ ```ts
42
+ import * as TestDatabase from "@smthrs/database/test/TestDatabase"
43
+ import * as Migrations from "@smthrs/run-store/Migrations"
44
+ import * as Duration from "effect/Duration"
45
+ import { TestClock } from "effect/testing"
46
+
47
+ const withTestClock = <A, E>(effect: Effect.Effect<A, E, RunStore.RunStore>) =>
48
+ effect.pipe(
49
+ Effect.provide(RunStore.layer),
50
+ Effect.provide(Migrations.layer),
51
+ Effect.provide(TestDatabase.layer),
52
+ Effect.provide(TestClock.layer())
53
+ )
54
+ ```
55
+
56
+ `TestDatabase` comes from [`@smthrs/database`](/api/database), which this
57
+ package depends on but does not re-export, so add it to your dev dependencies
58
+ for that import to resolve.
59
+
60
+ `TestClock.adjust(Duration.seconds(31))` walks a fresh lease past
61
+ `heartbeatStaleAfter` in one statement, which is what makes a steal test a
62
+ readable ten lines. Take `nowMs` from `Clock.currentTimeMillis` in the code under
63
+ test, never `Date.now()`: a reading more than `heartbeatSkewAllowance` ahead of
64
+ the store's clock fails with `invalid_run`, and a test that mixed the two would
65
+ be failing on that rather than on what it meant to assert.
66
+
67
+ To test the supervision loop, race it against work and assert on the exit:
68
+
69
+ ```ts
70
+ import { Ownership } from "@smthrs/run-store"
71
+ import type { OwnerId } from "@smthrs/run-store/Ownership"
72
+
73
+ const supervised = (runId: string, owner: OwnerId) =>
74
+ Effect.raceFirst(Effect.never, Ownership.heartbeatLoop(runId, owner))
75
+ ```
76
+
77
+ A pulse lands after each `TestClock.adjust(Ownership.heartbeatInterval)`. Once
78
+ the fence is gone, the loop interrupts itself and takes the raced work with it,
79
+ so the fiber's exit is an interrupt rather than a failure.
80
+
81
+ ## Assert on outcomes, not on rows
82
+
83
+ Every operation answers with a tagged value, so the assertion is the contract:
84
+
85
+ ```ts
86
+ const losesTheRace = Effect.gen(function*() {
87
+ const runs = yield* RunStore.RunStore
88
+ const outcome = yield* runs.claim("run-1", expected, ownerB, nowMs)
89
+ expect(outcome).toEqual({ _tag: "AlreadyClaimed" })
90
+ })
91
+ ```
92
+
93
+ Read the row back with `get` only for the fields the outcome does not carry:
94
+ `heartbeatAtMs` after a pulse, `finishedAtMs` after a terminal transition, or
95
+ `stateJson` to prove executable state round-tripped byte for byte. Counting
96
+ outcomes proves the other half; see
97
+ [Observe store outcomes](./observe-outcomes.md).
98
+
99
+ ## Use the noop layers for compositions that must not persist
100
+
101
+ Both services ship an explicit absence, so a test that is not about persistence
102
+ provides the smallest thing that type-checks:
103
+
104
+ | Layer | Behavior |
105
+ | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
106
+ | `RunStore.layerNoop()` | `create` and `get` fail with `persistence_failed`. Every compare-and-swap reports a loss: `NotFound`, or `ClaimLost` for `activate` and `abandonClaim`. |
107
+ | `AttemptStore.layerNoop()` | Every operation fails with the `unknown` code. |
108
+
109
+ Both take partial overrides, which is how a test forces one outcome without
110
+ building a store:
111
+
112
+ ```ts
113
+ const alwaysFenced = RunStore.layerNoop({
114
+ heartbeat: () => Effect.succeed({ _tag: "FenceLost" })
115
+ })
116
+ ```
117
+
118
+ Use an override to drive the caller's losing branch: a supervision test that
119
+ needs `FenceLost` on the third pulse gets it here, with no second process and no
120
+ clock arithmetic.
121
+
122
+ ## When a real file is the only honest test
123
+
124
+ The in-memory database lives in one process, so it cannot prove anything about
125
+ two processes contending for one run. Cross-process fencing needs a real SQLite
126
+ file and two real Node processes, which is what
127
+ [`test/RunStoreFileFence.integration.test.ts`](https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/run-store/test/RunStoreFileFence.integration.test.ts)
128
+ does: it spawns two children over one temporary database and steps them through a
129
+ takeover from the parent.
130
+
131
+ Write one of those only for a claim about concurrency across processes. Every
132
+ other rule in this package, the fence, the guards, the admission boundary, the
133
+ lease arithmetic, is fully decided inside one process.
134
+
135
+ ## Next steps
136
+
137
+ - [Compose the stores into a host](./compose-the-stores.md): the production
138
+ layer order the test layer stands in for.
139
+ - [Observe store outcomes](./observe-outcomes.md): asserting on counters.
140
+ - [Troubleshooting](../troubleshooting.md): the failures a test is most likely to
141
+ hit first.
@@ -0,0 +1,109 @@
1
+ ---
2
+ title: "Installation"
3
+ description: "Install @smthrs/run-store, the SQL driver and durable writer it needs, its import forms, and the subpaths that are deliberately not public."
4
+ sidebar:
5
+ order: 1
6
+ ---
7
+
8
+ ## Install the package
9
+
10
+ Not on npm yet; see [Installation](https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/flow/docs/installation.md#use-the-libraries).
11
+
12
+ The package requires Node.js 26.4.0 or later and ships as both ESM and
13
+ CommonJS with TypeScript declarations. Its dependencies install with it:
14
+ [`@smthrs/database`](/api/database) for the durable write contract,
15
+ [`@smthrs/journal`](/api/journal) for the `OwnerId` fencing token, and
16
+ [`@smthrs/observability`](/api/observability) for the shared throughput metric.
17
+
18
+ [`effect`](https://effect.website) is a peer dependency pinned at
19
+ `4.0.0-rc.115`, so install exactly that version. Two copies of `effect` in one
20
+ program are two sets of service tags, and a store layer built against one copy
21
+ cannot be provided to a program holding the other: the mismatch surfaces as a
22
+ missing service rather than as a version error.
23
+
24
+ Installed is not the same as importable. A package manager that isolates
25
+ transitive dependencies puts only `@smthrs/run-store` on your resolution path,
26
+ so install any of those packages you import by name as a direct dependency too.
27
+ `OwnerId` is the one you would otherwise reach for first: import it from
28
+ `@smthrs/run-store/Ownership`, which re-exports the journal's token, rather
29
+ than from `@smthrs/journal`.
30
+
31
+ [`@smthrs/flows`](/api/flows) is not a dependency of this package at all. The
32
+ snippets on these pages that type a `NodeRuntime.Options` value need it
33
+ installed alongside.
34
+
35
+ ## What a working composition adds
36
+
37
+ The stores are written against the driver-neutral `@smthrs/database` contract,
38
+ so they carry no database of their own. A composition supplies two services:
39
+
40
+ - `SqlClient.SqlClient` from `effect/unstable/sql/SqlClient`, which the SQL
41
+ driver provides. `@smthrs/database/node/NodeDatabase` opens a `node:sqlite`
42
+ database.
43
+ - `DurableWriter` from `@smthrs/database`, which serializes and retries the
44
+ write transactions the stores run inside.
45
+
46
+ The SQLite and PostgreSQL adapters share this package's migrations and
47
+ statements. Both must satisfy the `DurableWriter` serialization contract.
48
+ Use `PostgresDatabase` for schema-scoped advisory locking; a bare PostgreSQL
49
+ READ COMMITTED client does not serialize writers.
50
+
51
+ Using `NodeDatabase` or the in-memory `TestRunStore` helper selects the optional
52
+ Node SQLite driver:
53
+
54
+ ```bash
55
+ pnpm add @effect/sql-sqlite-node@4.0.0-rc.115
56
+ ```
57
+
58
+ Both `RunStore.layer` and `AttemptStore.layer` require exactly those two, and
59
+ `Migrations.layer` requires the `SqlClient` alone. The layer order that
60
+ satisfies them is in [Compose the stores into a host](./guides/compose-the-stores.md).
61
+
62
+ Most hosts do not wire this by hand. [`@smthrs/engine-store`](/api/engine-store)
63
+ composes these stores with the journal, the step cache, and the durable engine
64
+ state into one storage ladder, and `@smthrs/flows/NodeRuntime` builds that
65
+ ladder over a single SQLite file.
66
+
67
+ ## Import forms
68
+
69
+ The root entry point re-exports every module as a namespace:
70
+
71
+ ```ts
72
+ import { AttemptStore, Migrations, Ownership, RunStore, RunStoreMetrics } from "@smthrs/run-store"
73
+ ```
74
+
75
+ Each module is also importable from its own subpath, which is the form the API
76
+ reference uses:
77
+
78
+ ```ts
79
+ import * as AttemptStore from "@smthrs/run-store/AttemptStore"
80
+ import * as RunStore from "@smthrs/run-store/RunStore"
81
+ ```
82
+
83
+ Both forms reach the same modules. The root is driver-neutral and bundles for
84
+ the browser: nothing under it imports a `node:` built-in.
85
+
86
+ ## The two subpaths that are not namespaces of the root
87
+
88
+ | Import | Platform | What it holds |
89
+ | ------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------ |
90
+ | `@smthrs/run-store/Heartbeat` | any | The four lease durations: `heartbeatInterval`, `heartbeatStaleAfter`, `heartbeatSkewAllowance`, and `heartbeatWriteTolerance`. |
91
+ | `@smthrs/run-store/test/TestRunStore` | Node | `layer`, which provides migrated in-memory `RunStore` and `AttemptStore` services. |
92
+
93
+ `Ownership` re-exports all four durations, so the `Heartbeat` leaf exists for a
94
+ consumer that wants the numbers and no store at all:
95
+
96
+ ```ts
97
+ import * as Heartbeat from "@smthrs/run-store/Heartbeat"
98
+ import * as TestRunStore from "@smthrs/run-store/test/TestRunStore"
99
+ ```
100
+
101
+ `@smthrs/run-store/package.json` is exported. Three subpath families are
102
+ blocked in the export map and are not public API: `internal/*`, `migrations/*`,
103
+ and nested `*/index`. The migration implementations are blocked because the set
104
+ is the contract; import `Migrations.set` rather than a numbered file.
105
+
106
+ ## Next step
107
+
108
+ Run a full lifecycle against an in-memory database in the
109
+ [Quickstart](./quickstart.md).