@smthrs/journal 0.0.0-stage → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/CHANGELOG.md +149 -0
  2. package/LICENSE +21 -0
  3. package/README.md +206 -2
  4. package/dist/cjs/CallFact.d.ts +129 -0
  5. package/dist/cjs/CallFact.d.ts.map +1 -0
  6. package/dist/cjs/CallFact.js +106 -0
  7. package/dist/cjs/CallFact.js.map +7 -0
  8. package/dist/cjs/Consensus.d.ts +440 -0
  9. package/dist/cjs/Consensus.d.ts.map +1 -0
  10. package/dist/cjs/Consensus.js +251 -0
  11. package/dist/cjs/Consensus.js.map +7 -0
  12. package/dist/cjs/EngineEvent.d.ts +890 -0
  13. package/dist/cjs/EngineEvent.d.ts.map +1 -0
  14. package/dist/cjs/EngineEvent.js +383 -0
  15. package/dist/cjs/EngineEvent.js.map +7 -0
  16. package/dist/cjs/ExecutionFact.d.ts +216 -0
  17. package/dist/cjs/ExecutionFact.d.ts.map +1 -0
  18. package/dist/cjs/ExecutionFact.js +303 -0
  19. package/dist/cjs/ExecutionFact.js.map +7 -0
  20. package/dist/cjs/Journal.d.ts +617 -0
  21. package/dist/cjs/Journal.d.ts.map +1 -0
  22. package/dist/cjs/Journal.js +190 -0
  23. package/dist/cjs/Journal.js.map +7 -0
  24. package/dist/cjs/JournalEvent.d.ts +421 -0
  25. package/dist/cjs/JournalEvent.d.ts.map +1 -0
  26. package/dist/cjs/JournalEvent.js +281 -0
  27. package/dist/cjs/JournalEvent.js.map +7 -0
  28. package/dist/cjs/JournalGeneration.d.ts +34 -0
  29. package/dist/cjs/JournalGeneration.d.ts.map +1 -0
  30. package/dist/cjs/JournalGeneration.js +72 -0
  31. package/dist/cjs/JournalGeneration.js.map +7 -0
  32. package/dist/cjs/JournalMetrics.d.ts +72 -0
  33. package/dist/cjs/JournalMetrics.d.ts.map +1 -0
  34. package/dist/cjs/JournalMetrics.js +55 -0
  35. package/dist/cjs/JournalMetrics.js.map +7 -0
  36. package/dist/cjs/Migrations.d.ts +42 -0
  37. package/dist/cjs/Migrations.d.ts.map +1 -0
  38. package/dist/cjs/Migrations.js +58 -0
  39. package/dist/cjs/Migrations.js.map +7 -0
  40. package/dist/cjs/OwnerId.d.ts +40 -0
  41. package/dist/cjs/OwnerId.d.ts.map +1 -0
  42. package/dist/cjs/OwnerId.js +40 -0
  43. package/dist/cjs/OwnerId.js.map +7 -0
  44. package/dist/cjs/Projection.d.ts +29 -0
  45. package/dist/cjs/Projection.d.ts.map +1 -0
  46. package/dist/cjs/Projection.js +25 -0
  47. package/dist/cjs/Projection.js.map +7 -0
  48. package/dist/cjs/RedactedLogger.d.ts +121 -0
  49. package/dist/cjs/RedactedLogger.d.ts.map +1 -0
  50. package/dist/cjs/RedactedLogger.js +235 -0
  51. package/dist/cjs/RedactedLogger.js.map +7 -0
  52. package/dist/cjs/Redaction.d.ts +350 -0
  53. package/dist/cjs/Redaction.d.ts.map +1 -0
  54. package/dist/cjs/Redaction.js +928 -0
  55. package/dist/cjs/Redaction.js.map +7 -0
  56. package/dist/cjs/SqlConsensus.d.ts +44 -0
  57. package/dist/cjs/SqlConsensus.d.ts.map +1 -0
  58. package/dist/cjs/SqlConsensus.js +243 -0
  59. package/dist/cjs/SqlConsensus.js.map +7 -0
  60. package/dist/cjs/SqlJournal.d.ts +173 -0
  61. package/dist/cjs/SqlJournal.d.ts.map +1 -0
  62. package/dist/cjs/SqlJournal.js +1568 -0
  63. package/dist/cjs/SqlJournal.js.map +7 -0
  64. package/dist/cjs/StepFact.d.ts +66 -0
  65. package/dist/cjs/StepFact.d.ts.map +1 -0
  66. package/dist/cjs/StepFact.js +68 -0
  67. package/dist/cjs/StepFact.js.map +7 -0
  68. package/dist/cjs/index.d.ts +95 -0
  69. package/dist/cjs/index.d.ts.map +1 -0
  70. package/dist/cjs/index.js +63 -0
  71. package/dist/cjs/index.js.map +7 -0
  72. package/dist/cjs/internal/JournalInternalFault.d.ts +37 -0
  73. package/dist/cjs/internal/JournalInternalFault.d.ts.map +1 -0
  74. package/dist/cjs/internal/JournalInternalFault.js +43 -0
  75. package/dist/cjs/internal/JournalInternalFault.js.map +7 -0
  76. package/dist/cjs/migrations/0001_initial.d.ts +15 -0
  77. package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
  78. package/dist/cjs/migrations/0001_initial.js +54 -0
  79. package/dist/cjs/migrations/0001_initial.js.map +7 -0
  80. package/dist/cjs/migrations/0002_checkpoints.d.ts +27 -0
  81. package/dist/cjs/migrations/0002_checkpoints.d.ts.map +1 -0
  82. package/dist/cjs/migrations/0002_checkpoints.js +48 -0
  83. package/dist/cjs/migrations/0002_checkpoints.js.map +7 -0
  84. package/dist/cjs/migrations/0003_startup_index.d.ts +15 -0
  85. package/dist/cjs/migrations/0003_startup_index.d.ts.map +1 -0
  86. package/dist/cjs/migrations/0003_startup_index.js +40 -0
  87. package/dist/cjs/migrations/0003_startup_index.js.map +7 -0
  88. package/dist/cjs/migrations/0004_dedup.d.ts +15 -0
  89. package/dist/cjs/migrations/0004_dedup.d.ts.map +1 -0
  90. package/dist/cjs/migrations/0004_dedup.js +57 -0
  91. package/dist/cjs/migrations/0004_dedup.js.map +7 -0
  92. package/dist/cjs/migrations/0005_run_event_type.d.ts +15 -0
  93. package/dist/cjs/migrations/0005_run_event_type.d.ts.map +1 -0
  94. package/dist/cjs/migrations/0005_run_event_type.js +40 -0
  95. package/dist/cjs/migrations/0005_run_event_type.js.map +7 -0
  96. package/dist/cjs/migrations/0006_consensus.d.ts +22 -0
  97. package/dist/cjs/migrations/0006_consensus.d.ts.map +1 -0
  98. package/dist/cjs/migrations/0006_consensus.js +84 -0
  99. package/dist/cjs/migrations/0006_consensus.js.map +7 -0
  100. package/dist/cjs/package.json +1 -0
  101. package/dist/cjs/test/Notifying.d.ts +65 -0
  102. package/dist/cjs/test/Notifying.d.ts.map +1 -0
  103. package/dist/cjs/test/Notifying.js +62 -0
  104. package/dist/cjs/test/Notifying.js.map +7 -0
  105. package/dist/cjs/test/TestJournal.d.ts +43 -0
  106. package/dist/cjs/test/TestJournal.d.ts.map +1 -0
  107. package/dist/cjs/test/TestJournal.js +43 -0
  108. package/dist/cjs/test/TestJournal.js.map +7 -0
  109. package/dist/esm/CallFact.d.ts +129 -0
  110. package/dist/esm/CallFact.d.ts.map +1 -0
  111. package/dist/esm/CallFact.js +96 -0
  112. package/dist/esm/CallFact.js.map +1 -0
  113. package/dist/esm/Consensus.d.ts +440 -0
  114. package/dist/esm/Consensus.d.ts.map +1 -0
  115. package/dist/esm/Consensus.js +454 -0
  116. package/dist/esm/Consensus.js.map +1 -0
  117. package/dist/esm/EngineEvent.d.ts +890 -0
  118. package/dist/esm/EngineEvent.d.ts.map +1 -0
  119. package/dist/esm/EngineEvent.js +614 -0
  120. package/dist/esm/EngineEvent.js.map +1 -0
  121. package/dist/esm/ExecutionFact.d.ts +216 -0
  122. package/dist/esm/ExecutionFact.d.ts.map +1 -0
  123. package/dist/esm/ExecutionFact.js +374 -0
  124. package/dist/esm/ExecutionFact.js.map +1 -0
  125. package/dist/esm/Journal.d.ts +617 -0
  126. package/dist/esm/Journal.d.ts.map +1 -0
  127. package/dist/esm/Journal.js +323 -0
  128. package/dist/esm/Journal.js.map +1 -0
  129. package/dist/esm/JournalEvent.d.ts +421 -0
  130. package/dist/esm/JournalEvent.d.ts.map +1 -0
  131. package/dist/esm/JournalEvent.js +517 -0
  132. package/dist/esm/JournalEvent.js.map +1 -0
  133. package/dist/esm/JournalGeneration.d.ts +34 -0
  134. package/dist/esm/JournalGeneration.d.ts.map +1 -0
  135. package/dist/esm/JournalGeneration.js +65 -0
  136. package/dist/esm/JournalGeneration.js.map +1 -0
  137. package/dist/esm/JournalMetrics.d.ts +72 -0
  138. package/dist/esm/JournalMetrics.d.ts.map +1 -0
  139. package/dist/esm/JournalMetrics.js +79 -0
  140. package/dist/esm/JournalMetrics.js.map +1 -0
  141. package/dist/esm/Migrations.d.ts +42 -0
  142. package/dist/esm/Migrations.d.ts.map +1 -0
  143. package/dist/esm/Migrations.js +59 -0
  144. package/dist/esm/Migrations.js.map +1 -0
  145. package/dist/esm/OwnerId.d.ts +40 -0
  146. package/dist/esm/OwnerId.d.ts.map +1 -0
  147. package/dist/esm/OwnerId.js +33 -0
  148. package/dist/esm/OwnerId.js.map +1 -0
  149. package/dist/esm/Projection.d.ts +29 -0
  150. package/dist/esm/Projection.d.ts.map +1 -0
  151. package/dist/esm/Projection.js +13 -0
  152. package/dist/esm/Projection.js.map +1 -0
  153. package/dist/esm/RedactedLogger.d.ts +121 -0
  154. package/dist/esm/RedactedLogger.d.ts.map +1 -0
  155. package/dist/esm/RedactedLogger.js +445 -0
  156. package/dist/esm/RedactedLogger.js.map +1 -0
  157. package/dist/esm/Redaction.d.ts +350 -0
  158. package/dist/esm/Redaction.d.ts.map +1 -0
  159. package/dist/esm/Redaction.js +1584 -0
  160. package/dist/esm/Redaction.js.map +1 -0
  161. package/dist/esm/SqlConsensus.d.ts +44 -0
  162. package/dist/esm/SqlConsensus.d.ts.map +1 -0
  163. package/dist/esm/SqlConsensus.js +252 -0
  164. package/dist/esm/SqlConsensus.js.map +1 -0
  165. package/dist/esm/SqlJournal.d.ts +173 -0
  166. package/dist/esm/SqlJournal.d.ts.map +1 -0
  167. package/dist/esm/SqlJournal.js +1858 -0
  168. package/dist/esm/SqlJournal.js.map +1 -0
  169. package/dist/esm/StepFact.d.ts +66 -0
  170. package/dist/esm/StepFact.d.ts.map +1 -0
  171. package/dist/esm/StepFact.js +50 -0
  172. package/dist/esm/StepFact.js.map +1 -0
  173. package/dist/esm/index.d.ts +95 -0
  174. package/dist/esm/index.d.ts.map +1 -0
  175. package/dist/esm/index.js +95 -0
  176. package/dist/esm/index.js.map +1 -0
  177. package/dist/esm/internal/JournalInternalFault.d.ts +37 -0
  178. package/dist/esm/internal/JournalInternalFault.d.ts.map +1 -0
  179. package/dist/esm/internal/JournalInternalFault.js +28 -0
  180. package/dist/esm/internal/JournalInternalFault.js.map +1 -0
  181. package/dist/esm/migrations/0001_initial.d.ts +15 -0
  182. package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
  183. package/dist/esm/migrations/0001_initial.js +32 -0
  184. package/dist/esm/migrations/0001_initial.js.map +1 -0
  185. package/dist/esm/migrations/0002_checkpoints.d.ts +27 -0
  186. package/dist/esm/migrations/0002_checkpoints.d.ts.map +1 -0
  187. package/dist/esm/migrations/0002_checkpoints.js +38 -0
  188. package/dist/esm/migrations/0002_checkpoints.js.map +1 -0
  189. package/dist/esm/migrations/0003_startup_index.d.ts +15 -0
  190. package/dist/esm/migrations/0003_startup_index.d.ts.map +1 -0
  191. package/dist/esm/migrations/0003_startup_index.js +18 -0
  192. package/dist/esm/migrations/0003_startup_index.js.map +1 -0
  193. package/dist/esm/migrations/0004_dedup.d.ts +15 -0
  194. package/dist/esm/migrations/0004_dedup.d.ts.map +1 -0
  195. package/dist/esm/migrations/0004_dedup.js +38 -0
  196. package/dist/esm/migrations/0004_dedup.js.map +1 -0
  197. package/dist/esm/migrations/0005_run_event_type.d.ts +15 -0
  198. package/dist/esm/migrations/0005_run_event_type.d.ts.map +1 -0
  199. package/dist/esm/migrations/0005_run_event_type.js +18 -0
  200. package/dist/esm/migrations/0005_run_event_type.js.map +1 -0
  201. package/dist/esm/migrations/0006_consensus.d.ts +22 -0
  202. package/dist/esm/migrations/0006_consensus.d.ts.map +1 -0
  203. package/dist/esm/migrations/0006_consensus.js +69 -0
  204. package/dist/esm/migrations/0006_consensus.js.map +1 -0
  205. package/dist/esm/test/Notifying.d.ts +65 -0
  206. package/dist/esm/test/Notifying.d.ts.map +1 -0
  207. package/dist/esm/test/Notifying.js +62 -0
  208. package/dist/esm/test/Notifying.js.map +1 -0
  209. package/dist/esm/test/TestJournal.d.ts +43 -0
  210. package/dist/esm/test/TestJournal.d.ts.map +1 -0
  211. package/dist/esm/test/TestJournal.js +27 -0
  212. package/dist/esm/test/TestJournal.js.map +1 -0
  213. package/docs/README.md +164 -0
  214. package/docs/api.md +639 -0
  215. package/docs/concepts/compaction.md +157 -0
  216. package/docs/concepts/idempotency.md +127 -0
  217. package/docs/concepts/owner-fence.md +124 -0
  218. package/docs/concepts/redaction.md +218 -0
  219. package/docs/concepts/state-event-authority.md +159 -0
  220. package/docs/concepts/two-channels.md +167 -0
  221. package/docs/guides/commit-state-and-entry.md +114 -0
  222. package/docs/guides/compact-a-run.md +134 -0
  223. package/docs/guides/fold-a-projection.md +106 -0
  224. package/docs/guides/read-a-run.md +139 -0
  225. package/docs/guides/redact-log-output.md +94 -0
  226. package/docs/guides/testing.md +124 -0
  227. package/docs/guides/write-lifecycle-events.md +110 -0
  228. package/docs/installation.md +133 -0
  229. package/docs/quickstart.md +143 -0
  230. package/docs/troubleshooting.md +214 -0
  231. package/package.json +263 -3
  232. package/src/CallFact.ts +117 -0
  233. package/src/Consensus.ts +683 -0
  234. package/src/EngineEvent.ts +823 -0
  235. package/src/ExecutionFact.ts +433 -0
  236. package/src/Journal.ts +688 -0
  237. package/src/JournalEvent.ts +654 -0
  238. package/src/JournalGeneration.ts +77 -0
  239. package/src/JournalMetrics.ts +89 -0
  240. package/src/Migrations.ts +62 -0
  241. package/src/OwnerId.ts +42 -0
  242. package/src/Projection.ts +31 -0
  243. package/src/RedactedLogger.ts +483 -0
  244. package/src/Redaction.ts +1784 -0
  245. package/src/SqlConsensus.ts +344 -0
  246. package/src/SqlJournal.ts +2732 -0
  247. package/src/StepFact.ts +69 -0
  248. package/src/index.ts +109 -0
  249. package/src/internal/JournalInternalFault.ts +40 -0
  250. package/src/migrations/0001_initial.ts +41 -0
  251. package/src/migrations/0002_checkpoints.ts +46 -0
  252. package/src/migrations/0003_startup_index.ts +19 -0
  253. package/src/migrations/0004_dedup.ts +43 -0
  254. package/src/migrations/0005_run_event_type.ts +19 -0
  255. package/src/migrations/0006_consensus.ts +83 -0
  256. package/src/test/Notifying.ts +101 -0
  257. package/src/test/TestJournal.ts +52 -0
@@ -0,0 +1,2732 @@
1
+ /**
2
+ * SQLite-backed logical journal with durable and lossy channels.
3
+ *
4
+ * Lossy telemetry events remain optimistic until the single scoped writer
5
+ * commits them, so a process crash can lose accepted-but-unwritten telemetry.
6
+ * Lifecycle events use `emitDurable` and return after commit, except inside
7
+ * `transact`, where their savepoint completes before the outer commit.
8
+ *
9
+ * Governing design: `packages/smithers/flows/journal/docs/concepts/two-channels.md`.
10
+ * Prior-art decision: `packages/smithers/flows/sync/docs/concepts/replay-then-follow.md`.
11
+ *
12
+ * The replay-then-follow stream follows Effect's `EventJournal` and OpenCode's
13
+ * upstream event stream design. The bounded send queue deliberately deviates
14
+ * from their synchronous durable writes by reserving a provisional per-run
15
+ * sequence before admission. Both channels finalize canonical order at commit.
16
+ * SQLite retry and transaction behavior comes through `@smthrs/database`.
17
+ *
18
+ * @since 0.1.0
19
+ */
20
+
21
+ import * as Dialect from "@smthrs/database/Dialect"
22
+ import { afterCommit, DatabaseError, DurableWriter } from "@smthrs/database/DurableWriter"
23
+ import * as Cause from "effect/Cause"
24
+ import * as Clock from "effect/Clock"
25
+ import * as Deferred from "effect/Deferred"
26
+ import * as Effect from "effect/Effect"
27
+ import * as Exit from "effect/Exit"
28
+ import * as Layer from "effect/Layer"
29
+ import * as Metric from "effect/Metric"
30
+ import * as Option from "effect/Option"
31
+ import * as PubSub from "effect/PubSub"
32
+ import * as Queue from "effect/Queue"
33
+ import * as Result from "effect/Result"
34
+ import * as Schema from "effect/Schema"
35
+ import * as Scope from "effect/Scope"
36
+ import * as Semaphore from "effect/Semaphore"
37
+ import * as Stream from "effect/Stream"
38
+ import * as SqlClient from "effect/unstable/sql/SqlClient"
39
+ import type * as SqlError from "effect/unstable/sql/SqlError"
40
+ import { Consensus } from "./Consensus.ts"
41
+ import { JournalInternalFault } from "./internal/JournalInternalFault.ts"
42
+ import {
43
+ Checkpoint,
44
+ CheckpointOptions,
45
+ type Compacted,
46
+ CompactOptions,
47
+ type DurableReceipt,
48
+ type EmitReceipt,
49
+ EntriesOptions,
50
+ type EntriesPage,
51
+ Journal,
52
+ JournalError,
53
+ make as makeJournal,
54
+ maxEntriesLimit,
55
+ type OverflowPolicy,
56
+ type Service,
57
+ StreamOptions
58
+ } from "./Journal.ts"
59
+ import {
60
+ type Dedupe,
61
+ Entry,
62
+ Input,
63
+ makeEventId,
64
+ RunId,
65
+ type Seq,
66
+ type SourceId,
67
+ type SourceSeq
68
+ } from "./JournalEvent.ts"
69
+ import * as JournalGeneration from "./JournalGeneration.ts"
70
+ import * as JournalMetrics from "./JournalMetrics.ts"
71
+ import { OwnerId } from "./OwnerId.ts"
72
+ import type { Projection } from "./Projection.ts"
73
+ import * as Redaction from "./Redaction.ts"
74
+ import * as SqlConsensus from "./SqlConsensus.ts"
75
+
76
+ /** JSON text carrying an arbitrary decoded value. */
77
+ const UnknownFromJsonString = Schema.fromJsonString(Schema.Unknown)
78
+
79
+ /**
80
+ * Automatic checkpoint-and-compact policy for the SQL journal.
81
+ *
82
+ * Disabled unless set on {@link SqlJournalOptions}. Once a run's committed
83
+ * entry count reaches `entryThreshold`, the journal asks `capture` for the
84
+ * caller's replay state at the run's durable tail, writes it as a checkpoint
85
+ * at that sequence, and compacts the entries strictly below it.
86
+ *
87
+ * Both channels run the policy after their allocation permit is free. Lossy
88
+ * commits fork scoped maintenance so the sole queue consumer keeps draining;
89
+ * durable emits await their attempt. `flush` awaits registered maintenance.
90
+ * `capture` may read or emit without blocking unrelated allocation. A capture is interrupted
91
+ * after 30 seconds so caller code cannot wedge journal admission indefinitely.
92
+ *
93
+ * A failed or refused attempt (a live stream behind the boundary, a capture
94
+ * failure) is logged at warning, damped for `entryThreshold`
95
+ * further committed entries, and never surfaced to the emit that triggered
96
+ * it.
97
+ *
98
+ * @category models
99
+ * @since 0.1.0
100
+ */
101
+ export interface CompactionPolicy {
102
+ readonly entryThreshold: number
103
+ readonly capture: (runId: RunId, upTo: Seq) => Effect.Effect<unknown, unknown>
104
+ }
105
+
106
+ /**
107
+ * SQL journal queue and batching options.
108
+ *
109
+ * @category models
110
+ * @since 0.1.0
111
+ */
112
+ export interface SqlJournalOptions {
113
+ /**
114
+ * Bound on the number of ENTRIES held in two places at once: the lossy
115
+ * admission queue, where `overflow` decides what happens when it is full,
116
+ * and the sliding `changes` buffer, where a slow subscriber silently loses
117
+ * the entries that fall out of it.
118
+ *
119
+ * It bounds entries, not bytes. Set {@link SqlJournalOptions.maxEntryBytes}
120
+ * to bound the size of one entry as well; with it unset a handful of very
121
+ * large payloads is still the memory bill.
122
+ */
123
+ readonly capacity: number
124
+ /** Policy applied when the lossy admission queue is full. */
125
+ readonly overflow: OverflowPolicy
126
+ /** Entries the queued writer commits per transaction. */
127
+ readonly batchSize?: number | undefined
128
+ /**
129
+ * Upper bound on the in-process source-event index, the map that answers
130
+ * producer idempotency from memory.
131
+ *
132
+ * The index is a cache, never the authority, and a MISS costs a receipt, not
133
+ * correctness. On a miss for an explicit producer sequence the entry is
134
+ * admitted optimistically without any read: `emitLossy` returns `Accepted`
135
+ * where a resident entry would have returned `Duplicate`, and the insert
136
+ * collapses onto the committed row through the unique index
137
+ * `(run_id, source_id, source_seq)` rather than doubling it. A CHANGED retry
138
+ * behind an evicted entry therefore surfaces `idempotency_conflict` from
139
+ * `flush`, not from the emit. See `admitFromIndex` for why the read is gone:
140
+ * `emitLossy` is called from inside other write transactions, and the
141
+ * pre-admission SELECT deadlocked `@smthrs/agent`'s exit flush against the
142
+ * engine's own writer, measured at over 120 s versus about half a second
143
+ * without it.
144
+ *
145
+ * Bounding the cache keeps startup decode and resident memory O(bound) rather
146
+ * than O(total events ever written), mirroring Temporal's refusal to hold
147
+ * unbounded history in a shard (`service/history`). The separate
148
+ * sequence-floor maps start empty and grow only with runs and producers this
149
+ * layer instance touches; they are lazy, not governed by this bound.
150
+ *
151
+ * Idempotency compares canonical persisted JSON after redaction. Two inputs
152
+ * that redact to the same value are therefore the same event, and JSON's
153
+ * encoding intentionally treats `NaN` and `null` as the same `null` value.
154
+ */
155
+ readonly sourceEventCache?: number | undefined
156
+ /**
157
+ * Largest single entry the journal admits, measured in UTF-8 bytes of the
158
+ * encoded `payload` plus `meta`.
159
+ *
160
+ * Unset by default, which means unbounded: `capacity` bounds how MANY entries
161
+ * the admission queue and the `changes` buffer hold, never how large one is,
162
+ * so without this a handful of multi-megabyte payloads is the memory bill,
163
+ * and every one of them is replayed to each sync subscriber and time-travel
164
+ * consumer on every read. Set it where the journal is fed by untrusted or
165
+ * unbounded producers.
166
+ *
167
+ * The bound is checked after encoding and BEFORE any sequence is allocated or
168
+ * anything is queued, so a refused entry costs no sequence and leaves no gap.
169
+ * An entry over the bound fails `invalid_event`.
170
+ */
171
+ readonly maxEntryBytes?: number | undefined
172
+ /**
173
+ * Scrub applied to every `payload` and `meta` before it is encoded for
174
+ * persistence.
175
+ *
176
+ * Journal rows are permanent and are replayed verbatim to sync subscribers
177
+ * and time-travel consumers, so a credential that reaches `payload_json` is
178
+ * a durable, broadly readable leak. Redaction therefore defaults to
179
+ * `Redaction.make()`; pass `Redaction.makeNoop()` to persist payloads
180
+ * verbatim by choice.
181
+ */
182
+ readonly redact?: Redaction.Redactor | undefined
183
+ /**
184
+ * Automatic checkpoint-and-compact policy. Off by default: without it the
185
+ * journal never deletes an entry, and checkpointing stays a caller-driven
186
+ * `checkpoint` / `compact` call.
187
+ */
188
+ readonly compaction?: CompactionPolicy | undefined
189
+ }
190
+
191
+ /** Default retained window of the source-event index. */
192
+ const defaultSourceEventCache = 4096
193
+
194
+ /** Bounds caller-supplied compaction capture work before the attempt is damped. */
195
+ const compactionCaptureTimeout = "30 seconds"
196
+
197
+ interface QueuedEntry {
198
+ readonly runId: RunId
199
+ readonly seq: Seq
200
+ readonly eventId: string
201
+ readonly sourceId: SourceId
202
+ readonly sourceSeq: SourceSeq
203
+ readonly emittedAtMs: number
204
+ readonly eventType: string
205
+ readonly payloadJson: string
206
+ readonly metaJson: string
207
+ /** The producer's reading of a collision on this entry's identity. */
208
+ readonly dedupe: Dedupe
209
+ }
210
+
211
+ interface JournalRow {
212
+ readonly run_id: string
213
+ readonly seq: number
214
+ readonly event_id: string
215
+ readonly source_id: string
216
+ readonly source_seq: number
217
+ readonly emitted_at_ms: number
218
+ readonly event_type: string
219
+ readonly payload_json: string
220
+ readonly meta_json: string
221
+ }
222
+
223
+ interface CheckpointRow {
224
+ readonly run_id: string
225
+ readonly seq: number
226
+ readonly state_json: string
227
+ readonly created_at_ms: number
228
+ readonly compacted_at_ms: number | null
229
+ }
230
+
231
+ interface Prepared {
232
+ readonly validated: Input
233
+ readonly payloadJson: string
234
+ readonly metaJson: string
235
+ }
236
+
237
+ type Commit =
238
+ | { readonly entry: Entry; readonly inserted: true }
239
+ | { readonly entry: Pick<Entry, "seq">; readonly inserted: false }
240
+
241
+ interface SettledCommit {
242
+ readonly queued: QueuedEntry
243
+ readonly commit: Commit
244
+ }
245
+
246
+ interface EntryLoss {
247
+ readonly queued: QueuedEntry
248
+ readonly cause: JournalError
249
+ }
250
+
251
+ interface BatchOutcome {
252
+ readonly commits: ReadonlyArray<SettledCommit>
253
+ readonly losses: ReadonlyArray<EntryLoss>
254
+ }
255
+
256
+ interface RunBarrier {
257
+ /**
258
+ * Serializes one run's admission decisions against the moment a compaction
259
+ * closes the run, so "read the gate, then admit" is one critical section.
260
+ *
261
+ * Never held across a wait on the database outside a transaction the holder
262
+ * already owns; `admitFromIndex` in `SqlJournal.layer` states the lock order
263
+ * between this permit and the writer transaction.
264
+ */
265
+ readonly semaphore: Semaphore.Semaphore
266
+ /**
267
+ * Serializes compactions of one run inside their SQL transactions. Never
268
+ * held during the drain or while requesting a database connection.
269
+ */
270
+ readonly compactionLock: Semaphore.Semaphore
271
+ /**
272
+ * Serializes the compaction policy's per-run bookkeeping.
273
+ *
274
+ * A third lock rather than a reuse of either one above: the bookkeeping ends
275
+ * by running a compaction, and a compaction takes `compactionLock` and
276
+ * `semaphore` through `withCompactionBarrier`, so counting under either of
277
+ * those would deadlock against the work it schedules. No admission path takes
278
+ * this one.
279
+ */
280
+ readonly maintenance: Semaphore.Semaphore
281
+ /** Holders and waiters that can still use this exact barrier. */
282
+ users: number
283
+ /** Open while a compaction owns the run; every admission awaits it. */
284
+ compaction: Deferred.Deferred<void> | undefined
285
+ }
286
+
287
+ type RunAdmission<A> =
288
+ | { readonly _tag: "Done"; readonly value: A }
289
+ | { readonly _tag: "Wait"; readonly gate: Deferred.Deferred<void> }
290
+
291
+ interface SourceEvent {
292
+ readonly seq: Seq
293
+ readonly eventType: string
294
+ readonly payloadJson: string
295
+ readonly metaJson: string
296
+ readonly status: "pending" | "committed"
297
+ }
298
+
299
+ type Status = "open" | "closing" | "closed"
300
+
301
+ interface State {
302
+ status: Status
303
+ /**
304
+ * The most recent batch the optimistic writer lost. It is a *report*, not a
305
+ * latch: the writer survives a failed batch, so a transient outage must not
306
+ * revoke the lossy channel, and with it the durable delivery paths that call
307
+ * `flush`, for the rest of the process's life.
308
+ *
309
+ * Each loss is reported to whoever was waiting on it (`flushWaiters`, live
310
+ * streams) and, via `lossEpoch`, to at most one later `flush` and to every
311
+ * live stream that had not yet observed it. After that the report is spent
312
+ * and the journal is usable again the moment the database is.
313
+ */
314
+ sinkFailure: JournalError | undefined
315
+ /** Incremented on every lost batch; identifies an unreported loss. */
316
+ lossEpoch: number
317
+ /** The highest `lossEpoch` already reported to a `flush` caller. */
318
+ flushedLossEpoch: number
319
+ pending: number
320
+ readonly pendingByRun: Map<RunId, Set<QueuedEntry>>
321
+ readonly sequences: Map<RunId, number>
322
+ readonly sourceSequences: Map<string, number>
323
+ readonly sourceEvents: Map<string, SourceEvent>
324
+ readonly flushWaiters: Set<Deferred.Deferred<void, JournalError>>
325
+ }
326
+
327
+ const error = (code: JournalError["code"], message: string, cause?: unknown): JournalError =>
328
+ new JournalError({
329
+ code,
330
+ message,
331
+ ...(cause === undefined ? {} : { cause })
332
+ })
333
+
334
+ const sourceKey = (runId: RunId, sourceId: SourceId): string => `${runId.length}:${runId}${sourceId.length}:${sourceId}`
335
+
336
+ const sourceEventKey = (runId: RunId, sourceId: SourceId, sourceSeq: SourceSeq): string =>
337
+ `${sourceKey(runId, sourceId)}:${sourceSeq}`
338
+
339
+ /**
340
+ * Sorts a PARSED JSON tree so two encodings of one value compare equal.
341
+ *
342
+ * The argument is always `JSON.parse` output. That precondition is the whole
343
+ * safety argument: parse output has no prototype `toJSON`, no cycle, and no
344
+ * leaf JSON cannot represent, so this walk needs none of the handling
345
+ * {@link Redaction.redact} carries. Running it over a raw value instead is
346
+ * what broke: it rebuilt objects from `Object.keys`, so `{ at: new Date(...) }`
347
+ * persisted as `{"at":{}}` under `Redaction.makeNoop()` while the default
348
+ * redactor persisted the ISO string, and a redactor returning a cycle
349
+ * recursed until the stack gave out and killed the emit with a defect instead
350
+ * of a `JournalError`.
351
+ *
352
+ * `@smthrs/canonical` is the repository's general-purpose implementation, but
353
+ * journal cannot add that dependency while this release's lockfiles are frozen.
354
+ */
355
+ const canonicalizeFingerprint = (value: unknown, depth: number): unknown => {
356
+ if (depth > Redaction.maxDepth) {
357
+ throw new RangeError(`journal value nests deeper than ${Redaction.maxDepth} containers`)
358
+ }
359
+ if (Array.isArray(value)) return value.map((element) => canonicalizeFingerprint(element, depth + 1))
360
+ if (value === null || typeof value !== "object") return value
361
+ const record = value as Record<string, unknown>
362
+ return Object.fromEntries(
363
+ Object.keys(record).sort().map((key) => [key, canonicalizeFingerprint(record[key], depth + 1)])
364
+ )
365
+ }
366
+
367
+ /**
368
+ * Encodes a value as JSON, reporting anything the encoder refuses as a typed
369
+ * failure.
370
+ *
371
+ * The encoder's failure channel covers every value it declines, a `BigInt` and
372
+ * a cycle included, so nothing here has to catch a throw. What used to reach
373
+ * the emit as a `Die` came from the canonicalizer running BEFORE this, over a
374
+ * raw value it was never given the guards for; {@link encodeFingerprint} runs
375
+ * it after.
376
+ */
377
+ const encodeJson = (value: unknown, field: string): Result.Result<string, JournalError> =>
378
+ Schema.encodeUnknownResult(UnknownFromJsonString)(value).pipe(
379
+ Result.mapError((cause) => error("invalid_event", `${field} must be JSON-serializable`, cause))
380
+ )
381
+
382
+ /**
383
+ * The persisted bytes of one field: JSON, with object members in a stable key
384
+ * order.
385
+ *
386
+ * Encoding happens FIRST and canonicalization runs over the parsed result, so
387
+ * the bytes keep exactly the `JSON.stringify` semantics they had before
388
+ * idempotency became key-order independent: a `Date` is its ISO string, an
389
+ * `undefined` member is dropped, `NaN` is `null`. Canonicalizing the raw value
390
+ * instead silently destroyed every one of those.
391
+ *
392
+ * Nesting is bounded at `Redaction.maxDepth`, the same ceiling the default
393
+ * redactor enforces one step earlier, so a value that reaches this walk through
394
+ * `Redaction.makeNoop()` is refused as `invalid_event` rather than exhausting
395
+ * the stack as a defect.
396
+ */
397
+ const encodeFingerprint = (value: unknown, field: string): Result.Result<string, JournalError> =>
398
+ Result.flatMap(encodeJson(value, field), (json) =>
399
+ Result.try({
400
+ try: () => JSON.stringify(canonicalizeFingerprint(JSON.parse(json), 0)),
401
+ catch: (cause) => error("invalid_event", `${field} could not be canonicalized`, cause)
402
+ }))
403
+
404
+ /**
405
+ * Whether one write is fenced on a run's persisted ownership.
406
+ *
407
+ * The journal used to pass `OwnerId | undefined` and fence only when the value
408
+ * happened to be present, so an untyped caller that omitted the argument
409
+ * silently selected the unfenced path on three public methods. The tag makes
410
+ * the choice explicit: only `emitDurableUnfenced` and the internal
411
+ * auto-compaction path construct `unfenced`, and they say so at the call site.
412
+ */
413
+ type Fence =
414
+ | { readonly _tag: "Owned"; readonly owner: OwnerId }
415
+ | { readonly _tag: "Unfenced" }
416
+
417
+ /** The one unfenced write posture, named so a call site cannot mean it by accident. */
418
+ const unfenced: Fence = { _tag: "Unfenced" }
419
+
420
+ const decodeOwner = Schema.decodeUnknownResult(OwnerId)
421
+
422
+ /**
423
+ * Decodes the mandatory owner of a fenced method.
424
+ *
425
+ * A missing, null, or malformed owner is a caller contract violation, not a
426
+ * lost race: reporting it as `fence_lost` would tell the caller another
427
+ * process owns the run and send it looking for a conflict that never
428
+ * happened.
429
+ */
430
+ const requireFence = (owner: unknown, method: string): Result.Result<Fence, JournalError> =>
431
+ Result.match(decodeOwner(owner), {
432
+ onFailure: (cause) => Result.fail(error("invalid_event", `${method} requires a well-formed owner fence`, cause)),
433
+ onSuccess: (value) => Result.succeed<Fence>({ _tag: "Owned", owner: value })
434
+ })
435
+
436
+ const decodeInput = Schema.decodeUnknownResult(Input)
437
+ const decodeEntry = Schema.decodeUnknownEffect(Entry)
438
+ /**
439
+ * The read and maintenance boundaries decode the schemas they publish.
440
+ *
441
+ * `EntriesOptions`, `StreamOptions`, `CheckpointOptions`, `CompactOptions` and
442
+ * `RunId` already carry every invariant these methods need, and hand-checking a
443
+ * subset of them here let the read side disagree with the write side about what
444
+ * an identifier is.
445
+ */
446
+ const decodeEntriesOptions = Schema.decodeUnknownResult(EntriesOptions)
447
+ const decodeStreamOptions = Schema.decodeUnknownResult(StreamOptions)
448
+ const decodeCheckpointOptions = Schema.decodeUnknownResult(CheckpointOptions)
449
+ const decodeCompactOptions = Schema.decodeUnknownResult(CompactOptions)
450
+ const decodeRunId = Schema.decodeUnknownResult(RunId)
451
+
452
+ const decodeRow = (row: JournalRow): Effect.Effect<Entry, JournalError> =>
453
+ Effect.all({
454
+ payload: Schema.decodeUnknownEffect(UnknownFromJsonString)(row.payload_json),
455
+ meta: Schema.decodeUnknownEffect(UnknownFromJsonString)(row.meta_json)
456
+ }).pipe(
457
+ Effect.map(({ meta, payload }) => ({
458
+ runId: row.run_id as RunId,
459
+ seq: Number(row.seq),
460
+ eventId: row.event_id,
461
+ sourceId: row.source_id,
462
+ sourceSeq: Number(row.source_seq),
463
+ emittedAtMs: Number(row.emitted_at_ms),
464
+ eventType: row.event_type,
465
+ payload,
466
+ meta
467
+ })),
468
+ Effect.flatMap(decodeEntry),
469
+ Effect.mapError((cause) => error("decode_failed", "could not decode a durable journal row", cause))
470
+ )
471
+
472
+ const decodeCheckpoint = Schema.decodeUnknownEffect(Checkpoint)
473
+
474
+ const decodeCheckpointRow = (row: CheckpointRow): Effect.Effect<Checkpoint, JournalError> =>
475
+ Schema.decodeUnknownEffect(UnknownFromJsonString)(row.state_json).pipe(
476
+ Effect.flatMap((state) =>
477
+ decodeCheckpoint({
478
+ runId: row.run_id,
479
+ seq: Number(row.seq),
480
+ state,
481
+ createdAtMs: Number(row.created_at_ms),
482
+ compactedAtMs: row.compacted_at_ms === null ? null : Number(row.compacted_at_ms)
483
+ })
484
+ ),
485
+ Effect.mapError((cause) => error("decode_failed", "could not decode a durable checkpoint row", cause))
486
+ )
487
+
488
+ /** Freezes the one object graph shared by every changes subscriber. */
489
+ const freezePublished = <A>(value: A): A => {
490
+ const seen = new WeakSet<object>()
491
+ const walk = (node: unknown): void => {
492
+ if (node === null || typeof node !== "object") return
493
+ /* v8 ignore next -- decoded JSON and Entry instances are acyclic, but this guard keeps the walk total */
494
+ if (seen.has(node)) return
495
+ seen.add(node)
496
+ for (const key of Reflect.ownKeys(node)) {
497
+ walk((node as Record<PropertyKey, unknown>)[key])
498
+ }
499
+ Object.freeze(node)
500
+ }
501
+ walk(value)
502
+ return value
503
+ }
504
+
505
+ interface ValidatedOptions {
506
+ readonly batchSize: number
507
+ readonly sourceEventCache: number
508
+ readonly maxEntryBytes: number | undefined
509
+ readonly redact: Redaction.Redactor
510
+ }
511
+
512
+ /**
513
+ * UTF-8 size of the bytes an entry persists.
514
+ *
515
+ * `TextEncoder` rather than `Buffer.byteLength` because this module has no
516
+ * other `node:` import and the journal's browser-facing consumers bundle it.
517
+ */
518
+ const encodedBytes = (encoder: TextEncoder, payloadJson: string, metaJson: string): number =>
519
+ encoder.encode(payloadJson).length + encoder.encode(metaJson).length
520
+
521
+ const validateOptions = (options: SqlJournalOptions): Effect.Effect<ValidatedOptions, JournalError> =>
522
+ Effect.suspend(() => {
523
+ if (!Number.isSafeInteger(options.capacity) || options.capacity <= 0) {
524
+ return Effect.fail(error("invalid_event", "capacity must be a positive safe integer"))
525
+ }
526
+ const batchSize = options.batchSize ?? Math.min(options.capacity, 64)
527
+ if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {
528
+ return Effect.fail(error("invalid_event", "batchSize must be a positive safe integer"))
529
+ }
530
+ const sourceEventCache = options.sourceEventCache ?? defaultSourceEventCache
531
+ if (!Number.isSafeInteger(sourceEventCache) || sourceEventCache <= 0) {
532
+ return Effect.fail(error("invalid_event", "sourceEventCache must be a positive safe integer"))
533
+ }
534
+ if (
535
+ options.maxEntryBytes !== undefined &&
536
+ (!Number.isSafeInteger(options.maxEntryBytes) || options.maxEntryBytes <= 0)
537
+ ) {
538
+ return Effect.fail(error("invalid_event", "maxEntryBytes must be a positive safe integer"))
539
+ }
540
+ if (
541
+ options.compaction !== undefined &&
542
+ (!Number.isSafeInteger(options.compaction.entryThreshold) || options.compaction.entryThreshold <= 0)
543
+ ) {
544
+ return Effect.fail(error("invalid_event", "compaction.entryThreshold must be a positive safe integer"))
545
+ }
546
+ return Effect.succeed({
547
+ batchSize,
548
+ sourceEventCache,
549
+ maxEntryBytes: options.maxEntryBytes,
550
+ redact: options.redact ?? Redaction.make()
551
+ })
552
+ })
553
+
554
+ const isJournalError = Schema.is(JournalError)
555
+
556
+ /**
557
+ * Constructs the SQL journal over the context's SQL client, durable writer,
558
+ * and consensus strategy.
559
+ *
560
+ * `emitLossy` validates and admits telemetry to the non-blocking queue;
561
+ * `emitDurable` allocates and commits inside the database transaction, fenced
562
+ * through `Consensus.guard` in that same transaction.
563
+ *
564
+ * @category constructors
565
+ * @since 1.0.0
566
+ */
567
+ export const make = (
568
+ options: SqlJournalOptions
569
+ ): Effect.Effect<Service, JournalError, Consensus | DurableWriter | SqlClient.SqlClient | Scope.Scope> =>
570
+ Effect.gen(function*() {
571
+ const { batchSize, maxEntryBytes, redact, sourceEventCache } = yield* validateOptions(options)
572
+ // `batchSize` sizes the writer's transactions. `stream` pages the durable
573
+ // tail through `entries`, whose `limit` is bounded by `maxEntriesLimit`,
574
+ // so a larger batch must not leak into the read boundary: a layer that
575
+ // was accepted with `batchSize: 16384` streamed nothing but
576
+ // `invalid_event` before this clamp.
577
+ const readPageSize = Math.min(batchSize, maxEntriesLimit)
578
+ const sql = yield* Effect.service(SqlClient.SqlClient)
579
+ const writer = yield* DurableWriter
580
+ const consensus = yield* Consensus
581
+ yield* JournalGeneration.initialize.pipe(
582
+ Effect.mapError((cause) => error("read_failed", "could not initialize journal generations", cause))
583
+ )
584
+ /** One encoder for the layer: constructing one per emit measured slower. */
585
+ const byteCounter = new TextEncoder()
586
+
587
+ // Hash the same encoded content the ordinary dedup check compares.
588
+ // A JSON tuple keeps boundaries unambiguous without retaining payloads.
589
+ const contentFingerprint = (eventType: string, payloadJson: string, metaJson: string) =>
590
+ Effect.tryPromise({
591
+ try: () =>
592
+ crypto.subtle.digest("SHA-256", byteCounter.encode(JSON.stringify([eventType, payloadJson, metaJson]))),
593
+ catch: (cause) => error("sink_failed", "could not fingerprint journal content", cause)
594
+ }).pipe(
595
+ Effect.map((digest) =>
596
+ Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("")
597
+ )
598
+ )
599
+
600
+ const queue = yield* Queue.dropping<QueuedEntry>(options.capacity)
601
+ const changes = yield* PubSub.sliding<Entry>(options.capacity)
602
+ const wakes = new Map<RunId, Set<PubSub.PubSub<void>>>()
603
+ /**
604
+ * The durable cursor of every live in-process stream, per run: the
605
+ * highest committed sequence the stream has read from the store, or its
606
+ * starting `afterSequence`. `compact` refuses to truncate below a
607
+ * registered cursor, so a live follower's next durable page is never
608
+ * deleted out from under it. Readers this process cannot see, pagers
609
+ * of `entries`, followers in other processes, are covered by the
610
+ * read-side `compacted` guard instead.
611
+ */
612
+ const readers = new Map<RunId, Set<{ cursor: number }>>()
613
+ /**
614
+ * Only the most recent `sourceEventCache` events are decoded at startup.
615
+ * Older events stay durable-only: their idempotency is enforced by the
616
+ * writer's `(run_id, source_id, source_seq)` re-check, so the process
617
+ * never has to hold the whole history to stay correct.
618
+ */
619
+ const sourceEventRows = yield* sql<JournalRow>`
620
+ SELECT run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
621
+ event_type, payload_json, meta_json
622
+ FROM flows_journal_events
623
+ ORDER BY emitted_at_ms DESC, run_id DESC, seq DESC
624
+ LIMIT ${sourceEventCache}
625
+ `.pipe(
626
+ Effect.mapError((cause) => error("sink_failed", "could not initialize journal source events", cause))
627
+ )
628
+ const durableEntries = yield* Effect.forEach(sourceEventRows, decodeRow)
629
+ const initialized = yield* Effect.fromResult(
630
+ Result.gen(function*() {
631
+ // The two allocation floors start EMPTY and are filled one run at a
632
+ // time by `ensureFloors`. They used to be seeded by two unbounded
633
+ // `GROUP BY` aggregations, one entry per run that ever wrote an
634
+ // event, one per `(run_id, source_id)` pair, so layer construction
635
+ // scanned the whole table and built a map proportional to total
636
+ // history, which is exactly what the bound below exists to avoid.
637
+ const sequences = new Map<RunId, number>()
638
+ const sourceSequences = new Map<string, number>()
639
+ const sourceEvents = new Map<string, SourceEvent>()
640
+ // Seeded oldest-first so the map's insertion order stays the
641
+ // eviction order once `retain` starts adding newer events.
642
+ for (const entry of [...durableEntries].reverse()) {
643
+ if (
644
+ !Number.isSafeInteger(entry.seq) ||
645
+ !Number.isSafeInteger(entry.sourceSeq) ||
646
+ entry.seq === Number.MAX_SAFE_INTEGER ||
647
+ entry.sourceSeq === Number.MAX_SAFE_INTEGER
648
+ ) {
649
+ return yield* Result.fail(
650
+ error("decode_failed", "durable event sequence is outside the allocatable safe integer range")
651
+ )
652
+ }
653
+ sourceEvents.set(sourceEventKey(entry.runId, entry.sourceId, entry.sourceSeq), {
654
+ seq: entry.seq,
655
+ eventType: entry.eventType,
656
+ payloadJson: yield* encodeFingerprint(entry.payload, "payload"),
657
+ metaJson: yield* encodeFingerprint(entry.meta, "meta"),
658
+ status: "committed"
659
+ })
660
+ }
661
+ return {
662
+ sequences,
663
+ sourceSequences,
664
+ sourceEvents
665
+ }
666
+ })
667
+ )
668
+ const state: State = {
669
+ status: "open",
670
+ sinkFailure: undefined,
671
+ lossEpoch: 0,
672
+ flushedLossEpoch: 0,
673
+ pending: 0,
674
+ pendingByRun: new Map(),
675
+ sequences: initialized.sequences,
676
+ sourceSequences: initialized.sourceSequences,
677
+ sourceEvents: initialized.sourceEvents,
678
+ flushWaiters: new Set()
679
+ }
680
+ // The permit protects only synchronous in-memory reservation. It must
681
+ // never be held while waiting on SQLite: an enclosing `transact` owns a
682
+ // database transaction, so DB -> allocator and allocator -> DB ordering
683
+ // would otherwise deadlock concurrent lifecycle writers.
684
+ const allocation = yield* Semaphore.make(1)
685
+ // Close this scope AFTER the journal's flushing finalizer. Dynamically
686
+ // forking directly into the layer scope would register interruption
687
+ // finalizers ahead of the flush that must await these attempts.
688
+ const maintenanceScope = yield* Scope.make()
689
+ yield* Effect.addFinalizer((exit) => Scope.close(maintenanceScope, exit))
690
+ let pendingMaintenance = 0
691
+ const compactionCounts = new Map<RunId, number>()
692
+ const compactingRuns = new Set<RunId>()
693
+ const runBarriers = new Map<RunId, RunBarrier>()
694
+ const runDrainWaiters = new Map<RunId, Set<Deferred.Deferred<void>>>()
695
+ const activeWritesByRun = new Map<RunId, number>()
696
+
697
+ const barrierFor = (runId: RunId): RunBarrier => {
698
+ const existing = runBarriers.get(runId)
699
+ if (existing !== undefined) return existing
700
+ const created: RunBarrier = {
701
+ semaphore: Semaphore.makeUnsafe(1),
702
+ compactionLock: Semaphore.makeUnsafe(1),
703
+ maintenance: Semaphore.makeUnsafe(1),
704
+ users: 0,
705
+ compaction: undefined
706
+ }
707
+ runBarriers.set(runId, created)
708
+ return created
709
+ }
710
+
711
+ /** A waiter owns a reference too, before it can yield on any permit. */
712
+ const withRunBarrier = <A, E, R>(
713
+ runId: RunId,
714
+ use: (barrier: RunBarrier) => Effect.Effect<A, E, R>
715
+ ): Effect.Effect<A, E, R> =>
716
+ Effect.suspend(() => {
717
+ const barrier = barrierFor(runId)
718
+ barrier.users += 1
719
+ return use(barrier).pipe(Effect.ensuring(Effect.sync(() => {
720
+ barrier.users -= 1
721
+ })))
722
+ })
723
+
724
+ /** Flush is the retirement boundary; allocation floors remain monotonic. */
725
+ const retireQuiescentRuns = (): void => {
726
+ for (const [runId, barrier] of runBarriers) {
727
+ if (
728
+ barrier.users > 0 || barrier.compaction !== undefined ||
729
+ state.pendingByRun.has(runId) || activeWritesByRun.has(runId) ||
730
+ runDrainWaiters.has(runId) || readers.has(runId) || wakes.has(runId) ||
731
+ compactingRuns.has(runId)
732
+ ) continue
733
+ runBarriers.delete(runId)
734
+ compactionCounts.delete(runId)
735
+ }
736
+ }
737
+
738
+ /** Blocks new admissions while a compaction owns the run. */
739
+ const withRunAdmission = <A, E, R>(
740
+ runId: RunId,
741
+ effect: Effect.Effect<A, E, R>
742
+ ): Effect.Effect<A, E, R> =>
743
+ withRunBarrier(runId, (barrier) =>
744
+ barrier.semaphore.withPermit(
745
+ Effect.suspend((): Effect.Effect<RunAdmission<A>, E, R> => {
746
+ const gate = barrier.compaction
747
+ return gate === undefined
748
+ ? Effect.map(effect, (value): RunAdmission<A> => ({ _tag: "Done", value }))
749
+ : Effect.succeed<RunAdmission<A>>({ _tag: "Wait", gate })
750
+ })
751
+ ).pipe(
752
+ Effect.flatMap((attempt): Effect.Effect<A, E, R> =>
753
+ attempt._tag === "Done"
754
+ ? Effect.succeed(attempt.value)
755
+ : Deferred.await(attempt.gate).pipe(
756
+ Effect.andThen(withRunAdmission(runId, effect))
757
+ )
758
+ )
759
+ ))
760
+
761
+ /** Acquires batch run permits in stable order so mixed-run batches cannot deadlock. */
762
+ const withRunPermits = <A, E, R>(
763
+ runIds: ReadonlyArray<RunId>,
764
+ effect: Effect.Effect<A, E, R>
765
+ ): Effect.Effect<A, E, R> => {
766
+ const ordered = [...new Set(runIds)].sort()
767
+ const acquire = (index: number): Effect.Effect<A, E, R> =>
768
+ index === ordered.length
769
+ ? effect
770
+ : withRunBarrier(ordered[index]!, (barrier) => barrier.semaphore.withPermit(acquire(index + 1)))
771
+ return acquire(0)
772
+ }
773
+
774
+ const admitRunPending = (queued: QueuedEntry): void => {
775
+ const pending = state.pendingByRun.get(queued.runId) ?? new Set<QueuedEntry>()
776
+ pending.add(queued)
777
+ state.pendingByRun.set(queued.runId, pending)
778
+ }
779
+
780
+ const completeRunDrain = (runId: RunId): void => {
781
+ if ((state.pendingByRun.get(runId)?.size ?? 0) + (activeWritesByRun.get(runId) ?? 0) > 0) return
782
+ const waiters = runDrainWaiters.get(runId)
783
+ runDrainWaiters.delete(runId)
784
+ for (const waiter of waiters ?? []) {
785
+ Deferred.doneUnsafe(waiter, Effect.void)
786
+ }
787
+ }
788
+
789
+ const settleRunPending = (batch: ReadonlyArray<QueuedEntry>): void => {
790
+ for (const queued of batch) {
791
+ // Every batched entry was counted by `admitRunPending` when it was
792
+ // admitted, so the run always has a count to settle here.
793
+ const pending = state.pendingByRun.get(queued.runId)!
794
+ pending.delete(queued)
795
+ if (pending.size > 0) continue
796
+ state.pendingByRun.delete(queued.runId)
797
+ completeRunDrain(queued.runId)
798
+ }
799
+ }
800
+
801
+ const endRunWrite = (runId: RunId): void => {
802
+ // `withActiveRunWrite` counted this write in before running it, and
803
+ // releases through here exactly once.
804
+ const remaining = activeWritesByRun.get(runId)! - 1
805
+ if (remaining > 0) {
806
+ activeWritesByRun.set(runId, remaining)
807
+ return
808
+ }
809
+ activeWritesByRun.delete(runId)
810
+ completeRunDrain(runId)
811
+ }
812
+
813
+ /**
814
+ * Register only INSIDE the writer transaction, after SQL acquisition.
815
+ * A parked writer has touched neither SQL nor the allocation floors, so
816
+ * the SQL lock orders it after the compactor's commit. It then allocates
817
+ * its canonical seq above the surviving checkpoint/tail. An explicit
818
+ * sourceSeq chosen earlier is an identity, not a replay cursor; the
819
+ * insert still checks that identity against rows and retained tombstones.
820
+ * No SQL owner may drain a writer parked on that owner's connection.
821
+ */
822
+ const withActiveRunWrite = <A, E, R>(
823
+ runId: RunId,
824
+ effect: Effect.Effect<A, E, R>
825
+ ): Effect.Effect<A, E, R> =>
826
+ withRunBarrier(runId, () =>
827
+ Effect.suspend(() => {
828
+ activeWritesByRun.set(runId, (activeWritesByRun.get(runId) ?? 0) + 1)
829
+ return effect.pipe(Effect.ensuring(Effect.sync(() => endRunWrite(runId))))
830
+ }))
831
+
832
+ const awaitRunDrained = (runId: RunId): Effect.Effect<void> =>
833
+ Effect.suspend(() => {
834
+ if ((state.pendingByRun.get(runId)?.size ?? 0) + (activeWritesByRun.get(runId) ?? 0) === 0) {
835
+ return Effect.void
836
+ }
837
+ const waiter = Deferred.makeUnsafe<void>()
838
+ const waiters = runDrainWaiters.get(runId) ?? new Set()
839
+ waiters.add(waiter)
840
+ runDrainWaiters.set(runId, waiters)
841
+ // No cleanup on the way out: `completeRunDrain` deletes the whole
842
+ // set as it completes it, and an interrupted waiter is a Deferred
843
+ // nobody awaits that the next drain of this run discards. Deleting
844
+ // it here would only add an unreachable branch.
845
+ return Deferred.await(waiter)
846
+ })
847
+
848
+ /**
849
+ * Top-level compaction drains with admission OPEN, acquires SQL before
850
+ * either run lock or the gate, then rechecks the drain. Raced work makes
851
+ * it release SQL and retry. No connection owner waits on work needing SQL.
852
+ *
853
+ * An enclosing managed OR raw SQL transaction cannot release its SQL by
854
+ * returning from our savepoint. Skip both drains there and persist this
855
+ * run's pending entries inline, using the batch insert path under the
856
+ * same transaction and gate. The pending set includes entries already
857
+ * taken by the batch fiber but still waiting for SQL. Keep them queued
858
+ * and counted until that fiber settles: rollback must leave its work
859
+ * intact, and after commit it collapses onto the inline inserts by identity.
860
+ * Publication is owned by the outer commit, exactly as for durable emits.
861
+ * Raw SQL has no managed publication boundary; replay remains authoritative.
862
+ *
863
+ * A reservation allocated before compaction is NOT a committed cursor.
864
+ * The inline insert raises its canonical seq above the durable tail,
865
+ * even if its reservation is below the checkpoint being compacted. Thus
866
+ * it survives replay after that checkpoint with its source identity and
867
+ * content checks unchanged. Leaving it queued without inserting would
868
+ * instead let the new floor reject the old reservation as `compacted`.
869
+ * Already compacted reservations retain that existing loss behavior.
870
+ *
871
+ * The final drain check and gate closure share a short admission permit,
872
+ * released before SQL work. Cold floor reads hold no run permit. Durable
873
+ * writes register only after SQL acquisition. Concurrent compactors also
874
+ * acquire SQL before their compaction lock. All gates/locks unwind before
875
+ * retry or cancellation; no run lock or closed gate waits for SQL.
876
+ */
877
+ const withCompactionBarrier = <A, E, R>(
878
+ runId: RunId,
879
+ effect: Effect.Effect<A, E, R>
880
+ ): Effect.Effect<A, Exclude<E, SqlError.SqlError> | DatabaseError, R> =>
881
+ Effect.flatMap(Effect.serviceOption(sql.transactionService), (enclosing) =>
882
+ withRunBarrier(runId, (barrier) => {
883
+ const nested = Option.isSome(enclosing)
884
+ const attempt: Effect.Effect<A, Exclude<E, SqlError.SqlError> | DatabaseError, R> =
885
+ (nested ? Effect.void : awaitRunDrained(runId)).pipe(
886
+ Effect.andThen(writer.write(
887
+ barrier.compactionLock.withPermit(Effect.uninterruptibleMask((restore) =>
888
+ barrier.semaphore.withPermit(Effect.sync(() => {
889
+ if (
890
+ !nested && (state.pendingByRun.get(runId)?.size ?? 0) + (activeWritesByRun.get(runId) ?? 0) > 0
891
+ ) {
892
+ return undefined
893
+ }
894
+ const gate = Deferred.makeUnsafe<void>()
895
+ barrier.compaction = gate
896
+ return gate
897
+ })).pipe(Effect.flatMap((gate) =>
898
+ gate === undefined ?
899
+ Effect.succeed(Option.none<A>()) :
900
+ restore(Effect.gen(function*() {
901
+ if (nested) {
902
+ const outcome = yield* insertBatch([...state.pendingByRun.get(runId) ?? []])
903
+ // The batch still owns loss reporting and pending
904
+ // settlement; neither may escape an outer rollback.
905
+ for (const { queued, commit } of outcome.commits) {
906
+ yield* settleCommit(queued, commit, restore)
907
+ }
908
+ }
909
+ return yield* Effect.asSome(effect)
910
+ })).pipe(Effect.ensuring(Effect.sync(() => {
911
+ barrier.compaction = undefined
912
+ Deferred.doneUnsafe(gate, Effect.void)
913
+ })))
914
+ ))
915
+ ))
916
+ )),
917
+ Effect.flatMap((result) =>
918
+ Option.isSome(result) ? Effect.succeed(result.value) : attempt
919
+ )
920
+ )
921
+ return attempt
922
+ }))
923
+
924
+ /**
925
+ * Raises the in-process seq allocation floor for a run.
926
+ *
927
+ * Both emit paths call this at the moment they allocate, so the floor
928
+ * never names a seq some writer has already taken. It used to move only
929
+ * in `rememberCommitted`, which `settleCommit` parks until the outermost
930
+ * COMMIT, so a durable emit inside an open `transact` left the floor
931
+ * behind, `emitLossy` allocated the same seq from it, and the lossy
932
+ * INSERT hit `PRIMARY KEY (run_id, seq)`.
933
+ *
934
+ * The floor only ever rises. `insertOne` can settle on a raced duplicate
935
+ * whose row was written by another process at a higher seq, and that
936
+ * still has to raise the floor here.
937
+ */
938
+ const raiseSequenceFloor = (runId: RunId, seq: number): void => {
939
+ state.sequences.set(runId, Math.max(state.sequences.get(runId) ?? 0, seq + 1))
940
+ }
941
+
942
+ /** Raises the in-process producer-sequence allocation floor. */
943
+ const raiseSourceSequenceFloor = (runId: RunId, sourceId: SourceId, sourceSeq: number): void => {
944
+ const key = sourceKey(runId, sourceId)
945
+ state.sourceSequences.set(key, Math.max(state.sourceSequences.get(key) ?? 0, sourceSeq + 1))
946
+ }
947
+
948
+ /**
949
+ * Adds an entry to the bounded source-event index, evicting the
950
+ * least-recently added *committed* entry when the bound is exceeded.
951
+ *
952
+ * Uncommitted entries are never evicted: they are not in the database
953
+ * yet, so memory is the only place their identity exists. Committed ones
954
+ * are always re-derivable from `flows_journal_events`, which is what
955
+ * makes the bound safe.
956
+ */
957
+ const retain = (identity: string, event: SourceEvent): void => {
958
+ state.sourceEvents.delete(identity)
959
+ state.sourceEvents.set(identity, event)
960
+ if (state.sourceEvents.size <= sourceEventCache) {
961
+ return
962
+ }
963
+ for (const [candidate, retained] of state.sourceEvents) {
964
+ if (retained.status !== "committed" || candidate === identity) {
965
+ continue
966
+ }
967
+ state.sourceEvents.delete(candidate)
968
+ return
969
+ }
970
+ }
971
+
972
+ const completeFlushWaiters = (exit: Effect.Effect<void, JournalError>): void => {
973
+ const waiters = Array.from(state.flushWaiters)
974
+ state.flushWaiters.clear()
975
+ for (const waiter of waiters) {
976
+ Deferred.doneUnsafe(waiter, exit)
977
+ }
978
+ }
979
+
980
+ const flushInternal: Effect.Effect<void, JournalError> = Effect.suspend(() => {
981
+ // A loss that happened while nothing was registered still has to reach
982
+ // a caller, so the first flush after it reports it, once. A later
983
+ // flush has nothing to do with the lost batch and must succeed, or a
984
+ // single transient outage would stall every durable delivery that
985
+ // flushes (`DeferredPersistence.completeDeferred`, `recordClockScheduled`)
986
+ // for the process's lifetime.
987
+ if (state.sinkFailure !== undefined && state.lossEpoch > state.flushedLossEpoch) {
988
+ state.flushedLossEpoch = state.lossEpoch
989
+ return Effect.fail(state.sinkFailure)
990
+ }
991
+ if (state.status === "closed") {
992
+ return Effect.fail(error("journal_closed", "journal is closed"))
993
+ }
994
+ if (state.pending === 0 && pendingMaintenance === 0) {
995
+ return Effect.void
996
+ }
997
+ const waiter = Deferred.makeUnsafe<void, JournalError>()
998
+ state.flushWaiters.add(waiter)
999
+ return Deferred.await(waiter).pipe(
1000
+ Effect.ensuring(Effect.sync(() => {
1001
+ state.flushWaiters.delete(waiter)
1002
+ }))
1003
+ )
1004
+ }).pipe(Effect.ensuring(Effect.sync(retireQuiescentRuns)))
1005
+
1006
+ const prepare = (input: Input, emittedAtMs: number): Result.Result<Prepared, JournalError> =>
1007
+ Result.gen(function*() {
1008
+ if (state.status !== "open") {
1009
+ return yield* Result.fail(error("journal_closed", "journal is closed"))
1010
+ }
1011
+ // `RunId`, `SourceId`, and `Input.eventType` carry the non-empty and
1012
+ // well-formed-UTF-16 checks themselves, so decode is the whole
1013
+ // identifier contract. The service used to re-check emptiness here,
1014
+ // which let a caller hold an identifier that decoded and then failed
1015
+ // at the write.
1016
+ const validated = yield* Result.mapError(
1017
+ decodeInput(input),
1018
+ (cause) => error("invalid_event", "event violates the journal input contract", cause)
1019
+ )
1020
+ if (!Number.isSafeInteger(emittedAtMs) || emittedAtMs < 0) {
1021
+ return yield* Result.fail(
1022
+ error("invalid_event", "emittedAtMs must be a non-negative safe integer")
1023
+ )
1024
+ }
1025
+ // Redaction happens here, at the single point every channel funnels
1026
+ // through, so no write path can bypass it (issue #46). A redaction
1027
+ // failure carries no cause: whatever the redactor or a hostile getter
1028
+ // threw can quote the unredacted value, and that value never reached
1029
+ // the journal, so the error must not be the place it leaks from.
1030
+ const redactedPayload = yield* Result.try({
1031
+ try: () => redact(validated.payload),
1032
+ catch: () => error("invalid_event", "payload could not be redacted")
1033
+ })
1034
+ const redactedMeta = yield* Result.try({
1035
+ try: () => redact(validated.meta ?? null),
1036
+ catch: () => error("invalid_event", "meta could not be redacted")
1037
+ })
1038
+ const payloadJson = yield* encodeFingerprint(redactedPayload, "payload")
1039
+ const metaJson = yield* encodeFingerprint(redactedMeta, "meta")
1040
+ // Measured on the encoded bytes, and here rather than at admission,
1041
+ // so a refused entry costs no sequence and leaves no gap in the run.
1042
+ if (maxEntryBytes !== undefined) {
1043
+ const bytes = encodedBytes(byteCounter, payloadJson, metaJson)
1044
+ if (bytes > maxEntryBytes) {
1045
+ return yield* Result.fail(
1046
+ error(
1047
+ "invalid_event",
1048
+ `event is ${bytes} bytes, over the ${maxEntryBytes}-byte maxEntryBytes bound`
1049
+ )
1050
+ )
1051
+ }
1052
+ }
1053
+ return { validated, payloadJson, metaJson }
1054
+ })
1055
+
1056
+ const compareSourceEvent = (
1057
+ prepared: Prepared,
1058
+ sourceSeq: SourceSeq,
1059
+ existing: SourceEvent
1060
+ ): Result.Result<EmitReceipt, JournalError> => {
1061
+ const { metaJson, payloadJson, validated } = prepared
1062
+ if (
1063
+ validated.dedupe !== "identity" && (
1064
+ existing.eventType !== validated.eventType ||
1065
+ existing.payloadJson !== payloadJson ||
1066
+ existing.metaJson !== metaJson
1067
+ )
1068
+ ) {
1069
+ return Result.fail(error(
1070
+ "idempotency_conflict",
1071
+ `source event ${validated.sourceId}:${sourceSeq} for run ${validated.runId} was reused with different content`
1072
+ ))
1073
+ }
1074
+ return Result.succeed({
1075
+ _tag: "Duplicate",
1076
+ seq: existing.seq,
1077
+ sourceSeq,
1078
+ status: existing.status
1079
+ })
1080
+ }
1081
+
1082
+ /**
1083
+ * Answers an explicit producer identity from memory alone, before
1084
+ * anything is allocated.
1085
+ *
1086
+ * This performs NO read. That is the whole point of it: `emitLossy` is
1087
+ * the channel a producer reaches for precisely because it may be called
1088
+ * from inside somebody else's open write transaction, and this used to
1089
+ * issue a SELECT here for every explicit sequence. The executor's exit
1090
+ * flush in `@smthrs/agent` runs inside the engine's write transaction,
1091
+ * so that read waited on the writer that was waiting on the flush and
1092
+ * the run stalled at 0% CPU: measured, that case did not finish in
1093
+ * 120 s with the read and takes about half a second without it. What the
1094
+ * read used to answer, a producer identity that is already durable but
1095
+ * no longer in the bounded index, the unique index
1096
+ * `(run_id, source_id, source_seq)` answers at insert time instead: the
1097
+ * queued insert runs first and reads only the row it actually collided
1098
+ * with, inside the writer's own transaction where a read cannot
1099
+ * deadlock against it.
1100
+ *
1101
+ * A cache miss therefore admits optimistically. The receipt is
1102
+ * `Accepted` where it would once have been `Duplicate`, which the lossy
1103
+ * channel already allows for: `Accepted` is admission to the queue, not
1104
+ * a commit, and the entry still collapses onto the committed row rather
1105
+ * than doubling it.
1106
+ *
1107
+ * Lock order: lossy admission checks cached identities without SQL,
1108
+ * reads cold allocation floors outside run permits on a miss, then
1109
+ * rechecks the identity and reserves sequences under
1110
+ * the permit using max(read floor, in-memory floor). Durable writers
1111
+ * raise those caches at allocation time and take no run permit. Batches
1112
+ * take the writer transaction before run permits, so their SQL already
1113
+ * owns the connection. Compaction drains with admission open, acquires
1114
+ * SQL before its run locks, then rechecks the drain and closes the gate.
1115
+ * At top level, raced work makes it release SQL and drain again. Inside
1116
+ * an enclosing transaction it persists pending entries inline instead;
1117
+ * draining there would wait on work needing its own connection. Durable
1118
+ * writes register as active only after acquiring SQL. No permit or
1119
+ * closed gate ever waits for a SQL connection, and
1120
+ * a transaction-owning producer cannot encounter a compactor's gate or
1121
+ * permit: that compactor cannot acquire SQL until the producer commits.
1122
+ *
1123
+ * Reading floors under a permit would invert the batch's order: a new
1124
+ * producer waits on the batch's connection while the batch waits on its
1125
+ * permit. Moving batch permits before the transaction would instead
1126
+ * block even readless admission from inside an enclosing transaction.
1127
+ * Keep floor reads outside permits and collision reads inside the
1128
+ * writer transaction. `ensureFloors` documents cache lifetime, including
1129
+ * the quiescent truncation contract that permits floor invalidation.
1130
+ */
1131
+ const admitFromIndex = (prepared: Prepared): Result.Result<EmitReceipt | undefined, JournalError> => {
1132
+ const { validated } = prepared
1133
+ const sourceSeq = validated.sourceSeq
1134
+ if (sourceSeq === undefined) return Result.succeed(undefined)
1135
+ if (
1136
+ !Number.isSafeInteger(sourceSeq) ||
1137
+ sourceSeq < 0 ||
1138
+ sourceSeq === Number.MAX_SAFE_INTEGER
1139
+ ) {
1140
+ return Result.fail(
1141
+ error("invalid_event", "journal sequence is outside the allocatable safe integer range")
1142
+ )
1143
+ }
1144
+ const cached = state.sourceEvents.get(sourceEventKey(validated.runId, validated.sourceId, sourceSeq))
1145
+ return cached === undefined ? Result.succeed(undefined) : compareSourceEvent(prepared, sourceSeq, cached)
1146
+ }
1147
+
1148
+ /**
1149
+ * The in-memory half of a lossy admission: the identity check, the
1150
+ * allocation, and the queue offer. It runs under the run permit and
1151
+ * performs no I/O there; `admitFromIndex` documents the lock order that
1152
+ * keeps.
1153
+ */
1154
+ const admitQueued = (
1155
+ prepared: Prepared,
1156
+ floors: { readonly seq: number; readonly sourceSeq: number },
1157
+ emittedAtMs: number
1158
+ ): Effect.Effect<EmitReceipt, JournalError> =>
1159
+ Effect.suspend(() => {
1160
+ // Preparation precedes the floor read and admission waits. Check
1161
+ // closure and the identity again only after acquiring the permit,
1162
+ // before allocating or returning a cached duplicate.
1163
+ if (state.status !== "open") {
1164
+ return Effect.fail(error("journal_closed", "journal is closed"))
1165
+ }
1166
+ return Effect.flatMap(
1167
+ Effect.fromResult(admitFromIndex(prepared)),
1168
+ (indexed) =>
1169
+ indexed !== undefined
1170
+ ? Effect.succeed(indexed)
1171
+ : allocation.withPermit(Effect.fromResult(
1172
+ Result.gen(function*() {
1173
+ const { metaJson, payloadJson, validated } = prepared
1174
+ const key = sourceKey(validated.runId, validated.sourceId)
1175
+ const nextSourceSeq = Math.max(
1176
+ floors.sourceSeq,
1177
+ state.sourceSequences.get(key) ?? 0
1178
+ )
1179
+ // An explicit producer sequence was validated by
1180
+ // `admitFromIndex`, and an implicit one is the floor
1181
+ // `ensureFloors` already proved to be a non-negative
1182
+ // safe integer. Only exhaustion is left: the allocator
1183
+ // cannot advance past MAX_SAFE_INTEGER, and neither can
1184
+ // the identity that follows it.
1185
+ const sourceSeq: SourceSeq = validated.sourceSeq ?? (nextSourceSeq as SourceSeq)
1186
+ if (sourceSeq === Number.MAX_SAFE_INTEGER) {
1187
+ return yield* Result.fail(
1188
+ error(
1189
+ "invalid_event",
1190
+ "journal sequence is outside the allocatable safe integer range"
1191
+ )
1192
+ )
1193
+ }
1194
+ const nextSeq = Math.max(
1195
+ floors.seq,
1196
+ state.sequences.get(validated.runId) ?? 0
1197
+ )
1198
+ if (nextSeq === Number.MAX_SAFE_INTEGER) {
1199
+ return yield* Result.fail(
1200
+ error(
1201
+ "invalid_event",
1202
+ "journal sequence is outside the allocatable safe integer range"
1203
+ )
1204
+ )
1205
+ }
1206
+ const seq = nextSeq as Seq
1207
+ raiseSequenceFloor(validated.runId, seq)
1208
+ state.sourceSequences.set(key, Math.max(nextSourceSeq, sourceSeq + 1))
1209
+
1210
+ const queued: QueuedEntry = {
1211
+ runId: validated.runId,
1212
+ seq,
1213
+ eventId: makeEventId(validated.runId, validated.sourceId, sourceSeq),
1214
+ sourceId: validated.sourceId,
1215
+ sourceSeq,
1216
+ emittedAtMs,
1217
+ eventType: validated.eventType,
1218
+ payloadJson,
1219
+ metaJson,
1220
+ dedupe: validated.dedupe ?? "content"
1221
+ }
1222
+ let evicted: QueuedEntry | undefined
1223
+ if (Queue.sizeUnsafe(queue) >= options.capacity && options.overflow === "drop-oldest") {
1224
+ const exit = Queue.takeUnsafe(queue)
1225
+ /* v8 ignore next -- size and take run synchronously while the journal and queue are open */
1226
+ if (exit === undefined || !Exit.isSuccess(exit)) {
1227
+ return yield* Result.fail(
1228
+ error("journal_closed", "journal admission queue is unavailable")
1229
+ )
1230
+ }
1231
+ evicted = exit.value
1232
+ const evictedIdentity = sourceEventKey(
1233
+ evicted.runId,
1234
+ evicted.sourceId,
1235
+ evicted.sourceSeq
1236
+ )
1237
+ state.sourceEvents.delete(evictedIdentity)
1238
+ state.pending = Math.max(0, state.pending - 1)
1239
+ settleRunPending([evicted])
1240
+ }
1241
+ const accepted = Queue.offerUnsafe(queue, queued)
1242
+ if (!accepted) {
1243
+ if (options.overflow === "reject") {
1244
+ return yield* Result.fail(
1245
+ error("queue_overflow", "journal admission queue is full")
1246
+ )
1247
+ }
1248
+ return {
1249
+ _tag: "Dropped",
1250
+ seq,
1251
+ sourceSeq,
1252
+ policy: "drop-newest"
1253
+ } satisfies EmitReceipt
1254
+ }
1255
+
1256
+ retain(sourceEventKey(validated.runId, validated.sourceId, sourceSeq), {
1257
+ seq,
1258
+ eventType: validated.eventType,
1259
+ payloadJson,
1260
+ metaJson,
1261
+ status: "pending"
1262
+ })
1263
+ state.pending += 1
1264
+ admitRunPending(queued)
1265
+ if (evicted !== undefined) {
1266
+ return {
1267
+ _tag: "Accepted",
1268
+ seq,
1269
+ sourceSeq,
1270
+ evicted: {
1271
+ policy: "drop-oldest",
1272
+ count: 1
1273
+ }
1274
+ } satisfies EmitReceipt
1275
+ }
1276
+ return {
1277
+ _tag: "Accepted",
1278
+ seq,
1279
+ sourceSeq
1280
+ } satisfies EmitReceipt
1281
+ })
1282
+ ))
1283
+ )
1284
+ })
1285
+
1286
+ const queuedEmit: Service["emitLossy"] = Effect.fn("Journal.emitLossy")((input: Input) =>
1287
+ Effect.annotateCurrentSpan({
1288
+ runId: input.runId,
1289
+ sourceId: input.sourceId,
1290
+ eventType: input.eventType
1291
+ }).pipe(
1292
+ Effect.andThen(Clock.currentTimeMillis.pipe(Effect.map(Math.floor))),
1293
+ Effect.flatMap((emittedAtMs) =>
1294
+ Effect.flatMap(
1295
+ withRunAdmission(
1296
+ input.runId,
1297
+ Effect.suspend(() =>
1298
+ Effect.fromResult(Result.gen(function*() {
1299
+ const prepared = yield* prepare(input, emittedAtMs)
1300
+ const indexed = yield* admitFromIndex(prepared)
1301
+ return { prepared, indexed }
1302
+ }))
1303
+ )
1304
+ ),
1305
+ ({ prepared, indexed }) =>
1306
+ // A cached duplicate needs neither allocation floor, including
1307
+ // after startup when the identity index is warm but floors are
1308
+ // cold. On a miss, release the permit before reading SQL, then
1309
+ // recheck identity and allocation under a fresh admission.
1310
+ indexed !== undefined ? Effect.succeed(indexed) : Effect.flatMap(
1311
+ ensureFloors(prepared.validated),
1312
+ (floors) => withRunAdmission(prepared.validated.runId, admitQueued(prepared, floors, emittedAtMs))
1313
+ )
1314
+ )
1315
+ ),
1316
+ Effect.tap((receipt) => Metric.update(JournalMetrics.lossy[receipt._tag], 1))
1317
+ )
1318
+ )
1319
+
1320
+ /**
1321
+ * The run's compaction floor: the largest checkpoint sequence whose
1322
+ * lower entries have been truncated, or `undefined` while the run has
1323
+ * never been compacted.
1324
+ */
1325
+ const compactionFloor = (runId: RunId): Effect.Effect<number | undefined, SqlError.SqlError> =>
1326
+ Effect.map(
1327
+ sql<{ readonly floor: number | null }>`
1328
+ SELECT MAX(seq) AS floor FROM flows_journal_checkpoints
1329
+ WHERE run_id = ${runId} AND compacted_at_ms IS NOT NULL
1330
+ `,
1331
+ (rows) => {
1332
+ const floor = rows[0]?.floor
1333
+ return floor === null || floor === undefined ? undefined : Number(floor)
1334
+ }
1335
+ )
1336
+
1337
+ const readPage: Service["entries"] = Effect.fn("Journal.entries")((pageOptions) =>
1338
+ Effect.gen(function*() {
1339
+ // Snapshot the caller-owned list before the first suspension so its
1340
+ // validated bound is also the bound passed to the SQL client.
1341
+ const options = {
1342
+ ...pageOptions,
1343
+ ...(Array.isArray(pageOptions.eventTypes)
1344
+ ? { eventTypes: [...pageOptions.eventTypes] } :
1345
+ {})
1346
+ }
1347
+ yield* Effect.annotateCurrentSpan({
1348
+ runId: options.runId,
1349
+ limit: options.limit,
1350
+ ...(options.eventTypes === undefined ? {} : { eventTypes: options.eventTypes }),
1351
+ ...(options.after === undefined ? {} : { after: options.after })
1352
+ })
1353
+ // `EntriesOptions` already carries every one of these invariants: a
1354
+ // well-formed non-empty `runId`, a `Seq` cursor, and a `limit`
1355
+ // between 1 and `maxEntriesLimit`. Hand-checking them here let the
1356
+ // read boundary disagree with the write boundary, and it did: a
1357
+ // lone-surrogate run id was refused at `emit` and answered with an
1358
+ // empty page at `entries`.
1359
+ yield* Effect.fromResult(Result.mapError(
1360
+ decodeEntriesOptions(options),
1361
+ (cause) => error("invalid_event", "entries options violate the journal read contract", cause)
1362
+ ))
1363
+ const after = options.after ?? -1
1364
+ // With several types SQLite can prefer the run/sequence index to
1365
+ // avoid sorting, scanning every unrelated event instead. Require
1366
+ // the journal-owned type index; only matching rows may be sorted.
1367
+ const rows = yield* (options.eventTypes === undefined ?
1368
+ sql<JournalRow>`
1369
+ SELECT run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
1370
+ event_type, payload_json, meta_json
1371
+ FROM flows_journal_events
1372
+ WHERE run_id = ${options.runId} AND seq > ${after}
1373
+ ORDER BY seq ASC
1374
+ LIMIT ${options.limit + 1}
1375
+ ` :
1376
+ sql<JournalRow>`
1377
+ SELECT run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
1378
+ event_type, payload_json, meta_json
1379
+ FROM flows_journal_events ${Dialect.indexHint(sql, "flows_journal_events_run_event_type_idx")}
1380
+ WHERE run_id = ${options.runId} AND ${sql.in("event_type", options.eventTypes)} AND seq > ${after}
1381
+ ORDER BY seq ASC
1382
+ LIMIT ${options.limit + 1}
1383
+ `).pipe(
1384
+ Effect.mapError((cause) => error("read_failed", "durable journal read failed", cause))
1385
+ )
1386
+ // The floor is read AFTER the page. Truncation and the floor
1387
+ // advance commit atomically, so any deletion that could have
1388
+ // shortened the page above is visible in this floor read, and a
1389
+ // cursor at or above `floor - 1` therefore read a complete page.
1390
+ // The converse order would let a compaction commit between the two
1391
+ // reads and hand back a silently gapped history.
1392
+ const floor = yield* compactionFloor(options.runId).pipe(
1393
+ Effect.mapError((cause) => error("read_failed", "durable journal read failed", cause))
1394
+ )
1395
+ if (floor !== undefined && after < floor - 1) {
1396
+ return yield* Effect.fail(
1397
+ new JournalError({
1398
+ code: "compacted",
1399
+ message: `run ${options.runId} is compacted through sequence ${floor}; resync from its checkpoint`,
1400
+ checkpointSeq: floor as Seq
1401
+ })
1402
+ )
1403
+ }
1404
+ const page = rows.slice(0, options.limit)
1405
+ const entries = yield* Effect.forEach(page, decodeRow)
1406
+ return {
1407
+ entries,
1408
+ hasMore: rows.length > options.limit
1409
+ } satisfies EntriesPage
1410
+ })
1411
+ )
1412
+
1413
+ const subscribeRun = (runId: RunId) =>
1414
+ Effect.gen(function*() {
1415
+ const wake = yield* PubSub.sliding<void>(1)
1416
+ const subscription = yield* PubSub.subscribe(wake)
1417
+ yield* Effect.acquireRelease(
1418
+ Effect.sync(() => {
1419
+ const subscribers = wakes.get(runId) ?? new Set()
1420
+ subscribers.add(wake)
1421
+ wakes.set(runId, subscribers)
1422
+ }),
1423
+ () =>
1424
+ Effect.sync(() => {
1425
+ const subscribers = wakes.get(runId)
1426
+ subscribers?.delete(wake)
1427
+ if (subscribers?.size === 0) {
1428
+ wakes.delete(runId)
1429
+ }
1430
+ }).pipe(Effect.andThen(PubSub.shutdown(wake)))
1431
+ )
1432
+ return subscription
1433
+ })
1434
+
1435
+ const stream = (streamOptions: StreamOptions): Stream.Stream<Entry, JournalError> =>
1436
+ Stream.unwrap(
1437
+ Effect.fn("Journal.stream")(function*() {
1438
+ yield* Effect.annotateCurrentSpan({
1439
+ runId: streamOptions.runId,
1440
+ ...(streamOptions.afterSequence === undefined ? {} : { afterSequence: streamOptions.afterSequence })
1441
+ })
1442
+ yield* Effect.fromResult(Result.mapError(
1443
+ decodeStreamOptions(streamOptions),
1444
+ (cause) => error("invalid_event", "stream options violate the journal read contract", cause)
1445
+ ))
1446
+ const wake = yield* subscribeRun(streamOptions.runId)
1447
+ // The cursor lives in a registered box for the stream's lifetime
1448
+ // so `compact` can see how far every live follower has read.
1449
+ const reader = { cursor: streamOptions.afterSequence ?? -1 }
1450
+ yield* Effect.acquireRelease(
1451
+ Effect.sync(() => {
1452
+ const registered = readers.get(streamOptions.runId) ?? new Set()
1453
+ registered.add(reader)
1454
+ readers.set(streamOptions.runId, registered)
1455
+ }),
1456
+ () =>
1457
+ Effect.sync(() => {
1458
+ const registered = readers.get(streamOptions.runId)
1459
+ registered?.delete(reader)
1460
+ if (registered?.size === 0) {
1461
+ readers.delete(streamOptions.runId)
1462
+ }
1463
+ })
1464
+ )
1465
+ // A live consumer is told about losses that happen while it is
1466
+ // following, and only about those: a loss it never overlapped is
1467
+ // already spent by the time it subscribes.
1468
+ const subscribedLossEpoch = state.lossEpoch
1469
+ const readPages = (initialCursor: number): Stream.Stream<Entry, JournalError> =>
1470
+ Stream.paginate(initialCursor, (cursor) =>
1471
+ Effect.suspend(() =>
1472
+ state.sinkFailure !== undefined && state.lossEpoch > subscribedLossEpoch
1473
+ ? Effect.fail(state.sinkFailure)
1474
+ : readPage({
1475
+ runId: streamOptions.runId,
1476
+ ...(cursor < 0 ? {} : { after: cursor as Seq }),
1477
+ limit: readPageSize
1478
+ })
1479
+ ).pipe(
1480
+ Effect.map((page) => {
1481
+ const last = page.entries.at(-1)
1482
+ if (last !== undefined) {
1483
+ // Both channels append above the committed tail, so no
1484
+ // pending reservation can later commit below this cursor.
1485
+ reader.cursor = last.seq
1486
+ }
1487
+ return [
1488
+ page.entries,
1489
+ page.hasMore ? Option.some(reader.cursor) : Option.none<number>()
1490
+ ] as const
1491
+ })
1492
+ ))
1493
+ const historical = readPages(reader.cursor)
1494
+ // PubSub is only a local fast path. Another journal process cannot
1495
+ // publish into it, so a bounded poll must recheck both the durable
1496
+ // tail and the compaction floor while the follower is otherwise idle.
1497
+ // One raced wake keeps the live tail in the consumer fiber instead
1498
+ // of merging two background streams, which also preserves a plain
1499
+ // interruption cause when the consumer is cancelled.
1500
+ const live = Stream.fromEffectRepeat(
1501
+ Effect.raceFirst(PubSub.take(wake), Effect.sleep("1 second"))
1502
+ ).pipe(Stream.flatMap(() =>
1503
+ readPages(reader.cursor)
1504
+ ))
1505
+ return Stream.concat(historical, live)
1506
+ })()
1507
+ )
1508
+
1509
+ /**
1510
+ * The owner fence for the fenced append, for checkpoint, and for
1511
+ * compaction: the injected strategy's commit-time admission (R3),
1512
+ * evaluated inside the caller's write transaction. `DurableWriter`
1513
+ * serializes write transactions, so no reclaim can commit between the
1514
+ * guard and the statements that run beside it in the same transaction.
1515
+ * A strategy that cannot answer is a sink failure, never a lost fence;
1516
+ * its cause is preserved so the writer's retry classification still sees
1517
+ * a transient conflict through the wrapper.
1518
+ */
1519
+ const fenceGuard = (
1520
+ runId: RunId,
1521
+ owner: OwnerId
1522
+ ): Effect.Effect<void, JournalError> =>
1523
+ consensus.guard(runId, owner).pipe(
1524
+ Effect.mapError((cause) =>
1525
+ cause.code === "fence_lost"
1526
+ ? error("fence_lost", cause.message)
1527
+ : error("sink_failed", "consensus guard failed", cause)
1528
+ )
1529
+ )
1530
+
1531
+ /**
1532
+ * Reads the row or compacted identity a duplicate emit collides with.
1533
+ * The insert guard in migration 0004 extends the event table's unique
1534
+ * constraints to identities retained by compaction. Tombstones return
1535
+ * only a receipt sequence and never produce a replayable entry.
1536
+ *
1537
+ * On the queued channel this runs only AFTER the insert, on the row that
1538
+ * insert's `ON CONFLICT DO NOTHING` actually refused: the unique index is
1539
+ * the admission decision there, so a batch of entries that collide with
1540
+ * nothing costs no lookups at all, and the one a collision costs happens
1541
+ * inside the writer's own transaction. See {@link insertOne} for why the
1542
+ * durable channel still asks first.
1543
+ *
1544
+ * The lookup covers BOTH unique constraints the insert can conflict on:
1545
+ * `UNIQUE (event_id)` and `UNIQUE (run_id, source_id, source_seq)`. It is
1546
+ * tempting to keep only the first, because `makeEventId` is injective in
1547
+ * exactly that triple, but that argument holds only for rows this
1548
+ * journal minted. `TimeTravelStore.createFork` copies a parent's rows
1549
+ * under the child's `run_id` with `'fork:' || run_id || ':' || event_id`,
1550
+ * so a forked run carries rows whose triple is live and whose event id it
1551
+ * will never mint. Looking those up by event id alone finds nothing, and
1552
+ * the caller reports the resulting empty insert as `fence_lost`, a
1553
+ * healthy fork failing with "someone else owns this run".
1554
+ */
1555
+ const selectExisting = (
1556
+ queued: QueuedEntry
1557
+ ): Effect.Effect<Commit | undefined, JournalError | SqlError.SqlError> =>
1558
+ Effect.gen(function*() {
1559
+ const existing = yield* sql<JournalRow & { readonly content_hash: string | null }>`
1560
+ SELECT run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
1561
+ event_type, payload_json, meta_json, NULL AS content_hash
1562
+ FROM flows_journal_events
1563
+ WHERE event_id = ${queued.eventId}
1564
+ OR (
1565
+ run_id = ${queued.runId}
1566
+ AND source_id = ${queued.sourceId}
1567
+ AND source_seq = ${queued.sourceSeq}
1568
+ )
1569
+ UNION ALL
1570
+ SELECT run_id, seq, event_id, source_id, source_seq, 0 AS emitted_at_ms,
1571
+ '' AS event_type, '' AS payload_json, '' AS meta_json, content_hash
1572
+ FROM flows_journal_dedup
1573
+ WHERE event_id = ${queued.eventId}
1574
+ OR (
1575
+ run_id = ${queued.runId}
1576
+ AND source_id = ${queued.sourceId}
1577
+ AND source_seq = ${queued.sourceSeq}
1578
+ )
1579
+ ORDER BY seq ASC
1580
+ LIMIT 1
1581
+ `
1582
+ if (existing.length === 0) {
1583
+ return undefined
1584
+ }
1585
+ const row = existing[0]!
1586
+ if (
1587
+ queued.dedupe !== "identity" && (
1588
+ row.content_hash === null
1589
+ ? row.event_type !== queued.eventType ||
1590
+ row.payload_json !== queued.payloadJson ||
1591
+ row.meta_json !== queued.metaJson
1592
+ : row.content_hash !==
1593
+ (yield* contentFingerprint(queued.eventType, queued.payloadJson, queued.metaJson))
1594
+ )
1595
+ ) {
1596
+ return yield* Effect.fail(
1597
+ error(
1598
+ "idempotency_conflict",
1599
+ `source event ${queued.sourceId}:${queued.sourceSeq} for run ${queued.runId} was reused with different content`
1600
+ )
1601
+ )
1602
+ }
1603
+ return {
1604
+ entry: row.content_hash === null ? yield* decodeRow(row) : { seq: Number(row.seq) as Seq },
1605
+ inserted: false
1606
+ }
1607
+ })
1608
+
1609
+ /**
1610
+ * When the fence is `Owned` the insert is admitted by the consensus
1611
+ * strategy's `guard` in the same serialized write transaction, following
1612
+ * Temporal's shard `rangeID` check
1613
+ * (`service/history/shard/context_impl.go`, `renewRangeLocked`): a
1614
+ * zombie owner whose run was reclaimed cannot append, and fails with
1615
+ * `fence_lost` before any row is written.
1616
+ *
1617
+ * The fence outranks dedup. The guard runs BEFORE the INSERT, so a
1618
+ * fenced insert consults the dedup index only for an owner the strategy
1619
+ * has confirmed; Temporal conditions every request on the `rangeID`
1620
+ * before anything else. Answering a zombie's resubmission from the dedup
1621
+ * index would launder its lost fence into a `Duplicate` receipt for work
1622
+ * the live owner committed; a confirmed owner's conflict, by contrast,
1623
+ * is a genuine duplicate (its own earlier commit, or a forked run's
1624
+ * copied row).
1625
+ *
1626
+ * The queued channel is decided by the constraint alone. The ownerless
1627
+ * DURABLE path keeps its up-front lookup, and the reason is measured
1628
+ * rather than aesthetic: with two connections open on one file, removing
1629
+ * it left `emitDurableUnfenced` transactions overlapping, so one of them
1630
+ * blocked in SQLite's synchronous busy wait until the driver's
1631
+ * `busy_timeout` expired. The two-connection case in `JournalDurable`
1632
+ * went from 17 ms and no write retries to 5.4 s and one, three runs each
1633
+ * way. That is a read this channel is buying scheduling room with, and
1634
+ * it is affordable here: a durable emit already reads its allocation
1635
+ * floor in the same transaction, and its caller is by definition not
1636
+ * flushing from inside somebody else's.
1637
+ */
1638
+ const insertOne = (
1639
+ queued: QueuedEntry,
1640
+ fence: Fence,
1641
+ enforceCompactionFloor = false
1642
+ ): Effect.Effect<Commit, JournalError | SqlError.SqlError> =>
1643
+ Effect.gen(function*() {
1644
+ if (fence._tag === "Owned") {
1645
+ yield* fenceGuard(queued.runId, fence.owner)
1646
+ } else if (!enforceCompactionFloor) {
1647
+ const duplicate = yield* selectExisting(queued)
1648
+ if (duplicate !== undefined) {
1649
+ return duplicate
1650
+ }
1651
+ }
1652
+ // Admission reservations are provisional. Allocate above the durable
1653
+ // tail in this transaction so a queued entry can never commit behind
1654
+ // a reader's cursor, even when another connection overtook it.
1655
+ const insert = enforceCompactionFloor
1656
+ ? sql<JournalRow>`
1657
+ INSERT INTO flows_journal_events (
1658
+ run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
1659
+ event_type, payload_json, meta_json
1660
+ )
1661
+ SELECT
1662
+ ${queued.runId},
1663
+ (SELECT ${Dialect.greatest(sql)}(${queued.seq}, COALESCE(MAX(seq), -1) + 1)
1664
+ FROM flows_journal_events WHERE run_id = ${queued.runId}),
1665
+ ${queued.eventId},
1666
+ ${queued.sourceId},
1667
+ ${queued.sourceSeq},
1668
+ ${queued.emittedAtMs},
1669
+ ${queued.eventType},
1670
+ ${queued.payloadJson},
1671
+ ${queued.metaJson}
1672
+ WHERE NOT EXISTS (
1673
+ SELECT 1 FROM flows_journal_checkpoints
1674
+ WHERE run_id = ${queued.runId}
1675
+ AND compacted_at_ms IS NOT NULL
1676
+ AND seq >= ${queued.seq}
1677
+ ) AND NOT EXISTS (
1678
+ SELECT 1 FROM flows_journal_events
1679
+ WHERE run_id = ${queued.runId} AND seq >= ${Number.MAX_SAFE_INTEGER - 1}
1680
+ )
1681
+ ON CONFLICT DO NOTHING
1682
+ RETURNING run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
1683
+ event_type, payload_json, meta_json
1684
+ `
1685
+ : sql<JournalRow>`
1686
+ INSERT INTO flows_journal_events (
1687
+ run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
1688
+ event_type, payload_json, meta_json
1689
+ ) VALUES (
1690
+ ${queued.runId},
1691
+ ${queued.seq},
1692
+ ${queued.eventId},
1693
+ ${queued.sourceId},
1694
+ ${queued.sourceSeq},
1695
+ ${queued.emittedAtMs},
1696
+ ${queued.eventType},
1697
+ ${queued.payloadJson},
1698
+ ${queued.metaJson}
1699
+ )
1700
+ ON CONFLICT DO NOTHING
1701
+ RETURNING run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
1702
+ event_type, payload_json, meta_json
1703
+ `
1704
+ const inserted = yield* insert
1705
+ if (inserted.length > 0) {
1706
+ return {
1707
+ entry: yield* decodeRow(inserted[0]!),
1708
+ inserted: true
1709
+ }
1710
+ }
1711
+ // The INSERT produced no row, so a unique constraint fired: the
1712
+ // guard above already confirmed a fenced owner, so this collision
1713
+ // is a genuine duplicate, never a laundered lost fence.
1714
+ const racedDuplicate = yield* selectExisting(queued)
1715
+ if (racedDuplicate !== undefined) {
1716
+ return racedDuplicate
1717
+ }
1718
+ // The insert produced no row and names no committed identity. Before
1719
+ // reporting sequence exhaustion, ask whether compaction moved the
1720
+ // floor above this reservation: that is a drop, and
1721
+ // the caller needs the floor to resync from. A durable write cannot
1722
+ // reach this case, because it allocates above the surviving
1723
+ // checkpoint row, so the read costs nothing on that path.
1724
+ {
1725
+ const floor = yield* compactionFloor(queued.runId)
1726
+ if (floor !== undefined && queued.seq <= floor) {
1727
+ return yield* Effect.fail(
1728
+ new JournalError({
1729
+ code: "compacted",
1730
+ message:
1731
+ `queued sequence ${queued.seq} for run ${queued.runId} was dropped below compaction floor ${floor}`,
1732
+ checkpointSeq: floor as Seq
1733
+ })
1734
+ )
1735
+ }
1736
+ }
1737
+ return yield* Effect.fail(
1738
+ error(
1739
+ "invalid_event",
1740
+ "journal sequence is outside the allocatable safe integer range"
1741
+ )
1742
+ )
1743
+ })
1744
+
1745
+ /** Shared transactional insert path; the caller already owns SQL. */
1746
+ const insertBatch = (batch: ReadonlyArray<QueuedEntry>): Effect.Effect<BatchOutcome, SqlError.SqlError> =>
1747
+ Effect.gen(function*() {
1748
+ const results = yield* Effect.forEach(
1749
+ batch,
1750
+ (queued) => Effect.map(Effect.result(insertOne(queued, unfenced, true)), (result) => ({ queued, result }))
1751
+ )
1752
+ const commits: Array<SettledCommit> = []
1753
+ const losses: Array<EntryLoss> = []
1754
+ for (const settled of results) {
1755
+ if (Result.isSuccess(settled.result)) {
1756
+ commits.push({ queued: settled.queued, commit: settled.result.success })
1757
+ } else if (isJournalError(settled.result.failure)) {
1758
+ losses.push({ queued: settled.queued, cause: settled.result.failure })
1759
+ } else {
1760
+ return yield* Effect.fail(settled.result.failure)
1761
+ }
1762
+ }
1763
+ return { commits, losses }
1764
+ })
1765
+
1766
+ // Transaction first, run permits second. Admissions read floors before
1767
+ // taking a permit; compaction also acquires SQL before its run locks.
1768
+ // No holder of a run permit or closed gate can wait on this batch's
1769
+ // connection (see `admitFromIndex` and `withCompactionBarrier`).
1770
+ const persistBatch = (
1771
+ batch: ReadonlyArray<QueuedEntry>
1772
+ ): Effect.Effect<BatchOutcome, JournalError> =>
1773
+ writer.write(withRunPermits(
1774
+ batch.map((queued) => queued.runId),
1775
+ insertBatch(batch)
1776
+ )).pipe(
1777
+ // Every per-entry `JournalError` is settled inside the transaction
1778
+ // above, so the only failure that escapes is the transaction itself:
1779
+ // a database outage, which is what `sink_failed` names.
1780
+ Effect.mapError((cause) => error("sink_failed", "journal sink failed", cause))
1781
+ )
1782
+
1783
+ const publish = (commits: ReadonlyArray<Commit>): Effect.Effect<void> =>
1784
+ Effect.forEach(
1785
+ commits,
1786
+ (commit) => {
1787
+ if (!commit.inserted) {
1788
+ return Effect.void
1789
+ }
1790
+ return PubSub.publish(changes, freezePublished(commit.entry)).pipe(
1791
+ Effect.andThen(
1792
+ Effect.forEach(
1793
+ wakes.get(commit.entry.runId) ?? [],
1794
+ (wake) => PubSub.publish(wake, undefined),
1795
+ { discard: true }
1796
+ )
1797
+ ),
1798
+ Effect.asVoid
1799
+ )
1800
+ },
1801
+ { discard: true }
1802
+ )
1803
+
1804
+ const rememberCommitted = (queued: QueuedEntry, seq: Seq): void => {
1805
+ retain(sourceEventKey(queued.runId, queued.sourceId, queued.sourceSeq), {
1806
+ seq,
1807
+ eventType: queued.eventType,
1808
+ payloadJson: queued.payloadJson,
1809
+ metaJson: queued.metaJson,
1810
+ status: "committed"
1811
+ })
1812
+ raiseSequenceFloor(queued.runId, seq)
1813
+ raiseSourceSequenceFloor(queued.runId, queued.sourceId, queued.sourceSeq)
1814
+ }
1815
+
1816
+ /**
1817
+ * Reads the durable allocation floor for a run (or a producer). Durable
1818
+ * allocation calls this inside its transaction; lossy admission calls
1819
+ * it before taking any run permit.
1820
+ *
1821
+ * The pinned Node SQLite driver begins IMMEDIATE transactions, acquiring
1822
+ * the writer lock before this floor read. A caller-supplied SQL client
1823
+ * may instead use DEFERRED transactions, whose lock upgrade can fail
1824
+ * with SQLITE_BUSY_SNAPSHOT. In either case DurableWriter retries the
1825
+ * whole transaction, floor read included, against committed state.
1826
+ * The cross-connection allocation invariant is exercised by
1827
+ * `packages/smithers/flows/journal/test/JournalDurable.test.ts` ("emitDurable never
1828
+ * collides when two connections write one run concurrently").
1829
+ *
1830
+ * Governing design: `packages/smithers/flows/journal/docs/concepts/two-channels.md`.
1831
+ */
1832
+ const nextDurable = (
1833
+ column: "seq" | "source_seq",
1834
+ runId: RunId,
1835
+ sourceId: SourceId | undefined
1836
+ ): Effect.Effect<number, SqlError.SqlError> =>
1837
+ Effect.map(
1838
+ column === "seq"
1839
+ ? sql<{ readonly next: number | null }>`
1840
+ SELECT MAX(seq) + 1 AS next FROM flows_journal_events WHERE run_id = ${runId}
1841
+ `
1842
+ : sql<{ readonly next: number | null }>`
1843
+ SELECT MAX(source_seq) + 1 AS next FROM (
1844
+ SELECT MAX(source_seq) AS source_seq FROM flows_journal_events
1845
+ WHERE run_id = ${runId} AND source_id = ${sourceId!}
1846
+ UNION ALL
1847
+ SELECT MAX(source_seq) AS source_seq FROM flows_journal_dedup
1848
+ WHERE run_id = ${runId} AND source_id = ${sourceId!}
1849
+ )
1850
+ `,
1851
+ (rows) => Number(rows[0]?.next ?? 0)
1852
+ )
1853
+
1854
+ /**
1855
+ * Reads a run's durable allocation floors when the in-process cache has
1856
+ * not observed them yet.
1857
+ *
1858
+ * `emitLossy` allocates from the index alone, it queues rather than
1859
+ * writing, so it cannot read the database mid-allocation, which is why
1860
+ * the floors used to be seeded for every run at construction. Reading
1861
+ * them on first use instead keeps the index proportional to the runs this
1862
+ * process touches rather than to total history, and the durable read is
1863
+ * the same `MAX(...) + 1` the seed computed.
1864
+ *
1865
+ * This function never mutates the cache. Its SQL reads deliberately run
1866
+ * without the allocator and without the run permit; the caller later
1867
+ * takes the run permit and then the short allocator permit, re-checks
1868
+ * the monotonic cache, and reserves both sequences in one synchronous
1869
+ * step. A floor read outside the permit is a lower bound: every
1870
+ * in-process writer raises the cache at allocation time, and the only
1871
+ * cache invalidation is `JournalGeneration.forget`, whose contract
1872
+ * requires quiescent producers and flushed admissions, so it cannot
1873
+ * overlap this read-to-reservation interval. Retention evicts committed
1874
+ * identities only; run retirement and compaction leave allocation floors
1875
+ * intact. Compaction also preserves the checkpoint entry and producer
1876
+ * dedup rows for cold reads. Reading under the permit is what deadlocked
1877
+ * against the batch writer; see `admitFromIndex`.
1878
+ *
1879
+ * A producer that supplies its own `sourceSeq` allocates nothing from
1880
+ * the producer floor, so that floor is not read for it. This is what
1881
+ * makes the explicit path readless end to end once the run's own `seq`
1882
+ * floor is known, which it is from the first entry any producer writes
1883
+ * for the run.
1884
+ */
1885
+ const ensureFloors = (
1886
+ input: Input
1887
+ ): Effect.Effect<{ readonly seq: number; readonly sourceSeq: number }, JournalError> =>
1888
+ Effect.suspend(() => {
1889
+ const runId = input.runId
1890
+ const key = sourceKey(runId, input.sourceId)
1891
+ const seq = state.sequences.get(runId)
1892
+ const sourceSeq = input.sourceSeq === undefined ? state.sourceSequences.get(key) : 0
1893
+ if (seq !== undefined && sourceSeq !== undefined) {
1894
+ return Effect.succeed({ seq, sourceSeq })
1895
+ }
1896
+ return Effect.all({
1897
+ seq: seq === undefined ? nextDurable("seq", runId, undefined) : Effect.succeed(seq),
1898
+ sourceSeq: sourceSeq === undefined
1899
+ ? nextDurable("source_seq", runId, input.sourceId)
1900
+ : Effect.succeed(sourceSeq)
1901
+ }).pipe(
1902
+ Effect.mapError((cause) => error("sink_failed", "could not read journal allocation floor", cause)),
1903
+ Effect.flatMap((floors) => {
1904
+ if (
1905
+ !Number.isSafeInteger(floors.seq) ||
1906
+ floors.seq < 0 ||
1907
+ !Number.isSafeInteger(floors.sourceSeq) ||
1908
+ floors.sourceSeq < 0
1909
+ ) {
1910
+ return Effect.fail(
1911
+ error("invalid_event", "journal sequence is outside the allocatable safe integer range")
1912
+ )
1913
+ }
1914
+ return Effect.succeed(floors)
1915
+ })
1916
+ )
1917
+ })
1918
+
1919
+ /**
1920
+ * Records a committed entry in the in-process index and publishes it,
1921
+ * immediately outside a transaction, or deferred to its owning writer.
1922
+ * A raw SQL transaction has no managed commit boundary: skip this optional
1923
+ * publication rather than exposing uncommitted data. Database replay stays
1924
+ * authoritative, including for tail subscribers.
1925
+ */
1926
+ const settleCommit = (
1927
+ queued: QueuedEntry,
1928
+ commit: Commit,
1929
+ restoreMaintenance: (effect: Effect.Effect<void>) => Effect.Effect<void>
1930
+ ): Effect.Effect<Effect.Effect<void>> =>
1931
+ Effect.gen(function*() {
1932
+ const mandatory = Effect.sync(() => rememberCommitted(queued, commit.entry.seq)).pipe(
1933
+ Effect.andThen(publish([commit]))
1934
+ )
1935
+ // Restore the durable caller's policy, not unconditional interruption.
1936
+ // A cancellation finalizer may journal before releasing resources;
1937
+ // even interruptible(void) here re-delivers its pending interruption
1938
+ // after COMMIT and abandons that cleanup. Ordinary callers still make
1939
+ // a slow capture interruptible when the writer publishes after COMMIT.
1940
+ const maintenance = restoreMaintenance(noteCommitted(queued.runId, commit.inserted ? 1 : 0))
1941
+ const enclosing = yield* Effect.serviceOption(sql.transactionService)
1942
+ if (Option.isNone(enclosing)) {
1943
+ yield* mandatory
1944
+ return maintenance
1945
+ }
1946
+ yield* afterCommit(mandatory.pipe(Effect.andThen(maintenance)), sql)
1947
+ return Effect.void
1948
+ })
1949
+
1950
+ /**
1951
+ * Opens (or joins) the write transaction that keeps the logical WAL
1952
+ * atomic with the executable state it describes.
1953
+ *
1954
+ * The shared writer owns publication across retries, nested savepoint
1955
+ * rollbacks, and transactions opened by other stores on the same client.
1956
+ */
1957
+ const transact: Service["transact"] = <A, E, R>(
1958
+ effect: Effect.Effect<A, E, R>
1959
+ ): Effect.Effect<A, E | JournalError, R> =>
1960
+ writer.write(effect).pipe(
1961
+ Effect.catchIf(
1962
+ (cause): cause is DatabaseError => cause instanceof DatabaseError,
1963
+ (cause) => Effect.fail(error("sink_failed", "journal transaction failed", cause))
1964
+ )
1965
+ )
1966
+
1967
+ const writeDurable = (
1968
+ input: Input,
1969
+ fence: Fence
1970
+ ): Effect.Effect<DurableReceipt, JournalError> =>
1971
+ Effect.uninterruptibleMask((restoreMaintenance) =>
1972
+ restoreMaintenance(Effect.gen(function*() {
1973
+ yield* Effect.annotateCurrentSpan({
1974
+ runId: input.runId,
1975
+ sourceId: input.sourceId,
1976
+ eventType: input.eventType
1977
+ })
1978
+ const emittedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
1979
+ const prepared = yield* Effect.fromResult(prepare(input, emittedAtMs))
1980
+ const committed = yield* Effect.gen(function*() {
1981
+ const { metaJson, payloadJson, validated } = prepared
1982
+ return yield* Effect.uninterruptibleMask((restore) =>
1983
+ restore(writer.write(withActiveRunWrite(
1984
+ prepared.validated.runId,
1985
+ Effect.gen(function*() {
1986
+ // Read only for a producer that allocates from this floor. A
1987
+ // supplied sequence is not allocated, so reading the floor
1988
+ // for it costs a query whose answer is discarded.
1989
+ const durableSourceSeq = validated.sourceSeq === undefined
1990
+ ? yield* nextDurable("source_seq", validated.runId, validated.sourceId)
1991
+ : 0
1992
+ const durableSeq = yield* nextDurable("seq", validated.runId, undefined)
1993
+ const reserved = yield* allocation.withPermit(Effect.fromResult(Result.gen(function*() {
1994
+ const key = sourceKey(validated.runId, validated.sourceId)
1995
+ const sourceSeq: SourceSeq = validated.sourceSeq ??
1996
+ (Math.max(
1997
+ durableSourceSeq,
1998
+ state.sourceSequences.get(key) ?? 0
1999
+ ) as SourceSeq)
2000
+ if (
2001
+ !Number.isSafeInteger(sourceSeq) ||
2002
+ sourceSeq < 0 ||
2003
+ sourceSeq === Number.MAX_SAFE_INTEGER
2004
+ ) {
2005
+ return yield* Result.fail(
2006
+ error("invalid_event", "journal sequence is outside the allocatable safe integer range")
2007
+ )
2008
+ }
2009
+ const seq = Math.max(
2010
+ durableSeq,
2011
+ state.sequences.get(validated.runId) ?? 0
2012
+ ) as Seq
2013
+ if (!Number.isSafeInteger(seq) || seq === Number.MAX_SAFE_INTEGER) {
2014
+ return yield* Result.fail(
2015
+ error("invalid_event", "journal sequence is outside the allocatable safe integer range")
2016
+ )
2017
+ }
2018
+ // Claim the seq NOW, not at commit: a concurrent `emitLossy`
2019
+ // allocates from this floor alone, and `settleCommit` parks
2020
+ // the commit-time raise until the outermost COMMIT.
2021
+ // Re-entering the transaction body is idempotent because the
2022
+ // floor only rises. An abandoned attempt leaves the number
2023
+ // unused, which is a gap: allocation is `MAX(seq) + 1` and
2024
+ // replay is `ORDER BY seq`, so neither reads a gap as anything.
2025
+ raiseSequenceFloor(validated.runId, seq)
2026
+ // Claim the producer sequence at the same allocation seam.
2027
+ // Without this, a lossy emit from the same producer can read
2028
+ // the pre-transaction source floor and reuse this identity
2029
+ // while an enclosing `transact` is still open.
2030
+ raiseSourceSequenceFloor(validated.runId, validated.sourceId, sourceSeq)
2031
+ return { seq, sourceSeq }
2032
+ })))
2033
+ const { seq, sourceSeq } = reserved
2034
+ const queued: QueuedEntry = {
2035
+ runId: validated.runId,
2036
+ seq,
2037
+ eventId: makeEventId(validated.runId, validated.sourceId, sourceSeq),
2038
+ sourceId: validated.sourceId,
2039
+ sourceSeq,
2040
+ emittedAtMs,
2041
+ eventType: validated.eventType,
2042
+ payloadJson,
2043
+ metaJson,
2044
+ dedupe: validated.dedupe ?? "content"
2045
+ }
2046
+ const commit = yield* insertOne(queued, fence)
2047
+ return { commit, queued, sourceSeq }
2048
+ })
2049
+ ))).pipe(
2050
+ /**
2051
+ * `writer.write` is a retrying transaction: its body replays on
2052
+ * `SQLITE_BUSY(_SNAPSHOT)` and can still abort at COMMIT after the
2053
+ * body succeeded. Cache mutation and publication therefore happen
2054
+ * strictly after the transaction returns, so subscribers never
2055
+ * observe a rolled-back entry and a replayed body never publishes
2056
+ * twice. Mirrors the queued path, which publishes in a `.tap`
2057
+ * outside `persistBatch`. Only the transaction is restored: an
2058
+ * interruption that lands after it returns waits for the index
2059
+ * update and publication, so a committed row always reaches
2060
+ * `changes`.
2061
+ *
2062
+ * Under `transact` "after the transaction returns" is not yet
2063
+ * "after COMMIT": this write is a savepoint of the caller's
2064
+ * transaction, so `settleCommit` parks both effects until the
2065
+ * outermost transaction commits. The active-write reference
2066
+ * is released before that outer commit, which lets automatic
2067
+ * compaction drain the run before its maintenance transaction.
2068
+ */
2069
+ Effect.flatMap((written) =>
2070
+ Effect.gen(function*() {
2071
+ const maintenance = yield* settleCommit(written.queued, written.commit, restoreMaintenance)
2072
+ const receipt: DurableReceipt = written.commit.inserted
2073
+ ? { _tag: "Accepted", seq: written.commit.entry.seq, sourceSeq: written.sourceSeq }
2074
+ : {
2075
+ _tag: "Duplicate",
2076
+ seq: written.commit.entry.seq,
2077
+ sourceSeq: written.sourceSeq,
2078
+ status: "committed"
2079
+ }
2080
+ yield* Metric.update(JournalMetrics.durable[receipt._tag], 1)
2081
+ return { maintenance, receipt }
2082
+ })
2083
+ )
2084
+ )
2085
+ )
2086
+ }).pipe(
2087
+ Effect.mapError((cause) =>
2088
+ isJournalError(cause) ? cause : error("sink_failed", "durable journal write failed", cause)
2089
+ )
2090
+ )
2091
+ yield* committed.maintenance
2092
+ return committed.receipt
2093
+ }))
2094
+ )
2095
+
2096
+ const emitDurable: Service["emitDurable"] = Effect.fn("Journal.emitDurable")((
2097
+ input: Input,
2098
+ owner: OwnerId
2099
+ ) =>
2100
+ Effect.flatMap(
2101
+ Effect.fromResult(requireFence(owner, "emitDurable")),
2102
+ (fence) => writeDurable(input, fence)
2103
+ )
2104
+ )
2105
+
2106
+ const emitDurableUnfenced: Service["emitDurableUnfenced"] = Effect.fn("Journal.emitDurableUnfenced")((
2107
+ input: Input
2108
+ ) => writeDurable(input, unfenced))
2109
+
2110
+ const emitLossy: Service["emitLossy"] = queuedEmit
2111
+
2112
+ const checkpointInternal = (
2113
+ checkpointOptions: CheckpointOptions,
2114
+ fence: Fence
2115
+ ): Effect.Effect<Checkpoint, JournalError> =>
2116
+ Effect.gen(function*() {
2117
+ yield* Effect.annotateCurrentSpan({
2118
+ runId: checkpointOptions.runId,
2119
+ seq: checkpointOptions.seq
2120
+ })
2121
+ yield* Effect.fromResult(Result.mapError(
2122
+ decodeCheckpointOptions(checkpointOptions),
2123
+ (cause) => error("invalid_event", "checkpoint options violate the journal contract", cause)
2124
+ ))
2125
+ // The state round-trips verbatim: it is replay input, so redaction
2126
+ // deliberately does not apply, rewriting it would resume the run
2127
+ // with the wrong data. A secret that must not persist belongs in a
2128
+ // `Redacted` field of the caller's own state schema.
2129
+ const stateJson = yield* Effect.fromResult(encodeJson(checkpointOptions.state, "state"))
2130
+ const receiptState = yield* Schema.decodeUnknownEffect(UnknownFromJsonString)(stateJson).pipe(
2131
+ Effect.mapError(
2132
+ /* v8 ignore next -- encodeJson produced these valid JSON bytes immediately above */
2133
+ (cause) => error("decode_failed", "could not decode persisted checkpoint state", cause)
2134
+ )
2135
+ )
2136
+ const createdAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
2137
+ return yield* writer.write(Effect.gen(function*() {
2138
+ if (fence._tag === "Owned") {
2139
+ yield* fenceGuard(checkpointOptions.runId, fence.owner)
2140
+ }
2141
+ // The target must be a committed entry: the surviving row is what
2142
+ // keeps the run's durable `MAX(seq)` allocation floor at or above
2143
+ // the compaction boundary, so a process restarted after
2144
+ // compaction can never re-allocate a truncated sequence.
2145
+ const target = yield* sql<{ readonly ok: number }>`
2146
+ SELECT 1 AS ok FROM flows_journal_events
2147
+ WHERE run_id = ${checkpointOptions.runId} AND seq = ${checkpointOptions.seq}
2148
+ `
2149
+ if (target.length === 0) {
2150
+ return yield* Effect.fail(error(
2151
+ "checkpoint_invalid",
2152
+ `checkpoint sequence ${checkpointOptions.seq} names no committed entry of run ${checkpointOptions.runId}`
2153
+ ))
2154
+ }
2155
+ const floor = yield* compactionFloor(checkpointOptions.runId)
2156
+ if (floor !== undefined && checkpointOptions.seq <= floor) {
2157
+ return yield* Effect.fail(
2158
+ new JournalError({
2159
+ code: "checkpoint_invalid",
2160
+ message: `run ${checkpointOptions.runId} is already compacted through sequence ${floor}`,
2161
+ checkpointSeq: floor as Seq
2162
+ })
2163
+ )
2164
+ }
2165
+ yield* sql`
2166
+ INSERT INTO flows_journal_checkpoints (run_id, seq, state_json, created_at_ms)
2167
+ VALUES (${checkpointOptions.runId}, ${checkpointOptions.seq}, ${stateJson}, ${createdAtMs})
2168
+ ON CONFLICT (run_id, seq) DO UPDATE SET
2169
+ state_json = excluded.state_json,
2170
+ created_at_ms = excluded.created_at_ms
2171
+ `
2172
+ return new Checkpoint({
2173
+ runId: checkpointOptions.runId,
2174
+ seq: checkpointOptions.seq,
2175
+ state: receiptState,
2176
+ createdAtMs,
2177
+ compactedAtMs: null
2178
+ })
2179
+ })).pipe(
2180
+ Effect.mapError((cause) =>
2181
+ isJournalError(cause) ? cause : error("sink_failed", "durable checkpoint write failed", cause)
2182
+ )
2183
+ )
2184
+ })
2185
+
2186
+ const checkpoint: Service["checkpoint"] = Effect.fn("Journal.checkpoint")((
2187
+ checkpointOptions: CheckpointOptions,
2188
+ owner: OwnerId
2189
+ ) =>
2190
+ Effect.flatMap(
2191
+ Effect.fromResult(requireFence(owner, "checkpoint")),
2192
+ (fence) => checkpointInternal(checkpointOptions, fence)
2193
+ )
2194
+ )
2195
+
2196
+ const latestCheckpoint: Service["latestCheckpoint"] = Effect.fn("Journal.latestCheckpoint")((runId: RunId) =>
2197
+ Effect.gen(function*() {
2198
+ yield* Effect.annotateCurrentSpan({ runId })
2199
+ yield* Effect.fromResult(Result.mapError(
2200
+ decodeRunId(runId),
2201
+ (cause) => error("invalid_event", "runId violates the journal identifier contract", cause)
2202
+ ))
2203
+ const rows = yield* sql<CheckpointRow>`
2204
+ SELECT run_id, seq, state_json, created_at_ms, compacted_at_ms
2205
+ FROM flows_journal_checkpoints
2206
+ WHERE run_id = ${runId}
2207
+ ORDER BY seq DESC
2208
+ LIMIT 1
2209
+ `.pipe(Effect.mapError((cause) => error("read_failed", "durable checkpoint read failed", cause)))
2210
+ const row = rows[0]
2211
+ return row === undefined ? Option.none() : Option.some(yield* decodeCheckpointRow(row))
2212
+ })
2213
+ )
2214
+
2215
+ const compactInternal = (
2216
+ compactOptions: CompactOptions,
2217
+ fence: Fence
2218
+ ): Effect.Effect<Compacted, JournalError> =>
2219
+ Effect.gen(function*() {
2220
+ yield* Effect.annotateCurrentSpan({
2221
+ runId: compactOptions.runId,
2222
+ ...(compactOptions.upTo === undefined ? {} : { upTo: compactOptions.upTo })
2223
+ })
2224
+ yield* Effect.fromResult(Result.mapError(
2225
+ decodeCompactOptions(compactOptions),
2226
+ (cause) => error("invalid_event", "compact options violate the journal contract", cause)
2227
+ ))
2228
+ const compactedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
2229
+ return yield* withCompactionBarrier(
2230
+ compactOptions.runId,
2231
+ Effect.gen(function*() {
2232
+ if (fence._tag === "Owned") {
2233
+ yield* fenceGuard(compactOptions.runId, fence.owner)
2234
+ }
2235
+ const upTo = compactOptions.upTo
2236
+ const rows = upTo === undefined
2237
+ ? yield* sql<CheckpointRow>`
2238
+ SELECT run_id, seq, state_json, created_at_ms, compacted_at_ms
2239
+ FROM flows_journal_checkpoints
2240
+ WHERE run_id = ${compactOptions.runId}
2241
+ ORDER BY seq DESC
2242
+ LIMIT 1
2243
+ `
2244
+ : yield* sql<CheckpointRow>`
2245
+ SELECT run_id, seq, state_json, created_at_ms, compacted_at_ms
2246
+ FROM flows_journal_checkpoints
2247
+ WHERE run_id = ${compactOptions.runId} AND seq = ${upTo}
2248
+ `
2249
+ const row = rows[0]
2250
+ if (row === undefined) {
2251
+ return yield* Effect.fail(error(
2252
+ "checkpoint_invalid",
2253
+ `run ${compactOptions.runId} has no checkpoint${
2254
+ upTo === undefined ? "" : ` at sequence ${upTo}`
2255
+ } to compact to`
2256
+ ))
2257
+ }
2258
+ const checkpointSeq = Number(row.seq) as Seq
2259
+ if (row.compacted_at_ms !== null) {
2260
+ // A retried compaction: the floor is already here and the rows
2261
+ // below it are already gone.
2262
+ return { runId: compactOptions.runId, checkpointSeq, deleted: 0 } satisfies Compacted
2263
+ }
2264
+ for (const reader of readers.get(compactOptions.runId) ?? []) {
2265
+ if (reader.cursor < checkpointSeq - 1) {
2266
+ return yield* Effect.fail(
2267
+ new JournalError({
2268
+ code: "reader_behind",
2269
+ message:
2270
+ `a live stream of run ${compactOptions.runId} still needs sequences below checkpoint ${checkpointSeq}`,
2271
+ checkpointSeq
2272
+ })
2273
+ )
2274
+ }
2275
+ }
2276
+ const doomed = yield* sql<{ readonly total: number }>`
2277
+ SELECT COUNT(*) AS total FROM flows_journal_events
2278
+ WHERE run_id = ${compactOptions.runId} AND seq < ${checkpointSeq}
2279
+ `
2280
+ // Retain identities atomically with deletion, in bounded pages.
2281
+ // These records never participate in replay, but preserve exact
2282
+ // retries and producer allocation floors after a fresh open.
2283
+ let after = -1
2284
+ while (true) {
2285
+ const retiring = yield* sql<JournalRow>`
2286
+ SELECT run_id, seq, event_id, source_id, source_seq, emitted_at_ms,
2287
+ event_type, payload_json, meta_json
2288
+ FROM flows_journal_events
2289
+ WHERE run_id = ${compactOptions.runId} AND seq > ${after} AND seq < ${checkpointSeq}
2290
+ ORDER BY seq ASC LIMIT 256
2291
+ `
2292
+ if (retiring.length === 0) break
2293
+ for (const entry of retiring) {
2294
+ const fingerprint = yield* contentFingerprint(entry.event_type, entry.payload_json, entry.meta_json)
2295
+ yield* sql`
2296
+ INSERT INTO flows_journal_dedup (run_id, source_id, source_seq, event_id, seq, content_hash)
2297
+ VALUES (${entry.run_id}, ${entry.source_id}, ${entry.source_seq}, ${entry.event_id},
2298
+ ${entry.seq}, ${fingerprint})
2299
+ `
2300
+ }
2301
+ after = Number(retiring[retiring.length - 1]!.seq)
2302
+ }
2303
+ // Strictly below the checkpoint: the checkpointed entry survives,
2304
+ // holding the run's `MAX(seq)` allocation floor. Superseded
2305
+ // checkpoints go with their entries; the truncation and the floor
2306
+ // advance are one transaction, so a crash between them is
2307
+ // unrepresentable.
2308
+ yield* sql`
2309
+ DELETE FROM flows_journal_events
2310
+ WHERE run_id = ${compactOptions.runId} AND seq < ${checkpointSeq}
2311
+ `
2312
+ yield* sql`
2313
+ DELETE FROM flows_journal_checkpoints
2314
+ WHERE run_id = ${compactOptions.runId} AND seq < ${checkpointSeq}
2315
+ `
2316
+ yield* sql`
2317
+ UPDATE flows_journal_checkpoints
2318
+ SET compacted_at_ms = ${compactedAtMs}
2319
+ WHERE run_id = ${compactOptions.runId} AND seq = ${checkpointSeq}
2320
+ `
2321
+ return {
2322
+ runId: compactOptions.runId,
2323
+ checkpointSeq,
2324
+ deleted: Number(doomed[0]?.total ?? 0)
2325
+ } satisfies Compacted
2326
+ })
2327
+ ).pipe(
2328
+ Effect.mapError((cause) =>
2329
+ isJournalError(cause) ? cause : error("sink_failed", "journal compaction failed", cause)
2330
+ )
2331
+ )
2332
+ })
2333
+
2334
+ const compact: Service["compact"] = Effect.fn("Journal.compact")((
2335
+ compactOptions: CompactOptions,
2336
+ owner: OwnerId
2337
+ ) =>
2338
+ Effect.flatMap(
2339
+ Effect.fromResult(requireFence(owner, "compact")),
2340
+ (fence) => compactInternal(compactOptions, fence)
2341
+ )
2342
+ )
2343
+
2344
+ const compactionPolicy = options.compaction
2345
+
2346
+ const countEntries = (runId: RunId): Effect.Effect<number, SqlError.SqlError> =>
2347
+ Effect.map(
2348
+ sql<{ readonly total: number }>`
2349
+ SELECT COUNT(*) AS total FROM flows_journal_events WHERE run_id = ${runId}
2350
+ `,
2351
+ (rows) => Number(rows[0]?.total ?? 0)
2352
+ )
2353
+
2354
+ /**
2355
+ * One automatic checkpoint-and-compact attempt at the run's durable
2356
+ * tail. Runs post-commit; a failure or refusal is logged and damped,
2357
+ * the counter restarts, so the next attempt waits for another
2358
+ * `entryThreshold` commits, and is never surfaced to the emit whose
2359
+ * settlement crossed the threshold.
2360
+ */
2361
+ const policyCompact = (policy: CompactionPolicy, runId: RunId): Effect.Effect<void> =>
2362
+ Effect.gen(function*() {
2363
+ const tail = yield* sql<{ readonly last: number | null }>`
2364
+ SELECT MAX(seq) AS last FROM flows_journal_events WHERE run_id = ${runId}
2365
+ `
2366
+ const last = tail[0]?.last
2367
+ if (last === null || last === undefined) {
2368
+ return
2369
+ }
2370
+ const upTo = Number(last) as Seq
2371
+ const captured = yield* Effect.timeoutOrElse(policy.capture(runId, upTo), {
2372
+ duration: compactionCaptureTimeout,
2373
+ orElse: () =>
2374
+ Effect.fail(
2375
+ new JournalInternalFault({
2376
+ code: "compaction_capture_timeout",
2377
+ message: `journal compaction capture for run ${runId} exceeded ${compactionCaptureTimeout}`
2378
+ })
2379
+ )
2380
+ })
2381
+ // The policy is the journal's OWN post-commit maintenance, not a
2382
+ // caller's mutating entrypoint: it owns no run and holds no fence,
2383
+ // so it drives the internal channel. The attempt only ever
2384
+ // truncates below a tail the run's own commits produced, a retry is
2385
+ // idempotent (a re-attempt after a reclaim compacts the same
2386
+ // committed prefix the live owner also sees), and every failure is
2387
+ // damped below, never surfaced to the emit that crossed the
2388
+ // threshold.
2389
+ yield* checkpointInternal({ runId, seq: upTo, state: captured }, unfenced)
2390
+ yield* compactInternal({ runId, upTo }, unfenced)
2391
+ compactionCounts.set(runId, yield* countEntries(runId))
2392
+ }).pipe(
2393
+ Effect.catchCause((cause) =>
2394
+ Cause.hasInterruptsOnly(cause)
2395
+ ? Effect.failCause(cause as Cause.Cause<never>)
2396
+ : Effect.sync(() => {
2397
+ compactionCounts.set(runId, 0)
2398
+ }).pipe(
2399
+ Effect.andThen(
2400
+ Effect.logWarning("journal auto-compaction failed; retrying after the next threshold", cause)
2401
+ )
2402
+ )
2403
+ )
2404
+ )
2405
+
2406
+ /**
2407
+ * Counts a run's committed entries toward the compaction policy and
2408
+ * triggers an attempt at the threshold. Lossy settlements register and
2409
+ * fork it; durable settlements await it. Registration and handoff are
2410
+ * uninterruptible so cancellation cannot leak the maintenance count.
2411
+ *
2412
+ * The count is seeded lazily from the durable COUNT on the run's first
2413
+ * committed entry in this process, mirroring `ensureFloors`, so a
2414
+ * restarted process still compacts a long pre-existing history. The
2415
+ * durable COUNT already includes the rows the caller is reporting: they
2416
+ * committed before this settlement ran.
2417
+ *
2418
+ * The read-modify-write runs under the run's `maintenance` permit because
2419
+ * the seeding COUNT is an awaited SQL read. Without the permit two
2420
+ * settlements that overlapped both observed an unseeded counter, both
2421
+ * issued the COUNT, and the later reply overwrote the newer count with a
2422
+ * stale one: measured with `entryThreshold: 10`, ten committed events
2423
+ * produced zero capture calls and no checkpoint. A settlement that
2424
+ * commits BETWEEN the seeding COUNT and the counter write is now counted
2425
+ * twice instead, which only brings an attempt forward, and
2426
+ * {@link policyCompact} re-seeds the counter from a fresh COUNT when it
2427
+ * finishes, so the error does not accumulate.
2428
+ *
2429
+ * The permit is released before the compaction runs. Holding it across
2430
+ * `policyCompact` would make a slow `capture` block every other
2431
+ * settlement of the same run, which is the stall {@link settleCommit}
2432
+ * moved this work off the allocation permit to avoid.
2433
+ */
2434
+ const noteCommitted = (runId: RunId, committed: number, background = false): Effect.Effect<void> => {
2435
+ const policy = compactionPolicy
2436
+ if (policy === undefined || committed <= 0) {
2437
+ return Effect.void
2438
+ }
2439
+ return Effect.uninterruptibleMask((restore) =>
2440
+ withRunBarrier(runId, (barrier) =>
2441
+ restore(barrier.maintenance.withPermit(
2442
+ Effect.gen(function*() {
2443
+ if (compactingRuns.has(runId)) {
2444
+ return false
2445
+ }
2446
+ const known = compactionCounts.get(runId)
2447
+ const current = known === undefined ? yield* countEntries(runId) : known + committed
2448
+ compactionCounts.set(runId, current)
2449
+ if (current < policy.entryThreshold) {
2450
+ return false
2451
+ }
2452
+ compactingRuns.add(runId)
2453
+ pendingMaintenance += 1
2454
+ return true
2455
+ })
2456
+ )).pipe(
2457
+ Effect.flatMap((triggered) => {
2458
+ if (!triggered) return Effect.void
2459
+ const attempt = policyCompact(policy, runId).pipe(Effect.ensuring(Effect.sync(() => {
2460
+ compactingRuns.delete(runId)
2461
+ pendingMaintenance -= 1
2462
+ if (state.pending === 0 && pendingMaintenance === 0) completeFlushWaiters(Effect.void)
2463
+ })))
2464
+ // The sole queue consumer must never await a compaction barrier:
2465
+ // that barrier may need another batch from this very consumer.
2466
+ return background
2467
+ ? Effect.asVoid(Effect.forkIn(Effect.interruptible(attempt), maintenanceScope))
2468
+ : restore(attempt)
2469
+ }),
2470
+ Effect.catchCause((cause) =>
2471
+ Cause.hasInterruptsOnly(cause)
2472
+ ? Effect.failCause(cause as Cause.Cause<never>)
2473
+ : Effect.logWarning("journal compaction policy bookkeeping failed", cause)
2474
+ )
2475
+ ))
2476
+ )
2477
+ }
2478
+
2479
+ const recordCommits = (
2480
+ commits: ReadonlyArray<SettledCommit>
2481
+ ): void => {
2482
+ for (const { commit, queued } of commits) {
2483
+ rememberCommitted(queued, commit.entry.seq)
2484
+ }
2485
+ }
2486
+
2487
+ const settle = (count: number): void => {
2488
+ state.pending = Math.max(0, state.pending - count)
2489
+ if (state.pending === 0 && pendingMaintenance === 0) {
2490
+ completeFlushWaiters(Effect.void)
2491
+ }
2492
+ }
2493
+
2494
+ /** Reports entries that left the queue without durable commits. */
2495
+ const reportLoss = (cause: JournalError, batch: ReadonlyArray<QueuedEntry>): void => {
2496
+ for (const queued of batch) {
2497
+ const identity = sourceEventKey(queued.runId, queued.sourceId, queued.sourceSeq)
2498
+ // Allocation is serialized until this batch settles, so no newer
2499
+ // admission can replace this exact identity before the deletion.
2500
+ state.sourceEvents.delete(identity)
2501
+ }
2502
+ state.sinkFailure = cause
2503
+ state.lossEpoch += 1
2504
+ state.pending = Math.max(0, state.pending - batch.length)
2505
+ // A waiter that is already registered is the flush the loss belongs
2506
+ // to, so reporting it there spends the report; only a loss nobody was
2507
+ // waiting on is left for the next flush to pick up.
2508
+ if (state.flushWaiters.size > 0) {
2509
+ state.flushedLossEpoch = state.lossEpoch
2510
+ }
2511
+ completeFlushWaiters(Effect.fail(cause))
2512
+ for (const subscribers of wakes.values()) {
2513
+ for (const wake of subscribers) {
2514
+ PubSub.publishUnsafe(wake, undefined)
2515
+ }
2516
+ }
2517
+ }
2518
+
2519
+ /**
2520
+ * Makes a lost batch visible to an operator. `reportLoss` only reaches a
2521
+ * flush caller or a live stream, and a telemetry producer usually has
2522
+ * neither, so the loss is also logged and counted here.
2523
+ */
2524
+ // Callers supply one refused entry or a batch taken with a minimum of one.
2525
+ const observeLoss = (cause: JournalError, batch: ReadonlyArray<QueuedEntry>): Effect.Effect<void> =>
2526
+ Effect.andThen(
2527
+ Metric.update(JournalMetrics.lost(cause.code), batch.length),
2528
+ Effect.logWarning(
2529
+ `journal lost ${batch.length} lossy entries (${cause.code}) for runs ${
2530
+ [...new Set(batch.map((queued) => queued.runId))].join(", ")
2531
+ }`,
2532
+ cause
2533
+ )
2534
+ )
2535
+
2536
+ // A failed transaction loses the whole batch. Release its per-run drain
2537
+ // counts before reporting the failure so a waiting compactor can retry
2538
+ // against the same database outage instead of waiting forever.
2539
+ const failSink = (cause: JournalError, batch: ReadonlyArray<QueuedEntry>): void => {
2540
+ settleRunPending(batch)
2541
+ reportLoss(cause, batch)
2542
+ }
2543
+
2544
+ // One failed batch loses that batch and is reported as such; it never
2545
+ // ends the writer. Only interruption (scope closure) stops the loop, so
2546
+ // the queue keeps draining as soon as the database is healthy again.
2547
+ const writeBatch = Queue.takeBetween(queue, 1, batchSize).pipe(
2548
+ Effect.flatMap((batch) =>
2549
+ persistBatch(batch).pipe(
2550
+ Effect.tap((outcome) => Effect.sync(() => recordCommits(outcome.commits))),
2551
+ Effect.tap((outcome) => publish(outcome.commits.map(({ commit }) => commit))),
2552
+ // The transaction has committed and publication is complete. Drop
2553
+ // the barrier counts before policy compaction takes this run's
2554
+ // permit. Maintenance has its own global flush count.
2555
+ Effect.tap(() => Effect.sync(() => settleRunPending(batch))),
2556
+ // Register maintenance BEFORE settling the batch, so flush cannot
2557
+ // observe a gap between queued work and its policy attempt.
2558
+ Effect.tap((outcome) => {
2559
+ const perRun = new Map<RunId, number>()
2560
+ outcome.commits.forEach(({ commit, queued }) => {
2561
+ if (!commit.inserted) {
2562
+ return
2563
+ }
2564
+ perRun.set(queued.runId, (perRun.get(queued.runId) ?? 0) + 1)
2565
+ })
2566
+ return Effect.forEach(perRun, ([runId, committed]) => noteCommitted(runId, committed, true), {
2567
+ discard: true
2568
+ })
2569
+ }),
2570
+ Effect.tap((outcome) =>
2571
+ // Observed before it is reported, so a flush the report wakes
2572
+ // already sees the loss in the log and the registry.
2573
+ Effect.uninterruptible(Effect.andThen(
2574
+ Effect.forEach(outcome.losses, (loss) => observeLoss(loss.cause, [loss.queued]), { discard: true }),
2575
+ Effect.sync(() => {
2576
+ for (const loss of outcome.losses) {
2577
+ reportLoss(loss.cause, [loss.queued])
2578
+ }
2579
+ settle(outcome.commits.length)
2580
+ })
2581
+ ))
2582
+ ),
2583
+ Effect.catch((cause) =>
2584
+ Effect.uninterruptible(
2585
+ Effect.andThen(observeLoss(cause, batch), Effect.sync(() => failSink(cause, batch)))
2586
+ )
2587
+ ),
2588
+ // Defects only: an interruption is scope closure, and it must end
2589
+ // the writer rather than be reported as a lost batch.
2590
+ Effect.catchDefect((defect) => {
2591
+ const cause = error("sink_failed", "journal writer failed", Cause.die(defect))
2592
+ return Effect.uninterruptible(
2593
+ Effect.andThen(observeLoss(cause, batch), Effect.sync(() => failSink(cause, batch)))
2594
+ )
2595
+ })
2596
+ )
2597
+ )
2598
+ )
2599
+
2600
+ const drain = Effect.forever(writeBatch)
2601
+
2602
+ yield* Effect.forkScoped(drain)
2603
+ yield* Effect.addFinalizer(() =>
2604
+ Effect.gen(function*() {
2605
+ // A scope finalizer runs once, and nothing else moves the status, so
2606
+ // the journal is always `open` here.
2607
+ yield* Effect.sync(() => {
2608
+ state.status = "closing"
2609
+ })
2610
+ // The last flush is the last chance to say that queued entries were
2611
+ // lost; nothing is left to report the failure to once this returns.
2612
+ yield* flushInternal.pipe(
2613
+ Effect.catch((cause) => Effect.logWarning("journal final flush failed while closing", cause))
2614
+ )
2615
+ yield* Effect.sync(() => {
2616
+ state.status = "closed"
2617
+ })
2618
+ yield* Queue.shutdown(queue)
2619
+ yield* PubSub.shutdown(changes)
2620
+ wakes.clear()
2621
+ })
2622
+ )
2623
+
2624
+ const project = <S, E, R>(
2625
+ projection: Projection<S, E, R>,
2626
+ streamOptions: StreamOptions
2627
+ ): Stream.Stream<S, JournalError, R> =>
2628
+ Stream.unwrap(
2629
+ Effect.fn("Journal.project")(
2630
+ <S2, E2, R2>(
2631
+ activeProjection: Projection<S2, E2, R2>,
2632
+ activeOptions: StreamOptions
2633
+ ) =>
2634
+ Effect.annotateCurrentSpan({
2635
+ projection: activeProjection.name,
2636
+ runId: activeOptions.runId,
2637
+ ...(activeOptions.afterSequence === undefined ? {} : { afterSequence: activeOptions.afterSequence })
2638
+ }).pipe(Effect.andThen(Effect.succeed(
2639
+ stream(activeOptions).pipe(
2640
+ Stream.scanEffect(activeProjection.initial, (state, entry) =>
2641
+ Effect.suspend(() => activeProjection.reduce(state, entry)).pipe(
2642
+ Effect.catchCause((cause) =>
2643
+ Cause.hasInterruptsOnly(cause)
2644
+ ? Effect.failCause(cause as Cause.Cause<never>)
2645
+ : Effect.fail(
2646
+ error(
2647
+ "projection_failed",
2648
+ `projection ${activeProjection.name} failed`,
2649
+ cause
2650
+ )
2651
+ )
2652
+ )
2653
+ ))
2654
+ )
2655
+ )))
2656
+ )(projection, streamOptions)
2657
+ )
2658
+
2659
+ yield* JournalGeneration.onTruncate((runIds) => {
2660
+ for (const runId of runIds) {
2661
+ state.sequences.delete(runId as RunId)
2662
+ const prefix = `${runId.length}:${runId}`
2663
+ for (const key of state.sourceSequences.keys()) {
2664
+ if (key.startsWith(prefix)) state.sourceSequences.delete(key)
2665
+ }
2666
+ for (const key of state.sourceEvents.keys()) {
2667
+ if (key.startsWith(prefix)) state.sourceEvents.delete(key)
2668
+ }
2669
+ }
2670
+ })
2671
+
2672
+ return makeJournal({
2673
+ generation: (runId) =>
2674
+ sql<{ readonly generation: number; readonly afterSeq: number }>`
2675
+ SELECT generation, after_seq AS "afterSeq" FROM flows_journal_generations WHERE run_id = ${runId}
2676
+ `.pipe(
2677
+ Effect.map((rows) => rows[0] ?? { generation: 0, afterSeq: -1 }),
2678
+ Effect.mapError((cause) => error("read_failed", "could not read journal generation", cause))
2679
+ ),
2680
+ emitLossy,
2681
+ emitDurable,
2682
+ emitDurableUnfenced,
2683
+ transact,
2684
+ whenCommitted: (update) =>
2685
+ Effect.flatMap(
2686
+ Effect.serviceOption(sql.transactionService),
2687
+ (transaction) => Option.isNone(transaction) ? Effect.as(update, true) : afterCommit(update, sql)
2688
+ ),
2689
+ stream,
2690
+ entries: readPage,
2691
+ changes: PubSub.subscribe(changes),
2692
+ project,
2693
+ flush: Effect.fn("Journal.flush")(() =>
2694
+ Effect.suspend(() => Effect.annotateCurrentSpan({ pending: state.pending })).pipe(
2695
+ Effect.andThen(flushInternal)
2696
+ )
2697
+ )(),
2698
+ checkpoint,
2699
+ latestCheckpoint,
2700
+ compact
2701
+ })
2702
+ })
2703
+
2704
+ /**
2705
+ * Provides the SQL journal over an explicitly injected consensus strategy.
2706
+ *
2707
+ * The strategy in context arbitrates the owner fence: `emitDurable`,
2708
+ * `checkpoint`, and `compact` join its `guard` inside their write
2709
+ * transaction. Provide the SAME strategy instance to every service that must
2710
+ * agree on ownership — `RunStore.layerWith` in particular — because
2711
+ * `Consensus.layerLocal` keeps its leases per layer build.
2712
+ *
2713
+ * @category layers
2714
+ * @since 1.0.0
2715
+ */
2716
+ export const layerWith = (
2717
+ options: SqlJournalOptions
2718
+ ): Layer.Layer<Journal, JournalError, Consensus | DurableWriter | SqlClient.SqlClient> =>
2719
+ Layer.effect(Journal, make(options))
2720
+
2721
+ /**
2722
+ * Provides the SQL journal fenced by the database-backed strategy,
2723
+ * `SqlConsensus`, over the same client. Use {@link layerWith} to inject a
2724
+ * different strategy.
2725
+ *
2726
+ * @category layers
2727
+ * @since 0.1.0
2728
+ */
2729
+ export const layer = (
2730
+ options: SqlJournalOptions
2731
+ ): Layer.Layer<Journal, JournalError, DurableWriter | SqlClient.SqlClient> =>
2732
+ layerWith(options).pipe(Layer.provide(SqlConsensus.layer))