@bjornpagen/bumbledb 1.3.0 → 2.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (290) hide show
  1. package/COOKBOOK.md +224 -95
  2. package/README.md +114 -45
  3. package/dist/alternatives.d.ts +5 -6
  4. package/dist/alternatives.d.ts.map +1 -1
  5. package/dist/alternatives.js +5 -6
  6. package/dist/alternatives.js.map +1 -1
  7. package/dist/bin.d.ts +3 -0
  8. package/dist/bin.d.ts.map +1 -0
  9. package/dist/bin.js +85 -0
  10. package/dist/bin.js.map +1 -0
  11. package/dist/capacity.d.ts +14 -17
  12. package/dist/capacity.d.ts.map +1 -1
  13. package/dist/capacity.js +7 -7
  14. package/dist/capacity.js.map +1 -1
  15. package/dist/changes.d.ts +32 -41
  16. package/dist/changes.d.ts.map +1 -1
  17. package/dist/changes.js +86 -110
  18. package/dist/changes.js.map +1 -1
  19. package/dist/closed.d.ts +2 -2
  20. package/dist/closed.d.ts.map +1 -1
  21. package/dist/closed.js +5 -5
  22. package/dist/closed.js.map +1 -1
  23. package/dist/codec.d.ts +20 -27
  24. package/dist/codec.d.ts.map +1 -1
  25. package/dist/codec.js +48 -22
  26. package/dist/codec.js.map +1 -1
  27. package/dist/compile.d.ts +26 -50
  28. package/dist/compile.d.ts.map +1 -1
  29. package/dist/compile.js +75 -64
  30. package/dist/compile.js.map +1 -1
  31. package/dist/database/database.d.ts +98 -0
  32. package/dist/database/database.d.ts.map +1 -0
  33. package/dist/database/database.js +249 -0
  34. package/dist/database/database.js.map +1 -0
  35. package/dist/database/driver.d.ts +40 -0
  36. package/dist/database/driver.d.ts.map +1 -0
  37. package/dist/database/driver.js +68 -0
  38. package/dist/database/driver.js.map +1 -0
  39. package/dist/database/fs.d.ts +12 -0
  40. package/dist/database/fs.d.ts.map +1 -0
  41. package/dist/database/fs.js +120 -0
  42. package/dist/database/fs.js.map +1 -0
  43. package/dist/database/generate.d.ts +26 -0
  44. package/dist/database/generate.d.ts.map +1 -0
  45. package/dist/database/generate.js +102 -0
  46. package/dist/database/generate.js.map +1 -0
  47. package/dist/database/hosted.d.ts +32 -0
  48. package/dist/database/hosted.d.ts.map +1 -0
  49. package/dist/database/hosted.js +44 -0
  50. package/dist/database/hosted.js.map +1 -0
  51. package/dist/database/io.d.ts +94 -0
  52. package/dist/database/io.d.ts.map +1 -0
  53. package/dist/database/io.js +35 -0
  54. package/dist/database/io.js.map +1 -0
  55. package/dist/database/mem.d.ts +26 -0
  56. package/dist/database/mem.d.ts.map +1 -0
  57. package/dist/database/mem.js +67 -0
  58. package/dist/database/mem.js.map +1 -0
  59. package/dist/database/migration.d.ts +55 -0
  60. package/dist/database/migration.d.ts.map +1 -0
  61. package/dist/database/migration.js +46 -0
  62. package/dist/database/migration.js.map +1 -0
  63. package/dist/database/s3.d.ts +36 -0
  64. package/dist/database/s3.d.ts.map +1 -0
  65. package/dist/database/s3.js +141 -0
  66. package/dist/database/s3.js.map +1 -0
  67. package/dist/db.d.ts +44 -128
  68. package/dist/db.d.ts.map +1 -1
  69. package/dist/db.js +163 -287
  70. package/dist/db.js.map +1 -1
  71. package/dist/engine.d.ts +7 -0
  72. package/dist/engine.d.ts.map +1 -0
  73. package/dist/engine.js +2 -0
  74. package/dist/engine.js.map +1 -0
  75. package/dist/errors.d.ts +152 -46
  76. package/dist/errors.d.ts.map +1 -1
  77. package/dist/errors.js +107 -19
  78. package/dist/errors.js.map +1 -1
  79. package/dist/face.d.ts +5 -5
  80. package/dist/face.d.ts.map +1 -1
  81. package/dist/face.js +6 -6
  82. package/dist/face.js.map +1 -1
  83. package/dist/fields.d.ts +4 -5
  84. package/dist/fields.d.ts.map +1 -1
  85. package/dist/fields.js +3 -3
  86. package/dist/fields.js.map +1 -1
  87. package/dist/index.d.ts +60 -58
  88. package/dist/index.d.ts.map +1 -1
  89. package/dist/index.js +27 -29
  90. package/dist/index.js.map +1 -1
  91. package/dist/judgment.d.ts +1 -1
  92. package/dist/judgment.js +1 -1
  93. package/dist/law.d.ts +16 -68
  94. package/dist/law.d.ts.map +1 -1
  95. package/dist/law.js +3 -3
  96. package/dist/law.js.map +1 -1
  97. package/dist/lower.d.ts +3 -3
  98. package/dist/lower.d.ts.map +1 -1
  99. package/dist/lower.js +35 -39
  100. package/dist/lower.js.map +1 -1
  101. package/dist/native/addon.d.ts +23 -0
  102. package/dist/native/addon.d.ts.map +1 -0
  103. package/dist/native/addon.js +15 -0
  104. package/dist/native/addon.js.map +1 -0
  105. package/dist/native/binding.d.ts +996 -0
  106. package/dist/native/json.d.ts +14 -0
  107. package/dist/native/json.d.ts.map +1 -0
  108. package/dist/native/json.js +73 -0
  109. package/dist/native/json.js.map +1 -0
  110. package/dist/native/load.d.ts +9 -0
  111. package/dist/native/load.d.ts.map +1 -0
  112. package/dist/native/load.js +30 -0
  113. package/dist/native/load.js.map +1 -0
  114. package/dist/native/op.d.ts +29 -0
  115. package/dist/native/op.d.ts.map +1 -0
  116. package/dist/native/op.js +50 -0
  117. package/dist/native/op.js.map +1 -0
  118. package/dist/query/atom.d.ts +13 -14
  119. package/dist/query/atom.d.ts.map +1 -1
  120. package/dist/query/atom.js +2 -2
  121. package/dist/query/atom.js.map +1 -1
  122. package/dist/query/compute.d.ts +7 -7
  123. package/dist/query/compute.d.ts.map +1 -1
  124. package/dist/query/compute.js +7 -16
  125. package/dist/query/compute.js.map +1 -1
  126. package/dist/query/find.d.ts +8 -8
  127. package/dist/query/find.d.ts.map +1 -1
  128. package/dist/query/find.js +1 -1
  129. package/dist/query/lower.d.ts +31 -17
  130. package/dist/query/lower.d.ts.map +1 -1
  131. package/dist/query/lower.js +260 -368
  132. package/dist/query/lower.js.map +1 -1
  133. package/dist/query/run.d.ts +5 -5
  134. package/dist/query/run.d.ts.map +1 -1
  135. package/dist/query/run.js +32 -15
  136. package/dist/query/run.js.map +1 -1
  137. package/dist/query/scope.d.ts +20 -28
  138. package/dist/query/scope.d.ts.map +1 -1
  139. package/dist/query/scope.js +13 -8
  140. package/dist/query/scope.js.map +1 -1
  141. package/dist/query/segments.d.ts +2 -2
  142. package/dist/query/segments.d.ts.map +1 -1
  143. package/dist/query/segments.js +3 -3
  144. package/dist/query/segments.js.map +1 -1
  145. package/dist/relation.d.ts +1 -1
  146. package/dist/relation.d.ts.map +1 -1
  147. package/dist/relation.js +4 -4
  148. package/dist/relation.js.map +1 -1
  149. package/dist/result.d.ts +15 -22
  150. package/dist/result.d.ts.map +1 -1
  151. package/dist/result.js +30 -39
  152. package/dist/result.js.map +1 -1
  153. package/dist/rows.d.ts +9 -33
  154. package/dist/rows.d.ts.map +1 -1
  155. package/dist/rows.js +8 -12
  156. package/dist/rows.js.map +1 -1
  157. package/dist/runtime.d.ts +17 -43
  158. package/dist/runtime.d.ts.map +1 -1
  159. package/dist/runtime.js +41 -241
  160. package/dist/runtime.js.map +1 -1
  161. package/dist/scalar.d.ts +36 -106
  162. package/dist/scalar.d.ts.map +1 -1
  163. package/dist/scalar.js +36 -185
  164. package/dist/scalar.js.map +1 -1
  165. package/dist/schema.d.ts +7 -18
  166. package/dist/schema.d.ts.map +1 -1
  167. package/dist/schema.js +18 -228
  168. package/dist/schema.js.map +1 -1
  169. package/dist/selection.d.ts +4 -4
  170. package/dist/selection.d.ts.map +1 -1
  171. package/dist/selection.js +5 -5
  172. package/dist/selection.js.map +1 -1
  173. package/dist/shape.d.ts +5 -5
  174. package/dist/shape.d.ts.map +1 -1
  175. package/dist/spec.d.ts +7 -79
  176. package/dist/spec.d.ts.map +1 -1
  177. package/dist/spec.js.map +1 -1
  178. package/dist/statements.d.ts +3 -3
  179. package/dist/statements.d.ts.map +1 -1
  180. package/dist/statements.js +10 -10
  181. package/dist/statements.js.map +1 -1
  182. package/dist/uuid.d.ts +1 -1
  183. package/dist/uuid.d.ts.map +1 -1
  184. package/dist/uuid.js +1 -1
  185. package/dist/uuid.js.map +1 -1
  186. package/dist/values.d.ts +2 -2
  187. package/dist/values.d.ts.map +1 -1
  188. package/dist/values.js +2 -2
  189. package/dist/values.js.map +1 -1
  190. package/package.json +18 -18
  191. package/src/alternatives.ts +7 -8
  192. package/src/bin.ts +82 -0
  193. package/src/capacity.ts +20 -23
  194. package/src/changes.ts +134 -197
  195. package/src/closed.ts +5 -5
  196. package/src/codec.ts +77 -51
  197. package/src/compile.ts +91 -113
  198. package/src/database/database.ts +413 -0
  199. package/src/database/driver.ts +134 -0
  200. package/src/database/fs.ts +119 -0
  201. package/src/database/generate.ts +116 -0
  202. package/src/database/hosted.ts +102 -0
  203. package/src/database/io.ts +134 -0
  204. package/src/database/mem.ts +113 -0
  205. package/src/database/migration.ts +107 -0
  206. package/src/database/s3.ts +231 -0
  207. package/src/db.ts +290 -547
  208. package/src/engine.ts +14 -0
  209. package/src/errors.ts +118 -39
  210. package/src/face.ts +9 -9
  211. package/src/fields.ts +6 -7
  212. package/src/index.ts +102 -109
  213. package/src/judgment.ts +1 -1
  214. package/src/law.ts +18 -70
  215. package/src/lower.ts +61 -72
  216. package/src/native/addon.ts +54 -0
  217. package/src/native/binding.d.ts +996 -0
  218. package/src/native/json.ts +100 -0
  219. package/src/native/load.ts +31 -0
  220. package/src/native/op.ts +73 -0
  221. package/src/query/atom.ts +15 -16
  222. package/src/query/compute.ts +13 -22
  223. package/src/query/find.ts +8 -8
  224. package/src/query/lower.ts +323 -414
  225. package/src/query/run.ts +42 -21
  226. package/src/query/scope.ts +37 -38
  227. package/src/query/segments.ts +4 -4
  228. package/src/relation.ts +4 -4
  229. package/src/result.ts +53 -80
  230. package/src/rows.ts +18 -46
  231. package/src/runtime.ts +50 -346
  232. package/src/scalar.ts +76 -336
  233. package/src/schema.ts +23 -284
  234. package/src/selection.ts +6 -6
  235. package/src/shape.ts +5 -5
  236. package/src/spec.ts +7 -85
  237. package/src/statements.ts +11 -11
  238. package/src/uuid.ts +1 -1
  239. package/src/values.ts +4 -4
  240. package/dist/close.d.ts +0 -24
  241. package/dist/close.d.ts.map +0 -1
  242. package/dist/close.js +0 -34
  243. package/dist/close.js.map +0 -1
  244. package/dist/db-native.d.ts +0 -196
  245. package/dist/db-native.d.ts.map +0 -1
  246. package/dist/db-native.js +0 -13
  247. package/dist/db-native.js.map +0 -1
  248. package/dist/internal/log.d.ts +0 -26
  249. package/dist/internal/log.d.ts.map +0 -1
  250. package/dist/internal/log.js +0 -12
  251. package/dist/internal/log.js.map +0 -1
  252. package/dist/migration.d.ts +0 -31
  253. package/dist/migration.d.ts.map +0 -1
  254. package/dist/migration.js +0 -47
  255. package/dist/migration.js.map +0 -1
  256. package/dist/native.d.ts +0 -562
  257. package/dist/native.d.ts.map +0 -1
  258. package/dist/native.js +0 -123
  259. package/dist/native.js.map +0 -1
  260. package/dist/query/description.d.ts +0 -41
  261. package/dist/query/description.d.ts.map +0 -1
  262. package/dist/query/description.js +0 -311
  263. package/dist/query/description.js.map +0 -1
  264. package/dist/query/parse-ir.d.ts +0 -10
  265. package/dist/query/parse-ir.d.ts.map +0 -1
  266. package/dist/query/parse-ir.js +0 -280
  267. package/dist/query/parse-ir.js.map +0 -1
  268. package/dist/runtime-codes.d.ts +0 -2
  269. package/dist/runtime-codes.d.ts.map +0 -1
  270. package/dist/runtime-codes.js +0 -18
  271. package/dist/runtime-codes.js.map +0 -1
  272. package/dist/runtime-errors.d.ts +0 -118
  273. package/dist/runtime-errors.d.ts.map +0 -1
  274. package/dist/runtime-errors.js +0 -98
  275. package/dist/runtime-errors.js.map +0 -1
  276. package/dist/runtime-native.d.ts +0 -141
  277. package/dist/runtime-native.d.ts.map +0 -1
  278. package/dist/runtime-native.js +0 -5
  279. package/dist/runtime-native.js.map +0 -1
  280. package/pack-provenance.json +0 -7
  281. package/src/close.ts +0 -53
  282. package/src/db-native.ts +0 -243
  283. package/src/internal/log.ts +0 -48
  284. package/src/migration.ts +0 -61
  285. package/src/native.ts +0 -736
  286. package/src/query/description.ts +0 -424
  287. package/src/query/parse-ir.ts +0 -291
  288. package/src/runtime-codes.ts +0 -17
  289. package/src/runtime-errors.ts +0 -109
  290. package/src/runtime-native.ts +0 -189
@@ -0,0 +1,413 @@
1
+ /**
2
+ * A durable database: an immutable log in an object store, read through a local LMDB cache that is
3
+ * only a cache. Writes are submitted commands decided once per request id; reads pin snapshots of
4
+ * the cache at a chosen freshness. Migrations are bundled with the code and run on open (`migrate`)
5
+ * or are required to have run already (`verify`).
6
+ */
7
+ import { randomBytes } from "node:crypto"
8
+ import type { Scope } from "effect"
9
+ import { Context, Duration, Effect, Layer, Option, RcMap, Schema } from "effect"
10
+ import type { ChangeSet } from "../changes.ts"
11
+ import { ChangeSetLive, ChangeSet as ChangeSets } from "../changes.ts"
12
+ import { compiledOf } from "../compile.ts"
13
+ import type { QueryReader } from "../db.ts"
14
+ import { scopedSnapshot } from "../db.ts"
15
+ import { argumentError, DbError, dbError } from "../errors.ts"
16
+ import { lower } from "../lower.ts"
17
+ import { addon } from "../native/addon.ts"
18
+ import type { HeadOut, HostedOpenIn, ReceiptOut, RefusalOut, SettledOut } from "../native/binding.d.ts"
19
+ import { call, scoped } from "../native/op.ts"
20
+ import type { Bumble } from "../runtime.ts"
21
+ import { runtimeHandle } from "../runtime.ts"
22
+ import type { AnySchema } from "../schema.ts"
23
+ import { schemaDescriptor, schemasAgree } from "../schema.ts"
24
+ import { Driver } from "./driver.ts"
25
+ import type { HostedRef, HostedRequest } from "./hosted.ts"
26
+ import { hostedPort } from "./hosted.ts"
27
+ import type { ExecutorOptions, ObjectStore } from "./io.ts"
28
+ import { defaultExecutorOptions } from "./io.ts"
29
+ import type { Migrations } from "./migration.ts"
30
+ import { populateOf, verifyBundle } from "./migration.ts"
31
+
32
+ /** A submission's idempotency key: 32 lowercase hex digits. */
33
+ const RequestId = Schema.String.check(Schema.isPattern(/^[0-9a-f]{32}$/)).pipe(Schema.brand("RequestId"))
34
+ type RequestId = typeof RequestId.Type
35
+
36
+ /** How fresh a read must be: the cache as it is, the log's tip, or at least a submitted `seq`. */
37
+ type Consistency = "cached" | "latest" | { readonly atLeast: bigint }
38
+
39
+ /** A decided submission (its receipt), or a refusal. Every refusal but `Unknown` proves the log does not decide it. */
40
+ type SubmitOutcome = Extract<SettledOut, { readonly _tag: "Decided" | "Refused" }>
41
+
42
+ interface SubmitOptions {
43
+ readonly requestId: RequestId
44
+ /** Decide only if the database is exactly at this revision. */
45
+ readonly precondition?: bigint
46
+ }
47
+
48
+ /** A snapshot reader, with the log position and revision the cache had reached when it was pinned. */
49
+ interface DatabaseReader<S extends AnySchema> extends QueryReader<S> {
50
+ readonly seq: bigint
51
+ /** The committed revision; a submit can require it with `precondition`. */
52
+ readonly revision: bigint
53
+ }
54
+
55
+ interface Database<S extends AnySchema> {
56
+ readonly schema: S
57
+ /** The last decided state this process has seen. */
58
+ readonly head: Effect.Effect<HeadOut>
59
+ submit(changes: ChangeSet<S>, options: SubmitOptions): Effect.Effect<SubmitOutcome, DbError>
60
+ read(consistency?: Consistency): Effect.Effect<DatabaseReader<S>, DbError, Scope.Scope>
61
+ /** The receipt of `requestId`, after catching up with the log. */
62
+ resolve(requestId: RequestId): Effect.Effect<Option.Option<ReceiptOut>, DbError>
63
+ }
64
+
65
+ interface Tuning {
66
+ /** Parallel tail reads while catching up. */
67
+ readonly probeWindow: number
68
+ /** Log entries between automatic checkpoints. */
69
+ readonly checkpointEvery: bigint
70
+ readonly checkpointKeep: number
71
+ readonly maxEntryBytes: number
72
+ /** Store calls in flight at once. */
73
+ readonly concurrency: number
74
+ readonly executor: ExecutorOptions
75
+ /** Stale migration attempts before writers are frozen for one more attempt. */
76
+ readonly migrationAttempts: number
77
+ readonly freezeLease: Duration.Input
78
+ }
79
+
80
+ const defaultTuning: Tuning = {
81
+ probeWindow: 32,
82
+ checkpointEvery: 256n,
83
+ checkpointKeep: 3,
84
+ maxEntryBytes: 4 * 1024 * 1024,
85
+ concurrency: 32,
86
+ executor: defaultExecutorOptions,
87
+ migrationAttempts: 3,
88
+ freezeLease: "5 minutes"
89
+ }
90
+
91
+ interface DatabaseOptions<S extends AnySchema> {
92
+ /** The schema the code reads and writes; the last migration's schema. */
93
+ readonly schema: S
94
+ readonly migrations: Migrations
95
+ readonly store: ObjectStore
96
+ /** Where the disposable local cache lives. */
97
+ readonly cache: { readonly directory: string }
98
+ /** `migrate` creates the database and runs pending migrations; `verify` refuses to open with any pending. */
99
+ readonly onOpen: "verify" | "migrate"
100
+ readonly tuning?: Partial<Tuning>
101
+ }
102
+
103
+ function describeRefusal(refusal: RefusalOut): string {
104
+ switch (refusal._tag) {
105
+ case "Frozen":
106
+ return `writes are frozen until ${refusal.deadline}`
107
+ case "MigrationPending":
108
+ return `migration ${refusal.next} has not run`
109
+ case "MigrationRejected":
110
+ return `migration ${refusal.migration.name} was rejected: ${refusal.evidence.violations.map((violation) => violation.spelling).join("; ")}`
111
+ case "MigrationsDiverged":
112
+ return `bundled migration ${refusal.index} differs from the one the log applied`
113
+ case "Stale":
114
+ return `the log moved to ${refusal.head}`
115
+ case "RequestReused":
116
+ return `request ${refusal.request} was decided for another command`
117
+ case "Cache":
118
+ return refusal.message
119
+ case "Corrupt":
120
+ return `log entry ${refusal.seq} is corrupt`
121
+ default:
122
+ return refusal._tag
123
+ }
124
+ }
125
+
126
+ /** A refusal of the machine, as a `DbError`. */
127
+ function refused(operation: string, refusal: RefusalOut): DbError {
128
+ return new DbError({ operation, reason: { _tag: "Engine", kind: refusal._tag, message: describeRefusal(refusal) } })
129
+ }
130
+
131
+ function hex(bytes: number): string {
132
+ return randomBytes(bytes).toString("hex")
133
+ }
134
+
135
+ function settledAs<T extends SettledOut["_tag"]>(
136
+ operation: string,
137
+ settled: SettledOut,
138
+ tag: T
139
+ ): Effect.Effect<Extract<SettledOut, { readonly _tag: T }>, DbError> {
140
+ if (settled._tag === tag) return Effect.succeed(settled as Extract<SettledOut, { readonly _tag: T }>)
141
+ if (settled._tag === "Refused") return Effect.fail(refused(operation, settled.refusal))
142
+ return Effect.fail(dbError(operation, { _tag: "Internal" }))
143
+ }
144
+
145
+ class DatabaseLive<S extends AnySchema> implements Database<S> {
146
+ readonly schema: S
147
+ readonly #hosted: HostedRef
148
+ readonly #run: (request: HostedRequest) => Effect.Effect<SettledOut, DbError>
149
+ readonly #head: () => HeadOut | undefined
150
+
151
+ constructor(
152
+ schema: S,
153
+ hosted: HostedRef,
154
+ run: (request: HostedRequest) => Effect.Effect<SettledOut, DbError>,
155
+ head: () => HeadOut | undefined
156
+ ) {
157
+ this.schema = schema
158
+ this.#hosted = hosted
159
+ this.#run = run
160
+ this.#head = head
161
+ }
162
+
163
+ get head(): Effect.Effect<HeadOut> {
164
+ return Effect.suspend(() => {
165
+ const head = this.#head()
166
+ return head === undefined ? Effect.die(dbError("Database.head", { _tag: "Internal" })) : Effect.succeed(head)
167
+ })
168
+ }
169
+
170
+ submit(changes: ChangeSet<S>, options: SubmitOptions): Effect.Effect<SubmitOutcome, DbError> {
171
+ const schemaId = compiledOf(this.schema).schemaId
172
+ const run = this.#run
173
+ return Effect.gen(function* () {
174
+ const handle = ChangeSetLive.handle(changes, schemaId)
175
+ if (handle === undefined) {
176
+ return yield* Effect.fail(new DbError({ operation: "Database.submit", reason: { _tag: "InvalidArgument" } }))
177
+ }
178
+ const settled = yield* run({
179
+ _tag: "Submit",
180
+ request: options.requestId,
181
+ revision: options.precondition ?? null,
182
+ changes: handle
183
+ })
184
+ if (settled._tag !== "Decided" && settled._tag !== "Refused") {
185
+ return yield* Effect.fail(dbError("Database.submit", { _tag: "Internal" }))
186
+ }
187
+ return settled
188
+ }).pipe(Effect.withSpan("Database.submit"))
189
+ }
190
+
191
+ read(consistency: Consistency = "cached"): Effect.Effect<DatabaseReader<S>, DbError, Scope.Scope> {
192
+ const run = this.#run
193
+ const hosted = this.#hosted
194
+ const schema = this.schema
195
+ const seen = this.#head
196
+ return Effect.gen(function* () {
197
+ const known = seen()?.seq ?? 0n
198
+ if (consistency === "latest" || (typeof consistency === "object" && known < consistency.atLeast)) {
199
+ yield* Effect.flatMap(run({ _tag: "Sync" }), (settled) => settledAs("Database.read", settled, "Synced"))
200
+ }
201
+ const head = seen()
202
+ const snapshot = yield* scopedSnapshot("Database.read", schema, (done) => addon.hostedSnapshot(hosted, done))
203
+ return {
204
+ seq: head?.seq ?? 0n,
205
+ revision: head?.revision ?? 0n,
206
+ get: snapshot.get.bind(snapshot),
207
+ execute: snapshot.execute.bind(snapshot),
208
+ prepare: snapshot.prepare.bind(snapshot)
209
+ } satisfies DatabaseReader<S>
210
+ }).pipe(Effect.withSpan("Database.read"))
211
+ }
212
+
213
+ resolve(requestId: RequestId): Effect.Effect<Option.Option<ReceiptOut>, DbError> {
214
+ return this.#run({ _tag: "Resolve", request: requestId }).pipe(
215
+ Effect.flatMap((settled) => settledAs("Database.resolve", settled, "Resolved")),
216
+ Effect.map((resolved) => Option.fromNullishOr(resolved.receipt)),
217
+ Effect.withSpan("Database.resolve")
218
+ )
219
+ }
220
+ }
221
+
222
+ /** Runs bundled migration `step` to completion, freezing writers if it keeps losing to them. */
223
+ const migrateStep = (
224
+ migrations: Migrations,
225
+ step: number,
226
+ hosted: HostedRef,
227
+ run: (request: HostedRequest) => Effect.Effect<SettledOut, DbError>,
228
+ head: () => HeadOut | undefined,
229
+ tuning: Tuning
230
+ ) =>
231
+ Effect.gen(function* () {
232
+ const migration = migrations[step]
233
+ const previous = migrations[step - 1]
234
+ if (migration === undefined || previous === undefined) {
235
+ return yield* Effect.fail(dbError("Database.migrate", { _tag: "Internal" }))
236
+ }
237
+ const attempt = Effect.scoped(
238
+ Effect.gen(function* () {
239
+ const base = head()?.seq ?? 0n
240
+ const from = yield* scopedSnapshot("Database.migrate", previous.to, (done) =>
241
+ addon.hostedSnapshot(hosted, done)
242
+ )
243
+ const into = yield* ChangeSets.builder(migration.to)
244
+ const populate = populateOf(migration)
245
+ if (populate !== undefined) yield* populate({ from, into })
246
+ const rows = ChangeSetLive.handle(yield* into.finish(), compiledOf(migration.to).schemaId)
247
+ if (rows === undefined) return yield* Effect.fail(dbError("Database.migrate", { _tag: "Internal" }))
248
+ const copy = addon.hostedUnchanged(hosted, step)
249
+ return yield* run({ _tag: "Migrate", step, base, copy, rows })
250
+ })
251
+ )
252
+ let settled: SettledOut = { _tag: "Refused", refusal: { _tag: "NotPending" } }
253
+ for (let tries = 0; tries <= tuning.migrationAttempts; tries++) {
254
+ if (tries === tuning.migrationAttempts) {
255
+ const lease = BigInt(Math.ceil(Duration.toMillis(tuning.freezeLease)))
256
+ yield* Effect.flatMap(run({ _tag: "Freeze", leaseMillis: lease }), (frozen) =>
257
+ settledAs("Database.migrate", frozen, "Frozen")
258
+ )
259
+ }
260
+ settled = yield* attempt
261
+ if (settled._tag !== "Refused" || settled.refusal._tag !== "Stale") break
262
+ }
263
+ yield* settledAs("Database.migrate", settled, "Migrated")
264
+ }).pipe(Effect.withSpan("Database.migrate"))
265
+
266
+ /**
267
+ * Seeds a database nothing has written to yet with its initial migration's rows, after every
268
+ * migration ran: the seed's relations must still exist unchanged in the current schema. The request
269
+ * id comes from the migration's hash, so racing openers decide the seed once.
270
+ */
271
+ const seedInitial = <S extends AnySchema>(
272
+ database: Database<S>,
273
+ hash: string,
274
+ seed: NonNullable<ReturnType<typeof populateOf>>
275
+ ) =>
276
+ Effect.scoped(
277
+ Effect.gen(function* () {
278
+ const from = yield* database.read("cached")
279
+ const into = yield* ChangeSets.builder(database.schema)
280
+ yield* seed({ from, into } as never)
281
+ const outcome = yield* database.submit(yield* into.finish(), { requestId: RequestId.make(hash.slice(0, 32)) })
282
+ if (outcome._tag === "Refused" && outcome.refusal._tag !== "RequestReused") {
283
+ return yield* Effect.fail(refused("Database.seed", outcome.refusal))
284
+ }
285
+ })
286
+ ).pipe(Effect.withSpan("Database.seed"))
287
+
288
+ /** Opens a database for the current scope. */
289
+ const make = Effect.fn("Database.make")(function* <S extends AnySchema>(options: DatabaseOptions<S>) {
290
+ const tuning: Tuning = { ...defaultTuning, ...options.tuning }
291
+ const { migrations } = options
292
+ const last = migrations[migrations.length - 1]
293
+ const theory = yield* Effect.try({
294
+ try: () => {
295
+ verifyBundle(migrations)
296
+ const theory = schemaDescriptor(options.schema)
297
+ if (last === undefined || !schemasAgree(theory, last.to)) {
298
+ throw new DbError({ operation: "Database.make", reason: { _tag: "InvalidArgument" } })
299
+ }
300
+ return theory as S
301
+ },
302
+ catch: (cause) => argumentError("Database.make", cause)
303
+ })
304
+ const opening: HostedOpenIn = {
305
+ bundle: migrations.map((migration) => ({ name: migration.id, hash: migration.hash, schema: lower(migration.to) })),
306
+ seed: hex(16),
307
+ ...(options.onOpen === "migrate" ? { create: hex(16) } : {}),
308
+ probeWindow: tuning.probeWindow,
309
+ checkpointEvery: tuning.checkpointEvery.toString(),
310
+ checkpointKeep: tuning.checkpointKeep,
311
+ maxEntryBytes: tuning.maxEntryBytes
312
+ }
313
+ const runtime = yield* runtimeHandle
314
+ const directory = yield* scoped(
315
+ "Database.cacheRelease",
316
+ call(
317
+ "Database.make",
318
+ (done) => addon.runtimeDirectoryAcquire(runtime, options.cache.directory, done),
319
+ addon.runtimeDirectoryTake
320
+ ),
321
+ (owned) => (done) => addon.runtimeDirectoryClose(owned, false, done)
322
+ )
323
+ const hosted = yield* scoped(
324
+ "Database.release",
325
+ call(
326
+ "Database.make",
327
+ (done) => addon.hostedOpen(directory, "cache", JSON.stringify(opening), done),
328
+ addon.hostedTake
329
+ ),
330
+ (owned) => (done) => addon.hostedClose(owned, done)
331
+ )
332
+ let head: HeadOut | undefined
333
+ const seen = () => head
334
+ const driver = yield* Driver.make(
335
+ hostedPort(hosted, (next) => {
336
+ head = next
337
+ }),
338
+ options.store,
339
+ { concurrency: tuning.concurrency, executor: tuning.executor }
340
+ )
341
+ const opened = yield* Effect.flatMap(driver.run({ _tag: "Open" }), (settled) =>
342
+ settledAs("Database.open", settled, "Opened")
343
+ )
344
+ if (opened.pending > 0 && options.onOpen === "verify") {
345
+ return yield* Effect.fail(
346
+ refused("Database.open", { _tag: "MigrationPending", next: migrations.length - opened.pending })
347
+ )
348
+ }
349
+ const fresh = seen()?.revision === 0n
350
+ for (let step = migrations.length - opened.pending; step < migrations.length; step++) {
351
+ yield* migrateStep(migrations, step, hosted, driver.run, seen, tuning)
352
+ }
353
+ const database = new DatabaseLive(theory, hosted, driver.run, seen) as Database<S>
354
+ const initial = migrations[0]
355
+ const seed = populateOf(initial)
356
+ if (options.onOpen === "migrate" && seed !== undefined && fresh) {
357
+ yield* seedInitial(database, initial.hash, seed)
358
+ }
359
+ return database
360
+ })
361
+
362
+ /** A service key for one application's database. */
363
+ function tag<S extends AnySchema>(key: string): Context.Key<Database<S>, Database<S>> {
364
+ return Context.Service<Database<S>>(key)
365
+ }
366
+
367
+ /** Provides `key` with a database opened from `options` for the layer's lifetime. */
368
+ function layer<S extends AnySchema>(
369
+ key: Context.Key<Database<S>, Database<S>>,
370
+ options: DatabaseOptions<S>
371
+ ): Layer.Layer<Database<S>, DbError, Bumble> {
372
+ return Layer.effect(key, make(options))
373
+ }
374
+
375
+ interface PoolOptions<S extends AnySchema> extends Omit<DatabaseOptions<S>, "store" | "cache"> {
376
+ readonly store: (tenant: string) => ObjectStore
377
+ readonly cache: (tenant: string) => { readonly directory: string }
378
+ /** How long an unused tenant's database stays open. */
379
+ readonly idleTimeToLive?: Duration.Input
380
+ }
381
+
382
+ /** Databases opened on demand per tenant and closed once idle. */
383
+ interface DatabasePool<S extends AnySchema> {
384
+ get(tenant: string): Effect.Effect<Database<S>, DbError, Scope.Scope>
385
+ }
386
+
387
+ /** A pool of per-tenant databases owned by the current scope. */
388
+ const pool = Effect.fn("Database.pool")(function* <S extends AnySchema>(options: PoolOptions<S>) {
389
+ const map = yield* RcMap.make({
390
+ lookup: (tenant: string) => make({ ...options, store: options.store(tenant), cache: options.cache(tenant) }),
391
+ idleTimeToLive: options.idleTimeToLive ?? "5 minutes"
392
+ })
393
+ return { get: (tenant: string) => RcMap.get(map, tenant) } satisfies DatabasePool<S>
394
+ })
395
+
396
+ /** A fresh random request id. */
397
+ function requestId(): RequestId {
398
+ return RequestId.make(hex(16))
399
+ }
400
+
401
+ const Database = Object.freeze({ make, layer, pool, tag, requestId })
402
+
403
+ export type {
404
+ Consistency,
405
+ DatabaseOptions,
406
+ DatabasePool,
407
+ DatabaseReader,
408
+ PoolOptions,
409
+ SubmitOptions,
410
+ SubmitOutcome,
411
+ Tuning
412
+ }
413
+ export { Database, RequestId }
@@ -0,0 +1,134 @@
1
+ /**
2
+ * Drives one sans-IO log machine. Requests and store responses go through one mailbox, so the
3
+ * machine steps strictly one input at a time; every `IoRequest` a step emits runs concurrently
4
+ * (bounded) and comes back as another input. Each request carries a ticket that settles exactly
5
+ * once, and closing the driver settles every ticket still open.
6
+ */
7
+ import type { Scope } from "effect"
8
+ import { Deferred, Effect, Fiber, FiberSet, Queue, Semaphore } from "effect"
9
+ import { DbError } from "../errors.ts"
10
+ import type { ExecutorOptions, IoRequest, IoResponse, ObjectStore } from "./io.ts"
11
+ import { defaultExecutorOptions, execute } from "./io.ts"
12
+
13
+ interface Step<Settled> {
14
+ readonly io: readonly IoRequest[]
15
+ readonly done: ReadonlyArray<{ readonly ticket: bigint; readonly settled: Settled }>
16
+ }
17
+
18
+ /** The machine as the driver sees it. The driver never calls it concurrently. */
19
+ interface MachinePort<Request, Settled> {
20
+ readonly request: (ticket: bigint, request: Request) => Effect.Effect<Step<Settled>, DbError>
21
+ readonly respond: (response: IoResponse) => Effect.Effect<Step<Settled>, DbError>
22
+ /** Stops the machine; its step settles every open ticket. */
23
+ readonly close: Effect.Effect<Step<Settled>, DbError>
24
+ }
25
+
26
+ interface DriverOptions {
27
+ /** Store calls in flight at once. */
28
+ readonly concurrency: number
29
+ readonly executor: ExecutorOptions
30
+ }
31
+
32
+ interface Driver<Request, Settled> {
33
+ /** Submits one request and waits for its settlement. */
34
+ readonly run: (request: Request) => Effect.Effect<Settled, DbError>
35
+ }
36
+
37
+ type Message<Request> =
38
+ | { readonly _tag: "Request"; readonly ticket: bigint; readonly request: Request }
39
+ | { readonly _tag: "Response"; readonly response: IoResponse }
40
+ | { readonly _tag: "Close" }
41
+
42
+ const defaultDriverOptions: DriverOptions = { concurrency: 32, executor: defaultExecutorOptions }
43
+
44
+ /** Starts a driver owned by the current scope; closing the scope closes the machine. */
45
+ const make = Effect.fnUntraced(function* <Request, Settled>(
46
+ port: MachinePort<Request, Settled>,
47
+ store: ObjectStore,
48
+ options: DriverOptions = defaultDriverOptions
49
+ ): Effect.fn.Return<Driver<Request, Settled>, never, Scope.Scope> {
50
+ const mailbox = yield* Queue.unbounded<Message<Request>>()
51
+ const permits = yield* Semaphore.make(options.concurrency)
52
+ const io = yield* FiberSet.make()
53
+ const tickets = new Map<bigint, Deferred.Deferred<Settled, DbError>>()
54
+ let nextTicket = 0n
55
+ const closed = new DbError({ operation: "Database", reason: { _tag: "ClosedHandle" } })
56
+
57
+ const apply = (step: Step<Settled>) =>
58
+ Effect.gen(function* () {
59
+ for (const { ticket, settled } of step.done) {
60
+ const waiter = tickets.get(ticket)
61
+ tickets.delete(ticket)
62
+ if (waiter !== undefined) yield* Deferred.succeed(waiter, settled)
63
+ }
64
+ for (const request of step.io) {
65
+ yield* FiberSet.run(
66
+ io,
67
+ Semaphore.withPermit(permits, execute(store, request, options.executor)).pipe(
68
+ Effect.flatMap((response) => Queue.offer(mailbox, { _tag: "Response", response }))
69
+ )
70
+ )
71
+ }
72
+ })
73
+
74
+ let stopped: DbError | undefined
75
+
76
+ /** Ends the driver: later runs fail with `error`, and open tickets fail with it now. */
77
+ const stop = (error: DbError) =>
78
+ Effect.suspend(() => {
79
+ stopped ??= error
80
+ const open = [...tickets.values()]
81
+ tickets.clear()
82
+ return Effect.forEach(open, (waiter) => Deferred.fail(waiter, error), { discard: true })
83
+ })
84
+
85
+ const stepFor = (message: Message<Request>) => {
86
+ switch (message._tag) {
87
+ case "Request":
88
+ return port.request(message.ticket, message.request)
89
+ case "Response":
90
+ return port.respond(message.response)
91
+ case "Close":
92
+ return port.close
93
+ }
94
+ }
95
+
96
+ const loop: Effect.Effect<void> = Effect.gen(function* () {
97
+ while (true) {
98
+ const message = yield* Queue.take(mailbox)
99
+ const step = stepFor(message)
100
+ const failed = yield* step.pipe(
101
+ Effect.flatMap(apply),
102
+ Effect.as(false),
103
+ Effect.catch((error) => Effect.as(stop(error), true))
104
+ )
105
+ if (failed || message._tag === "Close") return yield* stop(closed)
106
+ }
107
+ })
108
+
109
+ const machine = yield* Effect.forkScoped(loop)
110
+ yield* Effect.addFinalizer(() =>
111
+ Queue.offer(mailbox, { _tag: "Close" }).pipe(
112
+ Effect.andThen(Fiber.await(machine)),
113
+ Effect.andThen(FiberSet.clear(io))
114
+ )
115
+ )
116
+
117
+ return {
118
+ run: (request: Request) =>
119
+ Effect.suspend(() => {
120
+ if (stopped !== undefined) return Effect.fail(stopped)
121
+ const waiter = Deferred.makeUnsafe<Settled, DbError>()
122
+ const ticket = nextTicket
123
+ nextTicket += 1n
124
+ tickets.set(ticket, waiter)
125
+ Queue.offerUnsafe(mailbox, { _tag: "Request", ticket, request })
126
+ return Deferred.await(waiter)
127
+ })
128
+ }
129
+ })
130
+
131
+ const Driver = Object.freeze({ make, defaultOptions: defaultDriverOptions })
132
+
133
+ export type { DriverOptions, MachinePort, Step }
134
+ export { Driver }
@@ -0,0 +1,119 @@
1
+ import { randomUUID } from "node:crypto"
2
+ import * as fs from "node:fs/promises"
3
+ import * as path from "node:path"
4
+ import { Effect } from "effect"
5
+ import { DbError } from "../errors.ts"
6
+ import type { Body, Bucket, Created, Deleted, Fetched, Listed, ObjectStore, Reply, Target } from "./io.ts"
7
+
8
+ function code(cause: unknown): string | undefined {
9
+ return typeof cause === "object" && cause !== null && "code" in cause && typeof cause.code === "string"
10
+ ? cause.code
11
+ : undefined
12
+ }
13
+
14
+ function attempt<A>(operation: string, run: () => Promise<A>): Effect.Effect<A, DbError> {
15
+ return Effect.tryPromise({
16
+ try: run,
17
+ catch: (cause) => new DbError({ operation, reason: { _tag: "Io", kind: code(cause) ?? "Unknown" } })
18
+ })
19
+ }
20
+
21
+ async function syncDirectory(directory: string): Promise<void> {
22
+ const handle = await fs.open(directory, "r")
23
+ try {
24
+ await handle.sync()
25
+ } finally {
26
+ await handle.close()
27
+ }
28
+ }
29
+
30
+ async function writeDurably(file: string, body: Body): Promise<void> {
31
+ if (body._tag === "File") await fs.copyFile(body.path, file, fs.constants.COPYFILE_EXCL)
32
+ const handle = await fs.open(file, body._tag === "File" ? "r+" : "wx")
33
+ try {
34
+ if (body._tag === "Bytes") await handle.writeFile(body.bytes)
35
+ await handle.sync()
36
+ } finally {
37
+ await handle.close()
38
+ }
39
+ }
40
+
41
+ /**
42
+ * A store in a local directory, one file per key under `root`. A create writes a temporary file,
43
+ * fsyncs it, hard-links it into place (the link fails if the key exists) and fsyncs the directory,
44
+ * so a created key is durable and never torn. The store clock is the local clock.
45
+ */
46
+ function make(root: string): ObjectStore {
47
+ const file = (key: string) => path.join(root, ...key.split("/"))
48
+ const now = () => BigInt(Date.now())
49
+ return {
50
+ get: (_bucket: Bucket, key: string, target: Target) =>
51
+ attempt("FsStore.get", async (): Promise<Reply<Fetched>> => {
52
+ try {
53
+ if (target._tag === "Memory") {
54
+ const [bytes, stat] = await Promise.all([fs.readFile(file(key)), fs.stat(file(key), { bigint: true })])
55
+ return {
56
+ date: now(),
57
+ result: { _tag: "Body", bytes: new Uint8Array(bytes), lastModified: stat.mtimeMs }
58
+ }
59
+ }
60
+ const stat = await fs.stat(file(key), { bigint: true })
61
+ await fs.copyFile(file(key), target.path)
62
+ return { date: now(), result: { _tag: "Saved", lastModified: stat.mtimeMs } }
63
+ } catch (cause) {
64
+ if (code(cause) === "ENOENT") return { date: now(), result: { _tag: "Missing" } }
65
+ throw cause
66
+ }
67
+ }),
68
+ putIfAbsent: (_bucket: Bucket, key: string, body: Body) =>
69
+ attempt("FsStore.putIfAbsent", async (): Promise<Reply<Created>> => {
70
+ const target = file(key)
71
+ const directory = path.dirname(target)
72
+ await fs.mkdir(directory, { recursive: true })
73
+ const temporary = path.join(directory, `.tmp-${randomUUID()}`)
74
+ try {
75
+ await writeDurably(temporary, body)
76
+ try {
77
+ await fs.link(temporary, target)
78
+ } catch (cause) {
79
+ if (code(cause) === "EEXIST") return { date: now(), result: { _tag: "Occupied" } }
80
+ throw cause
81
+ }
82
+ } finally {
83
+ await fs.rm(temporary, { force: true })
84
+ }
85
+ await syncDirectory(directory)
86
+ return { date: now(), result: { _tag: "Created" } }
87
+ }),
88
+ list: (prefix: string, startAfter: string | null, maxKeys: number) =>
89
+ attempt("FsStore.list", async (): Promise<Reply<Listed>> => {
90
+ const directory = path.posix.dirname(`${prefix}_`)
91
+ let names: string[]
92
+ try {
93
+ names = await fs.readdir(path.join(root, ...directory.split("/")))
94
+ } catch (cause) {
95
+ if (code(cause) === "ENOENT") return { date: now(), result: { _tag: "Keys", keys: [] } }
96
+ throw cause
97
+ }
98
+ const keys = names
99
+ .filter((name) => !name.startsWith(".tmp-"))
100
+ .map((name) => (directory === "." ? name : `${directory}/${name}`))
101
+ .filter((key) => key.startsWith(prefix) && (startAfter === null || key > startAfter))
102
+ .sort()
103
+ .slice(0, maxKeys)
104
+ return { date: now(), result: { _tag: "Keys", keys } }
105
+ }),
106
+ delete: (key: string) =>
107
+ attempt("FsStore.delete", async (): Promise<Reply<Deleted>> => {
108
+ await fs.rm(file(key), { force: true })
109
+ await syncDirectory(path.dirname(file(key))).catch((cause) => {
110
+ if (code(cause) !== "ENOENT") throw cause
111
+ })
112
+ return { date: now(), result: { _tag: "Deleted" } }
113
+ })
114
+ }
115
+ }
116
+
117
+ const FsStore = Object.freeze({ make })
118
+
119
+ export { FsStore }