@bjornpagen/bumbledb 1.3.0 → 2.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (290) hide show
  1. package/COOKBOOK.md +224 -95
  2. package/README.md +114 -45
  3. package/dist/alternatives.d.ts +5 -6
  4. package/dist/alternatives.d.ts.map +1 -1
  5. package/dist/alternatives.js +5 -6
  6. package/dist/alternatives.js.map +1 -1
  7. package/dist/bin.d.ts +3 -0
  8. package/dist/bin.d.ts.map +1 -0
  9. package/dist/bin.js +85 -0
  10. package/dist/bin.js.map +1 -0
  11. package/dist/capacity.d.ts +14 -17
  12. package/dist/capacity.d.ts.map +1 -1
  13. package/dist/capacity.js +7 -7
  14. package/dist/capacity.js.map +1 -1
  15. package/dist/changes.d.ts +32 -41
  16. package/dist/changes.d.ts.map +1 -1
  17. package/dist/changes.js +86 -110
  18. package/dist/changes.js.map +1 -1
  19. package/dist/closed.d.ts +2 -2
  20. package/dist/closed.d.ts.map +1 -1
  21. package/dist/closed.js +5 -5
  22. package/dist/closed.js.map +1 -1
  23. package/dist/codec.d.ts +20 -27
  24. package/dist/codec.d.ts.map +1 -1
  25. package/dist/codec.js +48 -22
  26. package/dist/codec.js.map +1 -1
  27. package/dist/compile.d.ts +26 -50
  28. package/dist/compile.d.ts.map +1 -1
  29. package/dist/compile.js +75 -64
  30. package/dist/compile.js.map +1 -1
  31. package/dist/database/database.d.ts +98 -0
  32. package/dist/database/database.d.ts.map +1 -0
  33. package/dist/database/database.js +249 -0
  34. package/dist/database/database.js.map +1 -0
  35. package/dist/database/driver.d.ts +40 -0
  36. package/dist/database/driver.d.ts.map +1 -0
  37. package/dist/database/driver.js +68 -0
  38. package/dist/database/driver.js.map +1 -0
  39. package/dist/database/fs.d.ts +12 -0
  40. package/dist/database/fs.d.ts.map +1 -0
  41. package/dist/database/fs.js +120 -0
  42. package/dist/database/fs.js.map +1 -0
  43. package/dist/database/generate.d.ts +26 -0
  44. package/dist/database/generate.d.ts.map +1 -0
  45. package/dist/database/generate.js +102 -0
  46. package/dist/database/generate.js.map +1 -0
  47. package/dist/database/hosted.d.ts +32 -0
  48. package/dist/database/hosted.d.ts.map +1 -0
  49. package/dist/database/hosted.js +44 -0
  50. package/dist/database/hosted.js.map +1 -0
  51. package/dist/database/io.d.ts +94 -0
  52. package/dist/database/io.d.ts.map +1 -0
  53. package/dist/database/io.js +35 -0
  54. package/dist/database/io.js.map +1 -0
  55. package/dist/database/mem.d.ts +26 -0
  56. package/dist/database/mem.d.ts.map +1 -0
  57. package/dist/database/mem.js +67 -0
  58. package/dist/database/mem.js.map +1 -0
  59. package/dist/database/migration.d.ts +55 -0
  60. package/dist/database/migration.d.ts.map +1 -0
  61. package/dist/database/migration.js +46 -0
  62. package/dist/database/migration.js.map +1 -0
  63. package/dist/database/s3.d.ts +36 -0
  64. package/dist/database/s3.d.ts.map +1 -0
  65. package/dist/database/s3.js +141 -0
  66. package/dist/database/s3.js.map +1 -0
  67. package/dist/db.d.ts +44 -128
  68. package/dist/db.d.ts.map +1 -1
  69. package/dist/db.js +163 -287
  70. package/dist/db.js.map +1 -1
  71. package/dist/engine.d.ts +7 -0
  72. package/dist/engine.d.ts.map +1 -0
  73. package/dist/engine.js +2 -0
  74. package/dist/engine.js.map +1 -0
  75. package/dist/errors.d.ts +152 -46
  76. package/dist/errors.d.ts.map +1 -1
  77. package/dist/errors.js +107 -19
  78. package/dist/errors.js.map +1 -1
  79. package/dist/face.d.ts +5 -5
  80. package/dist/face.d.ts.map +1 -1
  81. package/dist/face.js +6 -6
  82. package/dist/face.js.map +1 -1
  83. package/dist/fields.d.ts +4 -5
  84. package/dist/fields.d.ts.map +1 -1
  85. package/dist/fields.js +3 -3
  86. package/dist/fields.js.map +1 -1
  87. package/dist/index.d.ts +60 -58
  88. package/dist/index.d.ts.map +1 -1
  89. package/dist/index.js +27 -29
  90. package/dist/index.js.map +1 -1
  91. package/dist/judgment.d.ts +1 -1
  92. package/dist/judgment.js +1 -1
  93. package/dist/law.d.ts +16 -68
  94. package/dist/law.d.ts.map +1 -1
  95. package/dist/law.js +3 -3
  96. package/dist/law.js.map +1 -1
  97. package/dist/lower.d.ts +3 -3
  98. package/dist/lower.d.ts.map +1 -1
  99. package/dist/lower.js +35 -39
  100. package/dist/lower.js.map +1 -1
  101. package/dist/native/addon.d.ts +23 -0
  102. package/dist/native/addon.d.ts.map +1 -0
  103. package/dist/native/addon.js +15 -0
  104. package/dist/native/addon.js.map +1 -0
  105. package/dist/native/binding.d.ts +996 -0
  106. package/dist/native/json.d.ts +14 -0
  107. package/dist/native/json.d.ts.map +1 -0
  108. package/dist/native/json.js +73 -0
  109. package/dist/native/json.js.map +1 -0
  110. package/dist/native/load.d.ts +9 -0
  111. package/dist/native/load.d.ts.map +1 -0
  112. package/dist/native/load.js +30 -0
  113. package/dist/native/load.js.map +1 -0
  114. package/dist/native/op.d.ts +29 -0
  115. package/dist/native/op.d.ts.map +1 -0
  116. package/dist/native/op.js +50 -0
  117. package/dist/native/op.js.map +1 -0
  118. package/dist/query/atom.d.ts +13 -14
  119. package/dist/query/atom.d.ts.map +1 -1
  120. package/dist/query/atom.js +2 -2
  121. package/dist/query/atom.js.map +1 -1
  122. package/dist/query/compute.d.ts +7 -7
  123. package/dist/query/compute.d.ts.map +1 -1
  124. package/dist/query/compute.js +7 -16
  125. package/dist/query/compute.js.map +1 -1
  126. package/dist/query/find.d.ts +8 -8
  127. package/dist/query/find.d.ts.map +1 -1
  128. package/dist/query/find.js +1 -1
  129. package/dist/query/lower.d.ts +31 -17
  130. package/dist/query/lower.d.ts.map +1 -1
  131. package/dist/query/lower.js +260 -368
  132. package/dist/query/lower.js.map +1 -1
  133. package/dist/query/run.d.ts +5 -5
  134. package/dist/query/run.d.ts.map +1 -1
  135. package/dist/query/run.js +32 -15
  136. package/dist/query/run.js.map +1 -1
  137. package/dist/query/scope.d.ts +20 -28
  138. package/dist/query/scope.d.ts.map +1 -1
  139. package/dist/query/scope.js +13 -8
  140. package/dist/query/scope.js.map +1 -1
  141. package/dist/query/segments.d.ts +2 -2
  142. package/dist/query/segments.d.ts.map +1 -1
  143. package/dist/query/segments.js +3 -3
  144. package/dist/query/segments.js.map +1 -1
  145. package/dist/relation.d.ts +1 -1
  146. package/dist/relation.d.ts.map +1 -1
  147. package/dist/relation.js +4 -4
  148. package/dist/relation.js.map +1 -1
  149. package/dist/result.d.ts +15 -22
  150. package/dist/result.d.ts.map +1 -1
  151. package/dist/result.js +30 -39
  152. package/dist/result.js.map +1 -1
  153. package/dist/rows.d.ts +9 -33
  154. package/dist/rows.d.ts.map +1 -1
  155. package/dist/rows.js +8 -12
  156. package/dist/rows.js.map +1 -1
  157. package/dist/runtime.d.ts +17 -43
  158. package/dist/runtime.d.ts.map +1 -1
  159. package/dist/runtime.js +41 -241
  160. package/dist/runtime.js.map +1 -1
  161. package/dist/scalar.d.ts +36 -106
  162. package/dist/scalar.d.ts.map +1 -1
  163. package/dist/scalar.js +36 -185
  164. package/dist/scalar.js.map +1 -1
  165. package/dist/schema.d.ts +7 -18
  166. package/dist/schema.d.ts.map +1 -1
  167. package/dist/schema.js +18 -228
  168. package/dist/schema.js.map +1 -1
  169. package/dist/selection.d.ts +4 -4
  170. package/dist/selection.d.ts.map +1 -1
  171. package/dist/selection.js +5 -5
  172. package/dist/selection.js.map +1 -1
  173. package/dist/shape.d.ts +5 -5
  174. package/dist/shape.d.ts.map +1 -1
  175. package/dist/spec.d.ts +7 -79
  176. package/dist/spec.d.ts.map +1 -1
  177. package/dist/spec.js.map +1 -1
  178. package/dist/statements.d.ts +3 -3
  179. package/dist/statements.d.ts.map +1 -1
  180. package/dist/statements.js +10 -10
  181. package/dist/statements.js.map +1 -1
  182. package/dist/uuid.d.ts +1 -1
  183. package/dist/uuid.d.ts.map +1 -1
  184. package/dist/uuid.js +1 -1
  185. package/dist/uuid.js.map +1 -1
  186. package/dist/values.d.ts +2 -2
  187. package/dist/values.d.ts.map +1 -1
  188. package/dist/values.js +2 -2
  189. package/dist/values.js.map +1 -1
  190. package/package.json +18 -18
  191. package/src/alternatives.ts +7 -8
  192. package/src/bin.ts +82 -0
  193. package/src/capacity.ts +20 -23
  194. package/src/changes.ts +134 -197
  195. package/src/closed.ts +5 -5
  196. package/src/codec.ts +77 -51
  197. package/src/compile.ts +91 -113
  198. package/src/database/database.ts +413 -0
  199. package/src/database/driver.ts +134 -0
  200. package/src/database/fs.ts +119 -0
  201. package/src/database/generate.ts +116 -0
  202. package/src/database/hosted.ts +102 -0
  203. package/src/database/io.ts +134 -0
  204. package/src/database/mem.ts +113 -0
  205. package/src/database/migration.ts +107 -0
  206. package/src/database/s3.ts +231 -0
  207. package/src/db.ts +290 -547
  208. package/src/engine.ts +14 -0
  209. package/src/errors.ts +118 -39
  210. package/src/face.ts +9 -9
  211. package/src/fields.ts +6 -7
  212. package/src/index.ts +102 -109
  213. package/src/judgment.ts +1 -1
  214. package/src/law.ts +18 -70
  215. package/src/lower.ts +61 -72
  216. package/src/native/addon.ts +54 -0
  217. package/src/native/binding.d.ts +996 -0
  218. package/src/native/json.ts +100 -0
  219. package/src/native/load.ts +31 -0
  220. package/src/native/op.ts +73 -0
  221. package/src/query/atom.ts +15 -16
  222. package/src/query/compute.ts +13 -22
  223. package/src/query/find.ts +8 -8
  224. package/src/query/lower.ts +323 -414
  225. package/src/query/run.ts +42 -21
  226. package/src/query/scope.ts +37 -38
  227. package/src/query/segments.ts +4 -4
  228. package/src/relation.ts +4 -4
  229. package/src/result.ts +53 -80
  230. package/src/rows.ts +18 -46
  231. package/src/runtime.ts +50 -346
  232. package/src/scalar.ts +76 -336
  233. package/src/schema.ts +23 -284
  234. package/src/selection.ts +6 -6
  235. package/src/shape.ts +5 -5
  236. package/src/spec.ts +7 -85
  237. package/src/statements.ts +11 -11
  238. package/src/uuid.ts +1 -1
  239. package/src/values.ts +4 -4
  240. package/dist/close.d.ts +0 -24
  241. package/dist/close.d.ts.map +0 -1
  242. package/dist/close.js +0 -34
  243. package/dist/close.js.map +0 -1
  244. package/dist/db-native.d.ts +0 -196
  245. package/dist/db-native.d.ts.map +0 -1
  246. package/dist/db-native.js +0 -13
  247. package/dist/db-native.js.map +0 -1
  248. package/dist/internal/log.d.ts +0 -26
  249. package/dist/internal/log.d.ts.map +0 -1
  250. package/dist/internal/log.js +0 -12
  251. package/dist/internal/log.js.map +0 -1
  252. package/dist/migration.d.ts +0 -31
  253. package/dist/migration.d.ts.map +0 -1
  254. package/dist/migration.js +0 -47
  255. package/dist/migration.js.map +0 -1
  256. package/dist/native.d.ts +0 -562
  257. package/dist/native.d.ts.map +0 -1
  258. package/dist/native.js +0 -123
  259. package/dist/native.js.map +0 -1
  260. package/dist/query/description.d.ts +0 -41
  261. package/dist/query/description.d.ts.map +0 -1
  262. package/dist/query/description.js +0 -311
  263. package/dist/query/description.js.map +0 -1
  264. package/dist/query/parse-ir.d.ts +0 -10
  265. package/dist/query/parse-ir.d.ts.map +0 -1
  266. package/dist/query/parse-ir.js +0 -280
  267. package/dist/query/parse-ir.js.map +0 -1
  268. package/dist/runtime-codes.d.ts +0 -2
  269. package/dist/runtime-codes.d.ts.map +0 -1
  270. package/dist/runtime-codes.js +0 -18
  271. package/dist/runtime-codes.js.map +0 -1
  272. package/dist/runtime-errors.d.ts +0 -118
  273. package/dist/runtime-errors.d.ts.map +0 -1
  274. package/dist/runtime-errors.js +0 -98
  275. package/dist/runtime-errors.js.map +0 -1
  276. package/dist/runtime-native.d.ts +0 -141
  277. package/dist/runtime-native.d.ts.map +0 -1
  278. package/dist/runtime-native.js +0 -5
  279. package/dist/runtime-native.js.map +0 -1
  280. package/pack-provenance.json +0 -7
  281. package/src/close.ts +0 -53
  282. package/src/db-native.ts +0 -243
  283. package/src/internal/log.ts +0 -48
  284. package/src/migration.ts +0 -61
  285. package/src/native.ts +0 -736
  286. package/src/query/description.ts +0 -424
  287. package/src/query/parse-ir.ts +0 -291
  288. package/src/runtime-codes.ts +0 -17
  289. package/src/runtime-errors.ts +0 -109
  290. package/src/runtime-native.ts +0 -189
@@ -0,0 +1,116 @@
1
+ /**
2
+ * The migrations directory: one `NNNN_name/` per migration holding the schema it migrates to
3
+ * (`schema.json`, the engine's spec, and `schema.ts`, the same schema as SDK code) and its
4
+ * `migration.ts`; `index.ts` bundles them in order. A migration's hash covers its id and its
5
+ * `schema.json`, so editing either after it ran is caught by `check`.
6
+ */
7
+ import { createHash } from "node:crypto"
8
+ import * as fs from "node:fs"
9
+ import * as path from "node:path"
10
+ import { compiledOf } from "../compile.ts"
11
+ import { AuthoringError } from "../errors.ts"
12
+ import { lower } from "../lower.ts"
13
+ import { addon } from "../native/addon.ts"
14
+ import type { AnySchema } from "../schema.ts"
15
+ import { schemaDescriptor } from "../schema.ts"
16
+
17
+ const DIRECTORY = /^([0-9]{4})_([A-Za-z0-9_]+)$/
18
+
19
+ /** The `schema.json` text of a schema: its lowered spec, formatted for review. */
20
+ function specText(theory: AnySchema): string {
21
+ return `${JSON.stringify(lower(schemaDescriptor(theory)), null, "\t")}\n`
22
+ }
23
+
24
+ /** A migration's hash: sha256 over its id and its `schema.json` text. */
25
+ function migrationHash(id: string, spec: string): string {
26
+ return createHash("sha256").update(`${id}\n${spec}`).digest("hex")
27
+ }
28
+
29
+ /** The migration directories under `root`, in order. */
30
+ function migrationIds(root: string): string[] {
31
+ if (!fs.existsSync(root)) return []
32
+ return fs
33
+ .readdirSync(root, { withFileTypes: true })
34
+ .filter((entry) => entry.isDirectory() && DIRECTORY.test(entry.name))
35
+ .map((entry) => entry.name)
36
+ .sort()
37
+ }
38
+
39
+ function indexText(ids: readonly string[]): string {
40
+ const imports = ids.map((id) => `import m${id.slice(0, 4)} from "./${id}/migration.ts"`)
41
+ const list = ids.map((id) => `m${id.slice(0, 4)}`).join(", ")
42
+ return `${imports.join("\n")}\n\nexport const migrations = [${list}] as const\n`
43
+ }
44
+
45
+ function migrationText(id: string, hash: string, previous: string | undefined): string {
46
+ const lines = ['import { Migration } from "@bjornpagen/bumbledb"']
47
+ if (previous !== undefined) lines.push(`import { schema as from } from "../${previous}/schema.ts"`)
48
+ lines.push('import { schema as to } from "./schema.ts"', "")
49
+ lines.push(
50
+ previous === undefined
51
+ ? `export default Migration.make({ id: "${id}", hash: "${hash}", to })`
52
+ : `export default Migration.make({ id: "${id}", hash: "${hash}", from, to })`
53
+ )
54
+ return `${lines.join("\n")}\n`
55
+ }
56
+
57
+ type Generated = { readonly _tag: "Unchanged" } | { readonly _tag: "Created"; readonly id: string }
58
+
59
+ /**
60
+ * Adds a migration to `schema` under `root` when it differs from the last migration's schema, and
61
+ * rewrites `index.ts`.
62
+ */
63
+ function generate(root: string, theory: AnySchema, name: string): Generated {
64
+ if (!/^[A-Za-z0-9_]+$/.test(name))
65
+ throw new AuthoringError({ message: `migration name ${name}: letters, digits and _` })
66
+ const ids = migrationIds(root)
67
+ const spec = specText(theory)
68
+ const previous = ids.at(-1)
69
+ if (previous !== undefined && fs.readFileSync(path.join(root, previous, "schema.json"), "utf8") === spec) {
70
+ return { _tag: "Unchanged" }
71
+ }
72
+ const bindings = addon.schemaBindings(compiledOf(schemaDescriptor(theory)).handle)
73
+ if (bindings._tag === "Unrepresentable") {
74
+ throw new AuthoringError({ message: `schema ${theory.name}: ${bindings.coordinate}: ${bindings.reason}` })
75
+ }
76
+ const number = previous === undefined ? 1 : Number(previous.slice(0, 4)) + 1
77
+ const id = `${number.toString().padStart(4, "0")}_${name}`
78
+ const directory = path.join(root, id)
79
+ fs.mkdirSync(directory, { recursive: true })
80
+ fs.writeFileSync(path.join(directory, "schema.json"), spec)
81
+ fs.writeFileSync(path.join(directory, "schema.ts"), bindings.source)
82
+ fs.writeFileSync(path.join(directory, "migration.ts"), migrationText(id, migrationHash(id, spec), previous))
83
+ fs.writeFileSync(path.join(root, "index.ts"), indexText([...ids, id]))
84
+ return { _tag: "Created", id }
85
+ }
86
+
87
+ /** Everything `check` found wrong; empty when the directory is consistent with `schema`. */
88
+ function check(
89
+ root: string,
90
+ theory: AnySchema,
91
+ bundled: readonly { readonly id: string; readonly hash: string }[]
92
+ ): string[] {
93
+ const problems: string[] = []
94
+ const ids = migrationIds(root)
95
+ if (ids.length === 0) return [`${root}: no migrations; run bumbledb generate`]
96
+ if (fs.readFileSync(path.join(root, "index.ts"), "utf8") !== indexText(ids)) {
97
+ problems.push(`${root}/index.ts does not list the migration directories in order; run bumbledb generate`)
98
+ }
99
+ ids.forEach((id, index) => {
100
+ const spec = fs.readFileSync(path.join(root, id, "schema.json"), "utf8")
101
+ const migration = bundled[index]
102
+ if (migration === undefined || migration.id !== id) {
103
+ problems.push(`${id}: not bundled at position ${index + 1}`)
104
+ } else if (migration.hash !== migrationHash(id, spec)) {
105
+ problems.push(`${id}: its hash does not match its schema.json; a migration is never edited after it runs`)
106
+ }
107
+ })
108
+ const last = ids.at(-1)
109
+ if (last !== undefined && fs.readFileSync(path.join(root, last, "schema.json"), "utf8") !== specText(theory)) {
110
+ problems.push(`the schema differs from ${last}; run bumbledb generate`)
111
+ }
112
+ return problems
113
+ }
114
+
115
+ export type { Generated }
116
+ export { check, generate, migrationHash, migrationIds, specText }
@@ -0,0 +1,102 @@
1
+ /**
2
+ * The addon's hosted machine as a driver port. A hosted step is never cancelled: its output carries
3
+ * requests the machine then waits on, so every step runs to completion even when its caller is
4
+ * interrupted.
5
+ */
6
+ import { Effect } from "effect"
7
+ import type { DbError } from "../errors.ts"
8
+ import type { ChangesRef, OperationRef } from "../native/addon.ts"
9
+ import { addon } from "../native/addon.ts"
10
+ import type {
11
+ CopyPair,
12
+ ExternalObject,
13
+ HeadOut,
14
+ HostedHandle,
15
+ IoOutcomeIn,
16
+ SettledOut,
17
+ StepOut
18
+ } from "../native/binding.d.ts"
19
+ import { call } from "../native/op.ts"
20
+ import type { MachinePort, Step } from "./driver.ts"
21
+ import type { IoResponse, IoResult } from "./io.ts"
22
+
23
+ type HostedRef = ExternalObject<HostedHandle>
24
+
25
+ /** One machine request. `request` ids are 32 hex digits; `revision` present is an exact-revision precondition. */
26
+ type HostedRequest =
27
+ | { readonly _tag: "Open" }
28
+ | { readonly _tag: "Sync" }
29
+ | { readonly _tag: "Resolve"; readonly request: string }
30
+ | { readonly _tag: "Freeze"; readonly leaseMillis: bigint }
31
+ | {
32
+ readonly _tag: "Submit"
33
+ readonly request: string
34
+ readonly revision: bigint | null
35
+ readonly changes: ChangesRef
36
+ }
37
+ | {
38
+ readonly _tag: "Migrate"
39
+ readonly step: number
40
+ readonly base: bigint
41
+ readonly copy: readonly CopyPair[]
42
+ readonly rows: ChangesRef
43
+ }
44
+
45
+ function outcomeIn(result: Exclude<IoResult, { readonly _tag: "Body" }>): IoOutcomeIn {
46
+ return result._tag === "Keys" ? { _tag: "Keys", keys: [...result.keys] } : result
47
+ }
48
+
49
+ /** A port over one hosted machine; `onHead` sees every head a step reports. */
50
+ function hostedPort(hosted: HostedRef, onHead: (head: HeadOut) => void): MachinePort<HostedRequest, SettledOut> {
51
+ const step = (
52
+ operation: string,
53
+ start: (done: () => void) => OperationRef
54
+ ): Effect.Effect<Step<SettledOut>, DbError> =>
55
+ call(operation, start, addon.hostedStepTake).pipe(
56
+ Effect.tap((out: StepOut) =>
57
+ Effect.sync(() => {
58
+ if (out.head !== undefined) onHead(out.head)
59
+ })
60
+ ),
61
+ Effect.uninterruptible
62
+ )
63
+ return {
64
+ request: (ticket, request) => {
65
+ switch (request._tag) {
66
+ case "Open":
67
+ case "Sync":
68
+ return step(`Database.${request._tag}`, (done) =>
69
+ addon.hostedStep(hosted, { _tag: request._tag, ticket }, done)
70
+ )
71
+ case "Resolve":
72
+ return step("Database.resolve", (done) =>
73
+ addon.hostedStep(hosted, { _tag: "Resolve", ticket, request: request.request }, done)
74
+ )
75
+ case "Freeze":
76
+ return step("Database.freeze", (done) =>
77
+ addon.hostedStep(hosted, { _tag: "Freeze", ticket, leaseMillis: request.leaseMillis }, done)
78
+ )
79
+ case "Submit":
80
+ return step("Database.submit", (done) =>
81
+ addon.hostedSubmit(hosted, ticket, request.request, request.revision, request.changes, done)
82
+ )
83
+ case "Migrate":
84
+ return step("Database.migrate", (done) =>
85
+ addon.hostedMigrate(hosted, ticket, request.step, request.base, [...request.copy], request.rows, done)
86
+ )
87
+ }
88
+ },
89
+ respond: (response: IoResponse) => {
90
+ const { id, date, result } = response
91
+ return result._tag === "Body"
92
+ ? step("Database.respond", (done) =>
93
+ addon.hostedRespondBody(hosted, id, date, result.lastModified, result.bytes, done)
94
+ )
95
+ : step("Database.respond", (done) => addon.hostedRespond(hosted, id, date, outcomeIn(result), done))
96
+ },
97
+ close: step("Database.close", (done) => addon.hostedStep(hosted, { _tag: "Close" }, done))
98
+ }
99
+ }
100
+
101
+ export type { HostedRef, HostedRequest }
102
+ export { hostedPort }
@@ -0,0 +1,134 @@
1
+ /**
2
+ * The object-store side of the hosted log. The log machine emits `IoRequest`s as data; this module
3
+ * runs them against an `ObjectStore` and turns every outcome into an `IoResponse`. Reads are
4
+ * idempotent and retried here; a create-only PUT is never retried here, because the machine
5
+ * resolves an ambiguous PUT by reading the key back.
6
+ */
7
+ import type { Duration } from "effect"
8
+ import { Effect, Schedule } from "effect"
9
+ import { DbError } from "../errors.ts"
10
+ import type { BucketOut, IoRequestOut } from "../native/binding.d.ts"
11
+
12
+ /** `Log` is the commit log (`log/`, an S3 Express directory bucket); `Checkpoints` holds `ckpt/` and `mig/`. */
13
+ type Bucket = BucketOut
14
+
15
+ /** One request from the machine. `key` is relative to the database prefix. */
16
+ type IoRequest = IoRequestOut
17
+
18
+ /** Unix milliseconds on the store's clock. */
19
+ type Millis = bigint
20
+
21
+ /** Where a GET delivers the object: returned in memory, or written to a local file. */
22
+ type Target = { readonly _tag: "Memory" } | { readonly _tag: "File"; readonly path: string }
23
+
24
+ /** What a create-only PUT uploads: bytes in memory, or a local file. */
25
+ type Body = { readonly _tag: "Bytes"; readonly bytes: Uint8Array } | { readonly _tag: "File"; readonly path: string }
26
+
27
+ type Fetched =
28
+ | { readonly _tag: "Body"; readonly bytes: Uint8Array; readonly lastModified: Millis }
29
+ | { readonly _tag: "Saved"; readonly lastModified: Millis }
30
+ | { readonly _tag: "Missing" }
31
+
32
+ /** `Occupied`: the key exists, or, for a log object, another conditional write to it was in progress. */
33
+ type Created = { readonly _tag: "Created" } | { readonly _tag: "Occupied" }
34
+
35
+ type Listed = { readonly _tag: "Keys"; readonly keys: readonly string[] }
36
+
37
+ type Deleted = { readonly _tag: "Deleted" }
38
+
39
+ /** `Failed` covers everything a store could not answer definitively: 5xx, timeouts, transport. */
40
+ type IoResult = Fetched | Created | Listed | Deleted | { readonly _tag: "Failed" }
41
+
42
+ interface IoResponse {
43
+ readonly id: bigint
44
+ /** The store's `Date` header, when it sent one. */
45
+ readonly date: Millis | null
46
+ readonly result: IoResult
47
+ }
48
+
49
+ /** A store answer plus the store's clock reading. */
50
+ interface Reply<R> {
51
+ readonly date: Millis | null
52
+ readonly result: R
53
+ }
54
+
55
+ /**
56
+ * The four verbs the log needs. Keys arrive relative; a store owns its own prefix. LIST and DELETE
57
+ * exist only for `Checkpoints`: LIST order on an Express bucket is not lexicographic, and a `log/`
58
+ * key must never disappear.
59
+ */
60
+ interface ObjectStore {
61
+ readonly get: (bucket: Bucket, key: string, target: Target) => Effect.Effect<Reply<Fetched>, DbError>
62
+ readonly putIfAbsent: (bucket: Bucket, key: string, body: Body) => Effect.Effect<Reply<Created>, DbError>
63
+ /** Checkpoint keys under `prefix` after `startAfter`, in lexicographic order. */
64
+ readonly list: (prefix: string, startAfter: string | null, maxKeys: number) => Effect.Effect<Reply<Listed>, DbError>
65
+ /** Deletes one checkpoint key; deleting a missing key succeeds. */
66
+ readonly delete: (key: string) => Effect.Effect<Reply<Deleted>, DbError>
67
+ }
68
+
69
+ interface ExecutorOptions {
70
+ /** Bound on one store call; a call that exceeds it counts as `Failed` (or is retried, for a read). */
71
+ readonly timeout: Duration.Input
72
+ /** Retry policy for GET, LIST and DELETE. */
73
+ readonly readRetry: Schedule.Schedule<unknown, unknown>
74
+ }
75
+
76
+ const defaultExecutorOptions: ExecutorOptions = {
77
+ timeout: "10 seconds",
78
+ readRetry: Schedule.exponential("20 millis").pipe(Schedule.jittered, Schedule.upTo({ times: 4 }))
79
+ }
80
+
81
+ function refused(request: IoRequest): Effect.Effect<never, DbError> {
82
+ return Effect.fail(new DbError({ operation: `ObjectStore.${request.op._tag}`, reason: { _tag: "InvalidArgument" } }))
83
+ }
84
+
85
+ function dispatch(store: ObjectStore, request: IoRequest): Effect.Effect<Reply<IoResult>, DbError> {
86
+ const { bucket, key, op } = request
87
+ switch (op._tag) {
88
+ case "GetMemory":
89
+ return store.get(bucket, key, { _tag: "Memory" })
90
+ case "GetFile":
91
+ return store.get(bucket, key, { _tag: "File", path: op.path })
92
+ case "PutBytes":
93
+ return store.putIfAbsent(bucket, key, { _tag: "Bytes", bytes: op.bytes })
94
+ case "PutFile":
95
+ return store.putIfAbsent(bucket, key, { _tag: "File", path: op.path })
96
+ case "List":
97
+ return bucket === "Checkpoints" ? store.list(key, op.startAfter ?? null, op.maxKeys) : refused(request)
98
+ case "Delete":
99
+ return bucket === "Checkpoints" ? store.delete(key) : refused(request)
100
+ }
101
+ }
102
+
103
+ /** Runs one request to an `IoResponse`; it never fails. */
104
+ function execute(
105
+ store: ObjectStore,
106
+ request: IoRequest,
107
+ options: ExecutorOptions = defaultExecutorOptions
108
+ ): Effect.Effect<IoResponse> {
109
+ const once = Effect.timeout(dispatch(store, request), options.timeout)
110
+ const write = request.op._tag === "PutBytes" || request.op._tag === "PutFile"
111
+ const attempts = write ? once : Effect.retry(once, options.readRetry)
112
+ return attempts.pipe(
113
+ Effect.map((reply): IoResponse => ({ id: request.id, date: reply.date, result: reply.result })),
114
+ Effect.catch(() => Effect.succeed<IoResponse>({ id: request.id, date: null, result: { _tag: "Failed" } }))
115
+ )
116
+ }
117
+
118
+ export type {
119
+ Body,
120
+ Bucket,
121
+ Created,
122
+ Deleted,
123
+ ExecutorOptions,
124
+ Fetched,
125
+ IoRequest,
126
+ IoResponse,
127
+ IoResult,
128
+ Listed,
129
+ Millis,
130
+ ObjectStore,
131
+ Reply,
132
+ Target
133
+ }
134
+ export { defaultExecutorOptions, execute }
@@ -0,0 +1,113 @@
1
+ import * as fs from "node:fs/promises"
2
+ import { Effect } from "effect"
3
+ import { DbError } from "../errors.ts"
4
+ import type { Body, Bucket, Created, Deleted, Fetched, Listed, Millis, ObjectStore, Reply, Target } from "./io.ts"
5
+
6
+ type Verb = "get" | "putIfAbsent" | "list" | "delete"
7
+
8
+ /** `Fail`: the call fails before it has any effect. `Lose`: the call takes effect, then fails, like a lost response. */
9
+ type Fault = "Fail" | "Lose"
10
+
11
+ interface MemStoreOptions {
12
+ /** The store clock behind `Date` and `Last-Modified`. Defaults to a counter that ticks once per call. */
13
+ readonly clock?: () => Millis
14
+ /** Picks the fault, if any, for each call. */
15
+ readonly fault?: (call: { readonly verb: Verb; readonly bucket: Bucket; readonly key: string }) => Fault | undefined
16
+ }
17
+
18
+ interface Stored {
19
+ readonly bytes: Uint8Array
20
+ readonly lastModified: Millis
21
+ }
22
+
23
+ /** An in-memory store with the create-only semantics of S3, and deterministic fault injection for tests. */
24
+ interface MemStore extends ObjectStore {
25
+ /** A copy of the objects currently in `bucket`, by key. */
26
+ readonly objects: (bucket: Bucket) => ReadonlyMap<string, Uint8Array>
27
+ }
28
+
29
+ function failure(operation: string): DbError {
30
+ return new DbError({ operation, reason: { _tag: "Io", kind: "Injected" } })
31
+ }
32
+
33
+ function make(options: MemStoreOptions = {}): MemStore {
34
+ let ticks = 0n
35
+ const clock =
36
+ options.clock ??
37
+ (() => {
38
+ ticks += 1n
39
+ return ticks
40
+ })
41
+ const buckets: Record<Bucket, Map<string, Stored>> = { Log: new Map(), Checkpoints: new Map() }
42
+
43
+ function run<R>(
44
+ verb: Verb,
45
+ bucket: Bucket,
46
+ key: string,
47
+ body: (now: Millis) => Effect.Effect<R, DbError>
48
+ ): Effect.Effect<Reply<R>, DbError> {
49
+ return Effect.suspend(() => {
50
+ const fault = options.fault?.({ verb, bucket, key })
51
+ const operation = `MemStore.${verb}`
52
+ if (fault === "Fail") return Effect.fail(failure(operation))
53
+ const now = clock()
54
+ return Effect.flatMap(body(now), (result) =>
55
+ fault === "Lose" ? Effect.fail(failure(operation)) : Effect.succeed({ date: now, result })
56
+ )
57
+ })
58
+ }
59
+
60
+ const io = <A>(operation: string, f: () => Promise<A>) =>
61
+ Effect.tryPromise({
62
+ try: f,
63
+ catch: () => new DbError({ operation, reason: { _tag: "Io", kind: "File" } })
64
+ })
65
+
66
+ return {
67
+ get: (bucket: Bucket, key: string, target: Target) =>
68
+ run("get", bucket, key, (): Effect.Effect<Fetched, DbError> => {
69
+ const stored = buckets[bucket].get(key)
70
+ if (stored === undefined) return Effect.succeed({ _tag: "Missing" })
71
+ if (target._tag === "Memory") {
72
+ return Effect.succeed({ _tag: "Body", bytes: stored.bytes.slice(), lastModified: stored.lastModified })
73
+ }
74
+ return io("MemStore.get", () => fs.writeFile(target.path, stored.bytes)).pipe(
75
+ Effect.as<Fetched>({ _tag: "Saved", lastModified: stored.lastModified })
76
+ )
77
+ }),
78
+ putIfAbsent: (bucket: Bucket, key: string, body: Body) =>
79
+ run("putIfAbsent", bucket, key, (now): Effect.Effect<Created, DbError> => {
80
+ const bytes =
81
+ body._tag === "Bytes"
82
+ ? Effect.succeed(body.bytes.slice())
83
+ : io("MemStore.putIfAbsent", () => fs.readFile(body.path)).pipe(
84
+ Effect.map((buffer) => new Uint8Array(buffer))
85
+ )
86
+ return Effect.map(bytes, (owned): Created => {
87
+ const objects = buckets[bucket]
88
+ if (objects.has(key)) return { _tag: "Occupied" }
89
+ objects.set(key, { bytes: owned, lastModified: now })
90
+ return { _tag: "Created" }
91
+ })
92
+ }),
93
+ list: (prefix: string, startAfter: string | null, maxKeys: number) =>
94
+ run("list", "Checkpoints", prefix, (): Effect.Effect<Listed> => {
95
+ const keys = [...buckets.Checkpoints.keys()]
96
+ .filter((key) => key.startsWith(prefix) && (startAfter === null || key > startAfter))
97
+ .sort()
98
+ .slice(0, maxKeys)
99
+ return Effect.succeed({ _tag: "Keys", keys })
100
+ }),
101
+ delete: (key: string) =>
102
+ run("delete", "Checkpoints", key, (): Effect.Effect<Deleted> => {
103
+ buckets.Checkpoints.delete(key)
104
+ return Effect.succeed({ _tag: "Deleted" })
105
+ }),
106
+ objects: (bucket: Bucket) => new Map([...buckets[bucket]].map(([key, stored]) => [key, stored.bytes.slice()]))
107
+ }
108
+ }
109
+
110
+ const MemStore = Object.freeze({ make })
111
+
112
+ export type { Fault, MemStoreOptions }
113
+ export { MemStore }
@@ -0,0 +1,107 @@
1
+ import type { Effect, Scope } from "effect"
2
+ import type { ChangeDraft } from "../changes.ts"
3
+ import type { QueryReader } from "../db.ts"
4
+ import type { DbError } from "../errors.ts"
5
+ import { AuthoringError } from "../errors.ts"
6
+ import type { AnySchema } from "../schema.ts"
7
+ import { schemaDescriptor, schemasAgree } from "../schema.ts"
8
+
9
+ /** What `populate` reads from (the database at the old schema) and writes into (rows at the new one). */
10
+ interface PopulateContext<From extends AnySchema, To extends AnySchema> {
11
+ readonly from: QueryReader<From>
12
+ readonly into: ChangeDraft<To>
13
+ }
14
+
15
+ /**
16
+ * Computes the rows a migration adds at its new schema, reading the database at the old one.
17
+ * Relations that keep their name and fields are copied unchanged without code.
18
+ */
19
+ type Populate<From extends AnySchema, To extends AnySchema> = (
20
+ context: PopulateContext<From, To>
21
+ ) => Effect.Effect<void, DbError, Scope.Scope>
22
+
23
+ /**
24
+ * One bundled migration: its stable `id` (the directory name), the content `hash` that
25
+ * `bumbledb generate` wrote (64 hex digits), and the schema it migrates to. The first migration of a
26
+ * bundle has no `from`: it is the initial schema, and its `populate` seeds a newly created database
27
+ * once.
28
+ */
29
+ interface Migration<From extends AnySchema | undefined = AnySchema | undefined, To extends AnySchema = AnySchema> {
30
+ readonly id: string
31
+ readonly hash: string
32
+ readonly from: From
33
+ readonly to: To
34
+ readonly populate?: Populate<From extends AnySchema ? From : To, To>
35
+ }
36
+
37
+ /** Any migration of a bundle; its `populate` is read back through {@link populateOf}. */
38
+ interface AnyMigration {
39
+ readonly id: string
40
+ readonly hash: string
41
+ readonly from: AnySchema | undefined
42
+ readonly to: AnySchema
43
+ readonly populate?: unknown
44
+ }
45
+
46
+ /** A bundle: the initial migration first, then each migration from its predecessor's schema. */
47
+ type Migrations = readonly [AnyMigration, ...AnyMigration[]]
48
+
49
+ const HASH = /^[0-9a-f]{64}$/
50
+ const ID = /^[0-9]{4}_[A-Za-z0-9_]+$/
51
+
52
+ function make<To extends AnySchema, From extends AnySchema = never>(input: {
53
+ readonly id: string
54
+ readonly hash: string
55
+ readonly from?: From
56
+ readonly to: To
57
+ readonly populate?: Populate<[From] extends [never] ? To : From, To>
58
+ }): Migration<[From] extends [never] ? undefined : From, To>
59
+ function make(input: {
60
+ readonly id: string
61
+ readonly hash: string
62
+ readonly from?: AnySchema
63
+ readonly to: AnySchema
64
+ readonly populate?: Populate<AnySchema, AnySchema>
65
+ }): AnyMigration {
66
+ if (!ID.test(input.id)) throw new AuthoringError({ message: `migration ${input.id}: an id is NNNN_name` })
67
+ if (!HASH.test(input.hash)) throw new AuthoringError({ message: `migration ${input.id}: a hash is 64 hex digits` })
68
+ const migration = {
69
+ id: input.id,
70
+ hash: input.hash,
71
+ from: input.from === undefined ? undefined : schemaDescriptor(input.from),
72
+ to: schemaDescriptor(input.to),
73
+ ...(input.populate === undefined ? {} : { populate: input.populate })
74
+ }
75
+ return Object.freeze(migration)
76
+ }
77
+
78
+ /** The populate step of a migration `make` built. */
79
+ function populateOf(migration: AnyMigration): Populate<AnySchema, AnySchema> | undefined {
80
+ return migration.populate as Populate<AnySchema, AnySchema> | undefined
81
+ }
82
+
83
+ /** Checks that each migration starts from its predecessor's schema and that ids are unique. */
84
+ function verifyBundle(migrations: Migrations): void {
85
+ const ids = new Set<string>()
86
+ migrations.forEach((migration, index) => {
87
+ if (ids.has(migration.id)) throw new AuthoringError({ message: `migrations: ${migration.id} appears twice` })
88
+ ids.add(migration.id)
89
+ const previous = migrations[index - 1]
90
+ if (previous === undefined) {
91
+ if (migration.from !== undefined) {
92
+ throw new AuthoringError({ message: `migrations: the first migration ${migration.id} has no from schema` })
93
+ }
94
+ return
95
+ }
96
+ if (migration.from === undefined || !schemasAgree(migration.from, previous.to)) {
97
+ throw new AuthoringError({
98
+ message: `migrations: ${migration.id} must start from the schema ${previous.id} migrates to`
99
+ })
100
+ }
101
+ })
102
+ }
103
+
104
+ const Migration = Object.freeze({ make })
105
+
106
+ export type { AnyMigration, Migrations, Populate, PopulateContext }
107
+ export { Migration, populateOf, verifyBundle }