@gusto/baerly-storage 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +47 -124
  2. package/dist/API.md +161 -511
  3. package/dist/CHANGELOG.md +1148 -307
  4. package/dist/RECIPES.md +398 -80
  5. package/dist/THIRD-PARTY-LICENSES.txt +5 -26
  6. package/dist/{app-config-DyiFK3_k.d.ts → app-config-C6qW7m8S.d.ts} +119 -111
  7. package/dist/app-config.d.ts +31 -1
  8. package/dist/app-config.js +1 -2
  9. package/dist/app-config.js.map +1 -1
  10. package/dist/auth.d.ts +33 -28
  11. package/dist/auth.js +549 -985
  12. package/dist/auth.js.map +1 -1
  13. package/dist/baerly.js +130 -138
  14. package/dist/baerly.js.map +1 -1
  15. package/dist/baerly.spec.json +97 -22
  16. package/dist/capacity-warning-header-BlVopyiI.js +16 -0
  17. package/dist/capacity-warning-header-BlVopyiI.js.map +1 -0
  18. package/dist/{client-Fe8CwB73.d.ts → client-CHQTXpef.d.ts} +121 -36
  19. package/dist/client-react.d.ts +67 -20
  20. package/dist/client-react.js +240 -54
  21. package/dist/client-react.js.map +1 -1
  22. package/dist/client.d.ts +5 -4
  23. package/dist/client.js +51 -16
  24. package/dist/client.js.map +1 -1
  25. package/dist/{cloudflare-DzKBrUlg.js → cloudflare-CKHYXccw.js} +504 -124
  26. package/dist/cloudflare-CKHYXccw.js.map +1 -0
  27. package/dist/cloudflare.d.ts +106 -71
  28. package/dist/cloudflare.js +334 -271
  29. package/dist/cloudflare.js.map +1 -1
  30. package/dist/code-resolution-CL_7L0wS.js +22 -0
  31. package/dist/code-resolution-CL_7L0wS.js.map +1 -0
  32. package/dist/collection-api-DbyhLAY8.d.ts +1001 -0
  33. package/dist/config-DEzInG6M.js +4035 -0
  34. package/dist/config-DEzInG6M.js.map +1 -0
  35. package/dist/constants-D3GSH3tc.js +469 -0
  36. package/dist/constants-D3GSH3tc.js.map +1 -0
  37. package/dist/{context-DRZ3tZ5l.js → context-rlpSnHFA.js} +17 -17
  38. package/dist/context-rlpSnHFA.js.map +1 -0
  39. package/dist/cost-B52llVDx.js +330 -0
  40. package/dist/cost-B52llVDx.js.map +1 -0
  41. package/dist/{current-json-C6nhPOm1.js → current-json-CC6lE4Aj.js} +115 -32
  42. package/dist/current-json-CC6lE4Aj.js.map +1 -0
  43. package/dist/db-plFUNTSF.d.ts +130 -0
  44. package/dist/db-read-seams-BSs0m1z1.js +3261 -0
  45. package/dist/db-read-seams-BSs0m1z1.js.map +1 -0
  46. package/dist/{deploy-DonTr-CV.js → deploy-Obj4nE-Z.js} +12 -16
  47. package/dist/deploy-Obj4nE-Z.js.map +1 -0
  48. package/dist/{dev-banner-CWHIk8tH.d.ts → dev-banner-B5AUHmu1.d.ts} +1 -4
  49. package/dist/dev-landing-BI5UdmOb.d.ts +32 -0
  50. package/dist/dev-landing-CD-si4dW.js +50 -0
  51. package/dist/dev-landing-CD-si4dW.js.map +1 -0
  52. package/dist/dev-vite.d.ts +38 -39
  53. package/dist/dev-vite.js +67 -36
  54. package/dist/dev-vite.js.map +1 -1
  55. package/dist/dev.d.ts +3 -2
  56. package/dist/dev.js +2 -1
  57. package/dist/doctor-BA0i-YAN.js +270 -0
  58. package/dist/doctor-BA0i-YAN.js.map +1 -0
  59. package/dist/dump-D1vX8ozg.js +133 -0
  60. package/dist/dump-D1vX8ozg.js.map +1 -0
  61. package/dist/errors-7Rk1HGCE.js +160 -0
  62. package/dist/errors-7Rk1HGCE.js.map +1 -0
  63. package/dist/{validate-DtljsCh1.js → export-BVL2EvYE.js} +409 -19
  64. package/dist/export-BVL2EvYE.js.map +1 -0
  65. package/dist/fsck-BWkPgov4.js +145 -0
  66. package/dist/fsck-BWkPgov4.js.map +1 -0
  67. package/dist/gcs-admin-Dso1HWQj.d.ts +127 -0
  68. package/dist/gcs.d.ts +30 -0
  69. package/dist/gcs.js +1 -0
  70. package/dist/goog4-signer-CeLIABoE.js +331 -0
  71. package/dist/goog4-signer-CeLIABoE.js.map +1 -0
  72. package/dist/{http-DGd5tbNs.js → http-Z5rZyYMX.js} +898 -271
  73. package/dist/http-Z5rZyYMX.js.map +1 -0
  74. package/dist/http-transport-DNUM8kU-.js +1299 -0
  75. package/dist/http-transport-DNUM8kU-.js.map +1 -0
  76. package/dist/http.d.ts +366 -294
  77. package/dist/http.js +2 -2
  78. package/dist/index-JJrIu1pO.d.ts +135 -0
  79. package/dist/index-LEgHN0qJ.d.ts +50 -0
  80. package/dist/{index-CuA9Mi3K.d.ts → index-RiEYj8t0.d.ts} +291 -78
  81. package/dist/index.d.ts +27 -6
  82. package/dist/index.js +3 -5
  83. package/dist/inspect-BTsVYZKd.js +196 -0
  84. package/dist/inspect-BTsVYZKd.js.map +1 -0
  85. package/dist/{key-Di6yyxFu.js → key-C4FPZ9Io.js} +3 -4
  86. package/dist/{key-Di6yyxFu.js.map → key-C4FPZ9Io.js.map} +1 -1
  87. package/dist/ledger-Dj_bGgo0.js +192 -0
  88. package/dist/ledger-Dj_bGgo0.js.map +1 -0
  89. package/dist/legacy-names-DgZBtgOh.js +47 -0
  90. package/dist/legacy-names-DgZBtgOh.js.map +1 -0
  91. package/dist/log-C9W15Dfo.d.ts +94 -0
  92. package/dist/log-tail-DWJbneBF.js +142 -0
  93. package/dist/log-tail-DWJbneBF.js.map +1 -0
  94. package/dist/log-walk-C-vA-WGJ.js +111 -0
  95. package/dist/log-walk-C-vA-WGJ.js.map +1 -0
  96. package/dist/maintenance-Dd-Yakqj.js +1709 -0
  97. package/dist/maintenance-Dd-Yakqj.js.map +1 -0
  98. package/dist/maintenance-DevB6z4e.js +2237 -0
  99. package/dist/maintenance-DevB6z4e.js.map +1 -0
  100. package/dist/maintenance.d.ts +100 -3
  101. package/dist/maintenance.js +1 -3
  102. package/dist/node.d.ts +130 -142
  103. package/dist/node.js +4 -3
  104. package/dist/{observability-BmflLCdF.js → observability-C-ISCCLE.js} +1275 -332
  105. package/dist/observability-C-ISCCLE.js.map +1 -0
  106. package/dist/observability.d.ts +27 -3
  107. package/dist/observability.js +3 -3
  108. package/dist/path-segment-DBXG26PW.js +363 -0
  109. package/dist/path-segment-DBXG26PW.js.map +1 -0
  110. package/dist/{plan-a7PJHBCy.js → plan-BvQbaNMe.js} +19 -11
  111. package/dist/plan-BvQbaNMe.js.map +1 -0
  112. package/dist/request-CRWiTrza.js +272 -0
  113. package/dist/request-CRWiTrza.js.map +1 -0
  114. package/dist/restore-U6Im8V1V.js +1467 -0
  115. package/dist/restore-U6Im8V1V.js.map +1 -0
  116. package/dist/retry-OBtLB2GC.js +126 -0
  117. package/dist/retry-OBtLB2GC.js.map +1 -0
  118. package/dist/rolldown-runtime-D3KUYPsf.js +12 -0
  119. package/dist/{runtime-spec-Dsw2-n8S.js → runtime-spec-B5mg3pHF.js} +189 -28
  120. package/dist/runtime-spec-B5mg3pHF.js.map +1 -0
  121. package/dist/{s3-http-B-z8th6O.d.ts → s3-http-DBC_wiCL.d.ts} +18 -10
  122. package/dist/s3.d.ts +20 -7
  123. package/dist/s3.js +4 -5
  124. package/dist/s3.js.map +1 -1
  125. package/dist/sha256-4FfGQNPw.js +22 -0
  126. package/dist/sha256-4FfGQNPw.js.map +1 -0
  127. package/dist/{shared-secret-DgEicd3x.js → shared-secret-Vkur-8r0.js} +27 -15
  128. package/dist/shared-secret-Vkur-8r0.js.map +1 -0
  129. package/dist/sigv4-sign-5IUflUD_.js +507 -0
  130. package/dist/sigv4-sign-5IUflUD_.js.map +1 -0
  131. package/dist/snapshot-CknzkDM0.js +148 -0
  132. package/dist/snapshot-CknzkDM0.js.map +1 -0
  133. package/dist/{src-BL_JoPTn.js → src-Actga-bO.js} +1004 -378
  134. package/dist/src-Actga-bO.js.map +1 -0
  135. package/dist/src-D0MQfvOp.js +812 -0
  136. package/dist/src-D0MQfvOp.js.map +1 -0
  137. package/dist/src-Dr9uo3dc.js +501 -0
  138. package/dist/src-Dr9uo3dc.js.map +1 -0
  139. package/dist/{subcommand-DoDA-K7d.js → subcommand-BLoF66dG.js} +89 -16
  140. package/dist/subcommand-BLoF66dG.js.map +1 -0
  141. package/dist/{time-DUqvFMHq.js → time-DRxTP9iv.js} +15 -34
  142. package/dist/time-DRxTP9iv.js.map +1 -0
  143. package/dist/types-suVbsEWS.d.ts +116 -0
  144. package/dist/{usage-DBV1eQHC.js → usage-CPzVhrQr.js} +40 -45
  145. package/dist/usage-CPzVhrQr.js.map +1 -0
  146. package/dist/uuid-BuT_c404.js +333 -0
  147. package/dist/uuid-BuT_c404.js.map +1 -0
  148. package/dist/verifier-Bmlmndzo.d.ts +95 -0
  149. package/dist/{wire-izfGZDvr.js → wire-DndvKU-K.js} +2 -3
  150. package/dist/wire-DndvKU-K.js.map +1 -0
  151. package/package.json +96 -69
  152. package/dist/aws4fetch.esm-p7eEzLCs.js +0 -1702
  153. package/dist/aws4fetch.esm-p7eEzLCs.js.map +0 -1
  154. package/dist/bytes-BXdamO3L.js +0 -11
  155. package/dist/bytes-BXdamO3L.js.map +0 -1
  156. package/dist/cloudflare-DzKBrUlg.js.map +0 -1
  157. package/dist/config-1tv0-Yzm.js +0 -2693
  158. package/dist/config-1tv0-Yzm.js.map +0 -1
  159. package/dist/constants-CfeMWQWk.js +0 -222
  160. package/dist/constants-CfeMWQWk.js.map +0 -1
  161. package/dist/context-DRZ3tZ5l.js.map +0 -1
  162. package/dist/cost-DSMIVqzL.js +0 -203
  163. package/dist/cost-DSMIVqzL.js.map +0 -1
  164. package/dist/current-json-C6nhPOm1.js.map +0 -1
  165. package/dist/db-DsPkOVEH.d.ts +0 -254
  166. package/dist/deploy-DonTr-CV.js.map +0 -1
  167. package/dist/doctor-DSIyCDg_.js +0 -346
  168. package/dist/doctor-DSIyCDg_.js.map +0 -1
  169. package/dist/dump-Bh5k26VQ.js +0 -94
  170. package/dist/dump-Bh5k26VQ.js.map +0 -1
  171. package/dist/errors-Dfa3p78c.js +0 -78
  172. package/dist/errors-Dfa3p78c.js.map +0 -1
  173. package/dist/export-DtF5tDIM.js +0 -401
  174. package/dist/export-DtF5tDIM.js.map +0 -1
  175. package/dist/fsck-D_djMO5D.js +0 -279
  176. package/dist/fsck-D_djMO5D.js.map +0 -1
  177. package/dist/http-DGd5tbNs.js.map +0 -1
  178. package/dist/index-0rZcMBwy.d.ts +0 -115
  179. package/dist/index-BNRh23K9.d.ts +0 -162
  180. package/dist/index-BYk9oCNf.d.ts +0 -1112
  181. package/dist/indexes-Dn9hbGQG.d.ts +0 -22
  182. package/dist/inspect-6WZ-pC7-.js +0 -133
  183. package/dist/inspect-6WZ-pC7-.js.map +0 -1
  184. package/dist/log-tail-D98U1JBo.js +0 -522
  185. package/dist/log-tail-D98U1JBo.js.map +0 -1
  186. package/dist/maintenance-B4Dki6Ig.js +0 -2157
  187. package/dist/maintenance-B4Dki6Ig.js.map +0 -1
  188. package/dist/maintenance-B_e0T99L.d.ts +0 -537
  189. package/dist/maintenance-C7coQp4c.js +0 -1418
  190. package/dist/maintenance-C7coQp4c.js.map +0 -1
  191. package/dist/normalize-rjnZLwS4.js +0 -133
  192. package/dist/normalize-rjnZLwS4.js.map +0 -1
  193. package/dist/observability-BmflLCdF.js.map +0 -1
  194. package/dist/plan-a7PJHBCy.js.map +0 -1
  195. package/dist/query-rAtXNQQB.js +0 -1773
  196. package/dist/query-rAtXNQQB.js.map +0 -1
  197. package/dist/rebuild-index-BVzQSZlm.js +0 -81
  198. package/dist/rebuild-index-BVzQSZlm.js.map +0 -1
  199. package/dist/request-u0M51mhJ.js +0 -72
  200. package/dist/request-u0M51mhJ.js.map +0 -1
  201. package/dist/restore-R-4rbciV.js +0 -778
  202. package/dist/restore-R-4rbciV.js.map +0 -1
  203. package/dist/rolldown-runtime-EAsCxrDo.js +0 -14
  204. package/dist/runtime-spec-Dsw2-n8S.js.map +0 -1
  205. package/dist/shared-secret-DgEicd3x.js.map +0 -1
  206. package/dist/src-BL_JoPTn.js.map +0 -1
  207. package/dist/src-CYHFDrKz.js +0 -536
  208. package/dist/src-CYHFDrKz.js.map +0 -1
  209. package/dist/src-DYZ4H1aF.js +0 -392
  210. package/dist/src-DYZ4H1aF.js.map +0 -1
  211. package/dist/subcommand-DoDA-K7d.js.map +0 -1
  212. package/dist/time-DUqvFMHq.js.map +0 -1
  213. package/dist/usage-DBV1eQHC.js.map +0 -1
  214. package/dist/validate-DtljsCh1.js.map +0 -1
  215. package/dist/wire-izfGZDvr.js.map +0 -1
@@ -1,2157 +0,0 @@
1
- import { a as decodeJsonBytes, i as readCurrentJson, o as encodeJsonBytes, r as logSeqStartOf, t as casUpdateCurrentJson } from "./current-json-C6nhPOm1.js";
2
- import { a as GC_MAX_PENDING_CANDIDATES, l as MAINTENANCE_PROFILE_CF_FREE, o as GC_PENDING_CONTENT_TYPE, s as LOG_FORWARD_PROBE_CAP, u as MAINTENANCE_PROFILE_CF_PAID } from "./constants-CfeMWQWk.js";
3
- import { t as BaerlyError } from "./errors-Dfa3p78c.js";
4
- import { i as noopMetricsRecorder, n as getCurrentContext } from "./context-DRZ3tZ5l.js";
5
- //#region packages/protocol/src/coordination/gc-pending.ts
6
- /**
7
- * `gc/pending.json` — the per-collection GC candidate ledger. CAS-
8
- * protected; one per `(tenant, collection)` key — for example
9
- * `<tenant>/<collection>/gc/pending.json`. `runGc()` in
10
- * `packages/server/src/gc.ts` is the single writer that adds and
11
- * removes entries. Stays small in steady state because every
12
- * candidate is deleted once `due_at` passes.
13
- *
14
- * "Mark" appends candidates with a future `due_at`; "sweep" deletes
15
- * candidates whose `due_at` is in the past. The two phases run in
16
- * the same compactor pass — see `runGc()` in
17
- * `packages/server/src/gc.ts`.
18
- *
19
- * Lives next to {@link CurrentJson} in `coordination/` — same shape
20
- * of "CAS-protected small control object." Pure module; no Node
21
- * imports; Worker-bundleable.
22
- *
23
- * @see packages/server/src/gc.ts
24
- */
25
- /**
26
- * Read + parse `gc/pending.json` at `key`. Returns `null` on
27
- * not-found.
28
- *
29
- * @throws BaerlyError{code:"InvalidResponse"} — body is not valid
30
- * JSON, or `schema_version` is unknown, or the shape fails
31
- * the runtime guard.
32
- */
33
- const readGcPending = async (storage, key, opts) => {
34
- const got = await storage.get(key, opts);
35
- if (got === null) return null;
36
- let parsed;
37
- try {
38
- parsed = decodeJsonBytes(got.body);
39
- } catch (error) {
40
- throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: body is not valid JSON`, error);
41
- }
42
- return {
43
- json: assertGcPending(parsed, key),
44
- etag: got.etag
45
- };
46
- };
47
- /**
48
- * Create `gc/pending.json` if-and-only-if it does not exist (S3
49
- * `If-None-Match: "*"`). Use this once per collection at first GC
50
- * run; subsequent updates go through {@link casUpdateGcPending}.
51
- *
52
- * @throws BaerlyError{code:"Conflict"} — the key already exists.
53
- * @throws BaerlyError{code:"InvalidResponse"} — `initial` fails the
54
- * runtime shape guard.
55
- */
56
- const createGcPending = async (storage, key, initial, opts) => {
57
- assertGcPending(initial, key);
58
- const body = encodeJsonBytes(initial);
59
- const putOpts = {
60
- ifNoneMatch: "*",
61
- contentType: GC_PENDING_CONTENT_TYPE,
62
- ...opts?.signal !== void 0 && { signal: opts.signal }
63
- };
64
- let result;
65
- try {
66
- result = await storage.put(key, body, putOpts);
67
- } catch (error) {
68
- throw translateCasError(error, key);
69
- }
70
- return {
71
- json: initial,
72
- etag: result.etag
73
- };
74
- };
75
- /**
76
- * Read-modify-write `gc/pending.json` under CAS, with a bounded retry
77
- * loop. Each attempt re-reads the latest body, hands the `mutator`
78
- * that latest value, and writes the result with
79
- * `If-Match: <latest etag>`. On `Conflict` (a concurrent writer landed
80
- * between this attempt's read and write) the loop re-reads and
81
- * re-applies the mutator, up to {@link GC_PENDING_CAS_MAX_ATTEMPTS}
82
- * total attempts, then surfaces `Conflict`.
83
- *
84
- * `mutator` MUST be a pure, deterministic MERGE of this pass's results
85
- * INTO the `latest` value it is handed — it is called once per attempt
86
- * (so it must not have side effects), and it MUST read `latest` rather
87
- * than ignore it and write a precomputed set. Ignoring `latest` would
88
- * silently overwrite a concurrent pass's candidates: the
89
- * If-Match succeeds (it used the fresh etag), so no conflict is
90
- * raised, and the concurrent marks are lost. See {@link mergeGcPending}.
91
- *
92
- * @throws BaerlyError{code:"Conflict"} — another writer kept landing a
93
- * write between this function's read and write for every
94
- * attempt.
95
- * @throws BaerlyError{code:"InvalidResponse"} — `key` does not exist
96
- * (use {@link createGcPending} instead) or body doesn't
97
- * parse / fails the shape guard.
98
- */
99
- const casUpdateGcPending = async (storage, key, mutator, opts) => {
100
- let lastConflict;
101
- for (let attempt = 0; attempt < 3; attempt++) {
102
- const existing = await readGcPending(storage, key, opts);
103
- if (existing === null) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key} does not exist; use createGcPending first`);
104
- const next = mutator(structuredClone(existing.json));
105
- assertGcPending(next, key);
106
- const body = encodeJsonBytes(next);
107
- const putOpts = {
108
- ifMatch: existing.etag,
109
- contentType: GC_PENDING_CONTENT_TYPE,
110
- ...opts?.signal !== void 0 && { signal: opts.signal }
111
- };
112
- try {
113
- return {
114
- json: next,
115
- etag: (await storage.put(key, body, putOpts)).etag
116
- };
117
- } catch (error) {
118
- const translated = translateCasError(error, key);
119
- if (translated.code !== "Conflict") throw translated;
120
- lastConflict = translated;
121
- }
122
- }
123
- throw lastConflict;
124
- };
125
- /**
126
- * Pure merge of one GC pass's results INTO the `latest`
127
- * `gc/pending.json` value, used as the {@link casUpdateGcPending}
128
- * mutator. Deterministic, no I/O — the kernel unit + mutation test
129
- * surface. Splitting it out is what makes the CAS write correct under
130
- * concurrency: each retry re-merges against fresh `latest` rather than
131
- * overwriting it with a stale precomputed set.
132
- *
133
- * Semantics:
134
- * - **candidates**: start from `latest.candidates`, DROP any whose key
135
- * this pass already DELETED (`sweptKeys`), then APPEND every fresh
136
- * mark whose key is not already present AND was not itself swept
137
- * this pass (a `graceMillis: 0` pass can mark-and-sweep a key in one
138
- * go — re-adding it to the ledger after deleting it would be wrong;
139
- * dedup also drops a key a concurrent pass already marked). Capped at
140
- * `maxCandidates`.
141
- * - **last_swept_at**: the lexicographically LATER of `latest`'s and
142
- * this pass's (ISO-8601 UTC; `""` sorts lowest so a real timestamp
143
- * always wins). Never regress a concurrent pass's newer sweep time.
144
- * - **content_scan_cursor**: rotation cursor — liveness, not
145
- * correctness; just don't regress forward progress. The two sides
146
- * are NOT symmetric: `pass.nextContentCursor === undefined` means
147
- * THIS pass examined to the END of the keyspace (WRAP — the most-
148
- * advanced position), whereas `latest.content_scan_cursor` absent
149
- * means the stored ledger has no cursor yet (start-from-beginning).
150
- * So:
151
- * - our pass WRAPPED ⇒ result wraps (key omitted) — our pass is
152
- * the most advanced; this is the single-writer rotation reset.
153
- * - else (our pass advanced to a defined cursor):
154
- * - latest cursor defined ⇒ take the lexicographically GREATER
155
- * (don't regress a concurrent pass's advance);
156
- * - latest cursor absent ⇒ keep OUR advance (don't lose it).
157
- */
158
- const mergeGcPending = (latest, pass) => {
159
- const surviving = latest.candidates.filter((c) => !pass.sweptKeys.has(c.key));
160
- const present = new Set(surviving.map((c) => c.key));
161
- const appended = [];
162
- for (const c of pass.newCandidates) if (!pass.sweptKeys.has(c.key) && !present.has(c.key)) {
163
- present.add(c.key);
164
- appended.push(c);
165
- }
166
- const candidates = [...surviving, ...appended].slice(0, pass.maxCandidates);
167
- const lastSweptAt = pass.lastSweptAt > latest.last_swept_at ? pass.lastSweptAt : latest.last_swept_at;
168
- const latestCursor = latest.content_scan_cursor;
169
- const passCursor = pass.nextContentCursor;
170
- let cursor;
171
- if (passCursor === void 0) cursor = void 0;
172
- else if (latestCursor !== void 0) cursor = passCursor > latestCursor ? passCursor : latestCursor;
173
- else cursor = passCursor;
174
- return {
175
- schema_version: 1,
176
- candidates,
177
- last_swept_at: lastSweptAt,
178
- ...cursor !== void 0 && { content_scan_cursor: cursor }
179
- };
180
- };
181
- const VALID_REASONS = /* @__PURE__ */ new Set([
182
- "stale-log",
183
- "orphan-snapshot",
184
- "orphan-content"
185
- ]);
186
- /**
187
- * Runtime guard for `parsed` to be a {@link GcPending}. Throws
188
- * `BaerlyError{code:"InvalidResponse"}` rather than coercing.
189
- */
190
- const assertGcPending = (parsed, key) => {
191
- if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: parsed body is not an object`);
192
- const r = parsed;
193
- if (r["schema_version"] !== 1) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: unsupported schema_version ${String(r["schema_version"])}; expected 1`);
194
- if (!Array.isArray(r["candidates"])) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates must be an array`);
195
- for (let i = 0; i < r["candidates"].length; i++) {
196
- const c = r["candidates"][i];
197
- if (c === null || typeof c !== "object") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}] is not an object`);
198
- const cr = c;
199
- if (typeof cr["key"] !== "string" || cr["key"].length === 0) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].key must be a non-empty string`);
200
- if (typeof cr["due_at"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].due_at must be a string`);
201
- if (typeof cr["reason"] !== "string" || !VALID_REASONS.has(cr["reason"])) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].reason must be one of stale-log|orphan-snapshot|orphan-content`);
202
- }
203
- if (typeof r["last_swept_at"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: last_swept_at must be a string`);
204
- if (r["content_scan_cursor"] !== void 0 && typeof r["content_scan_cursor"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: content_scan_cursor must be a string when present`);
205
- return parsed;
206
- };
207
- /**
208
- * Wrap a storage-level error with `gc/pending.json at <key>`
209
- * location context. `Conflict` (the storage layer's CAS-lost signal)
210
- * is re-thrown with an annotated message; other `BaerlyError`s pass
211
- * through; non-`BaerlyError` falls through as `InvalidResponse`.
212
- */
213
- const translateCasError = (e, key) => {
214
- if (e instanceof BaerlyError && e.code === "Conflict") return new BaerlyError("Conflict", `gc/pending.json CAS lost at ${key}: ${e.message}`, e);
215
- if (e instanceof BaerlyError) return e;
216
- return new BaerlyError("InvalidResponse", `gc/pending.json write at ${key} failed: ${String(e)}`, e);
217
- };
218
- //#endregion
219
- //#region packages/protocol/src/query/_internals.ts
220
- /** @internal Same primitive type for comparison purposes. */
221
- const sameComparableType = (a, b) => typeof a === "string" && typeof b === "string" || typeof a === "number" && typeof b === "number";
222
- /**
223
- * @internal Returns negative / zero / positive for a<b / a==b / a>b.
224
- */
225
- const compareScalar = (a, b) => {
226
- if (typeof a === "number" && typeof b === "number") return a - b;
227
- if (typeof a === "string" && typeof b === "string") {
228
- if (a < b) return -1;
229
- if (a > b) return 1;
230
- return 0;
231
- }
232
- return 0;
233
- };
234
- /** @internal Render a predicate path for use in error messages. */
235
- const formatPath = (path) => path.length === 0 ? "<root>" : path.map((p) => JSON.stringify(p)).join(".");
236
- /**
237
- * Recursive structural equality for `DocumentValue` values. Lifted to
238
- * `export` so the filtered-index implication checker (in
239
- * `@baerly/server`'s query planner) and other callers can reuse the
240
- * same comparison rule the validator + merger already trust. Two
241
- * values `a` and `b` are equal iff:
242
- *
243
- * - they are reference-equal (covers primitives and object identity);
244
- * - or they are both non-`null` objects with the same key set, and
245
- * every value pair is recursively equal.
246
- *
247
- * Arrays are out-of-band for `DocumentValue`; this function rejects
248
- * them implicitly via the same key-set walk (an array reads as an
249
- * object with numeric-string keys plus a `length` exposure that
250
- * `Object.keys` happens to skip — defensive callers should still gate
251
- * with `Array.isArray` first).
252
- */
253
- const deepEqualDocumentValue = (a, b) => {
254
- if (a === b) return true;
255
- if (typeof a !== "object" || typeof b !== "object") return false;
256
- const aKeys = Object.keys(a);
257
- const bKeys = Object.keys(b);
258
- if (aKeys.length !== bKeys.length) return false;
259
- for (const key of aKeys) {
260
- if (!(key in b)) return false;
261
- const aSub = a[key];
262
- const bSub = b[key];
263
- if (aSub === void 0 || bSub === void 0) return false;
264
- if (!deepEqualDocumentValue(aSub, bSub)) return false;
265
- }
266
- return true;
267
- };
268
- //#endregion
269
- //#region packages/protocol/src/query/matches.ts
270
- /**
271
- * Evaluate a validated wire-form predicate against a `JSONObject`
272
- * document. Returns `true` iff every clause holds.
273
- *
274
- * @example
275
- * ```ts
276
- * matchesWire(
277
- * { clauses: [{ op: "eq", field: "status", value: "open" }] },
278
- * { status: "open", priority: "p1" },
279
- * ); // true
280
- *
281
- * matchesWire(
282
- * { clauses: [{ op: "in", field: "priority", value: ["p1", "p2"] }] },
283
- * { priority: "p3" },
284
- * ); // false
285
- *
286
- * matchesWire(
287
- * {
288
- * clauses: [
289
- * { op: "gte", field: "count", value: 1 },
290
- * { op: "lt", field: "count", value: 10 },
291
- * ],
292
- * },
293
- * { count: 5 },
294
- * ); // true
295
- * ```
296
- */
297
- const matchesWire = (wire, doc) => {
298
- for (const clause of wire.clauses) if (!matchesClause(clause, lookupPath(doc, clause.field))) return false;
299
- return true;
300
- };
301
- const lookupPath = (doc, field) => {
302
- if (!field.includes(".")) return doc[field];
303
- const segments = field.split(".");
304
- let cursor = doc;
305
- for (const segment of segments) {
306
- if (cursor === void 0 || cursor === null || typeof cursor !== "object" || Array.isArray(cursor)) return;
307
- cursor = cursor[segment];
308
- }
309
- return cursor;
310
- };
311
- const matchesClause = (clause, actual) => {
312
- switch (clause.op) {
313
- case "eq": return matchesEq(clause.value, actual);
314
- case "in": return matchesIn(clause.value, actual);
315
- case "gt": return compareGT(actual, clause.value, false);
316
- case "gte": return compareGT(actual, clause.value, true);
317
- case "lt": return compareLT(actual, clause.value, false);
318
- case "lte": return compareLT(actual, clause.value, true);
319
- }
320
- };
321
- const matchesEq = (expected, actual) => {
322
- if (typeof expected === "object") {
323
- if (actual === void 0 || actual === null || typeof actual !== "object" || Array.isArray(actual)) return false;
324
- for (const subKey of Object.keys(expected)) {
325
- const subExpected = expected[subKey];
326
- if (subExpected === void 0) continue;
327
- const subActual = actual[subKey];
328
- if (!matchesEq(subExpected, subActual)) return false;
329
- }
330
- return true;
331
- }
332
- return expected === actual;
333
- };
334
- const matchesIn = (members, actual) => {
335
- for (const m of members) if (typeof m === "object") {
336
- if (matchesEq(m, actual)) return true;
337
- } else if (m === actual) return true;
338
- return false;
339
- };
340
- const compareGT = (actual, bound, inclusive) => {
341
- if (typeof bound === "string") return typeof actual === "string" && (inclusive ? actual >= bound : actual > bound);
342
- if (typeof bound === "number") return typeof actual === "number" && (inclusive ? actual >= bound : actual > bound);
343
- return false;
344
- };
345
- const compareLT = (actual, bound, inclusive) => {
346
- if (typeof bound === "string") return typeof actual === "string" && (inclusive ? actual <= bound : actual < bound);
347
- if (typeof bound === "number") return typeof actual === "number" && (inclusive ? actual <= bound : actual < bound);
348
- return false;
349
- };
350
- //#endregion
351
- //#region packages/protocol/src/query/satisfiable.ts
352
- /**
353
- * Per-field satisfiability check for a {@link PredicateWire}. Groups
354
- * clauses by field, then runs the same logic the pre-redesign
355
- * `assertOpObjectSatisfiable` ran on a single op-object — but
356
- * generalised across N clauses on the same field (the wire merger
357
- * concatenates lists without coalescing).
358
- *
359
- * Triggers:
360
- * - Two `eq` clauses on the same field with different values →
361
- * `InvalidConfig` (the same conflict that pre-redesign
362
- * `mergePredicates` threw on shared primitive keys).
363
- * - Range bounds with `lo > hi` (or `lo == hi` with strict on
364
- * either side) → `UnsatisfiablePredicate`.
365
- * - `eq` value outside the residual range interval or absent from
366
- * every `in` set on the same field → `UnsatisfiablePredicate`.
367
- * - `in` sets that intersect to empty across multiple clauses →
368
- * `UnsatisfiablePredicate`.
369
- *
370
- * Used by both `./validate.ts` (wire-arrival check) and
371
- * `./merge.ts` (post-merge re-check, kept in lockstep).
372
- */
373
- const groupByField = (wire) => {
374
- const out = /* @__PURE__ */ new Map();
375
- for (const clause of wire.clauses) {
376
- let group = out.get(clause.field);
377
- if (group === void 0) {
378
- group = {
379
- eqs: [],
380
- ins: []
381
- };
382
- out.set(clause.field, group);
383
- }
384
- applyClause(group, clause.field, clause);
385
- }
386
- return out;
387
- };
388
- const applyClause = (group, field, clause) => {
389
- switch (clause.op) {
390
- case "eq":
391
- group.eqs.push(clause.value);
392
- return;
393
- case "in":
394
- group.ins.push(clause.value);
395
- return;
396
- case "gt":
397
- tightenLower(group, field, clause.value, false);
398
- return;
399
- case "gte":
400
- tightenLower(group, field, clause.value, true);
401
- return;
402
- case "lt":
403
- tightenUpper(group, field, clause.value, false);
404
- return;
405
- case "lte":
406
- tightenUpper(group, field, clause.value, true);
407
- return;
408
- }
409
- };
410
- const tightenLower = (group, field, value, inclusive) => {
411
- if (group.lo === void 0) {
412
- group.lo = {
413
- value,
414
- inclusive
415
- };
416
- return;
417
- }
418
- if (!sameComparableType(group.lo.value, value)) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} has type-incompatible lower bounds (${JSON.stringify(group.lo.value)} vs ${JSON.stringify(value)}).`);
419
- const c = compareScalar(group.lo.value, value);
420
- if (c < 0 || c === 0 && inclusive === false && group.lo.inclusive === true) group.lo = {
421
- value,
422
- inclusive
423
- };
424
- };
425
- const tightenUpper = (group, field, value, inclusive) => {
426
- if (group.hi === void 0) {
427
- group.hi = {
428
- value,
429
- inclusive
430
- };
431
- return;
432
- }
433
- if (!sameComparableType(group.hi.value, value)) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} has type-incompatible upper bounds (${JSON.stringify(group.hi.value)} vs ${JSON.stringify(value)}).`);
434
- const c = compareScalar(group.hi.value, value);
435
- if (c > 0 || c === 0 && inclusive === false && group.hi.inclusive === true) group.hi = {
436
- value,
437
- inclusive
438
- };
439
- };
440
- const intersectInSets = (sets) => {
441
- if (sets.length === 0) return [];
442
- let acc = sets[0];
443
- for (let i = 1; i < sets.length; i++) {
444
- const next = [];
445
- for (const a of acc) for (const b of sets[i]) if (deepEqualDocumentValue(a, b)) {
446
- next.push(a);
447
- break;
448
- }
449
- acc = next;
450
- }
451
- return acc;
452
- };
453
- /**
454
- * Check that every per-field clause group on `wire` is jointly
455
- * satisfiable. Throws `BaerlyError{code:"UnsatisfiablePredicate"}`
456
- * (or `"InvalidConfig"` for conflicting `eq` clauses) on the
457
- * first contradiction. Returns `void` on success.
458
- */
459
- const assertWireSatisfiable = (wire) => {
460
- const groups = groupByField(wire);
461
- for (const [field, group] of groups) {
462
- if (group.eqs.length > 1) {
463
- for (let i = 1; i < group.eqs.length; i++) if (!deepEqualDocumentValue(group.eqs[0], group.eqs[i])) throw new BaerlyError("InvalidConfig", `Conflicting equality clauses on field ${formatPath([field])} (${JSON.stringify(group.eqs[0])} vs ${JSON.stringify(group.eqs[i])}). Predicate chains must agree on shared equality.`);
464
- }
465
- const eq = group.eqs[0];
466
- if (group.lo !== void 0 && group.hi !== void 0 && sameComparableType(group.lo.value, group.hi.value)) {
467
- const c = compareScalar(group.lo.value, group.hi.value);
468
- if (c > 0 || c === 0 && (!group.lo.inclusive || !group.hi.inclusive)) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} has empty interval (lo=${JSON.stringify(group.lo.value)} ${group.lo.inclusive ? "gte" : "gt"}, hi=${JSON.stringify(group.hi.value)} ${group.hi.inclusive ? "lte" : "lt"}).`);
469
- }
470
- if (eq !== void 0) {
471
- if (group.lo !== void 0) {
472
- if (!sameComparableType(eq, group.lo.value)) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} eq=${JSON.stringify(eq)} (${typeof eq}) is type-incompatible with lower bound ${JSON.stringify(group.lo.value)} (${typeof group.lo.value}); range ops require matching primitive types.`);
473
- const c = compareScalar(eq, group.lo.value);
474
- if (c < 0 || c === 0 && !group.lo.inclusive) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} eq=${JSON.stringify(eq)} excluded by lower bound ${JSON.stringify(group.lo.value)} (${group.lo.inclusive ? "gte" : "gt"}).`);
475
- }
476
- if (group.hi !== void 0) {
477
- if (!sameComparableType(eq, group.hi.value)) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} eq=${JSON.stringify(eq)} (${typeof eq}) is type-incompatible with upper bound ${JSON.stringify(group.hi.value)} (${typeof group.hi.value}); range ops require matching primitive types.`);
478
- const c = compareScalar(eq, group.hi.value);
479
- if (c > 0 || c === 0 && !group.hi.inclusive) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} eq=${JSON.stringify(eq)} excluded by upper bound ${JSON.stringify(group.hi.value)} (${group.hi.inclusive ? "lte" : "lt"}).`);
480
- }
481
- if (group.ins.length > 0) for (const set of group.ins) {
482
- let found = false;
483
- for (const m of set) if (deepEqualDocumentValue(eq, m)) {
484
- found = true;
485
- break;
486
- }
487
- if (!found) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} eq=${JSON.stringify(eq)} not present in in() set.`);
488
- }
489
- }
490
- if (group.ins.length > 1) {
491
- if (intersectInSets(group.ins).length === 0) throw new BaerlyError("UnsatisfiablePredicate", `Predicate on field ${formatPath([field])} has empty in() intersection across multiple clauses.`);
492
- }
493
- }
494
- };
495
- //#endregion
496
- //#region packages/protocol/src/query/validate.ts
497
- /**
498
- * Wire predicate validator. Walks a {@link PredicateWire} and throws
499
- * on a structurally bad clause (`BaerlyError{code:"InvalidConfig"}`)
500
- * or a per-field clause group that contradicts itself
501
- * (`BaerlyError{code:"UnsatisfiablePredicate"}`).
502
- *
503
- * The object-form normaliser at `./normalize.ts` enforces shape on
504
- * the way in (rejecting `$`-keys / non-scalars / null / undefined);
505
- * the callback builder at `./builder.ts` enforces shape via the
506
- * type system. This validator is the wire-arrival check: it runs
507
- * over already-normalised clauses, defending against raw-`fetch`
508
- * callers who bypass the typed seam and submit arbitrary JSON.
509
- *
510
- * Companion modules: `./matches.ts` (evaluator), `./merge.ts`
511
- * (AND-merge). Shared helpers live in `./_internals.ts`.
512
- */
513
- const OP_NAMES = [
514
- "eq",
515
- "gt",
516
- "gte",
517
- "lt",
518
- "lte",
519
- "in"
520
- ];
521
- const RANGE_OPS = [
522
- "gt",
523
- "gte",
524
- "lt",
525
- "lte"
526
- ];
527
- /**
528
- * Validate a normalised predicate wire. Throws
529
- * `BaerlyError{code:"InvalidConfig"}` on a structurally bad clause,
530
- * or `BaerlyError{code:"UnsatisfiablePredicate"}` when a clause
531
- * group on one field contradicts itself.
532
- *
533
- * Returns the wire unchanged on success — callers can thread the
534
- * validated wire forward.
535
- *
536
- * @example
537
- * ```ts
538
- * validateWire({ clauses: [{ op: "eq", field: "status", value: "open" }] }); // ok
539
- * validateWire({ clauses: [{ op: "in", field: "priority", value: [] }] }); // throws UnsatisfiablePredicate
540
- * validateWire({ clauses: [{ op: "eq", field: "_id", value: "x" }] }); // throws InvalidConfig (top-level _id)
541
- * ```
542
- */
543
- const validateWire = (wire) => {
544
- if (wire === null || typeof wire !== "object" || !Array.isArray(wire.clauses)) throw new BaerlyError("InvalidConfig", `Predicate wire must be an object with a "clauses" array.`);
545
- for (let i = 0; i < wire.clauses.length; i++) validateClause(wire.clauses[i], i);
546
- assertWireSatisfiable(wire);
547
- return wire;
548
- };
549
- const validateClause = (clause, index) => {
550
- if (clause === null || typeof clause !== "object") throw new BaerlyError("InvalidConfig", `Predicate clause at index ${index} must be an object — got ${JSON.stringify(clause)}.`);
551
- const { op, field, value } = clause;
552
- if (typeof op !== "string" || !OP_NAMES.includes(op)) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${index} has unsupported op ${JSON.stringify(op)} — supported: ${OP_NAMES.join(" / ")}.`);
553
- if (typeof field !== "string" || field.length === 0) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${index} has empty / non-string field.`);
554
- if (field === "_id" || field.startsWith("_id.")) throw new BaerlyError("InvalidConfig", "Predicates may not key on \"_id\"; use .get(id) / .update(id, patch) / .replace(id, doc) / .delete(id) instead.");
555
- if (op === "in") {
556
- if (!Array.isArray(value)) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${index} (op="in", field=${JSON.stringify(field)}) must carry an array value.`);
557
- if (value.length === 0) throw new BaerlyError("UnsatisfiablePredicate", `Predicate clause at index ${index} (op="in", field=${JSON.stringify(field)}) has empty value array — no document can match.`);
558
- for (let i = 0; i < value.length; i++) validateMemberValue(value[i], field, index, i);
559
- return;
560
- }
561
- if (Array.isArray(value)) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${index} (op=${JSON.stringify(op)}, field=${JSON.stringify(field)}) must carry a primitive value, got an array — only op="in" accepts arrays.`);
562
- if (RANGE_OPS.includes(op)) {
563
- validateRangeBound(value, field, index);
564
- return;
565
- }
566
- validateScalar(value, field, index);
567
- };
568
- const validateMemberValue = (value, field, clauseIndex, memberIndex) => {
569
- if (value === null || value === void 0) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${clauseIndex} (op="in", field=${JSON.stringify(field)}) member ${memberIndex} is ${value === null ? "null" : "undefined"} — members must be string / number / boolean / nested object.`);
570
- if (Array.isArray(value)) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${clauseIndex} (op="in", field=${JSON.stringify(field)}) member ${memberIndex} is an array — members must be string / number / boolean / nested object.`);
571
- const t = typeof value;
572
- if (t === "object") return;
573
- if (t !== "string" && t !== "number" && t !== "boolean") throw new BaerlyError("InvalidConfig", `Predicate clause at index ${clauseIndex} (op="in", field=${JSON.stringify(field)}) member ${memberIndex} has unsupported type ${JSON.stringify(t)} — must be string / number / boolean / nested object.`);
574
- if (t === "number" && !Number.isFinite(value)) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${clauseIndex} (op="in", field=${JSON.stringify(field)}) member ${memberIndex} is ${String(value)} — finite numbers only (NaN / Infinity do not round-trip through JSON).`);
575
- };
576
- const validateScalar = (value, field, clauseIndex) => {
577
- if (value === null || value === void 0) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${clauseIndex} (field=${JSON.stringify(field)}) value is ${value === null ? "null" : "undefined"} — terminal values must be string / number / boolean / nested object.`);
578
- const t = typeof value;
579
- if (t === "object") return;
580
- if (t !== "string" && t !== "number" && t !== "boolean") throw new BaerlyError("InvalidConfig", `Predicate clause at index ${clauseIndex} (field=${JSON.stringify(field)}) value has unsupported type ${JSON.stringify(t)} — must be string / number / boolean / nested object.`);
581
- if (t === "number" && !Number.isFinite(value)) throw new BaerlyError("InvalidConfig", `Predicate clause at index ${clauseIndex} (field=${JSON.stringify(field)}) value is ${String(value)} — finite numbers only (NaN / Infinity do not round-trip through JSON).`);
582
- };
583
- const validateRangeBound = (value, field, clauseIndex) => {
584
- if (value === null || value === void 0) throw new BaerlyError("InvalidConfig", `Predicate range bound at clause ${clauseIndex} (field=${JSON.stringify(field)}) is ${value === null ? "null" : "undefined"} — range bounds must be string or finite number.`);
585
- const t = typeof value;
586
- if (t === "boolean") throw new BaerlyError("InvalidConfig", `Predicate range bound at clause ${clauseIndex} (field=${JSON.stringify(field)}) is a boolean — range bounds must be string or finite number (booleans are not ordered).`);
587
- if (t === "object") throw new BaerlyError("InvalidConfig", `Predicate range bound at clause ${clauseIndex} (field=${JSON.stringify(field)}) is a nested object — range bounds must be string or finite number.`);
588
- if (t !== "string" && t !== "number") throw new BaerlyError("InvalidConfig", `Predicate range bound at clause ${clauseIndex} (field=${JSON.stringify(field)}) has unsupported type ${JSON.stringify(t)} — must be string or finite number.`);
589
- if (t === "number" && !Number.isFinite(value)) throw new BaerlyError("InvalidConfig", `Predicate range bound at clause ${clauseIndex} (field=${JSON.stringify(field)}) is ${String(value)} — finite numbers only (NaN / Infinity do not round-trip through JSON).`);
590
- };
591
- //#endregion
592
- //#region packages/protocol/src/hashing.ts
593
- /**
594
- * Width of the {@link ContentVersionId} hex strings produced by
595
- * {@link versionFromContent}. 32 hex chars = 128 bits, matching the
596
- * information content of a v4 UUID. Birthday-bound collision probability
597
- * at N=10⁹ writes is ~1.5 × 10⁻²¹ (≈ N² / 2¹²⁹); ample for a
598
- * content-addressed version id. A collision is not detected at runtime —
599
- * two distinct bodies that share a truncated hash would alias to the same
600
- * content key, so a reader could observe the wrong body. The bound above
601
- * is the sole guard, and it sits far below any plausible write volume.
602
- *
603
- * @see docs/spec/log-entry-shape.md §"Content body layout"
604
- */
605
- const VERSION_HEX_LENGTH = 32;
606
- /**
607
- * Content-addressed {@link ContentVersionId}: SHA-256 of `body`,
608
- * lowercase hex, truncated to {@link VERSION_HEX_LENGTH}. Same body
609
- * bytes ⇒ same ContentVersionId — scoped to **single-writer idempotent
610
- * replay** (a crash-recovery rewrite of the same in-memory value by the
611
- * same writer reproduces the content key the manifest already
612
- * referenced). `body` is non-canonical (`JSON.stringify`, insertion
613
- * key order — see `encodeJsonBytes`), so this is NOT cross-writer
614
- * content dedup: different key order ⇒ different key for an equal value.
615
- *
616
- * Async because {@link crypto.subtle.digest} returns an `ArrayBuffer`
617
- * via Promise. Workers and browsers both expose `crypto.subtle`
618
- * synchronously enough that the await fits naturally in the write
619
- * pipeline.
620
- *
621
- * @see docs/spec/log-entry-shape.md §"Content body layout"
622
- */
623
- const versionFromContent = async (body) => {
624
- const view = new Uint8Array(body.byteLength);
625
- view.set(body);
626
- const digest = await crypto.subtle.digest("SHA-256", view);
627
- const bytes = new Uint8Array(digest);
628
- let hex = "";
629
- for (let i = 0; i < bytes.length; i++) hex += bytes[i].toString(16).padStart(2, "0");
630
- return hex.slice(0, VERSION_HEX_LENGTH);
631
- };
632
- /**
633
- * The single constructor for a log-object key:
634
- * `<manifestPrefix>/log/<seq>.json`. `manifestPrefix` is the
635
- * `<…>/manifests/<collection>` segment — this helper adds `/log/`.
636
- * Route all callers here so the key shape lives in one place.
637
- */
638
- const logObjectKey = (manifestPrefix, seq) => `${manifestPrefix}/log/${seq}.json`;
639
- //#endregion
640
- //#region packages/protocol/src/snapshot-hash.ts
641
- /**
642
- * SHA-256 of `bytes` as a 64-char lowercase hex string. Used to seal
643
- * snapshot filenames so a crashed mid-PUT can't produce a body that
644
- * readers consume as truth: the filename embeds the hash; readers
645
- * recompute on load and reject any mismatch as "file missing."
646
- *
647
- * Distinct from {@link versionFromContent} (truncated to 32 chars for
648
- * content keys) — snapshot bodies are larger and longer-lived;
649
- * 256-bit collision resistance matters here.
650
- */
651
- const snapshotHash = async (bytes) => {
652
- const view = new Uint8Array(bytes.byteLength);
653
- view.set(bytes);
654
- const digest = await crypto.subtle.digest("SHA-256", view);
655
- const out = new Uint8Array(digest);
656
- let hex = "";
657
- for (let i = 0; i < out.length; i++) hex += out[i].toString(16).padStart(2, "0");
658
- return hex;
659
- };
660
- //#endregion
661
- //#region packages/server/src/log-tail.ts
662
- /**
663
- * Tolerant forward-probe for the true committed log tail. The strict
664
- * `[log_seq_start, hint)` range is dense (a hole is corruption, owned
665
- * by `walkLogRange`); this owns only the tolerant `[hint, ∞)` probe,
666
- * where `hint` is a lower bound — the real tail may sit at or past it.
667
- */
668
- /**
669
- * Discover the true committed tail and fold entries in `[hint, tail)`.
670
- * `tail` is the first empty seq (>= hint); `entries` are the
671
- * LogEntries in `[hint, tail)` in seq order. `cap` bounds the walk;
672
- * exhausting it THROWS `Internal` (never silently truncates) — see
673
- * the cap-exhaustion comment below.
674
- */
675
- const probeTailFrom = async (storage, logPrefix, hint, opts) => {
676
- const cap = opts?.cap ?? 1e5;
677
- const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
678
- const entries = [];
679
- for (let i = 0; i < cap; i++) {
680
- const seq = hint + i;
681
- const key = logObjectKey(logPrefix, seq);
682
- const got = await storage.get(key, getOpts);
683
- if (got === null) return {
684
- tail: seq,
685
- entries
686
- };
687
- try {
688
- entries.push(decodeJsonBytes(got.body));
689
- } catch (error) {
690
- throw new BaerlyError("InvalidResponse", `probeTailFrom: malformed log entry at ${key}: ${error.message}`, { cause: error });
691
- }
692
- }
693
- throw new BaerlyError("Internal", `probeTailFrom: forward probe exceeded ${cap} from hint ${hint} on ${logPrefix}`);
694
- };
695
- /**
696
- * First empty log seq at/after `hint` — the slot a new commit creates —
697
- * found by galloping search (`O(log gap)` GETs vs `probeTailFrom`'s
698
- * `O(gap)`). Used by the writer's per-commit tail-find, where `tail_hint`
699
- * can lag the true tail. Position only; no bodies.
700
- *
701
- * PRECONDITION: occupancy MUST be a dense prefix from `hint` (galloping
702
- * assumes monotone occupied→empty; a hole ABOVE `hint` would be skipped
703
- * and the gallop could return a tail past live entries). Contrast
704
- * `probeTailFrom`, which stops at the first 404 and is hole-tolerant.
705
- */
706
- const findLogTail = async (storage, logPrefix, hint, opts) => {
707
- const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
708
- const exists = async (seq) => await storage.get(logObjectKey(logPrefix, seq), getOpts) !== null;
709
- if (!await exists(hint)) return hint;
710
- let lo = hint;
711
- let step = 1;
712
- let hi = hint + step;
713
- while (await exists(hi)) {
714
- lo = hi;
715
- step *= 2;
716
- hi = lo + step;
717
- if (step > 1e5) throw new BaerlyError("Internal", `findLogTail: galloping probe exceeded ${LOG_FORWARD_PROBE_CAP} from hint ${hint} on ${logPrefix}`);
718
- }
719
- while (hi - lo > 1) {
720
- const mid = lo + Math.floor((hi - lo) / 2);
721
- if (await exists(mid)) lo = mid;
722
- else hi = mid;
723
- }
724
- return hi;
725
- };
726
- //#endregion
727
- //#region packages/server/src/names.ts
728
- /** Reject names in the system-reserved leading-"_" namespace. See docs/adr/003-layout-versioning-cordon.md. */
729
- const assertNameNotReserved = (name, context) => {
730
- if (name.startsWith("_")) throw new BaerlyError("InvalidConfig", `${context}: "${name}" is reserved — names beginning with "_" are reserved for system use. See docs/adr/003-layout-versioning-cordon.md.`);
731
- };
732
- //#endregion
733
- //#region packages/server/src/snapshot.ts
734
- /**
735
- * Snapshot file shape + parse. Lifted out of `./compactor.ts` so that
736
- * the kernel barrel (`packages/server/src/index.ts`) — which re-exports
737
- * these four symbols — does not transitively pull `withObservability`
738
- * via `compact()`. `./compactor.ts` imports its own snapshot primitives
739
- * back from this module; the public API surface is unchanged.
740
- *
741
- * @see ../../../docs/spec/sync-protocol.md
742
- */
743
- /**
744
- * Single-level snapshot at L9; the level prefix is forward-compatible
745
- * with future L0..L9 rolling merges without a wire change.
746
- */
747
- const SNAPSHOT_LEVEL = 9;
748
- /**
749
- * `minSeq` and `maxSeq` are padded to {@link SEQ_DIGITS} digits so
750
- * `list(prefix)` returns snapshots in seq order without needing to
751
- * parse the filename. 12 digits caps at ~10^12 entries per collection;
752
- * a real bucket hits S3's per-prefix QPS ceiling long before that.
753
- */
754
- const SEQ_DIGITS = 12;
755
- const MAX_SEQ = 10 ** SEQ_DIGITS - 1;
756
- /**
757
- * Build the on-bucket key for a snapshot file. `sha256` must be the
758
- * lowercase hex digest of the canonical body
759
- * ({@link encodeSnapshotBody}); the filename embeds it so a crashed
760
- * mid-PUT body that doesn't match its own filename hash is rejected
761
- * by readers as "missing."
762
- *
763
- * @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
764
- * out of range or `sha256` is not 64 hex chars.
765
- */
766
- const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
767
- if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
768
- if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
769
- const pad = (n) => n.toString().padStart(SEQ_DIGITS, "0");
770
- return `${collectionPrefix}/snapshot/L${SNAPSHOT_LEVEL}/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
771
- };
772
- /**
773
- * Canonical byte encoding of a {@link SnapshotBody}. Only the
774
- * compactor produces snapshots, and only this function does the
775
- * serialisation — so the byte output is deterministic for a given
776
- * `SnapshotBody` (identical input ⇒ identical bytes ⇒ identical
777
- * filename hash).
778
- */
779
- const encodeSnapshotBody = (s) => encodeJsonBytes(s);
780
- /**
781
- * Load a snapshot file as a `Map<_id, DocumentData>`. Verifies the
782
- * body hash against the filename, checks the schema version, and
783
- * cross-checks `body.collection` against the caller's expectation.
784
- *
785
- * @example
786
- * ```ts
787
- * import { loadSnapshotAsMap } from "@gusto/baerly-storage";
788
- *
789
- * const map =
790
- * srcCurrent.snapshot === null
791
- * ? new Map<string, DocumentData>()
792
- * : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
793
- * ```
794
- *
795
- * @throws BaerlyError code="Internal" — pointer resolves to no body,
796
- * or the body's SHA-256 doesn't match the hash embedded in the
797
- * filename. (Protocol invariant violation; a corrupted snapshot
798
- * the reader cannot continue past.)
799
- * @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
800
- * isn't an object, has an unsupported `schema_version`, or carries a
801
- * `collection` that doesn't match `expectedCollection`.
802
- */
803
- const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
804
- const got = await storage.get(key, signal !== void 0 ? { signal } : void 0);
805
- if (got === null) throw new BaerlyError("Internal", `compact: snapshot pointer ${key} resolves to no body; protocol violation`);
806
- const filename = key.slice(key.lastIndexOf("/") + 1).replace(/\.json$/, "");
807
- const lastDash = filename.lastIndexOf("-");
808
- const expectedHash = filename.slice(lastDash + 1);
809
- const actualHash = await snapshotHash(got.body);
810
- if (actualHash !== expectedHash) throw new BaerlyError("Internal", `compact: snapshot ${key} body hash mismatch (got ${actualHash}); protocol violation`);
811
- let parsed;
812
- try {
813
- parsed = decodeJsonBytes(got.body);
814
- } catch (error) {
815
- throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
816
- }
817
- if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not an object`);
818
- const body = parsed;
819
- if (body.schema_version !== 1) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} has unsupported schema_version ${String(body.schema_version)}`);
820
- if (body.collection !== expectedCollection) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} carries collection ${body.collection}, expected ${expectedCollection}`);
821
- if (!Array.isArray(body.docs)) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body.docs is not an array`);
822
- const out = /* @__PURE__ */ new Map();
823
- for (const row of body.docs) out.set(row._id, row.body);
824
- return out;
825
- };
826
- //#endregion
827
- //#region packages/server/src/indexes.ts
828
- /**
829
- * Secondary indexes — declarative collection-config-driven
830
- * index entries that the writer emits inside the same CAS fence as
831
- * the log entry and content body.
832
- *
833
- * The {@link IndexDefinition} TYPE lives in `@baerly/protocol`
834
- * (cross-platform). This module owns the runtime helpers that
835
- * operate on that shape:
836
- *
837
- * - {@link validateIndexDefinition} — synchronous schema check
838
- * for path-segment safety; thrown at `Writer` construction.
839
- * - {@link encodeIndexValue} — value-order-preserving base-32 of a
840
- * type-tagged byte stream (null/bool tag-only, numbers as sortable
841
- * IEEE 754, strings as raw UTF-8, objects as `JSON.stringify`
842
- * bytes for equality only).
843
- * - {@link indexKeyFor} / {@link indexKeyPrefix} — the wire-key
844
- * shape consumed by the writer's index PUTs and the rebuild
845
- * command's idempotent reconciliation.
846
- * - {@link projectIndexValues} / {@link allIndexKeysFor} — pure
847
- * projections from a doc body to the set of index keys it
848
- * produces under a collection's declared indexes.
849
- *
850
- * Wire shape (single-field):
851
- *
852
- * `<logPrefix>/index/<indexName>/<value-b32>/<docId>.json`
853
- *
854
- * Wire shape (composite):
855
- *
856
- * `<logPrefix>/index/<indexName>/<a-b32>/<b-b32>/<docId>.json`
857
- *
858
- * Body is zero bytes. Each entry is a fact ("doc `<docId>` has
859
- * `<field> = <value>`"), not data. Readers list the prefix, extract
860
- * the doc id from each key, then issue a content GET on each.
861
- *
862
- * Encoding is RFC-4648 base-32 (lowercase alphabet `[0-9a-v]`,
863
- * matching `packages/protocol/src/log.ts:str2uintDesc` style) on a
864
- * type-tagged byte stream (see {@link encodeIndexValue}) so that:
865
- *
866
- * - String "5" and number 5 produce distinct encodings
867
- * (type-tag prefix keeps the slots disjoint).
868
- * - Equal-by-value inputs produce byte-equal segments.
869
- * - Lex order under storage `list(prefix)` matches value order:
870
- * `null < false < true < number < string < object`, with
871
- * numeric values ordered by IEEE 754 magnitude (range walks
872
- * and `$in` walks over numeric fields are first-class — no
873
- * planner fall-back to full-scan).
874
- *
875
- * Composite read routing is left-anchored. The planner may consume an
876
- * equality prefix plus a range / `$in` tail slot; predicates that do
877
- * not match a routable prefix fall back to the snapshot+log fold.
878
- *
879
- * @see docs/spec/sync-protocol.md — index emission is hybrid around
880
- * the commit: additive (new) keys are PUT before the committing
881
- * `log/<seq>` create, and stale-key DELETEs land after it.
882
- */
883
- /** Path-safe segment name. */
884
- const INDEX_NAME_RE = /^[a-z][a-z0-9_]*$/;
885
- /**
886
- * Lowercase base-32 alphabet, RFC 4648 variant matching the
887
- * `packages/protocol/src/log.ts:str2uintDesc` style (`[0-9a-v]`).
888
- * Lex-order-preserving under storage `list(prefix)` semantics.
889
- */
890
- const B32_ALPHABET = "0123456789abcdefghijklmnopqrstuv";
891
- /**
892
- * Reject malformed {@link IndexDefinition} shapes synchronously, at
893
- * writer construction. See docs/adr/003-layout-versioning-cordon.md.
894
- *
895
- * @throws BaerlyError code="InvalidConfig" — reserved leading-`_` `name`.
896
- * @throws BaerlyError code="SchemaError" — `name` fails the regex, or `on` is empty.
897
- */
898
- const validateIndexDefinition = (def) => {
899
- assertNameNotReserved(def.name, "index.name");
900
- if (!INDEX_NAME_RE.test(def.name)) throw new BaerlyError("SchemaError", `index.name must match /^[a-z][a-z0-9_]*$/; got ${JSON.stringify(def.name)}`);
901
- if (typeof def.on === "string") {
902
- if (def.on.length === 0) throw new BaerlyError("SchemaError", `index ${def.name}.on must be non-empty`);
903
- } else if (def.on.length === 0) throw new BaerlyError("SchemaError", `index ${def.name}.on must be non-empty array`);
904
- if (def.predicate !== void 0) try {
905
- validateWire(def.predicate);
906
- } catch (error) {
907
- if (error instanceof BaerlyError) throw new BaerlyError("SchemaError", `index ${def.name}.predicate is invalid: ${error.message}`, error);
908
- throw error;
909
- }
910
- };
911
- /**
912
- * Type-tag bytes for the value-order-preserving wire format. The
913
- * tag is the LEADING byte of the encoded payload; the values are
914
- * picked so the type order under byte-wise lex is
915
- * `null < false < true < number < string < object`.
916
- *
917
- * | Type | Byte-prefix | Payload |
918
- * |---------|-------------|----------------------------------------------------|
919
- * | null | 0x10 | (none) |
920
- * | false | 0x20 | (none) |
921
- * | true | 0x21 | (none) |
922
- * | number | 0x30 | 8-byte sortable big-endian IEEE 754 |
923
- * | string | 0x40 | UTF-8 bytes of the string (no quotes, no escapes) |
924
- * | object | 0x50 | UTF-8 bytes of `JSON.stringify(v)` |
925
- *
926
- * Local to the encoder by design — these are wire shape, not
927
- * cross-package protocol constants (see `CLAUDE.md` § "Magic
928
- * values").
929
- */
930
- const TYPE_NULL = 16;
931
- const TYPE_FALSE = 32;
932
- const TYPE_TRUE = 33;
933
- const TYPE_NUMBER = 48;
934
- const TYPE_STRING = 64;
935
- const TYPE_OBJECT = 80;
936
- /**
937
- * Encode a JS double as an 8-byte big-endian sortable IEEE 754
938
- * payload. Byte-wise lex order on the result matches numeric order
939
- * across all finite doubles plus `±Infinity`:
940
- *
941
- * 1. Get the 8-byte big-endian view of the double.
942
- * 2. If the sign bit is `0` (zero or positive), flip the sign bit
943
- * (XOR `0x80` on byte 0) so positives sort above negatives.
944
- * 3. If the sign bit is `1` (negative), flip every bit in all 8
945
- * bytes so more-negative sorts lower.
946
- *
947
- * `NaN` has no order-preserving slot and is rejected — see the
948
- * `encodeIndexValue` JSDoc for the wire-contract rationale.
949
- * `-0` and `+0` collapse because the `n === 0 ? 0 : n` coercion below
950
- * forces both into the positive-zero branch before `setFloat64`.
951
- */
952
- const encodeNumberPayload = (n) => {
953
- if (Number.isNaN(n)) throw new BaerlyError("SchemaError", `encodeIndexValue: NaN is not indexable (the byte encoding has no order-preserving slot for it)`);
954
- const buf = /* @__PURE__ */ new ArrayBuffer(8);
955
- new DataView(buf).setFloat64(0, n === 0 ? 0 : n, false);
956
- const bytes = new Uint8Array(buf);
957
- if ((bytes[0] & 128) === 0) bytes[0] = bytes[0] ^ 128;
958
- else for (let i = 0; i < 8; i++) bytes[i] = bytes[i] ^ 255;
959
- return bytes;
960
- };
961
- /**
962
- * Compose the typed byte stream for one value. The leading byte is
963
- * the type tag (see {@link TYPE_NULL} et al.); the payload shape is
964
- * type-dependent. Strings are emitted as their raw UTF-8 bytes (no
965
- * JSON quoting / escaping) so byte-wise lex order on encoded strings
966
- * matches byte-wise lex order on the source UTF-8. Objects fall back
967
- * to `JSON.stringify` for equality only.
968
- */
969
- const encodeBytes = (v) => {
970
- if (v === null || v === void 0) return new Uint8Array([TYPE_NULL]);
971
- if (v === false) return new Uint8Array([TYPE_FALSE]);
972
- if (v === true) return new Uint8Array([TYPE_TRUE]);
973
- if (typeof v === "number") {
974
- const payload = encodeNumberPayload(v);
975
- const out = new Uint8Array(1 + payload.length);
976
- out[0] = TYPE_NUMBER;
977
- out.set(payload, 1);
978
- return out;
979
- }
980
- if (typeof v === "string") {
981
- const utf8 = new TextEncoder().encode(v);
982
- const out = new Uint8Array(1 + utf8.length);
983
- out[0] = TYPE_STRING;
984
- out.set(utf8, 1);
985
- return out;
986
- }
987
- const utf8 = encodeJsonBytes(v);
988
- const out = new Uint8Array(1 + utf8.length);
989
- out[0] = TYPE_OBJECT;
990
- out.set(utf8, 1);
991
- return out;
992
- };
993
- /**
994
- * Encode an arbitrary `DocumentValue` value as a path-safe segment.
995
- *
996
- * Type-tagged byte stream, base-32-encoded for path safety. Lex
997
- * order on encoded bytes matches:
998
- *
999
- * - **Type order**: `null < false < true < number < string < object`.
1000
- * - **Within numbers**: IEEE 754 numeric order (sortable
1001
- * big-endian payload, sign-bit-flipped or all-bits-flipped).
1002
- * `-Infinity` sorts lowest, `+Infinity` highest.
1003
- * - **Within strings**: UTF-8 byte order.
1004
- * - **Within objects**: byte-equality only (still
1005
- * `JSON.stringify`-insertion-order dependent).
1006
- *
1007
- * Numbers ARE range-safe under this encoder — `9` sorts below `10`
1008
- * as expected. The query planner's old
1009
- * `numeric-range-on-byte-encoder` guard has been removed; range
1010
- * walks and `$in` walks over numeric fields are routed normally.
1011
- *
1012
- * `NaN` throws `BaerlyError{code:"SchemaError"}` at write time —
1013
- * the byte encoding has no order-preserving slot for it, and a
1014
- * silent collapse to the `null` key (as the prior
1015
- * `JSON.stringify`-based encoder did, since
1016
- * `JSON.stringify(NaN) === "null"`) is a worse failure mode than
1017
- * a loud reject.
1018
- *
1019
- * `-0` and `+0` collapse to the same encoding.
1020
- *
1021
- * Equal-by-value inputs (e.g. two `{a:1, b:2}` objects with the
1022
- * same key insertion order) produce byte-equal segments;
1023
- * structurally-different but visually-similar inputs (`"5"` vs `5`)
1024
- * are kept apart by the type-tag prefix.
1025
- *
1026
- * String ranges remain safe: `"2026-05-13" < "2026-05-14"`
1027
- * byte-wise matches semantic order. ISO 8601 timestamps,
1028
- * alphabetical priorities like `"p1"/"p2"/"p3"`, and any
1029
- * fixed-width string encoding are usable as range-walked indexed
1030
- * fields.
1031
- */
1032
- const encodeIndexValue = (v) => {
1033
- const bytes = encodeBytes(v);
1034
- let out = "";
1035
- let bits = 0;
1036
- let value = 0;
1037
- for (const b of bytes) {
1038
- value = value << 8 | b;
1039
- bits += 8;
1040
- while (bits >= 5) {
1041
- bits -= 5;
1042
- out += B32_ALPHABET[value >> bits & 31];
1043
- }
1044
- }
1045
- if (bits > 0) out += B32_ALPHABET[value << 5 - bits & 31];
1046
- return out;
1047
- };
1048
- /**
1049
- * Storage `list(prefix)` boundary for every entry under one index
1050
- * of a collection: `<logPrefix>/index/<name>/`. Used by the rebuild
1051
- * command's reconciliation walk and by the read-path index hint.
1052
- */
1053
- const indexKeyPrefix = (logPrefix, indexName) => `${logPrefix}/index/${indexName}/`;
1054
- /**
1055
- * Build the per-doc index key for one `(def, values, docId)`.
1056
- *
1057
- * Single-field: `<logPrefix>/index/<name>/<v0-b32>/<docId>.json`.
1058
- * Composite: `<logPrefix>/index/<name>/<v0-b32>/<v1-b32>/.../<docId>.json`.
1059
- *
1060
- * Composite entries are read-path routable when the planner can consume
1061
- * a left-anchored prefix of the indexed tuple.
1062
- */
1063
- const indexKeyFor = (logPrefix, def, values, docId) => {
1064
- const segments = values.map(encodeIndexValue).join("/");
1065
- return `${logPrefix}/index/${def.name}/${segments}/${docId}.json`;
1066
- };
1067
- /**
1068
- * Project the value tuple a doc body produces under one index
1069
- * definition. Returns `undefined` when any indexed field is
1070
- * null / undefined / missing — no index entry for that doc under
1071
- * that index (SQL "NULL values don't enter the index" semantics).
1072
- *
1073
- * @throws BaerlyError code="SchemaError" — `on` contains a dotted
1074
- * path, or the indexed field's value is an array. Both restrictions
1075
- * are current 0.x limitations; multi-value (array) indexes are a follow-up.
1076
- */
1077
- const projectIndexValues = (def, body) => {
1078
- if (body === void 0) return;
1079
- const fields = typeof def.on === "string" ? [def.on] : def.on;
1080
- const values = [];
1081
- for (const field of fields) {
1082
- if (field.includes(".")) throw new BaerlyError("SchemaError", `index ${def.name}: dotted-path "on" fields not yet supported (got ${JSON.stringify(field)})`);
1083
- const v = body[field];
1084
- if (v === null || v === void 0) return;
1085
- if (Array.isArray(v)) throw new BaerlyError("SchemaError", `index ${def.name}: cannot index array field "${field}" — multi-value indexes are not supported yet`);
1086
- values.push(v);
1087
- }
1088
- return values;
1089
- };
1090
- /**
1091
- * Compute the full index-key set a single doc produces under every
1092
- * declared index of its collection. Skips indexes whose projection
1093
- * yields `undefined` (null/missing indexed field). Used by both
1094
- * the writer (to PUT new keys) and the rebuild command (to compute
1095
- * the expected key set across the live doc map).
1096
- *
1097
- * Filter-aware (T4): when `def.predicate !== undefined`, the doc
1098
- * must satisfy `matchesWire(def.predicate, body)` or the def contributes
1099
- * zero keys. The writer's diff `oldKeys` vs `newKeys` then covers
1100
- * all four U-quadrants automatically — see the JSDoc on
1101
- * `writer.ts` above the index-emission block. `body ===
1102
- * undefined` short-circuits ahead of the filter check (no keys for
1103
- * an unknown pre-image, regardless of filter), preserving the prior
1104
- * D-quadrant behaviour.
1105
- */
1106
- const allIndexKeysFor = (logPrefix, defs, body, docId) => {
1107
- const keys = [];
1108
- for (const def of defs) {
1109
- if (def.predicate !== void 0 && body !== void 0) {
1110
- if (!matchesWire(def.predicate, body)) continue;
1111
- }
1112
- const values = projectIndexValues(def, body);
1113
- if (values !== void 0) keys.push(indexKeyFor(logPrefix, def, values, docId));
1114
- }
1115
- return keys;
1116
- };
1117
- //#endregion
1118
- //#region packages/server/src/log-walk.ts
1119
- /**
1120
- * Shared log-walk primitives consumed by the read path (`query.ts`),
1121
- * the compactor (`compactor.ts`), and the optional integrity walk in
1122
- * `Writer.#singleAttemptCommit` (`writer.ts`).
1123
- *
1124
- * Two operations: read one log entry, or walk a contiguous range of
1125
- * log entries with bounded parallelism. Each kernel call site
1126
- * previously inlined the same range-build + unbounded `Promise.all`
1127
- * pattern; on a free-tier collection where the compactor lags the
1128
- * tail to 50-100 entries, that fans out 50-100 concurrent GETs per
1129
- * call — and a contended writer multiplies that by the CAS retry
1130
- * budget. `walkLogRange` caps concurrency at
1131
- * {@link MAX_PARALLEL_LOG_READS} so a single call costs at most that
1132
- * many simultaneous GETs against the storage backend.
1133
- *
1134
- * Order invariant: results are returned in seq order regardless of
1135
- * resolution order. The read-side fold is order-sensitive (`U` then
1136
- * `D` on the same doc must apply in that order); callers may iterate
1137
- * the returned array as-is.
1138
- *
1139
- * Signal invariant: when the caller's `AbortSignal` fires, no further
1140
- * chunks dispatch. In-flight reads see the signal via the underlying
1141
- * `Storage.get` call.
1142
- */
1143
- /**
1144
- * Fetch a single log entry. Centralises the "missing entry inside the
1145
- * visible log range is a protocol-invariant violation" rule used by
1146
- * every caller.
1147
- *
1148
- * @throws BaerlyError code="Internal" when the key resolves to no
1149
- * body — a hole inside `[log_seq_start, tail_hint)` is a corruption
1150
- * the kernel can't continue past.
1151
- * @throws BaerlyError code="InvalidResponse" when the body isn't
1152
- * valid JSON.
1153
- */
1154
- const readLogEntry = async (storage, key, opts) => {
1155
- return (await readLogEntryWithBytes(storage, key, opts)).entry;
1156
- };
1157
- /**
1158
- * Like {@link readLogEntry}, but also surfaces `bytes` — the
1159
- * byteLength of the STORED log-object body exactly as it was GET back
1160
- * from storage (`got.body.byteLength`).
1161
- *
1162
- * The compactor uses this to sum the byte cost of the folded slice.
1163
- * The stored body byteLength IS what the writer PUT for each entry,
1164
- * so summing `got.body.byteLength` over the folded range yields the
1165
- * exact cost — no re-encode, no round-trip assumption (a re-encode
1166
- * of the parsed `LogEntry` could drift if key-order/whitespace ever
1167
- * diverged from what was PUT).
1168
- *
1169
- * Same error semantics as {@link readLogEntry}.
1170
- */
1171
- const readLogEntryWithBytes = async (storage, key, opts) => {
1172
- const got = await storage.get(key, opts?.signal !== void 0 ? { signal: opts.signal } : void 0);
1173
- if (got === null) throw new BaerlyError("Internal", `log-walk: missing log entry at ${key}; protocol invariant violation`);
1174
- try {
1175
- return {
1176
- entry: decodeJsonBytes(got.body),
1177
- bytes: got.body.byteLength
1178
- };
1179
- } catch (error) {
1180
- throw new BaerlyError("InvalidResponse", `log-walk: malformed log entry at ${key}`, error);
1181
- }
1182
- };
1183
- /**
1184
- * Walk `[fromSeq, toSeqExclusive)` of `${logPrefix}/log/<seq>.json`
1185
- * in chunks of {@link MAX_PARALLEL_LOG_READS} concurrent GETs.
1186
- * Returns the materialised entries in seq order.
1187
- *
1188
- * Empty range (`fromSeq >= toSeqExclusive`) returns `[]` and issues
1189
- * zero GETs.
1190
- *
1191
- * Same error semantics as {@link readLogEntry}: a missing entry
1192
- * throws `Internal`, a malformed body throws `InvalidResponse`.
1193
- */
1194
- const walkLogRange = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) => {
1195
- return (await walkLogRangeWithBytes(storage, logPrefix, fromSeq, toSeqExclusive, opts)).map((r) => r.entry);
1196
- };
1197
- /**
1198
- * Like {@link walkLogRange}, but each element also carries `bytes` —
1199
- * the STORED log-object body byteLength (see {@link readLogEntryWithBytes}).
1200
- * Same seq-order, empty-range, error, and bounded-parallelism
1201
- * ({@link MAX_PARALLEL_LOG_READS}) guarantees as {@link walkLogRange};
1202
- * this is the bytes-aware core `walkLogRange` is implemented on top of.
1203
- *
1204
- * The compactor consumes this so it gets BOTH the parsed entries (for
1205
- * the fold) AND the exact byte sum (for `mean_entry_bytes` stamping) from a
1206
- * single walk.
1207
- */
1208
- const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) => {
1209
- if (fromSeq >= toSeqExclusive) return [];
1210
- const total = toSeqExclusive - fromSeq;
1211
- const out = Array.from({ length: total });
1212
- for (let chunkStart = 0; chunkStart < total; chunkStart += 16) {
1213
- opts?.signal?.throwIfAborted();
1214
- const chunkEnd = Math.min(chunkStart + 16, total);
1215
- const promises = [];
1216
- for (let i = chunkStart; i < chunkEnd; i++) {
1217
- const seq = fromSeq + i;
1218
- promises.push(readLogEntryWithBytes(storage, logObjectKey(logPrefix, seq), opts));
1219
- }
1220
- const results = await Promise.all(promises);
1221
- for (let i = 0; i < results.length; i++) out[chunkStart + i] = results[i];
1222
- }
1223
- return out;
1224
- };
1225
- /**
1226
- * Fold a sequence of `LogEntry` records onto a doc-id-keyed map,
1227
- * applying the protocol's per-doc-replace semantics:
1228
- *
1229
- * - `I` / `U`: when `entry.after !== undefined`, set
1230
- * `map.set(entry.doc_id, entry.after)`. Entries with `after ===
1231
- * undefined` (the partial-merge writer's future patch-only shape)
1232
- * are ignored — today's writer always emits `after`, so this is a
1233
- * forward-compat guard, not a live branch.
1234
- * - `D`: tombstone — `map.delete(entry.doc_id)`.
1235
- *
1236
- * Entries whose `collection` does not match `opts.collection`, or
1237
- * whose `doc_id` is `undefined`, are skipped. When
1238
- * `opts.docIdFilter` is provided, only entries whose `doc_id` is
1239
- * present in the filter set are folded — used by the index-walk
1240
- * read path to scope the fold to the docs the planner already
1241
- * resolved.
1242
- *
1243
- * This is the canonical fold used by the read path
1244
- * (`query.ts`) and the index reconciler (`rebuild-index.ts`).
1245
- * Centralising it keeps the
1246
- * "straight `set` on I/U; tombstone on D" protocol invariant in one place.
1247
- *
1248
- * @see ../../../docs/spec/sync-protocol.md
1249
- */
1250
- const foldLogEntriesOnto = (map, entries, opts) => {
1251
- const { collection, docIdFilter } = opts;
1252
- for (const entry of entries) {
1253
- if (entry.collection !== collection) continue;
1254
- if (entry.doc_id === void 0) continue;
1255
- if (docIdFilter !== void 0 && !docIdFilter.has(entry.doc_id)) continue;
1256
- switch (entry.op) {
1257
- case "I":
1258
- case "U":
1259
- if (entry.after === void 0) continue;
1260
- map.set(entry.doc_id, entry.after);
1261
- break;
1262
- case "D":
1263
- map.delete(entry.doc_id);
1264
- break;
1265
- }
1266
- }
1267
- };
1268
- //#endregion
1269
- //#region packages/server/src/compactor.ts
1270
- /**
1271
- * Compactor: fold a prefix of the live log into a content-
1272
- * hashed snapshot file and CAS-advance `current.json` to swap the
1273
- * pointer.
1274
- *
1275
- * The hard correctness requirement is **atomic snapshot generation**:
1276
- * a compactor that crashes mid-write must not leave a corrupted body
1277
- * that readers consume as truth. The design: every snapshot's filename
1278
- * embeds the SHA-256 of its body. A crashed mid-PUT leaves a body that
1279
- * doesn't match its own filename hash, and readers reject it as
1280
- * "missing." The atomic moment is the CAS-swap of
1281
- * `current.json.snapshot`, which is a single conditional PUT.
1282
- *
1283
- * Order of operations: PUT snapshot body → CAS-advance `current.json`.
1284
- * A crash before the swap leaves an orphan snapshot file; the swap
1285
- * succeeds iff our captured ETag still matches.
1286
- *
1287
- * Why level-prefixed chunked snapshots over the alternatives: a
1288
- * single monolithic snapshot (every compaction rewrites one well-
1289
- * known key) makes partial writes catastrophic and serializes every
1290
- * writer on one key. WAL-checkpointing in the rqlite / Raft tradition
1291
- * requires a consensus layer baerly-storage doesn't have. The Litestream-
1292
- * style multi-level scheme — small fixed number of levels, chunked
1293
- * files keyed by sequence range — keeps random GETs cheap on
1294
- * S3-compatible storage and avoids LIST. Current ship is single-
1295
- * level at L9 (one snapshot replaces the prior; no multi-level merge
1296
- * yet); the key format is forward-compatible with future L0..L9
1297
- * rolling merges without a wire change.
1298
- *
1299
- * @see ../../../docs/spec/sync-protocol.md
1300
- */
1301
- const ctxMetrics$2 = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
1302
- const DEFAULT_MAX_PER_RUN = Number.MAX_SAFE_INTEGER;
1303
- const DEFAULT_MIN_TO_COMPACT = 100;
1304
- const APPLICATION_JSON$1 = "application/json";
1305
- /**
1306
- * Fold a prefix of the live log into a new snapshot and CAS-advance
1307
- * `current.json`. Idempotent: a second run with no new writes is a
1308
- * no-op (below-min-threshold).
1309
- *
1310
- * Single-attempt: on CAS conflict we return
1311
- * `{written: false, skippedReason: "cas-lost", newSnapshotKey}` — the
1312
- * caller (the cron handler invoking `runScheduledMaintenance` from
1313
- * `./maintenance.ts`) decides whether to schedule another run. The
1314
- * orphan snapshot file just written will be swept by `runGc()`.
1315
- *
1316
- * @throws BaerlyError code="InvalidResponse" — `current.json` is
1317
- * present but malformed, or a snapshot body fails its schema /
1318
- * collection cross-check.
1319
- * @throws BaerlyError code="Internal" — protocol-invariant violation:
1320
- * a log entry missing inside `[log_seq_start, foldEnd)`, or the
1321
- * prior-snapshot pointer resolves to no body / a body whose hash
1322
- * doesn't match its filename.
1323
- *
1324
- * @example
1325
- * ```ts
1326
- * import { compact } from "@gusto/baerly-storage";
1327
- *
1328
- * const res = await compact(
1329
- * { storage, currentJsonKey: "app/x/tenant/t/manifests/tickets/current.json" },
1330
- * { minEntriesToCompact: 100, maxEntriesPerRun: 40 },
1331
- * );
1332
- * if (res.written) console.log("snapshot landed at", res.newSnapshotKey);
1333
- * ```
1334
- */
1335
- const compact = async (args, options = {}) => {
1336
- const { storage, currentJsonKey } = args;
1337
- const internal = options;
1338
- const maxPerRun = internal.maxEntriesPerRun ?? DEFAULT_MAX_PER_RUN;
1339
- const minToCompact = options.minEntriesToCompact ?? DEFAULT_MIN_TO_COMPACT;
1340
- const ceilingBytes = internal.ceilingBytes ?? Number.POSITIVE_INFINITY;
1341
- const ceilingEntries = internal.ceilingEntries ?? Number.POSITIVE_INFINITY;
1342
- const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
1343
- const collectionName = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
1344
- const read = await readCurrentJson(storage, currentJsonKey, options.signal !== void 0 ? { signal: options.signal } : void 0);
1345
- if (read === null) return {
1346
- written: false,
1347
- skippedReason: "current-json-missing",
1348
- previousSnapshotKey: null,
1349
- logSeqStartBefore: 0,
1350
- logSeqStartAfter: 0,
1351
- entriesFolded: 0
1352
- };
1353
- const current = read.json;
1354
- const baseEtag = read.etag;
1355
- const logSeqStartBefore = logSeqStartOf(current);
1356
- const discoveredTail = (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStartBefore, current.tail_hint, internal.knownTail ?? 0), { ...options.signal !== void 0 && { signal: options.signal } })).tail;
1357
- const nextSeq = discoveredTail;
1358
- if (nextSeq - logSeqStartBefore < minToCompact) return {
1359
- written: false,
1360
- skippedReason: "below-min-threshold",
1361
- previousSnapshotKey: current.snapshot,
1362
- logSeqStartBefore,
1363
- logSeqStartAfter: logSeqStartBefore,
1364
- entriesFolded: 0
1365
- };
1366
- const foldEnd = Math.min(nextSeq, logSeqStartBefore + maxPerRun);
1367
- const base = current.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, current.snapshot, collectionName, options.signal);
1368
- const fetched = await walkLogRangeWithBytes(storage, collectionPrefix, logSeqStartBefore, foldEnd, { signal: options.signal });
1369
- const foldedSliceBytes = fetched.reduce((sum, f) => sum + f.bytes, 0);
1370
- for (const { entry } of fetched) {
1371
- if (entry.collection !== collectionName) continue;
1372
- if (entry.doc_id === void 0) continue;
1373
- switch (entry.op) {
1374
- case "I":
1375
- case "U":
1376
- if (entry.after !== void 0) base.set(entry.doc_id, entry.after);
1377
- break;
1378
- case "D":
1379
- base.delete(entry.doc_id);
1380
- break;
1381
- }
1382
- }
1383
- const bodyBytes = encodeSnapshotBody({
1384
- schema_version: 1,
1385
- min_seq: 0,
1386
- max_seq: foldEnd,
1387
- collection: collectionName,
1388
- docs: Array.from(base.entries()).toSorted(([a], [b]) => {
1389
- if (a < b) return -1;
1390
- if (a > b) return 1;
1391
- return 0;
1392
- }).map(([id, body]) => ({
1393
- _id: id,
1394
- body
1395
- }))
1396
- });
1397
- if (bodyBytes.byteLength > ceilingBytes || base.size > ceilingEntries) {
1398
- ctxMetrics$2().counter("db.compaction.deferred_total", 1, {
1399
- collection: collectionName,
1400
- dimension: bodyBytes.byteLength > ceilingBytes ? "bytes" : "rows"
1401
- });
1402
- return {
1403
- written: false,
1404
- skippedReason: "deferred",
1405
- deferred: true,
1406
- previousSnapshotKey: current.snapshot,
1407
- logSeqStartBefore,
1408
- logSeqStartAfter: logSeqStartBefore,
1409
- entriesFolded: 0
1410
- };
1411
- }
1412
- const newKey = snapshotKey(collectionPrefix, 0, foldEnd, await snapshotHash(bodyBytes));
1413
- const putOpts = {
1414
- contentType: APPLICATION_JSON$1,
1415
- ...options.signal !== void 0 && { signal: options.signal }
1416
- };
1417
- await storage.put(newKey, bodyBytes, putOpts);
1418
- const entriesFolded = foldEnd - logSeqStartBefore;
1419
- const nextBody = encodeJsonBytes({
1420
- ...current,
1421
- snapshot: newKey,
1422
- log_seq_start: foldEnd,
1423
- tail_hint: Math.max(current.tail_hint, discoveredTail),
1424
- snapshot_bytes: bodyBytes.byteLength,
1425
- snapshot_rows: base.size,
1426
- mean_entry_bytes: entriesFolded > 0 ? Math.round(foldedSliceBytes / entriesFolded) : current.mean_entry_bytes ?? 0
1427
- });
1428
- const casOpts = {
1429
- ifMatch: baseEtag,
1430
- contentType: APPLICATION_JSON$1,
1431
- ...options.signal !== void 0 && { signal: options.signal }
1432
- };
1433
- try {
1434
- await storage.put(currentJsonKey, nextBody, casOpts);
1435
- } catch (error) {
1436
- if (isCasConflict(error)) {
1437
- ctxMetrics$2().counter("db.compaction.cas_lost_total", 1, { collection: collectionName });
1438
- return {
1439
- written: false,
1440
- skippedReason: "cas-lost",
1441
- previousSnapshotKey: current.snapshot,
1442
- newSnapshotKey: newKey,
1443
- logSeqStartBefore,
1444
- logSeqStartAfter: logSeqStartBefore,
1445
- entriesFolded: foldEnd - logSeqStartBefore
1446
- };
1447
- }
1448
- throw error;
1449
- }
1450
- const metrics = ctxMetrics$2();
1451
- metrics.histogram("db.compact.entries_folded", entriesFolded, { collection: collectionName });
1452
- metrics.gauge("db.manifest.lag_window_depth", discoveredTail - foldEnd, { collection: collectionName });
1453
- return {
1454
- written: true,
1455
- previousSnapshotKey: current.snapshot,
1456
- newSnapshotKey: newKey,
1457
- logSeqStartBefore,
1458
- logSeqStartAfter: foldEnd,
1459
- entriesFolded
1460
- };
1461
- };
1462
- /**
1463
- * `true` when an `If-Match` CAS guard lost. Every in-tree
1464
- * {@link Storage} impl surfaces a lost CAS as
1465
- * `BaerlyError{code:"Conflict"}`.
1466
- */
1467
- const isCasConflict = (err) => err instanceof BaerlyError && err.code === "Conflict";
1468
- //#endregion
1469
- //#region packages/server/src/gc.ts
1470
- /**
1471
- * GC mark + sweep.
1472
- *
1473
- * `runGc` is a single-pass garbage collector that:
1474
- * 1. Reads `current.json` (and bootstraps `gc/pending.json` on first
1475
- * run).
1476
- * 2. Marks new orphan candidates by LISTing the three artifact
1477
- * prefixes — log, snapshot, content — and classifying each key.
1478
- * 3. Sweeps any already-pending candidate whose `due_at` has passed.
1479
- * 4. CAS-writes the updated `gc/pending.json`.
1480
- *
1481
- * Two-phase by design: every candidate sits in `gc/pending.json` for a
1482
- * grace period (default 7 days, see {@link GC_GRACE_PERIOD_MILLIS})
1483
- * before it is deleted. The grace bounds the worst plausible
1484
- * writer-retry window — a paused-process writer that resumes hours
1485
- * later still finds its idempotency anchor on the bucket.
1486
- *
1487
- * Idempotent: same input bucket state ⇒ same `pending.json` output.
1488
- * Unbounded by default — the run marks and sweeps the entire eligible
1489
- * set in one pass. Callers on the Cloudflare 50-subrequest free-tier
1490
- * budget opt INTO caps via the `CLOUDFLARE_FREE_TIER` profile's
1491
- * `gc.maxMarksPerRun` / `maxSweepsPerRun` knobs (`InternalRunGcOptions`,
1492
- * not on the public `RunGcOptions`).
1493
- *
1494
- * Three categories of orphan:
1495
- * - `stale-log`: `<collectionPrefix>/log/<seq>.json` with
1496
- * `seq < log_seq_start`. After `compact()` folds these into a
1497
- * snapshot, they're unreferenced.
1498
- * - `orphan-snapshot`: a `<collectionPrefix>/snapshot/L<n>/...` key not
1499
- * equal to `current.snapshot`. Each compactor run replaces the
1500
- * pointer; the prior file becomes unreferenced.
1501
- * - `orphan-content`: `<collectionPrefix>/content/<sha>.json` whose
1502
- * 32-hex truncated-SHA-256 hash is not in the live content-hash
1503
- * set (computed by hashing every live `entry.after` post-image —
1504
- * the same hash the writer's step 4 produces). Surfaces writer
1505
- * crashes between the content PUT and the log-entry PUT. Because
1506
- * content keys are hash-named (random lex order) and live content
1507
- * is never deleted, a bounded pass cannot rely on deletion to
1508
- * advance its LIST window the way `stale-log` does — so the
1509
- * orphan-content LIST carries a persisted rotation cursor
1510
- * (`content_scan_cursor` in `gc/pending.json`): each bounded pass
1511
- * resumes `startAfter` the prior pass's last examined key and wraps
1512
- * at end-of-keyspace, so the whole `content/` keyspace is swept
1513
- * over a rotation within the per-pass `maxMarksPerRun` budget.
1514
- *
1515
- * CAS-lost on `gc/pending.json` is non-fatal: the DELETEs already
1516
- * issued are durable, so we return a successful result and the next
1517
- * pass will pick up any work this pass lost.
1518
- */
1519
- const ctxMetrics$1 = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
1520
- const DEFAULT_MAX_MARKS = Number.MAX_SAFE_INTEGER;
1521
- const DEFAULT_MAX_SWEEPS = Number.MAX_SAFE_INTEGER;
1522
- /**
1523
- * Single GC pass — mark new orphans, sweep due-elapsed candidates,
1524
- * persist via CAS-update on `gc/pending.json`.
1525
- *
1526
- * Returns immediately if `current.json` is missing — there's nothing
1527
- * to do until a writer has bootstrapped the collection.
1528
- *
1529
- * @example
1530
- * ```ts
1531
- * import { runGc } from "@gusto/baerly-storage";
1532
- *
1533
- * const r = await runGc({ storage, currentJsonKey });
1534
- * console.log(`marked ${r.marked.stale_log} stale logs, swept ${r.swept}`);
1535
- * ```
1536
- */
1537
- const runGc = async (args, options = {}) => {
1538
- const { storage, currentJsonKey } = args;
1539
- const internal = options;
1540
- const grace = internal.graceMillis ?? 6048e5;
1541
- const maxMarks = internal.maxMarksPerRun ?? DEFAULT_MAX_MARKS;
1542
- const maxSweeps = internal.maxSweepsPerRun ?? DEFAULT_MAX_SWEEPS;
1543
- const now = internal.now ?? (() => /* @__PURE__ */ new Date());
1544
- const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
1545
- const collectionName = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
1546
- const gcPendingKey = `${collectionPrefix}/gc/pending.json`;
1547
- const signal = options.signal;
1548
- const signalOpts = signal !== void 0 ? { signal } : void 0;
1549
- const cur = await readCurrentJson(storage, currentJsonKey, signalOpts);
1550
- if (cur === null) return {
1551
- marked: {
1552
- stale_log: 0,
1553
- orphan_snapshot: 0,
1554
- orphan_content: 0
1555
- },
1556
- swept: 0,
1557
- pendingDepth: 0
1558
- };
1559
- const current = cur.json;
1560
- const logSeqStart = logSeqStartOf(current);
1561
- let pending = await readGcPending(storage, gcPendingKey, signalOpts);
1562
- if (pending === null) {
1563
- const initial = {
1564
- schema_version: 1,
1565
- candidates: [],
1566
- last_swept_at: ""
1567
- };
1568
- try {
1569
- pending = await createGcPending(storage, gcPendingKey, initial, signalOpts);
1570
- } catch (error) {
1571
- if (error instanceof BaerlyError && error.code === "Conflict") {
1572
- pending = await readGcPending(storage, gcPendingKey, signalOpts);
1573
- if (pending === null) throw error;
1574
- } else throw error;
1575
- }
1576
- }
1577
- const known = new Set(pending.json.candidates.map((c) => c.key));
1578
- const newCandidates = [];
1579
- let markedStaleLog = 0;
1580
- if (logSeqStart > 0) for await (const entry of listBounded(storage, `${collectionPrefix}/log/`, maxMarks, signal)) {
1581
- if (markedStaleLog >= maxMarks) break;
1582
- const seq = parseSeqFromLogKey(entry.key);
1583
- if (seq === null || seq >= logSeqStart) continue;
1584
- if (known.has(entry.key)) continue;
1585
- newCandidates.push({
1586
- key: entry.key,
1587
- due_at: computeDueAt(entry, now, grace),
1588
- reason: "stale-log"
1589
- });
1590
- markedStaleLog++;
1591
- }
1592
- let markedOrphanSnapshot = 0;
1593
- for await (const entry of listBounded(storage, `${collectionPrefix}/snapshot/`, maxMarks, signal)) {
1594
- if (markedOrphanSnapshot >= maxMarks) break;
1595
- if (entry.key === current.snapshot) continue;
1596
- if (known.has(entry.key)) continue;
1597
- newCandidates.push({
1598
- key: entry.key,
1599
- due_at: computeDueAt(entry, now, grace),
1600
- reason: "orphan-snapshot"
1601
- });
1602
- markedOrphanSnapshot++;
1603
- }
1604
- const liveHashes = await collectLiveContentHashes(storage, collectionPrefix, collectionName, current, logSeqStart, signal);
1605
- const cursor = pending.json.content_scan_cursor;
1606
- const contentPrefix = `${collectionPrefix}/content/`;
1607
- let markedOrphanContent = 0;
1608
- let examinedThisPass = 0;
1609
- let lastExaminedKey;
1610
- const listOpts = {
1611
- maxKeys: maxMarks,
1612
- ...cursor !== void 0 && { startAfter: cursor },
1613
- ...signal !== void 0 && { signal }
1614
- };
1615
- for await (const entry of storage.list(contentPrefix, listOpts)) {
1616
- examinedThisPass++;
1617
- lastExaminedKey = entry.key;
1618
- const hash = parseHashFromContentKey(entry.key);
1619
- if (hash === null || liveHashes.has(hash)) continue;
1620
- if (known.has(entry.key)) continue;
1621
- newCandidates.push({
1622
- key: entry.key,
1623
- due_at: computeDueAt(entry, now, grace),
1624
- reason: "orphan-content"
1625
- });
1626
- markedOrphanContent++;
1627
- }
1628
- const nextContentCursor = examinedThisPass < maxMarks ? void 0 : lastExaminedKey;
1629
- const nowMs = now().getTime();
1630
- const sweepCandidates = [...pending.json.candidates, ...newCandidates];
1631
- const toSweep = [];
1632
- const remaining = [];
1633
- for (const cand of sweepCandidates) if (toSweep.length < maxSweeps && Date.parse(cand.due_at) <= nowMs) toSweep.push(cand);
1634
- else remaining.push(cand);
1635
- await Promise.all(toSweep.map((c) => storage.delete(c.key, signalOpts)));
1636
- const sweptKeys = new Set(toSweep.map((c) => c.key));
1637
- const lastSweptAt = toSweep.length > 0 ? now().toISOString() : "";
1638
- const markedSummary = {
1639
- stale_log: markedStaleLog,
1640
- orphan_snapshot: markedOrphanSnapshot,
1641
- orphan_content: markedOrphanContent
1642
- };
1643
- let pendingDepth;
1644
- try {
1645
- pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, (latest) => mergeGcPending(latest, {
1646
- sweptKeys,
1647
- newCandidates,
1648
- lastSweptAt,
1649
- nextContentCursor,
1650
- maxCandidates: GC_MAX_PENDING_CANDIDATES
1651
- }), signalOpts)).json.candidates.length;
1652
- } catch (error) {
1653
- if (error instanceof BaerlyError && error.code === "Conflict") pendingDepth = remaining.length;
1654
- else throw error;
1655
- }
1656
- const labels = { collection: collectionName };
1657
- const metrics = ctxMetrics$1();
1658
- metrics.gauge("db.orphan.candidate_count", pendingDepth, labels);
1659
- metrics.gauge("db.gc.entries_swept_per_second", toSweep.length, labels);
1660
- if (toSweep.length > 0) {
1661
- const byReason = /* @__PURE__ */ new Map();
1662
- for (const c of toSweep) byReason.set(c.reason, (byReason.get(c.reason) ?? 0) + 1);
1663
- for (const [reason, count] of byReason) metrics.counter("db.gc.swept_total", count, {
1664
- collection: collectionName,
1665
- reason
1666
- });
1667
- }
1668
- return {
1669
- marked: markedSummary,
1670
- swept: toSweep.length,
1671
- pendingDepth
1672
- };
1673
- };
1674
- /**
1675
- * `Storage.list` with a hard ceiling — bounds the I/O cost of a
1676
- * single pass when a runaway prefix has accumulated many keys.
1677
- */
1678
- const listBounded = async function* (storage, prefix, cap, signal) {
1679
- let yielded = 0;
1680
- const opts = signal !== void 0 ? { signal } : void 0;
1681
- for await (const entry of storage.list(prefix, opts)) {
1682
- if (yielded >= cap) return;
1683
- yield entry;
1684
- yielded++;
1685
- }
1686
- };
1687
- /**
1688
- * Parse `<...>/log/<seq>.json` and return `seq`. Returns `null` on
1689
- * any shape that doesn't look like a log entry key — defensively
1690
- * tolerates an unrelated key under the log prefix.
1691
- */
1692
- const parseSeqFromLogKey = (key) => {
1693
- const match = /\/log\/(\d+)\.json$/.exec(key);
1694
- if (match === null) return null;
1695
- const n = Number.parseInt(match[1], 10);
1696
- return Number.isFinite(n) && n >= 0 ? n : null;
1697
- };
1698
- /**
1699
- * Parse `<...>/content/<sha-32>.json` and return the 32-hex hash.
1700
- * Returns `null` on any shape that doesn't match the writer's
1701
- * `versionFromContent`-produced key format.
1702
- */
1703
- const parseHashFromContentKey = (key) => {
1704
- const match = /\/content\/([0-9a-f]{32})\.json$/.exec(key);
1705
- return match === null ? null : match[1];
1706
- };
1707
- /**
1708
- * Anchor `due_at` to `lastModified` when the storage adapter
1709
- * surfaces it; fall back to the injected `now` so a Storage impl
1710
- * without a server clock still GCs after `grace` from the mark
1711
- * moment.
1712
- */
1713
- const computeDueAt = (entry, now, graceMs) => {
1714
- const base = entry.lastModified ?? now();
1715
- return new Date(base.getTime() + graceMs).toISOString();
1716
- };
1717
- /**
1718
- * Build the live content-hash set. The set covers every live
1719
- * post-image: every `entry.after` in `[logSeqStart, tail_hint)` plus
1720
- * every row body in the current snapshot.
1721
- *
1722
- * A snapshot read that throws (corrupt body, hash mismatch) is
1723
- * tolerated — the worst-case effect is missing some live hashes,
1724
- * which could mark a live content blob as orphan. The grace period
1725
- * absorbs the false-positive: a retry that re-PUTs the same hash
1726
- * recreates the key before the sweep deletes it.
1727
- */
1728
- const collectLiveContentHashes = async (storage, collectionPrefix, collectionName, current, logSeqStart, signal) => {
1729
- const hashes = /* @__PURE__ */ new Set();
1730
- const getOpts = signal !== void 0 ? { signal } : void 0;
1731
- const { tail } = await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, current.tail_hint), { signal });
1732
- const ingestLogEntry = async (s) => {
1733
- const got = await storage.get(logObjectKey(collectionPrefix, s), getOpts);
1734
- if (got === null) return;
1735
- let entry;
1736
- try {
1737
- entry = decodeJsonBytes(got.body);
1738
- } catch {
1739
- return;
1740
- }
1741
- if (entry.after === void 0) return;
1742
- const bodyBytes = encodeJsonBytes(entry.after);
1743
- hashes.add(await versionFromContent(bodyBytes));
1744
- };
1745
- for (let chunkStart = logSeqStart; chunkStart < tail; chunkStart += 16) {
1746
- signal?.throwIfAborted();
1747
- const chunkEnd = Math.min(chunkStart + 16, tail);
1748
- const chunk = [];
1749
- for (let s = chunkStart; s < chunkEnd; s++) chunk.push(ingestLogEntry(s));
1750
- await Promise.all(chunk);
1751
- }
1752
- if (current.snapshot !== null) try {
1753
- const map = await loadSnapshotAsMap(storage, current.snapshot, collectionName, signal);
1754
- const rowReads = [];
1755
- for (const body of map.values()) rowReads.push((async () => {
1756
- const bytes = encodeJsonBytes(body);
1757
- hashes.add(await versionFromContent(bytes));
1758
- })());
1759
- await Promise.all(rowReads);
1760
- } catch {}
1761
- return hashes;
1762
- };
1763
- //#endregion
1764
- //#region packages/server/src/rebuild-index.ts
1765
- /**
1766
- * Idempotent reconciliation for one secondary index.
1767
- *
1768
- * {@link rebuildIndex} folds the live `(snapshot, log_tail)` of a
1769
- * collection into the doc set the writer's index emission would
1770
- * have produced, then PUTs every missing index key and DELETEs every
1771
- * orphaned one. Same-key zero-byte PUTs are no-ops (the entries are
1772
- * content-addressed by composition); same-key DELETEs are
1773
- * contractually idempotent.
1774
- *
1775
- * Invariants:
1776
- *
1777
- * - **Re-running on a healthy index is a no-op.** `added = 0`,
1778
- * `removed = 0`.
1779
- * - **Re-running after a crashed mid-commit** reconciles toward
1780
- * the live `current.json` view — orphans deleted, missing keys
1781
- * PUT.
1782
- * - **Re-running while writes are in flight** is safe up to a
1783
- * race window: a write that lands during the rebuild may
1784
- * produce a transient "orphan" the rebuild sweeps, which the
1785
- * next live write then re-PUTs. The race converges on the next
1786
- * commit.
1787
- *
1788
- * The function reads the same fold the writer / reader use
1789
- * (snapshot via {@link loadSnapshotAsMap}; log tail by direct
1790
- * `get(<logPrefix>/log/<seq>.json)`) so it MUST agree with the
1791
- * writer on the live doc set by construction.
1792
- *
1793
- * @see ./indexes.ts — `IndexDefinition`, `allIndexKeysFor`,
1794
- * `indexKeyPrefix`.
1795
- * @see ./writer.ts — the hybrid index emission path this command
1796
- * reconciles.
1797
- */
1798
- const EMPTY_BODY = /* @__PURE__ */ new Uint8Array(0);
1799
- const APPLICATION_JSON = "application/json";
1800
- /**
1801
- * Idempotently reconcile one secondary index for one collection.
1802
- *
1803
- * @param storage Storage handle (any in-tree `Storage`).
1804
- * @param currentJsonKey The collection's `current.json` key.
1805
- * @param def The index to reconcile. Single-field today;
1806
- * composite definitions are accepted by the
1807
- * key encoder but the read path only consults
1808
- * single-field keys.
1809
- * @param opts Optional knobs — see {@link RebuildIndexOptions}.
1810
- * Strictly additive: existing positional-arg
1811
- * callers compile and behave unchanged when
1812
- * omitted.
1813
- *
1814
- * @throws BaerlyError code="InvalidResponse" — `current.json` missing.
1815
- * @throws BaerlyError code="Internal" — protocol-invariant
1816
- * violation (snapshot hash mismatch, missing log entry inside
1817
- * `[log_seq_start, tail_hint)`).
1818
- */
1819
- const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
1820
- const read = await readCurrentJson(storage, currentJsonKey);
1821
- if (read === null) throw new BaerlyError("InvalidResponse", `rebuildIndex: current.json missing at ${JSON.stringify(currentJsonKey)}`);
1822
- const lastSlash = currentJsonKey.lastIndexOf("/");
1823
- if (lastSlash < 0) throw new BaerlyError("InvalidResponse", `rebuildIndex: currentJsonKey ${JSON.stringify(currentJsonKey)} must contain "/"`);
1824
- const collectionPrefix = currentJsonKey.slice(0, lastSlash);
1825
- const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
1826
- const logSeqStart = logSeqStartOf(read.json);
1827
- const hint = read.json.tail_hint;
1828
- const live = read.json.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, read.json.snapshot, collection);
1829
- foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart, hint), { collection });
1830
- const probeOpts = opts.signal !== void 0 ? { signal: opts.signal } : void 0;
1831
- foldLogEntriesOnto(live, (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, hint), probeOpts)).entries, { collection });
1832
- const expected = /* @__PURE__ */ new Set();
1833
- for (const [docId, body] of live) for (const k of allIndexKeysFor(collectionPrefix, [def], body, docId)) expected.add(k);
1834
- const actual = /* @__PURE__ */ new Set();
1835
- const listOpts = {};
1836
- if (opts.signal !== void 0) listOpts.signal = opts.signal;
1837
- for await (const entry of storage.list(indexKeyPrefix(collectionPrefix, def.name), listOpts)) actual.add(entry.key);
1838
- const dryRun = opts.dryRun === true;
1839
- let added = 0;
1840
- let removed = 0;
1841
- let kept = 0;
1842
- for (const k of expected) {
1843
- if (actual.has(k)) {
1844
- kept += 1;
1845
- continue;
1846
- }
1847
- if (dryRun) {
1848
- added += 1;
1849
- continue;
1850
- }
1851
- try {
1852
- await storage.put(k, EMPTY_BODY, {
1853
- ifNoneMatch: "*",
1854
- contentType: APPLICATION_JSON
1855
- });
1856
- added += 1;
1857
- } catch (error) {
1858
- if (error instanceof BaerlyError && error.code === "Conflict") kept += 1;
1859
- else throw error;
1860
- }
1861
- }
1862
- for (const k of actual) {
1863
- if (expected.has(k)) continue;
1864
- if (dryRun) {
1865
- removed += 1;
1866
- continue;
1867
- }
1868
- await storage.delete(k);
1869
- removed += 1;
1870
- }
1871
- return {
1872
- added,
1873
- removed,
1874
- kept
1875
- };
1876
- };
1877
- //#endregion
1878
- //#region packages/server/src/maintenance.ts
1879
- /**
1880
- * `runScheduledMaintenance` — single-pass compose of `compact()` and
1881
- * `runGc()` over one collection. Designed to fit a single Cloudflare
1882
- * Cron Trigger invocation's 50-subrequest free-tier budget when
1883
- * called with the {@link CLOUDFLARE_FREE_TIER} profile (which the
1884
- * caller pairs with even/odd-minute alternation between compact and
1885
- * GC ticks); unbounded on Node.
1886
- *
1887
- * Single-attempt: returns the combined result. The caller (cron
1888
- * handler) is responsible for scheduling the next invocation; this
1889
- * function does not loop or retry. `compact()` and `runGc()` are
1890
- * already CAS-protected single-attempts — a restart safely retries
1891
- * next tick.
1892
- *
1893
- * Callers that want only one phase to fire this tick should invoke
1894
- * the underlying primitive directly (`compact()` or `runGc()` from
1895
- * this same module's re-exports). The even/odd-minute CF cron
1896
- * pattern uses exactly that.
1897
- */
1898
- const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
1899
- /**
1900
- * Single maintenance pass for one collection. Runs `compact()` then
1901
- * `runGc()` (in that order; the compactor's advance of
1902
- * `log_seq_start` produces the stale-log candidates the GC then
1903
- * marks). Callers wanting a single phase per tick invoke
1904
- * `compact()` or `runGc()` directly instead.
1905
- *
1906
- * Errors propagate — the caller's cron handler is responsible for
1907
- * logging them. The Cloudflare runtime ships uncaught Worker errors
1908
- * to the dashboard; Node operators wrap their `node-cron` callbacks
1909
- * themselves.
1910
- *
1911
- * @example
1912
- * ```ts
1913
- * import { runScheduledMaintenance, CLOUDFLARE_FREE_TIER } from "@gusto/baerly-storage/maintenance";
1914
- *
1915
- * // Node (unbounded — defaults fold the entire live tail):
1916
- * const res = await runScheduledMaintenance(
1917
- * { storage, currentJsonKey: "app/x/tenant/t/manifests/tickets/current.json" },
1918
- * );
1919
- * console.log("compacted:", res.compact.entriesFolded, "swept:", res.gc.swept);
1920
- *
1921
- * // Cloudflare free tier (50-subrequest cap, single phase per tick):
1922
- * await runScheduledMaintenance({ storage, currentJsonKey }, CLOUDFLARE_FREE_TIER);
1923
- * ```
1924
- */
1925
- const runScheduledMaintenance = async (args, options = {}) => {
1926
- return {
1927
- compact: await compact(args, {
1928
- ...options.compact,
1929
- ...options.signal !== void 0 && { signal: options.signal }
1930
- }),
1931
- gc: await runGc(args, {
1932
- ...options.gc,
1933
- ...options.signal !== void 0 && { signal: options.signal }
1934
- })
1935
- };
1936
- };
1937
- const profileToScheduledOptions = (profile) => ({
1938
- compact: {
1939
- maxEntriesPerRun: profile.maxFoldEntriesPerPass,
1940
- minEntriesToCompact: 50
1941
- },
1942
- gc: {
1943
- maxMarksPerRun: profile.gcMaxMarks,
1944
- maxSweepsPerRun: profile.gcMaxSweeps
1945
- }
1946
- });
1947
- /**
1948
- * Tuning profile for the 50-subrequest Cloudflare free-tier budget,
1949
- * derived from {@link MAINTENANCE_PROFILE_CF_FREE} (one source of truth).
1950
- *
1951
- * Budget arithmetic (worst case):
1952
- * compact: 1 GET current + 1 GET snapshot (if any) + N GETs log
1953
- * + 1 PUT snapshot + 1 PUT current = 3 + N (N = maxEntriesPerRun).
1954
- * runGc: 1 GET current + 1 GET pending + 3 LISTs (one page each)
1955
- * + M GETs log (live tail for content hashes) + S DELETEs + 1 PUT
1956
- * pending = 6 + M + S.
1957
- *
1958
- * With N=20, M=20, S=10 the totals are: compact ≈ 23, gc ≈ 36 — over
1959
- * the 50 cap combined. The Cloudflare scheduled handler therefore
1960
- * alternates phases per tick (even minute → compact, odd minute →
1961
- * GC) by calling `compact()` / `runGc()` directly instead of
1962
- * `runScheduledMaintenance`; the `maintenance-budget.test.ts`
1963
- * worst-case test proves each phase in isolation sits under 50 ops
1964
- * with these bounds.
1965
- */
1966
- const CLOUDFLARE_FREE_TIER = profileToScheduledOptions(MAINTENANCE_PROFILE_CF_FREE);
1967
- /**
1968
- * Tuning profile for the 10,000-subrequest Cloudflare paid-tier budget,
1969
- * derived from {@link MAINTENANCE_PROFILE_CF_PAID} (one source of truth).
1970
- * Keeps the single-phase CPU-killable shape but affords Node-tier per-pass
1971
- * throughput (compact ≈ 3+200, gc ≈ 6+200+100 — far under the 10k cap).
1972
- *
1973
- * To honour `BAERLY_MAINTENANCE_PROFILE=cf-paid` on the cron path, pass this
1974
- * constant to `runScheduledMaintenance` inside your
1975
- * {@link WorkerScheduledHandler}; the worked recipe lives in the `scheduled`
1976
- * handler docstring of `baerlyWorker` (`@gusto/baerly-storage/cloudflare`).
1977
- */
1978
- const CLOUDFLARE_PAID_TIER = profileToScheduledOptions(MAINTENANCE_PROFILE_CF_PAID);
1979
- /**
1980
- * Floor-based GC-cadence boundary crossing. Every commit advances the
1981
- * writer-observed tail by one logical slot; `tail_hint` remains a
1982
- * lower bound and may be refreshed later by maintenance. The floor test
1983
- * trips on any interval boundary inside `(prevSeq, nextSeq]`, so it
1984
- * stays correct whether the observed/probed tail advances by one or
1985
- * catches up by a larger step.
1986
- */
1987
- const crossesGcBoundary = (prevSeq, nextSeq, interval) => Math.floor(prevSeq / interval) !== Math.floor(nextSeq / interval);
1988
- /**
1989
- * Derived live-tail byte size — the ratio TRIGGER input that replaces the
1990
- * formerly-stored exact byte count. `(observedTail − log_seq_start) ×
1991
- * mean_entry_bytes`. `observedTail` is a param so write-tick callers can
1992
- * pass the writer-observed tail and scheduled callers can pass a
1993
- * probe-discovered true tail. Cold-start (no mean yet) falls back to
1994
- * {@link MAINTENANCE_COLD_START_ENTRY_BYTES}, never 0 (see that constant).
1995
- * TRIGGER-only — structurally barred from `foldViable` (the exact-bytes CPU-kill
1996
- * ceiling).
1997
- */
1998
- const estimateTailBytes = (current, observedTail) => Math.max(0, observedTail - current.log_seq_start) * (current.mean_entry_bytes ?? 128);
1999
- /**
2000
- * The cheap DISPATCH gate the writer calls after a commit — zero
2001
- * storage ops; it reads the post-commit {@link CurrentJson} already in
2002
- * scope. Ratio-OR-boundary, WITHOUT the entry floor (the floor is part
2003
- * of the fold-trigger Gate 1 checked inside {@link runBoundedMaintenance}).
2004
- * Returning `false` lets the writer skip the dispatch entirely. Ratio
2005
- * numerator is the DERIVED {@link estimateTailBytes};
2006
- * `observedTail` threads from the caller (the writer's in-memory observed
2007
- * tail `seq+1` under single-write commit). Both the boundary check and the
2008
- * ratio key off `observedTail`, since the stored `tail_hint` is now only a
2009
- * non-authoritative lower bound (compactor-advanced).
2010
- */
2011
- const shouldFireMaintenance = (current, prevSeq, gcInterval, observedTail) => crossesGcBoundary(prevSeq, observedTail, gcInterval) || estimateTailBytes(current, observedTail) / Math.max(current.snapshot_bytes, 65536) >= 1;
2012
- /**
2013
- * The write-tick maintenance runner: a bounded, single-attempt
2014
- * composition of {@link compact} and {@link runGc} for one collection,
2015
- * sized to fit a CPU-killable Cloudflare free-tier isolate by default.
2016
- *
2017
- * INVARIANT: a {@link MaintenanceProfile} changes only maintenance rate +
2018
- * defer threshold — never data/API/query/wire/correctness; reads stay pure.
2019
- * See `maintenance-profile-equivalence.test.ts`.
2020
- *
2021
- * Never throws — the write-tick caller must never see a maintenance
2022
- * failure. Expected signals (CAS contention, an over-ceiling deferral)
2023
- * are swallowed quietly; unexpected throws bump
2024
- * `db.maintenance.unexpected_error_total`, log a stack, and are
2025
- * swallowed.
2026
- *
2027
- * Control flow (fold-priority with a stateless GC-starvation guard):
2028
- * 0. `disabled` → skip fold/GC, but still allow the rate-limited
2029
- * tail_hint refresh that bounds read/write forward-probes.
2030
- * 1. Read `current.json` once.
2031
- * 2. Hard-GC starvation boundary (`"single"` only) → GC slice, skip fold.
2032
- * 3. Gate 1 (ratio AND entry floor) trips → fold if viable, else
2033
- * defer (and metric); on `"single"` a successful fold attempt is
2034
- * this tick's one phase.
2035
- * 4. Otherwise (or on `"both"`) → GC iff the cadence boundary crossed.
2036
- */
2037
- const runBoundedMaintenance = async (args, options) => {
2038
- const { storage, currentJsonKey, prevSeq } = args;
2039
- const profile = options?.profile ?? MAINTENANCE_PROFILE_CF_FREE;
2040
- const { maxFoldEntriesPerPass, gcMaxMarks, gcMaxSweeps, gcInterval } = profile;
2041
- const minEntriesToCompact = options?.minEntriesToCompact ?? 50;
2042
- const phasesPerTick = options?.phasesPerTick ?? "single";
2043
- const signal = options?.signal;
2044
- const C = args.maxFoldBytes ?? profile.maxFoldBytes;
2045
- const E = profile.maxFoldRows;
2046
- const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
2047
- const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
2048
- const gcOpts = {
2049
- maxMarksPerRun: gcMaxMarks,
2050
- maxSweepsPerRun: gcMaxSweeps,
2051
- ...signal !== void 0 && { signal },
2052
- ...options?.gcGraceMillis !== void 0 && { graceMillis: options.gcGraceMillis },
2053
- ...options?.now !== void 0 && { now: options.now }
2054
- };
2055
- try {
2056
- const read = await readCurrentJson(storage, currentJsonKey, signal !== void 0 ? { signal } : void 0);
2057
- if (read === null) return;
2058
- const current = read.json;
2059
- const snapshotBytes = current.snapshot_bytes;
2060
- const snapshotRows = current.snapshot_rows;
2061
- const logSeqStart = current.log_seq_start;
2062
- const probeFloor = Math.max(logSeqStart, current.tail_hint);
2063
- let nextSeq;
2064
- if (args.observedTail !== void 0) nextSeq = Math.max(args.observedTail, probeFloor);
2065
- else nextSeq = (await probeTailFrom(storage, collectionPrefix, probeFloor, signal !== void 0 ? { signal } : void 0)).tail;
2066
- const gate1 = estimateTailBytes(current, nextSeq) / Math.max(snapshotBytes, 65536) >= 1 && nextSeq - logSeqStart >= minEntriesToCompact;
2067
- const foldViable = snapshotBytes <= C && snapshotRows + maxFoldEntriesPerPass <= E;
2068
- const gcDue = crossesGcBoundary(prevSeq, nextSeq, gcInterval);
2069
- const refreshTailHintIfNeeded = async () => {
2070
- if (nextSeq - current.tail_hint < 128) return;
2071
- try {
2072
- await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
2073
- ...c,
2074
- tail_hint: Math.max(c.tail_hint, nextSeq)
2075
- }), signal !== void 0 ? { signal } : void 0);
2076
- } catch {}
2077
- };
2078
- if (args.disabled === true) {
2079
- await refreshTailHintIfNeeded();
2080
- return;
2081
- }
2082
- const hardGc = crossesGcBoundary(prevSeq, nextSeq, gcInterval * 4);
2083
- if (phasesPerTick === "single" && hardGc) {
2084
- await runGc({
2085
- storage,
2086
- currentJsonKey
2087
- }, gcOpts);
2088
- return;
2089
- }
2090
- if (gate1) if (foldViable) {
2091
- await compact({
2092
- storage,
2093
- currentJsonKey
2094
- }, {
2095
- maxEntriesPerRun: maxFoldEntriesPerPass,
2096
- minEntriesToCompact,
2097
- ceilingBytes: C,
2098
- ceilingEntries: E,
2099
- knownTail: nextSeq,
2100
- ...signal !== void 0 && { signal }
2101
- });
2102
- if (phasesPerTick === "single") return;
2103
- } else {
2104
- ctxMetrics().counter("db.compaction.deferred_total", 1, {
2105
- collection,
2106
- dimension: snapshotBytes > C ? "bytes" : "rows"
2107
- });
2108
- await refreshTailHintIfNeeded();
2109
- if (nextSeq - (current.last_warned_seq ?? 0) >= 1e3) {
2110
- console.warn(`baerly-storage: collection "${collection}" is deferring compaction — its snapshot exceeds the fold ceiling (${snapshotBytes > C ? `${C} bytes` : `${E} rows`}), so the tail keeps growing and read amplification will climb. The dataset has outgrown the maintenance envelope. On paid Cloudflare / Node you can raise BAERLY_MAINTENANCE_MAX_FOLD_BYTES — but on Cloudflare a cap above what a single isolate can fold makes folds get CPU-killed mid-flight and silently not land (no clean metric — watch snapshot age / object count); see docs/about/graduation.md. Otherwise, move to a database service (D1 / Postgres).`);
2111
- try {
2112
- await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
2113
- ...c,
2114
- last_warned_seq: nextSeq
2115
- }), signal !== void 0 ? { signal } : void 0);
2116
- } catch {}
2117
- }
2118
- }
2119
- if (gcDue) await runGc({
2120
- storage,
2121
- currentJsonKey
2122
- }, gcOpts);
2123
- } catch (error) {
2124
- if (error instanceof BaerlyError && error.code === "Conflict") return;
2125
- ctxMetrics().counter("db.maintenance.unexpected_error_total", 1, { collection });
2126
- console.error(error);
2127
- }
2128
- };
2129
- /**
2130
- * Default maintenance dispatcher: run the task inline and await it.
2131
- * Adapters that can keep running after the response (Cloudflare:
2132
- * `ctx.waitUntil`) override this with a fire-and-forget variant that
2133
- * returns `void`.
2134
- */
2135
- const dispatchInlineAwaited = (task) => task();
2136
- /**
2137
- * Parse the two host-agnostic ops-plane env vars into the
2138
- * {@link MaintenanceDispatch} overrides. Shared by every host adapter so the
2139
- * cross-host contract has one source of truth: `maxFoldBytes` (`C`) is the
2140
- * finite `BAERLY_MAINTENANCE_MAX_FOLD_BYTES` or undefined; `disabled` is
2141
- * `BAERLY_MAINTENANCE_DISABLE` truthy (set, non-empty, not `"0"`/`"false"`).
2142
- */
2143
- const parseMaintenanceEnv = (readEnv) => {
2144
- const rawFoldBytes = readEnv("BAERLY_MAINTENANCE_MAX_FOLD_BYTES");
2145
- const parsedFoldBytes = rawFoldBytes !== void 0 && rawFoldBytes !== "" ? Number(rawFoldBytes) : NaN;
2146
- const maxFoldBytes = Number.isFinite(parsedFoldBytes) ? parsedFoldBytes : void 0;
2147
- const rawDisable = readEnv("BAERLY_MAINTENANCE_DISABLE");
2148
- const disabled = rawDisable !== void 0 && rawDisable !== "" && rawDisable !== "0" && rawDisable.toLowerCase() !== "false";
2149
- return {
2150
- ...maxFoldBytes !== void 0 && { maxFoldBytes },
2151
- disabled
2152
- };
2153
- };
2154
- //#endregion
2155
- export { deepEqualDocumentValue as A, findLogTail as C, validateWire as D, versionFromContent as E, assertWireSatisfiable as O, assertNameNotReserved as S, logObjectKey as T, encodeIndexValue as _, estimateTailBytes as a, loadSnapshotAsMap as b, runScheduledMaintenance as c, runGc as d, compact as f, allIndexKeysFor as g, walkLogRange as h, dispatchInlineAwaited as i, matchesWire as k, shouldFireMaintenance as l, readLogEntry as m, CLOUDFLARE_PAID_TIER as n, parseMaintenanceEnv as o, foldLogEntriesOnto as p, crossesGcBoundary as r, runBoundedMaintenance as s, CLOUDFLARE_FREE_TIER as t, rebuildIndex as u, validateIndexDefinition as v, probeTailFrom as w, snapshotKey as x, encodeSnapshotBody as y };
2156
-
2157
- //# sourceMappingURL=maintenance-B4Dki6Ig.js.map