@gusto/baerly-storage 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +29 -0
  3. package/README.md +176 -70
  4. package/dist/API.md +213 -139
  5. package/dist/CHANGELOG.md +434 -0
  6. package/dist/RECIPES.md +125 -0
  7. package/dist/THIRD-PARTY-LICENSES.txt +238 -0
  8. package/dist/{app-config-C0Bbw_Z6.d.ts → app-config-DyiFK3_k.d.ts} +55 -11
  9. package/dist/app-config.d.ts +1 -1
  10. package/dist/auth.d.ts +1 -1
  11. package/dist/auth.js +301 -301
  12. package/dist/auth.js.map +1 -1
  13. package/dist/aws4fetch.esm-p7eEzLCs.js +1702 -0
  14. package/dist/aws4fetch.esm-p7eEzLCs.js.map +1 -0
  15. package/dist/baerly.js +35 -15
  16. package/dist/baerly.js.map +1 -1
  17. package/dist/baerly.spec.json +342 -0
  18. package/dist/bytes-BXdamO3L.js.map +1 -1
  19. package/dist/{client-DYZfoZb9.d.ts → client-Fe8CwB73.d.ts} +10 -10
  20. package/dist/client-react.d.ts +3 -3
  21. package/dist/client-react.js +4 -4
  22. package/dist/client-react.js.map +1 -1
  23. package/dist/client.d.ts +4 -4
  24. package/dist/client.js +3 -3
  25. package/dist/client.js.map +1 -1
  26. package/dist/{cloudflare-CINJPKCW.js → cloudflare-DzKBrUlg.js} +178 -2
  27. package/dist/cloudflare-DzKBrUlg.js.map +1 -0
  28. package/dist/cloudflare.d.ts +49 -6
  29. package/dist/cloudflare.js +59 -26
  30. package/dist/cloudflare.js.map +1 -1
  31. package/dist/config-1tv0-Yzm.js +2693 -0
  32. package/dist/config-1tv0-Yzm.js.map +1 -0
  33. package/dist/constants-CfeMWQWk.js +222 -0
  34. package/dist/constants-CfeMWQWk.js.map +1 -0
  35. package/dist/context-DRZ3tZ5l.js.map +1 -1
  36. package/dist/{cost-CQt1YRkA.js → cost-DSMIVqzL.js} +43 -42
  37. package/dist/cost-DSMIVqzL.js.map +1 -0
  38. package/dist/current-json-C6nhPOm1.js +190 -0
  39. package/dist/current-json-C6nhPOm1.js.map +1 -0
  40. package/dist/{db-BM3y2JQw.d.ts → db-DsPkOVEH.d.ts} +26 -108
  41. package/dist/{deploy-CrAbfgCq.js → deploy-DonTr-CV.js} +12 -5
  42. package/dist/deploy-DonTr-CV.js.map +1 -0
  43. package/dist/dev-vite.d.ts +11 -7
  44. package/dist/dev-vite.js +7 -6
  45. package/dist/dev-vite.js.map +1 -1
  46. package/dist/dev.d.ts +1 -1
  47. package/dist/dev.js +1 -1
  48. package/dist/{doctor-Db-hzPmE.js → doctor-DSIyCDg_.js} +14 -136
  49. package/dist/doctor-DSIyCDg_.js.map +1 -0
  50. package/dist/{dump-BYA55NLP.js → dump-Bh5k26VQ.js} +10 -31
  51. package/dist/dump-Bh5k26VQ.js.map +1 -0
  52. package/dist/{errors-DOdL95kK.js → errors-Dfa3p78c.js} +25 -5
  53. package/dist/errors-Dfa3p78c.js.map +1 -0
  54. package/dist/{export-Bhbz90-z.js → export-DtF5tDIM.js} +11 -32
  55. package/dist/{export-Bhbz90-z.js.map → export-DtF5tDIM.js.map} +1 -1
  56. package/dist/{fsck-CPPZWt9p.js → fsck-D_djMO5D.js} +28 -39
  57. package/dist/fsck-D_djMO5D.js.map +1 -0
  58. package/dist/{http-Cw8p6y5b.js → http-DxM9MflO.js} +151 -79
  59. package/dist/http-DxM9MflO.js.map +1 -0
  60. package/dist/http.d.ts +42 -26
  61. package/dist/http.js +3 -2
  62. package/dist/{index-DwDTWPSU.d.ts → index-0rZcMBwy.d.ts} +10 -7
  63. package/dist/{index-C3JAH1Yo.d.ts → index-B6oexfOB.d.ts} +9 -9
  64. package/dist/{index-cUY-36AN.d.ts → index-BNRh23K9.d.ts} +21 -5
  65. package/dist/{index-DSvwEUeF.d.ts → index-BYk9oCNf.d.ts} +240 -202
  66. package/dist/index.d.ts +6 -6
  67. package/dist/index.js +5 -5
  68. package/dist/{indexes-DAu2AkTW.d.ts → indexes-Dn9hbGQG.d.ts} +2 -2
  69. package/dist/{inspect-Bdbzvi-_.js → inspect-6WZ-pC7-.js} +21 -36
  70. package/dist/inspect-6WZ-pC7-.js.map +1 -0
  71. package/dist/key-Di6yyxFu.js +35 -0
  72. package/dist/key-Di6yyxFu.js.map +1 -0
  73. package/dist/{snapshot-SS8USDUF.js → log-tail-D98U1JBo.js} +179 -84
  74. package/dist/log-tail-D98U1JBo.js.map +1 -0
  75. package/dist/{maintenance-ClzBC1Vv.js → maintenance-B4Dki6Ig.js} +438 -176
  76. package/dist/maintenance-B4Dki6Ig.js.map +1 -0
  77. package/dist/{maintenance-BRrThz8V.d.ts → maintenance-B_e0T99L.d.ts} +114 -26
  78. package/dist/{maintenance-BvOv6BMM.js → maintenance-C7coQp4c.js} +217 -94
  79. package/dist/maintenance-C7coQp4c.js.map +1 -0
  80. package/dist/maintenance.d.ts +3 -2
  81. package/dist/maintenance.js +3 -2
  82. package/dist/node.d.ts +176 -68
  83. package/dist/node.js +3 -2
  84. package/dist/{normalize-XWY25_sG.js → normalize-rjnZLwS4.js} +2 -2
  85. package/dist/normalize-rjnZLwS4.js.map +1 -0
  86. package/dist/{observability-CpOiEN0k.js → observability-wzs3-BL_.js} +279 -109
  87. package/dist/observability-wzs3-BL_.js.map +1 -0
  88. package/dist/observability.d.ts +3 -3
  89. package/dist/observability.js +1 -1
  90. package/dist/{plan-DOTl21vq.js → plan-a7PJHBCy.js} +7 -4
  91. package/dist/plan-a7PJHBCy.js.map +1 -0
  92. package/dist/{query-BmiQy328.js → query-rAtXNQQB.js} +1063 -1202
  93. package/dist/query-rAtXNQQB.js.map +1 -0
  94. package/dist/{rebuild-index-DC_nR6wP.js → rebuild-index-BVzQSZlm.js} +10 -32
  95. package/dist/rebuild-index-BVzQSZlm.js.map +1 -0
  96. package/dist/{request-BEF_OvMh.js → request-u0M51mhJ.js} +3 -3
  97. package/dist/{request-BEF_OvMh.js.map → request-u0M51mhJ.js.map} +1 -1
  98. package/dist/restore-R-4rbciV.js +778 -0
  99. package/dist/restore-R-4rbciV.js.map +1 -0
  100. package/dist/runtime-spec-qL3_lTTU.js +391 -0
  101. package/dist/runtime-spec-qL3_lTTU.js.map +1 -0
  102. package/dist/s3-http-B-z8th6O.d.ts +60 -0
  103. package/dist/s3.d.ts +47 -0
  104. package/dist/s3.js +45 -0
  105. package/dist/s3.js.map +1 -0
  106. package/dist/{shared-secret-BJdSwpgw.js → shared-secret-DgEicd3x.js} +13 -5
  107. package/dist/shared-secret-DgEicd3x.js.map +1 -0
  108. package/dist/src-BG20LZs7.js +2271 -0
  109. package/dist/src-BG20LZs7.js.map +1 -0
  110. package/dist/{src-zhEAmmF7.js → src-CYHFDrKz.js} +82 -99
  111. package/dist/src-CYHFDrKz.js.map +1 -0
  112. package/dist/{src-QQ7jDeL2.js → src-DYZ4H1aF.js} +44 -23
  113. package/dist/src-DYZ4H1aF.js.map +1 -0
  114. package/dist/subcommand-DoDA-K7d.js +197 -0
  115. package/dist/subcommand-DoDA-K7d.js.map +1 -0
  116. package/dist/time-DUqvFMHq.js +119 -0
  117. package/dist/time-DUqvFMHq.js.map +1 -0
  118. package/dist/{usage-CLjirdD-.js → usage-DBV1eQHC.js} +7 -22
  119. package/dist/usage-DBV1eQHC.js.map +1 -0
  120. package/dist/{validate-DCq_0DIn.js → validate-DtljsCh1.js} +51 -51
  121. package/dist/{validate-DCq_0DIn.js.map → validate-DtljsCh1.js.map} +1 -1
  122. package/dist/wire-izfGZDvr.js.map +1 -1
  123. package/package.json +77 -42
  124. package/dist/cloudflare-CINJPKCW.js.map +0 -1
  125. package/dist/config-DVfDGyw1.js +0 -278
  126. package/dist/config-DVfDGyw1.js.map +0 -1
  127. package/dist/cost-CQt1YRkA.js.map +0 -1
  128. package/dist/current-json-C-EyJqs0.js +0 -354
  129. package/dist/current-json-C-EyJqs0.js.map +0 -1
  130. package/dist/deploy-CrAbfgCq.js.map +0 -1
  131. package/dist/doctor-Db-hzPmE.js.map +0 -1
  132. package/dist/dump-BYA55NLP.js.map +0 -1
  133. package/dist/errors-DOdL95kK.js.map +0 -1
  134. package/dist/fsck-CPPZWt9p.js.map +0 -1
  135. package/dist/http-Cw8p6y5b.js.map +0 -1
  136. package/dist/inspect-Bdbzvi-_.js.map +0 -1
  137. package/dist/maintenance-BvOv6BMM.js.map +0 -1
  138. package/dist/maintenance-ClzBC1Vv.js.map +0 -1
  139. package/dist/normalize-XWY25_sG.js.map +0 -1
  140. package/dist/observability-CpOiEN0k.js.map +0 -1
  141. package/dist/plan-DOTl21vq.js.map +0 -1
  142. package/dist/query-BmiQy328.js.map +0 -1
  143. package/dist/rebuild-index-DC_nR6wP.js.map +0 -1
  144. package/dist/restore-C6IcmLeA.js +0 -771
  145. package/dist/restore-C6IcmLeA.js.map +0 -1
  146. package/dist/shared-secret-BJdSwpgw.js.map +0 -1
  147. package/dist/snapshot-SS8USDUF.js.map +0 -1
  148. package/dist/src-DU1sCMlr.js +0 -6236
  149. package/dist/src-DU1sCMlr.js.map +0 -1
  150. package/dist/src-QQ7jDeL2.js.map +0 -1
  151. package/dist/src-zhEAmmF7.js.map +0 -1
  152. package/dist/subcommand-DZcHcsYp.js +0 -5122
  153. package/dist/subcommand-DZcHcsYp.js.map +0 -1
  154. package/dist/usage-CLjirdD-.js.map +0 -1
  155. /package/dist/{chunk-EAsCxrDo.js → rolldown-runtime-EAsCxrDo.js} +0 -0
@@ -1,771 +0,0 @@
1
- import { n as encodeJsonBytes, t as decodeJsonBytes } from "./bytes-BXdamO3L.js";
2
- import { a as readLogEntry, d as logSeqStartOf, f as readCurrentJson, o as walkLogRange, u as createCurrentJson } from "./snapshot-SS8USDUF.js";
3
- import { a as getCurrentContext, c as noopMetricsRecorder, l as versionFromContent, n as runBoundedMaintenance, o as allIndexKeysFor, r as shouldFireMaintenance, s as validateIndexDefinition, t as dispatchInlineAwaited } from "./maintenance-BvOv6BMM.js";
4
- import { a as uuid, i as countKey, n as parseBucketUri, r as timestamp, t as defineBaerlySubcommand } from "./subcommand-DZcHcsYp.js";
5
- import { o as BaerlyError, r as emitSuccess } from "./baerly.js";
6
- import { createInterface } from "node:readline";
7
- //#region ../server/src/log-conflict-adoption.ts
8
- /**
9
- * Decide whether the entry already at `/log/<seq>.json` is the
10
- * writer's own prior in-flight attempt and may be adopted. Pure
11
- * function; no I/O. The caller performs the GET (when the log PUT
12
- * surfaces 412) and passes the read-back body in `ctx.existing`.
13
- *
14
- * @see {@link AdoptionDecision} for the return shape.
15
- * @see The soundness invariant in this module's header docstring.
16
- */
17
- const tryAdoptOwnSessionLogEntry = (ctx) => {
18
- if (ctx.batchSize !== 1) return {
19
- adopt: false,
20
- reason: "batch"
21
- };
22
- if (ctx.existing.session !== ctx.self.session) return {
23
- adopt: false,
24
- reason: "foreign-session"
25
- };
26
- if (ctx.existing.seq !== ctx.self.seq) return {
27
- adopt: false,
28
- reason: "wrong-seq"
29
- };
30
- return {
31
- adopt: true,
32
- entry: ctx.existing
33
- };
34
- };
35
- //#endregion
36
- //#region ../server/src/writer.ts
37
- /**
38
- * `Writer` — stateless multi-instance write engine.
39
- *
40
- * Each {@link Writer.commit} call reads `current.json` FRESH from
41
- * the bucket, mints the next {@link LogEntry}, PUTs the content body
42
- * and the log entry, and CAS-advances `current.json` with `If-Match`.
43
- * Up to {@link S3_REQUEST_MAX_RETRIES} attempts on contention before
44
- * surfacing `BaerlyError{code:"Conflict"}`. The optional integrity
45
- * walk over the live log tail is gated by
46
- * {@link WriterOptions.verifyLogIntegrityOnCommit} (default off
47
- * — see the option's docstring).
48
- *
49
- * The instance carries no per-write cache: every commit re-reads
50
- * `current.json`, so N stateless server instances writing the same
51
- * tenant prefix contend at exactly one place — the conditional PUT
52
- * on `current.json` — and one loses cleanly with a 412.
53
- *
54
- * **Manifest-first ordering is REVERSED relative to the legacy
55
- * `src/syncer.ts` write loop.** Old loop: PUT manifest → PUT content
56
- * → CAS. New loop: PUT content → PUT log entry → CAS-advance
57
- * `current.json`. A crashed mid-loop writer leaves an unreferenced
58
- * content body (no log entry points at it), not an orphan log entry
59
- * with missing content. The compactor sweeps the orphan content
60
- * later.
61
- *
62
- * **`LogEntry` shape.** Emitted entries follow the contract in
63
- * `packages/protocol/src/log.ts` (see {@link LogEntry}) and
64
- * `docs/spec/log-entry-shape.md`. The on-bucket key is
65
- * `log/<seq>.json` (the integer `seq` is the load-bearing
66
- * identifier; the `lsn` string is schema-only).
67
- *
68
- * @see docs/spec/sync-protocol.md for the protocol invariants this
69
- * loop preserves.
70
- * @see docs/spec/log-entry-shape.md for the per-field contract.
71
- */
72
- const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
73
- const DEFAULT_INITIAL_BACKOFF_MS = 25;
74
- const MAX_BACKOFF_MS = 1500;
75
- const APPLICATION_JSON = "application/json";
76
- /**
77
- * Empty body for index entries. Each index entry is a fact ("doc
78
- * `<docId>` has `<field> = <value>`"), not data — readers list the
79
- * prefix, extract the doc id, then GET the content body separately.
80
- * Pre-allocated module-level constant so every index PUT shares one
81
- * zero-length buffer.
82
- */
83
- const EMPTY_BODY = new Uint8Array(0);
84
- /**
85
- * Stateless write engine for the multi-instance core.
86
- *
87
- * Construction is cheap and performs zero I/O — adapters build a
88
- * fresh `Writer` per request and discard it. All real work
89
- * happens in {@link commit}.
90
- *
91
- * @example
92
- * ```ts
93
- * import { Writer } from "@gusto/baerly-storage";
94
- * import { MemoryStorage } from "@gusto/baerly-storage";
95
- *
96
- * const writer = new Writer({
97
- * storage: new MemoryStorage(),
98
- * currentJsonKey: "app/tickets/tenant/acme/manifests/tickets/current.json",
99
- * });
100
- *
101
- * const result = await writer.commit({
102
- * op: "I",
103
- * collection: "tickets",
104
- * docId: "doc-1",
105
- * body: { _id: "doc-1", title: "hello" },
106
- * });
107
- * console.log(result.entry.seq, result.attempts);
108
- * ```
109
- */
110
- var Writer = class {
111
- #storage;
112
- #currentJsonKey;
113
- #maxRetries;
114
- #initialBackoffMs;
115
- #random;
116
- #indexes;
117
- #verifyLogIntegrityOnCommit;
118
- constructor(opts) {
119
- this.#storage = opts.storage;
120
- this.#currentJsonKey = opts.currentJsonKey;
121
- this.#maxRetries = opts.options?.maxRetries ?? 8;
122
- this.#initialBackoffMs = opts.options?.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS;
123
- this.#random = opts.options?.random ?? Math.random;
124
- const indexes = opts.options?.indexes ?? [];
125
- for (const def of indexes) validateIndexDefinition(def);
126
- this.#indexes = indexes;
127
- this.#verifyLogIntegrityOnCommit = opts.options?.verifyLogIntegrityOnCommit ?? false;
128
- }
129
- /**
130
- * Atomically commit one mutation. Reads `current.json` fresh, mints
131
- * the next log entry at `seq = current.next_seq`, PUTs the content
132
- * body and the log entry, then CAS-advances `current.json` with
133
- * `If-Match`. Retries on conflict up to `maxRetries`.
134
- *
135
- * The hot-path cost under no contention is 1 GET + 3 PUTs (4 ops);
136
- * an in-flight peer write costs one extra GET + the backoff sleep
137
- * per retry.
138
- *
139
- * Idempotency: PUT content uses `If-None-Match: "*"`, so a retry of
140
- * the same logical write produces the same content key and the
141
- * second PUT no-ops. PUT log entry also uses `If-None-Match: "*"`,
142
- * so two writers racing the same `next_seq` cause exactly one PUT
143
- * to land — the loser falls into the CAS retry path.
144
- *
145
- * @throws BaerlyError code="Conflict" when the retry budget is
146
- * exhausted (genuine high-contention case), or when the underlying
147
- * `current.json` CAS PUT lost.
148
- * @throws BaerlyError code="Conflict" — the writer fence
149
- * (`current.json.writer_fence.epoch`) was bumped by a concurrent
150
- * `claimWriter` call between this commit's read of
151
- * `current.json` and its CAS-advance. The stale writer aborts
152
- * to honour the new authority — the kernel does NOT retry under
153
- * the old epoch. See {@link claimWriter} for the rotation
154
- * recipe.
155
- * @throws BaerlyError code="Internal" when a log entry expected in
156
- * `[0, next_seq)` is missing — a protocol-invariant violation
157
- * (compactor bug or stale `current.json`).
158
- * @throws BaerlyError code="InvalidResponse" when a log-entry body
159
- * isn't valid JSON.
160
- *
161
- * **Implicit provisioning.** When `current.json` does not exist at
162
- * `currentJsonKey` (fresh bucket, fresh collection), the writer
163
- * auto-creates it with a zero-state manifest before the commit
164
- * lands. The cost is one extra Class A PUT on the very first
165
- * commit per collection; steady-state cost is unchanged. A peer
166
- * racing the same first-write loses cleanly on the create CAS
167
- * and re-reads to pick up the winner's manifest.
168
- */
169
- async commit(input) {
170
- const session = uuid().slice(0, 6);
171
- const logPrefix = this.#currentJsonKey.slice(0, this.#currentJsonKey.lastIndexOf("/"));
172
- for (let attempt = 1; attempt <= this.#maxRetries; attempt++) {
173
- let success;
174
- try {
175
- success = await this.#singleAttemptCommit([input], session, logPrefix, "Writer", true);
176
- } catch (error) {
177
- if (!isPreconditionFailed(error)) throw error;
178
- await this.#backoff(attempt);
179
- continue;
180
- }
181
- await this.#verifyFenceUnchanged(success.expectedEpoch, "Writer");
182
- this.#emitWriteMetrics(input.collection, success.classAOps + (attempt - 1));
183
- return {
184
- entry: success.entries[0],
185
- currentEtag: success.currentEtag,
186
- attempts: attempt
187
- };
188
- }
189
- throw new BaerlyError("Conflict", `Writer: CAS conflict on ${this.#currentJsonKey} after ${this.#maxRetries} attempts`);
190
- }
191
- /**
192
- * Single-attempt batched commit. Reads `current.json` ONCE, mints
193
- * `inputs.length` log entries with contiguous `seq` numbers
194
- * starting at the current `next_seq`, PUTs each content body and
195
- * each log entry, then CAS-advances `current.json` from
196
- * `next_seq = N` to `next_seq = N + inputs.length` with
197
- * `If-Match`.
198
- *
199
- * NO retry on CAS loss. On `current.json` 412, throws
200
- * `BaerlyError{code:"Conflict"}` immediately — the caller (here:
201
- * `Db.transaction`) decides whether to retry by re-running the
202
- * body, or to surface to the app. Likewise on a log-entry 412
203
- * (a peer wrote our seq).
204
- *
205
- * Empty `inputs`: returns `{ entries: [], currentEtag: undefined }`
206
- * after zero storage operations.
207
- *
208
- * @throws BaerlyError code="Conflict" — CAS lost on `current.json`,
209
- * or a log entry already exists at our seq (peer wrote ahead).
210
- * @throws BaerlyError code="Conflict" — the writer fence
211
- * (`current.json.writer_fence.epoch`) was bumped by a concurrent
212
- * `claimWriter` call between this batch's read of
213
- * `current.json` and its CAS-advance. The stale writer aborts
214
- * to honour the new authority — the kernel does NOT retry under
215
- * the old epoch. See {@link claimWriter} for the rotation
216
- * recipe.
217
- * @throws BaerlyError code="Internal" — protocol-invariant violation
218
- * (missing log entry inside `[0, next_seq)`).
219
- * @throws BaerlyError code="InvalidResponse" — malformed log body.
220
- *
221
- * **Implicit provisioning.** Like {@link commit}, this path auto-
222
- * creates `current.json` with a zero-state manifest on a missing
223
- * read. Because `commitBatch` does NOT retry on CAS loss, a peer
224
- * racing the same first-write surfaces `Conflict` to the caller
225
- * (`Db.transaction` re-runs the body); the manifest is durable
226
- * regardless of which writer won the create.
227
- */
228
- async commitBatch(inputs) {
229
- if (inputs.length === 0) return {
230
- entries: [],
231
- currentEtag: void 0
232
- };
233
- const session = uuid().slice(0, 6);
234
- const logPrefix = this.#currentJsonKey.slice(0, this.#currentJsonKey.lastIndexOf("/"));
235
- const success = await this.#singleAttemptCommit(inputs, session, logPrefix, "Writer.commitBatch", false);
236
- await this.#verifyFenceUnchanged(success.expectedEpoch, "Writer.commitBatch");
237
- this.#emitWriteMetrics(inputs[0].collection, success.classAOps);
238
- return {
239
- entries: success.entries,
240
- currentEtag: success.currentEtag
241
- };
242
- }
243
- /**
244
- * One full commit attempt — the shared body of {@link commit} and
245
- * {@link commitBatch}. Reads `current.json`, walks the log, mints N
246
- * entries, PUTs content + indexes in parallel, PUTs log entries,
247
- * CAS-advances `current.json`. Returns the success payload + the
248
- * pre-commit fence epoch so the caller can run
249
- * {@link #verifyFenceUnchanged} outside the retry-catch arm.
250
- *
251
- * Retryable failures (a peer wrote our seq; a peer won the CAS on
252
- * `current.json`) throw `BaerlyError{code:"Conflict"}`. The caller
253
- * catches via {@link isPreconditionFailed} and either retries
254
- * (`commit`) or surfaces (`commitBatch`). Fence-bump and protocol-
255
- * invariant violations are NOT thrown from this helper — the fence
256
- * check lives in the caller so its `Conflict` propagates past the
257
- * retry-catch arm. Other thrown `BaerlyError`s (Internal,
258
- * InvalidResponse, NetworkError) bypass `isPreconditionFailed` and
259
- * propagate naturally.
260
- *
261
- * Crash safety invariant: content / index PUTs are awaited before
262
- * log PUTs, which are awaited before the CAS. A crashed mid-attempt
263
- * writer leaves orphan content / index entries (no log entry
264
- * references them) — the compactor sweeps the orphans. The
265
- * inverse — orphan log entry with missing content — is never
266
- * produced.
267
- */
268
- async #singleAttemptCommit(inputs, session, logPrefix, errorPrefix, adoptOwnSessionOnLogConflict) {
269
- const read = await readCurrentJson(this.#storage, this.#currentJsonKey) ?? await this.#provisionCurrentJson();
270
- const current = read.json;
271
- const baseEtag = read.etag;
272
- const expectedEpoch = current.writer_fence.epoch;
273
- if (this.#verifyLogIntegrityOnCommit) await this.#walkLog(logPrefix, logSeqStartOf(current), current.next_seq);
274
- const commitNowMs = Date.now();
275
- const commitTs = new Date(commitNowMs).toISOString();
276
- const lsnTimestamp = timestamp(commitNowMs);
277
- const entries = [];
278
- for (let i = 0; i < inputs.length; i++) {
279
- const input = inputs[i];
280
- const seq = current.next_seq + i;
281
- const entry = {
282
- lsn: `${lsnTimestamp}_${session}_${countKey(seq)}`,
283
- commit_ts: commitTs,
284
- op: input.op,
285
- collection: input.collection,
286
- doc_id: input.docId,
287
- session,
288
- seq,
289
- ...input.op !== "D" && input.body !== void 0 ? { after: input.body } : {},
290
- ...input.origin !== void 0 ? { origin: input.origin } : {}
291
- };
292
- entries.push(entry);
293
- }
294
- let contentPutCount = 0;
295
- let indexClassA = 0;
296
- const parallelPuts = [];
297
- const inBatchImage = /* @__PURE__ */ new Map();
298
- for (const input of inputs) {
299
- if (input.op !== "D" && input.body !== void 0) {
300
- contentPutCount++;
301
- const bytes = encodeJsonBytes(input.body);
302
- const contentKey = `${logPrefix}/content/${await versionFromContent(bytes)}.json`;
303
- parallelPuts.push(this.#storage.put(contentKey, bytes, {
304
- ifNoneMatch: "*",
305
- contentType: APPLICATION_JSON
306
- }).catch((error) => {
307
- this.#observe429(error, input.collection);
308
- if (isPreconditionFailed(error)) return;
309
- throw error;
310
- }));
311
- }
312
- if (this.#indexes.length > 0) {
313
- let newKeys = [];
314
- let staleKeys = [];
315
- if (input.op === "I") newKeys = allIndexKeysFor(logPrefix, this.#indexes, input.body, input.docId);
316
- else if (input.op === "U") {
317
- const preImage = inBatchImage.has(input.docId) ? inBatchImage.get(input.docId) : await this.#readPreImage(logPrefix, input.collection, input.docId, current.next_seq);
318
- const oldKeys = allIndexKeysFor(logPrefix, this.#indexes, preImage, input.docId);
319
- newKeys = allIndexKeysFor(logPrefix, this.#indexes, input.body, input.docId);
320
- const newSet = new Set(newKeys);
321
- staleKeys = oldKeys.filter((k) => !newSet.has(k));
322
- } else {
323
- const preImage = inBatchImage.has(input.docId) ? inBatchImage.get(input.docId) : await this.#readPreImage(logPrefix, input.collection, input.docId, current.next_seq);
324
- staleKeys = allIndexKeysFor(logPrefix, this.#indexes, preImage, input.docId);
325
- }
326
- for (const k of newKeys) parallelPuts.push(this.#storage.put(k, EMPTY_BODY, {
327
- ifNoneMatch: "*",
328
- contentType: APPLICATION_JSON
329
- }).catch((error) => {
330
- this.#observe429(error, input.collection);
331
- if (isPreconditionFailed(error)) return;
332
- throw error;
333
- }));
334
- for (const k of staleKeys) parallelPuts.push(this.#storage.delete(k));
335
- indexClassA += newKeys.length + staleKeys.length;
336
- if (newKeys.length + staleKeys.length > 0) ctxMetrics().histogram("db.write.index_ops_per_logical_write", newKeys.length + staleKeys.length, { collection: input.collection });
337
- inBatchImage.set(input.docId, input.op === "D" ? void 0 : input.body);
338
- }
339
- }
340
- await Promise.all(parallelPuts);
341
- const logPutOne = async (entry) => {
342
- const logEntryKey = `${logPrefix}/log/${entry.seq}.json`;
343
- const logBytes = encodeJsonBytes(entry);
344
- try {
345
- await this.#storage.put(logEntryKey, logBytes, {
346
- ifNoneMatch: "*",
347
- contentType: APPLICATION_JSON
348
- });
349
- return { ok: true };
350
- } catch (error) {
351
- this.#observe429(error, entry.collection);
352
- if (!isPreconditionFailed(error)) throw error;
353
- ctxMetrics().counter("db.r2.put.412_total", 1, {
354
- collection: entry.collection,
355
- step: "log-put"
356
- });
357
- return { ok: false };
358
- }
359
- };
360
- const logPutResults = await Promise.all(entries.map(logPutOne));
361
- let committedEntries = entries;
362
- const firstConflictIdx = logPutResults.findIndex((r) => !r.ok);
363
- if (firstConflictIdx !== -1) {
364
- const conflictedEntry = entries[firstConflictIdx];
365
- const logEntryKey = `${logPrefix}/log/${conflictedEntry.seq}.json`;
366
- if (adoptOwnSessionOnLogConflict) {
367
- const decision = tryAdoptOwnSessionLogEntry({
368
- self: conflictedEntry,
369
- existing: await readLogEntry(this.#storage, logEntryKey),
370
- batchSize: inputs.length
371
- });
372
- if (decision.adopt) committedEntries = [decision.entry];
373
- else throw new BaerlyError("Conflict", `${errorPrefix}: log entry already exists at ${logEntryKey}; ${decision.reason}`);
374
- } else throw new BaerlyError("Conflict", `${errorPrefix}: log entry already exists at ${logEntryKey}; peer wrote our seq`);
375
- }
376
- const batchLogBytes = committedEntries.reduce((sum, e) => sum + encodeJsonBytes(e).byteLength, 0);
377
- const next = {
378
- ...current,
379
- next_seq: current.next_seq + inputs.length,
380
- tail_bytes: current.tail_bytes + batchLogBytes
381
- };
382
- if (next.next_seq - current.next_seq !== inputs.length) throw new BaerlyError("Internal", `${errorPrefix}: GC-cadence invariant violated — next_seq advanced by ${next.next_seq - current.next_seq} but ${inputs.length} input(s) were committed on ${this.#currentJsonKey}`);
383
- const nextBody = encodeJsonBytes(next);
384
- const putOpts = {
385
- ifMatch: baseEtag,
386
- contentType: APPLICATION_JSON
387
- };
388
- let result;
389
- try {
390
- result = await this.#storage.put(this.#currentJsonKey, nextBody, putOpts);
391
- } catch (error) {
392
- this.#observe429(error, inputs[0].collection);
393
- if (isPreconditionFailed(error)) {
394
- ctxMetrics().counter("db.r2.put.412_total", 1, {
395
- collection: inputs[0].collection,
396
- step: "current-json-cas"
397
- });
398
- throw new BaerlyError("Conflict", `${errorPrefix}: CAS conflict on ${this.#currentJsonKey}`);
399
- }
400
- throw error;
401
- }
402
- const prevSeq = current.next_seq;
403
- const maint = getCurrentContext()?.maintenance;
404
- if (maint?.disabled !== true) {
405
- if (shouldFireMaintenance(next, prevSeq, maint?.options?.gcInterval ?? 4)) await (maint?.dispatch ?? dispatchInlineAwaited)(() => runBoundedMaintenance({
406
- storage: this.#storage,
407
- currentJsonKey: this.#currentJsonKey,
408
- prevSeq,
409
- ...maint?.maxFoldBytes !== void 0 && { maxFoldBytes: maint.maxFoldBytes }
410
- }, maint?.options).then(() => {}, () => {}));
411
- }
412
- const classAOps = contentPutCount + entries.length + indexClassA + 1;
413
- return {
414
- entries: committedEntries,
415
- currentEtag: result.etag,
416
- classAOps,
417
- expectedEpoch
418
- };
419
- }
420
- /**
421
- * Read the pre-image content body for a doc by walking the live
422
- * log backwards from `currentNextSeq` looking for the
423
- * most-recent I/U entry on this `(collection, docId)`. Returns
424
- * `undefined` when:
425
- *
426
- * - the doc's most-recent op was `D` (tombstone — no live body);
427
- * - no entry for this doc lives inside the visible log range
428
- * `[log_seq_start, currentNextSeq)` (a fresh-insert race or
429
- * the doc has been folded into the snapshot but is no longer
430
- * referenced).
431
- *
432
- * Used ONLY by the index-emission path on `U` / `D` to compute
433
- * the stale-key set. Bounded by `log_seq_start`: entries below
434
- * have been folded into the snapshot and may already be swept
435
- * off the bucket — a snapshot fold is the rebuild command's job
436
- * (see `./rebuild-index.ts`), not the writer's.
437
- *
438
- * **Cost note:** linear walk, O(snapshot lag). For day-one Phase
439
- * 8 collections (helpdesk-shape: 100s of docs, <10k log entries)
440
- * this is fine — the compactor folds the live tail every ~100
441
- * entries (`packages/server/src/compactor.ts`). A follow-up ticket
442
- * caches per-doc index head in `current.json` for O(1) lookup.
443
- * The bound is documented; out of scope here.
444
- */
445
- /**
446
- * Auto-create the per-collection `current.json` at
447
- * `this.#currentJsonKey` with a zero-state initial manifest. Called
448
- * by `#singleAttemptCommit` when the read returned `null` (fresh
449
- * bucket / fresh collection).
450
- *
451
- * Concurrency: `createCurrentJson` uses `If-None-Match: "*"`. If a
452
- * peer raced us and won, the storage layer surfaces `Conflict`; we
453
- * recover by re-reading the now-present manifest. We do NOT retry
454
- * the create — a single recover-via-read covers every race that
455
- * could have produced our null read.
456
- *
457
- * The seed shape (`snapshot: null`, `next_seq: 0`, `log_seq_start:
458
- * 0`, `writer_fence: { epoch: 0, owner: "", claimed_at: "" }`)
459
- * matches `ensureTable` and `baerly deploy`'s pre-warm path, so an
460
- * operator who pre-provisions and a writer who auto-creates land
461
- * on byte-identical bytes.
462
- *
463
- * @throws BaerlyError code="InvalidResponse" — the post-recover
464
- * re-read also returned null. Defensive guard; in practice
465
- * unreachable because the Conflict implies the key now exists.
466
- */
467
- async #provisionCurrentJson() {
468
- const initial = {
469
- schema_version: 2,
470
- snapshot: null,
471
- next_seq: 0,
472
- log_seq_start: 0,
473
- writer_fence: {
474
- epoch: 0,
475
- owner: "",
476
- claimed_at: ""
477
- },
478
- tail_bytes: 0,
479
- snapshot_bytes: 0,
480
- snapshot_rows: 0
481
- };
482
- try {
483
- return await createCurrentJson(this.#storage, this.#currentJsonKey, initial);
484
- } catch (error) {
485
- if (!(error instanceof BaerlyError) || error.code !== "Conflict") throw error;
486
- }
487
- const recovered = await readCurrentJson(this.#storage, this.#currentJsonKey);
488
- if (recovered === null) throw new BaerlyError("InvalidResponse", `Writer: current.json missing at ${this.#currentJsonKey} after auto-create raced and recover read returned null`);
489
- return recovered;
490
- }
491
- async #readPreImage(logPrefix, collection, docId, currentNextSeq) {
492
- for (let s = currentNextSeq - 1; s >= 0; s--) {
493
- const logKey = `${logPrefix}/log/${s}.json`;
494
- const got = await this.#storage.get(logKey);
495
- if (got === null) continue;
496
- let entry;
497
- try {
498
- entry = decodeJsonBytes(got.body);
499
- } catch {
500
- continue;
501
- }
502
- if (entry.collection !== collection || entry.doc_id !== docId) continue;
503
- if (entry.op === "D") return;
504
- if ((entry.op === "I" || entry.op === "U") && entry.after !== void 0) return entry.after;
505
- }
506
- }
507
- /**
508
- * Walk `log/<logSeqStart>.json` … `log/<nextSeq - 1>.json` with
509
- * bounded parallelism via `walkLogRange`. Materialised entries are
510
- * discarded — the walk's only job is to surface a hole or a
511
- * malformed body inside the visible range as
512
- * `BaerlyError{code:"Internal"}` / `"InvalidResponse"`.
513
- *
514
- * Gated on `verifyLogIntegrityOnCommit`; default off in production
515
- * because the per-doc fold already lives in the read path. See the
516
- * option's docstring for the rationale and the opt-in convention.
517
- *
518
- * `logSeqStart` is the boundary set by the compactor on
519
- * `current.json.log_seq_start`. Entries below it have been folded
520
- * into the snapshot and may have been swept off the bucket, so the
521
- * walk MUST NOT GET-require them.
522
- */
523
- async #walkLog(logPrefix, logSeqStart, nextSeq) {
524
- await walkLogRange(this.#storage, logPrefix, logSeqStart, nextSeq);
525
- }
526
- /**
527
- * Bump the `db.r2.put.429_total` counter when `err` looks like an
528
- * R2 prefix-partition rate-limit. Best-effort: detected via
529
- * `BaerlyError{code:"NetworkError"}` with a `429` token in the
530
- * message. Called from every PUT call site's catch arm; mutually
531
- * exclusive in practice with `isPreconditionFailed` (412 ≠ 429).
532
- */
533
- #observe429(err, collection) {
534
- if (is429(err)) ctxMetrics().counter("db.r2.put.429_total", 1, { collection });
535
- }
536
- /**
537
- * Emit the per-logical-write `class_a_ops_per_logical_write`
538
- * histogram shared by `commit` and `commitBatch`. `classAOps`
539
- * differs between the two paths (retry-budget vs batch shape),
540
- * so each caller computes it.
541
- */
542
- #emitWriteMetrics(collection, classAOps) {
543
- ctxMetrics().histogram("db.write.class_a_ops_per_logical_write", classAOps, { collection });
544
- }
545
- /**
546
- * Re-read `current.json` after a successful CAS and assert the
547
- * writer-fence epoch is still ours. The CAS only mutated `next_seq`
548
- * (the fence is preserved from `current`), so a post-write epoch
549
- * mismatch can only mean another writer claimed the fence between
550
- * our step-1 read and step-6 PUT — the exact split-brain
551
- * `WriterFence` prevents. Throws `Conflict`; no retry, since the
552
- * stale writer must defer to the new authority.
553
- *
554
- * The fence-verify GET is Class B and intentionally NOT counted in
555
- * `class_a_ops_per_logical_write`.
556
- */
557
- async #verifyFenceUnchanged(expectedEpoch, where) {
558
- const postRead = await readCurrentJson(this.#storage, this.#currentJsonKey);
559
- if (postRead === null) return;
560
- if (postRead.json.writer_fence.epoch === expectedEpoch) return;
561
- throw new BaerlyError("Conflict", `${where}: writer fence bumped from epoch ${expectedEpoch} to ${postRead.json.writer_fence.epoch} during commit on ${this.#currentJsonKey}; stale writer aborting`);
562
- }
563
- /**
564
- * Exponential backoff with full jitter. `25 * 2^(attempt-1)` capped
565
- * at 1500ms; uniform jitter in `[0, base)` keeps colliding peers
566
- * from re-racing in lockstep. Worst-case total wait across the
567
- * default 8 attempts is roughly 2.5 s — well under a typical
568
- * Worker request budget.
569
- */
570
- async #backoff(attempt) {
571
- const sleepMs = Math.min(this.#initialBackoffMs * 2 ** (attempt - 1), MAX_BACKOFF_MS) * this.#random();
572
- if (sleepMs <= 0) return;
573
- await new Promise((r) => setTimeout(r, sleepMs));
574
- }
575
- };
576
- /**
577
- * `true` when the underlying storage surfaced a `PreconditionFailed`
578
- * 412 (or its non-S3 equivalents). Every in-tree {@link Storage} impl
579
- * surfaces a lost CAS as `BaerlyError{code:"Conflict"}`.
580
- */
581
- const isPreconditionFailed = (err) => err instanceof BaerlyError && err.code === "Conflict";
582
- /**
583
- * `true` when the underlying storage surfaced an R2 prefix-partition
584
- * rate-limit. {@link S3HttpStorage} stamps the upstream HTTP status
585
- * onto `BaerlyError.cause` as `{ status }`; we discriminate on that
586
- * structured field rather than regex-matching the message.
587
- */
588
- const is429 = (err) => {
589
- if (!(err instanceof BaerlyError) || err.code !== "NetworkError") return false;
590
- return err.cause?.status === 429;
591
- };
592
- //#endregion
593
- //#region src/admin/restore.ts
594
- /**
595
- * `baerly admin restore` — bulk-import NDJSON into a fresh collection.
596
- *
597
- * Reads NDJSON from stdin, reconstructs each row, and calls
598
- * `Writer.commit({op:"I"})` per row into the target bucket.
599
- * Programmatic callers (today: tests + the export round-trip
600
- * integration test) can divert the input by passing
601
- * `{ streams: { stdin } }` to {@link runRestore}. Idempotent on a fresh
602
- * bucket: re-running on a half-completed restore **refuses** unless
603
- * `--force` is set (which bumps the writer fence and seeds a fresh
604
- * `current.json` first).
605
- *
606
- * Args:
607
- * --bucket Required. Target bucket URI.
608
- * --app Required (or via baerly.config.ts).
609
- * --tenant Required (or via baerly.config.ts).
610
- * --collection Required. Target collection name.
611
- * --force Truncate the target if it exists (fence-bump + reseed).
612
- * --json JSON envelope.
613
- *
614
- * stdin: NDJSON, one `{_id, ...}` row per line. Empty lines tolerated.
615
- * EOF terminates.
616
- *
617
- * Cost shape: 1 PUT current.json (initial) + N × (1 PUT log + 1 PUT
618
- * content + 1 PUT current.json CAS) = 3N + 1 Class A ops for N rows.
619
- *
620
- * Partial-restore semantics: a mid-stream Network / Internal / parse
621
- * error leaves the target in a partial state (some rows committed,
622
- * `current.json` advanced to wherever the last successful commit
623
- * landed). Re-run with `--force` to truncate, or hand-clean. The
624
- * malformed-line case (missing `_id`, bad JSON) fails fast on that
625
- * line; rows committed BEFORE it survive.
626
- *
627
- * Exit codes:
628
- * 0 — every row committed.
629
- * 1 — InvalidConfig (bad bucket URI, missing args).
630
- * 2 — Network / storage error mid-stream, or a malformed NDJSON
631
- * line (raised as a plain `Error`, not `BaerlyError`).
632
- * 3 — Conflict from a pre-existing target without --force, or from a
633
- * concurrent writer when --force was used.
634
- */
635
- const RESTORE_OWNER = "baerly-restore";
636
- const bundle = defineBaerlySubcommand({
637
- name: "admin.restore",
638
- meta: { description: "Bulk-import canonical NDJSON into a fresh collection." },
639
- args: {
640
- bucket: {
641
- type: "string",
642
- required: true,
643
- description: "Target bucket URI (s3://<bucket>[/<prefix>], file:///<abs>, memory://<bucket>)",
644
- valueHint: "bucket-uri"
645
- },
646
- app: {
647
- type: "string",
648
- required: false,
649
- description: "Application name segment (defaults to baerly.config.ts).",
650
- valueHint: "app"
651
- },
652
- tenant: {
653
- type: "string",
654
- required: false,
655
- description: "Tenant name segment (defaults to baerly.config.ts).",
656
- valueHint: "tenant"
657
- },
658
- collection: {
659
- type: "string",
660
- required: true,
661
- description: "Target collection name.",
662
- valueHint: "name"
663
- },
664
- force: {
665
- type: "boolean",
666
- description: "Truncate the target if it exists (bump fence + reseed current.json)."
667
- },
668
- json: {
669
- type: "boolean",
670
- description: "Emit a structured JSON envelope to stdout (success) or stderr (error)"
671
- }
672
- },
673
- handler: async (args, ctx) => {
674
- const bucket = await parseBucketUri(args.bucket);
675
- const { app, tenant } = await ctx.resolveAppTenant({
676
- app: args.app,
677
- tenant: args.tenant
678
- });
679
- const currentJsonKey = `${bucket.keyPrefix}app/${app}/tenant/${tenant}/manifests/${args.collection}/current.json`;
680
- const head = await readCurrentJson(bucket.storage, currentJsonKey);
681
- if (head !== null && args.force !== true) throw new BaerlyError("Conflict", `baerly admin restore: ${currentJsonKey} exists; pass --force to truncate`);
682
- if (head !== null) {
683
- const truncatedNext = head.json.next_seq;
684
- const reseeded = {
685
- schema_version: 2,
686
- snapshot: null,
687
- next_seq: truncatedNext,
688
- log_seq_start: truncatedNext,
689
- writer_fence: {
690
- epoch: head.json.writer_fence.epoch + 1,
691
- owner: RESTORE_OWNER,
692
- claimed_at: ""
693
- },
694
- tail_bytes: 0,
695
- snapshot_bytes: 0,
696
- snapshot_rows: 0
697
- };
698
- try {
699
- await bucket.storage.put(currentJsonKey, encodeJsonBytes(reseeded), {
700
- ifMatch: head.etag,
701
- contentType: "application/json"
702
- });
703
- } catch (error) {
704
- if (error instanceof BaerlyError) throw error;
705
- throw new BaerlyError("NetworkError", `baerly admin restore: failed to reseed current.json: ${error.message}`);
706
- }
707
- } else {
708
- const seed = {
709
- schema_version: 2,
710
- snapshot: null,
711
- next_seq: 0,
712
- log_seq_start: 0,
713
- writer_fence: {
714
- epoch: 0,
715
- owner: RESTORE_OWNER,
716
- claimed_at: ""
717
- },
718
- tail_bytes: 0,
719
- snapshot_bytes: 0,
720
- snapshot_rows: 0
721
- };
722
- await createCurrentJson(bucket.storage, currentJsonKey, seed);
723
- }
724
- const writer = new Writer({
725
- storage: bucket.storage,
726
- currentJsonKey
727
- });
728
- const rl = createInterface({
729
- input: ctx.streams?.stdin ?? process.stdin,
730
- crlfDelay: Infinity
731
- });
732
- let count = 0;
733
- let lineNo = 0;
734
- for await (const line of rl) {
735
- lineNo++;
736
- if (line.length === 0) continue;
737
- let parsed;
738
- try {
739
- parsed = JSON.parse(line);
740
- } catch (error) {
741
- throw new Error(`baerly admin restore: line ${lineNo} is not valid JSON: ${error.message}`, { cause: error });
742
- }
743
- if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(`baerly admin restore: line ${lineNo} is not a JSON object`);
744
- const row = parsed;
745
- const id = row["_id"];
746
- if (typeof id !== "string" || id.length === 0) throw new Error(`baerly admin restore: line ${lineNo} missing non-empty string _id`);
747
- const body = row;
748
- await writer.commit({
749
- op: "I",
750
- collection: args.collection,
751
- docId: id,
752
- body
753
- });
754
- count++;
755
- }
756
- emitSuccess({
757
- command: "admin.restore",
758
- status: "ok",
759
- collection: args.collection,
760
- restored: count
761
- });
762
- return 0;
763
- }
764
- });
765
- /** citty `defineCommand` block for `baerly admin restore`. */
766
- const restoreCmd = bundle.cmd;
767
- bundle.run;
768
- //#endregion
769
- export { restoreCmd };
770
-
771
- //# sourceMappingURL=restore-C6IcmLeA.js.map