@gusto/baerly-storage 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +3 -2
  2. package/dist/API.md +59 -7
  3. package/dist/{app-config-Dj_nGBfj.d.ts → app-config-C0Bbw_Z6.d.ts} +9 -9
  4. package/dist/app-config.d.ts +1 -1
  5. package/dist/auth.d.ts +1 -1
  6. package/dist/baerly.js +10 -10
  7. package/dist/baerly.js.map +1 -1
  8. package/dist/bytes-BXdamO3L.js +11 -0
  9. package/dist/bytes-BXdamO3L.js.map +1 -0
  10. package/dist/{client-CC5yVs1S.d.ts → client-DYZfoZb9.d.ts} +3 -3
  11. package/dist/client-react.d.ts +3 -3
  12. package/dist/client.d.ts +3 -3
  13. package/dist/cloudflare.d.ts +31 -4
  14. package/dist/cloudflare.js +72 -6
  15. package/dist/cloudflare.js.map +1 -1
  16. package/dist/{config-Bsg8wgwo.js → config-DVfDGyw1.js} +82 -2
  17. package/dist/config-DVfDGyw1.js.map +1 -0
  18. package/dist/{context-Oj7ffwD-.js → context-DRZ3tZ5l.js} +3 -2
  19. package/dist/{context-Oj7ffwD-.js.map → context-DRZ3tZ5l.js.map} +1 -1
  20. package/dist/{cost-B1WiZxlf.js → cost-CQt1YRkA.js} +3 -4
  21. package/dist/{cost-B1WiZxlf.js.map → cost-CQt1YRkA.js.map} +1 -1
  22. package/dist/{current-json-BkatBzF7.js → current-json-C-EyJqs0.js} +107 -5
  23. package/dist/current-json-C-EyJqs0.js.map +1 -0
  24. package/dist/{db-aRMK_0R0.d.ts → db-BM3y2JQw.d.ts} +3 -3
  25. package/dist/{deploy-BB-wqbhf.js → deploy-CrAbfgCq.js} +2 -2
  26. package/dist/{deploy-BB-wqbhf.js.map → deploy-CrAbfgCq.js.map} +1 -1
  27. package/dist/dev-vite.d.ts +3 -3
  28. package/dist/dev-vite.js +3 -3
  29. package/dist/dev.d.ts +1 -1
  30. package/dist/dev.js +1 -1
  31. package/dist/{doctor-B8h9GCN9.js → doctor-Db-hzPmE.js} +144 -7
  32. package/dist/doctor-Db-hzPmE.js.map +1 -0
  33. package/dist/{dump-BHo_sIMZ.js → dump-BYA55NLP.js} +3 -4
  34. package/dist/{dump-BHo_sIMZ.js.map → dump-BYA55NLP.js.map} +1 -1
  35. package/dist/errors-DOdL95kK.js.map +1 -1
  36. package/dist/{export-Dz2PZKSR.js → export-Bhbz90-z.js} +4 -5
  37. package/dist/{export-Dz2PZKSR.js.map → export-Bhbz90-z.js.map} +1 -1
  38. package/dist/{fsck-DEz9lyCy.js → fsck-CPPZWt9p.js} +5 -8
  39. package/dist/fsck-CPPZWt9p.js.map +1 -0
  40. package/dist/{http-DeJEeUcO.js → http-Cw8p6y5b.js} +5 -5
  41. package/dist/{http-DeJEeUcO.js.map → http-Cw8p6y5b.js.map} +1 -1
  42. package/dist/http.d.ts +3 -3
  43. package/dist/http.js +1 -1
  44. package/dist/{index-m6pY8iRU.d.ts → index-C3JAH1Yo.d.ts} +14 -2
  45. package/dist/{index-5lX_5NxZ.d.ts → index-DSvwEUeF.d.ts} +63 -54
  46. package/dist/{index-DPaQhAro.d.ts → index-DwDTWPSU.d.ts} +2 -2
  47. package/dist/{index-spCBsIwm.d.ts → index-cUY-36AN.d.ts} +3 -3
  48. package/dist/index.d.ts +5 -5
  49. package/dist/index.js +3 -3
  50. package/dist/{indexes-BIH73Uv4.d.ts → indexes-DAu2AkTW.d.ts} +2 -2
  51. package/dist/{inspect-x02aFWnh.js → inspect-Bdbzvi-_.js} +5 -6
  52. package/dist/{inspect-x02aFWnh.js.map → inspect-Bdbzvi-_.js.map} +1 -1
  53. package/dist/maintenance-BRrThz8V.d.ts +449 -0
  54. package/dist/maintenance-BvOv6BMM.js +1295 -0
  55. package/dist/maintenance-BvOv6BMM.js.map +1 -0
  56. package/dist/maintenance-ClzBC1Vv.js +1895 -0
  57. package/dist/maintenance-ClzBC1Vv.js.map +1 -0
  58. package/dist/maintenance.d.ts +2 -323
  59. package/dist/maintenance.js +2 -2
  60. package/dist/node.d.ts +20 -79
  61. package/dist/node.js +2 -2
  62. package/dist/{observability-2IDYRuZK.js → observability-CpOiEN0k.js} +2 -2
  63. package/dist/{observability-2IDYRuZK.js.map → observability-CpOiEN0k.js.map} +1 -1
  64. package/dist/observability.d.ts +2 -2
  65. package/dist/observability.js +2 -2
  66. package/dist/{plan-Cs17VqjU.js → plan-DOTl21vq.js} +3 -4
  67. package/dist/{plan-Cs17VqjU.js.map → plan-DOTl21vq.js.map} +1 -1
  68. package/dist/{query-DcSbFLJt.js → query-BmiQy328.js} +58 -26
  69. package/dist/query-BmiQy328.js.map +1 -0
  70. package/dist/{rebuild-index-DwXM0WfA.js → rebuild-index-DC_nR6wP.js} +4 -5
  71. package/dist/{rebuild-index-DwXM0WfA.js.map → rebuild-index-DC_nR6wP.js.map} +1 -1
  72. package/dist/{restore-CbKVMMpm.js → restore-C6IcmLeA.js} +44 -120
  73. package/dist/restore-C6IcmLeA.js.map +1 -0
  74. package/dist/{log-walk-DIru96UT.js → snapshot-SS8USDUF.js} +187 -20
  75. package/dist/snapshot-SS8USDUF.js.map +1 -0
  76. package/dist/{src-Cr9C7_tT.js → src-DU1sCMlr.js} +315 -278
  77. package/dist/src-DU1sCMlr.js.map +1 -0
  78. package/dist/{src-n0Te1r98.js → src-QQ7jDeL2.js} +25 -6
  79. package/dist/src-QQ7jDeL2.js.map +1 -0
  80. package/dist/{src-Chafbw8r.js → src-zhEAmmF7.js} +35 -4
  81. package/dist/src-zhEAmmF7.js.map +1 -0
  82. package/dist/{bucket-uri-BrDEYvzy.js → subcommand-DZcHcsYp.js} +222 -22
  83. package/dist/subcommand-DZcHcsYp.js.map +1 -0
  84. package/dist/{usage-RVUQMLop.js → usage-CLjirdD-.js} +4 -4
  85. package/dist/{usage-RVUQMLop.js.map → usage-CLjirdD-.js.map} +1 -1
  86. package/dist/{validate-DhH1lO8I.js → validate-DCq_0DIn.js} +2 -2
  87. package/dist/validate-DCq_0DIn.js.map +1 -0
  88. package/package.json +13 -6
  89. package/dist/bucket-uri-BrDEYvzy.js.map +0 -1
  90. package/dist/config-Bsg8wgwo.js.map +0 -1
  91. package/dist/current-json-BkatBzF7.js.map +0 -1
  92. package/dist/doctor-B8h9GCN9.js.map +0 -1
  93. package/dist/fsck-DEz9lyCy.js.map +0 -1
  94. package/dist/indexes-BXn7a06a.js +0 -297
  95. package/dist/indexes-BXn7a06a.js.map +0 -1
  96. package/dist/log-walk-DIru96UT.js.map +0 -1
  97. package/dist/maintenance-Bn6ul9aV.js +0 -758
  98. package/dist/maintenance-Bn6ul9aV.js.map +0 -1
  99. package/dist/query-DcSbFLJt.js.map +0 -1
  100. package/dist/rebuild-index-C5gTXryd.js +0 -119
  101. package/dist/rebuild-index-C5gTXryd.js.map +0 -1
  102. package/dist/restore-CbKVMMpm.js.map +0 -1
  103. package/dist/snapshot-D7C4_FSl.js +0 -71
  104. package/dist/snapshot-D7C4_FSl.js.map +0 -1
  105. package/dist/snapshot-z_N9hpTe.js +0 -917
  106. package/dist/snapshot-z_N9hpTe.js.map +0 -1
  107. package/dist/src-Chafbw8r.js.map +0 -1
  108. package/dist/src-Cr9C7_tT.js.map +0 -1
  109. package/dist/src-n0Te1r98.js.map +0 -1
  110. package/dist/subcommand-BWs6K7zb.js +0 -110
  111. package/dist/subcommand-BWs6K7zb.js.map +0 -1
  112. package/dist/validate-DhH1lO8I.js.map +0 -1
@@ -1,758 +0,0 @@
1
- import { f as decodeJsonBytes, n as logSeqStartOf, o as GC_MAX_PENDING_CANDIDATES, p as encodeJsonBytes, r as readCurrentJson, s as GC_PENDING_CONTENT_TYPE } from "./current-json-BkatBzF7.js";
2
- import { t as BaerlyError } from "./errors-DOdL95kK.js";
3
- import { d as snapshotHash, f as versionFromContent, i as foldLogEntriesOnto, l as indexKeyPrefix, n as loadSnapshotAsMap, o as walkLogRange, r as snapshotKey, s as allIndexKeysFor, t as encodeSnapshotBody } from "./snapshot-z_N9hpTe.js";
4
- import { i as noopMetricsRecorder, n as getCurrentContext } from "./context-Oj7ffwD-.js";
5
- //#region packages/protocol/src/coordination/gc-pending.ts
6
- /**
7
- * `gc/pending.json` — the per-collection GC candidate ledger. CAS-
8
- * protected; one per `(tenant, collection)` key — for example
9
- * `<tenant>/<collection>/gc/pending.json`. `runGc()` in
10
- * `packages/server/src/gc.ts` is the single writer that adds and
11
- * removes entries. Stays small in steady state because every
12
- * candidate is deleted once `due_at` passes.
13
- *
14
- * "Mark" appends candidates with a future `due_at`; "sweep" deletes
15
- * candidates whose `due_at` is in the past. The two phases run in
16
- * the same compactor pass — see `runGc()` in
17
- * `packages/server/src/gc.ts`.
18
- *
19
- * Lives next to {@link CurrentJson} in `coordination/` — same shape
20
- * of "CAS-protected small control object." Pure module; no Node
21
- * imports; Worker-bundleable.
22
- *
23
- * @see packages/server/src/gc.ts
24
- */
25
- /**
26
- * Read + parse `gc/pending.json` at `key`. Returns `null` on
27
- * not-found.
28
- *
29
- * @throws BaerlyError{code:"InvalidResponse"} — body is not valid
30
- * JSON, or `schema_version` is unknown, or the shape fails
31
- * the runtime guard.
32
- */
33
- const readGcPending = async (storage, key, opts) => {
34
- const got = await storage.get(key, opts);
35
- if (got === null) return null;
36
- let parsed;
37
- try {
38
- parsed = decodeJsonBytes(got.body);
39
- } catch (error) {
40
- throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: body is not valid JSON`, error);
41
- }
42
- return {
43
- json: assertGcPending(parsed, key),
44
- etag: got.etag
45
- };
46
- };
47
- /**
48
- * Create `gc/pending.json` if-and-only-if it does not exist (S3
49
- * `If-None-Match: "*"`). Use this once per collection at first GC
50
- * run; subsequent updates go through {@link casUpdateGcPending}.
51
- *
52
- * @throws BaerlyError{code:"Conflict"} — the key already exists.
53
- * @throws BaerlyError{code:"InvalidResponse"} — `initial` fails the
54
- * runtime shape guard.
55
- */
56
- const createGcPending = async (storage, key, initial, opts) => {
57
- assertGcPending(initial, key);
58
- const body = encodeJsonBytes(initial);
59
- const putOpts = {
60
- ifNoneMatch: "*",
61
- contentType: GC_PENDING_CONTENT_TYPE,
62
- ...opts?.signal !== void 0 && { signal: opts.signal }
63
- };
64
- let result;
65
- try {
66
- result = await storage.put(key, body, putOpts);
67
- } catch (error) {
68
- throw translateCasError(error, key);
69
- }
70
- return {
71
- json: initial,
72
- etag: result.etag
73
- };
74
- };
75
- /**
76
- * Read-modify-write `gc/pending.json` under CAS. The `mutator`
77
- * receives a deep clone of the current parsed JSON and returns the
78
- * new state. The new state is written with `If-Match: <currentEtag>`;
79
- * on conflict, throws `BaerlyError{code:"Conflict"}`.
80
- *
81
- * `mutator` MUST be deterministic — it may be called multiple times
82
- * if the caller wraps this in a retry loop, so it must not have side
83
- * effects.
84
- *
85
- * @throws BaerlyError{code:"Conflict"} — another writer landed a write
86
- * between this function's read and write.
87
- * @throws BaerlyError{code:"InvalidResponse"} — `key` does not exist
88
- * (use {@link createGcPending} instead) or body doesn't
89
- * parse / fails the shape guard.
90
- */
91
- const casUpdateGcPending = async (storage, key, mutator, opts) => {
92
- const existing = await readGcPending(storage, key, opts);
93
- if (existing === null) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key} does not exist; use createGcPending first`);
94
- const next = mutator(structuredClone(existing.json));
95
- assertGcPending(next, key);
96
- const body = encodeJsonBytes(next);
97
- const putOpts = {
98
- ifMatch: existing.etag,
99
- contentType: GC_PENDING_CONTENT_TYPE,
100
- ...opts?.signal !== void 0 && { signal: opts.signal }
101
- };
102
- let result;
103
- try {
104
- result = await storage.put(key, body, putOpts);
105
- } catch (error) {
106
- throw translateCasError(error, key);
107
- }
108
- return {
109
- json: next,
110
- etag: result.etag
111
- };
112
- };
113
- const VALID_REASONS = new Set([
114
- "stale-log",
115
- "orphan-snapshot",
116
- "orphan-content"
117
- ]);
118
- /**
119
- * Runtime guard for `parsed` to be a {@link GcPending}. Throws
120
- * `BaerlyError{code:"InvalidResponse"}` rather than coercing.
121
- */
122
- const assertGcPending = (parsed, key) => {
123
- if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: parsed body is not an object`);
124
- const r = parsed;
125
- if (r["schema_version"] !== 1) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: unsupported schema_version ${String(r["schema_version"])}; expected 1`);
126
- if (!Array.isArray(r["candidates"])) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates must be an array`);
127
- for (let i = 0; i < r["candidates"].length; i++) {
128
- const c = r["candidates"][i];
129
- if (c === null || typeof c !== "object") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}] is not an object`);
130
- const cr = c;
131
- if (typeof cr["key"] !== "string" || cr["key"].length === 0) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].key must be a non-empty string`);
132
- if (typeof cr["due_at"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].due_at must be a string`);
133
- if (typeof cr["reason"] !== "string" || !VALID_REASONS.has(cr["reason"])) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].reason must be one of stale-log|orphan-snapshot|orphan-content`);
134
- }
135
- if (typeof r["last_swept_at"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: last_swept_at must be a string`);
136
- return parsed;
137
- };
138
- /**
139
- * Wrap a storage-level error with `gc/pending.json at <key>`
140
- * location context. `Conflict` (the storage layer's CAS-lost signal)
141
- * is re-thrown with an annotated message; other `BaerlyError`s pass
142
- * through; non-`BaerlyError` falls through as `InvalidResponse`.
143
- */
144
- const translateCasError = (e, key) => {
145
- if (e instanceof BaerlyError && e.code === "Conflict") return new BaerlyError("Conflict", `gc/pending.json CAS lost at ${key}: ${e.message}`, e);
146
- if (e instanceof BaerlyError) return e;
147
- return new BaerlyError("InvalidResponse", `gc/pending.json write at ${key} failed: ${String(e)}`, e);
148
- };
149
- //#endregion
150
- //#region packages/server/src/compactor.ts
151
- /**
152
- * Compactor: fold a prefix of the live log into a content-
153
- * hashed snapshot file and CAS-advance `current.json` to swap the
154
- * pointer.
155
- *
156
- * The hard correctness requirement is **atomic snapshot generation**:
157
- * a compactor that crashes mid-write must not leave a corrupted body
158
- * that readers consume as truth. The design: every snapshot's filename
159
- * embeds the SHA-256 of its body. A crashed mid-PUT leaves a body that
160
- * doesn't match its own filename hash, and readers reject it as
161
- * "missing." The atomic moment is the CAS-swap of
162
- * `current.json.snapshot`, which is a single conditional PUT.
163
- *
164
- * Order of operations (matches `Writer.commit`'s manifest-first-
165
- * REVERSED ordering): PUT snapshot body → CAS-advance `current.json`.
166
- * A crash before the swap leaves an orphan snapshot file; the swap
167
- * succeeds iff our captured ETag still matches.
168
- *
169
- * Why level-prefixed chunked snapshots over the alternatives: a
170
- * single monolithic snapshot (every compaction rewrites one well-
171
- * known key) makes partial writes catastrophic and serializes every
172
- * writer on one key. WAL-checkpointing in the rqlite / Raft tradition
173
- * requires a consensus layer Baerly doesn't have. The Litestream-
174
- * style multi-level scheme — small fixed number of levels, chunked
175
- * files keyed by sequence range — keeps random GETs cheap on
176
- * S3-compatible storage and avoids LIST. Current ship is single-
177
- * level at L9 (one snapshot replaces the prior; no multi-level merge
178
- * yet); the key format is forward-compatible with future L0..L9
179
- * rolling merges without a wire change.
180
- *
181
- * @see ../../../docs/spec/sync-protocol.md
182
- */
183
- const ctxMetrics$1 = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
184
- const DEFAULT_MAX_PER_RUN = Number.MAX_SAFE_INTEGER;
185
- const DEFAULT_MIN_TO_COMPACT = 100;
186
- const APPLICATION_JSON$1 = "application/json";
187
- /**
188
- * Fold a prefix of the live log into a new snapshot and CAS-advance
189
- * `current.json`. Idempotent: a second run with no new writes is a
190
- * no-op (below-min-threshold).
191
- *
192
- * Single-attempt: on CAS conflict we return
193
- * `{written: false, skippedReason: "cas-lost", newSnapshotKey}` — the
194
- * caller (the cron handler invoking `runScheduledMaintenance` from
195
- * `./maintenance.ts`) decides whether to schedule another run. The
196
- * orphan snapshot file just written will be swept by `runGc()`.
197
- *
198
- * @throws BaerlyError code="InvalidResponse" — `current.json` is
199
- * present but malformed, or a snapshot body fails its schema /
200
- * collection cross-check.
201
- * @throws BaerlyError code="Internal" — protocol-invariant violation:
202
- * a log entry missing inside `[log_seq_start, foldEnd)`, or the
203
- * prior-snapshot pointer resolves to no body / a body whose hash
204
- * doesn't match its filename.
205
- *
206
- * @example
207
- * ```ts
208
- * import { compact } from "@gusto/baerly-storage";
209
- *
210
- * const res = await compact(
211
- * { storage, currentJsonKey: "app/x/tenant/t/manifests/tickets/current.json" },
212
- * { minEntriesToCompact: 100, maxEntriesPerRun: 40 },
213
- * );
214
- * if (res.written) console.log("snapshot landed at", res.newSnapshotKey);
215
- * ```
216
- */
217
- const compact = async (args, options = {}) => {
218
- const { storage, currentJsonKey } = args;
219
- const maxPerRun = options.maxEntriesPerRun ?? DEFAULT_MAX_PER_RUN;
220
- const minToCompact = options.minEntriesToCompact ?? DEFAULT_MIN_TO_COMPACT;
221
- const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
222
- const collectionName = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
223
- const read = await readCurrentJson(storage, currentJsonKey, options.signal !== void 0 ? { signal: options.signal } : void 0);
224
- if (read === null) return {
225
- written: false,
226
- skippedReason: "current-json-missing",
227
- previousSnapshotKey: null,
228
- logSeqStartBefore: 0,
229
- logSeqStartAfter: 0,
230
- entriesFolded: 0
231
- };
232
- const current = read.json;
233
- const baseEtag = read.etag;
234
- const logSeqStartBefore = logSeqStartOf(current);
235
- const nextSeq = current.next_seq;
236
- if (nextSeq - logSeqStartBefore < minToCompact) return {
237
- written: false,
238
- skippedReason: "below-min-threshold",
239
- previousSnapshotKey: current.snapshot,
240
- logSeqStartBefore,
241
- logSeqStartAfter: logSeqStartBefore,
242
- entriesFolded: 0
243
- };
244
- const foldEnd = Math.min(nextSeq, logSeqStartBefore + maxPerRun);
245
- const base = current.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, current.snapshot, collectionName, options.signal);
246
- const entries = await walkLogRange(storage, collectionPrefix, logSeqStartBefore, foldEnd, { signal: options.signal });
247
- for (const entry of entries) {
248
- if (entry.collection !== collectionName) continue;
249
- if (entry.doc_id === void 0) continue;
250
- switch (entry.op) {
251
- case "I":
252
- case "U":
253
- if (entry.new !== void 0) base.set(entry.doc_id, entry.new);
254
- break;
255
- case "D":
256
- base.delete(entry.doc_id);
257
- break;
258
- case "T":
259
- case "M": break;
260
- }
261
- }
262
- const bodyBytes = encodeSnapshotBody({
263
- schema_version: 1,
264
- min_seq: 0,
265
- max_seq: foldEnd,
266
- collection: collectionName,
267
- docs: Array.from(base.entries()).toSorted(([a], [b]) => {
268
- if (a < b) return -1;
269
- if (a > b) return 1;
270
- return 0;
271
- }).map(([id, body]) => ({
272
- _id: id,
273
- body
274
- }))
275
- });
276
- const newKey = snapshotKey(collectionPrefix, 0, foldEnd, await snapshotHash(bodyBytes));
277
- const putOpts = {
278
- contentType: APPLICATION_JSON$1,
279
- ...options.signal !== void 0 && { signal: options.signal }
280
- };
281
- await storage.put(newKey, bodyBytes, putOpts);
282
- const nextBody = encodeJsonBytes({
283
- ...current,
284
- snapshot: newKey,
285
- log_seq_start: foldEnd
286
- });
287
- const casOpts = {
288
- ifMatch: baseEtag,
289
- contentType: APPLICATION_JSON$1,
290
- ...options.signal !== void 0 && { signal: options.signal }
291
- };
292
- try {
293
- await storage.put(currentJsonKey, nextBody, casOpts);
294
- } catch (error) {
295
- if (isCasConflict(error)) return {
296
- written: false,
297
- skippedReason: "cas-lost",
298
- previousSnapshotKey: current.snapshot,
299
- newSnapshotKey: newKey,
300
- logSeqStartBefore,
301
- logSeqStartAfter: logSeqStartBefore,
302
- entriesFolded: foldEnd - logSeqStartBefore
303
- };
304
- throw error;
305
- }
306
- const entriesFolded = foldEnd - logSeqStartBefore;
307
- const metrics = ctxMetrics$1();
308
- metrics.histogram("db.compact.entries_folded", entriesFolded, { collection: collectionName });
309
- metrics.gauge("db.manifest.lag_window_depth", current.next_seq - foldEnd, { collection: collectionName });
310
- return {
311
- written: true,
312
- previousSnapshotKey: current.snapshot,
313
- newSnapshotKey: newKey,
314
- logSeqStartBefore,
315
- logSeqStartAfter: foldEnd,
316
- entriesFolded
317
- };
318
- };
319
- /**
320
- * `true` when an `If-Match` CAS guard lost. Every in-tree
321
- * {@link Storage} impl surfaces a lost CAS as
322
- * `BaerlyError{code:"Conflict"}`.
323
- */
324
- const isCasConflict = (err) => err instanceof BaerlyError && err.code === "Conflict";
325
- //#endregion
326
- //#region packages/server/src/gc.ts
327
- /**
328
- * GC mark + sweep.
329
- *
330
- * `runGc` is a single-pass garbage collector that:
331
- * 1. Reads `current.json` (and bootstraps `gc/pending.json` on first
332
- * run).
333
- * 2. Marks new orphan candidates by LISTing the three artifact
334
- * prefixes — log, snapshot, content — and classifying each key.
335
- * 3. Sweeps any already-pending candidate whose `due_at` has passed.
336
- * 4. CAS-writes the updated `gc/pending.json`.
337
- *
338
- * Two-phase by design: every candidate sits in `gc/pending.json` for a
339
- * grace period (default 7 days, see {@link GC_GRACE_PERIOD_MILLIS})
340
- * before it is deleted. The grace bounds the worst plausible
341
- * writer-retry window — a paused-process writer that resumes hours
342
- * later still finds its idempotency anchor on the bucket.
343
- *
344
- * Idempotent: same input bucket state ⇒ same `pending.json` output.
345
- * Unbounded by default — the run marks and sweeps the entire eligible
346
- * set in one pass. Callers on the Cloudflare 50-subrequest free-tier
347
- * budget opt INTO caps via the `CLOUDFLARE_FREE_TIER` profile's
348
- * `gc.maxMarksPerRun` / `maxSweepsPerRun` knobs (`InternalRunGcOptions`,
349
- * not on the public `RunGcOptions`).
350
- *
351
- * Three categories of orphan:
352
- * - `stale-log`: `<collectionPrefix>/log/<seq>.json` with
353
- * `seq < log_seq_start`. After `compact()` folds these into a
354
- * snapshot, they're unreferenced.
355
- * - `orphan-snapshot`: a `<collectionPrefix>/snapshot/L<n>/...` key not
356
- * equal to `current.snapshot`. Each compactor run replaces the
357
- * pointer; the prior file becomes unreferenced.
358
- * - `orphan-content`: `<collectionPrefix>/content/<sha>.json` whose
359
- * 32-hex truncated-SHA-256 hash is not in the live content-hash
360
- * set (computed by hashing every live `entry.new` post-image —
361
- * the same hash the writer's step 4 produces). Surfaces writer
362
- * crashes between the content PUT and the log-entry PUT.
363
- *
364
- * CAS-lost on `gc/pending.json` is non-fatal: the DELETEs already
365
- * issued are durable, so we return a successful result and the next
366
- * pass will pick up any work this pass lost.
367
- */
368
- const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
369
- const DEFAULT_MAX_MARKS = Number.MAX_SAFE_INTEGER;
370
- const DEFAULT_MAX_SWEEPS = Number.MAX_SAFE_INTEGER;
371
- /**
372
- * Single GC pass — mark new orphans, sweep due-elapsed candidates,
373
- * persist via CAS-update on `gc/pending.json`.
374
- *
375
- * Returns immediately if `current.json` is missing — there's nothing
376
- * to do until a writer has bootstrapped the collection.
377
- *
378
- * @example
379
- * ```ts
380
- * import { runGc } from "@gusto/baerly-storage";
381
- *
382
- * const r = await runGc({ storage, currentJsonKey });
383
- * console.log(`marked ${r.marked.stale_log} stale logs, swept ${r.swept}`);
384
- * ```
385
- */
386
- const runGc = async (args, options = {}) => {
387
- const { storage, currentJsonKey } = args;
388
- const internal = options;
389
- const grace = internal.graceMillis ?? 6048e5;
390
- const maxMarks = internal.maxMarksPerRun ?? DEFAULT_MAX_MARKS;
391
- const maxSweeps = internal.maxSweepsPerRun ?? DEFAULT_MAX_SWEEPS;
392
- const now = internal.now ?? (() => /* @__PURE__ */ new Date());
393
- const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
394
- const collectionName = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
395
- const gcPendingKey = `${collectionPrefix}/gc/pending.json`;
396
- const signal = options.signal;
397
- const signalOpts = signal !== void 0 ? { signal } : void 0;
398
- const cur = await readCurrentJson(storage, currentJsonKey, signalOpts);
399
- if (cur === null) return {
400
- marked: {
401
- stale_log: 0,
402
- orphan_snapshot: 0,
403
- orphan_content: 0
404
- },
405
- swept: 0,
406
- pendingDepth: 0
407
- };
408
- const current = cur.json;
409
- const logSeqStart = logSeqStartOf(current);
410
- let pending = await readGcPending(storage, gcPendingKey, signalOpts);
411
- if (pending === null) {
412
- const initial = {
413
- schema_version: 1,
414
- candidates: [],
415
- last_swept_at: ""
416
- };
417
- try {
418
- pending = await createGcPending(storage, gcPendingKey, initial, signalOpts);
419
- } catch (error) {
420
- if (error instanceof BaerlyError && error.code === "Conflict") {
421
- pending = await readGcPending(storage, gcPendingKey, signalOpts);
422
- if (pending === null) throw error;
423
- } else throw error;
424
- }
425
- }
426
- const known = new Set(pending.json.candidates.map((c) => c.key));
427
- const newCandidates = [];
428
- let markedStaleLog = 0;
429
- if (logSeqStart > 0) for await (const entry of listBounded(storage, `${collectionPrefix}/log/`, maxMarks, signal)) {
430
- if (markedStaleLog >= maxMarks) break;
431
- const seq = parseSeqFromLogKey(entry.key);
432
- if (seq === null || seq >= logSeqStart) continue;
433
- if (known.has(entry.key)) continue;
434
- newCandidates.push({
435
- key: entry.key,
436
- due_at: computeDueAt(entry, now, grace),
437
- reason: "stale-log"
438
- });
439
- markedStaleLog++;
440
- }
441
- let markedOrphanSnapshot = 0;
442
- for await (const entry of listBounded(storage, `${collectionPrefix}/snapshot/`, maxMarks, signal)) {
443
- if (markedOrphanSnapshot >= maxMarks) break;
444
- if (entry.key === current.snapshot) continue;
445
- if (known.has(entry.key)) continue;
446
- newCandidates.push({
447
- key: entry.key,
448
- due_at: computeDueAt(entry, now, grace),
449
- reason: "orphan-snapshot"
450
- });
451
- markedOrphanSnapshot++;
452
- }
453
- const liveHashes = await collectLiveContentHashes(storage, collectionPrefix, collectionName, current, logSeqStart, signal);
454
- let markedOrphanContent = 0;
455
- for await (const entry of listBounded(storage, `${collectionPrefix}/content/`, maxMarks, signal)) {
456
- if (markedOrphanContent >= maxMarks) break;
457
- const hash = parseHashFromContentKey(entry.key);
458
- if (hash === null || liveHashes.has(hash)) continue;
459
- if (known.has(entry.key)) continue;
460
- newCandidates.push({
461
- key: entry.key,
462
- due_at: computeDueAt(entry, now, grace),
463
- reason: "orphan-content"
464
- });
465
- markedOrphanContent++;
466
- }
467
- const nowMs = now().getTime();
468
- const sweepCandidates = [...pending.json.candidates, ...newCandidates];
469
- const toSweep = [];
470
- const remaining = [];
471
- for (const cand of sweepCandidates) if (toSweep.length < maxSweeps && Date.parse(cand.due_at) <= nowMs) toSweep.push(cand);
472
- else remaining.push(cand);
473
- await Promise.all(toSweep.map((c) => storage.delete(c.key, signalOpts)));
474
- const merged = remaining.slice(0, GC_MAX_PENDING_CANDIDATES);
475
- const lastSweptAt = toSweep.length > 0 ? now().toISOString() : pending.json.last_swept_at;
476
- const markedSummary = {
477
- stale_log: markedStaleLog,
478
- orphan_snapshot: markedOrphanSnapshot,
479
- orphan_content: markedOrphanContent
480
- };
481
- let pendingDepth;
482
- try {
483
- pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, () => ({
484
- schema_version: 1,
485
- candidates: merged,
486
- last_swept_at: lastSweptAt
487
- }), signalOpts)).json.candidates.length;
488
- } catch (error) {
489
- if (error instanceof BaerlyError && error.code === "Conflict") pendingDepth = remaining.length;
490
- else throw error;
491
- }
492
- const labels = { collection: collectionName };
493
- const metrics = ctxMetrics();
494
- metrics.gauge("db.orphan.candidate_count", pendingDepth, labels);
495
- metrics.gauge("db.gc.entries_swept_per_second", toSweep.length, labels);
496
- if (toSweep.length > 0) {
497
- const byReason = /* @__PURE__ */ new Map();
498
- for (const c of toSweep) byReason.set(c.reason, (byReason.get(c.reason) ?? 0) + 1);
499
- for (const [reason, count] of byReason) metrics.counter("db.gc.swept_total", count, {
500
- collection: collectionName,
501
- reason
502
- });
503
- }
504
- return {
505
- marked: markedSummary,
506
- swept: toSweep.length,
507
- pendingDepth
508
- };
509
- };
510
- /**
511
- * `Storage.list` with a hard ceiling — bounds the I/O cost of a
512
- * single pass when a runaway prefix has accumulated many keys.
513
- */
514
- const listBounded = async function* (storage, prefix, cap, signal) {
515
- let yielded = 0;
516
- const opts = signal !== void 0 ? { signal } : void 0;
517
- for await (const entry of storage.list(prefix, opts)) {
518
- if (yielded >= cap) return;
519
- yield entry;
520
- yielded++;
521
- }
522
- };
523
- /**
524
- * Parse `<...>/log/<seq>.json` and return `seq`. Returns `null` on
525
- * any shape that doesn't look like a log entry key — defensively
526
- * tolerates an unrelated key under the log prefix.
527
- */
528
- const parseSeqFromLogKey = (key) => {
529
- const match = /\/log\/(\d+)\.json$/.exec(key);
530
- if (match === null) return null;
531
- const n = Number.parseInt(match[1], 10);
532
- return Number.isFinite(n) && n >= 0 ? n : null;
533
- };
534
- /**
535
- * Parse `<...>/content/<sha-32>.json` and return the 32-hex hash.
536
- * Returns `null` on any shape that doesn't match the writer's
537
- * `versionFromContent`-produced key format.
538
- */
539
- const parseHashFromContentKey = (key) => {
540
- const match = /\/content\/([0-9a-f]{32})\.json$/.exec(key);
541
- return match === null ? null : match[1];
542
- };
543
- /**
544
- * Anchor `due_at` to `lastModified` when the storage adapter
545
- * surfaces it; fall back to the injected `now` so a Storage impl
546
- * without a server clock still GCs after `grace` from the mark
547
- * moment.
548
- */
549
- const computeDueAt = (entry, now, graceMs) => {
550
- const base = entry.lastModified ?? now();
551
- return new Date(base.getTime() + graceMs).toISOString();
552
- };
553
- /**
554
- * Build the live content-hash set. The set covers every live
555
- * post-image: every `entry.new` in `[logSeqStart, next_seq)` plus
556
- * every row body in the current snapshot.
557
- *
558
- * A snapshot read that throws (corrupt body, hash mismatch) is
559
- * tolerated — the worst-case effect is missing some live hashes,
560
- * which could mark a live content blob as orphan. The grace period
561
- * absorbs the false-positive: a retry that re-PUTs the same hash
562
- * recreates the key before the sweep deletes it.
563
- */
564
- const collectLiveContentHashes = async (storage, collectionPrefix, collectionName, current, logSeqStart, signal) => {
565
- const hashes = /* @__PURE__ */ new Set();
566
- const getOpts = signal !== void 0 ? { signal } : void 0;
567
- const logReads = [];
568
- for (let s = logSeqStart; s < current.next_seq; s++) logReads.push((async () => {
569
- const got = await storage.get(`${collectionPrefix}/log/${s}.json`, getOpts);
570
- if (got === null) return;
571
- let entry;
572
- try {
573
- entry = decodeJsonBytes(got.body);
574
- } catch {
575
- return;
576
- }
577
- if (entry.new === void 0) return;
578
- const bodyBytes = encodeJsonBytes(entry.new);
579
- hashes.add(await versionFromContent(bodyBytes));
580
- })());
581
- await Promise.all(logReads);
582
- if (current.snapshot !== null) try {
583
- const map = await loadSnapshotAsMap(storage, current.snapshot, collectionName, signal);
584
- const rowReads = [];
585
- for (const body of map.values()) rowReads.push((async () => {
586
- const bytes = encodeJsonBytes(body);
587
- hashes.add(await versionFromContent(bytes));
588
- })());
589
- await Promise.all(rowReads);
590
- } catch {}
591
- return hashes;
592
- };
593
- //#endregion
594
- //#region packages/server/src/rebuild-index.ts
595
- /**
596
- * Idempotent reconciliation for one secondary index.
597
- *
598
- * {@link rebuildIndex} folds the live `(snapshot, log_tail)` of a
599
- * collection into the doc set the writer's index emission would
600
- * have produced, then PUTs every missing index key and DELETEs every
601
- * orphaned one. Same-key zero-byte PUTs are no-ops (the entries are
602
- * content-addressed by composition); same-key DELETEs are
603
- * contractually idempotent.
604
- *
605
- * Invariants:
606
- *
607
- * - **Re-running on a healthy index is a no-op.** `added = 0`,
608
- * `removed = 0`.
609
- * - **Re-running after a crashed mid-commit** reconciles toward
610
- * the live `current.json` view — orphans deleted, missing keys
611
- * PUT.
612
- * - **Re-running while writes are in flight** is safe up to a
613
- * race window: a write that lands during the rebuild may
614
- * produce a transient "orphan" the rebuild sweeps, which the
615
- * next live write then re-PUTs. The race converges on the next
616
- * commit.
617
- *
618
- * The function reads the same fold the writer / reader use
619
- * (snapshot via {@link loadSnapshotAsMap}; log tail by direct
620
- * `get(<logPrefix>/log/<seq>.json)`) so it MUST agree with the
621
- * writer on the live doc set by construction.
622
- *
623
- * @see ./indexes.ts — `IndexDefinition`, `allIndexKeysFor`,
624
- * `indexKeyPrefix`.
625
- * @see ./writer.ts — the fence-time emission path this
626
- * command reconciles.
627
- */
628
- const EMPTY_BODY = new Uint8Array(0);
629
- const APPLICATION_JSON = "application/json";
630
- /**
631
- * Idempotently reconcile one secondary index for one collection.
632
- *
633
- * @param storage Storage handle (any in-tree `Storage`).
634
- * @param currentJsonKey The collection's `current.json` key.
635
- * @param def The index to reconcile. Single-field today;
636
- * composite definitions are accepted by the
637
- * key encoder but the read path only consults
638
- * single-field keys.
639
- * @param opts Optional knobs — see {@link RebuildIndexOptions}.
640
- * Strictly additive: existing positional-arg
641
- * callers compile and behave unchanged when
642
- * omitted.
643
- *
644
- * @throws BaerlyError code="InvalidResponse" — `current.json` missing.
645
- * @throws BaerlyError code="Internal" — protocol-invariant
646
- * violation (snapshot hash mismatch, missing log entry inside
647
- * `[log_seq_start, next_seq)`).
648
- */
649
- const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
650
- const read = await readCurrentJson(storage, currentJsonKey);
651
- if (read === null) throw new BaerlyError("InvalidResponse", `rebuildIndex: current.json missing at ${JSON.stringify(currentJsonKey)}`);
652
- const lastSlash = currentJsonKey.lastIndexOf("/");
653
- if (lastSlash < 0) throw new BaerlyError("InvalidResponse", `rebuildIndex: currentJsonKey ${JSON.stringify(currentJsonKey)} must contain "/"`);
654
- const collectionPrefix = currentJsonKey.slice(0, lastSlash);
655
- const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
656
- const logSeqStart = logSeqStartOf(read.json);
657
- const nextSeq = read.json.next_seq;
658
- const live = read.json.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, read.json.snapshot, collection);
659
- foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart, nextSeq), { collection });
660
- const expected = /* @__PURE__ */ new Set();
661
- for (const [docId, body] of live) for (const k of allIndexKeysFor(collectionPrefix, [def], body, docId)) expected.add(k);
662
- const actual = /* @__PURE__ */ new Set();
663
- const listOpts = {};
664
- if (opts.signal !== void 0) listOpts.signal = opts.signal;
665
- for await (const entry of storage.list(indexKeyPrefix(collectionPrefix, def.name), listOpts)) actual.add(entry.key);
666
- const dryRun = opts.dryRun === true;
667
- let added = 0;
668
- let removed = 0;
669
- let kept = 0;
670
- for (const k of expected) {
671
- if (actual.has(k)) {
672
- kept += 1;
673
- continue;
674
- }
675
- if (dryRun) {
676
- added += 1;
677
- continue;
678
- }
679
- try {
680
- await storage.put(k, EMPTY_BODY, {
681
- ifNoneMatch: "*",
682
- contentType: APPLICATION_JSON
683
- });
684
- added += 1;
685
- } catch (error) {
686
- if (error instanceof BaerlyError && error.code === "Conflict") kept += 1;
687
- else throw error;
688
- }
689
- }
690
- for (const k of actual) {
691
- if (expected.has(k)) continue;
692
- if (dryRun) {
693
- removed += 1;
694
- continue;
695
- }
696
- await storage.delete(k);
697
- removed += 1;
698
- }
699
- return {
700
- added,
701
- removed,
702
- kept
703
- };
704
- };
705
- //#endregion
706
- //#region packages/server/src/maintenance.ts
707
- /**
708
- * Single maintenance pass for one collection. Runs `compact()` then
709
- * `runGc()` (in that order; the compactor's advance of
710
- * `log_seq_start` produces the stale-log candidates the GC then
711
- * marks). Callers wanting a single phase per tick invoke
712
- * `compact()` or `runGc()` directly instead.
713
- *
714
- * Errors propagate — the caller's cron handler is responsible for
715
- * logging them. The Cloudflare runtime ships uncaught Worker errors
716
- * to the dashboard; Node operators wrap their `node-cron` callbacks
717
- * themselves.
718
- *
719
- * @example
720
- * ```ts
721
- * import { runScheduledMaintenance, CLOUDFLARE_FREE_TIER } from "@gusto/baerly-storage/maintenance";
722
- *
723
- * // Node (unbounded — defaults fold the entire live tail):
724
- * const res = await runScheduledMaintenance(
725
- * { storage, currentJsonKey: "app/x/tenant/t/manifests/tickets/current.json" },
726
- * );
727
- * console.log("compacted:", res.compact.entriesFolded, "swept:", res.gc.swept);
728
- *
729
- * // Cloudflare free tier (50-subrequest cap, single phase per tick):
730
- * await runScheduledMaintenance({ storage, currentJsonKey }, CLOUDFLARE_FREE_TIER);
731
- * ```
732
- */
733
- const runScheduledMaintenance = async (args, options = {}) => {
734
- return {
735
- compact: await compact(args, {
736
- ...options.compact,
737
- ...options.signal !== void 0 && { signal: options.signal }
738
- }),
739
- gc: await runGc(args, {
740
- ...options.gc,
741
- ...options.signal !== void 0 && { signal: options.signal }
742
- })
743
- };
744
- };
745
- const CLOUDFLARE_FREE_TIER = {
746
- compact: {
747
- maxEntriesPerRun: 20,
748
- minEntriesToCompact: 50
749
- },
750
- gc: {
751
- maxMarksPerRun: 20,
752
- maxSweepsPerRun: 10
753
- }
754
- };
755
- //#endregion
756
- export { compact as a, runGc as i, runScheduledMaintenance as n, rebuildIndex as r, CLOUDFLARE_FREE_TIER as t };
757
-
758
- //# sourceMappingURL=maintenance-Bn6ul9aV.js.map