@gusto/baerly-storage 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +29 -0
  3. package/README.md +176 -70
  4. package/dist/API.md +213 -139
  5. package/dist/CHANGELOG.md +434 -0
  6. package/dist/RECIPES.md +125 -0
  7. package/dist/THIRD-PARTY-LICENSES.txt +238 -0
  8. package/dist/{app-config-C0Bbw_Z6.d.ts → app-config-DyiFK3_k.d.ts} +55 -11
  9. package/dist/app-config.d.ts +1 -1
  10. package/dist/auth.d.ts +1 -1
  11. package/dist/auth.js +301 -301
  12. package/dist/auth.js.map +1 -1
  13. package/dist/aws4fetch.esm-p7eEzLCs.js +1702 -0
  14. package/dist/aws4fetch.esm-p7eEzLCs.js.map +1 -0
  15. package/dist/baerly.js +35 -15
  16. package/dist/baerly.js.map +1 -1
  17. package/dist/baerly.spec.json +342 -0
  18. package/dist/bytes-BXdamO3L.js.map +1 -1
  19. package/dist/{client-DYZfoZb9.d.ts → client-Fe8CwB73.d.ts} +10 -10
  20. package/dist/client-react.d.ts +3 -3
  21. package/dist/client-react.js +4 -4
  22. package/dist/client-react.js.map +1 -1
  23. package/dist/client.d.ts +4 -4
  24. package/dist/client.js +3 -3
  25. package/dist/client.js.map +1 -1
  26. package/dist/{cloudflare-CINJPKCW.js → cloudflare-DzKBrUlg.js} +178 -2
  27. package/dist/cloudflare-DzKBrUlg.js.map +1 -0
  28. package/dist/cloudflare.d.ts +49 -6
  29. package/dist/cloudflare.js +59 -26
  30. package/dist/cloudflare.js.map +1 -1
  31. package/dist/config-1tv0-Yzm.js +2693 -0
  32. package/dist/config-1tv0-Yzm.js.map +1 -0
  33. package/dist/constants-CfeMWQWk.js +222 -0
  34. package/dist/constants-CfeMWQWk.js.map +1 -0
  35. package/dist/context-DRZ3tZ5l.js.map +1 -1
  36. package/dist/{cost-CQt1YRkA.js → cost-DSMIVqzL.js} +43 -42
  37. package/dist/cost-DSMIVqzL.js.map +1 -0
  38. package/dist/current-json-C6nhPOm1.js +190 -0
  39. package/dist/current-json-C6nhPOm1.js.map +1 -0
  40. package/dist/{db-BM3y2JQw.d.ts → db-DsPkOVEH.d.ts} +26 -108
  41. package/dist/{deploy-CrAbfgCq.js → deploy-DonTr-CV.js} +12 -5
  42. package/dist/deploy-DonTr-CV.js.map +1 -0
  43. package/dist/dev-vite.d.ts +11 -7
  44. package/dist/dev-vite.js +7 -6
  45. package/dist/dev-vite.js.map +1 -1
  46. package/dist/dev.d.ts +1 -1
  47. package/dist/dev.js +1 -1
  48. package/dist/{doctor-Db-hzPmE.js → doctor-DSIyCDg_.js} +14 -136
  49. package/dist/doctor-DSIyCDg_.js.map +1 -0
  50. package/dist/{dump-BYA55NLP.js → dump-Bh5k26VQ.js} +10 -31
  51. package/dist/dump-Bh5k26VQ.js.map +1 -0
  52. package/dist/{errors-DOdL95kK.js → errors-Dfa3p78c.js} +25 -5
  53. package/dist/errors-Dfa3p78c.js.map +1 -0
  54. package/dist/{export-Bhbz90-z.js → export-DtF5tDIM.js} +11 -32
  55. package/dist/{export-Bhbz90-z.js.map → export-DtF5tDIM.js.map} +1 -1
  56. package/dist/{fsck-CPPZWt9p.js → fsck-D_djMO5D.js} +28 -39
  57. package/dist/fsck-D_djMO5D.js.map +1 -0
  58. package/dist/{http-Cw8p6y5b.js → http-DxM9MflO.js} +151 -79
  59. package/dist/http-DxM9MflO.js.map +1 -0
  60. package/dist/http.d.ts +42 -26
  61. package/dist/http.js +3 -2
  62. package/dist/{index-DwDTWPSU.d.ts → index-0rZcMBwy.d.ts} +10 -7
  63. package/dist/{index-C3JAH1Yo.d.ts → index-B6oexfOB.d.ts} +9 -9
  64. package/dist/{index-cUY-36AN.d.ts → index-BNRh23K9.d.ts} +21 -5
  65. package/dist/{index-DSvwEUeF.d.ts → index-BYk9oCNf.d.ts} +240 -202
  66. package/dist/index.d.ts +6 -6
  67. package/dist/index.js +5 -5
  68. package/dist/{indexes-DAu2AkTW.d.ts → indexes-Dn9hbGQG.d.ts} +2 -2
  69. package/dist/{inspect-Bdbzvi-_.js → inspect-6WZ-pC7-.js} +21 -36
  70. package/dist/inspect-6WZ-pC7-.js.map +1 -0
  71. package/dist/key-Di6yyxFu.js +35 -0
  72. package/dist/key-Di6yyxFu.js.map +1 -0
  73. package/dist/{snapshot-SS8USDUF.js → log-tail-D98U1JBo.js} +179 -84
  74. package/dist/log-tail-D98U1JBo.js.map +1 -0
  75. package/dist/{maintenance-ClzBC1Vv.js → maintenance-B4Dki6Ig.js} +438 -176
  76. package/dist/maintenance-B4Dki6Ig.js.map +1 -0
  77. package/dist/{maintenance-BRrThz8V.d.ts → maintenance-B_e0T99L.d.ts} +114 -26
  78. package/dist/{maintenance-BvOv6BMM.js → maintenance-C7coQp4c.js} +217 -94
  79. package/dist/maintenance-C7coQp4c.js.map +1 -0
  80. package/dist/maintenance.d.ts +3 -2
  81. package/dist/maintenance.js +3 -2
  82. package/dist/node.d.ts +176 -68
  83. package/dist/node.js +3 -2
  84. package/dist/{normalize-XWY25_sG.js → normalize-rjnZLwS4.js} +2 -2
  85. package/dist/normalize-rjnZLwS4.js.map +1 -0
  86. package/dist/{observability-CpOiEN0k.js → observability-wzs3-BL_.js} +279 -109
  87. package/dist/observability-wzs3-BL_.js.map +1 -0
  88. package/dist/observability.d.ts +3 -3
  89. package/dist/observability.js +1 -1
  90. package/dist/{plan-DOTl21vq.js → plan-a7PJHBCy.js} +7 -4
  91. package/dist/plan-a7PJHBCy.js.map +1 -0
  92. package/dist/{query-BmiQy328.js → query-rAtXNQQB.js} +1063 -1202
  93. package/dist/query-rAtXNQQB.js.map +1 -0
  94. package/dist/{rebuild-index-DC_nR6wP.js → rebuild-index-BVzQSZlm.js} +10 -32
  95. package/dist/rebuild-index-BVzQSZlm.js.map +1 -0
  96. package/dist/{request-BEF_OvMh.js → request-u0M51mhJ.js} +3 -3
  97. package/dist/{request-BEF_OvMh.js.map → request-u0M51mhJ.js.map} +1 -1
  98. package/dist/restore-R-4rbciV.js +778 -0
  99. package/dist/restore-R-4rbciV.js.map +1 -0
  100. package/dist/runtime-spec-qL3_lTTU.js +391 -0
  101. package/dist/runtime-spec-qL3_lTTU.js.map +1 -0
  102. package/dist/s3-http-B-z8th6O.d.ts +60 -0
  103. package/dist/s3.d.ts +47 -0
  104. package/dist/s3.js +45 -0
  105. package/dist/s3.js.map +1 -0
  106. package/dist/{shared-secret-BJdSwpgw.js → shared-secret-DgEicd3x.js} +13 -5
  107. package/dist/shared-secret-DgEicd3x.js.map +1 -0
  108. package/dist/src-BG20LZs7.js +2271 -0
  109. package/dist/src-BG20LZs7.js.map +1 -0
  110. package/dist/{src-zhEAmmF7.js → src-CYHFDrKz.js} +82 -99
  111. package/dist/src-CYHFDrKz.js.map +1 -0
  112. package/dist/{src-QQ7jDeL2.js → src-DYZ4H1aF.js} +44 -23
  113. package/dist/src-DYZ4H1aF.js.map +1 -0
  114. package/dist/subcommand-DoDA-K7d.js +197 -0
  115. package/dist/subcommand-DoDA-K7d.js.map +1 -0
  116. package/dist/time-DUqvFMHq.js +119 -0
  117. package/dist/time-DUqvFMHq.js.map +1 -0
  118. package/dist/{usage-CLjirdD-.js → usage-DBV1eQHC.js} +7 -22
  119. package/dist/usage-DBV1eQHC.js.map +1 -0
  120. package/dist/{validate-DCq_0DIn.js → validate-DtljsCh1.js} +51 -51
  121. package/dist/{validate-DCq_0DIn.js.map → validate-DtljsCh1.js.map} +1 -1
  122. package/dist/wire-izfGZDvr.js.map +1 -1
  123. package/package.json +77 -42
  124. package/dist/cloudflare-CINJPKCW.js.map +0 -1
  125. package/dist/config-DVfDGyw1.js +0 -278
  126. package/dist/config-DVfDGyw1.js.map +0 -1
  127. package/dist/cost-CQt1YRkA.js.map +0 -1
  128. package/dist/current-json-C-EyJqs0.js +0 -354
  129. package/dist/current-json-C-EyJqs0.js.map +0 -1
  130. package/dist/deploy-CrAbfgCq.js.map +0 -1
  131. package/dist/doctor-Db-hzPmE.js.map +0 -1
  132. package/dist/dump-BYA55NLP.js.map +0 -1
  133. package/dist/errors-DOdL95kK.js.map +0 -1
  134. package/dist/fsck-CPPZWt9p.js.map +0 -1
  135. package/dist/http-Cw8p6y5b.js.map +0 -1
  136. package/dist/inspect-Bdbzvi-_.js.map +0 -1
  137. package/dist/maintenance-BvOv6BMM.js.map +0 -1
  138. package/dist/maintenance-ClzBC1Vv.js.map +0 -1
  139. package/dist/normalize-XWY25_sG.js.map +0 -1
  140. package/dist/observability-CpOiEN0k.js.map +0 -1
  141. package/dist/plan-DOTl21vq.js.map +0 -1
  142. package/dist/query-BmiQy328.js.map +0 -1
  143. package/dist/rebuild-index-DC_nR6wP.js.map +0 -1
  144. package/dist/restore-C6IcmLeA.js +0 -771
  145. package/dist/restore-C6IcmLeA.js.map +0 -1
  146. package/dist/shared-secret-BJdSwpgw.js.map +0 -1
  147. package/dist/snapshot-SS8USDUF.js.map +0 -1
  148. package/dist/src-DU1sCMlr.js +0 -6236
  149. package/dist/src-DU1sCMlr.js.map +0 -1
  150. package/dist/src-QQ7jDeL2.js.map +0 -1
  151. package/dist/src-zhEAmmF7.js.map +0 -1
  152. package/dist/subcommand-DZcHcsYp.js +0 -5122
  153. package/dist/subcommand-DZcHcsYp.js.map +0 -1
  154. package/dist/usage-CLjirdD-.js.map +0 -1
  155. /package/dist/{chunk-EAsCxrDo.js → rolldown-runtime-EAsCxrDo.js} +0 -0
@@ -1,5 +1,6 @@
1
- import { c as GC_MAX_PENDING_CANDIDATES, d as MAINTENANCE_MAX_FOLD_ROWS, i as readCurrentJson, l as GC_PENDING_CONTENT_TYPE, r as logSeqStartOf, t as casUpdateCurrentJson, v as decodeJsonBytes, y as encodeJsonBytes } from "./current-json-C-EyJqs0.js";
2
- import { t as BaerlyError } from "./errors-DOdL95kK.js";
1
+ import { a as decodeJsonBytes, i as readCurrentJson, o as encodeJsonBytes, r as logSeqStartOf, t as casUpdateCurrentJson } from "./current-json-C6nhPOm1.js";
2
+ import { a as GC_MAX_PENDING_CANDIDATES, l as MAINTENANCE_PROFILE_CF_FREE, o as GC_PENDING_CONTENT_TYPE, s as LOG_FORWARD_PROBE_CAP, u as MAINTENANCE_PROFILE_CF_PAID } from "./constants-CfeMWQWk.js";
3
+ import { t as BaerlyError } from "./errors-Dfa3p78c.js";
3
4
  import { i as noopMetricsRecorder, n as getCurrentContext } from "./context-DRZ3tZ5l.js";
4
5
  //#region packages/protocol/src/coordination/gc-pending.ts
5
6
  /**
@@ -72,44 +73,112 @@ const createGcPending = async (storage, key, initial, opts) => {
72
73
  };
73
74
  };
74
75
  /**
75
- * Read-modify-write `gc/pending.json` under CAS. The `mutator`
76
- * receives a deep clone of the current parsed JSON and returns the
77
- * new state. The new state is written with `If-Match: <currentEtag>`;
78
- * on conflict, throws `BaerlyError{code:"Conflict"}`.
79
- *
80
- * `mutator` MUST be deterministic — it may be called multiple times
81
- * if the caller wraps this in a retry loop, so it must not have side
82
- * effects.
83
- *
84
- * @throws BaerlyError{code:"Conflict"} — another writer landed a write
85
- * between this function's read and write.
76
+ * Read-modify-write `gc/pending.json` under CAS, with a bounded retry
77
+ * loop. Each attempt re-reads the latest body, hands the `mutator`
78
+ * that latest value, and writes the result with
79
+ * `If-Match: <latest etag>`. On `Conflict` (a concurrent writer landed
80
+ * between this attempt's read and write) the loop re-reads and
81
+ * re-applies the mutator, up to {@link GC_PENDING_CAS_MAX_ATTEMPTS}
82
+ * total attempts, then surfaces `Conflict`.
83
+ *
84
+ * `mutator` MUST be a pure, deterministic MERGE of this pass's results
85
+ * INTO the `latest` value it is handed — it is called once per attempt
86
+ * (so it must not have side effects), and it MUST read `latest` rather
87
+ * than ignore it and write a precomputed set. Ignoring `latest` would
88
+ * silently overwrite a concurrent pass's candidates: the
89
+ * If-Match succeeds (it used the fresh etag), so no conflict is
90
+ * raised, and the concurrent marks are lost. See {@link mergeGcPending}.
91
+ *
92
+ * @throws BaerlyError{code:"Conflict"} — another writer kept landing a
93
+ * write between this function's read and write for every
94
+ * attempt.
86
95
  * @throws BaerlyError{code:"InvalidResponse"} — `key` does not exist
87
96
  * (use {@link createGcPending} instead) or body doesn't
88
97
  * parse / fails the shape guard.
89
98
  */
90
99
  const casUpdateGcPending = async (storage, key, mutator, opts) => {
91
- const existing = await readGcPending(storage, key, opts);
92
- if (existing === null) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key} does not exist; use createGcPending first`);
93
- const next = mutator(structuredClone(existing.json));
94
- assertGcPending(next, key);
95
- const body = encodeJsonBytes(next);
96
- const putOpts = {
97
- ifMatch: existing.etag,
98
- contentType: GC_PENDING_CONTENT_TYPE,
99
- ...opts?.signal !== void 0 && { signal: opts.signal }
100
- };
101
- let result;
102
- try {
103
- result = await storage.put(key, body, putOpts);
104
- } catch (error) {
105
- throw translateCasError(error, key);
100
+ let lastConflict;
101
+ for (let attempt = 0; attempt < 3; attempt++) {
102
+ const existing = await readGcPending(storage, key, opts);
103
+ if (existing === null) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key} does not exist; use createGcPending first`);
104
+ const next = mutator(structuredClone(existing.json));
105
+ assertGcPending(next, key);
106
+ const body = encodeJsonBytes(next);
107
+ const putOpts = {
108
+ ifMatch: existing.etag,
109
+ contentType: GC_PENDING_CONTENT_TYPE,
110
+ ...opts?.signal !== void 0 && { signal: opts.signal }
111
+ };
112
+ try {
113
+ return {
114
+ json: next,
115
+ etag: (await storage.put(key, body, putOpts)).etag
116
+ };
117
+ } catch (error) {
118
+ const translated = translateCasError(error, key);
119
+ if (translated.code !== "Conflict") throw translated;
120
+ lastConflict = translated;
121
+ }
106
122
  }
123
+ throw lastConflict;
124
+ };
125
+ /**
126
+ * Pure merge of one GC pass's results INTO the `latest`
127
+ * `gc/pending.json` value, used as the {@link casUpdateGcPending}
128
+ * mutator. Deterministic, no I/O — the kernel unit + mutation test
129
+ * surface. Splitting it out is what makes the CAS write correct under
130
+ * concurrency: each retry re-merges against fresh `latest` rather than
131
+ * overwriting it with a stale precomputed set.
132
+ *
133
+ * Semantics:
134
+ * - **candidates**: start from `latest.candidates`, DROP any whose key
135
+ * this pass already DELETED (`sweptKeys`), then APPEND every fresh
136
+ * mark whose key is not already present AND was not itself swept
137
+ * this pass (a `graceMillis: 0` pass can mark-and-sweep a key in one
138
+ * go — re-adding it to the ledger after deleting it would be wrong;
139
+ * dedup also drops a key a concurrent pass already marked). Capped at
140
+ * `maxCandidates`.
141
+ * - **last_swept_at**: the lexicographically LATER of `latest`'s and
142
+ * this pass's (ISO-8601 UTC; `""` sorts lowest so a real timestamp
143
+ * always wins). Never regress a concurrent pass's newer sweep time.
144
+ * - **content_scan_cursor**: rotation cursor — liveness, not
145
+ * correctness; just don't regress forward progress. The two sides
146
+ * are NOT symmetric: `pass.nextContentCursor === undefined` means
147
+ * THIS pass examined to the END of the keyspace (WRAP — the most-
148
+ * advanced position), whereas `latest.content_scan_cursor` absent
149
+ * means the stored ledger has no cursor yet (start-from-beginning).
150
+ * So:
151
+ * - our pass WRAPPED ⇒ result wraps (key omitted) — our pass is
152
+ * the most advanced; this is the single-writer rotation reset.
153
+ * - else (our pass advanced to a defined cursor):
154
+ * - latest cursor defined ⇒ take the lexicographically GREATER
155
+ * (don't regress a concurrent pass's advance);
156
+ * - latest cursor absent ⇒ keep OUR advance (don't lose it).
157
+ */
158
+ const mergeGcPending = (latest, pass) => {
159
+ const surviving = latest.candidates.filter((c) => !pass.sweptKeys.has(c.key));
160
+ const present = new Set(surviving.map((c) => c.key));
161
+ const appended = [];
162
+ for (const c of pass.newCandidates) if (!pass.sweptKeys.has(c.key) && !present.has(c.key)) {
163
+ present.add(c.key);
164
+ appended.push(c);
165
+ }
166
+ const candidates = [...surviving, ...appended].slice(0, pass.maxCandidates);
167
+ const lastSweptAt = pass.lastSweptAt > latest.last_swept_at ? pass.lastSweptAt : latest.last_swept_at;
168
+ const latestCursor = latest.content_scan_cursor;
169
+ const passCursor = pass.nextContentCursor;
170
+ let cursor;
171
+ if (passCursor === void 0) cursor = void 0;
172
+ else if (latestCursor !== void 0) cursor = passCursor > latestCursor ? passCursor : latestCursor;
173
+ else cursor = passCursor;
107
174
  return {
108
- json: next,
109
- etag: result.etag
175
+ schema_version: 1,
176
+ candidates,
177
+ last_swept_at: lastSweptAt,
178
+ ...cursor !== void 0 && { content_scan_cursor: cursor }
110
179
  };
111
180
  };
112
- const VALID_REASONS = new Set([
181
+ const VALID_REASONS = /* @__PURE__ */ new Set([
113
182
  "stale-log",
114
183
  "orphan-snapshot",
115
184
  "orphan-content"
@@ -560,6 +629,13 @@ const versionFromContent = async (body) => {
560
629
  for (let i = 0; i < bytes.length; i++) hex += bytes[i].toString(16).padStart(2, "0");
561
630
  return hex.slice(0, VERSION_HEX_LENGTH);
562
631
  };
632
+ /**
633
+ * The single constructor for a log-object key:
634
+ * `<manifestPrefix>/log/<seq>.json`. `manifestPrefix` is the
635
+ * `<…>/manifests/<collection>` segment — this helper adds `/log/`.
636
+ * Route all callers here so the key shape lives in one place.
637
+ */
638
+ const logObjectKey = (manifestPrefix, seq) => `${manifestPrefix}/log/${seq}.json`;
563
639
  //#endregion
564
640
  //#region packages/protocol/src/snapshot-hash.ts
565
641
  /**
@@ -582,10 +658,170 @@ const snapshotHash = async (bytes) => {
582
658
  return hex;
583
659
  };
584
660
  //#endregion
661
+ //#region packages/server/src/log-tail.ts
662
+ /**
663
+ * Tolerant forward-probe for the true committed log tail. The strict
664
+ * `[log_seq_start, hint)` range is dense (a hole is corruption, owned
665
+ * by `walkLogRange`); this owns only the tolerant `[hint, ∞)` probe,
666
+ * where `hint` is a lower bound — the real tail may sit at or past it.
667
+ */
668
+ /**
669
+ * Discover the true committed tail and fold entries in `[hint, tail)`.
670
+ * `tail` is the first empty seq (>= hint); `entries` are the
671
+ * LogEntries in `[hint, tail)` in seq order. `cap` bounds the walk;
672
+ * exhausting it THROWS `Internal` (never silently truncates) — see
673
+ * the cap-exhaustion comment below.
674
+ */
675
+ const probeTailFrom = async (storage, logPrefix, hint, opts) => {
676
+ const cap = opts?.cap ?? 1e5;
677
+ const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
678
+ const entries = [];
679
+ for (let i = 0; i < cap; i++) {
680
+ const seq = hint + i;
681
+ const key = logObjectKey(logPrefix, seq);
682
+ const got = await storage.get(key, getOpts);
683
+ if (got === null) return {
684
+ tail: seq,
685
+ entries
686
+ };
687
+ try {
688
+ entries.push(decodeJsonBytes(got.body));
689
+ } catch (error) {
690
+ throw new BaerlyError("InvalidResponse", `probeTailFrom: malformed log entry at ${key}: ${error.message}`, { cause: error });
691
+ }
692
+ }
693
+ throw new BaerlyError("Internal", `probeTailFrom: forward probe exceeded ${cap} from hint ${hint} on ${logPrefix}`);
694
+ };
695
+ /**
696
+ * First empty log seq at/after `hint` — the slot a new commit creates —
697
+ * found by galloping search (`O(log gap)` GETs vs `probeTailFrom`'s
698
+ * `O(gap)`). Used by the writer's per-commit tail-find, where `tail_hint`
699
+ * can lag the true tail. Position only; no bodies.
700
+ *
701
+ * PRECONDITION: occupancy MUST be a dense prefix from `hint` (galloping
702
+ * assumes monotone occupied→empty; a hole ABOVE `hint` would be skipped
703
+ * and the gallop could return a tail past live entries). Contrast
704
+ * `probeTailFrom`, which stops at the first 404 and is hole-tolerant.
705
+ */
706
+ const findLogTail = async (storage, logPrefix, hint, opts) => {
707
+ const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
708
+ const exists = async (seq) => await storage.get(logObjectKey(logPrefix, seq), getOpts) !== null;
709
+ if (!await exists(hint)) return hint;
710
+ let lo = hint;
711
+ let step = 1;
712
+ let hi = hint + step;
713
+ while (await exists(hi)) {
714
+ lo = hi;
715
+ step *= 2;
716
+ hi = lo + step;
717
+ if (step > 1e5) throw new BaerlyError("Internal", `findLogTail: galloping probe exceeded ${LOG_FORWARD_PROBE_CAP} from hint ${hint} on ${logPrefix}`);
718
+ }
719
+ while (hi - lo > 1) {
720
+ const mid = lo + Math.floor((hi - lo) / 2);
721
+ if (await exists(mid)) lo = mid;
722
+ else hi = mid;
723
+ }
724
+ return hi;
725
+ };
726
+ //#endregion
585
727
  //#region packages/server/src/names.ts
586
- /** Reject names in the system-reserved leading-"_" namespace. See docs/adr/007-layout-versioning-cordon.md. */
728
+ /** Reject names in the system-reserved leading-"_" namespace. See docs/adr/003-layout-versioning-cordon.md. */
587
729
  const assertNameNotReserved = (name, context) => {
588
- if (name.startsWith("_")) throw new BaerlyError("InvalidConfig", `${context}: "${name}" is reserved — names beginning with "_" are reserved for system use. See docs/adr/007-layout-versioning-cordon.md.`);
730
+ if (name.startsWith("_")) throw new BaerlyError("InvalidConfig", `${context}: "${name}" is reserved — names beginning with "_" are reserved for system use. See docs/adr/003-layout-versioning-cordon.md.`);
731
+ };
732
+ //#endregion
733
+ //#region packages/server/src/snapshot.ts
734
+ /**
735
+ * Snapshot file shape + parse. Lifted out of `./compactor.ts` so that
736
+ * the kernel barrel (`packages/server/src/index.ts`) — which re-exports
737
+ * these four symbols — does not transitively pull `withObservability`
738
+ * via `compact()`. `./compactor.ts` imports its own snapshot primitives
739
+ * back from this module; the public API surface is unchanged.
740
+ *
741
+ * @see ../../../docs/spec/sync-protocol.md
742
+ */
743
+ /**
744
+ * Single-level snapshot at L9; the level prefix is forward-compatible
745
+ * with future L0..L9 rolling merges without a wire change.
746
+ */
747
+ const SNAPSHOT_LEVEL = 9;
748
+ /**
749
+ * `minSeq` and `maxSeq` are padded to {@link SEQ_DIGITS} digits so
750
+ * `list(prefix)` returns snapshots in seq order without needing to
751
+ * parse the filename. 12 digits caps at ~10^12 entries per collection;
752
+ * a real bucket hits S3's per-prefix QPS ceiling long before that.
753
+ */
754
+ const SEQ_DIGITS = 12;
755
+ const MAX_SEQ = 10 ** SEQ_DIGITS - 1;
756
+ /**
757
+ * Build the on-bucket key for a snapshot file. `sha256` must be the
758
+ * lowercase hex digest of the canonical body
759
+ * ({@link encodeSnapshotBody}); the filename embeds it so a crashed
760
+ * mid-PUT body that doesn't match its own filename hash is rejected
761
+ * by readers as "missing."
762
+ *
763
+ * @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
764
+ * out of range or `sha256` is not 64 hex chars.
765
+ */
766
+ const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
767
+ if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
768
+ if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
769
+ const pad = (n) => n.toString().padStart(SEQ_DIGITS, "0");
770
+ return `${collectionPrefix}/snapshot/L${SNAPSHOT_LEVEL}/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
771
+ };
772
+ /**
773
+ * Canonical byte encoding of a {@link SnapshotBody}. Only the
774
+ * compactor produces snapshots, and only this function does the
775
+ * serialisation — so the byte output is deterministic for a given
776
+ * `SnapshotBody` (identical input ⇒ identical bytes ⇒ identical
777
+ * filename hash).
778
+ */
779
+ const encodeSnapshotBody = (s) => encodeJsonBytes(s);
780
+ /**
781
+ * Load a snapshot file as a `Map<_id, DocumentData>`. Verifies the
782
+ * body hash against the filename, checks the schema version, and
783
+ * cross-checks `body.collection` against the caller's expectation.
784
+ *
785
+ * @example
786
+ * ```ts
787
+ * import { loadSnapshotAsMap } from "@gusto/baerly-storage";
788
+ *
789
+ * const map =
790
+ * srcCurrent.snapshot === null
791
+ * ? new Map<string, DocumentData>()
792
+ * : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
793
+ * ```
794
+ *
795
+ * @throws BaerlyError code="Internal" — pointer resolves to no body,
796
+ * or the body's SHA-256 doesn't match the hash embedded in the
797
+ * filename. (Protocol invariant violation; a corrupted snapshot
798
+ * the reader cannot continue past.)
799
+ * @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
800
+ * isn't an object, has an unsupported `schema_version`, or carries a
801
+ * `collection` that doesn't match `expectedCollection`.
802
+ */
803
+ const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
804
+ const got = await storage.get(key, signal !== void 0 ? { signal } : void 0);
805
+ if (got === null) throw new BaerlyError("Internal", `compact: snapshot pointer ${key} resolves to no body; protocol violation`);
806
+ const filename = key.slice(key.lastIndexOf("/") + 1).replace(/\.json$/, "");
807
+ const lastDash = filename.lastIndexOf("-");
808
+ const expectedHash = filename.slice(lastDash + 1);
809
+ const actualHash = await snapshotHash(got.body);
810
+ if (actualHash !== expectedHash) throw new BaerlyError("Internal", `compact: snapshot ${key} body hash mismatch (got ${actualHash}); protocol violation`);
811
+ let parsed;
812
+ try {
813
+ parsed = decodeJsonBytes(got.body);
814
+ } catch (error) {
815
+ throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
816
+ }
817
+ if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not an object`);
818
+ const body = parsed;
819
+ if (body.schema_version !== 1) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} has unsupported schema_version ${String(body.schema_version)}`);
820
+ if (body.collection !== expectedCollection) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} carries collection ${body.collection}, expected ${expectedCollection}`);
821
+ if (!Array.isArray(body.docs)) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body.docs is not an array`);
822
+ const out = /* @__PURE__ */ new Map();
823
+ for (const row of body.docs) out.set(row._id, row.body);
824
+ return out;
589
825
  };
590
826
  //#endregion
591
827
  //#region packages/server/src/indexes.ts
@@ -615,8 +851,7 @@ const assertNameNotReserved = (name, context) => {
615
851
  *
616
852
  * `<logPrefix>/index/<indexName>/<value-b32>/<docId>.json`
617
853
  *
618
- * Wire shape (composite — accepted by the encoder, not consulted by
619
- * the read path today):
854
+ * Wire shape (composite):
620
855
  *
621
856
  * `<logPrefix>/index/<indexName>/<a-b32>/<b-b32>/<docId>.json`
622
857
  *
@@ -637,11 +872,13 @@ const assertNameNotReserved = (name, context) => {
637
872
  * and `$in` walks over numeric fields are first-class — no
638
873
  * planner fall-back to full-scan).
639
874
  *
640
- * Locked: single-field indexes only on the read path. Composite
641
- * shape is documented and reserved (the path encoder accepts it).
875
+ * Composite read routing is left-anchored. The planner may consume an
876
+ * equality prefix plus a range / `$in` tail slot; predicates that do
877
+ * not match a routable prefix fall back to the snapshot+log fold.
642
878
  *
643
- * @see docs/spec/sync-protocol.md — fence model these PUTs land
644
- * inside (between log PUT and `current.json` CAS-advance).
879
+ * @see docs/spec/sync-protocol.md — index emission is hybrid around
880
+ * the commit: additive (new) keys are PUT before the committing
881
+ * `log/<seq>` create, and stale-key DELETEs land after it.
645
882
  */
646
883
  /** Path-safe segment name. */
647
884
  const INDEX_NAME_RE = /^[a-z][a-z0-9_]*$/;
@@ -653,7 +890,7 @@ const INDEX_NAME_RE = /^[a-z][a-z0-9_]*$/;
653
890
  const B32_ALPHABET = "0123456789abcdefghijklmnopqrstuv";
654
891
  /**
655
892
  * Reject malformed {@link IndexDefinition} shapes synchronously, at
656
- * writer construction. See docs/adr/007-layout-versioning-cordon.md.
893
+ * writer construction. See docs/adr/003-layout-versioning-cordon.md.
657
894
  *
658
895
  * @throws BaerlyError code="InvalidConfig" — reserved leading-`_` `name`.
659
896
  * @throws BaerlyError code="SchemaError" — `name` fails the regex, or `on` is empty.
@@ -820,8 +1057,8 @@ const indexKeyPrefix = (logPrefix, indexName) => `${logPrefix}/index/${indexName
820
1057
  * Single-field: `<logPrefix>/index/<name>/<v0-b32>/<docId>.json`.
821
1058
  * Composite: `<logPrefix>/index/<name>/<v0-b32>/<v1-b32>/.../<docId>.json`.
822
1059
  *
823
- * Composite shape is accepted but the read path today only
824
- * consults single-field entries.
1060
+ * Composite entries are read-path routable when the planner can consume
1061
+ * a left-anchored prefix of the indexed tuple.
825
1062
  */
826
1063
  const indexKeyFor = (logPrefix, def, values, docId) => {
827
1064
  const segments = values.map(encodeIndexValue).join("/");
@@ -835,7 +1072,7 @@ const indexKeyFor = (logPrefix, def, values, docId) => {
835
1072
  *
836
1073
  * @throws BaerlyError code="SchemaError" — `on` contains a dotted
837
1074
  * path, or the indexed field's value is an array. Both restrictions
838
- * are pre-launch; multi-value (array) indexes are a follow-up.
1075
+ * are current 0.x limitations; multi-value (array) indexes are a follow-up.
839
1076
  */
840
1077
  const projectIndexValues = (def, body) => {
841
1078
  if (body === void 0) return;
@@ -909,7 +1146,7 @@ const allIndexKeysFor = (logPrefix, defs, body, docId) => {
909
1146
  * every caller.
910
1147
  *
911
1148
  * @throws BaerlyError code="Internal" when the key resolves to no
912
- * body — a hole inside `[log_seq_start, next_seq)` is a corruption
1149
+ * body — a hole inside `[log_seq_start, tail_hint)` is a corruption
913
1150
  * the kernel can't continue past.
914
1151
  * @throws BaerlyError code="InvalidResponse" when the body isn't
915
1152
  * valid JSON.
@@ -922,14 +1159,12 @@ const readLogEntry = async (storage, key, opts) => {
922
1159
  * byteLength of the STORED log-object body exactly as it was GET back
923
1160
  * from storage (`got.body.byteLength`).
924
1161
  *
925
- * The compactor uses this to subtract from `current.json.tail_bytes`
926
- * the EXACT bytes the writer PUT for the folded slice. The writer
927
- * accumulated `encodeJsonBytes(entry).byteLength` per committed entry
928
- * (`writer.ts` Step 6); the stored body IS those bytes, so summing
929
- * `got.body.byteLength` over the folded range == the writer's
930
- * accumulated total for that range — no re-encode, no round-trip
931
- * assumption (a re-encode of the parsed `LogEntry` could drift if
932
- * key-order/whitespace ever diverged from what was PUT).
1162
+ * The compactor uses this to sum the byte cost of the folded slice.
1163
+ * The stored body byteLength IS what the writer PUT for each entry,
1164
+ * so summing `got.body.byteLength` over the folded range yields the
1165
+ * exact cost — no re-encode, no round-trip assumption (a re-encode
1166
+ * of the parsed `LogEntry` could drift if key-order/whitespace ever
1167
+ * diverged from what was PUT).
933
1168
  *
934
1169
  * Same error semantics as {@link readLogEntry}.
935
1170
  */
@@ -967,7 +1202,7 @@ const walkLogRange = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) =
967
1202
  * this is the bytes-aware core `walkLogRange` is implemented on top of.
968
1203
  *
969
1204
  * The compactor consumes this so it gets BOTH the parsed entries (for
970
- * the fold) AND the exact byte sum (to decrement `tail_bytes`) from a
1205
+ * the fold) AND the exact byte sum (for `mean_entry_bytes` stamping) from a
971
1206
  * single walk.
972
1207
  */
973
1208
  const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) => {
@@ -980,7 +1215,7 @@ const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive
980
1215
  const promises = [];
981
1216
  for (let i = chunkStart; i < chunkEnd; i++) {
982
1217
  const seq = fromSeq + i;
983
- promises.push(readLogEntryWithBytes(storage, `${logPrefix}/log/${seq}.json`, opts));
1218
+ promises.push(readLogEntryWithBytes(storage, logObjectKey(logPrefix, seq), opts));
984
1219
  }
985
1220
  const results = await Promise.all(promises);
986
1221
  for (let i = 0; i < results.length; i++) out[chunkStart + i] = results[i];
@@ -1030,76 +1265,6 @@ const foldLogEntriesOnto = (map, entries, opts) => {
1030
1265
  }
1031
1266
  }
1032
1267
  };
1033
- const MAX_SEQ = 10 ** 12 - 1;
1034
- /**
1035
- * Build the on-bucket key for a snapshot file. `sha256` must be the
1036
- * lowercase hex digest of the canonical body
1037
- * ({@link encodeSnapshotBody}); the filename embeds it so a crashed
1038
- * mid-PUT body that doesn't match its own filename hash is rejected
1039
- * by readers as "missing."
1040
- *
1041
- * @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
1042
- * out of range or `sha256` is not 64 hex chars.
1043
- */
1044
- const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
1045
- if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
1046
- if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
1047
- const pad = (n) => n.toString().padStart(12, "0");
1048
- return `${collectionPrefix}/snapshot/L9/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
1049
- };
1050
- /**
1051
- * Canonical byte encoding of a {@link SnapshotBody}. Only the
1052
- * compactor produces snapshots, and only this function does the
1053
- * serialisation — so the byte output is deterministic for a given
1054
- * `SnapshotBody` (identical input ⇒ identical bytes ⇒ identical
1055
- * filename hash).
1056
- */
1057
- const encodeSnapshotBody = (s) => encodeJsonBytes(s);
1058
- /**
1059
- * Load a snapshot file as a `Map<_id, DocumentData>`. Verifies the
1060
- * body hash against the filename, checks the schema version, and
1061
- * cross-checks `body.collection` against the caller's expectation.
1062
- *
1063
- * @example
1064
- * ```ts
1065
- * import { loadSnapshotAsMap } from "@gusto/baerly-storage";
1066
- *
1067
- * const map =
1068
- * srcCurrent.snapshot === null
1069
- * ? new Map<string, DocumentData>()
1070
- * : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
1071
- * ```
1072
- *
1073
- * @throws BaerlyError code="Internal" — pointer resolves to no body,
1074
- * or the body's SHA-256 doesn't match the hash embedded in the
1075
- * filename. (Protocol invariant violation; a corrupted snapshot
1076
- * the reader cannot continue past.)
1077
- * @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
1078
- * isn't an object, has an unsupported `schema_version`, or carries a
1079
- * `collection` that doesn't match `expectedCollection`.
1080
- */
1081
- const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
1082
- const got = await storage.get(key, signal !== void 0 ? { signal } : void 0);
1083
- if (got === null) throw new BaerlyError("Internal", `compact: snapshot pointer ${key} resolves to no body; protocol violation`);
1084
- const filename = key.slice(key.lastIndexOf("/") + 1).replace(/\.json$/, "");
1085
- const lastDash = filename.lastIndexOf("-");
1086
- const expectedHash = filename.slice(lastDash + 1);
1087
- const actualHash = await snapshotHash(got.body);
1088
- if (actualHash !== expectedHash) throw new BaerlyError("Internal", `compact: snapshot ${key} body hash mismatch (got ${actualHash}); protocol violation`);
1089
- let parsed;
1090
- try {
1091
- parsed = decodeJsonBytes(got.body);
1092
- } catch (error) {
1093
- throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
1094
- }
1095
- if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not an object`);
1096
- const body = parsed;
1097
- if (body.schema_version !== 1) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} has unsupported schema_version ${String(body.schema_version)}`);
1098
- if (body.collection !== expectedCollection) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} carries collection ${body.collection}, expected ${expectedCollection}`);
1099
- const out = /* @__PURE__ */ new Map();
1100
- for (const row of body.docs) out.set(row._id, row.body);
1101
- return out;
1102
- };
1103
1268
  //#endregion
1104
1269
  //#region packages/server/src/compactor.ts
1105
1270
  /**
@@ -1115,8 +1280,7 @@ const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
1115
1280
  * "missing." The atomic moment is the CAS-swap of
1116
1281
  * `current.json.snapshot`, which is a single conditional PUT.
1117
1282
  *
1118
- * Order of operations (matches `Writer.commit`'s manifest-first-
1119
- * REVERSED ordering): PUT snapshot body → CAS-advance `current.json`.
1283
+ * Order of operations: PUT snapshot body → CAS-advance `current.json`.
1120
1284
  * A crash before the swap leaves an orphan snapshot file; the swap
1121
1285
  * succeeds iff our captured ETag still matches.
1122
1286
  *
@@ -1124,7 +1288,7 @@ const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
1124
1288
  * single monolithic snapshot (every compaction rewrites one well-
1125
1289
  * known key) makes partial writes catastrophic and serializes every
1126
1290
  * writer on one key. WAL-checkpointing in the rqlite / Raft tradition
1127
- * requires a consensus layer Baerly doesn't have. The Litestream-
1291
+ * requires a consensus layer baerly-storage doesn't have. The Litestream-
1128
1292
  * style multi-level scheme — small fixed number of levels, chunked
1129
1293
  * files keyed by sequence range — keeps random GETs cheap on
1130
1294
  * S3-compatible storage and avoids LIST. Current ship is single-
@@ -1189,7 +1353,8 @@ const compact = async (args, options = {}) => {
1189
1353
  const current = read.json;
1190
1354
  const baseEtag = read.etag;
1191
1355
  const logSeqStartBefore = logSeqStartOf(current);
1192
- const nextSeq = current.next_seq;
1356
+ const discoveredTail = (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStartBefore, current.tail_hint, internal.knownTail ?? 0), { ...options.signal !== void 0 && { signal: options.signal } })).tail;
1357
+ const nextSeq = discoveredTail;
1193
1358
  if (nextSeq - logSeqStartBefore < minToCompact) return {
1194
1359
  written: false,
1195
1360
  skippedReason: "below-min-threshold",
@@ -1250,13 +1415,15 @@ const compact = async (args, options = {}) => {
1250
1415
  ...options.signal !== void 0 && { signal: options.signal }
1251
1416
  };
1252
1417
  await storage.put(newKey, bodyBytes, putOpts);
1418
+ const entriesFolded = foldEnd - logSeqStartBefore;
1253
1419
  const nextBody = encodeJsonBytes({
1254
1420
  ...current,
1255
1421
  snapshot: newKey,
1256
1422
  log_seq_start: foldEnd,
1423
+ tail_hint: Math.max(current.tail_hint, discoveredTail),
1257
1424
  snapshot_bytes: bodyBytes.byteLength,
1258
1425
  snapshot_rows: base.size,
1259
- tail_bytes: current.tail_bytes - foldedSliceBytes
1426
+ mean_entry_bytes: entriesFolded > 0 ? Math.round(foldedSliceBytes / entriesFolded) : current.mean_entry_bytes ?? 0
1260
1427
  });
1261
1428
  const casOpts = {
1262
1429
  ifMatch: baseEtag,
@@ -1280,10 +1447,9 @@ const compact = async (args, options = {}) => {
1280
1447
  }
1281
1448
  throw error;
1282
1449
  }
1283
- const entriesFolded = foldEnd - logSeqStartBefore;
1284
1450
  const metrics = ctxMetrics$2();
1285
1451
  metrics.histogram("db.compact.entries_folded", entriesFolded, { collection: collectionName });
1286
- metrics.gauge("db.manifest.lag_window_depth", current.next_seq - foldEnd, { collection: collectionName });
1452
+ metrics.gauge("db.manifest.lag_window_depth", discoveredTail - foldEnd, { collection: collectionName });
1287
1453
  return {
1288
1454
  written: true,
1289
1455
  previousSnapshotKey: current.snapshot,
@@ -1467,8 +1633,8 @@ const runGc = async (args, options = {}) => {
1467
1633
  for (const cand of sweepCandidates) if (toSweep.length < maxSweeps && Date.parse(cand.due_at) <= nowMs) toSweep.push(cand);
1468
1634
  else remaining.push(cand);
1469
1635
  await Promise.all(toSweep.map((c) => storage.delete(c.key, signalOpts)));
1470
- const merged = remaining.slice(0, GC_MAX_PENDING_CANDIDATES);
1471
- const lastSweptAt = toSweep.length > 0 ? now().toISOString() : pending.json.last_swept_at;
1636
+ const sweptKeys = new Set(toSweep.map((c) => c.key));
1637
+ const lastSweptAt = toSweep.length > 0 ? now().toISOString() : "";
1472
1638
  const markedSummary = {
1473
1639
  stale_log: markedStaleLog,
1474
1640
  orphan_snapshot: markedOrphanSnapshot,
@@ -1476,11 +1642,12 @@ const runGc = async (args, options = {}) => {
1476
1642
  };
1477
1643
  let pendingDepth;
1478
1644
  try {
1479
- pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, () => ({
1480
- schema_version: 1,
1481
- candidates: merged,
1482
- last_swept_at: lastSweptAt,
1483
- ...nextContentCursor !== void 0 && { content_scan_cursor: nextContentCursor }
1645
+ pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, (latest) => mergeGcPending(latest, {
1646
+ sweptKeys,
1647
+ newCandidates,
1648
+ lastSweptAt,
1649
+ nextContentCursor,
1650
+ maxCandidates: GC_MAX_PENDING_CANDIDATES
1484
1651
  }), signalOpts)).json.candidates.length;
1485
1652
  } catch (error) {
1486
1653
  if (error instanceof BaerlyError && error.code === "Conflict") pendingDepth = remaining.length;
@@ -1549,7 +1716,7 @@ const computeDueAt = (entry, now, graceMs) => {
1549
1716
  };
1550
1717
  /**
1551
1718
  * Build the live content-hash set. The set covers every live
1552
- * post-image: every `entry.after` in `[logSeqStart, next_seq)` plus
1719
+ * post-image: every `entry.after` in `[logSeqStart, tail_hint)` plus
1553
1720
  * every row body in the current snapshot.
1554
1721
  *
1555
1722
  * A snapshot read that throws (corrupt body, hash mismatch) is
@@ -1561,9 +1728,9 @@ const computeDueAt = (entry, now, graceMs) => {
1561
1728
  const collectLiveContentHashes = async (storage, collectionPrefix, collectionName, current, logSeqStart, signal) => {
1562
1729
  const hashes = /* @__PURE__ */ new Set();
1563
1730
  const getOpts = signal !== void 0 ? { signal } : void 0;
1564
- const logReads = [];
1565
- for (let s = logSeqStart; s < current.next_seq; s++) logReads.push((async () => {
1566
- const got = await storage.get(`${collectionPrefix}/log/${s}.json`, getOpts);
1731
+ const { tail } = await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, current.tail_hint), { signal });
1732
+ const ingestLogEntry = async (s) => {
1733
+ const got = await storage.get(logObjectKey(collectionPrefix, s), getOpts);
1567
1734
  if (got === null) return;
1568
1735
  let entry;
1569
1736
  try {
@@ -1574,8 +1741,14 @@ const collectLiveContentHashes = async (storage, collectionPrefix, collectionNam
1574
1741
  if (entry.after === void 0) return;
1575
1742
  const bodyBytes = encodeJsonBytes(entry.after);
1576
1743
  hashes.add(await versionFromContent(bodyBytes));
1577
- })());
1578
- await Promise.all(logReads);
1744
+ };
1745
+ for (let chunkStart = logSeqStart; chunkStart < tail; chunkStart += 16) {
1746
+ signal?.throwIfAborted();
1747
+ const chunkEnd = Math.min(chunkStart + 16, tail);
1748
+ const chunk = [];
1749
+ for (let s = chunkStart; s < chunkEnd; s++) chunk.push(ingestLogEntry(s));
1750
+ await Promise.all(chunk);
1751
+ }
1579
1752
  if (current.snapshot !== null) try {
1580
1753
  const map = await loadSnapshotAsMap(storage, current.snapshot, collectionName, signal);
1581
1754
  const rowReads = [];
@@ -1619,10 +1792,10 @@ const collectLiveContentHashes = async (storage, collectionPrefix, collectionNam
1619
1792
  *
1620
1793
  * @see ./indexes.ts — `IndexDefinition`, `allIndexKeysFor`,
1621
1794
  * `indexKeyPrefix`.
1622
- * @see ./writer.ts — the fence-time emission path this
1623
- * command reconciles.
1795
+ * @see ./writer.ts — the hybrid index emission path this command
1796
+ * reconciles.
1624
1797
  */
1625
- const EMPTY_BODY = new Uint8Array(0);
1798
+ const EMPTY_BODY = /* @__PURE__ */ new Uint8Array(0);
1626
1799
  const APPLICATION_JSON = "application/json";
1627
1800
  /**
1628
1801
  * Idempotently reconcile one secondary index for one collection.
@@ -1641,7 +1814,7 @@ const APPLICATION_JSON = "application/json";
1641
1814
  * @throws BaerlyError code="InvalidResponse" — `current.json` missing.
1642
1815
  * @throws BaerlyError code="Internal" — protocol-invariant
1643
1816
  * violation (snapshot hash mismatch, missing log entry inside
1644
- * `[log_seq_start, next_seq)`).
1817
+ * `[log_seq_start, tail_hint)`).
1645
1818
  */
1646
1819
  const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
1647
1820
  const read = await readCurrentJson(storage, currentJsonKey);
@@ -1651,9 +1824,11 @@ const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
1651
1824
  const collectionPrefix = currentJsonKey.slice(0, lastSlash);
1652
1825
  const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
1653
1826
  const logSeqStart = logSeqStartOf(read.json);
1654
- const nextSeq = read.json.next_seq;
1827
+ const hint = read.json.tail_hint;
1655
1828
  const live = read.json.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, read.json.snapshot, collection);
1656
- foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart, nextSeq), { collection });
1829
+ foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart, hint), { collection });
1830
+ const probeOpts = opts.signal !== void 0 ? { signal: opts.signal } : void 0;
1831
+ foldLogEntriesOnto(live, (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, hint), probeOpts)).entries, { collection });
1657
1832
  const expected = /* @__PURE__ */ new Set();
1658
1833
  for (const [docId, body] of live) for (const k of allIndexKeysFor(collectionPrefix, [def], body, docId)) expected.add(k);
1659
1834
  const actual = /* @__PURE__ */ new Set();
@@ -1759,36 +1934,90 @@ const runScheduledMaintenance = async (args, options = {}) => {
1759
1934
  })
1760
1935
  };
1761
1936
  };
1762
- const CLOUDFLARE_FREE_TIER = {
1937
+ const profileToScheduledOptions = (profile) => ({
1763
1938
  compact: {
1764
- maxEntriesPerRun: 20,
1939
+ maxEntriesPerRun: profile.maxFoldEntriesPerPass,
1765
1940
  minEntriesToCompact: 50
1766
1941
  },
1767
1942
  gc: {
1768
- maxMarksPerRun: 20,
1769
- maxSweepsPerRun: 10
1943
+ maxMarksPerRun: profile.gcMaxMarks,
1944
+ maxSweepsPerRun: profile.gcMaxSweeps
1770
1945
  }
1771
- };
1946
+ });
1772
1947
  /**
1773
- * Floor-based GC-cadence boundary crossing. BATCH-SAFE: a
1774
- * `commitBatch` that jumps `next_seq` by its whole batch size can step
1775
- * CLEAN OVER a `next_seq % interval === 0` modulo test; this floor test
1776
- * trips on any interval boundary inside `(prevSeq, nextSeq]`.
1948
+ * Tuning profile for the 50-subrequest Cloudflare free-tier budget,
1949
+ * derived from {@link MAINTENANCE_PROFILE_CF_FREE} (one source of truth).
1950
+ *
1951
+ * Budget arithmetic (worst case):
1952
+ * compact: 1 GET current + 1 GET snapshot (if any) + N GETs log
1953
+ * + 1 PUT snapshot + 1 PUT current = 3 + N (N = maxEntriesPerRun).
1954
+ * runGc: 1 GET current + 1 GET pending + 3 LISTs (one page each)
1955
+ * + M GETs log (live tail for content hashes) + S DELETEs + 1 PUT
1956
+ * pending = 6 + M + S.
1957
+ *
1958
+ * With N=20, M=20, S=10 the totals are: compact ≈ 23, gc ≈ 36 — over
1959
+ * the 50 cap combined. The Cloudflare scheduled handler therefore
1960
+ * alternates phases per tick (even minute → compact, odd minute →
1961
+ * GC) by calling `compact()` / `runGc()` directly instead of
1962
+ * `runScheduledMaintenance`; the `maintenance-budget.test.ts`
1963
+ * worst-case test proves each phase in isolation sits under 50 ops
1964
+ * with these bounds.
1965
+ */
1966
+ const CLOUDFLARE_FREE_TIER = profileToScheduledOptions(MAINTENANCE_PROFILE_CF_FREE);
1967
+ /**
1968
+ * Tuning profile for the 10,000-subrequest Cloudflare paid-tier budget,
1969
+ * derived from {@link MAINTENANCE_PROFILE_CF_PAID} (one source of truth).
1970
+ * Keeps the single-phase CPU-killable shape but affords Node-tier per-pass
1971
+ * throughput (compact ≈ 3+200, gc ≈ 6+200+100 — far under the 10k cap).
1972
+ *
1973
+ * To honour `BAERLY_MAINTENANCE_PROFILE=cf-paid` on the cron path, pass this
1974
+ * constant to `runScheduledMaintenance` inside your
1975
+ * {@link WorkerScheduledHandler}; the worked recipe lives in the `scheduled`
1976
+ * handler docstring of `baerlyWorker` (`@gusto/baerly-storage/cloudflare`).
1977
+ */
1978
+ const CLOUDFLARE_PAID_TIER = profileToScheduledOptions(MAINTENANCE_PROFILE_CF_PAID);
1979
+ /**
1980
+ * Floor-based GC-cadence boundary crossing. Every commit advances the
1981
+ * writer-observed tail by one logical slot; `tail_hint` remains a
1982
+ * lower bound and may be refreshed later by maintenance. The floor test
1983
+ * trips on any interval boundary inside `(prevSeq, nextSeq]`, so it
1984
+ * stays correct whether the observed/probed tail advances by one or
1985
+ * catches up by a larger step.
1777
1986
  */
1778
1987
  const crossesGcBoundary = (prevSeq, nextSeq, interval) => Math.floor(prevSeq / interval) !== Math.floor(nextSeq / interval);
1779
1988
  /**
1989
+ * Derived live-tail byte size — the ratio TRIGGER input that replaces the
1990
+ * formerly-stored exact byte count. `(observedTail − log_seq_start) ×
1991
+ * mean_entry_bytes`. `observedTail` is a param so write-tick callers can
1992
+ * pass the writer-observed tail and scheduled callers can pass a
1993
+ * probe-discovered true tail. Cold-start (no mean yet) falls back to
1994
+ * {@link MAINTENANCE_COLD_START_ENTRY_BYTES}, never 0 (see that constant).
1995
+ * TRIGGER-only — structurally barred from `foldViable` (the exact-bytes CPU-kill
1996
+ * ceiling).
1997
+ */
1998
+ const estimateTailBytes = (current, observedTail) => Math.max(0, observedTail - current.log_seq_start) * (current.mean_entry_bytes ?? 128);
1999
+ /**
1780
2000
  * The cheap DISPATCH gate the writer calls after a commit — zero
1781
- * storage ops; it reads the post-CAS {@link CurrentJson} already in
2001
+ * storage ops; it reads the post-commit {@link CurrentJson} already in
1782
2002
  * scope. Ratio-OR-boundary, WITHOUT the entry floor (the floor is part
1783
2003
  * of the fold-trigger Gate 1 checked inside {@link runBoundedMaintenance}).
1784
- * Returning `false` lets the writer skip the dispatch entirely.
2004
+ * Returning `false` lets the writer skip the dispatch entirely. Ratio
2005
+ * numerator is the DERIVED {@link estimateTailBytes};
2006
+ * `observedTail` threads from the caller (the writer's in-memory observed
2007
+ * tail `seq+1` under single-write commit). Both the boundary check and the
2008
+ * ratio key off `observedTail`, since the stored `tail_hint` is now only a
2009
+ * non-authoritative lower bound (compactor-advanced).
1785
2010
  */
1786
- const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundary(prevSeq, current.next_seq, gcInterval) || current.tail_bytes / Math.max(current.snapshot_bytes, 65536) >= 1;
2011
+ const shouldFireMaintenance = (current, prevSeq, gcInterval, observedTail) => crossesGcBoundary(prevSeq, observedTail, gcInterval) || estimateTailBytes(current, observedTail) / Math.max(current.snapshot_bytes, 65536) >= 1;
1787
2012
  /**
1788
2013
  * The write-tick maintenance runner: a bounded, single-attempt
1789
2014
  * composition of {@link compact} and {@link runGc} for one collection,
1790
2015
  * sized to fit a CPU-killable Cloudflare free-tier isolate by default.
1791
2016
  *
2017
+ * INVARIANT: a {@link MaintenanceProfile} changes only maintenance rate +
2018
+ * defer threshold — never data/API/query/wire/correctness; reads stay pure.
2019
+ * See `maintenance-profile-equivalence.test.ts`.
2020
+ *
1792
2021
  * Never throws — the write-tick caller must never see a maintenance
1793
2022
  * failure. Expected signals (CAS contention, an over-ceiling deferral)
1794
2023
  * are swallowed quietly; unexpected throws bump
@@ -1796,7 +2025,8 @@ const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundar
1796
2025
  * swallowed.
1797
2026
  *
1798
2027
  * Control flow (fold-priority with a stateless GC-starvation guard):
1799
- * 0. `disabled` → return with zero storage ops.
2028
+ * 0. `disabled` → skip fold/GC, but still allow the rate-limited
2029
+ * tail_hint refresh that bounds read/write forward-probes.
1800
2030
  * 1. Read `current.json` once.
1801
2031
  * 2. Hard-GC starvation boundary (`"single"` only) → GC slice, skip fold.
1802
2032
  * 3. Gate 1 (ratio AND entry floor) trips → fold if viable, else
@@ -1805,17 +2035,14 @@ const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundar
1805
2035
  * 4. Otherwise (or on `"both"`) → GC iff the cadence boundary crossed.
1806
2036
  */
1807
2037
  const runBoundedMaintenance = async (args, options) => {
1808
- if (args.disabled === true) return;
1809
2038
  const { storage, currentJsonKey, prevSeq } = args;
1810
- const maxFoldEntriesPerPass = options?.maxFoldEntriesPerPass ?? 20;
2039
+ const profile = options?.profile ?? MAINTENANCE_PROFILE_CF_FREE;
2040
+ const { maxFoldEntriesPerPass, gcMaxMarks, gcMaxSweeps, gcInterval } = profile;
1811
2041
  const minEntriesToCompact = options?.minEntriesToCompact ?? 50;
1812
- const gcMaxMarks = options?.gcMaxMarks ?? 20;
1813
- const gcMaxSweeps = options?.gcMaxSweeps ?? 10;
1814
- const gcInterval = options?.gcInterval ?? 4;
1815
2042
  const phasesPerTick = options?.phasesPerTick ?? "single";
1816
2043
  const signal = options?.signal;
1817
- const C = args.maxFoldBytes ?? 524288;
1818
- const E = MAINTENANCE_MAX_FOLD_ROWS;
2044
+ const C = args.maxFoldBytes ?? profile.maxFoldBytes;
2045
+ const E = profile.maxFoldRows;
1819
2046
  const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
1820
2047
  const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
1821
2048
  const gcOpts = {
@@ -1829,14 +2056,29 @@ const runBoundedMaintenance = async (args, options) => {
1829
2056
  const read = await readCurrentJson(storage, currentJsonKey, signal !== void 0 ? { signal } : void 0);
1830
2057
  if (read === null) return;
1831
2058
  const current = read.json;
1832
- const tailBytes = current.tail_bytes;
1833
2059
  const snapshotBytes = current.snapshot_bytes;
1834
2060
  const snapshotRows = current.snapshot_rows;
1835
- const nextSeq = current.next_seq;
1836
2061
  const logSeqStart = current.log_seq_start;
1837
- const gate1 = tailBytes / Math.max(snapshotBytes, 65536) >= 1 && nextSeq - logSeqStart >= minEntriesToCompact;
2062
+ const probeFloor = Math.max(logSeqStart, current.tail_hint);
2063
+ let nextSeq;
2064
+ if (args.observedTail !== void 0) nextSeq = Math.max(args.observedTail, probeFloor);
2065
+ else nextSeq = (await probeTailFrom(storage, collectionPrefix, probeFloor, signal !== void 0 ? { signal } : void 0)).tail;
2066
+ const gate1 = estimateTailBytes(current, nextSeq) / Math.max(snapshotBytes, 65536) >= 1 && nextSeq - logSeqStart >= minEntriesToCompact;
1838
2067
  const foldViable = snapshotBytes <= C && snapshotRows + maxFoldEntriesPerPass <= E;
1839
2068
  const gcDue = crossesGcBoundary(prevSeq, nextSeq, gcInterval);
2069
+ const refreshTailHintIfNeeded = async () => {
2070
+ if (nextSeq - current.tail_hint < 128) return;
2071
+ try {
2072
+ await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
2073
+ ...c,
2074
+ tail_hint: Math.max(c.tail_hint, nextSeq)
2075
+ }), signal !== void 0 ? { signal } : void 0);
2076
+ } catch {}
2077
+ };
2078
+ if (args.disabled === true) {
2079
+ await refreshTailHintIfNeeded();
2080
+ return;
2081
+ }
1840
2082
  const hardGc = crossesGcBoundary(prevSeq, nextSeq, gcInterval * 4);
1841
2083
  if (phasesPerTick === "single" && hardGc) {
1842
2084
  await runGc({
@@ -1854,6 +2096,7 @@ const runBoundedMaintenance = async (args, options) => {
1854
2096
  minEntriesToCompact,
1855
2097
  ceilingBytes: C,
1856
2098
  ceilingEntries: E,
2099
+ knownTail: nextSeq,
1857
2100
  ...signal !== void 0 && { signal }
1858
2101
  });
1859
2102
  if (phasesPerTick === "single") return;
@@ -1862,8 +2105,9 @@ const runBoundedMaintenance = async (args, options) => {
1862
2105
  collection,
1863
2106
  dimension: snapshotBytes > C ? "bytes" : "rows"
1864
2107
  });
2108
+ await refreshTailHintIfNeeded();
1865
2109
  if (nextSeq - (current.last_warned_seq ?? 0) >= 1e3) {
1866
- console.warn(`baerly-storage: collection "${collection}" is deferring compaction — its snapshot exceeds the fold ceiling (${snapshotBytes > C ? "bytes" : "rows"}), so the tail keeps growing and read amplification will climb. This is graduation-pending: the dataset has outgrown prototype-tier maintenance. On paid Cloudflare / Node you can raise BAERLY_MAINTENANCE_MAX_FOLD_BYTES — but on Cloudflare a cap above what a single isolate can fold makes folds get CPU-killed mid-flight and silently not land (no clean metric — watch snapshot age / object count); see docs/about/graduation.md. Otherwise, graduate to a server-backed database.`);
2110
+ console.warn(`baerly-storage: collection "${collection}" is deferring compaction — its snapshot exceeds the fold ceiling (${snapshotBytes > C ? `${C} bytes` : `${E} rows`}), so the tail keeps growing and read amplification will climb. The dataset has outgrown the maintenance envelope. On paid Cloudflare / Node you can raise BAERLY_MAINTENANCE_MAX_FOLD_BYTES — but on Cloudflare a cap above what a single isolate can fold makes folds get CPU-killed mid-flight and silently not land (no clean metric — watch snapshot age / object count); see docs/about/graduation.md. Otherwise, move to a database service (D1 / Postgres).`);
1867
2111
  try {
1868
2112
  await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
1869
2113
  ...c,
@@ -1889,7 +2133,25 @@ const runBoundedMaintenance = async (args, options) => {
1889
2133
  * returns `void`.
1890
2134
  */
1891
2135
  const dispatchInlineAwaited = (task) => task();
2136
+ /**
2137
+ * Parse the two host-agnostic ops-plane env vars into the
2138
+ * {@link MaintenanceDispatch} overrides. Shared by every host adapter so the
2139
+ * cross-host contract has one source of truth: `maxFoldBytes` (`C`) is the
2140
+ * finite `BAERLY_MAINTENANCE_MAX_FOLD_BYTES` or undefined; `disabled` is
2141
+ * `BAERLY_MAINTENANCE_DISABLE` truthy (set, non-empty, not `"0"`/`"false"`).
2142
+ */
2143
+ const parseMaintenanceEnv = (readEnv) => {
2144
+ const rawFoldBytes = readEnv("BAERLY_MAINTENANCE_MAX_FOLD_BYTES");
2145
+ const parsedFoldBytes = rawFoldBytes !== void 0 && rawFoldBytes !== "" ? Number(rawFoldBytes) : NaN;
2146
+ const maxFoldBytes = Number.isFinite(parsedFoldBytes) ? parsedFoldBytes : void 0;
2147
+ const rawDisable = readEnv("BAERLY_MAINTENANCE_DISABLE");
2148
+ const disabled = rawDisable !== void 0 && rawDisable !== "" && rawDisable !== "0" && rawDisable.toLowerCase() !== "false";
2149
+ return {
2150
+ ...maxFoldBytes !== void 0 && { maxFoldBytes },
2151
+ disabled
2152
+ };
2153
+ };
1892
2154
  //#endregion
1893
- export { matchesWire as C, assertWireSatisfiable as S, encodeIndexValue as _, runScheduledMaintenance as a, versionFromContent as b, runGc as c, loadSnapshotAsMap as d, snapshotKey as f, allIndexKeysFor as g, walkLogRange as h, runBoundedMaintenance as i, compact as l, readLogEntry as m, crossesGcBoundary as n, shouldFireMaintenance as o, foldLogEntriesOnto as p, dispatchInlineAwaited as r, rebuildIndex as s, CLOUDFLARE_FREE_TIER as t, encodeSnapshotBody as u, validateIndexDefinition as v, deepEqualDocumentValue as w, validateWire as x, assertNameNotReserved as y };
2155
+ export { deepEqualDocumentValue as A, findLogTail as C, validateWire as D, versionFromContent as E, assertWireSatisfiable as O, assertNameNotReserved as S, logObjectKey as T, encodeIndexValue as _, estimateTailBytes as a, loadSnapshotAsMap as b, runScheduledMaintenance as c, runGc as d, compact as f, allIndexKeysFor as g, walkLogRange as h, dispatchInlineAwaited as i, matchesWire as k, shouldFireMaintenance as l, readLogEntry as m, CLOUDFLARE_PAID_TIER as n, parseMaintenanceEnv as o, foldLogEntriesOnto as p, crossesGcBoundary as r, runBoundedMaintenance as s, CLOUDFLARE_FREE_TIER as t, rebuildIndex as u, validateIndexDefinition as v, probeTailFrom as w, snapshotKey as x, encodeSnapshotBody as y };
1894
2156
 
1895
- //# sourceMappingURL=maintenance-ClzBC1Vv.js.map
2157
+ //# sourceMappingURL=maintenance-B4Dki6Ig.js.map