@gusto/baerly-storage 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +29 -0
- package/README.md +176 -70
- package/dist/API.md +213 -139
- package/dist/CHANGELOG.md +434 -0
- package/dist/RECIPES.md +125 -0
- package/dist/THIRD-PARTY-LICENSES.txt +238 -0
- package/dist/{app-config-C0Bbw_Z6.d.ts → app-config-DyiFK3_k.d.ts} +55 -11
- package/dist/app-config.d.ts +1 -1
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +301 -301
- package/dist/auth.js.map +1 -1
- package/dist/aws4fetch.esm-p7eEzLCs.js +1702 -0
- package/dist/aws4fetch.esm-p7eEzLCs.js.map +1 -0
- package/dist/baerly.js +35 -15
- package/dist/baerly.js.map +1 -1
- package/dist/baerly.spec.json +342 -0
- package/dist/bytes-BXdamO3L.js.map +1 -1
- package/dist/{client-DYZfoZb9.d.ts → client-Fe8CwB73.d.ts} +10 -10
- package/dist/client-react.d.ts +3 -3
- package/dist/client-react.js +4 -4
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +4 -4
- package/dist/client.js +3 -3
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-CINJPKCW.js → cloudflare-DzKBrUlg.js} +178 -2
- package/dist/cloudflare-DzKBrUlg.js.map +1 -0
- package/dist/cloudflare.d.ts +49 -6
- package/dist/cloudflare.js +59 -26
- package/dist/cloudflare.js.map +1 -1
- package/dist/config-1tv0-Yzm.js +2693 -0
- package/dist/config-1tv0-Yzm.js.map +1 -0
- package/dist/constants-CfeMWQWk.js +222 -0
- package/dist/constants-CfeMWQWk.js.map +1 -0
- package/dist/context-DRZ3tZ5l.js.map +1 -1
- package/dist/{cost-CQt1YRkA.js → cost-DSMIVqzL.js} +43 -42
- package/dist/cost-DSMIVqzL.js.map +1 -0
- package/dist/current-json-C6nhPOm1.js +190 -0
- package/dist/current-json-C6nhPOm1.js.map +1 -0
- package/dist/{db-BM3y2JQw.d.ts → db-DsPkOVEH.d.ts} +26 -108
- package/dist/{deploy-CrAbfgCq.js → deploy-DonTr-CV.js} +12 -5
- package/dist/deploy-DonTr-CV.js.map +1 -0
- package/dist/dev-vite.d.ts +11 -7
- package/dist/dev-vite.js +7 -6
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/{doctor-Db-hzPmE.js → doctor-DSIyCDg_.js} +14 -136
- package/dist/doctor-DSIyCDg_.js.map +1 -0
- package/dist/{dump-BYA55NLP.js → dump-Bh5k26VQ.js} +10 -31
- package/dist/dump-Bh5k26VQ.js.map +1 -0
- package/dist/{errors-DOdL95kK.js → errors-Dfa3p78c.js} +25 -5
- package/dist/errors-Dfa3p78c.js.map +1 -0
- package/dist/{export-Bhbz90-z.js → export-DtF5tDIM.js} +11 -32
- package/dist/{export-Bhbz90-z.js.map → export-DtF5tDIM.js.map} +1 -1
- package/dist/{fsck-CPPZWt9p.js → fsck-D_djMO5D.js} +28 -39
- package/dist/fsck-D_djMO5D.js.map +1 -0
- package/dist/{http-Cw8p6y5b.js → http-DxM9MflO.js} +151 -79
- package/dist/http-DxM9MflO.js.map +1 -0
- package/dist/http.d.ts +42 -26
- package/dist/http.js +3 -2
- package/dist/{index-DwDTWPSU.d.ts → index-0rZcMBwy.d.ts} +10 -7
- package/dist/{index-C3JAH1Yo.d.ts → index-B6oexfOB.d.ts} +9 -9
- package/dist/{index-cUY-36AN.d.ts → index-BNRh23K9.d.ts} +21 -5
- package/dist/{index-DSvwEUeF.d.ts → index-BYk9oCNf.d.ts} +240 -202
- package/dist/index.d.ts +6 -6
- package/dist/index.js +5 -5
- package/dist/{indexes-DAu2AkTW.d.ts → indexes-Dn9hbGQG.d.ts} +2 -2
- package/dist/{inspect-Bdbzvi-_.js → inspect-6WZ-pC7-.js} +21 -36
- package/dist/inspect-6WZ-pC7-.js.map +1 -0
- package/dist/key-Di6yyxFu.js +35 -0
- package/dist/key-Di6yyxFu.js.map +1 -0
- package/dist/{snapshot-SS8USDUF.js → log-tail-D98U1JBo.js} +179 -84
- package/dist/log-tail-D98U1JBo.js.map +1 -0
- package/dist/{maintenance-ClzBC1Vv.js → maintenance-B4Dki6Ig.js} +438 -176
- package/dist/maintenance-B4Dki6Ig.js.map +1 -0
- package/dist/{maintenance-BRrThz8V.d.ts → maintenance-B_e0T99L.d.ts} +114 -26
- package/dist/{maintenance-BvOv6BMM.js → maintenance-C7coQp4c.js} +217 -94
- package/dist/maintenance-C7coQp4c.js.map +1 -0
- package/dist/maintenance.d.ts +3 -2
- package/dist/maintenance.js +3 -2
- package/dist/node.d.ts +176 -68
- package/dist/node.js +3 -2
- package/dist/{normalize-XWY25_sG.js → normalize-rjnZLwS4.js} +2 -2
- package/dist/normalize-rjnZLwS4.js.map +1 -0
- package/dist/{observability-CpOiEN0k.js → observability-wzs3-BL_.js} +279 -109
- package/dist/observability-wzs3-BL_.js.map +1 -0
- package/dist/observability.d.ts +3 -3
- package/dist/observability.js +1 -1
- package/dist/{plan-DOTl21vq.js → plan-a7PJHBCy.js} +7 -4
- package/dist/plan-a7PJHBCy.js.map +1 -0
- package/dist/{query-BmiQy328.js → query-rAtXNQQB.js} +1063 -1202
- package/dist/query-rAtXNQQB.js.map +1 -0
- package/dist/{rebuild-index-DC_nR6wP.js → rebuild-index-BVzQSZlm.js} +10 -32
- package/dist/rebuild-index-BVzQSZlm.js.map +1 -0
- package/dist/{request-BEF_OvMh.js → request-u0M51mhJ.js} +3 -3
- package/dist/{request-BEF_OvMh.js.map → request-u0M51mhJ.js.map} +1 -1
- package/dist/restore-R-4rbciV.js +778 -0
- package/dist/restore-R-4rbciV.js.map +1 -0
- package/dist/runtime-spec-qL3_lTTU.js +391 -0
- package/dist/runtime-spec-qL3_lTTU.js.map +1 -0
- package/dist/s3-http-B-z8th6O.d.ts +60 -0
- package/dist/s3.d.ts +47 -0
- package/dist/s3.js +45 -0
- package/dist/s3.js.map +1 -0
- package/dist/{shared-secret-BJdSwpgw.js → shared-secret-DgEicd3x.js} +13 -5
- package/dist/shared-secret-DgEicd3x.js.map +1 -0
- package/dist/src-BG20LZs7.js +2271 -0
- package/dist/src-BG20LZs7.js.map +1 -0
- package/dist/{src-zhEAmmF7.js → src-CYHFDrKz.js} +82 -99
- package/dist/src-CYHFDrKz.js.map +1 -0
- package/dist/{src-QQ7jDeL2.js → src-DYZ4H1aF.js} +44 -23
- package/dist/src-DYZ4H1aF.js.map +1 -0
- package/dist/subcommand-DoDA-K7d.js +197 -0
- package/dist/subcommand-DoDA-K7d.js.map +1 -0
- package/dist/time-DUqvFMHq.js +119 -0
- package/dist/time-DUqvFMHq.js.map +1 -0
- package/dist/{usage-CLjirdD-.js → usage-DBV1eQHC.js} +7 -22
- package/dist/usage-DBV1eQHC.js.map +1 -0
- package/dist/{validate-DCq_0DIn.js → validate-DtljsCh1.js} +51 -51
- package/dist/{validate-DCq_0DIn.js.map → validate-DtljsCh1.js.map} +1 -1
- package/dist/wire-izfGZDvr.js.map +1 -1
- package/package.json +77 -42
- package/dist/cloudflare-CINJPKCW.js.map +0 -1
- package/dist/config-DVfDGyw1.js +0 -278
- package/dist/config-DVfDGyw1.js.map +0 -1
- package/dist/cost-CQt1YRkA.js.map +0 -1
- package/dist/current-json-C-EyJqs0.js +0 -354
- package/dist/current-json-C-EyJqs0.js.map +0 -1
- package/dist/deploy-CrAbfgCq.js.map +0 -1
- package/dist/doctor-Db-hzPmE.js.map +0 -1
- package/dist/dump-BYA55NLP.js.map +0 -1
- package/dist/errors-DOdL95kK.js.map +0 -1
- package/dist/fsck-CPPZWt9p.js.map +0 -1
- package/dist/http-Cw8p6y5b.js.map +0 -1
- package/dist/inspect-Bdbzvi-_.js.map +0 -1
- package/dist/maintenance-BvOv6BMM.js.map +0 -1
- package/dist/maintenance-ClzBC1Vv.js.map +0 -1
- package/dist/normalize-XWY25_sG.js.map +0 -1
- package/dist/observability-CpOiEN0k.js.map +0 -1
- package/dist/plan-DOTl21vq.js.map +0 -1
- package/dist/query-BmiQy328.js.map +0 -1
- package/dist/rebuild-index-DC_nR6wP.js.map +0 -1
- package/dist/restore-C6IcmLeA.js +0 -771
- package/dist/restore-C6IcmLeA.js.map +0 -1
- package/dist/shared-secret-BJdSwpgw.js.map +0 -1
- package/dist/snapshot-SS8USDUF.js.map +0 -1
- package/dist/src-DU1sCMlr.js +0 -6236
- package/dist/src-DU1sCMlr.js.map +0 -1
- package/dist/src-QQ7jDeL2.js.map +0 -1
- package/dist/src-zhEAmmF7.js.map +0 -1
- package/dist/subcommand-DZcHcsYp.js +0 -5122
- package/dist/subcommand-DZcHcsYp.js.map +0 -1
- package/dist/usage-CLjirdD-.js.map +0 -1
- /package/dist/{chunk-EAsCxrDo.js → rolldown-runtime-EAsCxrDo.js} +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { a as decodeJsonBytes, i as readCurrentJson, o as encodeJsonBytes, r as logSeqStartOf, t as casUpdateCurrentJson } from "./current-json-C6nhPOm1.js";
|
|
2
|
+
import { a as GC_MAX_PENDING_CANDIDATES, l as MAINTENANCE_PROFILE_CF_FREE, o as GC_PENDING_CONTENT_TYPE, s as LOG_FORWARD_PROBE_CAP, u as MAINTENANCE_PROFILE_CF_PAID } from "./constants-CfeMWQWk.js";
|
|
3
|
+
import { t as BaerlyError } from "./errors-Dfa3p78c.js";
|
|
3
4
|
import { i as noopMetricsRecorder, n as getCurrentContext } from "./context-DRZ3tZ5l.js";
|
|
4
5
|
//#region packages/protocol/src/coordination/gc-pending.ts
|
|
5
6
|
/**
|
|
@@ -72,44 +73,112 @@ const createGcPending = async (storage, key, initial, opts) => {
|
|
|
72
73
|
};
|
|
73
74
|
};
|
|
74
75
|
/**
|
|
75
|
-
* Read-modify-write `gc/pending.json` under CAS
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
76
|
+
* Read-modify-write `gc/pending.json` under CAS, with a bounded retry
|
|
77
|
+
* loop. Each attempt re-reads the latest body, hands the `mutator`
|
|
78
|
+
* that latest value, and writes the result with
|
|
79
|
+
* `If-Match: <latest etag>`. On `Conflict` (a concurrent writer landed
|
|
80
|
+
* between this attempt's read and write) the loop re-reads and
|
|
81
|
+
* re-applies the mutator, up to {@link GC_PENDING_CAS_MAX_ATTEMPTS}
|
|
82
|
+
* total attempts, then surfaces `Conflict`.
|
|
83
|
+
*
|
|
84
|
+
* `mutator` MUST be a pure, deterministic MERGE of this pass's results
|
|
85
|
+
* INTO the `latest` value it is handed — it is called once per attempt
|
|
86
|
+
* (so it must not have side effects), and it MUST read `latest` rather
|
|
87
|
+
* than ignore it and write a precomputed set. Ignoring `latest` would
|
|
88
|
+
* silently overwrite a concurrent pass's candidates: the
|
|
89
|
+
* If-Match succeeds (it used the fresh etag), so no conflict is
|
|
90
|
+
* raised, and the concurrent marks are lost. See {@link mergeGcPending}.
|
|
91
|
+
*
|
|
92
|
+
* @throws BaerlyError{code:"Conflict"} — another writer kept landing a
|
|
93
|
+
* write between this function's read and write for every
|
|
94
|
+
* attempt.
|
|
86
95
|
* @throws BaerlyError{code:"InvalidResponse"} — `key` does not exist
|
|
87
96
|
* (use {@link createGcPending} instead) or body doesn't
|
|
88
97
|
* parse / fails the shape guard.
|
|
89
98
|
*/
|
|
90
99
|
const casUpdateGcPending = async (storage, key, mutator, opts) => {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
100
|
+
let lastConflict;
|
|
101
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
102
|
+
const existing = await readGcPending(storage, key, opts);
|
|
103
|
+
if (existing === null) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key} does not exist; use createGcPending first`);
|
|
104
|
+
const next = mutator(structuredClone(existing.json));
|
|
105
|
+
assertGcPending(next, key);
|
|
106
|
+
const body = encodeJsonBytes(next);
|
|
107
|
+
const putOpts = {
|
|
108
|
+
ifMatch: existing.etag,
|
|
109
|
+
contentType: GC_PENDING_CONTENT_TYPE,
|
|
110
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
111
|
+
};
|
|
112
|
+
try {
|
|
113
|
+
return {
|
|
114
|
+
json: next,
|
|
115
|
+
etag: (await storage.put(key, body, putOpts)).etag
|
|
116
|
+
};
|
|
117
|
+
} catch (error) {
|
|
118
|
+
const translated = translateCasError(error, key);
|
|
119
|
+
if (translated.code !== "Conflict") throw translated;
|
|
120
|
+
lastConflict = translated;
|
|
121
|
+
}
|
|
106
122
|
}
|
|
123
|
+
throw lastConflict;
|
|
124
|
+
};
|
|
125
|
+
/**
|
|
126
|
+
* Pure merge of one GC pass's results INTO the `latest`
|
|
127
|
+
* `gc/pending.json` value, used as the {@link casUpdateGcPending}
|
|
128
|
+
* mutator. Deterministic, no I/O — the kernel unit + mutation test
|
|
129
|
+
* surface. Splitting it out is what makes the CAS write correct under
|
|
130
|
+
* concurrency: each retry re-merges against fresh `latest` rather than
|
|
131
|
+
* overwriting it with a stale precomputed set.
|
|
132
|
+
*
|
|
133
|
+
* Semantics:
|
|
134
|
+
* - **candidates**: start from `latest.candidates`, DROP any whose key
|
|
135
|
+
* this pass already DELETED (`sweptKeys`), then APPEND every fresh
|
|
136
|
+
* mark whose key is not already present AND was not itself swept
|
|
137
|
+
* this pass (a `graceMillis: 0` pass can mark-and-sweep a key in one
|
|
138
|
+
* go — re-adding it to the ledger after deleting it would be wrong;
|
|
139
|
+
* dedup also drops a key a concurrent pass already marked). Capped at
|
|
140
|
+
* `maxCandidates`.
|
|
141
|
+
* - **last_swept_at**: the lexicographically LATER of `latest`'s and
|
|
142
|
+
* this pass's (ISO-8601 UTC; `""` sorts lowest so a real timestamp
|
|
143
|
+
* always wins). Never regress a concurrent pass's newer sweep time.
|
|
144
|
+
* - **content_scan_cursor**: rotation cursor — liveness, not
|
|
145
|
+
* correctness; just don't regress forward progress. The two sides
|
|
146
|
+
* are NOT symmetric: `pass.nextContentCursor === undefined` means
|
|
147
|
+
* THIS pass examined to the END of the keyspace (WRAP — the most-
|
|
148
|
+
* advanced position), whereas `latest.content_scan_cursor` absent
|
|
149
|
+
* means the stored ledger has no cursor yet (start-from-beginning).
|
|
150
|
+
* So:
|
|
151
|
+
* - our pass WRAPPED ⇒ result wraps (key omitted) — our pass is
|
|
152
|
+
* the most advanced; this is the single-writer rotation reset.
|
|
153
|
+
* - else (our pass advanced to a defined cursor):
|
|
154
|
+
* - latest cursor defined ⇒ take the lexicographically GREATER
|
|
155
|
+
* (don't regress a concurrent pass's advance);
|
|
156
|
+
* - latest cursor absent ⇒ keep OUR advance (don't lose it).
|
|
157
|
+
*/
|
|
158
|
+
const mergeGcPending = (latest, pass) => {
|
|
159
|
+
const surviving = latest.candidates.filter((c) => !pass.sweptKeys.has(c.key));
|
|
160
|
+
const present = new Set(surviving.map((c) => c.key));
|
|
161
|
+
const appended = [];
|
|
162
|
+
for (const c of pass.newCandidates) if (!pass.sweptKeys.has(c.key) && !present.has(c.key)) {
|
|
163
|
+
present.add(c.key);
|
|
164
|
+
appended.push(c);
|
|
165
|
+
}
|
|
166
|
+
const candidates = [...surviving, ...appended].slice(0, pass.maxCandidates);
|
|
167
|
+
const lastSweptAt = pass.lastSweptAt > latest.last_swept_at ? pass.lastSweptAt : latest.last_swept_at;
|
|
168
|
+
const latestCursor = latest.content_scan_cursor;
|
|
169
|
+
const passCursor = pass.nextContentCursor;
|
|
170
|
+
let cursor;
|
|
171
|
+
if (passCursor === void 0) cursor = void 0;
|
|
172
|
+
else if (latestCursor !== void 0) cursor = passCursor > latestCursor ? passCursor : latestCursor;
|
|
173
|
+
else cursor = passCursor;
|
|
107
174
|
return {
|
|
108
|
-
|
|
109
|
-
|
|
175
|
+
schema_version: 1,
|
|
176
|
+
candidates,
|
|
177
|
+
last_swept_at: lastSweptAt,
|
|
178
|
+
...cursor !== void 0 && { content_scan_cursor: cursor }
|
|
110
179
|
};
|
|
111
180
|
};
|
|
112
|
-
const VALID_REASONS = new Set([
|
|
181
|
+
const VALID_REASONS = /* @__PURE__ */ new Set([
|
|
113
182
|
"stale-log",
|
|
114
183
|
"orphan-snapshot",
|
|
115
184
|
"orphan-content"
|
|
@@ -560,6 +629,13 @@ const versionFromContent = async (body) => {
|
|
|
560
629
|
for (let i = 0; i < bytes.length; i++) hex += bytes[i].toString(16).padStart(2, "0");
|
|
561
630
|
return hex.slice(0, VERSION_HEX_LENGTH);
|
|
562
631
|
};
|
|
632
|
+
/**
|
|
633
|
+
* The single constructor for a log-object key:
|
|
634
|
+
* `<manifestPrefix>/log/<seq>.json`. `manifestPrefix` is the
|
|
635
|
+
* `<…>/manifests/<collection>` segment — this helper adds `/log/`.
|
|
636
|
+
* Route all callers here so the key shape lives in one place.
|
|
637
|
+
*/
|
|
638
|
+
const logObjectKey = (manifestPrefix, seq) => `${manifestPrefix}/log/${seq}.json`;
|
|
563
639
|
//#endregion
|
|
564
640
|
//#region packages/protocol/src/snapshot-hash.ts
|
|
565
641
|
/**
|
|
@@ -582,10 +658,170 @@ const snapshotHash = async (bytes) => {
|
|
|
582
658
|
return hex;
|
|
583
659
|
};
|
|
584
660
|
//#endregion
|
|
661
|
+
//#region packages/server/src/log-tail.ts
|
|
662
|
+
/**
|
|
663
|
+
* Tolerant forward-probe for the true committed log tail. The strict
|
|
664
|
+
* `[log_seq_start, hint)` range is dense (a hole is corruption, owned
|
|
665
|
+
* by `walkLogRange`); this owns only the tolerant `[hint, ∞)` probe,
|
|
666
|
+
* where `hint` is a lower bound — the real tail may sit at or past it.
|
|
667
|
+
*/
|
|
668
|
+
/**
|
|
669
|
+
* Discover the true committed tail and fold entries in `[hint, tail)`.
|
|
670
|
+
* `tail` is the first empty seq (>= hint); `entries` are the
|
|
671
|
+
* LogEntries in `[hint, tail)` in seq order. `cap` bounds the walk;
|
|
672
|
+
* exhausting it THROWS `Internal` (never silently truncates) — see
|
|
673
|
+
* the cap-exhaustion comment below.
|
|
674
|
+
*/
|
|
675
|
+
const probeTailFrom = async (storage, logPrefix, hint, opts) => {
|
|
676
|
+
const cap = opts?.cap ?? 1e5;
|
|
677
|
+
const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
678
|
+
const entries = [];
|
|
679
|
+
for (let i = 0; i < cap; i++) {
|
|
680
|
+
const seq = hint + i;
|
|
681
|
+
const key = logObjectKey(logPrefix, seq);
|
|
682
|
+
const got = await storage.get(key, getOpts);
|
|
683
|
+
if (got === null) return {
|
|
684
|
+
tail: seq,
|
|
685
|
+
entries
|
|
686
|
+
};
|
|
687
|
+
try {
|
|
688
|
+
entries.push(decodeJsonBytes(got.body));
|
|
689
|
+
} catch (error) {
|
|
690
|
+
throw new BaerlyError("InvalidResponse", `probeTailFrom: malformed log entry at ${key}: ${error.message}`, { cause: error });
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
throw new BaerlyError("Internal", `probeTailFrom: forward probe exceeded ${cap} from hint ${hint} on ${logPrefix}`);
|
|
694
|
+
};
|
|
695
|
+
/**
|
|
696
|
+
* First empty log seq at/after `hint` — the slot a new commit creates —
|
|
697
|
+
* found by galloping search (`O(log gap)` GETs vs `probeTailFrom`'s
|
|
698
|
+
* `O(gap)`). Used by the writer's per-commit tail-find, where `tail_hint`
|
|
699
|
+
* can lag the true tail. Position only; no bodies.
|
|
700
|
+
*
|
|
701
|
+
* PRECONDITION: occupancy MUST be a dense prefix from `hint` (galloping
|
|
702
|
+
* assumes monotone occupied→empty; a hole ABOVE `hint` would be skipped
|
|
703
|
+
* and the gallop could return a tail past live entries). Contrast
|
|
704
|
+
* `probeTailFrom`, which stops at the first 404 and is hole-tolerant.
|
|
705
|
+
*/
|
|
706
|
+
const findLogTail = async (storage, logPrefix, hint, opts) => {
|
|
707
|
+
const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
708
|
+
const exists = async (seq) => await storage.get(logObjectKey(logPrefix, seq), getOpts) !== null;
|
|
709
|
+
if (!await exists(hint)) return hint;
|
|
710
|
+
let lo = hint;
|
|
711
|
+
let step = 1;
|
|
712
|
+
let hi = hint + step;
|
|
713
|
+
while (await exists(hi)) {
|
|
714
|
+
lo = hi;
|
|
715
|
+
step *= 2;
|
|
716
|
+
hi = lo + step;
|
|
717
|
+
if (step > 1e5) throw new BaerlyError("Internal", `findLogTail: galloping probe exceeded ${LOG_FORWARD_PROBE_CAP} from hint ${hint} on ${logPrefix}`);
|
|
718
|
+
}
|
|
719
|
+
while (hi - lo > 1) {
|
|
720
|
+
const mid = lo + Math.floor((hi - lo) / 2);
|
|
721
|
+
if (await exists(mid)) lo = mid;
|
|
722
|
+
else hi = mid;
|
|
723
|
+
}
|
|
724
|
+
return hi;
|
|
725
|
+
};
|
|
726
|
+
//#endregion
|
|
585
727
|
//#region packages/server/src/names.ts
|
|
586
|
-
/** Reject names in the system-reserved leading-"_" namespace. See docs/adr/
|
|
728
|
+
/** Reject names in the system-reserved leading-"_" namespace. See docs/adr/003-layout-versioning-cordon.md. */
|
|
587
729
|
const assertNameNotReserved = (name, context) => {
|
|
588
|
-
if (name.startsWith("_")) throw new BaerlyError("InvalidConfig", `${context}: "${name}" is reserved — names beginning with "_" are reserved for system use. See docs/adr/
|
|
730
|
+
if (name.startsWith("_")) throw new BaerlyError("InvalidConfig", `${context}: "${name}" is reserved — names beginning with "_" are reserved for system use. See docs/adr/003-layout-versioning-cordon.md.`);
|
|
731
|
+
};
|
|
732
|
+
//#endregion
|
|
733
|
+
//#region packages/server/src/snapshot.ts
|
|
734
|
+
/**
|
|
735
|
+
* Snapshot file shape + parse. Lifted out of `./compactor.ts` so that
|
|
736
|
+
* the kernel barrel (`packages/server/src/index.ts`) — which re-exports
|
|
737
|
+
* these four symbols — does not transitively pull `withObservability`
|
|
738
|
+
* via `compact()`. `./compactor.ts` imports its own snapshot primitives
|
|
739
|
+
* back from this module; the public API surface is unchanged.
|
|
740
|
+
*
|
|
741
|
+
* @see ../../../docs/spec/sync-protocol.md
|
|
742
|
+
*/
|
|
743
|
+
/**
|
|
744
|
+
* Single-level snapshot at L9; the level prefix is forward-compatible
|
|
745
|
+
* with future L0..L9 rolling merges without a wire change.
|
|
746
|
+
*/
|
|
747
|
+
const SNAPSHOT_LEVEL = 9;
|
|
748
|
+
/**
|
|
749
|
+
* `minSeq` and `maxSeq` are padded to {@link SEQ_DIGITS} digits so
|
|
750
|
+
* `list(prefix)` returns snapshots in seq order without needing to
|
|
751
|
+
* parse the filename. 12 digits caps at ~10^12 entries per collection;
|
|
752
|
+
* a real bucket hits S3's per-prefix QPS ceiling long before that.
|
|
753
|
+
*/
|
|
754
|
+
const SEQ_DIGITS = 12;
|
|
755
|
+
const MAX_SEQ = 10 ** SEQ_DIGITS - 1;
|
|
756
|
+
/**
|
|
757
|
+
* Build the on-bucket key for a snapshot file. `sha256` must be the
|
|
758
|
+
* lowercase hex digest of the canonical body
|
|
759
|
+
* ({@link encodeSnapshotBody}); the filename embeds it so a crashed
|
|
760
|
+
* mid-PUT body that doesn't match its own filename hash is rejected
|
|
761
|
+
* by readers as "missing."
|
|
762
|
+
*
|
|
763
|
+
* @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
|
|
764
|
+
* out of range or `sha256` is not 64 hex chars.
|
|
765
|
+
*/
|
|
766
|
+
const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
|
|
767
|
+
if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
|
|
768
|
+
if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
|
|
769
|
+
const pad = (n) => n.toString().padStart(SEQ_DIGITS, "0");
|
|
770
|
+
return `${collectionPrefix}/snapshot/L${SNAPSHOT_LEVEL}/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
|
|
771
|
+
};
|
|
772
|
+
/**
|
|
773
|
+
* Canonical byte encoding of a {@link SnapshotBody}. Only the
|
|
774
|
+
* compactor produces snapshots, and only this function does the
|
|
775
|
+
* serialisation — so the byte output is deterministic for a given
|
|
776
|
+
* `SnapshotBody` (identical input ⇒ identical bytes ⇒ identical
|
|
777
|
+
* filename hash).
|
|
778
|
+
*/
|
|
779
|
+
const encodeSnapshotBody = (s) => encodeJsonBytes(s);
|
|
780
|
+
/**
|
|
781
|
+
* Load a snapshot file as a `Map<_id, DocumentData>`. Verifies the
|
|
782
|
+
* body hash against the filename, checks the schema version, and
|
|
783
|
+
* cross-checks `body.collection` against the caller's expectation.
|
|
784
|
+
*
|
|
785
|
+
* @example
|
|
786
|
+
* ```ts
|
|
787
|
+
* import { loadSnapshotAsMap } from "@gusto/baerly-storage";
|
|
788
|
+
*
|
|
789
|
+
* const map =
|
|
790
|
+
* srcCurrent.snapshot === null
|
|
791
|
+
* ? new Map<string, DocumentData>()
|
|
792
|
+
* : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
|
|
793
|
+
* ```
|
|
794
|
+
*
|
|
795
|
+
* @throws BaerlyError code="Internal" — pointer resolves to no body,
|
|
796
|
+
* or the body's SHA-256 doesn't match the hash embedded in the
|
|
797
|
+
* filename. (Protocol invariant violation; a corrupted snapshot
|
|
798
|
+
* the reader cannot continue past.)
|
|
799
|
+
* @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
|
|
800
|
+
* isn't an object, has an unsupported `schema_version`, or carries a
|
|
801
|
+
* `collection` that doesn't match `expectedCollection`.
|
|
802
|
+
*/
|
|
803
|
+
const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
|
|
804
|
+
const got = await storage.get(key, signal !== void 0 ? { signal } : void 0);
|
|
805
|
+
if (got === null) throw new BaerlyError("Internal", `compact: snapshot pointer ${key} resolves to no body; protocol violation`);
|
|
806
|
+
const filename = key.slice(key.lastIndexOf("/") + 1).replace(/\.json$/, "");
|
|
807
|
+
const lastDash = filename.lastIndexOf("-");
|
|
808
|
+
const expectedHash = filename.slice(lastDash + 1);
|
|
809
|
+
const actualHash = await snapshotHash(got.body);
|
|
810
|
+
if (actualHash !== expectedHash) throw new BaerlyError("Internal", `compact: snapshot ${key} body hash mismatch (got ${actualHash}); protocol violation`);
|
|
811
|
+
let parsed;
|
|
812
|
+
try {
|
|
813
|
+
parsed = decodeJsonBytes(got.body);
|
|
814
|
+
} catch (error) {
|
|
815
|
+
throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
|
|
816
|
+
}
|
|
817
|
+
if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not an object`);
|
|
818
|
+
const body = parsed;
|
|
819
|
+
if (body.schema_version !== 1) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} has unsupported schema_version ${String(body.schema_version)}`);
|
|
820
|
+
if (body.collection !== expectedCollection) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} carries collection ${body.collection}, expected ${expectedCollection}`);
|
|
821
|
+
if (!Array.isArray(body.docs)) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body.docs is not an array`);
|
|
822
|
+
const out = /* @__PURE__ */ new Map();
|
|
823
|
+
for (const row of body.docs) out.set(row._id, row.body);
|
|
824
|
+
return out;
|
|
589
825
|
};
|
|
590
826
|
//#endregion
|
|
591
827
|
//#region packages/server/src/indexes.ts
|
|
@@ -615,8 +851,7 @@ const assertNameNotReserved = (name, context) => {
|
|
|
615
851
|
*
|
|
616
852
|
* `<logPrefix>/index/<indexName>/<value-b32>/<docId>.json`
|
|
617
853
|
*
|
|
618
|
-
* Wire shape (composite
|
|
619
|
-
* the read path today):
|
|
854
|
+
* Wire shape (composite):
|
|
620
855
|
*
|
|
621
856
|
* `<logPrefix>/index/<indexName>/<a-b32>/<b-b32>/<docId>.json`
|
|
622
857
|
*
|
|
@@ -637,11 +872,13 @@ const assertNameNotReserved = (name, context) => {
|
|
|
637
872
|
* and `$in` walks over numeric fields are first-class — no
|
|
638
873
|
* planner fall-back to full-scan).
|
|
639
874
|
*
|
|
640
|
-
*
|
|
641
|
-
*
|
|
875
|
+
* Composite read routing is left-anchored. The planner may consume an
|
|
876
|
+
* equality prefix plus a range / `$in` tail slot; predicates that do
|
|
877
|
+
* not match a routable prefix fall back to the snapshot+log fold.
|
|
642
878
|
*
|
|
643
|
-
* @see docs/spec/sync-protocol.md —
|
|
644
|
-
*
|
|
879
|
+
* @see docs/spec/sync-protocol.md — index emission is hybrid around
|
|
880
|
+
* the commit: additive (new) keys are PUT before the committing
|
|
881
|
+
* `log/<seq>` create, and stale-key DELETEs land after it.
|
|
645
882
|
*/
|
|
646
883
|
/** Path-safe segment name. */
|
|
647
884
|
const INDEX_NAME_RE = /^[a-z][a-z0-9_]*$/;
|
|
@@ -653,7 +890,7 @@ const INDEX_NAME_RE = /^[a-z][a-z0-9_]*$/;
|
|
|
653
890
|
const B32_ALPHABET = "0123456789abcdefghijklmnopqrstuv";
|
|
654
891
|
/**
|
|
655
892
|
* Reject malformed {@link IndexDefinition} shapes synchronously, at
|
|
656
|
-
* writer construction. See docs/adr/
|
|
893
|
+
* writer construction. See docs/adr/003-layout-versioning-cordon.md.
|
|
657
894
|
*
|
|
658
895
|
* @throws BaerlyError code="InvalidConfig" — reserved leading-`_` `name`.
|
|
659
896
|
* @throws BaerlyError code="SchemaError" — `name` fails the regex, or `on` is empty.
|
|
@@ -820,8 +1057,8 @@ const indexKeyPrefix = (logPrefix, indexName) => `${logPrefix}/index/${indexName
|
|
|
820
1057
|
* Single-field: `<logPrefix>/index/<name>/<v0-b32>/<docId>.json`.
|
|
821
1058
|
* Composite: `<logPrefix>/index/<name>/<v0-b32>/<v1-b32>/.../<docId>.json`.
|
|
822
1059
|
*
|
|
823
|
-
* Composite
|
|
824
|
-
*
|
|
1060
|
+
* Composite entries are read-path routable when the planner can consume
|
|
1061
|
+
* a left-anchored prefix of the indexed tuple.
|
|
825
1062
|
*/
|
|
826
1063
|
const indexKeyFor = (logPrefix, def, values, docId) => {
|
|
827
1064
|
const segments = values.map(encodeIndexValue).join("/");
|
|
@@ -835,7 +1072,7 @@ const indexKeyFor = (logPrefix, def, values, docId) => {
|
|
|
835
1072
|
*
|
|
836
1073
|
* @throws BaerlyError code="SchemaError" — `on` contains a dotted
|
|
837
1074
|
* path, or the indexed field's value is an array. Both restrictions
|
|
838
|
-
* are
|
|
1075
|
+
* are current 0.x limitations; multi-value (array) indexes are a follow-up.
|
|
839
1076
|
*/
|
|
840
1077
|
const projectIndexValues = (def, body) => {
|
|
841
1078
|
if (body === void 0) return;
|
|
@@ -909,7 +1146,7 @@ const allIndexKeysFor = (logPrefix, defs, body, docId) => {
|
|
|
909
1146
|
* every caller.
|
|
910
1147
|
*
|
|
911
1148
|
* @throws BaerlyError code="Internal" when the key resolves to no
|
|
912
|
-
* body — a hole inside `[log_seq_start,
|
|
1149
|
+
* body — a hole inside `[log_seq_start, tail_hint)` is a corruption
|
|
913
1150
|
* the kernel can't continue past.
|
|
914
1151
|
* @throws BaerlyError code="InvalidResponse" when the body isn't
|
|
915
1152
|
* valid JSON.
|
|
@@ -922,14 +1159,12 @@ const readLogEntry = async (storage, key, opts) => {
|
|
|
922
1159
|
* byteLength of the STORED log-object body exactly as it was GET back
|
|
923
1160
|
* from storage (`got.body.byteLength`).
|
|
924
1161
|
*
|
|
925
|
-
* The compactor uses this to
|
|
926
|
-
*
|
|
927
|
-
*
|
|
928
|
-
*
|
|
929
|
-
* `
|
|
930
|
-
*
|
|
931
|
-
* assumption (a re-encode of the parsed `LogEntry` could drift if
|
|
932
|
-
* key-order/whitespace ever diverged from what was PUT).
|
|
1162
|
+
* The compactor uses this to sum the byte cost of the folded slice.
|
|
1163
|
+
* The stored body byteLength IS what the writer PUT for each entry,
|
|
1164
|
+
* so summing `got.body.byteLength` over the folded range yields the
|
|
1165
|
+
* exact cost — no re-encode, no round-trip assumption (a re-encode
|
|
1166
|
+
* of the parsed `LogEntry` could drift if key-order/whitespace ever
|
|
1167
|
+
* diverged from what was PUT).
|
|
933
1168
|
*
|
|
934
1169
|
* Same error semantics as {@link readLogEntry}.
|
|
935
1170
|
*/
|
|
@@ -967,7 +1202,7 @@ const walkLogRange = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) =
|
|
|
967
1202
|
* this is the bytes-aware core `walkLogRange` is implemented on top of.
|
|
968
1203
|
*
|
|
969
1204
|
* The compactor consumes this so it gets BOTH the parsed entries (for
|
|
970
|
-
* the fold) AND the exact byte sum (
|
|
1205
|
+
* the fold) AND the exact byte sum (for `mean_entry_bytes` stamping) from a
|
|
971
1206
|
* single walk.
|
|
972
1207
|
*/
|
|
973
1208
|
const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) => {
|
|
@@ -980,7 +1215,7 @@ const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive
|
|
|
980
1215
|
const promises = [];
|
|
981
1216
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
982
1217
|
const seq = fromSeq + i;
|
|
983
|
-
promises.push(readLogEntryWithBytes(storage,
|
|
1218
|
+
promises.push(readLogEntryWithBytes(storage, logObjectKey(logPrefix, seq), opts));
|
|
984
1219
|
}
|
|
985
1220
|
const results = await Promise.all(promises);
|
|
986
1221
|
for (let i = 0; i < results.length; i++) out[chunkStart + i] = results[i];
|
|
@@ -1030,76 +1265,6 @@ const foldLogEntriesOnto = (map, entries, opts) => {
|
|
|
1030
1265
|
}
|
|
1031
1266
|
}
|
|
1032
1267
|
};
|
|
1033
|
-
const MAX_SEQ = 10 ** 12 - 1;
|
|
1034
|
-
/**
|
|
1035
|
-
* Build the on-bucket key for a snapshot file. `sha256` must be the
|
|
1036
|
-
* lowercase hex digest of the canonical body
|
|
1037
|
-
* ({@link encodeSnapshotBody}); the filename embeds it so a crashed
|
|
1038
|
-
* mid-PUT body that doesn't match its own filename hash is rejected
|
|
1039
|
-
* by readers as "missing."
|
|
1040
|
-
*
|
|
1041
|
-
* @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
|
|
1042
|
-
* out of range or `sha256` is not 64 hex chars.
|
|
1043
|
-
*/
|
|
1044
|
-
const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
|
|
1045
|
-
if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
|
|
1046
|
-
if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
|
|
1047
|
-
const pad = (n) => n.toString().padStart(12, "0");
|
|
1048
|
-
return `${collectionPrefix}/snapshot/L9/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
|
|
1049
|
-
};
|
|
1050
|
-
/**
|
|
1051
|
-
* Canonical byte encoding of a {@link SnapshotBody}. Only the
|
|
1052
|
-
* compactor produces snapshots, and only this function does the
|
|
1053
|
-
* serialisation — so the byte output is deterministic for a given
|
|
1054
|
-
* `SnapshotBody` (identical input ⇒ identical bytes ⇒ identical
|
|
1055
|
-
* filename hash).
|
|
1056
|
-
*/
|
|
1057
|
-
const encodeSnapshotBody = (s) => encodeJsonBytes(s);
|
|
1058
|
-
/**
|
|
1059
|
-
* Load a snapshot file as a `Map<_id, DocumentData>`. Verifies the
|
|
1060
|
-
* body hash against the filename, checks the schema version, and
|
|
1061
|
-
* cross-checks `body.collection` against the caller's expectation.
|
|
1062
|
-
*
|
|
1063
|
-
* @example
|
|
1064
|
-
* ```ts
|
|
1065
|
-
* import { loadSnapshotAsMap } from "@gusto/baerly-storage";
|
|
1066
|
-
*
|
|
1067
|
-
* const map =
|
|
1068
|
-
* srcCurrent.snapshot === null
|
|
1069
|
-
* ? new Map<string, DocumentData>()
|
|
1070
|
-
* : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
|
|
1071
|
-
* ```
|
|
1072
|
-
*
|
|
1073
|
-
* @throws BaerlyError code="Internal" — pointer resolves to no body,
|
|
1074
|
-
* or the body's SHA-256 doesn't match the hash embedded in the
|
|
1075
|
-
* filename. (Protocol invariant violation; a corrupted snapshot
|
|
1076
|
-
* the reader cannot continue past.)
|
|
1077
|
-
* @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
|
|
1078
|
-
* isn't an object, has an unsupported `schema_version`, or carries a
|
|
1079
|
-
* `collection` that doesn't match `expectedCollection`.
|
|
1080
|
-
*/
|
|
1081
|
-
const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
|
|
1082
|
-
const got = await storage.get(key, signal !== void 0 ? { signal } : void 0);
|
|
1083
|
-
if (got === null) throw new BaerlyError("Internal", `compact: snapshot pointer ${key} resolves to no body; protocol violation`);
|
|
1084
|
-
const filename = key.slice(key.lastIndexOf("/") + 1).replace(/\.json$/, "");
|
|
1085
|
-
const lastDash = filename.lastIndexOf("-");
|
|
1086
|
-
const expectedHash = filename.slice(lastDash + 1);
|
|
1087
|
-
const actualHash = await snapshotHash(got.body);
|
|
1088
|
-
if (actualHash !== expectedHash) throw new BaerlyError("Internal", `compact: snapshot ${key} body hash mismatch (got ${actualHash}); protocol violation`);
|
|
1089
|
-
let parsed;
|
|
1090
|
-
try {
|
|
1091
|
-
parsed = decodeJsonBytes(got.body);
|
|
1092
|
-
} catch (error) {
|
|
1093
|
-
throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
|
|
1094
|
-
}
|
|
1095
|
-
if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not an object`);
|
|
1096
|
-
const body = parsed;
|
|
1097
|
-
if (body.schema_version !== 1) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} has unsupported schema_version ${String(body.schema_version)}`);
|
|
1098
|
-
if (body.collection !== expectedCollection) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} carries collection ${body.collection}, expected ${expectedCollection}`);
|
|
1099
|
-
const out = /* @__PURE__ */ new Map();
|
|
1100
|
-
for (const row of body.docs) out.set(row._id, row.body);
|
|
1101
|
-
return out;
|
|
1102
|
-
};
|
|
1103
1268
|
//#endregion
|
|
1104
1269
|
//#region packages/server/src/compactor.ts
|
|
1105
1270
|
/**
|
|
@@ -1115,8 +1280,7 @@ const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
|
|
|
1115
1280
|
* "missing." The atomic moment is the CAS-swap of
|
|
1116
1281
|
* `current.json.snapshot`, which is a single conditional PUT.
|
|
1117
1282
|
*
|
|
1118
|
-
* Order of operations
|
|
1119
|
-
* REVERSED ordering): PUT snapshot body → CAS-advance `current.json`.
|
|
1283
|
+
* Order of operations: PUT snapshot body → CAS-advance `current.json`.
|
|
1120
1284
|
* A crash before the swap leaves an orphan snapshot file; the swap
|
|
1121
1285
|
* succeeds iff our captured ETag still matches.
|
|
1122
1286
|
*
|
|
@@ -1124,7 +1288,7 @@ const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
|
|
|
1124
1288
|
* single monolithic snapshot (every compaction rewrites one well-
|
|
1125
1289
|
* known key) makes partial writes catastrophic and serializes every
|
|
1126
1290
|
* writer on one key. WAL-checkpointing in the rqlite / Raft tradition
|
|
1127
|
-
* requires a consensus layer
|
|
1291
|
+
* requires a consensus layer baerly-storage doesn't have. The Litestream-
|
|
1128
1292
|
* style multi-level scheme — small fixed number of levels, chunked
|
|
1129
1293
|
* files keyed by sequence range — keeps random GETs cheap on
|
|
1130
1294
|
* S3-compatible storage and avoids LIST. Current ship is single-
|
|
@@ -1189,7 +1353,8 @@ const compact = async (args, options = {}) => {
|
|
|
1189
1353
|
const current = read.json;
|
|
1190
1354
|
const baseEtag = read.etag;
|
|
1191
1355
|
const logSeqStartBefore = logSeqStartOf(current);
|
|
1192
|
-
const
|
|
1356
|
+
const discoveredTail = (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStartBefore, current.tail_hint, internal.knownTail ?? 0), { ...options.signal !== void 0 && { signal: options.signal } })).tail;
|
|
1357
|
+
const nextSeq = discoveredTail;
|
|
1193
1358
|
if (nextSeq - logSeqStartBefore < minToCompact) return {
|
|
1194
1359
|
written: false,
|
|
1195
1360
|
skippedReason: "below-min-threshold",
|
|
@@ -1250,13 +1415,15 @@ const compact = async (args, options = {}) => {
|
|
|
1250
1415
|
...options.signal !== void 0 && { signal: options.signal }
|
|
1251
1416
|
};
|
|
1252
1417
|
await storage.put(newKey, bodyBytes, putOpts);
|
|
1418
|
+
const entriesFolded = foldEnd - logSeqStartBefore;
|
|
1253
1419
|
const nextBody = encodeJsonBytes({
|
|
1254
1420
|
...current,
|
|
1255
1421
|
snapshot: newKey,
|
|
1256
1422
|
log_seq_start: foldEnd,
|
|
1423
|
+
tail_hint: Math.max(current.tail_hint, discoveredTail),
|
|
1257
1424
|
snapshot_bytes: bodyBytes.byteLength,
|
|
1258
1425
|
snapshot_rows: base.size,
|
|
1259
|
-
|
|
1426
|
+
mean_entry_bytes: entriesFolded > 0 ? Math.round(foldedSliceBytes / entriesFolded) : current.mean_entry_bytes ?? 0
|
|
1260
1427
|
});
|
|
1261
1428
|
const casOpts = {
|
|
1262
1429
|
ifMatch: baseEtag,
|
|
@@ -1280,10 +1447,9 @@ const compact = async (args, options = {}) => {
|
|
|
1280
1447
|
}
|
|
1281
1448
|
throw error;
|
|
1282
1449
|
}
|
|
1283
|
-
const entriesFolded = foldEnd - logSeqStartBefore;
|
|
1284
1450
|
const metrics = ctxMetrics$2();
|
|
1285
1451
|
metrics.histogram("db.compact.entries_folded", entriesFolded, { collection: collectionName });
|
|
1286
|
-
metrics.gauge("db.manifest.lag_window_depth",
|
|
1452
|
+
metrics.gauge("db.manifest.lag_window_depth", discoveredTail - foldEnd, { collection: collectionName });
|
|
1287
1453
|
return {
|
|
1288
1454
|
written: true,
|
|
1289
1455
|
previousSnapshotKey: current.snapshot,
|
|
@@ -1467,8 +1633,8 @@ const runGc = async (args, options = {}) => {
|
|
|
1467
1633
|
for (const cand of sweepCandidates) if (toSweep.length < maxSweeps && Date.parse(cand.due_at) <= nowMs) toSweep.push(cand);
|
|
1468
1634
|
else remaining.push(cand);
|
|
1469
1635
|
await Promise.all(toSweep.map((c) => storage.delete(c.key, signalOpts)));
|
|
1470
|
-
const
|
|
1471
|
-
const lastSweptAt = toSweep.length > 0 ? now().toISOString() :
|
|
1636
|
+
const sweptKeys = new Set(toSweep.map((c) => c.key));
|
|
1637
|
+
const lastSweptAt = toSweep.length > 0 ? now().toISOString() : "";
|
|
1472
1638
|
const markedSummary = {
|
|
1473
1639
|
stale_log: markedStaleLog,
|
|
1474
1640
|
orphan_snapshot: markedOrphanSnapshot,
|
|
@@ -1476,11 +1642,12 @@ const runGc = async (args, options = {}) => {
|
|
|
1476
1642
|
};
|
|
1477
1643
|
let pendingDepth;
|
|
1478
1644
|
try {
|
|
1479
|
-
pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, () => ({
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1645
|
+
pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, (latest) => mergeGcPending(latest, {
|
|
1646
|
+
sweptKeys,
|
|
1647
|
+
newCandidates,
|
|
1648
|
+
lastSweptAt,
|
|
1649
|
+
nextContentCursor,
|
|
1650
|
+
maxCandidates: GC_MAX_PENDING_CANDIDATES
|
|
1484
1651
|
}), signalOpts)).json.candidates.length;
|
|
1485
1652
|
} catch (error) {
|
|
1486
1653
|
if (error instanceof BaerlyError && error.code === "Conflict") pendingDepth = remaining.length;
|
|
@@ -1549,7 +1716,7 @@ const computeDueAt = (entry, now, graceMs) => {
|
|
|
1549
1716
|
};
|
|
1550
1717
|
/**
|
|
1551
1718
|
* Build the live content-hash set. The set covers every live
|
|
1552
|
-
* post-image: every `entry.after` in `[logSeqStart,
|
|
1719
|
+
* post-image: every `entry.after` in `[logSeqStart, tail_hint)` plus
|
|
1553
1720
|
* every row body in the current snapshot.
|
|
1554
1721
|
*
|
|
1555
1722
|
* A snapshot read that throws (corrupt body, hash mismatch) is
|
|
@@ -1561,9 +1728,9 @@ const computeDueAt = (entry, now, graceMs) => {
|
|
|
1561
1728
|
const collectLiveContentHashes = async (storage, collectionPrefix, collectionName, current, logSeqStart, signal) => {
|
|
1562
1729
|
const hashes = /* @__PURE__ */ new Set();
|
|
1563
1730
|
const getOpts = signal !== void 0 ? { signal } : void 0;
|
|
1564
|
-
const
|
|
1565
|
-
|
|
1566
|
-
const got = await storage.get(
|
|
1731
|
+
const { tail } = await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, current.tail_hint), { signal });
|
|
1732
|
+
const ingestLogEntry = async (s) => {
|
|
1733
|
+
const got = await storage.get(logObjectKey(collectionPrefix, s), getOpts);
|
|
1567
1734
|
if (got === null) return;
|
|
1568
1735
|
let entry;
|
|
1569
1736
|
try {
|
|
@@ -1574,8 +1741,14 @@ const collectLiveContentHashes = async (storage, collectionPrefix, collectionNam
|
|
|
1574
1741
|
if (entry.after === void 0) return;
|
|
1575
1742
|
const bodyBytes = encodeJsonBytes(entry.after);
|
|
1576
1743
|
hashes.add(await versionFromContent(bodyBytes));
|
|
1577
|
-
}
|
|
1578
|
-
|
|
1744
|
+
};
|
|
1745
|
+
for (let chunkStart = logSeqStart; chunkStart < tail; chunkStart += 16) {
|
|
1746
|
+
signal?.throwIfAborted();
|
|
1747
|
+
const chunkEnd = Math.min(chunkStart + 16, tail);
|
|
1748
|
+
const chunk = [];
|
|
1749
|
+
for (let s = chunkStart; s < chunkEnd; s++) chunk.push(ingestLogEntry(s));
|
|
1750
|
+
await Promise.all(chunk);
|
|
1751
|
+
}
|
|
1579
1752
|
if (current.snapshot !== null) try {
|
|
1580
1753
|
const map = await loadSnapshotAsMap(storage, current.snapshot, collectionName, signal);
|
|
1581
1754
|
const rowReads = [];
|
|
@@ -1619,10 +1792,10 @@ const collectLiveContentHashes = async (storage, collectionPrefix, collectionNam
|
|
|
1619
1792
|
*
|
|
1620
1793
|
* @see ./indexes.ts — `IndexDefinition`, `allIndexKeysFor`,
|
|
1621
1794
|
* `indexKeyPrefix`.
|
|
1622
|
-
* @see ./writer.ts — the
|
|
1623
|
-
*
|
|
1795
|
+
* @see ./writer.ts — the hybrid index emission path this command
|
|
1796
|
+
* reconciles.
|
|
1624
1797
|
*/
|
|
1625
|
-
const EMPTY_BODY = new Uint8Array(0);
|
|
1798
|
+
const EMPTY_BODY = /* @__PURE__ */ new Uint8Array(0);
|
|
1626
1799
|
const APPLICATION_JSON = "application/json";
|
|
1627
1800
|
/**
|
|
1628
1801
|
* Idempotently reconcile one secondary index for one collection.
|
|
@@ -1641,7 +1814,7 @@ const APPLICATION_JSON = "application/json";
|
|
|
1641
1814
|
* @throws BaerlyError code="InvalidResponse" — `current.json` missing.
|
|
1642
1815
|
* @throws BaerlyError code="Internal" — protocol-invariant
|
|
1643
1816
|
* violation (snapshot hash mismatch, missing log entry inside
|
|
1644
|
-
* `[log_seq_start,
|
|
1817
|
+
* `[log_seq_start, tail_hint)`).
|
|
1645
1818
|
*/
|
|
1646
1819
|
const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
|
|
1647
1820
|
const read = await readCurrentJson(storage, currentJsonKey);
|
|
@@ -1651,9 +1824,11 @@ const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
|
|
|
1651
1824
|
const collectionPrefix = currentJsonKey.slice(0, lastSlash);
|
|
1652
1825
|
const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
1653
1826
|
const logSeqStart = logSeqStartOf(read.json);
|
|
1654
|
-
const
|
|
1827
|
+
const hint = read.json.tail_hint;
|
|
1655
1828
|
const live = read.json.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, read.json.snapshot, collection);
|
|
1656
|
-
foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart,
|
|
1829
|
+
foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart, hint), { collection });
|
|
1830
|
+
const probeOpts = opts.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
1831
|
+
foldLogEntriesOnto(live, (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, hint), probeOpts)).entries, { collection });
|
|
1657
1832
|
const expected = /* @__PURE__ */ new Set();
|
|
1658
1833
|
for (const [docId, body] of live) for (const k of allIndexKeysFor(collectionPrefix, [def], body, docId)) expected.add(k);
|
|
1659
1834
|
const actual = /* @__PURE__ */ new Set();
|
|
@@ -1759,36 +1934,90 @@ const runScheduledMaintenance = async (args, options = {}) => {
|
|
|
1759
1934
|
})
|
|
1760
1935
|
};
|
|
1761
1936
|
};
|
|
1762
|
-
const
|
|
1937
|
+
const profileToScheduledOptions = (profile) => ({
|
|
1763
1938
|
compact: {
|
|
1764
|
-
maxEntriesPerRun:
|
|
1939
|
+
maxEntriesPerRun: profile.maxFoldEntriesPerPass,
|
|
1765
1940
|
minEntriesToCompact: 50
|
|
1766
1941
|
},
|
|
1767
1942
|
gc: {
|
|
1768
|
-
maxMarksPerRun:
|
|
1769
|
-
maxSweepsPerRun:
|
|
1943
|
+
maxMarksPerRun: profile.gcMaxMarks,
|
|
1944
|
+
maxSweepsPerRun: profile.gcMaxSweeps
|
|
1770
1945
|
}
|
|
1771
|
-
};
|
|
1946
|
+
});
|
|
1772
1947
|
/**
|
|
1773
|
-
*
|
|
1774
|
-
*
|
|
1775
|
-
*
|
|
1776
|
-
*
|
|
1948
|
+
* Tuning profile for the 50-subrequest Cloudflare free-tier budget,
|
|
1949
|
+
* derived from {@link MAINTENANCE_PROFILE_CF_FREE} (one source of truth).
|
|
1950
|
+
*
|
|
1951
|
+
* Budget arithmetic (worst case):
|
|
1952
|
+
* compact: 1 GET current + 1 GET snapshot (if any) + N GETs log
|
|
1953
|
+
* + 1 PUT snapshot + 1 PUT current = 3 + N (N = maxEntriesPerRun).
|
|
1954
|
+
* runGc: 1 GET current + 1 GET pending + 3 LISTs (one page each)
|
|
1955
|
+
* + M GETs log (live tail for content hashes) + S DELETEs + 1 PUT
|
|
1956
|
+
* pending = 6 + M + S.
|
|
1957
|
+
*
|
|
1958
|
+
* With N=20, M=20, S=10 the totals are: compact ≈ 23, gc ≈ 36 — over
|
|
1959
|
+
* the 50 cap combined. The Cloudflare scheduled handler therefore
|
|
1960
|
+
* alternates phases per tick (even minute → compact, odd minute →
|
|
1961
|
+
* GC) by calling `compact()` / `runGc()` directly instead of
|
|
1962
|
+
* `runScheduledMaintenance`; the `maintenance-budget.test.ts`
|
|
1963
|
+
* worst-case test proves each phase in isolation sits under 50 ops
|
|
1964
|
+
* with these bounds.
|
|
1965
|
+
*/
|
|
1966
|
+
const CLOUDFLARE_FREE_TIER = profileToScheduledOptions(MAINTENANCE_PROFILE_CF_FREE);
|
|
1967
|
+
/**
|
|
1968
|
+
* Tuning profile for the 10,000-subrequest Cloudflare paid-tier budget,
|
|
1969
|
+
* derived from {@link MAINTENANCE_PROFILE_CF_PAID} (one source of truth).
|
|
1970
|
+
* Keeps the single-phase CPU-killable shape but affords Node-tier per-pass
|
|
1971
|
+
* throughput (compact ≈ 3+200, gc ≈ 6+200+100 — far under the 10k cap).
|
|
1972
|
+
*
|
|
1973
|
+
* To honour `BAERLY_MAINTENANCE_PROFILE=cf-paid` on the cron path, pass this
|
|
1974
|
+
* constant to `runScheduledMaintenance` inside your
|
|
1975
|
+
* {@link WorkerScheduledHandler}; the worked recipe lives in the `scheduled`
|
|
1976
|
+
* handler docstring of `baerlyWorker` (`@gusto/baerly-storage/cloudflare`).
|
|
1977
|
+
*/
|
|
1978
|
+
const CLOUDFLARE_PAID_TIER = profileToScheduledOptions(MAINTENANCE_PROFILE_CF_PAID);
|
|
1979
|
+
/**
|
|
1980
|
+
* Floor-based GC-cadence boundary crossing. Every commit advances the
|
|
1981
|
+
* writer-observed tail by one logical slot; `tail_hint` remains a
|
|
1982
|
+
* lower bound and may be refreshed later by maintenance. The floor test
|
|
1983
|
+
* trips on any interval boundary inside `(prevSeq, nextSeq]`, so it
|
|
1984
|
+
* stays correct whether the observed/probed tail advances by one or
|
|
1985
|
+
* catches up by a larger step.
|
|
1777
1986
|
*/
|
|
1778
1987
|
const crossesGcBoundary = (prevSeq, nextSeq, interval) => Math.floor(prevSeq / interval) !== Math.floor(nextSeq / interval);
|
|
1779
1988
|
/**
|
|
1989
|
+
* Derived live-tail byte size — the ratio TRIGGER input that replaces the
|
|
1990
|
+
* formerly-stored exact byte count. `(observedTail − log_seq_start) ×
|
|
1991
|
+
* mean_entry_bytes`. `observedTail` is a param so write-tick callers can
|
|
1992
|
+
* pass the writer-observed tail and scheduled callers can pass a
|
|
1993
|
+
* probe-discovered true tail. Cold-start (no mean yet) falls back to
|
|
1994
|
+
* {@link MAINTENANCE_COLD_START_ENTRY_BYTES}, never 0 (see that constant).
|
|
1995
|
+
* TRIGGER-only — structurally barred from `foldViable` (the exact-bytes CPU-kill
|
|
1996
|
+
* ceiling).
|
|
1997
|
+
*/
|
|
1998
|
+
const estimateTailBytes = (current, observedTail) => Math.max(0, observedTail - current.log_seq_start) * (current.mean_entry_bytes ?? 128);
|
|
1999
|
+
/**
|
|
1780
2000
|
* The cheap DISPATCH gate the writer calls after a commit — zero
|
|
1781
|
-
* storage ops; it reads the post-
|
|
2001
|
+
* storage ops; it reads the post-commit {@link CurrentJson} already in
|
|
1782
2002
|
* scope. Ratio-OR-boundary, WITHOUT the entry floor (the floor is part
|
|
1783
2003
|
* of the fold-trigger Gate 1 checked inside {@link runBoundedMaintenance}).
|
|
1784
|
-
* Returning `false` lets the writer skip the dispatch entirely.
|
|
2004
|
+
* Returning `false` lets the writer skip the dispatch entirely. Ratio
|
|
2005
|
+
* numerator is the DERIVED {@link estimateTailBytes};
|
|
2006
|
+
* `observedTail` threads from the caller (the writer's in-memory observed
|
|
2007
|
+
* tail `seq+1` under single-write commit). Both the boundary check and the
|
|
2008
|
+
* ratio key off `observedTail`, since the stored `tail_hint` is now only a
|
|
2009
|
+
* non-authoritative lower bound (compactor-advanced).
|
|
1785
2010
|
*/
|
|
1786
|
-
const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundary(prevSeq,
|
|
2011
|
+
const shouldFireMaintenance = (current, prevSeq, gcInterval, observedTail) => crossesGcBoundary(prevSeq, observedTail, gcInterval) || estimateTailBytes(current, observedTail) / Math.max(current.snapshot_bytes, 65536) >= 1;
|
|
1787
2012
|
/**
|
|
1788
2013
|
* The write-tick maintenance runner: a bounded, single-attempt
|
|
1789
2014
|
* composition of {@link compact} and {@link runGc} for one collection,
|
|
1790
2015
|
* sized to fit a CPU-killable Cloudflare free-tier isolate by default.
|
|
1791
2016
|
*
|
|
2017
|
+
* INVARIANT: a {@link MaintenanceProfile} changes only maintenance rate +
|
|
2018
|
+
* defer threshold — never data/API/query/wire/correctness; reads stay pure.
|
|
2019
|
+
* See `maintenance-profile-equivalence.test.ts`.
|
|
2020
|
+
*
|
|
1792
2021
|
* Never throws — the write-tick caller must never see a maintenance
|
|
1793
2022
|
* failure. Expected signals (CAS contention, an over-ceiling deferral)
|
|
1794
2023
|
* are swallowed quietly; unexpected throws bump
|
|
@@ -1796,7 +2025,8 @@ const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundar
|
|
|
1796
2025
|
* swallowed.
|
|
1797
2026
|
*
|
|
1798
2027
|
* Control flow (fold-priority with a stateless GC-starvation guard):
|
|
1799
|
-
* 0. `disabled` →
|
|
2028
|
+
* 0. `disabled` → skip fold/GC, but still allow the rate-limited
|
|
2029
|
+
* tail_hint refresh that bounds read/write forward-probes.
|
|
1800
2030
|
* 1. Read `current.json` once.
|
|
1801
2031
|
* 2. Hard-GC starvation boundary (`"single"` only) → GC slice, skip fold.
|
|
1802
2032
|
* 3. Gate 1 (ratio AND entry floor) trips → fold if viable, else
|
|
@@ -1805,17 +2035,14 @@ const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundar
|
|
|
1805
2035
|
* 4. Otherwise (or on `"both"`) → GC iff the cadence boundary crossed.
|
|
1806
2036
|
*/
|
|
1807
2037
|
const runBoundedMaintenance = async (args, options) => {
|
|
1808
|
-
if (args.disabled === true) return;
|
|
1809
2038
|
const { storage, currentJsonKey, prevSeq } = args;
|
|
1810
|
-
const
|
|
2039
|
+
const profile = options?.profile ?? MAINTENANCE_PROFILE_CF_FREE;
|
|
2040
|
+
const { maxFoldEntriesPerPass, gcMaxMarks, gcMaxSweeps, gcInterval } = profile;
|
|
1811
2041
|
const minEntriesToCompact = options?.minEntriesToCompact ?? 50;
|
|
1812
|
-
const gcMaxMarks = options?.gcMaxMarks ?? 20;
|
|
1813
|
-
const gcMaxSweeps = options?.gcMaxSweeps ?? 10;
|
|
1814
|
-
const gcInterval = options?.gcInterval ?? 4;
|
|
1815
2042
|
const phasesPerTick = options?.phasesPerTick ?? "single";
|
|
1816
2043
|
const signal = options?.signal;
|
|
1817
|
-
const C = args.maxFoldBytes ??
|
|
1818
|
-
const E =
|
|
2044
|
+
const C = args.maxFoldBytes ?? profile.maxFoldBytes;
|
|
2045
|
+
const E = profile.maxFoldRows;
|
|
1819
2046
|
const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
|
|
1820
2047
|
const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
1821
2048
|
const gcOpts = {
|
|
@@ -1829,14 +2056,29 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1829
2056
|
const read = await readCurrentJson(storage, currentJsonKey, signal !== void 0 ? { signal } : void 0);
|
|
1830
2057
|
if (read === null) return;
|
|
1831
2058
|
const current = read.json;
|
|
1832
|
-
const tailBytes = current.tail_bytes;
|
|
1833
2059
|
const snapshotBytes = current.snapshot_bytes;
|
|
1834
2060
|
const snapshotRows = current.snapshot_rows;
|
|
1835
|
-
const nextSeq = current.next_seq;
|
|
1836
2061
|
const logSeqStart = current.log_seq_start;
|
|
1837
|
-
const
|
|
2062
|
+
const probeFloor = Math.max(logSeqStart, current.tail_hint);
|
|
2063
|
+
let nextSeq;
|
|
2064
|
+
if (args.observedTail !== void 0) nextSeq = Math.max(args.observedTail, probeFloor);
|
|
2065
|
+
else nextSeq = (await probeTailFrom(storage, collectionPrefix, probeFloor, signal !== void 0 ? { signal } : void 0)).tail;
|
|
2066
|
+
const gate1 = estimateTailBytes(current, nextSeq) / Math.max(snapshotBytes, 65536) >= 1 && nextSeq - logSeqStart >= minEntriesToCompact;
|
|
1838
2067
|
const foldViable = snapshotBytes <= C && snapshotRows + maxFoldEntriesPerPass <= E;
|
|
1839
2068
|
const gcDue = crossesGcBoundary(prevSeq, nextSeq, gcInterval);
|
|
2069
|
+
const refreshTailHintIfNeeded = async () => {
|
|
2070
|
+
if (nextSeq - current.tail_hint < 128) return;
|
|
2071
|
+
try {
|
|
2072
|
+
await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
|
|
2073
|
+
...c,
|
|
2074
|
+
tail_hint: Math.max(c.tail_hint, nextSeq)
|
|
2075
|
+
}), signal !== void 0 ? { signal } : void 0);
|
|
2076
|
+
} catch {}
|
|
2077
|
+
};
|
|
2078
|
+
if (args.disabled === true) {
|
|
2079
|
+
await refreshTailHintIfNeeded();
|
|
2080
|
+
return;
|
|
2081
|
+
}
|
|
1840
2082
|
const hardGc = crossesGcBoundary(prevSeq, nextSeq, gcInterval * 4);
|
|
1841
2083
|
if (phasesPerTick === "single" && hardGc) {
|
|
1842
2084
|
await runGc({
|
|
@@ -1854,6 +2096,7 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1854
2096
|
minEntriesToCompact,
|
|
1855
2097
|
ceilingBytes: C,
|
|
1856
2098
|
ceilingEntries: E,
|
|
2099
|
+
knownTail: nextSeq,
|
|
1857
2100
|
...signal !== void 0 && { signal }
|
|
1858
2101
|
});
|
|
1859
2102
|
if (phasesPerTick === "single") return;
|
|
@@ -1862,8 +2105,9 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1862
2105
|
collection,
|
|
1863
2106
|
dimension: snapshotBytes > C ? "bytes" : "rows"
|
|
1864
2107
|
});
|
|
2108
|
+
await refreshTailHintIfNeeded();
|
|
1865
2109
|
if (nextSeq - (current.last_warned_seq ?? 0) >= 1e3) {
|
|
1866
|
-
console.warn(`baerly-storage: collection "${collection}" is deferring compaction — its snapshot exceeds the fold ceiling (${snapshotBytes > C ?
|
|
2110
|
+
console.warn(`baerly-storage: collection "${collection}" is deferring compaction — its snapshot exceeds the fold ceiling (${snapshotBytes > C ? `${C} bytes` : `${E} rows`}), so the tail keeps growing and read amplification will climb. The dataset has outgrown the maintenance envelope. On paid Cloudflare / Node you can raise BAERLY_MAINTENANCE_MAX_FOLD_BYTES — but on Cloudflare a cap above what a single isolate can fold makes folds get CPU-killed mid-flight and silently not land (no clean metric — watch snapshot age / object count); see docs/about/graduation.md. Otherwise, move to a database service (D1 / Postgres).`);
|
|
1867
2111
|
try {
|
|
1868
2112
|
await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
|
|
1869
2113
|
...c,
|
|
@@ -1889,7 +2133,25 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1889
2133
|
* returns `void`.
|
|
1890
2134
|
*/
|
|
1891
2135
|
const dispatchInlineAwaited = (task) => task();
|
|
2136
|
+
/**
|
|
2137
|
+
* Parse the two host-agnostic ops-plane env vars into the
|
|
2138
|
+
* {@link MaintenanceDispatch} overrides. Shared by every host adapter so the
|
|
2139
|
+
* cross-host contract has one source of truth: `maxFoldBytes` (`C`) is the
|
|
2140
|
+
* finite `BAERLY_MAINTENANCE_MAX_FOLD_BYTES` or undefined; `disabled` is
|
|
2141
|
+
* `BAERLY_MAINTENANCE_DISABLE` truthy (set, non-empty, not `"0"`/`"false"`).
|
|
2142
|
+
*/
|
|
2143
|
+
const parseMaintenanceEnv = (readEnv) => {
|
|
2144
|
+
const rawFoldBytes = readEnv("BAERLY_MAINTENANCE_MAX_FOLD_BYTES");
|
|
2145
|
+
const parsedFoldBytes = rawFoldBytes !== void 0 && rawFoldBytes !== "" ? Number(rawFoldBytes) : NaN;
|
|
2146
|
+
const maxFoldBytes = Number.isFinite(parsedFoldBytes) ? parsedFoldBytes : void 0;
|
|
2147
|
+
const rawDisable = readEnv("BAERLY_MAINTENANCE_DISABLE");
|
|
2148
|
+
const disabled = rawDisable !== void 0 && rawDisable !== "" && rawDisable !== "0" && rawDisable.toLowerCase() !== "false";
|
|
2149
|
+
return {
|
|
2150
|
+
...maxFoldBytes !== void 0 && { maxFoldBytes },
|
|
2151
|
+
disabled
|
|
2152
|
+
};
|
|
2153
|
+
};
|
|
1892
2154
|
//#endregion
|
|
1893
|
-
export {
|
|
2155
|
+
export { deepEqualDocumentValue as A, findLogTail as C, validateWire as D, versionFromContent as E, assertWireSatisfiable as O, assertNameNotReserved as S, logObjectKey as T, encodeIndexValue as _, estimateTailBytes as a, loadSnapshotAsMap as b, runScheduledMaintenance as c, runGc as d, compact as f, allIndexKeysFor as g, walkLogRange as h, dispatchInlineAwaited as i, matchesWire as k, shouldFireMaintenance as l, readLogEntry as m, CLOUDFLARE_PAID_TIER as n, parseMaintenanceEnv as o, foldLogEntriesOnto as p, crossesGcBoundary as r, runBoundedMaintenance as s, CLOUDFLARE_FREE_TIER as t, rebuildIndex as u, validateIndexDefinition as v, probeTailFrom as w, snapshotKey as x, encodeSnapshotBody as y };
|
|
1894
2156
|
|
|
1895
|
-
//# sourceMappingURL=maintenance-
|
|
2157
|
+
//# sourceMappingURL=maintenance-B4Dki6Ig.js.map
|