@gusto/baerly-storage 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +29 -0
- package/README.md +93 -52
- package/dist/API.md +151 -109
- package/dist/CHANGELOG.md +144 -0
- package/dist/THIRD-PARTY-LICENSES.txt +312 -0
- package/dist/{app-config-C0Bbw_Z6.d.ts → app-config-BIzyAKzy.d.ts} +14 -2
- package/dist/app-config.d.ts +1 -1
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +288 -288
- package/dist/auth.js.map +1 -1
- package/dist/baerly.js +10 -10
- package/dist/baerly.js.map +1 -1
- package/dist/bytes-BXdamO3L.js.map +1 -1
- package/dist/{client-DYZfoZb9.d.ts → client-BJXF3XJA.d.ts} +10 -10
- package/dist/client-react.d.ts +3 -3
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +3 -3
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-CINJPKCW.js → cloudflare-Cn2ibODt.js} +178 -2
- package/dist/cloudflare-Cn2ibODt.js.map +1 -0
- package/dist/cloudflare.d.ts +4 -4
- package/dist/cloudflare.js +10 -11
- package/dist/cloudflare.js.map +1 -1
- package/dist/{subcommand-DZcHcsYp.js → config-CGyBopy3.js} +643 -439
- package/dist/config-CGyBopy3.js.map +1 -0
- package/dist/context-DRZ3tZ5l.js.map +1 -1
- package/dist/{cost-CQt1YRkA.js → cost-BZ_MDBmn.js} +6 -4
- package/dist/cost-BZ_MDBmn.js.map +1 -0
- package/dist/{current-json-C-EyJqs0.js → current-json-DsN7FNY1.js} +47 -13
- package/dist/current-json-DsN7FNY1.js.map +1 -0
- package/dist/{db-BM3y2JQw.d.ts → db-CCT6tDfY.d.ts} +26 -108
- package/dist/{deploy-CrAbfgCq.js → deploy-BKKyGlSJ.js} +11 -4
- package/dist/deploy-BKKyGlSJ.js.map +1 -0
- package/dist/dev-vite.d.ts +5 -5
- package/dist/dev-vite.js +5 -5
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/{doctor-Db-hzPmE.js → doctor-BVU4GNXN.js} +13 -132
- package/dist/doctor-BVU4GNXN.js.map +1 -0
- package/dist/{dump-BYA55NLP.js → dump-DPEAKub6.js} +5 -3
- package/dist/dump-DPEAKub6.js.map +1 -0
- package/dist/errors-DOdL95kK.js.map +1 -1
- package/dist/{export-Bhbz90-z.js → export-BcuCLFFe.js} +6 -4
- package/dist/{export-Bhbz90-z.js.map → export-BcuCLFFe.js.map} +1 -1
- package/dist/{fsck-CPPZWt9p.js → fsck-RXgLOaJG.js} +23 -11
- package/dist/fsck-RXgLOaJG.js.map +1 -0
- package/dist/{http-Cw8p6y5b.js → http-Bd4byR4n.js} +45 -61
- package/dist/http-Bd4byR4n.js.map +1 -0
- package/dist/http.d.ts +15 -15
- package/dist/http.js +1 -1
- package/dist/{index-cUY-36AN.d.ts → index-BDceQQi-.d.ts} +20 -4
- package/dist/{index-C3JAH1Yo.d.ts → index-Cmsxx-ZN.d.ts} +9 -9
- package/dist/{index-DSvwEUeF.d.ts → index-DW7GT6M2.d.ts} +50 -42
- package/dist/{index-DwDTWPSU.d.ts → index-DnBGSeuU.d.ts} +10 -7
- package/dist/index.d.ts +6 -6
- package/dist/index.js +4 -4
- package/dist/{indexes-DAu2AkTW.d.ts → indexes-Cx7uMXm0.d.ts} +2 -2
- package/dist/{inspect-Bdbzvi-_.js → inspect-DhI49PSi.js} +16 -8
- package/dist/inspect-DhI49PSi.js.map +1 -0
- package/dist/{snapshot-SS8USDUF.js → log-tail-DXeDkWEp.js} +154 -83
- package/dist/log-tail-DXeDkWEp.js.map +1 -0
- package/dist/{maintenance-BRrThz8V.d.ts → maintenance-BkhUr72A.d.ts} +91 -26
- package/dist/{maintenance-ClzBC1Vv.js → maintenance-CWBQpZ2y.js} +368 -161
- package/dist/maintenance-CWBQpZ2y.js.map +1 -0
- package/dist/{maintenance-BvOv6BMM.js → maintenance-C_KSkVjd.js} +204 -82
- package/dist/maintenance-C_KSkVjd.js.map +1 -0
- package/dist/maintenance.d.ts +3 -2
- package/dist/maintenance.js +3 -2
- package/dist/node.d.ts +13 -8
- package/dist/node.js +1 -1
- package/dist/normalize-XWY25_sG.js.map +1 -1
- package/dist/{observability-CpOiEN0k.js → observability-BYd-RzEx.js} +63 -55
- package/dist/observability-BYd-RzEx.js.map +1 -0
- package/dist/observability.d.ts +3 -3
- package/dist/observability.js +1 -1
- package/dist/{plan-DOTl21vq.js → plan-DIu_B6fy.js} +7 -4
- package/dist/plan-DIu_B6fy.js.map +1 -0
- package/dist/{query-BmiQy328.js → query-sQNO5vU_.js} +1071 -1099
- package/dist/query-sQNO5vU_.js.map +1 -0
- package/dist/{rebuild-index-DC_nR6wP.js → rebuild-index-DP4CqSSs.js} +5 -4
- package/dist/rebuild-index-DP4CqSSs.js.map +1 -0
- package/dist/{restore-C6IcmLeA.js → restore-DpQYef6r.js} +362 -328
- package/dist/restore-DpQYef6r.js.map +1 -0
- package/dist/shared-secret-BJdSwpgw.js.map +1 -1
- package/dist/{src-QQ7jDeL2.js → src-CWpkSbvJ.js} +40 -20
- package/dist/src-CWpkSbvJ.js.map +1 -0
- package/dist/{src-DU1sCMlr.js → src-D24wDpzR.js} +269 -258
- package/dist/src-D24wDpzR.js.map +1 -0
- package/dist/{src-zhEAmmF7.js → src-Db4-qVtM.js} +36 -96
- package/dist/src-Db4-qVtM.js.map +1 -0
- package/dist/subcommand-CP9NY2J4.js +159 -0
- package/dist/subcommand-CP9NY2J4.js.map +1 -0
- package/dist/{usage-CLjirdD-.js → usage-CTfDq9HR.js} +3 -3
- package/dist/{usage-CLjirdD-.js.map → usage-CTfDq9HR.js.map} +1 -1
- package/dist/{validate-DCq_0DIn.js → validate-DtljsCh1.js} +51 -51
- package/dist/{validate-DCq_0DIn.js.map → validate-DtljsCh1.js.map} +1 -1
- package/package.json +40 -29
- package/dist/cloudflare-CINJPKCW.js.map +0 -1
- package/dist/config-DVfDGyw1.js +0 -278
- package/dist/config-DVfDGyw1.js.map +0 -1
- package/dist/cost-CQt1YRkA.js.map +0 -1
- package/dist/current-json-C-EyJqs0.js.map +0 -1
- package/dist/deploy-CrAbfgCq.js.map +0 -1
- package/dist/doctor-Db-hzPmE.js.map +0 -1
- package/dist/dump-BYA55NLP.js.map +0 -1
- package/dist/fsck-CPPZWt9p.js.map +0 -1
- package/dist/http-Cw8p6y5b.js.map +0 -1
- package/dist/inspect-Bdbzvi-_.js.map +0 -1
- package/dist/maintenance-BvOv6BMM.js.map +0 -1
- package/dist/maintenance-ClzBC1Vv.js.map +0 -1
- package/dist/observability-CpOiEN0k.js.map +0 -1
- package/dist/plan-DOTl21vq.js.map +0 -1
- package/dist/query-BmiQy328.js.map +0 -1
- package/dist/rebuild-index-DC_nR6wP.js.map +0 -1
- package/dist/restore-C6IcmLeA.js.map +0 -1
- package/dist/snapshot-SS8USDUF.js.map +0 -1
- package/dist/src-DU1sCMlr.js.map +0 -1
- package/dist/src-QQ7jDeL2.js.map +0 -1
- package/dist/src-zhEAmmF7.js.map +0 -1
- package/dist/subcommand-DZcHcsYp.js.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { c as GC_MAX_PENDING_CANDIDATES,
|
|
1
|
+
import { b as decodeJsonBytes, c as GC_MAX_PENDING_CANDIDATES, f as MAINTENANCE_PROFILE_CF_FREE, i as readCurrentJson, l as GC_PENDING_CONTENT_TYPE, r as logSeqStartOf, t as casUpdateCurrentJson, u as LOG_FORWARD_PROBE_CAP, x as encodeJsonBytes } from "./current-json-DsN7FNY1.js";
|
|
2
2
|
import { t as BaerlyError } from "./errors-DOdL95kK.js";
|
|
3
3
|
import { i as noopMetricsRecorder, n as getCurrentContext } from "./context-DRZ3tZ5l.js";
|
|
4
4
|
//#region packages/protocol/src/coordination/gc-pending.ts
|
|
@@ -72,41 +72,109 @@ const createGcPending = async (storage, key, initial, opts) => {
|
|
|
72
72
|
};
|
|
73
73
|
};
|
|
74
74
|
/**
|
|
75
|
-
* Read-modify-write `gc/pending.json` under CAS
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
75
|
+
* Read-modify-write `gc/pending.json` under CAS, with a bounded retry
|
|
76
|
+
* loop. Each attempt re-reads the latest body, hands the `mutator`
|
|
77
|
+
* that latest value, and writes the result with
|
|
78
|
+
* `If-Match: <latest etag>`. On `Conflict` (a concurrent writer landed
|
|
79
|
+
* between this attempt's read and write) the loop re-reads and
|
|
80
|
+
* re-applies the mutator, up to {@link GC_PENDING_CAS_MAX_ATTEMPTS}
|
|
81
|
+
* total attempts, then surfaces `Conflict`.
|
|
82
|
+
*
|
|
83
|
+
* `mutator` MUST be a pure, deterministic MERGE of this pass's results
|
|
84
|
+
* INTO the `latest` value it is handed — it is called once per attempt
|
|
85
|
+
* (so it must not have side effects), and it MUST read `latest` rather
|
|
86
|
+
* than ignore it and write a precomputed set. Ignoring `latest` would
|
|
87
|
+
* silently overwrite a concurrent pass's candidates: the
|
|
88
|
+
* If-Match succeeds (it used the fresh etag), so no conflict is
|
|
89
|
+
* raised, and the concurrent marks are lost. See {@link mergeGcPending}.
|
|
90
|
+
*
|
|
91
|
+
* @throws BaerlyError{code:"Conflict"} — another writer kept landing a
|
|
92
|
+
* write between this function's read and write for every
|
|
93
|
+
* attempt.
|
|
86
94
|
* @throws BaerlyError{code:"InvalidResponse"} — `key` does not exist
|
|
87
95
|
* (use {@link createGcPending} instead) or body doesn't
|
|
88
96
|
* parse / fails the shape guard.
|
|
89
97
|
*/
|
|
90
98
|
const casUpdateGcPending = async (storage, key, mutator, opts) => {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
99
|
+
let lastConflict;
|
|
100
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
101
|
+
const existing = await readGcPending(storage, key, opts);
|
|
102
|
+
if (existing === null) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key} does not exist; use createGcPending first`);
|
|
103
|
+
const next = mutator(structuredClone(existing.json));
|
|
104
|
+
assertGcPending(next, key);
|
|
105
|
+
const body = encodeJsonBytes(next);
|
|
106
|
+
const putOpts = {
|
|
107
|
+
ifMatch: existing.etag,
|
|
108
|
+
contentType: GC_PENDING_CONTENT_TYPE,
|
|
109
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
110
|
+
};
|
|
111
|
+
try {
|
|
112
|
+
return {
|
|
113
|
+
json: next,
|
|
114
|
+
etag: (await storage.put(key, body, putOpts)).etag
|
|
115
|
+
};
|
|
116
|
+
} catch (error) {
|
|
117
|
+
const translated = translateCasError(error, key);
|
|
118
|
+
if (translated.code !== "Conflict") throw translated;
|
|
119
|
+
lastConflict = translated;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
throw lastConflict;
|
|
123
|
+
};
|
|
124
|
+
/**
|
|
125
|
+
* Pure merge of one GC pass's results INTO the `latest`
|
|
126
|
+
* `gc/pending.json` value, used as the {@link casUpdateGcPending}
|
|
127
|
+
* mutator. Deterministic, no I/O — the kernel unit + mutation test
|
|
128
|
+
* surface. Splitting it out is what makes the CAS write correct under
|
|
129
|
+
* concurrency: each retry re-merges against fresh `latest` rather than
|
|
130
|
+
* overwriting it with a stale precomputed set.
|
|
131
|
+
*
|
|
132
|
+
* Semantics:
|
|
133
|
+
* - **candidates**: start from `latest.candidates`, DROP any whose key
|
|
134
|
+
* this pass already DELETED (`sweptKeys`), then APPEND every fresh
|
|
135
|
+
* mark whose key is not already present AND was not itself swept
|
|
136
|
+
* this pass (a `graceMillis: 0` pass can mark-and-sweep a key in one
|
|
137
|
+
* go — re-adding it to the ledger after deleting it would be wrong;
|
|
138
|
+
* dedup also drops a key a concurrent pass already marked). Capped at
|
|
139
|
+
* `maxCandidates`.
|
|
140
|
+
* - **last_swept_at**: the lexicographically LATER of `latest`'s and
|
|
141
|
+
* this pass's (ISO-8601 UTC; `""` sorts lowest so a real timestamp
|
|
142
|
+
* always wins). Never regress a concurrent pass's newer sweep time.
|
|
143
|
+
* - **content_scan_cursor**: rotation cursor — liveness, not
|
|
144
|
+
* correctness; just don't regress forward progress. The two sides
|
|
145
|
+
* are NOT symmetric: `pass.nextContentCursor === undefined` means
|
|
146
|
+
* THIS pass examined to the END of the keyspace (WRAP — the most-
|
|
147
|
+
* advanced position), whereas `latest.content_scan_cursor` absent
|
|
148
|
+
* means the stored ledger has no cursor yet (start-from-beginning).
|
|
149
|
+
* So:
|
|
150
|
+
* - our pass WRAPPED ⇒ result wraps (key omitted) — our pass is
|
|
151
|
+
* the most advanced; this is the single-writer rotation reset.
|
|
152
|
+
* - else (our pass advanced to a defined cursor):
|
|
153
|
+
* - latest cursor defined ⇒ take the lexicographically GREATER
|
|
154
|
+
* (don't regress a concurrent pass's advance);
|
|
155
|
+
* - latest cursor absent ⇒ keep OUR advance (don't lose it).
|
|
156
|
+
*/
|
|
157
|
+
const mergeGcPending = (latest, pass) => {
|
|
158
|
+
const surviving = latest.candidates.filter((c) => !pass.sweptKeys.has(c.key));
|
|
159
|
+
const present = new Set(surviving.map((c) => c.key));
|
|
160
|
+
const appended = [];
|
|
161
|
+
for (const c of pass.newCandidates) if (!pass.sweptKeys.has(c.key) && !present.has(c.key)) {
|
|
162
|
+
present.add(c.key);
|
|
163
|
+
appended.push(c);
|
|
106
164
|
}
|
|
165
|
+
const candidates = [...surviving, ...appended].slice(0, pass.maxCandidates);
|
|
166
|
+
const lastSweptAt = pass.lastSweptAt > latest.last_swept_at ? pass.lastSweptAt : latest.last_swept_at;
|
|
167
|
+
const latestCursor = latest.content_scan_cursor;
|
|
168
|
+
const passCursor = pass.nextContentCursor;
|
|
169
|
+
let cursor;
|
|
170
|
+
if (passCursor === void 0) cursor = void 0;
|
|
171
|
+
else if (latestCursor !== void 0) cursor = passCursor > latestCursor ? passCursor : latestCursor;
|
|
172
|
+
else cursor = passCursor;
|
|
107
173
|
return {
|
|
108
|
-
|
|
109
|
-
|
|
174
|
+
schema_version: 1,
|
|
175
|
+
candidates,
|
|
176
|
+
last_swept_at: lastSweptAt,
|
|
177
|
+
...cursor !== void 0 && { content_scan_cursor: cursor }
|
|
110
178
|
};
|
|
111
179
|
};
|
|
112
180
|
const VALID_REASONS = new Set([
|
|
@@ -560,6 +628,13 @@ const versionFromContent = async (body) => {
|
|
|
560
628
|
for (let i = 0; i < bytes.length; i++) hex += bytes[i].toString(16).padStart(2, "0");
|
|
561
629
|
return hex.slice(0, VERSION_HEX_LENGTH);
|
|
562
630
|
};
|
|
631
|
+
/**
|
|
632
|
+
* The single constructor for a log-object key:
|
|
633
|
+
* `<manifestPrefix>/log/<seq>.json`. `manifestPrefix` is the
|
|
634
|
+
* `<…>/manifests/<collection>` segment — this helper adds `/log/`.
|
|
635
|
+
* Route all callers here so the key shape lives in one place.
|
|
636
|
+
*/
|
|
637
|
+
const logObjectKey = (manifestPrefix, seq) => `${manifestPrefix}/log/${seq}.json`;
|
|
563
638
|
//#endregion
|
|
564
639
|
//#region packages/protocol/src/snapshot-hash.ts
|
|
565
640
|
/**
|
|
@@ -582,11 +657,147 @@ const snapshotHash = async (bytes) => {
|
|
|
582
657
|
return hex;
|
|
583
658
|
};
|
|
584
659
|
//#endregion
|
|
660
|
+
//#region packages/server/src/log-tail.ts
|
|
661
|
+
/**
|
|
662
|
+
* Tolerant forward-probe for the true committed log tail. The strict
|
|
663
|
+
* `[log_seq_start, hint)` range is dense (a hole is corruption, owned
|
|
664
|
+
* by `walkLogRange`); this owns only the tolerant `[hint, ∞)` probe,
|
|
665
|
+
* where `hint` is a lower bound — the real tail may sit at or past it.
|
|
666
|
+
*/
|
|
667
|
+
/**
|
|
668
|
+
* Discover the true committed tail and fold entries in `[hint, tail)`.
|
|
669
|
+
* `tail` is the first empty seq (>= hint); `entries` are the
|
|
670
|
+
* LogEntries in `[hint, tail)` in seq order. `cap` bounds the walk;
|
|
671
|
+
* exhausting it THROWS `Internal` (never silently truncates) — see
|
|
672
|
+
* the cap-exhaustion comment below.
|
|
673
|
+
*/
|
|
674
|
+
const probeTailFrom = async (storage, logPrefix, hint, opts) => {
|
|
675
|
+
const cap = opts?.cap ?? 1e5;
|
|
676
|
+
const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
677
|
+
const entries = [];
|
|
678
|
+
for (let i = 0; i < cap; i++) {
|
|
679
|
+
const seq = hint + i;
|
|
680
|
+
const key = logObjectKey(logPrefix, seq);
|
|
681
|
+
const got = await storage.get(key, getOpts);
|
|
682
|
+
if (got === null) return {
|
|
683
|
+
tail: seq,
|
|
684
|
+
entries
|
|
685
|
+
};
|
|
686
|
+
try {
|
|
687
|
+
entries.push(decodeJsonBytes(got.body));
|
|
688
|
+
} catch (error) {
|
|
689
|
+
throw new BaerlyError("InvalidResponse", `probeTailFrom: malformed log entry at ${key}: ${error.message}`, { cause: error });
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
throw new BaerlyError("Internal", `probeTailFrom: forward probe exceeded ${cap} from hint ${hint} on ${logPrefix}`);
|
|
693
|
+
};
|
|
694
|
+
/**
|
|
695
|
+
* First empty log seq at/after `hint` — the slot a new commit creates —
|
|
696
|
+
* found by galloping search (`O(log gap)` GETs vs `probeTailFrom`'s
|
|
697
|
+
* `O(gap)`). Used by the writer's per-commit tail-find, where `tail_hint`
|
|
698
|
+
* can lag the true tail. Position only; no bodies.
|
|
699
|
+
*
|
|
700
|
+
* PRECONDITION: occupancy MUST be a dense prefix from `hint` (galloping
|
|
701
|
+
* assumes monotone occupied→empty; a hole ABOVE `hint` would be skipped
|
|
702
|
+
* and the gallop could return a tail past live entries). Contrast
|
|
703
|
+
* `probeTailFrom`, which stops at the first 404 and is hole-tolerant.
|
|
704
|
+
*/
|
|
705
|
+
const findLogTail = async (storage, logPrefix, hint, opts) => {
|
|
706
|
+
const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
707
|
+
const exists = async (seq) => await storage.get(logObjectKey(logPrefix, seq), getOpts) !== null;
|
|
708
|
+
if (!await exists(hint)) return hint;
|
|
709
|
+
let lo = hint;
|
|
710
|
+
let step = 1;
|
|
711
|
+
let hi = hint + step;
|
|
712
|
+
while (await exists(hi)) {
|
|
713
|
+
lo = hi;
|
|
714
|
+
step *= 2;
|
|
715
|
+
hi = lo + step;
|
|
716
|
+
if (step > 1e5) throw new BaerlyError("Internal", `findLogTail: galloping probe exceeded ${LOG_FORWARD_PROBE_CAP} from hint ${hint} on ${logPrefix}`);
|
|
717
|
+
}
|
|
718
|
+
while (hi - lo > 1) {
|
|
719
|
+
const mid = lo + Math.floor((hi - lo) / 2);
|
|
720
|
+
if (await exists(mid)) lo = mid;
|
|
721
|
+
else hi = mid;
|
|
722
|
+
}
|
|
723
|
+
return hi;
|
|
724
|
+
};
|
|
725
|
+
//#endregion
|
|
585
726
|
//#region packages/server/src/names.ts
|
|
586
727
|
/** Reject names in the system-reserved leading-"_" namespace. See docs/adr/007-layout-versioning-cordon.md. */
|
|
587
728
|
const assertNameNotReserved = (name, context) => {
|
|
588
729
|
if (name.startsWith("_")) throw new BaerlyError("InvalidConfig", `${context}: "${name}" is reserved — names beginning with "_" are reserved for system use. See docs/adr/007-layout-versioning-cordon.md.`);
|
|
589
730
|
};
|
|
731
|
+
const MAX_SEQ = 10 ** 12 - 1;
|
|
732
|
+
/**
|
|
733
|
+
* Build the on-bucket key for a snapshot file. `sha256` must be the
|
|
734
|
+
* lowercase hex digest of the canonical body
|
|
735
|
+
* ({@link encodeSnapshotBody}); the filename embeds it so a crashed
|
|
736
|
+
* mid-PUT body that doesn't match its own filename hash is rejected
|
|
737
|
+
* by readers as "missing."
|
|
738
|
+
*
|
|
739
|
+
* @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
|
|
740
|
+
* out of range or `sha256` is not 64 hex chars.
|
|
741
|
+
*/
|
|
742
|
+
const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
|
|
743
|
+
if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
|
|
744
|
+
if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
|
|
745
|
+
const pad = (n) => n.toString().padStart(12, "0");
|
|
746
|
+
return `${collectionPrefix}/snapshot/L9/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
|
|
747
|
+
};
|
|
748
|
+
/**
|
|
749
|
+
* Canonical byte encoding of a {@link SnapshotBody}. Only the
|
|
750
|
+
* compactor produces snapshots, and only this function does the
|
|
751
|
+
* serialisation — so the byte output is deterministic for a given
|
|
752
|
+
* `SnapshotBody` (identical input ⇒ identical bytes ⇒ identical
|
|
753
|
+
* filename hash).
|
|
754
|
+
*/
|
|
755
|
+
const encodeSnapshotBody = (s) => encodeJsonBytes(s);
|
|
756
|
+
/**
|
|
757
|
+
* Load a snapshot file as a `Map<_id, DocumentData>`. Verifies the
|
|
758
|
+
* body hash against the filename, checks the schema version, and
|
|
759
|
+
* cross-checks `body.collection` against the caller's expectation.
|
|
760
|
+
*
|
|
761
|
+
* @example
|
|
762
|
+
* ```ts
|
|
763
|
+
* import { loadSnapshotAsMap } from "@gusto/baerly-storage";
|
|
764
|
+
*
|
|
765
|
+
* const map =
|
|
766
|
+
* srcCurrent.snapshot === null
|
|
767
|
+
* ? new Map<string, DocumentData>()
|
|
768
|
+
* : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
|
|
769
|
+
* ```
|
|
770
|
+
*
|
|
771
|
+
* @throws BaerlyError code="Internal" — pointer resolves to no body,
|
|
772
|
+
* or the body's SHA-256 doesn't match the hash embedded in the
|
|
773
|
+
* filename. (Protocol invariant violation; a corrupted snapshot
|
|
774
|
+
* the reader cannot continue past.)
|
|
775
|
+
* @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
|
|
776
|
+
* isn't an object, has an unsupported `schema_version`, or carries a
|
|
777
|
+
* `collection` that doesn't match `expectedCollection`.
|
|
778
|
+
*/
|
|
779
|
+
const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
|
|
780
|
+
const got = await storage.get(key, signal !== void 0 ? { signal } : void 0);
|
|
781
|
+
if (got === null) throw new BaerlyError("Internal", `compact: snapshot pointer ${key} resolves to no body; protocol violation`);
|
|
782
|
+
const filename = key.slice(key.lastIndexOf("/") + 1).replace(/\.json$/, "");
|
|
783
|
+
const lastDash = filename.lastIndexOf("-");
|
|
784
|
+
const expectedHash = filename.slice(lastDash + 1);
|
|
785
|
+
const actualHash = await snapshotHash(got.body);
|
|
786
|
+
if (actualHash !== expectedHash) throw new BaerlyError("Internal", `compact: snapshot ${key} body hash mismatch (got ${actualHash}); protocol violation`);
|
|
787
|
+
let parsed;
|
|
788
|
+
try {
|
|
789
|
+
parsed = decodeJsonBytes(got.body);
|
|
790
|
+
} catch (error) {
|
|
791
|
+
throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
|
|
792
|
+
}
|
|
793
|
+
if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not an object`);
|
|
794
|
+
const body = parsed;
|
|
795
|
+
if (body.schema_version !== 1) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} has unsupported schema_version ${String(body.schema_version)}`);
|
|
796
|
+
if (body.collection !== expectedCollection) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} carries collection ${body.collection}, expected ${expectedCollection}`);
|
|
797
|
+
const out = /* @__PURE__ */ new Map();
|
|
798
|
+
for (const row of body.docs) out.set(row._id, row.body);
|
|
799
|
+
return out;
|
|
800
|
+
};
|
|
590
801
|
//#endregion
|
|
591
802
|
//#region packages/server/src/indexes.ts
|
|
592
803
|
/**
|
|
@@ -640,8 +851,9 @@ const assertNameNotReserved = (name, context) => {
|
|
|
640
851
|
* Locked: single-field indexes only on the read path. Composite
|
|
641
852
|
* shape is documented and reserved (the path encoder accepts it).
|
|
642
853
|
*
|
|
643
|
-
* @see docs/spec/sync-protocol.md —
|
|
644
|
-
*
|
|
854
|
+
* @see docs/spec/sync-protocol.md — index emission is hybrid around
|
|
855
|
+
* the commit: additive (new) keys are PUT before the committing
|
|
856
|
+
* `log/<seq>` create, and stale-key DELETEs land after it.
|
|
645
857
|
*/
|
|
646
858
|
/** Path-safe segment name. */
|
|
647
859
|
const INDEX_NAME_RE = /^[a-z][a-z0-9_]*$/;
|
|
@@ -909,7 +1121,7 @@ const allIndexKeysFor = (logPrefix, defs, body, docId) => {
|
|
|
909
1121
|
* every caller.
|
|
910
1122
|
*
|
|
911
1123
|
* @throws BaerlyError code="Internal" when the key resolves to no
|
|
912
|
-
* body — a hole inside `[log_seq_start,
|
|
1124
|
+
* body — a hole inside `[log_seq_start, tail_hint)` is a corruption
|
|
913
1125
|
* the kernel can't continue past.
|
|
914
1126
|
* @throws BaerlyError code="InvalidResponse" when the body isn't
|
|
915
1127
|
* valid JSON.
|
|
@@ -922,14 +1134,12 @@ const readLogEntry = async (storage, key, opts) => {
|
|
|
922
1134
|
* byteLength of the STORED log-object body exactly as it was GET back
|
|
923
1135
|
* from storage (`got.body.byteLength`).
|
|
924
1136
|
*
|
|
925
|
-
* The compactor uses this to
|
|
926
|
-
*
|
|
927
|
-
*
|
|
928
|
-
*
|
|
929
|
-
* `
|
|
930
|
-
*
|
|
931
|
-
* assumption (a re-encode of the parsed `LogEntry` could drift if
|
|
932
|
-
* key-order/whitespace ever diverged from what was PUT).
|
|
1137
|
+
* The compactor uses this to sum the byte cost of the folded slice.
|
|
1138
|
+
* The stored body byteLength IS what the writer PUT for each entry,
|
|
1139
|
+
* so summing `got.body.byteLength` over the folded range yields the
|
|
1140
|
+
* exact cost — no re-encode, no round-trip assumption (a re-encode
|
|
1141
|
+
* of the parsed `LogEntry` could drift if key-order/whitespace ever
|
|
1142
|
+
* diverged from what was PUT).
|
|
933
1143
|
*
|
|
934
1144
|
* Same error semantics as {@link readLogEntry}.
|
|
935
1145
|
*/
|
|
@@ -967,7 +1177,7 @@ const walkLogRange = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) =
|
|
|
967
1177
|
* this is the bytes-aware core `walkLogRange` is implemented on top of.
|
|
968
1178
|
*
|
|
969
1179
|
* The compactor consumes this so it gets BOTH the parsed entries (for
|
|
970
|
-
* the fold) AND the exact byte sum (
|
|
1180
|
+
* the fold) AND the exact byte sum (for `mean_entry_bytes` stamping) from a
|
|
971
1181
|
* single walk.
|
|
972
1182
|
*/
|
|
973
1183
|
const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) => {
|
|
@@ -980,7 +1190,7 @@ const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive
|
|
|
980
1190
|
const promises = [];
|
|
981
1191
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
982
1192
|
const seq = fromSeq + i;
|
|
983
|
-
promises.push(readLogEntryWithBytes(storage,
|
|
1193
|
+
promises.push(readLogEntryWithBytes(storage, logObjectKey(logPrefix, seq), opts));
|
|
984
1194
|
}
|
|
985
1195
|
const results = await Promise.all(promises);
|
|
986
1196
|
for (let i = 0; i < results.length; i++) out[chunkStart + i] = results[i];
|
|
@@ -1030,76 +1240,6 @@ const foldLogEntriesOnto = (map, entries, opts) => {
|
|
|
1030
1240
|
}
|
|
1031
1241
|
}
|
|
1032
1242
|
};
|
|
1033
|
-
const MAX_SEQ = 10 ** 12 - 1;
|
|
1034
|
-
/**
|
|
1035
|
-
* Build the on-bucket key for a snapshot file. `sha256` must be the
|
|
1036
|
-
* lowercase hex digest of the canonical body
|
|
1037
|
-
* ({@link encodeSnapshotBody}); the filename embeds it so a crashed
|
|
1038
|
-
* mid-PUT body that doesn't match its own filename hash is rejected
|
|
1039
|
-
* by readers as "missing."
|
|
1040
|
-
*
|
|
1041
|
-
* @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
|
|
1042
|
-
* out of range or `sha256` is not 64 hex chars.
|
|
1043
|
-
*/
|
|
1044
|
-
const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
|
|
1045
|
-
if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
|
|
1046
|
-
if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
|
|
1047
|
-
const pad = (n) => n.toString().padStart(12, "0");
|
|
1048
|
-
return `${collectionPrefix}/snapshot/L9/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
|
|
1049
|
-
};
|
|
1050
|
-
/**
|
|
1051
|
-
* Canonical byte encoding of a {@link SnapshotBody}. Only the
|
|
1052
|
-
* compactor produces snapshots, and only this function does the
|
|
1053
|
-
* serialisation — so the byte output is deterministic for a given
|
|
1054
|
-
* `SnapshotBody` (identical input ⇒ identical bytes ⇒ identical
|
|
1055
|
-
* filename hash).
|
|
1056
|
-
*/
|
|
1057
|
-
const encodeSnapshotBody = (s) => encodeJsonBytes(s);
|
|
1058
|
-
/**
|
|
1059
|
-
* Load a snapshot file as a `Map<_id, DocumentData>`. Verifies the
|
|
1060
|
-
* body hash against the filename, checks the schema version, and
|
|
1061
|
-
* cross-checks `body.collection` against the caller's expectation.
|
|
1062
|
-
*
|
|
1063
|
-
* @example
|
|
1064
|
-
* ```ts
|
|
1065
|
-
* import { loadSnapshotAsMap } from "@gusto/baerly-storage";
|
|
1066
|
-
*
|
|
1067
|
-
* const map =
|
|
1068
|
-
* srcCurrent.snapshot === null
|
|
1069
|
-
* ? new Map<string, DocumentData>()
|
|
1070
|
-
* : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
|
|
1071
|
-
* ```
|
|
1072
|
-
*
|
|
1073
|
-
* @throws BaerlyError code="Internal" — pointer resolves to no body,
|
|
1074
|
-
* or the body's SHA-256 doesn't match the hash embedded in the
|
|
1075
|
-
* filename. (Protocol invariant violation; a corrupted snapshot
|
|
1076
|
-
* the reader cannot continue past.)
|
|
1077
|
-
* @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
|
|
1078
|
-
* isn't an object, has an unsupported `schema_version`, or carries a
|
|
1079
|
-
* `collection` that doesn't match `expectedCollection`.
|
|
1080
|
-
*/
|
|
1081
|
-
const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
|
|
1082
|
-
const got = await storage.get(key, signal !== void 0 ? { signal } : void 0);
|
|
1083
|
-
if (got === null) throw new BaerlyError("Internal", `compact: snapshot pointer ${key} resolves to no body; protocol violation`);
|
|
1084
|
-
const filename = key.slice(key.lastIndexOf("/") + 1).replace(/\.json$/, "");
|
|
1085
|
-
const lastDash = filename.lastIndexOf("-");
|
|
1086
|
-
const expectedHash = filename.slice(lastDash + 1);
|
|
1087
|
-
const actualHash = await snapshotHash(got.body);
|
|
1088
|
-
if (actualHash !== expectedHash) throw new BaerlyError("Internal", `compact: snapshot ${key} body hash mismatch (got ${actualHash}); protocol violation`);
|
|
1089
|
-
let parsed;
|
|
1090
|
-
try {
|
|
1091
|
-
parsed = decodeJsonBytes(got.body);
|
|
1092
|
-
} catch (error) {
|
|
1093
|
-
throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
|
|
1094
|
-
}
|
|
1095
|
-
if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not an object`);
|
|
1096
|
-
const body = parsed;
|
|
1097
|
-
if (body.schema_version !== 1) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} has unsupported schema_version ${String(body.schema_version)}`);
|
|
1098
|
-
if (body.collection !== expectedCollection) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} carries collection ${body.collection}, expected ${expectedCollection}`);
|
|
1099
|
-
const out = /* @__PURE__ */ new Map();
|
|
1100
|
-
for (const row of body.docs) out.set(row._id, row.body);
|
|
1101
|
-
return out;
|
|
1102
|
-
};
|
|
1103
1243
|
//#endregion
|
|
1104
1244
|
//#region packages/server/src/compactor.ts
|
|
1105
1245
|
/**
|
|
@@ -1115,8 +1255,7 @@ const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
|
|
|
1115
1255
|
* "missing." The atomic moment is the CAS-swap of
|
|
1116
1256
|
* `current.json.snapshot`, which is a single conditional PUT.
|
|
1117
1257
|
*
|
|
1118
|
-
* Order of operations
|
|
1119
|
-
* REVERSED ordering): PUT snapshot body → CAS-advance `current.json`.
|
|
1258
|
+
* Order of operations: PUT snapshot body → CAS-advance `current.json`.
|
|
1120
1259
|
* A crash before the swap leaves an orphan snapshot file; the swap
|
|
1121
1260
|
* succeeds iff our captured ETag still matches.
|
|
1122
1261
|
*
|
|
@@ -1189,7 +1328,8 @@ const compact = async (args, options = {}) => {
|
|
|
1189
1328
|
const current = read.json;
|
|
1190
1329
|
const baseEtag = read.etag;
|
|
1191
1330
|
const logSeqStartBefore = logSeqStartOf(current);
|
|
1192
|
-
const
|
|
1331
|
+
const discoveredTail = (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStartBefore, current.tail_hint, internal.knownTail ?? 0), { ...options.signal !== void 0 && { signal: options.signal } })).tail;
|
|
1332
|
+
const nextSeq = discoveredTail;
|
|
1193
1333
|
if (nextSeq - logSeqStartBefore < minToCompact) return {
|
|
1194
1334
|
written: false,
|
|
1195
1335
|
skippedReason: "below-min-threshold",
|
|
@@ -1250,13 +1390,15 @@ const compact = async (args, options = {}) => {
|
|
|
1250
1390
|
...options.signal !== void 0 && { signal: options.signal }
|
|
1251
1391
|
};
|
|
1252
1392
|
await storage.put(newKey, bodyBytes, putOpts);
|
|
1393
|
+
const entriesFolded = foldEnd - logSeqStartBefore;
|
|
1253
1394
|
const nextBody = encodeJsonBytes({
|
|
1254
1395
|
...current,
|
|
1255
1396
|
snapshot: newKey,
|
|
1256
1397
|
log_seq_start: foldEnd,
|
|
1398
|
+
tail_hint: Math.max(current.tail_hint, discoveredTail),
|
|
1257
1399
|
snapshot_bytes: bodyBytes.byteLength,
|
|
1258
1400
|
snapshot_rows: base.size,
|
|
1259
|
-
|
|
1401
|
+
mean_entry_bytes: entriesFolded > 0 ? Math.round(foldedSliceBytes / entriesFolded) : current.mean_entry_bytes ?? 0
|
|
1260
1402
|
});
|
|
1261
1403
|
const casOpts = {
|
|
1262
1404
|
ifMatch: baseEtag,
|
|
@@ -1280,10 +1422,9 @@ const compact = async (args, options = {}) => {
|
|
|
1280
1422
|
}
|
|
1281
1423
|
throw error;
|
|
1282
1424
|
}
|
|
1283
|
-
const entriesFolded = foldEnd - logSeqStartBefore;
|
|
1284
1425
|
const metrics = ctxMetrics$2();
|
|
1285
1426
|
metrics.histogram("db.compact.entries_folded", entriesFolded, { collection: collectionName });
|
|
1286
|
-
metrics.gauge("db.manifest.lag_window_depth",
|
|
1427
|
+
metrics.gauge("db.manifest.lag_window_depth", discoveredTail - foldEnd, { collection: collectionName });
|
|
1287
1428
|
return {
|
|
1288
1429
|
written: true,
|
|
1289
1430
|
previousSnapshotKey: current.snapshot,
|
|
@@ -1467,8 +1608,8 @@ const runGc = async (args, options = {}) => {
|
|
|
1467
1608
|
for (const cand of sweepCandidates) if (toSweep.length < maxSweeps && Date.parse(cand.due_at) <= nowMs) toSweep.push(cand);
|
|
1468
1609
|
else remaining.push(cand);
|
|
1469
1610
|
await Promise.all(toSweep.map((c) => storage.delete(c.key, signalOpts)));
|
|
1470
|
-
const
|
|
1471
|
-
const lastSweptAt = toSweep.length > 0 ? now().toISOString() :
|
|
1611
|
+
const sweptKeys = new Set(toSweep.map((c) => c.key));
|
|
1612
|
+
const lastSweptAt = toSweep.length > 0 ? now().toISOString() : "";
|
|
1472
1613
|
const markedSummary = {
|
|
1473
1614
|
stale_log: markedStaleLog,
|
|
1474
1615
|
orphan_snapshot: markedOrphanSnapshot,
|
|
@@ -1476,11 +1617,12 @@ const runGc = async (args, options = {}) => {
|
|
|
1476
1617
|
};
|
|
1477
1618
|
let pendingDepth;
|
|
1478
1619
|
try {
|
|
1479
|
-
pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, () => ({
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1620
|
+
pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, (latest) => mergeGcPending(latest, {
|
|
1621
|
+
sweptKeys,
|
|
1622
|
+
newCandidates,
|
|
1623
|
+
lastSweptAt,
|
|
1624
|
+
nextContentCursor,
|
|
1625
|
+
maxCandidates: GC_MAX_PENDING_CANDIDATES
|
|
1484
1626
|
}), signalOpts)).json.candidates.length;
|
|
1485
1627
|
} catch (error) {
|
|
1486
1628
|
if (error instanceof BaerlyError && error.code === "Conflict") pendingDepth = remaining.length;
|
|
@@ -1549,7 +1691,7 @@ const computeDueAt = (entry, now, graceMs) => {
|
|
|
1549
1691
|
};
|
|
1550
1692
|
/**
|
|
1551
1693
|
* Build the live content-hash set. The set covers every live
|
|
1552
|
-
* post-image: every `entry.after` in `[logSeqStart,
|
|
1694
|
+
* post-image: every `entry.after` in `[logSeqStart, tail_hint)` plus
|
|
1553
1695
|
* every row body in the current snapshot.
|
|
1554
1696
|
*
|
|
1555
1697
|
* A snapshot read that throws (corrupt body, hash mismatch) is
|
|
@@ -1561,9 +1703,9 @@ const computeDueAt = (entry, now, graceMs) => {
|
|
|
1561
1703
|
const collectLiveContentHashes = async (storage, collectionPrefix, collectionName, current, logSeqStart, signal) => {
|
|
1562
1704
|
const hashes = /* @__PURE__ */ new Set();
|
|
1563
1705
|
const getOpts = signal !== void 0 ? { signal } : void 0;
|
|
1564
|
-
const
|
|
1565
|
-
|
|
1566
|
-
const got = await storage.get(
|
|
1706
|
+
const { tail } = await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, current.tail_hint), { signal });
|
|
1707
|
+
const ingestLogEntry = async (s) => {
|
|
1708
|
+
const got = await storage.get(logObjectKey(collectionPrefix, s), getOpts);
|
|
1567
1709
|
if (got === null) return;
|
|
1568
1710
|
let entry;
|
|
1569
1711
|
try {
|
|
@@ -1574,8 +1716,14 @@ const collectLiveContentHashes = async (storage, collectionPrefix, collectionNam
|
|
|
1574
1716
|
if (entry.after === void 0) return;
|
|
1575
1717
|
const bodyBytes = encodeJsonBytes(entry.after);
|
|
1576
1718
|
hashes.add(await versionFromContent(bodyBytes));
|
|
1577
|
-
}
|
|
1578
|
-
|
|
1719
|
+
};
|
|
1720
|
+
for (let chunkStart = logSeqStart; chunkStart < tail; chunkStart += 16) {
|
|
1721
|
+
signal?.throwIfAborted();
|
|
1722
|
+
const chunkEnd = Math.min(chunkStart + 16, tail);
|
|
1723
|
+
const chunk = [];
|
|
1724
|
+
for (let s = chunkStart; s < chunkEnd; s++) chunk.push(ingestLogEntry(s));
|
|
1725
|
+
await Promise.all(chunk);
|
|
1726
|
+
}
|
|
1579
1727
|
if (current.snapshot !== null) try {
|
|
1580
1728
|
const map = await loadSnapshotAsMap(storage, current.snapshot, collectionName, signal);
|
|
1581
1729
|
const rowReads = [];
|
|
@@ -1619,8 +1767,8 @@ const collectLiveContentHashes = async (storage, collectionPrefix, collectionNam
|
|
|
1619
1767
|
*
|
|
1620
1768
|
* @see ./indexes.ts — `IndexDefinition`, `allIndexKeysFor`,
|
|
1621
1769
|
* `indexKeyPrefix`.
|
|
1622
|
-
* @see ./writer.ts — the
|
|
1623
|
-
*
|
|
1770
|
+
* @see ./writer.ts — the hybrid index emission path this command
|
|
1771
|
+
* reconciles.
|
|
1624
1772
|
*/
|
|
1625
1773
|
const EMPTY_BODY = new Uint8Array(0);
|
|
1626
1774
|
const APPLICATION_JSON = "application/json";
|
|
@@ -1641,7 +1789,7 @@ const APPLICATION_JSON = "application/json";
|
|
|
1641
1789
|
* @throws BaerlyError code="InvalidResponse" — `current.json` missing.
|
|
1642
1790
|
* @throws BaerlyError code="Internal" — protocol-invariant
|
|
1643
1791
|
* violation (snapshot hash mismatch, missing log entry inside
|
|
1644
|
-
* `[log_seq_start,
|
|
1792
|
+
* `[log_seq_start, tail_hint)`).
|
|
1645
1793
|
*/
|
|
1646
1794
|
const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
|
|
1647
1795
|
const read = await readCurrentJson(storage, currentJsonKey);
|
|
@@ -1651,9 +1799,11 @@ const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
|
|
|
1651
1799
|
const collectionPrefix = currentJsonKey.slice(0, lastSlash);
|
|
1652
1800
|
const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
1653
1801
|
const logSeqStart = logSeqStartOf(read.json);
|
|
1654
|
-
const
|
|
1802
|
+
const hint = read.json.tail_hint;
|
|
1655
1803
|
const live = read.json.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, read.json.snapshot, collection);
|
|
1656
|
-
foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart,
|
|
1804
|
+
foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart, hint), { collection });
|
|
1805
|
+
const probeOpts = opts.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
1806
|
+
foldLogEntriesOnto(live, (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, hint), probeOpts)).entries, { collection });
|
|
1657
1807
|
const expected = /* @__PURE__ */ new Set();
|
|
1658
1808
|
for (const [docId, body] of live) for (const k of allIndexKeysFor(collectionPrefix, [def], body, docId)) expected.add(k);
|
|
1659
1809
|
const actual = /* @__PURE__ */ new Set();
|
|
@@ -1759,36 +1909,78 @@ const runScheduledMaintenance = async (args, options = {}) => {
|
|
|
1759
1909
|
})
|
|
1760
1910
|
};
|
|
1761
1911
|
};
|
|
1762
|
-
const
|
|
1912
|
+
const profileToScheduledOptions = (profile) => ({
|
|
1763
1913
|
compact: {
|
|
1764
|
-
maxEntriesPerRun:
|
|
1914
|
+
maxEntriesPerRun: profile.maxFoldEntriesPerPass,
|
|
1765
1915
|
minEntriesToCompact: 50
|
|
1766
1916
|
},
|
|
1767
1917
|
gc: {
|
|
1768
|
-
maxMarksPerRun:
|
|
1769
|
-
maxSweepsPerRun:
|
|
1918
|
+
maxMarksPerRun: profile.gcMaxMarks,
|
|
1919
|
+
maxSweepsPerRun: profile.gcMaxSweeps
|
|
1770
1920
|
}
|
|
1771
|
-
};
|
|
1921
|
+
});
|
|
1922
|
+
/**
|
|
1923
|
+
* Tuning profile for the 50-subrequest Cloudflare free-tier budget,
|
|
1924
|
+
* derived from {@link MAINTENANCE_PROFILE_CF_FREE} (one source of truth).
|
|
1925
|
+
*
|
|
1926
|
+
* Budget arithmetic (worst case):
|
|
1927
|
+
* compact: 1 GET current + 1 GET snapshot (if any) + N GETs log
|
|
1928
|
+
* + 1 PUT snapshot + 1 PUT current = 3 + N (N = maxEntriesPerRun).
|
|
1929
|
+
* runGc: 1 GET current + 1 GET pending + 3 LISTs (one page each)
|
|
1930
|
+
* + M GETs log (live tail for content hashes) + S DELETEs + 1 PUT
|
|
1931
|
+
* pending = 6 + M + S.
|
|
1932
|
+
*
|
|
1933
|
+
* With N=20, M=20, S=10 the totals are: compact ≈ 23, gc ≈ 36 — over
|
|
1934
|
+
* the 50 cap combined. The Cloudflare scheduled handler therefore
|
|
1935
|
+
* alternates phases per tick (even minute → compact, odd minute →
|
|
1936
|
+
* GC) by calling `compact()` / `runGc()` directly instead of
|
|
1937
|
+
* `runScheduledMaintenance`; the `maintenance.budget.test.ts`
|
|
1938
|
+
* worst-case test proves each phase in isolation sits under 50 ops
|
|
1939
|
+
* with these bounds.
|
|
1940
|
+
*/
|
|
1941
|
+
const CLOUDFLARE_FREE_TIER = profileToScheduledOptions(MAINTENANCE_PROFILE_CF_FREE);
|
|
1772
1942
|
/**
|
|
1773
|
-
* Floor-based GC-cadence boundary crossing.
|
|
1774
|
-
*
|
|
1775
|
-
*
|
|
1776
|
-
* trips on any interval boundary inside `(prevSeq, nextSeq]
|
|
1943
|
+
* Floor-based GC-cadence boundary crossing. Every commit advances the
|
|
1944
|
+
* writer-observed tail by one logical slot; `tail_hint` remains a
|
|
1945
|
+
* lower bound and may be refreshed later by maintenance. The floor test
|
|
1946
|
+
* trips on any interval boundary inside `(prevSeq, nextSeq]`, so it
|
|
1947
|
+
* stays correct whether the observed/probed tail advances by one or
|
|
1948
|
+
* catches up by a larger step.
|
|
1777
1949
|
*/
|
|
1778
1950
|
const crossesGcBoundary = (prevSeq, nextSeq, interval) => Math.floor(prevSeq / interval) !== Math.floor(nextSeq / interval);
|
|
1779
1951
|
/**
|
|
1952
|
+
* Derived live-tail byte size — the ratio TRIGGER input that replaces the
|
|
1953
|
+
* formerly-stored exact byte count. `(observedTail − log_seq_start) ×
|
|
1954
|
+
* mean_entry_bytes`. `observedTail` is a param so write-tick callers can
|
|
1955
|
+
* pass the writer-observed tail and scheduled callers can pass a
|
|
1956
|
+
* probe-discovered true tail. Cold-start (no mean yet) falls back to
|
|
1957
|
+
* {@link MAINTENANCE_COLD_START_ENTRY_BYTES}, never 0 (see that constant).
|
|
1958
|
+
* TRIGGER-only — structurally barred from `foldViable` (the exact-bytes CPU-kill
|
|
1959
|
+
* ceiling).
|
|
1960
|
+
*/
|
|
1961
|
+
const estimateTailBytes = (current, observedTail) => Math.max(0, observedTail - current.log_seq_start) * (current.mean_entry_bytes ?? 128);
|
|
1962
|
+
/**
|
|
1780
1963
|
* The cheap DISPATCH gate the writer calls after a commit — zero
|
|
1781
|
-
* storage ops; it reads the post-
|
|
1964
|
+
* storage ops; it reads the post-commit {@link CurrentJson} already in
|
|
1782
1965
|
* scope. Ratio-OR-boundary, WITHOUT the entry floor (the floor is part
|
|
1783
1966
|
* of the fold-trigger Gate 1 checked inside {@link runBoundedMaintenance}).
|
|
1784
|
-
* Returning `false` lets the writer skip the dispatch entirely.
|
|
1967
|
+
* Returning `false` lets the writer skip the dispatch entirely. Ratio
|
|
1968
|
+
* numerator is the DERIVED {@link estimateTailBytes};
|
|
1969
|
+
* `observedTail` threads from the caller (the writer's in-memory observed
|
|
1970
|
+
* tail `seq+1` under single-write commit). Both the boundary check and the
|
|
1971
|
+
* ratio key off `observedTail`, since the stored `tail_hint` is now only a
|
|
1972
|
+
* non-authoritative lower bound (compactor-advanced).
|
|
1785
1973
|
*/
|
|
1786
|
-
const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundary(prevSeq,
|
|
1974
|
+
const shouldFireMaintenance = (current, prevSeq, gcInterval, observedTail) => crossesGcBoundary(prevSeq, observedTail, gcInterval) || estimateTailBytes(current, observedTail) / Math.max(current.snapshot_bytes, 65536) >= 1;
|
|
1787
1975
|
/**
|
|
1788
1976
|
* The write-tick maintenance runner: a bounded, single-attempt
|
|
1789
1977
|
* composition of {@link compact} and {@link runGc} for one collection,
|
|
1790
1978
|
* sized to fit a CPU-killable Cloudflare free-tier isolate by default.
|
|
1791
1979
|
*
|
|
1980
|
+
* INVARIANT: a {@link MaintenanceProfile} changes only maintenance rate +
|
|
1981
|
+
* defer threshold — never data/API/query/wire/correctness; reads stay pure.
|
|
1982
|
+
* See `maintenance-profile-equivalence.test.ts`.
|
|
1983
|
+
*
|
|
1792
1984
|
* Never throws — the write-tick caller must never see a maintenance
|
|
1793
1985
|
* failure. Expected signals (CAS contention, an over-ceiling deferral)
|
|
1794
1986
|
* are swallowed quietly; unexpected throws bump
|
|
@@ -1796,7 +1988,8 @@ const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundar
|
|
|
1796
1988
|
* swallowed.
|
|
1797
1989
|
*
|
|
1798
1990
|
* Control flow (fold-priority with a stateless GC-starvation guard):
|
|
1799
|
-
* 0. `disabled` →
|
|
1991
|
+
* 0. `disabled` → skip fold/GC, but still allow the rate-limited
|
|
1992
|
+
* tail_hint refresh that bounds read/write forward-probes.
|
|
1800
1993
|
* 1. Read `current.json` once.
|
|
1801
1994
|
* 2. Hard-GC starvation boundary (`"single"` only) → GC slice, skip fold.
|
|
1802
1995
|
* 3. Gate 1 (ratio AND entry floor) trips → fold if viable, else
|
|
@@ -1805,17 +1998,14 @@ const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundar
|
|
|
1805
1998
|
* 4. Otherwise (or on `"both"`) → GC iff the cadence boundary crossed.
|
|
1806
1999
|
*/
|
|
1807
2000
|
const runBoundedMaintenance = async (args, options) => {
|
|
1808
|
-
if (args.disabled === true) return;
|
|
1809
2001
|
const { storage, currentJsonKey, prevSeq } = args;
|
|
1810
|
-
const
|
|
2002
|
+
const profile = options?.profile ?? MAINTENANCE_PROFILE_CF_FREE;
|
|
2003
|
+
const { maxFoldEntriesPerPass, gcMaxMarks, gcMaxSweeps, gcInterval } = profile;
|
|
1811
2004
|
const minEntriesToCompact = options?.minEntriesToCompact ?? 50;
|
|
1812
|
-
const gcMaxMarks = options?.gcMaxMarks ?? 20;
|
|
1813
|
-
const gcMaxSweeps = options?.gcMaxSweeps ?? 10;
|
|
1814
|
-
const gcInterval = options?.gcInterval ?? 4;
|
|
1815
2005
|
const phasesPerTick = options?.phasesPerTick ?? "single";
|
|
1816
2006
|
const signal = options?.signal;
|
|
1817
|
-
const C = args.maxFoldBytes ??
|
|
1818
|
-
const E =
|
|
2007
|
+
const C = args.maxFoldBytes ?? profile.maxFoldBytes;
|
|
2008
|
+
const E = profile.maxFoldRows;
|
|
1819
2009
|
const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
|
|
1820
2010
|
const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
1821
2011
|
const gcOpts = {
|
|
@@ -1829,14 +2019,29 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1829
2019
|
const read = await readCurrentJson(storage, currentJsonKey, signal !== void 0 ? { signal } : void 0);
|
|
1830
2020
|
if (read === null) return;
|
|
1831
2021
|
const current = read.json;
|
|
1832
|
-
const tailBytes = current.tail_bytes;
|
|
1833
2022
|
const snapshotBytes = current.snapshot_bytes;
|
|
1834
2023
|
const snapshotRows = current.snapshot_rows;
|
|
1835
|
-
const nextSeq = current.next_seq;
|
|
1836
2024
|
const logSeqStart = current.log_seq_start;
|
|
1837
|
-
const
|
|
2025
|
+
const probeFloor = Math.max(logSeqStart, current.tail_hint);
|
|
2026
|
+
let nextSeq;
|
|
2027
|
+
if (args.observedTail !== void 0) nextSeq = Math.max(args.observedTail, probeFloor);
|
|
2028
|
+
else nextSeq = (await probeTailFrom(storage, collectionPrefix, probeFloor, signal !== void 0 ? { signal } : void 0)).tail;
|
|
2029
|
+
const gate1 = estimateTailBytes(current, nextSeq) / Math.max(snapshotBytes, 65536) >= 1 && nextSeq - logSeqStart >= minEntriesToCompact;
|
|
1838
2030
|
const foldViable = snapshotBytes <= C && snapshotRows + maxFoldEntriesPerPass <= E;
|
|
1839
2031
|
const gcDue = crossesGcBoundary(prevSeq, nextSeq, gcInterval);
|
|
2032
|
+
const refreshTailHintIfNeeded = async () => {
|
|
2033
|
+
if (nextSeq - current.tail_hint < 128) return;
|
|
2034
|
+
try {
|
|
2035
|
+
await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
|
|
2036
|
+
...c,
|
|
2037
|
+
tail_hint: Math.max(c.tail_hint, nextSeq)
|
|
2038
|
+
}), signal !== void 0 ? { signal } : void 0);
|
|
2039
|
+
} catch {}
|
|
2040
|
+
};
|
|
2041
|
+
if (args.disabled === true) {
|
|
2042
|
+
await refreshTailHintIfNeeded();
|
|
2043
|
+
return;
|
|
2044
|
+
}
|
|
1840
2045
|
const hardGc = crossesGcBoundary(prevSeq, nextSeq, gcInterval * 4);
|
|
1841
2046
|
if (phasesPerTick === "single" && hardGc) {
|
|
1842
2047
|
await runGc({
|
|
@@ -1854,6 +2059,7 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1854
2059
|
minEntriesToCompact,
|
|
1855
2060
|
ceilingBytes: C,
|
|
1856
2061
|
ceilingEntries: E,
|
|
2062
|
+
knownTail: nextSeq,
|
|
1857
2063
|
...signal !== void 0 && { signal }
|
|
1858
2064
|
});
|
|
1859
2065
|
if (phasesPerTick === "single") return;
|
|
@@ -1862,6 +2068,7 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1862
2068
|
collection,
|
|
1863
2069
|
dimension: snapshotBytes > C ? "bytes" : "rows"
|
|
1864
2070
|
});
|
|
2071
|
+
await refreshTailHintIfNeeded();
|
|
1865
2072
|
if (nextSeq - (current.last_warned_seq ?? 0) >= 1e3) {
|
|
1866
2073
|
console.warn(`baerly-storage: collection "${collection}" is deferring compaction — its snapshot exceeds the fold ceiling (${snapshotBytes > C ? "bytes" : "rows"}), so the tail keeps growing and read amplification will climb. This is graduation-pending: the dataset has outgrown prototype-tier maintenance. On paid Cloudflare / Node you can raise BAERLY_MAINTENANCE_MAX_FOLD_BYTES — but on Cloudflare a cap above what a single isolate can fold makes folds get CPU-killed mid-flight and silently not land (no clean metric — watch snapshot age / object count); see docs/about/graduation.md. Otherwise, graduate to a server-backed database.`);
|
|
1867
2074
|
try {
|
|
@@ -1890,6 +2097,6 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1890
2097
|
*/
|
|
1891
2098
|
const dispatchInlineAwaited = (task) => task();
|
|
1892
2099
|
//#endregion
|
|
1893
|
-
export {
|
|
2100
|
+
export { logObjectKey as C, matchesWire as D, assertWireSatisfiable as E, deepEqualDocumentValue as O, probeTailFrom as S, validateWire as T, encodeSnapshotBody as _, runBoundedMaintenance as a, assertNameNotReserved as b, rebuildIndex as c, foldLogEntriesOnto as d, readLogEntry as f, validateIndexDefinition as g, encodeIndexValue as h, estimateTailBytes as i, runGc as l, allIndexKeysFor as m, crossesGcBoundary as n, runScheduledMaintenance as o, walkLogRange as p, dispatchInlineAwaited as r, shouldFireMaintenance as s, CLOUDFLARE_FREE_TIER as t, compact as u, loadSnapshotAsMap as v, versionFromContent as w, findLogTail as x, snapshotKey as y };
|
|
1894
2101
|
|
|
1895
|
-
//# sourceMappingURL=maintenance-
|
|
2102
|
+
//# sourceMappingURL=maintenance-CWBQpZ2y.js.map
|