@gusto/baerly-storage 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -0
- package/dist/API.md +850 -0
- package/dist/app-config-Cb5dO2s9.d.ts +480 -0
- package/dist/app-config.d.ts +2 -0
- package/dist/app-config.js +8 -0
- package/dist/app-config.js.map +1 -0
- package/dist/auth.d.ts +237 -0
- package/dist/auth.js +1335 -0
- package/dist/auth.js.map +1 -0
- package/dist/baerly.js +761 -0
- package/dist/baerly.js.map +1 -0
- package/dist/bucket-uri-BrDEYvzy.js +4922 -0
- package/dist/bucket-uri-BrDEYvzy.js.map +1 -0
- package/dist/chunk-EAsCxrDo.js +14 -0
- package/dist/client-BLf3Ae5s.d.ts +233 -0
- package/dist/client-react.d.ts +131 -0
- package/dist/client-react.js +559 -0
- package/dist/client-react.js.map +1 -0
- package/dist/client.d.ts +4 -0
- package/dist/client.js +160 -0
- package/dist/client.js.map +1 -0
- package/dist/cloudflare-CINJPKCW.js +911 -0
- package/dist/cloudflare-CINJPKCW.js.map +1 -0
- package/dist/cloudflare.d.ts +211 -0
- package/dist/cloudflare.js +445 -0
- package/dist/cloudflare.js.map +1 -0
- package/dist/config-Bsg8wgwo.js +198 -0
- package/dist/config-Bsg8wgwo.js.map +1 -0
- package/dist/context-Oj7ffwD-.js +187 -0
- package/dist/context-Oj7ffwD-.js.map +1 -0
- package/dist/cost-B1WiZxlf.js +203 -0
- package/dist/cost-B1WiZxlf.js.map +1 -0
- package/dist/current-json-BkatBzF7.js +252 -0
- package/dist/current-json-BkatBzF7.js.map +1 -0
- package/dist/db-D62JguBc.d.ts +336 -0
- package/dist/deploy-BB-wqbhf.js +83 -0
- package/dist/deploy-BB-wqbhf.js.map +1 -0
- package/dist/dev-banner-CWHIk8tH.d.ts +26 -0
- package/dist/dev-vite.d.ts +146 -0
- package/dist/dev-vite.js +223 -0
- package/dist/dev-vite.js.map +1 -0
- package/dist/dev.d.ts +3 -0
- package/dist/dev.js +2 -0
- package/dist/doctor-B8h9GCN9.js +331 -0
- package/dist/doctor-B8h9GCN9.js.map +1 -0
- package/dist/dump-BHo_sIMZ.js +116 -0
- package/dist/dump-BHo_sIMZ.js.map +1 -0
- package/dist/errors-DOdL95kK.js +58 -0
- package/dist/errors-DOdL95kK.js.map +1 -0
- package/dist/export-Dz2PZKSR.js +423 -0
- package/dist/export-Dz2PZKSR.js.map +1 -0
- package/dist/fsck-DEz9lyCy.js +293 -0
- package/dist/fsck-DEz9lyCy.js.map +1 -0
- package/dist/http-CjUO1Bec.js +1756 -0
- package/dist/http-CjUO1Bec.js.map +1 -0
- package/dist/http.d.ts +1619 -0
- package/dist/http.js +2 -0
- package/dist/index-BGC-ENnj.d.ts +1065 -0
- package/dist/index-BYv0hb8M.d.ts +1457 -0
- package/dist/index-BquQkhJi.d.ts +112 -0
- package/dist/index-COsQ-Mlz.d.ts +146 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +6 -0
- package/dist/indexes-BXn7a06a.js +297 -0
- package/dist/indexes-BXn7a06a.js.map +1 -0
- package/dist/indexes-DwOXkn1Y.d.ts +22 -0
- package/dist/inspect-x02aFWnh.js +149 -0
- package/dist/inspect-x02aFWnh.js.map +1 -0
- package/dist/log-walk-DIru96UT.js +260 -0
- package/dist/log-walk-DIru96UT.js.map +1 -0
- package/dist/maintenance-Bn6ul9aV.js +758 -0
- package/dist/maintenance-Bn6ul9aV.js.map +1 -0
- package/dist/maintenance.d.ts +323 -0
- package/dist/maintenance.js +2 -0
- package/dist/node.d.ts +422 -0
- package/dist/node.js +2 -0
- package/dist/normalize-XWY25_sG.js +133 -0
- package/dist/normalize-XWY25_sG.js.map +1 -0
- package/dist/observability-2IDYRuZK.js +2333 -0
- package/dist/observability-2IDYRuZK.js.map +1 -0
- package/dist/observability.d.ts +3 -0
- package/dist/observability.js +3 -0
- package/dist/plan-Cs17VqjU.js +224 -0
- package/dist/plan-Cs17VqjU.js.map +1 -0
- package/dist/query-DcSbFLJt.js +1880 -0
- package/dist/query-DcSbFLJt.js.map +1 -0
- package/dist/rebuild-index-C5gTXryd.js +119 -0
- package/dist/rebuild-index-C5gTXryd.js.map +1 -0
- package/dist/rebuild-index-DwXM0WfA.js +104 -0
- package/dist/rebuild-index-DwXM0WfA.js.map +1 -0
- package/dist/request-BEF_OvMh.js +72 -0
- package/dist/request-BEF_OvMh.js.map +1 -0
- package/dist/restore-CbKVMMpm.js +847 -0
- package/dist/restore-CbKVMMpm.js.map +1 -0
- package/dist/shared-secret-BJdSwpgw.js +71 -0
- package/dist/shared-secret-BJdSwpgw.js.map +1 -0
- package/dist/snapshot-D7C4_FSl.js +71 -0
- package/dist/snapshot-D7C4_FSl.js.map +1 -0
- package/dist/snapshot-z_N9hpTe.js +917 -0
- package/dist/snapshot-z_N9hpTe.js.map +1 -0
- package/dist/src-Chafbw8r.js +522 -0
- package/dist/src-Chafbw8r.js.map +1 -0
- package/dist/src-DyET_VPP.js +6199 -0
- package/dist/src-DyET_VPP.js.map +1 -0
- package/dist/src-n0Te1r98.js +352 -0
- package/dist/src-n0Te1r98.js.map +1 -0
- package/dist/subcommand-BWs6K7zb.js +110 -0
- package/dist/subcommand-BWs6K7zb.js.map +1 -0
- package/dist/usage-RVUQMLop.js +339 -0
- package/dist/usage-RVUQMLop.js.map +1 -0
- package/dist/validate-DhH1lO8I.js +377 -0
- package/dist/validate-DhH1lO8I.js.map +1 -0
- package/dist/wire-izfGZDvr.js +7 -0
- package/dist/wire-izfGZDvr.js.map +1 -0
- package/package.json +189 -0
|
@@ -0,0 +1,758 @@
|
|
|
1
|
+
import { f as decodeJsonBytes, n as logSeqStartOf, o as GC_MAX_PENDING_CANDIDATES, p as encodeJsonBytes, r as readCurrentJson, s as GC_PENDING_CONTENT_TYPE } from "./current-json-BkatBzF7.js";
|
|
2
|
+
import { t as BaerlyError } from "./errors-DOdL95kK.js";
|
|
3
|
+
import { d as snapshotHash, f as versionFromContent, i as foldLogEntriesOnto, l as indexKeyPrefix, n as loadSnapshotAsMap, o as walkLogRange, r as snapshotKey, s as allIndexKeysFor, t as encodeSnapshotBody } from "./snapshot-z_N9hpTe.js";
|
|
4
|
+
import { i as noopMetricsRecorder, n as getCurrentContext } from "./context-Oj7ffwD-.js";
|
|
5
|
+
//#region packages/protocol/src/coordination/gc-pending.ts
|
|
6
|
+
/**
|
|
7
|
+
* `gc/pending.json` — the per-collection GC candidate ledger. CAS-
|
|
8
|
+
* protected; one per `(tenant, collection)` key — for example
|
|
9
|
+
* `<tenant>/<collection>/gc/pending.json`. `runGc()` in
|
|
10
|
+
* `packages/server/src/gc.ts` is the single writer that adds and
|
|
11
|
+
* removes entries. Stays small in steady state because every
|
|
12
|
+
* candidate is deleted once `due_at` passes.
|
|
13
|
+
*
|
|
14
|
+
* "Mark" appends candidates with a future `due_at`; "sweep" deletes
|
|
15
|
+
* candidates whose `due_at` is in the past. The two phases run in
|
|
16
|
+
* the same compactor pass — see `runGc()` in
|
|
17
|
+
* `packages/server/src/gc.ts`.
|
|
18
|
+
*
|
|
19
|
+
* Lives next to {@link CurrentJson} in `coordination/` — same shape
|
|
20
|
+
* of "CAS-protected small control object." Pure module; no Node
|
|
21
|
+
* imports; Worker-bundleable.
|
|
22
|
+
*
|
|
23
|
+
* @see packages/server/src/gc.ts
|
|
24
|
+
*/
|
|
25
|
+
/**
|
|
26
|
+
* Read + parse `gc/pending.json` at `key`. Returns `null` on
|
|
27
|
+
* not-found.
|
|
28
|
+
*
|
|
29
|
+
* @throws BaerlyError{code:"InvalidResponse"} — body is not valid
|
|
30
|
+
* JSON, or `schema_version` is unknown, or the shape fails
|
|
31
|
+
* the runtime guard.
|
|
32
|
+
*/
|
|
33
|
+
const readGcPending = async (storage, key, opts) => {
|
|
34
|
+
const got = await storage.get(key, opts);
|
|
35
|
+
if (got === null) return null;
|
|
36
|
+
let parsed;
|
|
37
|
+
try {
|
|
38
|
+
parsed = decodeJsonBytes(got.body);
|
|
39
|
+
} catch (error) {
|
|
40
|
+
throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: body is not valid JSON`, error);
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
json: assertGcPending(parsed, key),
|
|
44
|
+
etag: got.etag
|
|
45
|
+
};
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Create `gc/pending.json` if-and-only-if it does not exist (S3
|
|
49
|
+
* `If-None-Match: "*"`). Use this once per collection at first GC
|
|
50
|
+
* run; subsequent updates go through {@link casUpdateGcPending}.
|
|
51
|
+
*
|
|
52
|
+
* @throws BaerlyError{code:"Conflict"} — the key already exists.
|
|
53
|
+
* @throws BaerlyError{code:"InvalidResponse"} — `initial` fails the
|
|
54
|
+
* runtime shape guard.
|
|
55
|
+
*/
|
|
56
|
+
const createGcPending = async (storage, key, initial, opts) => {
|
|
57
|
+
assertGcPending(initial, key);
|
|
58
|
+
const body = encodeJsonBytes(initial);
|
|
59
|
+
const putOpts = {
|
|
60
|
+
ifNoneMatch: "*",
|
|
61
|
+
contentType: GC_PENDING_CONTENT_TYPE,
|
|
62
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
63
|
+
};
|
|
64
|
+
let result;
|
|
65
|
+
try {
|
|
66
|
+
result = await storage.put(key, body, putOpts);
|
|
67
|
+
} catch (error) {
|
|
68
|
+
throw translateCasError(error, key);
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
json: initial,
|
|
72
|
+
etag: result.etag
|
|
73
|
+
};
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Read-modify-write `gc/pending.json` under CAS. The `mutator`
|
|
77
|
+
* receives a deep clone of the current parsed JSON and returns the
|
|
78
|
+
* new state. The new state is written with `If-Match: <currentEtag>`;
|
|
79
|
+
* on conflict, throws `BaerlyError{code:"Conflict"}`.
|
|
80
|
+
*
|
|
81
|
+
* `mutator` MUST be deterministic — it may be called multiple times
|
|
82
|
+
* if the caller wraps this in a retry loop, so it must not have side
|
|
83
|
+
* effects.
|
|
84
|
+
*
|
|
85
|
+
* @throws BaerlyError{code:"Conflict"} — another writer landed a write
|
|
86
|
+
* between this function's read and write.
|
|
87
|
+
* @throws BaerlyError{code:"InvalidResponse"} — `key` does not exist
|
|
88
|
+
* (use {@link createGcPending} instead) or body doesn't
|
|
89
|
+
* parse / fails the shape guard.
|
|
90
|
+
*/
|
|
91
|
+
const casUpdateGcPending = async (storage, key, mutator, opts) => {
|
|
92
|
+
const existing = await readGcPending(storage, key, opts);
|
|
93
|
+
if (existing === null) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key} does not exist; use createGcPending first`);
|
|
94
|
+
const next = mutator(structuredClone(existing.json));
|
|
95
|
+
assertGcPending(next, key);
|
|
96
|
+
const body = encodeJsonBytes(next);
|
|
97
|
+
const putOpts = {
|
|
98
|
+
ifMatch: existing.etag,
|
|
99
|
+
contentType: GC_PENDING_CONTENT_TYPE,
|
|
100
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
101
|
+
};
|
|
102
|
+
let result;
|
|
103
|
+
try {
|
|
104
|
+
result = await storage.put(key, body, putOpts);
|
|
105
|
+
} catch (error) {
|
|
106
|
+
throw translateCasError(error, key);
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
json: next,
|
|
110
|
+
etag: result.etag
|
|
111
|
+
};
|
|
112
|
+
};
|
|
113
|
+
const VALID_REASONS = new Set([
|
|
114
|
+
"stale-log",
|
|
115
|
+
"orphan-snapshot",
|
|
116
|
+
"orphan-content"
|
|
117
|
+
]);
|
|
118
|
+
/**
|
|
119
|
+
* Runtime guard for `parsed` to be a {@link GcPending}. Throws
|
|
120
|
+
* `BaerlyError{code:"InvalidResponse"}` rather than coercing.
|
|
121
|
+
*/
|
|
122
|
+
const assertGcPending = (parsed, key) => {
|
|
123
|
+
if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: parsed body is not an object`);
|
|
124
|
+
const r = parsed;
|
|
125
|
+
if (r["schema_version"] !== 1) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: unsupported schema_version ${String(r["schema_version"])}; expected 1`);
|
|
126
|
+
if (!Array.isArray(r["candidates"])) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates must be an array`);
|
|
127
|
+
for (let i = 0; i < r["candidates"].length; i++) {
|
|
128
|
+
const c = r["candidates"][i];
|
|
129
|
+
if (c === null || typeof c !== "object") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}] is not an object`);
|
|
130
|
+
const cr = c;
|
|
131
|
+
if (typeof cr["key"] !== "string" || cr["key"].length === 0) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].key must be a non-empty string`);
|
|
132
|
+
if (typeof cr["due_at"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].due_at must be a string`);
|
|
133
|
+
if (typeof cr["reason"] !== "string" || !VALID_REASONS.has(cr["reason"])) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].reason must be one of stale-log|orphan-snapshot|orphan-content`);
|
|
134
|
+
}
|
|
135
|
+
if (typeof r["last_swept_at"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: last_swept_at must be a string`);
|
|
136
|
+
return parsed;
|
|
137
|
+
};
|
|
138
|
+
/**
|
|
139
|
+
* Wrap a storage-level error with `gc/pending.json at <key>`
|
|
140
|
+
* location context. `Conflict` (the storage layer's CAS-lost signal)
|
|
141
|
+
* is re-thrown with an annotated message; other `BaerlyError`s pass
|
|
142
|
+
* through; non-`BaerlyError` falls through as `InvalidResponse`.
|
|
143
|
+
*/
|
|
144
|
+
const translateCasError = (e, key) => {
|
|
145
|
+
if (e instanceof BaerlyError && e.code === "Conflict") return new BaerlyError("Conflict", `gc/pending.json CAS lost at ${key}: ${e.message}`, e);
|
|
146
|
+
if (e instanceof BaerlyError) return e;
|
|
147
|
+
return new BaerlyError("InvalidResponse", `gc/pending.json write at ${key} failed: ${String(e)}`, e);
|
|
148
|
+
};
|
|
149
|
+
//#endregion
|
|
150
|
+
//#region packages/server/src/compactor.ts
|
|
151
|
+
/**
|
|
152
|
+
* Compactor: fold a prefix of the live log into a content-
|
|
153
|
+
* hashed snapshot file and CAS-advance `current.json` to swap the
|
|
154
|
+
* pointer.
|
|
155
|
+
*
|
|
156
|
+
* The hard correctness requirement is **atomic snapshot generation**:
|
|
157
|
+
* a compactor that crashes mid-write must not leave a corrupted body
|
|
158
|
+
* that readers consume as truth. The design: every snapshot's filename
|
|
159
|
+
* embeds the SHA-256 of its body. A crashed mid-PUT leaves a body that
|
|
160
|
+
* doesn't match its own filename hash, and readers reject it as
|
|
161
|
+
* "missing." The atomic moment is the CAS-swap of
|
|
162
|
+
* `current.json.snapshot`, which is a single conditional PUT.
|
|
163
|
+
*
|
|
164
|
+
* Order of operations (matches `Writer.commit`'s manifest-first-
|
|
165
|
+
* REVERSED ordering): PUT snapshot body → CAS-advance `current.json`.
|
|
166
|
+
* A crash before the swap leaves an orphan snapshot file; the swap
|
|
167
|
+
* succeeds iff our captured ETag still matches.
|
|
168
|
+
*
|
|
169
|
+
* Why level-prefixed chunked snapshots over the alternatives: a
|
|
170
|
+
* single monolithic snapshot (every compaction rewrites one well-
|
|
171
|
+
* known key) makes partial writes catastrophic and serializes every
|
|
172
|
+
* writer on one key. WAL-checkpointing in the rqlite / Raft tradition
|
|
173
|
+
* requires a consensus layer Baerly doesn't have. The Litestream-
|
|
174
|
+
* style multi-level scheme — small fixed number of levels, chunked
|
|
175
|
+
* files keyed by sequence range — keeps random GETs cheap on
|
|
176
|
+
* S3-compatible storage and avoids LIST. Current ship is single-
|
|
177
|
+
* level at L9 (one snapshot replaces the prior; no multi-level merge
|
|
178
|
+
* yet); the key format is forward-compatible with future L0..L9
|
|
179
|
+
* rolling merges without a wire change.
|
|
180
|
+
*
|
|
181
|
+
* @see ../../../docs/spec/sync-protocol.md
|
|
182
|
+
*/
|
|
183
|
+
const ctxMetrics$1 = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
|
|
184
|
+
const DEFAULT_MAX_PER_RUN = Number.MAX_SAFE_INTEGER;
|
|
185
|
+
const DEFAULT_MIN_TO_COMPACT = 100;
|
|
186
|
+
const APPLICATION_JSON$1 = "application/json";
|
|
187
|
+
/**
|
|
188
|
+
* Fold a prefix of the live log into a new snapshot and CAS-advance
|
|
189
|
+
* `current.json`. Idempotent: a second run with no new writes is a
|
|
190
|
+
* no-op (below-min-threshold).
|
|
191
|
+
*
|
|
192
|
+
* Single-attempt: on CAS conflict we return
|
|
193
|
+
* `{written: false, skippedReason: "cas-lost", newSnapshotKey}` — the
|
|
194
|
+
* caller (the cron handler invoking `runScheduledMaintenance` from
|
|
195
|
+
* `./maintenance.ts`) decides whether to schedule another run. The
|
|
196
|
+
* orphan snapshot file just written will be swept by `runGc()`.
|
|
197
|
+
*
|
|
198
|
+
* @throws BaerlyError code="InvalidResponse" — `current.json` is
|
|
199
|
+
* present but malformed, or a snapshot body fails its schema /
|
|
200
|
+
* collection cross-check.
|
|
201
|
+
* @throws BaerlyError code="Internal" — protocol-invariant violation:
|
|
202
|
+
* a log entry missing inside `[log_seq_start, foldEnd)`, or the
|
|
203
|
+
* prior-snapshot pointer resolves to no body / a body whose hash
|
|
204
|
+
* doesn't match its filename.
|
|
205
|
+
*
|
|
206
|
+
* @example
|
|
207
|
+
* ```ts
|
|
208
|
+
* import { compact } from "@gusto/baerly-storage";
|
|
209
|
+
*
|
|
210
|
+
* const res = await compact(
|
|
211
|
+
* { storage, currentJsonKey: "app/x/tenant/t/manifests/tickets/current.json" },
|
|
212
|
+
* { minEntriesToCompact: 100, maxEntriesPerRun: 40 },
|
|
213
|
+
* );
|
|
214
|
+
* if (res.written) console.log("snapshot landed at", res.newSnapshotKey);
|
|
215
|
+
* ```
|
|
216
|
+
*/
|
|
217
|
+
const compact = async (args, options = {}) => {
|
|
218
|
+
const { storage, currentJsonKey } = args;
|
|
219
|
+
const maxPerRun = options.maxEntriesPerRun ?? DEFAULT_MAX_PER_RUN;
|
|
220
|
+
const minToCompact = options.minEntriesToCompact ?? DEFAULT_MIN_TO_COMPACT;
|
|
221
|
+
const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
|
|
222
|
+
const collectionName = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
223
|
+
const read = await readCurrentJson(storage, currentJsonKey, options.signal !== void 0 ? { signal: options.signal } : void 0);
|
|
224
|
+
if (read === null) return {
|
|
225
|
+
written: false,
|
|
226
|
+
skippedReason: "current-json-missing",
|
|
227
|
+
previousSnapshotKey: null,
|
|
228
|
+
logSeqStartBefore: 0,
|
|
229
|
+
logSeqStartAfter: 0,
|
|
230
|
+
entriesFolded: 0
|
|
231
|
+
};
|
|
232
|
+
const current = read.json;
|
|
233
|
+
const baseEtag = read.etag;
|
|
234
|
+
const logSeqStartBefore = logSeqStartOf(current);
|
|
235
|
+
const nextSeq = current.next_seq;
|
|
236
|
+
if (nextSeq - logSeqStartBefore < minToCompact) return {
|
|
237
|
+
written: false,
|
|
238
|
+
skippedReason: "below-min-threshold",
|
|
239
|
+
previousSnapshotKey: current.snapshot,
|
|
240
|
+
logSeqStartBefore,
|
|
241
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
242
|
+
entriesFolded: 0
|
|
243
|
+
};
|
|
244
|
+
const foldEnd = Math.min(nextSeq, logSeqStartBefore + maxPerRun);
|
|
245
|
+
const base = current.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, current.snapshot, collectionName, options.signal);
|
|
246
|
+
const entries = await walkLogRange(storage, collectionPrefix, logSeqStartBefore, foldEnd, { signal: options.signal });
|
|
247
|
+
for (const entry of entries) {
|
|
248
|
+
if (entry.collection !== collectionName) continue;
|
|
249
|
+
if (entry.doc_id === void 0) continue;
|
|
250
|
+
switch (entry.op) {
|
|
251
|
+
case "I":
|
|
252
|
+
case "U":
|
|
253
|
+
if (entry.new !== void 0) base.set(entry.doc_id, entry.new);
|
|
254
|
+
break;
|
|
255
|
+
case "D":
|
|
256
|
+
base.delete(entry.doc_id);
|
|
257
|
+
break;
|
|
258
|
+
case "T":
|
|
259
|
+
case "M": break;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
const bodyBytes = encodeSnapshotBody({
|
|
263
|
+
schema_version: 1,
|
|
264
|
+
min_seq: 0,
|
|
265
|
+
max_seq: foldEnd,
|
|
266
|
+
collection: collectionName,
|
|
267
|
+
docs: Array.from(base.entries()).toSorted(([a], [b]) => {
|
|
268
|
+
if (a < b) return -1;
|
|
269
|
+
if (a > b) return 1;
|
|
270
|
+
return 0;
|
|
271
|
+
}).map(([id, body]) => ({
|
|
272
|
+
_id: id,
|
|
273
|
+
body
|
|
274
|
+
}))
|
|
275
|
+
});
|
|
276
|
+
const newKey = snapshotKey(collectionPrefix, 0, foldEnd, await snapshotHash(bodyBytes));
|
|
277
|
+
const putOpts = {
|
|
278
|
+
contentType: APPLICATION_JSON$1,
|
|
279
|
+
...options.signal !== void 0 && { signal: options.signal }
|
|
280
|
+
};
|
|
281
|
+
await storage.put(newKey, bodyBytes, putOpts);
|
|
282
|
+
const nextBody = encodeJsonBytes({
|
|
283
|
+
...current,
|
|
284
|
+
snapshot: newKey,
|
|
285
|
+
log_seq_start: foldEnd
|
|
286
|
+
});
|
|
287
|
+
const casOpts = {
|
|
288
|
+
ifMatch: baseEtag,
|
|
289
|
+
contentType: APPLICATION_JSON$1,
|
|
290
|
+
...options.signal !== void 0 && { signal: options.signal }
|
|
291
|
+
};
|
|
292
|
+
try {
|
|
293
|
+
await storage.put(currentJsonKey, nextBody, casOpts);
|
|
294
|
+
} catch (error) {
|
|
295
|
+
if (isCasConflict(error)) return {
|
|
296
|
+
written: false,
|
|
297
|
+
skippedReason: "cas-lost",
|
|
298
|
+
previousSnapshotKey: current.snapshot,
|
|
299
|
+
newSnapshotKey: newKey,
|
|
300
|
+
logSeqStartBefore,
|
|
301
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
302
|
+
entriesFolded: foldEnd - logSeqStartBefore
|
|
303
|
+
};
|
|
304
|
+
throw error;
|
|
305
|
+
}
|
|
306
|
+
const entriesFolded = foldEnd - logSeqStartBefore;
|
|
307
|
+
const metrics = ctxMetrics$1();
|
|
308
|
+
metrics.histogram("db.compact.entries_folded", entriesFolded, { collection: collectionName });
|
|
309
|
+
metrics.gauge("db.manifest.lag_window_depth", current.next_seq - foldEnd, { collection: collectionName });
|
|
310
|
+
return {
|
|
311
|
+
written: true,
|
|
312
|
+
previousSnapshotKey: current.snapshot,
|
|
313
|
+
newSnapshotKey: newKey,
|
|
314
|
+
logSeqStartBefore,
|
|
315
|
+
logSeqStartAfter: foldEnd,
|
|
316
|
+
entriesFolded
|
|
317
|
+
};
|
|
318
|
+
};
|
|
319
|
+
/**
|
|
320
|
+
* `true` when an `If-Match` CAS guard lost. Every in-tree
|
|
321
|
+
* {@link Storage} impl surfaces a lost CAS as
|
|
322
|
+
* `BaerlyError{code:"Conflict"}`.
|
|
323
|
+
*/
|
|
324
|
+
const isCasConflict = (err) => err instanceof BaerlyError && err.code === "Conflict";
|
|
325
|
+
//#endregion
|
|
326
|
+
//#region packages/server/src/gc.ts
|
|
327
|
+
/**
|
|
328
|
+
* GC mark + sweep.
|
|
329
|
+
*
|
|
330
|
+
* `runGc` is a single-pass garbage collector that:
|
|
331
|
+
* 1. Reads `current.json` (and bootstraps `gc/pending.json` on first
|
|
332
|
+
* run).
|
|
333
|
+
* 2. Marks new orphan candidates by LISTing the three artifact
|
|
334
|
+
* prefixes — log, snapshot, content — and classifying each key.
|
|
335
|
+
* 3. Sweeps any already-pending candidate whose `due_at` has passed.
|
|
336
|
+
* 4. CAS-writes the updated `gc/pending.json`.
|
|
337
|
+
*
|
|
338
|
+
* Two-phase by design: every candidate sits in `gc/pending.json` for a
|
|
339
|
+
* grace period (default 7 days, see {@link GC_GRACE_PERIOD_MILLIS})
|
|
340
|
+
* before it is deleted. The grace bounds the worst plausible
|
|
341
|
+
* writer-retry window — a paused-process writer that resumes hours
|
|
342
|
+
* later still finds its idempotency anchor on the bucket.
|
|
343
|
+
*
|
|
344
|
+
* Idempotent: same input bucket state ⇒ same `pending.json` output.
|
|
345
|
+
* Unbounded by default — the run marks and sweeps the entire eligible
|
|
346
|
+
* set in one pass. Callers on the Cloudflare 50-subrequest free-tier
|
|
347
|
+
* budget opt INTO caps via the `CLOUDFLARE_FREE_TIER` profile's
|
|
348
|
+
* `gc.maxMarksPerRun` / `maxSweepsPerRun` knobs (`InternalRunGcOptions`,
|
|
349
|
+
* not on the public `RunGcOptions`).
|
|
350
|
+
*
|
|
351
|
+
* Three categories of orphan:
|
|
352
|
+
* - `stale-log`: `<collectionPrefix>/log/<seq>.json` with
|
|
353
|
+
* `seq < log_seq_start`. After `compact()` folds these into a
|
|
354
|
+
* snapshot, they're unreferenced.
|
|
355
|
+
* - `orphan-snapshot`: a `<collectionPrefix>/snapshot/L<n>/...` key not
|
|
356
|
+
* equal to `current.snapshot`. Each compactor run replaces the
|
|
357
|
+
* pointer; the prior file becomes unreferenced.
|
|
358
|
+
* - `orphan-content`: `<collectionPrefix>/content/<sha>.json` whose
|
|
359
|
+
* 32-hex truncated-SHA-256 hash is not in the live content-hash
|
|
360
|
+
* set (computed by hashing every live `entry.new` post-image —
|
|
361
|
+
* the same hash the writer's step 4 produces). Surfaces writer
|
|
362
|
+
* crashes between the content PUT and the log-entry PUT.
|
|
363
|
+
*
|
|
364
|
+
* CAS-lost on `gc/pending.json` is non-fatal: the DELETEs already
|
|
365
|
+
* issued are durable, so we return a successful result and the next
|
|
366
|
+
* pass will pick up any work this pass lost.
|
|
367
|
+
*/
|
|
368
|
+
const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
|
|
369
|
+
const DEFAULT_MAX_MARKS = Number.MAX_SAFE_INTEGER;
|
|
370
|
+
const DEFAULT_MAX_SWEEPS = Number.MAX_SAFE_INTEGER;
|
|
371
|
+
/**
|
|
372
|
+
* Single GC pass — mark new orphans, sweep due-elapsed candidates,
|
|
373
|
+
* persist via CAS-update on `gc/pending.json`.
|
|
374
|
+
*
|
|
375
|
+
* Returns immediately if `current.json` is missing — there's nothing
|
|
376
|
+
* to do until a writer has bootstrapped the collection.
|
|
377
|
+
*
|
|
378
|
+
* @example
|
|
379
|
+
* ```ts
|
|
380
|
+
* import { runGc } from "@gusto/baerly-storage";
|
|
381
|
+
*
|
|
382
|
+
* const r = await runGc({ storage, currentJsonKey });
|
|
383
|
+
* console.log(`marked ${r.marked.stale_log} stale logs, swept ${r.swept}`);
|
|
384
|
+
* ```
|
|
385
|
+
*/
|
|
386
|
+
const runGc = async (args, options = {}) => {
|
|
387
|
+
const { storage, currentJsonKey } = args;
|
|
388
|
+
const internal = options;
|
|
389
|
+
const grace = internal.graceMillis ?? 6048e5;
|
|
390
|
+
const maxMarks = internal.maxMarksPerRun ?? DEFAULT_MAX_MARKS;
|
|
391
|
+
const maxSweeps = internal.maxSweepsPerRun ?? DEFAULT_MAX_SWEEPS;
|
|
392
|
+
const now = internal.now ?? (() => /* @__PURE__ */ new Date());
|
|
393
|
+
const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
|
|
394
|
+
const collectionName = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
395
|
+
const gcPendingKey = `${collectionPrefix}/gc/pending.json`;
|
|
396
|
+
const signal = options.signal;
|
|
397
|
+
const signalOpts = signal !== void 0 ? { signal } : void 0;
|
|
398
|
+
const cur = await readCurrentJson(storage, currentJsonKey, signalOpts);
|
|
399
|
+
if (cur === null) return {
|
|
400
|
+
marked: {
|
|
401
|
+
stale_log: 0,
|
|
402
|
+
orphan_snapshot: 0,
|
|
403
|
+
orphan_content: 0
|
|
404
|
+
},
|
|
405
|
+
swept: 0,
|
|
406
|
+
pendingDepth: 0
|
|
407
|
+
};
|
|
408
|
+
const current = cur.json;
|
|
409
|
+
const logSeqStart = logSeqStartOf(current);
|
|
410
|
+
let pending = await readGcPending(storage, gcPendingKey, signalOpts);
|
|
411
|
+
if (pending === null) {
|
|
412
|
+
const initial = {
|
|
413
|
+
schema_version: 1,
|
|
414
|
+
candidates: [],
|
|
415
|
+
last_swept_at: ""
|
|
416
|
+
};
|
|
417
|
+
try {
|
|
418
|
+
pending = await createGcPending(storage, gcPendingKey, initial, signalOpts);
|
|
419
|
+
} catch (error) {
|
|
420
|
+
if (error instanceof BaerlyError && error.code === "Conflict") {
|
|
421
|
+
pending = await readGcPending(storage, gcPendingKey, signalOpts);
|
|
422
|
+
if (pending === null) throw error;
|
|
423
|
+
} else throw error;
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
const known = new Set(pending.json.candidates.map((c) => c.key));
|
|
427
|
+
const newCandidates = [];
|
|
428
|
+
let markedStaleLog = 0;
|
|
429
|
+
if (logSeqStart > 0) for await (const entry of listBounded(storage, `${collectionPrefix}/log/`, maxMarks, signal)) {
|
|
430
|
+
if (markedStaleLog >= maxMarks) break;
|
|
431
|
+
const seq = parseSeqFromLogKey(entry.key);
|
|
432
|
+
if (seq === null || seq >= logSeqStart) continue;
|
|
433
|
+
if (known.has(entry.key)) continue;
|
|
434
|
+
newCandidates.push({
|
|
435
|
+
key: entry.key,
|
|
436
|
+
due_at: computeDueAt(entry, now, grace),
|
|
437
|
+
reason: "stale-log"
|
|
438
|
+
});
|
|
439
|
+
markedStaleLog++;
|
|
440
|
+
}
|
|
441
|
+
let markedOrphanSnapshot = 0;
|
|
442
|
+
for await (const entry of listBounded(storage, `${collectionPrefix}/snapshot/`, maxMarks, signal)) {
|
|
443
|
+
if (markedOrphanSnapshot >= maxMarks) break;
|
|
444
|
+
if (entry.key === current.snapshot) continue;
|
|
445
|
+
if (known.has(entry.key)) continue;
|
|
446
|
+
newCandidates.push({
|
|
447
|
+
key: entry.key,
|
|
448
|
+
due_at: computeDueAt(entry, now, grace),
|
|
449
|
+
reason: "orphan-snapshot"
|
|
450
|
+
});
|
|
451
|
+
markedOrphanSnapshot++;
|
|
452
|
+
}
|
|
453
|
+
const liveHashes = await collectLiveContentHashes(storage, collectionPrefix, collectionName, current, logSeqStart, signal);
|
|
454
|
+
let markedOrphanContent = 0;
|
|
455
|
+
for await (const entry of listBounded(storage, `${collectionPrefix}/content/`, maxMarks, signal)) {
|
|
456
|
+
if (markedOrphanContent >= maxMarks) break;
|
|
457
|
+
const hash = parseHashFromContentKey(entry.key);
|
|
458
|
+
if (hash === null || liveHashes.has(hash)) continue;
|
|
459
|
+
if (known.has(entry.key)) continue;
|
|
460
|
+
newCandidates.push({
|
|
461
|
+
key: entry.key,
|
|
462
|
+
due_at: computeDueAt(entry, now, grace),
|
|
463
|
+
reason: "orphan-content"
|
|
464
|
+
});
|
|
465
|
+
markedOrphanContent++;
|
|
466
|
+
}
|
|
467
|
+
const nowMs = now().getTime();
|
|
468
|
+
const sweepCandidates = [...pending.json.candidates, ...newCandidates];
|
|
469
|
+
const toSweep = [];
|
|
470
|
+
const remaining = [];
|
|
471
|
+
for (const cand of sweepCandidates) if (toSweep.length < maxSweeps && Date.parse(cand.due_at) <= nowMs) toSweep.push(cand);
|
|
472
|
+
else remaining.push(cand);
|
|
473
|
+
await Promise.all(toSweep.map((c) => storage.delete(c.key, signalOpts)));
|
|
474
|
+
const merged = remaining.slice(0, GC_MAX_PENDING_CANDIDATES);
|
|
475
|
+
const lastSweptAt = toSweep.length > 0 ? now().toISOString() : pending.json.last_swept_at;
|
|
476
|
+
const markedSummary = {
|
|
477
|
+
stale_log: markedStaleLog,
|
|
478
|
+
orphan_snapshot: markedOrphanSnapshot,
|
|
479
|
+
orphan_content: markedOrphanContent
|
|
480
|
+
};
|
|
481
|
+
let pendingDepth;
|
|
482
|
+
try {
|
|
483
|
+
pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, () => ({
|
|
484
|
+
schema_version: 1,
|
|
485
|
+
candidates: merged,
|
|
486
|
+
last_swept_at: lastSweptAt
|
|
487
|
+
}), signalOpts)).json.candidates.length;
|
|
488
|
+
} catch (error) {
|
|
489
|
+
if (error instanceof BaerlyError && error.code === "Conflict") pendingDepth = remaining.length;
|
|
490
|
+
else throw error;
|
|
491
|
+
}
|
|
492
|
+
const labels = { collection: collectionName };
|
|
493
|
+
const metrics = ctxMetrics();
|
|
494
|
+
metrics.gauge("db.orphan.candidate_count", pendingDepth, labels);
|
|
495
|
+
metrics.gauge("db.gc.entries_swept_per_second", toSweep.length, labels);
|
|
496
|
+
if (toSweep.length > 0) {
|
|
497
|
+
const byReason = /* @__PURE__ */ new Map();
|
|
498
|
+
for (const c of toSweep) byReason.set(c.reason, (byReason.get(c.reason) ?? 0) + 1);
|
|
499
|
+
for (const [reason, count] of byReason) metrics.counter("db.gc.swept_total", count, {
|
|
500
|
+
collection: collectionName,
|
|
501
|
+
reason
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
return {
|
|
505
|
+
marked: markedSummary,
|
|
506
|
+
swept: toSweep.length,
|
|
507
|
+
pendingDepth
|
|
508
|
+
};
|
|
509
|
+
};
|
|
510
|
+
/**
|
|
511
|
+
* `Storage.list` with a hard ceiling — bounds the I/O cost of a
|
|
512
|
+
* single pass when a runaway prefix has accumulated many keys.
|
|
513
|
+
*/
|
|
514
|
+
const listBounded = async function* (storage, prefix, cap, signal) {
|
|
515
|
+
let yielded = 0;
|
|
516
|
+
const opts = signal !== void 0 ? { signal } : void 0;
|
|
517
|
+
for await (const entry of storage.list(prefix, opts)) {
|
|
518
|
+
if (yielded >= cap) return;
|
|
519
|
+
yield entry;
|
|
520
|
+
yielded++;
|
|
521
|
+
}
|
|
522
|
+
};
|
|
523
|
+
/**
|
|
524
|
+
* Parse `<...>/log/<seq>.json` and return `seq`. Returns `null` on
|
|
525
|
+
* any shape that doesn't look like a log entry key — defensively
|
|
526
|
+
* tolerates an unrelated key under the log prefix.
|
|
527
|
+
*/
|
|
528
|
+
const parseSeqFromLogKey = (key) => {
|
|
529
|
+
const match = /\/log\/(\d+)\.json$/.exec(key);
|
|
530
|
+
if (match === null) return null;
|
|
531
|
+
const n = Number.parseInt(match[1], 10);
|
|
532
|
+
return Number.isFinite(n) && n >= 0 ? n : null;
|
|
533
|
+
};
|
|
534
|
+
/**
|
|
535
|
+
* Parse `<...>/content/<sha-32>.json` and return the 32-hex hash.
|
|
536
|
+
* Returns `null` on any shape that doesn't match the writer's
|
|
537
|
+
* `versionFromContent`-produced key format.
|
|
538
|
+
*/
|
|
539
|
+
const parseHashFromContentKey = (key) => {
|
|
540
|
+
const match = /\/content\/([0-9a-f]{32})\.json$/.exec(key);
|
|
541
|
+
return match === null ? null : match[1];
|
|
542
|
+
};
|
|
543
|
+
/**
|
|
544
|
+
* Anchor `due_at` to `lastModified` when the storage adapter
|
|
545
|
+
* surfaces it; fall back to the injected `now` so a Storage impl
|
|
546
|
+
* without a server clock still GCs after `grace` from the mark
|
|
547
|
+
* moment.
|
|
548
|
+
*/
|
|
549
|
+
const computeDueAt = (entry, now, graceMs) => {
|
|
550
|
+
const base = entry.lastModified ?? now();
|
|
551
|
+
return new Date(base.getTime() + graceMs).toISOString();
|
|
552
|
+
};
|
|
553
|
+
/**
|
|
554
|
+
* Build the live content-hash set. The set covers every live
|
|
555
|
+
* post-image: every `entry.new` in `[logSeqStart, next_seq)` plus
|
|
556
|
+
* every row body in the current snapshot.
|
|
557
|
+
*
|
|
558
|
+
* A snapshot read that throws (corrupt body, hash mismatch) is
|
|
559
|
+
* tolerated — the worst-case effect is missing some live hashes,
|
|
560
|
+
* which could mark a live content blob as orphan. The grace period
|
|
561
|
+
* absorbs the false-positive: a retry that re-PUTs the same hash
|
|
562
|
+
* recreates the key before the sweep deletes it.
|
|
563
|
+
*/
|
|
564
|
+
const collectLiveContentHashes = async (storage, collectionPrefix, collectionName, current, logSeqStart, signal) => {
|
|
565
|
+
const hashes = /* @__PURE__ */ new Set();
|
|
566
|
+
const getOpts = signal !== void 0 ? { signal } : void 0;
|
|
567
|
+
const logReads = [];
|
|
568
|
+
for (let s = logSeqStart; s < current.next_seq; s++) logReads.push((async () => {
|
|
569
|
+
const got = await storage.get(`${collectionPrefix}/log/${s}.json`, getOpts);
|
|
570
|
+
if (got === null) return;
|
|
571
|
+
let entry;
|
|
572
|
+
try {
|
|
573
|
+
entry = decodeJsonBytes(got.body);
|
|
574
|
+
} catch {
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
if (entry.new === void 0) return;
|
|
578
|
+
const bodyBytes = encodeJsonBytes(entry.new);
|
|
579
|
+
hashes.add(await versionFromContent(bodyBytes));
|
|
580
|
+
})());
|
|
581
|
+
await Promise.all(logReads);
|
|
582
|
+
if (current.snapshot !== null) try {
|
|
583
|
+
const map = await loadSnapshotAsMap(storage, current.snapshot, collectionName, signal);
|
|
584
|
+
const rowReads = [];
|
|
585
|
+
for (const body of map.values()) rowReads.push((async () => {
|
|
586
|
+
const bytes = encodeJsonBytes(body);
|
|
587
|
+
hashes.add(await versionFromContent(bytes));
|
|
588
|
+
})());
|
|
589
|
+
await Promise.all(rowReads);
|
|
590
|
+
} catch {}
|
|
591
|
+
return hashes;
|
|
592
|
+
};
|
|
593
|
+
//#endregion
|
|
594
|
+
//#region packages/server/src/rebuild-index.ts
|
|
595
|
+
/**
|
|
596
|
+
* Idempotent reconciliation for one secondary index.
|
|
597
|
+
*
|
|
598
|
+
* {@link rebuildIndex} folds the live `(snapshot, log_tail)` of a
|
|
599
|
+
* collection into the doc set the writer's index emission would
|
|
600
|
+
* have produced, then PUTs every missing index key and DELETEs every
|
|
601
|
+
* orphaned one. Same-key zero-byte PUTs are no-ops (the entries are
|
|
602
|
+
* content-addressed by composition); same-key DELETEs are
|
|
603
|
+
* contractually idempotent.
|
|
604
|
+
*
|
|
605
|
+
* Invariants:
|
|
606
|
+
*
|
|
607
|
+
* - **Re-running on a healthy index is a no-op.** `added = 0`,
|
|
608
|
+
* `removed = 0`.
|
|
609
|
+
* - **Re-running after a crashed mid-commit** reconciles toward
|
|
610
|
+
* the live `current.json` view — orphans deleted, missing keys
|
|
611
|
+
* PUT.
|
|
612
|
+
* - **Re-running while writes are in flight** is safe up to a
|
|
613
|
+
* race window: a write that lands during the rebuild may
|
|
614
|
+
* produce a transient "orphan" the rebuild sweeps, which the
|
|
615
|
+
* next live write then re-PUTs. The race converges on the next
|
|
616
|
+
* commit.
|
|
617
|
+
*
|
|
618
|
+
* The function reads the same fold the writer / reader use
|
|
619
|
+
* (snapshot via {@link loadSnapshotAsMap}; log tail by direct
|
|
620
|
+
* `get(<logPrefix>/log/<seq>.json)`) so it MUST agree with the
|
|
621
|
+
* writer on the live doc set by construction.
|
|
622
|
+
*
|
|
623
|
+
* @see ./indexes.ts — `IndexDefinition`, `allIndexKeysFor`,
|
|
624
|
+
* `indexKeyPrefix`.
|
|
625
|
+
* @see ./writer.ts — the fence-time emission path this
|
|
626
|
+
* command reconciles.
|
|
627
|
+
*/
|
|
628
|
+
const EMPTY_BODY = new Uint8Array(0);
|
|
629
|
+
const APPLICATION_JSON = "application/json";
|
|
630
|
+
/**
|
|
631
|
+
* Idempotently reconcile one secondary index for one collection.
|
|
632
|
+
*
|
|
633
|
+
* @param storage Storage handle (any in-tree `Storage`).
|
|
634
|
+
* @param currentJsonKey The collection's `current.json` key.
|
|
635
|
+
* @param def The index to reconcile. Single-field today;
|
|
636
|
+
* composite definitions are accepted by the
|
|
637
|
+
* key encoder but the read path only consults
|
|
638
|
+
* single-field keys.
|
|
639
|
+
* @param opts Optional knobs — see {@link RebuildIndexOptions}.
|
|
640
|
+
* Strictly additive: existing positional-arg
|
|
641
|
+
* callers compile and behave unchanged when
|
|
642
|
+
* omitted.
|
|
643
|
+
*
|
|
644
|
+
* @throws BaerlyError code="InvalidResponse" — `current.json` missing.
|
|
645
|
+
* @throws BaerlyError code="Internal" — protocol-invariant
|
|
646
|
+
* violation (snapshot hash mismatch, missing log entry inside
|
|
647
|
+
* `[log_seq_start, next_seq)`).
|
|
648
|
+
*/
|
|
649
|
+
const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
|
|
650
|
+
const read = await readCurrentJson(storage, currentJsonKey);
|
|
651
|
+
if (read === null) throw new BaerlyError("InvalidResponse", `rebuildIndex: current.json missing at ${JSON.stringify(currentJsonKey)}`);
|
|
652
|
+
const lastSlash = currentJsonKey.lastIndexOf("/");
|
|
653
|
+
if (lastSlash < 0) throw new BaerlyError("InvalidResponse", `rebuildIndex: currentJsonKey ${JSON.stringify(currentJsonKey)} must contain "/"`);
|
|
654
|
+
const collectionPrefix = currentJsonKey.slice(0, lastSlash);
|
|
655
|
+
const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
656
|
+
const logSeqStart = logSeqStartOf(read.json);
|
|
657
|
+
const nextSeq = read.json.next_seq;
|
|
658
|
+
const live = read.json.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, read.json.snapshot, collection);
|
|
659
|
+
foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart, nextSeq), { collection });
|
|
660
|
+
const expected = /* @__PURE__ */ new Set();
|
|
661
|
+
for (const [docId, body] of live) for (const k of allIndexKeysFor(collectionPrefix, [def], body, docId)) expected.add(k);
|
|
662
|
+
const actual = /* @__PURE__ */ new Set();
|
|
663
|
+
const listOpts = {};
|
|
664
|
+
if (opts.signal !== void 0) listOpts.signal = opts.signal;
|
|
665
|
+
for await (const entry of storage.list(indexKeyPrefix(collectionPrefix, def.name), listOpts)) actual.add(entry.key);
|
|
666
|
+
const dryRun = opts.dryRun === true;
|
|
667
|
+
let added = 0;
|
|
668
|
+
let removed = 0;
|
|
669
|
+
let kept = 0;
|
|
670
|
+
for (const k of expected) {
|
|
671
|
+
if (actual.has(k)) {
|
|
672
|
+
kept += 1;
|
|
673
|
+
continue;
|
|
674
|
+
}
|
|
675
|
+
if (dryRun) {
|
|
676
|
+
added += 1;
|
|
677
|
+
continue;
|
|
678
|
+
}
|
|
679
|
+
try {
|
|
680
|
+
await storage.put(k, EMPTY_BODY, {
|
|
681
|
+
ifNoneMatch: "*",
|
|
682
|
+
contentType: APPLICATION_JSON
|
|
683
|
+
});
|
|
684
|
+
added += 1;
|
|
685
|
+
} catch (error) {
|
|
686
|
+
if (error instanceof BaerlyError && error.code === "Conflict") kept += 1;
|
|
687
|
+
else throw error;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
for (const k of actual) {
|
|
691
|
+
if (expected.has(k)) continue;
|
|
692
|
+
if (dryRun) {
|
|
693
|
+
removed += 1;
|
|
694
|
+
continue;
|
|
695
|
+
}
|
|
696
|
+
await storage.delete(k);
|
|
697
|
+
removed += 1;
|
|
698
|
+
}
|
|
699
|
+
return {
|
|
700
|
+
added,
|
|
701
|
+
removed,
|
|
702
|
+
kept
|
|
703
|
+
};
|
|
704
|
+
};
|
|
705
|
+
//#endregion
|
|
706
|
+
//#region packages/server/src/maintenance.ts
|
|
707
|
+
/**
|
|
708
|
+
* Single maintenance pass for one collection. Runs `compact()` then
|
|
709
|
+
* `runGc()` (in that order; the compactor's advance of
|
|
710
|
+
* `log_seq_start` produces the stale-log candidates the GC then
|
|
711
|
+
* marks). Callers wanting a single phase per tick invoke
|
|
712
|
+
* `compact()` or `runGc()` directly instead.
|
|
713
|
+
*
|
|
714
|
+
* Errors propagate — the caller's cron handler is responsible for
|
|
715
|
+
* logging them. The Cloudflare runtime ships uncaught Worker errors
|
|
716
|
+
* to the dashboard; Node operators wrap their `node-cron` callbacks
|
|
717
|
+
* themselves.
|
|
718
|
+
*
|
|
719
|
+
* @example
|
|
720
|
+
* ```ts
|
|
721
|
+
* import { runScheduledMaintenance, CLOUDFLARE_FREE_TIER } from "@gusto/baerly-storage/maintenance";
|
|
722
|
+
*
|
|
723
|
+
* // Node (unbounded — defaults fold the entire live tail):
|
|
724
|
+
* const res = await runScheduledMaintenance(
|
|
725
|
+
* { storage, currentJsonKey: "app/x/tenant/t/manifests/tickets/current.json" },
|
|
726
|
+
* );
|
|
727
|
+
* console.log("compacted:", res.compact.entriesFolded, "swept:", res.gc.swept);
|
|
728
|
+
*
|
|
729
|
+
* // Cloudflare free tier (50-subrequest cap, single phase per tick):
|
|
730
|
+
* await runScheduledMaintenance({ storage, currentJsonKey }, CLOUDFLARE_FREE_TIER);
|
|
731
|
+
* ```
|
|
732
|
+
*/
|
|
733
|
+
const runScheduledMaintenance = async (args, options = {}) => {
|
|
734
|
+
return {
|
|
735
|
+
compact: await compact(args, {
|
|
736
|
+
...options.compact,
|
|
737
|
+
...options.signal !== void 0 && { signal: options.signal }
|
|
738
|
+
}),
|
|
739
|
+
gc: await runGc(args, {
|
|
740
|
+
...options.gc,
|
|
741
|
+
...options.signal !== void 0 && { signal: options.signal }
|
|
742
|
+
})
|
|
743
|
+
};
|
|
744
|
+
};
|
|
745
|
+
const CLOUDFLARE_FREE_TIER = {
|
|
746
|
+
compact: {
|
|
747
|
+
maxEntriesPerRun: 20,
|
|
748
|
+
minEntriesToCompact: 50
|
|
749
|
+
},
|
|
750
|
+
gc: {
|
|
751
|
+
maxMarksPerRun: 20,
|
|
752
|
+
maxSweepsPerRun: 10
|
|
753
|
+
}
|
|
754
|
+
};
|
|
755
|
+
//#endregion
|
|
756
|
+
export { compact as a, runGc as i, runScheduledMaintenance as n, rebuildIndex as r, CLOUDFLARE_FREE_TIER as t };
|
|
757
|
+
|
|
758
|
+
//# sourceMappingURL=maintenance-Bn6ul9aV.js.map
|