@gusto/baerly-storage 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +29 -0
- package/README.md +108 -44
- package/dist/API.md +192 -121
- package/dist/CHANGELOG.md +144 -0
- package/dist/THIRD-PARTY-LICENSES.txt +312 -0
- package/dist/{app-config-C0Bbw_Z6.d.ts → app-config-BIzyAKzy.d.ts} +14 -2
- package/dist/app-config.d.ts +1 -1
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +288 -288
- package/dist/auth.js.map +1 -1
- package/dist/baerly.js +10 -10
- package/dist/baerly.js.map +1 -1
- package/dist/bytes-BXdamO3L.js.map +1 -1
- package/dist/{client-DYZfoZb9.d.ts → client-BJXF3XJA.d.ts} +10 -10
- package/dist/client-react.d.ts +76 -85
- package/dist/client-react.js +95 -53
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +3 -3
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-CINJPKCW.js → cloudflare-Cn2ibODt.js} +178 -2
- package/dist/cloudflare-Cn2ibODt.js.map +1 -0
- package/dist/cloudflare.d.ts +4 -4
- package/dist/cloudflare.js +10 -11
- package/dist/cloudflare.js.map +1 -1
- package/dist/{subcommand-DZcHcsYp.js → config-CGyBopy3.js} +643 -439
- package/dist/config-CGyBopy3.js.map +1 -0
- package/dist/context-DRZ3tZ5l.js.map +1 -1
- package/dist/{cost-CQt1YRkA.js → cost-BZ_MDBmn.js} +6 -4
- package/dist/cost-BZ_MDBmn.js.map +1 -0
- package/dist/{current-json-C-EyJqs0.js → current-json-DsN7FNY1.js} +47 -13
- package/dist/current-json-DsN7FNY1.js.map +1 -0
- package/dist/{db-BM3y2JQw.d.ts → db-CCT6tDfY.d.ts} +26 -108
- package/dist/{deploy-CrAbfgCq.js → deploy-BKKyGlSJ.js} +11 -4
- package/dist/deploy-BKKyGlSJ.js.map +1 -0
- package/dist/dev-vite.d.ts +5 -5
- package/dist/dev-vite.js +5 -5
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/{doctor-Db-hzPmE.js → doctor-BVU4GNXN.js} +13 -132
- package/dist/doctor-BVU4GNXN.js.map +1 -0
- package/dist/{dump-BYA55NLP.js → dump-DPEAKub6.js} +5 -3
- package/dist/dump-DPEAKub6.js.map +1 -0
- package/dist/errors-DOdL95kK.js.map +1 -1
- package/dist/{export-Bhbz90-z.js → export-BcuCLFFe.js} +6 -4
- package/dist/{export-Bhbz90-z.js.map → export-BcuCLFFe.js.map} +1 -1
- package/dist/{fsck-CPPZWt9p.js → fsck-RXgLOaJG.js} +23 -11
- package/dist/fsck-RXgLOaJG.js.map +1 -0
- package/dist/{http-Cw8p6y5b.js → http-Bd4byR4n.js} +45 -61
- package/dist/http-Bd4byR4n.js.map +1 -0
- package/dist/http.d.ts +15 -15
- package/dist/http.js +1 -1
- package/dist/{index-cUY-36AN.d.ts → index-BDceQQi-.d.ts} +20 -4
- package/dist/{index-C3JAH1Yo.d.ts → index-Cmsxx-ZN.d.ts} +9 -9
- package/dist/{index-DSvwEUeF.d.ts → index-DW7GT6M2.d.ts} +50 -42
- package/dist/{index-DwDTWPSU.d.ts → index-DnBGSeuU.d.ts} +10 -7
- package/dist/index.d.ts +6 -6
- package/dist/index.js +4 -4
- package/dist/{indexes-DAu2AkTW.d.ts → indexes-Cx7uMXm0.d.ts} +2 -2
- package/dist/{inspect-Bdbzvi-_.js → inspect-DhI49PSi.js} +16 -8
- package/dist/inspect-DhI49PSi.js.map +1 -0
- package/dist/{snapshot-SS8USDUF.js → log-tail-DXeDkWEp.js} +154 -83
- package/dist/log-tail-DXeDkWEp.js.map +1 -0
- package/dist/{maintenance-BRrThz8V.d.ts → maintenance-BkhUr72A.d.ts} +91 -26
- package/dist/{maintenance-ClzBC1Vv.js → maintenance-CWBQpZ2y.js} +368 -161
- package/dist/maintenance-CWBQpZ2y.js.map +1 -0
- package/dist/{maintenance-BvOv6BMM.js → maintenance-C_KSkVjd.js} +204 -82
- package/dist/maintenance-C_KSkVjd.js.map +1 -0
- package/dist/maintenance.d.ts +3 -2
- package/dist/maintenance.js +3 -2
- package/dist/node.d.ts +13 -8
- package/dist/node.js +1 -1
- package/dist/normalize-XWY25_sG.js.map +1 -1
- package/dist/{observability-CpOiEN0k.js → observability-BYd-RzEx.js} +63 -55
- package/dist/observability-BYd-RzEx.js.map +1 -0
- package/dist/observability.d.ts +3 -3
- package/dist/observability.js +1 -1
- package/dist/{plan-DOTl21vq.js → plan-DIu_B6fy.js} +7 -4
- package/dist/plan-DIu_B6fy.js.map +1 -0
- package/dist/{query-BmiQy328.js → query-sQNO5vU_.js} +1071 -1099
- package/dist/query-sQNO5vU_.js.map +1 -0
- package/dist/{rebuild-index-DC_nR6wP.js → rebuild-index-DP4CqSSs.js} +5 -4
- package/dist/rebuild-index-DP4CqSSs.js.map +1 -0
- package/dist/{restore-C6IcmLeA.js → restore-DpQYef6r.js} +362 -328
- package/dist/restore-DpQYef6r.js.map +1 -0
- package/dist/shared-secret-BJdSwpgw.js.map +1 -1
- package/dist/{src-QQ7jDeL2.js → src-CWpkSbvJ.js} +40 -20
- package/dist/src-CWpkSbvJ.js.map +1 -0
- package/dist/{src-DU1sCMlr.js → src-D24wDpzR.js} +269 -258
- package/dist/src-D24wDpzR.js.map +1 -0
- package/dist/{src-zhEAmmF7.js → src-Db4-qVtM.js} +36 -96
- package/dist/src-Db4-qVtM.js.map +1 -0
- package/dist/subcommand-CP9NY2J4.js +159 -0
- package/dist/subcommand-CP9NY2J4.js.map +1 -0
- package/dist/{usage-CLjirdD-.js → usage-CTfDq9HR.js} +3 -3
- package/dist/{usage-CLjirdD-.js.map → usage-CTfDq9HR.js.map} +1 -1
- package/dist/{validate-DCq_0DIn.js → validate-DtljsCh1.js} +51 -51
- package/dist/{validate-DCq_0DIn.js.map → validate-DtljsCh1.js.map} +1 -1
- package/package.json +42 -31
- package/dist/cloudflare-CINJPKCW.js.map +0 -1
- package/dist/config-DVfDGyw1.js +0 -278
- package/dist/config-DVfDGyw1.js.map +0 -1
- package/dist/cost-CQt1YRkA.js.map +0 -1
- package/dist/current-json-C-EyJqs0.js.map +0 -1
- package/dist/deploy-CrAbfgCq.js.map +0 -1
- package/dist/doctor-Db-hzPmE.js.map +0 -1
- package/dist/dump-BYA55NLP.js.map +0 -1
- package/dist/fsck-CPPZWt9p.js.map +0 -1
- package/dist/http-Cw8p6y5b.js.map +0 -1
- package/dist/inspect-Bdbzvi-_.js.map +0 -1
- package/dist/maintenance-BvOv6BMM.js.map +0 -1
- package/dist/maintenance-ClzBC1Vv.js.map +0 -1
- package/dist/observability-CpOiEN0k.js.map +0 -1
- package/dist/plan-DOTl21vq.js.map +0 -1
- package/dist/query-BmiQy328.js.map +0 -1
- package/dist/rebuild-index-DC_nR6wP.js.map +0 -1
- package/dist/restore-C6IcmLeA.js.map +0 -1
- package/dist/snapshot-SS8USDUF.js.map +0 -1
- package/dist/src-DU1sCMlr.js.map +0 -1
- package/dist/src-QQ7jDeL2.js.map +0 -1
- package/dist/src-zhEAmmF7.js.map +0 -1
- package/dist/subcommand-DZcHcsYp.js.map +0 -1
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { n as encodeJsonBytes, t as decodeJsonBytes } from "./bytes-BXdamO3L.js";
|
|
2
|
-
import {
|
|
3
|
-
import { c as
|
|
4
|
-
import { n as matchesWire, t as validateWire } from "./validate-
|
|
2
|
+
import { l as GC_MAX_PENDING_CANDIDATES, p as MAINTENANCE_PROFILE_CF_FREE, u as GC_PENDING_CONTENT_TYPE } from "./config-CGyBopy3.js";
|
|
3
|
+
import { a as walkLogRange, c as loadSnapshotAsMap, d as logObjectKey, f as casUpdateCurrentJson, h as readCurrentJson, l as snapshotKey, m as logSeqStartOf, n as probeTailFrom, o as walkLogRangeWithBytes, r as foldLogEntriesOnto, s as encodeSnapshotBody, u as snapshotHash } from "./log-tail-DXeDkWEp.js";
|
|
4
|
+
import { n as matchesWire, t as validateWire } from "./validate-DtljsCh1.js";
|
|
5
5
|
import { o as BaerlyError } from "./baerly.js";
|
|
6
|
+
import { i as assertNameNotReserved } from "./subcommand-CP9NY2J4.js";
|
|
6
7
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
7
8
|
//#region ../protocol/src/coordination/gc-pending.ts
|
|
8
9
|
/**
|
|
@@ -75,41 +76,109 @@ const createGcPending = async (storage, key, initial, opts) => {
|
|
|
75
76
|
};
|
|
76
77
|
};
|
|
77
78
|
/**
|
|
78
|
-
* Read-modify-write `gc/pending.json` under CAS
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
79
|
+
* Read-modify-write `gc/pending.json` under CAS, with a bounded retry
|
|
80
|
+
* loop. Each attempt re-reads the latest body, hands the `mutator`
|
|
81
|
+
* that latest value, and writes the result with
|
|
82
|
+
* `If-Match: <latest etag>`. On `Conflict` (a concurrent writer landed
|
|
83
|
+
* between this attempt's read and write) the loop re-reads and
|
|
84
|
+
* re-applies the mutator, up to {@link GC_PENDING_CAS_MAX_ATTEMPTS}
|
|
85
|
+
* total attempts, then surfaces `Conflict`.
|
|
86
|
+
*
|
|
87
|
+
* `mutator` MUST be a pure, deterministic MERGE of this pass's results
|
|
88
|
+
* INTO the `latest` value it is handed — it is called once per attempt
|
|
89
|
+
* (so it must not have side effects), and it MUST read `latest` rather
|
|
90
|
+
* than ignore it and write a precomputed set. Ignoring `latest` would
|
|
91
|
+
* silently overwrite a concurrent pass's candidates: the
|
|
92
|
+
* If-Match succeeds (it used the fresh etag), so no conflict is
|
|
93
|
+
* raised, and the concurrent marks are lost. See {@link mergeGcPending}.
|
|
94
|
+
*
|
|
95
|
+
* @throws BaerlyError{code:"Conflict"} — another writer kept landing a
|
|
96
|
+
* write between this function's read and write for every
|
|
97
|
+
* attempt.
|
|
89
98
|
* @throws BaerlyError{code:"InvalidResponse"} — `key` does not exist
|
|
90
99
|
* (use {@link createGcPending} instead) or body doesn't
|
|
91
100
|
* parse / fails the shape guard.
|
|
92
101
|
*/
|
|
93
102
|
const casUpdateGcPending = async (storage, key, mutator, opts) => {
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
103
|
+
let lastConflict;
|
|
104
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
105
|
+
const existing = await readGcPending(storage, key, opts);
|
|
106
|
+
if (existing === null) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key} does not exist; use createGcPending first`);
|
|
107
|
+
const next = mutator(structuredClone(existing.json));
|
|
108
|
+
assertGcPending(next, key);
|
|
109
|
+
const body = encodeJsonBytes(next);
|
|
110
|
+
const putOpts = {
|
|
111
|
+
ifMatch: existing.etag,
|
|
112
|
+
contentType: GC_PENDING_CONTENT_TYPE,
|
|
113
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
114
|
+
};
|
|
115
|
+
try {
|
|
116
|
+
return {
|
|
117
|
+
json: next,
|
|
118
|
+
etag: (await storage.put(key, body, putOpts)).etag
|
|
119
|
+
};
|
|
120
|
+
} catch (error) {
|
|
121
|
+
const translated = translateCasError(error, key);
|
|
122
|
+
if (translated.code !== "Conflict") throw translated;
|
|
123
|
+
lastConflict = translated;
|
|
124
|
+
}
|
|
109
125
|
}
|
|
126
|
+
throw lastConflict;
|
|
127
|
+
};
|
|
128
|
+
/**
|
|
129
|
+
* Pure merge of one GC pass's results INTO the `latest`
|
|
130
|
+
* `gc/pending.json` value, used as the {@link casUpdateGcPending}
|
|
131
|
+
* mutator. Deterministic, no I/O — the kernel unit + mutation test
|
|
132
|
+
* surface. Splitting it out is what makes the CAS write correct under
|
|
133
|
+
* concurrency: each retry re-merges against fresh `latest` rather than
|
|
134
|
+
* overwriting it with a stale precomputed set.
|
|
135
|
+
*
|
|
136
|
+
* Semantics:
|
|
137
|
+
* - **candidates**: start from `latest.candidates`, DROP any whose key
|
|
138
|
+
* this pass already DELETED (`sweptKeys`), then APPEND every fresh
|
|
139
|
+
* mark whose key is not already present AND was not itself swept
|
|
140
|
+
* this pass (a `graceMillis: 0` pass can mark-and-sweep a key in one
|
|
141
|
+
* go — re-adding it to the ledger after deleting it would be wrong;
|
|
142
|
+
* dedup also drops a key a concurrent pass already marked). Capped at
|
|
143
|
+
* `maxCandidates`.
|
|
144
|
+
* - **last_swept_at**: the lexicographically LATER of `latest`'s and
|
|
145
|
+
* this pass's (ISO-8601 UTC; `""` sorts lowest so a real timestamp
|
|
146
|
+
* always wins). Never regress a concurrent pass's newer sweep time.
|
|
147
|
+
* - **content_scan_cursor**: rotation cursor — liveness, not
|
|
148
|
+
* correctness; just don't regress forward progress. The two sides
|
|
149
|
+
* are NOT symmetric: `pass.nextContentCursor === undefined` means
|
|
150
|
+
* THIS pass examined to the END of the keyspace (WRAP — the most-
|
|
151
|
+
* advanced position), whereas `latest.content_scan_cursor` absent
|
|
152
|
+
* means the stored ledger has no cursor yet (start-from-beginning).
|
|
153
|
+
* So:
|
|
154
|
+
* - our pass WRAPPED ⇒ result wraps (key omitted) — our pass is
|
|
155
|
+
* the most advanced; this is the single-writer rotation reset.
|
|
156
|
+
* - else (our pass advanced to a defined cursor):
|
|
157
|
+
* - latest cursor defined ⇒ take the lexicographically GREATER
|
|
158
|
+
* (don't regress a concurrent pass's advance);
|
|
159
|
+
* - latest cursor absent ⇒ keep OUR advance (don't lose it).
|
|
160
|
+
*/
|
|
161
|
+
const mergeGcPending = (latest, pass) => {
|
|
162
|
+
const surviving = latest.candidates.filter((c) => !pass.sweptKeys.has(c.key));
|
|
163
|
+
const present = new Set(surviving.map((c) => c.key));
|
|
164
|
+
const appended = [];
|
|
165
|
+
for (const c of pass.newCandidates) if (!pass.sweptKeys.has(c.key) && !present.has(c.key)) {
|
|
166
|
+
present.add(c.key);
|
|
167
|
+
appended.push(c);
|
|
168
|
+
}
|
|
169
|
+
const candidates = [...surviving, ...appended].slice(0, pass.maxCandidates);
|
|
170
|
+
const lastSweptAt = pass.lastSweptAt > latest.last_swept_at ? pass.lastSweptAt : latest.last_swept_at;
|
|
171
|
+
const latestCursor = latest.content_scan_cursor;
|
|
172
|
+
const passCursor = pass.nextContentCursor;
|
|
173
|
+
let cursor;
|
|
174
|
+
if (passCursor === void 0) cursor = void 0;
|
|
175
|
+
else if (latestCursor !== void 0) cursor = passCursor > latestCursor ? passCursor : latestCursor;
|
|
176
|
+
else cursor = passCursor;
|
|
110
177
|
return {
|
|
111
|
-
|
|
112
|
-
|
|
178
|
+
schema_version: 1,
|
|
179
|
+
candidates,
|
|
180
|
+
last_swept_at: lastSweptAt,
|
|
181
|
+
...cursor !== void 0 && { content_scan_cursor: cursor }
|
|
113
182
|
};
|
|
114
183
|
};
|
|
115
184
|
const VALID_REASONS = new Set([
|
|
@@ -203,12 +272,6 @@ const noopMetricsRecorder = {
|
|
|
203
272
|
histogram: () => {}
|
|
204
273
|
};
|
|
205
274
|
//#endregion
|
|
206
|
-
//#region ../server/src/names.ts
|
|
207
|
-
/** Reject names in the system-reserved leading-"_" namespace. See docs/adr/007-layout-versioning-cordon.md. */
|
|
208
|
-
const assertNameNotReserved = (name, context) => {
|
|
209
|
-
if (name.startsWith("_")) throw new BaerlyError("InvalidConfig", `${context}: "${name}" is reserved — names beginning with "_" are reserved for system use. See docs/adr/007-layout-versioning-cordon.md.`);
|
|
210
|
-
};
|
|
211
|
-
//#endregion
|
|
212
275
|
//#region ../server/src/indexes.ts
|
|
213
276
|
/**
|
|
214
277
|
* Secondary indexes — declarative collection-config-driven
|
|
@@ -261,8 +324,9 @@ const assertNameNotReserved = (name, context) => {
|
|
|
261
324
|
* Locked: single-field indexes only on the read path. Composite
|
|
262
325
|
* shape is documented and reserved (the path encoder accepts it).
|
|
263
326
|
*
|
|
264
|
-
* @see docs/spec/sync-protocol.md —
|
|
265
|
-
*
|
|
327
|
+
* @see docs/spec/sync-protocol.md — index emission is hybrid around
|
|
328
|
+
* the commit: additive (new) keys are PUT before the committing
|
|
329
|
+
* `log/<seq>` create, and stale-key DELETEs land after it.
|
|
266
330
|
*/
|
|
267
331
|
/** Path-safe segment name. */
|
|
268
332
|
const INDEX_NAME_RE = /^[a-z][a-z0-9_]*$/;
|
|
@@ -563,8 +627,7 @@ const getCurrentContext = () => als.getStore();
|
|
|
563
627
|
* "missing." The atomic moment is the CAS-swap of
|
|
564
628
|
* `current.json.snapshot`, which is a single conditional PUT.
|
|
565
629
|
*
|
|
566
|
-
* Order of operations
|
|
567
|
-
* REVERSED ordering): PUT snapshot body → CAS-advance `current.json`.
|
|
630
|
+
* Order of operations: PUT snapshot body → CAS-advance `current.json`.
|
|
568
631
|
* A crash before the swap leaves an orphan snapshot file; the swap
|
|
569
632
|
* succeeds iff our captured ETag still matches.
|
|
570
633
|
*
|
|
@@ -637,7 +700,8 @@ const compact = async (args, options = {}) => {
|
|
|
637
700
|
const current = read.json;
|
|
638
701
|
const baseEtag = read.etag;
|
|
639
702
|
const logSeqStartBefore = logSeqStartOf(current);
|
|
640
|
-
const
|
|
703
|
+
const discoveredTail = (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStartBefore, current.tail_hint, internal.knownTail ?? 0), { ...options.signal !== void 0 && { signal: options.signal } })).tail;
|
|
704
|
+
const nextSeq = discoveredTail;
|
|
641
705
|
if (nextSeq - logSeqStartBefore < minToCompact) return {
|
|
642
706
|
written: false,
|
|
643
707
|
skippedReason: "below-min-threshold",
|
|
@@ -698,13 +762,15 @@ const compact = async (args, options = {}) => {
|
|
|
698
762
|
...options.signal !== void 0 && { signal: options.signal }
|
|
699
763
|
};
|
|
700
764
|
await storage.put(newKey, bodyBytes, putOpts);
|
|
765
|
+
const entriesFolded = foldEnd - logSeqStartBefore;
|
|
701
766
|
const nextBody = encodeJsonBytes({
|
|
702
767
|
...current,
|
|
703
768
|
snapshot: newKey,
|
|
704
769
|
log_seq_start: foldEnd,
|
|
770
|
+
tail_hint: Math.max(current.tail_hint, discoveredTail),
|
|
705
771
|
snapshot_bytes: bodyBytes.byteLength,
|
|
706
772
|
snapshot_rows: base.size,
|
|
707
|
-
|
|
773
|
+
mean_entry_bytes: entriesFolded > 0 ? Math.round(foldedSliceBytes / entriesFolded) : current.mean_entry_bytes ?? 0
|
|
708
774
|
});
|
|
709
775
|
const casOpts = {
|
|
710
776
|
ifMatch: baseEtag,
|
|
@@ -728,10 +794,9 @@ const compact = async (args, options = {}) => {
|
|
|
728
794
|
}
|
|
729
795
|
throw error;
|
|
730
796
|
}
|
|
731
|
-
const entriesFolded = foldEnd - logSeqStartBefore;
|
|
732
797
|
const metrics = ctxMetrics$2();
|
|
733
798
|
metrics.histogram("db.compact.entries_folded", entriesFolded, { collection: collectionName });
|
|
734
|
-
metrics.gauge("db.manifest.lag_window_depth",
|
|
799
|
+
metrics.gauge("db.manifest.lag_window_depth", discoveredTail - foldEnd, { collection: collectionName });
|
|
735
800
|
return {
|
|
736
801
|
written: true,
|
|
737
802
|
previousSnapshotKey: current.snapshot,
|
|
@@ -915,8 +980,8 @@ const runGc = async (args, options = {}) => {
|
|
|
915
980
|
for (const cand of sweepCandidates) if (toSweep.length < maxSweeps && Date.parse(cand.due_at) <= nowMs) toSweep.push(cand);
|
|
916
981
|
else remaining.push(cand);
|
|
917
982
|
await Promise.all(toSweep.map((c) => storage.delete(c.key, signalOpts)));
|
|
918
|
-
const
|
|
919
|
-
const lastSweptAt = toSweep.length > 0 ? now().toISOString() :
|
|
983
|
+
const sweptKeys = new Set(toSweep.map((c) => c.key));
|
|
984
|
+
const lastSweptAt = toSweep.length > 0 ? now().toISOString() : "";
|
|
920
985
|
const markedSummary = {
|
|
921
986
|
stale_log: markedStaleLog,
|
|
922
987
|
orphan_snapshot: markedOrphanSnapshot,
|
|
@@ -924,11 +989,12 @@ const runGc = async (args, options = {}) => {
|
|
|
924
989
|
};
|
|
925
990
|
let pendingDepth;
|
|
926
991
|
try {
|
|
927
|
-
pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, () => ({
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
992
|
+
pendingDepth = (await casUpdateGcPending(storage, gcPendingKey, (latest) => mergeGcPending(latest, {
|
|
993
|
+
sweptKeys,
|
|
994
|
+
newCandidates,
|
|
995
|
+
lastSweptAt,
|
|
996
|
+
nextContentCursor,
|
|
997
|
+
maxCandidates: GC_MAX_PENDING_CANDIDATES
|
|
932
998
|
}), signalOpts)).json.candidates.length;
|
|
933
999
|
} catch (error) {
|
|
934
1000
|
if (error instanceof BaerlyError && error.code === "Conflict") pendingDepth = remaining.length;
|
|
@@ -997,7 +1063,7 @@ const computeDueAt = (entry, now, graceMs) => {
|
|
|
997
1063
|
};
|
|
998
1064
|
/**
|
|
999
1065
|
* Build the live content-hash set. The set covers every live
|
|
1000
|
-
* post-image: every `entry.after` in `[logSeqStart,
|
|
1066
|
+
* post-image: every `entry.after` in `[logSeqStart, tail_hint)` plus
|
|
1001
1067
|
* every row body in the current snapshot.
|
|
1002
1068
|
*
|
|
1003
1069
|
* A snapshot read that throws (corrupt body, hash mismatch) is
|
|
@@ -1009,9 +1075,9 @@ const computeDueAt = (entry, now, graceMs) => {
|
|
|
1009
1075
|
const collectLiveContentHashes = async (storage, collectionPrefix, collectionName, current, logSeqStart, signal) => {
|
|
1010
1076
|
const hashes = /* @__PURE__ */ new Set();
|
|
1011
1077
|
const getOpts = signal !== void 0 ? { signal } : void 0;
|
|
1012
|
-
const
|
|
1013
|
-
|
|
1014
|
-
const got = await storage.get(
|
|
1078
|
+
const { tail } = await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, current.tail_hint), { signal });
|
|
1079
|
+
const ingestLogEntry = async (s) => {
|
|
1080
|
+
const got = await storage.get(logObjectKey(collectionPrefix, s), getOpts);
|
|
1015
1081
|
if (got === null) return;
|
|
1016
1082
|
let entry;
|
|
1017
1083
|
try {
|
|
@@ -1022,8 +1088,14 @@ const collectLiveContentHashes = async (storage, collectionPrefix, collectionNam
|
|
|
1022
1088
|
if (entry.after === void 0) return;
|
|
1023
1089
|
const bodyBytes = encodeJsonBytes(entry.after);
|
|
1024
1090
|
hashes.add(await versionFromContent(bodyBytes));
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1091
|
+
};
|
|
1092
|
+
for (let chunkStart = logSeqStart; chunkStart < tail; chunkStart += 16) {
|
|
1093
|
+
signal?.throwIfAborted();
|
|
1094
|
+
const chunkEnd = Math.min(chunkStart + 16, tail);
|
|
1095
|
+
const chunk = [];
|
|
1096
|
+
for (let s = chunkStart; s < chunkEnd; s++) chunk.push(ingestLogEntry(s));
|
|
1097
|
+
await Promise.all(chunk);
|
|
1098
|
+
}
|
|
1027
1099
|
if (current.snapshot !== null) try {
|
|
1028
1100
|
const map = await loadSnapshotAsMap(storage, current.snapshot, collectionName, signal);
|
|
1029
1101
|
const rowReads = [];
|
|
@@ -1067,8 +1139,8 @@ const collectLiveContentHashes = async (storage, collectionPrefix, collectionNam
|
|
|
1067
1139
|
*
|
|
1068
1140
|
* @see ./indexes.ts — `IndexDefinition`, `allIndexKeysFor`,
|
|
1069
1141
|
* `indexKeyPrefix`.
|
|
1070
|
-
* @see ./writer.ts — the
|
|
1071
|
-
*
|
|
1142
|
+
* @see ./writer.ts — the hybrid index emission path this command
|
|
1143
|
+
* reconciles.
|
|
1072
1144
|
*/
|
|
1073
1145
|
const EMPTY_BODY = new Uint8Array(0);
|
|
1074
1146
|
const APPLICATION_JSON = "application/json";
|
|
@@ -1089,7 +1161,7 @@ const APPLICATION_JSON = "application/json";
|
|
|
1089
1161
|
* @throws BaerlyError code="InvalidResponse" — `current.json` missing.
|
|
1090
1162
|
* @throws BaerlyError code="Internal" — protocol-invariant
|
|
1091
1163
|
* violation (snapshot hash mismatch, missing log entry inside
|
|
1092
|
-
* `[log_seq_start,
|
|
1164
|
+
* `[log_seq_start, tail_hint)`).
|
|
1093
1165
|
*/
|
|
1094
1166
|
const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
|
|
1095
1167
|
const read = await readCurrentJson(storage, currentJsonKey);
|
|
@@ -1099,9 +1171,11 @@ const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
|
|
|
1099
1171
|
const collectionPrefix = currentJsonKey.slice(0, lastSlash);
|
|
1100
1172
|
const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
1101
1173
|
const logSeqStart = logSeqStartOf(read.json);
|
|
1102
|
-
const
|
|
1174
|
+
const hint = read.json.tail_hint;
|
|
1103
1175
|
const live = read.json.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, read.json.snapshot, collection);
|
|
1104
|
-
foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart,
|
|
1176
|
+
foldLogEntriesOnto(live, await walkLogRange(storage, collectionPrefix, logSeqStart, hint), { collection });
|
|
1177
|
+
const probeOpts = opts.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
1178
|
+
foldLogEntriesOnto(live, (await probeTailFrom(storage, collectionPrefix, Math.max(logSeqStart, hint), probeOpts)).entries, { collection });
|
|
1105
1179
|
const expected = /* @__PURE__ */ new Set();
|
|
1106
1180
|
for (const [docId, body] of live) for (const k of allIndexKeysFor(collectionPrefix, [def], body, docId)) expected.add(k);
|
|
1107
1181
|
const actual = /* @__PURE__ */ new Set();
|
|
@@ -1169,26 +1243,59 @@ const rebuildIndex = async (storage, currentJsonKey, def, opts = {}) => {
|
|
|
1169
1243
|
* pattern uses exactly that.
|
|
1170
1244
|
*/
|
|
1171
1245
|
const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
|
|
1246
|
+
const profileToScheduledOptions = (profile) => ({
|
|
1247
|
+
compact: {
|
|
1248
|
+
maxEntriesPerRun: profile.maxFoldEntriesPerPass,
|
|
1249
|
+
minEntriesToCompact: 50
|
|
1250
|
+
},
|
|
1251
|
+
gc: {
|
|
1252
|
+
maxMarksPerRun: profile.gcMaxMarks,
|
|
1253
|
+
maxSweepsPerRun: profile.gcMaxSweeps
|
|
1254
|
+
}
|
|
1255
|
+
});
|
|
1256
|
+
profileToScheduledOptions(MAINTENANCE_PROFILE_CF_FREE);
|
|
1172
1257
|
/**
|
|
1173
|
-
* Floor-based GC-cadence boundary crossing.
|
|
1174
|
-
*
|
|
1175
|
-
*
|
|
1176
|
-
* trips on any interval boundary inside `(prevSeq, nextSeq]
|
|
1258
|
+
* Floor-based GC-cadence boundary crossing. Every commit advances the
|
|
1259
|
+
* writer-observed tail by one logical slot; `tail_hint` remains a
|
|
1260
|
+
* lower bound and may be refreshed later by maintenance. The floor test
|
|
1261
|
+
* trips on any interval boundary inside `(prevSeq, nextSeq]`, so it
|
|
1262
|
+
* stays correct whether the observed/probed tail advances by one or
|
|
1263
|
+
* catches up by a larger step.
|
|
1177
1264
|
*/
|
|
1178
1265
|
const crossesGcBoundary = (prevSeq, nextSeq, interval) => Math.floor(prevSeq / interval) !== Math.floor(nextSeq / interval);
|
|
1179
1266
|
/**
|
|
1267
|
+
* Derived live-tail byte size — the ratio TRIGGER input that replaces the
|
|
1268
|
+
* formerly-stored exact byte count. `(observedTail − log_seq_start) ×
|
|
1269
|
+
* mean_entry_bytes`. `observedTail` is a param so write-tick callers can
|
|
1270
|
+
* pass the writer-observed tail and scheduled callers can pass a
|
|
1271
|
+
* probe-discovered true tail. Cold-start (no mean yet) falls back to
|
|
1272
|
+
* {@link MAINTENANCE_COLD_START_ENTRY_BYTES}, never 0 (see that constant).
|
|
1273
|
+
* TRIGGER-only — structurally barred from `foldViable` (the exact-bytes CPU-kill
|
|
1274
|
+
* ceiling).
|
|
1275
|
+
*/
|
|
1276
|
+
const estimateTailBytes = (current, observedTail) => Math.max(0, observedTail - current.log_seq_start) * (current.mean_entry_bytes ?? 128);
|
|
1277
|
+
/**
|
|
1180
1278
|
* The cheap DISPATCH gate the writer calls after a commit — zero
|
|
1181
|
-
* storage ops; it reads the post-
|
|
1279
|
+
* storage ops; it reads the post-commit {@link CurrentJson} already in
|
|
1182
1280
|
* scope. Ratio-OR-boundary, WITHOUT the entry floor (the floor is part
|
|
1183
1281
|
* of the fold-trigger Gate 1 checked inside {@link runBoundedMaintenance}).
|
|
1184
|
-
* Returning `false` lets the writer skip the dispatch entirely.
|
|
1282
|
+
* Returning `false` lets the writer skip the dispatch entirely. Ratio
|
|
1283
|
+
* numerator is the DERIVED {@link estimateTailBytes};
|
|
1284
|
+
* `observedTail` threads from the caller (the writer's in-memory observed
|
|
1285
|
+
* tail `seq+1` under single-write commit). Both the boundary check and the
|
|
1286
|
+
* ratio key off `observedTail`, since the stored `tail_hint` is now only a
|
|
1287
|
+
* non-authoritative lower bound (compactor-advanced).
|
|
1185
1288
|
*/
|
|
1186
|
-
const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundary(prevSeq,
|
|
1289
|
+
const shouldFireMaintenance = (current, prevSeq, gcInterval, observedTail) => crossesGcBoundary(prevSeq, observedTail, gcInterval) || estimateTailBytes(current, observedTail) / Math.max(current.snapshot_bytes, 65536) >= 1;
|
|
1187
1290
|
/**
|
|
1188
1291
|
* The write-tick maintenance runner: a bounded, single-attempt
|
|
1189
1292
|
* composition of {@link compact} and {@link runGc} for one collection,
|
|
1190
1293
|
* sized to fit a CPU-killable Cloudflare free-tier isolate by default.
|
|
1191
1294
|
*
|
|
1295
|
+
* INVARIANT: a {@link MaintenanceProfile} changes only maintenance rate +
|
|
1296
|
+
* defer threshold — never data/API/query/wire/correctness; reads stay pure.
|
|
1297
|
+
* See `maintenance-profile-equivalence.test.ts`.
|
|
1298
|
+
*
|
|
1192
1299
|
* Never throws — the write-tick caller must never see a maintenance
|
|
1193
1300
|
* failure. Expected signals (CAS contention, an over-ceiling deferral)
|
|
1194
1301
|
* are swallowed quietly; unexpected throws bump
|
|
@@ -1196,7 +1303,8 @@ const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundar
|
|
|
1196
1303
|
* swallowed.
|
|
1197
1304
|
*
|
|
1198
1305
|
* Control flow (fold-priority with a stateless GC-starvation guard):
|
|
1199
|
-
* 0. `disabled` →
|
|
1306
|
+
* 0. `disabled` → skip fold/GC, but still allow the rate-limited
|
|
1307
|
+
* tail_hint refresh that bounds read/write forward-probes.
|
|
1200
1308
|
* 1. Read `current.json` once.
|
|
1201
1309
|
* 2. Hard-GC starvation boundary (`"single"` only) → GC slice, skip fold.
|
|
1202
1310
|
* 3. Gate 1 (ratio AND entry floor) trips → fold if viable, else
|
|
@@ -1205,17 +1313,14 @@ const shouldFireMaintenance = (current, prevSeq, gcInterval) => crossesGcBoundar
|
|
|
1205
1313
|
* 4. Otherwise (or on `"both"`) → GC iff the cadence boundary crossed.
|
|
1206
1314
|
*/
|
|
1207
1315
|
const runBoundedMaintenance = async (args, options) => {
|
|
1208
|
-
if (args.disabled === true) return;
|
|
1209
1316
|
const { storage, currentJsonKey, prevSeq } = args;
|
|
1210
|
-
const
|
|
1317
|
+
const profile = options?.profile ?? MAINTENANCE_PROFILE_CF_FREE;
|
|
1318
|
+
const { maxFoldEntriesPerPass, gcMaxMarks, gcMaxSweeps, gcInterval } = profile;
|
|
1211
1319
|
const minEntriesToCompact = options?.minEntriesToCompact ?? 50;
|
|
1212
|
-
const gcMaxMarks = options?.gcMaxMarks ?? 20;
|
|
1213
|
-
const gcMaxSweeps = options?.gcMaxSweeps ?? 10;
|
|
1214
|
-
const gcInterval = options?.gcInterval ?? 4;
|
|
1215
1320
|
const phasesPerTick = options?.phasesPerTick ?? "single";
|
|
1216
1321
|
const signal = options?.signal;
|
|
1217
|
-
const C = args.maxFoldBytes ??
|
|
1218
|
-
const E =
|
|
1322
|
+
const C = args.maxFoldBytes ?? profile.maxFoldBytes;
|
|
1323
|
+
const E = profile.maxFoldRows;
|
|
1219
1324
|
const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
|
|
1220
1325
|
const collection = collectionPrefix.slice(collectionPrefix.lastIndexOf("/") + 1);
|
|
1221
1326
|
const gcOpts = {
|
|
@@ -1229,14 +1334,29 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1229
1334
|
const read = await readCurrentJson(storage, currentJsonKey, signal !== void 0 ? { signal } : void 0);
|
|
1230
1335
|
if (read === null) return;
|
|
1231
1336
|
const current = read.json;
|
|
1232
|
-
const tailBytes = current.tail_bytes;
|
|
1233
1337
|
const snapshotBytes = current.snapshot_bytes;
|
|
1234
1338
|
const snapshotRows = current.snapshot_rows;
|
|
1235
|
-
const nextSeq = current.next_seq;
|
|
1236
1339
|
const logSeqStart = current.log_seq_start;
|
|
1237
|
-
const
|
|
1340
|
+
const probeFloor = Math.max(logSeqStart, current.tail_hint);
|
|
1341
|
+
let nextSeq;
|
|
1342
|
+
if (args.observedTail !== void 0) nextSeq = Math.max(args.observedTail, probeFloor);
|
|
1343
|
+
else nextSeq = (await probeTailFrom(storage, collectionPrefix, probeFloor, signal !== void 0 ? { signal } : void 0)).tail;
|
|
1344
|
+
const gate1 = estimateTailBytes(current, nextSeq) / Math.max(snapshotBytes, 65536) >= 1 && nextSeq - logSeqStart >= minEntriesToCompact;
|
|
1238
1345
|
const foldViable = snapshotBytes <= C && snapshotRows + maxFoldEntriesPerPass <= E;
|
|
1239
1346
|
const gcDue = crossesGcBoundary(prevSeq, nextSeq, gcInterval);
|
|
1347
|
+
const refreshTailHintIfNeeded = async () => {
|
|
1348
|
+
if (nextSeq - current.tail_hint < 128) return;
|
|
1349
|
+
try {
|
|
1350
|
+
await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
|
|
1351
|
+
...c,
|
|
1352
|
+
tail_hint: Math.max(c.tail_hint, nextSeq)
|
|
1353
|
+
}), signal !== void 0 ? { signal } : void 0);
|
|
1354
|
+
} catch {}
|
|
1355
|
+
};
|
|
1356
|
+
if (args.disabled === true) {
|
|
1357
|
+
await refreshTailHintIfNeeded();
|
|
1358
|
+
return;
|
|
1359
|
+
}
|
|
1240
1360
|
const hardGc = crossesGcBoundary(prevSeq, nextSeq, gcInterval * 4);
|
|
1241
1361
|
if (phasesPerTick === "single" && hardGc) {
|
|
1242
1362
|
await runGc({
|
|
@@ -1254,6 +1374,7 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1254
1374
|
minEntriesToCompact,
|
|
1255
1375
|
ceilingBytes: C,
|
|
1256
1376
|
ceilingEntries: E,
|
|
1377
|
+
knownTail: nextSeq,
|
|
1257
1378
|
...signal !== void 0 && { signal }
|
|
1258
1379
|
});
|
|
1259
1380
|
if (phasesPerTick === "single") return;
|
|
@@ -1262,6 +1383,7 @@ const runBoundedMaintenance = async (args, options) => {
|
|
|
1262
1383
|
collection,
|
|
1263
1384
|
dimension: snapshotBytes > C ? "bytes" : "rows"
|
|
1264
1385
|
});
|
|
1386
|
+
await refreshTailHintIfNeeded();
|
|
1265
1387
|
if (nextSeq - (current.last_warned_seq ?? 0) >= 1e3) {
|
|
1266
1388
|
console.warn(`baerly-storage: collection "${collection}" is deferring compaction — its snapshot exceeds the fold ceiling (${snapshotBytes > C ? "bytes" : "rows"}), so the tail keeps growing and read amplification will climb. This is graduation-pending: the dataset has outgrown prototype-tier maintenance. On paid Cloudflare / Node you can raise BAERLY_MAINTENANCE_MAX_FOLD_BYTES — but on Cloudflare a cap above what a single isolate can fold makes folds get CPU-killed mid-flight and silently not land (no clean metric — watch snapshot age / object count); see docs/about/graduation.md. Otherwise, graduate to a server-backed database.`);
|
|
1267
1389
|
try {
|
|
@@ -1292,4 +1414,4 @@ const dispatchInlineAwaited = (task) => task();
|
|
|
1292
1414
|
//#endregion
|
|
1293
1415
|
export { getCurrentContext as a, noopMetricsRecorder as c, rebuildIndex as i, versionFromContent as l, runBoundedMaintenance as n, allIndexKeysFor as o, shouldFireMaintenance as r, validateIndexDefinition as s, dispatchInlineAwaited as t };
|
|
1294
1416
|
|
|
1295
|
-
//# sourceMappingURL=maintenance-
|
|
1417
|
+
//# sourceMappingURL=maintenance-C_KSkVjd.js.map
|