@gusto/baerly-storage 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -124
- package/dist/API.md +161 -512
- package/dist/CHANGELOG.md +1137 -378
- package/dist/RECIPES.md +398 -80
- package/dist/THIRD-PARTY-LICENSES.txt +5 -26
- package/dist/{app-config-DyiFK3_k.d.ts → app-config-C6qW7m8S.d.ts} +119 -111
- package/dist/app-config.d.ts +31 -1
- package/dist/app-config.js +1 -2
- package/dist/app-config.js.map +1 -1
- package/dist/auth.d.ts +33 -28
- package/dist/auth.js +549 -985
- package/dist/auth.js.map +1 -1
- package/dist/baerly.js +130 -138
- package/dist/baerly.js.map +1 -1
- package/dist/baerly.spec.json +97 -22
- package/dist/capacity-warning-header-BlVopyiI.js +16 -0
- package/dist/capacity-warning-header-BlVopyiI.js.map +1 -0
- package/dist/{client-ypCm3zjt.d.ts → client-CHQTXpef.d.ts} +121 -36
- package/dist/client-react.d.ts +67 -20
- package/dist/client-react.js +240 -54
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +5 -4
- package/dist/client.js +51 -16
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-wrVYf4bb.js → cloudflare-CKHYXccw.js} +61 -82
- package/dist/cloudflare-CKHYXccw.js.map +1 -0
- package/dist/cloudflare.d.ts +106 -71
- package/dist/cloudflare.js +334 -271
- package/dist/cloudflare.js.map +1 -1
- package/dist/code-resolution-CL_7L0wS.js +22 -0
- package/dist/code-resolution-CL_7L0wS.js.map +1 -0
- package/dist/collection-api-DbyhLAY8.d.ts +1001 -0
- package/dist/config-DEzInG6M.js +4035 -0
- package/dist/config-DEzInG6M.js.map +1 -0
- package/dist/constants-D3GSH3tc.js +469 -0
- package/dist/constants-D3GSH3tc.js.map +1 -0
- package/dist/{context-DRZ3tZ5l.js → context-rlpSnHFA.js} +17 -17
- package/dist/context-rlpSnHFA.js.map +1 -0
- package/dist/cost-B52llVDx.js +330 -0
- package/dist/cost-B52llVDx.js.map +1 -0
- package/dist/{current-json-C6nhPOm1.js → current-json-CC6lE4Aj.js} +115 -32
- package/dist/current-json-CC6lE4Aj.js.map +1 -0
- package/dist/db-plFUNTSF.d.ts +130 -0
- package/dist/db-read-seams-BSs0m1z1.js +3261 -0
- package/dist/db-read-seams-BSs0m1z1.js.map +1 -0
- package/dist/{deploy-Di_Q01qx.js → deploy-Obj4nE-Z.js} +10 -14
- package/dist/deploy-Obj4nE-Z.js.map +1 -0
- package/dist/{dev-banner-CWHIk8tH.d.ts → dev-banner-B5AUHmu1.d.ts} +1 -4
- package/dist/dev-landing-BI5UdmOb.d.ts +32 -0
- package/dist/dev-landing-CD-si4dW.js +50 -0
- package/dist/dev-landing-CD-si4dW.js.map +1 -0
- package/dist/dev-vite.d.ts +9 -14
- package/dist/dev-vite.js +11 -10
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +3 -2
- package/dist/dev.js +2 -1
- package/dist/doctor-BA0i-YAN.js +270 -0
- package/dist/doctor-BA0i-YAN.js.map +1 -0
- package/dist/dump-D1vX8ozg.js +133 -0
- package/dist/dump-D1vX8ozg.js.map +1 -0
- package/dist/errors-7Rk1HGCE.js +160 -0
- package/dist/errors-7Rk1HGCE.js.map +1 -0
- package/dist/{validate-DtljsCh1.js → export-BVL2EvYE.js} +409 -19
- package/dist/export-BVL2EvYE.js.map +1 -0
- package/dist/fsck-BWkPgov4.js +145 -0
- package/dist/fsck-BWkPgov4.js.map +1 -0
- package/dist/{gcs-admin-Bx6xSNJj.d.ts → gcs-admin-Dso1HWQj.d.ts} +18 -14
- package/dist/gcs.d.ts +19 -7
- package/dist/gcs.js +1 -2
- package/dist/{goog4-signer-Z7KnJ1su.js → goog4-signer-CeLIABoE.js} +102 -92
- package/dist/goog4-signer-CeLIABoE.js.map +1 -0
- package/dist/{http-7zT29K3P.js → http-Z5rZyYMX.js} +898 -271
- package/dist/http-Z5rZyYMX.js.map +1 -0
- package/dist/http-transport-DNUM8kU-.js +1299 -0
- package/dist/http-transport-DNUM8kU-.js.map +1 -0
- package/dist/http.d.ts +366 -294
- package/dist/http.js +2 -2
- package/dist/index-JJrIu1pO.d.ts +135 -0
- package/dist/index-LEgHN0qJ.d.ts +50 -0
- package/dist/{index-DP3ClvcA.d.ts → index-RiEYj8t0.d.ts} +291 -78
- package/dist/index.d.ts +27 -6
- package/dist/index.js +3 -5
- package/dist/inspect-BTsVYZKd.js +196 -0
- package/dist/inspect-BTsVYZKd.js.map +1 -0
- package/dist/{key-Di6yyxFu.js → key-C4FPZ9Io.js} +3 -4
- package/dist/{key-Di6yyxFu.js.map → key-C4FPZ9Io.js.map} +1 -1
- package/dist/ledger-Dj_bGgo0.js +192 -0
- package/dist/ledger-Dj_bGgo0.js.map +1 -0
- package/dist/legacy-names-DgZBtgOh.js +47 -0
- package/dist/legacy-names-DgZBtgOh.js.map +1 -0
- package/dist/log-C9W15Dfo.d.ts +94 -0
- package/dist/log-tail-DWJbneBF.js +142 -0
- package/dist/log-tail-DWJbneBF.js.map +1 -0
- package/dist/log-walk-C-vA-WGJ.js +111 -0
- package/dist/log-walk-C-vA-WGJ.js.map +1 -0
- package/dist/maintenance-Dd-Yakqj.js +1709 -0
- package/dist/maintenance-Dd-Yakqj.js.map +1 -0
- package/dist/maintenance-DevB6z4e.js +2237 -0
- package/dist/maintenance-DevB6z4e.js.map +1 -0
- package/dist/maintenance.d.ts +100 -3
- package/dist/maintenance.js +1 -3
- package/dist/node.d.ts +119 -92
- package/dist/node.js +3 -3
- package/dist/{observability-Ch-VQJyE.js → observability-C-ISCCLE.js} +1247 -326
- package/dist/observability-C-ISCCLE.js.map +1 -0
- package/dist/observability.d.ts +27 -3
- package/dist/observability.js +3 -3
- package/dist/path-segment-DBXG26PW.js +363 -0
- package/dist/path-segment-DBXG26PW.js.map +1 -0
- package/dist/{plan-C4qgiXSg.js → plan-BvQbaNMe.js} +19 -11
- package/dist/plan-BvQbaNMe.js.map +1 -0
- package/dist/request-CRWiTrza.js +272 -0
- package/dist/request-CRWiTrza.js.map +1 -0
- package/dist/restore-U6Im8V1V.js +1467 -0
- package/dist/restore-U6Im8V1V.js.map +1 -0
- package/dist/retry-OBtLB2GC.js +126 -0
- package/dist/retry-OBtLB2GC.js.map +1 -0
- package/dist/rolldown-runtime-D3KUYPsf.js +12 -0
- package/dist/{runtime-spec--LykAdl1.js → runtime-spec-B5mg3pHF.js} +189 -28
- package/dist/runtime-spec-B5mg3pHF.js.map +1 -0
- package/dist/{s3-http-DD5gDc7t.d.ts → s3-http-DBC_wiCL.d.ts} +14 -8
- package/dist/s3.d.ts +20 -7
- package/dist/s3.js +4 -5
- package/dist/s3.js.map +1 -1
- package/dist/{sha256-CqcKTjw9.js → sha256-4FfGQNPw.js} +9 -9
- package/dist/sha256-4FfGQNPw.js.map +1 -0
- package/dist/{shared-secret-DgEicd3x.js → shared-secret-Vkur-8r0.js} +27 -15
- package/dist/shared-secret-Vkur-8r0.js.map +1 -0
- package/dist/{aws4fetch.esm-CXE5ZbAa.js → sigv4-sign-5IUflUD_.js} +143 -96
- package/dist/sigv4-sign-5IUflUD_.js.map +1 -0
- package/dist/snapshot-CknzkDM0.js +148 -0
- package/dist/snapshot-CknzkDM0.js.map +1 -0
- package/dist/{src-BMFfDUI2.js → src-Actga-bO.js} +949 -363
- package/dist/src-Actga-bO.js.map +1 -0
- package/dist/src-D0MQfvOp.js +812 -0
- package/dist/src-D0MQfvOp.js.map +1 -0
- package/dist/src-Dr9uo3dc.js +501 -0
- package/dist/src-Dr9uo3dc.js.map +1 -0
- package/dist/{subcommand-BV68pqwJ.js → subcommand-BLoF66dG.js} +89 -16
- package/dist/subcommand-BLoF66dG.js.map +1 -0
- package/dist/{time-DUqvFMHq.js → time-DRxTP9iv.js} +15 -34
- package/dist/time-DRxTP9iv.js.map +1 -0
- package/dist/types-suVbsEWS.d.ts +116 -0
- package/dist/{usage-BMZR8o6U.js → usage-CPzVhrQr.js} +40 -45
- package/dist/usage-CPzVhrQr.js.map +1 -0
- package/dist/uuid-BuT_c404.js +333 -0
- package/dist/uuid-BuT_c404.js.map +1 -0
- package/dist/verifier-Bmlmndzo.d.ts +95 -0
- package/dist/{wire-izfGZDvr.js → wire-DndvKU-K.js} +2 -3
- package/dist/wire-DndvKU-K.js.map +1 -0
- package/package.json +90 -69
- package/dist/aws4fetch.esm-CXE5ZbAa.js.map +0 -1
- package/dist/bytes-BXdamO3L.js +0 -11
- package/dist/bytes-BXdamO3L.js.map +0 -1
- package/dist/cloudflare-wrVYf4bb.js.map +0 -1
- package/dist/config-B6nTy8e2.js +0 -3095
- package/dist/config-B6nTy8e2.js.map +0 -1
- package/dist/constants-CfeMWQWk.js +0 -222
- package/dist/constants-CfeMWQWk.js.map +0 -1
- package/dist/context-DRZ3tZ5l.js.map +0 -1
- package/dist/cost-_SE_SVgZ.js +0 -203
- package/dist/cost-_SE_SVgZ.js.map +0 -1
- package/dist/current-json-C6nhPOm1.js.map +0 -1
- package/dist/db-DjjthkTi.d.ts +0 -254
- package/dist/deploy-Di_Q01qx.js.map +0 -1
- package/dist/doctor-RWVRpvat.js +0 -102
- package/dist/doctor-RWVRpvat.js.map +0 -1
- package/dist/dump-m2vEYDNY.js +0 -94
- package/dist/dump-m2vEYDNY.js.map +0 -1
- package/dist/errors-Dfa3p78c.js +0 -78
- package/dist/errors-Dfa3p78c.js.map +0 -1
- package/dist/export-DenKwEQu.js +0 -401
- package/dist/export-DenKwEQu.js.map +0 -1
- package/dist/fsck-2Vq-7-e2.js +0 -279
- package/dist/fsck-2Vq-7-e2.js.map +0 -1
- package/dist/goog4-signer-Z7KnJ1su.js.map +0 -1
- package/dist/http-7zT29K3P.js.map +0 -1
- package/dist/http-transport-BTgVCY5r.js +0 -1250
- package/dist/http-transport-BTgVCY5r.js.map +0 -1
- package/dist/index-BHz92Acj.d.ts +0 -162
- package/dist/index-CEZKQsKt.d.ts +0 -1113
- package/dist/index-uE-w2dgj.d.ts +0 -115
- package/dist/indexes-Dn9hbGQG.d.ts +0 -22
- package/dist/inspect-BpYbH1DM.js +0 -133
- package/dist/inspect-BpYbH1DM.js.map +0 -1
- package/dist/log-tail-QWU4wlu-.js +0 -514
- package/dist/log-tail-QWU4wlu-.js.map +0 -1
- package/dist/maintenance-BGHGt2Yg.d.ts +0 -537
- package/dist/maintenance-DULFcpd1.js +0 -2144
- package/dist/maintenance-DULFcpd1.js.map +0 -1
- package/dist/maintenance-fEi-UZ2o.js +0 -1412
- package/dist/maintenance-fEi-UZ2o.js.map +0 -1
- package/dist/normalize-rjnZLwS4.js +0 -133
- package/dist/normalize-rjnZLwS4.js.map +0 -1
- package/dist/observability-Ch-VQJyE.js.map +0 -1
- package/dist/plan-C4qgiXSg.js.map +0 -1
- package/dist/query-C43r-6ib.js +0 -1773
- package/dist/query-C43r-6ib.js.map +0 -1
- package/dist/rebuild-index-BH1dc1p6.js +0 -81
- package/dist/rebuild-index-BH1dc1p6.js.map +0 -1
- package/dist/request-u0M51mhJ.js +0 -72
- package/dist/request-u0M51mhJ.js.map +0 -1
- package/dist/restore-D0VYWzfP.js +0 -778
- package/dist/restore-D0VYWzfP.js.map +0 -1
- package/dist/rolldown-runtime-EAsCxrDo.js +0 -14
- package/dist/runtime-spec--LykAdl1.js.map +0 -1
- package/dist/sha256-CqcKTjw9.js.map +0 -1
- package/dist/shared-secret-DgEicd3x.js.map +0 -1
- package/dist/src-BMFfDUI2.js.map +0 -1
- package/dist/src-DYZ4H1aF.js +0 -392
- package/dist/src-DYZ4H1aF.js.map +0 -1
- package/dist/src-DvbTEya2.js +0 -536
- package/dist/src-DvbTEya2.js.map +0 -1
- package/dist/subcommand-BV68pqwJ.js.map +0 -1
- package/dist/time-DUqvFMHq.js.map +0 -1
- package/dist/usage-BMZR8o6U.js.map +0 -1
- package/dist/validate-DtljsCh1.js.map +0 -1
- package/dist/wire-izfGZDvr.js.map +0 -1
|
@@ -0,0 +1,1709 @@
|
|
|
1
|
+
import { c as isWriteConflict, s as BaerlyError } from "./baerly.js";
|
|
2
|
+
import { a as logDeleteFloorOf, c as readCurrentJson, d as encodeJsonBytes, l as unsupportedSchemaVersionMessage, n as casUpdateCurrentJson, o as logSeqStartOf, r as certifiedDeleteFloor, t as assertCurrentJsonTransition, u as decodeJsonBytes } from "./current-json-CC6lE4Aj.js";
|
|
3
|
+
import { C as MAINTENANCE_PROFILE_NODE, S as MAINTENANCE_PROFILE_CF_PAID, _ as GC_PENDING_CONTENT_TYPE, a as setRequestHoldResolver, b as MAINTENANCE_MIN_LIVE_BYTES, d as CF_PAID_COMPACT_TAIL_PROBE_GETS, g as GC_PASS_GRACE_FRACTION, h as GC_MAX_PENDING_CANDIDATES, m as GC_MARK_LIST_PAGE_KEYS, o as setRequestLedgerResolver, x as MAINTENANCE_PROFILE_CF_FREE } from "./config-DEzInG6M.js";
|
|
4
|
+
import { a as snapshotHash, i as isOwnCollection, n as loadSnapshotAsMap, r as snapshotKey, t as encodeSnapshotBody } from "./snapshot-CknzkDM0.js";
|
|
5
|
+
import { n as walkLogRangeWithBytes } from "./log-walk-C-vA-WGJ.js";
|
|
6
|
+
import { i as logObjectKey, n as probeTailChunk, r as probeTailFrom } from "./log-tail-DWJbneBF.js";
|
|
7
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
8
|
+
// ../protocol/src/coordination/gc-pending.ts
|
|
9
|
+
/**
|
|
10
|
+
* `gc/pending.json` — the per-collection GC candidate ledger. CAS-
|
|
11
|
+
* protected; one per `(tenant, collection)` key — for example
|
|
12
|
+
* `<tenant>/<collection>/gc/pending.json`. `runGc()` is the single
|
|
13
|
+
* writer that adds and removes entries. Stays small in steady state
|
|
14
|
+
* because every candidate is eventually resolved out of the ledger.
|
|
15
|
+
*
|
|
16
|
+
* "Mark" appends candidates with a future `due_at`; "sweep" revalidates
|
|
17
|
+
* due candidates and either DELETEs the named object or resolves the
|
|
18
|
+
* entry out of the ledger without one. Both run in the same GC pass, not
|
|
19
|
+
* separate passes.
|
|
20
|
+
*
|
|
21
|
+
* Lives next to {@link CurrentJson} in `coordination/` — same shape
|
|
22
|
+
* of "CAS-protected small control object." Pure module; no Node
|
|
23
|
+
* imports; Worker-bundleable.
|
|
24
|
+
*
|
|
25
|
+
* @see packages/server/src/gc.ts
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* The single constructor for a `gc/pending.json` key:
|
|
29
|
+
* `<collectionPrefix>/gc/pending.json`. `collectionPrefix` is the
|
|
30
|
+
* `<…>/manifests/<collection>` segment — this helper adds the ledger's
|
|
31
|
+
* relative path. Route all callers here so the key shape lives in one
|
|
32
|
+
* place, mirroring {@link logObjectKey} in `log-key.ts` for the sibling
|
|
33
|
+
* `log/<seq>.json` shape.
|
|
34
|
+
*/
|
|
35
|
+
const gcPendingKey = (collectionPrefix) => `${collectionPrefix}/gc/pending.json`;
|
|
36
|
+
/**
|
|
37
|
+
* Read + parse `gc/pending.json` at `key`. Returns `null` on
|
|
38
|
+
* not-found.
|
|
39
|
+
*
|
|
40
|
+
* @throws BaerlyError{code:"InvalidResponse"} — body is not valid
|
|
41
|
+
* JSON, or `schema_version` is unknown, or the shape fails
|
|
42
|
+
* the runtime guard.
|
|
43
|
+
*/
|
|
44
|
+
const readGcPending = async (storage, key, opts) => {
|
|
45
|
+
const got = await storage.get(key, opts);
|
|
46
|
+
if (got === null) return null;
|
|
47
|
+
let parsed;
|
|
48
|
+
try {
|
|
49
|
+
parsed = decodeJsonBytes(got.body);
|
|
50
|
+
} catch (error) {
|
|
51
|
+
throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: body is not valid JSON`, { cause: error });
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
json: assertGcPending(parsed, key),
|
|
55
|
+
etag: got.etag
|
|
56
|
+
};
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Create `gc/pending.json` if-and-only-if it does not exist (S3
|
|
60
|
+
* `If-None-Match: "*"`). Use this once per collection at first GC
|
|
61
|
+
* run; subsequent updates go through {@link casUpdateGcPending}.
|
|
62
|
+
*
|
|
63
|
+
* @throws BaerlyError{code:"WriteConflict"} — the key already exists.
|
|
64
|
+
* @throws BaerlyError{code:"InvalidResponse"} — `initial` fails the
|
|
65
|
+
* runtime shape guard.
|
|
66
|
+
*/
|
|
67
|
+
const createGcPending = async (storage, key, initial, opts) => {
|
|
68
|
+
assertGcPending(initial, key);
|
|
69
|
+
const body = encodeJsonBytes(initial);
|
|
70
|
+
const putOpts = {
|
|
71
|
+
ifNoneMatch: "*",
|
|
72
|
+
contentType: GC_PENDING_CONTENT_TYPE,
|
|
73
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
74
|
+
};
|
|
75
|
+
let result;
|
|
76
|
+
try {
|
|
77
|
+
result = await storage.put(key, body, putOpts);
|
|
78
|
+
} catch (error) {
|
|
79
|
+
throw translateCasError(error, key);
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
json: initial,
|
|
83
|
+
etag: result.etag
|
|
84
|
+
};
|
|
85
|
+
};
|
|
86
|
+
/**
|
|
87
|
+
* Read-modify-write `gc/pending.json` under CAS, with a bounded retry
|
|
88
|
+
* loop. Each attempt re-reads the latest body, hands the `mutator`
|
|
89
|
+
* that latest value, and writes the result with
|
|
90
|
+
* `If-Match: <latest etag>`. On `WriteConflict` (a concurrent writer landed
|
|
91
|
+
* between this attempt's read and write) the loop re-reads and
|
|
92
|
+
* re-applies the mutator, up to {@link GC_PENDING_CAS_MAX_ATTEMPTS}
|
|
93
|
+
* total attempts, then surfaces `WriteConflict`.
|
|
94
|
+
*
|
|
95
|
+
* `mutator` MUST be a pure, deterministic MERGE of this pass's results
|
|
96
|
+
* INTO the `latest` value it is handed — it is called once per attempt
|
|
97
|
+
* (so it must not have side effects), and it MUST read `latest` rather
|
|
98
|
+
* than ignore it and write a precomputed set. Ignoring `latest` would
|
|
99
|
+
* silently overwrite a concurrent pass's candidates: the
|
|
100
|
+
* If-Match succeeds (it used the fresh etag), so no conflict is
|
|
101
|
+
* raised, and the concurrent marks are lost. See {@link mergeGcPending}.
|
|
102
|
+
*
|
|
103
|
+
* @throws BaerlyError{code:"WriteConflict"} — another writer kept landing a
|
|
104
|
+
* write between this function's read and write for every
|
|
105
|
+
* attempt, OR `key` does not exist. The missing-key case is a
|
|
106
|
+
* CAS precondition failure, not a separate class of fault: the
|
|
107
|
+
* pre-read is an optimisation, and a bare
|
|
108
|
+
* `If-Match: <etag>` PUT against a key another actor deleted
|
|
109
|
+
* would have surfaced `412` → `WriteConflict` from the storage layer
|
|
110
|
+
* anyway. It is deliberately NOT retried — a deleted ledger
|
|
111
|
+
* stays deleted until someone calls {@link createGcPending},
|
|
112
|
+
* which the next `runGc` pass does. `baerly admin restore`
|
|
113
|
+
* deletes `gc/pending.json` on both reseed branches, so a
|
|
114
|
+
* concurrent pass reaches this routinely; `runGc` treats
|
|
115
|
+
* `WriteConflict` as benign best-effort.
|
|
116
|
+
* @throws BaerlyError{code:"InvalidResponse"} — the stored body doesn't
|
|
117
|
+
* parse or fails the shape guard. Kept distinct from the
|
|
118
|
+
* missing-key case on purpose: a caller that tolerates
|
|
119
|
+
* `WriteConflict` as a race must not thereby swallow a corrupt
|
|
120
|
+
* ledger, which fails identically on every future pass.
|
|
121
|
+
*/
|
|
122
|
+
const casUpdateGcPending = async (storage, key, mutator, opts) => {
|
|
123
|
+
let lastConflict;
|
|
124
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
125
|
+
const existing = await readGcPending(storage, key, opts);
|
|
126
|
+
if (existing === null) throw new BaerlyError("WriteConflict", `gc/pending.json at ${key} does not exist; it was deleted concurrently, or was never created — see createGcPending`);
|
|
127
|
+
const next = mutator(structuredClone(existing.json));
|
|
128
|
+
assertGcPending(next, key);
|
|
129
|
+
const body = encodeJsonBytes(next);
|
|
130
|
+
const putOpts = {
|
|
131
|
+
ifMatch: existing.etag,
|
|
132
|
+
contentType: GC_PENDING_CONTENT_TYPE,
|
|
133
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
134
|
+
};
|
|
135
|
+
try {
|
|
136
|
+
return {
|
|
137
|
+
json: next,
|
|
138
|
+
etag: (await storage.put(key, body, putOpts)).etag
|
|
139
|
+
};
|
|
140
|
+
} catch (error) {
|
|
141
|
+
const translated = translateCasError(error, key);
|
|
142
|
+
if (!isWriteConflict(translated)) throw translated;
|
|
143
|
+
lastConflict = translated;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
throw lastConflict;
|
|
147
|
+
};
|
|
148
|
+
/**
|
|
149
|
+
* Pure merge of one GC pass's results INTO the `latest`
|
|
150
|
+
* `gc/pending.json` value, used as the {@link casUpdateGcPending}
|
|
151
|
+
* mutator. Deterministic, no I/O — the kernel unit + mutation test
|
|
152
|
+
* surface. Splitting it out is what makes the CAS write correct under
|
|
153
|
+
* concurrency: each retry re-merges against fresh `latest` rather than
|
|
154
|
+
* overwriting it with a stale precomputed set.
|
|
155
|
+
*
|
|
156
|
+
* Semantics:
|
|
157
|
+
* - **candidates**: start from `latest.candidates`, DROP any whose key
|
|
158
|
+
* this pass already RESOLVED (`sweptKeys` carries both object DELETEs
|
|
159
|
+
* and ledger-only drops/evictions), then APPEND every fresh mark whose
|
|
160
|
+
* key is not already present AND was not itself resolved this pass. A
|
|
161
|
+
* `graceMillis: 0` pass can mark-and-sweep a key in one go; re-adding a
|
|
162
|
+
* terminal key would be wrong. Dedup also drops a key a concurrent pass
|
|
163
|
+
* already marked. Capped at `maxCandidates`.
|
|
164
|
+
* - **last_swept_at**: the lexicographically LATER of `latest`'s and
|
|
165
|
+
* this pass's (ISO-8601 UTC; `""` sorts lowest so a real timestamp
|
|
166
|
+
* always wins). Never regress a concurrent pass's newer sweep time.
|
|
167
|
+
*/
|
|
168
|
+
const mergeGcPending = (latest, pass) => {
|
|
169
|
+
const surviving = latest.candidates.filter((c) => !pass.sweptKeys.has(c.key));
|
|
170
|
+
const present = new Set(surviving.map((c) => c.key));
|
|
171
|
+
const appended = [];
|
|
172
|
+
for (const c of pass.newCandidates) if (!pass.sweptKeys.has(c.key) && !present.has(c.key)) {
|
|
173
|
+
present.add(c.key);
|
|
174
|
+
appended.push(c);
|
|
175
|
+
}
|
|
176
|
+
return {
|
|
177
|
+
schema_version: 1,
|
|
178
|
+
candidates: [...surviving, ...appended].slice(0, pass.maxCandidates),
|
|
179
|
+
last_swept_at: pass.lastSweptAt > latest.last_swept_at ? pass.lastSweptAt : latest.last_swept_at
|
|
180
|
+
};
|
|
181
|
+
};
|
|
182
|
+
/**
|
|
183
|
+
* Accepted `reason` values. Only `"orphan-snapshot"` is still emitted; the
|
|
184
|
+
* other two are legacy-ledger inputs with different sweep fates — see
|
|
185
|
+
* {@link GcCandidate.reason}.
|
|
186
|
+
*/
|
|
187
|
+
const VALID_REASONS = /* @__PURE__ */ new Set([
|
|
188
|
+
"stale-log",
|
|
189
|
+
"orphan-snapshot",
|
|
190
|
+
"orphan-content"
|
|
191
|
+
]);
|
|
192
|
+
/**
|
|
193
|
+
* Runtime guard for `parsed` to be a {@link GcPending}. Throws
|
|
194
|
+
* `BaerlyError{code:"InvalidResponse"}` rather than coercing.
|
|
195
|
+
*/
|
|
196
|
+
const assertGcPending = (parsed, key) => {
|
|
197
|
+
if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: parsed body is not an object`);
|
|
198
|
+
const r = parsed;
|
|
199
|
+
if (r["schema_version"] !== 1) throw new BaerlyError("InvalidResponse", unsupportedSchemaVersionMessage(`gc/pending.json at ${key}`, r["schema_version"], 1));
|
|
200
|
+
if (!Array.isArray(r["candidates"])) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates must be an array`);
|
|
201
|
+
for (let i = 0; i < r["candidates"].length; i++) {
|
|
202
|
+
const c = r["candidates"][i];
|
|
203
|
+
if (c === null || typeof c !== "object") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}] is not an object`);
|
|
204
|
+
const cr = c;
|
|
205
|
+
if (typeof cr["key"] !== "string" || cr["key"].length === 0) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].key must be a non-empty string`);
|
|
206
|
+
if (typeof cr["due_at"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].due_at must be a string`);
|
|
207
|
+
if (typeof cr["reason"] !== "string" || !VALID_REASONS.has(cr["reason"])) throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].reason must be one of stale-log|orphan-snapshot|orphan-content`);
|
|
208
|
+
if (cr["generation"] !== void 0 && typeof cr["generation"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: candidates[${String(i)}].generation must be a string when present`);
|
|
209
|
+
}
|
|
210
|
+
if (typeof r["last_swept_at"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: last_swept_at must be a string`);
|
|
211
|
+
if (r["log_scan_cursor"] !== void 0 && typeof r["log_scan_cursor"] !== "string") throw new BaerlyError("InvalidResponse", `gc/pending.json at ${key}: log_scan_cursor must be a string when present`);
|
|
212
|
+
return parsed;
|
|
213
|
+
};
|
|
214
|
+
/**
|
|
215
|
+
* Wrap a storage-level error with `gc/pending.json at <key>`
|
|
216
|
+
* location context. `WriteConflict` (the storage layer's CAS-lost signal)
|
|
217
|
+
* is re-thrown with an annotated message; other `BaerlyError`s pass
|
|
218
|
+
* through; non-`BaerlyError` falls through as `InvalidResponse`.
|
|
219
|
+
*/
|
|
220
|
+
const translateCasError = (e, key) => {
|
|
221
|
+
if (isWriteConflict(e)) return new BaerlyError("WriteConflict", `gc/pending.json CAS lost at ${key}: ${e.message}`, { cause: e });
|
|
222
|
+
if (e instanceof BaerlyError) return e;
|
|
223
|
+
return new BaerlyError("InvalidResponse", `gc/pending.json write at ${key} failed: ${String(e)}`, { cause: e });
|
|
224
|
+
};
|
|
225
|
+
// ../protocol/src/metrics.ts
|
|
226
|
+
/**
|
|
227
|
+
* No-op recorder. Safe default when no operator sink is wired —
|
|
228
|
+
* emission sites that read `getCurrentContext()?.recorder` fall
|
|
229
|
+
* back to this outside an active `runWithContext` scope.
|
|
230
|
+
*/
|
|
231
|
+
const noopMetricsRecorder = {
|
|
232
|
+
counter: () => {},
|
|
233
|
+
gauge: () => {},
|
|
234
|
+
histogram: () => {}
|
|
235
|
+
};
|
|
236
|
+
// ../server/src/option-guards.ts
|
|
237
|
+
/**
|
|
238
|
+
* Shared pre-I/O validation for the internal maintenance option seams
|
|
239
|
+
* (`InternalCompactOptions`, `InternalRunGcOptions`).
|
|
240
|
+
*
|
|
241
|
+
* Those seams are unvalidated by construction — reached by JS callers,
|
|
242
|
+
* in-repo casts, and the `_internal/testing` subpath — yet their values
|
|
243
|
+
* feed seq arithmetic and range bounds. A non-finite value must fail
|
|
244
|
+
* before the first storage operation, so a rejected run costs nothing
|
|
245
|
+
* durable. `compactor.ts` explains the specific NaN corruption at stake.
|
|
246
|
+
*
|
|
247
|
+
* Zero-import leaf beyond `BaerlyError`: both maintenance entry points
|
|
248
|
+
* already carry that import, so consolidating here removes duplication
|
|
249
|
+
* without widening either closure.
|
|
250
|
+
*/
|
|
251
|
+
/**
|
|
252
|
+
* Throw `InvalidConfig` unless `value` is an integer at or above `min`.
|
|
253
|
+
* `min` is `0` for a seq-shaped or count-shaped option (zero is a
|
|
254
|
+
* meaningful "do nothing" bound) and `1` for one that must make at least
|
|
255
|
+
* one unit of progress to be well-defined.
|
|
256
|
+
*
|
|
257
|
+
* Callers own the `undefined ⇒ default` decision before calling; this
|
|
258
|
+
* guard rejects rather than defaults, so passing an absent option is a
|
|
259
|
+
* caller bug, not a silently-tolerated input.
|
|
260
|
+
*/
|
|
261
|
+
const requireIntegerOption = (fnName, collectionName, optionName, value, min) => {
|
|
262
|
+
if (!Number.isInteger(value) || value < min) throw new BaerlyError("InvalidConfig", `${fnName}(${collectionName}): ${optionName} must be a ${min === 0 ? "non-negative" : "positive"} integer, got ${String(value)}`);
|
|
263
|
+
};
|
|
264
|
+
// ../server/src/observability/context.ts
|
|
265
|
+
/**
|
|
266
|
+
* Per-request observability context.
|
|
267
|
+
*
|
|
268
|
+
* One `ObservabilityContext` is created at the entry of each unit-of-
|
|
269
|
+
* work (HTTP request, maintenance tick, GC sweep, rebuild run) and
|
|
270
|
+
* carried through the call tree via {@link runWithContext} /
|
|
271
|
+
* {@link getCurrentContext}, which delegates to a single module-level
|
|
272
|
+
* `AsyncLocalStorage` instance.
|
|
273
|
+
*
|
|
274
|
+
* The context is intentionally tiny: an externally-set
|
|
275
|
+
* {@link request_id}, a high-resolution `started_at`, and a mutable
|
|
276
|
+
* {@link fields} bag used to accumulate canonical-line attributes
|
|
277
|
+
* during the unit's lifecycle.
|
|
278
|
+
*
|
|
279
|
+
* The context shape is `readonly` everywhere it can be — only the
|
|
280
|
+
* contents of `fields` are intended to mutate during a request. The
|
|
281
|
+
* {@link fields} map itself is not `readonly` because callers (the
|
|
282
|
+
* canonical-line emitter, middleware, the collection API once Dispatch
|
|
283
|
+
* 3+ wires it up) need to set entries.
|
|
284
|
+
*
|
|
285
|
+
* The context carries no sampling state — every unit-of-work emits
|
|
286
|
+
* one canonical line unconditionally; there is nothing to gate.
|
|
287
|
+
*
|
|
288
|
+
* ## Time source
|
|
289
|
+
*
|
|
290
|
+
* `started_at` uses `performance.now()` so durations can be
|
|
291
|
+
* computed without worrying about wall-clock skew (NTP step,
|
|
292
|
+
* leap-second smearing). `performance.now()` is part of both Node
|
|
293
|
+
* 18+ and the Workers Runtime; we don't fall back to `Date.now()`.
|
|
294
|
+
*
|
|
295
|
+
* @example
|
|
296
|
+
* ```ts
|
|
297
|
+
* const ctx = createObservabilityContext({});
|
|
298
|
+
* await runWithContext(ctx, async () => {
|
|
299
|
+
* const inner = getCurrentContext();
|
|
300
|
+
* inner?.fields.set("collection", "tickets");
|
|
301
|
+
* await doWork();
|
|
302
|
+
* });
|
|
303
|
+
* ```
|
|
304
|
+
*/
|
|
305
|
+
/**
|
|
306
|
+
* Module-level slot. One instance per process — the Workers
|
|
307
|
+
* Runtime's `nodejs_compat` flag wires `node:async_hooks` to a
|
|
308
|
+
* per-request scope, so cross-request bleed is impossible there.
|
|
309
|
+
*/
|
|
310
|
+
const als = new AsyncLocalStorage();
|
|
311
|
+
setRequestLedgerResolver(() => als.getStore()?.ledger);
|
|
312
|
+
setRequestHoldResolver(() => als.getStore()?.hold);
|
|
313
|
+
/**
|
|
314
|
+
* Run `fn` with `ctx` active as the current observability context.
|
|
315
|
+
* Inside `fn` (and any `await`-chained continuation rooted in `fn`),
|
|
316
|
+
* {@link getCurrentContext} returns `ctx`. Outside, it returns the
|
|
317
|
+
* previously-active context (or `undefined`).
|
|
318
|
+
*
|
|
319
|
+
* `runWithContext` is async-safe via `AsyncLocalStorage`; it propagates
|
|
320
|
+
* through `await`, `Promise.all`, `setTimeout`, and `queueMicrotask`.
|
|
321
|
+
*
|
|
322
|
+
* Returns `fn`'s return value verbatim — including its promise
|
|
323
|
+
* identity, so callers can `await` or chain off it without extra
|
|
324
|
+
* indirection.
|
|
325
|
+
*/
|
|
326
|
+
const runWithContext = (ctx, fn) => als.run(ctx, fn);
|
|
327
|
+
/** Returns the currently-active context, or `undefined` outside one. */
|
|
328
|
+
const getCurrentContext = () => als.getStore();
|
|
329
|
+
/** The active context's recorder, or the no-op recorder outside one. */
|
|
330
|
+
const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
|
|
331
|
+
// ../server/src/collection-key.ts
|
|
332
|
+
/**
|
|
333
|
+
* Splits a `current.json` key (`<collectionPrefix>/current.json`) into the
|
|
334
|
+
* collection's key prefix and its trailing name segment. `compact()`,
|
|
335
|
+
* `runGc()`, `retireLogRange()` and the maintenance runner all take the
|
|
336
|
+
* key and derive these the same way.
|
|
337
|
+
*/
|
|
338
|
+
const collectionFromCurrentJsonKey = (currentJsonKey) => {
|
|
339
|
+
const prefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
|
|
340
|
+
return {
|
|
341
|
+
prefix,
|
|
342
|
+
name: prefix.slice(prefix.lastIndexOf("/") + 1)
|
|
343
|
+
};
|
|
344
|
+
};
|
|
345
|
+
// ../server/src/snapshot-projection.ts
|
|
346
|
+
/**
|
|
347
|
+
* Byte-exact projection of the snapshot a fold rebuilds, entry by entry,
|
|
348
|
+
* as the compactor encodes it. Shared by the growth-refusal check
|
|
349
|
+
* (`./capacity.ts`) and the compactor's bytes cut (`./compactor.ts`), so
|
|
350
|
+
* the two agree on where a fold can stop; a module of its own because the
|
|
351
|
+
* compactor cannot import `./capacity.ts` without a cycle through
|
|
352
|
+
* `./maintenance.ts`.
|
|
353
|
+
*/
|
|
354
|
+
/** Encoded size of one snapshot row: the `{ _id, body }` element the compactor writes. */
|
|
355
|
+
const rowBytes = (docId, body) => encodeJsonBytes({
|
|
356
|
+
_id: docId,
|
|
357
|
+
body
|
|
358
|
+
}).byteLength;
|
|
359
|
+
/** Separators between `rows` elements of the snapshot's `docs` array. */
|
|
360
|
+
const separators = (rows) => Math.max(rows - 1, 0);
|
|
361
|
+
/**
|
|
362
|
+
* Encoded size of an empty snapshot body for `collection` whose header
|
|
363
|
+
* records `max_seq`. Rows add their own bytes plus one separator between
|
|
364
|
+
* each pair, inside the `docs` array.
|
|
365
|
+
*/
|
|
366
|
+
const headerBytes = (collection, maxSeq) => encodeSnapshotBody({
|
|
367
|
+
schema_version: 1,
|
|
368
|
+
min_seq: 0,
|
|
369
|
+
max_seq: maxSeq,
|
|
370
|
+
collection,
|
|
371
|
+
docs: []
|
|
372
|
+
}).byteLength;
|
|
373
|
+
/**
|
|
374
|
+
* Project the snapshot a fold would build from `head`'s stored figures and
|
|
375
|
+
* its rows (`base`), one entry at a time, byte for byte as the compactor
|
|
376
|
+
* encodes it: each row, one separator between rows, and the header, whose
|
|
377
|
+
* `max_seq` is the fold's end. A stored snapshot's `max_seq` is
|
|
378
|
+
* `log_seq_start` (the compactor writes both from the same fold end); a
|
|
379
|
+
* never-folded collection (`snapshot_bytes === 0`) starts empty at 0.
|
|
380
|
+
*
|
|
381
|
+
* `bytes` keeps the stored header, so a growth check made now does not
|
|
382
|
+
* know the `max_seq` its fold will record; `CAPACITY_BYTES_MARGIN` covers
|
|
383
|
+
* that header growth. `bytesAt` is exact for a fold whose end is known.
|
|
384
|
+
*
|
|
385
|
+
* @internal — exported for `capacity.test.ts`.
|
|
386
|
+
*/
|
|
387
|
+
const projectSnapshot = (head, base, collection) => {
|
|
388
|
+
const overlay = /* @__PURE__ */ new Map();
|
|
389
|
+
const sizeOf = (docId) => {
|
|
390
|
+
const known = overlay.get(docId);
|
|
391
|
+
if (known !== void 0) return known ?? void 0;
|
|
392
|
+
const body = base.get(docId);
|
|
393
|
+
return body === void 0 ? void 0 : rowBytes(docId, body);
|
|
394
|
+
};
|
|
395
|
+
const folded = head.snapshot_bytes !== 0;
|
|
396
|
+
let header = headerBytes(collection, folded ? head.log_seq_start : 0);
|
|
397
|
+
let rows = head.snapshot_rows;
|
|
398
|
+
let rowSum = folded ? head.snapshot_bytes - header - separators(rows) : 0;
|
|
399
|
+
const sizeNow = () => header + rowSum + separators(rows);
|
|
400
|
+
let peakRows = rows;
|
|
401
|
+
let peakBytes = sizeNow();
|
|
402
|
+
return {
|
|
403
|
+
get rows() {
|
|
404
|
+
return rows;
|
|
405
|
+
},
|
|
406
|
+
get bytes() {
|
|
407
|
+
return sizeNow();
|
|
408
|
+
},
|
|
409
|
+
get peakRows() {
|
|
410
|
+
return peakRows;
|
|
411
|
+
},
|
|
412
|
+
get peakBytes() {
|
|
413
|
+
return peakBytes;
|
|
414
|
+
},
|
|
415
|
+
bytesAt(maxSeq) {
|
|
416
|
+
return headerBytes(collection, maxSeq) + rowSum + separators(rows);
|
|
417
|
+
},
|
|
418
|
+
addsRow(op, docId, after) {
|
|
419
|
+
return op !== "D" && after !== void 0 && sizeOf(docId) === void 0;
|
|
420
|
+
},
|
|
421
|
+
apply(op, docId, after) {
|
|
422
|
+
if (op !== "D" && after === void 0) return;
|
|
423
|
+
const before = sizeOf(docId);
|
|
424
|
+
if (op === "D" || after === void 0) {
|
|
425
|
+
if (before !== void 0) {
|
|
426
|
+
rows -= 1;
|
|
427
|
+
rowSum -= before;
|
|
428
|
+
}
|
|
429
|
+
overlay.set(docId, null);
|
|
430
|
+
return;
|
|
431
|
+
}
|
|
432
|
+
const next = rowBytes(docId, after);
|
|
433
|
+
if (before === void 0) rows += 1;
|
|
434
|
+
rowSum += next - (before ?? 0);
|
|
435
|
+
overlay.set(docId, next);
|
|
436
|
+
peakRows = Math.max(peakRows, rows);
|
|
437
|
+
peakBytes = Math.max(peakBytes, sizeNow());
|
|
438
|
+
},
|
|
439
|
+
settle(maxSeq) {
|
|
440
|
+
header = headerBytes(collection, maxSeq);
|
|
441
|
+
peakRows = rows;
|
|
442
|
+
peakBytes = sizeNow();
|
|
443
|
+
}
|
|
444
|
+
};
|
|
445
|
+
};
|
|
446
|
+
// ../server/src/compactor.ts
|
|
447
|
+
/**
|
|
448
|
+
* Compactor: fold a prefix of the live log into a content-
|
|
449
|
+
* hashed snapshot file and CAS-advance `current.json` to swap the
|
|
450
|
+
* pointer.
|
|
451
|
+
*
|
|
452
|
+
* The hard correctness requirement is **atomic snapshot generation**:
|
|
453
|
+
* a compactor that crashes mid-write must not leave a corrupted body
|
|
454
|
+
* that readers consume as truth. The design: every snapshot's filename
|
|
455
|
+
* embeds the SHA-256 of its body. A crashed mid-PUT leaves a body that
|
|
456
|
+
* doesn't match its own filename hash, and readers reject it as
|
|
457
|
+
* "missing." The atomic moment is the CAS-swap of
|
|
458
|
+
* `current.json.snapshot`, which is a single conditional PUT.
|
|
459
|
+
*
|
|
460
|
+
* Order of operations: PUT snapshot body → CAS-advance `current.json`.
|
|
461
|
+
* A crash before the swap leaves an orphan snapshot file; the swap
|
|
462
|
+
* succeeds iff our captured ETag still matches.
|
|
463
|
+
*
|
|
464
|
+
* Why level-prefixed chunked snapshots over the alternatives: a
|
|
465
|
+
* single monolithic snapshot (every compaction rewrites one well-
|
|
466
|
+
* known key) makes partial writes catastrophic and serializes every
|
|
467
|
+
* writer on one key. WAL-checkpointing in the rqlite / Raft tradition
|
|
468
|
+
* requires a consensus layer baerly-storage doesn't have. The Litestream-
|
|
469
|
+
* style multi-level scheme — small fixed number of levels, chunked
|
|
470
|
+
* files keyed by sequence range — keeps random GETs cheap on
|
|
471
|
+
* S3-compatible storage and avoids LIST. Current ship is single-
|
|
472
|
+
* level at L9 (one snapshot replaces the prior; no multi-level merge
|
|
473
|
+
* yet); the key format is forward-compatible with future L0..L9
|
|
474
|
+
* rolling merges without a wire change.
|
|
475
|
+
*
|
|
476
|
+
* @see ../../../docs/spec/sync-protocol.md
|
|
477
|
+
*/
|
|
478
|
+
const DEFAULT_MAX_PER_RUN = Number.MAX_SAFE_INTEGER;
|
|
479
|
+
const DEFAULT_MIN_TO_COMPACT = 100;
|
|
480
|
+
const APPLICATION_JSON = "application/json";
|
|
481
|
+
/**
|
|
482
|
+
* Fold a prefix of the live log into a new snapshot and CAS-advance
|
|
483
|
+
* `current.json`. Idempotent: a second run with no new writes is a
|
|
484
|
+
* no-op (below-min-threshold).
|
|
485
|
+
*
|
|
486
|
+
* Single-attempt: on CAS conflict we return
|
|
487
|
+
* `{written: false, skippedReason: "cas-lost"}` — the
|
|
488
|
+
* caller (the cron handler invoking `runScheduledMaintenance` from
|
|
489
|
+
* `./maintenance.ts`) decides whether to schedule another run. The
|
|
490
|
+
* orphan snapshot file just written will be swept by `runGc()`.
|
|
491
|
+
*
|
|
492
|
+
* @throws BaerlyError code="InvalidConfig" — `maxEntriesPerRun` or
|
|
493
|
+
* `knownTail` (the `InternalCompactOptions` seam) is not a
|
|
494
|
+
* non-negative integer, or `maxTailProbeGets` is not a positive
|
|
495
|
+
* integer. Checked before any I/O: a non-finite value would otherwise
|
|
496
|
+
* slip the range guards and write an unreadable `current.json`.
|
|
497
|
+
* @throws BaerlyError code="InvalidResponse" — `current.json` is
|
|
498
|
+
* present but malformed, or a snapshot body fails its schema /
|
|
499
|
+
* collection cross-check.
|
|
500
|
+
* @throws BaerlyError code="Internal" — protocol-invariant violation:
|
|
501
|
+
* a log entry missing inside `[log_seq_start, foldEnd)`, or the
|
|
502
|
+
* prior-snapshot pointer resolves to no body / a body whose hash
|
|
503
|
+
* doesn't match its filename.
|
|
504
|
+
*
|
|
505
|
+
* @example
|
|
506
|
+
* ```ts
|
|
507
|
+
* import { compact } from "@baerly/server/_internal/maintenance";
|
|
508
|
+
*
|
|
509
|
+
* const res = await compact(
|
|
510
|
+
* { storage, currentJsonKey: "app/x/tenant/t/manifests/tickets/current.json" },
|
|
511
|
+
* { minEntriesToCompact: 100 },
|
|
512
|
+
* );
|
|
513
|
+
* if (res.written) console.log("folded", res.entriesFolded, "entries");
|
|
514
|
+
* ```
|
|
515
|
+
*/
|
|
516
|
+
const compact = (args, options) => compactInternal(args, options);
|
|
517
|
+
/**
|
|
518
|
+
* {@link compact}, typed to return the {@link InternalCompactResult}
|
|
519
|
+
* snapshot keys. Tests and in-repo tooling reach it via the
|
|
520
|
+
* `@baerly/server/_internal/testing` subpath.
|
|
521
|
+
*
|
|
522
|
+
* @internal
|
|
523
|
+
*/
|
|
524
|
+
const compactInternal = async (args, options = {}) => {
|
|
525
|
+
const { storage, currentJsonKey } = args;
|
|
526
|
+
const internal = options;
|
|
527
|
+
const maxPerRun = internal.maxEntriesPerRun ?? DEFAULT_MAX_PER_RUN;
|
|
528
|
+
const maxTailProbeGets = internal.maxTailProbeGets;
|
|
529
|
+
const minToCompact = options.minEntriesToCompact ?? DEFAULT_MIN_TO_COMPACT;
|
|
530
|
+
const ceilingBytes = internal.ceilingBytes ?? Number.POSITIVE_INFINITY;
|
|
531
|
+
const ceilingEntries = internal.ceilingEntries ?? Number.POSITIVE_INFINITY;
|
|
532
|
+
const { prefix: collectionPrefix, name: collectionName } = collectionFromCurrentJsonKey(currentJsonKey);
|
|
533
|
+
const requireSeqOption = (name, value) => requireIntegerOption("compact", collectionName, name, value, 0);
|
|
534
|
+
requireSeqOption("maxEntriesPerRun", maxPerRun);
|
|
535
|
+
requireSeqOption("knownTail", internal.knownTail ?? 0);
|
|
536
|
+
if (maxTailProbeGets !== void 0) requireIntegerOption("compact", collectionName, "maxTailProbeGets", maxTailProbeGets, 1);
|
|
537
|
+
const read = await readCurrentJson(storage, currentJsonKey, options.signal !== void 0 ? { signal: options.signal } : void 0);
|
|
538
|
+
if (read === null) return {
|
|
539
|
+
written: false,
|
|
540
|
+
skippedReason: "current-json-missing",
|
|
541
|
+
previousSnapshotKey: null,
|
|
542
|
+
logSeqStartBefore: 0,
|
|
543
|
+
logSeqStartAfter: 0,
|
|
544
|
+
entriesFolded: 0
|
|
545
|
+
};
|
|
546
|
+
const current = read.json;
|
|
547
|
+
const baseEtag = read.etag;
|
|
548
|
+
const logSeqStartBefore = logSeqStartOf(current);
|
|
549
|
+
const probeFloor = Math.max(logSeqStartBefore, current.tail_hint, internal.knownTail ?? 0);
|
|
550
|
+
let probe;
|
|
551
|
+
if (maxTailProbeGets === void 0) probe = {
|
|
552
|
+
kind: "exact",
|
|
553
|
+
...await probeTailFrom(storage, collectionPrefix, probeFloor, { ...options.signal !== void 0 && { signal: options.signal } }),
|
|
554
|
+
complete: true
|
|
555
|
+
};
|
|
556
|
+
else {
|
|
557
|
+
const foldReserve = (current.snapshot === null ? 0 : 1) + maxPerRun + 2;
|
|
558
|
+
const requestsLeft = getCurrentContext()?.ledger?.storageRemaining() ?? 1e400;
|
|
559
|
+
probe = await probeTailChunk(storage, collectionPrefix, probeFloor, {
|
|
560
|
+
...options.signal !== void 0 && { signal: options.signal },
|
|
561
|
+
cap: Math.max(0, Math.min(maxTailProbeGets, requestsLeft - foldReserve))
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
const tailIsExact = probe.kind === "exact";
|
|
565
|
+
const discoveredTail = probe.kind === "exact" ? probe.tail : probe.lowerBound;
|
|
566
|
+
const nextSeq = discoveredTail;
|
|
567
|
+
const available = nextSeq - logSeqStartBefore;
|
|
568
|
+
if (available < minToCompact) {
|
|
569
|
+
if (!tailIsExact && discoveredTail <= current.tail_hint) return {
|
|
570
|
+
written: false,
|
|
571
|
+
skippedReason: "probe-budget-checkpointed",
|
|
572
|
+
previousSnapshotKey: current.snapshot,
|
|
573
|
+
logSeqStartBefore,
|
|
574
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
575
|
+
entriesFolded: 0
|
|
576
|
+
};
|
|
577
|
+
if (!tailIsExact) {
|
|
578
|
+
const checkpoint = {
|
|
579
|
+
...current,
|
|
580
|
+
tail_hint: Math.max(current.tail_hint, discoveredTail)
|
|
581
|
+
};
|
|
582
|
+
assertCurrentJsonTransition(current, checkpoint, currentJsonKey);
|
|
583
|
+
const checkpointBody = encodeJsonBytes(checkpoint);
|
|
584
|
+
const checkpointCasOpts = {
|
|
585
|
+
ifMatch: baseEtag,
|
|
586
|
+
contentType: APPLICATION_JSON,
|
|
587
|
+
...options.signal !== void 0 && { signal: options.signal }
|
|
588
|
+
};
|
|
589
|
+
try {
|
|
590
|
+
await storage.put(currentJsonKey, checkpointBody, checkpointCasOpts);
|
|
591
|
+
} catch (error) {
|
|
592
|
+
if (isWriteConflict(error)) {
|
|
593
|
+
ctxMetrics().counter("db.compaction.cas_lost_total", 1, { collection: collectionName });
|
|
594
|
+
return {
|
|
595
|
+
written: false,
|
|
596
|
+
skippedReason: "cas-lost",
|
|
597
|
+
previousSnapshotKey: current.snapshot,
|
|
598
|
+
logSeqStartBefore,
|
|
599
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
600
|
+
entriesFolded: 0
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
throw error;
|
|
604
|
+
}
|
|
605
|
+
return {
|
|
606
|
+
written: false,
|
|
607
|
+
skippedReason: "probe-budget-checkpointed",
|
|
608
|
+
previousSnapshotKey: current.snapshot,
|
|
609
|
+
logSeqStartBefore,
|
|
610
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
611
|
+
entriesFolded: 0
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
if (available > 0) ctxMetrics().counter("db.compaction.below_min_total", 1, { collection: collectionName });
|
|
615
|
+
return {
|
|
616
|
+
written: false,
|
|
617
|
+
skippedReason: "below-min-threshold",
|
|
618
|
+
previousSnapshotKey: current.snapshot,
|
|
619
|
+
logSeqStartBefore,
|
|
620
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
621
|
+
entriesFolded: 0
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
let foldEnd = Math.min(nextSeq, logSeqStartBefore + maxPerRun);
|
|
625
|
+
const base = current.snapshot === null ? /* @__PURE__ */ new Map() : await loadSnapshotAsMap(storage, current.snapshot, collectionName, options.signal);
|
|
626
|
+
const fetched = await walkLogRangeWithBytes(storage, collectionPrefix, logSeqStartBefore, foldEnd, { signal: options.signal });
|
|
627
|
+
const own = (entry) => isOwnCollection(entry.collection, collectionName) && entry.doc_id !== void 0;
|
|
628
|
+
let applied = fetched.length;
|
|
629
|
+
let rowsStopped = false;
|
|
630
|
+
if (ceilingBytes !== Number.POSITIVE_INFINITY || ceilingEntries !== Number.POSITIVE_INFINITY) {
|
|
631
|
+
const projection = projectSnapshot({
|
|
632
|
+
...current,
|
|
633
|
+
snapshot_rows: base.size
|
|
634
|
+
}, base, collectionName);
|
|
635
|
+
let rowsEnd = 0;
|
|
636
|
+
let fits = 0;
|
|
637
|
+
for (const { entry } of fetched) {
|
|
638
|
+
if (own(entry)) {
|
|
639
|
+
if (projection.addsRow(entry.op, entry.doc_id, entry.after) && !(projection.rows < ceilingEntries)) {
|
|
640
|
+
rowsStopped = true;
|
|
641
|
+
break;
|
|
642
|
+
}
|
|
643
|
+
projection.apply(entry.op, entry.doc_id, entry.after);
|
|
644
|
+
}
|
|
645
|
+
rowsEnd++;
|
|
646
|
+
if (projection.bytesAt(logSeqStartBefore + rowsEnd) <= ceilingBytes) fits = rowsEnd;
|
|
647
|
+
}
|
|
648
|
+
applied = fits;
|
|
649
|
+
if (applied === 0 && fetched.length > 0) {
|
|
650
|
+
const byRows = rowsStopped && rowsEnd === 0;
|
|
651
|
+
return {
|
|
652
|
+
written: false,
|
|
653
|
+
skippedReason: "deferred",
|
|
654
|
+
deferred: true,
|
|
655
|
+
deferral: {
|
|
656
|
+
dimension: byRows ? "rows" : "bytes",
|
|
657
|
+
snapshotBytes: byRows ? current.snapshot_bytes : projection.bytesAt(logSeqStartBefore + rowsEnd),
|
|
658
|
+
snapshotRows: byRows ? base.size : projection.rows
|
|
659
|
+
},
|
|
660
|
+
previousSnapshotKey: current.snapshot,
|
|
661
|
+
logSeqStartBefore,
|
|
662
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
663
|
+
entriesFolded: 0
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
for (const { entry } of fetched.slice(0, applied)) {
|
|
668
|
+
if (!own(entry)) continue;
|
|
669
|
+
switch (entry.op) {
|
|
670
|
+
case "I":
|
|
671
|
+
case "U":
|
|
672
|
+
if (entry.after !== void 0) base.set(entry.doc_id, entry.after);
|
|
673
|
+
break;
|
|
674
|
+
case "D": base.delete(entry.doc_id);
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
foldEnd = logSeqStartBefore + applied;
|
|
678
|
+
const foldedSliceBytes = fetched.slice(0, applied).reduce((sum, f) => sum + f.bytes, 0);
|
|
679
|
+
const sortedDocs = Array.from(base.entries()).toSorted(([a], [b]) => {
|
|
680
|
+
if (a < b) return -1;
|
|
681
|
+
if (a > b) return 1;
|
|
682
|
+
return 0;
|
|
683
|
+
}).map(([id, body]) => ({
|
|
684
|
+
_id: id,
|
|
685
|
+
body
|
|
686
|
+
}));
|
|
687
|
+
const bodyBytes = encodeSnapshotBody({
|
|
688
|
+
schema_version: 1,
|
|
689
|
+
min_seq: 0,
|
|
690
|
+
max_seq: foldEnd,
|
|
691
|
+
collection: collectionName,
|
|
692
|
+
docs: sortedDocs
|
|
693
|
+
});
|
|
694
|
+
const bytesFit = bodyBytes.byteLength <= ceilingBytes;
|
|
695
|
+
if (!(bytesFit && base.size <= ceilingEntries)) return {
|
|
696
|
+
written: false,
|
|
697
|
+
skippedReason: "deferred",
|
|
698
|
+
deferred: true,
|
|
699
|
+
deferral: {
|
|
700
|
+
dimension: bytesFit ? "rows" : "bytes",
|
|
701
|
+
snapshotBytes: bodyBytes.byteLength,
|
|
702
|
+
snapshotRows: base.size
|
|
703
|
+
},
|
|
704
|
+
previousSnapshotKey: current.snapshot,
|
|
705
|
+
logSeqStartBefore,
|
|
706
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
707
|
+
entriesFolded: 0
|
|
708
|
+
};
|
|
709
|
+
const sha256 = await snapshotHash(bodyBytes);
|
|
710
|
+
const newKey = snapshotKey(collectionPrefix, 0, foldEnd, sha256);
|
|
711
|
+
const putOpts = {
|
|
712
|
+
contentType: APPLICATION_JSON,
|
|
713
|
+
...options.signal !== void 0 && { signal: options.signal }
|
|
714
|
+
};
|
|
715
|
+
await storage.put(newKey, bodyBytes, putOpts);
|
|
716
|
+
const entriesFolded = foldEnd - logSeqStartBefore;
|
|
717
|
+
const next = {
|
|
718
|
+
...current,
|
|
719
|
+
snapshot: newKey,
|
|
720
|
+
log_seq_start: foldEnd,
|
|
721
|
+
tail_hint: Math.max(current.tail_hint, discoveredTail),
|
|
722
|
+
snapshot_bytes: bodyBytes.byteLength,
|
|
723
|
+
snapshot_rows: base.size,
|
|
724
|
+
mean_entry_bytes: entriesFolded > 0 ? Math.round(foldedSliceBytes / entriesFolded) : current.mean_entry_bytes ?? 0
|
|
725
|
+
};
|
|
726
|
+
assertCurrentJsonTransition(current, next, currentJsonKey);
|
|
727
|
+
const nextBody = encodeJsonBytes(next);
|
|
728
|
+
const casOpts = {
|
|
729
|
+
ifMatch: baseEtag,
|
|
730
|
+
contentType: APPLICATION_JSON,
|
|
731
|
+
...options.signal !== void 0 && { signal: options.signal }
|
|
732
|
+
};
|
|
733
|
+
try {
|
|
734
|
+
await storage.put(currentJsonKey, nextBody, casOpts);
|
|
735
|
+
} catch (error) {
|
|
736
|
+
if (isWriteConflict(error)) {
|
|
737
|
+
ctxMetrics().counter("db.compaction.cas_lost_total", 1, { collection: collectionName });
|
|
738
|
+
return {
|
|
739
|
+
written: false,
|
|
740
|
+
skippedReason: "cas-lost",
|
|
741
|
+
previousSnapshotKey: current.snapshot,
|
|
742
|
+
newSnapshotKey: newKey,
|
|
743
|
+
logSeqStartBefore,
|
|
744
|
+
logSeqStartAfter: logSeqStartBefore,
|
|
745
|
+
entriesFolded: foldEnd - logSeqStartBefore
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
throw error;
|
|
749
|
+
}
|
|
750
|
+
const metrics = ctxMetrics();
|
|
751
|
+
metrics.histogram("db.compact.entries_folded", entriesFolded, { collection: collectionName });
|
|
752
|
+
if (tailIsExact) metrics.gauge("db.manifest.lag_window_depth", discoveredTail - foldEnd, { collection: collectionName });
|
|
753
|
+
return {
|
|
754
|
+
written: true,
|
|
755
|
+
previousSnapshotKey: current.snapshot,
|
|
756
|
+
newSnapshotKey: newKey,
|
|
757
|
+
logSeqStartBefore,
|
|
758
|
+
logSeqStartAfter: foldEnd,
|
|
759
|
+
entriesFolded
|
|
760
|
+
};
|
|
761
|
+
};
|
|
762
|
+
// ../server/src/log-retention.ts
|
|
763
|
+
/**
|
|
764
|
+
* Sequence-based log retention.
|
|
765
|
+
*
|
|
766
|
+
* Log keys are derivable, so retirement uses a half-open sequence range rather
|
|
767
|
+
* than LIST-based discovery. This module computes the range and owns the
|
|
768
|
+
* CAS-then-delete retirement pass. Every maintenance trigger runs it through
|
|
769
|
+
* the shared `retireLogs` helper in `maintenance.ts`: `runScheduledMaintenance`
|
|
770
|
+
* after its GC phase, and `runBoundedMaintenance` on the hard-GC early-return
|
|
771
|
+
* path and as its final step.
|
|
772
|
+
*/
|
|
773
|
+
/** Compute the budgeted prefix of the currently retirable log range. */
|
|
774
|
+
const computeRetirableRange = (current, opts) => {
|
|
775
|
+
const window = opts?.window ?? 1024;
|
|
776
|
+
const maxDeletes = opts?.maxDeletes ?? 20;
|
|
777
|
+
const liveFloor = logSeqStartOf(current);
|
|
778
|
+
const start = certifiedDeleteFloor(current);
|
|
779
|
+
const boundary = Math.min(liveFloor - window, liveFloor);
|
|
780
|
+
return {
|
|
781
|
+
start,
|
|
782
|
+
end: Math.max(start, Math.min(boundary, start + maxDeletes))
|
|
783
|
+
};
|
|
784
|
+
};
|
|
785
|
+
/**
|
|
786
|
+
* Delete a budgeted, computed slice of the stale log prefix and advance
|
|
787
|
+
* `log_delete_floor` to match. Sequence-based: log keys are derivable, so this
|
|
788
|
+
* never LISTs.
|
|
789
|
+
*
|
|
790
|
+
* **The authorized range is computed inside the CAS mutator, against the state
|
|
791
|
+
* the CAS validates — do not hoist it out.** {@link casUpdateCurrentJson}
|
|
792
|
+
* performs its own read and CASes against *that* etag, so a range computed from
|
|
793
|
+
* an earlier read is a plan the CAS never validated. Computing it in the mutator
|
|
794
|
+
* makes plan and validation atomic: whatever floor this call publishes is the
|
|
795
|
+
* floor its own DELETEs are authorized by. If the fresh state yields an empty
|
|
796
|
+
* range (a `restore --force` reseed lowered `log_seq_start` — the one writer
|
|
797
|
+
* allowed to, `restore.ts:191-206`), the pass aborts with `WriteConflict` and zero
|
|
798
|
+
* DELETEs rather than publishing a floor above the live one.
|
|
799
|
+
*
|
|
800
|
+
* **CAS before deleting, never after.** Every floor this function publishes
|
|
801
|
+
* satisfies `end <= log_seq_start - window`, where `window` is the *effective*
|
|
802
|
+
* window — {@link LOG_RETENTION_SEQ_WINDOW} by default, or `opts.window` when
|
|
803
|
+
* a caller overrides it (via {@link computeRetirableRange}, which also clamps
|
|
804
|
+
* a malformed stored floor per `docs/spec/sync-protocol.md` invariant 12).
|
|
805
|
+
* That bound holds only at the default window: `opts.window` is a test seam
|
|
806
|
+
* that can shrink or erase the folded entries the window keeps readable for
|
|
807
|
+
* a windowed commit and steepen the retirement rate it limits (this module's
|
|
808
|
+
* own `{window: 0}` tests push the floor to `log_seq_start` exactly).
|
|
809
|
+
* `log_delete_floor` is monotone under the transition validator regardless of
|
|
810
|
+
* `window`. So by the time a slot is physically deleted, a durable
|
|
811
|
+
* certificate that it is gone already exists — which is what makes
|
|
812
|
+
* `log_delete_floor` usable as a witness of "reclaimed" rather than "folded".
|
|
813
|
+
* A crash between the CAS and the end of the DELETE loop leaks the remaining
|
|
814
|
+
* slice permanently below the newly-advanced floor: deliberate, and the
|
|
815
|
+
* fail-safe direction this program picked — leak, never corruption.
|
|
816
|
+
*
|
|
817
|
+
* The writer-side counterpart lives on the commit path, not here.
|
|
818
|
+
* {@link LOG_RETENTION_SEQ_WINDOW} is a cost margin and a rate limit, never a
|
|
819
|
+
* fence — see its JSDoc. What stops a paused or uncertain writer's create from
|
|
820
|
+
* being acknowledged in a slot this pass has already retired is
|
|
821
|
+
* `Writer#assertCommitAboveDeleteFloor`, which re-reads the manifest after the
|
|
822
|
+
* committing create resolves and fails a sub-floor `seq` with
|
|
823
|
+
* `BaerlyError{code:"AmbiguousCommit"}`. See
|
|
824
|
+
* `docs/adr/002-ephemeral-coordination.md` § Closed paths for the two
|
|
825
|
+
* approaches that were tried and rejected, and why that check is what remains.
|
|
826
|
+
*
|
|
827
|
+
* @param opts.signal Threaded to every storage call this makes (the gate
|
|
828
|
+
* read, the CAS's own read + write, and each DELETE in the loop) — an
|
|
829
|
+
* already-aborted signal, or one aborted mid-call, rejects the in-flight
|
|
830
|
+
* {@link Storage} operation.
|
|
831
|
+
* @param opts.window Test seam that overrides {@link LOG_RETENTION_SEQ_WINDOW}.
|
|
832
|
+
* NOT a production tuning knob — do not promote this to an env var the way
|
|
833
|
+
* `BAERLY_MAINTENANCE_*` tunes the fold/GC budgets. Shrinking it (this
|
|
834
|
+
* module's own tests use `{window: 0}`) erases the pre-image reach and
|
|
835
|
+
* steepens the retirement rate described above; production call sites
|
|
836
|
+
* should omit it and take the default.
|
|
837
|
+
* @param opts.maxDeletes Test seam that overrides
|
|
838
|
+
* {@link LOG_RETENTION_MAX_DELETES_PER_TICK}. Same caveat as `opts.window`:
|
|
839
|
+
* a scoped override for exercising the budget boundary in tests, not a
|
|
840
|
+
* knob meant to reach a config surface.
|
|
841
|
+
* @throws BaerlyError{code:"WriteConflict"} if the floor CAS loses to another
|
|
842
|
+
* `current.json` writer, or if the fresh state authorizes nothing. The
|
|
843
|
+
* maintenance call sites' shared `retireLogs` helper (`maintenance.ts`)
|
|
844
|
+
* swallows exactly this code and re-throws everything else, matching how
|
|
845
|
+
* `WriteConflict` from `compact()` / `runGc()` is treated at every call site;
|
|
846
|
+
* returning it unwrapped here is what keeps that policy in one place.
|
|
847
|
+
* @throws BaerlyError{code:"Internal"} if the floor CAS completes without
|
|
848
|
+
* having authorized the range it published — a wiring tripwire, not a
|
|
849
|
+
* runtime state. `retireLogs` re-throws it into `runBoundedMaintenance`'s
|
|
850
|
+
* outer catch (`db.maintenance.unexpected_error_total`), so the failure is
|
|
851
|
+
* observable rather than silent.
|
|
852
|
+
*/
|
|
853
|
+
async function retireLogRange(storage, currentJsonKey, opts) {
|
|
854
|
+
const readOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
855
|
+
const read = await readCurrentJson(storage, currentJsonKey, readOpts);
|
|
856
|
+
if (read === null) return { deleted: 0 };
|
|
857
|
+
const gate = computeRetirableRange(read.json, opts);
|
|
858
|
+
if (gate.start >= gate.end) return { deleted: 0 };
|
|
859
|
+
let authorized;
|
|
860
|
+
const cas = await casUpdateCurrentJson(storage, currentJsonKey, (current) => {
|
|
861
|
+
authorized = computeRetirableRange(current, opts);
|
|
862
|
+
if (authorized.start >= authorized.end) throw new BaerlyError("WriteConflict", `retireLogRange: ${currentJsonKey} authorizes no retirable range at CAS time; a concurrent writer moved log_seq_start`);
|
|
863
|
+
return {
|
|
864
|
+
...current,
|
|
865
|
+
log_delete_floor: authorized.end
|
|
866
|
+
};
|
|
867
|
+
}, readOpts);
|
|
868
|
+
if (authorized === void 0 || logDeleteFloorOf(cas.json) !== authorized.end) {
|
|
869
|
+
const published = logDeleteFloorOf(cas.json);
|
|
870
|
+
const authorizedEnd = authorized === void 0 ? "none" : String(authorized.end);
|
|
871
|
+
throw new BaerlyError("Internal", `retireLogRange: ${currentJsonKey}: the floor CAS published ${String(published)} against authorized end ${authorizedEnd}; refusing to DELETE a range the CAS never validated`);
|
|
872
|
+
}
|
|
873
|
+
const range = authorized;
|
|
874
|
+
const { prefix: collectionPrefix } = collectionFromCurrentJsonKey(currentJsonKey);
|
|
875
|
+
await Promise.all(Array.from({ length: range.end - range.start }, (_, i) => storage.delete(logObjectKey(collectionPrefix, range.start + i), readOpts)));
|
|
876
|
+
return { deleted: range.end - range.start };
|
|
877
|
+
}
|
|
878
|
+
// ../server/src/gc.ts
|
|
879
|
+
/**
|
|
880
|
+
* GC mark + sweep.
|
|
881
|
+
*
|
|
882
|
+
* `runGc` is a single-pass garbage collector that:
|
|
883
|
+
* 1. Reads `current.json` (and bootstraps `gc/pending.json` on first
|
|
884
|
+
* run).
|
|
885
|
+
* 2. Marks new orphan candidates by LISTing the snapshot prefix and
|
|
886
|
+
* classifying each key.
|
|
887
|
+
* 3. Rescues candidates proven live, then sweeps due candidates that
|
|
888
|
+
* are safe to classify.
|
|
889
|
+
* 4. CAS-writes the updated `gc/pending.json`.
|
|
890
|
+
*
|
|
891
|
+
* Two-phase by design: each mark sits in `gc/pending.json` for a grace
|
|
892
|
+
* period (default 7 days, see {@link GC_GRACE_PERIOD_MILLIS}) before
|
|
893
|
+
* sweep may delete it. The grace bounds the worst plausible writer-retry
|
|
894
|
+
* window — a paused-process writer that resumes hours later still finds
|
|
895
|
+
* its idempotency anchor on the bucket. Sweep can also resolve a
|
|
896
|
+
* candidate OUT of the ledger without deleting it, when a generation or
|
|
897
|
+
* liveness recheck fails.
|
|
898
|
+
*
|
|
899
|
+
* **A pass is bounded by the grace it relies on.** Sweep DELETEs go out
|
|
900
|
+
* in batches of {@link GC_SWEEP_DELETE_BATCH_KEYS}. Before each batch,
|
|
901
|
+
* a pass whose sweep-gate `current.json` read is older than
|
|
902
|
+
* {@link GC_PASS_GRACE_FRACTION} of the grace issues no further DELETEs;
|
|
903
|
+
* the unissued candidates stay pending, still due, for a later pass.
|
|
904
|
+
* That bounds how stale a DELETE's liveness proof can be (sync-protocol
|
|
905
|
+
* invariant 14) without a lease or a scheduler.
|
|
906
|
+
*
|
|
907
|
+
* Stale log objects are reclaimed by a sequence window rather than a
|
|
908
|
+
* timed mark-and-sweep. See `log-retention.ts` for the computed-range
|
|
909
|
+
* deletion path.
|
|
910
|
+
*
|
|
911
|
+
* Unbounded by default — the run marks and sweeps the entire eligible
|
|
912
|
+
* set in one pass. Callers on a per-invocation request limit (Cloudflare
|
|
913
|
+
* Workers) opt INTO caps via the `CLOUDFLARE_FREE_TIER` /
|
|
914
|
+
* `CLOUDFLARE_PAID_TIER` profiles' mark and sweep knobs
|
|
915
|
+
* (`InternalRunGcOptions`, not on the public `RunGcOptions`); the
|
|
916
|
+
* per-pass worst cases are on `CLOUDFLARE_FREE_TIER`.
|
|
917
|
+
*
|
|
918
|
+
* One category of orphan:
|
|
919
|
+
* - `orphan-snapshot`: a canonical
|
|
920
|
+
* `<collectionPrefix>/snapshot/L9/...` key not equal to
|
|
921
|
+
* `current.snapshot`. Each compactor run replaces the pointer; the
|
|
922
|
+
* prior file becomes unreferenced. Discovery is scoped to this
|
|
923
|
+
* binary's owned `snapshot/L9/` namespace. Future L0-L8 layouts
|
|
924
|
+
* stay opaque to this binary and do not consume the bounded mark
|
|
925
|
+
* window. Opaque L9 keys are ignored during mark and evicted from
|
|
926
|
+
* legacy ledgers without a DELETE; no current writer emits one.
|
|
927
|
+
*
|
|
928
|
+
* **Mark window rule.** Every pass LISTs `snapshot/L9/` from the
|
|
929
|
+
* lexicographic start — no persisted cursor, so no orphan can sit
|
|
930
|
+
* behind a position the scan never revisits (a restore that deletes the
|
|
931
|
+
* ledger, a stale-generation drop, or a lost CAS simply re-marks on the
|
|
932
|
+
* next pass). The window is `maxMarks` plus every key the pass will
|
|
933
|
+
* certainly skip: the snapshot keys already in `gc/pending.json` and the
|
|
934
|
+
* one live `current.snapshot`. The loop stops after `maxMarks` NEW
|
|
935
|
+
* marks. Without the widening, older keys sort first and a cohort
|
|
936
|
+
* waiting out its 7-day grace fills a `maxMarks`-key window, capping
|
|
937
|
+
* marking at `maxMarks` per grace while folds keep orphaning snapshots.
|
|
938
|
+
* The widening is capped at {@link GC_MARK_LIST_PAGE_KEYS} (one full
|
|
939
|
+
* provider page) and never shrinks the window below `maxMarks`, so it
|
|
940
|
+
* adds no LIST page request beyond what `maxMarks` alone costs: one
|
|
941
|
+
* request per 1000 keys, more only if the provider truncates a page
|
|
942
|
+
* short. Marking only adds ledger entries; deletion authority stays with
|
|
943
|
+
* the sweep gate below.
|
|
944
|
+
*
|
|
945
|
+
* Progress: within the protocol-written L9 keyspace at most ONE key is
|
|
946
|
+
* permanently unmarkable (the live `current.snapshot`), so for any
|
|
947
|
+
* `maxMarks >= 1` a pass whose ledger has room marks an unpending orphan
|
|
948
|
+
* if one exists in the window. Opaque L9 keys (non-canonical names) are
|
|
949
|
+
* skipped but still occupy window slots; no current writer emits one.
|
|
950
|
+
*
|
|
951
|
+
* Ceiling: `GC_MAX_PENDING_CANDIDATES` (1000) bounds the in-flight
|
|
952
|
+
* cohort, so a collection reclaims at most ~1000 replaced snapshots per
|
|
953
|
+
* 7-day grace — about 143 folds/day. At the cap the window holds only
|
|
954
|
+
* pending keys and `mergeGcPending` drops appends past the cap, so
|
|
955
|
+
* marking stops until the oldest cohort comes due and is swept; a
|
|
956
|
+
* collection folding faster than that accumulates the excess.
|
|
957
|
+
*
|
|
958
|
+
* **`content/` is deliberately NOT a category.** Legacy
|
|
959
|
+
* `<collectionPrefix>/content/<sha>.json` side objects — written by
|
|
960
|
+
* v0.6.0, and still written by a v0.6.0 node during a mixed rollout —
|
|
961
|
+
* are inert: no current writer creates one, no reader opens one. GC
|
|
962
|
+
* never LISTs, reads, or deletes under that prefix, so they cost storage
|
|
963
|
+
* and nothing else, and an operator who wants the bytes back deletes the
|
|
964
|
+
* prefix directly (`docs/guide/backups.md`). An `orphan-content` entry
|
|
965
|
+
* found in a legacy ledger is EVICTED, not swept — see the sweep gate's
|
|
966
|
+
* legacy arm.
|
|
967
|
+
*
|
|
968
|
+
* CAS-lost on `gc/pending.json` is non-fatal: the DELETEs already
|
|
969
|
+
* issued are durable, so we return a successful result and the next
|
|
970
|
+
* pass will pick up any work this pass lost.
|
|
971
|
+
*
|
|
972
|
+
* **Not fault-isolated, by design.** Every storage call in Steps 1-5 —
|
|
973
|
+
* the `current.json`/`gc/pending.json` reads, the mark-phase `LIST`s,
|
|
974
|
+
* the sweep DELETEs — is uncaught within `runGc` itself; a throw from
|
|
975
|
+
* any of them aborts the whole pass. That's a deliberate fail-safe, not
|
|
976
|
+
* a gap to close with a `try`/`catch` around one call: Steps 3-4 re-LIST
|
|
977
|
+
* from scratch every pass rather than trusting partial progress, sweep
|
|
978
|
+
* DELETEs are idempotent, and the pending-ledger CAS in Step 5 hasn't
|
|
979
|
+
* happened yet when an earlier step throws — so an abort mid-pass loses
|
|
980
|
+
* no durable state and deletes nothing it shouldn't. This is fuzz-tested
|
|
981
|
+
* directly: `maintenance-crash-fuzz.test.ts` aborts the K-th storage op
|
|
982
|
+
* inside `runGc` and asserts the reader still returns every live row.
|
|
983
|
+
*/
|
|
984
|
+
const DEFAULT_MAX_MARKS = Number.MAX_SAFE_INTEGER;
|
|
985
|
+
const DEFAULT_MAX_SWEEPS = Number.MAX_SAFE_INTEGER;
|
|
986
|
+
const zeroGcResult = () => ({
|
|
987
|
+
marked: { orphan_snapshot: 0 },
|
|
988
|
+
swept: 0,
|
|
989
|
+
dropped: {
|
|
990
|
+
stale_generation: 0,
|
|
991
|
+
still_live: 0
|
|
992
|
+
},
|
|
993
|
+
pendingDepth: 0
|
|
994
|
+
});
|
|
995
|
+
/**
|
|
996
|
+
* Single GC pass — mark new orphans, sweep due-elapsed candidates,
|
|
997
|
+
* persist via CAS-update on `gc/pending.json`.
|
|
998
|
+
*
|
|
999
|
+
* Returns immediately if `current.json` is missing — there's nothing
|
|
1000
|
+
* to do until a writer has bootstrapped the collection.
|
|
1001
|
+
*
|
|
1002
|
+
* @example
|
|
1003
|
+
* ```ts
|
|
1004
|
+
* import { runGc } from "@baerly/server/_internal/maintenance";
|
|
1005
|
+
*
|
|
1006
|
+
* const r = await runGc({ storage, currentJsonKey });
|
|
1007
|
+
* console.log(`marked ${r.marked.orphan_snapshot} orphan snapshots, swept ${r.swept}`);
|
|
1008
|
+
* ```
|
|
1009
|
+
*/
|
|
1010
|
+
const runGc = async (args, options = {}) => {
|
|
1011
|
+
const { storage, currentJsonKey } = args;
|
|
1012
|
+
const internal = options;
|
|
1013
|
+
const grace = internal.graceMillis ?? 6048e5;
|
|
1014
|
+
const maxMarks = internal.maxMarksPerRun ?? DEFAULT_MAX_MARKS;
|
|
1015
|
+
const maxSweeps = internal.maxSweepsPerRun ?? DEFAULT_MAX_SWEEPS;
|
|
1016
|
+
const now = internal.now ?? (() => /* @__PURE__ */ new Date());
|
|
1017
|
+
const monotonicNow = internal.monotonicNow ?? (() => performance.now());
|
|
1018
|
+
/** Both clocks at one instant; see {@link elapsedSince}. */
|
|
1019
|
+
const stamp = () => ({
|
|
1020
|
+
wall: now().getTime(),
|
|
1021
|
+
mono: monotonicNow()
|
|
1022
|
+
});
|
|
1023
|
+
/** The larger of the two clocks' elapsed times, so neither can understate it. */
|
|
1024
|
+
const elapsedSince = (from) => Math.max(now().getTime() - from.wall, monotonicNow() - from.mono);
|
|
1025
|
+
const { prefix: collectionPrefix, name: collectionName } = collectionFromCurrentJsonKey(currentJsonKey);
|
|
1026
|
+
const pendingKey = gcPendingKey(collectionPrefix);
|
|
1027
|
+
const signal = options.signal;
|
|
1028
|
+
const signalOpts = signal !== void 0 ? { signal } : void 0;
|
|
1029
|
+
const currentReadAt = stamp();
|
|
1030
|
+
const cur = await readCurrentJson(storage, currentJsonKey, signalOpts);
|
|
1031
|
+
if (cur === null) return zeroGcResult();
|
|
1032
|
+
const current = cur.json;
|
|
1033
|
+
let pending = await readGcPending(storage, pendingKey, signalOpts);
|
|
1034
|
+
if (pending === null) {
|
|
1035
|
+
const initial = {
|
|
1036
|
+
schema_version: 1,
|
|
1037
|
+
candidates: [],
|
|
1038
|
+
last_swept_at: ""
|
|
1039
|
+
};
|
|
1040
|
+
try {
|
|
1041
|
+
pending = await createGcPending(storage, pendingKey, initial, signalOpts);
|
|
1042
|
+
} catch (error) {
|
|
1043
|
+
if (isWriteConflict(error)) {
|
|
1044
|
+
pending = await readGcPending(storage, pendingKey, signalOpts);
|
|
1045
|
+
if (pending === null) throw error;
|
|
1046
|
+
} else throw error;
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
const known = new Set(pending.json.candidates.map((c) => c.key));
|
|
1050
|
+
const markGeneration = current.generation !== void 0 ? { generation: current.generation } : {};
|
|
1051
|
+
const newCandidates = [];
|
|
1052
|
+
let markedOrphanSnapshot = 0;
|
|
1053
|
+
const snapshotPrefix = `${collectionPrefix}/snapshot/L9/`;
|
|
1054
|
+
const pendingSnapshotKeys = pending.json.candidates.filter((c) => c.key.startsWith(snapshotPrefix)).length;
|
|
1055
|
+
for await (const entry of storage.list(snapshotPrefix, listWindow(markWindowKeys(maxMarks, pendingSnapshotKeys), void 0, signal))) {
|
|
1056
|
+
if (markedOrphanSnapshot >= maxMarks) break;
|
|
1057
|
+
if (entry.key === current.snapshot) continue;
|
|
1058
|
+
if (parseMaxSeqFromCanonicalSnapshotKey(entry.key, collectionPrefix) === null) continue;
|
|
1059
|
+
if (known.has(entry.key)) continue;
|
|
1060
|
+
newCandidates.push({
|
|
1061
|
+
key: entry.key,
|
|
1062
|
+
due_at: computeDueAt(now, grace),
|
|
1063
|
+
reason: "orphan-snapshot",
|
|
1064
|
+
...markGeneration
|
|
1065
|
+
});
|
|
1066
|
+
markedOrphanSnapshot++;
|
|
1067
|
+
}
|
|
1068
|
+
const nowMs = now().getTime();
|
|
1069
|
+
const sweepCandidates = [...pending.json.candidates, ...newCandidates];
|
|
1070
|
+
const hasDueSnapshot = sweepCandidates.some((candidate) => candidate.reason === "orphan-snapshot" && Date.parse(candidate.due_at) <= nowMs);
|
|
1071
|
+
const freshReadAt = hasDueSnapshot ? stamp() : currentReadAt;
|
|
1072
|
+
const freshCurrent = hasDueSnapshot ? await readCurrentJson(storage, currentJsonKey, signalOpts) : void 0;
|
|
1073
|
+
const fenceCurrent = freshCurrent?.json ?? current;
|
|
1074
|
+
const fenceReadAt = freshCurrent?.json !== void 0 ? freshReadAt : currentReadAt;
|
|
1075
|
+
const fenceGeneration = fenceCurrent.generation ?? "-";
|
|
1076
|
+
const fenceFloor = logSeqStartOf(fenceCurrent);
|
|
1077
|
+
const rescuedKeys = /* @__PURE__ */ new Set();
|
|
1078
|
+
const staleGenerationKeys = /* @__PURE__ */ new Set();
|
|
1079
|
+
const evictedKeys = /* @__PURE__ */ new Set();
|
|
1080
|
+
for (const candidate of sweepCandidates) {
|
|
1081
|
+
if (candidate.reason === "orphan-content") {
|
|
1082
|
+
evictedKeys.add(candidate.key);
|
|
1083
|
+
continue;
|
|
1084
|
+
}
|
|
1085
|
+
if (candidate.reason === "stale-log" && parseSeqFromCanonicalLogKey(candidate.key, collectionPrefix) === null) {
|
|
1086
|
+
evictedKeys.add(candidate.key);
|
|
1087
|
+
continue;
|
|
1088
|
+
}
|
|
1089
|
+
if (candidate.reason === "orphan-snapshot" && parseMaxSeqFromCanonicalSnapshotKey(candidate.key, collectionPrefix) === null) {
|
|
1090
|
+
evictedKeys.add(candidate.key);
|
|
1091
|
+
continue;
|
|
1092
|
+
}
|
|
1093
|
+
if ((candidate.generation ?? "-") !== fenceGeneration) {
|
|
1094
|
+
staleGenerationKeys.add(candidate.key);
|
|
1095
|
+
continue;
|
|
1096
|
+
}
|
|
1097
|
+
if (candidate.reason === "orphan-snapshot" && candidate.key === freshCurrent?.json.snapshot) {
|
|
1098
|
+
rescuedKeys.add(candidate.key);
|
|
1099
|
+
continue;
|
|
1100
|
+
}
|
|
1101
|
+
if (candidate.reason === "stale-log") {
|
|
1102
|
+
const seq = parseSeqFromCanonicalLogKey(candidate.key, collectionPrefix);
|
|
1103
|
+
if (seq !== null && seq >= fenceFloor) rescuedKeys.add(candidate.key);
|
|
1104
|
+
continue;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
const droppedKeys = /* @__PURE__ */ new Set([
|
|
1108
|
+
...staleGenerationKeys,
|
|
1109
|
+
...rescuedKeys,
|
|
1110
|
+
...evictedKeys
|
|
1111
|
+
]);
|
|
1112
|
+
const toSweep = [];
|
|
1113
|
+
const remaining = [];
|
|
1114
|
+
for (const cand of sweepCandidates) {
|
|
1115
|
+
if (droppedKeys.has(cand.key)) continue;
|
|
1116
|
+
if (toSweep.length < maxSweeps && Date.parse(cand.due_at) <= nowMs) {
|
|
1117
|
+
if (cand.reason === "orphan-snapshot") {
|
|
1118
|
+
const maxSeq = parseMaxSeqFromCanonicalSnapshotKey(cand.key, collectionPrefix);
|
|
1119
|
+
if (maxSeq === null || maxSeq >= (freshCurrent?.json.log_seq_start ?? 0)) {
|
|
1120
|
+
remaining.push(cand);
|
|
1121
|
+
continue;
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
toSweep.push(cand);
|
|
1125
|
+
} else remaining.push(cand);
|
|
1126
|
+
}
|
|
1127
|
+
const sweepBudgetMillis = grace * GC_PASS_GRACE_FRACTION;
|
|
1128
|
+
const swept = [];
|
|
1129
|
+
for (let start = 0; start < toSweep.length; start += 100) {
|
|
1130
|
+
if (grace > 0 && elapsedSince(fenceReadAt) > sweepBudgetMillis) break;
|
|
1131
|
+
const batch = toSweep.slice(start, start + 100);
|
|
1132
|
+
await Promise.all(batch.map((c) => storage.delete(c.key, signalOpts)));
|
|
1133
|
+
swept.push(...batch);
|
|
1134
|
+
}
|
|
1135
|
+
const abandonedCount = toSweep.length - swept.length;
|
|
1136
|
+
const sweptKeys = /* @__PURE__ */ new Set([...swept.map((candidate) => candidate.key), ...droppedKeys]);
|
|
1137
|
+
const lastSweptAt = swept.length > 0 ? now().toISOString() : "";
|
|
1138
|
+
const markedSummary = { orphan_snapshot: markedOrphanSnapshot };
|
|
1139
|
+
let pendingDepth;
|
|
1140
|
+
try {
|
|
1141
|
+
pendingDepth = (await casUpdateGcPending(storage, pendingKey, (latest) => mergeGcPending(latest, {
|
|
1142
|
+
sweptKeys,
|
|
1143
|
+
newCandidates,
|
|
1144
|
+
lastSweptAt,
|
|
1145
|
+
maxCandidates: GC_MAX_PENDING_CANDIDATES
|
|
1146
|
+
}), signalOpts)).json.candidates.length;
|
|
1147
|
+
} catch (error) {
|
|
1148
|
+
if (isWriteConflict(error)) pendingDepth = remaining.length + abandonedCount;
|
|
1149
|
+
else throw error;
|
|
1150
|
+
}
|
|
1151
|
+
const labels = { collection: collectionName };
|
|
1152
|
+
const metrics = ctxMetrics();
|
|
1153
|
+
metrics.gauge("db.orphan.candidate_count", pendingDepth, labels);
|
|
1154
|
+
metrics.gauge("db.gc.entries_swept_per_second", swept.length, labels);
|
|
1155
|
+
if (swept.length > 0) {
|
|
1156
|
+
const byReason = /* @__PURE__ */ new Map();
|
|
1157
|
+
for (const c of swept) byReason.set(c.reason, (byReason.get(c.reason) ?? 0) + 1);
|
|
1158
|
+
for (const [reason, count] of byReason) metrics.counter("db.gc.swept_total", count, {
|
|
1159
|
+
collection: collectionName,
|
|
1160
|
+
reason
|
|
1161
|
+
});
|
|
1162
|
+
}
|
|
1163
|
+
if (staleGenerationKeys.size > 0) metrics.counter("db.gc.dropped_total", staleGenerationKeys.size, {
|
|
1164
|
+
collection: collectionName,
|
|
1165
|
+
cause: "stale-generation"
|
|
1166
|
+
});
|
|
1167
|
+
if (rescuedKeys.size > 0) metrics.counter("db.gc.dropped_total", rescuedKeys.size, {
|
|
1168
|
+
collection: collectionName,
|
|
1169
|
+
cause: "still-live"
|
|
1170
|
+
});
|
|
1171
|
+
return {
|
|
1172
|
+
marked: markedSummary,
|
|
1173
|
+
swept: swept.length,
|
|
1174
|
+
dropped: {
|
|
1175
|
+
stale_generation: staleGenerationKeys.size,
|
|
1176
|
+
still_live: rescuedKeys.size
|
|
1177
|
+
},
|
|
1178
|
+
pendingDepth
|
|
1179
|
+
};
|
|
1180
|
+
};
|
|
1181
|
+
/**
|
|
1182
|
+
* `Storage.list` options for one bounded rotation window: at most
|
|
1183
|
+
* `maxKeys` keys, resuming strictly after `cursor` when the ledger
|
|
1184
|
+
* carries one. `maxKeys` is a hard cross-page total on every adapter
|
|
1185
|
+
* (`docs/spec/storage-compatibility.md`), which is what makes
|
|
1186
|
+
* "yielded fewer than we asked for ⇒ end of keyspace" a sound wrap
|
|
1187
|
+
* test. `startAfter` is strict-greater on the key VALUE, so a cursor
|
|
1188
|
+
* naming a key this pass already deleted still resumes correctly —
|
|
1189
|
+
* never probe whether it still exists.
|
|
1190
|
+
*/
|
|
1191
|
+
const listWindow = (maxKeys, cursor, signal) => ({
|
|
1192
|
+
maxKeys,
|
|
1193
|
+
...cursor !== void 0 && { startAfter: cursor },
|
|
1194
|
+
...signal !== void 0 && { signal }
|
|
1195
|
+
});
|
|
1196
|
+
/**
|
|
1197
|
+
* `maxKeys` for the orphan-snapshot mark LIST: `maxMarks` plus every key
|
|
1198
|
+
* the pass is certain to skip without marking — the `pendingSnapshotKeys`
|
|
1199
|
+
* already in the ledger under `snapshot/L9/` and the one live
|
|
1200
|
+
* `current.snapshot`. Without the widening, a ledger cohort waiting out
|
|
1201
|
+
* its grace fills the lex-first window and marking stalls at `maxMarks`
|
|
1202
|
+
* per grace period.
|
|
1203
|
+
*
|
|
1204
|
+
* Capped at {@link GC_MARK_LIST_PAGE_KEYS} so the widening never adds a
|
|
1205
|
+
* full LIST page: a pass with `maxMarks` at or under one page fits one
|
|
1206
|
+
* page request unless the provider truncates short. The cap is never
|
|
1207
|
+
* below `maxMarks`, so a bounded pass scans at least what it always did.
|
|
1208
|
+
*/
|
|
1209
|
+
const markWindowKeys = (maxMarks, pendingSnapshotKeys) => Math.max(maxMarks, Math.min(maxMarks + pendingSnapshotKeys + 1, GC_MARK_LIST_PAGE_KEYS));
|
|
1210
|
+
/**
|
|
1211
|
+
* Parse the exact canonical `log/<seq>.json` key for this collection.
|
|
1212
|
+
* Returns `null` for another prefix, malformed or noncanonical decimal,
|
|
1213
|
+
* or a sequence outside JavaScript's safe-integer range.
|
|
1214
|
+
*/
|
|
1215
|
+
const parseSeqFromCanonicalLogKey = (key, collectionPrefix) => {
|
|
1216
|
+
const logPrefix = `${collectionPrefix}/log/`;
|
|
1217
|
+
if (!key.startsWith(logPrefix)) return null;
|
|
1218
|
+
const match = /^(\d+)\.json$/.exec(key.slice(logPrefix.length));
|
|
1219
|
+
if (match === null) return null;
|
|
1220
|
+
const seq = Number(match[1]);
|
|
1221
|
+
return Number.isSafeInteger(seq) && seq >= 0 && logObjectKey(collectionPrefix, seq) === key ? seq : null;
|
|
1222
|
+
};
|
|
1223
|
+
/**
|
|
1224
|
+
* Parse the exclusive `max_seq` from the exact key shape emitted by
|
|
1225
|
+
* `snapshotKey`. Unknown levels/layouts and invalid ranges stay opaque: GC
|
|
1226
|
+
* never deletes the object and never retains its candidate in the bounded
|
|
1227
|
+
* ledger.
|
|
1228
|
+
*/
|
|
1229
|
+
const parseMaxSeqFromCanonicalSnapshotKey = (key, collectionPrefix) => {
|
|
1230
|
+
const snapshotPrefix = `${collectionPrefix}/snapshot/L9/`;
|
|
1231
|
+
if (!key.startsWith(snapshotPrefix)) return null;
|
|
1232
|
+
const match = /^(\d{12})-(\d{12})-[0-9a-f]{64}\.json$/.exec(key.slice(snapshotPrefix.length));
|
|
1233
|
+
if (match === null) return null;
|
|
1234
|
+
const minSeq = Number(match[1]);
|
|
1235
|
+
const maxSeq = Number(match[2]);
|
|
1236
|
+
return minSeq <= maxSeq ? maxSeq : null;
|
|
1237
|
+
};
|
|
1238
|
+
/**
|
|
1239
|
+
* Anchor `due_at` on the MARK: `now() + graceMs`. Grace measures the
|
|
1240
|
+
* writer-retry window from when we judged a key dead, which is
|
|
1241
|
+
* unrelated to when the object was written.
|
|
1242
|
+
*
|
|
1243
|
+
* Never anchor on the object's own timestamp. Doing so made the
|
|
1244
|
+
* effective grace `max(0, graceMs − object age)` — zero past the
|
|
1245
|
+
* 7-day default, for exactly the old objects GC marks. It read a
|
|
1246
|
+
* `lastModified` that only the S3/GCS/R2 adapters populated, so the
|
|
1247
|
+
* suite's backends showed a full window and no test could see it.
|
|
1248
|
+
* `StorageListEntry` no longer carries a timestamp; `gc.test.ts`
|
|
1249
|
+
* still pins the behaviour against an entry that fakes one.
|
|
1250
|
+
*
|
|
1251
|
+
* Called per candidate, not hoisted per pass, so the local-clock
|
|
1252
|
+
* backends keep their exact previous horizon.
|
|
1253
|
+
*/
|
|
1254
|
+
const computeDueAt = (now, graceMs) => new Date(now().getTime() + graceMs).toISOString();
|
|
1255
|
+
// ../server/src/maintenance.ts
|
|
1256
|
+
/**
|
|
1257
|
+
* Maintenance passes over one collection. `runScheduledMaintenance` is the
|
|
1258
|
+
* published opt-in scheduled pass (addressed by collection, sized by host
|
|
1259
|
+
* tier); `runMaintenancePass` is the internal composition it resolves to,
|
|
1260
|
+
* which the adapters, CLI, bench and tests drive with per-phase knobs;
|
|
1261
|
+
* `runBoundedMaintenance` is the in-band write-tick runner.
|
|
1262
|
+
*
|
|
1263
|
+
* Every pass is a single attempt: `compact()` and `runGc()` are
|
|
1264
|
+
* CAS-protected, so a pass cut short is retried safely by the next one.
|
|
1265
|
+
* The caller schedules the next pass; nothing here loops or retries.
|
|
1266
|
+
*/
|
|
1267
|
+
/**
|
|
1268
|
+
* One log-retirement pass: run {@link retireLogRange} and emit its
|
|
1269
|
+
* counter. Shared by every maintenance trigger that retires stale logs
|
|
1270
|
+
* (`runScheduledMaintenance` and both retirement points inside
|
|
1271
|
+
* `runBoundedMaintenance`), so the metric name, option threading, and
|
|
1272
|
+
* error policy cannot drift between call sites.
|
|
1273
|
+
*
|
|
1274
|
+
* Error policy — deliberately NOT a bare swallow: `WriteConflict` (the floor
|
|
1275
|
+
* CAS lost to another writer, or a concurrent fold moved `log_seq_start`)
|
|
1276
|
+
* is routine contention, swallowed here and retried by the next pass.
|
|
1277
|
+
* Anything else is re-thrown for the caller's own error path, matching
|
|
1278
|
+
* how `compact()` / `runGc()` failures are treated at every call site:
|
|
1279
|
+
* `runBoundedMaintenance`'s outer catch counts
|
|
1280
|
+
* `db.maintenance.unexpected_error_total` and logs the stack, and
|
|
1281
|
+
* `runScheduledMaintenance` propagates per its documented "errors
|
|
1282
|
+
* propagate" contract. Retirement is the only mechanism bounding the
|
|
1283
|
+
* steady-state log-object count — a persistent unexpected failure must
|
|
1284
|
+
* not fail silently while objects accumulate unbounded.
|
|
1285
|
+
*/
|
|
1286
|
+
const retireLogs = async (storage, currentJsonKey, opts) => {
|
|
1287
|
+
const { name: collection } = collectionFromCurrentJsonKey(currentJsonKey);
|
|
1288
|
+
try {
|
|
1289
|
+
const retired = await retireLogRange(storage, currentJsonKey, {
|
|
1290
|
+
...opts?.logRetention,
|
|
1291
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
1292
|
+
});
|
|
1293
|
+
ctxMetrics().counter("db.log_retention.deleted_total", retired.deleted, { collection });
|
|
1294
|
+
return retired.deleted;
|
|
1295
|
+
} catch (error) {
|
|
1296
|
+
if (isWriteConflict(error)) return 0;
|
|
1297
|
+
throw error;
|
|
1298
|
+
}
|
|
1299
|
+
};
|
|
1300
|
+
const profileToScheduledOptions = (profile, compactMaxTailProbeGets, minEntriesToCompact = 1) => ({
|
|
1301
|
+
compact: {
|
|
1302
|
+
maxEntriesPerRun: profile.maxFoldEntriesPerPass,
|
|
1303
|
+
minEntriesToCompact,
|
|
1304
|
+
...compactMaxTailProbeGets !== void 0 && { maxTailProbeGets: compactMaxTailProbeGets }
|
|
1305
|
+
},
|
|
1306
|
+
gc: {
|
|
1307
|
+
maxMarksPerRun: profile.gcMaxMarks,
|
|
1308
|
+
maxSweepsPerRun: profile.gcMaxSweeps
|
|
1309
|
+
}
|
|
1310
|
+
});
|
|
1311
|
+
profileToScheduledOptions(MAINTENANCE_PROFILE_CF_FREE, 25, MAINTENANCE_PROFILE_CF_FREE.maxFoldEntriesPerPass);
|
|
1312
|
+
profileToScheduledOptions(MAINTENANCE_PROFILE_CF_PAID, CF_PAID_COMPACT_TAIL_PROBE_GETS);
|
|
1313
|
+
/**
|
|
1314
|
+
* Floor-based GC-cadence boundary crossing. Every commit advances the
|
|
1315
|
+
* writer-observed tail by one logical slot; `tail_hint` remains a
|
|
1316
|
+
* lower bound and may be refreshed later by maintenance. The floor test
|
|
1317
|
+
* trips on any interval boundary inside `(prevSeq, nextSeq]`, so it
|
|
1318
|
+
* stays correct whether the observed/probed tail advances by one or
|
|
1319
|
+
* catches up by a larger step.
|
|
1320
|
+
*/
|
|
1321
|
+
const crossesGcBoundary = (prevSeq, nextSeq, interval) => Math.floor(prevSeq / interval) !== Math.floor(nextSeq / interval);
|
|
1322
|
+
/**
|
|
1323
|
+
* The write-tick runner's pre-fold projection off STORED figures: the
|
|
1324
|
+
* snapshot it would load is within `C` bytes and `E` rows. It does not
|
|
1325
|
+
* project the pass: `compact()` counts real rows as it applies the range
|
|
1326
|
+
* and folds the longest prefix that stays within `E`, so an update/delete
|
|
1327
|
+
* tail keeps folding at the ceiling and only an entry that would add a row
|
|
1328
|
+
* past `E` stops it. The runner, `baerly inspect` and the backlog bench
|
|
1329
|
+
* share this predicate. Both comparisons are written so a NaN ceiling
|
|
1330
|
+
* fails closed.
|
|
1331
|
+
*/
|
|
1332
|
+
const isFoldViable = (snapshotBytes, snapshotRows, ceilings) => snapshotBytes <= ceilings.maxFoldBytes && snapshotRows <= ceilings.maxFoldRows;
|
|
1333
|
+
/**
|
|
1334
|
+
* Resolve the serving host's fold ceilings: `profile` (absent ⇒
|
|
1335
|
+
* {@link MAINTENANCE_PROFILE_CF_FREE}) with `C` overridden by
|
|
1336
|
+
* `BAERLY_MAINTENANCE_MAX_FOLD_BYTES` (`maxFoldBytes`). `E` has no
|
|
1337
|
+
* override. The write-tick runner and growth refusal (`capacity.ts`) both
|
|
1338
|
+
* resolve through here, so a refusal and a fold never see different
|
|
1339
|
+
* ceilings.
|
|
1340
|
+
*/
|
|
1341
|
+
const resolveFoldCeilings = (profile, maxFoldBytes) => {
|
|
1342
|
+
const resolved = profile ?? MAINTENANCE_PROFILE_CF_FREE;
|
|
1343
|
+
return {
|
|
1344
|
+
maxFoldBytes: maxFoldBytes ?? resolved.maxFoldBytes,
|
|
1345
|
+
maxFoldRows: resolved.maxFoldRows
|
|
1346
|
+
};
|
|
1347
|
+
};
|
|
1348
|
+
/**
|
|
1349
|
+
* Derived live-tail byte size — the ratio TRIGGER input that replaces the
|
|
1350
|
+
* formerly-stored exact byte count. `(observedTail − log_seq_start) ×
|
|
1351
|
+
* mean_entry_bytes`. `observedTail` is a param so write-tick callers can
|
|
1352
|
+
* pass the writer-observed tail and scheduled callers can pass a
|
|
1353
|
+
* probe-discovered true tail. Cold-start (no mean yet) falls back to
|
|
1354
|
+
* {@link MAINTENANCE_COLD_START_ENTRY_BYTES}, never 0 (see that constant).
|
|
1355
|
+
* TRIGGER-only — structurally barred from `foldViable` (the exact-bytes fold
|
|
1356
|
+
* ceiling).
|
|
1357
|
+
*/
|
|
1358
|
+
const estimateTailBytes = (current, observedTail) => Math.max(0, observedTail - current.log_seq_start) * (current.mean_entry_bytes ?? 128);
|
|
1359
|
+
/** Derived live-tail bytes over the snapshot's bytes (floored at {@link MAINTENANCE_MIN_LIVE_BYTES}). */
|
|
1360
|
+
const tailRatio = (current, observedTail) => estimateTailBytes(current, observedTail) / Math.max(current.snapshot_bytes, MAINTENANCE_MIN_LIVE_BYTES);
|
|
1361
|
+
/**
|
|
1362
|
+
* The cheap DISPATCH gate the writer calls after a commit — zero
|
|
1363
|
+
* storage ops; it reads the post-commit {@link CurrentJson} already in
|
|
1364
|
+
* scope. Ratio-OR-boundary, WITHOUT the entry floor (the floor is part
|
|
1365
|
+
* of the fold-trigger Gate 1 checked inside {@link runBoundedMaintenance}).
|
|
1366
|
+
* Returning `false` lets the writer skip the dispatch entirely. Ratio
|
|
1367
|
+
* numerator is the DERIVED {@link estimateTailBytes};
|
|
1368
|
+
* `observedTail` threads from the caller (the writer's in-memory observed
|
|
1369
|
+
* tail `seq+1` under single-write commit). Both the boundary check and the
|
|
1370
|
+
* ratio key off `observedTail`, since the stored `tail_hint` is now only a
|
|
1371
|
+
* non-authoritative lower bound (compactor-advanced).
|
|
1372
|
+
*/
|
|
1373
|
+
const shouldFireMaintenance = (current, prevSeq, gcInterval, observedTail) => crossesGcBoundary(prevSeq, observedTail, gcInterval) || tailRatio(current, observedTail) >= 1;
|
|
1374
|
+
/**
|
|
1375
|
+
* Human-readable line for a fold-wall deferral. Names both ceilings, which
|
|
1376
|
+
* arm tripped, that `log_seq_start` and log retention are frozen, and the
|
|
1377
|
+
* remedy that actually applies to that arm — the bytes override does
|
|
1378
|
+
* nothing for rows.
|
|
1379
|
+
*/
|
|
1380
|
+
const foldDeferredWarning = (f) => {
|
|
1381
|
+
const which = f.gate === "rebuild" ? "rebuilt" : "stored";
|
|
1382
|
+
const rowsTripped = f.gate === "rebuild" ? `the rows arm tripped (the next log entry adds a row past E, or the rebuilt snapshot is over E)` : `the rows arm tripped (snapshot_rows > E)`;
|
|
1383
|
+
const tripped = f.dimension === "bytes" ? `the bytes arm tripped (snapshot_bytes > C)` : rowsTripped;
|
|
1384
|
+
const remedy = f.dimension === "bytes" ? `On Workers Paid set BAERLY_MAINTENANCE_PROFILE=cf-paid (C=${MAINTENANCE_PROFILE_CF_PAID.maxFoldBytes}); on paid Cloudflare / Node you can also raise BAERLY_MAINTENANCE_MAX_FOLD_BYTES — but on Cloudflare a cap above the configured default may exceed an invocation's CPU or memory resources, so such folds may not land. Otherwise, move to a database service (D1 / Postgres).` : `E has no env override, and BAERLY_MAINTENANCE_MAX_FOLD_BYTES does not apply to rows. On Cloudflare, move to Workers Paid and set BAERLY_MAINTENANCE_PROFILE=cf-paid (E=${MAINTENANCE_PROFILE_CF_PAID.maxFoldRows}); on Node (E=${MAINTENANCE_PROFILE_NODE.maxFoldRows}) or past the paid ceiling, split the collection or move to a database service (D1 / Postgres).`;
|
|
1385
|
+
return `baerly-storage: collection "${f.collection}" has stopped compacting — the ${which} snapshot is ${f.snapshot_bytes} bytes / ${f.snapshot_rows} rows against this host's fold ceiling C=${f.ceiling_bytes} bytes, E=${f.ceiling_rows} rows, and ${tripped}. Until it folds again log_seq_start stays at ${f.log_seq_start}, log retention is paused, the live tail (${f.live_log_tail} entries) and the bucket grow without bound, and every read replays the tail. ${remedy} See docs/about/graduation.md.`;
|
|
1386
|
+
};
|
|
1387
|
+
/**
|
|
1388
|
+
* The write-tick maintenance runner: a bounded, single-attempt
|
|
1389
|
+
* composition of {@link compact} and {@link runGc} for one collection,
|
|
1390
|
+
* sized by default to a conservative Cloudflare free-tier CPU budget.
|
|
1391
|
+
*
|
|
1392
|
+
* INVARIANT: a {@link MaintenanceProfile} changes only maintenance rate +
|
|
1393
|
+
* defer threshold — never data/API/query/wire/correctness; reads stay pure.
|
|
1394
|
+
* See `maintenance-profile-equivalence.test.ts`.
|
|
1395
|
+
*
|
|
1396
|
+
* Never throws — the write-tick caller must never see a maintenance
|
|
1397
|
+
* failure. CAS contention is swallowed quietly, and so is a call the
|
|
1398
|
+
* invocation's request ledger refused (each phase is sized from the
|
|
1399
|
+
* ledger's remainder first; see the request-budget note inside, and
|
|
1400
|
+
* `docs/spec/sync-protocol.md` "Per-invocation request budget"); an over-ceiling deferral
|
|
1401
|
+
* is not an error but is signalled loudly (see `signalFoldDeferred`
|
|
1402
|
+
* inside); unexpected throws bump
|
|
1403
|
+
* `db.maintenance.unexpected_error_total`, log a stack, and are
|
|
1404
|
+
* swallowed.
|
|
1405
|
+
*
|
|
1406
|
+
* Control flow (fold-priority with a stateless GC-starvation guard):
|
|
1407
|
+
* 0. `disabled` → skip fold/GC, but still allow the rate-limited
|
|
1408
|
+
* tail_hint refresh that bounds read/write forward-probes.
|
|
1409
|
+
* 1. Read `current.json` once.
|
|
1410
|
+
* 2. Hard-GC starvation boundary (`"single"` only) → GC slice, skip fold.
|
|
1411
|
+
* 3. Gate 1 (ratio AND entry floor, or a forced tick) trips → fold if the pre-fold
|
|
1412
|
+
* projection is viable, else defer and signal. On `"single"` a fold
|
|
1413
|
+
* attempt is this tick's one phase — including one that compact()
|
|
1414
|
+
* deferred after rebuilding, which signals the same way and ends the
|
|
1415
|
+
* tick. A pre-fold defer falls through to Step 4.
|
|
1416
|
+
* 4. Otherwise (or on `"both"`) → GC iff the cadence boundary crossed.
|
|
1417
|
+
*
|
|
1418
|
+
* Steps 0, 2, 3-defer, and 4 all take the rate-limited `tail_hint`
|
|
1419
|
+
* refresh, but the tick publishes the hint AT MOST ONCE — a landed fold
|
|
1420
|
+
* (Step-7 CAS) and the refresh itself both count as that publication.
|
|
1421
|
+
* Ordering matters: Step 3's defer refreshes and then falls through to
|
|
1422
|
+
* Step 4, so without the shared guard that one tick would CAS
|
|
1423
|
+
* `current.json` twice with identical bytes.
|
|
1424
|
+
*/
|
|
1425
|
+
const runBoundedMaintenance = async (args, options) => {
|
|
1426
|
+
const { storage, currentJsonKey, prevSeq } = args;
|
|
1427
|
+
const profile = options?.profile ?? MAINTENANCE_PROFILE_CF_FREE;
|
|
1428
|
+
const { maxFoldEntriesPerPass, gcMaxMarks, gcMaxSweeps, gcInterval } = profile;
|
|
1429
|
+
const forceFold = args.forceFold === true;
|
|
1430
|
+
const minEntriesToCompact = forceFold ? 1 : options?.minEntriesToCompact ?? 50;
|
|
1431
|
+
const phasesPerTick = options?.phasesPerTick ?? "single";
|
|
1432
|
+
const signal = options?.signal;
|
|
1433
|
+
const ceilings = resolveFoldCeilings(profile, args.maxFoldBytes);
|
|
1434
|
+
const C = ceilings.maxFoldBytes;
|
|
1435
|
+
const E = ceilings.maxFoldRows;
|
|
1436
|
+
const { prefix: collectionPrefix, name: collection } = collectionFromCurrentJsonKey(currentJsonKey);
|
|
1437
|
+
const gcOpts = {
|
|
1438
|
+
maxMarksPerRun: gcMaxMarks,
|
|
1439
|
+
maxSweepsPerRun: gcMaxSweeps,
|
|
1440
|
+
...signal !== void 0 && { signal },
|
|
1441
|
+
...options?.gcGraceMillis !== void 0 && { graceMillis: options.gcGraceMillis },
|
|
1442
|
+
...options?.now !== void 0 && { now: options.now }
|
|
1443
|
+
};
|
|
1444
|
+
const ledger = getCurrentContext()?.ledger;
|
|
1445
|
+
const requestsLeft = () => ledger?.storageRemaining() ?? 1e400;
|
|
1446
|
+
const skipForBudget = (phase) => {
|
|
1447
|
+
ctxMetrics().counter("db.maintenance.request_budget_skipped_total", 1, {
|
|
1448
|
+
collection,
|
|
1449
|
+
phase
|
|
1450
|
+
});
|
|
1451
|
+
};
|
|
1452
|
+
const retireWithinBudget = async () => {
|
|
1453
|
+
const deletes = Math.floor((requestsLeft() - 7) / 3);
|
|
1454
|
+
if (deletes < 1) {
|
|
1455
|
+
skipForBudget("retire");
|
|
1456
|
+
return;
|
|
1457
|
+
}
|
|
1458
|
+
const logRetention = options?.logRetention;
|
|
1459
|
+
const maxDeletes = logRetention?.maxDeletes ?? 20;
|
|
1460
|
+
await retireLogs(storage, currentJsonKey, {
|
|
1461
|
+
logRetention: deletes < maxDeletes ? {
|
|
1462
|
+
...logRetention,
|
|
1463
|
+
maxDeletes: deletes
|
|
1464
|
+
} : logRetention,
|
|
1465
|
+
...signal !== void 0 && { signal }
|
|
1466
|
+
});
|
|
1467
|
+
};
|
|
1468
|
+
try {
|
|
1469
|
+
const read = await readCurrentJson(storage, currentJsonKey, signal !== void 0 ? { signal } : void 0);
|
|
1470
|
+
if (read === null) return;
|
|
1471
|
+
const current = read.json;
|
|
1472
|
+
const snapshotBytes = current.snapshot_bytes;
|
|
1473
|
+
const snapshotRows = current.snapshot_rows;
|
|
1474
|
+
const logSeqStart = current.log_seq_start;
|
|
1475
|
+
const probeFloor = Math.max(logSeqStart, current.tail_hint);
|
|
1476
|
+
let nextSeq;
|
|
1477
|
+
if (args.observedTail !== void 0) nextSeq = Math.max(args.observedTail, probeFloor);
|
|
1478
|
+
else nextSeq = (await probeTailFrom(storage, collectionPrefix, probeFloor, signal !== void 0 ? { signal } : void 0)).tail;
|
|
1479
|
+
const ratio = tailRatio(current, nextSeq);
|
|
1480
|
+
const foldAtTailEntries = options?.foldAtTailEntries;
|
|
1481
|
+
const gate1 = forceFold || foldAtTailEntries !== void 0 && nextSeq - logSeqStart >= foldAtTailEntries || ratio >= 1 && nextSeq - logSeqStart >= minEntriesToCompact;
|
|
1482
|
+
const foldViable = isFoldViable(snapshotBytes, snapshotRows, ceilings);
|
|
1483
|
+
const gcDue = crossesGcBoundary(prevSeq, nextSeq, gcInterval);
|
|
1484
|
+
let tailHintPublished = false;
|
|
1485
|
+
const refreshTailHintIfNeeded = async (stampWarnedSeq = false) => {
|
|
1486
|
+
if (!stampWarnedSeq) {
|
|
1487
|
+
if (tailHintPublished) return;
|
|
1488
|
+
if (nextSeq - current.tail_hint < 128) return;
|
|
1489
|
+
}
|
|
1490
|
+
tailHintPublished = true;
|
|
1491
|
+
if (requestsLeft() < 2) {
|
|
1492
|
+
skipForBudget("tail_hint");
|
|
1493
|
+
return;
|
|
1494
|
+
}
|
|
1495
|
+
try {
|
|
1496
|
+
await casUpdateCurrentJson(storage, currentJsonKey, (c) => ({
|
|
1497
|
+
...c,
|
|
1498
|
+
tail_hint: Math.max(c.tail_hint, nextSeq),
|
|
1499
|
+
...stampWarnedSeq && { last_warned_seq: nextSeq }
|
|
1500
|
+
}), signal !== void 0 ? { signal } : void 0);
|
|
1501
|
+
} catch {}
|
|
1502
|
+
};
|
|
1503
|
+
/**
|
|
1504
|
+
* The one fold-wall deferral signal, shared by both gates so a
|
|
1505
|
+
* deferral counts once and reads the same wherever it closed:
|
|
1506
|
+
*
|
|
1507
|
+
* - `db.compaction.deferred_total` (labels `collection`, `dimension`,
|
|
1508
|
+
* `gate`) plus `db.compaction.deferred_{bytes,rows}_total` — distinct
|
|
1509
|
+
* names, because the canonical line keys counters by name alone and
|
|
1510
|
+
* drops labels.
|
|
1511
|
+
* - A `fold_deferred` structured warning through the adapter's
|
|
1512
|
+
* {@link MaintenanceDispatch.warn} sink, carrying the human text as
|
|
1513
|
+
* `detail` — or, with no sink or one that throws, that text as a
|
|
1514
|
+
* `console.warn` line. Rate-limited by the SHARED `current.json.last_warned_seq` (not a
|
|
1515
|
+
* per-isolate Set — a fresh isolate must honour it). A collection
|
|
1516
|
+
* that has never warned warns on its first deferral. On Cloudflare
|
|
1517
|
+
* the tick runs in `ctx.waitUntil` after the canonical line flushed,
|
|
1518
|
+
* so the log record is the only signal an operator sees.
|
|
1519
|
+
* - The bounded `tail_hint` refresh (HR-2): a deferring collection
|
|
1520
|
+
* never reaches compact()'s Step-7 CAS, so without it every read
|
|
1521
|
+
* would re-walk the whole growing tail. Write-tick only; reads stay
|
|
1522
|
+
* pure.
|
|
1523
|
+
*/
|
|
1524
|
+
const signalFoldDeferred = async (d) => {
|
|
1525
|
+
const metrics = ctxMetrics();
|
|
1526
|
+
metrics.counter("db.compaction.deferred_total", 1, {
|
|
1527
|
+
collection,
|
|
1528
|
+
dimension: d.dimension,
|
|
1529
|
+
gate: d.gate
|
|
1530
|
+
});
|
|
1531
|
+
metrics.counter(d.dimension === "bytes" ? "db.compaction.deferred_bytes_total" : "db.compaction.deferred_rows_total", 1, {
|
|
1532
|
+
collection,
|
|
1533
|
+
gate: d.gate
|
|
1534
|
+
});
|
|
1535
|
+
const warnDue = current.last_warned_seq === void 0 || nextSeq - current.last_warned_seq >= 1e3;
|
|
1536
|
+
if (warnDue) {
|
|
1537
|
+
const fields = {
|
|
1538
|
+
collection,
|
|
1539
|
+
dimension: d.dimension,
|
|
1540
|
+
gate: d.gate,
|
|
1541
|
+
snapshot_bytes: d.snapshotBytes,
|
|
1542
|
+
snapshot_rows: d.snapshotRows,
|
|
1543
|
+
ceiling_bytes: C,
|
|
1544
|
+
ceiling_rows: E,
|
|
1545
|
+
live_log_tail: nextSeq - logSeqStart,
|
|
1546
|
+
log_seq_start: logSeqStart
|
|
1547
|
+
};
|
|
1548
|
+
const detail = foldDeferredWarning(fields);
|
|
1549
|
+
const sink = getCurrentContext()?.maintenance?.warn;
|
|
1550
|
+
let delivered = false;
|
|
1551
|
+
if (sink !== void 0) try {
|
|
1552
|
+
sink("fold_deferred", {
|
|
1553
|
+
...fields,
|
|
1554
|
+
detail
|
|
1555
|
+
});
|
|
1556
|
+
delivered = true;
|
|
1557
|
+
} catch {}
|
|
1558
|
+
if (!delivered) console.warn(detail);
|
|
1559
|
+
}
|
|
1560
|
+
await refreshTailHintIfNeeded(warnDue);
|
|
1561
|
+
};
|
|
1562
|
+
const runGcAndRefreshTailHint = async () => {
|
|
1563
|
+
const sweeps = requestsLeft() - 6;
|
|
1564
|
+
if (sweeps < 0) skipForBudget("gc");
|
|
1565
|
+
else await runGc({
|
|
1566
|
+
storage,
|
|
1567
|
+
currentJsonKey
|
|
1568
|
+
}, sweeps < gcMaxSweeps ? {
|
|
1569
|
+
...gcOpts,
|
|
1570
|
+
maxSweepsPerRun: sweeps
|
|
1571
|
+
} : gcOpts);
|
|
1572
|
+
await refreshTailHintIfNeeded();
|
|
1573
|
+
};
|
|
1574
|
+
try {
|
|
1575
|
+
if (args.disabled === true) {
|
|
1576
|
+
await refreshTailHintIfNeeded();
|
|
1577
|
+
return;
|
|
1578
|
+
}
|
|
1579
|
+
const hardGc = crossesGcBoundary(prevSeq, nextSeq, gcInterval * 4);
|
|
1580
|
+
if (phasesPerTick === "single" && hardGc) {
|
|
1581
|
+
await runGcAndRefreshTailHint();
|
|
1582
|
+
await retireWithinBudget();
|
|
1583
|
+
return;
|
|
1584
|
+
}
|
|
1585
|
+
const foldFits = requestsLeft() >= 5 + Math.min(maxFoldEntriesPerPass, Math.max(0, nextSeq - logSeqStart));
|
|
1586
|
+
if (gate1 && foldViable && !foldFits) skipForBudget("fold");
|
|
1587
|
+
else if (gate1) {
|
|
1588
|
+
if (foldViable) {
|
|
1589
|
+
const foldRes = await compact({
|
|
1590
|
+
storage,
|
|
1591
|
+
currentJsonKey
|
|
1592
|
+
}, {
|
|
1593
|
+
maxEntriesPerRun: maxFoldEntriesPerPass,
|
|
1594
|
+
minEntriesToCompact: foldAtTailEntries === void 0 ? minEntriesToCompact : Math.min(minEntriesToCompact, foldAtTailEntries),
|
|
1595
|
+
ceilingBytes: C,
|
|
1596
|
+
ceilingEntries: E,
|
|
1597
|
+
knownTail: nextSeq,
|
|
1598
|
+
...signal !== void 0 && { signal }
|
|
1599
|
+
});
|
|
1600
|
+
if (foldRes.written) tailHintPublished = true;
|
|
1601
|
+
if (foldRes.deferral !== void 0) await signalFoldDeferred({
|
|
1602
|
+
gate: "rebuild",
|
|
1603
|
+
dimension: foldRes.deferral.dimension,
|
|
1604
|
+
snapshotBytes: foldRes.deferral.snapshotBytes,
|
|
1605
|
+
snapshotRows: foldRes.deferral.snapshotRows
|
|
1606
|
+
});
|
|
1607
|
+
if (phasesPerTick === "single") return;
|
|
1608
|
+
} else await signalFoldDeferred({
|
|
1609
|
+
gate: "pre-fold",
|
|
1610
|
+
dimension: snapshotBytes > C ? "bytes" : "rows",
|
|
1611
|
+
snapshotBytes,
|
|
1612
|
+
snapshotRows
|
|
1613
|
+
});
|
|
1614
|
+
}
|
|
1615
|
+
if (gcDue) await runGcAndRefreshTailHint();
|
|
1616
|
+
await retireWithinBudget();
|
|
1617
|
+
} finally {
|
|
1618
|
+
await refreshTailHintIfNeeded();
|
|
1619
|
+
}
|
|
1620
|
+
} catch (error) {
|
|
1621
|
+
if (isWriteConflict(error)) return;
|
|
1622
|
+
if (error instanceof BaerlyError && error.code === "CapacityExceeded" && error.cause?.refused === true) {
|
|
1623
|
+
ctxMetrics().counter("db.maintenance.request_budget_refused_total", 1, { collection });
|
|
1624
|
+
return;
|
|
1625
|
+
}
|
|
1626
|
+
ctxMetrics().counter("db.maintenance.unexpected_error_total", 1, { collection });
|
|
1627
|
+
console.error(error);
|
|
1628
|
+
}
|
|
1629
|
+
};
|
|
1630
|
+
/**
|
|
1631
|
+
* Dispatch one write-tick {@link runBoundedMaintenance} through the
|
|
1632
|
+
* request's {@link MaintenanceDispatch}: inline and awaited by default,
|
|
1633
|
+
* `ctx.waitUntil` on Cloudflare. Never throws: the tick swallows its own
|
|
1634
|
+
* failures, and a failing dispatcher bumps
|
|
1635
|
+
* `db.write.maintenance_dispatch_failed_total`. The writer calls it after
|
|
1636
|
+
* a commit; a write refused with `MaintenancePending` calls it with
|
|
1637
|
+
* `forceFold` and nothing committed (`prevSeq === observedTail`).
|
|
1638
|
+
*/
|
|
1639
|
+
const dispatchWriteTick = async (args) => {
|
|
1640
|
+
const maint = getCurrentContext()?.maintenance;
|
|
1641
|
+
const dispatch = maint?.dispatch ?? dispatchInlineAwaited;
|
|
1642
|
+
try {
|
|
1643
|
+
await dispatch(() => runBoundedMaintenance({
|
|
1644
|
+
storage: args.storage,
|
|
1645
|
+
currentJsonKey: args.currentJsonKey,
|
|
1646
|
+
prevSeq: args.prevSeq,
|
|
1647
|
+
observedTail: args.observedTail,
|
|
1648
|
+
...maint?.disabled === true && { disabled: true },
|
|
1649
|
+
...maint?.maxFoldBytes !== void 0 && { maxFoldBytes: maint.maxFoldBytes },
|
|
1650
|
+
...args.forceFold === true && { forceFold: true }
|
|
1651
|
+
}, maint?.options).then(() => {}, () => {}));
|
|
1652
|
+
} catch {
|
|
1653
|
+
ctxMetrics().counter("db.write.maintenance_dispatch_failed_total", 1, { collection: args.collection });
|
|
1654
|
+
}
|
|
1655
|
+
};
|
|
1656
|
+
/**
|
|
1657
|
+
* Default maintenance dispatcher: run the task inline and await it.
|
|
1658
|
+
* Adapters that can keep running after the response (Cloudflare:
|
|
1659
|
+
* `ctx.waitUntil`) override this with a fire-and-forget variant that
|
|
1660
|
+
* returns `void`.
|
|
1661
|
+
*/
|
|
1662
|
+
const dispatchInlineAwaited = (task) => task();
|
|
1663
|
+
/**
|
|
1664
|
+
* Parse the two host-agnostic ops-plane env vars into the
|
|
1665
|
+
* {@link MaintenanceDispatch} overrides. Shared by every host adapter so the
|
|
1666
|
+
* cross-host contract has one source of truth.
|
|
1667
|
+
*
|
|
1668
|
+
* `maxFoldBytes` (`C`) is `BAERLY_MAINTENANCE_MAX_FOLD_BYTES` written as a
|
|
1669
|
+
* plain positive decimal integer (`/^[1-9][0-9]*$/`, at most
|
|
1670
|
+
* `Number.MAX_SAFE_INTEGER`), or undefined when unset or `""` (the active
|
|
1671
|
+
* profile's default applies). Anything else throws, untrimmed: `" 1024"`,
|
|
1672
|
+
* `"0x400"`, `"1e6"`, `"1.5"`, `"8mb"`. A unit-suffixed or mistyped value
|
|
1673
|
+
* would otherwise run silently at the default ceiling; zero or negative
|
|
1674
|
+
* would make `snapshot_bytes <= C` false forever, so the fold would defer
|
|
1675
|
+
* on every tick and the live tail would grow unbounded.
|
|
1676
|
+
*
|
|
1677
|
+
* `disabled` reads `BAERLY_MAINTENANCE_DISABLE`, untrimmed, against a closed,
|
|
1678
|
+
* case-insensitive vocabulary: unset, `""`, `"0"`, `"false"` → `false`;
|
|
1679
|
+
* `"1"`, `"true"` → `true`; anything else throws. The prior truthy-if-
|
|
1680
|
+
* non-empty parsing silently disabled maintenance on `"no"` / `"off"` —
|
|
1681
|
+
* the two most natural ways an operator would try to spell "don't disable".
|
|
1682
|
+
*
|
|
1683
|
+
* @throws {BaerlyError} `InvalidConfig` for a
|
|
1684
|
+
* `BAERLY_MAINTENANCE_MAX_FOLD_BYTES` that is not a plain positive decimal
|
|
1685
|
+
* integer, or a `BAERLY_MAINTENANCE_DISABLE`
|
|
1686
|
+
* value outside the closed vocabulary above.
|
|
1687
|
+
*/
|
|
1688
|
+
const parseMaintenanceEnv = (readEnv) => {
|
|
1689
|
+
const rawFoldBytes = readEnv("BAERLY_MAINTENANCE_MAX_FOLD_BYTES");
|
|
1690
|
+
let maxFoldBytes;
|
|
1691
|
+
if (rawFoldBytes !== void 0 && rawFoldBytes !== "") {
|
|
1692
|
+
const parsedFoldBytes = Number(rawFoldBytes);
|
|
1693
|
+
if (!/^[1-9][0-9]*$/.test(rawFoldBytes) || !Number.isSafeInteger(parsedFoldBytes)) throw new BaerlyError("InvalidConfig", `BAERLY_MAINTENANCE_MAX_FOLD_BYTES must be unset, "", or a positive decimal integer byte count no greater than ${Number.MAX_SAFE_INTEGER}, got ${JSON.stringify(rawFoldBytes)}`);
|
|
1694
|
+
maxFoldBytes = parsedFoldBytes;
|
|
1695
|
+
}
|
|
1696
|
+
const rawDisable = readEnv("BAERLY_MAINTENANCE_DISABLE");
|
|
1697
|
+
const normalizedDisable = rawDisable?.toLowerCase();
|
|
1698
|
+
let disabled;
|
|
1699
|
+
if (rawDisable === void 0 || normalizedDisable === "" || normalizedDisable === "0" || normalizedDisable === "false") disabled = false;
|
|
1700
|
+
else if (normalizedDisable === "1" || normalizedDisable === "true") disabled = true;
|
|
1701
|
+
else throw new BaerlyError("InvalidConfig", `BAERLY_MAINTENANCE_DISABLE must be one of "", "0", "false", "1", "true" (case-insensitive), got ${JSON.stringify(rawDisable)}`);
|
|
1702
|
+
return {
|
|
1703
|
+
...maxFoldBytes !== void 0 && { maxFoldBytes },
|
|
1704
|
+
disabled
|
|
1705
|
+
};
|
|
1706
|
+
};
|
|
1707
|
+
export { getCurrentContext as a, gcPendingKey as c, shouldFireMaintenance as i, isFoldViable as n, runWithContext as o, parseMaintenanceEnv as r, noopMetricsRecorder as s, dispatchWriteTick as t };
|
|
1708
|
+
|
|
1709
|
+
//# sourceMappingURL=maintenance-Dd-Yakqj.js.map
|