@gusto/baerly-storage 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +29 -0
- package/README.md +176 -70
- package/dist/API.md +213 -139
- package/dist/CHANGELOG.md +434 -0
- package/dist/RECIPES.md +125 -0
- package/dist/THIRD-PARTY-LICENSES.txt +238 -0
- package/dist/{app-config-C0Bbw_Z6.d.ts → app-config-DyiFK3_k.d.ts} +55 -11
- package/dist/app-config.d.ts +1 -1
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +301 -301
- package/dist/auth.js.map +1 -1
- package/dist/aws4fetch.esm-p7eEzLCs.js +1702 -0
- package/dist/aws4fetch.esm-p7eEzLCs.js.map +1 -0
- package/dist/baerly.js +35 -15
- package/dist/baerly.js.map +1 -1
- package/dist/baerly.spec.json +342 -0
- package/dist/bytes-BXdamO3L.js.map +1 -1
- package/dist/{client-DYZfoZb9.d.ts → client-Fe8CwB73.d.ts} +10 -10
- package/dist/client-react.d.ts +3 -3
- package/dist/client-react.js +4 -4
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +4 -4
- package/dist/client.js +3 -3
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-CINJPKCW.js → cloudflare-DzKBrUlg.js} +178 -2
- package/dist/cloudflare-DzKBrUlg.js.map +1 -0
- package/dist/cloudflare.d.ts +49 -6
- package/dist/cloudflare.js +59 -26
- package/dist/cloudflare.js.map +1 -1
- package/dist/config-1tv0-Yzm.js +2693 -0
- package/dist/config-1tv0-Yzm.js.map +1 -0
- package/dist/constants-CfeMWQWk.js +222 -0
- package/dist/constants-CfeMWQWk.js.map +1 -0
- package/dist/context-DRZ3tZ5l.js.map +1 -1
- package/dist/{cost-CQt1YRkA.js → cost-DSMIVqzL.js} +43 -42
- package/dist/cost-DSMIVqzL.js.map +1 -0
- package/dist/current-json-C6nhPOm1.js +190 -0
- package/dist/current-json-C6nhPOm1.js.map +1 -0
- package/dist/{db-BM3y2JQw.d.ts → db-DsPkOVEH.d.ts} +26 -108
- package/dist/{deploy-CrAbfgCq.js → deploy-DonTr-CV.js} +12 -5
- package/dist/deploy-DonTr-CV.js.map +1 -0
- package/dist/dev-vite.d.ts +11 -7
- package/dist/dev-vite.js +7 -6
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/{doctor-Db-hzPmE.js → doctor-DSIyCDg_.js} +14 -136
- package/dist/doctor-DSIyCDg_.js.map +1 -0
- package/dist/{dump-BYA55NLP.js → dump-Bh5k26VQ.js} +10 -31
- package/dist/dump-Bh5k26VQ.js.map +1 -0
- package/dist/{errors-DOdL95kK.js → errors-Dfa3p78c.js} +25 -5
- package/dist/errors-Dfa3p78c.js.map +1 -0
- package/dist/{export-Bhbz90-z.js → export-DtF5tDIM.js} +11 -32
- package/dist/{export-Bhbz90-z.js.map → export-DtF5tDIM.js.map} +1 -1
- package/dist/{fsck-CPPZWt9p.js → fsck-D_djMO5D.js} +28 -39
- package/dist/fsck-D_djMO5D.js.map +1 -0
- package/dist/{http-Cw8p6y5b.js → http-DxM9MflO.js} +151 -79
- package/dist/http-DxM9MflO.js.map +1 -0
- package/dist/http.d.ts +42 -26
- package/dist/http.js +3 -2
- package/dist/{index-DwDTWPSU.d.ts → index-0rZcMBwy.d.ts} +10 -7
- package/dist/{index-C3JAH1Yo.d.ts → index-B6oexfOB.d.ts} +9 -9
- package/dist/{index-cUY-36AN.d.ts → index-BNRh23K9.d.ts} +21 -5
- package/dist/{index-DSvwEUeF.d.ts → index-BYk9oCNf.d.ts} +240 -202
- package/dist/index.d.ts +6 -6
- package/dist/index.js +5 -5
- package/dist/{indexes-DAu2AkTW.d.ts → indexes-Dn9hbGQG.d.ts} +2 -2
- package/dist/{inspect-Bdbzvi-_.js → inspect-6WZ-pC7-.js} +21 -36
- package/dist/inspect-6WZ-pC7-.js.map +1 -0
- package/dist/key-Di6yyxFu.js +35 -0
- package/dist/key-Di6yyxFu.js.map +1 -0
- package/dist/{snapshot-SS8USDUF.js → log-tail-D98U1JBo.js} +179 -84
- package/dist/log-tail-D98U1JBo.js.map +1 -0
- package/dist/{maintenance-ClzBC1Vv.js → maintenance-B4Dki6Ig.js} +438 -176
- package/dist/maintenance-B4Dki6Ig.js.map +1 -0
- package/dist/{maintenance-BRrThz8V.d.ts → maintenance-B_e0T99L.d.ts} +114 -26
- package/dist/{maintenance-BvOv6BMM.js → maintenance-C7coQp4c.js} +217 -94
- package/dist/maintenance-C7coQp4c.js.map +1 -0
- package/dist/maintenance.d.ts +3 -2
- package/dist/maintenance.js +3 -2
- package/dist/node.d.ts +176 -68
- package/dist/node.js +3 -2
- package/dist/{normalize-XWY25_sG.js → normalize-rjnZLwS4.js} +2 -2
- package/dist/normalize-rjnZLwS4.js.map +1 -0
- package/dist/{observability-CpOiEN0k.js → observability-wzs3-BL_.js} +279 -109
- package/dist/observability-wzs3-BL_.js.map +1 -0
- package/dist/observability.d.ts +3 -3
- package/dist/observability.js +1 -1
- package/dist/{plan-DOTl21vq.js → plan-a7PJHBCy.js} +7 -4
- package/dist/plan-a7PJHBCy.js.map +1 -0
- package/dist/{query-BmiQy328.js → query-rAtXNQQB.js} +1063 -1202
- package/dist/query-rAtXNQQB.js.map +1 -0
- package/dist/{rebuild-index-DC_nR6wP.js → rebuild-index-BVzQSZlm.js} +10 -32
- package/dist/rebuild-index-BVzQSZlm.js.map +1 -0
- package/dist/{request-BEF_OvMh.js → request-u0M51mhJ.js} +3 -3
- package/dist/{request-BEF_OvMh.js.map → request-u0M51mhJ.js.map} +1 -1
- package/dist/restore-R-4rbciV.js +778 -0
- package/dist/restore-R-4rbciV.js.map +1 -0
- package/dist/runtime-spec-qL3_lTTU.js +391 -0
- package/dist/runtime-spec-qL3_lTTU.js.map +1 -0
- package/dist/s3-http-B-z8th6O.d.ts +60 -0
- package/dist/s3.d.ts +47 -0
- package/dist/s3.js +45 -0
- package/dist/s3.js.map +1 -0
- package/dist/{shared-secret-BJdSwpgw.js → shared-secret-DgEicd3x.js} +13 -5
- package/dist/shared-secret-DgEicd3x.js.map +1 -0
- package/dist/src-BG20LZs7.js +2271 -0
- package/dist/src-BG20LZs7.js.map +1 -0
- package/dist/{src-zhEAmmF7.js → src-CYHFDrKz.js} +82 -99
- package/dist/src-CYHFDrKz.js.map +1 -0
- package/dist/{src-QQ7jDeL2.js → src-DYZ4H1aF.js} +44 -23
- package/dist/src-DYZ4H1aF.js.map +1 -0
- package/dist/subcommand-DoDA-K7d.js +197 -0
- package/dist/subcommand-DoDA-K7d.js.map +1 -0
- package/dist/time-DUqvFMHq.js +119 -0
- package/dist/time-DUqvFMHq.js.map +1 -0
- package/dist/{usage-CLjirdD-.js → usage-DBV1eQHC.js} +7 -22
- package/dist/usage-DBV1eQHC.js.map +1 -0
- package/dist/{validate-DCq_0DIn.js → validate-DtljsCh1.js} +51 -51
- package/dist/{validate-DCq_0DIn.js.map → validate-DtljsCh1.js.map} +1 -1
- package/dist/wire-izfGZDvr.js.map +1 -1
- package/package.json +77 -42
- package/dist/cloudflare-CINJPKCW.js.map +0 -1
- package/dist/config-DVfDGyw1.js +0 -278
- package/dist/config-DVfDGyw1.js.map +0 -1
- package/dist/cost-CQt1YRkA.js.map +0 -1
- package/dist/current-json-C-EyJqs0.js +0 -354
- package/dist/current-json-C-EyJqs0.js.map +0 -1
- package/dist/deploy-CrAbfgCq.js.map +0 -1
- package/dist/doctor-Db-hzPmE.js.map +0 -1
- package/dist/dump-BYA55NLP.js.map +0 -1
- package/dist/errors-DOdL95kK.js.map +0 -1
- package/dist/fsck-CPPZWt9p.js.map +0 -1
- package/dist/http-Cw8p6y5b.js.map +0 -1
- package/dist/inspect-Bdbzvi-_.js.map +0 -1
- package/dist/maintenance-BvOv6BMM.js.map +0 -1
- package/dist/maintenance-ClzBC1Vv.js.map +0 -1
- package/dist/normalize-XWY25_sG.js.map +0 -1
- package/dist/observability-CpOiEN0k.js.map +0 -1
- package/dist/plan-DOTl21vq.js.map +0 -1
- package/dist/query-BmiQy328.js.map +0 -1
- package/dist/rebuild-index-DC_nR6wP.js.map +0 -1
- package/dist/restore-C6IcmLeA.js +0 -771
- package/dist/restore-C6IcmLeA.js.map +0 -1
- package/dist/shared-secret-BJdSwpgw.js.map +0 -1
- package/dist/snapshot-SS8USDUF.js.map +0 -1
- package/dist/src-DU1sCMlr.js +0 -6236
- package/dist/src-DU1sCMlr.js.map +0 -1
- package/dist/src-QQ7jDeL2.js.map +0 -1
- package/dist/src-zhEAmmF7.js.map +0 -1
- package/dist/subcommand-DZcHcsYp.js +0 -5122
- package/dist/subcommand-DZcHcsYp.js.map +0 -1
- package/dist/usage-CLjirdD-.js.map +0 -1
- /package/dist/{chunk-EAsCxrDo.js → rolldown-runtime-EAsCxrDo.js} +0 -0
|
@@ -0,0 +1,778 @@
|
|
|
1
|
+
import { n as encodeJsonBytes, t as decodeJsonBytes } from "./bytes-BXdamO3L.js";
|
|
2
|
+
import { a as countKey, d as LOG_FORWARD_PROBE_CAP, g as MAX_KEY_BYTES, i as timestamp, o as uuid, p as MAINTENANCE_PROFILE_CF_FREE, r as parseBucketUri } from "./config-1tv0-Yzm.js";
|
|
3
|
+
import { a as walkLogRange, d as logObjectKey, h as readCurrentJson, i as readLogEntry, m as logSeqStartOf, p as createCurrentJson, t as findLogTail } from "./log-tail-D98U1JBo.js";
|
|
4
|
+
import { a as getCurrentContext, c as noopMetricsRecorder, l as versionFromContent, n as runBoundedMaintenance, o as allIndexKeysFor, r as shouldFireMaintenance, s as validateIndexDefinition, t as dispatchInlineAwaited } from "./maintenance-C7coQp4c.js";
|
|
5
|
+
import { o as BaerlyError, r as emitSuccess } from "./baerly.js";
|
|
6
|
+
import { a as TENANT_ARG, c as assertPathSegment, i as JSON_ARG, o as assertCollectionArg, s as defineBaerlySubcommand, t as APP_ARG } from "./subcommand-DoDA-K7d.js";
|
|
7
|
+
import { createInterface } from "node:readline";
|
|
8
|
+
//#region ../server/src/doc-id.ts
|
|
9
|
+
/**
|
|
10
|
+
* Guard a caller-supplied `_id` before it becomes a written key segment
|
|
11
|
+
* (index keys, log `doc_id`) or filesystem path (`LocalFsStorage`). The
|
|
12
|
+
* `_id` analogue of `assertKeySegment` (`db.ts`). Routes through the
|
|
13
|
+
* single shared {@link assertPathSegment} rule (empty / `"/"` /
|
|
14
|
+
* `"."`|`".."` / C0-C1 control chars / leading `"_"` (ADR-003) / overlong,
|
|
15
|
+
* all as `BaerlyError{code:"InvalidConfig"}`).
|
|
16
|
+
*/
|
|
17
|
+
const assertDocId = (id) => {
|
|
18
|
+
assertPathSegment(id, "_id");
|
|
19
|
+
};
|
|
20
|
+
//#endregion
|
|
21
|
+
//#region ../server/src/key-limit.ts
|
|
22
|
+
const utf8 = new TextEncoder();
|
|
23
|
+
/**
|
|
24
|
+
* Reject an assembled object key over the S3/R2 full-key ceiling
|
|
25
|
+
* ({@link MAX_KEY_BYTES}) as `InvalidConfig`. Per-segment caps don't
|
|
26
|
+
* bound the sum, and index values skip `assertPathSegment` entirely, so
|
|
27
|
+
* this post-encoding check at the writer's PUT sites is their only
|
|
28
|
+
* overflow guard. @see docs/spec/storage-compatibility.md
|
|
29
|
+
*/
|
|
30
|
+
const assertKeyWithinLimit = (key) => {
|
|
31
|
+
const bytes = utf8.encode(key).length;
|
|
32
|
+
if (bytes > 1024) throw new BaerlyError("InvalidConfig", `object key exceeds ${MAX_KEY_BYTES} bytes (got ${bytes}): ${JSON.stringify(key.slice(0, 80))}…`);
|
|
33
|
+
};
|
|
34
|
+
//#endregion
|
|
35
|
+
//#region ../server/src/log-conflict-adoption.ts
|
|
36
|
+
/**
|
|
37
|
+
* Decide whether the entry already at `/log/<seq>.json` is the
|
|
38
|
+
* writer's own prior in-flight attempt and may be adopted. Pure
|
|
39
|
+
* function; no I/O. The caller performs the GET (when the log PUT
|
|
40
|
+
* surfaces 412) and passes the read-back body in `ctx.existing`.
|
|
41
|
+
*
|
|
42
|
+
* ## Correctness precondition
|
|
43
|
+
*
|
|
44
|
+
* A fresh `session` id is minted per `commit()` call, seq-stable
|
|
45
|
+
* across that call's retries. Adoption compares the writer-minted
|
|
46
|
+
* intent/body fields as well as session and seq: a dropped-ack retry
|
|
47
|
+
* reuses the same attempted entry and adopts, while a same-session /
|
|
48
|
+
* same-seq occupant carrying a different logical mutation is treated as
|
|
49
|
+
* a conflict and skipped by the caller's forward-probe loop.
|
|
50
|
+
*
|
|
51
|
+
* @see {@link AdoptionDecision} for the return shape.
|
|
52
|
+
* @see The soundness invariant in this module's header docstring.
|
|
53
|
+
*/
|
|
54
|
+
const tryAdoptOwnSessionLogEntry = (ctx) => {
|
|
55
|
+
if (ctx.existing.session !== ctx.self.session) return {
|
|
56
|
+
adopt: false,
|
|
57
|
+
reason: "foreign-session"
|
|
58
|
+
};
|
|
59
|
+
if (ctx.existing.seq !== ctx.self.seq) return {
|
|
60
|
+
adopt: false,
|
|
61
|
+
reason: "wrong-seq"
|
|
62
|
+
};
|
|
63
|
+
if (JSON.stringify(ctx.existing) !== JSON.stringify(ctx.self)) return {
|
|
64
|
+
adopt: false,
|
|
65
|
+
reason: "intent-mismatch"
|
|
66
|
+
};
|
|
67
|
+
return {
|
|
68
|
+
adopt: true,
|
|
69
|
+
entry: ctx.existing
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
//#endregion
|
|
73
|
+
//#region ../server/src/writer.ts
|
|
74
|
+
/**
|
|
75
|
+
* `Writer` — stateless multi-instance write engine (single-write commit).
|
|
76
|
+
*
|
|
77
|
+
* Each {@link Writer.commit} reads `current.json` FRESH (for
|
|
78
|
+
* `log_seq_start` / `tail_hint` as a probe FLOOR — a lower bound, NOT a
|
|
79
|
+
* CAS precondition), PUTs the content body, then **creates `log/<seq>`
|
|
80
|
+
* via `If-None-Match: "*"`. That create IS the commit / linearization
|
|
81
|
+
* point.** The writer writes NOTHING to `current.json` on the commit
|
|
82
|
+
* path; `tail_hint` advances outside ordinary commits via compaction
|
|
83
|
+
* folds, write-tick refreshes, and explicit operator/import paths. The
|
|
84
|
+
* commit model and its invariants are recorded in ADR-004
|
|
85
|
+
* (`docs/adr/004-single-write-commit.md`). The
|
|
86
|
+
* optional integrity walk is gated by
|
|
87
|
+
* {@link WriterOptions.verifyLogIntegrityOnCommit} (default off).
|
|
88
|
+
*
|
|
89
|
+
* No per-write cache: every commit re-reads + re-probes, so N stateless
|
|
90
|
+
* instances contend at one place — the `If-None-Match` create of
|
|
91
|
+
* `log/<seq>`. The loser gets a 412 and re-probes forward to the next
|
|
92
|
+
* empty slot. A 412 from the writer's OWN session at the same seq is its
|
|
93
|
+
* own lost-ack / crashed-but-durable commit → adopted (not retried), so
|
|
94
|
+
* the write lands at EXACTLY `seq`, never duplicated.
|
|
95
|
+
*
|
|
96
|
+
* **Crash safety.** Order is content + additive index `newKeys` →
|
|
97
|
+
* `log/<seq>` create → stale index-key DELETEs. A crash before the
|
|
98
|
+
* create leaves an unreferenced content body + orphan additive index
|
|
99
|
+
* keys (no log entry references them), not an orphan log entry with
|
|
100
|
+
* missing content — the compactor sweeps the content, and a stray
|
|
101
|
+
* additive key only ever yields a false-POSITIVE candidate that
|
|
102
|
+
* `matchesWire` (read path) drops. Emitting `newKeys` before the commit
|
|
103
|
+
* means a committed row is ALWAYS index-findable; the stale-key DELETE
|
|
104
|
+
* stays after the commit so a crash can never de-index a committed
|
|
105
|
+
* doc. See `index-emit-order.test.ts` and ADR-004 Q4.
|
|
106
|
+
*
|
|
107
|
+
* **`LogEntry` shape.** Emitted entries follow the contract in
|
|
108
|
+
* `packages/protocol/src/log.ts` (see {@link LogEntry}) and
|
|
109
|
+
* `docs/spec/log-entry-shape.md`. The on-bucket key is
|
|
110
|
+
* `log/<seq>.json` (the integer `seq` is the load-bearing
|
|
111
|
+
* identifier; the `lsn` string is schema-only).
|
|
112
|
+
*
|
|
113
|
+
* @see docs/spec/sync-protocol.md for the protocol invariants this
|
|
114
|
+
* loop preserves.
|
|
115
|
+
* @see docs/spec/log-entry-shape.md for the per-field contract.
|
|
116
|
+
*/
|
|
117
|
+
const CAS_CONFLICT_RESOLUTION = "Re-read and retry.";
|
|
118
|
+
const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
|
|
119
|
+
const DEFAULT_INITIAL_BACKOFF_MS = 25;
|
|
120
|
+
const MAX_BACKOFF_MS = 1500;
|
|
121
|
+
const APPLICATION_JSON = "application/json";
|
|
122
|
+
/**
|
|
123
|
+
* Empty body for index entries. Each index entry is a fact ("doc
|
|
124
|
+
* `<docId>` has `<field> = <value>`"), not data — readers list the
|
|
125
|
+
* prefix, extract the doc id, then GET the content body separately.
|
|
126
|
+
* Pre-allocated module-level constant so every index PUT shares one
|
|
127
|
+
* zero-length buffer.
|
|
128
|
+
*/
|
|
129
|
+
const EMPTY_BODY = /* @__PURE__ */ new Uint8Array(0);
|
|
130
|
+
/**
|
|
131
|
+
* Stateless write engine for the multi-instance core.
|
|
132
|
+
*
|
|
133
|
+
* Construction is cheap and performs zero I/O — adapters build a
|
|
134
|
+
* fresh `Writer` per request and discard it. All real work
|
|
135
|
+
* happens in {@link commit}.
|
|
136
|
+
*
|
|
137
|
+
* @example
|
|
138
|
+
* ```ts
|
|
139
|
+
* import { Writer } from "@gusto/baerly-storage";
|
|
140
|
+
* import { MemoryStorage } from "@gusto/baerly-storage";
|
|
141
|
+
*
|
|
142
|
+
* const writer = new Writer({
|
|
143
|
+
* storage: new MemoryStorage(),
|
|
144
|
+
* currentJsonKey: "app/tickets/tenant/acme/manifests/tickets/current.json",
|
|
145
|
+
* });
|
|
146
|
+
*
|
|
147
|
+
* const result = await writer.commit({
|
|
148
|
+
* op: "I",
|
|
149
|
+
* collection: "tickets",
|
|
150
|
+
* docId: "doc-1",
|
|
151
|
+
* body: { _id: "doc-1", title: "hello" },
|
|
152
|
+
* });
|
|
153
|
+
* console.log(result.entry.seq, result.attempts);
|
|
154
|
+
* ```
|
|
155
|
+
*/
|
|
156
|
+
var Writer = class {
|
|
157
|
+
#storage;
|
|
158
|
+
#currentJsonKey;
|
|
159
|
+
#maxRetries;
|
|
160
|
+
#initialBackoffMs;
|
|
161
|
+
#random;
|
|
162
|
+
#indexes;
|
|
163
|
+
#verifyLogIntegrityOnCommit;
|
|
164
|
+
constructor(opts) {
|
|
165
|
+
this.#storage = opts.storage;
|
|
166
|
+
this.#currentJsonKey = opts.currentJsonKey;
|
|
167
|
+
this.#maxRetries = opts.options?.maxRetries ?? 8;
|
|
168
|
+
this.#initialBackoffMs = opts.options?.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS;
|
|
169
|
+
this.#random = opts.options?.random ?? Math.random;
|
|
170
|
+
const indexes = opts.options?.indexes ?? [];
|
|
171
|
+
for (const def of indexes) validateIndexDefinition(def);
|
|
172
|
+
this.#indexes = indexes;
|
|
173
|
+
this.#verifyLogIntegrityOnCommit = opts.options?.verifyLogIntegrityOnCommit ?? false;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Atomically commit one mutation. Reads `current.json` fresh (as a
|
|
177
|
+
* probe FLOOR, not a CAS precondition), PUTs the content body and any
|
|
178
|
+
* additive index `newKeys`, then creates `log/<seq>` via
|
|
179
|
+
* `If-None-Match: "*"` — that create IS the commit (single-write
|
|
180
|
+
* commit; no `current.json` CAS follows it). Stale index keys are
|
|
181
|
+
* DELETE'd after the commit.
|
|
182
|
+
*
|
|
183
|
+
* The hot-path cost under no contention on an unindexed collection is
|
|
184
|
+
* 1 GET + 2 PUTs (content + the committing `log/<seq>` create); each
|
|
185
|
+
* declared index that projects a key on this write adds one PUT
|
|
186
|
+
* (new key, before the commit) and, on U/D, up to one DELETE (stale
|
|
187
|
+
* key, after the commit). An in-flight peer write costs one extra GET
|
|
188
|
+
* + the backoff sleep per retry.
|
|
189
|
+
*
|
|
190
|
+
* Idempotency: PUT content uses `If-None-Match: "*"`, so a retry of
|
|
191
|
+
* the same logical write produces the same content key and the
|
|
192
|
+
* second PUT no-ops. The `log/<seq>` create also uses
|
|
193
|
+
* `If-None-Match: "*"`, so two writers racing the same tail cause
|
|
194
|
+
* exactly one create to win; the loser reads back the occupant,
|
|
195
|
+
* adopts its own lost-ack write or re-probes forward to the next
|
|
196
|
+
* empty slot.
|
|
197
|
+
*
|
|
198
|
+
* @throws BaerlyError code="Conflict" when a retryable conflict
|
|
199
|
+
* escapes the single-attempt helper and exhausts the retry budget
|
|
200
|
+
* (presently unreachable on the log-create 412 path).
|
|
201
|
+
* @throws BaerlyError code="Internal" when a log entry expected in
|
|
202
|
+
* `[0, tail_hint)` is missing — a protocol-invariant violation
|
|
203
|
+
* (compactor bug or stale `current.json`).
|
|
204
|
+
* @throws BaerlyError code="InvalidResponse" when a log-entry body
|
|
205
|
+
* isn't valid JSON.
|
|
206
|
+
*
|
|
207
|
+
* **Implicit provisioning.** When `current.json` does not exist at
|
|
208
|
+
* `currentJsonKey` (fresh bucket, fresh collection), the writer
|
|
209
|
+
* auto-creates it with a zero-state manifest before the commit
|
|
210
|
+
* lands. The cost is one extra Class A PUT on the very first
|
|
211
|
+
* commit per collection; steady-state cost is unchanged. A peer
|
|
212
|
+
* racing the same first-write loses cleanly on the create CAS
|
|
213
|
+
* and re-reads to pick up the winner's manifest.
|
|
214
|
+
*/
|
|
215
|
+
async commit(input) {
|
|
216
|
+
assertDocId(input.docId);
|
|
217
|
+
const session = uuid().slice(0, 6);
|
|
218
|
+
const logPrefix = this.#currentJsonKey.slice(0, this.#currentJsonKey.lastIndexOf("/"));
|
|
219
|
+
for (let attempt = 1; attempt <= this.#maxRetries; attempt++) {
|
|
220
|
+
let success;
|
|
221
|
+
try {
|
|
222
|
+
success = await this.#singleAttemptCommit(input, session, logPrefix, "Writer");
|
|
223
|
+
} catch (error) {
|
|
224
|
+
if (!isPreconditionFailed(error)) throw error;
|
|
225
|
+
await this.#backoff(attempt);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
this.#emitWriteMetrics(input.collection, success.classAOps + (attempt - 1));
|
|
229
|
+
return {
|
|
230
|
+
entry: success.entries[0],
|
|
231
|
+
currentEtag: success.currentEtag,
|
|
232
|
+
attempts: attempt
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
throw new BaerlyError("Conflict", `Writer: CAS conflict on ${this.#currentJsonKey} after ${this.#maxRetries} attempts`, void 0, void 0, void 0, CAS_CONFLICT_RESOLUTION, true);
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* One full commit attempt — the body of {@link commit}. Reads
|
|
239
|
+
* `current.json`, PUTs content + additive index `newKeys` in parallel,
|
|
240
|
+
* then forward-probes the tail and creates `log/<seq>` once via
|
|
241
|
+
* `If-None-Match: "*"` — the numbered log create IS the commit
|
|
242
|
+
* (single-write commit; no `current.json` CAS follows it). Stale index
|
|
243
|
+
* keys are DELETE'd after the commit. Returns the success payload.
|
|
244
|
+
*
|
|
245
|
+
* Commits exactly one {@link CommitInput} — the numbered log create is
|
|
246
|
+
* the single linearization point per call.
|
|
247
|
+
*
|
|
248
|
+
* A log-create 412 is disambiguated by session read-back: own session
|
|
249
|
+
* / own seq ⇒ our crashed-or-lost-ack commit is already durable ⇒
|
|
250
|
+
* adopt; foreign session ⇒ re-probe forward and retry at the next
|
|
251
|
+
* empty slot. Thrown `BaerlyError`s (Internal, InvalidResponse,
|
|
252
|
+
* NetworkError) propagate.
|
|
253
|
+
*
|
|
254
|
+
* Crash safety invariant: content + additive index `newKeys` PUTs are
|
|
255
|
+
* awaited before the log create. A crash before the create leaves
|
|
256
|
+
* orphan content + additive index keys (no log entry references them)
|
|
257
|
+
* — the compactor sweeps the content, and a stray additive key only
|
|
258
|
+
* yields a false-positive candidate that `matchesWire` drops. A crash
|
|
259
|
+
* AFTER the create is a durable, committed write at `seq` whose index
|
|
260
|
+
* `newKeys` are already present (so it's index-findable); the only
|
|
261
|
+
* residual is a possibly-undeleted stale OLD-value key, dropped by
|
|
262
|
+
* `matchesWire` and cleaned by a later write / `rebuild-index`. The
|
|
263
|
+
* inverse — committed log entry with missing content, or a de-indexed
|
|
264
|
+
* committed doc — is never produced.
|
|
265
|
+
*/
|
|
266
|
+
async #singleAttemptCommit(input, session, logPrefix, errorPrefix) {
|
|
267
|
+
const read = await readCurrentJson(this.#storage, this.#currentJsonKey) ?? await this.#provisionCurrentJson();
|
|
268
|
+
const current = read.json;
|
|
269
|
+
const baseEtag = read.etag;
|
|
270
|
+
if (this.#verifyLogIntegrityOnCommit) await this.#walkLog(logPrefix, logSeqStartOf(current), current.tail_hint);
|
|
271
|
+
const probeFloor = Math.max(logSeqStartOf(current), current.tail_hint);
|
|
272
|
+
const preCommitTail = await findLogTail(this.#storage, logPrefix, probeFloor);
|
|
273
|
+
let seq = preCommitTail;
|
|
274
|
+
const commitNowMs = Date.now();
|
|
275
|
+
const commitTs = new Date(commitNowMs).toISOString();
|
|
276
|
+
const lsnTimestamp = timestamp(commitNowMs);
|
|
277
|
+
const mintEntry = (atSeq) => ({
|
|
278
|
+
lsn: `${lsnTimestamp}_${session}_${countKey(atSeq)}`,
|
|
279
|
+
commit_ts: commitTs,
|
|
280
|
+
op: input.op,
|
|
281
|
+
collection: input.collection,
|
|
282
|
+
doc_id: input.docId,
|
|
283
|
+
session,
|
|
284
|
+
seq: atSeq,
|
|
285
|
+
...input.op !== "D" && input.body !== void 0 ? { after: input.body } : {},
|
|
286
|
+
...input.origin !== void 0 ? { origin: input.origin } : {}
|
|
287
|
+
});
|
|
288
|
+
let entry = mintEntry(seq);
|
|
289
|
+
let contentPutCount = 0;
|
|
290
|
+
let newKeysClassA = 0;
|
|
291
|
+
const parallelPuts = [];
|
|
292
|
+
if (input.op !== "D" && input.body !== void 0) {
|
|
293
|
+
contentPutCount++;
|
|
294
|
+
const bytes = encodeJsonBytes(input.body);
|
|
295
|
+
const contentKey = `${logPrefix}/content/${await versionFromContent(bytes)}.json`;
|
|
296
|
+
assertKeyWithinLimit(contentKey);
|
|
297
|
+
parallelPuts.push(this.#storage.put(contentKey, bytes, {
|
|
298
|
+
ifNoneMatch: "*",
|
|
299
|
+
contentType: APPLICATION_JSON
|
|
300
|
+
}).catch((error) => {
|
|
301
|
+
this.#observe429(error, input.collection);
|
|
302
|
+
if (isPreconditionFailed(error)) return;
|
|
303
|
+
throw error;
|
|
304
|
+
}));
|
|
305
|
+
}
|
|
306
|
+
if (this.#indexes.length > 0 && input.op !== "D") {
|
|
307
|
+
const newKeys = allIndexKeysFor(logPrefix, this.#indexes, input.body, input.docId);
|
|
308
|
+
for (const k of newKeys) {
|
|
309
|
+
assertKeyWithinLimit(k);
|
|
310
|
+
newKeysClassA++;
|
|
311
|
+
parallelPuts.push(this.#storage.put(k, EMPTY_BODY, {
|
|
312
|
+
ifNoneMatch: "*",
|
|
313
|
+
contentType: APPLICATION_JSON
|
|
314
|
+
}).catch((error) => {
|
|
315
|
+
this.#observe429(error, input.collection);
|
|
316
|
+
if (isPreconditionFailed(error)) return;
|
|
317
|
+
throw error;
|
|
318
|
+
}));
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
await Promise.all(parallelPuts);
|
|
322
|
+
let committedEntry = entry;
|
|
323
|
+
let probes = 0;
|
|
324
|
+
let transientRetries = 0;
|
|
325
|
+
for (;;) {
|
|
326
|
+
const logEntryKey = logObjectKey(logPrefix, seq);
|
|
327
|
+
assertKeyWithinLimit(logEntryKey);
|
|
328
|
+
let conflicted = false;
|
|
329
|
+
try {
|
|
330
|
+
await this.#storage.put(logEntryKey, encodeJsonBytes(entry), {
|
|
331
|
+
ifNoneMatch: "*",
|
|
332
|
+
contentType: APPLICATION_JSON
|
|
333
|
+
});
|
|
334
|
+
} catch (error) {
|
|
335
|
+
this.#observe429(error, entry.collection);
|
|
336
|
+
if (isPreconditionFailed(error)) {
|
|
337
|
+
ctxMetrics().counter("db.r2.put.412_total", 1, {
|
|
338
|
+
collection: entry.collection,
|
|
339
|
+
step: "log-put"
|
|
340
|
+
});
|
|
341
|
+
conflicted = true;
|
|
342
|
+
} else if (isTransientWrite(error) && transientRetries < this.#maxRetries) {
|
|
343
|
+
transientRetries++;
|
|
344
|
+
await this.#backoff(transientRetries);
|
|
345
|
+
continue;
|
|
346
|
+
} else throw error;
|
|
347
|
+
}
|
|
348
|
+
if (!conflicted) {
|
|
349
|
+
committedEntry = entry;
|
|
350
|
+
break;
|
|
351
|
+
}
|
|
352
|
+
const existing = await readLogEntry(this.#storage, logEntryKey);
|
|
353
|
+
const decision = tryAdoptOwnSessionLogEntry({
|
|
354
|
+
self: entry,
|
|
355
|
+
existing
|
|
356
|
+
});
|
|
357
|
+
if (decision.adopt) {
|
|
358
|
+
committedEntry = decision.entry;
|
|
359
|
+
break;
|
|
360
|
+
}
|
|
361
|
+
if (++probes > 1e5) throw new BaerlyError("Internal", `${errorPrefix}: log-tail forward-probe exceeded ${LOG_FORWARD_PROBE_CAP} on ${this.#currentJsonKey}`);
|
|
362
|
+
seq = await findLogTail(this.#storage, logPrefix, seq + 1);
|
|
363
|
+
entry = mintEntry(seq);
|
|
364
|
+
}
|
|
365
|
+
const committedEntries = [committedEntry];
|
|
366
|
+
const staleKeysClassA = await this.#cleanupStaleIndexKeysBestEffort({
|
|
367
|
+
input,
|
|
368
|
+
logPrefix,
|
|
369
|
+
seq
|
|
370
|
+
});
|
|
371
|
+
const indexClassA = newKeysClassA + staleKeysClassA;
|
|
372
|
+
if (this.#indexes.length > 0 && indexClassA > 0) ctxMetrics().histogram("db.write.index_ops_per_logical_write", indexClassA, { collection: input.collection });
|
|
373
|
+
const prevSeq = preCommitTail;
|
|
374
|
+
const observedTail = seq + 1;
|
|
375
|
+
const maint = getCurrentContext()?.maintenance;
|
|
376
|
+
const gcInterval = maint?.options?.profile?.gcInterval ?? MAINTENANCE_PROFILE_CF_FREE.gcInterval;
|
|
377
|
+
if (maint?.disabled === true ? observedTail - current.tail_hint >= 128 : shouldFireMaintenance(current, prevSeq, gcInterval, observedTail)) await (maint?.dispatch ?? dispatchInlineAwaited)(() => runBoundedMaintenance({
|
|
378
|
+
storage: this.#storage,
|
|
379
|
+
currentJsonKey: this.#currentJsonKey,
|
|
380
|
+
prevSeq,
|
|
381
|
+
observedTail,
|
|
382
|
+
...maint?.disabled === true && { disabled: true },
|
|
383
|
+
...maint?.maxFoldBytes !== void 0 && { maxFoldBytes: maint.maxFoldBytes }
|
|
384
|
+
}, maint?.options).then(() => {}, () => {}));
|
|
385
|
+
return {
|
|
386
|
+
entries: committedEntries,
|
|
387
|
+
currentEtag: baseEtag,
|
|
388
|
+
classAOps: contentPutCount + 1 + indexClassA
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* Auto-create the per-collection `current.json` at
|
|
393
|
+
* `this.#currentJsonKey` with a zero-state initial manifest. Called
|
|
394
|
+
* by `#singleAttemptCommit` when the read returned `null` (fresh
|
|
395
|
+
* bucket / fresh collection).
|
|
396
|
+
*
|
|
397
|
+
* Concurrency: `createCurrentJson` uses `If-None-Match: "*"`. If a
|
|
398
|
+
* peer raced us and won, the storage layer surfaces `Conflict`; we
|
|
399
|
+
* recover by re-reading the now-present manifest. We do NOT retry
|
|
400
|
+
* the create — a single recover-via-read covers every race that
|
|
401
|
+
* could have produced our null read.
|
|
402
|
+
*
|
|
403
|
+
* The seed shape (`snapshot: null`, `tail_hint: 0`, `log_seq_start:
|
|
404
|
+
* 0`, `writer_fence: { epoch: 0, owner: "", claimed_at: "" }`)
|
|
405
|
+
* matches `ensureTable` and `baerly deploy`'s pre-warm path, so an
|
|
406
|
+
* operator who pre-provisions and a writer who auto-creates land
|
|
407
|
+
* on byte-identical bytes.
|
|
408
|
+
*
|
|
409
|
+
* @throws BaerlyError code="InvalidResponse" — the post-recover
|
|
410
|
+
* re-read also returned null. Defensive guard; in practice
|
|
411
|
+
* unreachable because the Conflict implies the key now exists.
|
|
412
|
+
*/
|
|
413
|
+
async #provisionCurrentJson() {
|
|
414
|
+
const initial = {
|
|
415
|
+
schema_version: 3,
|
|
416
|
+
snapshot: null,
|
|
417
|
+
tail_hint: 0,
|
|
418
|
+
log_seq_start: 0,
|
|
419
|
+
writer_fence: {
|
|
420
|
+
epoch: 0,
|
|
421
|
+
owner: "",
|
|
422
|
+
claimed_at: ""
|
|
423
|
+
},
|
|
424
|
+
snapshot_bytes: 0,
|
|
425
|
+
snapshot_rows: 0
|
|
426
|
+
};
|
|
427
|
+
try {
|
|
428
|
+
return await createCurrentJson(this.#storage, this.#currentJsonKey, initial);
|
|
429
|
+
} catch (error) {
|
|
430
|
+
if (!(error instanceof BaerlyError) || error.code !== "Conflict") throw error;
|
|
431
|
+
}
|
|
432
|
+
const recovered = await readCurrentJson(this.#storage, this.#currentJsonKey);
|
|
433
|
+
if (recovered === null) throw new BaerlyError("InvalidResponse", `Writer: current.json missing at ${this.#currentJsonKey} after auto-create raced and recover read returned null`);
|
|
434
|
+
return recovered;
|
|
435
|
+
}
|
|
436
|
+
async #cleanupStaleIndexKeysBestEffort(args) {
|
|
437
|
+
if (this.#indexes.length === 0) return 0;
|
|
438
|
+
const { input } = args;
|
|
439
|
+
let staleKeys = [];
|
|
440
|
+
try {
|
|
441
|
+
if (input.op === "U") {
|
|
442
|
+
const preImage = await this.#readPreImage(args.logPrefix, input.collection, input.docId, args.seq);
|
|
443
|
+
const oldKeys = allIndexKeysFor(args.logPrefix, this.#indexes, preImage, input.docId);
|
|
444
|
+
const newSet = new Set(allIndexKeysFor(args.logPrefix, this.#indexes, input.body, input.docId));
|
|
445
|
+
staleKeys = oldKeys.filter((k) => !newSet.has(k));
|
|
446
|
+
} else if (input.op === "D") {
|
|
447
|
+
const preImage = await this.#readPreImage(args.logPrefix, input.collection, input.docId, args.seq);
|
|
448
|
+
staleKeys = allIndexKeysFor(args.logPrefix, this.#indexes, preImage, input.docId);
|
|
449
|
+
}
|
|
450
|
+
} catch {
|
|
451
|
+
ctxMetrics().counter("db.write.index_cleanup_errors_total", 1, {
|
|
452
|
+
collection: input.collection,
|
|
453
|
+
step: "pre-image-read"
|
|
454
|
+
});
|
|
455
|
+
return 0;
|
|
456
|
+
}
|
|
457
|
+
const indexDeletes = [];
|
|
458
|
+
for (const k of staleKeys) indexDeletes.push(this.#storage.delete(k).catch(() => {
|
|
459
|
+
ctxMetrics().counter("db.write.index_cleanup_errors_total", 1, {
|
|
460
|
+
collection: input.collection,
|
|
461
|
+
step: "delete"
|
|
462
|
+
});
|
|
463
|
+
}));
|
|
464
|
+
await Promise.all(indexDeletes);
|
|
465
|
+
return staleKeys.length;
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Read the pre-image content body for a doc by walking the live log
|
|
469
|
+
* backwards from `currentNextSeq` looking for the most-recent I/U
|
|
470
|
+
* entry on this `(collection, docId)`. Returns `undefined` when:
|
|
471
|
+
*
|
|
472
|
+
* - the doc's most-recent op was `D` (tombstone — no live body);
|
|
473
|
+
* - no entry for this doc lives inside the visible log range
|
|
474
|
+
* `[log_seq_start, currentNextSeq)` (a fresh-insert race or the
|
|
475
|
+
* doc has been folded into the snapshot but is no longer
|
|
476
|
+
* referenced).
|
|
477
|
+
*
|
|
478
|
+
* Used ONLY by the index-emission path's stale-key half (Step 5b, on
|
|
479
|
+
* `U` / `D`) to compute which OLD-value keys to DELETE. Bounded by
|
|
480
|
+
* `log_seq_start`: entries below have been folded into the snapshot
|
|
481
|
+
* and may already be swept off the bucket — a snapshot fold is the
|
|
482
|
+
* rebuild command's job (see `./rebuild-index.ts`), not the writer's.
|
|
483
|
+
*
|
|
484
|
+
* **Cost note:** linear walk, O(snapshot lag). For helpdesk-shape
|
|
485
|
+
* collections (100s of docs, <10k log entries) this is fine — the
|
|
486
|
+
* compactor folds the live tail every ~100 entries
|
|
487
|
+
* (`packages/server/src/compactor.ts`). A follow-up ticket caches
|
|
488
|
+
* per-doc index head in `current.json` for O(1) lookup.
|
|
489
|
+
*/
|
|
490
|
+
async #readPreImage(logPrefix, collection, docId, currentNextSeq) {
|
|
491
|
+
for (let s = currentNextSeq - 1; s >= 0; s--) {
|
|
492
|
+
const logKey = logObjectKey(logPrefix, s);
|
|
493
|
+
const got = await this.#storage.get(logKey);
|
|
494
|
+
if (got === null) continue;
|
|
495
|
+
let entry;
|
|
496
|
+
try {
|
|
497
|
+
entry = decodeJsonBytes(got.body);
|
|
498
|
+
} catch {
|
|
499
|
+
continue;
|
|
500
|
+
}
|
|
501
|
+
if (entry.collection !== collection || entry.doc_id !== docId) continue;
|
|
502
|
+
if (entry.op === "D") return;
|
|
503
|
+
if ((entry.op === "I" || entry.op === "U") && entry.after !== void 0) return entry.after;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* Walk `log/<logSeqStart>.json` … `log/<nextSeq - 1>.json` with
|
|
508
|
+
* bounded parallelism via `walkLogRange`. Materialised entries are
|
|
509
|
+
* discarded — the walk's only job is to surface a hole or a
|
|
510
|
+
* malformed body inside the visible range as
|
|
511
|
+
* `BaerlyError{code:"Internal"}` / `"InvalidResponse"`.
|
|
512
|
+
*
|
|
513
|
+
* Gated on `verifyLogIntegrityOnCommit`; default off in production
|
|
514
|
+
* because the per-doc fold already lives in the read path. See the
|
|
515
|
+
* option's docstring for the rationale and the opt-in convention.
|
|
516
|
+
*
|
|
517
|
+
* `logSeqStart` is the boundary set by the compactor on
|
|
518
|
+
* `current.json.log_seq_start`. Entries below it have been folded
|
|
519
|
+
* into the snapshot and may have been swept off the bucket, so the
|
|
520
|
+
* walk MUST NOT GET-require them.
|
|
521
|
+
*/
|
|
522
|
+
async #walkLog(logPrefix, logSeqStart, nextSeq) {
|
|
523
|
+
await walkLogRange(this.#storage, logPrefix, logSeqStart, nextSeq);
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Bump the `db.r2.put.429_total` counter when `err` looks like an
|
|
527
|
+
* R2 prefix-partition rate-limit. Best-effort: detected via
|
|
528
|
+
* `BaerlyError{code:"NetworkError"}` with a `429` token in the
|
|
529
|
+
* message. Called from every PUT call site's catch arm; mutually
|
|
530
|
+
* exclusive in practice with `isPreconditionFailed` (412 ≠ 429).
|
|
531
|
+
*/
|
|
532
|
+
#observe429(err, collection) {
|
|
533
|
+
if (is429(err)) ctxMetrics().counter("db.r2.put.429_total", 1, { collection });
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* Emit the per-logical-write `class_a_ops_per_logical_write`
|
|
537
|
+
* histogram for `commit`. `classAOps` is the per-attempt base plus
|
|
538
|
+
* a charge for prior failed attempts (the retry-budget cost model),
|
|
539
|
+
* computed by the caller.
|
|
540
|
+
*/
|
|
541
|
+
#emitWriteMetrics(collection, classAOps) {
|
|
542
|
+
ctxMetrics().histogram("db.write.class_a_ops_per_logical_write", classAOps, { collection });
|
|
543
|
+
}
|
|
544
|
+
/**
|
|
545
|
+
* Exponential backoff with full jitter. `25 * 2^(attempt-1)` capped
|
|
546
|
+
* at 1500ms; uniform jitter in `[0, base)` keeps colliding peers
|
|
547
|
+
* from re-racing in lockstep. Worst-case total wait across the
|
|
548
|
+
* default 8 attempts is roughly 2.5 s — well under a typical
|
|
549
|
+
* Worker request budget.
|
|
550
|
+
*/
|
|
551
|
+
async #backoff(attempt) {
|
|
552
|
+
const sleepMs = Math.min(this.#initialBackoffMs * 2 ** (attempt - 1), MAX_BACKOFF_MS) * this.#random();
|
|
553
|
+
if (sleepMs <= 0) return;
|
|
554
|
+
await new Promise((r) => setTimeout(r, sleepMs));
|
|
555
|
+
}
|
|
556
|
+
};
|
|
557
|
+
/**
|
|
558
|
+
* `true` when the underlying storage surfaced a `PreconditionFailed`
|
|
559
|
+
* 412 (or its non-S3 equivalents). Every in-tree {@link Storage} impl
|
|
560
|
+
* surfaces a lost CAS as `BaerlyError{code:"Conflict"}`.
|
|
561
|
+
*/
|
|
562
|
+
const isPreconditionFailed = (err) => err instanceof BaerlyError && err.code === "Conflict";
|
|
563
|
+
/**
|
|
564
|
+
* `true` when a log-create PUT failed transiently (a `NetworkError`) —
|
|
565
|
+
* the write may or may not have landed (a dropped ack). Safe to retry
|
|
566
|
+
* the SAME seq: own-session adoption closes the double-commit window if
|
|
567
|
+
* it did land. NOT a 412 (that's handled by adoption directly).
|
|
568
|
+
*/
|
|
569
|
+
const isTransientWrite = (err) => err instanceof BaerlyError && err.code === "NetworkError";
|
|
570
|
+
/**
|
|
571
|
+
* `true` when the underlying storage surfaced an R2 prefix-partition
|
|
572
|
+
* rate-limit. {@link S3HttpStorage} stamps the upstream HTTP status
|
|
573
|
+
* onto `BaerlyError.cause` as `{ status }`; we discriminate on that
|
|
574
|
+
* structured field rather than regex-matching the message.
|
|
575
|
+
*/
|
|
576
|
+
const is429 = (err) => {
|
|
577
|
+
if (!(err instanceof BaerlyError) || err.code !== "NetworkError") return false;
|
|
578
|
+
return err.cause?.status === 429;
|
|
579
|
+
};
|
|
580
|
+
//#endregion
|
|
581
|
+
//#region src/admin/restore.ts
|
|
582
|
+
/**
|
|
583
|
+
* `baerly admin restore` — bulk-import NDJSON into a fresh collection.
|
|
584
|
+
*
|
|
585
|
+
* Reads NDJSON from stdin, reconstructs each row, and calls
|
|
586
|
+
* `Writer.commit({op:"I"})` per row into the target bucket.
|
|
587
|
+
* Programmatic callers (today: tests + the export round-trip
|
|
588
|
+
* integration test) can divert the input by passing
|
|
589
|
+
* `{ streams: { stdin } }` to {@link runRestore}. Idempotent on a fresh
|
|
590
|
+
* bucket: re-running on a half-completed restore **refuses** unless
|
|
591
|
+
* `--force` is set (which reseeds a fresh `current.json` above the old
|
|
592
|
+
* tail first).
|
|
593
|
+
*
|
|
594
|
+
* Args:
|
|
595
|
+
* --bucket Required. Target bucket URI.
|
|
596
|
+
* --app Required (or via baerly.config.ts).
|
|
597
|
+
* --tenant Required (or via baerly.config.ts).
|
|
598
|
+
* --collection Required. Target collection name.
|
|
599
|
+
* --force Truncate the target if it exists (reseed above the old tail).
|
|
600
|
+
* --json JSON envelope.
|
|
601
|
+
*
|
|
602
|
+
* stdin: NDJSON, one `{_id, ...}` row per line. Empty lines tolerated.
|
|
603
|
+
* EOF terminates.
|
|
604
|
+
*
|
|
605
|
+
* Cost shape: under single-write commit each `Writer.commit` is 2 Class
|
|
606
|
+
* A PUTs (content + the committing `log/<seq>` create — no per-row
|
|
607
|
+
* `current.json` write). So: 1 PUT current.json (initial seed) + N × 2
|
|
608
|
+
* + 1 PUT current.json (final `tail_hint` stamp) = 2N + 2 Class A ops
|
|
609
|
+
* for N rows.
|
|
610
|
+
*
|
|
611
|
+
* Partial-restore semantics: a mid-stream Network / Internal / parse
|
|
612
|
+
* error leaves the target in a partial state (some rows committed,
|
|
613
|
+
* `current.json` advanced to wherever the last successful commit
|
|
614
|
+
* landed). Re-run with `--force` to truncate, or hand-clean. The
|
|
615
|
+
* malformed-line case (missing `_id`, bad JSON) fails fast on that
|
|
616
|
+
* line; rows committed BEFORE it survive.
|
|
617
|
+
*
|
|
618
|
+
* Exit codes:
|
|
619
|
+
* 0 — every row committed.
|
|
620
|
+
* 1 — InvalidConfig (bad bucket URI, missing args).
|
|
621
|
+
* 2 — Network / storage error mid-stream, or a malformed NDJSON
|
|
622
|
+
* line (raised as a plain `Error`, not `BaerlyError`).
|
|
623
|
+
* 3 — Conflict from a pre-existing target without --force, or from a
|
|
624
|
+
* concurrent writer when --force was used.
|
|
625
|
+
*/
|
|
626
|
+
const RESTORE_OWNER = "baerly-restore";
|
|
627
|
+
const RESTORE_ARGS = {
|
|
628
|
+
bucket: {
|
|
629
|
+
type: "string",
|
|
630
|
+
required: true,
|
|
631
|
+
description: "Target bucket URI (s3://<bucket>[/<prefix>], file:///<abs>, memory://<bucket>)",
|
|
632
|
+
valueHint: "bucket-uri"
|
|
633
|
+
},
|
|
634
|
+
app: APP_ARG,
|
|
635
|
+
tenant: TENANT_ARG,
|
|
636
|
+
collection: {
|
|
637
|
+
type: "string",
|
|
638
|
+
required: true,
|
|
639
|
+
description: "Target collection name.",
|
|
640
|
+
valueHint: "name"
|
|
641
|
+
},
|
|
642
|
+
force: {
|
|
643
|
+
type: "boolean",
|
|
644
|
+
description: "Truncate the target if it exists (bump fence + reseed current.json)."
|
|
645
|
+
},
|
|
646
|
+
json: JSON_ARG
|
|
647
|
+
};
|
|
648
|
+
const tailFromListedLogKeys = async (storage, collectionPrefix) => {
|
|
649
|
+
const logPrefix = `${collectionPrefix}/log/`;
|
|
650
|
+
let maxSeq = -1;
|
|
651
|
+
for await (const entry of storage.list(logPrefix)) {
|
|
652
|
+
const tail = entry.key.slice(logPrefix.length);
|
|
653
|
+
const match = /^(\d+)\.json$/.exec(tail);
|
|
654
|
+
if (match === null) continue;
|
|
655
|
+
const seq = Number.parseInt(match[1], 10);
|
|
656
|
+
if (Number.isFinite(seq) && seq > maxSeq) maxSeq = seq;
|
|
657
|
+
}
|
|
658
|
+
return maxSeq + 1;
|
|
659
|
+
};
|
|
660
|
+
const bundle = defineBaerlySubcommand({
|
|
661
|
+
name: "admin.restore",
|
|
662
|
+
meta: { description: "Bulk-import canonical NDJSON into a fresh collection." },
|
|
663
|
+
args: RESTORE_ARGS,
|
|
664
|
+
handler: async (args, ctx) => {
|
|
665
|
+
const bucket = await parseBucketUri(args.bucket);
|
|
666
|
+
const { app, tenant } = await ctx.resolveAppTenant({
|
|
667
|
+
app: args.app,
|
|
668
|
+
tenant: args.tenant
|
|
669
|
+
});
|
|
670
|
+
assertCollectionArg(args.collection, "baerly admin restore");
|
|
671
|
+
const currentJsonKey = `${bucket.keyPrefix}app/${app}/tenant/${tenant}/manifests/${args.collection}/current.json`;
|
|
672
|
+
let baseSeq = 0;
|
|
673
|
+
const head = await readCurrentJson(bucket.storage, currentJsonKey);
|
|
674
|
+
if (head !== null && args.force !== true) throw new BaerlyError("Conflict", `baerly admin restore: ${currentJsonKey} exists; pass --force to truncate`, void 0, void 0, void 0, "Pass --force to truncate the existing collection, or choose an empty target.", false);
|
|
675
|
+
if (head !== null) {
|
|
676
|
+
const collectionPrefix = currentJsonKey.slice(0, currentJsonKey.lastIndexOf("/"));
|
|
677
|
+
const truncatedNext = await tailFromListedLogKeys(bucket.storage, collectionPrefix);
|
|
678
|
+
baseSeq = truncatedNext;
|
|
679
|
+
const reseeded = {
|
|
680
|
+
schema_version: 3,
|
|
681
|
+
snapshot: null,
|
|
682
|
+
tail_hint: truncatedNext,
|
|
683
|
+
log_seq_start: truncatedNext,
|
|
684
|
+
writer_fence: {
|
|
685
|
+
epoch: head.json.writer_fence.epoch + 1,
|
|
686
|
+
owner: RESTORE_OWNER,
|
|
687
|
+
claimed_at: ""
|
|
688
|
+
},
|
|
689
|
+
snapshot_bytes: 0,
|
|
690
|
+
snapshot_rows: 0
|
|
691
|
+
};
|
|
692
|
+
try {
|
|
693
|
+
await bucket.storage.put(currentJsonKey, encodeJsonBytes(reseeded), {
|
|
694
|
+
ifMatch: head.etag,
|
|
695
|
+
contentType: "application/json"
|
|
696
|
+
});
|
|
697
|
+
} catch (error) {
|
|
698
|
+
if (error instanceof BaerlyError) throw error;
|
|
699
|
+
throw new BaerlyError("NetworkError", `baerly admin restore: failed to reseed current.json: ${error.message}`);
|
|
700
|
+
}
|
|
701
|
+
} else {
|
|
702
|
+
const seed = {
|
|
703
|
+
schema_version: 3,
|
|
704
|
+
snapshot: null,
|
|
705
|
+
tail_hint: 0,
|
|
706
|
+
log_seq_start: 0,
|
|
707
|
+
writer_fence: {
|
|
708
|
+
epoch: 0,
|
|
709
|
+
owner: RESTORE_OWNER,
|
|
710
|
+
claimed_at: ""
|
|
711
|
+
},
|
|
712
|
+
snapshot_bytes: 0,
|
|
713
|
+
snapshot_rows: 0
|
|
714
|
+
};
|
|
715
|
+
await createCurrentJson(bucket.storage, currentJsonKey, seed);
|
|
716
|
+
}
|
|
717
|
+
const writer = new Writer({
|
|
718
|
+
storage: bucket.storage,
|
|
719
|
+
currentJsonKey
|
|
720
|
+
});
|
|
721
|
+
const rl = createInterface({
|
|
722
|
+
input: ctx.streams?.stdin ?? process.stdin,
|
|
723
|
+
crlfDelay: Infinity
|
|
724
|
+
});
|
|
725
|
+
let count = 0;
|
|
726
|
+
let lineNo = 0;
|
|
727
|
+
for await (const line of rl) {
|
|
728
|
+
lineNo++;
|
|
729
|
+
if (line.length === 0) continue;
|
|
730
|
+
let parsed;
|
|
731
|
+
try {
|
|
732
|
+
parsed = JSON.parse(line);
|
|
733
|
+
} catch (error) {
|
|
734
|
+
throw new Error(`baerly admin restore: line ${lineNo} is not valid JSON: ${error.message}`, { cause: error });
|
|
735
|
+
}
|
|
736
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(`baerly admin restore: line ${lineNo} is not a JSON object`);
|
|
737
|
+
const row = parsed;
|
|
738
|
+
const id = row["_id"];
|
|
739
|
+
if (typeof id !== "string" || id.length === 0) throw new Error(`baerly admin restore: line ${lineNo} missing non-empty string _id`);
|
|
740
|
+
const body = row;
|
|
741
|
+
await writer.commit({
|
|
742
|
+
op: "I",
|
|
743
|
+
collection: args.collection,
|
|
744
|
+
docId: id,
|
|
745
|
+
body
|
|
746
|
+
});
|
|
747
|
+
count++;
|
|
748
|
+
}
|
|
749
|
+
if (count > 0) {
|
|
750
|
+
const afterLoad = await readCurrentJson(bucket.storage, currentJsonKey);
|
|
751
|
+
if (afterLoad !== null) {
|
|
752
|
+
const clampedTail = Math.max(baseSeq + count, afterLoad.json.tail_hint, afterLoad.json.log_seq_start);
|
|
753
|
+
const stamped = {
|
|
754
|
+
...afterLoad.json,
|
|
755
|
+
tail_hint: clampedTail
|
|
756
|
+
};
|
|
757
|
+
await bucket.storage.put(currentJsonKey, encodeJsonBytes(stamped), {
|
|
758
|
+
ifMatch: afterLoad.etag,
|
|
759
|
+
contentType: "application/json"
|
|
760
|
+
});
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
emitSuccess({
|
|
764
|
+
command: "admin.restore",
|
|
765
|
+
status: "ok",
|
|
766
|
+
collection: args.collection,
|
|
767
|
+
restored: count
|
|
768
|
+
});
|
|
769
|
+
return 0;
|
|
770
|
+
}
|
|
771
|
+
});
|
|
772
|
+
/** citty `defineCommand` block for `baerly admin restore`. */
|
|
773
|
+
const restoreCmd = bundle.cmd;
|
|
774
|
+
bundle.run;
|
|
775
|
+
//#endregion
|
|
776
|
+
export { restoreCmd };
|
|
777
|
+
|
|
778
|
+
//# sourceMappingURL=restore-R-4rbciV.js.map
|