@gusto/baerly-storage 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +29 -0
- package/README.md +176 -70
- package/dist/API.md +213 -139
- package/dist/CHANGELOG.md +434 -0
- package/dist/RECIPES.md +125 -0
- package/dist/THIRD-PARTY-LICENSES.txt +238 -0
- package/dist/{app-config-C0Bbw_Z6.d.ts → app-config-DyiFK3_k.d.ts} +55 -11
- package/dist/app-config.d.ts +1 -1
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +301 -301
- package/dist/auth.js.map +1 -1
- package/dist/aws4fetch.esm-p7eEzLCs.js +1702 -0
- package/dist/aws4fetch.esm-p7eEzLCs.js.map +1 -0
- package/dist/baerly.js +35 -15
- package/dist/baerly.js.map +1 -1
- package/dist/baerly.spec.json +342 -0
- package/dist/bytes-BXdamO3L.js.map +1 -1
- package/dist/{client-DYZfoZb9.d.ts → client-Fe8CwB73.d.ts} +10 -10
- package/dist/client-react.d.ts +3 -3
- package/dist/client-react.js +4 -4
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +4 -4
- package/dist/client.js +3 -3
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-CINJPKCW.js → cloudflare-DzKBrUlg.js} +178 -2
- package/dist/cloudflare-DzKBrUlg.js.map +1 -0
- package/dist/cloudflare.d.ts +49 -6
- package/dist/cloudflare.js +59 -26
- package/dist/cloudflare.js.map +1 -1
- package/dist/config-1tv0-Yzm.js +2693 -0
- package/dist/config-1tv0-Yzm.js.map +1 -0
- package/dist/constants-CfeMWQWk.js +222 -0
- package/dist/constants-CfeMWQWk.js.map +1 -0
- package/dist/context-DRZ3tZ5l.js.map +1 -1
- package/dist/{cost-CQt1YRkA.js → cost-DSMIVqzL.js} +43 -42
- package/dist/cost-DSMIVqzL.js.map +1 -0
- package/dist/current-json-C6nhPOm1.js +190 -0
- package/dist/current-json-C6nhPOm1.js.map +1 -0
- package/dist/{db-BM3y2JQw.d.ts → db-DsPkOVEH.d.ts} +26 -108
- package/dist/{deploy-CrAbfgCq.js → deploy-DonTr-CV.js} +12 -5
- package/dist/deploy-DonTr-CV.js.map +1 -0
- package/dist/dev-vite.d.ts +11 -7
- package/dist/dev-vite.js +7 -6
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/{doctor-Db-hzPmE.js → doctor-DSIyCDg_.js} +14 -136
- package/dist/doctor-DSIyCDg_.js.map +1 -0
- package/dist/{dump-BYA55NLP.js → dump-Bh5k26VQ.js} +10 -31
- package/dist/dump-Bh5k26VQ.js.map +1 -0
- package/dist/{errors-DOdL95kK.js → errors-Dfa3p78c.js} +25 -5
- package/dist/errors-Dfa3p78c.js.map +1 -0
- package/dist/{export-Bhbz90-z.js → export-DtF5tDIM.js} +11 -32
- package/dist/{export-Bhbz90-z.js.map → export-DtF5tDIM.js.map} +1 -1
- package/dist/{fsck-CPPZWt9p.js → fsck-D_djMO5D.js} +28 -39
- package/dist/fsck-D_djMO5D.js.map +1 -0
- package/dist/{http-Cw8p6y5b.js → http-DxM9MflO.js} +151 -79
- package/dist/http-DxM9MflO.js.map +1 -0
- package/dist/http.d.ts +42 -26
- package/dist/http.js +3 -2
- package/dist/{index-DwDTWPSU.d.ts → index-0rZcMBwy.d.ts} +10 -7
- package/dist/{index-C3JAH1Yo.d.ts → index-B6oexfOB.d.ts} +9 -9
- package/dist/{index-cUY-36AN.d.ts → index-BNRh23K9.d.ts} +21 -5
- package/dist/{index-DSvwEUeF.d.ts → index-BYk9oCNf.d.ts} +240 -202
- package/dist/index.d.ts +6 -6
- package/dist/index.js +5 -5
- package/dist/{indexes-DAu2AkTW.d.ts → indexes-Dn9hbGQG.d.ts} +2 -2
- package/dist/{inspect-Bdbzvi-_.js → inspect-6WZ-pC7-.js} +21 -36
- package/dist/inspect-6WZ-pC7-.js.map +1 -0
- package/dist/key-Di6yyxFu.js +35 -0
- package/dist/key-Di6yyxFu.js.map +1 -0
- package/dist/{snapshot-SS8USDUF.js → log-tail-D98U1JBo.js} +179 -84
- package/dist/log-tail-D98U1JBo.js.map +1 -0
- package/dist/{maintenance-ClzBC1Vv.js → maintenance-B4Dki6Ig.js} +438 -176
- package/dist/maintenance-B4Dki6Ig.js.map +1 -0
- package/dist/{maintenance-BRrThz8V.d.ts → maintenance-B_e0T99L.d.ts} +114 -26
- package/dist/{maintenance-BvOv6BMM.js → maintenance-C7coQp4c.js} +217 -94
- package/dist/maintenance-C7coQp4c.js.map +1 -0
- package/dist/maintenance.d.ts +3 -2
- package/dist/maintenance.js +3 -2
- package/dist/node.d.ts +176 -68
- package/dist/node.js +3 -2
- package/dist/{normalize-XWY25_sG.js → normalize-rjnZLwS4.js} +2 -2
- package/dist/normalize-rjnZLwS4.js.map +1 -0
- package/dist/{observability-CpOiEN0k.js → observability-wzs3-BL_.js} +279 -109
- package/dist/observability-wzs3-BL_.js.map +1 -0
- package/dist/observability.d.ts +3 -3
- package/dist/observability.js +1 -1
- package/dist/{plan-DOTl21vq.js → plan-a7PJHBCy.js} +7 -4
- package/dist/plan-a7PJHBCy.js.map +1 -0
- package/dist/{query-BmiQy328.js → query-rAtXNQQB.js} +1063 -1202
- package/dist/query-rAtXNQQB.js.map +1 -0
- package/dist/{rebuild-index-DC_nR6wP.js → rebuild-index-BVzQSZlm.js} +10 -32
- package/dist/rebuild-index-BVzQSZlm.js.map +1 -0
- package/dist/{request-BEF_OvMh.js → request-u0M51mhJ.js} +3 -3
- package/dist/{request-BEF_OvMh.js.map → request-u0M51mhJ.js.map} +1 -1
- package/dist/restore-R-4rbciV.js +778 -0
- package/dist/restore-R-4rbciV.js.map +1 -0
- package/dist/runtime-spec-qL3_lTTU.js +391 -0
- package/dist/runtime-spec-qL3_lTTU.js.map +1 -0
- package/dist/s3-http-B-z8th6O.d.ts +60 -0
- package/dist/s3.d.ts +47 -0
- package/dist/s3.js +45 -0
- package/dist/s3.js.map +1 -0
- package/dist/{shared-secret-BJdSwpgw.js → shared-secret-DgEicd3x.js} +13 -5
- package/dist/shared-secret-DgEicd3x.js.map +1 -0
- package/dist/src-BG20LZs7.js +2271 -0
- package/dist/src-BG20LZs7.js.map +1 -0
- package/dist/{src-zhEAmmF7.js → src-CYHFDrKz.js} +82 -99
- package/dist/src-CYHFDrKz.js.map +1 -0
- package/dist/{src-QQ7jDeL2.js → src-DYZ4H1aF.js} +44 -23
- package/dist/src-DYZ4H1aF.js.map +1 -0
- package/dist/subcommand-DoDA-K7d.js +197 -0
- package/dist/subcommand-DoDA-K7d.js.map +1 -0
- package/dist/time-DUqvFMHq.js +119 -0
- package/dist/time-DUqvFMHq.js.map +1 -0
- package/dist/{usage-CLjirdD-.js → usage-DBV1eQHC.js} +7 -22
- package/dist/usage-DBV1eQHC.js.map +1 -0
- package/dist/{validate-DCq_0DIn.js → validate-DtljsCh1.js} +51 -51
- package/dist/{validate-DCq_0DIn.js.map → validate-DtljsCh1.js.map} +1 -1
- package/dist/wire-izfGZDvr.js.map +1 -1
- package/package.json +77 -42
- package/dist/cloudflare-CINJPKCW.js.map +0 -1
- package/dist/config-DVfDGyw1.js +0 -278
- package/dist/config-DVfDGyw1.js.map +0 -1
- package/dist/cost-CQt1YRkA.js.map +0 -1
- package/dist/current-json-C-EyJqs0.js +0 -354
- package/dist/current-json-C-EyJqs0.js.map +0 -1
- package/dist/deploy-CrAbfgCq.js.map +0 -1
- package/dist/doctor-Db-hzPmE.js.map +0 -1
- package/dist/dump-BYA55NLP.js.map +0 -1
- package/dist/errors-DOdL95kK.js.map +0 -1
- package/dist/fsck-CPPZWt9p.js.map +0 -1
- package/dist/http-Cw8p6y5b.js.map +0 -1
- package/dist/inspect-Bdbzvi-_.js.map +0 -1
- package/dist/maintenance-BvOv6BMM.js.map +0 -1
- package/dist/maintenance-ClzBC1Vv.js.map +0 -1
- package/dist/normalize-XWY25_sG.js.map +0 -1
- package/dist/observability-CpOiEN0k.js.map +0 -1
- package/dist/plan-DOTl21vq.js.map +0 -1
- package/dist/query-BmiQy328.js.map +0 -1
- package/dist/rebuild-index-DC_nR6wP.js.map +0 -1
- package/dist/restore-C6IcmLeA.js +0 -771
- package/dist/restore-C6IcmLeA.js.map +0 -1
- package/dist/shared-secret-BJdSwpgw.js.map +0 -1
- package/dist/snapshot-SS8USDUF.js.map +0 -1
- package/dist/src-DU1sCMlr.js +0 -6236
- package/dist/src-DU1sCMlr.js.map +0 -1
- package/dist/src-QQ7jDeL2.js.map +0 -1
- package/dist/src-zhEAmmF7.js.map +0 -1
- package/dist/subcommand-DZcHcsYp.js +0 -5122
- package/dist/subcommand-DZcHcsYp.js.map +0 -1
- package/dist/usage-CLjirdD-.js.map +0 -1
- /package/dist/{chunk-EAsCxrDo.js → rolldown-runtime-EAsCxrDo.js} +0 -0
package/dist/restore-C6IcmLeA.js
DELETED
|
@@ -1,771 +0,0 @@
|
|
|
1
|
-
import { n as encodeJsonBytes, t as decodeJsonBytes } from "./bytes-BXdamO3L.js";
|
|
2
|
-
import { a as readLogEntry, d as logSeqStartOf, f as readCurrentJson, o as walkLogRange, u as createCurrentJson } from "./snapshot-SS8USDUF.js";
|
|
3
|
-
import { a as getCurrentContext, c as noopMetricsRecorder, l as versionFromContent, n as runBoundedMaintenance, o as allIndexKeysFor, r as shouldFireMaintenance, s as validateIndexDefinition, t as dispatchInlineAwaited } from "./maintenance-BvOv6BMM.js";
|
|
4
|
-
import { a as uuid, i as countKey, n as parseBucketUri, r as timestamp, t as defineBaerlySubcommand } from "./subcommand-DZcHcsYp.js";
|
|
5
|
-
import { o as BaerlyError, r as emitSuccess } from "./baerly.js";
|
|
6
|
-
import { createInterface } from "node:readline";
|
|
7
|
-
//#region ../server/src/log-conflict-adoption.ts
|
|
8
|
-
/**
|
|
9
|
-
* Decide whether the entry already at `/log/<seq>.json` is the
|
|
10
|
-
* writer's own prior in-flight attempt and may be adopted. Pure
|
|
11
|
-
* function; no I/O. The caller performs the GET (when the log PUT
|
|
12
|
-
* surfaces 412) and passes the read-back body in `ctx.existing`.
|
|
13
|
-
*
|
|
14
|
-
* @see {@link AdoptionDecision} for the return shape.
|
|
15
|
-
* @see The soundness invariant in this module's header docstring.
|
|
16
|
-
*/
|
|
17
|
-
const tryAdoptOwnSessionLogEntry = (ctx) => {
|
|
18
|
-
if (ctx.batchSize !== 1) return {
|
|
19
|
-
adopt: false,
|
|
20
|
-
reason: "batch"
|
|
21
|
-
};
|
|
22
|
-
if (ctx.existing.session !== ctx.self.session) return {
|
|
23
|
-
adopt: false,
|
|
24
|
-
reason: "foreign-session"
|
|
25
|
-
};
|
|
26
|
-
if (ctx.existing.seq !== ctx.self.seq) return {
|
|
27
|
-
adopt: false,
|
|
28
|
-
reason: "wrong-seq"
|
|
29
|
-
};
|
|
30
|
-
return {
|
|
31
|
-
adopt: true,
|
|
32
|
-
entry: ctx.existing
|
|
33
|
-
};
|
|
34
|
-
};
|
|
35
|
-
//#endregion
|
|
36
|
-
//#region ../server/src/writer.ts
|
|
37
|
-
/**
|
|
38
|
-
* `Writer` — stateless multi-instance write engine.
|
|
39
|
-
*
|
|
40
|
-
* Each {@link Writer.commit} call reads `current.json` FRESH from
|
|
41
|
-
* the bucket, mints the next {@link LogEntry}, PUTs the content body
|
|
42
|
-
* and the log entry, and CAS-advances `current.json` with `If-Match`.
|
|
43
|
-
* Up to {@link S3_REQUEST_MAX_RETRIES} attempts on contention before
|
|
44
|
-
* surfacing `BaerlyError{code:"Conflict"}`. The optional integrity
|
|
45
|
-
* walk over the live log tail is gated by
|
|
46
|
-
* {@link WriterOptions.verifyLogIntegrityOnCommit} (default off
|
|
47
|
-
* — see the option's docstring).
|
|
48
|
-
*
|
|
49
|
-
* The instance carries no per-write cache: every commit re-reads
|
|
50
|
-
* `current.json`, so N stateless server instances writing the same
|
|
51
|
-
* tenant prefix contend at exactly one place — the conditional PUT
|
|
52
|
-
* on `current.json` — and one loses cleanly with a 412.
|
|
53
|
-
*
|
|
54
|
-
* **Manifest-first ordering is REVERSED relative to the legacy
|
|
55
|
-
* `src/syncer.ts` write loop.** Old loop: PUT manifest → PUT content
|
|
56
|
-
* → CAS. New loop: PUT content → PUT log entry → CAS-advance
|
|
57
|
-
* `current.json`. A crashed mid-loop writer leaves an unreferenced
|
|
58
|
-
* content body (no log entry points at it), not an orphan log entry
|
|
59
|
-
* with missing content. The compactor sweeps the orphan content
|
|
60
|
-
* later.
|
|
61
|
-
*
|
|
62
|
-
* **`LogEntry` shape.** Emitted entries follow the contract in
|
|
63
|
-
* `packages/protocol/src/log.ts` (see {@link LogEntry}) and
|
|
64
|
-
* `docs/spec/log-entry-shape.md`. The on-bucket key is
|
|
65
|
-
* `log/<seq>.json` (the integer `seq` is the load-bearing
|
|
66
|
-
* identifier; the `lsn` string is schema-only).
|
|
67
|
-
*
|
|
68
|
-
* @see docs/spec/sync-protocol.md for the protocol invariants this
|
|
69
|
-
* loop preserves.
|
|
70
|
-
* @see docs/spec/log-entry-shape.md for the per-field contract.
|
|
71
|
-
*/
|
|
72
|
-
const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
|
|
73
|
-
const DEFAULT_INITIAL_BACKOFF_MS = 25;
|
|
74
|
-
const MAX_BACKOFF_MS = 1500;
|
|
75
|
-
const APPLICATION_JSON = "application/json";
|
|
76
|
-
/**
|
|
77
|
-
* Empty body for index entries. Each index entry is a fact ("doc
|
|
78
|
-
* `<docId>` has `<field> = <value>`"), not data — readers list the
|
|
79
|
-
* prefix, extract the doc id, then GET the content body separately.
|
|
80
|
-
* Pre-allocated module-level constant so every index PUT shares one
|
|
81
|
-
* zero-length buffer.
|
|
82
|
-
*/
|
|
83
|
-
const EMPTY_BODY = new Uint8Array(0);
|
|
84
|
-
/**
|
|
85
|
-
* Stateless write engine for the multi-instance core.
|
|
86
|
-
*
|
|
87
|
-
* Construction is cheap and performs zero I/O — adapters build a
|
|
88
|
-
* fresh `Writer` per request and discard it. All real work
|
|
89
|
-
* happens in {@link commit}.
|
|
90
|
-
*
|
|
91
|
-
* @example
|
|
92
|
-
* ```ts
|
|
93
|
-
* import { Writer } from "@gusto/baerly-storage";
|
|
94
|
-
* import { MemoryStorage } from "@gusto/baerly-storage";
|
|
95
|
-
*
|
|
96
|
-
* const writer = new Writer({
|
|
97
|
-
* storage: new MemoryStorage(),
|
|
98
|
-
* currentJsonKey: "app/tickets/tenant/acme/manifests/tickets/current.json",
|
|
99
|
-
* });
|
|
100
|
-
*
|
|
101
|
-
* const result = await writer.commit({
|
|
102
|
-
* op: "I",
|
|
103
|
-
* collection: "tickets",
|
|
104
|
-
* docId: "doc-1",
|
|
105
|
-
* body: { _id: "doc-1", title: "hello" },
|
|
106
|
-
* });
|
|
107
|
-
* console.log(result.entry.seq, result.attempts);
|
|
108
|
-
* ```
|
|
109
|
-
*/
|
|
110
|
-
var Writer = class {
|
|
111
|
-
#storage;
|
|
112
|
-
#currentJsonKey;
|
|
113
|
-
#maxRetries;
|
|
114
|
-
#initialBackoffMs;
|
|
115
|
-
#random;
|
|
116
|
-
#indexes;
|
|
117
|
-
#verifyLogIntegrityOnCommit;
|
|
118
|
-
constructor(opts) {
|
|
119
|
-
this.#storage = opts.storage;
|
|
120
|
-
this.#currentJsonKey = opts.currentJsonKey;
|
|
121
|
-
this.#maxRetries = opts.options?.maxRetries ?? 8;
|
|
122
|
-
this.#initialBackoffMs = opts.options?.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS;
|
|
123
|
-
this.#random = opts.options?.random ?? Math.random;
|
|
124
|
-
const indexes = opts.options?.indexes ?? [];
|
|
125
|
-
for (const def of indexes) validateIndexDefinition(def);
|
|
126
|
-
this.#indexes = indexes;
|
|
127
|
-
this.#verifyLogIntegrityOnCommit = opts.options?.verifyLogIntegrityOnCommit ?? false;
|
|
128
|
-
}
|
|
129
|
-
/**
|
|
130
|
-
* Atomically commit one mutation. Reads `current.json` fresh, mints
|
|
131
|
-
* the next log entry at `seq = current.next_seq`, PUTs the content
|
|
132
|
-
* body and the log entry, then CAS-advances `current.json` with
|
|
133
|
-
* `If-Match`. Retries on conflict up to `maxRetries`.
|
|
134
|
-
*
|
|
135
|
-
* The hot-path cost under no contention is 1 GET + 3 PUTs (4 ops);
|
|
136
|
-
* an in-flight peer write costs one extra GET + the backoff sleep
|
|
137
|
-
* per retry.
|
|
138
|
-
*
|
|
139
|
-
* Idempotency: PUT content uses `If-None-Match: "*"`, so a retry of
|
|
140
|
-
* the same logical write produces the same content key and the
|
|
141
|
-
* second PUT no-ops. PUT log entry also uses `If-None-Match: "*"`,
|
|
142
|
-
* so two writers racing the same `next_seq` cause exactly one PUT
|
|
143
|
-
* to land — the loser falls into the CAS retry path.
|
|
144
|
-
*
|
|
145
|
-
* @throws BaerlyError code="Conflict" when the retry budget is
|
|
146
|
-
* exhausted (genuine high-contention case), or when the underlying
|
|
147
|
-
* `current.json` CAS PUT lost.
|
|
148
|
-
* @throws BaerlyError code="Conflict" — the writer fence
|
|
149
|
-
* (`current.json.writer_fence.epoch`) was bumped by a concurrent
|
|
150
|
-
* `claimWriter` call between this commit's read of
|
|
151
|
-
* `current.json` and its CAS-advance. The stale writer aborts
|
|
152
|
-
* to honour the new authority — the kernel does NOT retry under
|
|
153
|
-
* the old epoch. See {@link claimWriter} for the rotation
|
|
154
|
-
* recipe.
|
|
155
|
-
* @throws BaerlyError code="Internal" when a log entry expected in
|
|
156
|
-
* `[0, next_seq)` is missing — a protocol-invariant violation
|
|
157
|
-
* (compactor bug or stale `current.json`).
|
|
158
|
-
* @throws BaerlyError code="InvalidResponse" when a log-entry body
|
|
159
|
-
* isn't valid JSON.
|
|
160
|
-
*
|
|
161
|
-
* **Implicit provisioning.** When `current.json` does not exist at
|
|
162
|
-
* `currentJsonKey` (fresh bucket, fresh collection), the writer
|
|
163
|
-
* auto-creates it with a zero-state manifest before the commit
|
|
164
|
-
* lands. The cost is one extra Class A PUT on the very first
|
|
165
|
-
* commit per collection; steady-state cost is unchanged. A peer
|
|
166
|
-
* racing the same first-write loses cleanly on the create CAS
|
|
167
|
-
* and re-reads to pick up the winner's manifest.
|
|
168
|
-
*/
|
|
169
|
-
async commit(input) {
|
|
170
|
-
const session = uuid().slice(0, 6);
|
|
171
|
-
const logPrefix = this.#currentJsonKey.slice(0, this.#currentJsonKey.lastIndexOf("/"));
|
|
172
|
-
for (let attempt = 1; attempt <= this.#maxRetries; attempt++) {
|
|
173
|
-
let success;
|
|
174
|
-
try {
|
|
175
|
-
success = await this.#singleAttemptCommit([input], session, logPrefix, "Writer", true);
|
|
176
|
-
} catch (error) {
|
|
177
|
-
if (!isPreconditionFailed(error)) throw error;
|
|
178
|
-
await this.#backoff(attempt);
|
|
179
|
-
continue;
|
|
180
|
-
}
|
|
181
|
-
await this.#verifyFenceUnchanged(success.expectedEpoch, "Writer");
|
|
182
|
-
this.#emitWriteMetrics(input.collection, success.classAOps + (attempt - 1));
|
|
183
|
-
return {
|
|
184
|
-
entry: success.entries[0],
|
|
185
|
-
currentEtag: success.currentEtag,
|
|
186
|
-
attempts: attempt
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
throw new BaerlyError("Conflict", `Writer: CAS conflict on ${this.#currentJsonKey} after ${this.#maxRetries} attempts`);
|
|
190
|
-
}
|
|
191
|
-
/**
|
|
192
|
-
* Single-attempt batched commit. Reads `current.json` ONCE, mints
|
|
193
|
-
* `inputs.length` log entries with contiguous `seq` numbers
|
|
194
|
-
* starting at the current `next_seq`, PUTs each content body and
|
|
195
|
-
* each log entry, then CAS-advances `current.json` from
|
|
196
|
-
* `next_seq = N` to `next_seq = N + inputs.length` with
|
|
197
|
-
* `If-Match`.
|
|
198
|
-
*
|
|
199
|
-
* NO retry on CAS loss. On `current.json` 412, throws
|
|
200
|
-
* `BaerlyError{code:"Conflict"}` immediately — the caller (here:
|
|
201
|
-
* `Db.transaction`) decides whether to retry by re-running the
|
|
202
|
-
* body, or to surface to the app. Likewise on a log-entry 412
|
|
203
|
-
* (a peer wrote our seq).
|
|
204
|
-
*
|
|
205
|
-
* Empty `inputs`: returns `{ entries: [], currentEtag: undefined }`
|
|
206
|
-
* after zero storage operations.
|
|
207
|
-
*
|
|
208
|
-
* @throws BaerlyError code="Conflict" — CAS lost on `current.json`,
|
|
209
|
-
* or a log entry already exists at our seq (peer wrote ahead).
|
|
210
|
-
* @throws BaerlyError code="Conflict" — the writer fence
|
|
211
|
-
* (`current.json.writer_fence.epoch`) was bumped by a concurrent
|
|
212
|
-
* `claimWriter` call between this batch's read of
|
|
213
|
-
* `current.json` and its CAS-advance. The stale writer aborts
|
|
214
|
-
* to honour the new authority — the kernel does NOT retry under
|
|
215
|
-
* the old epoch. See {@link claimWriter} for the rotation
|
|
216
|
-
* recipe.
|
|
217
|
-
* @throws BaerlyError code="Internal" — protocol-invariant violation
|
|
218
|
-
* (missing log entry inside `[0, next_seq)`).
|
|
219
|
-
* @throws BaerlyError code="InvalidResponse" — malformed log body.
|
|
220
|
-
*
|
|
221
|
-
* **Implicit provisioning.** Like {@link commit}, this path auto-
|
|
222
|
-
* creates `current.json` with a zero-state manifest on a missing
|
|
223
|
-
* read. Because `commitBatch` does NOT retry on CAS loss, a peer
|
|
224
|
-
* racing the same first-write surfaces `Conflict` to the caller
|
|
225
|
-
* (`Db.transaction` re-runs the body); the manifest is durable
|
|
226
|
-
* regardless of which writer won the create.
|
|
227
|
-
*/
|
|
228
|
-
async commitBatch(inputs) {
|
|
229
|
-
if (inputs.length === 0) return {
|
|
230
|
-
entries: [],
|
|
231
|
-
currentEtag: void 0
|
|
232
|
-
};
|
|
233
|
-
const session = uuid().slice(0, 6);
|
|
234
|
-
const logPrefix = this.#currentJsonKey.slice(0, this.#currentJsonKey.lastIndexOf("/"));
|
|
235
|
-
const success = await this.#singleAttemptCommit(inputs, session, logPrefix, "Writer.commitBatch", false);
|
|
236
|
-
await this.#verifyFenceUnchanged(success.expectedEpoch, "Writer.commitBatch");
|
|
237
|
-
this.#emitWriteMetrics(inputs[0].collection, success.classAOps);
|
|
238
|
-
return {
|
|
239
|
-
entries: success.entries,
|
|
240
|
-
currentEtag: success.currentEtag
|
|
241
|
-
};
|
|
242
|
-
}
|
|
243
|
-
/**
|
|
244
|
-
* One full commit attempt — the shared body of {@link commit} and
|
|
245
|
-
* {@link commitBatch}. Reads `current.json`, walks the log, mints N
|
|
246
|
-
* entries, PUTs content + indexes in parallel, PUTs log entries,
|
|
247
|
-
* CAS-advances `current.json`. Returns the success payload + the
|
|
248
|
-
* pre-commit fence epoch so the caller can run
|
|
249
|
-
* {@link #verifyFenceUnchanged} outside the retry-catch arm.
|
|
250
|
-
*
|
|
251
|
-
* Retryable failures (a peer wrote our seq; a peer won the CAS on
|
|
252
|
-
* `current.json`) throw `BaerlyError{code:"Conflict"}`. The caller
|
|
253
|
-
* catches via {@link isPreconditionFailed} and either retries
|
|
254
|
-
* (`commit`) or surfaces (`commitBatch`). Fence-bump and protocol-
|
|
255
|
-
* invariant violations are NOT thrown from this helper — the fence
|
|
256
|
-
* check lives in the caller so its `Conflict` propagates past the
|
|
257
|
-
* retry-catch arm. Other thrown `BaerlyError`s (Internal,
|
|
258
|
-
* InvalidResponse, NetworkError) bypass `isPreconditionFailed` and
|
|
259
|
-
* propagate naturally.
|
|
260
|
-
*
|
|
261
|
-
* Crash safety invariant: content / index PUTs are awaited before
|
|
262
|
-
* log PUTs, which are awaited before the CAS. A crashed mid-attempt
|
|
263
|
-
* writer leaves orphan content / index entries (no log entry
|
|
264
|
-
* references them) — the compactor sweeps the orphans. The
|
|
265
|
-
* inverse — orphan log entry with missing content — is never
|
|
266
|
-
* produced.
|
|
267
|
-
*/
|
|
268
|
-
async #singleAttemptCommit(inputs, session, logPrefix, errorPrefix, adoptOwnSessionOnLogConflict) {
|
|
269
|
-
const read = await readCurrentJson(this.#storage, this.#currentJsonKey) ?? await this.#provisionCurrentJson();
|
|
270
|
-
const current = read.json;
|
|
271
|
-
const baseEtag = read.etag;
|
|
272
|
-
const expectedEpoch = current.writer_fence.epoch;
|
|
273
|
-
if (this.#verifyLogIntegrityOnCommit) await this.#walkLog(logPrefix, logSeqStartOf(current), current.next_seq);
|
|
274
|
-
const commitNowMs = Date.now();
|
|
275
|
-
const commitTs = new Date(commitNowMs).toISOString();
|
|
276
|
-
const lsnTimestamp = timestamp(commitNowMs);
|
|
277
|
-
const entries = [];
|
|
278
|
-
for (let i = 0; i < inputs.length; i++) {
|
|
279
|
-
const input = inputs[i];
|
|
280
|
-
const seq = current.next_seq + i;
|
|
281
|
-
const entry = {
|
|
282
|
-
lsn: `${lsnTimestamp}_${session}_${countKey(seq)}`,
|
|
283
|
-
commit_ts: commitTs,
|
|
284
|
-
op: input.op,
|
|
285
|
-
collection: input.collection,
|
|
286
|
-
doc_id: input.docId,
|
|
287
|
-
session,
|
|
288
|
-
seq,
|
|
289
|
-
...input.op !== "D" && input.body !== void 0 ? { after: input.body } : {},
|
|
290
|
-
...input.origin !== void 0 ? { origin: input.origin } : {}
|
|
291
|
-
};
|
|
292
|
-
entries.push(entry);
|
|
293
|
-
}
|
|
294
|
-
let contentPutCount = 0;
|
|
295
|
-
let indexClassA = 0;
|
|
296
|
-
const parallelPuts = [];
|
|
297
|
-
const inBatchImage = /* @__PURE__ */ new Map();
|
|
298
|
-
for (const input of inputs) {
|
|
299
|
-
if (input.op !== "D" && input.body !== void 0) {
|
|
300
|
-
contentPutCount++;
|
|
301
|
-
const bytes = encodeJsonBytes(input.body);
|
|
302
|
-
const contentKey = `${logPrefix}/content/${await versionFromContent(bytes)}.json`;
|
|
303
|
-
parallelPuts.push(this.#storage.put(contentKey, bytes, {
|
|
304
|
-
ifNoneMatch: "*",
|
|
305
|
-
contentType: APPLICATION_JSON
|
|
306
|
-
}).catch((error) => {
|
|
307
|
-
this.#observe429(error, input.collection);
|
|
308
|
-
if (isPreconditionFailed(error)) return;
|
|
309
|
-
throw error;
|
|
310
|
-
}));
|
|
311
|
-
}
|
|
312
|
-
if (this.#indexes.length > 0) {
|
|
313
|
-
let newKeys = [];
|
|
314
|
-
let staleKeys = [];
|
|
315
|
-
if (input.op === "I") newKeys = allIndexKeysFor(logPrefix, this.#indexes, input.body, input.docId);
|
|
316
|
-
else if (input.op === "U") {
|
|
317
|
-
const preImage = inBatchImage.has(input.docId) ? inBatchImage.get(input.docId) : await this.#readPreImage(logPrefix, input.collection, input.docId, current.next_seq);
|
|
318
|
-
const oldKeys = allIndexKeysFor(logPrefix, this.#indexes, preImage, input.docId);
|
|
319
|
-
newKeys = allIndexKeysFor(logPrefix, this.#indexes, input.body, input.docId);
|
|
320
|
-
const newSet = new Set(newKeys);
|
|
321
|
-
staleKeys = oldKeys.filter((k) => !newSet.has(k));
|
|
322
|
-
} else {
|
|
323
|
-
const preImage = inBatchImage.has(input.docId) ? inBatchImage.get(input.docId) : await this.#readPreImage(logPrefix, input.collection, input.docId, current.next_seq);
|
|
324
|
-
staleKeys = allIndexKeysFor(logPrefix, this.#indexes, preImage, input.docId);
|
|
325
|
-
}
|
|
326
|
-
for (const k of newKeys) parallelPuts.push(this.#storage.put(k, EMPTY_BODY, {
|
|
327
|
-
ifNoneMatch: "*",
|
|
328
|
-
contentType: APPLICATION_JSON
|
|
329
|
-
}).catch((error) => {
|
|
330
|
-
this.#observe429(error, input.collection);
|
|
331
|
-
if (isPreconditionFailed(error)) return;
|
|
332
|
-
throw error;
|
|
333
|
-
}));
|
|
334
|
-
for (const k of staleKeys) parallelPuts.push(this.#storage.delete(k));
|
|
335
|
-
indexClassA += newKeys.length + staleKeys.length;
|
|
336
|
-
if (newKeys.length + staleKeys.length > 0) ctxMetrics().histogram("db.write.index_ops_per_logical_write", newKeys.length + staleKeys.length, { collection: input.collection });
|
|
337
|
-
inBatchImage.set(input.docId, input.op === "D" ? void 0 : input.body);
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
await Promise.all(parallelPuts);
|
|
341
|
-
const logPutOne = async (entry) => {
|
|
342
|
-
const logEntryKey = `${logPrefix}/log/${entry.seq}.json`;
|
|
343
|
-
const logBytes = encodeJsonBytes(entry);
|
|
344
|
-
try {
|
|
345
|
-
await this.#storage.put(logEntryKey, logBytes, {
|
|
346
|
-
ifNoneMatch: "*",
|
|
347
|
-
contentType: APPLICATION_JSON
|
|
348
|
-
});
|
|
349
|
-
return { ok: true };
|
|
350
|
-
} catch (error) {
|
|
351
|
-
this.#observe429(error, entry.collection);
|
|
352
|
-
if (!isPreconditionFailed(error)) throw error;
|
|
353
|
-
ctxMetrics().counter("db.r2.put.412_total", 1, {
|
|
354
|
-
collection: entry.collection,
|
|
355
|
-
step: "log-put"
|
|
356
|
-
});
|
|
357
|
-
return { ok: false };
|
|
358
|
-
}
|
|
359
|
-
};
|
|
360
|
-
const logPutResults = await Promise.all(entries.map(logPutOne));
|
|
361
|
-
let committedEntries = entries;
|
|
362
|
-
const firstConflictIdx = logPutResults.findIndex((r) => !r.ok);
|
|
363
|
-
if (firstConflictIdx !== -1) {
|
|
364
|
-
const conflictedEntry = entries[firstConflictIdx];
|
|
365
|
-
const logEntryKey = `${logPrefix}/log/${conflictedEntry.seq}.json`;
|
|
366
|
-
if (adoptOwnSessionOnLogConflict) {
|
|
367
|
-
const decision = tryAdoptOwnSessionLogEntry({
|
|
368
|
-
self: conflictedEntry,
|
|
369
|
-
existing: await readLogEntry(this.#storage, logEntryKey),
|
|
370
|
-
batchSize: inputs.length
|
|
371
|
-
});
|
|
372
|
-
if (decision.adopt) committedEntries = [decision.entry];
|
|
373
|
-
else throw new BaerlyError("Conflict", `${errorPrefix}: log entry already exists at ${logEntryKey}; ${decision.reason}`);
|
|
374
|
-
} else throw new BaerlyError("Conflict", `${errorPrefix}: log entry already exists at ${logEntryKey}; peer wrote our seq`);
|
|
375
|
-
}
|
|
376
|
-
const batchLogBytes = committedEntries.reduce((sum, e) => sum + encodeJsonBytes(e).byteLength, 0);
|
|
377
|
-
const next = {
|
|
378
|
-
...current,
|
|
379
|
-
next_seq: current.next_seq + inputs.length,
|
|
380
|
-
tail_bytes: current.tail_bytes + batchLogBytes
|
|
381
|
-
};
|
|
382
|
-
if (next.next_seq - current.next_seq !== inputs.length) throw new BaerlyError("Internal", `${errorPrefix}: GC-cadence invariant violated — next_seq advanced by ${next.next_seq - current.next_seq} but ${inputs.length} input(s) were committed on ${this.#currentJsonKey}`);
|
|
383
|
-
const nextBody = encodeJsonBytes(next);
|
|
384
|
-
const putOpts = {
|
|
385
|
-
ifMatch: baseEtag,
|
|
386
|
-
contentType: APPLICATION_JSON
|
|
387
|
-
};
|
|
388
|
-
let result;
|
|
389
|
-
try {
|
|
390
|
-
result = await this.#storage.put(this.#currentJsonKey, nextBody, putOpts);
|
|
391
|
-
} catch (error) {
|
|
392
|
-
this.#observe429(error, inputs[0].collection);
|
|
393
|
-
if (isPreconditionFailed(error)) {
|
|
394
|
-
ctxMetrics().counter("db.r2.put.412_total", 1, {
|
|
395
|
-
collection: inputs[0].collection,
|
|
396
|
-
step: "current-json-cas"
|
|
397
|
-
});
|
|
398
|
-
throw new BaerlyError("Conflict", `${errorPrefix}: CAS conflict on ${this.#currentJsonKey}`);
|
|
399
|
-
}
|
|
400
|
-
throw error;
|
|
401
|
-
}
|
|
402
|
-
const prevSeq = current.next_seq;
|
|
403
|
-
const maint = getCurrentContext()?.maintenance;
|
|
404
|
-
if (maint?.disabled !== true) {
|
|
405
|
-
if (shouldFireMaintenance(next, prevSeq, maint?.options?.gcInterval ?? 4)) await (maint?.dispatch ?? dispatchInlineAwaited)(() => runBoundedMaintenance({
|
|
406
|
-
storage: this.#storage,
|
|
407
|
-
currentJsonKey: this.#currentJsonKey,
|
|
408
|
-
prevSeq,
|
|
409
|
-
...maint?.maxFoldBytes !== void 0 && { maxFoldBytes: maint.maxFoldBytes }
|
|
410
|
-
}, maint?.options).then(() => {}, () => {}));
|
|
411
|
-
}
|
|
412
|
-
const classAOps = contentPutCount + entries.length + indexClassA + 1;
|
|
413
|
-
return {
|
|
414
|
-
entries: committedEntries,
|
|
415
|
-
currentEtag: result.etag,
|
|
416
|
-
classAOps,
|
|
417
|
-
expectedEpoch
|
|
418
|
-
};
|
|
419
|
-
}
|
|
420
|
-
/**
|
|
421
|
-
* Read the pre-image content body for a doc by walking the live
|
|
422
|
-
* log backwards from `currentNextSeq` looking for the
|
|
423
|
-
* most-recent I/U entry on this `(collection, docId)`. Returns
|
|
424
|
-
* `undefined` when:
|
|
425
|
-
*
|
|
426
|
-
* - the doc's most-recent op was `D` (tombstone — no live body);
|
|
427
|
-
* - no entry for this doc lives inside the visible log range
|
|
428
|
-
* `[log_seq_start, currentNextSeq)` (a fresh-insert race or
|
|
429
|
-
* the doc has been folded into the snapshot but is no longer
|
|
430
|
-
* referenced).
|
|
431
|
-
*
|
|
432
|
-
* Used ONLY by the index-emission path on `U` / `D` to compute
|
|
433
|
-
* the stale-key set. Bounded by `log_seq_start`: entries below
|
|
434
|
-
* have been folded into the snapshot and may already be swept
|
|
435
|
-
* off the bucket — a snapshot fold is the rebuild command's job
|
|
436
|
-
* (see `./rebuild-index.ts`), not the writer's.
|
|
437
|
-
*
|
|
438
|
-
* **Cost note:** linear walk, O(snapshot lag). For day-one Phase
|
|
439
|
-
* 8 collections (helpdesk-shape: 100s of docs, <10k log entries)
|
|
440
|
-
* this is fine — the compactor folds the live tail every ~100
|
|
441
|
-
* entries (`packages/server/src/compactor.ts`). A follow-up ticket
|
|
442
|
-
* caches per-doc index head in `current.json` for O(1) lookup.
|
|
443
|
-
* The bound is documented; out of scope here.
|
|
444
|
-
*/
|
|
445
|
-
/**
|
|
446
|
-
* Auto-create the per-collection `current.json` at
|
|
447
|
-
* `this.#currentJsonKey` with a zero-state initial manifest. Called
|
|
448
|
-
* by `#singleAttemptCommit` when the read returned `null` (fresh
|
|
449
|
-
* bucket / fresh collection).
|
|
450
|
-
*
|
|
451
|
-
* Concurrency: `createCurrentJson` uses `If-None-Match: "*"`. If a
|
|
452
|
-
* peer raced us and won, the storage layer surfaces `Conflict`; we
|
|
453
|
-
* recover by re-reading the now-present manifest. We do NOT retry
|
|
454
|
-
* the create — a single recover-via-read covers every race that
|
|
455
|
-
* could have produced our null read.
|
|
456
|
-
*
|
|
457
|
-
* The seed shape (`snapshot: null`, `next_seq: 0`, `log_seq_start:
|
|
458
|
-
* 0`, `writer_fence: { epoch: 0, owner: "", claimed_at: "" }`)
|
|
459
|
-
* matches `ensureTable` and `baerly deploy`'s pre-warm path, so an
|
|
460
|
-
* operator who pre-provisions and a writer who auto-creates land
|
|
461
|
-
* on byte-identical bytes.
|
|
462
|
-
*
|
|
463
|
-
* @throws BaerlyError code="InvalidResponse" — the post-recover
|
|
464
|
-
* re-read also returned null. Defensive guard; in practice
|
|
465
|
-
* unreachable because the Conflict implies the key now exists.
|
|
466
|
-
*/
|
|
467
|
-
async #provisionCurrentJson() {
|
|
468
|
-
const initial = {
|
|
469
|
-
schema_version: 2,
|
|
470
|
-
snapshot: null,
|
|
471
|
-
next_seq: 0,
|
|
472
|
-
log_seq_start: 0,
|
|
473
|
-
writer_fence: {
|
|
474
|
-
epoch: 0,
|
|
475
|
-
owner: "",
|
|
476
|
-
claimed_at: ""
|
|
477
|
-
},
|
|
478
|
-
tail_bytes: 0,
|
|
479
|
-
snapshot_bytes: 0,
|
|
480
|
-
snapshot_rows: 0
|
|
481
|
-
};
|
|
482
|
-
try {
|
|
483
|
-
return await createCurrentJson(this.#storage, this.#currentJsonKey, initial);
|
|
484
|
-
} catch (error) {
|
|
485
|
-
if (!(error instanceof BaerlyError) || error.code !== "Conflict") throw error;
|
|
486
|
-
}
|
|
487
|
-
const recovered = await readCurrentJson(this.#storage, this.#currentJsonKey);
|
|
488
|
-
if (recovered === null) throw new BaerlyError("InvalidResponse", `Writer: current.json missing at ${this.#currentJsonKey} after auto-create raced and recover read returned null`);
|
|
489
|
-
return recovered;
|
|
490
|
-
}
|
|
491
|
-
async #readPreImage(logPrefix, collection, docId, currentNextSeq) {
|
|
492
|
-
for (let s = currentNextSeq - 1; s >= 0; s--) {
|
|
493
|
-
const logKey = `${logPrefix}/log/${s}.json`;
|
|
494
|
-
const got = await this.#storage.get(logKey);
|
|
495
|
-
if (got === null) continue;
|
|
496
|
-
let entry;
|
|
497
|
-
try {
|
|
498
|
-
entry = decodeJsonBytes(got.body);
|
|
499
|
-
} catch {
|
|
500
|
-
continue;
|
|
501
|
-
}
|
|
502
|
-
if (entry.collection !== collection || entry.doc_id !== docId) continue;
|
|
503
|
-
if (entry.op === "D") return;
|
|
504
|
-
if ((entry.op === "I" || entry.op === "U") && entry.after !== void 0) return entry.after;
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
/**
|
|
508
|
-
* Walk `log/<logSeqStart>.json` … `log/<nextSeq - 1>.json` with
|
|
509
|
-
* bounded parallelism via `walkLogRange`. Materialised entries are
|
|
510
|
-
* discarded — the walk's only job is to surface a hole or a
|
|
511
|
-
* malformed body inside the visible range as
|
|
512
|
-
* `BaerlyError{code:"Internal"}` / `"InvalidResponse"`.
|
|
513
|
-
*
|
|
514
|
-
* Gated on `verifyLogIntegrityOnCommit`; default off in production
|
|
515
|
-
* because the per-doc fold already lives in the read path. See the
|
|
516
|
-
* option's docstring for the rationale and the opt-in convention.
|
|
517
|
-
*
|
|
518
|
-
* `logSeqStart` is the boundary set by the compactor on
|
|
519
|
-
* `current.json.log_seq_start`. Entries below it have been folded
|
|
520
|
-
* into the snapshot and may have been swept off the bucket, so the
|
|
521
|
-
* walk MUST NOT GET-require them.
|
|
522
|
-
*/
|
|
523
|
-
async #walkLog(logPrefix, logSeqStart, nextSeq) {
|
|
524
|
-
await walkLogRange(this.#storage, logPrefix, logSeqStart, nextSeq);
|
|
525
|
-
}
|
|
526
|
-
/**
|
|
527
|
-
* Bump the `db.r2.put.429_total` counter when `err` looks like an
|
|
528
|
-
* R2 prefix-partition rate-limit. Best-effort: detected via
|
|
529
|
-
* `BaerlyError{code:"NetworkError"}` with a `429` token in the
|
|
530
|
-
* message. Called from every PUT call site's catch arm; mutually
|
|
531
|
-
* exclusive in practice with `isPreconditionFailed` (412 ≠ 429).
|
|
532
|
-
*/
|
|
533
|
-
#observe429(err, collection) {
|
|
534
|
-
if (is429(err)) ctxMetrics().counter("db.r2.put.429_total", 1, { collection });
|
|
535
|
-
}
|
|
536
|
-
/**
|
|
537
|
-
* Emit the per-logical-write `class_a_ops_per_logical_write`
|
|
538
|
-
* histogram shared by `commit` and `commitBatch`. `classAOps`
|
|
539
|
-
* differs between the two paths (retry-budget vs batch shape),
|
|
540
|
-
* so each caller computes it.
|
|
541
|
-
*/
|
|
542
|
-
#emitWriteMetrics(collection, classAOps) {
|
|
543
|
-
ctxMetrics().histogram("db.write.class_a_ops_per_logical_write", classAOps, { collection });
|
|
544
|
-
}
|
|
545
|
-
/**
|
|
546
|
-
* Re-read `current.json` after a successful CAS and assert the
|
|
547
|
-
* writer-fence epoch is still ours. The CAS only mutated `next_seq`
|
|
548
|
-
* (the fence is preserved from `current`), so a post-write epoch
|
|
549
|
-
* mismatch can only mean another writer claimed the fence between
|
|
550
|
-
* our step-1 read and step-6 PUT — the exact split-brain
|
|
551
|
-
* `WriterFence` prevents. Throws `Conflict`; no retry, since the
|
|
552
|
-
* stale writer must defer to the new authority.
|
|
553
|
-
*
|
|
554
|
-
* The fence-verify GET is Class B and intentionally NOT counted in
|
|
555
|
-
* `class_a_ops_per_logical_write`.
|
|
556
|
-
*/
|
|
557
|
-
async #verifyFenceUnchanged(expectedEpoch, where) {
|
|
558
|
-
const postRead = await readCurrentJson(this.#storage, this.#currentJsonKey);
|
|
559
|
-
if (postRead === null) return;
|
|
560
|
-
if (postRead.json.writer_fence.epoch === expectedEpoch) return;
|
|
561
|
-
throw new BaerlyError("Conflict", `${where}: writer fence bumped from epoch ${expectedEpoch} to ${postRead.json.writer_fence.epoch} during commit on ${this.#currentJsonKey}; stale writer aborting`);
|
|
562
|
-
}
|
|
563
|
-
/**
|
|
564
|
-
* Exponential backoff with full jitter. `25 * 2^(attempt-1)` capped
|
|
565
|
-
* at 1500ms; uniform jitter in `[0, base)` keeps colliding peers
|
|
566
|
-
* from re-racing in lockstep. Worst-case total wait across the
|
|
567
|
-
* default 8 attempts is roughly 2.5 s — well under a typical
|
|
568
|
-
* Worker request budget.
|
|
569
|
-
*/
|
|
570
|
-
async #backoff(attempt) {
|
|
571
|
-
const sleepMs = Math.min(this.#initialBackoffMs * 2 ** (attempt - 1), MAX_BACKOFF_MS) * this.#random();
|
|
572
|
-
if (sleepMs <= 0) return;
|
|
573
|
-
await new Promise((r) => setTimeout(r, sleepMs));
|
|
574
|
-
}
|
|
575
|
-
};
|
|
576
|
-
/**
|
|
577
|
-
* `true` when the underlying storage surfaced a `PreconditionFailed`
|
|
578
|
-
* 412 (or its non-S3 equivalents). Every in-tree {@link Storage} impl
|
|
579
|
-
* surfaces a lost CAS as `BaerlyError{code:"Conflict"}`.
|
|
580
|
-
*/
|
|
581
|
-
const isPreconditionFailed = (err) => err instanceof BaerlyError && err.code === "Conflict";
|
|
582
|
-
/**
|
|
583
|
-
* `true` when the underlying storage surfaced an R2 prefix-partition
|
|
584
|
-
* rate-limit. {@link S3HttpStorage} stamps the upstream HTTP status
|
|
585
|
-
* onto `BaerlyError.cause` as `{ status }`; we discriminate on that
|
|
586
|
-
* structured field rather than regex-matching the message.
|
|
587
|
-
*/
|
|
588
|
-
const is429 = (err) => {
|
|
589
|
-
if (!(err instanceof BaerlyError) || err.code !== "NetworkError") return false;
|
|
590
|
-
return err.cause?.status === 429;
|
|
591
|
-
};
|
|
592
|
-
//#endregion
|
|
593
|
-
//#region src/admin/restore.ts
|
|
594
|
-
/**
|
|
595
|
-
* `baerly admin restore` — bulk-import NDJSON into a fresh collection.
|
|
596
|
-
*
|
|
597
|
-
* Reads NDJSON from stdin, reconstructs each row, and calls
|
|
598
|
-
* `Writer.commit({op:"I"})` per row into the target bucket.
|
|
599
|
-
* Programmatic callers (today: tests + the export round-trip
|
|
600
|
-
* integration test) can divert the input by passing
|
|
601
|
-
* `{ streams: { stdin } }` to {@link runRestore}. Idempotent on a fresh
|
|
602
|
-
* bucket: re-running on a half-completed restore **refuses** unless
|
|
603
|
-
* `--force` is set (which bumps the writer fence and seeds a fresh
|
|
604
|
-
* `current.json` first).
|
|
605
|
-
*
|
|
606
|
-
* Args:
|
|
607
|
-
* --bucket Required. Target bucket URI.
|
|
608
|
-
* --app Required (or via baerly.config.ts).
|
|
609
|
-
* --tenant Required (or via baerly.config.ts).
|
|
610
|
-
* --collection Required. Target collection name.
|
|
611
|
-
* --force Truncate the target if it exists (fence-bump + reseed).
|
|
612
|
-
* --json JSON envelope.
|
|
613
|
-
*
|
|
614
|
-
* stdin: NDJSON, one `{_id, ...}` row per line. Empty lines tolerated.
|
|
615
|
-
* EOF terminates.
|
|
616
|
-
*
|
|
617
|
-
* Cost shape: 1 PUT current.json (initial) + N × (1 PUT log + 1 PUT
|
|
618
|
-
* content + 1 PUT current.json CAS) = 3N + 1 Class A ops for N rows.
|
|
619
|
-
*
|
|
620
|
-
* Partial-restore semantics: a mid-stream Network / Internal / parse
|
|
621
|
-
* error leaves the target in a partial state (some rows committed,
|
|
622
|
-
* `current.json` advanced to wherever the last successful commit
|
|
623
|
-
* landed). Re-run with `--force` to truncate, or hand-clean. The
|
|
624
|
-
* malformed-line case (missing `_id`, bad JSON) fails fast on that
|
|
625
|
-
* line; rows committed BEFORE it survive.
|
|
626
|
-
*
|
|
627
|
-
* Exit codes:
|
|
628
|
-
* 0 — every row committed.
|
|
629
|
-
* 1 — InvalidConfig (bad bucket URI, missing args).
|
|
630
|
-
* 2 — Network / storage error mid-stream, or a malformed NDJSON
|
|
631
|
-
* line (raised as a plain `Error`, not `BaerlyError`).
|
|
632
|
-
* 3 — Conflict from a pre-existing target without --force, or from a
|
|
633
|
-
* concurrent writer when --force was used.
|
|
634
|
-
*/
|
|
635
|
-
const RESTORE_OWNER = "baerly-restore";
|
|
636
|
-
const bundle = defineBaerlySubcommand({
|
|
637
|
-
name: "admin.restore",
|
|
638
|
-
meta: { description: "Bulk-import canonical NDJSON into a fresh collection." },
|
|
639
|
-
args: {
|
|
640
|
-
bucket: {
|
|
641
|
-
type: "string",
|
|
642
|
-
required: true,
|
|
643
|
-
description: "Target bucket URI (s3://<bucket>[/<prefix>], file:///<abs>, memory://<bucket>)",
|
|
644
|
-
valueHint: "bucket-uri"
|
|
645
|
-
},
|
|
646
|
-
app: {
|
|
647
|
-
type: "string",
|
|
648
|
-
required: false,
|
|
649
|
-
description: "Application name segment (defaults to baerly.config.ts).",
|
|
650
|
-
valueHint: "app"
|
|
651
|
-
},
|
|
652
|
-
tenant: {
|
|
653
|
-
type: "string",
|
|
654
|
-
required: false,
|
|
655
|
-
description: "Tenant name segment (defaults to baerly.config.ts).",
|
|
656
|
-
valueHint: "tenant"
|
|
657
|
-
},
|
|
658
|
-
collection: {
|
|
659
|
-
type: "string",
|
|
660
|
-
required: true,
|
|
661
|
-
description: "Target collection name.",
|
|
662
|
-
valueHint: "name"
|
|
663
|
-
},
|
|
664
|
-
force: {
|
|
665
|
-
type: "boolean",
|
|
666
|
-
description: "Truncate the target if it exists (bump fence + reseed current.json)."
|
|
667
|
-
},
|
|
668
|
-
json: {
|
|
669
|
-
type: "boolean",
|
|
670
|
-
description: "Emit a structured JSON envelope to stdout (success) or stderr (error)"
|
|
671
|
-
}
|
|
672
|
-
},
|
|
673
|
-
handler: async (args, ctx) => {
|
|
674
|
-
const bucket = await parseBucketUri(args.bucket);
|
|
675
|
-
const { app, tenant } = await ctx.resolveAppTenant({
|
|
676
|
-
app: args.app,
|
|
677
|
-
tenant: args.tenant
|
|
678
|
-
});
|
|
679
|
-
const currentJsonKey = `${bucket.keyPrefix}app/${app}/tenant/${tenant}/manifests/${args.collection}/current.json`;
|
|
680
|
-
const head = await readCurrentJson(bucket.storage, currentJsonKey);
|
|
681
|
-
if (head !== null && args.force !== true) throw new BaerlyError("Conflict", `baerly admin restore: ${currentJsonKey} exists; pass --force to truncate`);
|
|
682
|
-
if (head !== null) {
|
|
683
|
-
const truncatedNext = head.json.next_seq;
|
|
684
|
-
const reseeded = {
|
|
685
|
-
schema_version: 2,
|
|
686
|
-
snapshot: null,
|
|
687
|
-
next_seq: truncatedNext,
|
|
688
|
-
log_seq_start: truncatedNext,
|
|
689
|
-
writer_fence: {
|
|
690
|
-
epoch: head.json.writer_fence.epoch + 1,
|
|
691
|
-
owner: RESTORE_OWNER,
|
|
692
|
-
claimed_at: ""
|
|
693
|
-
},
|
|
694
|
-
tail_bytes: 0,
|
|
695
|
-
snapshot_bytes: 0,
|
|
696
|
-
snapshot_rows: 0
|
|
697
|
-
};
|
|
698
|
-
try {
|
|
699
|
-
await bucket.storage.put(currentJsonKey, encodeJsonBytes(reseeded), {
|
|
700
|
-
ifMatch: head.etag,
|
|
701
|
-
contentType: "application/json"
|
|
702
|
-
});
|
|
703
|
-
} catch (error) {
|
|
704
|
-
if (error instanceof BaerlyError) throw error;
|
|
705
|
-
throw new BaerlyError("NetworkError", `baerly admin restore: failed to reseed current.json: ${error.message}`);
|
|
706
|
-
}
|
|
707
|
-
} else {
|
|
708
|
-
const seed = {
|
|
709
|
-
schema_version: 2,
|
|
710
|
-
snapshot: null,
|
|
711
|
-
next_seq: 0,
|
|
712
|
-
log_seq_start: 0,
|
|
713
|
-
writer_fence: {
|
|
714
|
-
epoch: 0,
|
|
715
|
-
owner: RESTORE_OWNER,
|
|
716
|
-
claimed_at: ""
|
|
717
|
-
},
|
|
718
|
-
tail_bytes: 0,
|
|
719
|
-
snapshot_bytes: 0,
|
|
720
|
-
snapshot_rows: 0
|
|
721
|
-
};
|
|
722
|
-
await createCurrentJson(bucket.storage, currentJsonKey, seed);
|
|
723
|
-
}
|
|
724
|
-
const writer = new Writer({
|
|
725
|
-
storage: bucket.storage,
|
|
726
|
-
currentJsonKey
|
|
727
|
-
});
|
|
728
|
-
const rl = createInterface({
|
|
729
|
-
input: ctx.streams?.stdin ?? process.stdin,
|
|
730
|
-
crlfDelay: Infinity
|
|
731
|
-
});
|
|
732
|
-
let count = 0;
|
|
733
|
-
let lineNo = 0;
|
|
734
|
-
for await (const line of rl) {
|
|
735
|
-
lineNo++;
|
|
736
|
-
if (line.length === 0) continue;
|
|
737
|
-
let parsed;
|
|
738
|
-
try {
|
|
739
|
-
parsed = JSON.parse(line);
|
|
740
|
-
} catch (error) {
|
|
741
|
-
throw new Error(`baerly admin restore: line ${lineNo} is not valid JSON: ${error.message}`, { cause: error });
|
|
742
|
-
}
|
|
743
|
-
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(`baerly admin restore: line ${lineNo} is not a JSON object`);
|
|
744
|
-
const row = parsed;
|
|
745
|
-
const id = row["_id"];
|
|
746
|
-
if (typeof id !== "string" || id.length === 0) throw new Error(`baerly admin restore: line ${lineNo} missing non-empty string _id`);
|
|
747
|
-
const body = row;
|
|
748
|
-
await writer.commit({
|
|
749
|
-
op: "I",
|
|
750
|
-
collection: args.collection,
|
|
751
|
-
docId: id,
|
|
752
|
-
body
|
|
753
|
-
});
|
|
754
|
-
count++;
|
|
755
|
-
}
|
|
756
|
-
emitSuccess({
|
|
757
|
-
command: "admin.restore",
|
|
758
|
-
status: "ok",
|
|
759
|
-
collection: args.collection,
|
|
760
|
-
restored: count
|
|
761
|
-
});
|
|
762
|
-
return 0;
|
|
763
|
-
}
|
|
764
|
-
});
|
|
765
|
-
/** citty `defineCommand` block for `baerly admin restore`. */
|
|
766
|
-
const restoreCmd = bundle.cmd;
|
|
767
|
-
bundle.run;
|
|
768
|
-
//#endregion
|
|
769
|
-
export { restoreCmd };
|
|
770
|
-
|
|
771
|
-
//# sourceMappingURL=restore-C6IcmLeA.js.map
|