@gusto/baerly-storage 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -124
- package/dist/API.md +161 -512
- package/dist/CHANGELOG.md +1137 -378
- package/dist/RECIPES.md +398 -80
- package/dist/THIRD-PARTY-LICENSES.txt +5 -26
- package/dist/{app-config-DyiFK3_k.d.ts → app-config-C6qW7m8S.d.ts} +119 -111
- package/dist/app-config.d.ts +31 -1
- package/dist/app-config.js +1 -2
- package/dist/app-config.js.map +1 -1
- package/dist/auth.d.ts +33 -28
- package/dist/auth.js +549 -985
- package/dist/auth.js.map +1 -1
- package/dist/baerly.js +130 -138
- package/dist/baerly.js.map +1 -1
- package/dist/baerly.spec.json +97 -22
- package/dist/capacity-warning-header-BlVopyiI.js +16 -0
- package/dist/capacity-warning-header-BlVopyiI.js.map +1 -0
- package/dist/{client-ypCm3zjt.d.ts → client-CHQTXpef.d.ts} +121 -36
- package/dist/client-react.d.ts +67 -20
- package/dist/client-react.js +240 -54
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +5 -4
- package/dist/client.js +51 -16
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-wrVYf4bb.js → cloudflare-CKHYXccw.js} +61 -82
- package/dist/cloudflare-CKHYXccw.js.map +1 -0
- package/dist/cloudflare.d.ts +106 -71
- package/dist/cloudflare.js +334 -271
- package/dist/cloudflare.js.map +1 -1
- package/dist/code-resolution-CL_7L0wS.js +22 -0
- package/dist/code-resolution-CL_7L0wS.js.map +1 -0
- package/dist/collection-api-DbyhLAY8.d.ts +1001 -0
- package/dist/config-DEzInG6M.js +4035 -0
- package/dist/config-DEzInG6M.js.map +1 -0
- package/dist/constants-D3GSH3tc.js +469 -0
- package/dist/constants-D3GSH3tc.js.map +1 -0
- package/dist/{context-DRZ3tZ5l.js → context-rlpSnHFA.js} +17 -17
- package/dist/context-rlpSnHFA.js.map +1 -0
- package/dist/cost-B52llVDx.js +330 -0
- package/dist/cost-B52llVDx.js.map +1 -0
- package/dist/{current-json-C6nhPOm1.js → current-json-CC6lE4Aj.js} +115 -32
- package/dist/current-json-CC6lE4Aj.js.map +1 -0
- package/dist/db-plFUNTSF.d.ts +130 -0
- package/dist/db-read-seams-BSs0m1z1.js +3261 -0
- package/dist/db-read-seams-BSs0m1z1.js.map +1 -0
- package/dist/{deploy-Di_Q01qx.js → deploy-Obj4nE-Z.js} +10 -14
- package/dist/deploy-Obj4nE-Z.js.map +1 -0
- package/dist/{dev-banner-CWHIk8tH.d.ts → dev-banner-B5AUHmu1.d.ts} +1 -4
- package/dist/dev-landing-BI5UdmOb.d.ts +32 -0
- package/dist/dev-landing-CD-si4dW.js +50 -0
- package/dist/dev-landing-CD-si4dW.js.map +1 -0
- package/dist/dev-vite.d.ts +9 -14
- package/dist/dev-vite.js +11 -10
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +3 -2
- package/dist/dev.js +2 -1
- package/dist/doctor-BA0i-YAN.js +270 -0
- package/dist/doctor-BA0i-YAN.js.map +1 -0
- package/dist/dump-D1vX8ozg.js +133 -0
- package/dist/dump-D1vX8ozg.js.map +1 -0
- package/dist/errors-7Rk1HGCE.js +160 -0
- package/dist/errors-7Rk1HGCE.js.map +1 -0
- package/dist/{validate-DtljsCh1.js → export-BVL2EvYE.js} +409 -19
- package/dist/export-BVL2EvYE.js.map +1 -0
- package/dist/fsck-BWkPgov4.js +145 -0
- package/dist/fsck-BWkPgov4.js.map +1 -0
- package/dist/{gcs-admin-Bx6xSNJj.d.ts → gcs-admin-Dso1HWQj.d.ts} +18 -14
- package/dist/gcs.d.ts +19 -7
- package/dist/gcs.js +1 -2
- package/dist/{goog4-signer-Z7KnJ1su.js → goog4-signer-CeLIABoE.js} +102 -92
- package/dist/goog4-signer-CeLIABoE.js.map +1 -0
- package/dist/{http-7zT29K3P.js → http-Z5rZyYMX.js} +898 -271
- package/dist/http-Z5rZyYMX.js.map +1 -0
- package/dist/http-transport-DNUM8kU-.js +1299 -0
- package/dist/http-transport-DNUM8kU-.js.map +1 -0
- package/dist/http.d.ts +366 -294
- package/dist/http.js +2 -2
- package/dist/index-JJrIu1pO.d.ts +135 -0
- package/dist/index-LEgHN0qJ.d.ts +50 -0
- package/dist/{index-DP3ClvcA.d.ts → index-RiEYj8t0.d.ts} +291 -78
- package/dist/index.d.ts +27 -6
- package/dist/index.js +3 -5
- package/dist/inspect-BTsVYZKd.js +196 -0
- package/dist/inspect-BTsVYZKd.js.map +1 -0
- package/dist/{key-Di6yyxFu.js → key-C4FPZ9Io.js} +3 -4
- package/dist/{key-Di6yyxFu.js.map → key-C4FPZ9Io.js.map} +1 -1
- package/dist/ledger-Dj_bGgo0.js +192 -0
- package/dist/ledger-Dj_bGgo0.js.map +1 -0
- package/dist/legacy-names-DgZBtgOh.js +47 -0
- package/dist/legacy-names-DgZBtgOh.js.map +1 -0
- package/dist/log-C9W15Dfo.d.ts +94 -0
- package/dist/log-tail-DWJbneBF.js +142 -0
- package/dist/log-tail-DWJbneBF.js.map +1 -0
- package/dist/log-walk-C-vA-WGJ.js +111 -0
- package/dist/log-walk-C-vA-WGJ.js.map +1 -0
- package/dist/maintenance-Dd-Yakqj.js +1709 -0
- package/dist/maintenance-Dd-Yakqj.js.map +1 -0
- package/dist/maintenance-DevB6z4e.js +2237 -0
- package/dist/maintenance-DevB6z4e.js.map +1 -0
- package/dist/maintenance.d.ts +100 -3
- package/dist/maintenance.js +1 -3
- package/dist/node.d.ts +119 -92
- package/dist/node.js +3 -3
- package/dist/{observability-Ch-VQJyE.js → observability-C-ISCCLE.js} +1247 -326
- package/dist/observability-C-ISCCLE.js.map +1 -0
- package/dist/observability.d.ts +27 -3
- package/dist/observability.js +3 -3
- package/dist/path-segment-DBXG26PW.js +363 -0
- package/dist/path-segment-DBXG26PW.js.map +1 -0
- package/dist/{plan-C4qgiXSg.js → plan-BvQbaNMe.js} +19 -11
- package/dist/plan-BvQbaNMe.js.map +1 -0
- package/dist/request-CRWiTrza.js +272 -0
- package/dist/request-CRWiTrza.js.map +1 -0
- package/dist/restore-U6Im8V1V.js +1467 -0
- package/dist/restore-U6Im8V1V.js.map +1 -0
- package/dist/retry-OBtLB2GC.js +126 -0
- package/dist/retry-OBtLB2GC.js.map +1 -0
- package/dist/rolldown-runtime-D3KUYPsf.js +12 -0
- package/dist/{runtime-spec--LykAdl1.js → runtime-spec-B5mg3pHF.js} +189 -28
- package/dist/runtime-spec-B5mg3pHF.js.map +1 -0
- package/dist/{s3-http-DD5gDc7t.d.ts → s3-http-DBC_wiCL.d.ts} +14 -8
- package/dist/s3.d.ts +20 -7
- package/dist/s3.js +4 -5
- package/dist/s3.js.map +1 -1
- package/dist/{sha256-CqcKTjw9.js → sha256-4FfGQNPw.js} +9 -9
- package/dist/sha256-4FfGQNPw.js.map +1 -0
- package/dist/{shared-secret-DgEicd3x.js → shared-secret-Vkur-8r0.js} +27 -15
- package/dist/shared-secret-Vkur-8r0.js.map +1 -0
- package/dist/{aws4fetch.esm-CXE5ZbAa.js → sigv4-sign-5IUflUD_.js} +143 -96
- package/dist/sigv4-sign-5IUflUD_.js.map +1 -0
- package/dist/snapshot-CknzkDM0.js +148 -0
- package/dist/snapshot-CknzkDM0.js.map +1 -0
- package/dist/{src-BMFfDUI2.js → src-Actga-bO.js} +949 -363
- package/dist/src-Actga-bO.js.map +1 -0
- package/dist/src-D0MQfvOp.js +812 -0
- package/dist/src-D0MQfvOp.js.map +1 -0
- package/dist/src-Dr9uo3dc.js +501 -0
- package/dist/src-Dr9uo3dc.js.map +1 -0
- package/dist/{subcommand-BV68pqwJ.js → subcommand-BLoF66dG.js} +89 -16
- package/dist/subcommand-BLoF66dG.js.map +1 -0
- package/dist/{time-DUqvFMHq.js → time-DRxTP9iv.js} +15 -34
- package/dist/time-DRxTP9iv.js.map +1 -0
- package/dist/types-suVbsEWS.d.ts +116 -0
- package/dist/{usage-BMZR8o6U.js → usage-CPzVhrQr.js} +40 -45
- package/dist/usage-CPzVhrQr.js.map +1 -0
- package/dist/uuid-BuT_c404.js +333 -0
- package/dist/uuid-BuT_c404.js.map +1 -0
- package/dist/verifier-Bmlmndzo.d.ts +95 -0
- package/dist/{wire-izfGZDvr.js → wire-DndvKU-K.js} +2 -3
- package/dist/wire-DndvKU-K.js.map +1 -0
- package/package.json +90 -69
- package/dist/aws4fetch.esm-CXE5ZbAa.js.map +0 -1
- package/dist/bytes-BXdamO3L.js +0 -11
- package/dist/bytes-BXdamO3L.js.map +0 -1
- package/dist/cloudflare-wrVYf4bb.js.map +0 -1
- package/dist/config-B6nTy8e2.js +0 -3095
- package/dist/config-B6nTy8e2.js.map +0 -1
- package/dist/constants-CfeMWQWk.js +0 -222
- package/dist/constants-CfeMWQWk.js.map +0 -1
- package/dist/context-DRZ3tZ5l.js.map +0 -1
- package/dist/cost-_SE_SVgZ.js +0 -203
- package/dist/cost-_SE_SVgZ.js.map +0 -1
- package/dist/current-json-C6nhPOm1.js.map +0 -1
- package/dist/db-DjjthkTi.d.ts +0 -254
- package/dist/deploy-Di_Q01qx.js.map +0 -1
- package/dist/doctor-RWVRpvat.js +0 -102
- package/dist/doctor-RWVRpvat.js.map +0 -1
- package/dist/dump-m2vEYDNY.js +0 -94
- package/dist/dump-m2vEYDNY.js.map +0 -1
- package/dist/errors-Dfa3p78c.js +0 -78
- package/dist/errors-Dfa3p78c.js.map +0 -1
- package/dist/export-DenKwEQu.js +0 -401
- package/dist/export-DenKwEQu.js.map +0 -1
- package/dist/fsck-2Vq-7-e2.js +0 -279
- package/dist/fsck-2Vq-7-e2.js.map +0 -1
- package/dist/goog4-signer-Z7KnJ1su.js.map +0 -1
- package/dist/http-7zT29K3P.js.map +0 -1
- package/dist/http-transport-BTgVCY5r.js +0 -1250
- package/dist/http-transport-BTgVCY5r.js.map +0 -1
- package/dist/index-BHz92Acj.d.ts +0 -162
- package/dist/index-CEZKQsKt.d.ts +0 -1113
- package/dist/index-uE-w2dgj.d.ts +0 -115
- package/dist/indexes-Dn9hbGQG.d.ts +0 -22
- package/dist/inspect-BpYbH1DM.js +0 -133
- package/dist/inspect-BpYbH1DM.js.map +0 -1
- package/dist/log-tail-QWU4wlu-.js +0 -514
- package/dist/log-tail-QWU4wlu-.js.map +0 -1
- package/dist/maintenance-BGHGt2Yg.d.ts +0 -537
- package/dist/maintenance-DULFcpd1.js +0 -2144
- package/dist/maintenance-DULFcpd1.js.map +0 -1
- package/dist/maintenance-fEi-UZ2o.js +0 -1412
- package/dist/maintenance-fEi-UZ2o.js.map +0 -1
- package/dist/normalize-rjnZLwS4.js +0 -133
- package/dist/normalize-rjnZLwS4.js.map +0 -1
- package/dist/observability-Ch-VQJyE.js.map +0 -1
- package/dist/plan-C4qgiXSg.js.map +0 -1
- package/dist/query-C43r-6ib.js +0 -1773
- package/dist/query-C43r-6ib.js.map +0 -1
- package/dist/rebuild-index-BH1dc1p6.js +0 -81
- package/dist/rebuild-index-BH1dc1p6.js.map +0 -1
- package/dist/request-u0M51mhJ.js +0 -72
- package/dist/request-u0M51mhJ.js.map +0 -1
- package/dist/restore-D0VYWzfP.js +0 -778
- package/dist/restore-D0VYWzfP.js.map +0 -1
- package/dist/rolldown-runtime-EAsCxrDo.js +0 -14
- package/dist/runtime-spec--LykAdl1.js.map +0 -1
- package/dist/sha256-CqcKTjw9.js.map +0 -1
- package/dist/shared-secret-DgEicd3x.js.map +0 -1
- package/dist/src-BMFfDUI2.js.map +0 -1
- package/dist/src-DYZ4H1aF.js +0 -392
- package/dist/src-DYZ4H1aF.js.map +0 -1
- package/dist/src-DvbTEya2.js +0 -536
- package/dist/src-DvbTEya2.js.map +0 -1
- package/dist/subcommand-BV68pqwJ.js.map +0 -1
- package/dist/time-DUqvFMHq.js.map +0 -1
- package/dist/usage-BMZR8o6U.js.map +0 -1
- package/dist/validate-DtljsCh1.js.map +0 -1
- package/dist/wire-izfGZDvr.js.map +0 -1
|
@@ -0,0 +1,1467 @@
|
|
|
1
|
+
import { c as readCurrentJson, d as encodeJsonBytes, i as createCurrentJson, o as logSeqStartOf, r as certifiedDeleteFloor, s as mintGeneration, t as assertCurrentJsonTransition, u as decodeJsonBytes } from "./current-json-CC6lE4Aj.js";
|
|
2
|
+
import { E as MAX_KEY_BYTES, T as MAX_DOCUMENT_BYTES, c as countKey, l as uuid, n as parseBucketUri, s as timestamp, v as LOG_FORWARD_PROBE_CAP, x as MAINTENANCE_PROFILE_CF_FREE } from "./config-DEzInG6M.js";
|
|
3
|
+
import { c as isWriteConflict, r as emitSuccess, s as BaerlyError } from "./baerly.js";
|
|
4
|
+
import { a as getCurrentContext, c as gcPendingKey, i as shouldFireMaintenance, o as runWithContext, s as noopMetricsRecorder, t as dispatchWriteTick } from "./maintenance-Dd-Yakqj.js";
|
|
5
|
+
import { a as TENANT_ARG, c as defineBaerlySubcommand, d as assertWellFormedSegment, i as JSON_ARG, l as assertNfcSegment, o as assertCollectionArg, s as assertWritableNames, t as APP_ARG, u as assertPathSegment } from "./subcommand-BLoF66dG.js";
|
|
6
|
+
import { i as logObjectKey, r as probeTailFrom, t as findLogTail } from "./log-tail-DWJbneBF.js";
|
|
7
|
+
import { t as walkLogRange } from "./log-walk-C-vA-WGJ.js";
|
|
8
|
+
import { i as isOwnCollection } from "./snapshot-CknzkDM0.js";
|
|
9
|
+
import { collectionPrefixOf } from "./usage-CPzVhrQr.js";
|
|
10
|
+
import { createInterface } from "node:readline";
|
|
11
|
+
// ../protocol/src/code-resolution.ts
|
|
12
|
+
/** Canonical per-code corrective action; `errorEnvelope` falls back to this and it crosses the wire as `resolution`. Actionable codes only. One canonical action per actionable code; a throw site may pass a more specific resolution (CLI `restore`), never a different retry meaning. */
|
|
13
|
+
const CODE_RESOLUTIONS = {
|
|
14
|
+
WriteConflict: "Another write changed this data first and this commit did not land. Re-read, then retry.",
|
|
15
|
+
AlreadyExists: "Do not retry. A row with this _id is already committed: insert with a different _id or omit _id to auto-generate one, or use update or replace to change the existing row. If this insert omitted _id or followed an AmbiguousCommit, the row may be this insert's own earlier attempt: re-read it, and do not re-mint by inserting again without an _id.",
|
|
16
|
+
AmbiguousCommit: "Do not retry automatically. Re-read the row named by the error's _id; absence does not prove the write failed. A deliberate retry may repeat logical work; retry an insert with the same _id, and treat AlreadyExists on that retry as a committed row to re-read.",
|
|
17
|
+
CapacityExceeded: "Do not retry unchanged. Delete or shrink rows to make room; if every write is refused, the collection has outgrown this host: export it with baerly admin dump, or move it to a larger host.",
|
|
18
|
+
MaintenancePending: "Retry after the delay in retryAfter (Retry-After header, seconds): maintenance is catching up on this collection and this write was not committed.",
|
|
19
|
+
CursorExpired: "Do not retry this cursor. The changes after it are no longer retained: re-read the rows you need, then resume /v1/since with cursor: \"\".",
|
|
20
|
+
PayloadTooLarge: "Reduce the request body below the server's body-size cap (1 MiB default).",
|
|
21
|
+
UnsupportedMediaType: "Send the request body as JSON with a Content-Type: application/json header.",
|
|
22
|
+
Unauthorized: "Send a valid Authorization header for the server's configured auth method.",
|
|
23
|
+
AccessDenied: "These credentials are denied for this tenant prefix or bucket policy.",
|
|
24
|
+
NotFound: "No row matches this id; create it first or treat this as a miss."
|
|
25
|
+
};
|
|
26
|
+
// ../protocol/src/document-input.ts
|
|
27
|
+
const NON_FINITE_RESOLUTION = "JSON cannot represent NaN or Infinity. Store a finite number or null there, or omit the field.";
|
|
28
|
+
const UNDEFINED_MEMBER_RESOLUTION = "JSON drops a member whose value is undefined. Omit the field, or store null.";
|
|
29
|
+
const UNDEFINED_ELEMENT_RESOLUTION = "JSON turns an undefined or missing array element into null. Store null there, or remove the element.";
|
|
30
|
+
const NON_PLAIN_RESOLUTION = "Store plain JSON: convert a Date with date.toISOString(), a Map with Object.fromEntries(map), a Set or typed array with Array.from(value), a boxed primitive with value.valueOf(), and copy a class instance's fields into a plain object.";
|
|
31
|
+
const NON_JSON_TYPE_RESOLUTION = "Store a string, finite number, boolean, null, plain object or array there. Convert a bigint with String(value); remove functions and symbols.";
|
|
32
|
+
const TO_JSON_RESOLUTION = "Remove the toJSON hook and pass its intended JSON data explicitly. Documents and patches must not customize JSON serialization.";
|
|
33
|
+
const CIRCULAR_RESOLUTION = "Remove the circular reference: a document is a tree of plain objects and arrays.";
|
|
34
|
+
const DEPTH_RESOLUTION = `Keep each document at most 100 levels deep (each object or array is a level, the document itself included): flatten nested data into compound keys or arrays of records, or move a deep subtree into rows of its own.`;
|
|
35
|
+
/** Inspect descriptors so even a toJSON getter is refused without running it. */
|
|
36
|
+
const hasJsonHook = (value) => {
|
|
37
|
+
for (let at = value; at !== null; at = Object.getPrototypeOf(at)) {
|
|
38
|
+
const descriptor = Object.getOwnPropertyDescriptor(at, "toJSON");
|
|
39
|
+
if (descriptor !== void 0) return !("value" in descriptor) || typeof descriptor.value === "function";
|
|
40
|
+
}
|
|
41
|
+
return false;
|
|
42
|
+
};
|
|
43
|
+
const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;
|
|
44
|
+
const renderSegment = (key) => {
|
|
45
|
+
if (typeof key === "number") return `[${key}]`;
|
|
46
|
+
if (key.length > 64) return `[${JSON.stringify(key.slice(0, 64))}…]`;
|
|
47
|
+
return IDENTIFIER.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
|
|
48
|
+
};
|
|
49
|
+
/** The keys from the root to `at`. */
|
|
50
|
+
const keysOf = (at) => {
|
|
51
|
+
const keys = [];
|
|
52
|
+
for (let step = at; step?.key !== void 0; step = step.parent) keys.push(step.key);
|
|
53
|
+
keys.reverse();
|
|
54
|
+
return keys;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* The path from the root to `at`, as `a.b[2]["c d"]`. A deep path keeps
|
|
58
|
+
* its first and last {@link DOCUMENT_ERROR_PATH_EDGE_SEGMENTS} segments
|
|
59
|
+
* around a `.…` marker, so the message stays bounded.
|
|
60
|
+
*/
|
|
61
|
+
const pathOf = (at) => {
|
|
62
|
+
const keys = keysOf(at);
|
|
63
|
+
const path = keys.length > 8 ? `${keys.slice(0, 4).map(renderSegment).join("")}.…${keys.slice(-4).map(renderSegment).join("")}` : keys.map(renderSegment).join("");
|
|
64
|
+
return path.startsWith(".") ? path.slice(1) : path;
|
|
65
|
+
};
|
|
66
|
+
const refuse = (verb, at, what, resolution) => {
|
|
67
|
+
const path = pathOf(at);
|
|
68
|
+
throw new BaerlyError("InvalidConfig", path === "" ? `${verb}: the document is ${what}` : `${verb}: document value at ${path} is ${what}`, { resolution });
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* The first container past {@link MAX_DOCUMENT_DEPTH}. `SchemaError`, not
|
|
72
|
+
* `InvalidConfig`: the value is valid JSON, but its shape is outside what
|
|
73
|
+
* a document may be. `issues` carries the full path.
|
|
74
|
+
*/
|
|
75
|
+
const refuseDepth = (verb, at) => {
|
|
76
|
+
const message = `nests deeper than the 100-level document limit`;
|
|
77
|
+
throw new BaerlyError("SchemaError", `${verb}: document value at ${pathOf(at)} ${message}`, {
|
|
78
|
+
resolution: DEPTH_RESOLUTION,
|
|
79
|
+
issues: [{
|
|
80
|
+
path: keysOf(at),
|
|
81
|
+
message
|
|
82
|
+
}]
|
|
83
|
+
});
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* Reject a document (or update patch) holding a value JSON cannot store
|
|
87
|
+
* as written. `JSON.stringify` would silently turn `NaN` and `±Infinity`
|
|
88
|
+
* into `null`, drop `undefined` members, turn `undefined` or missing array
|
|
89
|
+
* elements into `null`, and reduce a `Date`, `Map`, typed array, boxed
|
|
90
|
+
* primitive or class instance to a different shape; a function, symbol or
|
|
91
|
+
* bigint would be dropped or throw. Each of those throws here instead,
|
|
92
|
+
* naming the path of the first offending value (capped, see
|
|
93
|
+
* {@link DOCUMENT_ERROR_PATH_EDGE_SEGMENTS}), as do circular references.
|
|
94
|
+
* Callable or accessor `toJSON` properties (own or inherited) are refused
|
|
95
|
+
* without invoking them, including on the root and on arrays. A value
|
|
96
|
+
* nested deeper than {@link MAX_DOCUMENT_DEPTH} levels is refused too, as
|
|
97
|
+
* `SchemaError`.
|
|
98
|
+
*
|
|
99
|
+
* Allowed: strings, finite numbers (`-0` included), booleans, `null` (at
|
|
100
|
+
* any depth), arrays, and plain objects (prototype `Object.prototype` or
|
|
101
|
+
* `null`). The root itself may be any object; its own
|
|
102
|
+
* enumerable fields are checked. A top-level `_id: undefined` counts as
|
|
103
|
+
* absent, as on `insert`; with `undefinedMembersAbsent`, every
|
|
104
|
+
* `undefined` object member does. No `null` limits are applied.
|
|
105
|
+
* Size is checked separately, on the whole post-image a write commits
|
|
106
|
+
* (`assertDocumentSize`).
|
|
107
|
+
*
|
|
108
|
+
* The input graph must remain stable until the write completes. Ordinary
|
|
109
|
+
* field getters are read during traversal; changing getters, proxies and
|
|
110
|
+
* concurrent mutation are not supported as a way to supply document data.
|
|
111
|
+
* This check does not snapshot the graph or execute serialization hooks.
|
|
112
|
+
*
|
|
113
|
+
* I/O-free. `insert` and `replace` call it on the caller's
|
|
114
|
+
* document before any storage call, and `update` on each merged
|
|
115
|
+
* post-image before any write; the HTTP client calls it before
|
|
116
|
+
* serializing (on the patch, for `update`), because the coercions above
|
|
117
|
+
* happen in `JSON.stringify` and a server only ever sees their result.
|
|
118
|
+
*
|
|
119
|
+
* @param verb the calling verb, used as the message prefix
|
|
120
|
+
* (`"insert"`, `"replace"`, `"update"`, …).
|
|
121
|
+
* @throws BaerlyError code="InvalidConfig" — `value` holds a value outside
|
|
122
|
+
* the JSON document domain. The message names its path.
|
|
123
|
+
* @throws BaerlyError code="SchemaError" — `value` nests an object or
|
|
124
|
+
* array deeper than {@link MAX_DOCUMENT_DEPTH} levels. The message names
|
|
125
|
+
* the limit and the (capped) path; `issues` holds the full path.
|
|
126
|
+
*/
|
|
127
|
+
const assertDocumentValues = (value, verb, options) => {
|
|
128
|
+
const undefinedMembersAbsent = options?.undefinedMembersAbsent === true;
|
|
129
|
+
const pending = [{
|
|
130
|
+
value,
|
|
131
|
+
parent: void 0,
|
|
132
|
+
key: void 0,
|
|
133
|
+
depth: 1
|
|
134
|
+
}];
|
|
135
|
+
const ancestors = /* @__PURE__ */ new Set();
|
|
136
|
+
while (pending.length > 0) {
|
|
137
|
+
const next = pending.pop();
|
|
138
|
+
if ("exit" in next) {
|
|
139
|
+
ancestors.delete(next.exit);
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const current = next.value;
|
|
143
|
+
if (typeof current === "string" || typeof current === "boolean" || current === null) continue;
|
|
144
|
+
if (typeof current === "number") {
|
|
145
|
+
if (Number.isFinite(current)) continue;
|
|
146
|
+
return refuse(verb, next, String(current), NON_FINITE_RESOLUTION);
|
|
147
|
+
}
|
|
148
|
+
if (current === void 0) {
|
|
149
|
+
const elementOrMember = typeof next.key === "number";
|
|
150
|
+
return refuse(verb, next, "undefined", elementOrMember ? UNDEFINED_ELEMENT_RESOLUTION : UNDEFINED_MEMBER_RESOLUTION);
|
|
151
|
+
}
|
|
152
|
+
if (typeof current !== "object") return refuse(verb, next, `a ${typeof current}`, NON_JSON_TYPE_RESOLUTION);
|
|
153
|
+
if (ancestors.has(current)) return refuse(verb, next, "a circular reference", CIRCULAR_RESOLUTION);
|
|
154
|
+
if (next.depth > 100) return refuseDepth(verb, next);
|
|
155
|
+
if (hasJsonHook(current)) return refuse(verb, next, "an object with a toJSON hook", TO_JSON_RESOLUTION);
|
|
156
|
+
if (Array.isArray(current)) {
|
|
157
|
+
ancestors.add(current);
|
|
158
|
+
pending.push({ exit: current });
|
|
159
|
+
for (let i = current.length - 1; i >= 0; i--) pending.push({
|
|
160
|
+
value: current[i],
|
|
161
|
+
parent: next,
|
|
162
|
+
key: i,
|
|
163
|
+
depth: next.depth + 1
|
|
164
|
+
});
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
const prototype = Object.getPrototypeOf(current);
|
|
168
|
+
if (next.parent !== void 0 && prototype !== Object.prototype && prototype !== null) {
|
|
169
|
+
const name = current.constructor?.name;
|
|
170
|
+
return refuse(verb, next, typeof name === "string" && name !== "" ? `a ${name}, not a plain object` : "not a plain object", NON_PLAIN_RESOLUTION);
|
|
171
|
+
}
|
|
172
|
+
ancestors.add(current);
|
|
173
|
+
pending.push({ exit: current });
|
|
174
|
+
const keys = Object.keys(current);
|
|
175
|
+
for (let i = keys.length - 1; i >= 0; i--) {
|
|
176
|
+
const key = keys[i];
|
|
177
|
+
const member = current[key];
|
|
178
|
+
if (member === void 0 && (undefinedMembersAbsent || key === "_id" && next.parent === void 0)) continue;
|
|
179
|
+
pending.push({
|
|
180
|
+
value: member,
|
|
181
|
+
parent: next,
|
|
182
|
+
key,
|
|
183
|
+
depth: next.depth + 1
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
// ../protocol/src/document-size.ts
|
|
189
|
+
const utf8Encoder = new TextEncoder();
|
|
190
|
+
const DOCUMENT_SIZE_RESOLUTION = `Keep each document at or under ${MAX_DOCUMENT_BYTES} bytes (1 MiB) of JSON: move large or growing data (attachments, long histories, arrays that keep growing) into rows of their own, or store blobs outside the database and keep a reference.`;
|
|
191
|
+
/**
|
|
192
|
+
* UTF-8 byte length of `json` when it is over `limit`, else `undefined`.
|
|
193
|
+
* Skips the encode when the string is short enough to decide it: a UTF-16
|
|
194
|
+
* code unit encodes to at most 3 bytes.
|
|
195
|
+
*/
|
|
196
|
+
const jsonByteLengthAbove = (json, limit) => {
|
|
197
|
+
if (json.length * 3 <= limit) return;
|
|
198
|
+
const bytes = utf8Encoder.encode(json).byteLength;
|
|
199
|
+
return bytes > limit ? bytes : void 0;
|
|
200
|
+
};
|
|
201
|
+
/**
|
|
202
|
+
* Refuse a post-image larger than {@link MAX_DOCUMENT_BYTES} once encoded
|
|
203
|
+
* as stored (`encodeJsonBytes`, which sorts nothing, so key order does not
|
|
204
|
+
* change the size). Call it on the whole document a write would commit,
|
|
205
|
+
* not on an update patch: a small patch can still grow a row past the cap.
|
|
206
|
+
* A row already stored larger is not checked here; only a write producing
|
|
207
|
+
* an over-cap post-image is refused.
|
|
208
|
+
*
|
|
209
|
+
* I/O-free. The value must already be in the JSON document domain (see
|
|
210
|
+
* `assertDocumentValues`); a value `JSON.stringify` cannot encode
|
|
211
|
+
* throws its own error.
|
|
212
|
+
*
|
|
213
|
+
* @param verb the calling verb, used as the message prefix.
|
|
214
|
+
* @throws BaerlyError code="InvalidConfig" — the encoded post-image is over
|
|
215
|
+
* the cap. The message states its size and the limit.
|
|
216
|
+
*/
|
|
217
|
+
const assertDocumentSize = (value, verb) => {
|
|
218
|
+
const bytes = jsonByteLengthAbove(JSON.stringify(value), MAX_DOCUMENT_BYTES);
|
|
219
|
+
if (bytes !== void 0) throw new BaerlyError("InvalidConfig", `${verb}: the document encodes to ${bytes} bytes of JSON, over the ${MAX_DOCUMENT_BYTES}-byte (1 MiB) document limit`, { resolution: DOCUMENT_SIZE_RESOLUTION });
|
|
220
|
+
};
|
|
221
|
+
// ../protocol/src/json.ts
|
|
222
|
+
/**
|
|
223
|
+
* Structural equality of two JSON values.
|
|
224
|
+
*
|
|
225
|
+
* Both inputs must already be JSON-normalized, i.e. the result of a
|
|
226
|
+
* `JSON.parse` — typically `JSON.parse(JSON.stringify(body))` compared
|
|
227
|
+
* against a stored row decoded from JSON. The round-trip drops
|
|
228
|
+
* `undefined` properties (as the write path's encoder does) and turns
|
|
229
|
+
* `-0` into `0`; that normalization is the caller's responsibility.
|
|
230
|
+
* Raw inputs holding `undefined` or `-0` are outside the contract.
|
|
231
|
+
*
|
|
232
|
+
* Uses an explicit work stack so comparison does not consume the
|
|
233
|
+
* JavaScript call stack with each nesting level. No depth cap is enforced.
|
|
234
|
+
*
|
|
235
|
+
* - Object key order is ignored; the key sets must match exactly.
|
|
236
|
+
* - Arrays compare element-wise in order, and an array is never equal
|
|
237
|
+
* to an object (`[1]` vs `{"0":1}` is `false`).
|
|
238
|
+
* - Primitives compare with `===`.
|
|
239
|
+
*
|
|
240
|
+
* Deliberately not `deepEqualDocumentValue`, which cannot tell an array
|
|
241
|
+
* from an index-keyed object.
|
|
242
|
+
*/
|
|
243
|
+
function jsonValueEqual(a, b) {
|
|
244
|
+
const pending = [[a, b]];
|
|
245
|
+
while (pending.length > 0) {
|
|
246
|
+
const [left, right] = pending.pop();
|
|
247
|
+
if (left === right) continue;
|
|
248
|
+
if (typeof left !== "object" || typeof right !== "object" || left === null || right === null) return false;
|
|
249
|
+
if (Array.isArray(left)) {
|
|
250
|
+
if (!Array.isArray(right) || left.length !== right.length) return false;
|
|
251
|
+
for (let i = 0; i < left.length; i++) pending.push([left[i], right[i]]);
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
if (Array.isArray(right)) return false;
|
|
255
|
+
const keys = Object.keys(left);
|
|
256
|
+
if (keys.length !== Object.keys(right).length) return false;
|
|
257
|
+
for (const key of keys) {
|
|
258
|
+
if (!Object.prototype.hasOwnProperty.call(right, key)) return false;
|
|
259
|
+
pending.push([left[key], right[key]]);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return true;
|
|
263
|
+
}
|
|
264
|
+
// ../server/src/doc-id.ts
|
|
265
|
+
/**
|
|
266
|
+
* Resolution for a non-NFC `_id`. A row stored under one by an earlier
|
|
267
|
+
* release stays readable and deletable, so it is migrated by copy.
|
|
268
|
+
*/
|
|
269
|
+
const DOC_ID_NFC_RESOLUTION = "Pass id.normalize(\"NFC\") for a new _id. To migrate a stored non-NFC row, insert a copy under a fresh NFC id that is not canonically equivalent to the original (for example, omit _id for a UUID), then delete the original.";
|
|
270
|
+
/**
|
|
271
|
+
* Resolution for an `_id` with an unpaired surrogate. A row stored under
|
|
272
|
+
* one by an earlier release stays readable and deletable in-process, so it
|
|
273
|
+
* is migrated by copy.
|
|
274
|
+
*/
|
|
275
|
+
const DOC_ID_WELL_FORMED_RESOLUTION = "Pass an _id with no unpaired UTF-16 surrogate (id.isWellFormed() is true), or omit _id for a UUID. To migrate a stored row with such an _id, read it by _id, remove _id from the copy so insert generates a fresh UUID, then delete the original.";
|
|
276
|
+
/**
|
|
277
|
+
* Guard an `_id` a write is about to store a row under: an insert, a
|
|
278
|
+
* replace, an update, or any `I` / `U` commit. The `_id` analogue of
|
|
279
|
+
* `assertKeySegment` (`db.ts`).
|
|
280
|
+
*
|
|
281
|
+
* Applies {@link assertStoredDocId}, then requires the id to be
|
|
282
|
+
* well-formed UTF-16 (no unpaired surrogate, which no URL or S3 key can
|
|
283
|
+
* carry) and in Unicode Normalization Form C (`id === id.normalize("NFC")`),
|
|
284
|
+
* so each canonical-equivalence class (NFC `é` and NFD `e` + U+0301) has one
|
|
285
|
+
* spelling and one byte form. R2 stores canonically-equivalent keys as one
|
|
286
|
+
* object, so any object a future format keys by `_id` (an index-aware
|
|
287
|
+
* snapshot, say) would otherwise merge two rows whose ids differ only by
|
|
288
|
+
* normalization; the rule keeps that door open, and loosening it later is
|
|
289
|
+
* not a breaking change. Case and compatibility variants (`A` / `a`, `fi` / `fi`)
|
|
290
|
+
* are distinct NFC strings and stay distinct ids.
|
|
291
|
+
*
|
|
292
|
+
* @throws BaerlyError code="InvalidConfig" — the id breaks the key-segment
|
|
293
|
+
* rule, holds an unpaired surrogate, or is not NFC.
|
|
294
|
+
*/
|
|
295
|
+
const assertDocId = (id) => {
|
|
296
|
+
assertStoredDocId(id);
|
|
297
|
+
assertWellFormedSegment(id, "_id", "", DOC_ID_WELL_FORMED_RESOLUTION);
|
|
298
|
+
assertNfcSegment(id, "_id", "", DOC_ID_NFC_RESOLUTION);
|
|
299
|
+
};
|
|
300
|
+
/**
|
|
301
|
+
* Guard an `_id` that names an existing row: the `doc_id` of a delete.
|
|
302
|
+
* Only the key-segment rule applies (the single shared
|
|
303
|
+
* {@link assertPathSegment}: empty / `"/"` / `"."`|`".."` / C0-C1 control
|
|
304
|
+
* chars / leading `"_"` (ADR-003) / overlong, all as
|
|
305
|
+
* `BaerlyError{code:"InvalidConfig"}`), so a row stored under a non-NFC
|
|
306
|
+
* id, or an id with an unpaired surrogate, before {@link assertDocId}
|
|
307
|
+
* refused them can still be deleted.
|
|
308
|
+
*/
|
|
309
|
+
const assertStoredDocId = (id) => {
|
|
310
|
+
assertPathSegment(id, "_id");
|
|
311
|
+
};
|
|
312
|
+
/**
|
|
313
|
+
* The `AlreadyExists` an insert throws when a live row holds `docId`.
|
|
314
|
+
* Shared by `runInsert`'s check read (`./query.ts`) and the writer's
|
|
315
|
+
* insert guard (`./writer.ts`), so a caller sees one error whichever
|
|
316
|
+
* side caught the collision. Not retriable; `err._id` is `docId`.
|
|
317
|
+
*/
|
|
318
|
+
const insertAlreadyExists = (collection, docId) => new BaerlyError("AlreadyExists", `insert: _id ${JSON.stringify(docId)} already exists in collection ${JSON.stringify(collection)}`, {
|
|
319
|
+
resolution: CODE_RESOLUTIONS.AlreadyExists,
|
|
320
|
+
_id: docId
|
|
321
|
+
});
|
|
322
|
+
/**
|
|
323
|
+
* `true` when the stored live row `stored` is the row an insert of `body`
|
|
324
|
+
* would have written: structurally equal after the JSON round-trip the
|
|
325
|
+
* write path's encoder applies (key order ignored, `undefined` properties
|
|
326
|
+
* dropped). `body` is the insert's post-image `{ ...doc, _id }`; the
|
|
327
|
+
* stored row carries no other server-stamped field (log metadata lives on
|
|
328
|
+
* the `LogEntry`, not in `after`).
|
|
329
|
+
*
|
|
330
|
+
* Shared by `runInsert`'s check read and the writer's insert guard, so an
|
|
331
|
+
* SDK-minted insert replays on the same rule whichever side found the row.
|
|
332
|
+
*/
|
|
333
|
+
const insertReplaysRow = (body, stored) => stored !== void 0 && jsonValueEqual(JSON.parse(JSON.stringify(body)), stored);
|
|
334
|
+
/**
|
|
335
|
+
* Count an SDK-minted insert answered as a replay of its stored row on
|
|
336
|
+
* `db.write.insert_replayed_total` (labelled by `collection`). A replay
|
|
337
|
+
* commits nothing and records no write metric, so without this counter it
|
|
338
|
+
* is invisible on the canonical line. Shared by `runInsert`'s check read
|
|
339
|
+
* and the writer's insert guard, the two places a replay is decided.
|
|
340
|
+
*/
|
|
341
|
+
const recordInsertReplay = (collection) => {
|
|
342
|
+
getCurrentContext()?.recorder.counter("db.write.insert_replayed_total", 1, { collection });
|
|
343
|
+
};
|
|
344
|
+
// ../server/src/key-limit.ts
|
|
345
|
+
const utf8 = new TextEncoder();
|
|
346
|
+
/**
|
|
347
|
+
* Reject an assembled object key over the S3/R2 full-key ceiling
|
|
348
|
+
* ({@link MAX_KEY_BYTES}) as `InvalidConfig`. Per-segment caps don't
|
|
349
|
+
* bound the sum, and index values skip `assertPathSegment` entirely, so
|
|
350
|
+
* this post-encoding check at the writer's PUT sites is their only
|
|
351
|
+
* overflow guard. @see docs/spec/storage-compatibility.md
|
|
352
|
+
*/
|
|
353
|
+
const assertKeyWithinLimit = (key) => {
|
|
354
|
+
const bytes = utf8.encode(key).length;
|
|
355
|
+
if (bytes > 1024) throw new BaerlyError("InvalidConfig", `object key exceeds ${MAX_KEY_BYTES} bytes (got ${bytes}): ${JSON.stringify(key.slice(0, 80))}…`);
|
|
356
|
+
};
|
|
357
|
+
// ../server/src/log-conflict-adoption.ts
|
|
358
|
+
/**
|
|
359
|
+
* Decide whether the entry already at `/log/<seq>.json` is the
|
|
360
|
+
* writer's own prior in-flight attempt and may be adopted. Pure
|
|
361
|
+
* function; no I/O. The caller performs the GET (when the log PUT
|
|
362
|
+
* surfaces 412) and passes the read-back body in `ctx.existing`.
|
|
363
|
+
*
|
|
364
|
+
* ## Correctness precondition
|
|
365
|
+
*
|
|
366
|
+
* A fresh `session` id is minted per `commit()` call, seq-stable
|
|
367
|
+
* across that call's retries. Adoption compares the writer-minted
|
|
368
|
+
* intent/body fields as well as session and seq: a dropped-ack retry
|
|
369
|
+
* reuses the same attempted entry and adopts, while a same-session /
|
|
370
|
+
* same-seq occupant carrying a different logical mutation is treated as
|
|
371
|
+
* a conflict and skipped by the caller's forward-probe loop.
|
|
372
|
+
*
|
|
373
|
+
* @see {@link AdoptionDecision} for the return shape.
|
|
374
|
+
* @see The soundness invariant in this module's header docstring.
|
|
375
|
+
*/
|
|
376
|
+
const tryAdoptOwnSessionLogEntry = (ctx) => {
|
|
377
|
+
if (ctx.existing.session !== ctx.self.session) return {
|
|
378
|
+
adopt: false,
|
|
379
|
+
reason: "foreign-session"
|
|
380
|
+
};
|
|
381
|
+
if (ctx.existing.seq !== ctx.self.seq) return {
|
|
382
|
+
adopt: false,
|
|
383
|
+
reason: "wrong-seq"
|
|
384
|
+
};
|
|
385
|
+
if (JSON.stringify(ctx.existing) !== JSON.stringify(ctx.self)) return {
|
|
386
|
+
adopt: false,
|
|
387
|
+
reason: "intent-mismatch"
|
|
388
|
+
};
|
|
389
|
+
return {
|
|
390
|
+
adopt: true,
|
|
391
|
+
entry: ctx.existing
|
|
392
|
+
};
|
|
393
|
+
};
|
|
394
|
+
// ../server/src/post-image-identity.ts
|
|
395
|
+
const throwBaerlyError = (code, message, cause) => {
|
|
396
|
+
throw new BaerlyError(code, `post-image: ${message}`, { cause });
|
|
397
|
+
};
|
|
398
|
+
/**
|
|
399
|
+
* Assert that an insert/update post-image names the document it is stored
|
|
400
|
+
* under: `after` is a plain object (prototype `Object.prototype` or `null`,
|
|
401
|
+
* not an array) whose own enumerable data property `_id` is a string
|
|
402
|
+
* `=== docId`.
|
|
403
|
+
*
|
|
404
|
+
* This checks identity only. It does not validate `docId` against the
|
|
405
|
+
* `utf8-scalar-v1` domain, and it does not validate any other field of the
|
|
406
|
+
* document. Callers that need the scalar domain call `assertSnapshotDocId`
|
|
407
|
+
* first.
|
|
408
|
+
*
|
|
409
|
+
* `code` selects the failure polarity: `InvalidConfig` for caller input,
|
|
410
|
+
* `InvalidResponse` for stored input. Failures go through `fail` when given,
|
|
411
|
+
* so a codec keeps its own message prefix; otherwise this throws a
|
|
412
|
+
* `BaerlyError` with `code`.
|
|
413
|
+
*/
|
|
414
|
+
function assertPostImageIdentity(docId, after, code, fail = throwBaerlyError) {
|
|
415
|
+
if (after === null || typeof after !== "object" || Array.isArray(after)) fail(code, "after must be a document object");
|
|
416
|
+
const prototype = Object.getPrototypeOf(after);
|
|
417
|
+
if (prototype !== Object.prototype && prototype !== null) fail(code, "after must be a plain object");
|
|
418
|
+
const descriptor = Object.getOwnPropertyDescriptor(after, "_id");
|
|
419
|
+
if (descriptor === void 0 || !descriptor.enumerable || !("value" in descriptor)) fail(code, "after._id must be an own enumerable data property");
|
|
420
|
+
if (typeof descriptor.value !== "string" || descriptor.value !== docId) fail(code, "after._id must equal doc_id");
|
|
421
|
+
}
|
|
422
|
+
// ../server/src/writer.ts
|
|
423
|
+
/**
|
|
424
|
+
* `Writer` — stateless multi-instance write engine (single-write commit).
|
|
425
|
+
*
|
|
426
|
+
* Each {@link Writer.commit} reads `current.json` FRESH (for
|
|
427
|
+
* `log_seq_start` / `tail_hint` as a probe FLOOR — a lower bound, NOT a
|
|
428
|
+
* CAS precondition), then **creates `log/<seq>` via `If-None-Match: "*"`.
|
|
429
|
+
* That create IS the commit / linearization point.** The writer writes NOTHING to `current.json` on the commit
|
|
430
|
+
* path; `tail_hint` advances outside ordinary commits via compaction
|
|
431
|
+
* folds, write-tick refreshes, and explicit operator/import paths. The
|
|
432
|
+
* commit model and its invariants are recorded in ADR-004
|
|
433
|
+
* (`docs/adr/004-single-write-commit.md`). The
|
|
434
|
+
* optional integrity walk is gated by
|
|
435
|
+
* {@link WriterOptions.verifyLogIntegrityOnCommit} (default off).
|
|
436
|
+
*
|
|
437
|
+
* No per-write cache: every commit re-reads + re-probes, so N stateless
|
|
438
|
+
* instances contend at one place — the `If-None-Match` create of
|
|
439
|
+
* `log/<seq>`. The loser gets a 412 and re-probes forward to the next
|
|
440
|
+
* empty slot. A 412 from the writer's OWN session at the same seq is its
|
|
441
|
+
* own lost-ack / crashed-but-durable commit → adopted (not retried), so
|
|
442
|
+
* the write lands at EXACTLY `seq`, never duplicated.
|
|
443
|
+
*
|
|
444
|
+
* **First insert wins.** An `I` carrying {@link CommitInput.insertGuard}
|
|
445
|
+
* reads the log window its caller's absence check could not see and
|
|
446
|
+
* refuses to commit over a live row a peer's `I` put there
|
|
447
|
+
* (`AlreadyExists`), so of concurrent inserts of one `doc_id` at most one
|
|
448
|
+
* commits (exactly one, absent storage failures). See {@link InsertGuard}
|
|
449
|
+
* and `docs/spec/sync-protocol.md` §"Insert: the first `I` wins".
|
|
450
|
+
*
|
|
451
|
+
* **Caller re-decisions.** A {@link CommitAdmission} re-runs the caller's
|
|
452
|
+
* decision over the same window; its verdict may replace a `U`'s
|
|
453
|
+
* post-image, which the write verbs use to re-apply an update to the
|
|
454
|
+
* newest version of its row. See {@link CommitVerdict} and
|
|
455
|
+
* `docs/spec/sync-protocol.md` §"Update and delete: decided on the newest
|
|
456
|
+
* row version".
|
|
457
|
+
*
|
|
458
|
+
* **Crash safety.** The `log/<seq>` create is the commit's only write.
|
|
459
|
+
* `LogEntry.after` is the authoritative post-image, so a crash before the
|
|
460
|
+
* create leaves nothing of the commit on the bucket, and a crash after it
|
|
461
|
+
* leaves a durable commit. The writer maintains no secondary objects:
|
|
462
|
+
* declared indexes change no storage call (ADR-004 Q4, superseded in 0.7).
|
|
463
|
+
*
|
|
464
|
+
* **`LogEntry` shape.** Emitted entries follow the contract in
|
|
465
|
+
* `packages/protocol/src/log.ts` (see {@link LogEntry}) and
|
|
466
|
+
* `docs/spec/log-entry-shape.md`. The on-bucket key is
|
|
467
|
+
* `log/<seq>.json` (the integer `seq` is the load-bearing
|
|
468
|
+
* identifier; the `lsn` string is schema-only).
|
|
469
|
+
*
|
|
470
|
+
* @see docs/spec/sync-protocol.md for the protocol invariants this
|
|
471
|
+
* loop preserves.
|
|
472
|
+
* @see docs/spec/log-entry-shape.md for the per-field contract.
|
|
473
|
+
*/
|
|
474
|
+
const ctxMetrics = () => getCurrentContext()?.recorder ?? noopMetricsRecorder;
|
|
475
|
+
const APPLICATION_JSON = "application/json";
|
|
476
|
+
/**
|
|
477
|
+
* Stateless write engine for the multi-instance core.
|
|
478
|
+
*
|
|
479
|
+
* Construction is cheap and performs zero I/O — adapters build a
|
|
480
|
+
* fresh `Writer` per request and discard it. All real work
|
|
481
|
+
* happens in {@link commit}.
|
|
482
|
+
*
|
|
483
|
+
* @example
|
|
484
|
+
* ```ts
|
|
485
|
+
* import { Writer } from "@gusto/baerly-storage";
|
|
486
|
+
* import { MemoryStorage } from "@gusto/baerly-storage";
|
|
487
|
+
*
|
|
488
|
+
* const writer = new Writer({
|
|
489
|
+
* storage: new MemoryStorage(),
|
|
490
|
+
* currentJsonKey: "app/tickets/tenant/acme/manifests/tickets/current.json",
|
|
491
|
+
* });
|
|
492
|
+
*
|
|
493
|
+
* const result = await writer.commit({
|
|
494
|
+
* op: "I",
|
|
495
|
+
* collection: "tickets",
|
|
496
|
+
* docId: "doc-1",
|
|
497
|
+
* body: { _id: "doc-1", title: "hello" },
|
|
498
|
+
* });
|
|
499
|
+
* console.log(result.entry.seq);
|
|
500
|
+
* ```
|
|
501
|
+
*/
|
|
502
|
+
var Writer = class {
|
|
503
|
+
#storage;
|
|
504
|
+
#currentJsonKey;
|
|
505
|
+
#verifyLogIntegrityOnCommit;
|
|
506
|
+
constructor(opts) {
|
|
507
|
+
this.#storage = opts.storage;
|
|
508
|
+
this.#currentJsonKey = opts.currentJsonKey;
|
|
509
|
+
this.#verifyLogIntegrityOnCommit = opts.options?.verifyLogIntegrityOnCommit ?? false;
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Atomically commit one mutation. Reads `current.json` fresh (as a
|
|
513
|
+
* probe FLOOR, not a CAS precondition), then creates `log/<seq>` via
|
|
514
|
+
* `If-None-Match: "*"` — that create IS the commit (single-write
|
|
515
|
+
* commit; no `current.json` CAS follows it).
|
|
516
|
+
*
|
|
517
|
+
* The hot-path cost under no contention is 3 GETs + 1 PUT, whatever
|
|
518
|
+
* indexes the collection declares: the Step 1 `current.json` read,
|
|
519
|
+
* `findLogTail`'s forward probe (one 404 GET when `tail_hint` is
|
|
520
|
+
* accurate, `O(log gap)` GETs when it lags a fold interval), the
|
|
521
|
+
* committing `log/<seq>` create, and the post-create `current.json`
|
|
522
|
+
* re-read that rejects a create landing below the certified delete
|
|
523
|
+
* floor (`#assertCommitAboveDeleteFloor`). A peer that won the slot
|
|
524
|
+
* costs one occupant readback GET, the forward probe's GETs, and one
|
|
525
|
+
* more create at the next slot.
|
|
526
|
+
*
|
|
527
|
+
* Idempotency: `LogEntry.after` carries the full post-image, and the
|
|
528
|
+
* `log/<seq>` create uses `If-None-Match: "*"`, so two writers racing
|
|
529
|
+
* the same tail cause
|
|
530
|
+
* exactly one create to win; the loser reads back the occupant,
|
|
531
|
+
* adopts its own lost-ack write or re-probes forward to the next
|
|
532
|
+
* empty slot.
|
|
533
|
+
*
|
|
534
|
+
* **Uncertain commit fate.** If the committing create wins at a
|
|
535
|
+
* `seq` the collection has already certified deleted — reachable when a
|
|
536
|
+
* create PUT or a lost-ack retry straddles a fold plus a retirement pass —
|
|
537
|
+
* the object it wrote is beneath every reader's floor. The commit then fails
|
|
538
|
+
* with `AmbiguousCommit` rather than acknowledging a mutation no fresh
|
|
539
|
+
* reader can see. The writer cannot tell whether an earlier attempt of its
|
|
540
|
+
* own landed and was folded before retirement removed the slot, so the
|
|
541
|
+
* mutation may or may not have been applied. The same uncertainty exists
|
|
542
|
+
* when a rejected create cannot be resolved by reading its slot, including
|
|
543
|
+
* rejections that follow a send of unknown outcome. The Writer never
|
|
544
|
+
* resends a create at the same slot (the adapter already retried it); it
|
|
545
|
+
* reads the slot back once instead. A missing readback does not prove that
|
|
546
|
+
* an earlier send will not land. Exact own-entry readback is adopted; an
|
|
547
|
+
* unresolved fate throws `AmbiguousCommit` with `err.retriable === false`.
|
|
548
|
+
* Reconcile state before deliberately retrying: a new logical call can
|
|
549
|
+
* duplicate work, including minting a second auto-ID document.
|
|
550
|
+
*
|
|
551
|
+
* @throws BaerlyError code="AmbiguousCommit" when the committing create
|
|
552
|
+
* landed below `min(log_delete_floor, log_seq_start)`, or a possibly
|
|
553
|
+
* durable create could not be resolved. The mutation's fate is unknown;
|
|
554
|
+
* `err._id` is `input.docId`.
|
|
555
|
+
* @throws BaerlyError code="Internal" when a log entry expected in
|
|
556
|
+
* `[0, tail_hint)` is missing — a protocol-invariant violation
|
|
557
|
+
* (compactor bug or stale `current.json`).
|
|
558
|
+
* @throws BaerlyError code="InvalidResponse" when a log-entry body
|
|
559
|
+
* isn't valid JSON.
|
|
560
|
+
* @throws BaerlyError code="InvalidConfig" when `docId` breaks the
|
|
561
|
+
* key-segment rule, when an `I` / `U` `docId` is not NFC, when an
|
|
562
|
+
* `I` / `U` body is not a plain object whose own `_id` is the string
|
|
563
|
+
* `docId`, when an `I` / `U` post-image encodes to more than
|
|
564
|
+
* `MAX_DOCUMENT_BYTES`, or when `insertGuard` is set on a non-`I` op.
|
|
565
|
+
* Checked before any I/O (an admission verdict's replacement
|
|
566
|
+
* post-image is size-checked when it is returned).
|
|
567
|
+
* @throws BaerlyError code="AlreadyExists" when `insertGuard` is set and
|
|
568
|
+
* the log window `[absentBelowSeq, seq)` holds a live `I` / `U` for
|
|
569
|
+
* this doc (unless `replayIdentical` is set and its `after` equals the
|
|
570
|
+
* body, which returns that entry instead). Nothing was committed;
|
|
571
|
+
* `err._id` is `input.docId`.
|
|
572
|
+
* @throws BaerlyError code="WriteConflict" when `insertGuard` or
|
|
573
|
+
* `admission` is set and part of its window was folded and retired (or
|
|
574
|
+
* is longer than `LOG_FORWARD_PROBE_CAP`) before the writer could read
|
|
575
|
+
* it. Nothing was committed; re-read and retry.
|
|
576
|
+
* @throws BaerlyError code="CapacityExceeded" when the request ledger
|
|
577
|
+
* cannot afford this commit's create and `COMMIT_RESOLUTION_RESERVE`,
|
|
578
|
+
* checked before each create; or when the ledger refused the create
|
|
579
|
+
* itself before any attempt was sent (`cause.notApplied`), or any
|
|
580
|
+
* storage call before it. Nothing was committed.
|
|
581
|
+
* @throws whatever `admission.check` throws (the write verbs throw
|
|
582
|
+
* `CapacityExceeded` or `MaintenancePending`). Nothing was committed.
|
|
583
|
+
*
|
|
584
|
+
* **Window cost.** A commit with `insertGuard` or `admission` probes the
|
|
585
|
+
* tail linearly from `max(windowFrom, log_seq_start)` with
|
|
586
|
+
* `probeTailFrom` instead of galloping from
|
|
587
|
+
* `max(log_seq_start, tail_hint)`. Uncontended, `windowFrom` is the
|
|
588
|
+
* tail, so that is one 404 GET. Each entry a peer committed after the
|
|
589
|
+
* caller's read costs one more GET, and if a fold moved `log_seq_start`
|
|
590
|
+
* past `windowFrom`, each folded slot in between costs one GET too.
|
|
591
|
+
*
|
|
592
|
+
* **Implicit provisioning.** When `current.json` does not exist at
|
|
593
|
+
* `currentJsonKey` (fresh bucket, fresh collection), the writer
|
|
594
|
+
* auto-creates it with a zero-state manifest before the commit
|
|
595
|
+
* lands. The cost is one extra Class A PUT on the very first
|
|
596
|
+
* commit per collection; steady-state cost is unchanged. A peer
|
|
597
|
+
* racing the same first-write loses cleanly on the create CAS
|
|
598
|
+
* and re-reads to pick up the winner's manifest.
|
|
599
|
+
*/
|
|
600
|
+
async commit(input) {
|
|
601
|
+
if (input.op === "D") assertStoredDocId(input.docId);
|
|
602
|
+
else assertDocId(input.docId);
|
|
603
|
+
if (input.op !== "D") {
|
|
604
|
+
assertPostImageIdentity(input.docId, input.body, "InvalidConfig");
|
|
605
|
+
assertDocumentSize(input.body, "Writer");
|
|
606
|
+
}
|
|
607
|
+
if (input.insertGuard !== void 0) assertInsertGuardShape(input);
|
|
608
|
+
if (input.admission !== void 0) assertAdmissionShape(input);
|
|
609
|
+
const session = uuid().slice(0, 6);
|
|
610
|
+
const logPrefix = this.#currentJsonKey.slice(0, this.#currentJsonKey.lastIndexOf("/"));
|
|
611
|
+
const success = await this.#singleAttemptCommit(input, session, logPrefix, "Writer");
|
|
612
|
+
if (success.replayed) recordInsertReplay(input.collection);
|
|
613
|
+
else this.#emitWriteMetrics(input.collection, success.classAOps);
|
|
614
|
+
return {
|
|
615
|
+
entry: success.entries[0],
|
|
616
|
+
currentEtag: success.currentEtag
|
|
617
|
+
};
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* One full commit attempt — the body of {@link commit}. Reads
|
|
621
|
+
* `current.json`, then forward-probes the tail and creates `log/<seq>`
|
|
622
|
+
* once via `If-None-Match: "*"` — the numbered log create IS the commit
|
|
623
|
+
* (single-write commit; no `current.json` CAS follows it). Returns the
|
|
624
|
+
* success payload.
|
|
625
|
+
*
|
|
626
|
+
* Commits exactly one {@link CommitInput} — the numbered log create is
|
|
627
|
+
* the single linearization point per call.
|
|
628
|
+
*
|
|
629
|
+
* A log-create 412 is disambiguated by session read-back: own session
|
|
630
|
+
* / own seq ⇒ our crashed-or-lost-ack commit is already durable ⇒
|
|
631
|
+
* adopt; foreign session ⇒ re-probe forward and create at the next
|
|
632
|
+
* empty slot. Each slot gets one `Storage.put` and never a same-slot
|
|
633
|
+
* resend, because the adapter already retried it. A definitive
|
|
634
|
+
* `AccessDenied` / `InvalidConfig` propagates as-is. A 412 or an uncertain
|
|
635
|
+
* failure resolves by one occupant readback, because adapters can hide
|
|
636
|
+
* earlier sends. When the call reports every attempt refused unapplied
|
|
637
|
+
* (`cause.notApplied`), the same readback runs because a throttle can hide
|
|
638
|
+
* a lost race: a foreign occupant forward-probes, a slot read back as
|
|
639
|
+
* empty or unreadable propagates the retriable `NetworkError`: the
|
|
640
|
+
* adapter already guarantees no attempt applied or can still apply.
|
|
641
|
+
* Unresolved creates throw non-retriable `AmbiguousCommit`; failures
|
|
642
|
+
* before the create propagate.
|
|
643
|
+
*
|
|
644
|
+
* Crash safety: the create is the attempt's only write. A crash before
|
|
645
|
+
* it leaves nothing; a crash after it is a durable, committed write at
|
|
646
|
+
* `seq`, carrying its full post-image in `LogEntry.after`.
|
|
647
|
+
*/
|
|
648
|
+
async #singleAttemptCommit(input, session, logPrefix, errorPrefix) {
|
|
649
|
+
const read = await readCurrentJson(this.#storage, this.#currentJsonKey) ?? await this.#provisionCurrentJson();
|
|
650
|
+
const current = read.json;
|
|
651
|
+
const baseEtag = read.etag;
|
|
652
|
+
if (this.#verifyLogIntegrityOnCommit) await this.#walkLog(logPrefix, logSeqStartOf(current), current.tail_hint);
|
|
653
|
+
const guard = input.insertGuard;
|
|
654
|
+
const admission = input.admission;
|
|
655
|
+
const windowFrom = guard?.absentBelowSeq ?? admission?.fromSeq;
|
|
656
|
+
const logSeqStart = logSeqStartOf(current);
|
|
657
|
+
let commitWindow;
|
|
658
|
+
let admissionForces = false;
|
|
659
|
+
let body = input.body;
|
|
660
|
+
const decide = async (window, tail) => {
|
|
661
|
+
if (admission === void 0) return;
|
|
662
|
+
const verdict = await admission.check(window, tail);
|
|
663
|
+
admissionForces = verdict?.forceMaintenance === true;
|
|
664
|
+
if (verdict?.body !== void 0) {
|
|
665
|
+
if (input.op !== "U") throw new BaerlyError("InvalidConfig", `${errorPrefix}: an admission verdict may replace only a U post-image; got op ${JSON.stringify(input.op)}`);
|
|
666
|
+
assertPostImageIdentity(input.docId, verdict.body, "InvalidConfig");
|
|
667
|
+
assertDocumentSize(verdict.body, errorPrefix);
|
|
668
|
+
body = verdict.body;
|
|
669
|
+
}
|
|
670
|
+
};
|
|
671
|
+
let preCommitTail;
|
|
672
|
+
if (windowFrom === void 0) {
|
|
673
|
+
const probeFloor = Math.max(logSeqStart, current.tail_hint);
|
|
674
|
+
preCommitTail = await findLogTail(this.#storage, logPrefix, probeFloor);
|
|
675
|
+
} else {
|
|
676
|
+
const folded = await this.#readFoldedWindow(logPrefix, windowFrom, logSeqStart, errorPrefix);
|
|
677
|
+
const probe = await probeTailFrom(this.#storage, logPrefix, Math.max(windowFrom, logSeqStart));
|
|
678
|
+
commitWindow = [...folded, ...probe.entries];
|
|
679
|
+
if (guard !== void 0) {
|
|
680
|
+
const replayed = decideInsertTarget(commitWindow, input);
|
|
681
|
+
if (replayed !== void 0) return {
|
|
682
|
+
replayed: true,
|
|
683
|
+
entries: [replayed],
|
|
684
|
+
currentEtag: baseEtag
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
await decide(commitWindow, probe.tail);
|
|
688
|
+
preCommitTail = probe.tail;
|
|
689
|
+
}
|
|
690
|
+
let seq = preCommitTail;
|
|
691
|
+
const commitNowMs = Date.now();
|
|
692
|
+
const commitTs = new Date(commitNowMs).toISOString();
|
|
693
|
+
const lsnTimestamp = timestamp(commitNowMs);
|
|
694
|
+
const mintEntry = (atSeq) => ({
|
|
695
|
+
lsn: `${lsnTimestamp}_${session}_${countKey(atSeq)}`,
|
|
696
|
+
commit_ts: commitTs,
|
|
697
|
+
op: input.op,
|
|
698
|
+
collection: input.collection,
|
|
699
|
+
doc_id: input.docId,
|
|
700
|
+
session,
|
|
701
|
+
seq: atSeq,
|
|
702
|
+
...input.op !== "D" && body !== void 0 ? { after: body } : {},
|
|
703
|
+
...input.origin !== void 0 ? { origin: input.origin } : {}
|
|
704
|
+
});
|
|
705
|
+
let entry = mintEntry(seq);
|
|
706
|
+
let committedEntry;
|
|
707
|
+
let probes = 0;
|
|
708
|
+
const ambiguous = (cause) => {
|
|
709
|
+
ctxMetrics().counter("db.write.unresolved_create_total", 1, { collection: entry.collection });
|
|
710
|
+
return new BaerlyError("AmbiguousCommit", `${errorPrefix}: the committing create at seq ${seq} on ${this.#currentJsonKey} was rejected and its fate could not be resolved; an earlier send may already have landed or may still land, so retrying may duplicate logical work`, {
|
|
711
|
+
cause,
|
|
712
|
+
resolution: CODE_RESOLUTIONS.AmbiguousCommit,
|
|
713
|
+
_id: entry.doc_id
|
|
714
|
+
});
|
|
715
|
+
};
|
|
716
|
+
for (;;) {
|
|
717
|
+
const outcome = await this.#withCommitHold(errorPrefix, async () => {
|
|
718
|
+
const logEntryKey = logObjectKey(logPrefix, seq);
|
|
719
|
+
assertKeyWithinLimit(logEntryKey);
|
|
720
|
+
const entryBody = encodeJsonBytes(entry);
|
|
721
|
+
let resolved;
|
|
722
|
+
let createFailure;
|
|
723
|
+
let notApplied = false;
|
|
724
|
+
try {
|
|
725
|
+
await this.#storage.put(logEntryKey, entryBody, {
|
|
726
|
+
ifNoneMatch: "*",
|
|
727
|
+
contentType: APPLICATION_JSON
|
|
728
|
+
});
|
|
729
|
+
resolved = entry;
|
|
730
|
+
} catch (error) {
|
|
731
|
+
createFailure = error;
|
|
732
|
+
this.#observe429(error, entry.collection);
|
|
733
|
+
if (isWriteConflict(error)) ctxMetrics().counter("db.r2.put.412_total", 1, {
|
|
734
|
+
collection: entry.collection,
|
|
735
|
+
step: "log-put"
|
|
736
|
+
});
|
|
737
|
+
else if (isDefinitiveCreateRejection(error)) throw error;
|
|
738
|
+
else if (isCreateNotApplied(error)) notApplied = true;
|
|
739
|
+
}
|
|
740
|
+
if (resolved === void 0) {
|
|
741
|
+
let occupant;
|
|
742
|
+
let decision;
|
|
743
|
+
try {
|
|
744
|
+
occupant = await this.#readLogOccupant(logEntryKey);
|
|
745
|
+
decision = tryAdoptOwnSessionLogEntry({
|
|
746
|
+
self: entry,
|
|
747
|
+
existing: occupant
|
|
748
|
+
});
|
|
749
|
+
} catch (error) {
|
|
750
|
+
if (notApplied) throw createFailure;
|
|
751
|
+
throw ambiguous(new AggregateError([createFailure, error], "The create and its readback could not be resolved"));
|
|
752
|
+
}
|
|
753
|
+
if (!decision.adopt) return { foreign: occupant };
|
|
754
|
+
resolved = decision.entry;
|
|
755
|
+
}
|
|
756
|
+
await this.#assertCommitAboveDeleteFloor(logPrefix, seq, input.collection, input.docId, errorPrefix);
|
|
757
|
+
return { committed: resolved };
|
|
758
|
+
});
|
|
759
|
+
if ("committed" in outcome) {
|
|
760
|
+
committedEntry = outcome.committed;
|
|
761
|
+
break;
|
|
762
|
+
}
|
|
763
|
+
const occupant = outcome.foreign;
|
|
764
|
+
if (++probes > 1e5) throw new BaerlyError("Internal", `${errorPrefix}: log-tail forward-probe exceeded ${LOG_FORWARD_PROBE_CAP} on ${this.#currentJsonKey}`);
|
|
765
|
+
if (commitWindow === void 0) seq = await findLogTail(this.#storage, logPrefix, seq + 1);
|
|
766
|
+
else {
|
|
767
|
+
commitWindow.push(occupant);
|
|
768
|
+
const probe = await probeTailFrom(this.#storage, logPrefix, seq + 1);
|
|
769
|
+
commitWindow.push(...probe.entries);
|
|
770
|
+
if (guard !== void 0) {
|
|
771
|
+
const replayed = decideInsertTarget(commitWindow, input);
|
|
772
|
+
if (replayed !== void 0) return {
|
|
773
|
+
replayed: true,
|
|
774
|
+
entries: [replayed],
|
|
775
|
+
currentEtag: baseEtag
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
await decide(commitWindow, probe.tail);
|
|
779
|
+
seq = probe.tail;
|
|
780
|
+
}
|
|
781
|
+
entry = mintEntry(seq);
|
|
782
|
+
}
|
|
783
|
+
const committedEntries = [committedEntry];
|
|
784
|
+
const prevSeq = preCommitTail;
|
|
785
|
+
const observedTail = seq + 1;
|
|
786
|
+
const maint = getCurrentContext()?.maintenance;
|
|
787
|
+
const gcInterval = maint?.options?.profile?.gcInterval ?? MAINTENANCE_PROFILE_CF_FREE.gcInterval;
|
|
788
|
+
const forceFold = (input.forceMaintenance === true || admissionForces) && maint?.disabled !== true;
|
|
789
|
+
const foldAtTailEntries = (maint?.options)?.foldAtTailEntries;
|
|
790
|
+
if (maint?.disabled === true ? observedTail - current.tail_hint >= 128 : forceFold || foldAtTailEntries !== void 0 && observedTail - current.log_seq_start >= foldAtTailEntries || shouldFireMaintenance(current, prevSeq, gcInterval, observedTail)) await dispatchWriteTick({
|
|
791
|
+
storage: this.#storage,
|
|
792
|
+
currentJsonKey: this.#currentJsonKey,
|
|
793
|
+
collection: input.collection,
|
|
794
|
+
prevSeq,
|
|
795
|
+
observedTail,
|
|
796
|
+
...forceFold && { forceFold: true }
|
|
797
|
+
});
|
|
798
|
+
return {
|
|
799
|
+
replayed: false,
|
|
800
|
+
entries: committedEntries,
|
|
801
|
+
currentEtag: baseEtag,
|
|
802
|
+
classAOps: 1
|
|
803
|
+
};
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* Auto-create the per-collection `current.json` at
|
|
807
|
+
* `this.#currentJsonKey` with a zero-state initial manifest. Called
|
|
808
|
+
* by `#singleAttemptCommit` when the read returned `null` (fresh
|
|
809
|
+
* bucket / fresh collection).
|
|
810
|
+
*
|
|
811
|
+
* Concurrency: `createCurrentJson` uses `If-None-Match: "*"`. If a
|
|
812
|
+
* peer raced us and won, the storage layer surfaces `WriteConflict`; we
|
|
813
|
+
* recover by re-reading the now-present manifest. We do NOT retry
|
|
814
|
+
* the create — a single recover-via-read covers every race that
|
|
815
|
+
* could have produced our null read.
|
|
816
|
+
*
|
|
817
|
+
* The seed shape (`snapshot: null`, `tail_hint: 0`, `log_seq_start:
|
|
818
|
+
* 0`, `writer_fence: { epoch: 0, owner: "", claimed_at: "" }`)
|
|
819
|
+
* matches `ensureTable`, so an
|
|
820
|
+
* operator who pre-provisions and a writer who auto-creates land on
|
|
821
|
+
* equivalent state. NOT byte-identical any more: `generation` is a
|
|
822
|
+
* fresh nonce per provisioner. That is harmless — `If-None-Match`
|
|
823
|
+
* means exactly one of them ever lands, and no reader compares a
|
|
824
|
+
* generation across provisioners, only against the cursor it minted.
|
|
825
|
+
*
|
|
826
|
+
* @throws BaerlyError code="InvalidResponse" — the post-recover
|
|
827
|
+
* re-read also returned null. Defensive guard; in practice
|
|
828
|
+
* unreachable because the WriteConflict implies the key now exists.
|
|
829
|
+
*/
|
|
830
|
+
async #provisionCurrentJson() {
|
|
831
|
+
const initial = {
|
|
832
|
+
schema_version: 3,
|
|
833
|
+
snapshot: null,
|
|
834
|
+
tail_hint: 0,
|
|
835
|
+
log_seq_start: 0,
|
|
836
|
+
writer_fence: {
|
|
837
|
+
epoch: 0,
|
|
838
|
+
owner: "",
|
|
839
|
+
claimed_at: ""
|
|
840
|
+
},
|
|
841
|
+
snapshot_bytes: 0,
|
|
842
|
+
snapshot_rows: 0,
|
|
843
|
+
generation: mintGeneration()
|
|
844
|
+
};
|
|
845
|
+
let createFailure;
|
|
846
|
+
try {
|
|
847
|
+
return await createCurrentJson(this.#storage, this.#currentJsonKey, initial);
|
|
848
|
+
} catch (error) {
|
|
849
|
+
if (!isWriteConflict(error) && !isCreateNotApplied(error)) throw error;
|
|
850
|
+
createFailure = error;
|
|
851
|
+
}
|
|
852
|
+
const recovered = await readCurrentJson(this.#storage, this.#currentJsonKey);
|
|
853
|
+
if (recovered === null && isCreateNotApplied(createFailure)) throw createFailure;
|
|
854
|
+
if (recovered === null) throw new BaerlyError("InvalidResponse", `Writer: current.json missing at ${this.#currentJsonKey} after auto-create raced and recover read returned null`);
|
|
855
|
+
return recovered;
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Fail the commit if the create landed at a sequence the collection has
|
|
859
|
+
* already certified deleted.
|
|
860
|
+
*
|
|
861
|
+
* **What a sub-floor `seq` proves, and what it does not.** A winning
|
|
862
|
+
* `If-None-Match: "*"` create means the slot was empty immediately before the
|
|
863
|
+
* PUT, so the object this commit just wrote sits beneath every reader's floor
|
|
864
|
+
* and no reader will ever consult it. What a sub-floor `seq` does NOT prove
|
|
865
|
+
* is that the *logical mutation* is invisible. Two histories reach this
|
|
866
|
+
* branch: the create re-filled a hole retirement had already certified
|
|
867
|
+
* deleted (invisible — the defect this check exists for), or it landed in a
|
|
868
|
+
* live empty slot that a fold and then a retirement pass swept out from under
|
|
869
|
+
* it (visible through the snapshot). That second history is a real
|
|
870
|
+
* over-rejection, pinned as contract by a test in
|
|
871
|
+
* `./log-retirement-aba.test.ts`, not an accident. It is the safe direction,
|
|
872
|
+
* and "What the caller is being told" below is why guessing between the two
|
|
873
|
+
* is worse than reporting both as ambiguous.
|
|
874
|
+
*
|
|
875
|
+
* **`log_delete_floor <= seq < log_seq_start` is NOT a failure.** Not because
|
|
876
|
+
* that band is provably safe — it is undecidable in the very same way. A
|
|
877
|
+
* winning create means the slot was empty just before the PUT, so what a fold
|
|
878
|
+
* absorbed there was either an earlier attempt of ours (mutation visible) or
|
|
879
|
+
* a foreign writer's entry (ours invisible), and the fold keeps no writer
|
|
880
|
+
* identity to tell them apart. The band is accepted because keying the check
|
|
881
|
+
* on `log_seq_start` would reject ordinary post-fold commits — a far
|
|
882
|
+
* larger harm than the residual it would catch. That residual is the
|
|
883
|
+
* invisible half of the undecidable band `docs/spec/sync-protocol.md`
|
|
884
|
+
* invariant 5 describes; it is not reachable by the certified-floor check,
|
|
885
|
+
* by construction.
|
|
886
|
+
*
|
|
887
|
+
* The floor is clamped to `min(log_delete_floor, log_seq_start)` for the
|
|
888
|
+
* reason invariant 12 gives: the `<= log_seq_start` bound is
|
|
889
|
+
* transition-scoped, so an out-of-bound stored floor is readable off disk
|
|
890
|
+
* and would otherwise fail a perfectly good commit.
|
|
891
|
+
*
|
|
892
|
+
* **What the caller is being told.** Not "your write failed" —
|
|
893
|
+
* `AmbiguousCommit`. An earlier attempt by this same writer may have landed
|
|
894
|
+
* and been folded before retirement removed the slot, and nothing durable
|
|
895
|
+
* distinguishes that history from a foreign writer's entry being folded
|
|
896
|
+
* instead: the fold keeps no writer identity, tombstones are purged,
|
|
897
|
+
* `current.json` holds no per-writer completion state, and `writer_fence` is
|
|
898
|
+
* not a replay filter (invariant 11). So the write contract under this fault
|
|
899
|
+
* is at-least-once, and re-probing-and-re-applying is emphatically not a
|
|
900
|
+
* safe default: `LogEntry.after` is a full post-image, so a duplicate
|
|
901
|
+
* clobbers a concurrent newer value and mints a second `lsn` for one logical
|
|
902
|
+
* mutation.
|
|
903
|
+
*
|
|
904
|
+
* The phantom object is DELETEd best-effort, and it is the only thing
|
|
905
|
+
* standing between a rejected commit and a permanently leaked object.
|
|
906
|
+
* Nothing reclaims the phantom afterwards: it sits at a seq *below*
|
|
907
|
+
* `log_delete_floor` — that is the precondition for reaching this branch —
|
|
908
|
+
* and `computeRetirableRange` (`./log-retention.ts`) *starts* at the
|
|
909
|
+
* certified floor, so retirement never revisits a slot beneath its own
|
|
910
|
+
* floor. GC does not reclaim log objects at all. So a `cleanup: "failed"`
|
|
911
|
+
* on `db.write.ambiguous_commit_total` is a genuine permanent leak, not a
|
|
912
|
+
* deferral. Deleting it is safe: no reader may read below the certified
|
|
913
|
+
* floor (invariant 5). A failed cleanup is recorded on the counter and never masks the
|
|
914
|
+
* throw.
|
|
915
|
+
*
|
|
916
|
+
* **A failed manifest read skips the check rather than failing the commit —
|
|
917
|
+
* except an abort, which propagates.** The check runs over an already-durable
|
|
918
|
+
* create, and an unreadable manifest is no evidence of a sub-floor `seq`, so a
|
|
919
|
+
* transient read failure is swallowed. An `AbortError` is rethrown instead: it
|
|
920
|
+
* is the caller's own cancellation rather than a blip, and it carries none of
|
|
921
|
+
* the `retriable` re-run hazard that motivates swallowing the rest. The skip
|
|
922
|
+
* is recorded on `db.write.delete_floor_check_skipped_total` so a sustained
|
|
923
|
+
* rate is visible to an operator rather than silent.
|
|
924
|
+
*
|
|
925
|
+
* Cost: one Class B GET per commit.
|
|
926
|
+
* `billableClassAOps = puts + lists` excludes GET, so billing is unchanged;
|
|
927
|
+
* it is one of the calls `COMMIT_RESOLUTION_RESERVE` keeps on the request
|
|
928
|
+
* ledger.
|
|
929
|
+
*
|
|
930
|
+
* @throws BaerlyError code="AmbiguousCommit"
|
|
931
|
+
*/
|
|
932
|
+
async #assertCommitAboveDeleteFloor(logPrefix, seq, collection, docId, errorPrefix) {
|
|
933
|
+
let read;
|
|
934
|
+
try {
|
|
935
|
+
read = await readCurrentJson(this.#storage, this.#currentJsonKey);
|
|
936
|
+
} catch (error) {
|
|
937
|
+
if (isAbortError(error)) throw error;
|
|
938
|
+
ctxMetrics().counter("db.write.delete_floor_check_skipped_total", 1, { collection });
|
|
939
|
+
countPostCommitRefusal(error, collection, "delete-floor-check");
|
|
940
|
+
return;
|
|
941
|
+
}
|
|
942
|
+
if (read === null) return;
|
|
943
|
+
const deleteFloor = certifiedDeleteFloor(read.json);
|
|
944
|
+
if (deleteFloor <= 0 || seq >= deleteFloor) return;
|
|
945
|
+
let cleanup = "deleted";
|
|
946
|
+
await this.#storage.delete(logObjectKey(logPrefix, seq)).catch(() => {
|
|
947
|
+
cleanup = "failed";
|
|
948
|
+
});
|
|
949
|
+
ctxMetrics().counter("db.write.ambiguous_commit_total", 1, {
|
|
950
|
+
collection,
|
|
951
|
+
cleanup
|
|
952
|
+
});
|
|
953
|
+
throw new BaerlyError("AmbiguousCommit", `${errorPrefix}: the committing create at seq ${seq} on ${this.#currentJsonKey} landed below the certified delete floor ${deleteFloor}; that log object is beneath every reader's floor, and this mutation may or may not have been applied by an earlier attempt — retry is at-least-once`, {
|
|
954
|
+
resolution: CODE_RESOLUTIONS.AmbiguousCommit,
|
|
955
|
+
_id: docId
|
|
956
|
+
});
|
|
957
|
+
}
|
|
958
|
+
/**
|
|
959
|
+
* Run one commit slot's attempt, `fn` (the committing create and the
|
|
960
|
+
* calls that resolve it), under a hold on the active request ledger of
|
|
961
|
+
* everything it may spend: the create at full retries, plus
|
|
962
|
+
* {@link COMMIT_RESOLUTION_RESERVE} for the readback and the delete-floor
|
|
963
|
+
* check. `fn` runs in a child observability context carrying the hold, so
|
|
964
|
+
* only its own storage calls draw on it; work outside that context (a
|
|
965
|
+
* `ctx.waitUntil` write tick of an earlier row, the read cache) sees the
|
|
966
|
+
* ledger net of it and cannot spend it. The hold is released when `fn`
|
|
967
|
+
* settles.
|
|
968
|
+
*
|
|
969
|
+
* Refuses, before `fn` sends anything, when the ledger's room is short
|
|
970
|
+
* of the hold. No create of this commit has landed or can land then:
|
|
971
|
+
* before the first slot nothing was sent, and before a later one a
|
|
972
|
+
* foreign occupant holds the slot this commit's only send targeted. A host without request
|
|
973
|
+
* limits (Node, or no ledger) never refuses and takes no hold.
|
|
974
|
+
*
|
|
975
|
+
* @throws BaerlyError code="CapacityExceeded", cause `{ refused: true }`.
|
|
976
|
+
*/
|
|
977
|
+
/**
|
|
978
|
+
* The collection segment of `current.json`'s key, for messages that
|
|
979
|
+
* reach HTTP callers unscrubbed (`CapacityExceeded` is `public`), which
|
|
980
|
+
* must not carry the storage key.
|
|
981
|
+
*/
|
|
982
|
+
#collectionName() {
|
|
983
|
+
const parts = this.#currentJsonKey.split("/");
|
|
984
|
+
return parts[parts.length - 2] ?? "";
|
|
985
|
+
}
|
|
986
|
+
async #withCommitHold(errorPrefix, fn) {
|
|
987
|
+
const ctx = getCurrentContext();
|
|
988
|
+
const ledger = ctx?.ledger;
|
|
989
|
+
const remaining = ledger?.storageRemaining() ?? 1e400;
|
|
990
|
+
if (ctx === void 0 || ledger === void 0 || remaining === 1e400) return await fn();
|
|
991
|
+
const needed = 9;
|
|
992
|
+
if (remaining < needed) throw new BaerlyError("CapacityExceeded", `${errorPrefix}: this request has ${String(remaining)} storage calls left, fewer than the ${String(needed)} a commit to collection ${JSON.stringify(this.#collectionName())} may need (the committing create, with retries, plus resolving it), so nothing was committed; the work this request needs does not fit in one request on this host`, { cause: {
|
|
993
|
+
refused: true,
|
|
994
|
+
needed,
|
|
995
|
+
remaining
|
|
996
|
+
} });
|
|
997
|
+
const hold = ledger.hold(needed);
|
|
998
|
+
try {
|
|
999
|
+
return await runWithContext({
|
|
1000
|
+
...ctx,
|
|
1001
|
+
hold
|
|
1002
|
+
}, fn);
|
|
1003
|
+
} finally {
|
|
1004
|
+
hold.release();
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Walk `log/<logSeqStart>.json` … `log/<nextSeq - 1>.json` with
|
|
1009
|
+
* bounded parallelism via `walkLogRange`. Materialised entries are
|
|
1010
|
+
* discarded — the walk's only job is to surface a hole or a
|
|
1011
|
+
* malformed body inside the visible range as
|
|
1012
|
+
* `BaerlyError{code:"Internal"}` / `"InvalidResponse"`.
|
|
1013
|
+
*
|
|
1014
|
+
* Gated on `verifyLogIntegrityOnCommit`; default off in production
|
|
1015
|
+
* because the per-doc fold already lives in the read path. See the
|
|
1016
|
+
* option's docstring for the rationale and the opt-in convention.
|
|
1017
|
+
*
|
|
1018
|
+
* `logSeqStart` is the boundary set by the compactor on
|
|
1019
|
+
* `current.json.log_seq_start`. Entries below it have been folded
|
|
1020
|
+
* into the snapshot and may have been swept off the bucket, so the
|
|
1021
|
+
* walk MUST NOT GET-require them.
|
|
1022
|
+
*/
|
|
1023
|
+
async #walkLog(logPrefix, logSeqStart, nextSeq) {
|
|
1024
|
+
await walkLogRange(this.#storage, logPrefix, logSeqStart, nextSeq);
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Read `[fromSeq, logSeqStart)` for a windowed commit (insert guard or
|
|
1028
|
+
* admission) whose window starts below the floor of this commit's
|
|
1029
|
+
* manifest read: a fold absorbed those entries after the caller's read. Slots below the floor stay
|
|
1030
|
+
* readable until log retirement deletes them, so a missing slot means
|
|
1031
|
+
* this commit can no longer check its window: `WriteConflict`, nothing
|
|
1032
|
+
* committed, and the caller's next read sees the folded state. A window
|
|
1033
|
+
* longer than {@link LOG_FORWARD_PROBE_CAP} is refused the same way
|
|
1034
|
+
* rather than spending an unbounded run of GETs on the write path.
|
|
1035
|
+
*
|
|
1036
|
+
* @throws BaerlyError code="WriteConflict" — a slot is missing, or the
|
|
1037
|
+
* range exceeds the cap.
|
|
1038
|
+
* @throws BaerlyError code="InvalidResponse" — a slot's body is not JSON.
|
|
1039
|
+
*/
|
|
1040
|
+
async #readFoldedWindow(logPrefix, fromSeq, logSeqStart, errorPrefix) {
|
|
1041
|
+
if (fromSeq >= logSeqStart) return [];
|
|
1042
|
+
const refuse = (detail) => new BaerlyError("WriteConflict", `${errorPrefix}: commit window [${fromSeq}, ${logSeqStart}) on ${this.#currentJsonKey} ${detail}; nothing was committed`);
|
|
1043
|
+
if (logSeqStart - fromSeq > 1e5) throw refuse(`exceeds ${LOG_FORWARD_PROBE_CAP} folded entries`);
|
|
1044
|
+
const entries = [];
|
|
1045
|
+
for (let chunk = fromSeq; chunk < logSeqStart; chunk += 16) {
|
|
1046
|
+
const end = Math.min(chunk + 16, logSeqStart);
|
|
1047
|
+
const keys = Array.from({ length: end - chunk }, (_, i) => logObjectKey(logPrefix, chunk + i));
|
|
1048
|
+
const bodies = await Promise.all(keys.map((key) => this.#storage.get(key)));
|
|
1049
|
+
for (let i = 0; i < bodies.length; i++) {
|
|
1050
|
+
const got = bodies[i];
|
|
1051
|
+
if (got === null || got === void 0) throw refuse(`was retired at seq ${chunk + i} before it could be checked`);
|
|
1052
|
+
try {
|
|
1053
|
+
entries.push(decodeJsonBytes(got.body));
|
|
1054
|
+
} catch (error) {
|
|
1055
|
+
throw new BaerlyError("InvalidResponse", `${errorPrefix}: malformed log entry at ${keys[i]}`, { cause: error });
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
return entries;
|
|
1060
|
+
}
|
|
1061
|
+
/**
|
|
1062
|
+
* Read the occupant of a slot whose committing create was rejected, with
|
|
1063
|
+
* one `Storage.get` (the adapter retries it; the Writer does not). A failed
|
|
1064
|
+
* read, an empty slot, or an occupant without a writer identity throws:
|
|
1065
|
+
* none of them proves the slot holds a foreign entry rather than a pending
|
|
1066
|
+
* send of our own.
|
|
1067
|
+
*/
|
|
1068
|
+
async #readLogOccupant(key) {
|
|
1069
|
+
const got = await this.#storage.get(key);
|
|
1070
|
+
if (got === null) throw new BaerlyError("NotFound", `Writer: ${key} is empty after its create was rejected`);
|
|
1071
|
+
let existing;
|
|
1072
|
+
try {
|
|
1073
|
+
existing = decodeJsonBytes(got.body);
|
|
1074
|
+
} catch (error) {
|
|
1075
|
+
throw new BaerlyError("InvalidResponse", `Writer: malformed log occupant at ${key}`, { cause: error });
|
|
1076
|
+
}
|
|
1077
|
+
if (!hasWriterIdentity(existing)) throw new BaerlyError("InvalidResponse", `Writer: malformed log occupant at ${key}`);
|
|
1078
|
+
return existing;
|
|
1079
|
+
}
|
|
1080
|
+
/**
|
|
1081
|
+
* Bump the `db.r2.put.429_total` counter when `err` looks like an
|
|
1082
|
+
* R2 prefix-partition rate-limit. Best-effort: detected via
|
|
1083
|
+
* `BaerlyError{code:"NetworkError"}` with a `429` token in the
|
|
1084
|
+
* message. Called from every PUT call site's catch arm; mutually
|
|
1085
|
+
* exclusive in practice with `isWriteConflict` (412 ≠ 429).
|
|
1086
|
+
*/
|
|
1087
|
+
#observe429(err, collection) {
|
|
1088
|
+
if (is429(err)) ctxMetrics().counter("db.r2.put.429_total", 1, { collection });
|
|
1089
|
+
}
|
|
1090
|
+
/**
|
|
1091
|
+
* Emit the per-logical-write `class_a_ops_per_logical_write`
|
|
1092
|
+
* histogram for `commit`. `classAOps` is the committing create, computed
|
|
1093
|
+
* by `#singleAttemptCommit`.
|
|
1094
|
+
*/
|
|
1095
|
+
#emitWriteMetrics(collection, classAOps) {
|
|
1096
|
+
ctxMetrics().histogram("db.write.class_a_ops_per_logical_write", classAOps, { collection });
|
|
1097
|
+
}
|
|
1098
|
+
};
|
|
1099
|
+
/**
|
|
1100
|
+
* `true` when a create rejection proves the create did not land. Under the
|
|
1101
|
+
* `Storage.put` contract an adapter that retried internally reports such a
|
|
1102
|
+
* rejection as `NetworkError`, so these codes describe the only attempt.
|
|
1103
|
+
*/
|
|
1104
|
+
const isDefinitiveCreateRejection = (err) => err instanceof BaerlyError && (err.code === "AccessDenied" || err.code === "InvalidConfig");
|
|
1105
|
+
/**
|
|
1106
|
+
* `true` when a create-only put reports, per the `Storage.put` contract,
|
|
1107
|
+
* that no attempt of the call was applied: a `NetworkError` after every
|
|
1108
|
+
* attempt was throttled or refused before issue, or the request ledger's
|
|
1109
|
+
* `CapacityExceeded` when it refused the call before any attempt of
|
|
1110
|
+
* unknown outcome. `cause.notApplied` is the proof either way.
|
|
1111
|
+
*/
|
|
1112
|
+
const isCreateNotApplied = (err) => err instanceof BaerlyError && (err.code === "NetworkError" || err.code === "CapacityExceeded") && err.cause?.notApplied === true;
|
|
1113
|
+
/**
|
|
1114
|
+
* Count a post-commit step the request ledger refused on
|
|
1115
|
+
* `db.write.post_commit_refused_total`. The step runs after a known
|
|
1116
|
+
* commit and swallows its failure, so the refusal leaves the commit
|
|
1117
|
+
* acknowledged; `phase` names the step that went without (the delete-floor
|
|
1118
|
+
* check is the only one).
|
|
1119
|
+
*/
|
|
1120
|
+
const countPostCommitRefusal = (err, collection, phase) => {
|
|
1121
|
+
if (err instanceof BaerlyError && err.code === "CapacityExceeded" && err.cause?.refused === true) ctxMetrics().counter("db.write.post_commit_refused_total", 1, {
|
|
1122
|
+
collection,
|
|
1123
|
+
phase
|
|
1124
|
+
});
|
|
1125
|
+
};
|
|
1126
|
+
/**
|
|
1127
|
+
* Reject an `insertGuard` on anything but an `I`, or with an
|
|
1128
|
+
* `absentBelowSeq` that is not a log seq. Runs before any I/O.
|
|
1129
|
+
*/
|
|
1130
|
+
const assertInsertGuardShape = (input) => {
|
|
1131
|
+
if (input.op !== "I") throw new BaerlyError("InvalidConfig", `Writer: insertGuard is valid only with op "I"; got op ${JSON.stringify(input.op)}`);
|
|
1132
|
+
const seq = input.insertGuard?.absentBelowSeq;
|
|
1133
|
+
if (typeof seq !== "number" || !Number.isSafeInteger(seq) || seq < 0) throw new BaerlyError("InvalidConfig", `Writer: insertGuard.absentBelowSeq must be a non-negative safe integer; got ${String(seq)}`);
|
|
1134
|
+
const replay = input.insertGuard?.replayIdentical;
|
|
1135
|
+
if (replay !== void 0 && typeof replay !== "boolean") throw new BaerlyError("InvalidConfig", `Writer: insertGuard.replayIdentical must be a boolean; got ${typeof replay}`);
|
|
1136
|
+
};
|
|
1137
|
+
/**
|
|
1138
|
+
* Reject an `admission` whose `fromSeq` is not a log seq, or differs from
|
|
1139
|
+
* the insert guard's window start. Runs before any I/O.
|
|
1140
|
+
*/
|
|
1141
|
+
const assertAdmissionShape = (input) => {
|
|
1142
|
+
const seq = input.admission?.fromSeq;
|
|
1143
|
+
if (typeof seq !== "number" || !Number.isSafeInteger(seq) || seq < 0) throw new BaerlyError("InvalidConfig", `Writer: admission.fromSeq must be a non-negative safe integer; got ${String(seq)}`);
|
|
1144
|
+
if (typeof input.admission?.check !== "function") throw new BaerlyError("InvalidConfig", "Writer: admission.check must be a function");
|
|
1145
|
+
const guarded = input.insertGuard?.absentBelowSeq;
|
|
1146
|
+
if (guarded !== void 0 && guarded !== seq) throw new BaerlyError("InvalidConfig", `Writer: admission.fromSeq ${seq} must equal insertGuard.absentBelowSeq ${guarded}`);
|
|
1147
|
+
};
|
|
1148
|
+
/**
|
|
1149
|
+
* Window decision for a guarded insert. `window` holds the log entries
|
|
1150
|
+
* in `[absentBelowSeq, seq)` in seq order, unfiltered. The LAST entry for
|
|
1151
|
+
* this `(collection, doc_id)` decides: a `D`, or no entry, means the row
|
|
1152
|
+
* is absent and the insert may commit (`undefined`). An `I` or `U` means a
|
|
1153
|
+
* row is live: with `replayIdentical` set and that entry's `after` equal
|
|
1154
|
+
* to the body, the insert is a replay of its own earlier success and the
|
|
1155
|
+
* entry is returned; otherwise the insert throws `AlreadyExists`.
|
|
1156
|
+
*
|
|
1157
|
+
* @throws BaerlyError code="AlreadyExists"
|
|
1158
|
+
*/
|
|
1159
|
+
const decideInsertTarget = (window, input) => {
|
|
1160
|
+
for (let i = window.length - 1; i >= 0; i--) {
|
|
1161
|
+
const e = window[i];
|
|
1162
|
+
if (!isOwnCollection(e.collection, input.collection) || e.doc_id !== input.docId) continue;
|
|
1163
|
+
if (e.op === "D") return;
|
|
1164
|
+
if (input.insertGuard?.replayIdentical === true && input.body !== void 0 && insertReplaysRow(input.body, e.after)) return e;
|
|
1165
|
+
throw insertAlreadyExists(input.collection, input.docId);
|
|
1166
|
+
}
|
|
1167
|
+
};
|
|
1168
|
+
/** `true` when a decoded log occupant carries the identity adoption compares. */
|
|
1169
|
+
const hasWriterIdentity = (value) => {
|
|
1170
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
1171
|
+
const { session, seq } = value;
|
|
1172
|
+
return typeof session === "string" && session.length > 0 && typeof seq === "number" && Number.isSafeInteger(seq) && seq >= 0;
|
|
1173
|
+
};
|
|
1174
|
+
/**
|
|
1175
|
+
* `true` when a rejection is an `AbortError` — the caller's own cancellation
|
|
1176
|
+
* through an `AbortSignal`, never a transient storage failure. Matched
|
|
1177
|
+
* structurally on `name` rather than `instanceof DOMException` because a host
|
|
1178
|
+
* may surface an abort as a plain `Error` so named (as `Storage` impls and the
|
|
1179
|
+
* test fixtures do). `@baerly/client` carries the same predicate; it is
|
|
1180
|
+
* duplicated rather than shared because `server` may not import `client`
|
|
1181
|
+
* (`scripts/lint-package-layers.mjs`) and one predicate does not earn a new
|
|
1182
|
+
* module in `@baerly/protocol`.
|
|
1183
|
+
*/
|
|
1184
|
+
const isAbortError = (err) => typeof err === "object" && err !== null && "name" in err && err.name === "AbortError";
|
|
1185
|
+
/**
|
|
1186
|
+
* `true` when the underlying storage surfaced an R2 prefix-partition
|
|
1187
|
+
* rate-limit. {@link S3HttpStorage} stamps the upstream HTTP status
|
|
1188
|
+
* onto `BaerlyError.cause` as `{ status }`; we discriminate on that
|
|
1189
|
+
* structured field rather than regex-matching the message.
|
|
1190
|
+
*/
|
|
1191
|
+
const is429 = (err) => {
|
|
1192
|
+
if (!(err instanceof BaerlyError) || err.code !== "NetworkError") return false;
|
|
1193
|
+
return err.cause?.status === 429;
|
|
1194
|
+
};
|
|
1195
|
+
// src/admin/restore.ts
|
|
1196
|
+
/**
|
|
1197
|
+
* `baerly admin restore` — bulk-import NDJSON into a fresh collection.
|
|
1198
|
+
*
|
|
1199
|
+
* Reads NDJSON from stdin, reconstructs each row, and calls
|
|
1200
|
+
* `Writer.commit({op:"I"})` per row into the target bucket.
|
|
1201
|
+
* Programmatic callers (today: tests + the export round-trip
|
|
1202
|
+
* integration test) can divert the input by passing
|
|
1203
|
+
* `{ streams: { stdin } }` to {@link runRestore}. Idempotent on a fresh
|
|
1204
|
+
* bucket: re-running on a half-completed restore **refuses** unless
|
|
1205
|
+
* `--force` is set (which reseeds a fresh `current.json` above the old
|
|
1206
|
+
* tail first).
|
|
1207
|
+
*
|
|
1208
|
+
* Args:
|
|
1209
|
+
* --bucket Required. Target bucket URI.
|
|
1210
|
+
* --app Required (or via baerly.config.ts).
|
|
1211
|
+
* --tenant Required (or via baerly.config.ts).
|
|
1212
|
+
* --collection Required. Target collection name.
|
|
1213
|
+
* --force Truncate the target if it exists (reseed above the old tail).
|
|
1214
|
+
* --json JSON envelope.
|
|
1215
|
+
*
|
|
1216
|
+
* stdin: NDJSON, one `{_id, ...}` row per line. Empty lines tolerated.
|
|
1217
|
+
* EOF terminates.
|
|
1218
|
+
*
|
|
1219
|
+
* Side effect: both reseed branches also delete any leftover
|
|
1220
|
+
* `gc/pending.json` for the collection before the first row commits. A
|
|
1221
|
+
* candidate marked before the reseed names a key from the old
|
|
1222
|
+
* incarnation of the log, which the commits below may re-create at the
|
|
1223
|
+
* same seq; the stale ledger's pending depth no longer describes this
|
|
1224
|
+
* collection either. A missing ledger is a no-op.
|
|
1225
|
+
*
|
|
1226
|
+
* Cost shape: before any in-band maintenance triggered by a long import,
|
|
1227
|
+
* the base billable shapes are fresh non-empty `N + 2` (initial seed,
|
|
1228
|
+
* one committing `log/<seq>` create per row, final `tail_hint` stamp),
|
|
1229
|
+
* force non-empty `N + 2 + P`, fresh empty `1`, and force empty
|
|
1230
|
+
* `1 + P`, where `P` is the number of provider LIST pages consumed by
|
|
1231
|
+
* the one `Storage.list()` iterator in `tailFromListedLogKeys`.
|
|
1232
|
+
* Maintenance adds the same measured profile overhead as the main
|
|
1233
|
+
* cost model.
|
|
1234
|
+
*
|
|
1235
|
+
* Partial-restore semantics: validation is per row, in stream order. A
|
|
1236
|
+
* line that fails (bad JSON, missing/duplicate/invalid-domain `_id`, or a
|
|
1237
|
+
* storage error) stops the run; rows BEFORE it stay committed, that line
|
|
1238
|
+
* and later ones are not, and the error message says how many rows
|
|
1239
|
+
* committed. The final `tail_hint` stamp may not have landed. Restore
|
|
1240
|
+
* again into a fresh target, or with `--force`. A whole-stream pre-pass
|
|
1241
|
+
* is deliberately not done (stdin is a stream). `_id`s are never
|
|
1242
|
+
* rewritten.
|
|
1243
|
+
*
|
|
1244
|
+
* Exit codes:
|
|
1245
|
+
* 0 — every row committed.
|
|
1246
|
+
* 1 — InvalidConfig (bad bucket URI, missing args, duplicate `_id`, or
|
|
1247
|
+
* an `_id` outside the write domain: key-segment rule + NFC).
|
|
1248
|
+
* 2 — Network / storage error mid-stream, or a malformed NDJSON
|
|
1249
|
+
* line (raised as a plain `Error`, not `BaerlyError`).
|
|
1250
|
+
* 3 — AlreadyExists from a pre-existing target without --force, or a
|
|
1251
|
+
* WriteConflict from a lost `current.json` CAS against
|
|
1252
|
+
* maintenance/compaction during --force.
|
|
1253
|
+
*/
|
|
1254
|
+
const RESTORE_OWNER = "baerly-restore";
|
|
1255
|
+
const RESTORE_ARGS = {
|
|
1256
|
+
bucket: {
|
|
1257
|
+
type: "string",
|
|
1258
|
+
required: true,
|
|
1259
|
+
description: "Target bucket URI (s3://<bucket>[/<prefix>], file:///<abs>, memory://<bucket>)",
|
|
1260
|
+
valueHint: "bucket-uri"
|
|
1261
|
+
},
|
|
1262
|
+
app: APP_ARG,
|
|
1263
|
+
tenant: TENANT_ARG,
|
|
1264
|
+
collection: {
|
|
1265
|
+
type: "string",
|
|
1266
|
+
required: true,
|
|
1267
|
+
description: "Target collection name.",
|
|
1268
|
+
valueHint: "name"
|
|
1269
|
+
},
|
|
1270
|
+
force: {
|
|
1271
|
+
type: "boolean",
|
|
1272
|
+
description: "Reseed the target when it already exists."
|
|
1273
|
+
},
|
|
1274
|
+
json: JSON_ARG
|
|
1275
|
+
};
|
|
1276
|
+
const tailFromListedLogKeys = async (storage, collectionPrefix) => {
|
|
1277
|
+
const logPrefix = `${collectionPrefix}/log/`;
|
|
1278
|
+
let maxSeq = -1;
|
|
1279
|
+
for await (const entry of storage.list(logPrefix)) {
|
|
1280
|
+
const tail = entry.key.slice(logPrefix.length);
|
|
1281
|
+
const match = /^(\d+)\.json$/.exec(tail);
|
|
1282
|
+
if (match === null) continue;
|
|
1283
|
+
const seq = Number.parseInt(match[1], 10);
|
|
1284
|
+
if (Number.isFinite(seq) && seq > maxSeq) maxSeq = seq;
|
|
1285
|
+
}
|
|
1286
|
+
return maxSeq + 1;
|
|
1287
|
+
};
|
|
1288
|
+
const bundle = defineBaerlySubcommand({
|
|
1289
|
+
name: "admin.restore",
|
|
1290
|
+
meta: { description: "Bulk-import canonical NDJSON into a fresh collection. Validates per row: rows before a failing line stay committed, and the error reports the count (re-run into a fresh target or with --force). Duplicate or non-NFC/invalid _ids are refused, never rewritten." },
|
|
1291
|
+
args: RESTORE_ARGS,
|
|
1292
|
+
handler: async (args, ctx) => {
|
|
1293
|
+
const bucket = await parseBucketUri(args.bucket);
|
|
1294
|
+
const { app, tenant } = await ctx.resolveAppTenant({
|
|
1295
|
+
app: args.app,
|
|
1296
|
+
tenant: args.tenant
|
|
1297
|
+
});
|
|
1298
|
+
assertCollectionArg(args.collection, "baerly admin restore");
|
|
1299
|
+
assertWritableNames({
|
|
1300
|
+
app,
|
|
1301
|
+
tenant,
|
|
1302
|
+
collection: args.collection
|
|
1303
|
+
}, "baerly admin restore");
|
|
1304
|
+
const currentJsonKey = `${bucket.keyPrefix}app/${app}/tenant/${tenant}/manifests/${args.collection}/current.json`;
|
|
1305
|
+
const collectionPrefix = collectionPrefixOf(currentJsonKey);
|
|
1306
|
+
const pendingKey = gcPendingKey(collectionPrefix);
|
|
1307
|
+
let baseSeq = 0;
|
|
1308
|
+
const head = await readCurrentJson(bucket.storage, currentJsonKey);
|
|
1309
|
+
if (head !== null && args.force !== true) throw new BaerlyError("AlreadyExists", `baerly admin restore: collection ${JSON.stringify(args.collection)} (app ${JSON.stringify(app)}, tenant ${JSON.stringify(tenant)}) already exists; pass --force to truncate it`, { resolution: "Pass --force to truncate the existing collection, or restore into an empty target." });
|
|
1310
|
+
if (head !== null) {
|
|
1311
|
+
const truncatedNext = await tailFromListedLogKeys(bucket.storage, collectionPrefix);
|
|
1312
|
+
baseSeq = truncatedNext;
|
|
1313
|
+
const reseeded = {
|
|
1314
|
+
schema_version: 3,
|
|
1315
|
+
snapshot: null,
|
|
1316
|
+
tail_hint: truncatedNext,
|
|
1317
|
+
log_seq_start: truncatedNext,
|
|
1318
|
+
writer_fence: {
|
|
1319
|
+
epoch: head.json.writer_fence.epoch + 1,
|
|
1320
|
+
owner: RESTORE_OWNER,
|
|
1321
|
+
claimed_at: ""
|
|
1322
|
+
},
|
|
1323
|
+
snapshot_bytes: 0,
|
|
1324
|
+
snapshot_rows: 0,
|
|
1325
|
+
generation: mintGeneration()
|
|
1326
|
+
};
|
|
1327
|
+
try {
|
|
1328
|
+
await bucket.storage.put(currentJsonKey, encodeJsonBytes(reseeded), {
|
|
1329
|
+
ifMatch: head.etag,
|
|
1330
|
+
contentType: "application/json"
|
|
1331
|
+
});
|
|
1332
|
+
} catch (error) {
|
|
1333
|
+
if (error instanceof BaerlyError) throw error;
|
|
1334
|
+
throw new BaerlyError("NetworkError", `baerly admin restore: failed to reseed current.json: ${error.message}`);
|
|
1335
|
+
}
|
|
1336
|
+
await bucket.storage.delete(pendingKey);
|
|
1337
|
+
} else {
|
|
1338
|
+
const seed = {
|
|
1339
|
+
schema_version: 3,
|
|
1340
|
+
snapshot: null,
|
|
1341
|
+
tail_hint: 0,
|
|
1342
|
+
log_seq_start: 0,
|
|
1343
|
+
writer_fence: {
|
|
1344
|
+
epoch: 0,
|
|
1345
|
+
owner: RESTORE_OWNER,
|
|
1346
|
+
claimed_at: ""
|
|
1347
|
+
},
|
|
1348
|
+
snapshot_bytes: 0,
|
|
1349
|
+
snapshot_rows: 0,
|
|
1350
|
+
generation: mintGeneration()
|
|
1351
|
+
};
|
|
1352
|
+
await createCurrentJson(bucket.storage, currentJsonKey, seed);
|
|
1353
|
+
await bucket.storage.delete(pendingKey);
|
|
1354
|
+
}
|
|
1355
|
+
const writer = new Writer({
|
|
1356
|
+
storage: bucket.storage,
|
|
1357
|
+
currentJsonKey
|
|
1358
|
+
});
|
|
1359
|
+
const input = ctx.streams?.stdin ?? process.stdin;
|
|
1360
|
+
const rl = createInterface({
|
|
1361
|
+
input,
|
|
1362
|
+
crlfDelay: 1e400
|
|
1363
|
+
});
|
|
1364
|
+
let count = 0;
|
|
1365
|
+
let lineNo = 0;
|
|
1366
|
+
const seenIds = /* @__PURE__ */ new Map();
|
|
1367
|
+
const committedNote = (when = "before this line") => `; ${count} row${count === 1 ? "" : "s"} committed ${when} (they remain in the target; restore again into a fresh target, or with --force)`;
|
|
1368
|
+
const idFix = "Restore never rewrites an _id. Fix the dump line, or rewrite it to a fresh NFC _id that is not canonically equivalent to the original.";
|
|
1369
|
+
const lines = (async function* () {
|
|
1370
|
+
try {
|
|
1371
|
+
yield* rl;
|
|
1372
|
+
} catch (error) {
|
|
1373
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
1374
|
+
failure.message += committedNote("before input failed");
|
|
1375
|
+
throw failure;
|
|
1376
|
+
}
|
|
1377
|
+
})();
|
|
1378
|
+
try {
|
|
1379
|
+
for await (const line of lines) {
|
|
1380
|
+
lineNo++;
|
|
1381
|
+
if (line.length === 0) continue;
|
|
1382
|
+
let parsed;
|
|
1383
|
+
try {
|
|
1384
|
+
parsed = JSON.parse(line);
|
|
1385
|
+
} catch (error) {
|
|
1386
|
+
throw new Error(`baerly admin restore: line ${lineNo} is not valid JSON: ${error.message}${committedNote()}`, { cause: error });
|
|
1387
|
+
}
|
|
1388
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(`baerly admin restore: line ${lineNo} is not a JSON object${committedNote()}`);
|
|
1389
|
+
const row = parsed;
|
|
1390
|
+
const id = row["_id"];
|
|
1391
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
1392
|
+
const fix = "Add a non-empty string _id to the dump line.";
|
|
1393
|
+
throw new BaerlyError("InvalidConfig", `baerly admin restore: line ${lineNo} missing non-empty string _id${committedNote()}. ${fix}`, { resolution: fix });
|
|
1394
|
+
}
|
|
1395
|
+
const firstLine = seenIds.get(id);
|
|
1396
|
+
if (firstLine !== void 0) throw new BaerlyError("InvalidConfig", `baerly admin restore: line ${lineNo} repeats _id ${JSON.stringify(id)} first seen on line ${firstLine}${committedNote()}. ${idFix}`, { resolution: idFix });
|
|
1397
|
+
try {
|
|
1398
|
+
assertDocId(id);
|
|
1399
|
+
} catch (error) {
|
|
1400
|
+
throw new BaerlyError("InvalidConfig", `baerly admin restore: line ${lineNo} has an invalid _id ${JSON.stringify(id)}: ${error.message}${committedNote()}. ${idFix}`, {
|
|
1401
|
+
cause: error,
|
|
1402
|
+
resolution: idFix
|
|
1403
|
+
});
|
|
1404
|
+
}
|
|
1405
|
+
seenIds.set(id, lineNo);
|
|
1406
|
+
try {
|
|
1407
|
+
assertDocumentValues(row, "restore");
|
|
1408
|
+
assertDocumentSize(row, "restore");
|
|
1409
|
+
} catch (error) {
|
|
1410
|
+
if (error instanceof BaerlyError) throw new BaerlyError("InvalidConfig", `baerly admin restore: line ${lineNo} (_id ${JSON.stringify(id)}) has an invalid document: ${error.message}${committedNote()}${error.resolution === void 0 ? "" : `. ${error.resolution}`}`, {
|
|
1411
|
+
cause: error,
|
|
1412
|
+
resolution: error.resolution
|
|
1413
|
+
});
|
|
1414
|
+
throw error;
|
|
1415
|
+
}
|
|
1416
|
+
const body = row;
|
|
1417
|
+
try {
|
|
1418
|
+
await writer.commit({
|
|
1419
|
+
op: "I",
|
|
1420
|
+
collection: args.collection,
|
|
1421
|
+
docId: id,
|
|
1422
|
+
body
|
|
1423
|
+
});
|
|
1424
|
+
} catch (error) {
|
|
1425
|
+
if (error instanceof Error) {
|
|
1426
|
+
if (error.code === "AmbiguousCommit") error.message += `; line ${lineNo} may or may not be committed (between ${count} and ${count + 1} rows are in the target); restore again with --force to get an exact copy of the dump`;
|
|
1427
|
+
else error.message += `; line ${lineNo} was not committed${committedNote()}`;
|
|
1428
|
+
}
|
|
1429
|
+
throw error;
|
|
1430
|
+
}
|
|
1431
|
+
count++;
|
|
1432
|
+
}
|
|
1433
|
+
} finally {
|
|
1434
|
+
rl.close();
|
|
1435
|
+
}
|
|
1436
|
+
if (count > 0) try {
|
|
1437
|
+
const afterLoad = await readCurrentJson(bucket.storage, currentJsonKey);
|
|
1438
|
+
if (afterLoad !== null) {
|
|
1439
|
+
const clampedTail = Math.max(baseSeq + count, afterLoad.json.tail_hint, afterLoad.json.log_seq_start);
|
|
1440
|
+
const stamped = {
|
|
1441
|
+
...afterLoad.json,
|
|
1442
|
+
tail_hint: clampedTail
|
|
1443
|
+
};
|
|
1444
|
+
assertCurrentJsonTransition(afterLoad.json, stamped, currentJsonKey);
|
|
1445
|
+
await bucket.storage.put(currentJsonKey, encodeJsonBytes(stamped), {
|
|
1446
|
+
ifMatch: afterLoad.etag,
|
|
1447
|
+
contentType: "application/json"
|
|
1448
|
+
});
|
|
1449
|
+
}
|
|
1450
|
+
} catch (error) {
|
|
1451
|
+
throw new BaerlyError(error instanceof BaerlyError ? error.code : "NetworkError", `baerly admin restore: all ${count} row${count === 1 ? "" : "s"} were committed, but stamping tail_hint on current.json failed: ${error.message}. Rerun restore with --force to truncate and restore the dump again.`, { cause: error });
|
|
1452
|
+
}
|
|
1453
|
+
emitSuccess({
|
|
1454
|
+
command: "admin.restore",
|
|
1455
|
+
status: "ok",
|
|
1456
|
+
collection: args.collection,
|
|
1457
|
+
restored: count
|
|
1458
|
+
});
|
|
1459
|
+
return 0;
|
|
1460
|
+
}
|
|
1461
|
+
});
|
|
1462
|
+
/** citty `defineCommand` block for `baerly admin restore`. */
|
|
1463
|
+
const restoreCmd = bundle.cmd;
|
|
1464
|
+
bundle.run;
|
|
1465
|
+
export { restoreCmd };
|
|
1466
|
+
|
|
1467
|
+
//# sourceMappingURL=restore-U6Im8V1V.js.map
|