@gusto/baerly-storage 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +29 -0
- package/README.md +176 -70
- package/dist/API.md +213 -139
- package/dist/CHANGELOG.md +434 -0
- package/dist/RECIPES.md +125 -0
- package/dist/THIRD-PARTY-LICENSES.txt +238 -0
- package/dist/{app-config-C0Bbw_Z6.d.ts → app-config-DyiFK3_k.d.ts} +55 -11
- package/dist/app-config.d.ts +1 -1
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +301 -301
- package/dist/auth.js.map +1 -1
- package/dist/aws4fetch.esm-p7eEzLCs.js +1702 -0
- package/dist/aws4fetch.esm-p7eEzLCs.js.map +1 -0
- package/dist/baerly.js +35 -15
- package/dist/baerly.js.map +1 -1
- package/dist/baerly.spec.json +342 -0
- package/dist/bytes-BXdamO3L.js.map +1 -1
- package/dist/{client-DYZfoZb9.d.ts → client-Fe8CwB73.d.ts} +10 -10
- package/dist/client-react.d.ts +3 -3
- package/dist/client-react.js +4 -4
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +4 -4
- package/dist/client.js +3 -3
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-CINJPKCW.js → cloudflare-DzKBrUlg.js} +178 -2
- package/dist/cloudflare-DzKBrUlg.js.map +1 -0
- package/dist/cloudflare.d.ts +49 -6
- package/dist/cloudflare.js +59 -26
- package/dist/cloudflare.js.map +1 -1
- package/dist/config-1tv0-Yzm.js +2693 -0
- package/dist/config-1tv0-Yzm.js.map +1 -0
- package/dist/constants-CfeMWQWk.js +222 -0
- package/dist/constants-CfeMWQWk.js.map +1 -0
- package/dist/context-DRZ3tZ5l.js.map +1 -1
- package/dist/{cost-CQt1YRkA.js → cost-DSMIVqzL.js} +43 -42
- package/dist/cost-DSMIVqzL.js.map +1 -0
- package/dist/current-json-C6nhPOm1.js +190 -0
- package/dist/current-json-C6nhPOm1.js.map +1 -0
- package/dist/{db-BM3y2JQw.d.ts → db-DsPkOVEH.d.ts} +26 -108
- package/dist/{deploy-CrAbfgCq.js → deploy-DonTr-CV.js} +12 -5
- package/dist/deploy-DonTr-CV.js.map +1 -0
- package/dist/dev-vite.d.ts +11 -7
- package/dist/dev-vite.js +7 -6
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/{doctor-Db-hzPmE.js → doctor-DSIyCDg_.js} +14 -136
- package/dist/doctor-DSIyCDg_.js.map +1 -0
- package/dist/{dump-BYA55NLP.js → dump-Bh5k26VQ.js} +10 -31
- package/dist/dump-Bh5k26VQ.js.map +1 -0
- package/dist/{errors-DOdL95kK.js → errors-Dfa3p78c.js} +25 -5
- package/dist/errors-Dfa3p78c.js.map +1 -0
- package/dist/{export-Bhbz90-z.js → export-DtF5tDIM.js} +11 -32
- package/dist/{export-Bhbz90-z.js.map → export-DtF5tDIM.js.map} +1 -1
- package/dist/{fsck-CPPZWt9p.js → fsck-D_djMO5D.js} +28 -39
- package/dist/fsck-D_djMO5D.js.map +1 -0
- package/dist/{http-Cw8p6y5b.js → http-DxM9MflO.js} +151 -79
- package/dist/http-DxM9MflO.js.map +1 -0
- package/dist/http.d.ts +42 -26
- package/dist/http.js +3 -2
- package/dist/{index-DwDTWPSU.d.ts → index-0rZcMBwy.d.ts} +10 -7
- package/dist/{index-C3JAH1Yo.d.ts → index-B6oexfOB.d.ts} +9 -9
- package/dist/{index-cUY-36AN.d.ts → index-BNRh23K9.d.ts} +21 -5
- package/dist/{index-DSvwEUeF.d.ts → index-BYk9oCNf.d.ts} +240 -202
- package/dist/index.d.ts +6 -6
- package/dist/index.js +5 -5
- package/dist/{indexes-DAu2AkTW.d.ts → indexes-Dn9hbGQG.d.ts} +2 -2
- package/dist/{inspect-Bdbzvi-_.js → inspect-6WZ-pC7-.js} +21 -36
- package/dist/inspect-6WZ-pC7-.js.map +1 -0
- package/dist/key-Di6yyxFu.js +35 -0
- package/dist/key-Di6yyxFu.js.map +1 -0
- package/dist/{snapshot-SS8USDUF.js → log-tail-D98U1JBo.js} +179 -84
- package/dist/log-tail-D98U1JBo.js.map +1 -0
- package/dist/{maintenance-ClzBC1Vv.js → maintenance-B4Dki6Ig.js} +438 -176
- package/dist/maintenance-B4Dki6Ig.js.map +1 -0
- package/dist/{maintenance-BRrThz8V.d.ts → maintenance-B_e0T99L.d.ts} +114 -26
- package/dist/{maintenance-BvOv6BMM.js → maintenance-C7coQp4c.js} +217 -94
- package/dist/maintenance-C7coQp4c.js.map +1 -0
- package/dist/maintenance.d.ts +3 -2
- package/dist/maintenance.js +3 -2
- package/dist/node.d.ts +176 -68
- package/dist/node.js +3 -2
- package/dist/{normalize-XWY25_sG.js → normalize-rjnZLwS4.js} +2 -2
- package/dist/normalize-rjnZLwS4.js.map +1 -0
- package/dist/{observability-CpOiEN0k.js → observability-wzs3-BL_.js} +279 -109
- package/dist/observability-wzs3-BL_.js.map +1 -0
- package/dist/observability.d.ts +3 -3
- package/dist/observability.js +1 -1
- package/dist/{plan-DOTl21vq.js → plan-a7PJHBCy.js} +7 -4
- package/dist/plan-a7PJHBCy.js.map +1 -0
- package/dist/{query-BmiQy328.js → query-rAtXNQQB.js} +1063 -1202
- package/dist/query-rAtXNQQB.js.map +1 -0
- package/dist/{rebuild-index-DC_nR6wP.js → rebuild-index-BVzQSZlm.js} +10 -32
- package/dist/rebuild-index-BVzQSZlm.js.map +1 -0
- package/dist/{request-BEF_OvMh.js → request-u0M51mhJ.js} +3 -3
- package/dist/{request-BEF_OvMh.js.map → request-u0M51mhJ.js.map} +1 -1
- package/dist/restore-R-4rbciV.js +778 -0
- package/dist/restore-R-4rbciV.js.map +1 -0
- package/dist/runtime-spec-qL3_lTTU.js +391 -0
- package/dist/runtime-spec-qL3_lTTU.js.map +1 -0
- package/dist/s3-http-B-z8th6O.d.ts +60 -0
- package/dist/s3.d.ts +47 -0
- package/dist/s3.js +45 -0
- package/dist/s3.js.map +1 -0
- package/dist/{shared-secret-BJdSwpgw.js → shared-secret-DgEicd3x.js} +13 -5
- package/dist/shared-secret-DgEicd3x.js.map +1 -0
- package/dist/src-BG20LZs7.js +2271 -0
- package/dist/src-BG20LZs7.js.map +1 -0
- package/dist/{src-zhEAmmF7.js → src-CYHFDrKz.js} +82 -99
- package/dist/src-CYHFDrKz.js.map +1 -0
- package/dist/{src-QQ7jDeL2.js → src-DYZ4H1aF.js} +44 -23
- package/dist/src-DYZ4H1aF.js.map +1 -0
- package/dist/subcommand-DoDA-K7d.js +197 -0
- package/dist/subcommand-DoDA-K7d.js.map +1 -0
- package/dist/time-DUqvFMHq.js +119 -0
- package/dist/time-DUqvFMHq.js.map +1 -0
- package/dist/{usage-CLjirdD-.js → usage-DBV1eQHC.js} +7 -22
- package/dist/usage-DBV1eQHC.js.map +1 -0
- package/dist/{validate-DCq_0DIn.js → validate-DtljsCh1.js} +51 -51
- package/dist/{validate-DCq_0DIn.js.map → validate-DtljsCh1.js.map} +1 -1
- package/dist/wire-izfGZDvr.js.map +1 -1
- package/package.json +77 -42
- package/dist/cloudflare-CINJPKCW.js.map +0 -1
- package/dist/config-DVfDGyw1.js +0 -278
- package/dist/config-DVfDGyw1.js.map +0 -1
- package/dist/cost-CQt1YRkA.js.map +0 -1
- package/dist/current-json-C-EyJqs0.js +0 -354
- package/dist/current-json-C-EyJqs0.js.map +0 -1
- package/dist/deploy-CrAbfgCq.js.map +0 -1
- package/dist/doctor-Db-hzPmE.js.map +0 -1
- package/dist/dump-BYA55NLP.js.map +0 -1
- package/dist/errors-DOdL95kK.js.map +0 -1
- package/dist/fsck-CPPZWt9p.js.map +0 -1
- package/dist/http-Cw8p6y5b.js.map +0 -1
- package/dist/inspect-Bdbzvi-_.js.map +0 -1
- package/dist/maintenance-BvOv6BMM.js.map +0 -1
- package/dist/maintenance-ClzBC1Vv.js.map +0 -1
- package/dist/normalize-XWY25_sG.js.map +0 -1
- package/dist/observability-CpOiEN0k.js.map +0 -1
- package/dist/plan-DOTl21vq.js.map +0 -1
- package/dist/query-BmiQy328.js.map +0 -1
- package/dist/rebuild-index-DC_nR6wP.js.map +0 -1
- package/dist/restore-C6IcmLeA.js +0 -771
- package/dist/restore-C6IcmLeA.js.map +0 -1
- package/dist/shared-secret-BJdSwpgw.js.map +0 -1
- package/dist/snapshot-SS8USDUF.js.map +0 -1
- package/dist/src-DU1sCMlr.js +0 -6236
- package/dist/src-DU1sCMlr.js.map +0 -1
- package/dist/src-QQ7jDeL2.js.map +0 -1
- package/dist/src-zhEAmmF7.js.map +0 -1
- package/dist/subcommand-DZcHcsYp.js +0 -5122
- package/dist/subcommand-DZcHcsYp.js.map +0 -1
- package/dist/usage-CLjirdD-.js.map +0 -1
- /package/dist/{chunk-EAsCxrDo.js → rolldown-runtime-EAsCxrDo.js} +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { d as IndexDefinition, f as DocumentData } from "./app-config-
|
|
1
|
+
import { d as IndexDefinition, f as DocumentData } from "./app-config-DyiFK3_k.js";
|
|
2
2
|
//#region packages/server/src/indexes.d.ts
|
|
3
3
|
/**
|
|
4
4
|
* Compute the full index-key set a single doc produces under every
|
|
@@ -19,4 +19,4 @@ import { d as IndexDefinition, f as DocumentData } from "./app-config-C0Bbw_Z6.j
|
|
|
19
19
|
declare const allIndexKeysFor: (logPrefix: string, defs: ReadonlyArray<IndexDefinition>, body: DocumentData | undefined, docId: string) => string[];
|
|
20
20
|
//#endregion
|
|
21
21
|
export { allIndexKeysFor as t };
|
|
22
|
-
//# sourceMappingURL=indexes-
|
|
22
|
+
//# sourceMappingURL=indexes-Dn9hbGQG.d.ts.map
|
|
@@ -1,44 +1,21 @@
|
|
|
1
|
-
import { n as loadCollectionIndexes } from "./config-
|
|
2
|
-
import {
|
|
3
|
-
import { n as parseBucketUri, t as defineBaerlySubcommand } from "./subcommand-DZcHcsYp.js";
|
|
1
|
+
import { n as loadCollectionIndexes, r as parseBucketUri } from "./config-1tv0-Yzm.js";
|
|
2
|
+
import { h as readCurrentJson, n as probeTailFrom } from "./log-tail-D98U1JBo.js";
|
|
4
3
|
import { i as isJsonMode, o as BaerlyError, r as emitSuccess } from "./baerly.js";
|
|
5
|
-
import { n as loadMaterialisedView } from "./plan-
|
|
4
|
+
import { n as loadMaterialisedView } from "./plan-a7PJHBCy.js";
|
|
5
|
+
import { a as TENANT_ARG, i as JSON_ARG, n as BUCKET_ARG, o as assertCollectionArg, r as COLLECTION_ARG, s as defineBaerlySubcommand, t as APP_ARG } from "./subcommand-DoDA-K7d.js";
|
|
6
6
|
//#region src/inspect.ts
|
|
7
7
|
const INSPECT_ARGS = {
|
|
8
|
-
bucket:
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
valueHint: "bucket-uri"
|
|
13
|
-
},
|
|
14
|
-
app: {
|
|
15
|
-
type: "string",
|
|
16
|
-
required: false,
|
|
17
|
-
description: "Application name segment (defaults to baerly.config.ts).",
|
|
18
|
-
valueHint: "app"
|
|
19
|
-
},
|
|
20
|
-
tenant: {
|
|
21
|
-
type: "string",
|
|
22
|
-
required: false,
|
|
23
|
-
description: "Tenant name segment (defaults to baerly.config.ts).",
|
|
24
|
-
valueHint: "tenant"
|
|
25
|
-
},
|
|
26
|
-
collection: {
|
|
27
|
-
type: "string",
|
|
28
|
-
required: true,
|
|
29
|
-
description: "Collection name.",
|
|
30
|
-
valueHint: "name"
|
|
31
|
-
},
|
|
8
|
+
bucket: BUCKET_ARG,
|
|
9
|
+
app: APP_ARG,
|
|
10
|
+
tenant: TENANT_ARG,
|
|
11
|
+
collection: COLLECTION_ARG,
|
|
32
12
|
config: {
|
|
33
13
|
type: "string",
|
|
34
14
|
required: false,
|
|
35
15
|
description: "Path to baerly.config.{js,mjs,json}; pulls declared indexes for key-count probes.",
|
|
36
16
|
valueHint: "path"
|
|
37
17
|
},
|
|
38
|
-
json:
|
|
39
|
-
type: "boolean",
|
|
40
|
-
description: "Emit a structured JSON envelope to stdout (success) or stderr (error)"
|
|
41
|
-
}
|
|
18
|
+
json: JSON_ARG
|
|
42
19
|
};
|
|
43
20
|
const countListEntries = async (storage, prefix) => {
|
|
44
21
|
let count = 0;
|
|
@@ -57,10 +34,13 @@ const renderText = (r, collection) => {
|
|
|
57
34
|
lines.push(`baerly inspect ${collection}`);
|
|
58
35
|
lines.push(` current.json: ${r.currentJsonKey}`);
|
|
59
36
|
lines.push(` schema_version: ${r.schema_version}`);
|
|
60
|
-
lines.push(`
|
|
37
|
+
lines.push(` tail_hint (stored): ${r.tail_hint}`);
|
|
38
|
+
lines.push(` discovered_tail: ${r.discovered_tail}`);
|
|
61
39
|
lines.push(` log_seq_start: ${r.log_seq_start}`);
|
|
62
40
|
lines.push(` live_log_tail: ${r.live_log_tail}`);
|
|
63
41
|
lines.push(` snapshot: ${r.snapshot ?? "(none)"}`);
|
|
42
|
+
lines.push(` snapshot_bytes: ${r.snapshot_bytes}`);
|
|
43
|
+
lines.push(` snapshot_rows: ${r.snapshot_rows}`);
|
|
64
44
|
lines.push(` writer_fence: epoch=${r.writer_fence.epoch} owner=${JSON.stringify(r.writer_fence.owner)} claimed_at=${JSON.stringify(r.writer_fence.claimed_at)}`);
|
|
65
45
|
lines.push(` materialised_rows: ${r.materialised_rows}`);
|
|
66
46
|
if (r.indexes.length > 0) {
|
|
@@ -81,12 +61,14 @@ const bundle = defineBaerlySubcommand({
|
|
|
81
61
|
app: args.app,
|
|
82
62
|
tenant: args.tenant
|
|
83
63
|
});
|
|
64
|
+
assertCollectionArg(args.collection, "baerly inspect");
|
|
84
65
|
const collectionPrefix = `${bucket.keyPrefix}app/${app}/tenant/${tenant}/manifests/${args.collection}`;
|
|
85
66
|
const currentJsonKey = `${collectionPrefix}/current.json`;
|
|
86
67
|
const read = await readCurrentJson(bucket.storage, currentJsonKey);
|
|
87
68
|
if (read === null) throw new BaerlyError("InvalidConfig", `baerly inspect: current.json not found at ${currentJsonKey}`);
|
|
88
69
|
const cur = read.json;
|
|
89
70
|
const log_seq_start = cur.log_seq_start ?? 0;
|
|
71
|
+
const discoveredTail = (await probeTailFrom(bucket.storage, collectionPrefix, Math.max(log_seq_start, cur.tail_hint))).tail;
|
|
90
72
|
const errors = [];
|
|
91
73
|
let materialisedRows = 0;
|
|
92
74
|
try {
|
|
@@ -121,10 +103,13 @@ const bundle = defineBaerlySubcommand({
|
|
|
121
103
|
const result = {
|
|
122
104
|
currentJsonKey,
|
|
123
105
|
schema_version: cur.schema_version,
|
|
124
|
-
|
|
106
|
+
tail_hint: cur.tail_hint,
|
|
107
|
+
discovered_tail: discoveredTail,
|
|
125
108
|
log_seq_start,
|
|
126
|
-
live_log_tail:
|
|
109
|
+
live_log_tail: discoveredTail - log_seq_start,
|
|
127
110
|
snapshot: cur.snapshot,
|
|
111
|
+
snapshot_bytes: cur.snapshot_bytes,
|
|
112
|
+
snapshot_rows: cur.snapshot_rows,
|
|
128
113
|
writer_fence: cur.writer_fence,
|
|
129
114
|
materialised_rows: materialisedRows,
|
|
130
115
|
indexes,
|
|
@@ -145,4 +130,4 @@ bundle.run;
|
|
|
145
130
|
//#endregion
|
|
146
131
|
export { inspect };
|
|
147
132
|
|
|
148
|
-
//# sourceMappingURL=inspect-
|
|
133
|
+
//# sourceMappingURL=inspect-6WZ-pC7-.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inspect-6WZ-pC7-.js","names":[],"sources":["../packages/cli/src/inspect.ts"],"sourcesContent":["/**\n * `baerly inspect` — read-only summary for one collection.\n *\n * Args:\n * --bucket Required. Bucket URI.\n * --app Required (or via baerly.config.ts).\n * --tenant Required (or via baerly.config.ts).\n * --collection Required. Collection name.\n * --config Optional path to baerly.config.{js,mjs,json} for index info.\n * --json JSON envelope.\n *\n * Reports, per-collection:\n * - currentJsonKey path.\n * - schema_version, tail_hint, log_seq_start (default 0).\n * - live_log_tail (= tail_hint - log_seq_start).\n * - snapshot key (or null).\n * - snapshot_bytes / snapshot_rows (the fold-ceiling inputs C / E\n * are compared against; see graduation.md).\n * - writer_fence (epoch, owner, claimed_at).\n * - materialised row count (via export/loadMaterialisedView).\n * - per-declared-index key count (when --config supplied).\n * - well-formed status: \"ok\" or \"error\" with an `errors` array.\n *\n * For projected monthly Class A / $ trajectories, use\n * `baerly cost --bucket=<uri> --collection=<name>` — the projection\n * used to live as a footer here but moved to its own verb so inspect\n * stays a glance command (no GET-storm over the trailing log sample).\n *\n * Cost shape:\n * 1 GET current.json\n * + 1 GET snapshot (if any)\n * + N GETs log tail (= tail_hint - log_seq_start)\n * + K LISTs (one per declared index, when --config supplied)\n * + 1 LIST snapshot/ prefix (orphan detection)\n *\n * Never mutates anything.\n *\n * Exit codes:\n * 0 — inspection ran (status may be \"ok\" or \"error\"; both exit 0).\n * 1 — InvalidConfig (bad bucket URI, missing args, unknown flag).\n * 2 — Storage / Network error.\n * 3 — Protocol invariant (Conflict / Internal / InvalidResponse).\n */\n\nimport { type ArgsDef } from \"citty\";\nimport { BaerlyError, readCurrentJson, type CurrentJson, type Storage } from \"@baerly/protocol\";\nimport { probeTailFrom } from \"@baerly/server\";\nimport { loadMaterialisedView } from \"./export/index.ts\";\nimport { loadCollectionIndexes } from \"./config.ts\";\nimport { parseBucketUri } from \"./bucket-uri.ts\";\nimport { emitSuccess, isJsonMode } from \"./output.ts\";\nimport {\n APP_ARG,\n assertCollectionArg,\n BUCKET_ARG,\n COLLECTION_ARG,\n defineBaerlySubcommand,\n JSON_ARG,\n TENANT_ARG,\n} from \"./subcommand.ts\";\n\nconst INSPECT_ARGS = {\n bucket: BUCKET_ARG,\n app: APP_ARG,\n tenant: TENANT_ARG,\n collection: COLLECTION_ARG,\n config: {\n type: \"string\",\n required: false,\n description:\n \"Path to baerly.config.{js,mjs,json}; pulls declared indexes for key-count probes.\",\n valueHint: \"path\",\n },\n json: JSON_ARG,\n} as const satisfies ArgsDef;\n\nconst countListEntries = async (\n storage: Storage,\n prefix: string,\n): Promise<{ count: number; keys: string[] }> => {\n let count = 0;\n const keys: string[] = [];\n for await (const entry of storage.list(prefix)) {\n count++;\n keys.push(entry.key);\n }\n return { count, keys };\n};\n\ninterface InspectResult {\n currentJsonKey: string;\n schema_version: number;\n /** The stored, compactor-advanced lower-bound hint (`current.json.tail_hint`). */\n tail_hint: number;\n /** The TRUE committed tail, discovered by forward-probe past `tail_hint`. */\n discovered_tail: number;\n log_seq_start: number;\n live_log_tail: number;\n snapshot: string | null;\n snapshot_bytes: number;\n snapshot_rows: number;\n writer_fence: CurrentJson[\"writer_fence\"];\n materialised_rows: number;\n indexes: { name: string; count: number }[];\n status: \"ok\" | \"error\";\n errors: string[];\n}\n\nconst renderText = (r: InspectResult, collection: string): string => {\n const lines: string[] = [];\n lines.push(`baerly inspect ${collection}`);\n lines.push(` current.json: ${r.currentJsonKey}`);\n lines.push(` schema_version: ${r.schema_version}`);\n lines.push(` tail_hint (stored): ${r.tail_hint}`);\n lines.push(` discovered_tail: ${r.discovered_tail}`);\n lines.push(` log_seq_start: ${r.log_seq_start}`);\n lines.push(` live_log_tail: ${r.live_log_tail}`);\n lines.push(` snapshot: ${r.snapshot ?? \"(none)\"}`);\n lines.push(` snapshot_bytes: ${r.snapshot_bytes}`);\n lines.push(` snapshot_rows: ${r.snapshot_rows}`);\n lines.push(\n ` writer_fence: epoch=${r.writer_fence.epoch} owner=${JSON.stringify(r.writer_fence.owner)} claimed_at=${JSON.stringify(r.writer_fence.claimed_at)}`,\n );\n lines.push(` materialised_rows: ${r.materialised_rows}`);\n if (r.indexes.length > 0) {\n const formatted = r.indexes.map((i) => `${i.name} (count=${i.count})`).join(\", \");\n lines.push(` indexes: ${formatted}`);\n } else {\n lines.push(` indexes: (none declared)`);\n }\n lines.push(` status: ${r.status}`);\n if (r.errors.length > 0) {\n for (const e of r.errors) {\n lines.push(` error: ${e}`);\n }\n }\n return lines.join(\"\\n\") + \"\\n\";\n};\n\nconst bundle = defineBaerlySubcommand({\n name: \"inspect\",\n meta: {\n description: \"Read-only summary of one collection's snapshot + log state.\",\n },\n args: INSPECT_ARGS,\n handler: async (args, ctx) => {\n const bucket = await parseBucketUri(args.bucket);\n const { app, tenant } = await ctx.resolveAppTenant({ app: args.app, tenant: args.tenant });\n assertCollectionArg(args.collection, \"baerly inspect\");\n const collectionPrefix = `${bucket.keyPrefix}app/${app}/tenant/${tenant}/manifests/${args.collection}`;\n const currentJsonKey = `${collectionPrefix}/current.json`;\n\n const read = await readCurrentJson(bucket.storage, currentJsonKey);\n if (read === null) {\n throw new BaerlyError(\n \"InvalidConfig\",\n `baerly inspect: current.json not found at ${currentJsonKey}`,\n );\n }\n const cur = read.json;\n const log_seq_start = cur.log_seq_start ?? 0;\n // Discover the TRUE tail by forward-probe — under single-write commit\n // the stored `tail_hint` is only a lower bound (compactor-advanced),\n // so an accurate operator summary must probe past it.\n const tailProbe = await probeTailFrom(\n bucket.storage,\n collectionPrefix,\n Math.max(log_seq_start, cur.tail_hint),\n );\n const discoveredTail = tailProbe.tail;\n\n const errors: string[] = [];\n let materialisedRows = 0;\n try {\n const view = await loadMaterialisedView({\n storage: bucket.storage,\n currentJsonKey,\n collection: args.collection,\n });\n materialisedRows = view?.size ?? 0;\n } catch (error) {\n if (error instanceof BaerlyError) {\n errors.push(`${error.code}: ${error.message}`);\n } else {\n errors.push((error as Error).message);\n }\n }\n\n // Orphan-snapshot detection: list every file under <collectionPrefix>/snapshot/\n // and flag anything that isn't the currently-pointed-at snapshot.\n try {\n const snapshotPrefix = `${collectionPrefix}/snapshot/`;\n const { keys } = await countListEntries(bucket.storage, snapshotPrefix);\n for (const k of keys) {\n if (cur.snapshot !== k) {\n errors.push(`orphan snapshot: ${k}`);\n }\n }\n } catch (error) {\n if (error instanceof BaerlyError) {\n errors.push(`${error.code}: ${error.message}`);\n }\n }\n\n const indexes: { name: string; count: number }[] = [];\n if (typeof args.config === \"string\" && args.config.length > 0) {\n const defs = await loadCollectionIndexes(args.config, args.collection, \"baerly inspect\");\n for (const def of defs) {\n const prefix = `${collectionPrefix}/index/${def.name}/`;\n const { count } = await countListEntries(bucket.storage, prefix);\n indexes.push({ name: def.name, count });\n }\n }\n\n const result: InspectResult = {\n currentJsonKey,\n schema_version: cur.schema_version,\n tail_hint: cur.tail_hint,\n discovered_tail: discoveredTail,\n log_seq_start,\n live_log_tail: discoveredTail - log_seq_start,\n snapshot: cur.snapshot,\n snapshot_bytes: cur.snapshot_bytes,\n snapshot_rows: cur.snapshot_rows,\n writer_fence: cur.writer_fence,\n materialised_rows: materialisedRows,\n indexes,\n status: errors.length === 0 ? \"ok\" : \"error\",\n errors,\n };\n\n if (isJsonMode()) {\n emitSuccess({ command: \"inspect\", ...result });\n } else {\n process.stdout.write(renderText(result, args.collection));\n }\n return 0;\n },\n});\n\n/** citty `defineCommand` block for `baerly inspect`. */\nexport const inspect = bundle.cmd;\n\n/**\n * Programmatic entry used by tests. Bypasses citty's `run` wrapper\n * (which would call `process.exit` and kill vitest) and returns the\n * integer exit code directly.\n */\nexport const runInspect = bundle.run;\n"],"mappings":";;;;;;AA6DA,MAAM,eAAe;CACnB,QAAQ;CACR,KAAK;CACL,QAAQ;CACR,YAAY;CACZ,QAAQ;EACN,MAAM;EACN,UAAU;EACV,aACE;EACF,WAAW;CACb;CACA,MAAM;AACR;AAEA,MAAM,mBAAmB,OACvB,SACA,WAC+C;CAC/C,IAAI,QAAQ;CACZ,MAAM,OAAiB,CAAC;CACxB,WAAW,MAAM,SAAS,QAAQ,KAAK,MAAM,GAAG;EAC9C;EACA,KAAK,KAAK,MAAM,GAAG;CACrB;CACA,OAAO;EAAE;EAAO;CAAK;AACvB;AAqBA,MAAM,cAAc,GAAkB,eAA+B;CACnE,MAAM,QAAkB,CAAC;CACzB,MAAM,KAAK,kBAAkB,YAAY;CACzC,MAAM,KAAK,0BAA0B,EAAE,gBAAgB;CACvD,MAAM,KAAK,0BAA0B,EAAE,gBAAgB;CACvD,MAAM,KAAK,0BAA0B,EAAE,WAAW;CAClD,MAAM,KAAK,0BAA0B,EAAE,iBAAiB;CACxD,MAAM,KAAK,0BAA0B,EAAE,eAAe;CACtD,MAAM,KAAK,0BAA0B,EAAE,eAAe;CACtD,MAAM,KAAK,0BAA0B,EAAE,YAAY,UAAU;CAC7D,MAAM,KAAK,0BAA0B,EAAE,gBAAgB;CACvD,MAAM,KAAK,0BAA0B,EAAE,eAAe;CACtD,MAAM,KACJ,gCAAgC,EAAE,aAAa,MAAM,SAAS,KAAK,UAAU,EAAE,aAAa,KAAK,EAAE,cAAc,KAAK,UAAU,EAAE,aAAa,UAAU,GAC3J;CACA,MAAM,KAAK,0BAA0B,EAAE,mBAAmB;CAC1D,IAAI,EAAE,QAAQ,SAAS,GAAG;EACxB,MAAM,YAAY,EAAE,QAAQ,KAAK,MAAM,GAAG,EAAE,KAAK,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,IAAI;EAChF,MAAM,KAAK,0BAA0B,WAAW;CAClD,OACE,MAAM,KAAK,wCAAwC;CAErD,MAAM,KAAK,0BAA0B,EAAE,QAAQ;CAC/C,IAAI,EAAE,OAAO,SAAS,GACpB,KAAK,MAAM,KAAK,EAAE,QAChB,MAAM,KAAK,0BAA0B,GAAG;CAG5C,OAAO,MAAM,KAAK,IAAI,IAAI;AAC5B;AAEA,MAAM,SAAS,uBAAuB;CACpC,MAAM;CACN,MAAM,EACJ,aAAa,8DACf;CACA,MAAM;CACN,SAAS,OAAO,MAAM,QAAQ;EAC5B,MAAM,SAAS,MAAM,eAAe,KAAK,MAAM;EAC/C,MAAM,EAAE,KAAK,WAAW,MAAM,IAAI,iBAAiB;GAAE,KAAK,KAAK;GAAK,QAAQ,KAAK;EAAO,CAAC;EACzF,oBAAoB,KAAK,YAAY,gBAAgB;EACrD,MAAM,mBAAmB,GAAG,OAAO,UAAU,MAAM,IAAI,UAAU,OAAO,aAAa,KAAK;EAC1F,MAAM,iBAAiB,GAAG,iBAAiB;EAE3C,MAAM,OAAO,MAAM,gBAAgB,OAAO,SAAS,cAAc;EACjE,IAAI,SAAS,MACX,MAAM,IAAI,YACR,iBACA,6CAA6C,gBAC/C;EAEF,MAAM,MAAM,KAAK;EACjB,MAAM,gBAAgB,IAAI,iBAAiB;EAS3C,MAAM,kBAAiB,MALC,cACtB,OAAO,SACP,kBACA,KAAK,IAAI,eAAe,IAAI,SAAS,CACvC,EAAA,CACiC;EAEjC,MAAM,SAAmB,CAAC;EAC1B,IAAI,mBAAmB;EACvB,IAAI;GAMF,oBAAmB,MALA,qBAAqB;IACtC,SAAS,OAAO;IAChB;IACA,YAAY,KAAK;GACnB,CAAC,EAAA,EACwB,QAAQ;EACnC,SAAS,OAAO;GACd,IAAI,iBAAiB,aACnB,OAAO,KAAK,GAAG,MAAM,KAAK,IAAI,MAAM,SAAS;QAE7C,OAAO,KAAM,MAAgB,OAAO;EAExC;EAIA,IAAI;GACF,MAAM,iBAAiB,GAAG,iBAAiB;GAC3C,MAAM,EAAE,SAAS,MAAM,iBAAiB,OAAO,SAAS,cAAc;GACtE,KAAK,MAAM,KAAK,MACd,IAAI,IAAI,aAAa,GACnB,OAAO,KAAK,oBAAoB,GAAG;EAGzC,SAAS,OAAO;GACd,IAAI,iBAAiB,aACnB,OAAO,KAAK,GAAG,MAAM,KAAK,IAAI,MAAM,SAAS;EAEjD;EAEA,MAAM,UAA6C,CAAC;EACpD,IAAI,OAAO,KAAK,WAAW,YAAY,KAAK,OAAO,SAAS,GAAG;GAC7D,MAAM,OAAO,MAAM,sBAAsB,KAAK,QAAQ,KAAK,YAAY,gBAAgB;GACvF,KAAK,MAAM,OAAO,MAAM;IACtB,MAAM,SAAS,GAAG,iBAAiB,SAAS,IAAI,KAAK;IACrD,MAAM,EAAE,UAAU,MAAM,iBAAiB,OAAO,SAAS,MAAM;IAC/D,QAAQ,KAAK;KAAE,MAAM,IAAI;KAAM;IAAM,CAAC;GACxC;EACF;EAEA,MAAM,SAAwB;GAC5B;GACA,gBAAgB,IAAI;GACpB,WAAW,IAAI;GACf,iBAAiB;GACjB;GACA,eAAe,iBAAiB;GAChC,UAAU,IAAI;GACd,gBAAgB,IAAI;GACpB,eAAe,IAAI;GACnB,cAAc,IAAI;GAClB,mBAAmB;GACnB;GACA,QAAQ,OAAO,WAAW,IAAI,OAAO;GACrC;EACF;EAEA,IAAI,WAAW,GACb,YAAY;GAAE,SAAS;GAAW,GAAG;EAAO,CAAC;OAE7C,QAAQ,OAAO,MAAM,WAAW,QAAQ,KAAK,UAAU,CAAC;EAE1D,OAAO;CACT;AACF,CAAC;;AAGD,MAAa,UAAU,OAAO;AAOJ,OAAO"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { t as BaerlyError } from "./errors-Dfa3p78c.js";
|
|
2
|
+
//#region packages/protocol/src/storage/key.ts
|
|
3
|
+
/**
|
|
4
|
+
* Validate a raw {@link Storage} object key against the portable baerly key
|
|
5
|
+
* grammar before it is used to address an object. A key MUST be non-empty
|
|
6
|
+
* and `/`-delimited, with **no path segment equal to `.` or `..`**.
|
|
7
|
+
*
|
|
8
|
+
* The dot-segment rule is not a vendor quirk — it is a *client-side*
|
|
9
|
+
* universal: RFC 3986 §5.2.4 "remove dot segments" is mandatory for every
|
|
10
|
+
* conformant URL parser, so `<endpoint>/<bucket>/.` normalizes to
|
|
11
|
+
* `<endpoint>/<bucket>/` and `<endpoint>/<bucket>/..` escapes the bucket
|
|
12
|
+
* entirely — *before* the request is signed or sent, in TypeScript, Go,
|
|
13
|
+
* Rust, Python, or any language a `Storage` port is written in. Such a key
|
|
14
|
+
* cannot be addressed over the S3/R2 HTTP API regardless of backend; a bare
|
|
15
|
+
* `.` PUT surfaces as a confusing bucket-root 403 rather than a clear
|
|
16
|
+
* rejection. The kernel never emits such keys (caller-controlled segments
|
|
17
|
+
* are already screened by `assertPathSegment` one layer up), so this is the
|
|
18
|
+
* boundary guard that makes every adapter reject them *identically* and
|
|
19
|
+
* gives ports a single, testable contract.
|
|
20
|
+
*
|
|
21
|
+
* Thrown as `BaerlyError{code:"InvalidConfig"}` to match the sibling
|
|
22
|
+
* key/segment guards (`assertKeyWithinLimit`, `assertPathSegment`). The
|
|
23
|
+
* full-key 1024-byte ceiling is enforced separately on the write path
|
|
24
|
+
* (`assertKeyWithinLimit`), where multi-segment keys are assembled.
|
|
25
|
+
*
|
|
26
|
+
* @see docs/spec/storage-compatibility.md — "Key namespace"
|
|
27
|
+
*/
|
|
28
|
+
function assertValidStorageKey(key) {
|
|
29
|
+
if (key.length === 0) throw new BaerlyError("InvalidConfig", "storage key must be a non-empty string");
|
|
30
|
+
for (const segment of key.split("/")) if (segment === "." || segment === "..") throw new BaerlyError("InvalidConfig", `storage key may not contain a "." or ".." path segment — RFC 3986 dot-segment removal makes it unaddressable over the S3/R2 HTTP API: ${JSON.stringify(key)}`);
|
|
31
|
+
}
|
|
32
|
+
//#endregion
|
|
33
|
+
export { assertValidStorageKey as t };
|
|
34
|
+
|
|
35
|
+
//# sourceMappingURL=key-Di6yyxFu.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"key-Di6yyxFu.js","names":[],"sources":["../packages/protocol/src/storage/key.ts"],"sourcesContent":["import { BaerlyError } from \"../errors.ts\";\n\n/**\n * Validate a raw {@link Storage} object key against the portable baerly key\n * grammar before it is used to address an object. A key MUST be non-empty\n * and `/`-delimited, with **no path segment equal to `.` or `..`**.\n *\n * The dot-segment rule is not a vendor quirk — it is a *client-side*\n * universal: RFC 3986 §5.2.4 \"remove dot segments\" is mandatory for every\n * conformant URL parser, so `<endpoint>/<bucket>/.` normalizes to\n * `<endpoint>/<bucket>/` and `<endpoint>/<bucket>/..` escapes the bucket\n * entirely — *before* the request is signed or sent, in TypeScript, Go,\n * Rust, Python, or any language a `Storage` port is written in. Such a key\n * cannot be addressed over the S3/R2 HTTP API regardless of backend; a bare\n * `.` PUT surfaces as a confusing bucket-root 403 rather than a clear\n * rejection. The kernel never emits such keys (caller-controlled segments\n * are already screened by `assertPathSegment` one layer up), so this is the\n * boundary guard that makes every adapter reject them *identically* and\n * gives ports a single, testable contract.\n *\n * Thrown as `BaerlyError{code:\"InvalidConfig\"}` to match the sibling\n * key/segment guards (`assertKeyWithinLimit`, `assertPathSegment`). The\n * full-key 1024-byte ceiling is enforced separately on the write path\n * (`assertKeyWithinLimit`), where multi-segment keys are assembled.\n *\n * @see docs/spec/storage-compatibility.md — \"Key namespace\"\n */\nexport function assertValidStorageKey(key: string): void {\n if (key.length === 0) {\n throw new BaerlyError(\"InvalidConfig\", \"storage key must be a non-empty string\");\n }\n for (const segment of key.split(\"/\")) {\n if (segment === \".\" || segment === \"..\") {\n throw new BaerlyError(\n \"InvalidConfig\",\n `storage key may not contain a \".\" or \"..\" path segment — RFC 3986 dot-segment ` +\n `removal makes it unaddressable over the S3/R2 HTTP API: ${JSON.stringify(key)}`,\n );\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BA,SAAgB,sBAAsB,KAAmB;CACvD,IAAI,IAAI,WAAW,GACjB,MAAM,IAAI,YAAY,iBAAiB,wCAAwC;CAEjF,KAAK,MAAM,WAAW,IAAI,MAAM,GAAG,GACjC,IAAI,YAAY,OAAO,YAAY,MACjC,MAAM,IAAI,YACR,iBACA,yIAC6D,KAAK,UAAU,GAAG,GACjF;AAGN"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { n as encodeJsonBytes, t as decodeJsonBytes } from "./bytes-BXdamO3L.js";
|
|
2
|
-
import {
|
|
2
|
+
import { d as LOG_FORWARD_PROBE_CAP, s as CURRENT_JSON_CONTENT_TYPE } from "./config-1tv0-Yzm.js";
|
|
3
3
|
import { o as BaerlyError } from "./baerly.js";
|
|
4
4
|
//#region ../protocol/src/coordination/current-json.ts
|
|
5
5
|
/**
|
|
6
|
-
* `current.json` — the per-collection
|
|
7
|
-
* its embedded {@link WriterFence} epoch token.
|
|
6
|
+
* `current.json` — the per-collection compaction-state control object —
|
|
7
|
+
* and its embedded, currently dormant {@link WriterFence} epoch token.
|
|
8
8
|
*
|
|
9
9
|
* A dedicated `coordination/` namespace (parallel to `storage/`) holds
|
|
10
10
|
* primitives whose contract is "atomic agreement over a small JSON
|
|
@@ -129,7 +129,7 @@ async function casUpdateCurrentJson(storage, key, mutator, opts) {
|
|
|
129
129
|
}
|
|
130
130
|
/**
|
|
131
131
|
* Read `current.json.log_seq_start` — the low-water mark of the live
|
|
132
|
-
* log range `[log_seq_start,
|
|
132
|
+
* log range `[log_seq_start, tail_hint)`. The field is always present
|
|
133
133
|
* on disk (every `createCurrentJson` write seeds it; the compactor
|
|
134
134
|
* advances it on every fold); this helper exists to document intent
|
|
135
135
|
* at call sites that walk the log range or assert the snapshot
|
|
@@ -141,17 +141,17 @@ const encodeJson = (json) => encodeJsonBytes(json);
|
|
|
141
141
|
* Runtime guard for `parsed` to be a {@link CurrentJson}. Throws
|
|
142
142
|
* `BaerlyError{code:"InvalidResponse"}` rather than coercing. Tolerant
|
|
143
143
|
* reader: unknown keys ignored, new required fields bump
|
|
144
|
-
* `schema_version` (move (b), ADR-
|
|
144
|
+
* `schema_version` (move (b), ADR-003).
|
|
145
145
|
*/
|
|
146
146
|
const assertCurrentJson = (parsed, key) => {
|
|
147
147
|
if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `current.json at ${key}: parsed body is not an object`);
|
|
148
148
|
const r = parsed;
|
|
149
|
-
if (r["schema_version"] === 1) throw new BaerlyError("InvalidResponse", `current.json at ${key}
|
|
150
|
-
if (r["schema_version"] !==
|
|
149
|
+
if (r["schema_version"] === 1 || r["schema_version"] === 2) throw new BaerlyError("InvalidResponse", `current.json at ${key} uses schema v${String(r["schema_version"])} from a pre-0.3.0 internal build; this build requires schema v3. Dump/restore with the old build, or re-provision scratch/local buckets.`);
|
|
150
|
+
if (r["schema_version"] !== 3) throw new BaerlyError("InvalidResponse", `current.json at ${key}: unsupported schema_version ${String(r["schema_version"])}; this build requires 3. This almost always means stale scratch data from a different build — wipe the local dev bucket (e.g. \`rm -rf .baerly-data\`) or recreate the R2/S3 bucket, then retry.`);
|
|
151
151
|
if (!(typeof r["snapshot"] === "string" || r["snapshot"] === null)) throw new BaerlyError("InvalidResponse", `current.json at ${key}: snapshot must be string|null`);
|
|
152
|
-
if (typeof r["
|
|
152
|
+
if (typeof r["tail_hint"] !== "number" || !Number.isInteger(r["tail_hint"]) || r["tail_hint"] < 0) throw new BaerlyError("InvalidResponse", `current.json at ${key}: tail_hint must be a non-negative integer`);
|
|
153
153
|
if (typeof r["log_seq_start"] !== "number" || !Number.isInteger(r["log_seq_start"]) || r["log_seq_start"] < 0) throw new BaerlyError("InvalidResponse", `current.json at ${key}: log_seq_start must be a non-negative integer`);
|
|
154
|
-
if (r["log_seq_start"] > r["
|
|
154
|
+
if (r["log_seq_start"] > r["tail_hint"]) throw new BaerlyError("InvalidResponse", `current.json at ${key}: log_seq_start ${String(r["log_seq_start"])} > tail_hint ${String(r["tail_hint"])}`);
|
|
155
155
|
const fence = r["writer_fence"];
|
|
156
156
|
if (fence === null || typeof fence !== "object") throw new BaerlyError("InvalidResponse", `current.json at ${key}: writer_fence missing`);
|
|
157
157
|
const f = fence;
|
|
@@ -159,10 +159,10 @@ const assertCurrentJson = (parsed, key) => {
|
|
|
159
159
|
if (typeof f["owner"] !== "string") throw new BaerlyError("InvalidResponse", `current.json at ${key}: writer_fence.owner must be a string`);
|
|
160
160
|
if (typeof f["claimed_at"] !== "string") throw new BaerlyError("InvalidResponse", `current.json at ${key}: writer_fence.claimed_at must be a string`);
|
|
161
161
|
if (f["lease_until"] !== void 0 && typeof f["lease_until"] !== "string") throw new BaerlyError("InvalidResponse", `current.json at ${key}: writer_fence.lease_until must be string if present`);
|
|
162
|
-
if (typeof r["tail_bytes"] !== "number" || !Number.isInteger(r["tail_bytes"]) || r["tail_bytes"] < 0) throw new BaerlyError("InvalidResponse", `current.json at ${key}: tail_bytes must be a non-negative integer`);
|
|
163
162
|
if (typeof r["snapshot_bytes"] !== "number" || !Number.isInteger(r["snapshot_bytes"]) || r["snapshot_bytes"] < 0) throw new BaerlyError("InvalidResponse", `current.json at ${key}: snapshot_bytes must be a non-negative integer`);
|
|
164
163
|
if (typeof r["snapshot_rows"] !== "number" || !Number.isInteger(r["snapshot_rows"]) || r["snapshot_rows"] < 0) throw new BaerlyError("InvalidResponse", `current.json at ${key}: snapshot_rows must be a non-negative integer`);
|
|
165
164
|
if (r["last_warned_seq"] !== void 0 && (typeof r["last_warned_seq"] !== "number" || !Number.isInteger(r["last_warned_seq"]) || r["last_warned_seq"] < 0)) throw new BaerlyError("InvalidResponse", `current.json at ${key}: last_warned_seq must be a non-negative integer if present`);
|
|
165
|
+
if (r["mean_entry_bytes"] !== void 0 && (typeof r["mean_entry_bytes"] !== "number" || !Number.isInteger(r["mean_entry_bytes"]) || r["mean_entry_bytes"] < 0)) throw new BaerlyError("InvalidResponse", `current.json at ${key}: mean_entry_bytes must be a non-negative integer if present`);
|
|
166
166
|
return parsed;
|
|
167
167
|
};
|
|
168
168
|
/**
|
|
@@ -177,6 +177,13 @@ const translateCasError = (e, key) => {
|
|
|
177
177
|
if (e instanceof BaerlyError) return e;
|
|
178
178
|
return new BaerlyError("InvalidResponse", `current.json write at ${key} failed: ${String(e)}`, e);
|
|
179
179
|
};
|
|
180
|
+
/**
|
|
181
|
+
* The single constructor for a log-object key:
|
|
182
|
+
* `<manifestPrefix>/log/<seq>.json`. `manifestPrefix` is the
|
|
183
|
+
* `<…>/manifests/<collection>` segment — this helper adds `/log/`.
|
|
184
|
+
* Route all callers here so the key shape lives in one place.
|
|
185
|
+
*/
|
|
186
|
+
const logObjectKey = (manifestPrefix, seq) => `${manifestPrefix}/log/${seq}.json`;
|
|
180
187
|
//#endregion
|
|
181
188
|
//#region ../protocol/src/snapshot-hash.ts
|
|
182
189
|
/**
|
|
@@ -199,6 +206,100 @@ const snapshotHash = async (bytes) => {
|
|
|
199
206
|
return hex;
|
|
200
207
|
};
|
|
201
208
|
//#endregion
|
|
209
|
+
//#region ../server/src/snapshot.ts
|
|
210
|
+
/**
|
|
211
|
+
* Snapshot file shape + parse. Lifted out of `./compactor.ts` so that
|
|
212
|
+
* the kernel barrel (`packages/server/src/index.ts`) — which re-exports
|
|
213
|
+
* these four symbols — does not transitively pull `withObservability`
|
|
214
|
+
* via `compact()`. `./compactor.ts` imports its own snapshot primitives
|
|
215
|
+
* back from this module; the public API surface is unchanged.
|
|
216
|
+
*
|
|
217
|
+
* @see ../../../docs/spec/sync-protocol.md
|
|
218
|
+
*/
|
|
219
|
+
/**
|
|
220
|
+
* Single-level snapshot at L9; the level prefix is forward-compatible
|
|
221
|
+
* with future L0..L9 rolling merges without a wire change.
|
|
222
|
+
*/
|
|
223
|
+
const SNAPSHOT_LEVEL = 9;
|
|
224
|
+
/**
|
|
225
|
+
* `minSeq` and `maxSeq` are padded to {@link SEQ_DIGITS} digits so
|
|
226
|
+
* `list(prefix)` returns snapshots in seq order without needing to
|
|
227
|
+
* parse the filename. 12 digits caps at ~10^12 entries per collection;
|
|
228
|
+
* a real bucket hits S3's per-prefix QPS ceiling long before that.
|
|
229
|
+
*/
|
|
230
|
+
const SEQ_DIGITS = 12;
|
|
231
|
+
const MAX_SEQ = 10 ** SEQ_DIGITS - 1;
|
|
232
|
+
/**
|
|
233
|
+
* Build the on-bucket key for a snapshot file. `sha256` must be the
|
|
234
|
+
* lowercase hex digest of the canonical body
|
|
235
|
+
* ({@link encodeSnapshotBody}); the filename embeds it so a crashed
|
|
236
|
+
* mid-PUT body that doesn't match its own filename hash is rejected
|
|
237
|
+
* by readers as "missing."
|
|
238
|
+
*
|
|
239
|
+
* @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
|
|
240
|
+
* out of range or `sha256` is not 64 hex chars.
|
|
241
|
+
*/
|
|
242
|
+
const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
|
|
243
|
+
if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
|
|
244
|
+
if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
|
|
245
|
+
const pad = (n) => n.toString().padStart(SEQ_DIGITS, "0");
|
|
246
|
+
return `${collectionPrefix}/snapshot/L${SNAPSHOT_LEVEL}/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
|
|
247
|
+
};
|
|
248
|
+
/**
|
|
249
|
+
* Canonical byte encoding of a {@link SnapshotBody}. Only the
|
|
250
|
+
* compactor produces snapshots, and only this function does the
|
|
251
|
+
* serialisation — so the byte output is deterministic for a given
|
|
252
|
+
* `SnapshotBody` (identical input ⇒ identical bytes ⇒ identical
|
|
253
|
+
* filename hash).
|
|
254
|
+
*/
|
|
255
|
+
const encodeSnapshotBody = (s) => encodeJsonBytes(s);
|
|
256
|
+
/**
|
|
257
|
+
* Load a snapshot file as a `Map<_id, DocumentData>`. Verifies the
|
|
258
|
+
* body hash against the filename, checks the schema version, and
|
|
259
|
+
* cross-checks `body.collection` against the caller's expectation.
|
|
260
|
+
*
|
|
261
|
+
* @example
|
|
262
|
+
* ```ts
|
|
263
|
+
* import { loadSnapshotAsMap } from "@gusto/baerly-storage";
|
|
264
|
+
*
|
|
265
|
+
* const map =
|
|
266
|
+
* srcCurrent.snapshot === null
|
|
267
|
+
* ? new Map<string, DocumentData>()
|
|
268
|
+
* : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
|
|
269
|
+
* ```
|
|
270
|
+
*
|
|
271
|
+
* @throws BaerlyError code="Internal" — pointer resolves to no body,
|
|
272
|
+
* or the body's SHA-256 doesn't match the hash embedded in the
|
|
273
|
+
* filename. (Protocol invariant violation; a corrupted snapshot
|
|
274
|
+
* the reader cannot continue past.)
|
|
275
|
+
* @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
|
|
276
|
+
* isn't an object, has an unsupported `schema_version`, or carries a
|
|
277
|
+
* `collection` that doesn't match `expectedCollection`.
|
|
278
|
+
*/
|
|
279
|
+
const loadSnapshotAsMap = async (storage, key, expectedCollection, signal) => {
|
|
280
|
+
const got = await storage.get(key, signal !== void 0 ? { signal } : void 0);
|
|
281
|
+
if (got === null) throw new BaerlyError("Internal", `compact: snapshot pointer ${key} resolves to no body; protocol violation`);
|
|
282
|
+
const filename = key.slice(key.lastIndexOf("/") + 1).replace(/\.json$/, "");
|
|
283
|
+
const lastDash = filename.lastIndexOf("-");
|
|
284
|
+
const expectedHash = filename.slice(lastDash + 1);
|
|
285
|
+
const actualHash = await snapshotHash(got.body);
|
|
286
|
+
if (actualHash !== expectedHash) throw new BaerlyError("Internal", `compact: snapshot ${key} body hash mismatch (got ${actualHash}); protocol violation`);
|
|
287
|
+
let parsed;
|
|
288
|
+
try {
|
|
289
|
+
parsed = decodeJsonBytes(got.body);
|
|
290
|
+
} catch (error) {
|
|
291
|
+
throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
|
|
292
|
+
}
|
|
293
|
+
if (parsed === null || typeof parsed !== "object") throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not an object`);
|
|
294
|
+
const body = parsed;
|
|
295
|
+
if (body.schema_version !== 1) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} has unsupported schema_version ${String(body.schema_version)}`);
|
|
296
|
+
if (body.collection !== expectedCollection) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} carries collection ${body.collection}, expected ${expectedCollection}`);
|
|
297
|
+
if (!Array.isArray(body.docs)) throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body.docs is not an array`);
|
|
298
|
+
const out = /* @__PURE__ */ new Map();
|
|
299
|
+
for (const row of body.docs) out.set(row._id, row.body);
|
|
300
|
+
return out;
|
|
301
|
+
};
|
|
302
|
+
//#endregion
|
|
202
303
|
//#region ../server/src/log-walk.ts
|
|
203
304
|
/**
|
|
204
305
|
* Shared log-walk primitives consumed by the read path (`query.ts`),
|
|
@@ -230,7 +331,7 @@ const snapshotHash = async (bytes) => {
|
|
|
230
331
|
* every caller.
|
|
231
332
|
*
|
|
232
333
|
* @throws BaerlyError code="Internal" when the key resolves to no
|
|
233
|
-
* body — a hole inside `[log_seq_start,
|
|
334
|
+
* body — a hole inside `[log_seq_start, tail_hint)` is a corruption
|
|
234
335
|
* the kernel can't continue past.
|
|
235
336
|
* @throws BaerlyError code="InvalidResponse" when the body isn't
|
|
236
337
|
* valid JSON.
|
|
@@ -243,14 +344,12 @@ const readLogEntry = async (storage, key, opts) => {
|
|
|
243
344
|
* byteLength of the STORED log-object body exactly as it was GET back
|
|
244
345
|
* from storage (`got.body.byteLength`).
|
|
245
346
|
*
|
|
246
|
-
* The compactor uses this to
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
* `
|
|
251
|
-
*
|
|
252
|
-
* assumption (a re-encode of the parsed `LogEntry` could drift if
|
|
253
|
-
* key-order/whitespace ever diverged from what was PUT).
|
|
347
|
+
* The compactor uses this to sum the byte cost of the folded slice.
|
|
348
|
+
* The stored body byteLength IS what the writer PUT for each entry,
|
|
349
|
+
* so summing `got.body.byteLength` over the folded range yields the
|
|
350
|
+
* exact cost — no re-encode, no round-trip assumption (a re-encode
|
|
351
|
+
* of the parsed `LogEntry` could drift if key-order/whitespace ever
|
|
352
|
+
* diverged from what was PUT).
|
|
254
353
|
*
|
|
255
354
|
* Same error semantics as {@link readLogEntry}.
|
|
256
355
|
*/
|
|
@@ -288,7 +387,7 @@ const walkLogRange = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) =
|
|
|
288
387
|
* this is the bytes-aware core `walkLogRange` is implemented on top of.
|
|
289
388
|
*
|
|
290
389
|
* The compactor consumes this so it gets BOTH the parsed entries (for
|
|
291
|
-
* the fold) AND the exact byte sum (
|
|
390
|
+
* the fold) AND the exact byte sum (for `mean_entry_bytes` stamping) from a
|
|
292
391
|
* single walk.
|
|
293
392
|
*/
|
|
294
393
|
const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive, opts) => {
|
|
@@ -301,7 +400,7 @@ const walkLogRangeWithBytes = async (storage, logPrefix, fromSeq, toSeqExclusive
|
|
|
301
400
|
const promises = [];
|
|
302
401
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
303
402
|
const seq = fromSeq + i;
|
|
304
|
-
promises.push(readLogEntryWithBytes(storage,
|
|
403
|
+
promises.push(readLogEntryWithBytes(storage, logObjectKey(logPrefix, seq), opts));
|
|
305
404
|
}
|
|
306
405
|
const results = await Promise.all(promises);
|
|
307
406
|
for (let i = 0; i < results.length; i++) out[chunkStart + i] = results[i];
|
|
@@ -351,77 +450,73 @@ const foldLogEntriesOnto = (map, entries, opts) => {
|
|
|
351
450
|
}
|
|
352
451
|
}
|
|
353
452
|
};
|
|
354
|
-
|
|
453
|
+
//#endregion
|
|
454
|
+
//#region ../server/src/log-tail.ts
|
|
355
455
|
/**
|
|
356
|
-
*
|
|
357
|
-
*
|
|
358
|
-
*
|
|
359
|
-
*
|
|
360
|
-
* by readers as "missing."
|
|
361
|
-
*
|
|
362
|
-
* @throws BaerlyError code="InvalidConfig" — `minSeq`/`maxSeq` are
|
|
363
|
-
* out of range or `sha256` is not 64 hex chars.
|
|
456
|
+
* Tolerant forward-probe for the true committed log tail. The strict
|
|
457
|
+
* `[log_seq_start, hint)` range is dense (a hole is corruption, owned
|
|
458
|
+
* by `walkLogRange`); this owns only the tolerant `[hint, ∞)` probe,
|
|
459
|
+
* where `hint` is a lower bound — the real tail may sit at or past it.
|
|
364
460
|
*/
|
|
365
|
-
const snapshotKey = (collectionPrefix, minSeq, maxSeq, sha256) => {
|
|
366
|
-
if (minSeq < 0 || maxSeq < 0 || minSeq > maxSeq || maxSeq > MAX_SEQ) throw new BaerlyError("InvalidConfig", `snapshotKey: invalid range [${minSeq}, ${maxSeq})`);
|
|
367
|
-
if (!/^[0-9a-f]{64}$/.test(sha256)) throw new BaerlyError("InvalidConfig", "snapshotKey: sha256 must be 64 hex chars");
|
|
368
|
-
const pad = (n) => n.toString().padStart(12, "0");
|
|
369
|
-
return `${collectionPrefix}/snapshot/L9/${pad(minSeq)}-${pad(maxSeq)}-${sha256}.json`;
|
|
370
|
-
};
|
|
371
461
|
/**
|
|
372
|
-
*
|
|
373
|
-
*
|
|
374
|
-
*
|
|
375
|
-
* `
|
|
376
|
-
*
|
|
462
|
+
* Discover the true committed tail and fold entries in `[hint, tail)`.
|
|
463
|
+
* `tail` is the first empty seq (>= hint); `entries` are the
|
|
464
|
+
* LogEntries in `[hint, tail)` in seq order. `cap` bounds the walk;
|
|
465
|
+
* exhausting it THROWS `Internal` (never silently truncates) — see
|
|
466
|
+
* the cap-exhaustion comment below.
|
|
377
467
|
*/
|
|
378
|
-
const
|
|
468
|
+
const probeTailFrom = async (storage, logPrefix, hint, opts) => {
|
|
469
|
+
const cap = opts?.cap ?? 1e5;
|
|
470
|
+
const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
471
|
+
const entries = [];
|
|
472
|
+
for (let i = 0; i < cap; i++) {
|
|
473
|
+
const seq = hint + i;
|
|
474
|
+
const key = logObjectKey(logPrefix, seq);
|
|
475
|
+
const got = await storage.get(key, getOpts);
|
|
476
|
+
if (got === null) return {
|
|
477
|
+
tail: seq,
|
|
478
|
+
entries
|
|
479
|
+
};
|
|
480
|
+
try {
|
|
481
|
+
entries.push(decodeJsonBytes(got.body));
|
|
482
|
+
} catch (error) {
|
|
483
|
+
throw new BaerlyError("InvalidResponse", `probeTailFrom: malformed log entry at ${key}: ${error.message}`, { cause: error });
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
throw new BaerlyError("Internal", `probeTailFrom: forward probe exceeded ${cap} from hint ${hint} on ${logPrefix}`);
|
|
487
|
+
};
|
|
379
488
|
/**
|
|
380
|
-
*
|
|
381
|
-
*
|
|
382
|
-
*
|
|
489
|
+
* First empty log seq at/after `hint` — the slot a new commit creates —
|
|
490
|
+
* found by galloping search (`O(log gap)` GETs vs `probeTailFrom`'s
|
|
491
|
+
* `O(gap)`). Used by the writer's per-commit tail-find, where `tail_hint`
|
|
492
|
+
* can lag the true tail. Position only; no bodies.
|
|
383
493
|
*
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
*
|
|
388
|
-
* const map =
|
|
389
|
-
* srcCurrent.snapshot === null
|
|
390
|
-
* ? new Map<string, DocumentData>()
|
|
391
|
-
* : await loadSnapshotAsMap(src.storage, srcCurrent.snapshot, "tickets");
|
|
392
|
-
* ```
|
|
393
|
-
*
|
|
394
|
-
* @throws BaerlyError code="Internal" — pointer resolves to no body,
|
|
395
|
-
* or the body's SHA-256 doesn't match the hash embedded in the
|
|
396
|
-
* filename. (Protocol invariant violation; a corrupted snapshot
|
|
397
|
-
* the reader cannot continue past.)
|
|
398
|
-
* @throws BaerlyError code="InvalidResponse" — body isn't valid JSON,
|
|
399
|
-
* isn't an object, has an unsupported `schema_version`, or carries a
|
|
400
|
-
* `collection` that doesn't match `expectedCollection`.
|
|
494
|
+
* PRECONDITION: occupancy MUST be a dense prefix from `hint` (galloping
|
|
495
|
+
* assumes monotone occupied→empty; a hole ABOVE `hint` would be skipped
|
|
496
|
+
* and the gallop could return a tail past live entries). Contrast
|
|
497
|
+
* `probeTailFrom`, which stops at the first 404 and is hole-tolerant.
|
|
401
498
|
*/
|
|
402
|
-
const
|
|
403
|
-
const
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
throw new BaerlyError("InvalidResponse", `compact: snapshot ${key} body is not valid JSON`, error);
|
|
499
|
+
const findLogTail = async (storage, logPrefix, hint, opts) => {
|
|
500
|
+
const getOpts = opts?.signal !== void 0 ? { signal: opts.signal } : void 0;
|
|
501
|
+
const exists = async (seq) => await storage.get(logObjectKey(logPrefix, seq), getOpts) !== null;
|
|
502
|
+
if (!await exists(hint)) return hint;
|
|
503
|
+
let lo = hint;
|
|
504
|
+
let step = 1;
|
|
505
|
+
let hi = hint + step;
|
|
506
|
+
while (await exists(hi)) {
|
|
507
|
+
lo = hi;
|
|
508
|
+
step *= 2;
|
|
509
|
+
hi = lo + step;
|
|
510
|
+
if (step > 1e5) throw new BaerlyError("Internal", `findLogTail: galloping probe exceeded ${LOG_FORWARD_PROBE_CAP} from hint ${hint} on ${logPrefix}`);
|
|
415
511
|
}
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
return out;
|
|
512
|
+
while (hi - lo > 1) {
|
|
513
|
+
const mid = lo + Math.floor((hi - lo) / 2);
|
|
514
|
+
if (await exists(mid)) lo = mid;
|
|
515
|
+
else hi = mid;
|
|
516
|
+
}
|
|
517
|
+
return hi;
|
|
423
518
|
};
|
|
424
519
|
//#endregion
|
|
425
|
-
export {
|
|
520
|
+
export { walkLogRange as a, loadSnapshotAsMap as c, logObjectKey as d, casUpdateCurrentJson as f, readCurrentJson as h, readLogEntry as i, snapshotKey as l, logSeqStartOf as m, probeTailFrom as n, walkLogRangeWithBytes as o, createCurrentJson as p, foldLogEntriesOnto as r, encodeSnapshotBody as s, findLogTail as t, snapshotHash as u };
|
|
426
521
|
|
|
427
|
-
//# sourceMappingURL=
|
|
522
|
+
//# sourceMappingURL=log-tail-D98U1JBo.js.map
|