@amalgm/shell 0.1.111 → 0.1.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
3
3
  import { open, readFile } from "node:fs/promises";
4
4
  import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
5
5
  import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
6
- import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, deriveUploadManifest, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, planCargoWaves, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
6
+ import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, deriveUploadManifest, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, pathIsSuspect, planCargoWaves, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, GRAPH_HEAD_OPERATION_KIND, } from "@amalgm/live";
7
7
  import Database from "better-sqlite3";
8
8
  import { atomicCopy, atomicWrite, ensurePrivateDir, ensureUserDir } from "./filesystem.js";
9
9
  import { emitFilesStage, } from "./files-observability.js";
@@ -22,9 +22,9 @@ import { WORKSPACE_UUID as UUID, createFilesRegisterPorts, ensureWorkspaceBindin
22
22
  import { applyRepositoryFiles, captureRepository, inspectRepositoryTransportFile, hasGitMarker, readCachedRepositoryIdentity, } from "./git-repository-host.js";
23
23
  import { gitEvidenceFingerprint, inspectGitRegistration, sameGitIdentity, } from "./git-registration-host.js";
24
24
  import { projectMaterializedGraph } from "./materialized-graph.js";
25
+ import { graphPageIdentities, registryGraphFromSnapshot, } from "./registry-graph.js";
25
26
  import { NodeWatchHost, } from "./watching/index.js";
26
27
  import { GroundCoordinator } from "./ground-coordination.js";
27
- import { submitMutationOperation } from "./paged-mutation-submit.js";
28
28
  import { adoptSnapshotDeliveryBaseline } from "./snapshot-delivery-baseline.js";
29
29
  import { WireClient, WireRequestError } from "./wire-client.js";
30
30
  /** Cargo waves hold up to 16MiB of payload each; two in flight bounds the
@@ -40,6 +40,10 @@ const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = 128;
40
40
  // on the 1 GiB runtimes used for ordinary Files materialization. Small
41
41
  // objects still use the full request-concurrency window.
42
42
  const DIRECT_OBJECT_DOWNLOAD_MAX_IN_FLIGHT_BYTES = 2 * CHUNK_BYTES;
43
+ /** Graph pages are at most a few megabytes each and a whole account is a
44
+ * handful of them, so a narrow lane keeps the cold read fast without holding
45
+ * the whole graph twice while it assembles. */
46
+ const GRAPH_PAGE_READ_CONCURRENCY = 8;
43
47
  const DETECT_QUIET_MS = 12;
44
48
  const DETECT_MAX_DEFERRAL_MS = 75;
45
49
  const DETECT_RETRY_MS = 250;
@@ -55,13 +59,19 @@ const WALK_CAPTURE_CONCURRENCY = 8;
55
59
  const REGISTER_STALE_CAPTURE_ATTEMPTS = 3;
56
60
  const DOWNLOAD_CONTENT_RETRY_ATTEMPTS = 3;
57
61
  const APPLY_GROUND_WAIT_TIMEOUT_MS = 15_000;
62
+ /** Load a journaled publication for replay. The records whose cargo must
63
+ * re-upload are already inside the publication's own bytes — its pages, or,
64
+ * for a row converted from the pre-page journal, the replacement operation
65
+ * itself — so nothing is stored twice and nothing is re-read from ground. */
58
66
  function loadSnapshotPublication(entry) {
59
- const snapshotRecords = snapshotFromRecords((JSON.parse(entry.snapshotBlob.toString("utf8")).records ?? [])).records;
60
67
  const uploadIds = new Set(entry.uploadUuids);
68
+ const published = uploadIds.size === 0 ? [] : entry.pages.length > 0
69
+ ? entry.pages.flatMap((page) => JSON.parse(page.bytes.toString("utf8")))
70
+ : (JSON.parse(entry.operationBlob.toString("utf8")).records ?? []);
61
71
  return {
62
72
  ...entry,
63
- snapshotRecords,
64
- uploadRecords: snapshotRecords.filter((record) => uploadIds.has(record.uuid)),
73
+ uploadRecords: published.filter((record) => uploadIds.has(record.uuid)),
74
+ graph: null,
65
75
  };
66
76
  }
67
77
  const GROUND_ROW_COLUMNS = [
@@ -205,6 +215,11 @@ export class UserGroundHost {
205
215
  stage(input) {
206
216
  emitFilesStage(this.options.onFilesStage, input);
207
217
  }
218
+ /** Whether a counter worth real time on the critical path may be computed. */
219
+ measuring() {
220
+ return this.options.filesTraceActive?.()
221
+ ?? (this.options.onFilesStage !== undefined || this.options.onDetectScan !== undefined);
222
+ }
208
223
  reportDetectScan(evidence) {
209
224
  this.options.onDetectScan?.(evidence);
210
225
  this.stage({
@@ -222,7 +237,19 @@ export class UserGroundHost {
222
237
  gitInspections: evidence.gitInspections,
223
238
  repositoryCaptures: evidence.repositoryCaptures,
224
239
  records: evidence.records ?? 0,
225
- recordBytes: evidence.recordBytes ?? 0,
240
+ ...(evidence.recordBytes === undefined ? {} : { recordBytes: evidence.recordBytes }),
241
+ // Where the scan actually went. `captureBusyMs / captureMs` is the
242
+ // capture pool's occupancy; at ~1 the pool ran one file at a time.
243
+ walkMs: Math.round(evidence.walkMs ?? 0),
244
+ gitMs: Math.round(evidence.gitMs ?? 0),
245
+ captureMs: Math.round(evidence.captureMs ?? 0),
246
+ captureBusyMs: Math.round(evidence.captureBusyMs ?? 0),
247
+ captureBytes: evidence.captureBytes ?? 0,
248
+ repoIdentityMs: Math.round(evidence.repositoryIdentityMs ?? 0),
249
+ repoCaptureMs: Math.round(evidence.repositoryCaptureMs ?? 0),
250
+ repoSealMs: Math.round(evidence.repositorySealMs ?? 0),
251
+ reconcileMs: Math.round(evidence.reconcileMs ?? 0),
252
+ persistMs: Math.round(evidence.persistMs ?? 0),
226
253
  },
227
254
  });
228
255
  }
@@ -541,7 +568,7 @@ export class UserGroundHost {
541
568
  && record.status === "active"
542
569
  && record.payloadVersion === workspaceId);
543
570
  if (hasRegisteredBoundaryInDatabase(database, workspaceId)
544
- && hasRegisteredBoundary(this.cloudState.records)) {
571
+ && hasRegisteredBoundary(this.cloudState.graph.records())) {
545
572
  // Already declared — but Register may only return once cargo closure
546
573
  // holds under the current content contract. The graph head can be
547
574
  // unchanged while a contract cutover moved its cargo namespace, so
@@ -620,6 +647,7 @@ export class UserGroundHost {
620
647
  database: this.databasePath(identity),
621
648
  cacheDir: this.cacheDir(identity),
622
649
  onScan: (evidence) => this.reportDetectScan(evidence),
650
+ measuring: this.measuring(),
623
651
  registrationWorkspaceId: workspaceId,
624
652
  suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
625
653
  });
@@ -722,7 +750,7 @@ export class UserGroundHost {
722
750
  if (!hasRegisteredBoundaryInDatabase(database, workspaceId)) {
723
751
  throw stageError("entity registration", new Error(`workspace ${workspaceId} and its durable reference were not recorded`));
724
752
  }
725
- if (!hasRegisteredBoundary(this.cloudState.records)) {
753
+ if (!hasRegisteredBoundary(this.cloudState.graph.records())) {
726
754
  throw stageError("cloud graph commit", new Error(`workspace ${workspaceId} or its durable reference is absent from the committed head`));
727
755
  }
728
756
  return {
@@ -804,9 +832,10 @@ export class UserGroundHost {
804
832
  throw new Error("entity Record rail already started");
805
833
  await this.wire.open();
806
834
  // One floor, no per-feature gates: this shell and its gateway release
807
- // together, and the gateway ships first. 12 = bounded cold-Register cargo
808
- // waves; an older gateway cannot verify and seal that upload frame.
809
- if (this.wire.protocolVersion < 12) {
835
+ // together, and the gateway ships first. 13 = immutable registry graph
836
+ // pages plus one graph-head commit; an older gateway cannot publish or
837
+ // resolve that representation.
838
+ if (this.wire.protocolVersion < 13) {
810
839
  throw new Error("gateway speaks an older wire protocol than this shell");
811
840
  }
812
841
  this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
@@ -1078,7 +1107,7 @@ export class UserGroundHost {
1078
1107
  }, ["private.resource.ready", "private.resource.missing"]);
1079
1108
  if (frame.type === "private.resource.missing")
1080
1109
  return null;
1081
- const ready = readyFromFrame(frame);
1110
+ const ready = readyFromFrame(frame, await this.materializeRegistry(identity, frame));
1082
1111
  this.cloudState = ready.state;
1083
1112
  return ready.cloud;
1084
1113
  },
@@ -1093,7 +1122,7 @@ export class UserGroundHost {
1093
1122
  snapshot_checksum: input.checksum,
1094
1123
  snapshot_bytes: bytes.length,
1095
1124
  }, ["private.resource.ready"]);
1096
- const ready = readyFromFrame(frame);
1125
+ const ready = readyFromFrame(frame, await this.materializeRegistry(identity, frame));
1097
1126
  const current = ready.cloud;
1098
1127
  this.cloudState = ready.state;
1099
1128
  const created = frame.duplicate === false;
@@ -1103,6 +1132,61 @@ export class UserGroundHost {
1103
1132
  },
1104
1133
  };
1105
1134
  }
1135
+ /** The authority's stored snapshot as this machine's graph. A head names
1136
+ * immutable pages, so every page this machine already cached is a local read
1137
+ * and an unchanged root never crosses the network twice. */
1138
+ async materializeRegistry(identity, frame) {
1139
+ const resourceId = String(frame.resource_id || "");
1140
+ return registryGraphFromSnapshot(Buffer.from(String(frame.snapshot_b64 || ""), "base64"), (refs) => this.readGraphPages(identity, resourceId, refs), sha256Hex);
1141
+ }
1142
+ /** Read graph pages by identity: cached pages come off local disk, the rest
1143
+ * are fetched straight from content storage and cached on the way in. Every
1144
+ * page is re-proven against its declared identity before it is kept. */
1145
+ async readGraphPages(identity, resourceId, refs) {
1146
+ const directory = graphPageCacheDir(this.cacheDir(identity));
1147
+ const pages = new Map();
1148
+ const missing = [];
1149
+ for (const ref of new Map(refs.map((ref) => [ref.sha256, ref])).values()) {
1150
+ let cached = null;
1151
+ try {
1152
+ cached = readFileSync(join(directory, `${ref.sha256}.bin`));
1153
+ }
1154
+ catch {
1155
+ cached = null;
1156
+ }
1157
+ // A cache copy of the wrong size is torn evidence, never a page: fetch
1158
+ // the immutable original instead of assembling from it.
1159
+ if (cached && cached.byteLength === ref.bytes)
1160
+ pages.set(ref.sha256, cached);
1161
+ else
1162
+ missing.push(ref);
1163
+ }
1164
+ if (missing.length === 0)
1165
+ return pages;
1166
+ const started = performance.now();
1167
+ let fetchedBytes = 0;
1168
+ await boundedForEach(missing, GRAPH_PAGE_READ_CONCURRENCY, async (ref) => {
1169
+ const bytes = await retryContent(() => this.direct.fetchObject(resourceId, { kind: "block", sha256: ref.sha256 }), true);
1170
+ if (bytes.byteLength !== ref.bytes || sha256Hex(bytes) !== ref.sha256) {
1171
+ throw new Error(`cloud graph page ${ref.sha256} does not match its bytes`);
1172
+ }
1173
+ atomicWrite(join(directory, `${ref.sha256}.bin`), bytes);
1174
+ fetchedBytes += bytes.byteLength;
1175
+ pages.set(ref.sha256, bytes);
1176
+ });
1177
+ this.stage({
1178
+ primitive: "download",
1179
+ stage: "graph-pages",
1180
+ status: "completed",
1181
+ durationMs: performance.now() - started,
1182
+ measurements: {
1183
+ pages: refs.length,
1184
+ fetchedPages: missing.length,
1185
+ fetchedBytes,
1186
+ },
1187
+ });
1188
+ return pages;
1189
+ }
1106
1190
  localPort(identity) {
1107
1191
  const userRoot = this.userRoot(identity);
1108
1192
  const database = this.databasePath(identity);
@@ -1137,6 +1221,7 @@ export class UserGroundHost {
1137
1221
  await reconcileGround({
1138
1222
  identity, userRoot, database, cacheDir,
1139
1223
  onScan: (evidence) => this.reportDetectScan(evidence),
1224
+ measuring: this.measuring(),
1140
1225
  });
1141
1226
  return localValue(identity, userRoot, database);
1142
1227
  },
@@ -1851,6 +1936,7 @@ export class UserGroundHost {
1851
1936
  database: this.databasePath(identity),
1852
1937
  cacheDir: this.cacheDir(identity),
1853
1938
  onScan: (evidence) => this.reportDetectScan(evidence),
1939
+ measuring: this.measuring(),
1854
1940
  suspicions: reconcileSuspicions,
1855
1941
  coreScan,
1856
1942
  });
@@ -1942,14 +2028,6 @@ export class UserGroundHost {
1942
2028
  if (first.resourceId !== resourceId) {
1943
2029
  throw new Error("local snapshot publication belongs to a different user-ground resource");
1944
2030
  }
1945
- this.cloudState = {
1946
- resourceId: first.resourceId,
1947
- authorityEpoch: first.authorityEpoch,
1948
- headVersion: first.baseVersion,
1949
- checksum: "",
1950
- deliveryWatermark: first.deliveryWatermark,
1951
- records: loadSnapshotPublication(first).snapshotRecords,
1952
- };
1953
2031
  try {
1954
2032
  await this.publishPendingSnapshots(identity);
1955
2033
  }
@@ -1981,6 +2059,12 @@ export class UserGroundHost {
1981
2059
  throw error;
1982
2060
  }
1983
2061
  await this.recordRail?.flush();
2062
+ // A replayed publication leaves no graph behind — it only ever journaled
2063
+ // the roots it rebuilt — so the machine re-reads the authority's head
2064
+ // before deriving the next boundary on top of it.
2065
+ if (!this.cloudState) {
2066
+ await this.cloudPort(identity).lookup(privateEntityResourceId(identity.userId, sha256Hex));
2067
+ }
1984
2068
  const state = this.cloudState;
1985
2069
  if (!state)
1986
2070
  throw new Error("cloud state is unavailable for graph publication");
@@ -1999,24 +2083,43 @@ export class UserGroundHost {
1999
2083
  const state = this.cloudState;
2000
2084
  if (!state)
2001
2085
  throw new Error("cloud state is unavailable for graph publication");
2002
- const officialRecords = recordsAtDeliveryPrefix(state.records, readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark));
2003
- const localRecords = recordsRetainingMissingEvidence(readGroundRowsForRoots(this.databasePath(identity), "entities", publicationRootIds)
2086
+ // Graph preparation is the leg between a committed local ground and the
2087
+ // first wire frame. It is pure CPU and SQLite, so it is measured by its
2088
+ // own parts: whoever reads this ledger must see which part is slow.
2089
+ const preparationStarted = performance.now();
2090
+ const leg = {};
2091
+ const timed = (name, work) => {
2092
+ const started = performance.now();
2093
+ const value = work();
2094
+ leg[`${name}Ms`] = performance.now() - started;
2095
+ return value;
2096
+ };
2097
+ if (trace?.journey === "register") {
2098
+ this.stage({
2099
+ primitive: "register",
2100
+ stage: "graph-preparation",
2101
+ status: "started",
2102
+ ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2103
+ });
2104
+ }
2105
+ const localRecords = timed("groundRows", () => recordsRetainingMissingEvidence(readGroundRowsForRoots(this.databasePath(identity), "entities", publicationRootIds)
2004
2106
  .map(portableRecord), readGroundRowsForRoots(this.databasePath(identity), "detection_notebook", publicationRootIds)
2005
- .map(portableRecord));
2006
- const merged = mergeMaterializedRoots(officialRecords, localRecords);
2007
- const traveling = travelingRecords(merged, sha256Hex);
2107
+ .map(portableRecord)));
2108
+ const accepted = timed("officialRows", () => readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark));
2109
+ const rebuilt = timed("merge", () => publicationRoots(state.graph, localRecords, accepted));
2110
+ const projected = timed("project", () => new Map([...rebuilt].map(([rootId, records]) => [rootId, records.length === 0 ? records : travelingRecords(records, sha256Hex)])));
2111
+ const traveling = [...projected.values()].flat();
2008
2112
  const impossibleGitLeaf = traveling.find((record) => (record.type === "file.text" || record.type === "file.binary" || record.type === "link")
2009
2113
  && record.payloadVersion?.startsWith("git:"));
2010
2114
  if (impossibleGitLeaf) {
2011
- const parent = merged.find((record) => record.uuid === impossibleGitLeaf.parentUUID);
2115
+ const parent = traveling.find((record) => record.uuid === impossibleGitLeaf.parentUUID);
2012
2116
  throw new Error(`repository-owned leaf escaped its rail: ${impossibleGitLeaf.name} parent=${parent?.type ?? "missing"}`);
2013
2117
  }
2014
- const snapshot = snapshotFromRecords(traveling);
2015
- // ONE canonical serialization: these exact bytes are the journal blob
2016
- // and the checksum's input; they are never re-derived or re-parsed on
2017
- // the in-process path.
2018
- const snapshotBlob = Buffer.from(stableJson(snapshot), "utf8");
2019
- const checksum = sha256Hex(snapshotBlob);
2118
+ // ONE canonical serialization per rebuilt root: the page bytes are the
2119
+ // journal's payload, the upload's payload, and the head's identities.
2120
+ // Untouched roots are not serialized, hashed, or journaled at all.
2121
+ const publication = timed("pages", () => state.graph.rebuild(projected, sha256Hex));
2122
+ const checksum = publication.checksum;
2020
2123
  const localIds = new Set(localRecords.map((record) => record.uuid));
2021
2124
  // Graph membership is never proof of sealed cargo: a record in the cloud
2022
2125
  // graph may name content sealed only under a prior content contract,
@@ -2025,8 +2128,14 @@ export class UserGroundHost {
2025
2128
  // sealed-first upload asks that exact question itself and moves only
2026
2129
  // what is unsealed, so Register always re-establishes cargo closure
2027
2130
  // before returning, even when the graph itself is unchanged.
2028
- const cargoRecords = snapshot.records.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
2131
+ const cargoRecords = traveling.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
2029
2132
  if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
2133
+ this.finishPreparation(trace, preparationStarted, leg, {
2134
+ rebuiltRoots: rebuilt.size,
2135
+ records: traveling.length,
2136
+ cargoRecords: cargoRecords.length,
2137
+ graphUnchanged: true,
2138
+ });
2030
2139
  // The graph is already committed; only cargo can be missing. Repair
2031
2140
  // uploads need no journal row — a crash simply re-runs the same
2032
2141
  // repair on the next Register.
@@ -2040,18 +2149,12 @@ export class UserGroundHost {
2040
2149
  }
2041
2150
  return;
2042
2151
  }
2043
- const replacementRootIds = localRecords
2044
- .filter((record) => record.parentUUID === null)
2045
- .map((record) => record.uuid);
2046
- const replacementRoots = new Set(replacementRootIds);
2047
- const rootOf = recordRootResolver(snapshot.records);
2048
- const replacementRecords = snapshot.records.filter((record) => {
2049
- const root = rootOf(record);
2050
- return root !== null && replacementRoots.has(root);
2051
- });
2052
- const replacement = rootReplacementFromRecords(replacementRootIds, replacementRecords);
2053
- // The operation's canonical bytes: staged, hashed, and submitted as-is.
2054
- const operationBlob = Buffer.from(stableJson(replacement), "utf8");
2152
+ // The operation's canonical bytes: journaled and submitted as-is.
2153
+ const operationBlob = timed("operation", () => Buffer.from(stableJson(publication.operation), "utf8"));
2154
+ // Only pages the committed head does not already name still have to be
2155
+ // sealed; a rebuilt root that came out byte-identical costs nothing.
2156
+ const committed = graphPageIdentities(state.graph.head);
2157
+ const fresh = publication.pages.filter((page) => !committed.has(page.ref.sha256));
2055
2158
  const entry = {
2056
2159
  mutationId: randomUUID(),
2057
2160
  resourceId: state.resourceId,
@@ -2059,24 +2162,54 @@ export class UserGroundHost {
2059
2162
  baseVersion: state.headVersion,
2060
2163
  snapshotChecksum: checksum,
2061
2164
  deliveryWatermark,
2062
- operationKind: replacement.kind,
2063
- replacementRootIds: replacement.rootIds,
2165
+ operationKind: publication.operation.kind,
2166
+ replacementRootIds: publication.operation.replacedRootIds,
2064
2167
  uploadUuids: cargoRecords.map((record) => record.uuid),
2065
- snapshotBlob,
2066
2168
  operationBlob,
2169
+ pages: fresh.map((page) => ({
2170
+ sha256: page.ref.sha256,
2171
+ bytes: Buffer.from(page.bytes.buffer, page.bytes.byteOffset, page.bytes.byteLength),
2172
+ })),
2067
2173
  status: "pending",
2068
2174
  submitted: false,
2069
2175
  };
2070
- supersedeSnapshotJournal(this.databasePath(identity), entry);
2071
- // Publish the freshly journaled intent directly: the records are already
2072
- // in memory and the operation bytes already exist — nothing is re-read,
2073
- // re-parsed, or re-serialized between prepare and the wire.
2176
+ timed("journal", () => supersedeSnapshotJournal(this.databasePath(identity), entry));
2177
+ this.finishPreparation(trace, preparationStarted, leg, {
2178
+ rebuiltRoots: rebuilt.size,
2179
+ records: traveling.length,
2180
+ pages: publication.pages.length,
2181
+ freshPages: fresh.length,
2182
+ pageBytes: fresh.reduce((total, page) => total + page.ref.bytes, 0),
2183
+ operationBytes: operationBlob.byteLength,
2184
+ cargoRecords: cargoRecords.length,
2185
+ graphUnchanged: false,
2186
+ });
2187
+ // Publish the freshly journaled intent directly: the pages and the
2188
+ // operation bytes already exist — nothing is re-read, re-parsed, or
2189
+ // re-serialized between prepare and the wire.
2074
2190
  await this.publishLoadedSnapshot(identity, {
2075
2191
  ...entry,
2076
- snapshotRecords: snapshot.records,
2077
2192
  uploadRecords: cargoRecords,
2193
+ graph: publication.graph,
2078
2194
  }, trace, manifestHints);
2079
2195
  }
2196
+ /** Close the graph-preparation stage with every leg it spent, so a slow
2197
+ * preparation names the slow part instead of one opaque number. */
2198
+ finishPreparation(trace, startedAt, legs, counters) {
2199
+ if (trace?.journey !== "register")
2200
+ return;
2201
+ this.stage({
2202
+ primitive: "register",
2203
+ stage: "graph-preparation",
2204
+ status: "completed",
2205
+ durationMs: performance.now() - startedAt,
2206
+ ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2207
+ measurements: {
2208
+ ...counters,
2209
+ ...Object.fromEntries(Object.entries(legs).map(([name, value]) => [name, Math.round(value)])),
2210
+ },
2211
+ });
2212
+ }
2080
2213
  /** Which of these heads does the authority hold sealed under the current
2081
2214
  * content contract? Batched so a whole workspace costs a handful of
2082
2215
  * frames; the answer is the only permission to skip an upload. */
@@ -2110,8 +2243,61 @@ export class UserGroundHost {
2110
2243
  await this.publishLoadedSnapshot(identity, loadSnapshotPublication(pending), trace, manifestHints);
2111
2244
  }
2112
2245
  }
2246
+ /** Seal this publication's graph pages. Pages are immutable, content
2247
+ * addressed, and already in memory, so the authority's seal inventory alone
2248
+ * decides what still has to move and whole pages ride cargo waves. The head
2249
+ * may not commit until every page it newly names is sealed. */
2250
+ async uploadGraphPages(resourceId, pages, trace) {
2251
+ if (pages.length === 0)
2252
+ return;
2253
+ const started = performance.now();
2254
+ const sealed = await this.sealedContentHashes(resourceId, pages.map((page) => page.sha256));
2255
+ const unsealed = pages
2256
+ .filter((page) => !sealed.has(page.sha256))
2257
+ .map((page) => ({ page, bytes: page.bytes.byteLength }));
2258
+ let uploadedBytes = 0;
2259
+ await boundedForEach([...planCargoWaves(unsealed)], CARGO_UPLOAD_CONCURRENCY, async (wave) => {
2260
+ const payload = Buffer.concat(wave.items.map((item) => item.page.bytes));
2261
+ const encoded = await encodeContentWireBytes(payload);
2262
+ const frame = await retryContent(() => this.wire.requestBinary({
2263
+ type: "private.entity-content.put-cargo",
2264
+ resource_id: resourceId,
2265
+ // A page is a single-chunk artifact: its identity, its only chunk,
2266
+ // and its bytes are one and the same sha.
2267
+ artifacts: wave.items.map((item) => ({
2268
+ content_hash: item.page.sha256,
2269
+ bytes: item.bytes,
2270
+ chunks: [{ sha256: item.page.sha256, bytes: item.bytes }],
2271
+ })),
2272
+ bytes: payload.byteLength,
2273
+ wire_bytes: encoded.bytes.byteLength,
2274
+ ...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
2275
+ }, encoded.bytes, ["private.entity-content.stored-cargo"]));
2276
+ const stored = new Set(Array.isArray(frame.sealed) ? frame.sealed.map(String) : []);
2277
+ for (const item of wave.items) {
2278
+ if (!stored.has(item.page.sha256)) {
2279
+ throw new Error(`content authority did not seal graph page ${item.page.sha256}`);
2280
+ }
2281
+ }
2282
+ uploadedBytes += encoded.bytes.byteLength;
2283
+ });
2284
+ this.stage({
2285
+ primitive: "upload",
2286
+ stage: "graph-pages",
2287
+ status: "completed",
2288
+ ...(trace?.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2289
+ durationMs: performance.now() - started,
2290
+ measurements: {
2291
+ journey: trace?.journey ?? "bootstrap",
2292
+ pages: pages.length,
2293
+ uploadedPages: unsealed.length,
2294
+ uploadedBytes,
2295
+ },
2296
+ });
2297
+ }
2113
2298
  async publishLoadedSnapshot(identity, publication, trace, manifestHints) {
2114
2299
  try {
2300
+ await this.uploadGraphPages(publication.resourceId, publication.pages, trace);
2115
2301
  await this.uploadSnapshotContent(identity, publication.resourceId, publication.uploadRecords, trace, manifestHints);
2116
2302
  }
2117
2303
  catch (error) {
@@ -2136,22 +2322,36 @@ export class UserGroundHost {
2136
2322
  ...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
2137
2323
  });
2138
2324
  }
2325
+ const legMs = {};
2326
+ const local = (name, work) => {
2327
+ const at = performance.now();
2328
+ const value = work();
2329
+ legMs[`${name}Ms`] = performance.now() - at;
2330
+ return value;
2331
+ };
2139
2332
  if (!publication.submitted) {
2140
- markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId);
2333
+ local("markSubmitted", () => markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId));
2141
2334
  }
2142
- const legMs = {};
2143
2335
  try {
2144
- frame = await submitMutationOperation(this.wire, {
2145
- resourceId: publication.resourceId,
2146
- authorityEpoch: publication.authorityEpoch,
2147
- mutationId: publication.mutationId,
2148
- deviceId: identity.deviceId,
2149
- baseVersion: publication.baseVersion,
2150
- contract: ENTITY_CLOUD_CONTRACT,
2151
- schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
2152
- operationKind: publication.operationKind,
2153
- deliveryWatermark: publication.deliveryWatermark,
2154
- }, publication.operationBlob, (leg, durationMs) => { legMs[`${leg}Ms`] = durationMs; });
2336
+ // A head commit names pages instead of carrying records, so it is small
2337
+ // by construction and always travels inline in one frame.
2338
+ const submitStarted = performance.now();
2339
+ frame = await this.wire.request({
2340
+ type: "shared.mutation.submit",
2341
+ envelope: {
2342
+ resourceId: publication.resourceId,
2343
+ authorityEpoch: publication.authorityEpoch,
2344
+ mutationId: publication.mutationId,
2345
+ deviceId: identity.deviceId,
2346
+ baseVersion: publication.baseVersion,
2347
+ contract: ENTITY_CLOUD_CONTRACT,
2348
+ schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
2349
+ operationKind: publication.operationKind,
2350
+ deliveryWatermark: publication.deliveryWatermark,
2351
+ operation: JSON.parse(publication.operationBlob.toString("utf8")),
2352
+ },
2353
+ }, ["shared.mutation.ack"]);
2354
+ legMs.submitMs = performance.now() - submitStarted;
2155
2355
  }
2156
2356
  catch (error) {
2157
2357
  if (trace?.journey === "register") {
@@ -2172,7 +2372,30 @@ export class UserGroundHost {
2172
2372
  }
2173
2373
  throw pipelineStageError("cloud graph commit", error);
2174
2374
  }
2375
+ // A replay carries no graph: clearing the state makes the next reader
2376
+ // fetch the authority's own head instead of a graph missing every root
2377
+ // this publication did not rebuild.
2378
+ this.cloudState = publication.graph === null ? null : {
2379
+ resourceId: publication.resourceId,
2380
+ authorityEpoch: Number(frame.authority_epoch),
2381
+ headVersion: Number(frame.version),
2382
+ checksum: publication.snapshotChecksum,
2383
+ deliveryWatermark: Number(frame.delivery_watermark),
2384
+ graph: publication.graph,
2385
+ };
2386
+ local("clearJournal", () => deleteJournalEntry(this.databasePath(identity), publication.mutationId));
2175
2387
  if (trace?.journey === "register") {
2388
+ // The authority reports where its side of the commit went; carrying
2389
+ // its legs here gives one ledger line the whole story: local journal
2390
+ // marks, wire time, gateway assemble/validate, and the database RPC.
2391
+ for (const [wire, leg] of [
2392
+ ["verify_ms", "serverVerifyMs"],
2393
+ ["validate_ms", "serverValidateMs"],
2394
+ ["database_ms", "serverDatabaseMs"],
2395
+ ]) {
2396
+ if (typeof frame[wire] === "number")
2397
+ legMs[leg] = frame[wire];
2398
+ }
2176
2399
  this.stage({
2177
2400
  primitive: "register",
2178
2401
  stage: "cloud-graph-commit",
@@ -2182,19 +2405,11 @@ export class UserGroundHost {
2182
2405
  measurements: {
2183
2406
  version: Number(frame.version),
2184
2407
  operationBytes: publication.operationBlob.byteLength,
2185
- ...legMs,
2408
+ ...Object.fromEntries(Object.entries(legMs)
2409
+ .map(([name, value]) => [name, Math.round(value)])),
2186
2410
  },
2187
2411
  });
2188
2412
  }
2189
- this.cloudState = {
2190
- resourceId: publication.resourceId,
2191
- authorityEpoch: Number(frame.authority_epoch),
2192
- headVersion: Number(frame.version),
2193
- checksum: publication.snapshotChecksum,
2194
- deliveryWatermark: Number(frame.delivery_watermark),
2195
- records: publication.snapshotRecords,
2196
- };
2197
- deleteJournalEntry(this.databasePath(identity), publication.mutationId);
2198
2413
  }
2199
2414
  }
2200
2415
  const sha256Hex = (input) => createHash("sha256").update(input).digest("hex");
@@ -2213,6 +2428,12 @@ function contentCacheFile(cacheDir, contentHash) {
2213
2428
  function manifestCacheFile(cacheDir, contentHash) {
2214
2429
  return join(cacheDir, "manifests", `${contentHash}.json`);
2215
2430
  }
2431
+ /** Graph pages are immutable and content-addressed, so they cache exactly like
2432
+ * artifacts do — in their own directory, because they are the registry's own
2433
+ * bytes rather than any entity's cargo. */
2434
+ function graphPageCacheDir(cacheDir) {
2435
+ return join(cacheDir, "graph-pages");
2436
+ }
2216
2437
  function readCachedManifest(cacheDir, contentHash) {
2217
2438
  try {
2218
2439
  const checked = checkContentManifest(JSON.parse(readFileSync(manifestCacheFile(cacheDir, contentHash), "utf8")), contentHash);
@@ -2286,78 +2507,110 @@ function assertHealthyWatch(evidence, rootId) {
2286
2507
  * file over a dozen times; the file itself is never deleted by the host, so
2287
2508
  * every later open needs only the connection and its pragmas. */
2288
2509
  const migratedDatabases = new Set();
2510
+ /** A publication row holds only what the authority must be told: the small
2511
+ * meta facts and the head-commit operation naming this graph's pages. The
2512
+ * pages themselves are write-once rows in their own table, because a status
2513
+ * transition rewrites its whole row — overflow pages included — through the
2514
+ * WAL at synchronous=FULL, and page bytes must never ride that rewrite. */
2289
2515
  const SNAPSHOT_PUBLICATIONS_SCHEMA = `
2290
2516
  CREATE TABLE IF NOT EXISTS snapshot_publications (
2291
2517
  sequence INTEGER PRIMARY KEY AUTOINCREMENT,
2292
2518
  mutation_id TEXT NOT NULL UNIQUE,
2293
2519
  resource_id TEXT NOT NULL,
2294
2520
  meta_json TEXT NOT NULL,
2295
- snapshot_blob BLOB NOT NULL,
2296
2521
  operation_blob BLOB NOT NULL,
2297
2522
  created_at INTEGER NOT NULL,
2298
2523
  status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
2299
2524
  submitted_at INTEGER
2525
+ );
2526
+ CREATE TABLE IF NOT EXISTS snapshot_publication_pages (
2527
+ mutation_id TEXT NOT NULL,
2528
+ page_index INTEGER NOT NULL,
2529
+ sha256 TEXT NOT NULL,
2530
+ bytes BLOB NOT NULL,
2531
+ PRIMARY KEY (mutation_id, page_index)
2300
2532
  );`;
2301
- /** Converts an old-shape publication journal (one nested publication_json
2302
- * text per row) into the blob journal, atomically, preserving every row's
2303
- * meaning. A pending row may be a submitted mutation whose outcome exists
2304
- * only at the authority — dropping it would violate the retry law that
2305
- * recovers a lost acknowledgement — so conversion is structural: the nested
2306
- * snapshot text becomes the snapshot blob verbatim, the stored replacement
2307
- * becomes the operation blob, and semantic judgment stays where it always
2308
- * was, with the reader (an unlawful row still blocks registration rather
2309
- * than being silently replaced).
2533
+ function legacyJournalShape(columns) {
2534
+ if (columns.size === 0)
2535
+ return null;
2536
+ if (columns.has("publication_json"))
2537
+ return "nested-text";
2538
+ if (columns.has("snapshot_blob"))
2539
+ return "row-blobs";
2540
+ // The paged journal and the side-table one both keep only small columns
2541
+ // here; only the paged one names the operation on the row itself.
2542
+ return columns.has("operation_blob") ? null : "split-blobs";
2543
+ }
2544
+ /** Converts a pre-paged publication journal into the paged one, atomically,
2545
+ * preserving every row's meaning. A pending row may be a submitted mutation
2546
+ * whose outcome exists only at the authority — dropping it would violate the
2547
+ * retry law that recovers a lost acknowledgement — so conversion is
2548
+ * structural: each row keeps its identity, its meta facts, and the exact
2549
+ * operation bytes it would submit, and semantic judgment stays where it
2550
+ * always was, with the reader (an unlawful row still blocks registration
2551
+ * rather than being silently replaced). A converted row rebuilt whole roots
2552
+ * rather than pages, so it carries no page rows; its replayed upload comes
2553
+ * from the records inside its own operation.
2310
2554
  *
2311
2555
  * The transform runs entirely inside SQLite. A production journal row is a
2312
2556
  * register-scale graph serialized three times over — the exact text whose
2313
2557
  * repeated JavaScript materialization blew the 512 MB heap — so no part of
2314
- * it may ever become a JavaScript value here: json_extract lifts the nested
2315
- * snapshot text straight into its blob, and the meta and operation rows are
2316
- * json_object over subtrees, all in engine memory outside the V8 heap.
2558
+ * it may ever become a JavaScript value here: json_extract lifts nested text
2559
+ * straight into its blob and meta rows are json_object over subtrees, all in
2560
+ * engine memory outside the V8 heap. The snapshot blob is simply not carried:
2561
+ * the authority stores the head now, and a replay re-reads it.
2317
2562
  *
2318
2563
  * Tables older than the three-state journal carry no status or submitted_at;
2319
2564
  * their rows were written before the pre-submit durability mark existed, so
2320
2565
  * the only safe reading is "possibly submitted": keep them pending and
2321
2566
  * marked submitted, which replays the same mutation id — always idempotent —
2322
- * and never supersedes. A row whose publication_json does not parse fails
2323
- * the open loudly with the journal intact; nothing is ever deleted on a
2324
- * conversion failure. */
2325
- function convertLegacySnapshotJournal(database, columns) {
2567
+ * and never supersedes. A row whose stored operation does not resolve fails
2568
+ * the open loudly against the NOT NULL law with the journal intact; nothing
2569
+ * is ever deleted on a conversion failure. */
2570
+ function convertLegacySnapshotJournal(database, shape, columns) {
2571
+ const metaJson = shape === "nested-text"
2572
+ ? `json_object(
2573
+ 'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
2574
+ 'baseVersion', json_extract(publication_json, '$.baseVersion'),
2575
+ 'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
2576
+ 'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
2577
+ 'operationKind', 'entity.roots.replace',
2578
+ 'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2579
+ 'uploadUuids', coalesce((
2580
+ SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
2581
+ FROM json_each(publication_json, '$.uploadRecords') AS uploaded
2582
+ ), json_array())
2583
+ )`
2584
+ : "meta_json";
2585
+ const operationBlob = {
2586
+ "nested-text": `CAST(json_object(
2587
+ 'kind', 'entity.roots.replace',
2588
+ 'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2589
+ 'records', json(json_extract(publication_json, '$.replacementRecords'))
2590
+ ) AS BLOB)`,
2591
+ "row-blobs": "operation_blob",
2592
+ "split-blobs": `(SELECT blobs.operation_blob FROM snapshot_publication_blobs AS blobs
2593
+ WHERE blobs.mutation_id = legacy.mutation_id)`,
2594
+ }[shape];
2326
2595
  database.transaction(() => {
2327
2596
  database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
2328
2597
  database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
2329
2598
  database.exec(`
2330
2599
  INSERT INTO snapshot_publications(
2331
- mutation_id, resource_id, meta_json, snapshot_blob, operation_blob,
2332
- created_at, status, submitted_at
2600
+ mutation_id, resource_id, meta_json, operation_blob, created_at, status, submitted_at
2333
2601
  )
2334
2602
  SELECT
2335
2603
  mutation_id,
2336
2604
  resource_id,
2337
- json_object(
2338
- 'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
2339
- 'baseVersion', json_extract(publication_json, '$.baseVersion'),
2340
- 'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
2341
- 'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
2342
- 'operationKind', 'entity.roots.replace',
2343
- 'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2344
- 'uploadUuids', coalesce((
2345
- SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
2346
- FROM json_each(publication_json, '$.uploadRecords') AS uploaded
2347
- ), json_array())
2348
- ),
2349
- CAST(json_extract(publication_json, '$.snapshotJson') AS BLOB),
2350
- CAST(json_object(
2351
- 'kind', 'entity.roots.replace',
2352
- 'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2353
- 'records', json(json_extract(publication_json, '$.replacementRecords'))
2354
- ) AS BLOB),
2605
+ ${metaJson},
2606
+ ${operationBlob},
2355
2607
  created_at,
2356
2608
  ${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
2357
2609
  ${columns.has("submitted_at") ? "submitted_at" : "created_at"}
2358
- FROM snapshot_publications_legacy ORDER BY sequence
2610
+ FROM snapshot_publications_legacy AS legacy ORDER BY sequence
2359
2611
  `);
2360
2612
  database.exec("DROP TABLE snapshot_publications_legacy");
2613
+ database.exec("DROP TABLE IF EXISTS snapshot_publication_blobs");
2361
2614
  })();
2362
2615
  }
2363
2616
  /** The single opener and schema-migration entry for a user ground database.
@@ -2376,9 +2629,9 @@ export function initializeDatabase(file) {
2376
2629
  // how this file arrived at this process. One PRAGMA against the in-memory
2377
2630
  // catalog costs nothing next to the connection open itself.
2378
2631
  const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
2379
- if (publicationColumns.has("publication_json")) {
2380
- convertLegacySnapshotJournal(database, publicationColumns);
2381
- }
2632
+ const legacyShape = legacyJournalShape(publicationColumns);
2633
+ if (legacyShape)
2634
+ convertLegacySnapshotJournal(database, legacyShape, publicationColumns);
2382
2635
  if (migratedDatabases.has(file))
2383
2636
  return database;
2384
2637
  database.exec(`
@@ -2954,18 +3207,36 @@ function readSnapshotPublications(file) {
2954
3207
  return [];
2955
3208
  const database = initializeDatabase(file);
2956
3209
  try {
3210
+ const pages = database.prepare(`
3211
+ SELECT mutation_id AS mutationId, sha256, bytes
3212
+ FROM snapshot_publication_pages ORDER BY mutation_id, page_index
3213
+ `).all();
3214
+ const pagesByMutation = new Map();
3215
+ for (const page of pages) {
3216
+ const carried = pagesByMutation.get(page.mutationId) ?? [];
3217
+ carried.push({ sha256: page.sha256, bytes: page.bytes });
3218
+ pagesByMutation.set(page.mutationId, carried);
3219
+ }
2957
3220
  return database.prepare(`
2958
- SELECT mutation_id AS mutationId, resource_id AS resourceId,
2959
- meta_json AS metaJson, snapshot_blob AS snapshotBlob,
2960
- operation_blob AS operationBlob, status,
3221
+ SELECT mutation_id AS mutationId,
3222
+ resource_id AS resourceId,
3223
+ meta_json AS metaJson,
3224
+ operation_blob AS operationBlob,
3225
+ status,
2961
3226
  submitted_at AS submittedAt
2962
3227
  FROM snapshot_publications ORDER BY sequence
2963
3228
  `).all().map((row) => {
2964
3229
  const meta = JSON.parse(row.metaJson);
2965
- // No lawful publication carries zero roots — rootReplacementFromRecords
2966
- // refuses them at build time, so a rootless row here is damage.
3230
+ // No lawful publication carries zero roots — a head commit that replaces
3231
+ // nothing would prove nothing — so a rootless row here is damage, never
3232
+ // a skippable row: silently dropping it would let a new publication
3233
+ // supersede a submitted mutation whose outcome exists only at the
3234
+ // authority. `entity.roots.replace` appears only on rows converted from
3235
+ // a pre-paged journal, which no longer produce but must still replay.
2967
3236
  if (!Array.isArray(meta.replacementRootIds) || meta.replacementRootIds.length === 0
2968
- || !Array.isArray(meta.uploadUuids) || meta.operationKind !== "entity.roots.replace") {
3237
+ || !Array.isArray(meta.uploadUuids)
3238
+ || (meta.operationKind !== GRAPH_HEAD_OPERATION_KIND
3239
+ && meta.operationKind !== "entity.roots.replace")) {
2969
3240
  throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
2970
3241
  }
2971
3242
  const deliveryWatermark = Number(meta.deliveryWatermark);
@@ -2979,11 +3250,11 @@ function readSnapshotPublications(file) {
2979
3250
  baseVersion: Number(meta.baseVersion),
2980
3251
  deliveryWatermark,
2981
3252
  snapshotChecksum: String(meta.snapshotChecksum),
2982
- operationKind: meta.operationKind,
3253
+ operationKind: String(meta.operationKind),
2983
3254
  replacementRootIds: meta.replacementRootIds.map(String),
2984
3255
  uploadUuids: meta.uploadUuids.map(String),
2985
- snapshotBlob: row.snapshotBlob,
2986
3256
  operationBlob: row.operationBlob,
3257
+ pages: pagesByMutation.get(row.mutationId) ?? [],
2987
3258
  status: row.status === "rejected" ? "rejected" : "pending",
2988
3259
  submitted: row.submittedAt !== null,
2989
3260
  };
@@ -3150,12 +3421,17 @@ function supersedeSnapshotJournal(file, entry) {
3150
3421
  if (unresolved) {
3151
3422
  throw new Error("a submitted publication with an unknown outcome must resolve before supersession");
3152
3423
  }
3424
+ database.prepare(`
3425
+ DELETE FROM snapshot_publication_pages WHERE mutation_id IN (
3426
+ SELECT mutation_id FROM snapshot_publications WHERE resource_id = ?
3427
+ )
3428
+ `).run(entry.resourceId);
3153
3429
  database.prepare("DELETE FROM snapshot_publications WHERE resource_id = ?")
3154
3430
  .run(entry.resourceId);
3155
3431
  database.prepare(`
3156
3432
  INSERT INTO snapshot_publications(
3157
- mutation_id, resource_id, meta_json, snapshot_blob, operation_blob, created_at
3158
- ) VALUES (?, ?, ?, ?, ?, ?)
3433
+ mutation_id, resource_id, meta_json, operation_blob, created_at
3434
+ ) VALUES (?, ?, ?, ?, ?)
3159
3435
  `).run(entry.mutationId, entry.resourceId, stableJson({
3160
3436
  authorityEpoch: entry.authorityEpoch,
3161
3437
  baseVersion: entry.baseVersion,
@@ -3164,7 +3440,12 @@ function supersedeSnapshotJournal(file, entry) {
3164
3440
  operationKind: entry.operationKind,
3165
3441
  replacementRootIds: entry.replacementRootIds,
3166
3442
  uploadUuids: entry.uploadUuids,
3167
- }), entry.snapshotBlob, entry.operationBlob, Date.now());
3443
+ }), entry.operationBlob, Date.now());
3444
+ const insertPage = database.prepare(`
3445
+ INSERT INTO snapshot_publication_pages(mutation_id, page_index, sha256, bytes)
3446
+ VALUES (?, ?, ?, ?)
3447
+ `);
3448
+ entry.pages.forEach((page, index) => insertPage.run(entry.mutationId, index, page.sha256, page.bytes));
3168
3449
  })();
3169
3450
  }
3170
3451
  finally {
@@ -3234,7 +3515,10 @@ function captureChurn(error) {
3234
3515
  function deleteJournalEntry(file, mutationId) {
3235
3516
  const database = initializeDatabase(file);
3236
3517
  try {
3237
- database.prepare("DELETE FROM snapshot_publications WHERE mutation_id = ?").run(mutationId);
3518
+ database.transaction(() => {
3519
+ database.prepare("DELETE FROM snapshot_publication_pages WHERE mutation_id = ?").run(mutationId);
3520
+ database.prepare("DELETE FROM snapshot_publications WHERE mutation_id = ?").run(mutationId);
3521
+ })();
3238
3522
  }
3239
3523
  finally {
3240
3524
  database.close();
@@ -3614,7 +3898,7 @@ function materializedWorkspaceBindings(bindingDir) {
3614
3898
  return bindings.sort((left, right) => left.workspaceId.localeCompare(right.workspaceId));
3615
3899
  }
3616
3900
  async function reconcileGround(input) {
3617
- const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, } = input;
3901
+ const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, measuring = false, } = input;
3618
3902
  const resourceId = privateEntityResourceId(identity.userId, sha256Hex);
3619
3903
  const bindingDir = workspaceBindingDir(userRoot, identity.deviceId);
3620
3904
  const boundRootEntries = materializedWorkspaceBindings(bindingDir);
@@ -3709,9 +3993,24 @@ async function reconcileGround(input) {
3709
3993
  contentReads: 0,
3710
3994
  gitInspections: 0,
3711
3995
  repositoryCaptures: 0,
3996
+ walkMs: 0,
3997
+ captureMs: 0,
3998
+ captureBusyMs: 0,
3999
+ captureBytes: 0,
4000
+ gitMs: 0,
4001
+ repositoryIdentityMs: 0,
4002
+ repositoryCaptureMs: 0,
4003
+ repositorySealMs: 0,
4004
+ reconcileMs: 0,
4005
+ persistMs: 0,
3712
4006
  };
3713
4007
  const started = performance.now();
3714
- const result = await reconcileRoot({ ...parameters, evidence: counters, materializedUUIDs });
4008
+ const result = await reconcileRoot({
4009
+ ...parameters,
4010
+ evidence: counters,
4011
+ measureEvidence: measuring,
4012
+ materializedUUIDs,
4013
+ });
3715
4014
  if (result.transferFailure !== null) {
3716
4015
  transferFailures.push({ rootId: parameters.rootUUID, reason: result.transferFailure });
3717
4016
  }
@@ -3749,7 +4048,14 @@ async function reconcileGround(input) {
3749
4048
  durationMs: performance.now() - started,
3750
4049
  plannedRows: proposals.length,
3751
4050
  records: proposals.length,
3752
- recordBytes: proposals.reduce((bytes, proposal) => bytes + encodedRecordBytes(parameters.resourceId, proposal), 0),
4051
+ // Encoding every proposal to learn its wire size costs one record
4052
+ // construction per entity — real time on the critical path of a cold
4053
+ // Register. It is evidence, so it is paid only when read.
4054
+ ...(measuring
4055
+ ? {
4056
+ recordBytes: proposals.reduce((bytes, proposal) => bytes + encodedRecordBytes(parameters.resourceId, proposal), 0),
4057
+ }
4058
+ : {}),
3753
4059
  });
3754
4060
  // Each root commits its local projection before Detect yields. Native
3755
4061
  // callbacks can therefore preserve new suspicion between large roots.
@@ -3848,7 +4154,7 @@ async function reconcileGround(input) {
3848
4154
  };
3849
4155
  }
3850
4156
  async function reconcileRoot(input) {
3851
- const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, plansSendRecords, largeManifests, } = input;
4157
+ const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, } = input;
3852
4158
  // Prior rows reach this scan from every binding this root encloses. A row's
3853
4159
  // stored relativePath is relative to the root that last committed it, which
3854
4160
  // is a different frame when an enclosing boundary was just registered
@@ -4021,6 +4327,33 @@ async function reconcileRoot(input) {
4021
4327
  // its own entry in place; every task is awaited before entries are read.
4022
4328
  const captureTasks = [];
4023
4329
  const captureGate = concurrencyGate(WALK_CAPTURE_CONCURRENCY);
4330
+ // The scan's three costs are metadata traversal, content capture, and Git
4331
+ // inspection. They interleave, so each is accumulated where it is spent
4332
+ // rather than inferred from the total afterwards.
4333
+ let captureOpenedAt = 0;
4334
+ let captureClosedAt = 0;
4335
+ const spent = (work, into) => {
4336
+ if (!measureEvidence || !evidence)
4337
+ return work();
4338
+ const at = performance.now();
4339
+ try {
4340
+ return work();
4341
+ }
4342
+ finally {
4343
+ evidence[into] += performance.now() - at;
4344
+ }
4345
+ };
4346
+ const spentAsync = async (work, into) => {
4347
+ if (!measureEvidence || !evidence)
4348
+ return work();
4349
+ const at = performance.now();
4350
+ try {
4351
+ return await work();
4352
+ }
4353
+ finally {
4354
+ evidence[into] += performance.now() - at;
4355
+ }
4356
+ };
4024
4357
  const visit = async (directory, parentUUID, parentType, base = "", activeRepository = parentType === "repo.git"
4025
4358
  ? {
4026
4359
  root: directory,
@@ -4028,8 +4361,8 @@ async function reconcileRoot(input) {
4028
4361
  evidence: directory === rootPath ? rootRepositoryEvidence : null,
4029
4362
  }
4030
4363
  : null, forceObserve = rootRailChanged) => {
4031
- const children = readdirSync(directory, { withFileTypes: true })
4032
- .sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
4364
+ const children = spent(() => readdirSync(directory, { withFileTypes: true })
4365
+ .sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0), "walkMs");
4033
4366
  for (const child of children) {
4034
4367
  // Every materialization boundary reserves its root .amalgm/ directory
4035
4368
  // for machine intents, staging, and recovery. This is structural host
@@ -4045,7 +4378,7 @@ async function reconcileRoot(input) {
4045
4378
  if (!policy(relativePath))
4046
4379
  continue;
4047
4380
  const absolutePath = join(directory, child.name);
4048
- const stats = lstatSync(absolutePath);
4381
+ const stats = spent(() => lstatSync(absolutePath), "walkMs");
4049
4382
  evidence && (evidence.entries += 1);
4050
4383
  // The walk is synchronous metadata work. Yield periodically so the
4051
4384
  // wire, Receive, and Apply keep running while a large root is proved.
@@ -4076,7 +4409,7 @@ async function reconcileRoot(input) {
4076
4409
  if (observe && repositoryPath && activeRepository && !activeRepository.evidence) {
4077
4410
  evidence && (evidence.gitInspections += 1);
4078
4411
  try {
4079
- activeRepository.evidence = inspectGitRegistration(activeRepository.root, activeRepository.evidencePaths);
4412
+ activeRepository.evidence = spent(() => inspectGitRegistration(activeRepository.root, activeRepository.evidencePaths), "gitMs");
4080
4413
  }
4081
4414
  catch {
4082
4415
  // Classification already settled from the marker. If Git cannot
@@ -4169,11 +4502,19 @@ async function reconcileRoot(input) {
4169
4502
  });
4170
4503
  if (deferredCapture) {
4171
4504
  const entry = entries[entries.length - 1];
4505
+ if (measureEvidence && evidence && captureOpenedAt === 0)
4506
+ captureOpenedAt = performance.now();
4172
4507
  captureTasks.push(captureGate(async () => {
4508
+ const taskStarted = measureEvidence ? performance.now() : 0;
4173
4509
  let current = stats;
4174
4510
  for (let attempt = 1;; attempt++) {
4175
4511
  try {
4176
4512
  const captured = await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir);
4513
+ if (measureEvidence && evidence) {
4514
+ evidence.captureBusyMs += performance.now() - taskStarted;
4515
+ evidence.captureBytes += captured.bytes;
4516
+ captureClosedAt = performance.now();
4517
+ }
4177
4518
  entry.type = classifyFile({ binary: captured.binary });
4178
4519
  entry.payloadVersion = captured.contentHash;
4179
4520
  entry.contentVerifiedAtMs = Date.now();
@@ -4219,6 +4560,10 @@ async function reconcileRoot(input) {
4219
4560
  };
4220
4561
  await visit(rootPath, rootUUID, rootType);
4221
4562
  await Promise.all(captureTasks);
4563
+ if (measureEvidence && evidence && captureOpenedAt !== 0) {
4564
+ evidence.captureMs += captureClosedAt - captureOpenedAt;
4565
+ }
4566
+ const reconcileStarted = measureEvidence ? performance.now() : 0;
4222
4567
  // When every entry sits at its stored address with its stored type and
4223
4568
  // physical fingerprint (honoring any fixed UUID), path-first claiming makes
4224
4569
  // the multi-pass reconciliation the identity map — resolve directly. Any
@@ -4252,6 +4597,8 @@ async function reconcileRoot(input) {
4252
4597
  inode: entry.inode,
4253
4598
  ...(entry.fixedUUID ? { fixedUUID: entry.fixedUUID } : {}),
4254
4599
  })));
4600
+ if (measureEvidence && evidence)
4601
+ evidence.reconcileMs += performance.now() - reconcileStarted;
4255
4602
  const resolvedByProvisional = new Map();
4256
4603
  for (const entry of entries) {
4257
4604
  const resolved = resolvedIdentity.get(entry.relativePath) ?? entry.uuid;
@@ -4318,13 +4665,13 @@ async function reconcileRoot(input) {
4318
4665
  contentVerifiedAtMs: entry.contentVerifiedAtMs,
4319
4666
  };
4320
4667
  });
4321
- const persistCurrentRows = () => {
4668
+ const persistCurrentRows = () => spent(() => {
4322
4669
  const rows = rowsFromEntries();
4323
4670
  if (input.suspicion === undefined) {
4324
4671
  persistRows(database, identity, resourceId, rootUUID, rows, existingRows);
4325
4672
  }
4326
4673
  return rows;
4327
- };
4674
+ }, "persistMs");
4328
4675
  // Seal nested repositories first so their independent rows are complete
4329
4676
  // before the enclosing graph is persisted. Their state is not an input to
4330
4677
  // the enclosing identity map; only their address boundary is.
@@ -4452,7 +4799,7 @@ async function reconcileRoot(input) {
4452
4799
  }
4453
4800
  }
4454
4801
  const fingerprintBefore = gitEvidenceFingerprint(repository.absolutePath);
4455
- let repoIdentity = await refreshRepositoryIdentity();
4802
+ let repoIdentity = await spentAsync(refreshRepositoryIdentity, "repositoryIdentityMs");
4456
4803
  /* The rows and the transport identity are derived from the same fresh
4457
4804
  * worktree evidence. A later Watch ring for one of these dirty paths is
4458
4805
  * therefore ordinary equality, not a transport-only echo. */
@@ -4490,8 +4837,8 @@ async function reconcileRoot(input) {
4490
4837
  let captured = null;
4491
4838
  let coherent = false;
4492
4839
  for (let attempt = 0; attempt < 3; attempt += 1) {
4493
- captured = captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity);
4494
- const after = await refreshRepositoryIdentity();
4840
+ captured = spent(() => captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity), "repositoryCaptureMs");
4841
+ const after = await spentAsync(refreshRepositoryIdentity, "repositoryIdentityMs");
4495
4842
  if (sameGitIdentity(after, repoIdentity)) {
4496
4843
  repoIdentity = [...captured.identity];
4497
4844
  coherent = true;
@@ -4513,15 +4860,18 @@ async function reconcileRoot(input) {
4513
4860
  }
4514
4861
  repository.payloadVersion = captured.stateId;
4515
4862
  repository.transportVersion = captured.transportVersion;
4516
- if (captured.bytes) {
4517
- sealCapturedArtifact(cacheDir, {
4518
- entityId: repository.uuid,
4519
- entityType: "repo.git",
4520
- kind: "git",
4521
- contentHash: captured.transportVersion,
4522
- }, captured.bytes);
4523
- }
4524
- const currentLayout = inspectRepositoryTransportFile(join(cacheDir, `${captured.transportVersion}.bin`));
4863
+ const settled = captured;
4864
+ const currentLayout = spent(() => {
4865
+ if (settled.bytes) {
4866
+ sealCapturedArtifact(cacheDir, {
4867
+ entityId: repository.uuid,
4868
+ entityType: "repo.git",
4869
+ kind: "git",
4870
+ contentHash: settled.transportVersion,
4871
+ }, settled.bytes);
4872
+ }
4873
+ return inspectRepositoryTransportFile(join(cacheDir, `${settled.transportVersion}.bin`));
4874
+ }, "repositorySealMs");
4525
4875
  replayByUuid.set(repository.uuid, {
4526
4876
  kind: "repo.git",
4527
4877
  stateId: currentLayout.stateId,
@@ -4760,19 +5110,79 @@ function recordRootResolver(records) {
4760
5110
  };
4761
5111
  return rootOf;
4762
5112
  }
4763
- function mergeMaterializedRoots(cloudRecords, localRecords) {
4764
- const localRoots = new Set(localRecords
5113
+ /** The roots one publication rebuilds, each with its complete new records.
5114
+ *
5115
+ * A walked root is rebuilt from its ground alone: materialized ground is the
5116
+ * whole truth for the roots it covers, so nothing of the committed graph
5117
+ * survives underneath one. Any other root the newly delivered authority prefix
5118
+ * touched is rebuilt from the committed graph with those results folded in, so
5119
+ * the published graph never declares a watermark it has not actually applied.
5120
+ * Every root outside this map keeps its committed pages untouched. */
5121
+ function publicationRoots(graph, localRecords, accepted) {
5122
+ const rebuilt = new Map();
5123
+ const walked = new Map(localRecords
4765
5124
  .filter((record) => record.parentUUID === null)
4766
- .map((record) => record.uuid));
4767
- const localIds = new Set(localRecords.map((record) => record.uuid));
4768
- const rootOf = recordRootResolver(cloudRecords);
4769
- const preserved = cloudRecords.filter((record) => {
4770
- if (localIds.has(record.uuid))
4771
- return false;
4772
- const root = rootOf(record);
4773
- return root === null || !localRoots.has(root);
4774
- });
4775
- return [...preserved, ...localRecords];
5125
+ .map((record) => [record.uuid, []]));
5126
+ if (walked.size > 0) {
5127
+ const rootOf = recordRootResolver(localRecords);
5128
+ for (const record of localRecords) {
5129
+ const rootId = rootOf(record);
5130
+ const group = rootId === null ? undefined : walked.get(rootId);
5131
+ if (!group)
5132
+ throw new Error(`walked record ${record.uuid} descends from no walked root`);
5133
+ group.push(record);
5134
+ }
5135
+ for (const [rootId, records] of walked)
5136
+ rebuilt.set(rootId, records);
5137
+ }
5138
+ const overlays = new Map();
5139
+ for (const result of accepted) {
5140
+ const rootId = graph.rootOf(result.uuid);
5141
+ if (rootId === null || walked.has(rootId))
5142
+ continue;
5143
+ const overlay = overlays.get(rootId);
5144
+ if (overlay)
5145
+ overlay.push(result);
5146
+ else
5147
+ overlays.set(rootId, [result]);
5148
+ }
5149
+ for (const [rootId, results] of overlays) {
5150
+ const byUuid = new Map(graph.recordsFor(rootId).map((record) => [record.uuid, record]));
5151
+ for (const result of results)
5152
+ byUuid.set(result.uuid, result);
5153
+ // Entity channels are independently ordered, so container membership is
5154
+ // derived once from the final prefix rather than by arrival order.
5155
+ rebuilt.set(rootId, recordsRetainingMissingEvidence([...byUuid.values()], []));
5156
+ }
5157
+ // A records-shaped authority base carries no page identities, so it can
5158
+ // prove nothing about a root the commit leaves alone: the first head pages
5159
+ // the whole account at once, and never again.
5160
+ if (graph.head === null) {
5161
+ for (const rootId of graph.rootIds()) {
5162
+ if (!rebuilt.has(rootId))
5163
+ rebuilt.set(rootId, graph.recordsFor(rootId));
5164
+ }
5165
+ }
5166
+ // Re-rooting is a move, not a copy: an enclosing Register walks records the
5167
+ // committed graph still files under the root that used to hold them. Every
5168
+ // such root is rebuilt without them, and a root that keeps nothing is left
5169
+ // with no records at all — the rebuild then drops it from the graph.
5170
+ const vacated = new Map();
5171
+ for (const record of localRecords) {
5172
+ const rootId = graph.rootOf(record.uuid);
5173
+ if (rootId === null || walked.has(rootId))
5174
+ continue;
5175
+ const gone = vacated.get(rootId);
5176
+ if (gone)
5177
+ gone.add(record.uuid);
5178
+ else
5179
+ vacated.set(rootId, new Set([record.uuid]));
5180
+ }
5181
+ for (const [rootId, gone] of vacated) {
5182
+ const held = rebuilt.get(rootId) ?? graph.recordsFor(rootId);
5183
+ rebuilt.set(rootId, held.filter((record) => !gone.has(record.uuid)));
5184
+ }
5185
+ return rebuilt;
4776
5186
  }
4777
5187
  /** Filesystem disappearance is evidence, not cloud lifecycle authority. Keep
4778
5188
  * the last logical records in the outgoing graph while their materialized
@@ -5345,14 +5755,14 @@ async function installCloudWorkspace(input) {
5345
5755
  alreadyMaterialized: false,
5346
5756
  };
5347
5757
  }
5348
- function readyFromFrame(frame) {
5349
- const bytes = Buffer.from(String(frame.snapshot_b64 || ""), "base64");
5758
+ function readyFromFrame(frame, graph) {
5350
5759
  const cloud = {
5351
5760
  resourceId: String(frame.resource_id || ""),
5352
5761
  version: Number(frame.head_version),
5353
5762
  deliveryWatermark: Number(frame.delivery_watermark),
5354
5763
  checksum: String(frame.snapshot_checksum || ""),
5355
- snapshot: parseSnapshot(bytes),
5764
+ snapshot: graph.snapshot(),
5765
+ ...(graph.head ? { head: graph.head } : {}),
5356
5766
  };
5357
5767
  return {
5358
5768
  cloud,
@@ -5362,7 +5772,7 @@ function readyFromFrame(frame) {
5362
5772
  headVersion: cloud.version,
5363
5773
  checksum: cloud.checksum,
5364
5774
  deliveryWatermark: cloud.deliveryWatermark,
5365
- records: cloud.snapshot.records,
5775
+ graph,
5366
5776
  },
5367
5777
  };
5368
5778
  }