@amalgm/shell 0.1.112 → 0.1.114

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/PURPOSE.md +75 -6
  2. package/README.md +36 -7
  3. package/dist/authorization/device-key.d.ts +19 -0
  4. package/dist/authorization/device-key.js +81 -0
  5. package/dist/authorization/device-key.js.map +1 -0
  6. package/dist/authorization/index.d.ts +65 -0
  7. package/dist/authorization/index.js +63 -0
  8. package/dist/authorization/index.js.map +1 -0
  9. package/dist/authorization/issuer.d.ts +11 -0
  10. package/dist/authorization/issuer.js +76 -0
  11. package/dist/authorization/issuer.js.map +1 -0
  12. package/dist/authorization/key-store.d.ts +15 -0
  13. package/dist/authorization/key-store.js +148 -0
  14. package/dist/authorization/key-store.js.map +1 -0
  15. package/dist/authorization/recovery-client.d.ts +19 -0
  16. package/dist/authorization/recovery-client.js +102 -0
  17. package/dist/authorization/recovery-client.js.map +1 -0
  18. package/dist/authorization/recovery-host.d.ts +28 -0
  19. package/dist/authorization/recovery-host.js +93 -0
  20. package/dist/authorization/recovery-host.js.map +1 -0
  21. package/dist/authorization/recovery.d.ts +13 -0
  22. package/dist/authorization/recovery.js +27 -0
  23. package/dist/authorization/recovery.js.map +1 -0
  24. package/dist/authorization/secret-process.d.ts +2 -0
  25. package/dist/authorization/secret-process.js +32 -0
  26. package/dist/authorization/secret-process.js.map +1 -0
  27. package/dist/boot.d.ts +19 -11
  28. package/dist/boot.js +34 -25
  29. package/dist/boot.js.map +1 -1
  30. package/dist/builtin-agents.d.ts +3 -0
  31. package/dist/builtin-agents.js +15 -0
  32. package/dist/builtin-agents.js.map +1 -0
  33. package/dist/chat-composition.d.ts +19 -0
  34. package/dist/chat-composition.js +83 -0
  35. package/dist/chat-composition.js.map +1 -0
  36. package/dist/chat-host.d.ts +9 -25
  37. package/dist/chat-host.js +12 -158
  38. package/dist/chat-host.js.map +1 -1
  39. package/dist/cli.js +21 -13
  40. package/dist/cli.js.map +1 -1
  41. package/dist/content-cache-host.d.ts +7 -0
  42. package/dist/content-cache-host.js +23 -1
  43. package/dist/content-cache-host.js.map +1 -1
  44. package/dist/credentials-host.d.ts +1 -5
  45. package/dist/credentials-host.js +1 -5
  46. package/dist/credentials-host.js.map +1 -1
  47. package/dist/entity-record-store.d.ts +2 -0
  48. package/dist/entity-record-store.js +15 -0
  49. package/dist/entity-record-store.js.map +1 -1
  50. package/dist/forward.js +1 -1
  51. package/dist/forward.js.map +1 -1
  52. package/dist/index.d.ts +7 -6
  53. package/dist/index.js +2 -2
  54. package/dist/index.js.map +1 -1
  55. package/dist/machine-host.d.ts +29 -0
  56. package/dist/machine-host.js +240 -10
  57. package/dist/machine-host.js.map +1 -1
  58. package/dist/parity.d.ts +1 -5
  59. package/dist/parity.js +4 -41
  60. package/dist/parity.js.map +1 -1
  61. package/dist/registry-graph.d.ts +61 -0
  62. package/dist/registry-graph.js +156 -0
  63. package/dist/registry-graph.js.map +1 -0
  64. package/dist/release-catalog.js +7 -4
  65. package/dist/release-catalog.js.map +1 -1
  66. package/dist/runtime-command-host.d.ts +11 -0
  67. package/dist/runtime-command-host.js +65 -0
  68. package/dist/runtime-command-host.js.map +1 -1
  69. package/dist/user-ground-host.d.ts +13 -0
  70. package/dist/user-ground-host.js +449 -205
  71. package/dist/user-ground-host.js.map +1 -1
  72. package/dist/wire-client.js +1 -1
  73. package/package.json +15 -6
  74. package/dist/chat-payloads-host.d.ts +0 -8
  75. package/dist/chat-payloads-host.js +0 -76
  76. package/dist/chat-payloads-host.js.map +0 -1
  77. package/dist/chat-relay-host.d.ts +0 -24
  78. package/dist/chat-relay-host.js +0 -90
  79. package/dist/chat-relay-host.js.map +0 -1
  80. package/dist/drivers/claude-code.d.ts +0 -27
  81. package/dist/drivers/claude-code.js +0 -72
  82. package/dist/drivers/claude-code.js.map +0 -1
  83. package/dist/paged-mutation-submit.d.ts +0 -28
  84. package/dist/paged-mutation-submit.js +0 -61
  85. package/dist/paged-mutation-submit.js.map +0 -1
@@ -3,7 +3,7 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
3
3
  import { open, readFile } from "node:fs/promises";
4
4
  import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
5
5
  import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
6
- import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, deriveUploadManifest, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, planCargoWaves, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
6
+ import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, deriveUploadManifest, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, pathIsSuspect, planCargoWaves, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, GRAPH_HEAD_OPERATION_KIND, } from "@amalgm/live";
7
7
  import Database from "better-sqlite3";
8
8
  import { atomicCopy, atomicWrite, ensurePrivateDir, ensureUserDir } from "./filesystem.js";
9
9
  import { emitFilesStage, } from "./files-observability.js";
@@ -11,7 +11,7 @@ import { EntityApplyHost, } from "./entity-apply-host.js";
11
11
  import { EntityRecordSqliteStore } from "./entity-record-store.js";
12
12
  import { observeEntityRecordPorts } from "./entity-record-observability.js";
13
13
  import { indexRowsByOutermostRoot } from "./ground-root-index.js";
14
- import { ContentCacheDownloadSet, captureContentFile, hashContentFile, sealCapturedArtifact, } from "./content-cache-host.js";
14
+ import { ContentCacheDownloadSet, captureContentFile, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
15
15
  import { DirectContentPlane } from "./content-direct.js";
16
16
  import { encodeContentWireBytes } from "./content-wire-codec.js";
17
17
  import { exactTextReplay, planGroundDetection, reconcileGroundUUIDs, } from "./detection/portable.js";
@@ -22,9 +22,9 @@ import { WORKSPACE_UUID as UUID, createFilesRegisterPorts, ensureWorkspaceBindin
22
22
  import { applyRepositoryFiles, captureRepository, inspectRepositoryTransportFile, hasGitMarker, readCachedRepositoryIdentity, } from "./git-repository-host.js";
23
23
  import { gitEvidenceFingerprint, inspectGitRegistration, sameGitIdentity, } from "./git-registration-host.js";
24
24
  import { projectMaterializedGraph } from "./materialized-graph.js";
25
+ import { graphPageIdentities, registryGraphFromSnapshot, } from "./registry-graph.js";
25
26
  import { NodeWatchHost, } from "./watching/index.js";
26
27
  import { GroundCoordinator } from "./ground-coordination.js";
27
- import { submitMutationOperation } from "./paged-mutation-submit.js";
28
28
  import { adoptSnapshotDeliveryBaseline } from "./snapshot-delivery-baseline.js";
29
29
  import { WireClient, WireRequestError } from "./wire-client.js";
30
30
  /** Cargo waves hold up to 16MiB of payload each; two in flight bounds the
@@ -40,6 +40,10 @@ const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = 128;
40
40
  // on the 1 GiB runtimes used for ordinary Files materialization. Small
41
41
  // objects still use the full request-concurrency window.
42
42
  const DIRECT_OBJECT_DOWNLOAD_MAX_IN_FLIGHT_BYTES = 2 * CHUNK_BYTES;
43
+ /** Graph pages are at most a few megabytes each and a whole account is a
44
+ * handful of them, so a narrow lane keeps the cold read fast without holding
45
+ * the whole graph twice while it assembles. */
46
+ const GRAPH_PAGE_READ_CONCURRENCY = 8;
43
47
  const DETECT_QUIET_MS = 12;
44
48
  const DETECT_MAX_DEFERRAL_MS = 75;
45
49
  const DETECT_RETRY_MS = 250;
@@ -55,13 +59,19 @@ const WALK_CAPTURE_CONCURRENCY = 8;
55
59
  const REGISTER_STALE_CAPTURE_ATTEMPTS = 3;
56
60
  const DOWNLOAD_CONTENT_RETRY_ATTEMPTS = 3;
57
61
  const APPLY_GROUND_WAIT_TIMEOUT_MS = 15_000;
62
+ /** Load a journaled publication for replay. The records whose cargo must
63
+ * re-upload are already inside the publication's own bytes — its pages, or,
64
+ * for a row converted from the pre-page journal, the replacement operation
65
+ * itself — so nothing is stored twice and nothing is re-read from ground. */
58
66
  function loadSnapshotPublication(entry) {
59
- const snapshotRecords = snapshotFromRecords((JSON.parse(entry.snapshotBlob.toString("utf8")).records ?? [])).records;
60
67
  const uploadIds = new Set(entry.uploadUuids);
68
+ const published = uploadIds.size === 0 ? [] : entry.pages.length > 0
69
+ ? entry.pages.flatMap((page) => JSON.parse(page.bytes.toString("utf8")))
70
+ : (JSON.parse(entry.operationBlob.toString("utf8")).records ?? []);
61
71
  return {
62
72
  ...entry,
63
- snapshotRecords,
64
- uploadRecords: snapshotRecords.filter((record) => uploadIds.has(record.uuid)),
73
+ uploadRecords: published.filter((record) => uploadIds.has(record.uuid)),
74
+ graph: null,
65
75
  };
66
76
  }
67
77
  const GROUND_ROW_COLUMNS = [
@@ -558,7 +568,7 @@ export class UserGroundHost {
558
568
  && record.status === "active"
559
569
  && record.payloadVersion === workspaceId);
560
570
  if (hasRegisteredBoundaryInDatabase(database, workspaceId)
561
- && hasRegisteredBoundary(this.cloudState.records)) {
571
+ && hasRegisteredBoundary(this.cloudState.graph.records())) {
562
572
  // Already declared — but Register may only return once cargo closure
563
573
  // holds under the current content contract. The graph head can be
564
574
  // unchanged while a contract cutover moved its cargo namespace, so
@@ -740,7 +750,7 @@ export class UserGroundHost {
740
750
  if (!hasRegisteredBoundaryInDatabase(database, workspaceId)) {
741
751
  throw stageError("entity registration", new Error(`workspace ${workspaceId} and its durable reference were not recorded`));
742
752
  }
743
- if (!hasRegisteredBoundary(this.cloudState.records)) {
753
+ if (!hasRegisteredBoundary(this.cloudState.graph.records())) {
744
754
  throw stageError("cloud graph commit", new Error(`workspace ${workspaceId} or its durable reference is absent from the committed head`));
745
755
  }
746
756
  return {
@@ -822,13 +832,45 @@ export class UserGroundHost {
822
832
  throw new Error("entity Record rail already started");
823
833
  await this.wire.open();
824
834
  // One floor, no per-feature gates: this shell and its gateway release
825
- // together, and the gateway ships first. 12 = bounded cold-Register cargo
826
- // waves; an older gateway cannot verify and seal that upload frame.
827
- if (this.wire.protocolVersion < 12) {
835
+ // together, and the gateway ships first. 13 = immutable registry graph
836
+ // pages plus one graph-head commit; an older gateway cannot publish or
837
+ // resolve that representation.
838
+ if (this.wire.protocolVersion < 13) {
828
839
  throw new Error("gateway speaks an older wire protocol than this shell");
829
840
  }
830
841
  this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
831
- const store = new EntityRecordSqliteStore(initializeDatabase(this.databasePath(identity)));
842
+ const recordDatabase = initializeDatabase(this.databasePath(identity));
843
+ const store = new EntityRecordSqliteStore(recordDatabase);
844
+ const pruneAcceptedCargo = () => {
845
+ const retained = new Set(store.pendingContentHashes());
846
+ const currentReplayBases = recordDatabase.prepare(`
847
+ SELECT type,
848
+ payload_version AS payloadVersion,
849
+ transport_version AS transportVersion
850
+ FROM entities
851
+ WHERE type IN ('file.text', 'repo.git')
852
+ `).all();
853
+ for (const row of currentReplayBases) {
854
+ const contentHash = row.type === "file.text" ? row.payloadVersion : row.transportVersion;
855
+ if (contentHash && /^[0-9a-f]{64}$/.test(contentHash))
856
+ retained.add(contentHash);
857
+ }
858
+ pruneSettledContentCache(this.cacheDir(identity), retained);
859
+ };
860
+ // Accepted Send cargo is no longer retry responsibility. Keep only the
861
+ // current verified text and repository heads needed for the next exact
862
+ // delta; reclaim all other unreferenced history before start and after
863
+ // every acceptance.
864
+ pruneAcceptedCargo();
865
+ const cacheReleasingStore = {
866
+ readPending: (limit) => store.readPending(limit),
867
+ stampAccepted: async (...args) => {
868
+ store.stampAccepted(...args);
869
+ pruneAcceptedCargo();
870
+ },
871
+ readDeliveryCursor: (authorityId) => store.readDeliveryCursor(authorityId),
872
+ receive: (input) => store.receive(input),
873
+ };
832
874
  const wireAuthority = createEntityRecordAuthorityPort({
833
875
  request: (frame, acceptedTypes, binaryBody) => binaryBody === undefined
834
876
  ? this.wire.request({ ...frame }, acceptedTypes)
@@ -837,7 +879,7 @@ export class UserGroundHost {
837
879
  onDisconnect: (listener) => this.wire.onDisconnect(listener),
838
880
  });
839
881
  const ports = observeEntityRecordPorts({
840
- store,
882
+ store: cacheReleasingStore,
841
883
  authority: wireAuthority,
842
884
  cargo: {
843
885
  ensure: async (record) => {
@@ -1096,7 +1138,7 @@ export class UserGroundHost {
1096
1138
  }, ["private.resource.ready", "private.resource.missing"]);
1097
1139
  if (frame.type === "private.resource.missing")
1098
1140
  return null;
1099
- const ready = readyFromFrame(frame);
1141
+ const ready = readyFromFrame(frame, await this.materializeRegistry(identity, frame));
1100
1142
  this.cloudState = ready.state;
1101
1143
  return ready.cloud;
1102
1144
  },
@@ -1111,7 +1153,7 @@ export class UserGroundHost {
1111
1153
  snapshot_checksum: input.checksum,
1112
1154
  snapshot_bytes: bytes.length,
1113
1155
  }, ["private.resource.ready"]);
1114
- const ready = readyFromFrame(frame);
1156
+ const ready = readyFromFrame(frame, await this.materializeRegistry(identity, frame));
1115
1157
  const current = ready.cloud;
1116
1158
  this.cloudState = ready.state;
1117
1159
  const created = frame.duplicate === false;
@@ -1121,6 +1163,61 @@ export class UserGroundHost {
1121
1163
  },
1122
1164
  };
1123
1165
  }
1166
+ /** The authority's stored snapshot as this machine's graph. A head names
1167
+ * immutable pages, so every page this machine already cached is a local read
1168
+ * and an unchanged root never crosses the network twice. */
1169
+ async materializeRegistry(identity, frame) {
1170
+ const resourceId = String(frame.resource_id || "");
1171
+ return registryGraphFromSnapshot(Buffer.from(String(frame.snapshot_b64 || ""), "base64"), (refs) => this.readGraphPages(identity, resourceId, refs), sha256Hex);
1172
+ }
1173
+ /** Read graph pages by identity: cached pages come off local disk, the rest
1174
+ * are fetched straight from content storage and cached on the way in. Every
1175
+ * page is re-proven against its declared identity before it is kept. */
1176
+ async readGraphPages(identity, resourceId, refs) {
1177
+ const directory = graphPageCacheDir(this.cacheDir(identity));
1178
+ const pages = new Map();
1179
+ const missing = [];
1180
+ for (const ref of new Map(refs.map((ref) => [ref.sha256, ref])).values()) {
1181
+ let cached = null;
1182
+ try {
1183
+ cached = readFileSync(join(directory, `${ref.sha256}.bin`));
1184
+ }
1185
+ catch {
1186
+ cached = null;
1187
+ }
1188
+ // A cache copy of the wrong size is torn evidence, never a page: fetch
1189
+ // the immutable original instead of assembling from it.
1190
+ if (cached && cached.byteLength === ref.bytes)
1191
+ pages.set(ref.sha256, cached);
1192
+ else
1193
+ missing.push(ref);
1194
+ }
1195
+ if (missing.length === 0)
1196
+ return pages;
1197
+ const started = performance.now();
1198
+ let fetchedBytes = 0;
1199
+ await boundedForEach(missing, GRAPH_PAGE_READ_CONCURRENCY, async (ref) => {
1200
+ const bytes = await retryContent(() => this.direct.fetchObject(resourceId, { kind: "block", sha256: ref.sha256 }), true);
1201
+ if (bytes.byteLength !== ref.bytes || sha256Hex(bytes) !== ref.sha256) {
1202
+ throw new Error(`cloud graph page ${ref.sha256} does not match its bytes`);
1203
+ }
1204
+ atomicWrite(join(directory, `${ref.sha256}.bin`), bytes);
1205
+ fetchedBytes += bytes.byteLength;
1206
+ pages.set(ref.sha256, bytes);
1207
+ });
1208
+ this.stage({
1209
+ primitive: "download",
1210
+ stage: "graph-pages",
1211
+ status: "completed",
1212
+ durationMs: performance.now() - started,
1213
+ measurements: {
1214
+ pages: refs.length,
1215
+ fetchedPages: missing.length,
1216
+ fetchedBytes,
1217
+ },
1218
+ });
1219
+ return pages;
1220
+ }
1124
1221
  localPort(identity) {
1125
1222
  const userRoot = this.userRoot(identity);
1126
1223
  const database = this.databasePath(identity);
@@ -1772,7 +1869,22 @@ export class UserGroundHost {
1772
1869
  durationMs: performance.now() - waitStarted,
1773
1870
  });
1774
1871
  try {
1775
- await this.syncLocalChangesUnlocked(identity, pending, coreScan);
1872
+ let retriedAdvancedBasis = false;
1873
+ for (;;) {
1874
+ try {
1875
+ await this.syncLocalChangesUnlocked(identity, pending, coreScan);
1876
+ break;
1877
+ }
1878
+ catch (error) {
1879
+ if (retriedAdvancedBasis || !detectionBasisAdvanced(error))
1880
+ throw error;
1881
+ // The complete generation remains unsettled. Re-read its truth once
1882
+ // while this pass still owns every affected ground cone; another
1883
+ // conflict means an invariant outside this finite pass moved again
1884
+ // and must remain visible to the caller.
1885
+ retriedAdvancedBasis = true;
1886
+ }
1887
+ }
1776
1888
  }
1777
1889
  finally {
1778
1890
  release();
@@ -1908,7 +2020,7 @@ export class UserGroundHost {
1908
2020
  proposals: detectedRecords,
1909
2021
  });
1910
2022
  if (!committed) {
1911
- throw pipelineStageError("detection", new Error("the verified notebook advanced while Detect was observing; retrying fresh truth"));
2023
+ throw pipelineStageError("detection", new DetectionBasisAdvancedError());
1912
2024
  }
1913
2025
  const recordCommitDuration = performance.now() - recordCommitStarted;
1914
2026
  if (detectedRecords.length === 0) {
@@ -1962,14 +2074,6 @@ export class UserGroundHost {
1962
2074
  if (first.resourceId !== resourceId) {
1963
2075
  throw new Error("local snapshot publication belongs to a different user-ground resource");
1964
2076
  }
1965
- this.cloudState = {
1966
- resourceId: first.resourceId,
1967
- authorityEpoch: first.authorityEpoch,
1968
- headVersion: first.baseVersion,
1969
- checksum: "",
1970
- deliveryWatermark: first.deliveryWatermark,
1971
- records: loadSnapshotPublication(first).snapshotRecords,
1972
- };
1973
2077
  try {
1974
2078
  await this.publishPendingSnapshots(identity);
1975
2079
  }
@@ -2001,6 +2105,12 @@ export class UserGroundHost {
2001
2105
  throw error;
2002
2106
  }
2003
2107
  await this.recordRail?.flush();
2108
+ // A replayed publication leaves no graph behind — it only ever journaled
2109
+ // the roots it rebuilt — so the machine re-reads the authority's head
2110
+ // before deriving the next boundary on top of it.
2111
+ if (!this.cloudState) {
2112
+ await this.cloudPort(identity).lookup(privateEntityResourceId(identity.userId, sha256Hex));
2113
+ }
2004
2114
  const state = this.cloudState;
2005
2115
  if (!state)
2006
2116
  throw new Error("cloud state is unavailable for graph publication");
@@ -2038,24 +2148,24 @@ export class UserGroundHost {
2038
2148
  ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2039
2149
  });
2040
2150
  }
2041
- const officialRecords = timed("officialRows", () => recordsAtDeliveryPrefix(state.records, readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark)));
2042
2151
  const localRecords = timed("groundRows", () => recordsRetainingMissingEvidence(readGroundRowsForRoots(this.databasePath(identity), "entities", publicationRootIds)
2043
2152
  .map(portableRecord), readGroundRowsForRoots(this.databasePath(identity), "detection_notebook", publicationRootIds)
2044
2153
  .map(portableRecord)));
2045
- const merged = timed("merge", () => mergeMaterializedRoots(officialRecords, localRecords));
2046
- const traveling = timed("project", () => travelingRecords(merged, sha256Hex));
2154
+ const accepted = timed("officialRows", () => readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark));
2155
+ const rebuilt = timed("merge", () => publicationRoots(state.graph, localRecords, accepted));
2156
+ const projected = timed("project", () => new Map([...rebuilt].map(([rootId, records]) => [rootId, records.length === 0 ? records : travelingRecords(records, sha256Hex)])));
2157
+ const traveling = [...projected.values()].flat();
2047
2158
  const impossibleGitLeaf = traveling.find((record) => (record.type === "file.text" || record.type === "file.binary" || record.type === "link")
2048
2159
  && record.payloadVersion?.startsWith("git:"));
2049
2160
  if (impossibleGitLeaf) {
2050
- const parent = merged.find((record) => record.uuid === impossibleGitLeaf.parentUUID);
2161
+ const parent = traveling.find((record) => record.uuid === impossibleGitLeaf.parentUUID);
2051
2162
  throw new Error(`repository-owned leaf escaped its rail: ${impossibleGitLeaf.name} parent=${parent?.type ?? "missing"}`);
2052
2163
  }
2053
- const snapshot = timed("normalize", () => snapshotFromRecords(traveling));
2054
- // ONE canonical serialization: these exact bytes are the journal blob
2055
- // and the checksum's input; they are never re-derived or re-parsed on
2056
- // the in-process path.
2057
- const snapshotBlob = timed("serialize", () => Buffer.from(stableJson(snapshot), "utf8"));
2058
- const checksum = timed("checksum", () => sha256Hex(snapshotBlob));
2164
+ // ONE canonical serialization per rebuilt root: the page bytes are the
2165
+ // journal's payload, the upload's payload, and the head's identities.
2166
+ // Untouched roots are not serialized, hashed, or journaled at all.
2167
+ const publication = timed("pages", () => state.graph.rebuild(projected, sha256Hex));
2168
+ const checksum = publication.checksum;
2059
2169
  const localIds = new Set(localRecords.map((record) => record.uuid));
2060
2170
  // Graph membership is never proof of sealed cargo: a record in the cloud
2061
2171
  // graph may name content sealed only under a prior content contract,
@@ -2064,11 +2174,11 @@ export class UserGroundHost {
2064
2174
  // sealed-first upload asks that exact question itself and moves only
2065
2175
  // what is unsealed, so Register always re-establishes cargo closure
2066
2176
  // before returning, even when the graph itself is unchanged.
2067
- const cargoRecords = snapshot.records.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
2177
+ const cargoRecords = traveling.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
2068
2178
  if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
2069
2179
  this.finishPreparation(trace, preparationStarted, leg, {
2070
- records: snapshot.records.length,
2071
- snapshotBytes: snapshotBlob.byteLength,
2180
+ rebuiltRoots: rebuilt.size,
2181
+ records: traveling.length,
2072
2182
  cargoRecords: cargoRecords.length,
2073
2183
  graphUnchanged: true,
2074
2184
  });
@@ -2085,19 +2195,12 @@ export class UserGroundHost {
2085
2195
  }
2086
2196
  return;
2087
2197
  }
2088
- const replacement = timed("replacement", () => {
2089
- const rootIds = localRecords
2090
- .filter((record) => record.parentUUID === null)
2091
- .map((record) => record.uuid);
2092
- const roots = new Set(rootIds);
2093
- const rootOf = recordRootResolver(snapshot.records);
2094
- return rootReplacementFromRecords(rootIds, snapshot.records.filter((record) => {
2095
- const root = rootOf(record);
2096
- return root !== null && roots.has(root);
2097
- }));
2098
- });
2099
- // The operation's canonical bytes: staged, hashed, and submitted as-is.
2100
- const operationBlob = timed("operation", () => Buffer.from(stableJson(replacement), "utf8"));
2198
+ // The operation's canonical bytes: journaled and submitted as-is.
2199
+ const operationBlob = timed("operation", () => Buffer.from(stableJson(publication.operation), "utf8"));
2200
+ // Only pages the committed head does not already name still have to be
2201
+ // sealed; a rebuilt root that came out byte-identical costs nothing.
2202
+ const committed = graphPageIdentities(state.graph.head);
2203
+ const fresh = publication.pages.filter((page) => !committed.has(page.ref.sha256));
2101
2204
  const entry = {
2102
2205
  mutationId: randomUUID(),
2103
2206
  resourceId: state.resourceId,
@@ -2105,30 +2208,35 @@ export class UserGroundHost {
2105
2208
  baseVersion: state.headVersion,
2106
2209
  snapshotChecksum: checksum,
2107
2210
  deliveryWatermark,
2108
- operationKind: replacement.kind,
2109
- replacementRootIds: replacement.rootIds,
2211
+ operationKind: publication.operation.kind,
2212
+ replacementRootIds: publication.operation.replacedRootIds,
2110
2213
  uploadUuids: cargoRecords.map((record) => record.uuid),
2111
- snapshotBlob,
2112
2214
  operationBlob,
2215
+ pages: fresh.map((page) => ({
2216
+ sha256: page.ref.sha256,
2217
+ bytes: Buffer.from(page.bytes.buffer, page.bytes.byteOffset, page.bytes.byteLength),
2218
+ })),
2113
2219
  status: "pending",
2114
2220
  submitted: false,
2115
2221
  };
2116
2222
  timed("journal", () => supersedeSnapshotJournal(this.databasePath(identity), entry));
2117
2223
  this.finishPreparation(trace, preparationStarted, leg, {
2118
- records: snapshot.records.length,
2119
- snapshotBytes: snapshotBlob.byteLength,
2224
+ rebuiltRoots: rebuilt.size,
2225
+ records: traveling.length,
2226
+ pages: publication.pages.length,
2227
+ freshPages: fresh.length,
2228
+ pageBytes: fresh.reduce((total, page) => total + page.ref.bytes, 0),
2120
2229
  operationBytes: operationBlob.byteLength,
2121
- operationRoots: replacement.rootIds.length,
2122
2230
  cargoRecords: cargoRecords.length,
2123
2231
  graphUnchanged: false,
2124
2232
  });
2125
- // Publish the freshly journaled intent directly: the records are already
2126
- // in memory and the operation bytes already exist — nothing is re-read,
2127
- // re-parsed, or re-serialized between prepare and the wire.
2233
+ // Publish the freshly journaled intent directly: the pages and the
2234
+ // operation bytes already exist — nothing is re-read, re-parsed, or
2235
+ // re-serialized between prepare and the wire.
2128
2236
  await this.publishLoadedSnapshot(identity, {
2129
2237
  ...entry,
2130
- snapshotRecords: snapshot.records,
2131
2238
  uploadRecords: cargoRecords,
2239
+ graph: publication.graph,
2132
2240
  }, trace, manifestHints);
2133
2241
  }
2134
2242
  /** Close the graph-preparation stage with every leg it spent, so a slow
@@ -2181,8 +2289,61 @@ export class UserGroundHost {
2181
2289
  await this.publishLoadedSnapshot(identity, loadSnapshotPublication(pending), trace, manifestHints);
2182
2290
  }
2183
2291
  }
2292
+ /** Seal this publication's graph pages. Pages are immutable, content
2293
+ * addressed, and already in memory, so the authority's seal inventory alone
2294
+ * decides what still has to move and whole pages ride cargo waves. The head
2295
+ * may not commit until every page it newly names is sealed. */
2296
+ async uploadGraphPages(resourceId, pages, trace) {
2297
+ if (pages.length === 0)
2298
+ return;
2299
+ const started = performance.now();
2300
+ const sealed = await this.sealedContentHashes(resourceId, pages.map((page) => page.sha256));
2301
+ const unsealed = pages
2302
+ .filter((page) => !sealed.has(page.sha256))
2303
+ .map((page) => ({ page, bytes: page.bytes.byteLength }));
2304
+ let uploadedBytes = 0;
2305
+ await boundedForEach([...planCargoWaves(unsealed)], CARGO_UPLOAD_CONCURRENCY, async (wave) => {
2306
+ const payload = Buffer.concat(wave.items.map((item) => item.page.bytes));
2307
+ const encoded = await encodeContentWireBytes(payload);
2308
+ const frame = await retryContent(() => this.wire.requestBinary({
2309
+ type: "private.entity-content.put-cargo",
2310
+ resource_id: resourceId,
2311
+ // A page is a single-chunk artifact: its identity, its only chunk,
2312
+ // and its bytes are one and the same sha.
2313
+ artifacts: wave.items.map((item) => ({
2314
+ content_hash: item.page.sha256,
2315
+ bytes: item.bytes,
2316
+ chunks: [{ sha256: item.page.sha256, bytes: item.bytes }],
2317
+ })),
2318
+ bytes: payload.byteLength,
2319
+ wire_bytes: encoded.bytes.byteLength,
2320
+ ...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
2321
+ }, encoded.bytes, ["private.entity-content.stored-cargo"]));
2322
+ const stored = new Set(Array.isArray(frame.sealed) ? frame.sealed.map(String) : []);
2323
+ for (const item of wave.items) {
2324
+ if (!stored.has(item.page.sha256)) {
2325
+ throw new Error(`content authority did not seal graph page ${item.page.sha256}`);
2326
+ }
2327
+ }
2328
+ uploadedBytes += encoded.bytes.byteLength;
2329
+ });
2330
+ this.stage({
2331
+ primitive: "upload",
2332
+ stage: "graph-pages",
2333
+ status: "completed",
2334
+ ...(trace?.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2335
+ durationMs: performance.now() - started,
2336
+ measurements: {
2337
+ journey: trace?.journey ?? "bootstrap",
2338
+ pages: pages.length,
2339
+ uploadedPages: unsealed.length,
2340
+ uploadedBytes,
2341
+ },
2342
+ });
2343
+ }
2184
2344
  async publishLoadedSnapshot(identity, publication, trace, manifestHints) {
2185
2345
  try {
2346
+ await this.uploadGraphPages(publication.resourceId, publication.pages, trace);
2186
2347
  await this.uploadSnapshotContent(identity, publication.resourceId, publication.uploadRecords, trace, manifestHints);
2187
2348
  }
2188
2349
  catch (error) {
@@ -2218,17 +2379,25 @@ export class UserGroundHost {
2218
2379
  local("markSubmitted", () => markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId));
2219
2380
  }
2220
2381
  try {
2221
- frame = await submitMutationOperation(this.wire, {
2222
- resourceId: publication.resourceId,
2223
- authorityEpoch: publication.authorityEpoch,
2224
- mutationId: publication.mutationId,
2225
- deviceId: identity.deviceId,
2226
- baseVersion: publication.baseVersion,
2227
- contract: ENTITY_CLOUD_CONTRACT,
2228
- schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
2229
- operationKind: publication.operationKind,
2230
- deliveryWatermark: publication.deliveryWatermark,
2231
- }, publication.operationBlob, (leg, durationMs) => { legMs[`${leg}Ms`] = durationMs; });
2382
+ // A head commit names pages instead of carrying records, so it is small
2383
+ // by construction and always travels inline in one frame.
2384
+ const submitStarted = performance.now();
2385
+ frame = await this.wire.request({
2386
+ type: "shared.mutation.submit",
2387
+ envelope: {
2388
+ resourceId: publication.resourceId,
2389
+ authorityEpoch: publication.authorityEpoch,
2390
+ mutationId: publication.mutationId,
2391
+ deviceId: identity.deviceId,
2392
+ baseVersion: publication.baseVersion,
2393
+ contract: ENTITY_CLOUD_CONTRACT,
2394
+ schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
2395
+ operationKind: publication.operationKind,
2396
+ deliveryWatermark: publication.deliveryWatermark,
2397
+ operation: JSON.parse(publication.operationBlob.toString("utf8")),
2398
+ },
2399
+ }, ["shared.mutation.ack"]);
2400
+ legMs.submitMs = performance.now() - submitStarted;
2232
2401
  }
2233
2402
  catch (error) {
2234
2403
  if (trace?.journey === "register") {
@@ -2249,13 +2418,16 @@ export class UserGroundHost {
2249
2418
  }
2250
2419
  throw pipelineStageError("cloud graph commit", error);
2251
2420
  }
2252
- this.cloudState = {
2421
+ // A replay carries no graph: clearing the state makes the next reader
2422
+ // fetch the authority's own head instead of a graph missing every root
2423
+ // this publication did not rebuild.
2424
+ this.cloudState = publication.graph === null ? null : {
2253
2425
  resourceId: publication.resourceId,
2254
2426
  authorityEpoch: Number(frame.authority_epoch),
2255
2427
  headVersion: Number(frame.version),
2256
2428
  checksum: publication.snapshotChecksum,
2257
2429
  deliveryWatermark: Number(frame.delivery_watermark),
2258
- records: publication.snapshotRecords,
2430
+ graph: publication.graph,
2259
2431
  };
2260
2432
  local("clearJournal", () => deleteJournalEntry(this.databasePath(identity), publication.mutationId));
2261
2433
  if (trace?.journey === "register") {
@@ -2263,7 +2435,7 @@ export class UserGroundHost {
2263
2435
  // its legs here gives one ledger line the whole story: local journal
2264
2436
  // marks, wire time, gateway assemble/validate, and the database RPC.
2265
2437
  for (const [wire, leg] of [
2266
- ["assemble_ms", "serverAssembleMs"],
2438
+ ["verify_ms", "serverVerifyMs"],
2267
2439
  ["validate_ms", "serverValidateMs"],
2268
2440
  ["database_ms", "serverDatabaseMs"],
2269
2441
  ]) {
@@ -2302,6 +2474,12 @@ function contentCacheFile(cacheDir, contentHash) {
2302
2474
  function manifestCacheFile(cacheDir, contentHash) {
2303
2475
  return join(cacheDir, "manifests", `${contentHash}.json`);
2304
2476
  }
2477
+ /** Graph pages are immutable and content-addressed, so they cache exactly like
2478
+ * artifacts do — in their own directory, because they are the registry's own
2479
+ * bytes rather than any entity's cargo. */
2480
+ function graphPageCacheDir(cacheDir) {
2481
+ return join(cacheDir, "graph-pages");
2482
+ }
2305
2483
  function readCachedManifest(cacheDir, contentHash) {
2306
2484
  try {
2307
2485
  const checked = checkContentManifest(JSON.parse(readFileSync(manifestCacheFile(cacheDir, contentHash), "utf8")), contentHash);
@@ -2333,6 +2511,15 @@ function pipelineStageError(stage, error) {
2333
2511
  cause: error,
2334
2512
  });
2335
2513
  }
2514
+ class DetectionBasisAdvancedError extends Error {
2515
+ constructor() {
2516
+ super("the verified notebook advanced while Detect was observing; retrying fresh truth");
2517
+ }
2518
+ }
2519
+ function detectionBasisAdvanced(error) {
2520
+ return error instanceof DetectionBasisAdvancedError
2521
+ || (error instanceof Error && error.cause instanceof DetectionBasisAdvancedError);
2522
+ }
2336
2523
  function filesPipelineStage(error) {
2337
2524
  const stage = error instanceof Error
2338
2525
  ? error.filesStage
@@ -2375,118 +2562,110 @@ function assertHealthyWatch(evidence, rootId) {
2375
2562
  * file over a dozen times; the file itself is never deleted by the host, so
2376
2563
  * every later open needs only the connection and its pragmas. */
2377
2564
  const migratedDatabases = new Set();
2378
- /** The journal's mutable facts and its immutable payload live in separate
2379
- * tables on purpose: SQLite rewrites a row's entire record — overflow pages
2380
- * included — on any column update, so flipping submitted_at on a row that
2381
- * carried a register-scale snapshot cost a full multi-megabyte rewrite
2382
- * through the WAL at synchronous=FULL (measured ~7.5s of the production
2383
- * cloud-commit stage on the 1-GB machine). The blob row is written exactly
2384
- * once and read only on restart replay; every status transition touches only
2385
- * the small row. */
2565
+ /** A publication row holds only what the authority must be told: the small
2566
+ * meta facts and the head-commit operation naming this graph's pages. The
2567
+ * pages themselves are write-once rows in their own table, because a status
2568
+ * transition rewrites its whole row — overflow pages included — through the
2569
+ * WAL at synchronous=FULL, and page bytes must never ride that rewrite. */
2386
2570
  const SNAPSHOT_PUBLICATIONS_SCHEMA = `
2387
2571
  CREATE TABLE IF NOT EXISTS snapshot_publications (
2388
2572
  sequence INTEGER PRIMARY KEY AUTOINCREMENT,
2389
2573
  mutation_id TEXT NOT NULL UNIQUE,
2390
2574
  resource_id TEXT NOT NULL,
2391
2575
  meta_json TEXT NOT NULL,
2576
+ operation_blob BLOB NOT NULL,
2392
2577
  created_at INTEGER NOT NULL,
2393
2578
  status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
2394
2579
  submitted_at INTEGER
2395
2580
  );
2396
- CREATE TABLE IF NOT EXISTS snapshot_publication_blobs (
2397
- mutation_id TEXT PRIMARY KEY,
2398
- snapshot_blob BLOB NOT NULL,
2399
- operation_blob BLOB NOT NULL
2581
+ CREATE TABLE IF NOT EXISTS snapshot_publication_pages (
2582
+ mutation_id TEXT NOT NULL,
2583
+ page_index INTEGER NOT NULL,
2584
+ sha256 TEXT NOT NULL,
2585
+ bytes BLOB NOT NULL,
2586
+ PRIMARY KEY (mutation_id, page_index)
2400
2587
  );`;
2401
- /** Converts an old-shape publication journal (one nested publication_json
2402
- * text per row) into the blob journal, atomically, preserving every row's
2403
- * meaning. A pending row may be a submitted mutation whose outcome exists
2404
- * only at the authority — dropping it would violate the retry law that
2405
- * recovers a lost acknowledgement — so conversion is structural: the nested
2406
- * snapshot text becomes the snapshot blob verbatim, the stored replacement
2407
- * becomes the operation blob, and semantic judgment stays where it always
2408
- * was, with the reader (an unlawful row still blocks registration rather
2409
- * than being silently replaced).
2588
+ function legacyJournalShape(columns) {
2589
+ if (columns.size === 0)
2590
+ return null;
2591
+ if (columns.has("publication_json"))
2592
+ return "nested-text";
2593
+ if (columns.has("snapshot_blob"))
2594
+ return "row-blobs";
2595
+ // The paged journal and the side-table one both keep only small columns
2596
+ // here; only the paged one names the operation on the row itself.
2597
+ return columns.has("operation_blob") ? null : "split-blobs";
2598
+ }
2599
+ /** Converts a pre-paged publication journal into the paged one, atomically,
2600
+ * preserving every row's meaning. A pending row may be a submitted mutation
2601
+ * whose outcome exists only at the authority — dropping it would violate the
2602
+ * retry law that recovers a lost acknowledgement — so conversion is
2603
+ * structural: each row keeps its identity, its meta facts, and the exact
2604
+ * operation bytes it would submit, and semantic judgment stays where it
2605
+ * always was, with the reader (an unlawful row still blocks registration
2606
+ * rather than being silently replaced). A converted row rebuilt whole roots
2607
+ * rather than pages, so it carries no page rows; its replayed upload comes
2608
+ * from the records inside its own operation.
2410
2609
  *
2411
2610
  * The transform runs entirely inside SQLite. A production journal row is a
2412
2611
  * register-scale graph serialized three times over — the exact text whose
2413
2612
  * repeated JavaScript materialization blew the 512 MB heap — so no part of
2414
- * it may ever become a JavaScript value here: json_extract lifts the nested
2415
- * snapshot text straight into its blob, and the meta and operation rows are
2416
- * json_object over subtrees, all in engine memory outside the V8 heap.
2613
+ * it may ever become a JavaScript value here: json_extract lifts nested text
2614
+ * straight into its blob and meta rows are json_object over subtrees, all in
2615
+ * engine memory outside the V8 heap. The snapshot blob is simply not carried:
2616
+ * the authority stores the head now, and a replay re-reads it.
2417
2617
  *
2418
2618
  * Tables older than the three-state journal carry no status or submitted_at;
2419
2619
  * their rows were written before the pre-submit durability mark existed, so
2420
2620
  * the only safe reading is "possibly submitted": keep them pending and
2421
2621
  * marked submitted, which replays the same mutation id — always idempotent —
2422
- * and never supersedes. A row whose publication_json does not parse fails
2423
- * the open loudly with the journal intact; nothing is ever deleted on a
2424
- * conversion failure. */
2425
- function convertLegacySnapshotJournal(database, columns) {
2622
+ * and never supersedes. A row whose stored operation does not resolve fails
2623
+ * the open loudly against the NOT NULL law with the journal intact; nothing
2624
+ * is ever deleted on a conversion failure. */
2625
+ function convertLegacySnapshotJournal(database, shape, columns) {
2626
+ const metaJson = shape === "nested-text"
2627
+ ? `json_object(
2628
+ 'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
2629
+ 'baseVersion', json_extract(publication_json, '$.baseVersion'),
2630
+ 'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
2631
+ 'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
2632
+ 'operationKind', 'entity.roots.replace',
2633
+ 'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2634
+ 'uploadUuids', coalesce((
2635
+ SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
2636
+ FROM json_each(publication_json, '$.uploadRecords') AS uploaded
2637
+ ), json_array())
2638
+ )`
2639
+ : "meta_json";
2640
+ const operationBlob = {
2641
+ "nested-text": `CAST(json_object(
2642
+ 'kind', 'entity.roots.replace',
2643
+ 'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2644
+ 'records', json(json_extract(publication_json, '$.replacementRecords'))
2645
+ ) AS BLOB)`,
2646
+ "row-blobs": "operation_blob",
2647
+ "split-blobs": `(SELECT blobs.operation_blob FROM snapshot_publication_blobs AS blobs
2648
+ WHERE blobs.mutation_id = legacy.mutation_id)`,
2649
+ }[shape];
2426
2650
  database.transaction(() => {
2427
2651
  database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
2428
2652
  database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
2429
2653
  database.exec(`
2430
2654
  INSERT INTO snapshot_publications(
2431
- mutation_id, resource_id, meta_json, created_at, status, submitted_at
2655
+ mutation_id, resource_id, meta_json, operation_blob, created_at, status, submitted_at
2432
2656
  )
2433
2657
  SELECT
2434
2658
  mutation_id,
2435
2659
  resource_id,
2436
- json_object(
2437
- 'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
2438
- 'baseVersion', json_extract(publication_json, '$.baseVersion'),
2439
- 'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
2440
- 'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
2441
- 'operationKind', 'entity.roots.replace',
2442
- 'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2443
- 'uploadUuids', coalesce((
2444
- SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
2445
- FROM json_each(publication_json, '$.uploadRecords') AS uploaded
2446
- ), json_array())
2447
- ),
2660
+ ${metaJson},
2661
+ ${operationBlob},
2448
2662
  created_at,
2449
2663
  ${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
2450
2664
  ${columns.has("submitted_at") ? "submitted_at" : "created_at"}
2451
- FROM snapshot_publications_legacy ORDER BY sequence
2452
- `);
2453
- database.exec(`
2454
- INSERT INTO snapshot_publication_blobs(mutation_id, snapshot_blob, operation_blob)
2455
- SELECT
2456
- mutation_id,
2457
- CAST(json_extract(publication_json, '$.snapshotJson') AS BLOB),
2458
- CAST(json_object(
2459
- 'kind', 'entity.roots.replace',
2460
- 'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2461
- 'records', json(json_extract(publication_json, '$.replacementRecords'))
2462
- ) AS BLOB)
2463
- FROM snapshot_publications_legacy ORDER BY sequence
2464
- `);
2465
- database.exec("DROP TABLE snapshot_publications_legacy");
2466
- })();
2467
- }
2468
- /** Converts the single-table blob journal (blobs stored on the publication
2469
- * row itself) into the split journal, atomically and entirely inside SQLite —
2470
- * the same engine-memory law as the nested-text conversion above. Every
2471
- * field already has its final meaning; the blobs simply move to their
2472
- * write-once table so status transitions stop rewriting them. */
2473
- function splitSnapshotJournalBlobs(database) {
2474
- database.transaction(() => {
2475
- database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
2476
- database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
2477
- database.exec(`
2478
- INSERT INTO snapshot_publications(
2479
- mutation_id, resource_id, meta_json, created_at, status, submitted_at
2480
- )
2481
- SELECT mutation_id, resource_id, meta_json, created_at, status, submitted_at
2482
- FROM snapshot_publications_legacy ORDER BY sequence
2483
- `);
2484
- database.exec(`
2485
- INSERT INTO snapshot_publication_blobs(mutation_id, snapshot_blob, operation_blob)
2486
- SELECT mutation_id, snapshot_blob, operation_blob
2487
- FROM snapshot_publications_legacy ORDER BY sequence
2665
+ FROM snapshot_publications_legacy AS legacy ORDER BY sequence
2488
2666
  `);
2489
2667
  database.exec("DROP TABLE snapshot_publications_legacy");
2668
+ database.exec("DROP TABLE IF EXISTS snapshot_publication_blobs");
2490
2669
  })();
2491
2670
  }
2492
2671
  /** The single opener and schema-migration entry for a user ground database.
@@ -2505,12 +2684,9 @@ export function initializeDatabase(file) {
2505
2684
  // how this file arrived at this process. One PRAGMA against the in-memory
2506
2685
  // catalog costs nothing next to the connection open itself.
2507
2686
  const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
2508
- if (publicationColumns.has("publication_json")) {
2509
- convertLegacySnapshotJournal(database, publicationColumns);
2510
- }
2511
- else if (publicationColumns.has("snapshot_blob")) {
2512
- splitSnapshotJournalBlobs(database);
2513
- }
2687
+ const legacyShape = legacyJournalShape(publicationColumns);
2688
+ if (legacyShape)
2689
+ convertLegacySnapshotJournal(database, legacyShape, publicationColumns);
2514
2690
  if (migratedDatabases.has(file))
2515
2691
  return database;
2516
2692
  database.exec(`
@@ -3086,29 +3262,36 @@ function readSnapshotPublications(file) {
3086
3262
  return [];
3087
3263
  const database = initializeDatabase(file);
3088
3264
  try {
3265
+ const pages = database.prepare(`
3266
+ SELECT mutation_id AS mutationId, sha256, bytes
3267
+ FROM snapshot_publication_pages ORDER BY mutation_id, page_index
3268
+ `).all();
3269
+ const pagesByMutation = new Map();
3270
+ for (const page of pages) {
3271
+ const carried = pagesByMutation.get(page.mutationId) ?? [];
3272
+ carried.push({ sha256: page.sha256, bytes: page.bytes });
3273
+ pagesByMutation.set(page.mutationId, carried);
3274
+ }
3089
3275
  return database.prepare(`
3090
- SELECT publications.mutation_id AS mutationId,
3091
- publications.resource_id AS resourceId,
3092
- publications.meta_json AS metaJson,
3093
- blobs.snapshot_blob AS snapshotBlob,
3094
- blobs.operation_blob AS operationBlob,
3095
- publications.status,
3096
- publications.submitted_at AS submittedAt
3097
- FROM snapshot_publications AS publications
3098
- LEFT JOIN snapshot_publication_blobs AS blobs USING (mutation_id)
3099
- ORDER BY publications.sequence
3276
+ SELECT mutation_id AS mutationId,
3277
+ resource_id AS resourceId,
3278
+ meta_json AS metaJson,
3279
+ operation_blob AS operationBlob,
3280
+ status,
3281
+ submitted_at AS submittedAt
3282
+ FROM snapshot_publications ORDER BY sequence
3100
3283
  `).all().map((row) => {
3101
- // A publication without its payload is damage, never a skippable row —
3102
- // silently dropping it would let a new publication supersede a
3103
- // submitted mutation whose outcome exists only at the authority.
3104
- if (!row.snapshotBlob || !row.operationBlob) {
3105
- throw new Error(`snapshot publication ${row.mutationId} has no journaled payload`);
3106
- }
3107
3284
  const meta = JSON.parse(row.metaJson);
3108
- // No lawful publication carries zero roots — rootReplacementFromRecords
3109
- // refuses them at build time, so a rootless row here is damage.
3285
+ // No lawful publication carries zero roots — a head commit that replaces
3286
+ // nothing would prove nothing — so a rootless row here is damage, never
3287
+ // a skippable row: silently dropping it would let a new publication
3288
+ // supersede a submitted mutation whose outcome exists only at the
3289
+ // authority. `entity.roots.replace` appears only on rows converted from
3290
+ // a pre-paged journal, which no longer produce but must still replay.
3110
3291
  if (!Array.isArray(meta.replacementRootIds) || meta.replacementRootIds.length === 0
3111
- || !Array.isArray(meta.uploadUuids) || meta.operationKind !== "entity.roots.replace") {
3292
+ || !Array.isArray(meta.uploadUuids)
3293
+ || (meta.operationKind !== GRAPH_HEAD_OPERATION_KIND
3294
+ && meta.operationKind !== "entity.roots.replace")) {
3112
3295
  throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
3113
3296
  }
3114
3297
  const deliveryWatermark = Number(meta.deliveryWatermark);
@@ -3122,11 +3305,11 @@ function readSnapshotPublications(file) {
3122
3305
  baseVersion: Number(meta.baseVersion),
3123
3306
  deliveryWatermark,
3124
3307
  snapshotChecksum: String(meta.snapshotChecksum),
3125
- operationKind: meta.operationKind,
3308
+ operationKind: String(meta.operationKind),
3126
3309
  replacementRootIds: meta.replacementRootIds.map(String),
3127
3310
  uploadUuids: meta.uploadUuids.map(String),
3128
- snapshotBlob: row.snapshotBlob,
3129
3311
  operationBlob: row.operationBlob,
3312
+ pages: pagesByMutation.get(row.mutationId) ?? [],
3130
3313
  status: row.status === "rejected" ? "rejected" : "pending",
3131
3314
  submitted: row.submittedAt !== null,
3132
3315
  };
@@ -3294,7 +3477,7 @@ function supersedeSnapshotJournal(file, entry) {
3294
3477
  throw new Error("a submitted publication with an unknown outcome must resolve before supersession");
3295
3478
  }
3296
3479
  database.prepare(`
3297
- DELETE FROM snapshot_publication_blobs WHERE mutation_id IN (
3480
+ DELETE FROM snapshot_publication_pages WHERE mutation_id IN (
3298
3481
  SELECT mutation_id FROM snapshot_publications WHERE resource_id = ?
3299
3482
  )
3300
3483
  `).run(entry.resourceId);
@@ -3302,8 +3485,8 @@ function supersedeSnapshotJournal(file, entry) {
3302
3485
  .run(entry.resourceId);
3303
3486
  database.prepare(`
3304
3487
  INSERT INTO snapshot_publications(
3305
- mutation_id, resource_id, meta_json, created_at
3306
- ) VALUES (?, ?, ?, ?)
3488
+ mutation_id, resource_id, meta_json, operation_blob, created_at
3489
+ ) VALUES (?, ?, ?, ?, ?)
3307
3490
  `).run(entry.mutationId, entry.resourceId, stableJson({
3308
3491
  authorityEpoch: entry.authorityEpoch,
3309
3492
  baseVersion: entry.baseVersion,
@@ -3312,11 +3495,12 @@ function supersedeSnapshotJournal(file, entry) {
3312
3495
  operationKind: entry.operationKind,
3313
3496
  replacementRootIds: entry.replacementRootIds,
3314
3497
  uploadUuids: entry.uploadUuids,
3315
- }), Date.now());
3316
- database.prepare(`
3317
- INSERT INTO snapshot_publication_blobs(mutation_id, snapshot_blob, operation_blob)
3318
- VALUES (?, ?, ?)
3319
- `).run(entry.mutationId, entry.snapshotBlob, entry.operationBlob);
3498
+ }), entry.operationBlob, Date.now());
3499
+ const insertPage = database.prepare(`
3500
+ INSERT INTO snapshot_publication_pages(mutation_id, page_index, sha256, bytes)
3501
+ VALUES (?, ?, ?, ?)
3502
+ `);
3503
+ entry.pages.forEach((page, index) => insertPage.run(entry.mutationId, index, page.sha256, page.bytes));
3320
3504
  })();
3321
3505
  }
3322
3506
  finally {
@@ -3387,7 +3571,7 @@ function deleteJournalEntry(file, mutationId) {
3387
3571
  const database = initializeDatabase(file);
3388
3572
  try {
3389
3573
  database.transaction(() => {
3390
- database.prepare("DELETE FROM snapshot_publication_blobs WHERE mutation_id = ?").run(mutationId);
3574
+ database.prepare("DELETE FROM snapshot_publication_pages WHERE mutation_id = ?").run(mutationId);
3391
3575
  database.prepare("DELETE FROM snapshot_publications WHERE mutation_id = ?").run(mutationId);
3392
3576
  })();
3393
3577
  }
@@ -4981,19 +5165,79 @@ function recordRootResolver(records) {
4981
5165
  };
4982
5166
  return rootOf;
4983
5167
  }
4984
- function mergeMaterializedRoots(cloudRecords, localRecords) {
4985
- const localRoots = new Set(localRecords
5168
+ /** The roots one publication rebuilds, each with its complete new records.
5169
+ *
5170
+ * A walked root is rebuilt from its ground alone: materialized ground is the
5171
+ * whole truth for the roots it covers, so nothing of the committed graph
5172
+ * survives underneath one. Any other root the newly delivered authority prefix
5173
+ * touched is rebuilt from the committed graph with those results folded in, so
5174
+ * the published graph never declares a watermark it has not actually applied.
5175
+ * Every root outside this map keeps its committed pages untouched. */
5176
+ function publicationRoots(graph, localRecords, accepted) {
5177
+ const rebuilt = new Map();
5178
+ const walked = new Map(localRecords
4986
5179
  .filter((record) => record.parentUUID === null)
4987
- .map((record) => record.uuid));
4988
- const localIds = new Set(localRecords.map((record) => record.uuid));
4989
- const rootOf = recordRootResolver(cloudRecords);
4990
- const preserved = cloudRecords.filter((record) => {
4991
- if (localIds.has(record.uuid))
4992
- return false;
4993
- const root = rootOf(record);
4994
- return root === null || !localRoots.has(root);
4995
- });
4996
- return [...preserved, ...localRecords];
5180
+ .map((record) => [record.uuid, []]));
5181
+ if (walked.size > 0) {
5182
+ const rootOf = recordRootResolver(localRecords);
5183
+ for (const record of localRecords) {
5184
+ const rootId = rootOf(record);
5185
+ const group = rootId === null ? undefined : walked.get(rootId);
5186
+ if (!group)
5187
+ throw new Error(`walked record ${record.uuid} descends from no walked root`);
5188
+ group.push(record);
5189
+ }
5190
+ for (const [rootId, records] of walked)
5191
+ rebuilt.set(rootId, records);
5192
+ }
5193
+ const overlays = new Map();
5194
+ for (const result of accepted) {
5195
+ const rootId = graph.rootOf(result.uuid);
5196
+ if (rootId === null || walked.has(rootId))
5197
+ continue;
5198
+ const overlay = overlays.get(rootId);
5199
+ if (overlay)
5200
+ overlay.push(result);
5201
+ else
5202
+ overlays.set(rootId, [result]);
5203
+ }
5204
+ for (const [rootId, results] of overlays) {
5205
+ const byUuid = new Map(graph.recordsFor(rootId).map((record) => [record.uuid, record]));
5206
+ for (const result of results)
5207
+ byUuid.set(result.uuid, result);
5208
+ // Entity channels are independently ordered, so container membership is
5209
+ // derived once from the final prefix rather than by arrival order.
5210
+ rebuilt.set(rootId, recordsRetainingMissingEvidence([...byUuid.values()], []));
5211
+ }
5212
+ // A records-shaped authority base carries no page identities, so it can
5213
+ // prove nothing about a root the commit leaves alone: the first head pages
5214
+ // the whole account at once, and never again.
5215
+ if (graph.head === null) {
5216
+ for (const rootId of graph.rootIds()) {
5217
+ if (!rebuilt.has(rootId))
5218
+ rebuilt.set(rootId, graph.recordsFor(rootId));
5219
+ }
5220
+ }
5221
+ // Re-rooting is a move, not a copy: an enclosing Register walks records the
5222
+ // committed graph still files under the root that used to hold them. Every
5223
+ // such root is rebuilt without them, and a root that keeps nothing is left
5224
+ // with no records at all — the rebuild then drops it from the graph.
5225
+ const vacated = new Map();
5226
+ for (const record of localRecords) {
5227
+ const rootId = graph.rootOf(record.uuid);
5228
+ if (rootId === null || walked.has(rootId))
5229
+ continue;
5230
+ const gone = vacated.get(rootId);
5231
+ if (gone)
5232
+ gone.add(record.uuid);
5233
+ else
5234
+ vacated.set(rootId, new Set([record.uuid]));
5235
+ }
5236
+ for (const [rootId, gone] of vacated) {
5237
+ const held = rebuilt.get(rootId) ?? graph.recordsFor(rootId);
5238
+ rebuilt.set(rootId, held.filter((record) => !gone.has(record.uuid)));
5239
+ }
5240
+ return rebuilt;
4997
5241
  }
4998
5242
  /** Filesystem disappearance is evidence, not cloud lifecycle authority. Keep
4999
5243
  * the last logical records in the outgoing graph while their materialized
@@ -5566,14 +5810,14 @@ async function installCloudWorkspace(input) {
5566
5810
  alreadyMaterialized: false,
5567
5811
  };
5568
5812
  }
5569
- function readyFromFrame(frame) {
5570
- const bytes = Buffer.from(String(frame.snapshot_b64 || ""), "base64");
5813
+ function readyFromFrame(frame, graph) {
5571
5814
  const cloud = {
5572
5815
  resourceId: String(frame.resource_id || ""),
5573
5816
  version: Number(frame.head_version),
5574
5817
  deliveryWatermark: Number(frame.delivery_watermark),
5575
5818
  checksum: String(frame.snapshot_checksum || ""),
5576
- snapshot: parseSnapshot(bytes),
5819
+ snapshot: graph.snapshot(),
5820
+ ...(graph.head ? { head: graph.head } : {}),
5577
5821
  };
5578
5822
  return {
5579
5823
  cloud,
@@ -5583,7 +5827,7 @@ function readyFromFrame(frame) {
5583
5827
  headVersion: cloud.version,
5584
5828
  checksum: cloud.checksum,
5585
5829
  deliveryWatermark: cloud.deliveryWatermark,
5586
- records: cloud.snapshot.records,
5830
+ graph,
5587
5831
  },
5588
5832
  };
5589
5833
  }