@amalgm/shell 0.1.110 → 0.1.111

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,8 +9,10 @@ type Frame = Record<string, unknown> & {
9
9
  };
10
10
  /** Publishes one mutation operation of any size as ONE atomic commit.
11
11
  *
12
- * A small operation travels inline in the submit frame, byte-identical to
13
- * the pre-paging wire. A larger one stages as byte pages of its canonical
12
+ * The operation arrives as its canonical UTF-8 bytes, serialized exactly
13
+ * once by the caller — this function never re-serializes a graph. A small
14
+ * operation travels inline in the submit frame, byte-identical to the
15
+ * pre-paging wire. A larger one stages as byte pages of its canonical
14
16
  * JSON, then submits a reference the gateway assembles into the same commit
15
17
  * path — transport pages, the operation never does. Every attempt stages
16
18
  * afresh, so a replay after a lost acknowledgement or dropped connection is
@@ -22,5 +24,5 @@ type Frame = Record<string, unknown> & {
22
24
  * bridge — the gateway reassembles the whole graph and the database journals
23
25
  * it as one large operation; the eventual endpoint is immutable graph pages
24
26
  * with a small atomically-committed graph-head pointer. */
25
- export declare function submitMutationOperation(wire: Pick<WireClient, "request" | "requestBinary">, envelope: Record<string, unknown>, operation: unknown): Promise<Frame>;
27
+ export declare function submitMutationOperation(wire: Pick<WireClient, "request" | "requestBinary">, envelope: Record<string, unknown>, operation: Buffer, onLeg?: (leg: "stage" | "submit", durationMs: number) => void): Promise<Frame>;
26
28
  export {};
@@ -6,8 +6,10 @@ export const MAX_INLINE_OPERATION_BYTES = 8 * 1024 * 1024;
6
6
  export const MUTATION_PAGE_BYTES = 4 * 1024 * 1024;
7
7
  /** Publishes one mutation operation of any size as ONE atomic commit.
8
8
  *
9
- * A small operation travels inline in the submit frame, byte-identical to
10
- * the pre-paging wire. A larger one stages as byte pages of its canonical
9
+ * The operation arrives as its canonical UTF-8 bytes, serialized exactly
10
+ * once by the caller — this function never re-serializes a graph. A small
11
+ * operation travels inline in the submit frame, byte-identical to the
12
+ * pre-paging wire. A larger one stages as byte pages of its canonical
11
13
  * JSON, then submits a reference the gateway assembles into the same commit
12
14
  * path — transport pages, the operation never does. Every attempt stages
13
15
  * afresh, so a replay after a lost acknowledgement or dropped connection is
@@ -19,13 +21,16 @@ export const MUTATION_PAGE_BYTES = 4 * 1024 * 1024;
19
21
  * bridge — the gateway reassembles the whole graph and the database journals
20
22
  * it as one large operation; the eventual endpoint is immutable graph pages
21
23
  * with a small atomically-committed graph-head pointer. */
22
- export async function submitMutationOperation(wire, envelope, operation) {
23
- const canonical = Buffer.from(JSON.stringify(operation), "utf8");
24
+ export async function submitMutationOperation(wire, envelope, operation, onLeg) {
25
+ const canonical = operation;
26
+ const started = performance.now();
24
27
  if (canonical.byteLength <= MAX_INLINE_OPERATION_BYTES) {
25
- return wire.request({
28
+ const frame = await wire.request({
26
29
  type: "shared.mutation.submit",
27
- envelope: { ...envelope, operation },
30
+ envelope: { ...envelope, operation: JSON.parse(canonical.toString("utf8")) },
28
31
  }, ["shared.mutation.ack"]);
32
+ onLeg?.("submit", performance.now() - started);
33
+ return frame;
29
34
  }
30
35
  const staged = {
31
36
  page_count: Math.ceil(canonical.byteLength / MUTATION_PAGE_BYTES),
@@ -43,10 +48,14 @@ export async function submitMutationOperation(wire, envelope, operation) {
43
48
  bytes: page.byteLength,
44
49
  }, page, ["shared.mutation.staged"]);
45
50
  }
46
- return wire.request({
51
+ const submitStarted = performance.now();
52
+ onLeg?.("stage", submitStarted - started);
53
+ const frame = await wire.request({
47
54
  type: "shared.mutation.submit",
48
55
  envelope,
49
56
  staged,
50
57
  }, ["shared.mutation.ack"]);
58
+ onLeg?.("submit", performance.now() - submitStarted);
59
+ return frame;
51
60
  }
52
61
  //# sourceMappingURL=paged-mutation-submit.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"paged-mutation-submit.js","sourceRoot":"","sources":["../src/paged-mutation-submit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC;0EAC0E;AAC1E,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAE1D,+EAA+E;AAC/E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAInD;;;;;;;;;;;;;;2DAc2D;AAC3D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAmD,EACnD,QAAiC,EACjC,SAAkB;IAElB,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IACjE,IAAI,SAAS,CAAC,UAAU,IAAI,0BAA0B,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,OAAO,CAAC;YAClB,IAAI,EAAE,wBAAwB;YAC9B,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,SAAS,EAAE;SACrC,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAC9B,CAAC;IACD,MAAM,MAAM,GAAG;QACb,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,mBAAmB,CAAC;QACjE,eAAe,EAAE,SAAS,CAAC,UAAU;QACrC,gBAAgB,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;KACvE,CAAC;IACF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAC7B,KAAK,GAAG,mBAAmB,EAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,mBAAmB,EAAE,SAAS,CAAC,UAAU,CAAC,CAClE,CAAC;QACF,MAAM,IAAI,CAAC,aAAa,CAAC;YACvB,IAAI,EAAE,uBAAuB;YAC7B,WAAW,EAAE,QAAQ,CAAC,UAAU;YAChC,WAAW,EAAE,QAAQ,CAAC,UAAU;YAChC,UAAU,EAAE,KAAK;YACjB,GAAG,MAAM;YACT,KAAK,EAAE,IAAI,CAAC,UAAU;SACvB,EAAE,IAAI,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,CAAC;QAClB,IAAI,EAAE,wBAAwB;QAC9B,QAAQ;QACR,MAAM;KACP,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;AAC9B,CAAC"}
1
+ {"version":3,"file":"paged-mutation-submit.js","sourceRoot":"","sources":["../src/paged-mutation-submit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC;0EAC0E;AAC1E,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAE1D,+EAA+E;AAC/E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAInD;;;;;;;;;;;;;;;;2DAgB2D;AAC3D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAmD,EACnD,QAAiC,EACjC,SAAiB,EACjB,KAA6D;IAE7D,MAAM,SAAS,GAAG,SAAS,CAAC;IAC5B,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,IAAI,SAAS,CAAC,UAAU,IAAI,0BAA0B,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAC/B,IAAI,EAAE,wBAAwB;YAC9B,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE;SAC7E,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC5B,KAAK,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG;QACb,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,mBAAmB,CAAC;QACjE,eAAe,EAAE,SAAS,CAAC,UAAU;QACrC,gBAAgB,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;KACvE,CAAC;IACF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAC7B,KAAK,GAAG,mBAAmB,EAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,mBAAmB,EAAE,SAAS,CAAC,UAAU,CAAC,CAClE,CAAC;QACF,MAAM,IAAI,CAAC,aAAa,CAAC;YACvB,IAAI,EAAE,uBAAuB;YAC7B,WAAW,EAAE,QAAQ,CAAC,UAAU;YAChC,WAAW,EAAE,QAAQ,CAAC,UAAU;YAChC,UAAU,EAAE,KAAK;YACjB,GAAG,MAAM;YACT,KAAK,EAAE,IAAI,CAAC,UAAU;SACvB,EAAE,IAAI,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACxC,KAAK,EAAE,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;QAC/B,IAAI,EAAE,wBAAwB;QAC9B,QAAQ;QACR,MAAM;KACP,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAC5B,KAAK,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -1,5 +1,6 @@
1
1
  import { type UserGroundPort, type UserHomeDeclaration } from "@amalgm/core/identity";
2
2
  import { type EntityType, type FilesAddPorts, type FilesRegisterPorts, type UserGroundIdentity } from "@amalgm/live";
3
+ import Database from "better-sqlite3";
3
4
  import { type FilesStageObserver } from "./files-observability.js";
4
5
  import { type EntityApplyStageEvidence } from "./entity-apply-host.js";
5
6
  import { type OpenNativeWatch, type WatchCoverageEvidence, type WatchHealth } from "./watching/index.js";
@@ -184,5 +185,11 @@ export declare class UserGroundHost {
184
185
  * frames; the answer is the only permission to skip an upload. */
185
186
  private sealedContentHashes;
186
187
  private publishPendingSnapshots;
188
+ private publishLoadedSnapshot;
187
189
  }
190
+ /** The single opener and schema-migration entry for a user ground database.
191
+ * Exported so the upgrade path is provable in isolation — the journal
192
+ * conversion law test opens a legacy file through this exact entry inside a
193
+ * heap-capped child process. */
194
+ export declare function initializeDatabase(file: string): Database.Database;
188
195
  export {};
@@ -55,6 +55,15 @@ const WALK_CAPTURE_CONCURRENCY = 8;
55
55
  const REGISTER_STALE_CAPTURE_ATTEMPTS = 3;
56
56
  const DOWNLOAD_CONTENT_RETRY_ATTEMPTS = 3;
57
57
  const APPLY_GROUND_WAIT_TIMEOUT_MS = 15_000;
58
+ function loadSnapshotPublication(entry) {
59
+ const snapshotRecords = snapshotFromRecords((JSON.parse(entry.snapshotBlob.toString("utf8")).records ?? [])).records;
60
+ const uploadIds = new Set(entry.uploadUuids);
61
+ return {
62
+ ...entry,
63
+ snapshotRecords,
64
+ uploadRecords: snapshotRecords.filter((record) => uploadIds.has(record.uuid)),
65
+ };
66
+ }
58
67
  const GROUND_ROW_COLUMNS = [
59
68
  "uuid", "resource_id", "root_uuid", "type", "parent_uuid", "name", "status", "version",
60
69
  "payload_version", "transport_version", "relative_path", "absolute_path", "device_number",
@@ -1939,7 +1948,7 @@ export class UserGroundHost {
1939
1948
  headVersion: first.baseVersion,
1940
1949
  checksum: "",
1941
1950
  deliveryWatermark: first.deliveryWatermark,
1942
- records: snapshotFromRecords(JSON.parse(first.snapshotJson).records || []).records,
1951
+ records: loadSnapshotPublication(first).snapshotRecords,
1943
1952
  };
1944
1953
  try {
1945
1954
  await this.publishPendingSnapshots(identity);
@@ -2003,32 +2012,27 @@ export class UserGroundHost {
2003
2012
  throw new Error(`repository-owned leaf escaped its rail: ${impossibleGitLeaf.name} parent=${parent?.type ?? "missing"}`);
2004
2013
  }
2005
2014
  const snapshot = snapshotFromRecords(traveling);
2006
- const snapshotJson = stableJson(snapshot);
2007
- const checksum = sha256Hex(snapshotJson);
2015
+ // ONE canonical serialization: these exact bytes are the journal blob
2016
+ // and the checksum's input; they are never re-derived or re-parsed on
2017
+ // the in-process path.
2018
+ const snapshotBlob = Buffer.from(stableJson(snapshot), "utf8");
2019
+ const checksum = sha256Hex(snapshotBlob);
2008
2020
  const localIds = new Set(localRecords.map((record) => record.uuid));
2009
2021
  // Graph membership is never proof of sealed cargo: a record in the cloud
2010
2022
  // graph may name content sealed only under a prior content contract,
2011
2023
  // whose namespace the current readers never touch. Only the authority's
2012
- // (resource, hash, contract) seal evidence may skip an upload — and the
2013
- // question is asked even when the graph itself is unchanged, so Register
2014
- // always re-establishes cargo closure before returning.
2015
- const cargo = snapshot.records.flatMap((record) => {
2016
- if (!localIds.has(record.uuid))
2017
- return [];
2018
- const artifact = artifactForRecord(record);
2019
- return artifact ? [{ record, contentHash: artifact.contentHash }] : [];
2020
- });
2021
- const sealedArtifacts = await this.sealedContentHashes(state.resourceId, [...new Set(cargo.map((entry) => entry.contentHash))]);
2022
- const uploadRecords = cargo
2023
- .filter((entry) => !sealedArtifacts.has(entry.contentHash))
2024
- .map((entry) => entry.record);
2024
+ // (resource, hash, contract) seal evidence may skip an upload — the
2025
+ // sealed-first upload asks that exact question itself and moves only
2026
+ // what is unsealed, so Register always re-establishes cargo closure
2027
+ // before returning, even when the graph itself is unchanged.
2028
+ const cargoRecords = snapshot.records.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
2025
2029
  if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
2026
2030
  // The graph is already committed; only cargo can be missing. Repair
2027
2031
  // uploads need no journal row — a crash simply re-runs the same
2028
2032
  // repair on the next Register.
2029
- if (uploadRecords.length > 0) {
2033
+ if (cargoRecords.length > 0) {
2030
2034
  try {
2031
- await this.uploadSnapshotContent(identity, state.resourceId, uploadRecords, trace, manifestHints);
2035
+ await this.uploadSnapshotContent(identity, state.resourceId, cargoRecords, trace, manifestHints);
2032
2036
  }
2033
2037
  catch (error) {
2034
2038
  throw pipelineStageError("cloud upload", error);
@@ -2046,20 +2050,32 @@ export class UserGroundHost {
2046
2050
  return root !== null && replacementRoots.has(root);
2047
2051
  });
2048
2052
  const replacement = rootReplacementFromRecords(replacementRootIds, replacementRecords);
2049
- supersedeSnapshotJournal(this.databasePath(identity), {
2053
+ // The operation's canonical bytes: staged, hashed, and submitted as-is.
2054
+ const operationBlob = Buffer.from(stableJson(replacement), "utf8");
2055
+ const entry = {
2050
2056
  mutationId: randomUUID(),
2051
2057
  resourceId: state.resourceId,
2052
2058
  authorityEpoch: state.authorityEpoch,
2053
2059
  baseVersion: state.headVersion,
2054
2060
  snapshotChecksum: checksum,
2055
- snapshotJson,
2056
2061
  deliveryWatermark,
2057
- uploadRecords,
2062
+ operationKind: replacement.kind,
2058
2063
  replacementRootIds: replacement.rootIds,
2059
- replacementRecords: replacement.records,
2064
+ uploadUuids: cargoRecords.map((record) => record.uuid),
2065
+ snapshotBlob,
2066
+ operationBlob,
2060
2067
  status: "pending",
2061
- });
2062
- await this.publishPendingSnapshots(identity, trace, manifestHints);
2068
+ submitted: false,
2069
+ };
2070
+ supersedeSnapshotJournal(this.databasePath(identity), entry);
2071
+ // Publish the freshly journaled intent directly: the records are already
2072
+ // in memory and the operation bytes already exist — nothing is re-read,
2073
+ // re-parsed, or re-serialized between prepare and the wire.
2074
+ await this.publishLoadedSnapshot(identity, {
2075
+ ...entry,
2076
+ snapshotRecords: snapshot.records,
2077
+ uploadRecords: cargoRecords,
2078
+ }, trace, manifestHints);
2063
2079
  }
2064
2080
  /** Which of these heads does the authority hold sealed under the current
2065
2081
  * content contract? Batched so a whole workspace costs a handful of
@@ -2091,88 +2107,94 @@ export class UserGroundHost {
2091
2107
  // snapshot superseding it can publish this resource again.
2092
2108
  if (pending.status === "rejected")
2093
2109
  continue;
2094
- const snapshot = snapshotFromRecords(JSON.parse(pending.snapshotJson).records || []);
2095
- const replacement = rootReplacementFromRecords(pending.replacementRootIds, pending.replacementRecords);
2096
- try {
2097
- await this.uploadSnapshotContent(identity, pending.resourceId, pending.uploadRecords, trace, manifestHints);
2098
- }
2099
- catch (error) {
2100
- // An unsubmitted pending row is a local intent the authority has
2101
- // never seen. When its captured ground moved before upload, the
2102
- // intent is simply stale: delete it and let the caller re-derive
2103
- // one from current ground. A submitted row never lands here — its
2104
- // upload sealed completely before the first submit frame left.
2105
- if (staleCapture(error) && !pending.submitted) {
2106
- deleteJournalEntry(this.databasePath(identity), pending.mutationId);
2107
- }
2108
- throw pipelineStageError("cloud upload", error);
2109
- }
2110
- let frame;
2111
- const graphCommitStarted = performance.now();
2112
- if (trace?.journey === "register") {
2113
- this.stage({
2114
- primitive: "register",
2115
- stage: "cloud-graph-commit",
2116
- status: "started",
2117
- ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2118
- ...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
2119
- });
2120
- }
2121
- if (!pending.submitted) {
2122
- markSnapshotJournalSubmitted(this.databasePath(identity), pending.mutationId);
2123
- }
2124
- try {
2125
- frame = await submitMutationOperation(this.wire, {
2126
- resourceId: pending.resourceId,
2127
- authorityEpoch: pending.authorityEpoch,
2128
- mutationId: pending.mutationId,
2129
- deviceId: identity.deviceId,
2130
- baseVersion: pending.baseVersion,
2131
- contract: ENTITY_CLOUD_CONTRACT,
2132
- schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
2133
- operationKind: replacement.kind,
2134
- deliveryWatermark: pending.deliveryWatermark,
2135
- }, replacement);
2136
- }
2137
- catch (error) {
2138
- if (trace?.journey === "register") {
2139
- this.stage({
2140
- primitive: "register",
2141
- stage: "cloud-graph-commit",
2142
- status: "failed",
2143
- ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2144
- durationMs: performance.now() - graphCommitStarted,
2145
- });
2146
- }
2147
- // A non-retryable error frame is the authority's answer: this exact
2148
- // mutation is refused forever. Settle the row so it is never
2149
- // replayed and a newer snapshot may supersede it. Every other
2150
- // failure leaves the outcome unknown and the row pending.
2151
- if (error instanceof WireRequestError && error.retryable === false) {
2152
- markSnapshotJournalRejected(this.databasePath(identity), pending.mutationId);
2153
- }
2154
- throw pipelineStageError("cloud graph commit", error);
2110
+ await this.publishLoadedSnapshot(identity, loadSnapshotPublication(pending), trace, manifestHints);
2111
+ }
2112
+ }
2113
+ async publishLoadedSnapshot(identity, publication, trace, manifestHints) {
2114
+ try {
2115
+ await this.uploadSnapshotContent(identity, publication.resourceId, publication.uploadRecords, trace, manifestHints);
2116
+ }
2117
+ catch (error) {
2118
+ // An unsubmitted pending row is a local intent the authority has
2119
+ // never seen. When its captured ground moved before upload, the
2120
+ // intent is simply stale: delete it and let the caller re-derive
2121
+ // one from current ground. A submitted row never lands here — its
2122
+ // upload sealed completely before the first submit frame left.
2123
+ if (staleCapture(error) && !publication.submitted) {
2124
+ deleteJournalEntry(this.databasePath(identity), publication.mutationId);
2155
2125
  }
2126
+ throw pipelineStageError("cloud upload", error);
2127
+ }
2128
+ let frame;
2129
+ const graphCommitStarted = performance.now();
2130
+ if (trace?.journey === "register") {
2131
+ this.stage({
2132
+ primitive: "register",
2133
+ stage: "cloud-graph-commit",
2134
+ status: "started",
2135
+ ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2136
+ ...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
2137
+ });
2138
+ }
2139
+ if (!publication.submitted) {
2140
+ markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId);
2141
+ }
2142
+ const legMs = {};
2143
+ try {
2144
+ frame = await submitMutationOperation(this.wire, {
2145
+ resourceId: publication.resourceId,
2146
+ authorityEpoch: publication.authorityEpoch,
2147
+ mutationId: publication.mutationId,
2148
+ deviceId: identity.deviceId,
2149
+ baseVersion: publication.baseVersion,
2150
+ contract: ENTITY_CLOUD_CONTRACT,
2151
+ schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
2152
+ operationKind: publication.operationKind,
2153
+ deliveryWatermark: publication.deliveryWatermark,
2154
+ }, publication.operationBlob, (leg, durationMs) => { legMs[`${leg}Ms`] = durationMs; });
2155
+ }
2156
+ catch (error) {
2156
2157
  if (trace?.journey === "register") {
2157
2158
  this.stage({
2158
2159
  primitive: "register",
2159
2160
  stage: "cloud-graph-commit",
2160
- status: "completed",
2161
+ status: "failed",
2161
2162
  ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2162
2163
  durationMs: performance.now() - graphCommitStarted,
2163
- measurements: { version: Number(frame.version) },
2164
2164
  });
2165
2165
  }
2166
- this.cloudState = {
2167
- resourceId: pending.resourceId,
2168
- authorityEpoch: Number(frame.authority_epoch),
2169
- headVersion: Number(frame.version),
2170
- checksum: pending.snapshotChecksum,
2171
- deliveryWatermark: Number(frame.delivery_watermark),
2172
- records: snapshot.records,
2173
- };
2174
- deleteJournalEntry(this.databasePath(identity), pending.mutationId);
2166
+ // A non-retryable error frame is the authority's answer: this exact
2167
+ // mutation is refused forever. Settle the row so it is never
2168
+ // replayed and a newer snapshot may supersede it. Every other
2169
+ // failure leaves the outcome unknown and the row pending.
2170
+ if (error instanceof WireRequestError && error.retryable === false) {
2171
+ markSnapshotJournalRejected(this.databasePath(identity), publication.mutationId);
2172
+ }
2173
+ throw pipelineStageError("cloud graph commit", error);
2175
2174
  }
2175
+ if (trace?.journey === "register") {
2176
+ this.stage({
2177
+ primitive: "register",
2178
+ stage: "cloud-graph-commit",
2179
+ status: "completed",
2180
+ ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2181
+ durationMs: performance.now() - graphCommitStarted,
2182
+ measurements: {
2183
+ version: Number(frame.version),
2184
+ operationBytes: publication.operationBlob.byteLength,
2185
+ ...legMs,
2186
+ },
2187
+ });
2188
+ }
2189
+ this.cloudState = {
2190
+ resourceId: publication.resourceId,
2191
+ authorityEpoch: Number(frame.authority_epoch),
2192
+ headVersion: Number(frame.version),
2193
+ checksum: publication.snapshotChecksum,
2194
+ deliveryWatermark: Number(frame.delivery_watermark),
2195
+ records: publication.snapshotRecords,
2196
+ };
2197
+ deleteJournalEntry(this.databasePath(identity), publication.mutationId);
2176
2198
  }
2177
2199
  }
2178
2200
  const sha256Hex = (input) => createHash("sha256").update(input).digest("hex");
@@ -2264,12 +2286,99 @@ function assertHealthyWatch(evidence, rootId) {
2264
2286
  * file over a dozen times; the file itself is never deleted by the host, so
2265
2287
  * every later open needs only the connection and its pragmas. */
2266
2288
  const migratedDatabases = new Set();
2267
- function initializeDatabase(file) {
2289
+ const SNAPSHOT_PUBLICATIONS_SCHEMA = `
2290
+ CREATE TABLE IF NOT EXISTS snapshot_publications (
2291
+ sequence INTEGER PRIMARY KEY AUTOINCREMENT,
2292
+ mutation_id TEXT NOT NULL UNIQUE,
2293
+ resource_id TEXT NOT NULL,
2294
+ meta_json TEXT NOT NULL,
2295
+ snapshot_blob BLOB NOT NULL,
2296
+ operation_blob BLOB NOT NULL,
2297
+ created_at INTEGER NOT NULL,
2298
+ status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
2299
+ submitted_at INTEGER
2300
+ );`;
2301
+ /** Converts an old-shape publication journal (one nested publication_json
2302
+ * text per row) into the blob journal, atomically, preserving every row's
2303
+ * meaning. A pending row may be a submitted mutation whose outcome exists
2304
+ * only at the authority — dropping it would violate the retry law that
2305
+ * recovers a lost acknowledgement — so conversion is structural: the nested
2306
+ * snapshot text becomes the snapshot blob verbatim, the stored replacement
2307
+ * becomes the operation blob, and semantic judgment stays where it always
2308
+ * was, with the reader (an unlawful row still blocks registration rather
2309
+ * than being silently replaced).
2310
+ *
2311
+ * The transform runs entirely inside SQLite. A production journal row is a
2312
+ * register-scale graph serialized three times over — the exact text whose
2313
+ * repeated JavaScript materialization blew the 512 MB heap — so no part of
2314
+ * it may ever become a JavaScript value here: json_extract lifts the nested
2315
+ * snapshot text straight into its blob, and the meta and operation rows are
2316
+ * json_object over subtrees, all in engine memory outside the V8 heap.
2317
+ *
2318
+ * Tables older than the three-state journal carry no status or submitted_at;
2319
+ * their rows were written before the pre-submit durability mark existed, so
2320
+ * the only safe reading is "possibly submitted": keep them pending and
2321
+ * marked submitted, which replays the same mutation id — always idempotent —
2322
+ * and never supersedes. A row whose publication_json does not parse fails
2323
+ * the open loudly with the journal intact; nothing is ever deleted on a
2324
+ * conversion failure. */
2325
+ function convertLegacySnapshotJournal(database, columns) {
2326
+ database.transaction(() => {
2327
+ database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
2328
+ database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
2329
+ database.exec(`
2330
+ INSERT INTO snapshot_publications(
2331
+ mutation_id, resource_id, meta_json, snapshot_blob, operation_blob,
2332
+ created_at, status, submitted_at
2333
+ )
2334
+ SELECT
2335
+ mutation_id,
2336
+ resource_id,
2337
+ json_object(
2338
+ 'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
2339
+ 'baseVersion', json_extract(publication_json, '$.baseVersion'),
2340
+ 'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
2341
+ 'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
2342
+ 'operationKind', 'entity.roots.replace',
2343
+ 'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2344
+ 'uploadUuids', coalesce((
2345
+ SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
2346
+ FROM json_each(publication_json, '$.uploadRecords') AS uploaded
2347
+ ), json_array())
2348
+ ),
2349
+ CAST(json_extract(publication_json, '$.snapshotJson') AS BLOB),
2350
+ CAST(json_object(
2351
+ 'kind', 'entity.roots.replace',
2352
+ 'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2353
+ 'records', json(json_extract(publication_json, '$.replacementRecords'))
2354
+ ) AS BLOB),
2355
+ created_at,
2356
+ ${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
2357
+ ${columns.has("submitted_at") ? "submitted_at" : "created_at"}
2358
+ FROM snapshot_publications_legacy ORDER BY sequence
2359
+ `);
2360
+ database.exec("DROP TABLE snapshot_publications_legacy");
2361
+ })();
2362
+ }
2363
+ /** The single opener and schema-migration entry for a user ground database.
2364
+ * Exported so the upgrade path is provable in isolation — the journal
2365
+ * conversion law test opens a legacy file through this exact entry inside a
2366
+ * heap-capped child process. */
2367
+ export function initializeDatabase(file) {
2268
2368
  ensurePrivateDir(dirname(file));
2269
2369
  const database = new Database(file);
2270
2370
  database.pragma("busy_timeout = 5000");
2271
2371
  database.pragma("journal_mode = WAL");
2272
2372
  database.pragma("synchronous = FULL");
2373
+ // The legacy probe runs on every open, before the migration memo: a journal
2374
+ // row may be a submitted publication whose outcome exists only at the
2375
+ // authority, so an old-shape table must convert — never drop — no matter
2376
+ // how this file arrived at this process. One PRAGMA against the in-memory
2377
+ // catalog costs nothing next to the connection open itself.
2378
+ const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
2379
+ if (publicationColumns.has("publication_json")) {
2380
+ convertLegacySnapshotJournal(database, publicationColumns);
2381
+ }
2273
2382
  if (migratedDatabases.has(file))
2274
2383
  return database;
2275
2384
  database.exec(`
@@ -2361,15 +2470,7 @@ function initializeDatabase(file) {
2361
2470
  authority_id TEXT PRIMARY KEY,
2362
2471
  delivery_sequence INTEGER NOT NULL CHECK(delivery_sequence >= 0)
2363
2472
  );
2364
- CREATE TABLE IF NOT EXISTS snapshot_publications (
2365
- sequence INTEGER PRIMARY KEY AUTOINCREMENT,
2366
- mutation_id TEXT NOT NULL UNIQUE,
2367
- resource_id TEXT NOT NULL,
2368
- publication_json TEXT NOT NULL,
2369
- created_at INTEGER NOT NULL,
2370
- status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
2371
- submitted_at INTEGER
2372
- );
2473
+ ${SNAPSHOT_PUBLICATIONS_SCHEMA}
2373
2474
  CREATE TABLE IF NOT EXISTS workspace_add_intents (
2374
2475
  workspace_uuid TEXT PRIMARY KEY,
2375
2476
  destination_path TEXT NOT NULL UNIQUE,
@@ -2405,13 +2506,6 @@ function initializeDatabase(file) {
2405
2506
  if (!inboxColumns.has("resolved_at")) {
2406
2507
  database.exec("ALTER TABLE record_inbox ADD COLUMN resolved_at INTEGER");
2407
2508
  }
2408
- const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
2409
- if (!publicationColumns.has("status")) {
2410
- database.exec("ALTER TABLE snapshot_publications ADD COLUMN status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected'))");
2411
- }
2412
- if (!publicationColumns.has("submitted_at")) {
2413
- database.exec("ALTER TABLE snapshot_publications ADD COLUMN submitted_at INTEGER");
2414
- }
2415
2509
  database.exec("DROP INDEX IF EXISTS record_inbox_delivery");
2416
2510
  database.exec("DROP TABLE IF EXISTS mutation_journal");
2417
2511
  migratedDatabases.add(file);
@@ -2862,33 +2956,34 @@ function readSnapshotPublications(file) {
2862
2956
  try {
2863
2957
  return database.prepare(`
2864
2958
  SELECT mutation_id AS mutationId, resource_id AS resourceId,
2865
- publication_json AS publicationJson, status,
2959
+ meta_json AS metaJson, snapshot_blob AS snapshotBlob,
2960
+ operation_blob AS operationBlob, status,
2866
2961
  submitted_at AS submittedAt
2867
2962
  FROM snapshot_publications ORDER BY sequence
2868
2963
  `).all().map((row) => {
2869
- const publication = JSON.parse(row.publicationJson);
2870
- if (!Array.isArray(publication.uploadRecords)) {
2871
- throw new Error(`snapshot publication ${row.mutationId} has no exact upload subset`);
2872
- }
2873
- if (!Array.isArray(publication.replacementRootIds)
2874
- || !Array.isArray(publication.replacementRecords)) {
2964
+ const meta = JSON.parse(row.metaJson);
2965
+ // No lawful publication carries zero roots — rootReplacementFromRecords
2966
+ // refuses them at build time, so a rootless row here is damage.
2967
+ if (!Array.isArray(meta.replacementRootIds) || meta.replacementRootIds.length === 0
2968
+ || !Array.isArray(meta.uploadUuids) || meta.operationKind !== "entity.roots.replace") {
2875
2969
  throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
2876
2970
  }
2877
- const deliveryWatermark = Number(publication.deliveryWatermark);
2971
+ const deliveryWatermark = Number(meta.deliveryWatermark);
2878
2972
  if (!Number.isSafeInteger(deliveryWatermark) || deliveryWatermark < 0) {
2879
2973
  throw new Error(`snapshot publication ${row.mutationId} has no delivery watermark`);
2880
2974
  }
2881
2975
  return {
2882
2976
  mutationId: row.mutationId,
2883
2977
  resourceId: row.resourceId,
2884
- authorityEpoch: Number(publication.authorityEpoch),
2885
- baseVersion: Number(publication.baseVersion),
2978
+ authorityEpoch: Number(meta.authorityEpoch),
2979
+ baseVersion: Number(meta.baseVersion),
2886
2980
  deliveryWatermark,
2887
- snapshotChecksum: String(publication.snapshotChecksum),
2888
- snapshotJson: String(publication.snapshotJson),
2889
- uploadRecords: snapshotFromRecords(publication.uploadRecords).records,
2890
- replacementRootIds: publication.replacementRootIds.map(String),
2891
- replacementRecords: snapshotFromRecords(publication.replacementRecords).records,
2981
+ snapshotChecksum: String(meta.snapshotChecksum),
2982
+ operationKind: meta.operationKind,
2983
+ replacementRootIds: meta.replacementRootIds.map(String),
2984
+ uploadUuids: meta.uploadUuids.map(String),
2985
+ snapshotBlob: row.snapshotBlob,
2986
+ operationBlob: row.operationBlob,
2892
2987
  status: row.status === "rejected" ? "rejected" : "pending",
2893
2988
  submitted: row.submittedAt !== null,
2894
2989
  };
@@ -3059,18 +3154,17 @@ function supersedeSnapshotJournal(file, entry) {
3059
3154
  .run(entry.resourceId);
3060
3155
  database.prepare(`
3061
3156
  INSERT INTO snapshot_publications(
3062
- mutation_id, resource_id, publication_json, created_at
3063
- ) VALUES (?, ?, ?, ?)
3157
+ mutation_id, resource_id, meta_json, snapshot_blob, operation_blob, created_at
3158
+ ) VALUES (?, ?, ?, ?, ?, ?)
3064
3159
  `).run(entry.mutationId, entry.resourceId, stableJson({
3065
3160
  authorityEpoch: entry.authorityEpoch,
3066
3161
  baseVersion: entry.baseVersion,
3067
3162
  deliveryWatermark: entry.deliveryWatermark,
3068
3163
  snapshotChecksum: entry.snapshotChecksum,
3069
- snapshotJson: entry.snapshotJson,
3070
- uploadRecords: entry.uploadRecords,
3164
+ operationKind: entry.operationKind,
3071
3165
  replacementRootIds: entry.replacementRootIds,
3072
- replacementRecords: entry.replacementRecords,
3073
- }), Date.now());
3166
+ uploadUuids: entry.uploadUuids,
3167
+ }), entry.snapshotBlob, entry.operationBlob, Date.now());
3074
3168
  })();
3075
3169
  }
3076
3170
  finally {