@amalgm/shell 0.1.110 → 0.1.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -9,8 +9,10 @@ type Frame = Record<string, unknown> & {
|
|
|
9
9
|
};
|
|
10
10
|
/** Publishes one mutation operation of any size as ONE atomic commit.
|
|
11
11
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
12
|
+
* The operation arrives as its canonical UTF-8 bytes, serialized exactly
|
|
13
|
+
* once by the caller — this function never re-serializes a graph. A small
|
|
14
|
+
* operation travels inline in the submit frame, byte-identical to the
|
|
15
|
+
* pre-paging wire. A larger one stages as byte pages of its canonical
|
|
14
16
|
* JSON, then submits a reference the gateway assembles into the same commit
|
|
15
17
|
* path — transport pages, the operation never does. Every attempt stages
|
|
16
18
|
* afresh, so a replay after a lost acknowledgement or dropped connection is
|
|
@@ -22,5 +24,5 @@ type Frame = Record<string, unknown> & {
|
|
|
22
24
|
* bridge — the gateway reassembles the whole graph and the database journals
|
|
23
25
|
* it as one large operation; the eventual endpoint is immutable graph pages
|
|
24
26
|
* with a small atomically-committed graph-head pointer. */
|
|
25
|
-
export declare function submitMutationOperation(wire: Pick<WireClient, "request" | "requestBinary">, envelope: Record<string, unknown>, operation:
|
|
27
|
+
export declare function submitMutationOperation(wire: Pick<WireClient, "request" | "requestBinary">, envelope: Record<string, unknown>, operation: Buffer, onLeg?: (leg: "stage" | "submit", durationMs: number) => void): Promise<Frame>;
|
|
26
28
|
export {};
|
|
@@ -6,8 +6,10 @@ export const MAX_INLINE_OPERATION_BYTES = 8 * 1024 * 1024;
|
|
|
6
6
|
export const MUTATION_PAGE_BYTES = 4 * 1024 * 1024;
|
|
7
7
|
/** Publishes one mutation operation of any size as ONE atomic commit.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
9
|
+
* The operation arrives as its canonical UTF-8 bytes, serialized exactly
|
|
10
|
+
* once by the caller — this function never re-serializes a graph. A small
|
|
11
|
+
* operation travels inline in the submit frame, byte-identical to the
|
|
12
|
+
* pre-paging wire. A larger one stages as byte pages of its canonical
|
|
11
13
|
* JSON, then submits a reference the gateway assembles into the same commit
|
|
12
14
|
* path — transport pages, the operation never does. Every attempt stages
|
|
13
15
|
* afresh, so a replay after a lost acknowledgement or dropped connection is
|
|
@@ -19,13 +21,16 @@ export const MUTATION_PAGE_BYTES = 4 * 1024 * 1024;
|
|
|
19
21
|
* bridge — the gateway reassembles the whole graph and the database journals
|
|
20
22
|
* it as one large operation; the eventual endpoint is immutable graph pages
|
|
21
23
|
* with a small atomically-committed graph-head pointer. */
|
|
22
|
-
export async function submitMutationOperation(wire, envelope, operation) {
|
|
23
|
-
const canonical =
|
|
24
|
+
export async function submitMutationOperation(wire, envelope, operation, onLeg) {
|
|
25
|
+
const canonical = operation;
|
|
26
|
+
const started = performance.now();
|
|
24
27
|
if (canonical.byteLength <= MAX_INLINE_OPERATION_BYTES) {
|
|
25
|
-
|
|
28
|
+
const frame = await wire.request({
|
|
26
29
|
type: "shared.mutation.submit",
|
|
27
|
-
envelope: { ...envelope, operation },
|
|
30
|
+
envelope: { ...envelope, operation: JSON.parse(canonical.toString("utf8")) },
|
|
28
31
|
}, ["shared.mutation.ack"]);
|
|
32
|
+
onLeg?.("submit", performance.now() - started);
|
|
33
|
+
return frame;
|
|
29
34
|
}
|
|
30
35
|
const staged = {
|
|
31
36
|
page_count: Math.ceil(canonical.byteLength / MUTATION_PAGE_BYTES),
|
|
@@ -43,10 +48,14 @@ export async function submitMutationOperation(wire, envelope, operation) {
|
|
|
43
48
|
bytes: page.byteLength,
|
|
44
49
|
}, page, ["shared.mutation.staged"]);
|
|
45
50
|
}
|
|
46
|
-
|
|
51
|
+
const submitStarted = performance.now();
|
|
52
|
+
onLeg?.("stage", submitStarted - started);
|
|
53
|
+
const frame = await wire.request({
|
|
47
54
|
type: "shared.mutation.submit",
|
|
48
55
|
envelope,
|
|
49
56
|
staged,
|
|
50
57
|
}, ["shared.mutation.ack"]);
|
|
58
|
+
onLeg?.("submit", performance.now() - submitStarted);
|
|
59
|
+
return frame;
|
|
51
60
|
}
|
|
52
61
|
//# sourceMappingURL=paged-mutation-submit.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"paged-mutation-submit.js","sourceRoot":"","sources":["../src/paged-mutation-submit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC;0EAC0E;AAC1E,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAE1D,+EAA+E;AAC/E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAInD
|
|
1
|
+
{"version":3,"file":"paged-mutation-submit.js","sourceRoot":"","sources":["../src/paged-mutation-submit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC;0EAC0E;AAC1E,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAE1D,+EAA+E;AAC/E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAInD;;;;;;;;;;;;;;;;2DAgB2D;AAC3D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAmD,EACnD,QAAiC,EACjC,SAAiB,EACjB,KAA6D;IAE7D,MAAM,SAAS,GAAG,SAAS,CAAC;IAC5B,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,IAAI,SAAS,CAAC,UAAU,IAAI,0BAA0B,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAC/B,IAAI,EAAE,wBAAwB;YAC9B,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE;SAC7E,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC5B,KAAK,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG;QACb,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,mBAAmB,CAAC;QACjE,eAAe,EAAE,SAAS,CAAC,UAAU;QACrC,gBAAgB,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;KACvE,CAAC;IACF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAC7B,KAAK,GAAG,mBAAmB,EAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,mBAAmB,EAAE,SAAS,CAAC,UAAU,CAAC,CAClE,CAAC;QACF,MAAM,IAAI,CAAC,aAAa,CAAC;YACvB,IAAI,EAAE,uBAAuB;YAC7B,WAAW,EAAE,QAAQ,CAAC,UAAU;YAChC,WAAW,EAAE,QAAQ,CAAC,UAAU;YAChC,UAAU,EAAE,KAAK;YACjB,GAAG,MAAM;YACT,KAAK,EAAE,IAAI,CAAC,UAAU;SACvB,EAAE,IAAI,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACxC,KAAK,EAAE,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;QAC/B,IAAI,EAAE,wBAAwB;QAC9B,QAAQ;QACR,MAAM;KACP,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAC5B,KAAK,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type UserGroundPort, type UserHomeDeclaration } from "@amalgm/core/identity";
|
|
2
2
|
import { type EntityType, type FilesAddPorts, type FilesRegisterPorts, type UserGroundIdentity } from "@amalgm/live";
|
|
3
|
+
import Database from "better-sqlite3";
|
|
3
4
|
import { type FilesStageObserver } from "./files-observability.js";
|
|
4
5
|
import { type EntityApplyStageEvidence } from "./entity-apply-host.js";
|
|
5
6
|
import { type OpenNativeWatch, type WatchCoverageEvidence, type WatchHealth } from "./watching/index.js";
|
|
@@ -184,5 +185,11 @@ export declare class UserGroundHost {
|
|
|
184
185
|
* frames; the answer is the only permission to skip an upload. */
|
|
185
186
|
private sealedContentHashes;
|
|
186
187
|
private publishPendingSnapshots;
|
|
188
|
+
private publishLoadedSnapshot;
|
|
187
189
|
}
|
|
190
|
+
/** The single opener and schema-migration entry for a user ground database.
|
|
191
|
+
* Exported so the upgrade path is provable in isolation — the journal
|
|
192
|
+
* conversion law test opens a legacy file through this exact entry inside a
|
|
193
|
+
* heap-capped child process. */
|
|
194
|
+
export declare function initializeDatabase(file: string): Database.Database;
|
|
188
195
|
export {};
|
package/dist/user-ground-host.js
CHANGED
|
@@ -55,6 +55,15 @@ const WALK_CAPTURE_CONCURRENCY = 8;
|
|
|
55
55
|
const REGISTER_STALE_CAPTURE_ATTEMPTS = 3;
|
|
56
56
|
const DOWNLOAD_CONTENT_RETRY_ATTEMPTS = 3;
|
|
57
57
|
const APPLY_GROUND_WAIT_TIMEOUT_MS = 15_000;
|
|
58
|
+
function loadSnapshotPublication(entry) {
|
|
59
|
+
const snapshotRecords = snapshotFromRecords((JSON.parse(entry.snapshotBlob.toString("utf8")).records ?? [])).records;
|
|
60
|
+
const uploadIds = new Set(entry.uploadUuids);
|
|
61
|
+
return {
|
|
62
|
+
...entry,
|
|
63
|
+
snapshotRecords,
|
|
64
|
+
uploadRecords: snapshotRecords.filter((record) => uploadIds.has(record.uuid)),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
58
67
|
const GROUND_ROW_COLUMNS = [
|
|
59
68
|
"uuid", "resource_id", "root_uuid", "type", "parent_uuid", "name", "status", "version",
|
|
60
69
|
"payload_version", "transport_version", "relative_path", "absolute_path", "device_number",
|
|
@@ -1939,7 +1948,7 @@ export class UserGroundHost {
|
|
|
1939
1948
|
headVersion: first.baseVersion,
|
|
1940
1949
|
checksum: "",
|
|
1941
1950
|
deliveryWatermark: first.deliveryWatermark,
|
|
1942
|
-
records:
|
|
1951
|
+
records: loadSnapshotPublication(first).snapshotRecords,
|
|
1943
1952
|
};
|
|
1944
1953
|
try {
|
|
1945
1954
|
await this.publishPendingSnapshots(identity);
|
|
@@ -2003,32 +2012,27 @@ export class UserGroundHost {
|
|
|
2003
2012
|
throw new Error(`repository-owned leaf escaped its rail: ${impossibleGitLeaf.name} parent=${parent?.type ?? "missing"}`);
|
|
2004
2013
|
}
|
|
2005
2014
|
const snapshot = snapshotFromRecords(traveling);
|
|
2006
|
-
|
|
2007
|
-
|
|
2015
|
+
// ONE canonical serialization: these exact bytes are the journal blob
|
|
2016
|
+
// and the checksum's input; they are never re-derived or re-parsed on
|
|
2017
|
+
// the in-process path.
|
|
2018
|
+
const snapshotBlob = Buffer.from(stableJson(snapshot), "utf8");
|
|
2019
|
+
const checksum = sha256Hex(snapshotBlob);
|
|
2008
2020
|
const localIds = new Set(localRecords.map((record) => record.uuid));
|
|
2009
2021
|
// Graph membership is never proof of sealed cargo: a record in the cloud
|
|
2010
2022
|
// graph may name content sealed only under a prior content contract,
|
|
2011
2023
|
// whose namespace the current readers never touch. Only the authority's
|
|
2012
|
-
// (resource, hash, contract) seal evidence may skip an upload —
|
|
2013
|
-
//
|
|
2014
|
-
// always re-establishes cargo closure
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
return [];
|
|
2018
|
-
const artifact = artifactForRecord(record);
|
|
2019
|
-
return artifact ? [{ record, contentHash: artifact.contentHash }] : [];
|
|
2020
|
-
});
|
|
2021
|
-
const sealedArtifacts = await this.sealedContentHashes(state.resourceId, [...new Set(cargo.map((entry) => entry.contentHash))]);
|
|
2022
|
-
const uploadRecords = cargo
|
|
2023
|
-
.filter((entry) => !sealedArtifacts.has(entry.contentHash))
|
|
2024
|
-
.map((entry) => entry.record);
|
|
2024
|
+
// (resource, hash, contract) seal evidence may skip an upload — the
|
|
2025
|
+
// sealed-first upload asks that exact question itself and moves only
|
|
2026
|
+
// what is unsealed, so Register always re-establishes cargo closure
|
|
2027
|
+
// before returning, even when the graph itself is unchanged.
|
|
2028
|
+
const cargoRecords = snapshot.records.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
|
|
2025
2029
|
if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
|
|
2026
2030
|
// The graph is already committed; only cargo can be missing. Repair
|
|
2027
2031
|
// uploads need no journal row — a crash simply re-runs the same
|
|
2028
2032
|
// repair on the next Register.
|
|
2029
|
-
if (
|
|
2033
|
+
if (cargoRecords.length > 0) {
|
|
2030
2034
|
try {
|
|
2031
|
-
await this.uploadSnapshotContent(identity, state.resourceId,
|
|
2035
|
+
await this.uploadSnapshotContent(identity, state.resourceId, cargoRecords, trace, manifestHints);
|
|
2032
2036
|
}
|
|
2033
2037
|
catch (error) {
|
|
2034
2038
|
throw pipelineStageError("cloud upload", error);
|
|
@@ -2046,20 +2050,32 @@ export class UserGroundHost {
|
|
|
2046
2050
|
return root !== null && replacementRoots.has(root);
|
|
2047
2051
|
});
|
|
2048
2052
|
const replacement = rootReplacementFromRecords(replacementRootIds, replacementRecords);
|
|
2049
|
-
|
|
2053
|
+
// The operation's canonical bytes: staged, hashed, and submitted as-is.
|
|
2054
|
+
const operationBlob = Buffer.from(stableJson(replacement), "utf8");
|
|
2055
|
+
const entry = {
|
|
2050
2056
|
mutationId: randomUUID(),
|
|
2051
2057
|
resourceId: state.resourceId,
|
|
2052
2058
|
authorityEpoch: state.authorityEpoch,
|
|
2053
2059
|
baseVersion: state.headVersion,
|
|
2054
2060
|
snapshotChecksum: checksum,
|
|
2055
|
-
snapshotJson,
|
|
2056
2061
|
deliveryWatermark,
|
|
2057
|
-
|
|
2062
|
+
operationKind: replacement.kind,
|
|
2058
2063
|
replacementRootIds: replacement.rootIds,
|
|
2059
|
-
|
|
2064
|
+
uploadUuids: cargoRecords.map((record) => record.uuid),
|
|
2065
|
+
snapshotBlob,
|
|
2066
|
+
operationBlob,
|
|
2060
2067
|
status: "pending",
|
|
2061
|
-
|
|
2062
|
-
|
|
2068
|
+
submitted: false,
|
|
2069
|
+
};
|
|
2070
|
+
supersedeSnapshotJournal(this.databasePath(identity), entry);
|
|
2071
|
+
// Publish the freshly journaled intent directly: the records are already
|
|
2072
|
+
// in memory and the operation bytes already exist — nothing is re-read,
|
|
2073
|
+
// re-parsed, or re-serialized between prepare and the wire.
|
|
2074
|
+
await this.publishLoadedSnapshot(identity, {
|
|
2075
|
+
...entry,
|
|
2076
|
+
snapshotRecords: snapshot.records,
|
|
2077
|
+
uploadRecords: cargoRecords,
|
|
2078
|
+
}, trace, manifestHints);
|
|
2063
2079
|
}
|
|
2064
2080
|
/** Which of these heads does the authority hold sealed under the current
|
|
2065
2081
|
* content contract? Batched so a whole workspace costs a handful of
|
|
@@ -2091,88 +2107,94 @@ export class UserGroundHost {
|
|
|
2091
2107
|
// snapshot superseding it can publish this resource again.
|
|
2092
2108
|
if (pending.status === "rejected")
|
|
2093
2109
|
continue;
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
}
|
|
2110
|
-
let frame;
|
|
2111
|
-
const graphCommitStarted = performance.now();
|
|
2112
|
-
if (trace?.journey === "register") {
|
|
2113
|
-
this.stage({
|
|
2114
|
-
primitive: "register",
|
|
2115
|
-
stage: "cloud-graph-commit",
|
|
2116
|
-
status: "started",
|
|
2117
|
-
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2118
|
-
...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
|
|
2119
|
-
});
|
|
2120
|
-
}
|
|
2121
|
-
if (!pending.submitted) {
|
|
2122
|
-
markSnapshotJournalSubmitted(this.databasePath(identity), pending.mutationId);
|
|
2123
|
-
}
|
|
2124
|
-
try {
|
|
2125
|
-
frame = await submitMutationOperation(this.wire, {
|
|
2126
|
-
resourceId: pending.resourceId,
|
|
2127
|
-
authorityEpoch: pending.authorityEpoch,
|
|
2128
|
-
mutationId: pending.mutationId,
|
|
2129
|
-
deviceId: identity.deviceId,
|
|
2130
|
-
baseVersion: pending.baseVersion,
|
|
2131
|
-
contract: ENTITY_CLOUD_CONTRACT,
|
|
2132
|
-
schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
|
|
2133
|
-
operationKind: replacement.kind,
|
|
2134
|
-
deliveryWatermark: pending.deliveryWatermark,
|
|
2135
|
-
}, replacement);
|
|
2136
|
-
}
|
|
2137
|
-
catch (error) {
|
|
2138
|
-
if (trace?.journey === "register") {
|
|
2139
|
-
this.stage({
|
|
2140
|
-
primitive: "register",
|
|
2141
|
-
stage: "cloud-graph-commit",
|
|
2142
|
-
status: "failed",
|
|
2143
|
-
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2144
|
-
durationMs: performance.now() - graphCommitStarted,
|
|
2145
|
-
});
|
|
2146
|
-
}
|
|
2147
|
-
// A non-retryable error frame is the authority's answer: this exact
|
|
2148
|
-
// mutation is refused forever. Settle the row so it is never
|
|
2149
|
-
// replayed and a newer snapshot may supersede it. Every other
|
|
2150
|
-
// failure leaves the outcome unknown and the row pending.
|
|
2151
|
-
if (error instanceof WireRequestError && error.retryable === false) {
|
|
2152
|
-
markSnapshotJournalRejected(this.databasePath(identity), pending.mutationId);
|
|
2153
|
-
}
|
|
2154
|
-
throw pipelineStageError("cloud graph commit", error);
|
|
2110
|
+
await this.publishLoadedSnapshot(identity, loadSnapshotPublication(pending), trace, manifestHints);
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
async publishLoadedSnapshot(identity, publication, trace, manifestHints) {
|
|
2114
|
+
try {
|
|
2115
|
+
await this.uploadSnapshotContent(identity, publication.resourceId, publication.uploadRecords, trace, manifestHints);
|
|
2116
|
+
}
|
|
2117
|
+
catch (error) {
|
|
2118
|
+
// An unsubmitted pending row is a local intent the authority has
|
|
2119
|
+
// never seen. When its captured ground moved before upload, the
|
|
2120
|
+
// intent is simply stale: delete it and let the caller re-derive
|
|
2121
|
+
// one from current ground. A submitted row never lands here — its
|
|
2122
|
+
// upload sealed completely before the first submit frame left.
|
|
2123
|
+
if (staleCapture(error) && !publication.submitted) {
|
|
2124
|
+
deleteJournalEntry(this.databasePath(identity), publication.mutationId);
|
|
2155
2125
|
}
|
|
2126
|
+
throw pipelineStageError("cloud upload", error);
|
|
2127
|
+
}
|
|
2128
|
+
let frame;
|
|
2129
|
+
const graphCommitStarted = performance.now();
|
|
2130
|
+
if (trace?.journey === "register") {
|
|
2131
|
+
this.stage({
|
|
2132
|
+
primitive: "register",
|
|
2133
|
+
stage: "cloud-graph-commit",
|
|
2134
|
+
status: "started",
|
|
2135
|
+
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2136
|
+
...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
|
|
2137
|
+
});
|
|
2138
|
+
}
|
|
2139
|
+
if (!publication.submitted) {
|
|
2140
|
+
markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId);
|
|
2141
|
+
}
|
|
2142
|
+
const legMs = {};
|
|
2143
|
+
try {
|
|
2144
|
+
frame = await submitMutationOperation(this.wire, {
|
|
2145
|
+
resourceId: publication.resourceId,
|
|
2146
|
+
authorityEpoch: publication.authorityEpoch,
|
|
2147
|
+
mutationId: publication.mutationId,
|
|
2148
|
+
deviceId: identity.deviceId,
|
|
2149
|
+
baseVersion: publication.baseVersion,
|
|
2150
|
+
contract: ENTITY_CLOUD_CONTRACT,
|
|
2151
|
+
schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
|
|
2152
|
+
operationKind: publication.operationKind,
|
|
2153
|
+
deliveryWatermark: publication.deliveryWatermark,
|
|
2154
|
+
}, publication.operationBlob, (leg, durationMs) => { legMs[`${leg}Ms`] = durationMs; });
|
|
2155
|
+
}
|
|
2156
|
+
catch (error) {
|
|
2156
2157
|
if (trace?.journey === "register") {
|
|
2157
2158
|
this.stage({
|
|
2158
2159
|
primitive: "register",
|
|
2159
2160
|
stage: "cloud-graph-commit",
|
|
2160
|
-
status: "
|
|
2161
|
+
status: "failed",
|
|
2161
2162
|
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2162
2163
|
durationMs: performance.now() - graphCommitStarted,
|
|
2163
|
-
measurements: { version: Number(frame.version) },
|
|
2164
2164
|
});
|
|
2165
2165
|
}
|
|
2166
|
-
this
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
deleteJournalEntry(this.databasePath(identity), pending.mutationId);
|
|
2166
|
+
// A non-retryable error frame is the authority's answer: this exact
|
|
2167
|
+
// mutation is refused forever. Settle the row so it is never
|
|
2168
|
+
// replayed and a newer snapshot may supersede it. Every other
|
|
2169
|
+
// failure leaves the outcome unknown and the row pending.
|
|
2170
|
+
if (error instanceof WireRequestError && error.retryable === false) {
|
|
2171
|
+
markSnapshotJournalRejected(this.databasePath(identity), publication.mutationId);
|
|
2172
|
+
}
|
|
2173
|
+
throw pipelineStageError("cloud graph commit", error);
|
|
2175
2174
|
}
|
|
2175
|
+
if (trace?.journey === "register") {
|
|
2176
|
+
this.stage({
|
|
2177
|
+
primitive: "register",
|
|
2178
|
+
stage: "cloud-graph-commit",
|
|
2179
|
+
status: "completed",
|
|
2180
|
+
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2181
|
+
durationMs: performance.now() - graphCommitStarted,
|
|
2182
|
+
measurements: {
|
|
2183
|
+
version: Number(frame.version),
|
|
2184
|
+
operationBytes: publication.operationBlob.byteLength,
|
|
2185
|
+
...legMs,
|
|
2186
|
+
},
|
|
2187
|
+
});
|
|
2188
|
+
}
|
|
2189
|
+
this.cloudState = {
|
|
2190
|
+
resourceId: publication.resourceId,
|
|
2191
|
+
authorityEpoch: Number(frame.authority_epoch),
|
|
2192
|
+
headVersion: Number(frame.version),
|
|
2193
|
+
checksum: publication.snapshotChecksum,
|
|
2194
|
+
deliveryWatermark: Number(frame.delivery_watermark),
|
|
2195
|
+
records: publication.snapshotRecords,
|
|
2196
|
+
};
|
|
2197
|
+
deleteJournalEntry(this.databasePath(identity), publication.mutationId);
|
|
2176
2198
|
}
|
|
2177
2199
|
}
|
|
2178
2200
|
const sha256Hex = (input) => createHash("sha256").update(input).digest("hex");
|
|
@@ -2264,12 +2286,99 @@ function assertHealthyWatch(evidence, rootId) {
|
|
|
2264
2286
|
* file over a dozen times; the file itself is never deleted by the host, so
|
|
2265
2287
|
* every later open needs only the connection and its pragmas. */
|
|
2266
2288
|
const migratedDatabases = new Set();
|
|
2267
|
-
|
|
2289
|
+
const SNAPSHOT_PUBLICATIONS_SCHEMA = `
|
|
2290
|
+
CREATE TABLE IF NOT EXISTS snapshot_publications (
|
|
2291
|
+
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2292
|
+
mutation_id TEXT NOT NULL UNIQUE,
|
|
2293
|
+
resource_id TEXT NOT NULL,
|
|
2294
|
+
meta_json TEXT NOT NULL,
|
|
2295
|
+
snapshot_blob BLOB NOT NULL,
|
|
2296
|
+
operation_blob BLOB NOT NULL,
|
|
2297
|
+
created_at INTEGER NOT NULL,
|
|
2298
|
+
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
|
|
2299
|
+
submitted_at INTEGER
|
|
2300
|
+
);`;
|
|
2301
|
+
/** Converts an old-shape publication journal (one nested publication_json
|
|
2302
|
+
* text per row) into the blob journal, atomically, preserving every row's
|
|
2303
|
+
* meaning. A pending row may be a submitted mutation whose outcome exists
|
|
2304
|
+
* only at the authority — dropping it would violate the retry law that
|
|
2305
|
+
* recovers a lost acknowledgement — so conversion is structural: the nested
|
|
2306
|
+
* snapshot text becomes the snapshot blob verbatim, the stored replacement
|
|
2307
|
+
* becomes the operation blob, and semantic judgment stays where it always
|
|
2308
|
+
* was, with the reader (an unlawful row still blocks registration rather
|
|
2309
|
+
* than being silently replaced).
|
|
2310
|
+
*
|
|
2311
|
+
* The transform runs entirely inside SQLite. A production journal row is a
|
|
2312
|
+
* register-scale graph serialized three times over — the exact text whose
|
|
2313
|
+
* repeated JavaScript materialization blew the 512 MB heap — so no part of
|
|
2314
|
+
* it may ever become a JavaScript value here: json_extract lifts the nested
|
|
2315
|
+
* snapshot text straight into its blob, and the meta and operation rows are
|
|
2316
|
+
* json_object over subtrees, all in engine memory outside the V8 heap.
|
|
2317
|
+
*
|
|
2318
|
+
* Tables older than the three-state journal carry no status or submitted_at;
|
|
2319
|
+
* their rows were written before the pre-submit durability mark existed, so
|
|
2320
|
+
* the only safe reading is "possibly submitted": keep them pending and
|
|
2321
|
+
* marked submitted, which replays the same mutation id — always idempotent —
|
|
2322
|
+
* and never supersedes. A row whose publication_json does not parse fails
|
|
2323
|
+
* the open loudly with the journal intact; nothing is ever deleted on a
|
|
2324
|
+
* conversion failure. */
|
|
2325
|
+
function convertLegacySnapshotJournal(database, columns) {
|
|
2326
|
+
database.transaction(() => {
|
|
2327
|
+
database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
|
|
2328
|
+
database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
|
|
2329
|
+
database.exec(`
|
|
2330
|
+
INSERT INTO snapshot_publications(
|
|
2331
|
+
mutation_id, resource_id, meta_json, snapshot_blob, operation_blob,
|
|
2332
|
+
created_at, status, submitted_at
|
|
2333
|
+
)
|
|
2334
|
+
SELECT
|
|
2335
|
+
mutation_id,
|
|
2336
|
+
resource_id,
|
|
2337
|
+
json_object(
|
|
2338
|
+
'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
|
|
2339
|
+
'baseVersion', json_extract(publication_json, '$.baseVersion'),
|
|
2340
|
+
'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
|
|
2341
|
+
'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
|
|
2342
|
+
'operationKind', 'entity.roots.replace',
|
|
2343
|
+
'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2344
|
+
'uploadUuids', coalesce((
|
|
2345
|
+
SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
|
|
2346
|
+
FROM json_each(publication_json, '$.uploadRecords') AS uploaded
|
|
2347
|
+
), json_array())
|
|
2348
|
+
),
|
|
2349
|
+
CAST(json_extract(publication_json, '$.snapshotJson') AS BLOB),
|
|
2350
|
+
CAST(json_object(
|
|
2351
|
+
'kind', 'entity.roots.replace',
|
|
2352
|
+
'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2353
|
+
'records', json(json_extract(publication_json, '$.replacementRecords'))
|
|
2354
|
+
) AS BLOB),
|
|
2355
|
+
created_at,
|
|
2356
|
+
${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
|
|
2357
|
+
${columns.has("submitted_at") ? "submitted_at" : "created_at"}
|
|
2358
|
+
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2359
|
+
`);
|
|
2360
|
+
database.exec("DROP TABLE snapshot_publications_legacy");
|
|
2361
|
+
})();
|
|
2362
|
+
}
|
|
2363
|
+
/** The single opener and schema-migration entry for a user ground database.
|
|
2364
|
+
* Exported so the upgrade path is provable in isolation — the journal
|
|
2365
|
+
* conversion law test opens a legacy file through this exact entry inside a
|
|
2366
|
+
* heap-capped child process. */
|
|
2367
|
+
export function initializeDatabase(file) {
|
|
2268
2368
|
ensurePrivateDir(dirname(file));
|
|
2269
2369
|
const database = new Database(file);
|
|
2270
2370
|
database.pragma("busy_timeout = 5000");
|
|
2271
2371
|
database.pragma("journal_mode = WAL");
|
|
2272
2372
|
database.pragma("synchronous = FULL");
|
|
2373
|
+
// The legacy probe runs on every open, before the migration memo: a journal
|
|
2374
|
+
// row may be a submitted publication whose outcome exists only at the
|
|
2375
|
+
// authority, so an old-shape table must convert — never drop — no matter
|
|
2376
|
+
// how this file arrived at this process. One PRAGMA against the in-memory
|
|
2377
|
+
// catalog costs nothing next to the connection open itself.
|
|
2378
|
+
const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
|
|
2379
|
+
if (publicationColumns.has("publication_json")) {
|
|
2380
|
+
convertLegacySnapshotJournal(database, publicationColumns);
|
|
2381
|
+
}
|
|
2273
2382
|
if (migratedDatabases.has(file))
|
|
2274
2383
|
return database;
|
|
2275
2384
|
database.exec(`
|
|
@@ -2361,15 +2470,7 @@ function initializeDatabase(file) {
|
|
|
2361
2470
|
authority_id TEXT PRIMARY KEY,
|
|
2362
2471
|
delivery_sequence INTEGER NOT NULL CHECK(delivery_sequence >= 0)
|
|
2363
2472
|
);
|
|
2364
|
-
|
|
2365
|
-
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2366
|
-
mutation_id TEXT NOT NULL UNIQUE,
|
|
2367
|
-
resource_id TEXT NOT NULL,
|
|
2368
|
-
publication_json TEXT NOT NULL,
|
|
2369
|
-
created_at INTEGER NOT NULL,
|
|
2370
|
-
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
|
|
2371
|
-
submitted_at INTEGER
|
|
2372
|
-
);
|
|
2473
|
+
${SNAPSHOT_PUBLICATIONS_SCHEMA}
|
|
2373
2474
|
CREATE TABLE IF NOT EXISTS workspace_add_intents (
|
|
2374
2475
|
workspace_uuid TEXT PRIMARY KEY,
|
|
2375
2476
|
destination_path TEXT NOT NULL UNIQUE,
|
|
@@ -2405,13 +2506,6 @@ function initializeDatabase(file) {
|
|
|
2405
2506
|
if (!inboxColumns.has("resolved_at")) {
|
|
2406
2507
|
database.exec("ALTER TABLE record_inbox ADD COLUMN resolved_at INTEGER");
|
|
2407
2508
|
}
|
|
2408
|
-
const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
|
|
2409
|
-
if (!publicationColumns.has("status")) {
|
|
2410
|
-
database.exec("ALTER TABLE snapshot_publications ADD COLUMN status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected'))");
|
|
2411
|
-
}
|
|
2412
|
-
if (!publicationColumns.has("submitted_at")) {
|
|
2413
|
-
database.exec("ALTER TABLE snapshot_publications ADD COLUMN submitted_at INTEGER");
|
|
2414
|
-
}
|
|
2415
2509
|
database.exec("DROP INDEX IF EXISTS record_inbox_delivery");
|
|
2416
2510
|
database.exec("DROP TABLE IF EXISTS mutation_journal");
|
|
2417
2511
|
migratedDatabases.add(file);
|
|
@@ -2862,33 +2956,34 @@ function readSnapshotPublications(file) {
|
|
|
2862
2956
|
try {
|
|
2863
2957
|
return database.prepare(`
|
|
2864
2958
|
SELECT mutation_id AS mutationId, resource_id AS resourceId,
|
|
2865
|
-
|
|
2959
|
+
meta_json AS metaJson, snapshot_blob AS snapshotBlob,
|
|
2960
|
+
operation_blob AS operationBlob, status,
|
|
2866
2961
|
submitted_at AS submittedAt
|
|
2867
2962
|
FROM snapshot_publications ORDER BY sequence
|
|
2868
2963
|
`).all().map((row) => {
|
|
2869
|
-
const
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|| !Array.isArray(publication.replacementRecords)) {
|
|
2964
|
+
const meta = JSON.parse(row.metaJson);
|
|
2965
|
+
// No lawful publication carries zero roots — rootReplacementFromRecords
|
|
2966
|
+
// refuses them at build time, so a rootless row here is damage.
|
|
2967
|
+
if (!Array.isArray(meta.replacementRootIds) || meta.replacementRootIds.length === 0
|
|
2968
|
+
|| !Array.isArray(meta.uploadUuids) || meta.operationKind !== "entity.roots.replace") {
|
|
2875
2969
|
throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
|
|
2876
2970
|
}
|
|
2877
|
-
const deliveryWatermark = Number(
|
|
2971
|
+
const deliveryWatermark = Number(meta.deliveryWatermark);
|
|
2878
2972
|
if (!Number.isSafeInteger(deliveryWatermark) || deliveryWatermark < 0) {
|
|
2879
2973
|
throw new Error(`snapshot publication ${row.mutationId} has no delivery watermark`);
|
|
2880
2974
|
}
|
|
2881
2975
|
return {
|
|
2882
2976
|
mutationId: row.mutationId,
|
|
2883
2977
|
resourceId: row.resourceId,
|
|
2884
|
-
authorityEpoch: Number(
|
|
2885
|
-
baseVersion: Number(
|
|
2978
|
+
authorityEpoch: Number(meta.authorityEpoch),
|
|
2979
|
+
baseVersion: Number(meta.baseVersion),
|
|
2886
2980
|
deliveryWatermark,
|
|
2887
|
-
snapshotChecksum: String(
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2981
|
+
snapshotChecksum: String(meta.snapshotChecksum),
|
|
2982
|
+
operationKind: meta.operationKind,
|
|
2983
|
+
replacementRootIds: meta.replacementRootIds.map(String),
|
|
2984
|
+
uploadUuids: meta.uploadUuids.map(String),
|
|
2985
|
+
snapshotBlob: row.snapshotBlob,
|
|
2986
|
+
operationBlob: row.operationBlob,
|
|
2892
2987
|
status: row.status === "rejected" ? "rejected" : "pending",
|
|
2893
2988
|
submitted: row.submittedAt !== null,
|
|
2894
2989
|
};
|
|
@@ -3059,18 +3154,17 @@ function supersedeSnapshotJournal(file, entry) {
|
|
|
3059
3154
|
.run(entry.resourceId);
|
|
3060
3155
|
database.prepare(`
|
|
3061
3156
|
INSERT INTO snapshot_publications(
|
|
3062
|
-
mutation_id, resource_id,
|
|
3063
|
-
) VALUES (?, ?, ?, ?)
|
|
3157
|
+
mutation_id, resource_id, meta_json, snapshot_blob, operation_blob, created_at
|
|
3158
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
3064
3159
|
`).run(entry.mutationId, entry.resourceId, stableJson({
|
|
3065
3160
|
authorityEpoch: entry.authorityEpoch,
|
|
3066
3161
|
baseVersion: entry.baseVersion,
|
|
3067
3162
|
deliveryWatermark: entry.deliveryWatermark,
|
|
3068
3163
|
snapshotChecksum: entry.snapshotChecksum,
|
|
3069
|
-
|
|
3070
|
-
uploadRecords: entry.uploadRecords,
|
|
3164
|
+
operationKind: entry.operationKind,
|
|
3071
3165
|
replacementRootIds: entry.replacementRootIds,
|
|
3072
|
-
|
|
3073
|
-
}), Date.now());
|
|
3166
|
+
uploadUuids: entry.uploadUuids,
|
|
3167
|
+
}), entry.snapshotBlob, entry.operationBlob, Date.now());
|
|
3074
3168
|
})();
|
|
3075
3169
|
}
|
|
3076
3170
|
finally {
|