@amalgm/shell 0.1.111 → 0.1.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PURPOSE.md +8 -0
- package/dist/cli.js +24 -1
- package/dist/cli.js.map +1 -1
- package/dist/files-trace.d.ts +41 -0
- package/dist/files-trace.js +138 -0
- package/dist/files-trace.js.map +1 -0
- package/dist/gateway.js +7 -1
- package/dist/gateway.js.map +1 -1
- package/dist/machine-host.d.ts +6 -0
- package/dist/machine-host.js +25 -11
- package/dist/machine-host.js.map +1 -1
- package/dist/registry-graph.d.ts +61 -0
- package/dist/registry-graph.js +156 -0
- package/dist/registry-graph.js.map +1 -0
- package/dist/runtime-command-host.js +7 -1
- package/dist/runtime-command-host.js.map +1 -1
- package/dist/user-ground-host.d.ts +47 -0
- package/dist/user-ground-host.js +578 -168
- package/dist/user-ground-host.js.map +1 -1
- package/dist/wire-client.js +1 -1
- package/package.json +2 -2
- package/dist/paged-mutation-submit.d.ts +0 -28
- package/dist/paged-mutation-submit.js +0 -61
- package/dist/paged-mutation-submit.js.map +0 -1
package/dist/user-ground-host.js
CHANGED
|
@@ -3,7 +3,7 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
|
|
|
3
3
|
import { open, readFile } from "node:fs/promises";
|
|
4
4
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
5
|
import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
|
|
6
|
-
import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, deriveUploadManifest, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash,
|
|
6
|
+
import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, deriveUploadManifest, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, pathIsSuspect, planCargoWaves, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, GRAPH_HEAD_OPERATION_KIND, } from "@amalgm/live";
|
|
7
7
|
import Database from "better-sqlite3";
|
|
8
8
|
import { atomicCopy, atomicWrite, ensurePrivateDir, ensureUserDir } from "./filesystem.js";
|
|
9
9
|
import { emitFilesStage, } from "./files-observability.js";
|
|
@@ -22,9 +22,9 @@ import { WORKSPACE_UUID as UUID, createFilesRegisterPorts, ensureWorkspaceBindin
|
|
|
22
22
|
import { applyRepositoryFiles, captureRepository, inspectRepositoryTransportFile, hasGitMarker, readCachedRepositoryIdentity, } from "./git-repository-host.js";
|
|
23
23
|
import { gitEvidenceFingerprint, inspectGitRegistration, sameGitIdentity, } from "./git-registration-host.js";
|
|
24
24
|
import { projectMaterializedGraph } from "./materialized-graph.js";
|
|
25
|
+
import { graphPageIdentities, registryGraphFromSnapshot, } from "./registry-graph.js";
|
|
25
26
|
import { NodeWatchHost, } from "./watching/index.js";
|
|
26
27
|
import { GroundCoordinator } from "./ground-coordination.js";
|
|
27
|
-
import { submitMutationOperation } from "./paged-mutation-submit.js";
|
|
28
28
|
import { adoptSnapshotDeliveryBaseline } from "./snapshot-delivery-baseline.js";
|
|
29
29
|
import { WireClient, WireRequestError } from "./wire-client.js";
|
|
30
30
|
/** Cargo waves hold up to 16MiB of payload each; two in flight bounds the
|
|
@@ -40,6 +40,10 @@ const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = 128;
|
|
|
40
40
|
// on the 1 GiB runtimes used for ordinary Files materialization. Small
|
|
41
41
|
// objects still use the full request-concurrency window.
|
|
42
42
|
const DIRECT_OBJECT_DOWNLOAD_MAX_IN_FLIGHT_BYTES = 2 * CHUNK_BYTES;
|
|
43
|
+
/** Graph pages are at most a few megabytes each and a whole account is a
|
|
44
|
+
* handful of them, so a narrow lane keeps the cold read fast without holding
|
|
45
|
+
* the whole graph twice while it assembles. */
|
|
46
|
+
const GRAPH_PAGE_READ_CONCURRENCY = 8;
|
|
43
47
|
const DETECT_QUIET_MS = 12;
|
|
44
48
|
const DETECT_MAX_DEFERRAL_MS = 75;
|
|
45
49
|
const DETECT_RETRY_MS = 250;
|
|
@@ -55,13 +59,19 @@ const WALK_CAPTURE_CONCURRENCY = 8;
|
|
|
55
59
|
const REGISTER_STALE_CAPTURE_ATTEMPTS = 3;
|
|
56
60
|
const DOWNLOAD_CONTENT_RETRY_ATTEMPTS = 3;
|
|
57
61
|
const APPLY_GROUND_WAIT_TIMEOUT_MS = 15_000;
|
|
62
|
+
/** Load a journaled publication for replay. The records whose cargo must
|
|
63
|
+
* re-upload are already inside the publication's own bytes — its pages, or,
|
|
64
|
+
* for a row converted from the pre-page journal, the replacement operation
|
|
65
|
+
* itself — so nothing is stored twice and nothing is re-read from ground. */
|
|
58
66
|
function loadSnapshotPublication(entry) {
|
|
59
|
-
const snapshotRecords = snapshotFromRecords((JSON.parse(entry.snapshotBlob.toString("utf8")).records ?? [])).records;
|
|
60
67
|
const uploadIds = new Set(entry.uploadUuids);
|
|
68
|
+
const published = uploadIds.size === 0 ? [] : entry.pages.length > 0
|
|
69
|
+
? entry.pages.flatMap((page) => JSON.parse(page.bytes.toString("utf8")))
|
|
70
|
+
: (JSON.parse(entry.operationBlob.toString("utf8")).records ?? []);
|
|
61
71
|
return {
|
|
62
72
|
...entry,
|
|
63
|
-
|
|
64
|
-
|
|
73
|
+
uploadRecords: published.filter((record) => uploadIds.has(record.uuid)),
|
|
74
|
+
graph: null,
|
|
65
75
|
};
|
|
66
76
|
}
|
|
67
77
|
const GROUND_ROW_COLUMNS = [
|
|
@@ -205,6 +215,11 @@ export class UserGroundHost {
|
|
|
205
215
|
stage(input) {
|
|
206
216
|
emitFilesStage(this.options.onFilesStage, input);
|
|
207
217
|
}
|
|
218
|
+
/** Whether a counter worth real time on the critical path may be computed. */
|
|
219
|
+
measuring() {
|
|
220
|
+
return this.options.filesTraceActive?.()
|
|
221
|
+
?? (this.options.onFilesStage !== undefined || this.options.onDetectScan !== undefined);
|
|
222
|
+
}
|
|
208
223
|
reportDetectScan(evidence) {
|
|
209
224
|
this.options.onDetectScan?.(evidence);
|
|
210
225
|
this.stage({
|
|
@@ -222,7 +237,19 @@ export class UserGroundHost {
|
|
|
222
237
|
gitInspections: evidence.gitInspections,
|
|
223
238
|
repositoryCaptures: evidence.repositoryCaptures,
|
|
224
239
|
records: evidence.records ?? 0,
|
|
225
|
-
recordBytes: evidence.recordBytes
|
|
240
|
+
...(evidence.recordBytes === undefined ? {} : { recordBytes: evidence.recordBytes }),
|
|
241
|
+
// Where the scan actually went. `captureBusyMs / captureMs` is the
|
|
242
|
+
// capture pool's occupancy; at ~1 the pool ran one file at a time.
|
|
243
|
+
walkMs: Math.round(evidence.walkMs ?? 0),
|
|
244
|
+
gitMs: Math.round(evidence.gitMs ?? 0),
|
|
245
|
+
captureMs: Math.round(evidence.captureMs ?? 0),
|
|
246
|
+
captureBusyMs: Math.round(evidence.captureBusyMs ?? 0),
|
|
247
|
+
captureBytes: evidence.captureBytes ?? 0,
|
|
248
|
+
repoIdentityMs: Math.round(evidence.repositoryIdentityMs ?? 0),
|
|
249
|
+
repoCaptureMs: Math.round(evidence.repositoryCaptureMs ?? 0),
|
|
250
|
+
repoSealMs: Math.round(evidence.repositorySealMs ?? 0),
|
|
251
|
+
reconcileMs: Math.round(evidence.reconcileMs ?? 0),
|
|
252
|
+
persistMs: Math.round(evidence.persistMs ?? 0),
|
|
226
253
|
},
|
|
227
254
|
});
|
|
228
255
|
}
|
|
@@ -541,7 +568,7 @@ export class UserGroundHost {
|
|
|
541
568
|
&& record.status === "active"
|
|
542
569
|
&& record.payloadVersion === workspaceId);
|
|
543
570
|
if (hasRegisteredBoundaryInDatabase(database, workspaceId)
|
|
544
|
-
&& hasRegisteredBoundary(this.cloudState.records)) {
|
|
571
|
+
&& hasRegisteredBoundary(this.cloudState.graph.records())) {
|
|
545
572
|
// Already declared — but Register may only return once cargo closure
|
|
546
573
|
// holds under the current content contract. The graph head can be
|
|
547
574
|
// unchanged while a contract cutover moved its cargo namespace, so
|
|
@@ -620,6 +647,7 @@ export class UserGroundHost {
|
|
|
620
647
|
database: this.databasePath(identity),
|
|
621
648
|
cacheDir: this.cacheDir(identity),
|
|
622
649
|
onScan: (evidence) => this.reportDetectScan(evidence),
|
|
650
|
+
measuring: this.measuring(),
|
|
623
651
|
registrationWorkspaceId: workspaceId,
|
|
624
652
|
suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
|
|
625
653
|
});
|
|
@@ -722,7 +750,7 @@ export class UserGroundHost {
|
|
|
722
750
|
if (!hasRegisteredBoundaryInDatabase(database, workspaceId)) {
|
|
723
751
|
throw stageError("entity registration", new Error(`workspace ${workspaceId} and its durable reference were not recorded`));
|
|
724
752
|
}
|
|
725
|
-
if (!hasRegisteredBoundary(this.cloudState.records)) {
|
|
753
|
+
if (!hasRegisteredBoundary(this.cloudState.graph.records())) {
|
|
726
754
|
throw stageError("cloud graph commit", new Error(`workspace ${workspaceId} or its durable reference is absent from the committed head`));
|
|
727
755
|
}
|
|
728
756
|
return {
|
|
@@ -804,9 +832,10 @@ export class UserGroundHost {
|
|
|
804
832
|
throw new Error("entity Record rail already started");
|
|
805
833
|
await this.wire.open();
|
|
806
834
|
// One floor, no per-feature gates: this shell and its gateway release
|
|
807
|
-
// together, and the gateway ships first.
|
|
808
|
-
//
|
|
809
|
-
|
|
835
|
+
// together, and the gateway ships first. 13 = immutable registry graph
|
|
836
|
+
// pages plus one graph-head commit; an older gateway cannot publish or
|
|
837
|
+
// resolve that representation.
|
|
838
|
+
if (this.wire.protocolVersion < 13) {
|
|
810
839
|
throw new Error("gateway speaks an older wire protocol than this shell");
|
|
811
840
|
}
|
|
812
841
|
this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
|
|
@@ -1078,7 +1107,7 @@ export class UserGroundHost {
|
|
|
1078
1107
|
}, ["private.resource.ready", "private.resource.missing"]);
|
|
1079
1108
|
if (frame.type === "private.resource.missing")
|
|
1080
1109
|
return null;
|
|
1081
|
-
const ready = readyFromFrame(frame);
|
|
1110
|
+
const ready = readyFromFrame(frame, await this.materializeRegistry(identity, frame));
|
|
1082
1111
|
this.cloudState = ready.state;
|
|
1083
1112
|
return ready.cloud;
|
|
1084
1113
|
},
|
|
@@ -1093,7 +1122,7 @@ export class UserGroundHost {
|
|
|
1093
1122
|
snapshot_checksum: input.checksum,
|
|
1094
1123
|
snapshot_bytes: bytes.length,
|
|
1095
1124
|
}, ["private.resource.ready"]);
|
|
1096
|
-
const ready = readyFromFrame(frame);
|
|
1125
|
+
const ready = readyFromFrame(frame, await this.materializeRegistry(identity, frame));
|
|
1097
1126
|
const current = ready.cloud;
|
|
1098
1127
|
this.cloudState = ready.state;
|
|
1099
1128
|
const created = frame.duplicate === false;
|
|
@@ -1103,6 +1132,61 @@ export class UserGroundHost {
|
|
|
1103
1132
|
},
|
|
1104
1133
|
};
|
|
1105
1134
|
}
|
|
1135
|
+
/** The authority's stored snapshot as this machine's graph. A head names
|
|
1136
|
+
* immutable pages, so every page this machine already cached is a local read
|
|
1137
|
+
* and an unchanged root never crosses the network twice. */
|
|
1138
|
+
async materializeRegistry(identity, frame) {
|
|
1139
|
+
const resourceId = String(frame.resource_id || "");
|
|
1140
|
+
return registryGraphFromSnapshot(Buffer.from(String(frame.snapshot_b64 || ""), "base64"), (refs) => this.readGraphPages(identity, resourceId, refs), sha256Hex);
|
|
1141
|
+
}
|
|
1142
|
+
/** Read graph pages by identity: cached pages come off local disk, the rest
|
|
1143
|
+
* are fetched straight from content storage and cached on the way in. Every
|
|
1144
|
+
* page is re-proven against its declared identity before it is kept. */
|
|
1145
|
+
async readGraphPages(identity, resourceId, refs) {
|
|
1146
|
+
const directory = graphPageCacheDir(this.cacheDir(identity));
|
|
1147
|
+
const pages = new Map();
|
|
1148
|
+
const missing = [];
|
|
1149
|
+
for (const ref of new Map(refs.map((ref) => [ref.sha256, ref])).values()) {
|
|
1150
|
+
let cached = null;
|
|
1151
|
+
try {
|
|
1152
|
+
cached = readFileSync(join(directory, `${ref.sha256}.bin`));
|
|
1153
|
+
}
|
|
1154
|
+
catch {
|
|
1155
|
+
cached = null;
|
|
1156
|
+
}
|
|
1157
|
+
// A cache copy of the wrong size is torn evidence, never a page: fetch
|
|
1158
|
+
// the immutable original instead of assembling from it.
|
|
1159
|
+
if (cached && cached.byteLength === ref.bytes)
|
|
1160
|
+
pages.set(ref.sha256, cached);
|
|
1161
|
+
else
|
|
1162
|
+
missing.push(ref);
|
|
1163
|
+
}
|
|
1164
|
+
if (missing.length === 0)
|
|
1165
|
+
return pages;
|
|
1166
|
+
const started = performance.now();
|
|
1167
|
+
let fetchedBytes = 0;
|
|
1168
|
+
await boundedForEach(missing, GRAPH_PAGE_READ_CONCURRENCY, async (ref) => {
|
|
1169
|
+
const bytes = await retryContent(() => this.direct.fetchObject(resourceId, { kind: "block", sha256: ref.sha256 }), true);
|
|
1170
|
+
if (bytes.byteLength !== ref.bytes || sha256Hex(bytes) !== ref.sha256) {
|
|
1171
|
+
throw new Error(`cloud graph page ${ref.sha256} does not match its bytes`);
|
|
1172
|
+
}
|
|
1173
|
+
atomicWrite(join(directory, `${ref.sha256}.bin`), bytes);
|
|
1174
|
+
fetchedBytes += bytes.byteLength;
|
|
1175
|
+
pages.set(ref.sha256, bytes);
|
|
1176
|
+
});
|
|
1177
|
+
this.stage({
|
|
1178
|
+
primitive: "download",
|
|
1179
|
+
stage: "graph-pages",
|
|
1180
|
+
status: "completed",
|
|
1181
|
+
durationMs: performance.now() - started,
|
|
1182
|
+
measurements: {
|
|
1183
|
+
pages: refs.length,
|
|
1184
|
+
fetchedPages: missing.length,
|
|
1185
|
+
fetchedBytes,
|
|
1186
|
+
},
|
|
1187
|
+
});
|
|
1188
|
+
return pages;
|
|
1189
|
+
}
|
|
1106
1190
|
localPort(identity) {
|
|
1107
1191
|
const userRoot = this.userRoot(identity);
|
|
1108
1192
|
const database = this.databasePath(identity);
|
|
@@ -1137,6 +1221,7 @@ export class UserGroundHost {
|
|
|
1137
1221
|
await reconcileGround({
|
|
1138
1222
|
identity, userRoot, database, cacheDir,
|
|
1139
1223
|
onScan: (evidence) => this.reportDetectScan(evidence),
|
|
1224
|
+
measuring: this.measuring(),
|
|
1140
1225
|
});
|
|
1141
1226
|
return localValue(identity, userRoot, database);
|
|
1142
1227
|
},
|
|
@@ -1851,6 +1936,7 @@ export class UserGroundHost {
|
|
|
1851
1936
|
database: this.databasePath(identity),
|
|
1852
1937
|
cacheDir: this.cacheDir(identity),
|
|
1853
1938
|
onScan: (evidence) => this.reportDetectScan(evidence),
|
|
1939
|
+
measuring: this.measuring(),
|
|
1854
1940
|
suspicions: reconcileSuspicions,
|
|
1855
1941
|
coreScan,
|
|
1856
1942
|
});
|
|
@@ -1942,14 +2028,6 @@ export class UserGroundHost {
|
|
|
1942
2028
|
if (first.resourceId !== resourceId) {
|
|
1943
2029
|
throw new Error("local snapshot publication belongs to a different user-ground resource");
|
|
1944
2030
|
}
|
|
1945
|
-
this.cloudState = {
|
|
1946
|
-
resourceId: first.resourceId,
|
|
1947
|
-
authorityEpoch: first.authorityEpoch,
|
|
1948
|
-
headVersion: first.baseVersion,
|
|
1949
|
-
checksum: "",
|
|
1950
|
-
deliveryWatermark: first.deliveryWatermark,
|
|
1951
|
-
records: loadSnapshotPublication(first).snapshotRecords,
|
|
1952
|
-
};
|
|
1953
2031
|
try {
|
|
1954
2032
|
await this.publishPendingSnapshots(identity);
|
|
1955
2033
|
}
|
|
@@ -1981,6 +2059,12 @@ export class UserGroundHost {
|
|
|
1981
2059
|
throw error;
|
|
1982
2060
|
}
|
|
1983
2061
|
await this.recordRail?.flush();
|
|
2062
|
+
// A replayed publication leaves no graph behind — it only ever journaled
|
|
2063
|
+
// the roots it rebuilt — so the machine re-reads the authority's head
|
|
2064
|
+
// before deriving the next boundary on top of it.
|
|
2065
|
+
if (!this.cloudState) {
|
|
2066
|
+
await this.cloudPort(identity).lookup(privateEntityResourceId(identity.userId, sha256Hex));
|
|
2067
|
+
}
|
|
1984
2068
|
const state = this.cloudState;
|
|
1985
2069
|
if (!state)
|
|
1986
2070
|
throw new Error("cloud state is unavailable for graph publication");
|
|
@@ -1999,24 +2083,43 @@ export class UserGroundHost {
|
|
|
1999
2083
|
const state = this.cloudState;
|
|
2000
2084
|
if (!state)
|
|
2001
2085
|
throw new Error("cloud state is unavailable for graph publication");
|
|
2002
|
-
|
|
2003
|
-
|
|
2086
|
+
// Graph preparation is the leg between a committed local ground and the
|
|
2087
|
+
// first wire frame. It is pure CPU and SQLite, so it is measured by its
|
|
2088
|
+
// own parts: whoever reads this ledger must see which part is slow.
|
|
2089
|
+
const preparationStarted = performance.now();
|
|
2090
|
+
const leg = {};
|
|
2091
|
+
const timed = (name, work) => {
|
|
2092
|
+
const started = performance.now();
|
|
2093
|
+
const value = work();
|
|
2094
|
+
leg[`${name}Ms`] = performance.now() - started;
|
|
2095
|
+
return value;
|
|
2096
|
+
};
|
|
2097
|
+
if (trace?.journey === "register") {
|
|
2098
|
+
this.stage({
|
|
2099
|
+
primitive: "register",
|
|
2100
|
+
stage: "graph-preparation",
|
|
2101
|
+
status: "started",
|
|
2102
|
+
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2103
|
+
});
|
|
2104
|
+
}
|
|
2105
|
+
const localRecords = timed("groundRows", () => recordsRetainingMissingEvidence(readGroundRowsForRoots(this.databasePath(identity), "entities", publicationRootIds)
|
|
2004
2106
|
.map(portableRecord), readGroundRowsForRoots(this.databasePath(identity), "detection_notebook", publicationRootIds)
|
|
2005
|
-
.map(portableRecord));
|
|
2006
|
-
const
|
|
2007
|
-
const
|
|
2107
|
+
.map(portableRecord)));
|
|
2108
|
+
const accepted = timed("officialRows", () => readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark));
|
|
2109
|
+
const rebuilt = timed("merge", () => publicationRoots(state.graph, localRecords, accepted));
|
|
2110
|
+
const projected = timed("project", () => new Map([...rebuilt].map(([rootId, records]) => [rootId, records.length === 0 ? records : travelingRecords(records, sha256Hex)])));
|
|
2111
|
+
const traveling = [...projected.values()].flat();
|
|
2008
2112
|
const impossibleGitLeaf = traveling.find((record) => (record.type === "file.text" || record.type === "file.binary" || record.type === "link")
|
|
2009
2113
|
&& record.payloadVersion?.startsWith("git:"));
|
|
2010
2114
|
if (impossibleGitLeaf) {
|
|
2011
|
-
const parent =
|
|
2115
|
+
const parent = traveling.find((record) => record.uuid === impossibleGitLeaf.parentUUID);
|
|
2012
2116
|
throw new Error(`repository-owned leaf escaped its rail: ${impossibleGitLeaf.name} parent=${parent?.type ?? "missing"}`);
|
|
2013
2117
|
}
|
|
2014
|
-
|
|
2015
|
-
//
|
|
2016
|
-
//
|
|
2017
|
-
|
|
2018
|
-
const
|
|
2019
|
-
const checksum = sha256Hex(snapshotBlob);
|
|
2118
|
+
// ONE canonical serialization per rebuilt root: the page bytes are the
|
|
2119
|
+
// journal's payload, the upload's payload, and the head's identities.
|
|
2120
|
+
// Untouched roots are not serialized, hashed, or journaled at all.
|
|
2121
|
+
const publication = timed("pages", () => state.graph.rebuild(projected, sha256Hex));
|
|
2122
|
+
const checksum = publication.checksum;
|
|
2020
2123
|
const localIds = new Set(localRecords.map((record) => record.uuid));
|
|
2021
2124
|
// Graph membership is never proof of sealed cargo: a record in the cloud
|
|
2022
2125
|
// graph may name content sealed only under a prior content contract,
|
|
@@ -2025,8 +2128,14 @@ export class UserGroundHost {
|
|
|
2025
2128
|
// sealed-first upload asks that exact question itself and moves only
|
|
2026
2129
|
// what is unsealed, so Register always re-establishes cargo closure
|
|
2027
2130
|
// before returning, even when the graph itself is unchanged.
|
|
2028
|
-
const cargoRecords =
|
|
2131
|
+
const cargoRecords = traveling.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
|
|
2029
2132
|
if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
|
|
2133
|
+
this.finishPreparation(trace, preparationStarted, leg, {
|
|
2134
|
+
rebuiltRoots: rebuilt.size,
|
|
2135
|
+
records: traveling.length,
|
|
2136
|
+
cargoRecords: cargoRecords.length,
|
|
2137
|
+
graphUnchanged: true,
|
|
2138
|
+
});
|
|
2030
2139
|
// The graph is already committed; only cargo can be missing. Repair
|
|
2031
2140
|
// uploads need no journal row — a crash simply re-runs the same
|
|
2032
2141
|
// repair on the next Register.
|
|
@@ -2040,18 +2149,12 @@ export class UserGroundHost {
|
|
|
2040
2149
|
}
|
|
2041
2150
|
return;
|
|
2042
2151
|
}
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
const
|
|
2048
|
-
const
|
|
2049
|
-
const root = rootOf(record);
|
|
2050
|
-
return root !== null && replacementRoots.has(root);
|
|
2051
|
-
});
|
|
2052
|
-
const replacement = rootReplacementFromRecords(replacementRootIds, replacementRecords);
|
|
2053
|
-
// The operation's canonical bytes: staged, hashed, and submitted as-is.
|
|
2054
|
-
const operationBlob = Buffer.from(stableJson(replacement), "utf8");
|
|
2152
|
+
// The operation's canonical bytes: journaled and submitted as-is.
|
|
2153
|
+
const operationBlob = timed("operation", () => Buffer.from(stableJson(publication.operation), "utf8"));
|
|
2154
|
+
// Only pages the committed head does not already name still have to be
|
|
2155
|
+
// sealed; a rebuilt root that came out byte-identical costs nothing.
|
|
2156
|
+
const committed = graphPageIdentities(state.graph.head);
|
|
2157
|
+
const fresh = publication.pages.filter((page) => !committed.has(page.ref.sha256));
|
|
2055
2158
|
const entry = {
|
|
2056
2159
|
mutationId: randomUUID(),
|
|
2057
2160
|
resourceId: state.resourceId,
|
|
@@ -2059,24 +2162,54 @@ export class UserGroundHost {
|
|
|
2059
2162
|
baseVersion: state.headVersion,
|
|
2060
2163
|
snapshotChecksum: checksum,
|
|
2061
2164
|
deliveryWatermark,
|
|
2062
|
-
operationKind:
|
|
2063
|
-
replacementRootIds:
|
|
2165
|
+
operationKind: publication.operation.kind,
|
|
2166
|
+
replacementRootIds: publication.operation.replacedRootIds,
|
|
2064
2167
|
uploadUuids: cargoRecords.map((record) => record.uuid),
|
|
2065
|
-
snapshotBlob,
|
|
2066
2168
|
operationBlob,
|
|
2169
|
+
pages: fresh.map((page) => ({
|
|
2170
|
+
sha256: page.ref.sha256,
|
|
2171
|
+
bytes: Buffer.from(page.bytes.buffer, page.bytes.byteOffset, page.bytes.byteLength),
|
|
2172
|
+
})),
|
|
2067
2173
|
status: "pending",
|
|
2068
2174
|
submitted: false,
|
|
2069
2175
|
};
|
|
2070
|
-
supersedeSnapshotJournal(this.databasePath(identity), entry);
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2176
|
+
timed("journal", () => supersedeSnapshotJournal(this.databasePath(identity), entry));
|
|
2177
|
+
this.finishPreparation(trace, preparationStarted, leg, {
|
|
2178
|
+
rebuiltRoots: rebuilt.size,
|
|
2179
|
+
records: traveling.length,
|
|
2180
|
+
pages: publication.pages.length,
|
|
2181
|
+
freshPages: fresh.length,
|
|
2182
|
+
pageBytes: fresh.reduce((total, page) => total + page.ref.bytes, 0),
|
|
2183
|
+
operationBytes: operationBlob.byteLength,
|
|
2184
|
+
cargoRecords: cargoRecords.length,
|
|
2185
|
+
graphUnchanged: false,
|
|
2186
|
+
});
|
|
2187
|
+
// Publish the freshly journaled intent directly: the pages and the
|
|
2188
|
+
// operation bytes already exist — nothing is re-read, re-parsed, or
|
|
2189
|
+
// re-serialized between prepare and the wire.
|
|
2074
2190
|
await this.publishLoadedSnapshot(identity, {
|
|
2075
2191
|
...entry,
|
|
2076
|
-
snapshotRecords: snapshot.records,
|
|
2077
2192
|
uploadRecords: cargoRecords,
|
|
2193
|
+
graph: publication.graph,
|
|
2078
2194
|
}, trace, manifestHints);
|
|
2079
2195
|
}
|
|
2196
|
+
/** Close the graph-preparation stage with every leg it spent, so a slow
|
|
2197
|
+
* preparation names the slow part instead of one opaque number. */
|
|
2198
|
+
finishPreparation(trace, startedAt, legs, counters) {
|
|
2199
|
+
if (trace?.journey !== "register")
|
|
2200
|
+
return;
|
|
2201
|
+
this.stage({
|
|
2202
|
+
primitive: "register",
|
|
2203
|
+
stage: "graph-preparation",
|
|
2204
|
+
status: "completed",
|
|
2205
|
+
durationMs: performance.now() - startedAt,
|
|
2206
|
+
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2207
|
+
measurements: {
|
|
2208
|
+
...counters,
|
|
2209
|
+
...Object.fromEntries(Object.entries(legs).map(([name, value]) => [name, Math.round(value)])),
|
|
2210
|
+
},
|
|
2211
|
+
});
|
|
2212
|
+
}
|
|
2080
2213
|
/** Which of these heads does the authority hold sealed under the current
|
|
2081
2214
|
* content contract? Batched so a whole workspace costs a handful of
|
|
2082
2215
|
* frames; the answer is the only permission to skip an upload. */
|
|
@@ -2110,8 +2243,61 @@ export class UserGroundHost {
|
|
|
2110
2243
|
await this.publishLoadedSnapshot(identity, loadSnapshotPublication(pending), trace, manifestHints);
|
|
2111
2244
|
}
|
|
2112
2245
|
}
|
|
2246
|
+
/** Seal this publication's graph pages. Pages are immutable, content
|
|
2247
|
+
* addressed, and already in memory, so the authority's seal inventory alone
|
|
2248
|
+
* decides what still has to move and whole pages ride cargo waves. The head
|
|
2249
|
+
* may not commit until every page it newly names is sealed. */
|
|
2250
|
+
async uploadGraphPages(resourceId, pages, trace) {
|
|
2251
|
+
if (pages.length === 0)
|
|
2252
|
+
return;
|
|
2253
|
+
const started = performance.now();
|
|
2254
|
+
const sealed = await this.sealedContentHashes(resourceId, pages.map((page) => page.sha256));
|
|
2255
|
+
const unsealed = pages
|
|
2256
|
+
.filter((page) => !sealed.has(page.sha256))
|
|
2257
|
+
.map((page) => ({ page, bytes: page.bytes.byteLength }));
|
|
2258
|
+
let uploadedBytes = 0;
|
|
2259
|
+
await boundedForEach([...planCargoWaves(unsealed)], CARGO_UPLOAD_CONCURRENCY, async (wave) => {
|
|
2260
|
+
const payload = Buffer.concat(wave.items.map((item) => item.page.bytes));
|
|
2261
|
+
const encoded = await encodeContentWireBytes(payload);
|
|
2262
|
+
const frame = await retryContent(() => this.wire.requestBinary({
|
|
2263
|
+
type: "private.entity-content.put-cargo",
|
|
2264
|
+
resource_id: resourceId,
|
|
2265
|
+
// A page is a single-chunk artifact: its identity, its only chunk,
|
|
2266
|
+
// and its bytes are one and the same sha.
|
|
2267
|
+
artifacts: wave.items.map((item) => ({
|
|
2268
|
+
content_hash: item.page.sha256,
|
|
2269
|
+
bytes: item.bytes,
|
|
2270
|
+
chunks: [{ sha256: item.page.sha256, bytes: item.bytes }],
|
|
2271
|
+
})),
|
|
2272
|
+
bytes: payload.byteLength,
|
|
2273
|
+
wire_bytes: encoded.bytes.byteLength,
|
|
2274
|
+
...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
|
|
2275
|
+
}, encoded.bytes, ["private.entity-content.stored-cargo"]));
|
|
2276
|
+
const stored = new Set(Array.isArray(frame.sealed) ? frame.sealed.map(String) : []);
|
|
2277
|
+
for (const item of wave.items) {
|
|
2278
|
+
if (!stored.has(item.page.sha256)) {
|
|
2279
|
+
throw new Error(`content authority did not seal graph page ${item.page.sha256}`);
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
uploadedBytes += encoded.bytes.byteLength;
|
|
2283
|
+
});
|
|
2284
|
+
this.stage({
|
|
2285
|
+
primitive: "upload",
|
|
2286
|
+
stage: "graph-pages",
|
|
2287
|
+
status: "completed",
|
|
2288
|
+
...(trace?.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2289
|
+
durationMs: performance.now() - started,
|
|
2290
|
+
measurements: {
|
|
2291
|
+
journey: trace?.journey ?? "bootstrap",
|
|
2292
|
+
pages: pages.length,
|
|
2293
|
+
uploadedPages: unsealed.length,
|
|
2294
|
+
uploadedBytes,
|
|
2295
|
+
},
|
|
2296
|
+
});
|
|
2297
|
+
}
|
|
2113
2298
|
async publishLoadedSnapshot(identity, publication, trace, manifestHints) {
|
|
2114
2299
|
try {
|
|
2300
|
+
await this.uploadGraphPages(publication.resourceId, publication.pages, trace);
|
|
2115
2301
|
await this.uploadSnapshotContent(identity, publication.resourceId, publication.uploadRecords, trace, manifestHints);
|
|
2116
2302
|
}
|
|
2117
2303
|
catch (error) {
|
|
@@ -2136,22 +2322,36 @@ export class UserGroundHost {
|
|
|
2136
2322
|
...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
|
|
2137
2323
|
});
|
|
2138
2324
|
}
|
|
2325
|
+
const legMs = {};
|
|
2326
|
+
const local = (name, work) => {
|
|
2327
|
+
const at = performance.now();
|
|
2328
|
+
const value = work();
|
|
2329
|
+
legMs[`${name}Ms`] = performance.now() - at;
|
|
2330
|
+
return value;
|
|
2331
|
+
};
|
|
2139
2332
|
if (!publication.submitted) {
|
|
2140
|
-
markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId);
|
|
2333
|
+
local("markSubmitted", () => markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId));
|
|
2141
2334
|
}
|
|
2142
|
-
const legMs = {};
|
|
2143
2335
|
try {
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2336
|
+
// A head commit names pages instead of carrying records, so it is small
|
|
2337
|
+
// by construction and always travels inline in one frame.
|
|
2338
|
+
const submitStarted = performance.now();
|
|
2339
|
+
frame = await this.wire.request({
|
|
2340
|
+
type: "shared.mutation.submit",
|
|
2341
|
+
envelope: {
|
|
2342
|
+
resourceId: publication.resourceId,
|
|
2343
|
+
authorityEpoch: publication.authorityEpoch,
|
|
2344
|
+
mutationId: publication.mutationId,
|
|
2345
|
+
deviceId: identity.deviceId,
|
|
2346
|
+
baseVersion: publication.baseVersion,
|
|
2347
|
+
contract: ENTITY_CLOUD_CONTRACT,
|
|
2348
|
+
schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
|
|
2349
|
+
operationKind: publication.operationKind,
|
|
2350
|
+
deliveryWatermark: publication.deliveryWatermark,
|
|
2351
|
+
operation: JSON.parse(publication.operationBlob.toString("utf8")),
|
|
2352
|
+
},
|
|
2353
|
+
}, ["shared.mutation.ack"]);
|
|
2354
|
+
legMs.submitMs = performance.now() - submitStarted;
|
|
2155
2355
|
}
|
|
2156
2356
|
catch (error) {
|
|
2157
2357
|
if (trace?.journey === "register") {
|
|
@@ -2172,7 +2372,30 @@ export class UserGroundHost {
|
|
|
2172
2372
|
}
|
|
2173
2373
|
throw pipelineStageError("cloud graph commit", error);
|
|
2174
2374
|
}
|
|
2375
|
+
// A replay carries no graph: clearing the state makes the next reader
|
|
2376
|
+
// fetch the authority's own head instead of a graph missing every root
|
|
2377
|
+
// this publication did not rebuild.
|
|
2378
|
+
this.cloudState = publication.graph === null ? null : {
|
|
2379
|
+
resourceId: publication.resourceId,
|
|
2380
|
+
authorityEpoch: Number(frame.authority_epoch),
|
|
2381
|
+
headVersion: Number(frame.version),
|
|
2382
|
+
checksum: publication.snapshotChecksum,
|
|
2383
|
+
deliveryWatermark: Number(frame.delivery_watermark),
|
|
2384
|
+
graph: publication.graph,
|
|
2385
|
+
};
|
|
2386
|
+
local("clearJournal", () => deleteJournalEntry(this.databasePath(identity), publication.mutationId));
|
|
2175
2387
|
if (trace?.journey === "register") {
|
|
2388
|
+
// The authority reports where its side of the commit went; carrying
|
|
2389
|
+
// its legs here gives one ledger line the whole story: local journal
|
|
2390
|
+
// marks, wire time, gateway assemble/validate, and the database RPC.
|
|
2391
|
+
for (const [wire, leg] of [
|
|
2392
|
+
["verify_ms", "serverVerifyMs"],
|
|
2393
|
+
["validate_ms", "serverValidateMs"],
|
|
2394
|
+
["database_ms", "serverDatabaseMs"],
|
|
2395
|
+
]) {
|
|
2396
|
+
if (typeof frame[wire] === "number")
|
|
2397
|
+
legMs[leg] = frame[wire];
|
|
2398
|
+
}
|
|
2176
2399
|
this.stage({
|
|
2177
2400
|
primitive: "register",
|
|
2178
2401
|
stage: "cloud-graph-commit",
|
|
@@ -2182,19 +2405,11 @@ export class UserGroundHost {
|
|
|
2182
2405
|
measurements: {
|
|
2183
2406
|
version: Number(frame.version),
|
|
2184
2407
|
operationBytes: publication.operationBlob.byteLength,
|
|
2185
|
-
...legMs
|
|
2408
|
+
...Object.fromEntries(Object.entries(legMs)
|
|
2409
|
+
.map(([name, value]) => [name, Math.round(value)])),
|
|
2186
2410
|
},
|
|
2187
2411
|
});
|
|
2188
2412
|
}
|
|
2189
|
-
this.cloudState = {
|
|
2190
|
-
resourceId: publication.resourceId,
|
|
2191
|
-
authorityEpoch: Number(frame.authority_epoch),
|
|
2192
|
-
headVersion: Number(frame.version),
|
|
2193
|
-
checksum: publication.snapshotChecksum,
|
|
2194
|
-
deliveryWatermark: Number(frame.delivery_watermark),
|
|
2195
|
-
records: publication.snapshotRecords,
|
|
2196
|
-
};
|
|
2197
|
-
deleteJournalEntry(this.databasePath(identity), publication.mutationId);
|
|
2198
2413
|
}
|
|
2199
2414
|
}
|
|
2200
2415
|
const sha256Hex = (input) => createHash("sha256").update(input).digest("hex");
|
|
@@ -2213,6 +2428,12 @@ function contentCacheFile(cacheDir, contentHash) {
|
|
|
2213
2428
|
function manifestCacheFile(cacheDir, contentHash) {
|
|
2214
2429
|
return join(cacheDir, "manifests", `${contentHash}.json`);
|
|
2215
2430
|
}
|
|
2431
|
+
/** Graph pages are immutable and content-addressed, so they cache exactly like
|
|
2432
|
+
* artifacts do — in their own directory, because they are the registry's own
|
|
2433
|
+
* bytes rather than any entity's cargo. */
|
|
2434
|
+
function graphPageCacheDir(cacheDir) {
|
|
2435
|
+
return join(cacheDir, "graph-pages");
|
|
2436
|
+
}
|
|
2216
2437
|
function readCachedManifest(cacheDir, contentHash) {
|
|
2217
2438
|
try {
|
|
2218
2439
|
const checked = checkContentManifest(JSON.parse(readFileSync(manifestCacheFile(cacheDir, contentHash), "utf8")), contentHash);
|
|
@@ -2286,78 +2507,110 @@ function assertHealthyWatch(evidence, rootId) {
|
|
|
2286
2507
|
* file over a dozen times; the file itself is never deleted by the host, so
|
|
2287
2508
|
* every later open needs only the connection and its pragmas. */
|
|
2288
2509
|
const migratedDatabases = new Set();
|
|
2510
|
+
/** A publication row holds only what the authority must be told: the small
|
|
2511
|
+
* meta facts and the head-commit operation naming this graph's pages. The
|
|
2512
|
+
* pages themselves are write-once rows in their own table, because a status
|
|
2513
|
+
* transition rewrites its whole row — overflow pages included — through the
|
|
2514
|
+
* WAL at synchronous=FULL, and page bytes must never ride that rewrite. */
|
|
2289
2515
|
const SNAPSHOT_PUBLICATIONS_SCHEMA = `
|
|
2290
2516
|
CREATE TABLE IF NOT EXISTS snapshot_publications (
|
|
2291
2517
|
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2292
2518
|
mutation_id TEXT NOT NULL UNIQUE,
|
|
2293
2519
|
resource_id TEXT NOT NULL,
|
|
2294
2520
|
meta_json TEXT NOT NULL,
|
|
2295
|
-
snapshot_blob BLOB NOT NULL,
|
|
2296
2521
|
operation_blob BLOB NOT NULL,
|
|
2297
2522
|
created_at INTEGER NOT NULL,
|
|
2298
2523
|
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
|
|
2299
2524
|
submitted_at INTEGER
|
|
2525
|
+
);
|
|
2526
|
+
CREATE TABLE IF NOT EXISTS snapshot_publication_pages (
|
|
2527
|
+
mutation_id TEXT NOT NULL,
|
|
2528
|
+
page_index INTEGER NOT NULL,
|
|
2529
|
+
sha256 TEXT NOT NULL,
|
|
2530
|
+
bytes BLOB NOT NULL,
|
|
2531
|
+
PRIMARY KEY (mutation_id, page_index)
|
|
2300
2532
|
);`;
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2533
|
+
function legacyJournalShape(columns) {
|
|
2534
|
+
if (columns.size === 0)
|
|
2535
|
+
return null;
|
|
2536
|
+
if (columns.has("publication_json"))
|
|
2537
|
+
return "nested-text";
|
|
2538
|
+
if (columns.has("snapshot_blob"))
|
|
2539
|
+
return "row-blobs";
|
|
2540
|
+
// The paged journal and the side-table one both keep only small columns
|
|
2541
|
+
// here; only the paged one names the operation on the row itself.
|
|
2542
|
+
return columns.has("operation_blob") ? null : "split-blobs";
|
|
2543
|
+
}
|
|
2544
|
+
/** Converts a pre-paged publication journal into the paged one, atomically,
|
|
2545
|
+
* preserving every row's meaning. A pending row may be a submitted mutation
|
|
2546
|
+
* whose outcome exists only at the authority — dropping it would violate the
|
|
2547
|
+
* retry law that recovers a lost acknowledgement — so conversion is
|
|
2548
|
+
* structural: each row keeps its identity, its meta facts, and the exact
|
|
2549
|
+
* operation bytes it would submit, and semantic judgment stays where it
|
|
2550
|
+
* always was, with the reader (an unlawful row still blocks registration
|
|
2551
|
+
* rather than being silently replaced). A converted row rebuilt whole roots
|
|
2552
|
+
* rather than pages, so it carries no page rows; its replayed upload comes
|
|
2553
|
+
* from the records inside its own operation.
|
|
2310
2554
|
*
|
|
2311
2555
|
* The transform runs entirely inside SQLite. A production journal row is a
|
|
2312
2556
|
* register-scale graph serialized three times over — the exact text whose
|
|
2313
2557
|
* repeated JavaScript materialization blew the 512 MB heap — so no part of
|
|
2314
|
-
* it may ever become a JavaScript value here: json_extract lifts
|
|
2315
|
-
*
|
|
2316
|
-
*
|
|
2558
|
+
* it may ever become a JavaScript value here: json_extract lifts nested text
|
|
2559
|
+
* straight into its blob and meta rows are json_object over subtrees, all in
|
|
2560
|
+
* engine memory outside the V8 heap. The snapshot blob is simply not carried:
|
|
2561
|
+
* the authority stores the head now, and a replay re-reads it.
|
|
2317
2562
|
*
|
|
2318
2563
|
* Tables older than the three-state journal carry no status or submitted_at;
|
|
2319
2564
|
* their rows were written before the pre-submit durability mark existed, so
|
|
2320
2565
|
* the only safe reading is "possibly submitted": keep them pending and
|
|
2321
2566
|
* marked submitted, which replays the same mutation id — always idempotent —
|
|
2322
|
-
* and never supersedes. A row whose
|
|
2323
|
-
* the open loudly with the journal intact; nothing
|
|
2324
|
-
* conversion failure. */
|
|
2325
|
-
function convertLegacySnapshotJournal(database, columns) {
|
|
2567
|
+
* and never supersedes. A row whose stored operation does not resolve fails
|
|
2568
|
+
* the open loudly against the NOT NULL law with the journal intact; nothing
|
|
2569
|
+
* is ever deleted on a conversion failure. */
|
|
2570
|
+
function convertLegacySnapshotJournal(database, shape, columns) {
|
|
2571
|
+
const metaJson = shape === "nested-text"
|
|
2572
|
+
? `json_object(
|
|
2573
|
+
'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
|
|
2574
|
+
'baseVersion', json_extract(publication_json, '$.baseVersion'),
|
|
2575
|
+
'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
|
|
2576
|
+
'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
|
|
2577
|
+
'operationKind', 'entity.roots.replace',
|
|
2578
|
+
'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2579
|
+
'uploadUuids', coalesce((
|
|
2580
|
+
SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
|
|
2581
|
+
FROM json_each(publication_json, '$.uploadRecords') AS uploaded
|
|
2582
|
+
), json_array())
|
|
2583
|
+
)`
|
|
2584
|
+
: "meta_json";
|
|
2585
|
+
const operationBlob = {
|
|
2586
|
+
"nested-text": `CAST(json_object(
|
|
2587
|
+
'kind', 'entity.roots.replace',
|
|
2588
|
+
'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2589
|
+
'records', json(json_extract(publication_json, '$.replacementRecords'))
|
|
2590
|
+
) AS BLOB)`,
|
|
2591
|
+
"row-blobs": "operation_blob",
|
|
2592
|
+
"split-blobs": `(SELECT blobs.operation_blob FROM snapshot_publication_blobs AS blobs
|
|
2593
|
+
WHERE blobs.mutation_id = legacy.mutation_id)`,
|
|
2594
|
+
}[shape];
|
|
2326
2595
|
database.transaction(() => {
|
|
2327
2596
|
database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
|
|
2328
2597
|
database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
|
|
2329
2598
|
database.exec(`
|
|
2330
2599
|
INSERT INTO snapshot_publications(
|
|
2331
|
-
mutation_id, resource_id, meta_json,
|
|
2332
|
-
created_at, status, submitted_at
|
|
2600
|
+
mutation_id, resource_id, meta_json, operation_blob, created_at, status, submitted_at
|
|
2333
2601
|
)
|
|
2334
2602
|
SELECT
|
|
2335
2603
|
mutation_id,
|
|
2336
2604
|
resource_id,
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
'baseVersion', json_extract(publication_json, '$.baseVersion'),
|
|
2340
|
-
'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
|
|
2341
|
-
'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
|
|
2342
|
-
'operationKind', 'entity.roots.replace',
|
|
2343
|
-
'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2344
|
-
'uploadUuids', coalesce((
|
|
2345
|
-
SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
|
|
2346
|
-
FROM json_each(publication_json, '$.uploadRecords') AS uploaded
|
|
2347
|
-
), json_array())
|
|
2348
|
-
),
|
|
2349
|
-
CAST(json_extract(publication_json, '$.snapshotJson') AS BLOB),
|
|
2350
|
-
CAST(json_object(
|
|
2351
|
-
'kind', 'entity.roots.replace',
|
|
2352
|
-
'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2353
|
-
'records', json(json_extract(publication_json, '$.replacementRecords'))
|
|
2354
|
-
) AS BLOB),
|
|
2605
|
+
${metaJson},
|
|
2606
|
+
${operationBlob},
|
|
2355
2607
|
created_at,
|
|
2356
2608
|
${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
|
|
2357
2609
|
${columns.has("submitted_at") ? "submitted_at" : "created_at"}
|
|
2358
|
-
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2610
|
+
FROM snapshot_publications_legacy AS legacy ORDER BY sequence
|
|
2359
2611
|
`);
|
|
2360
2612
|
database.exec("DROP TABLE snapshot_publications_legacy");
|
|
2613
|
+
database.exec("DROP TABLE IF EXISTS snapshot_publication_blobs");
|
|
2361
2614
|
})();
|
|
2362
2615
|
}
|
|
2363
2616
|
/** The single opener and schema-migration entry for a user ground database.
|
|
@@ -2376,9 +2629,9 @@ export function initializeDatabase(file) {
|
|
|
2376
2629
|
// how this file arrived at this process. One PRAGMA against the in-memory
|
|
2377
2630
|
// catalog costs nothing next to the connection open itself.
|
|
2378
2631
|
const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2632
|
+
const legacyShape = legacyJournalShape(publicationColumns);
|
|
2633
|
+
if (legacyShape)
|
|
2634
|
+
convertLegacySnapshotJournal(database, legacyShape, publicationColumns);
|
|
2382
2635
|
if (migratedDatabases.has(file))
|
|
2383
2636
|
return database;
|
|
2384
2637
|
database.exec(`
|
|
@@ -2954,18 +3207,36 @@ function readSnapshotPublications(file) {
|
|
|
2954
3207
|
return [];
|
|
2955
3208
|
const database = initializeDatabase(file);
|
|
2956
3209
|
try {
|
|
3210
|
+
const pages = database.prepare(`
|
|
3211
|
+
SELECT mutation_id AS mutationId, sha256, bytes
|
|
3212
|
+
FROM snapshot_publication_pages ORDER BY mutation_id, page_index
|
|
3213
|
+
`).all();
|
|
3214
|
+
const pagesByMutation = new Map();
|
|
3215
|
+
for (const page of pages) {
|
|
3216
|
+
const carried = pagesByMutation.get(page.mutationId) ?? [];
|
|
3217
|
+
carried.push({ sha256: page.sha256, bytes: page.bytes });
|
|
3218
|
+
pagesByMutation.set(page.mutationId, carried);
|
|
3219
|
+
}
|
|
2957
3220
|
return database.prepare(`
|
|
2958
|
-
SELECT mutation_id AS mutationId,
|
|
2959
|
-
|
|
2960
|
-
|
|
3221
|
+
SELECT mutation_id AS mutationId,
|
|
3222
|
+
resource_id AS resourceId,
|
|
3223
|
+
meta_json AS metaJson,
|
|
3224
|
+
operation_blob AS operationBlob,
|
|
3225
|
+
status,
|
|
2961
3226
|
submitted_at AS submittedAt
|
|
2962
3227
|
FROM snapshot_publications ORDER BY sequence
|
|
2963
3228
|
`).all().map((row) => {
|
|
2964
3229
|
const meta = JSON.parse(row.metaJson);
|
|
2965
|
-
// No lawful publication carries zero roots —
|
|
2966
|
-
//
|
|
3230
|
+
// No lawful publication carries zero roots — a head commit that replaces
|
|
3231
|
+
// nothing would prove nothing — so a rootless row here is damage, never
|
|
3232
|
+
// a skippable row: silently dropping it would let a new publication
|
|
3233
|
+
// supersede a submitted mutation whose outcome exists only at the
|
|
3234
|
+
// authority. `entity.roots.replace` appears only on rows converted from
|
|
3235
|
+
// a pre-paged journal, which no longer produce but must still replay.
|
|
2967
3236
|
if (!Array.isArray(meta.replacementRootIds) || meta.replacementRootIds.length === 0
|
|
2968
|
-
|| !Array.isArray(meta.uploadUuids)
|
|
3237
|
+
|| !Array.isArray(meta.uploadUuids)
|
|
3238
|
+
|| (meta.operationKind !== GRAPH_HEAD_OPERATION_KIND
|
|
3239
|
+
&& meta.operationKind !== "entity.roots.replace")) {
|
|
2969
3240
|
throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
|
|
2970
3241
|
}
|
|
2971
3242
|
const deliveryWatermark = Number(meta.deliveryWatermark);
|
|
@@ -2979,11 +3250,11 @@ function readSnapshotPublications(file) {
|
|
|
2979
3250
|
baseVersion: Number(meta.baseVersion),
|
|
2980
3251
|
deliveryWatermark,
|
|
2981
3252
|
snapshotChecksum: String(meta.snapshotChecksum),
|
|
2982
|
-
operationKind: meta.operationKind,
|
|
3253
|
+
operationKind: String(meta.operationKind),
|
|
2983
3254
|
replacementRootIds: meta.replacementRootIds.map(String),
|
|
2984
3255
|
uploadUuids: meta.uploadUuids.map(String),
|
|
2985
|
-
snapshotBlob: row.snapshotBlob,
|
|
2986
3256
|
operationBlob: row.operationBlob,
|
|
3257
|
+
pages: pagesByMutation.get(row.mutationId) ?? [],
|
|
2987
3258
|
status: row.status === "rejected" ? "rejected" : "pending",
|
|
2988
3259
|
submitted: row.submittedAt !== null,
|
|
2989
3260
|
};
|
|
@@ -3150,12 +3421,17 @@ function supersedeSnapshotJournal(file, entry) {
|
|
|
3150
3421
|
if (unresolved) {
|
|
3151
3422
|
throw new Error("a submitted publication with an unknown outcome must resolve before supersession");
|
|
3152
3423
|
}
|
|
3424
|
+
database.prepare(`
|
|
3425
|
+
DELETE FROM snapshot_publication_pages WHERE mutation_id IN (
|
|
3426
|
+
SELECT mutation_id FROM snapshot_publications WHERE resource_id = ?
|
|
3427
|
+
)
|
|
3428
|
+
`).run(entry.resourceId);
|
|
3153
3429
|
database.prepare("DELETE FROM snapshot_publications WHERE resource_id = ?")
|
|
3154
3430
|
.run(entry.resourceId);
|
|
3155
3431
|
database.prepare(`
|
|
3156
3432
|
INSERT INTO snapshot_publications(
|
|
3157
|
-
mutation_id, resource_id, meta_json,
|
|
3158
|
-
) VALUES (?, ?, ?, ?,
|
|
3433
|
+
mutation_id, resource_id, meta_json, operation_blob, created_at
|
|
3434
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
3159
3435
|
`).run(entry.mutationId, entry.resourceId, stableJson({
|
|
3160
3436
|
authorityEpoch: entry.authorityEpoch,
|
|
3161
3437
|
baseVersion: entry.baseVersion,
|
|
@@ -3164,7 +3440,12 @@ function supersedeSnapshotJournal(file, entry) {
|
|
|
3164
3440
|
operationKind: entry.operationKind,
|
|
3165
3441
|
replacementRootIds: entry.replacementRootIds,
|
|
3166
3442
|
uploadUuids: entry.uploadUuids,
|
|
3167
|
-
}), entry.
|
|
3443
|
+
}), entry.operationBlob, Date.now());
|
|
3444
|
+
const insertPage = database.prepare(`
|
|
3445
|
+
INSERT INTO snapshot_publication_pages(mutation_id, page_index, sha256, bytes)
|
|
3446
|
+
VALUES (?, ?, ?, ?)
|
|
3447
|
+
`);
|
|
3448
|
+
entry.pages.forEach((page, index) => insertPage.run(entry.mutationId, index, page.sha256, page.bytes));
|
|
3168
3449
|
})();
|
|
3169
3450
|
}
|
|
3170
3451
|
finally {
|
|
@@ -3234,7 +3515,10 @@ function captureChurn(error) {
|
|
|
3234
3515
|
function deleteJournalEntry(file, mutationId) {
|
|
3235
3516
|
const database = initializeDatabase(file);
|
|
3236
3517
|
try {
|
|
3237
|
-
database.
|
|
3518
|
+
database.transaction(() => {
|
|
3519
|
+
database.prepare("DELETE FROM snapshot_publication_pages WHERE mutation_id = ?").run(mutationId);
|
|
3520
|
+
database.prepare("DELETE FROM snapshot_publications WHERE mutation_id = ?").run(mutationId);
|
|
3521
|
+
})();
|
|
3238
3522
|
}
|
|
3239
3523
|
finally {
|
|
3240
3524
|
database.close();
|
|
@@ -3614,7 +3898,7 @@ function materializedWorkspaceBindings(bindingDir) {
|
|
|
3614
3898
|
return bindings.sort((left, right) => left.workspaceId.localeCompare(right.workspaceId));
|
|
3615
3899
|
}
|
|
3616
3900
|
async function reconcileGround(input) {
|
|
3617
|
-
const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, } = input;
|
|
3901
|
+
const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, measuring = false, } = input;
|
|
3618
3902
|
const resourceId = privateEntityResourceId(identity.userId, sha256Hex);
|
|
3619
3903
|
const bindingDir = workspaceBindingDir(userRoot, identity.deviceId);
|
|
3620
3904
|
const boundRootEntries = materializedWorkspaceBindings(bindingDir);
|
|
@@ -3709,9 +3993,24 @@ async function reconcileGround(input) {
|
|
|
3709
3993
|
contentReads: 0,
|
|
3710
3994
|
gitInspections: 0,
|
|
3711
3995
|
repositoryCaptures: 0,
|
|
3996
|
+
walkMs: 0,
|
|
3997
|
+
captureMs: 0,
|
|
3998
|
+
captureBusyMs: 0,
|
|
3999
|
+
captureBytes: 0,
|
|
4000
|
+
gitMs: 0,
|
|
4001
|
+
repositoryIdentityMs: 0,
|
|
4002
|
+
repositoryCaptureMs: 0,
|
|
4003
|
+
repositorySealMs: 0,
|
|
4004
|
+
reconcileMs: 0,
|
|
4005
|
+
persistMs: 0,
|
|
3712
4006
|
};
|
|
3713
4007
|
const started = performance.now();
|
|
3714
|
-
const result = await reconcileRoot({
|
|
4008
|
+
const result = await reconcileRoot({
|
|
4009
|
+
...parameters,
|
|
4010
|
+
evidence: counters,
|
|
4011
|
+
measureEvidence: measuring,
|
|
4012
|
+
materializedUUIDs,
|
|
4013
|
+
});
|
|
3715
4014
|
if (result.transferFailure !== null) {
|
|
3716
4015
|
transferFailures.push({ rootId: parameters.rootUUID, reason: result.transferFailure });
|
|
3717
4016
|
}
|
|
@@ -3749,7 +4048,14 @@ async function reconcileGround(input) {
|
|
|
3749
4048
|
durationMs: performance.now() - started,
|
|
3750
4049
|
plannedRows: proposals.length,
|
|
3751
4050
|
records: proposals.length,
|
|
3752
|
-
|
|
4051
|
+
// Encoding every proposal to learn its wire size costs one record
|
|
4052
|
+
// construction per entity — real time on the critical path of a cold
|
|
4053
|
+
// Register. It is evidence, so it is paid only when read.
|
|
4054
|
+
...(measuring
|
|
4055
|
+
? {
|
|
4056
|
+
recordBytes: proposals.reduce((bytes, proposal) => bytes + encodedRecordBytes(parameters.resourceId, proposal), 0),
|
|
4057
|
+
}
|
|
4058
|
+
: {}),
|
|
3753
4059
|
});
|
|
3754
4060
|
// Each root commits its local projection before Detect yields. Native
|
|
3755
4061
|
// callbacks can therefore preserve new suspicion between large roots.
|
|
@@ -3848,7 +4154,7 @@ async function reconcileGround(input) {
|
|
|
3848
4154
|
};
|
|
3849
4155
|
}
|
|
3850
4156
|
async function reconcileRoot(input) {
|
|
3851
|
-
const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, plansSendRecords, largeManifests, } = input;
|
|
4157
|
+
const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, } = input;
|
|
3852
4158
|
// Prior rows reach this scan from every binding this root encloses. A row's
|
|
3853
4159
|
// stored relativePath is relative to the root that last committed it, which
|
|
3854
4160
|
// is a different frame when an enclosing boundary was just registered
|
|
@@ -4021,6 +4327,33 @@ async function reconcileRoot(input) {
|
|
|
4021
4327
|
// its own entry in place; every task is awaited before entries are read.
|
|
4022
4328
|
const captureTasks = [];
|
|
4023
4329
|
const captureGate = concurrencyGate(WALK_CAPTURE_CONCURRENCY);
|
|
4330
|
+
// The scan's three costs are metadata traversal, content capture, and Git
|
|
4331
|
+
// inspection. They interleave, so each is accumulated where it is spent
|
|
4332
|
+
// rather than inferred from the total afterwards.
|
|
4333
|
+
let captureOpenedAt = 0;
|
|
4334
|
+
let captureClosedAt = 0;
|
|
4335
|
+
const spent = (work, into) => {
|
|
4336
|
+
if (!measureEvidence || !evidence)
|
|
4337
|
+
return work();
|
|
4338
|
+
const at = performance.now();
|
|
4339
|
+
try {
|
|
4340
|
+
return work();
|
|
4341
|
+
}
|
|
4342
|
+
finally {
|
|
4343
|
+
evidence[into] += performance.now() - at;
|
|
4344
|
+
}
|
|
4345
|
+
};
|
|
4346
|
+
const spentAsync = async (work, into) => {
|
|
4347
|
+
if (!measureEvidence || !evidence)
|
|
4348
|
+
return work();
|
|
4349
|
+
const at = performance.now();
|
|
4350
|
+
try {
|
|
4351
|
+
return await work();
|
|
4352
|
+
}
|
|
4353
|
+
finally {
|
|
4354
|
+
evidence[into] += performance.now() - at;
|
|
4355
|
+
}
|
|
4356
|
+
};
|
|
4024
4357
|
const visit = async (directory, parentUUID, parentType, base = "", activeRepository = parentType === "repo.git"
|
|
4025
4358
|
? {
|
|
4026
4359
|
root: directory,
|
|
@@ -4028,8 +4361,8 @@ async function reconcileRoot(input) {
|
|
|
4028
4361
|
evidence: directory === rootPath ? rootRepositoryEvidence : null,
|
|
4029
4362
|
}
|
|
4030
4363
|
: null, forceObserve = rootRailChanged) => {
|
|
4031
|
-
const children = readdirSync(directory, { withFileTypes: true })
|
|
4032
|
-
.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
|
|
4364
|
+
const children = spent(() => readdirSync(directory, { withFileTypes: true })
|
|
4365
|
+
.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0), "walkMs");
|
|
4033
4366
|
for (const child of children) {
|
|
4034
4367
|
// Every materialization boundary reserves its root .amalgm/ directory
|
|
4035
4368
|
// for machine intents, staging, and recovery. This is structural host
|
|
@@ -4045,7 +4378,7 @@ async function reconcileRoot(input) {
|
|
|
4045
4378
|
if (!policy(relativePath))
|
|
4046
4379
|
continue;
|
|
4047
4380
|
const absolutePath = join(directory, child.name);
|
|
4048
|
-
const stats = lstatSync(absolutePath);
|
|
4381
|
+
const stats = spent(() => lstatSync(absolutePath), "walkMs");
|
|
4049
4382
|
evidence && (evidence.entries += 1);
|
|
4050
4383
|
// The walk is synchronous metadata work. Yield periodically so the
|
|
4051
4384
|
// wire, Receive, and Apply keep running while a large root is proved.
|
|
@@ -4076,7 +4409,7 @@ async function reconcileRoot(input) {
|
|
|
4076
4409
|
if (observe && repositoryPath && activeRepository && !activeRepository.evidence) {
|
|
4077
4410
|
evidence && (evidence.gitInspections += 1);
|
|
4078
4411
|
try {
|
|
4079
|
-
activeRepository.evidence = inspectGitRegistration(activeRepository.root, activeRepository.evidencePaths);
|
|
4412
|
+
activeRepository.evidence = spent(() => inspectGitRegistration(activeRepository.root, activeRepository.evidencePaths), "gitMs");
|
|
4080
4413
|
}
|
|
4081
4414
|
catch {
|
|
4082
4415
|
// Classification already settled from the marker. If Git cannot
|
|
@@ -4169,11 +4502,19 @@ async function reconcileRoot(input) {
|
|
|
4169
4502
|
});
|
|
4170
4503
|
if (deferredCapture) {
|
|
4171
4504
|
const entry = entries[entries.length - 1];
|
|
4505
|
+
if (measureEvidence && evidence && captureOpenedAt === 0)
|
|
4506
|
+
captureOpenedAt = performance.now();
|
|
4172
4507
|
captureTasks.push(captureGate(async () => {
|
|
4508
|
+
const taskStarted = measureEvidence ? performance.now() : 0;
|
|
4173
4509
|
let current = stats;
|
|
4174
4510
|
for (let attempt = 1;; attempt++) {
|
|
4175
4511
|
try {
|
|
4176
4512
|
const captured = await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir);
|
|
4513
|
+
if (measureEvidence && evidence) {
|
|
4514
|
+
evidence.captureBusyMs += performance.now() - taskStarted;
|
|
4515
|
+
evidence.captureBytes += captured.bytes;
|
|
4516
|
+
captureClosedAt = performance.now();
|
|
4517
|
+
}
|
|
4177
4518
|
entry.type = classifyFile({ binary: captured.binary });
|
|
4178
4519
|
entry.payloadVersion = captured.contentHash;
|
|
4179
4520
|
entry.contentVerifiedAtMs = Date.now();
|
|
@@ -4219,6 +4560,10 @@ async function reconcileRoot(input) {
|
|
|
4219
4560
|
};
|
|
4220
4561
|
await visit(rootPath, rootUUID, rootType);
|
|
4221
4562
|
await Promise.all(captureTasks);
|
|
4563
|
+
if (measureEvidence && evidence && captureOpenedAt !== 0) {
|
|
4564
|
+
evidence.captureMs += captureClosedAt - captureOpenedAt;
|
|
4565
|
+
}
|
|
4566
|
+
const reconcileStarted = measureEvidence ? performance.now() : 0;
|
|
4222
4567
|
// When every entry sits at its stored address with its stored type and
|
|
4223
4568
|
// physical fingerprint (honoring any fixed UUID), path-first claiming makes
|
|
4224
4569
|
// the multi-pass reconciliation the identity map — resolve directly. Any
|
|
@@ -4252,6 +4597,8 @@ async function reconcileRoot(input) {
|
|
|
4252
4597
|
inode: entry.inode,
|
|
4253
4598
|
...(entry.fixedUUID ? { fixedUUID: entry.fixedUUID } : {}),
|
|
4254
4599
|
})));
|
|
4600
|
+
if (measureEvidence && evidence)
|
|
4601
|
+
evidence.reconcileMs += performance.now() - reconcileStarted;
|
|
4255
4602
|
const resolvedByProvisional = new Map();
|
|
4256
4603
|
for (const entry of entries) {
|
|
4257
4604
|
const resolved = resolvedIdentity.get(entry.relativePath) ?? entry.uuid;
|
|
@@ -4318,13 +4665,13 @@ async function reconcileRoot(input) {
|
|
|
4318
4665
|
contentVerifiedAtMs: entry.contentVerifiedAtMs,
|
|
4319
4666
|
};
|
|
4320
4667
|
});
|
|
4321
|
-
const persistCurrentRows = () => {
|
|
4668
|
+
const persistCurrentRows = () => spent(() => {
|
|
4322
4669
|
const rows = rowsFromEntries();
|
|
4323
4670
|
if (input.suspicion === undefined) {
|
|
4324
4671
|
persistRows(database, identity, resourceId, rootUUID, rows, existingRows);
|
|
4325
4672
|
}
|
|
4326
4673
|
return rows;
|
|
4327
|
-
};
|
|
4674
|
+
}, "persistMs");
|
|
4328
4675
|
// Seal nested repositories first so their independent rows are complete
|
|
4329
4676
|
// before the enclosing graph is persisted. Their state is not an input to
|
|
4330
4677
|
// the enclosing identity map; only their address boundary is.
|
|
@@ -4452,7 +4799,7 @@ async function reconcileRoot(input) {
|
|
|
4452
4799
|
}
|
|
4453
4800
|
}
|
|
4454
4801
|
const fingerprintBefore = gitEvidenceFingerprint(repository.absolutePath);
|
|
4455
|
-
let repoIdentity = await refreshRepositoryIdentity
|
|
4802
|
+
let repoIdentity = await spentAsync(refreshRepositoryIdentity, "repositoryIdentityMs");
|
|
4456
4803
|
/* The rows and the transport identity are derived from the same fresh
|
|
4457
4804
|
* worktree evidence. A later Watch ring for one of these dirty paths is
|
|
4458
4805
|
* therefore ordinary equality, not a transport-only echo. */
|
|
@@ -4490,8 +4837,8 @@ async function reconcileRoot(input) {
|
|
|
4490
4837
|
let captured = null;
|
|
4491
4838
|
let coherent = false;
|
|
4492
4839
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4493
|
-
captured = captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity);
|
|
4494
|
-
const after = await refreshRepositoryIdentity
|
|
4840
|
+
captured = spent(() => captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity), "repositoryCaptureMs");
|
|
4841
|
+
const after = await spentAsync(refreshRepositoryIdentity, "repositoryIdentityMs");
|
|
4495
4842
|
if (sameGitIdentity(after, repoIdentity)) {
|
|
4496
4843
|
repoIdentity = [...captured.identity];
|
|
4497
4844
|
coherent = true;
|
|
@@ -4513,15 +4860,18 @@ async function reconcileRoot(input) {
|
|
|
4513
4860
|
}
|
|
4514
4861
|
repository.payloadVersion = captured.stateId;
|
|
4515
4862
|
repository.transportVersion = captured.transportVersion;
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4863
|
+
const settled = captured;
|
|
4864
|
+
const currentLayout = spent(() => {
|
|
4865
|
+
if (settled.bytes) {
|
|
4866
|
+
sealCapturedArtifact(cacheDir, {
|
|
4867
|
+
entityId: repository.uuid,
|
|
4868
|
+
entityType: "repo.git",
|
|
4869
|
+
kind: "git",
|
|
4870
|
+
contentHash: settled.transportVersion,
|
|
4871
|
+
}, settled.bytes);
|
|
4872
|
+
}
|
|
4873
|
+
return inspectRepositoryTransportFile(join(cacheDir, `${settled.transportVersion}.bin`));
|
|
4874
|
+
}, "repositorySealMs");
|
|
4525
4875
|
replayByUuid.set(repository.uuid, {
|
|
4526
4876
|
kind: "repo.git",
|
|
4527
4877
|
stateId: currentLayout.stateId,
|
|
@@ -4760,19 +5110,79 @@ function recordRootResolver(records) {
|
|
|
4760
5110
|
};
|
|
4761
5111
|
return rootOf;
|
|
4762
5112
|
}
|
|
4763
|
-
|
|
4764
|
-
|
|
5113
|
+
/** The roots one publication rebuilds, each with its complete new records.
|
|
5114
|
+
*
|
|
5115
|
+
* A walked root is rebuilt from its ground alone: materialized ground is the
|
|
5116
|
+
* whole truth for the roots it covers, so nothing of the committed graph
|
|
5117
|
+
* survives underneath one. Any other root the newly delivered authority prefix
|
|
5118
|
+
* touched is rebuilt from the committed graph with those results folded in, so
|
|
5119
|
+
* the published graph never declares a watermark it has not actually applied.
|
|
5120
|
+
* Every root outside this map keeps its committed pages untouched. */
|
|
5121
|
+
function publicationRoots(graph, localRecords, accepted) {
|
|
5122
|
+
const rebuilt = new Map();
|
|
5123
|
+
const walked = new Map(localRecords
|
|
4765
5124
|
.filter((record) => record.parentUUID === null)
|
|
4766
|
-
.map((record) => record.uuid));
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
5125
|
+
.map((record) => [record.uuid, []]));
|
|
5126
|
+
if (walked.size > 0) {
|
|
5127
|
+
const rootOf = recordRootResolver(localRecords);
|
|
5128
|
+
for (const record of localRecords) {
|
|
5129
|
+
const rootId = rootOf(record);
|
|
5130
|
+
const group = rootId === null ? undefined : walked.get(rootId);
|
|
5131
|
+
if (!group)
|
|
5132
|
+
throw new Error(`walked record ${record.uuid} descends from no walked root`);
|
|
5133
|
+
group.push(record);
|
|
5134
|
+
}
|
|
5135
|
+
for (const [rootId, records] of walked)
|
|
5136
|
+
rebuilt.set(rootId, records);
|
|
5137
|
+
}
|
|
5138
|
+
const overlays = new Map();
|
|
5139
|
+
for (const result of accepted) {
|
|
5140
|
+
const rootId = graph.rootOf(result.uuid);
|
|
5141
|
+
if (rootId === null || walked.has(rootId))
|
|
5142
|
+
continue;
|
|
5143
|
+
const overlay = overlays.get(rootId);
|
|
5144
|
+
if (overlay)
|
|
5145
|
+
overlay.push(result);
|
|
5146
|
+
else
|
|
5147
|
+
overlays.set(rootId, [result]);
|
|
5148
|
+
}
|
|
5149
|
+
for (const [rootId, results] of overlays) {
|
|
5150
|
+
const byUuid = new Map(graph.recordsFor(rootId).map((record) => [record.uuid, record]));
|
|
5151
|
+
for (const result of results)
|
|
5152
|
+
byUuid.set(result.uuid, result);
|
|
5153
|
+
// Entity channels are independently ordered, so container membership is
|
|
5154
|
+
// derived once from the final prefix rather than by arrival order.
|
|
5155
|
+
rebuilt.set(rootId, recordsRetainingMissingEvidence([...byUuid.values()], []));
|
|
5156
|
+
}
|
|
5157
|
+
// A records-shaped authority base carries no page identities, so it can
|
|
5158
|
+
// prove nothing about a root the commit leaves alone: the first head pages
|
|
5159
|
+
// the whole account at once, and never again.
|
|
5160
|
+
if (graph.head === null) {
|
|
5161
|
+
for (const rootId of graph.rootIds()) {
|
|
5162
|
+
if (!rebuilt.has(rootId))
|
|
5163
|
+
rebuilt.set(rootId, graph.recordsFor(rootId));
|
|
5164
|
+
}
|
|
5165
|
+
}
|
|
5166
|
+
// Re-rooting is a move, not a copy: an enclosing Register walks records the
|
|
5167
|
+
// committed graph still files under the root that used to hold them. Every
|
|
5168
|
+
// such root is rebuilt without them, and a root that keeps nothing is left
|
|
5169
|
+
// with no records at all — the rebuild then drops it from the graph.
|
|
5170
|
+
const vacated = new Map();
|
|
5171
|
+
for (const record of localRecords) {
|
|
5172
|
+
const rootId = graph.rootOf(record.uuid);
|
|
5173
|
+
if (rootId === null || walked.has(rootId))
|
|
5174
|
+
continue;
|
|
5175
|
+
const gone = vacated.get(rootId);
|
|
5176
|
+
if (gone)
|
|
5177
|
+
gone.add(record.uuid);
|
|
5178
|
+
else
|
|
5179
|
+
vacated.set(rootId, new Set([record.uuid]));
|
|
5180
|
+
}
|
|
5181
|
+
for (const [rootId, gone] of vacated) {
|
|
5182
|
+
const held = rebuilt.get(rootId) ?? graph.recordsFor(rootId);
|
|
5183
|
+
rebuilt.set(rootId, held.filter((record) => !gone.has(record.uuid)));
|
|
5184
|
+
}
|
|
5185
|
+
return rebuilt;
|
|
4776
5186
|
}
|
|
4777
5187
|
/** Filesystem disappearance is evidence, not cloud lifecycle authority. Keep
|
|
4778
5188
|
* the last logical records in the outgoing graph while their materialized
|
|
@@ -5345,14 +5755,14 @@ async function installCloudWorkspace(input) {
|
|
|
5345
5755
|
alreadyMaterialized: false,
|
|
5346
5756
|
};
|
|
5347
5757
|
}
|
|
5348
|
-
function readyFromFrame(frame) {
|
|
5349
|
-
const bytes = Buffer.from(String(frame.snapshot_b64 || ""), "base64");
|
|
5758
|
+
function readyFromFrame(frame, graph) {
|
|
5350
5759
|
const cloud = {
|
|
5351
5760
|
resourceId: String(frame.resource_id || ""),
|
|
5352
5761
|
version: Number(frame.head_version),
|
|
5353
5762
|
deliveryWatermark: Number(frame.delivery_watermark),
|
|
5354
5763
|
checksum: String(frame.snapshot_checksum || ""),
|
|
5355
|
-
snapshot:
|
|
5764
|
+
snapshot: graph.snapshot(),
|
|
5765
|
+
...(graph.head ? { head: graph.head } : {}),
|
|
5356
5766
|
};
|
|
5357
5767
|
return {
|
|
5358
5768
|
cloud,
|
|
@@ -5362,7 +5772,7 @@ function readyFromFrame(frame) {
|
|
|
5362
5772
|
headVersion: cloud.version,
|
|
5363
5773
|
checksum: cloud.checksum,
|
|
5364
5774
|
deliveryWatermark: cloud.deliveryWatermark,
|
|
5365
|
-
|
|
5775
|
+
graph,
|
|
5366
5776
|
},
|
|
5367
5777
|
};
|
|
5368
5778
|
}
|