@amalgm/shell 0.1.112 → 0.1.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PURPOSE.md +75 -6
- package/README.md +36 -7
- package/dist/authorization/device-key.d.ts +19 -0
- package/dist/authorization/device-key.js +81 -0
- package/dist/authorization/device-key.js.map +1 -0
- package/dist/authorization/index.d.ts +65 -0
- package/dist/authorization/index.js +63 -0
- package/dist/authorization/index.js.map +1 -0
- package/dist/authorization/issuer.d.ts +11 -0
- package/dist/authorization/issuer.js +76 -0
- package/dist/authorization/issuer.js.map +1 -0
- package/dist/authorization/key-store.d.ts +15 -0
- package/dist/authorization/key-store.js +148 -0
- package/dist/authorization/key-store.js.map +1 -0
- package/dist/authorization/recovery-client.d.ts +19 -0
- package/dist/authorization/recovery-client.js +102 -0
- package/dist/authorization/recovery-client.js.map +1 -0
- package/dist/authorization/recovery-host.d.ts +28 -0
- package/dist/authorization/recovery-host.js +93 -0
- package/dist/authorization/recovery-host.js.map +1 -0
- package/dist/authorization/recovery.d.ts +13 -0
- package/dist/authorization/recovery.js +27 -0
- package/dist/authorization/recovery.js.map +1 -0
- package/dist/authorization/secret-process.d.ts +2 -0
- package/dist/authorization/secret-process.js +32 -0
- package/dist/authorization/secret-process.js.map +1 -0
- package/dist/boot.d.ts +19 -11
- package/dist/boot.js +34 -25
- package/dist/boot.js.map +1 -1
- package/dist/builtin-agents.d.ts +3 -0
- package/dist/builtin-agents.js +15 -0
- package/dist/builtin-agents.js.map +1 -0
- package/dist/chat-composition.d.ts +19 -0
- package/dist/chat-composition.js +83 -0
- package/dist/chat-composition.js.map +1 -0
- package/dist/chat-host.d.ts +9 -25
- package/dist/chat-host.js +12 -158
- package/dist/chat-host.js.map +1 -1
- package/dist/cli.js +21 -13
- package/dist/cli.js.map +1 -1
- package/dist/content-cache-host.d.ts +7 -0
- package/dist/content-cache-host.js +23 -1
- package/dist/content-cache-host.js.map +1 -1
- package/dist/credentials-host.d.ts +1 -5
- package/dist/credentials-host.js +1 -5
- package/dist/credentials-host.js.map +1 -1
- package/dist/entity-record-store.d.ts +2 -0
- package/dist/entity-record-store.js +15 -0
- package/dist/entity-record-store.js.map +1 -1
- package/dist/forward.js +1 -1
- package/dist/forward.js.map +1 -1
- package/dist/index.d.ts +7 -6
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/machine-host.d.ts +29 -0
- package/dist/machine-host.js +240 -10
- package/dist/machine-host.js.map +1 -1
- package/dist/parity.d.ts +1 -5
- package/dist/parity.js +4 -41
- package/dist/parity.js.map +1 -1
- package/dist/registry-graph.d.ts +61 -0
- package/dist/registry-graph.js +156 -0
- package/dist/registry-graph.js.map +1 -0
- package/dist/release-catalog.js +7 -4
- package/dist/release-catalog.js.map +1 -1
- package/dist/runtime-command-host.d.ts +11 -0
- package/dist/runtime-command-host.js +65 -0
- package/dist/runtime-command-host.js.map +1 -1
- package/dist/user-ground-host.d.ts +13 -0
- package/dist/user-ground-host.js +449 -205
- package/dist/user-ground-host.js.map +1 -1
- package/dist/wire-client.js +1 -1
- package/package.json +15 -6
- package/dist/chat-payloads-host.d.ts +0 -8
- package/dist/chat-payloads-host.js +0 -76
- package/dist/chat-payloads-host.js.map +0 -1
- package/dist/chat-relay-host.d.ts +0 -24
- package/dist/chat-relay-host.js +0 -90
- package/dist/chat-relay-host.js.map +0 -1
- package/dist/drivers/claude-code.d.ts +0 -27
- package/dist/drivers/claude-code.js +0 -72
- package/dist/drivers/claude-code.js.map +0 -1
- package/dist/paged-mutation-submit.d.ts +0 -28
- package/dist/paged-mutation-submit.js +0 -61
- package/dist/paged-mutation-submit.js.map +0 -1
package/dist/user-ground-host.js
CHANGED
|
@@ -3,7 +3,7 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
|
|
|
3
3
|
import { open, readFile } from "node:fs/promises";
|
|
4
4
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
5
|
import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
|
|
6
|
-
import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, deriveUploadManifest, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash,
|
|
6
|
+
import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, deriveUploadManifest, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, pathIsSuspect, planCargoWaves, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, GRAPH_HEAD_OPERATION_KIND, } from "@amalgm/live";
|
|
7
7
|
import Database from "better-sqlite3";
|
|
8
8
|
import { atomicCopy, atomicWrite, ensurePrivateDir, ensureUserDir } from "./filesystem.js";
|
|
9
9
|
import { emitFilesStage, } from "./files-observability.js";
|
|
@@ -11,7 +11,7 @@ import { EntityApplyHost, } from "./entity-apply-host.js";
|
|
|
11
11
|
import { EntityRecordSqliteStore } from "./entity-record-store.js";
|
|
12
12
|
import { observeEntityRecordPorts } from "./entity-record-observability.js";
|
|
13
13
|
import { indexRowsByOutermostRoot } from "./ground-root-index.js";
|
|
14
|
-
import { ContentCacheDownloadSet, captureContentFile, hashContentFile, sealCapturedArtifact, } from "./content-cache-host.js";
|
|
14
|
+
import { ContentCacheDownloadSet, captureContentFile, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
|
|
15
15
|
import { DirectContentPlane } from "./content-direct.js";
|
|
16
16
|
import { encodeContentWireBytes } from "./content-wire-codec.js";
|
|
17
17
|
import { exactTextReplay, planGroundDetection, reconcileGroundUUIDs, } from "./detection/portable.js";
|
|
@@ -22,9 +22,9 @@ import { WORKSPACE_UUID as UUID, createFilesRegisterPorts, ensureWorkspaceBindin
|
|
|
22
22
|
import { applyRepositoryFiles, captureRepository, inspectRepositoryTransportFile, hasGitMarker, readCachedRepositoryIdentity, } from "./git-repository-host.js";
|
|
23
23
|
import { gitEvidenceFingerprint, inspectGitRegistration, sameGitIdentity, } from "./git-registration-host.js";
|
|
24
24
|
import { projectMaterializedGraph } from "./materialized-graph.js";
|
|
25
|
+
import { graphPageIdentities, registryGraphFromSnapshot, } from "./registry-graph.js";
|
|
25
26
|
import { NodeWatchHost, } from "./watching/index.js";
|
|
26
27
|
import { GroundCoordinator } from "./ground-coordination.js";
|
|
27
|
-
import { submitMutationOperation } from "./paged-mutation-submit.js";
|
|
28
28
|
import { adoptSnapshotDeliveryBaseline } from "./snapshot-delivery-baseline.js";
|
|
29
29
|
import { WireClient, WireRequestError } from "./wire-client.js";
|
|
30
30
|
/** Cargo waves hold up to 16MiB of payload each; two in flight bounds the
|
|
@@ -40,6 +40,10 @@ const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = 128;
|
|
|
40
40
|
// on the 1 GiB runtimes used for ordinary Files materialization. Small
|
|
41
41
|
// objects still use the full request-concurrency window.
|
|
42
42
|
const DIRECT_OBJECT_DOWNLOAD_MAX_IN_FLIGHT_BYTES = 2 * CHUNK_BYTES;
|
|
43
|
+
/** Graph pages are at most a few megabytes each and a whole account is a
|
|
44
|
+
* handful of them, so a narrow lane keeps the cold read fast without holding
|
|
45
|
+
* the whole graph twice while it assembles. */
|
|
46
|
+
const GRAPH_PAGE_READ_CONCURRENCY = 8;
|
|
43
47
|
const DETECT_QUIET_MS = 12;
|
|
44
48
|
const DETECT_MAX_DEFERRAL_MS = 75;
|
|
45
49
|
const DETECT_RETRY_MS = 250;
|
|
@@ -55,13 +59,19 @@ const WALK_CAPTURE_CONCURRENCY = 8;
|
|
|
55
59
|
const REGISTER_STALE_CAPTURE_ATTEMPTS = 3;
|
|
56
60
|
const DOWNLOAD_CONTENT_RETRY_ATTEMPTS = 3;
|
|
57
61
|
const APPLY_GROUND_WAIT_TIMEOUT_MS = 15_000;
|
|
62
|
+
/** Load a journaled publication for replay. The records whose cargo must
|
|
63
|
+
* re-upload are already inside the publication's own bytes — its pages, or,
|
|
64
|
+
* for a row converted from the pre-page journal, the replacement operation
|
|
65
|
+
* itself — so nothing is stored twice and nothing is re-read from ground. */
|
|
58
66
|
function loadSnapshotPublication(entry) {
|
|
59
|
-
const snapshotRecords = snapshotFromRecords((JSON.parse(entry.snapshotBlob.toString("utf8")).records ?? [])).records;
|
|
60
67
|
const uploadIds = new Set(entry.uploadUuids);
|
|
68
|
+
const published = uploadIds.size === 0 ? [] : entry.pages.length > 0
|
|
69
|
+
? entry.pages.flatMap((page) => JSON.parse(page.bytes.toString("utf8")))
|
|
70
|
+
: (JSON.parse(entry.operationBlob.toString("utf8")).records ?? []);
|
|
61
71
|
return {
|
|
62
72
|
...entry,
|
|
63
|
-
|
|
64
|
-
|
|
73
|
+
uploadRecords: published.filter((record) => uploadIds.has(record.uuid)),
|
|
74
|
+
graph: null,
|
|
65
75
|
};
|
|
66
76
|
}
|
|
67
77
|
const GROUND_ROW_COLUMNS = [
|
|
@@ -558,7 +568,7 @@ export class UserGroundHost {
|
|
|
558
568
|
&& record.status === "active"
|
|
559
569
|
&& record.payloadVersion === workspaceId);
|
|
560
570
|
if (hasRegisteredBoundaryInDatabase(database, workspaceId)
|
|
561
|
-
&& hasRegisteredBoundary(this.cloudState.records)) {
|
|
571
|
+
&& hasRegisteredBoundary(this.cloudState.graph.records())) {
|
|
562
572
|
// Already declared — but Register may only return once cargo closure
|
|
563
573
|
// holds under the current content contract. The graph head can be
|
|
564
574
|
// unchanged while a contract cutover moved its cargo namespace, so
|
|
@@ -740,7 +750,7 @@ export class UserGroundHost {
|
|
|
740
750
|
if (!hasRegisteredBoundaryInDatabase(database, workspaceId)) {
|
|
741
751
|
throw stageError("entity registration", new Error(`workspace ${workspaceId} and its durable reference were not recorded`));
|
|
742
752
|
}
|
|
743
|
-
if (!hasRegisteredBoundary(this.cloudState.records)) {
|
|
753
|
+
if (!hasRegisteredBoundary(this.cloudState.graph.records())) {
|
|
744
754
|
throw stageError("cloud graph commit", new Error(`workspace ${workspaceId} or its durable reference is absent from the committed head`));
|
|
745
755
|
}
|
|
746
756
|
return {
|
|
@@ -822,13 +832,45 @@ export class UserGroundHost {
|
|
|
822
832
|
throw new Error("entity Record rail already started");
|
|
823
833
|
await this.wire.open();
|
|
824
834
|
// One floor, no per-feature gates: this shell and its gateway release
|
|
825
|
-
// together, and the gateway ships first.
|
|
826
|
-
//
|
|
827
|
-
|
|
835
|
+
// together, and the gateway ships first. 13 = immutable registry graph
|
|
836
|
+
// pages plus one graph-head commit; an older gateway cannot publish or
|
|
837
|
+
// resolve that representation.
|
|
838
|
+
if (this.wire.protocolVersion < 13) {
|
|
828
839
|
throw new Error("gateway speaks an older wire protocol than this shell");
|
|
829
840
|
}
|
|
830
841
|
this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
|
|
831
|
-
const
|
|
842
|
+
const recordDatabase = initializeDatabase(this.databasePath(identity));
|
|
843
|
+
const store = new EntityRecordSqliteStore(recordDatabase);
|
|
844
|
+
const pruneAcceptedCargo = () => {
|
|
845
|
+
const retained = new Set(store.pendingContentHashes());
|
|
846
|
+
const currentReplayBases = recordDatabase.prepare(`
|
|
847
|
+
SELECT type,
|
|
848
|
+
payload_version AS payloadVersion,
|
|
849
|
+
transport_version AS transportVersion
|
|
850
|
+
FROM entities
|
|
851
|
+
WHERE type IN ('file.text', 'repo.git')
|
|
852
|
+
`).all();
|
|
853
|
+
for (const row of currentReplayBases) {
|
|
854
|
+
const contentHash = row.type === "file.text" ? row.payloadVersion : row.transportVersion;
|
|
855
|
+
if (contentHash && /^[0-9a-f]{64}$/.test(contentHash))
|
|
856
|
+
retained.add(contentHash);
|
|
857
|
+
}
|
|
858
|
+
pruneSettledContentCache(this.cacheDir(identity), retained);
|
|
859
|
+
};
|
|
860
|
+
// Accepted Send cargo is no longer retry responsibility. Keep only the
|
|
861
|
+
// current verified text and repository heads needed for the next exact
|
|
862
|
+
// delta; reclaim all other unreferenced history before start and after
|
|
863
|
+
// every acceptance.
|
|
864
|
+
pruneAcceptedCargo();
|
|
865
|
+
const cacheReleasingStore = {
|
|
866
|
+
readPending: (limit) => store.readPending(limit),
|
|
867
|
+
stampAccepted: async (...args) => {
|
|
868
|
+
store.stampAccepted(...args);
|
|
869
|
+
pruneAcceptedCargo();
|
|
870
|
+
},
|
|
871
|
+
readDeliveryCursor: (authorityId) => store.readDeliveryCursor(authorityId),
|
|
872
|
+
receive: (input) => store.receive(input),
|
|
873
|
+
};
|
|
832
874
|
const wireAuthority = createEntityRecordAuthorityPort({
|
|
833
875
|
request: (frame, acceptedTypes, binaryBody) => binaryBody === undefined
|
|
834
876
|
? this.wire.request({ ...frame }, acceptedTypes)
|
|
@@ -837,7 +879,7 @@ export class UserGroundHost {
|
|
|
837
879
|
onDisconnect: (listener) => this.wire.onDisconnect(listener),
|
|
838
880
|
});
|
|
839
881
|
const ports = observeEntityRecordPorts({
|
|
840
|
-
store,
|
|
882
|
+
store: cacheReleasingStore,
|
|
841
883
|
authority: wireAuthority,
|
|
842
884
|
cargo: {
|
|
843
885
|
ensure: async (record) => {
|
|
@@ -1096,7 +1138,7 @@ export class UserGroundHost {
|
|
|
1096
1138
|
}, ["private.resource.ready", "private.resource.missing"]);
|
|
1097
1139
|
if (frame.type === "private.resource.missing")
|
|
1098
1140
|
return null;
|
|
1099
|
-
const ready = readyFromFrame(frame);
|
|
1141
|
+
const ready = readyFromFrame(frame, await this.materializeRegistry(identity, frame));
|
|
1100
1142
|
this.cloudState = ready.state;
|
|
1101
1143
|
return ready.cloud;
|
|
1102
1144
|
},
|
|
@@ -1111,7 +1153,7 @@ export class UserGroundHost {
|
|
|
1111
1153
|
snapshot_checksum: input.checksum,
|
|
1112
1154
|
snapshot_bytes: bytes.length,
|
|
1113
1155
|
}, ["private.resource.ready"]);
|
|
1114
|
-
const ready = readyFromFrame(frame);
|
|
1156
|
+
const ready = readyFromFrame(frame, await this.materializeRegistry(identity, frame));
|
|
1115
1157
|
const current = ready.cloud;
|
|
1116
1158
|
this.cloudState = ready.state;
|
|
1117
1159
|
const created = frame.duplicate === false;
|
|
@@ -1121,6 +1163,61 @@ export class UserGroundHost {
|
|
|
1121
1163
|
},
|
|
1122
1164
|
};
|
|
1123
1165
|
}
|
|
1166
|
+
/** The authority's stored snapshot as this machine's graph. A head names
|
|
1167
|
+
* immutable pages, so every page this machine already cached is a local read
|
|
1168
|
+
* and an unchanged root never crosses the network twice. */
|
|
1169
|
+
async materializeRegistry(identity, frame) {
|
|
1170
|
+
const resourceId = String(frame.resource_id || "");
|
|
1171
|
+
return registryGraphFromSnapshot(Buffer.from(String(frame.snapshot_b64 || ""), "base64"), (refs) => this.readGraphPages(identity, resourceId, refs), sha256Hex);
|
|
1172
|
+
}
|
|
1173
|
+
/** Read graph pages by identity: cached pages come off local disk, the rest
|
|
1174
|
+
* are fetched straight from content storage and cached on the way in. Every
|
|
1175
|
+
* page is re-proven against its declared identity before it is kept. */
|
|
1176
|
+
async readGraphPages(identity, resourceId, refs) {
|
|
1177
|
+
const directory = graphPageCacheDir(this.cacheDir(identity));
|
|
1178
|
+
const pages = new Map();
|
|
1179
|
+
const missing = [];
|
|
1180
|
+
for (const ref of new Map(refs.map((ref) => [ref.sha256, ref])).values()) {
|
|
1181
|
+
let cached = null;
|
|
1182
|
+
try {
|
|
1183
|
+
cached = readFileSync(join(directory, `${ref.sha256}.bin`));
|
|
1184
|
+
}
|
|
1185
|
+
catch {
|
|
1186
|
+
cached = null;
|
|
1187
|
+
}
|
|
1188
|
+
// A cache copy of the wrong size is torn evidence, never a page: fetch
|
|
1189
|
+
// the immutable original instead of assembling from it.
|
|
1190
|
+
if (cached && cached.byteLength === ref.bytes)
|
|
1191
|
+
pages.set(ref.sha256, cached);
|
|
1192
|
+
else
|
|
1193
|
+
missing.push(ref);
|
|
1194
|
+
}
|
|
1195
|
+
if (missing.length === 0)
|
|
1196
|
+
return pages;
|
|
1197
|
+
const started = performance.now();
|
|
1198
|
+
let fetchedBytes = 0;
|
|
1199
|
+
await boundedForEach(missing, GRAPH_PAGE_READ_CONCURRENCY, async (ref) => {
|
|
1200
|
+
const bytes = await retryContent(() => this.direct.fetchObject(resourceId, { kind: "block", sha256: ref.sha256 }), true);
|
|
1201
|
+
if (bytes.byteLength !== ref.bytes || sha256Hex(bytes) !== ref.sha256) {
|
|
1202
|
+
throw new Error(`cloud graph page ${ref.sha256} does not match its bytes`);
|
|
1203
|
+
}
|
|
1204
|
+
atomicWrite(join(directory, `${ref.sha256}.bin`), bytes);
|
|
1205
|
+
fetchedBytes += bytes.byteLength;
|
|
1206
|
+
pages.set(ref.sha256, bytes);
|
|
1207
|
+
});
|
|
1208
|
+
this.stage({
|
|
1209
|
+
primitive: "download",
|
|
1210
|
+
stage: "graph-pages",
|
|
1211
|
+
status: "completed",
|
|
1212
|
+
durationMs: performance.now() - started,
|
|
1213
|
+
measurements: {
|
|
1214
|
+
pages: refs.length,
|
|
1215
|
+
fetchedPages: missing.length,
|
|
1216
|
+
fetchedBytes,
|
|
1217
|
+
},
|
|
1218
|
+
});
|
|
1219
|
+
return pages;
|
|
1220
|
+
}
|
|
1124
1221
|
localPort(identity) {
|
|
1125
1222
|
const userRoot = this.userRoot(identity);
|
|
1126
1223
|
const database = this.databasePath(identity);
|
|
@@ -1772,7 +1869,22 @@ export class UserGroundHost {
|
|
|
1772
1869
|
durationMs: performance.now() - waitStarted,
|
|
1773
1870
|
});
|
|
1774
1871
|
try {
|
|
1775
|
-
|
|
1872
|
+
let retriedAdvancedBasis = false;
|
|
1873
|
+
for (;;) {
|
|
1874
|
+
try {
|
|
1875
|
+
await this.syncLocalChangesUnlocked(identity, pending, coreScan);
|
|
1876
|
+
break;
|
|
1877
|
+
}
|
|
1878
|
+
catch (error) {
|
|
1879
|
+
if (retriedAdvancedBasis || !detectionBasisAdvanced(error))
|
|
1880
|
+
throw error;
|
|
1881
|
+
// The complete generation remains unsettled. Re-read its truth once
|
|
1882
|
+
// while this pass still owns every affected ground cone; another
|
|
1883
|
+
// conflict means an invariant outside this finite pass moved again
|
|
1884
|
+
// and must remain visible to the caller.
|
|
1885
|
+
retriedAdvancedBasis = true;
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1776
1888
|
}
|
|
1777
1889
|
finally {
|
|
1778
1890
|
release();
|
|
@@ -1908,7 +2020,7 @@ export class UserGroundHost {
|
|
|
1908
2020
|
proposals: detectedRecords,
|
|
1909
2021
|
});
|
|
1910
2022
|
if (!committed) {
|
|
1911
|
-
throw pipelineStageError("detection", new
|
|
2023
|
+
throw pipelineStageError("detection", new DetectionBasisAdvancedError());
|
|
1912
2024
|
}
|
|
1913
2025
|
const recordCommitDuration = performance.now() - recordCommitStarted;
|
|
1914
2026
|
if (detectedRecords.length === 0) {
|
|
@@ -1962,14 +2074,6 @@ export class UserGroundHost {
|
|
|
1962
2074
|
if (first.resourceId !== resourceId) {
|
|
1963
2075
|
throw new Error("local snapshot publication belongs to a different user-ground resource");
|
|
1964
2076
|
}
|
|
1965
|
-
this.cloudState = {
|
|
1966
|
-
resourceId: first.resourceId,
|
|
1967
|
-
authorityEpoch: first.authorityEpoch,
|
|
1968
|
-
headVersion: first.baseVersion,
|
|
1969
|
-
checksum: "",
|
|
1970
|
-
deliveryWatermark: first.deliveryWatermark,
|
|
1971
|
-
records: loadSnapshotPublication(first).snapshotRecords,
|
|
1972
|
-
};
|
|
1973
2077
|
try {
|
|
1974
2078
|
await this.publishPendingSnapshots(identity);
|
|
1975
2079
|
}
|
|
@@ -2001,6 +2105,12 @@ export class UserGroundHost {
|
|
|
2001
2105
|
throw error;
|
|
2002
2106
|
}
|
|
2003
2107
|
await this.recordRail?.flush();
|
|
2108
|
+
// A replayed publication leaves no graph behind — it only ever journaled
|
|
2109
|
+
// the roots it rebuilt — so the machine re-reads the authority's head
|
|
2110
|
+
// before deriving the next boundary on top of it.
|
|
2111
|
+
if (!this.cloudState) {
|
|
2112
|
+
await this.cloudPort(identity).lookup(privateEntityResourceId(identity.userId, sha256Hex));
|
|
2113
|
+
}
|
|
2004
2114
|
const state = this.cloudState;
|
|
2005
2115
|
if (!state)
|
|
2006
2116
|
throw new Error("cloud state is unavailable for graph publication");
|
|
@@ -2038,24 +2148,24 @@ export class UserGroundHost {
|
|
|
2038
2148
|
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2039
2149
|
});
|
|
2040
2150
|
}
|
|
2041
|
-
const officialRecords = timed("officialRows", () => recordsAtDeliveryPrefix(state.records, readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark)));
|
|
2042
2151
|
const localRecords = timed("groundRows", () => recordsRetainingMissingEvidence(readGroundRowsForRoots(this.databasePath(identity), "entities", publicationRootIds)
|
|
2043
2152
|
.map(portableRecord), readGroundRowsForRoots(this.databasePath(identity), "detection_notebook", publicationRootIds)
|
|
2044
2153
|
.map(portableRecord)));
|
|
2045
|
-
const
|
|
2046
|
-
const
|
|
2154
|
+
const accepted = timed("officialRows", () => readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark));
|
|
2155
|
+
const rebuilt = timed("merge", () => publicationRoots(state.graph, localRecords, accepted));
|
|
2156
|
+
const projected = timed("project", () => new Map([...rebuilt].map(([rootId, records]) => [rootId, records.length === 0 ? records : travelingRecords(records, sha256Hex)])));
|
|
2157
|
+
const traveling = [...projected.values()].flat();
|
|
2047
2158
|
const impossibleGitLeaf = traveling.find((record) => (record.type === "file.text" || record.type === "file.binary" || record.type === "link")
|
|
2048
2159
|
&& record.payloadVersion?.startsWith("git:"));
|
|
2049
2160
|
if (impossibleGitLeaf) {
|
|
2050
|
-
const parent =
|
|
2161
|
+
const parent = traveling.find((record) => record.uuid === impossibleGitLeaf.parentUUID);
|
|
2051
2162
|
throw new Error(`repository-owned leaf escaped its rail: ${impossibleGitLeaf.name} parent=${parent?.type ?? "missing"}`);
|
|
2052
2163
|
}
|
|
2053
|
-
|
|
2054
|
-
//
|
|
2055
|
-
//
|
|
2056
|
-
|
|
2057
|
-
const
|
|
2058
|
-
const checksum = timed("checksum", () => sha256Hex(snapshotBlob));
|
|
2164
|
+
// ONE canonical serialization per rebuilt root: the page bytes are the
|
|
2165
|
+
// journal's payload, the upload's payload, and the head's identities.
|
|
2166
|
+
// Untouched roots are not serialized, hashed, or journaled at all.
|
|
2167
|
+
const publication = timed("pages", () => state.graph.rebuild(projected, sha256Hex));
|
|
2168
|
+
const checksum = publication.checksum;
|
|
2059
2169
|
const localIds = new Set(localRecords.map((record) => record.uuid));
|
|
2060
2170
|
// Graph membership is never proof of sealed cargo: a record in the cloud
|
|
2061
2171
|
// graph may name content sealed only under a prior content contract,
|
|
@@ -2064,11 +2174,11 @@ export class UserGroundHost {
|
|
|
2064
2174
|
// sealed-first upload asks that exact question itself and moves only
|
|
2065
2175
|
// what is unsealed, so Register always re-establishes cargo closure
|
|
2066
2176
|
// before returning, even when the graph itself is unchanged.
|
|
2067
|
-
const cargoRecords =
|
|
2177
|
+
const cargoRecords = traveling.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
|
|
2068
2178
|
if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
|
|
2069
2179
|
this.finishPreparation(trace, preparationStarted, leg, {
|
|
2070
|
-
|
|
2071
|
-
|
|
2180
|
+
rebuiltRoots: rebuilt.size,
|
|
2181
|
+
records: traveling.length,
|
|
2072
2182
|
cargoRecords: cargoRecords.length,
|
|
2073
2183
|
graphUnchanged: true,
|
|
2074
2184
|
});
|
|
@@ -2085,19 +2195,12 @@ export class UserGroundHost {
|
|
|
2085
2195
|
}
|
|
2086
2196
|
return;
|
|
2087
2197
|
}
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
return rootReplacementFromRecords(rootIds, snapshot.records.filter((record) => {
|
|
2095
|
-
const root = rootOf(record);
|
|
2096
|
-
return root !== null && roots.has(root);
|
|
2097
|
-
}));
|
|
2098
|
-
});
|
|
2099
|
-
// The operation's canonical bytes: staged, hashed, and submitted as-is.
|
|
2100
|
-
const operationBlob = timed("operation", () => Buffer.from(stableJson(replacement), "utf8"));
|
|
2198
|
+
// The operation's canonical bytes: journaled and submitted as-is.
|
|
2199
|
+
const operationBlob = timed("operation", () => Buffer.from(stableJson(publication.operation), "utf8"));
|
|
2200
|
+
// Only pages the committed head does not already name still have to be
|
|
2201
|
+
// sealed; a rebuilt root that came out byte-identical costs nothing.
|
|
2202
|
+
const committed = graphPageIdentities(state.graph.head);
|
|
2203
|
+
const fresh = publication.pages.filter((page) => !committed.has(page.ref.sha256));
|
|
2101
2204
|
const entry = {
|
|
2102
2205
|
mutationId: randomUUID(),
|
|
2103
2206
|
resourceId: state.resourceId,
|
|
@@ -2105,30 +2208,35 @@ export class UserGroundHost {
|
|
|
2105
2208
|
baseVersion: state.headVersion,
|
|
2106
2209
|
snapshotChecksum: checksum,
|
|
2107
2210
|
deliveryWatermark,
|
|
2108
|
-
operationKind:
|
|
2109
|
-
replacementRootIds:
|
|
2211
|
+
operationKind: publication.operation.kind,
|
|
2212
|
+
replacementRootIds: publication.operation.replacedRootIds,
|
|
2110
2213
|
uploadUuids: cargoRecords.map((record) => record.uuid),
|
|
2111
|
-
snapshotBlob,
|
|
2112
2214
|
operationBlob,
|
|
2215
|
+
pages: fresh.map((page) => ({
|
|
2216
|
+
sha256: page.ref.sha256,
|
|
2217
|
+
bytes: Buffer.from(page.bytes.buffer, page.bytes.byteOffset, page.bytes.byteLength),
|
|
2218
|
+
})),
|
|
2113
2219
|
status: "pending",
|
|
2114
2220
|
submitted: false,
|
|
2115
2221
|
};
|
|
2116
2222
|
timed("journal", () => supersedeSnapshotJournal(this.databasePath(identity), entry));
|
|
2117
2223
|
this.finishPreparation(trace, preparationStarted, leg, {
|
|
2118
|
-
|
|
2119
|
-
|
|
2224
|
+
rebuiltRoots: rebuilt.size,
|
|
2225
|
+
records: traveling.length,
|
|
2226
|
+
pages: publication.pages.length,
|
|
2227
|
+
freshPages: fresh.length,
|
|
2228
|
+
pageBytes: fresh.reduce((total, page) => total + page.ref.bytes, 0),
|
|
2120
2229
|
operationBytes: operationBlob.byteLength,
|
|
2121
|
-
operationRoots: replacement.rootIds.length,
|
|
2122
2230
|
cargoRecords: cargoRecords.length,
|
|
2123
2231
|
graphUnchanged: false,
|
|
2124
2232
|
});
|
|
2125
|
-
// Publish the freshly journaled intent directly: the
|
|
2126
|
-
//
|
|
2127
|
-
// re-
|
|
2233
|
+
// Publish the freshly journaled intent directly: the pages and the
|
|
2234
|
+
// operation bytes already exist — nothing is re-read, re-parsed, or
|
|
2235
|
+
// re-serialized between prepare and the wire.
|
|
2128
2236
|
await this.publishLoadedSnapshot(identity, {
|
|
2129
2237
|
...entry,
|
|
2130
|
-
snapshotRecords: snapshot.records,
|
|
2131
2238
|
uploadRecords: cargoRecords,
|
|
2239
|
+
graph: publication.graph,
|
|
2132
2240
|
}, trace, manifestHints);
|
|
2133
2241
|
}
|
|
2134
2242
|
/** Close the graph-preparation stage with every leg it spent, so a slow
|
|
@@ -2181,8 +2289,61 @@ export class UserGroundHost {
|
|
|
2181
2289
|
await this.publishLoadedSnapshot(identity, loadSnapshotPublication(pending), trace, manifestHints);
|
|
2182
2290
|
}
|
|
2183
2291
|
}
|
|
2292
|
+
/** Seal this publication's graph pages. Pages are immutable, content
|
|
2293
|
+
* addressed, and already in memory, so the authority's seal inventory alone
|
|
2294
|
+
* decides what still has to move and whole pages ride cargo waves. The head
|
|
2295
|
+
* may not commit until every page it newly names is sealed. */
|
|
2296
|
+
async uploadGraphPages(resourceId, pages, trace) {
|
|
2297
|
+
if (pages.length === 0)
|
|
2298
|
+
return;
|
|
2299
|
+
const started = performance.now();
|
|
2300
|
+
const sealed = await this.sealedContentHashes(resourceId, pages.map((page) => page.sha256));
|
|
2301
|
+
const unsealed = pages
|
|
2302
|
+
.filter((page) => !sealed.has(page.sha256))
|
|
2303
|
+
.map((page) => ({ page, bytes: page.bytes.byteLength }));
|
|
2304
|
+
let uploadedBytes = 0;
|
|
2305
|
+
await boundedForEach([...planCargoWaves(unsealed)], CARGO_UPLOAD_CONCURRENCY, async (wave) => {
|
|
2306
|
+
const payload = Buffer.concat(wave.items.map((item) => item.page.bytes));
|
|
2307
|
+
const encoded = await encodeContentWireBytes(payload);
|
|
2308
|
+
const frame = await retryContent(() => this.wire.requestBinary({
|
|
2309
|
+
type: "private.entity-content.put-cargo",
|
|
2310
|
+
resource_id: resourceId,
|
|
2311
|
+
// A page is a single-chunk artifact: its identity, its only chunk,
|
|
2312
|
+
// and its bytes are one and the same sha.
|
|
2313
|
+
artifacts: wave.items.map((item) => ({
|
|
2314
|
+
content_hash: item.page.sha256,
|
|
2315
|
+
bytes: item.bytes,
|
|
2316
|
+
chunks: [{ sha256: item.page.sha256, bytes: item.bytes }],
|
|
2317
|
+
})),
|
|
2318
|
+
bytes: payload.byteLength,
|
|
2319
|
+
wire_bytes: encoded.bytes.byteLength,
|
|
2320
|
+
...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
|
|
2321
|
+
}, encoded.bytes, ["private.entity-content.stored-cargo"]));
|
|
2322
|
+
const stored = new Set(Array.isArray(frame.sealed) ? frame.sealed.map(String) : []);
|
|
2323
|
+
for (const item of wave.items) {
|
|
2324
|
+
if (!stored.has(item.page.sha256)) {
|
|
2325
|
+
throw new Error(`content authority did not seal graph page ${item.page.sha256}`);
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
uploadedBytes += encoded.bytes.byteLength;
|
|
2329
|
+
});
|
|
2330
|
+
this.stage({
|
|
2331
|
+
primitive: "upload",
|
|
2332
|
+
stage: "graph-pages",
|
|
2333
|
+
status: "completed",
|
|
2334
|
+
...(trace?.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2335
|
+
durationMs: performance.now() - started,
|
|
2336
|
+
measurements: {
|
|
2337
|
+
journey: trace?.journey ?? "bootstrap",
|
|
2338
|
+
pages: pages.length,
|
|
2339
|
+
uploadedPages: unsealed.length,
|
|
2340
|
+
uploadedBytes,
|
|
2341
|
+
},
|
|
2342
|
+
});
|
|
2343
|
+
}
|
|
2184
2344
|
async publishLoadedSnapshot(identity, publication, trace, manifestHints) {
|
|
2185
2345
|
try {
|
|
2346
|
+
await this.uploadGraphPages(publication.resourceId, publication.pages, trace);
|
|
2186
2347
|
await this.uploadSnapshotContent(identity, publication.resourceId, publication.uploadRecords, trace, manifestHints);
|
|
2187
2348
|
}
|
|
2188
2349
|
catch (error) {
|
|
@@ -2218,17 +2379,25 @@ export class UserGroundHost {
|
|
|
2218
2379
|
local("markSubmitted", () => markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId));
|
|
2219
2380
|
}
|
|
2220
2381
|
try {
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2382
|
+
// A head commit names pages instead of carrying records, so it is small
|
|
2383
|
+
// by construction and always travels inline in one frame.
|
|
2384
|
+
const submitStarted = performance.now();
|
|
2385
|
+
frame = await this.wire.request({
|
|
2386
|
+
type: "shared.mutation.submit",
|
|
2387
|
+
envelope: {
|
|
2388
|
+
resourceId: publication.resourceId,
|
|
2389
|
+
authorityEpoch: publication.authorityEpoch,
|
|
2390
|
+
mutationId: publication.mutationId,
|
|
2391
|
+
deviceId: identity.deviceId,
|
|
2392
|
+
baseVersion: publication.baseVersion,
|
|
2393
|
+
contract: ENTITY_CLOUD_CONTRACT,
|
|
2394
|
+
schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
|
|
2395
|
+
operationKind: publication.operationKind,
|
|
2396
|
+
deliveryWatermark: publication.deliveryWatermark,
|
|
2397
|
+
operation: JSON.parse(publication.operationBlob.toString("utf8")),
|
|
2398
|
+
},
|
|
2399
|
+
}, ["shared.mutation.ack"]);
|
|
2400
|
+
legMs.submitMs = performance.now() - submitStarted;
|
|
2232
2401
|
}
|
|
2233
2402
|
catch (error) {
|
|
2234
2403
|
if (trace?.journey === "register") {
|
|
@@ -2249,13 +2418,16 @@ export class UserGroundHost {
|
|
|
2249
2418
|
}
|
|
2250
2419
|
throw pipelineStageError("cloud graph commit", error);
|
|
2251
2420
|
}
|
|
2252
|
-
|
|
2421
|
+
// A replay carries no graph: clearing the state makes the next reader
|
|
2422
|
+
// fetch the authority's own head instead of a graph missing every root
|
|
2423
|
+
// this publication did not rebuild.
|
|
2424
|
+
this.cloudState = publication.graph === null ? null : {
|
|
2253
2425
|
resourceId: publication.resourceId,
|
|
2254
2426
|
authorityEpoch: Number(frame.authority_epoch),
|
|
2255
2427
|
headVersion: Number(frame.version),
|
|
2256
2428
|
checksum: publication.snapshotChecksum,
|
|
2257
2429
|
deliveryWatermark: Number(frame.delivery_watermark),
|
|
2258
|
-
|
|
2430
|
+
graph: publication.graph,
|
|
2259
2431
|
};
|
|
2260
2432
|
local("clearJournal", () => deleteJournalEntry(this.databasePath(identity), publication.mutationId));
|
|
2261
2433
|
if (trace?.journey === "register") {
|
|
@@ -2263,7 +2435,7 @@ export class UserGroundHost {
|
|
|
2263
2435
|
// its legs here gives one ledger line the whole story: local journal
|
|
2264
2436
|
// marks, wire time, gateway assemble/validate, and the database RPC.
|
|
2265
2437
|
for (const [wire, leg] of [
|
|
2266
|
-
["
|
|
2438
|
+
["verify_ms", "serverVerifyMs"],
|
|
2267
2439
|
["validate_ms", "serverValidateMs"],
|
|
2268
2440
|
["database_ms", "serverDatabaseMs"],
|
|
2269
2441
|
]) {
|
|
@@ -2302,6 +2474,12 @@ function contentCacheFile(cacheDir, contentHash) {
|
|
|
2302
2474
|
function manifestCacheFile(cacheDir, contentHash) {
|
|
2303
2475
|
return join(cacheDir, "manifests", `${contentHash}.json`);
|
|
2304
2476
|
}
|
|
2477
|
+
/** Graph pages are immutable and content-addressed, so they cache exactly like
|
|
2478
|
+
* artifacts do — in their own directory, because they are the registry's own
|
|
2479
|
+
* bytes rather than any entity's cargo. */
|
|
2480
|
+
function graphPageCacheDir(cacheDir) {
|
|
2481
|
+
return join(cacheDir, "graph-pages");
|
|
2482
|
+
}
|
|
2305
2483
|
function readCachedManifest(cacheDir, contentHash) {
|
|
2306
2484
|
try {
|
|
2307
2485
|
const checked = checkContentManifest(JSON.parse(readFileSync(manifestCacheFile(cacheDir, contentHash), "utf8")), contentHash);
|
|
@@ -2333,6 +2511,15 @@ function pipelineStageError(stage, error) {
|
|
|
2333
2511
|
cause: error,
|
|
2334
2512
|
});
|
|
2335
2513
|
}
|
|
2514
|
+
class DetectionBasisAdvancedError extends Error {
|
|
2515
|
+
constructor() {
|
|
2516
|
+
super("the verified notebook advanced while Detect was observing; retrying fresh truth");
|
|
2517
|
+
}
|
|
2518
|
+
}
|
|
2519
|
+
function detectionBasisAdvanced(error) {
|
|
2520
|
+
return error instanceof DetectionBasisAdvancedError
|
|
2521
|
+
|| (error instanceof Error && error.cause instanceof DetectionBasisAdvancedError);
|
|
2522
|
+
}
|
|
2336
2523
|
function filesPipelineStage(error) {
|
|
2337
2524
|
const stage = error instanceof Error
|
|
2338
2525
|
? error.filesStage
|
|
@@ -2375,118 +2562,110 @@ function assertHealthyWatch(evidence, rootId) {
|
|
|
2375
2562
|
* file over a dozen times; the file itself is never deleted by the host, so
|
|
2376
2563
|
* every later open needs only the connection and its pragmas. */
|
|
2377
2564
|
const migratedDatabases = new Set();
|
|
2378
|
-
/**
|
|
2379
|
-
*
|
|
2380
|
-
*
|
|
2381
|
-
*
|
|
2382
|
-
*
|
|
2383
|
-
* cloud-commit stage on the 1-GB machine). The blob row is written exactly
|
|
2384
|
-
* once and read only on restart replay; every status transition touches only
|
|
2385
|
-
* the small row. */
|
|
2565
|
+
/** A publication row holds only what the authority must be told: the small
|
|
2566
|
+
* meta facts and the head-commit operation naming this graph's pages. The
|
|
2567
|
+
* pages themselves are write-once rows in their own table, because a status
|
|
2568
|
+
* transition rewrites its whole row — overflow pages included — through the
|
|
2569
|
+
* WAL at synchronous=FULL, and page bytes must never ride that rewrite. */
|
|
2386
2570
|
const SNAPSHOT_PUBLICATIONS_SCHEMA = `
|
|
2387
2571
|
CREATE TABLE IF NOT EXISTS snapshot_publications (
|
|
2388
2572
|
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2389
2573
|
mutation_id TEXT NOT NULL UNIQUE,
|
|
2390
2574
|
resource_id TEXT NOT NULL,
|
|
2391
2575
|
meta_json TEXT NOT NULL,
|
|
2576
|
+
operation_blob BLOB NOT NULL,
|
|
2392
2577
|
created_at INTEGER NOT NULL,
|
|
2393
2578
|
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
|
|
2394
2579
|
submitted_at INTEGER
|
|
2395
2580
|
);
|
|
2396
|
-
CREATE TABLE IF NOT EXISTS
|
|
2397
|
-
mutation_id TEXT
|
|
2398
|
-
|
|
2399
|
-
|
|
2581
|
+
CREATE TABLE IF NOT EXISTS snapshot_publication_pages (
|
|
2582
|
+
mutation_id TEXT NOT NULL,
|
|
2583
|
+
page_index INTEGER NOT NULL,
|
|
2584
|
+
sha256 TEXT NOT NULL,
|
|
2585
|
+
bytes BLOB NOT NULL,
|
|
2586
|
+
PRIMARY KEY (mutation_id, page_index)
|
|
2400
2587
|
);`;
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2588
|
+
function legacyJournalShape(columns) {
|
|
2589
|
+
if (columns.size === 0)
|
|
2590
|
+
return null;
|
|
2591
|
+
if (columns.has("publication_json"))
|
|
2592
|
+
return "nested-text";
|
|
2593
|
+
if (columns.has("snapshot_blob"))
|
|
2594
|
+
return "row-blobs";
|
|
2595
|
+
// The paged journal and the side-table one both keep only small columns
|
|
2596
|
+
// here; only the paged one names the operation on the row itself.
|
|
2597
|
+
return columns.has("operation_blob") ? null : "split-blobs";
|
|
2598
|
+
}
|
|
2599
|
+
/** Converts a pre-paged publication journal into the paged one, atomically,
|
|
2600
|
+
* preserving every row's meaning. A pending row may be a submitted mutation
|
|
2601
|
+
* whose outcome exists only at the authority — dropping it would violate the
|
|
2602
|
+
* retry law that recovers a lost acknowledgement — so conversion is
|
|
2603
|
+
* structural: each row keeps its identity, its meta facts, and the exact
|
|
2604
|
+
* operation bytes it would submit, and semantic judgment stays where it
|
|
2605
|
+
* always was, with the reader (an unlawful row still blocks registration
|
|
2606
|
+
* rather than being silently replaced). A converted row rebuilt whole roots
|
|
2607
|
+
* rather than pages, so it carries no page rows; its replayed upload comes
|
|
2608
|
+
* from the records inside its own operation.
|
|
2410
2609
|
*
|
|
2411
2610
|
* The transform runs entirely inside SQLite. A production journal row is a
|
|
2412
2611
|
* register-scale graph serialized three times over — the exact text whose
|
|
2413
2612
|
* repeated JavaScript materialization blew the 512 MB heap — so no part of
|
|
2414
|
-
* it may ever become a JavaScript value here: json_extract lifts
|
|
2415
|
-
*
|
|
2416
|
-
*
|
|
2613
|
+
* it may ever become a JavaScript value here: json_extract lifts nested text
|
|
2614
|
+
* straight into its blob and meta rows are json_object over subtrees, all in
|
|
2615
|
+
* engine memory outside the V8 heap. The snapshot blob is simply not carried:
|
|
2616
|
+
* the authority stores the head now, and a replay re-reads it.
|
|
2417
2617
|
*
|
|
2418
2618
|
* Tables older than the three-state journal carry no status or submitted_at;
|
|
2419
2619
|
* their rows were written before the pre-submit durability mark existed, so
|
|
2420
2620
|
* the only safe reading is "possibly submitted": keep them pending and
|
|
2421
2621
|
* marked submitted, which replays the same mutation id — always idempotent —
|
|
2422
|
-
* and never supersedes. A row whose
|
|
2423
|
-
* the open loudly with the journal intact; nothing
|
|
2424
|
-
* conversion failure. */
|
|
2425
|
-
function convertLegacySnapshotJournal(database, columns) {
|
|
2622
|
+
* and never supersedes. A row whose stored operation does not resolve fails
|
|
2623
|
+
* the open loudly against the NOT NULL law with the journal intact; nothing
|
|
2624
|
+
* is ever deleted on a conversion failure. */
|
|
2625
|
+
function convertLegacySnapshotJournal(database, shape, columns) {
|
|
2626
|
+
const metaJson = shape === "nested-text"
|
|
2627
|
+
? `json_object(
|
|
2628
|
+
'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
|
|
2629
|
+
'baseVersion', json_extract(publication_json, '$.baseVersion'),
|
|
2630
|
+
'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
|
|
2631
|
+
'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
|
|
2632
|
+
'operationKind', 'entity.roots.replace',
|
|
2633
|
+
'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2634
|
+
'uploadUuids', coalesce((
|
|
2635
|
+
SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
|
|
2636
|
+
FROM json_each(publication_json, '$.uploadRecords') AS uploaded
|
|
2637
|
+
), json_array())
|
|
2638
|
+
)`
|
|
2639
|
+
: "meta_json";
|
|
2640
|
+
const operationBlob = {
|
|
2641
|
+
"nested-text": `CAST(json_object(
|
|
2642
|
+
'kind', 'entity.roots.replace',
|
|
2643
|
+
'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2644
|
+
'records', json(json_extract(publication_json, '$.replacementRecords'))
|
|
2645
|
+
) AS BLOB)`,
|
|
2646
|
+
"row-blobs": "operation_blob",
|
|
2647
|
+
"split-blobs": `(SELECT blobs.operation_blob FROM snapshot_publication_blobs AS blobs
|
|
2648
|
+
WHERE blobs.mutation_id = legacy.mutation_id)`,
|
|
2649
|
+
}[shape];
|
|
2426
2650
|
database.transaction(() => {
|
|
2427
2651
|
database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
|
|
2428
2652
|
database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
|
|
2429
2653
|
database.exec(`
|
|
2430
2654
|
INSERT INTO snapshot_publications(
|
|
2431
|
-
mutation_id, resource_id, meta_json, created_at, status, submitted_at
|
|
2655
|
+
mutation_id, resource_id, meta_json, operation_blob, created_at, status, submitted_at
|
|
2432
2656
|
)
|
|
2433
2657
|
SELECT
|
|
2434
2658
|
mutation_id,
|
|
2435
2659
|
resource_id,
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
'baseVersion', json_extract(publication_json, '$.baseVersion'),
|
|
2439
|
-
'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
|
|
2440
|
-
'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
|
|
2441
|
-
'operationKind', 'entity.roots.replace',
|
|
2442
|
-
'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2443
|
-
'uploadUuids', coalesce((
|
|
2444
|
-
SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
|
|
2445
|
-
FROM json_each(publication_json, '$.uploadRecords') AS uploaded
|
|
2446
|
-
), json_array())
|
|
2447
|
-
),
|
|
2660
|
+
${metaJson},
|
|
2661
|
+
${operationBlob},
|
|
2448
2662
|
created_at,
|
|
2449
2663
|
${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
|
|
2450
2664
|
${columns.has("submitted_at") ? "submitted_at" : "created_at"}
|
|
2451
|
-
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2452
|
-
`);
|
|
2453
|
-
database.exec(`
|
|
2454
|
-
INSERT INTO snapshot_publication_blobs(mutation_id, snapshot_blob, operation_blob)
|
|
2455
|
-
SELECT
|
|
2456
|
-
mutation_id,
|
|
2457
|
-
CAST(json_extract(publication_json, '$.snapshotJson') AS BLOB),
|
|
2458
|
-
CAST(json_object(
|
|
2459
|
-
'kind', 'entity.roots.replace',
|
|
2460
|
-
'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2461
|
-
'records', json(json_extract(publication_json, '$.replacementRecords'))
|
|
2462
|
-
) AS BLOB)
|
|
2463
|
-
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2464
|
-
`);
|
|
2465
|
-
database.exec("DROP TABLE snapshot_publications_legacy");
|
|
2466
|
-
})();
|
|
2467
|
-
}
|
|
2468
|
-
/** Converts the single-table blob journal (blobs stored on the publication
|
|
2469
|
-
* row itself) into the split journal, atomically and entirely inside SQLite —
|
|
2470
|
-
* the same engine-memory law as the nested-text conversion above. Every
|
|
2471
|
-
* field already has its final meaning; the blobs simply move to their
|
|
2472
|
-
* write-once table so status transitions stop rewriting them. */
|
|
2473
|
-
function splitSnapshotJournalBlobs(database) {
|
|
2474
|
-
database.transaction(() => {
|
|
2475
|
-
database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
|
|
2476
|
-
database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
|
|
2477
|
-
database.exec(`
|
|
2478
|
-
INSERT INTO snapshot_publications(
|
|
2479
|
-
mutation_id, resource_id, meta_json, created_at, status, submitted_at
|
|
2480
|
-
)
|
|
2481
|
-
SELECT mutation_id, resource_id, meta_json, created_at, status, submitted_at
|
|
2482
|
-
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2483
|
-
`);
|
|
2484
|
-
database.exec(`
|
|
2485
|
-
INSERT INTO snapshot_publication_blobs(mutation_id, snapshot_blob, operation_blob)
|
|
2486
|
-
SELECT mutation_id, snapshot_blob, operation_blob
|
|
2487
|
-
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2665
|
+
FROM snapshot_publications_legacy AS legacy ORDER BY sequence
|
|
2488
2666
|
`);
|
|
2489
2667
|
database.exec("DROP TABLE snapshot_publications_legacy");
|
|
2668
|
+
database.exec("DROP TABLE IF EXISTS snapshot_publication_blobs");
|
|
2490
2669
|
})();
|
|
2491
2670
|
}
|
|
2492
2671
|
/** The single opener and schema-migration entry for a user ground database.
|
|
@@ -2505,12 +2684,9 @@ export function initializeDatabase(file) {
|
|
|
2505
2684
|
// how this file arrived at this process. One PRAGMA against the in-memory
|
|
2506
2685
|
// catalog costs nothing next to the connection open itself.
|
|
2507
2686
|
const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
else if (publicationColumns.has("snapshot_blob")) {
|
|
2512
|
-
splitSnapshotJournalBlobs(database);
|
|
2513
|
-
}
|
|
2687
|
+
const legacyShape = legacyJournalShape(publicationColumns);
|
|
2688
|
+
if (legacyShape)
|
|
2689
|
+
convertLegacySnapshotJournal(database, legacyShape, publicationColumns);
|
|
2514
2690
|
if (migratedDatabases.has(file))
|
|
2515
2691
|
return database;
|
|
2516
2692
|
database.exec(`
|
|
@@ -3086,29 +3262,36 @@ function readSnapshotPublications(file) {
|
|
|
3086
3262
|
return [];
|
|
3087
3263
|
const database = initializeDatabase(file);
|
|
3088
3264
|
try {
|
|
3265
|
+
const pages = database.prepare(`
|
|
3266
|
+
SELECT mutation_id AS mutationId, sha256, bytes
|
|
3267
|
+
FROM snapshot_publication_pages ORDER BY mutation_id, page_index
|
|
3268
|
+
`).all();
|
|
3269
|
+
const pagesByMutation = new Map();
|
|
3270
|
+
for (const page of pages) {
|
|
3271
|
+
const carried = pagesByMutation.get(page.mutationId) ?? [];
|
|
3272
|
+
carried.push({ sha256: page.sha256, bytes: page.bytes });
|
|
3273
|
+
pagesByMutation.set(page.mutationId, carried);
|
|
3274
|
+
}
|
|
3089
3275
|
return database.prepare(`
|
|
3090
|
-
SELECT
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
FROM snapshot_publications AS publications
|
|
3098
|
-
LEFT JOIN snapshot_publication_blobs AS blobs USING (mutation_id)
|
|
3099
|
-
ORDER BY publications.sequence
|
|
3276
|
+
SELECT mutation_id AS mutationId,
|
|
3277
|
+
resource_id AS resourceId,
|
|
3278
|
+
meta_json AS metaJson,
|
|
3279
|
+
operation_blob AS operationBlob,
|
|
3280
|
+
status,
|
|
3281
|
+
submitted_at AS submittedAt
|
|
3282
|
+
FROM snapshot_publications ORDER BY sequence
|
|
3100
3283
|
`).all().map((row) => {
|
|
3101
|
-
// A publication without its payload is damage, never a skippable row —
|
|
3102
|
-
// silently dropping it would let a new publication supersede a
|
|
3103
|
-
// submitted mutation whose outcome exists only at the authority.
|
|
3104
|
-
if (!row.snapshotBlob || !row.operationBlob) {
|
|
3105
|
-
throw new Error(`snapshot publication ${row.mutationId} has no journaled payload`);
|
|
3106
|
-
}
|
|
3107
3284
|
const meta = JSON.parse(row.metaJson);
|
|
3108
|
-
// No lawful publication carries zero roots —
|
|
3109
|
-
//
|
|
3285
|
+
// No lawful publication carries zero roots — a head commit that replaces
|
|
3286
|
+
// nothing would prove nothing — so a rootless row here is damage, never
|
|
3287
|
+
// a skippable row: silently dropping it would let a new publication
|
|
3288
|
+
// supersede a submitted mutation whose outcome exists only at the
|
|
3289
|
+
// authority. `entity.roots.replace` appears only on rows converted from
|
|
3290
|
+
// a pre-paged journal, which no longer produce but must still replay.
|
|
3110
3291
|
if (!Array.isArray(meta.replacementRootIds) || meta.replacementRootIds.length === 0
|
|
3111
|
-
|| !Array.isArray(meta.uploadUuids)
|
|
3292
|
+
|| !Array.isArray(meta.uploadUuids)
|
|
3293
|
+
|| (meta.operationKind !== GRAPH_HEAD_OPERATION_KIND
|
|
3294
|
+
&& meta.operationKind !== "entity.roots.replace")) {
|
|
3112
3295
|
throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
|
|
3113
3296
|
}
|
|
3114
3297
|
const deliveryWatermark = Number(meta.deliveryWatermark);
|
|
@@ -3122,11 +3305,11 @@ function readSnapshotPublications(file) {
|
|
|
3122
3305
|
baseVersion: Number(meta.baseVersion),
|
|
3123
3306
|
deliveryWatermark,
|
|
3124
3307
|
snapshotChecksum: String(meta.snapshotChecksum),
|
|
3125
|
-
operationKind: meta.operationKind,
|
|
3308
|
+
operationKind: String(meta.operationKind),
|
|
3126
3309
|
replacementRootIds: meta.replacementRootIds.map(String),
|
|
3127
3310
|
uploadUuids: meta.uploadUuids.map(String),
|
|
3128
|
-
snapshotBlob: row.snapshotBlob,
|
|
3129
3311
|
operationBlob: row.operationBlob,
|
|
3312
|
+
pages: pagesByMutation.get(row.mutationId) ?? [],
|
|
3130
3313
|
status: row.status === "rejected" ? "rejected" : "pending",
|
|
3131
3314
|
submitted: row.submittedAt !== null,
|
|
3132
3315
|
};
|
|
@@ -3294,7 +3477,7 @@ function supersedeSnapshotJournal(file, entry) {
|
|
|
3294
3477
|
throw new Error("a submitted publication with an unknown outcome must resolve before supersession");
|
|
3295
3478
|
}
|
|
3296
3479
|
database.prepare(`
|
|
3297
|
-
DELETE FROM
|
|
3480
|
+
DELETE FROM snapshot_publication_pages WHERE mutation_id IN (
|
|
3298
3481
|
SELECT mutation_id FROM snapshot_publications WHERE resource_id = ?
|
|
3299
3482
|
)
|
|
3300
3483
|
`).run(entry.resourceId);
|
|
@@ -3302,8 +3485,8 @@ function supersedeSnapshotJournal(file, entry) {
|
|
|
3302
3485
|
.run(entry.resourceId);
|
|
3303
3486
|
database.prepare(`
|
|
3304
3487
|
INSERT INTO snapshot_publications(
|
|
3305
|
-
mutation_id, resource_id, meta_json, created_at
|
|
3306
|
-
) VALUES (?, ?, ?, ?)
|
|
3488
|
+
mutation_id, resource_id, meta_json, operation_blob, created_at
|
|
3489
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
3307
3490
|
`).run(entry.mutationId, entry.resourceId, stableJson({
|
|
3308
3491
|
authorityEpoch: entry.authorityEpoch,
|
|
3309
3492
|
baseVersion: entry.baseVersion,
|
|
@@ -3312,11 +3495,12 @@ function supersedeSnapshotJournal(file, entry) {
|
|
|
3312
3495
|
operationKind: entry.operationKind,
|
|
3313
3496
|
replacementRootIds: entry.replacementRootIds,
|
|
3314
3497
|
uploadUuids: entry.uploadUuids,
|
|
3315
|
-
}), Date.now());
|
|
3316
|
-
database.prepare(`
|
|
3317
|
-
INSERT INTO
|
|
3318
|
-
VALUES (?, ?, ?)
|
|
3319
|
-
`)
|
|
3498
|
+
}), entry.operationBlob, Date.now());
|
|
3499
|
+
const insertPage = database.prepare(`
|
|
3500
|
+
INSERT INTO snapshot_publication_pages(mutation_id, page_index, sha256, bytes)
|
|
3501
|
+
VALUES (?, ?, ?, ?)
|
|
3502
|
+
`);
|
|
3503
|
+
entry.pages.forEach((page, index) => insertPage.run(entry.mutationId, index, page.sha256, page.bytes));
|
|
3320
3504
|
})();
|
|
3321
3505
|
}
|
|
3322
3506
|
finally {
|
|
@@ -3387,7 +3571,7 @@ function deleteJournalEntry(file, mutationId) {
|
|
|
3387
3571
|
const database = initializeDatabase(file);
|
|
3388
3572
|
try {
|
|
3389
3573
|
database.transaction(() => {
|
|
3390
|
-
database.prepare("DELETE FROM
|
|
3574
|
+
database.prepare("DELETE FROM snapshot_publication_pages WHERE mutation_id = ?").run(mutationId);
|
|
3391
3575
|
database.prepare("DELETE FROM snapshot_publications WHERE mutation_id = ?").run(mutationId);
|
|
3392
3576
|
})();
|
|
3393
3577
|
}
|
|
@@ -4981,19 +5165,79 @@ function recordRootResolver(records) {
|
|
|
4981
5165
|
};
|
|
4982
5166
|
return rootOf;
|
|
4983
5167
|
}
|
|
4984
|
-
|
|
4985
|
-
|
|
5168
|
+
/** The roots one publication rebuilds, each with its complete new records.
|
|
5169
|
+
*
|
|
5170
|
+
* A walked root is rebuilt from its ground alone: materialized ground is the
|
|
5171
|
+
* whole truth for the roots it covers, so nothing of the committed graph
|
|
5172
|
+
* survives underneath one. Any other root the newly delivered authority prefix
|
|
5173
|
+
* touched is rebuilt from the committed graph with those results folded in, so
|
|
5174
|
+
* the published graph never declares a watermark it has not actually applied.
|
|
5175
|
+
* Every root outside this map keeps its committed pages untouched. */
|
|
5176
|
+
function publicationRoots(graph, localRecords, accepted) {
|
|
5177
|
+
const rebuilt = new Map();
|
|
5178
|
+
const walked = new Map(localRecords
|
|
4986
5179
|
.filter((record) => record.parentUUID === null)
|
|
4987
|
-
.map((record) => record.uuid));
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
4993
|
-
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
5180
|
+
.map((record) => [record.uuid, []]));
|
|
5181
|
+
if (walked.size > 0) {
|
|
5182
|
+
const rootOf = recordRootResolver(localRecords);
|
|
5183
|
+
for (const record of localRecords) {
|
|
5184
|
+
const rootId = rootOf(record);
|
|
5185
|
+
const group = rootId === null ? undefined : walked.get(rootId);
|
|
5186
|
+
if (!group)
|
|
5187
|
+
throw new Error(`walked record ${record.uuid} descends from no walked root`);
|
|
5188
|
+
group.push(record);
|
|
5189
|
+
}
|
|
5190
|
+
for (const [rootId, records] of walked)
|
|
5191
|
+
rebuilt.set(rootId, records);
|
|
5192
|
+
}
|
|
5193
|
+
const overlays = new Map();
|
|
5194
|
+
for (const result of accepted) {
|
|
5195
|
+
const rootId = graph.rootOf(result.uuid);
|
|
5196
|
+
if (rootId === null || walked.has(rootId))
|
|
5197
|
+
continue;
|
|
5198
|
+
const overlay = overlays.get(rootId);
|
|
5199
|
+
if (overlay)
|
|
5200
|
+
overlay.push(result);
|
|
5201
|
+
else
|
|
5202
|
+
overlays.set(rootId, [result]);
|
|
5203
|
+
}
|
|
5204
|
+
for (const [rootId, results] of overlays) {
|
|
5205
|
+
const byUuid = new Map(graph.recordsFor(rootId).map((record) => [record.uuid, record]));
|
|
5206
|
+
for (const result of results)
|
|
5207
|
+
byUuid.set(result.uuid, result);
|
|
5208
|
+
// Entity channels are independently ordered, so container membership is
|
|
5209
|
+
// derived once from the final prefix rather than by arrival order.
|
|
5210
|
+
rebuilt.set(rootId, recordsRetainingMissingEvidence([...byUuid.values()], []));
|
|
5211
|
+
}
|
|
5212
|
+
// A records-shaped authority base carries no page identities, so it can
|
|
5213
|
+
// prove nothing about a root the commit leaves alone: the first head pages
|
|
5214
|
+
// the whole account at once, and never again.
|
|
5215
|
+
if (graph.head === null) {
|
|
5216
|
+
for (const rootId of graph.rootIds()) {
|
|
5217
|
+
if (!rebuilt.has(rootId))
|
|
5218
|
+
rebuilt.set(rootId, graph.recordsFor(rootId));
|
|
5219
|
+
}
|
|
5220
|
+
}
|
|
5221
|
+
// Re-rooting is a move, not a copy: an enclosing Register walks records the
|
|
5222
|
+
// committed graph still files under the root that used to hold them. Every
|
|
5223
|
+
// such root is rebuilt without them, and a root that keeps nothing is left
|
|
5224
|
+
// with no records at all — the rebuild then drops it from the graph.
|
|
5225
|
+
const vacated = new Map();
|
|
5226
|
+
for (const record of localRecords) {
|
|
5227
|
+
const rootId = graph.rootOf(record.uuid);
|
|
5228
|
+
if (rootId === null || walked.has(rootId))
|
|
5229
|
+
continue;
|
|
5230
|
+
const gone = vacated.get(rootId);
|
|
5231
|
+
if (gone)
|
|
5232
|
+
gone.add(record.uuid);
|
|
5233
|
+
else
|
|
5234
|
+
vacated.set(rootId, new Set([record.uuid]));
|
|
5235
|
+
}
|
|
5236
|
+
for (const [rootId, gone] of vacated) {
|
|
5237
|
+
const held = rebuilt.get(rootId) ?? graph.recordsFor(rootId);
|
|
5238
|
+
rebuilt.set(rootId, held.filter((record) => !gone.has(record.uuid)));
|
|
5239
|
+
}
|
|
5240
|
+
return rebuilt;
|
|
4997
5241
|
}
|
|
4998
5242
|
/** Filesystem disappearance is evidence, not cloud lifecycle authority. Keep
|
|
4999
5243
|
* the last logical records in the outgoing graph while their materialized
|
|
@@ -5566,14 +5810,14 @@ async function installCloudWorkspace(input) {
|
|
|
5566
5810
|
alreadyMaterialized: false,
|
|
5567
5811
|
};
|
|
5568
5812
|
}
|
|
5569
|
-
function readyFromFrame(frame) {
|
|
5570
|
-
const bytes = Buffer.from(String(frame.snapshot_b64 || ""), "base64");
|
|
5813
|
+
function readyFromFrame(frame, graph) {
|
|
5571
5814
|
const cloud = {
|
|
5572
5815
|
resourceId: String(frame.resource_id || ""),
|
|
5573
5816
|
version: Number(frame.head_version),
|
|
5574
5817
|
deliveryWatermark: Number(frame.delivery_watermark),
|
|
5575
5818
|
checksum: String(frame.snapshot_checksum || ""),
|
|
5576
|
-
snapshot:
|
|
5819
|
+
snapshot: graph.snapshot(),
|
|
5820
|
+
...(graph.head ? { head: graph.head } : {}),
|
|
5577
5821
|
};
|
|
5578
5822
|
return {
|
|
5579
5823
|
cloud,
|
|
@@ -5583,7 +5827,7 @@ function readyFromFrame(frame) {
|
|
|
5583
5827
|
headVersion: cloud.version,
|
|
5584
5828
|
checksum: cloud.checksum,
|
|
5585
5829
|
deliveryWatermark: cloud.deliveryWatermark,
|
|
5586
|
-
|
|
5830
|
+
graph,
|
|
5587
5831
|
},
|
|
5588
5832
|
};
|
|
5589
5833
|
}
|