@amalgm/shell 0.1.93 → 0.1.95

package/dist/user-ground-host.js

@@ -3,7 +3,7 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
 import { open } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
-import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, download as downloadArtifact, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
+import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, download as downloadArtifact, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
 import Database from "better-sqlite3";
 import { atomicCopy, atomicWrite, ensurePrivateDir, ensureUserDir } from "./filesystem.js";
 import { emitFilesStage, } from "./files-observability.js";
@@ -12,7 +12,8 @@ import { EntityRecordSqliteStore } from "./entity-record-store.js";
 import { observeEntityRecordPorts } from "./entity-record-observability.js";
 import { indexRowsByOutermostRoot } from "./ground-root-index.js";
 import { ContentCacheDownload, captureContentFile, hashContentFile, sealCapturedArtifact, } from "./content-cache-host.js";
-import { decodeContentWireBytes, encodeContentWireBytes } from "./content-wire-codec.js";
+import { DirectContentPlane } from "./content-direct.js";
+import { encodeContentWireBytes } from "./content-wire-codec.js";
 import { exactTextReplay, planGroundDetection, reconcileGroundUUIDs, } from "./detection/portable.js";
 import { AcceptedInboxResultIndex } from "./detection/accepted-result.js";
 import { sameNotebookBasisRow } from "./detection/notebook-basis.js";
@@ -26,8 +27,13 @@ import { GroundCoordinator } from "./ground-coordination.js";
 import { submitMutationOperation } from "./paged-mutation-submit.js";
 import { WireClient, WireRequestError } from "./wire-client.js";
 const SMALL_CONTENT_UPLOAD_CONCURRENCY = 16;
-const FILE_CONTENT_DOWNLOAD_CONCURRENCY = 4;
-const FILE_BATCH_DOWNLOAD_CONCURRENCY = 2;
+/** Artifact lanes for a download phase. Direct storage fetches cost the
+ * gateway nothing, so parallelism is bounded by this machine's pipe rather
+ * than authority fairness. */
+const FILE_CONTENT_DOWNLOAD_CONCURRENCY = 16;
+/** Block lanes within one artifact. Most files are a single block; large
+ * repository packs fan their chunks across this. */
+const DIRECT_BLOCK_DOWNLOAD_CONCURRENCY = 8;
 const DETECT_QUIET_MS = 12;
 const DETECT_MAX_DEFERRAL_MS = 75;
 const DETECT_RETRY_MS = 250;
@@ -47,6 +53,7 @@ const GROUND_ROW_COLUMNS = [
 export class UserGroundHost {
     options;
     wire;
+    direct;
     watchHost;
     activeIdentity = null;
     rescanTimer = null;
@@ -75,6 +82,9 @@ export class UserGroundHost {
             runtimeVersion: options.runtimeVersion,
             requestTimeoutMs: options.wireRequestTimeoutMs,
         });
+        this.direct = new DirectContentPlane({
+            request: (frame, accept) => this.wire.request(frame, accept),
+        });
         this.watchHost = new NodeWatchHost({
             open: options.openWatch,
             onSignal: () => this.scheduleRescan(),
@@ -461,7 +471,17 @@ export class UserGroundHost {
                 && record.payloadVersion === workspaceId);
         if (hasRegisteredBoundaryInDatabase(database, workspaceId)
             && hasRegisteredBoundary(this.cloudState.records)) {
-            this.finishColdOperation();
+            // Already declared — but Register may only return once cargo closure
+            // holds under the current content contract. The graph head can be
+            // unchanged while a contract cutover moved its cargo namespace, so
+            // the publish path re-proves seal evidence and repairs any missing
+            // uploads before this answer is lawful.
+            try {
+                await this.publishMaterializedSnapshot(identity, new Set([workspaceId]), { journey: "register", workspaceId });
+            }
+            finally {
+                this.finishColdOperation();
+            }
             return {
                 registered: true,
                 entityCount: countDescendantRows(database, workspaceId),
@@ -670,10 +690,10 @@ export class UserGroundHost {
             throw new Error("entity Record rail already started");
         await this.wire.open();
         // One floor, no per-feature gates: this shell and its gateway release
-        // together, and the gateway ships first. 8 = paged mutation operations;
-        // an older gateway drops the stage frame and a large graph could never
-        // publish.
-        if (this.wire.protocolVersion < 8) {
+        // together, and the gateway ships first. 10 = contract-aware cargo
+        // closure; an older gateway drops the sealed-evidence frame the upload
+        // planner depends on, so nothing this shell registers could publish.
+        if (this.wire.protocolVersion < 10) {
             throw new Error("gateway speaks an older wire protocol than this shell");
         }
         const store = new EntityRecordSqliteStore(initializeDatabase(this.databasePath(identity)));
@@ -1220,42 +1240,13 @@ export class UserGroundHost {
             measurements: { journey: trace?.journey ?? "bootstrap", kind: artifact.kind },
         });
         const cache = { target: null };
-        // A cold artifact — no local manifest — is fetched whole in one round
-        // trip: get-artifact answers with the manifest and, for artifacts within
-        // the batch caps, every block packed in part order. Chunk reads then
-        // slice this buffer instead of paying a second round trip.
-        const prefetched = { bytes: null };
-        const sliceChunks = (packed, manifest, indexes) => {
-            const offsets = [];
-            let offset = 0;
-            for (const chunk of manifest.chunks) {
-                offsets.push(offset);
-                offset += chunk.bytes;
-            }
-            return Buffer.concat(indexes.map((index) => {
-                const chunk = manifest.chunks[index];
-                const start = offsets[index];
-                if (!chunk || start === undefined)
-                    throw new Error(`content manifest has no chunk ${index}`);
-                return packed.subarray(start, start + chunk.bytes);
-            }));
-        };
-        const getChunk = async (manifest, index) => {
-            if (prefetched.bytes)
-                return sliceChunks(prefetched.bytes, manifest, [index]);
-            const chunk = manifest.chunks[index];
-            if (!chunk)
-                throw new Error(`content manifest has no chunk ${index}`);
+        // Every storage object — the manifest and each block — is fetched
+        // straight from content storage over a presigned URL and opened locally;
+        // the wire carries only the key handoff and signing requests. A retry
+        // asks for a fresh URL every attempt.
+        const fetchObject = async (object) => {
             requests += 1;
-            const frame = await retryContent(() => this.wire.request({
-                type: "private.entity-content.get",
-                resource_id: resourceId,
-                content_hash: artifact.contentHash,
-                kind: "chunk",
-                part_index: index,
-                sha256: chunk.sha256,
-            }, ["private.entity-content.data"]), true, retryAttempts);
-            const bytes = binaryFrameBytes(frame);
+            const bytes = await retryContent(() => this.direct.fetchObject(resourceId, object), true, retryAttempts);
             downloadedBytes += bytes.byteLength;
             return bytes;
         };
@@ -1267,73 +1258,39 @@ export class UserGroundHost {
                     // proves chunk identity; downloaded bytes are still hash-verified,
                     // so a stale local manifest can only fail loudly, never corrupt.
                     const local = knownManifest ?? readCachedManifest(cacheDir, artifact.contentHash);
-                    if (local) {
-                        cache.target = new ContentCacheDownload(cacheDir, artifact, local);
-                        return local;
-                    }
-                    requests += 1;
-                    const frame = await retryContent(() => this.wire.request({
-                        type: "private.entity-content.get-artifact",
-                        resource_id: resourceId,
-                        content_hash: artifact.contentHash,
-                    }, ["private.entity-content.artifact"]), true, retryAttempts);
-                    const manifestJson = String(frame.manifest_json ?? "");
-                    downloadedBytes += Buffer.byteLength(manifestJson);
-                    const checked = checkContentManifest(JSON.parse(manifestJson), artifact.contentHash);
-                    if (!checked.ok)
-                        throw new Error(checked.error);
-                    if (frame.complete === true) {
-                        const decoded = await decodeContentWireBytes(binaryWireFrameBytes(frame), frame.content_encoding, Number(frame.bytes));
-                        if (decoded.byteLength !== checked.value.bytes) {
-                            throw new Error("artifact payload does not match its manifest");
-                        }
-                        prefetched.bytes = decoded;
-                        downloadedBytes += Number(frame.wire_bytes ?? frame.bytes);
-                    }
-                    cache.target = new ContentCacheDownload(cacheDir, artifact, checked.value);
-                    return checked.value;
+                    const manifest = local ?? await (async () => {
+                        const bytes = await fetchObject({ kind: "manifest", sha256: artifact.contentHash });
+                        const checked = checkContentManifest(JSON.parse(bytes.toString("utf8")), artifact.contentHash);
+                        if (!checked.ok)
+                            throw new Error(checked.error);
+                        return checked.value;
+                    })();
+                    cache.target = new ContentCacheDownload(cacheDir, artifact, manifest);
+                    return manifest;
                 },
                 hasChunk: async (_candidate, _manifest, index) => {
                     if (!cache.target)
                         throw new Error("download cache has no content manifest");
                     return cache.target.hasChunk(index);
                 },
-                getChunk: async (_candidate, manifest, index) => getChunk(manifest, index),
-                getChunkBatch: async (_candidate, manifest, indexes) => {
-                    if (prefetched.bytes)
-                        return sliceChunks(prefetched.bytes, manifest, indexes);
-                    const frame = await retryContent(() => this.wire.request({
-                        type: "private.entity-content.get-batch",
-                        resource_id: resourceId,
-                        content_hash: artifact.contentHash,
-                        chunks: indexes.map((index) => ({
-                            part_index: index,
-                            sha256: manifest.chunks[index]?.sha256,
-                            bytes: manifest.chunks[index]?.bytes,
-                        })),
-                    }, ["private.entity-content.data-batch"]), true, retryAttempts);
-                    requests += 1;
-                    const decoded = await decodeContentWireBytes(binaryWireFrameBytes(frame), frame.content_encoding, Number(frame.bytes));
-                    downloadedBytes += Number(frame.wire_bytes ?? frame.bytes);
-                    return decoded;
+                getChunk: async (_candidate, manifest, index) => {
+                    const chunk = manifest.chunks[index];
+                    if (!chunk)
+                        throw new Error(`content manifest has no chunk ${index}`);
+                    return fetchObject({ kind: "block", sha256: chunk.sha256 });
                 },
                 putChunk: async (_candidate, _manifest, index, bytes) => {
                     if (!cache.target)
                         throw new Error("download cache has no content manifest");
                     await cache.target.putChunk(index, bytes);
                 },
-                putChunkBatch: async (_candidate, _manifest, indexes, bytes) => {
-                    if (!cache.target)
-                        throw new Error("download cache has no content manifest");
-                    await cache.target.putChunkBatch(indexes, bytes);
-                },
                 sealArtifact: async () => {
                     if (!cache.target)
                         throw new Error("download cache has no content manifest");
                     const sealed = await cache.target.seal();
                     return { local: sealed, bytes: sealed.bytes, contentHash: sealed.contentHash };
                 },
-            }, artifact.kind === "file" ? { concurrency: FILE_BATCH_DOWNLOAD_CONCURRENCY } : {});
+            }, { concurrency: DIRECT_BLOCK_DOWNLOAD_CONCURRENCY });
             this.stage({
                 primitive: "download",
                 stage: "immutable-content",
@@ -1757,9 +1714,37 @@ export class UserGroundHost {
         const snapshot = snapshotFromRecords(traveling);
         const snapshotJson = stableJson(snapshot);
         const checksum = sha256Hex(snapshotJson);
-        if (checksum === state.checksum)
-            return;
         const localIds = new Set(localRecords.map((record) => record.uuid));
+        // Graph membership is never proof of sealed cargo: a record in the cloud
+        // graph may name content sealed only under a prior content contract,
+        // whose namespace the current readers never touch. Only the authority's
+        // (resource, hash, contract) seal evidence may skip an upload — and the
+        // question is asked even when the graph itself is unchanged, so Register
+        // always re-establishes cargo closure before returning.
+        const cargo = snapshot.records.flatMap((record) => {
+            if (!localIds.has(record.uuid))
+                return [];
+            const artifact = artifactForRecord(record);
+            return artifact ? [{ record, contentHash: artifact.contentHash }] : [];
+        });
+        const sealedArtifacts = await this.sealedContentHashes(state.resourceId, [...new Set(cargo.map((entry) => entry.contentHash))]);
+        const uploadRecords = cargo
+            .filter((entry) => !sealedArtifacts.has(entry.contentHash))
+            .map((entry) => entry.record);
+        if (checksum === state.checksum) {
+            // The graph is already committed; only cargo can be missing. Repair
+            // uploads need no journal row — a crash simply re-runs the same
+            // repair on the next Register.
+            if (uploadRecords.length > 0) {
+                try {
+                    await this.uploadSnapshotContent(identity, state.resourceId, uploadRecords, trace);
+                }
+                catch (error) {
+                    throw pipelineStageError("cloud upload", error);
+                }
+            }
+            return;
+        }
         const replacementRootIds = localRecords
             .filter((record) => record.parentUUID === null)
             .map((record) => record.uuid);
@@ -1770,16 +1755,6 @@ export class UserGroundHost {
             return root !== null && replacementRoots.has(root);
         });
         const replacement = rootReplacementFromRecords(replacementRootIds, replacementRecords);
-        const publishedArtifacts = new Set(state.records.flatMap((record) => {
-            const artifact = artifactForRecord(record);
-            return artifact ? [artifact.contentHash] : [];
-        }));
-        const uploadRecords = snapshot.records.filter((record) => {
-            if (!localIds.has(record.uuid))
-                return false;
-            const artifact = artifactForRecord(record);
-            return artifact !== null && !publishedArtifacts.has(artifact.contentHash);
-        });
         supersedeSnapshotJournal(this.databasePath(identity), {
             mutationId: randomUUID(),
             resourceId: state.resourceId,
@@ -1794,6 +1769,29 @@ export class UserGroundHost {
         });
         await this.publishPendingSnapshots(identity, trace);
     }
+    /** Which of these heads does the authority hold sealed under the current
+     * content contract? Batched so a whole workspace costs a handful of
+     * frames; the answer is the only permission to skip an upload. */
+    async sealedContentHashes(resourceId, hashes) {
+        const batches = [];
+        for (let start = 0; start < hashes.length; start += PRESIGN_BATCH_OBJECTS) {
+            batches.push(hashes.slice(start, start + PRESIGN_BATCH_OBJECTS));
+        }
+        const sealed = new Set();
+        const answers = await Promise.all(batches.map((batch) => retryContent(() => this.wire.request({
+            type: "private.entity-content.sealed",
+            resource_id: resourceId,
+            content_hashes: batch,
+        }, ["private.entity-content.sealed-result"]))));
+        for (const frame of answers) {
+            if (!Array.isArray(frame.sealed)) {
+                throw new Error("content authority answered an invalid seal inventory");
+            }
+            for (const hash of frame.sealed)
+                sealed.add(String(hash));
+        }
+        return sealed;
+    }
     async publishPendingSnapshots(identity, trace) {
         for (const pending of readSnapshotPublications(this.databasePath(identity))) {
             // A rejected row is a settled fact, not an intent: the authority
@@ -4689,28 +4687,6 @@ function readyFromFrame(frame) {
         },
     };
 }
-/** Expose the WebSocket payload as a view. Live verifies content blocks
- * against their manifest, so the host must not copy or hash those bytes a
- * second time before returning them to the portable operation. */
-function binaryFrameBytes(frame) {
-    const bytes = binaryWireFrameBytes(frame);
-    if (frame.content_encoding !== undefined || bytes.length !== Number(frame.bytes)) {
-        throw new Error("wire content byte count does not match its header");
-    }
-    return bytes;
-}
-function binaryWireFrameBytes(frame) {
-    if (!(frame.data_bin instanceof Uint8Array)) {
-        throw new Error("wire content response is not a binary frame");
-    }
-    const bytes = Buffer.isBuffer(frame.data_bin)
-        ? frame.data_bin
-        : Buffer.from(frame.data_bin.buffer, frame.data_bin.byteOffset, frame.data_bin.byteLength);
-    if (bytes.length !== Number(frame.wire_bytes ?? frame.bytes)) {
-        throw new Error("wire content byte count does not match its header");
-    }
-    return bytes;
-}
 /** Immutable content reads and writes are idempotent. Retry transient wire or
  * authority failures; reads additionally tolerate the brief not-found window
  * between an immutable object acknowledgement and globally visible R2 state. */