@amalgm/shell 0.1.133 → 0.1.134

package/dist/user-ground-host.js

@@ -11,7 +11,8 @@ import { EntityApplyHost, } from "./entity-apply-host.js";
 import { EntityRecordSqliteStore } from "./entity-record-store.js";
 import { observeEntityRecordPorts } from "./entity-record-observability.js";
 import { indexRowsByOutermostRoot } from "./ground-root-index.js";
-import { ContentCacheDownloadSet, captureContentFile, hasSealedContent, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
+import { ContentCacheDownloadSet, captureContentFile, contentBlockCollector, hasSealedContent, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
+import { RegisterUploadFusion } from "./register-upload-fusion.js";
 import { StagedContentSet, addStagingEvidenceDir, clearAddStagingEvidence, } from "./workspace-add-staging.js";
 import { DirectContentPlane } from "./content-direct.js";
 import { encodeContentWireBytes } from "./content-wire-codec.js";
@@ -676,20 +677,75 @@ export class UserGroundHost {
                     this.stage({
                         primitive: "register", stage: "entity-registration", status: "started", workspaceId,
                     });
-                    const scanned = await reconcileGround({
-                        identity,
-                        userRoot: this.userRoot(identity),
-                        database: this.databasePath(identity),
-                        cacheDir: this.cacheDir(identity),
-                        onScan: (evidence) => this.reportDetectScan(evidence),
-                        measuring: this.measuring(),
-                        registrationWorkspaceId: workspaceId,
-                        suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
+                    // The walk's own read is also the upload: smalls ride cargo
+                    // segments, larges stream blocks and seal last. The flush settles
+                    // every fused effect before publication plans, so the upload
+                    // stage's seal inventory — the guarantor for anything fusion
+                    // could not ship — already sees these seals.
+                    const fusionResourceId = this.cloudState.resourceId;
+                    const fusionLegs = new TransferLegs();
+                    const fusion = new RegisterUploadFusion({
+                        sealedContentHashes: (hashes) => this.sealedContentHashes(fusionResourceId, hashes),
+                        sendCargo: async (artifacts, payload) => {
+                            const { sealed } = await this.putCargo(fusionResourceId, artifacts, payload, fusionLegs);
+                            return sealed;
+                        },
+                        putBlocks: async (chunks, bytes) => {
+                            await this.putContentBlocks(fusionResourceId, chunks, bytes, fusionLegs);
+                        },
+                        sealManifest: async (manifest) => {
+                            await this.putContentManifest(fusionResourceId, manifest, fusionLegs);
+                        },
                     });
+                    const scanned = await (async () => {
+                        try {
+                            return await reconcileGround({
+                                identity,
+                                userRoot: this.userRoot(identity),
+                                database: this.databasePath(identity),
+                                cacheDir: this.cacheDir(identity),
+                                onScan: (evidence) => this.reportDetectScan(evidence),
+                                measuring: this.measuring(),
+                                registrationWorkspaceId: workspaceId,
+                                suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
+                                fusion,
+                            });
+                        }
+                        finally {
+                            // Even an abandoned walk settles its in-flight cargo, and a
+                            // stale-capture re-walk then skips whatever already sealed.
+                            const fused = await fusion.flush();
+                            this.stage({
+                                primitive: "register",
+                                stage: "fused-upload",
+                                status: "completed",
+                                workspaceId,
+                                measurements: {
+                                    smallArtifacts: fused.smallArtifacts,
+                                    smallBytes: fused.smallBytes,
+                                    cargoSegments: fused.cargoSegments,
+                                    sealedSkipped: fused.sealedSkipped,
+                                    largeArtifacts: fused.largeArtifacts,
+                                    largeBytes: fused.largeBytes,
+                                    ...(fused.disabledReason !== null ? { disabled: fused.disabledReason } : {}),
+                                    ...fusionLegs.report(),
+                                },
+                            });
+                        }
+                    })();
                     const retry = scanned.detection.find((plan) => plan.kind === "retry");
                     if (retry?.kind === "retry") {
                         throw new Error(retry.reasons.join("; "));
                     }
+                    // The classic upload re-proved every source byte against its
+                    // captured hash when it read it. The fused rail already shipped
+                    // those bytes, so the equivalent proof is the fingerprint law at
+                    // stat cost: any fused source that moved after capture makes this
+                    // attempt stale, and the re-walk publishes what stands there now.
+                    const movedGround = await fusion.movedGround();
+                    if (movedGround !== null) {
+                        throw new StaleCaptureError(`${movedGround} changed after its fused upload`);
+                    }
                     const registrationDuration = performance.now() - registrationStarted;
                     const commitStarted = performance.now();
                     this.stage({
@@ -867,10 +923,10 @@ export class UserGroundHost {
             throw new Error("entity Record rail already started");
         await this.wire.open();
         // One floor, no per-feature gates: this shell and its gateway release
-        // together, and the gateway ships first. 14 = immutable content packs
-        // with a refs frame answering locations; an older gateway still speaks
-        // presign and cannot resolve a packed object at all.
-        if (this.wire.protocolVersion < 14) {
+        // together, and the gateway ships first. 15 = one-read Register capture:
+        // large-file blocks become durable while capture owns their bytes, then
+        // the manifest seals last. An older gateway cannot accept that rail.
+        if (this.wire.protocolVersion < 15) {
             throw new Error("gateway speaks an older wire protocol than this shell");
         }
         this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
@@ -1413,6 +1469,40 @@ export class UserGroundHost {
             wireBytes: encoded.bytes.byteLength,
         };
     }
+    /** Store one wave of provisional content blocks, addressed by their own
+     * digests and bound to no artifact. Resolving means every block is
+     * durable at the authority. Answers the wire bytes actually sent. */
+    async putContentBlocks(resourceId, blocks, payload, legs) {
+        const encoded = await legs.time("encodeMs", () => encodeContentWireBytes(payload));
+        const frame = await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
+            type: "private.entity-content.put-blocks",
+            resource_id: resourceId,
+            blocks,
+            bytes: payload.byteLength,
+            wire_bytes: encoded.bytes.byteLength,
+            ...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
+        }, encoded.bytes, ["private.entity-content.stored-blocks"])));
+        legs.add("gatewayVerifyMs", Number(frame.verify_ms));
+        legs.add("gatewayStoreMs", Number(frame.store_ms));
+        return encoded.bytes.byteLength;
+    }
+    /** Seal one artifact's manifest. The authority re-proves every named
+     * block durable before anything becomes visible, so this frame is only
+     * lawful after the complete file went over the wire. */
+    async putContentManifest(resourceId, manifest, legs) {
+        const bytes = Buffer.from(stableJson(manifest), "utf8");
+        const frame = await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
+            type: "private.entity-content.put",
+            resource_id: resourceId,
+            content_hash: manifest.contentHash,
+            sha256: sha256Hex(bytes),
+            bytes: bytes.length,
+        }, bytes, ["private.entity-content.stored"])));
+        legs.add("gatewayVerifyMs", Number(frame.verify_ms));
+        legs.add("gatewayStoreMs", Number(frame.store_ms));
+        legs.add("gatewaySealMs", Number(frame.seal_ms));
+        return bytes.length;
+    }
     async uploadSnapshotContent(identity, resourceId, records, trace, manifestHints) {
         const cache = this.cacheDir(identity);
         const byHash = new Map();
@@ -1550,22 +1640,12 @@ export class UserGroundHost {
             const { artifact, file, manifest: knownManifest, bytes: fileBytes } = source;
             const sourceFile = { handle: null };
             let manifestPresent = false;
-            const putChunk = async (manifest, index, bytes) => {
+            const chunkBlocks = (manifest, indexes) => indexes.map((index) => {
                 const chunk = manifest.chunks[index];
                 if (!chunk)
                     throw new Error(`content manifest has no chunk ${index}`);
-                await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
-                    type: "private.entity-content.put",
-                    resource_id: resourceId,
-                    content_hash: artifact.contentHash,
-                    kind: "chunk",
-                    part_index: index,
-                    part_count: manifest.chunks.length,
-                    sha256: chunk.sha256,
-                    bytes: chunk.bytes,
-                }, bytes, ["private.entity-content.stored"])));
-                uploadedBytes += bytes.byteLength;
-            };
+                return { sha256: chunk.sha256, bytes: chunk.bytes };
+            });
             try {
                 const receipt = await uploadArtifact(artifact, {
                     bytes: fileBytes,
@@ -1595,40 +1675,16 @@ export class UserGroundHost {
                         }
                         return frame.missing;
                     },
-                    putChunk: async (_candidate, manifest, index, bytes) => putChunk(manifest, index, bytes),
+                    putChunk: async (_candidate, manifest, index, bytes) => {
+                        uploadedBytes += await this.putContentBlocks(resourceId, chunkBlocks(manifest, [index]), bytes, legs);
+                    },
                     putChunkBatch: async (_candidate, manifest, indexes, bytes) => {
-                        const encoded = await legs.time("encodeMs", () => encodeContentWireBytes(bytes));
-                        await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
-                            type: "private.entity-content.put-batch",
-                            resource_id: resourceId,
-                            content_hash: artifact.contentHash,
-                            part_count: manifest.chunks.length,
-                            chunks: indexes.map((index) => ({
-                                part_index: index,
-                                sha256: manifest.chunks[index]?.sha256,
-                                bytes: manifest.chunks[index]?.bytes,
-                            })),
-                            bytes: bytes.byteLength,
-                            wire_bytes: encoded.bytes.byteLength,
-                            ...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
-                        }, encoded.bytes, ["private.entity-content.stored-batch"])));
-                        uploadedBytes += encoded.bytes.byteLength;
+                        uploadedBytes += await this.putContentBlocks(resourceId, chunkBlocks(manifest, indexes), bytes, legs);
                     },
                     putManifest: async (_candidate, manifest) => {
                         if (manifestPresent)
                             return;
-                        const bytes = Buffer.from(stableJson(manifest), "utf8");
-                        await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
-                            type: "private.entity-content.put",
-                            resource_id: resourceId,
-                            content_hash: artifact.contentHash,
-                            kind: "manifest",
-                            part_index: manifest.chunks.length,
-                            part_count: manifest.chunks.length,
-                            sha256: sha256Hex(bytes),
-                            bytes: bytes.length,
-                        }, bytes, ["private.entity-content.stored"])));
-                        uploadedBytes += bytes.byteLength;
+                        uploadedBytes += await this.putContentManifest(resourceId, manifest, legs);
                     },
                 });
                 uploadedChunks += receipt.uploadedChunks;
@@ -4102,7 +4158,7 @@ function materializedWorkspaceBindings(bindingDir) {
     return bindings.sort((left, right) => left.workspaceId.localeCompare(right.workspaceId));
 }
 async function reconcileGround(input) {
-    const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, measuring = false, } = input;
+    const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, fusion, onScan, measuring = false, } = input;
     const resourceId = privateEntityResourceId(identity.userId, sha256Hex);
     const bindingDir = workspaceBindingDir(userRoot, identity.deviceId);
     const boundRootEntries = materializedWorkspaceBindings(bindingDir);
@@ -4328,9 +4384,11 @@ async function reconcileGround(input) {
             suspicion: observationFor(rootPath),
             // A register scan publishes one complete snapshot; it never mints Send
             // records for the same initial contents, so it plans no detection and
-            // stages no cache bytes — upload reads the ground itself.
+            // stages no cache bytes — the fused rail ships content from the walk's
+            // own read, and upload re-reads only what fusion could not seal.
             plansSendRecords: suspicions !== undefined && registrationWorkspaceId === undefined,
             largeManifests,
+            fusion,
         });
     }
     const changedUUIDs = new Set([
@@ -4358,7 +4416,7 @@ async function reconcileGround(input) {
     };
 }
 async function reconcileRoot(input) {
-    const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, } = input;
+    const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, fusion, } = input;
     // Prior rows reach this scan from every binding this root encloses. A row's
     // stored relativePath is relative to the root that last committed it, which
     // is a different frame when an enclosing boundary was just registered
@@ -4711,9 +4769,20 @@ async function reconcileRoot(input) {
                 captureTasks.push(captureGate(async () => {
                     const taskStarted = measureEvidence ? performance.now() : 0;
                     let current = stats;
+                    // Repository leaves ship on the Git-native rail; everything else
+                    // may fuse its capture read with upload. Fusion is opportunistic:
+                    // once disabled, captures continue exactly as the classic path
+                    // and the upload stage ships what the walk could not.
+                    const fused = activeRepository ? undefined : fusion;
                     for (let attempt = 1;; attempt++) {
                         try {
-                            const captured = await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir);
+                            const collect = fused?.enabled && current.size <= CHUNK_BYTES
+                                ? contentBlockCollector(current.size)
+                                : null;
+                            const captured = fused && current.size > CHUNK_BYTES
+                                ? (await fused.captureLarge(absolutePath, current))
+                                    ?? await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir)
+                                : await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir, collect);
                             if (measureEvidence && evidence) {
                                 evidence.captureBusyMs += performance.now() - taskStarted;
                                 evidence.captureBytes += captured.bytes;
@@ -4726,6 +4795,12 @@ async function reconcileRoot(input) {
                             if (captured.bytes > CHUNK_BYTES) {
                                 largeManifests?.set(captured.contentHash, captured.manifest);
                             }
+                            // The capture proved these exact bytes against the fingerprint
+                            // law; hand them to the cargo rail. Waiting here is window
+                            // backpressure, bounding memory by the capture pool's width.
+                            if (collect && fused) {
+                                await fused.addSmall(absolutePath, current, captured, collect.collected);
+                            }
                             return;
                         }
                         catch (error) {