@amalgm/shell 0.1.93 → 0.1.94

package/dist/user-ground-host.js

@@ -12,7 +12,8 @@ import { EntityRecordSqliteStore } from "./entity-record-store.js";
 import { observeEntityRecordPorts } from "./entity-record-observability.js";
 import { indexRowsByOutermostRoot } from "./ground-root-index.js";
 import { ContentCacheDownload, captureContentFile, hashContentFile, sealCapturedArtifact, } from "./content-cache-host.js";
-import { decodeContentWireBytes, encodeContentWireBytes } from "./content-wire-codec.js";
+import { DirectContentPlane } from "./content-direct.js";
+import { encodeContentWireBytes } from "./content-wire-codec.js";
 import { exactTextReplay, planGroundDetection, reconcileGroundUUIDs, } from "./detection/portable.js";
 import { AcceptedInboxResultIndex } from "./detection/accepted-result.js";
 import { sameNotebookBasisRow } from "./detection/notebook-basis.js";
@@ -26,8 +27,13 @@ import { GroundCoordinator } from "./ground-coordination.js";
 import { submitMutationOperation } from "./paged-mutation-submit.js";
 import { WireClient, WireRequestError } from "./wire-client.js";
 const SMALL_CONTENT_UPLOAD_CONCURRENCY = 16;
-const FILE_CONTENT_DOWNLOAD_CONCURRENCY = 4;
-const FILE_BATCH_DOWNLOAD_CONCURRENCY = 2;
+/** Artifact lanes for a download phase. Direct storage fetches cost the
+ * gateway nothing, so parallelism is bounded by this machine's pipe rather
+ * than authority fairness. */
+const FILE_CONTENT_DOWNLOAD_CONCURRENCY = 16;
+/** Block lanes within one artifact. Most files are a single block; large
+ * repository packs fan their chunks across this. */
+const DIRECT_BLOCK_DOWNLOAD_CONCURRENCY = 8;
 const DETECT_QUIET_MS = 12;
 const DETECT_MAX_DEFERRAL_MS = 75;
 const DETECT_RETRY_MS = 250;
@@ -47,6 +53,7 @@ const GROUND_ROW_COLUMNS = [
 export class UserGroundHost {
     options;
     wire;
+    direct;
     watchHost;
     activeIdentity = null;
     rescanTimer = null;
@@ -75,6 +82,9 @@ export class UserGroundHost {
             runtimeVersion: options.runtimeVersion,
             requestTimeoutMs: options.wireRequestTimeoutMs,
         });
+        this.direct = new DirectContentPlane({
+            request: (frame, accept) => this.wire.request(frame, accept),
+        });
         this.watchHost = new NodeWatchHost({
             open: options.openWatch,
             onSignal: () => this.scheduleRescan(),
@@ -670,10 +680,10 @@ export class UserGroundHost {
             throw new Error("entity Record rail already started");
         await this.wire.open();
         // One floor, no per-feature gates: this shell and its gateway release
-        // together, and the gateway ships first. 8 = paged mutation operations;
-        // an older gateway drops the stage frame and a large graph could never
-        // publish.
-        if (this.wire.protocolVersion < 8) {
+        // together, and the gateway ships first. 9 = direct content downloads;
+        // an older gateway still relays download bytes itself and drops the key
+        // and presign frames this shell depends on.
+        if (this.wire.protocolVersion < 9) {
             throw new Error("gateway speaks an older wire protocol than this shell");
         }
         const store = new EntityRecordSqliteStore(initializeDatabase(this.databasePath(identity)));
@@ -1220,42 +1230,13 @@ export class UserGroundHost {
             measurements: { journey: trace?.journey ?? "bootstrap", kind: artifact.kind },
         });
         const cache = { target: null };
-        // A cold artifact — no local manifest — is fetched whole in one round
-        // trip: get-artifact answers with the manifest and, for artifacts within
-        // the batch caps, every block packed in part order. Chunk reads then
-        // slice this buffer instead of paying a second round trip.
-        const prefetched = { bytes: null };
-        const sliceChunks = (packed, manifest, indexes) => {
-            const offsets = [];
-            let offset = 0;
-            for (const chunk of manifest.chunks) {
-                offsets.push(offset);
-                offset += chunk.bytes;
-            }
-            return Buffer.concat(indexes.map((index) => {
-                const chunk = manifest.chunks[index];
-                const start = offsets[index];
-                if (!chunk || start === undefined)
-                    throw new Error(`content manifest has no chunk ${index}`);
-                return packed.subarray(start, start + chunk.bytes);
-            }));
-        };
-        const getChunk = async (manifest, index) => {
-            if (prefetched.bytes)
-                return sliceChunks(prefetched.bytes, manifest, [index]);
-            const chunk = manifest.chunks[index];
-            if (!chunk)
-                throw new Error(`content manifest has no chunk ${index}`);
+        // Every storage object — the manifest and each block — is fetched
+        // straight from content storage over a presigned URL and opened locally;
+        // the wire carries only the key handoff and signing requests. A retry
+        // asks for a fresh URL every attempt.
+        const fetchObject = async (object) => {
             requests += 1;
-            const frame = await retryContent(() => this.wire.request({
-                type: "private.entity-content.get",
-                resource_id: resourceId,
-                content_hash: artifact.contentHash,
-                kind: "chunk",
-                part_index: index,
-                sha256: chunk.sha256,
-            }, ["private.entity-content.data"]), true, retryAttempts);
-            const bytes = binaryFrameBytes(frame);
+            const bytes = await retryContent(() => this.direct.fetchObject(resourceId, object), true, retryAttempts);
             downloadedBytes += bytes.byteLength;
             return bytes;
         };
@@ -1267,73 +1248,39 @@ export class UserGroundHost {
                     // proves chunk identity; downloaded bytes are still hash-verified,
                     // so a stale local manifest can only fail loudly, never corrupt.
                     const local = knownManifest ?? readCachedManifest(cacheDir, artifact.contentHash);
-                    if (local) {
-                        cache.target = new ContentCacheDownload(cacheDir, artifact, local);
-                        return local;
-                    }
-                    requests += 1;
-                    const frame = await retryContent(() => this.wire.request({
-                        type: "private.entity-content.get-artifact",
-                        resource_id: resourceId,
-                        content_hash: artifact.contentHash,
-                    }, ["private.entity-content.artifact"]), true, retryAttempts);
-                    const manifestJson = String(frame.manifest_json ?? "");
-                    downloadedBytes += Buffer.byteLength(manifestJson);
-                    const checked = checkContentManifest(JSON.parse(manifestJson), artifact.contentHash);
-                    if (!checked.ok)
-                        throw new Error(checked.error);
-                    if (frame.complete === true) {
-                        const decoded = await decodeContentWireBytes(binaryWireFrameBytes(frame), frame.content_encoding, Number(frame.bytes));
-                        if (decoded.byteLength !== checked.value.bytes) {
-                            throw new Error("artifact payload does not match its manifest");
-                        }
-                        prefetched.bytes = decoded;
-                        downloadedBytes += Number(frame.wire_bytes ?? frame.bytes);
-                    }
-                    cache.target = new ContentCacheDownload(cacheDir, artifact, checked.value);
-                    return checked.value;
+                    const manifest = local ?? await (async () => {
+                        const bytes = await fetchObject({ kind: "manifest", sha256: artifact.contentHash });
+                        const checked = checkContentManifest(JSON.parse(bytes.toString("utf8")), artifact.contentHash);
+                        if (!checked.ok)
+                            throw new Error(checked.error);
+                        return checked.value;
+                    })();
+                    cache.target = new ContentCacheDownload(cacheDir, artifact, manifest);
+                    return manifest;
                 },
                 hasChunk: async (_candidate, _manifest, index) => {
                     if (!cache.target)
                         throw new Error("download cache has no content manifest");
                     return cache.target.hasChunk(index);
                 },
-                getChunk: async (_candidate, manifest, index) => getChunk(manifest, index),
-                getChunkBatch: async (_candidate, manifest, indexes) => {
-                    if (prefetched.bytes)
-                        return sliceChunks(prefetched.bytes, manifest, indexes);
-                    const frame = await retryContent(() => this.wire.request({
-                        type: "private.entity-content.get-batch",
-                        resource_id: resourceId,
-                        content_hash: artifact.contentHash,
-                        chunks: indexes.map((index) => ({
-                            part_index: index,
-                            sha256: manifest.chunks[index]?.sha256,
-                            bytes: manifest.chunks[index]?.bytes,
-                        })),
-                    }, ["private.entity-content.data-batch"]), true, retryAttempts);
-                    requests += 1;
-                    const decoded = await decodeContentWireBytes(binaryWireFrameBytes(frame), frame.content_encoding, Number(frame.bytes));
-                    downloadedBytes += Number(frame.wire_bytes ?? frame.bytes);
-                    return decoded;
+                getChunk: async (_candidate, manifest, index) => {
+                    const chunk = manifest.chunks[index];
+                    if (!chunk)
+                        throw new Error(`content manifest has no chunk ${index}`);
+                    return fetchObject({ kind: "block", sha256: chunk.sha256 });
                 },
                 putChunk: async (_candidate, _manifest, index, bytes) => {
                     if (!cache.target)
                         throw new Error("download cache has no content manifest");
                     await cache.target.putChunk(index, bytes);
                 },
-                putChunkBatch: async (_candidate, _manifest, indexes, bytes) => {
-                    if (!cache.target)
-                        throw new Error("download cache has no content manifest");
-                    await cache.target.putChunkBatch(indexes, bytes);
-                },
                 sealArtifact: async () => {
                     if (!cache.target)
                         throw new Error("download cache has no content manifest");
                     const sealed = await cache.target.seal();
                     return { local: sealed, bytes: sealed.bytes, contentHash: sealed.contentHash };
                 },
-            }, artifact.kind === "file" ? { concurrency: FILE_BATCH_DOWNLOAD_CONCURRENCY } : {});
+            }, { concurrency: DIRECT_BLOCK_DOWNLOAD_CONCURRENCY });
             this.stage({
                 primitive: "download",
                 stage: "immutable-content",
@@ -4689,28 +4636,6 @@ function readyFromFrame(frame) {
         },
     };
 }
-/** Expose the WebSocket payload as a view. Live verifies content blocks
- * against their manifest, so the host must not copy or hash those bytes a
- * second time before returning them to the portable operation. */
-function binaryFrameBytes(frame) {
-    const bytes = binaryWireFrameBytes(frame);
-    if (frame.content_encoding !== undefined || bytes.length !== Number(frame.bytes)) {
-        throw new Error("wire content byte count does not match its header");
-    }
-    return bytes;
-}
-function binaryWireFrameBytes(frame) {
-    if (!(frame.data_bin instanceof Uint8Array)) {
-        throw new Error("wire content response is not a binary frame");
-    }
-    const bytes = Buffer.isBuffer(frame.data_bin)
-        ? frame.data_bin
-        : Buffer.from(frame.data_bin.buffer, frame.data_bin.byteOffset, frame.data_bin.byteLength);
-    if (bytes.length !== Number(frame.wire_bytes ?? frame.bytes)) {
-        throw new Error("wire content byte count does not match its header");
-    }
-    return bytes;
-}
 /** Immutable content reads and writes are idempotent. Retry transient wire or
  * authority failures; reads additionally tolerate the brief not-found window
  * between an immutable object acknowledgement and globally visible R2 state. */