@amalgm/shell 0.1.134 → 0.1.135

package/dist/user-ground-host.js

@@ -38,8 +38,8 @@ const CARGO_SEGMENT_LOAD_CONCURRENCY = 8;
 /** Storage-object lanes for one download set. Tiny objects use the wide
  * request window, while the separate byte budget narrows large chunks so
  * request concurrency can never silently become an unbounded memory claim. */
-/** Admission is sized to the refs batch law so one same-turn wave coalesces
- * into one refs frame and the span fetcher merges each pack's members into a
+/** Admission is sized to the refs batch law so one planned wave resolves in
+ * one refs frame and the span planner merges each pack's members into a
  * few broad ranged reads. Memory is bounded by the byte laws below, never by
  * this count. */
 const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = CONTENT_REFS_BATCH_OBJECTS;
@@ -685,7 +685,14 @@ export class UserGroundHost {
                     const fusionResourceId = this.cloudState.resourceId;
                     const fusionLegs = new TransferLegs();
                     const fusion = new RegisterUploadFusion({
-                        sealedContentHashes: (hashes) => this.sealedContentHashes(fusionResourceId, hashes),
+                        // Fusion ships whole small artifacts into packs; skipping one
+                        // requires it to be both sealed and packed, so sealed-but-
+                        // standalone smalls re-ship here and placement converges on
+                        // packs without a separate migration.
+                        sealedContentHashes: async (hashes) => {
+                            const inventory = await this.sealInventory(fusionResourceId, hashes);
+                            return new Set([...inventory.sealed].filter((hash) => inventory.packed.has(hash)));
+                        },
                         sendCargo: async (artifacts, payload) => {
                             const { sealed } = await this.putCargo(fusionResourceId, artifacts, payload, fusionLegs);
                             return sealed;
@@ -923,10 +930,11 @@ export class UserGroundHost {
             throw new Error("entity Record rail already started");
         await this.wire.open();
         // One floor, no per-feature gates: this shell and its gateway release
-        // together, and the gateway ships first. 15 = one-read Register capture:
-        // large-file blocks become durable while capture owns their bytes, then
-        // the manifest seals last. An older gateway cannot accept that rail.
-        if (this.wire.protocolVersion < 15) {
+        // together, and the gateway ships first. 16 = placement answers on the
+        // seal inventory; an older gateway cannot say which sealed artifacts are
+        // packed, and this shell plans pack-span downloads and repacks against
+        // exactly that answer.
+        if (this.wire.protocolVersion < 16) {
             throw new Error("gateway speaks an older wire protocol than this shell");
         }
         this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
@@ -1538,11 +1546,16 @@ export class UserGroundHost {
         // The authority's seal inventory is the only permission to skip an
         // upload — asked before any source is touched, so sealed artifacts cost
         // zero local reads and a retried publication whose upload already sealed
-        // completely never needs its captured ground to still exist.
-        const alreadySealed = await legs.time("planMs", () => this.sealedContentHashes(resourceId, [...byHash.keys()]));
+        // completely never needs its captured ground to still exist. Placement
+        // rides the same answer: a sealed-but-unpacked small artifact is
+        // correctness-complete but stored inefficiently, so this guarantor also
+        // re-ships it into a pack when its source is still at hand (axiom 44).
+        const inventory = await legs.time("planMs", () => this.sealInventory(resourceId, [...byHash.keys()]));
         const unsealed = [...byHash.values()]
-            .filter(({ artifact }) => !alreadySealed.has(artifact.contentHash));
-        const rows = legs.timeSync("planMs", () => readGroundRowsByUuid(this.databasePath(identity), unsealed.map(({ uuid }) => uuid)));
+            .filter(({ artifact }) => !inventory.sealed.has(artifact.contentHash));
+        const unpacked = [...byHash.values()].filter(({ artifact }) => inventory.sealed.has(artifact.contentHash)
+            && !inventory.packed.has(artifact.contentHash));
+        const rows = legs.timeSync("planMs", () => readGroundRowsByUuid(this.databasePath(identity), [...unsealed, ...unpacked].map(({ uuid }) => uuid)));
         // Each artifact uploads from the strongest evidence available: a cache
         // copy when another journey pinned one, otherwise the ground entity
         // itself. Register stages nothing, so the ground path is the common
@@ -1589,6 +1602,42 @@ export class UserGroundHost {
                 bytes,
             };
         }));
+        // Repack candidates are sealed already, so their sources are optional:
+        // anything missing, oversized, or unreadable skips silently — a frozen
+        // workspace simply keeps standalone placement until its next actual
+        // publication. What readers see never changes either way.
+        const repackSources = legs.timeSync("planMs", () => unpacked.flatMap(({ artifact, uuid }) => {
+            const cached = contentCacheFile(cache, artifact.contentHash);
+            const fromCache = existsSync(cached);
+            const row = rows.get(uuid);
+            if (!fromCache && (artifact.kind === "git" || !row))
+                return [];
+            const file = fromCache ? cached : row.absolutePath;
+            let linkTarget;
+            let bytes;
+            try {
+                if (!fromCache && artifact.kind === "link") {
+                    linkTarget = Buffer.from(readlinkSync(file), "utf8");
+                    bytes = linkTarget.byteLength;
+                }
+                else {
+                    bytes = statSync(file).size;
+                }
+            }
+            catch {
+                return [];
+            }
+            if (bytes > CHUNK_BYTES)
+                return [];
+            return [{
+                    artifact,
+                    file,
+                    fromCache,
+                    ...(linkTarget ? { linkTarget } : {}),
+                    bytes,
+                    repack: true,
+                }];
+        }));
         const stale = (source, error) => {
             if (source.fromCache) {
                 // A cache copy that no longer proves its hash is torn evidence:
@@ -1608,8 +1657,11 @@ export class UserGroundHost {
         const small = sources.filter((source) => source.bytes <= CHUNK_BYTES);
         const large = sources.filter((source) => source.bytes > CHUNK_BYTES);
         let cargoSegments = 0;
+        let repackedArtifacts = 0;
+        let repackedBytes = 0;
         const sendCargoSegment = async (segment) => {
-            const loaded = new Array(segment.items.length);
+            const loaded = new Array(segment.items.length).fill(null);
+            const dropped = [];
             await boundedForEach(segment.items.map((item, index) => ({ source: item.source, index })), CARGO_SEGMENT_LOAD_CONCURRENCY, async ({ source, index }) => {
                 try {
                     const bytes = source.linkTarget ?? await legs.time("loadMs", () => readFile(source.file));
@@ -1620,21 +1672,39 @@ export class UserGroundHost {
                     };
                 }
                 catch (error) {
+                    // A repack source that churned or vanished under this read is
+                    // dropped, never stale: the artifact stays sealed standalone,
+                    // and it is answered as sealed because it truthfully is.
+                    if (source.repack) {
+                        dropped.push(source.artifact.contentHash);
+                        return;
+                    }
                     if (sourceMismatch(error))
                         stale(source, error);
                     throw error;
                 }
             });
-            const { sealed, wireBytes } = await this.putCargo(resourceId, loaded.map((item, index) => ({
-                content_hash: segment.items[index].contentHash,
-                bytes: item.manifest.bytes,
-                chunks: item.manifest.chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
-            })), Buffer.concat(loaded.map((item) => item.bytes)), legs);
-            for (const item of loaded)
-                uploadedChunks += item.manifest.chunks.length;
+            const shipping = segment.items.flatMap((item, index) => {
+                const load = loaded[index];
+                return load === null ? [] : [{ item, load }];
+            });
+            if (shipping.length === 0)
+                return dropped;
+            const { sealed, wireBytes } = await this.putCargo(resourceId, shipping.map(({ item, load }) => ({
+                content_hash: item.contentHash,
+                bytes: load.manifest.bytes,
+                chunks: load.manifest.chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
+            })), Buffer.concat(shipping.map(({ load }) => load.bytes)), legs);
+            for (const { item, load } of shipping) {
+                uploadedChunks += load.manifest.chunks.length;
+                if (item.source.repack) {
+                    repackedArtifacts += 1;
+                    repackedBytes += load.manifest.bytes;
+                }
+            }
             uploadedBytes += wireBytes;
             cargoSegments += 1;
-            return sealed;
+            return [...sealed, ...dropped];
         };
         const transfer = async (source) => {
             const { artifact, file, manifest: knownManifest, bytes: fileBytes } = source;
@@ -1701,7 +1771,7 @@ export class UserGroundHost {
             }
         };
         try {
-            await streamCargo(small.map((source) => ({
+            await streamCargo([...small, ...repackSources].map((source) => ({
                 contentHash: source.artifact.contentHash,
                 bytes: source.bytes,
                 source,
@@ -1728,6 +1798,12 @@ export class UserGroundHost {
                     cargoBytes: small.reduce((total, source) => total + source.bytes, 0),
                     largeArtifacts: large.length,
                     largeBytes: large.reduce((total, source) => total + source.bytes, 0),
+                    // Placement work: sealed-but-standalone smalls re-shipped into
+                    // packs by this guarantor. Candidates counts what the inventory
+                    // named; repacked counts what actually shipped this pass.
+                    repackCandidates: unpacked.length,
+                    repackedArtifacts,
+                    repackedBytes,
                     uploadedBytes,
                     loadedBytes,
                     uploadedChunks,
@@ -1757,10 +1833,11 @@ export class UserGroundHost {
         }
     }
     /** Download a set of immutable artifacts straight from content storage.
-     * The set is the primitive: every storage object the set still needs —
-     * manifests and blocks alike — shares one wide lane bound, so no artifact
-     * serializes behind another's plan or seal. Each object is still its own
-     * storage request. A lone artifact is a set of one. */
+     * The set is the primitive and the plan is physical: the engine resolves
+     * the whole want-list in two planned waves — manifests grouped by the
+     * pack that holds them, then blocks the same way — so all members of one
+     * pack collapse into a few broad ranged reads while standalone objects
+     * stay one read each. A lone artifact is a set of one. */
     async downloadContentSet(resourceId, cacheDir, wants, retryAttempts = 10, trace, staged) {
         const unique = new Map(wants.map((want) => [want.artifact.contentHash, want]));
         const sole = unique.size === 1 ? [...unique.values()][0]?.artifact : undefined;
@@ -1801,56 +1878,75 @@ export class UserGroundHost {
                 throw new Error("download cache has no content manifest");
             return found;
         };
-        // Every storage object — the manifest and each block — is fetched
-        // straight from content storage over a presigned URL and opened locally;
-        // packed objects arrive as merged byte ranges of their immutable pack.
-        // The wire carries only the key handoff and refs requests. A retry asks
-        // for a fresh ref every attempt.
-        const fetchObject = async (object) => {
-            requests += 1;
-            const bytes = await legs.time("fetchMs", () => retryContent(() => this.direct.fetchObject(resourceId, object), true, retryAttempts));
-            downloadedBytes += bytes.byteLength;
-            return bytes;
+        // An Add stages ordinary files directly at their hidden final
+        // destination — except content this machine already holds sealed,
+        // which stays on the cache rail so a warm re-Add remains a local
+        // clone instead of a fresh download.
+        const route = (artifact, manifest) => {
+            const stagedPaths = stagedSet && staged?.destinations.get(artifact.contentHash);
+            if (stagedPaths && !hasSealedContent(cacheDir, artifact.contentHash, manifest.bytes)) {
+                stagedArtifacts += 1;
+                targets.set(artifact.contentHash, stagedSet.target(artifact, manifest, stagedPaths));
+            }
+            else {
+                targets.set(artifact.contentHash, downloads.target(artifact, manifest));
+            }
         };
         try {
-            const receipts = await downloadAllArtifacts([...unique.values()].map((want) => want.artifact), {
+            // Every storage object — manifests and blocks alike — is fetched
+            // straight from content storage over a presigned URL and opened
+            // locally; the engine plans whole waves against these primitives, so
+            // the wire carries only the key handoff and batched refs requests. A
+            // retried read refreshes its URL but never its geometry: placement is
+            // immutable, and a moved answer fails loudly instead of retrying.
+            const { receipts, shape } = await downloadAllArtifacts([...unique.values()].map((want) => want.artifact), {
                 sha256Hex: timedSha256Hex,
-                getManifest: async (artifact) => {
+                localManifest: (artifact) => legs.timeSync("planMs", () => {
                     // The record's plan or this machine's own manifest cache already
                     // proves chunk identity; downloaded bytes are still hash-verified,
                     // so a stale local manifest can only fail loudly, never corrupt.
-                    const local = legs.timeSync("planMs", () => unique.get(artifact.contentHash)?.manifest
-                        ?? readCachedManifest(cacheDir, artifact.contentHash));
-                    const manifest = local ?? await (async () => {
-                        const bytes = await fetchObject({ kind: "manifest", sha256: artifact.contentHash });
-                        const checked = checkContentManifest(JSON.parse(bytes.toString("utf8")), artifact.contentHash);
-                        if (!checked.ok)
-                            throw new Error(checked.error);
-                        return checked.value;
-                    })();
-                    // An Add stages ordinary files directly at their hidden final
-                    // destination — except content this machine already holds sealed,
-                    // which stays on the cache rail so a warm re-Add remains a local
-                    // clone instead of a fresh download.
-                    legs.timeSync("planMs", () => {
-                        const stagedPaths = stagedSet && staged?.destinations.get(artifact.contentHash);
-                        if (stagedPaths && !hasSealedContent(cacheDir, artifact.contentHash, manifest.bytes)) {
-                            stagedArtifacts += 1;
-                            targets.set(artifact.contentHash, stagedSet.target(artifact, manifest, stagedPaths));
-                        }
-                        else {
-                            targets.set(artifact.contentHash, downloads.target(artifact, manifest));
+                    const local = unique.get(artifact.contentHash)?.manifest
+                        ?? readCachedManifest(cacheDir, artifact.contentHash);
+                    if (local)
+                        route(artifact, local);
+                    return local ?? null;
+                }),
+                requestRefs: async (objects) => legs.time("fetchMs", () => retryContent(() => this.direct.requestRefs(resourceId, objects), true, retryAttempts)),
+                fetchSpan: async (url, span, refresh) => legs.time("fetchMs", async () => {
+                    let attempt = 0;
+                    return retryContent(async () => {
+                        let signed = url;
+                        if (attempt > 0) {
+                            try {
+                                signed = await refresh();
+                            }
+                            catch (error) {
+                                // A moved placement (or a refused re-resolution) is a
+                                // planning fact, not a transient fetch fault — surface it
+                                // without burning the remaining attempts.
+                                throw new WireRequestError({
+                                    type: "shared.error",
+                                    code: "entity_content_moved",
+                                    message: error instanceof Error ? error.message : String(error),
+                                    status: 409,
+                                    retryable: false,
+                                });
+                            }
                         }
-                    });
-                    return manifest;
+                        attempt += 1;
+                        return this.direct.fetchSpan(signed, span);
+                    }, true, retryAttempts);
+                }),
+                openEnvelope: async (_object, sealed) => {
+                    requests += 1;
+                    const plain = await legs.time("fetchMs", () => this.direct.openEnvelope(resourceId, sealed));
+                    downloadedBytes += plain.byteLength;
+                    return plain;
                 },
-                hasChunk: async (artifact, _manifest, index) => legs.time("reuseMs", () => target(artifact).hasChunk(index)),
-                getChunk: async (_artifact, manifest, index) => {
-                    const chunk = manifest.chunks[index];
-                    if (!chunk)
-                        throw new Error(`content manifest has no chunk ${index}`);
-                    return fetchObject({ kind: "block", sha256: chunk.sha256 });
+                manifestProven: async (artifact, manifest) => {
+                    legs.timeSync("planMs", () => route(artifact, manifest));
                 },
+                hasChunk: async (artifact, _manifest, index) => legs.time("reuseMs", () => target(artifact).hasChunk(index)),
                 putChunk: async (artifact, _manifest, index, bytes) => legs.time("writeMs", () => target(artifact).putChunk(index, bytes)),
                 sealArtifact: async (artifact) => {
                     const sealed = await legs.time("sealMs", () => target(artifact).seal());
@@ -1891,6 +1987,10 @@ export class UserGroundHost {
                     // nothing in flight. The staged rail decomposes its own write and
                     // seal internals below; `stagedArtifacts` counts its placements.
                     requests,
+                    // The engine's own geometry: how manifests were known (local vs
+                    // fetched, packed vs standalone) and how many planned pack spans
+                    // carried the packed objects.
+                    ...shape,
                     ...physicalPlaneDelta(planeBefore, this.direct.metrics()),
                     ...legs.report(),
                     ...(stagedSet ? stagedSet.report() : {}),
@@ -2471,27 +2571,35 @@ export class UserGroundHost {
         });
     }
     /** Which of these heads does the authority hold sealed under the current
-     * content contract? Batched so a whole workspace costs a handful of
-     * frames; the answer is the only permission to skip an upload. */
-    async sealedContentHashes(resourceId, hashes) {
+     * content contract — and which of the sealed ones are packed, their whole
+     * closure inside immutable packs? Batched so a whole workspace costs a
+     * handful of frames. `sealed` is the only permission to skip an upload
+     * for correctness; `packed` is placement knowledge only. The packed
+     * answer is a protocol 16 capability and is required: a missing field
+     * means an older gateway, and this machine refuses to guess placement. */
+    async sealInventory(resourceId, hashes) {
         const batches = [];
         for (let start = 0; start < hashes.length; start += CONTENT_REFS_BATCH_OBJECTS) {
             batches.push(hashes.slice(start, start + CONTENT_REFS_BATCH_OBJECTS));
         }
         const sealed = new Set();
+        const packed = new Set();
         const answers = await Promise.all(batches.map((batch) => retryContent(() => this.wire.request({
             type: "private.entity-content.sealed",
             resource_id: resourceId,
             content_hashes: batch,
         }, ["private.entity-content.sealed-result"]))));
         for (const frame of answers) {
-            if (!Array.isArray(frame.sealed)) {
-                throw new Error("content authority answered an invalid seal inventory");
+            if (!Array.isArray(frame.sealed) || !Array.isArray(frame.packed)) {
+                throw new Error("content authority answered a seal inventory without "
+                    + "placement; protocol 16 requires the gateway to deploy before this machine");
             }
             for (const hash of frame.sealed)
                 sealed.add(String(hash));
+            for (const hash of frame.packed)
+                packed.add(String(hash));
         }
-        return sealed;
+        return { sealed, packed };
     }
     async publishPendingSnapshots(identity, trace, manifestHints) {
         for (const pending of readSnapshotPublications(this.databasePath(identity))) {
@@ -2512,7 +2620,7 @@ export class UserGroundHost {
             return;
         const started = performance.now();
         const legs = new TransferLegs();
-        const sealed = await legs.time("planMs", () => this.sealedContentHashes(resourceId, pages.map((page) => page.sha256)));
+        const { sealed } = await legs.time("planMs", () => this.sealInventory(resourceId, pages.map((page) => page.sha256)));
         const unsealed = pages
             .filter((page) => !sealed.has(page.sha256))
             .map((page) => ({ contentHash: page.sha256, bytes: page.bytes.byteLength, page }));