@amalgm/shell 0.1.100 → 0.1.102

package/PURPOSE.md

@@ -284,10 +284,13 @@ shell only assembles.
     baseline: the newest retained Inbox result that exactly equals each
     installed row advances that entity's resolved and materialized sequence
     without replaying historical cargo, while every later different result
-    remains ordinary Apply work. Startup adopts that baseline while holding
-    every watched ground cone: an older Detect transaction settles first and
-    any newer Watch generation waits until adoption finishes. Apply starts
-    only after that finite boundary. Register
+    remains ordinary Apply work. Watch may establish native coverage before
+    startup Apply is ready, but its suspicions remain unconsumed until Apply
+    has adopted or resumed durable materialization. Startup adopts that
+    baseline while holding every watched ground cone; only after Apply starts
+    does one Detect generation consume the rings accumulated during boot. An
+    older Detect transaction settles first and any newer Watch generation
+    waits until adoption finishes. Register
     and Watch enumerate the same durable binding inventory, so a missed
     notification for rendering the portable reference cannot make either the
     reference row or its declared external

package/dist/user-ground-host.d.ts

@@ -154,6 +154,12 @@ export declare class UserGroundHost {
      * runs unchanged. */
     private readInlineCargo;
     private uploadSnapshotContent;
+    /** Download a set of immutable artifacts straight from content storage.
+     * The set is the primitive: every storage object the set still needs —
+     * manifests and blocks alike — shares one wide lane bound, so no artifact
+     * serializes behind another's plan or seal. Each object is still its own
+     * storage request. A lone artifact is a set of one. */
+    private downloadContentSet;
     private downloadContent;
     private ensureWatchers;
     private scheduleRescan;

package/dist/user-ground-host.js

@@ -3,7 +3,7 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
 import { open } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
-import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, download as downloadArtifact, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
+import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
 import Database from "better-sqlite3";
 import { atomicCopy, atomicWrite, ensurePrivateDir, ensureUserDir } from "./filesystem.js";
 import { emitFilesStage, } from "./files-observability.js";
@@ -27,13 +27,12 @@ import { GroundCoordinator } from "./ground-coordination.js";
 import { submitMutationOperation } from "./paged-mutation-submit.js";
 import { WireClient, WireRequestError } from "./wire-client.js";
 const SMALL_CONTENT_UPLOAD_CONCURRENCY = 16;
-/** Artifact lanes for a download phase. Direct storage fetches cost the
- * gateway nothing, so parallelism is bounded by this machine's pipe rather
- * than authority fairness. */
-const FILE_CONTENT_DOWNLOAD_CONCURRENCY = 16;
-/** Block lanes within one artifact. Most files are a single block; large
- * repository packs fan their chunks across this. */
-const DIRECT_BLOCK_DOWNLOAD_CONCURRENCY = 8;
+/** Storage-object lanes for one download set. Manifests and blocks are all
+ * independent content-addressed objects fetched straight from storage, so
+ * twelve thousand small files and one large pack share the same wide bound;
+ * direct fetches cost the gateway nothing, and memory stays bounded by
+ * lanes × one block. */
+const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = 128;
 const DETECT_QUIET_MS = 12;
 const DETECT_MAX_DEFERRAL_MS = 75;
 const DETECT_RETRY_MS = 250;
@@ -87,7 +86,14 @@ export class UserGroundHost {
         });
         this.watchHost = new NodeWatchHost({
             open: options.openWatch,
-            onSignal: () => this.scheduleRescan(),
+            // Watch establishes the native evidence boundary during convergence,
+            // before the durable Apply baseline has been adopted. Keep every ring
+            // in Watch, but do not let Detect consume that evidence until Apply has
+            // resumed any revealed intent and owns materialization truth.
+            onSignal: () => {
+                if (this.converged)
+                    this.scheduleRescan();
+            },
             onEvent: (evidence) => this.stage({
                 primitive: "watch",
                 stage: "native-suspicion",
@@ -164,6 +170,11 @@ export class UserGroundHost {
                 await this.startRecordRail(this.activeIdentity);
                 await this.startApplyRail(this.activeIdentity);
                 this.converged = true;
+                // Native rings collected while convergence was establishing Watch
+                // coverage are ordinary evidence, not a startup exception. They get
+                // one Detect generation now that Apply recovery is authoritative.
+                if (this.watchHost.hasPending)
+                    this.scheduleRescan();
                 return evidence;
             },
         };
@@ -288,7 +299,7 @@ export class UserGroundHost {
                         records: input.records,
                         destinationParent: input.destinationParent,
                         cloudHead: input.cloudHead,
-                        readContent: (artifact) => this.downloadContent(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifact, 10, { journey: "add", workspaceId: input.workspace.uuid }),
+                        readContent: (artifacts) => this.downloadContentSet(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifacts.map((artifact) => ({ artifact })), 10, { journey: "add", workspaceId: input.workspace.uuid }),
                         adoptMaterialization: (rootUUID) => {
                             const settled = this.applyHost?.adoptVerifiedMaterialization(rootUUID) ?? 0;
                             this.applyRail?.recordsAvailable();
@@ -414,7 +425,7 @@ export class UserGroundHost {
                 records: intent.records,
                 destinationParent: dirname(intent.destinationPath),
                 cloudHead: intent.cloudHead,
-                readContent: (artifact) => this.downloadContent(intent.resourceId, this.cacheDir(identity), artifact, 10, { journey: "add", workspaceId: intent.workspaceId }),
+                readContent: (artifacts) => this.downloadContentSet(intent.resourceId, this.cacheDir(identity), artifacts.map((artifact) => ({ artifact })), 10, { journey: "add", workspaceId: intent.workspaceId }),
                 adoptMaterialization: (rootUUID) => {
                     this.applyHost?.adoptVerifiedMaterialization(rootUUID);
                     this.applyRail?.recordsAvailable();
@@ -1050,7 +1061,7 @@ export class UserGroundHost {
                     database,
                     cacheDir,
                     cloud,
-                    readContent: (artifact) => this.downloadContent(cloud.resourceId, cacheDir, artifact),
+                    readContent: (artifacts) => this.downloadContentSet(cloud.resourceId, cacheDir, artifacts.map((artifact) => ({ artifact }))),
                 });
                 return localValue(identity, userRoot, database);
             },
@@ -1268,13 +1279,24 @@ export class UserGroundHost {
             throw error;
         }
     }
-    async downloadContent(resourceId, cacheDir, artifact, retryAttempts = 10, trace, knownManifest) {
+    /** Download a set of immutable artifacts straight from content storage.
+     * The set is the primitive: every storage object the set still needs —
+     * manifests and blocks alike — shares one wide lane bound, so no artifact
+     * serializes behind another's plan or seal. Each object is still its own
+     * storage request. A lone artifact is a set of one. */
+    async downloadContentSet(resourceId, cacheDir, wants, retryAttempts = 10, trace) {
+        const unique = new Map(wants.map((want) => [want.artifact.contentHash, want]));
+        const sole = unique.size === 1 ? [...unique.values()][0]?.artifact : undefined;
         const started = performance.now();
+        let declaredBytes = 0;
         let downloadedBytes = 0;
+        let downloadedChunks = 0;
+        let reusedChunks = 0;
         let requests = 0;
+        const entityId = trace?.entityId ?? sole?.entityId;
         const identifiers = {
             ...(trace?.workspaceId ? { workspaceId: trace.workspaceId } : {}),
-            entityId: trace?.entityId ?? artifact.entityId,
+            ...(entityId ? { entityId } : {}),
             ...(trace?.mutationId ? { mutationId: trace.mutationId } : {}),
             ...(trace?.globalSequence ? { globalSequence: trace.globalSequence } : {}),
         };
@@ -1283,9 +1305,19 @@ export class UserGroundHost {
             stage: "immutable-content",
             status: "started",
             ...identifiers,
-            measurements: { journey: trace?.journey ?? "bootstrap", kind: artifact.kind },
+            measurements: {
+                journey: trace?.journey ?? "bootstrap",
+                artifacts: unique.size,
+                ...(sole ? { kind: sole.kind } : {}),
+            },
         });
-        const cache = { target: null };
+        const targets = new Map();
+        const target = (artifact) => {
+            const found = targets.get(artifact.contentHash);
+            if (!found)
+                throw new Error("download cache has no content manifest");
+            return found;
+        };
         // Every storage object — the manifest and each block — is fetched
         // straight from content storage over a presigned URL and opened locally;
         // the wire carries only the key handoff and signing requests. A retry
@@ -1297,13 +1329,14 @@ export class UserGroundHost {
             return bytes;
         };
         try {
-            const receipt = await downloadArtifact(artifact, {
+            const receipts = await downloadAllArtifacts([...unique.values()].map((want) => want.artifact), {
                 sha256Hex,
-                getManifest: async () => {
+                getManifest: async (artifact) => {
                     // The record's plan or this machine's own manifest cache already
                     // proves chunk identity; downloaded bytes are still hash-verified,
                     // so a stale local manifest can only fail loudly, never corrupt.
-                    const local = knownManifest ?? readCachedManifest(cacheDir, artifact.contentHash);
+                    const local = unique.get(artifact.contentHash)?.manifest
+                        ?? readCachedManifest(cacheDir, artifact.contentHash);
                     const manifest = local ?? await (async () => {
                         const bytes = await fetchObject({ kind: "manifest", sha256: artifact.contentHash });
                         const checked = checkContentManifest(JSON.parse(bytes.toString("utf8")), artifact.contentHash);
@@ -1311,32 +1344,31 @@ export class UserGroundHost {
                             throw new Error(checked.error);
                         return checked.value;
                     })();
-                    cache.target = new ContentCacheDownload(cacheDir, artifact, manifest);
+                    targets.set(artifact.contentHash, new ContentCacheDownload(cacheDir, artifact, manifest));
                     return manifest;
                 },
-                hasChunk: async (_candidate, _manifest, index) => {
-                    if (!cache.target)
-                        throw new Error("download cache has no content manifest");
-                    return cache.target.hasChunk(index);
-                },
-                getChunk: async (_candidate, manifest, index) => {
+                hasChunk: async (artifact, _manifest, index) => target(artifact).hasChunk(index),
+                getChunk: async (_artifact, manifest, index) => {
                     const chunk = manifest.chunks[index];
                     if (!chunk)
                         throw new Error(`content manifest has no chunk ${index}`);
                     return fetchObject({ kind: "block", sha256: chunk.sha256 });
                 },
-                putChunk: async (_candidate, _manifest, index, bytes) => {
-                    if (!cache.target)
-                        throw new Error("download cache has no content manifest");
-                    await cache.target.putChunk(index, bytes);
+                putChunk: async (artifact, _manifest, index, bytes) => {
+                    await target(artifact).putChunk(index, bytes);
                 },
-                sealArtifact: async () => {
-                    if (!cache.target)
-                        throw new Error("download cache has no content manifest");
-                    const sealed = await cache.target.seal();
+                sealArtifact: async (artifact) => {
+                    const sealed = await target(artifact).seal();
                     return { local: sealed, bytes: sealed.bytes, contentHash: sealed.contentHash };
                 },
-            }, { concurrency: DIRECT_BLOCK_DOWNLOAD_CONCURRENCY });
+            }, { concurrency: DIRECT_OBJECT_DOWNLOAD_CONCURRENCY });
+            const content = new Map();
+            for (const receipt of receipts) {
+                declaredBytes += receipt.manifest.bytes;
+                downloadedChunks += receipt.downloadedChunks;
+                reusedChunks += receipt.reusedChunks;
+                content.set(receipt.artifact.contentHash, receipt.local);
+            }
             this.stage({
                 primitive: "download",
                 stage: "immutable-content",
@@ -1345,15 +1377,16 @@ export class UserGroundHost {
                 durationMs: performance.now() - started,
                 measurements: {
                     journey: trace?.journey ?? "bootstrap",
-                    kind: artifact.kind,
-                    declaredBytes: receipt.manifest.bytes,
+                    artifacts: unique.size,
+                    ...(sole ? { kind: sole.kind } : {}),
+                    declaredBytes,
                     downloadedBytes,
-                    downloadedChunks: receipt.downloadedChunks,
-                    reusedChunks: receipt.reusedChunks,
+                    downloadedChunks,
+                    reusedChunks,
                     requests,
                 },
             });
-            return receipt.local;
+            return content;
         }
         catch (error) {
             this.stage({
@@ -1362,14 +1395,26 @@ export class UserGroundHost {
                 status: "failed",
                 ...identifiers,
                 durationMs: performance.now() - started,
-                measurements: { journey: trace?.journey ?? "bootstrap", kind: artifact.kind, requests },
+                measurements: {
+                    journey: trace?.journey ?? "bootstrap",
+                    artifacts: unique.size,
+                    ...(sole ? { kind: sole.kind } : {}),
+                    requests,
+                },
             });
             throw error;
         }
         finally {
-            await cache.target?.close();
+            await Promise.all([...targets.values()].map((open) => open.close()));
         }
     }
+    async downloadContent(resourceId, cacheDir, artifact, retryAttempts = 10, trace, knownManifest) {
+        const content = await this.downloadContentSet(resourceId, cacheDir, [{ artifact, ...(knownManifest ? { manifest: knownManifest } : {}) }], retryAttempts, trace);
+        const local = content.get(artifact.contentHash);
+        if (!local)
+            throw new Error(`download set did not seal ${artifact.contentHash}`);
+        return local;
+    }
     ensureWatchers(identity, authoritativeBaselineRootIds = []) {
         if (this.closing)
             return this.watchHost.health();
@@ -4259,62 +4304,76 @@ async function materializeRecords(input) {
     const { records, rootPath, cacheDir, bindingDir, readContent } = input;
     const repositories = [];
     const repositoriesByUuid = new Map();
-    const fileArtifacts = new Map();
+    // One download set covers everything the wire owes this add up front:
+    // every ordinary file artifact and every repository transport head, side
+    // by side under one wide object bound. Only transport parents — links a
+    // chain reveals inside downloaded history — cost further rounds, and each
+    // round is again one set across every chain. Disk application below keeps
+    // its ordered, validated walk.
+    const wanted = new Map();
+    const repositoryHeads = [];
     for (const record of records) {
-        if (record.type !== "file.text" && record.type !== "file.binary" && record.type !== "link")
-            continue;
-        const artifact = artifactForRecord(record);
-        if (artifact)
-            fileArtifacts.set(artifact.contentHash, artifact);
-    }
-    const resolveRepositoryChain = async (record) => {
-        if (!record.payloadVersion || !record.transportVersion) {
-            throw new Error(`cloud repository ${record.uuid} has no state transport`);
+        if (record.type === "repo.git") {
+            if (!record.payloadVersion || !record.transportVersion) {
+                throw new Error(`cloud repository ${record.uuid} has no state transport`);
+            }
+            const artifact = artifactForRecord(record);
+            if (!artifact)
+                throw new Error(`cloud repository ${record.uuid} has no transfer artifact`);
+            wanted.set(artifact.contentHash, artifact);
+            repositoryHeads.push({ record, artifact });
         }
-        const artifact = artifactForRecord(record);
-        if (!artifact)
-            throw new Error(`cloud repository ${record.uuid} has no transfer artifact`);
-        const local = await readContent(artifact);
+        else if (["file.text", "file.binary", "link"].includes(record.type)) {
+            const artifact = artifactForRecord(record);
+            if (artifact)
+                wanted.set(artifact.contentHash, artifact);
+        }
+    }
+    const fileContent = await readContent([...wanted.values()]);
+    const repositoryChains = new Map();
+    const walking = repositoryHeads.map(({ record, artifact }) => {
+        const local = fileContent.get(artifact.contentHash);
+        if (!local)
+            throw new Error(`cloud repository ${record.uuid} transport was not downloaded`);
         if (local.contentHash !== record.transportVersion) {
             throw new Error(`cloud repository ${record.uuid} transport is corrupt`);
         }
-        const chain = [{ file: local.file, transportVersion: local.contentHash }];
-        const versions = new Set([record.transportVersion]);
-        let state = inspectRepositoryTransportFile(local.file);
-        while (state.parentTransportVersion) {
-            if (versions.has(state.parentTransportVersion)) {
-                throw new Error(`cloud repository ${record.uuid} transport chain has a cycle`);
-            }
-            const parentVersion = state.parentTransportVersion;
-            versions.add(parentVersion);
-            const parent = await readContent({ ...artifact, contentHash: parentVersion });
-            if (parent.contentHash !== parentVersion) {
-                throw new Error(`cloud repository ${record.uuid} parent transport is corrupt`);
-            }
-            chain.unshift({ file: parent.file, transportVersion: parent.contentHash });
-            state = inspectRepositoryTransportFile(parent.file);
-        }
         const latest = inspectRepositoryTransportFile(local.file);
         if (latest.stateId !== record.payloadVersion) {
             throw new Error(`cloud repository ${record.uuid} transport does not name its declared state`);
         }
-        return chain;
-    };
-    // One download phase covers everything the wire owes this add: ordinary
-    // file artifacts and every repository transport chain, side by side under
-    // one bound. A chain's own links stay serial (each names its parent), but
-    // no chain waits behind the file wave or another repository's chain.
-    // Disk application below keeps its ordered, validated walk.
-    const fileContent = new Map();
-    const repositoryChains = new Map();
-    await boundedForEach([
-        ...[...fileArtifacts.values()].map((artifact) => async () => {
-            fileContent.set(artifact.contentHash, await readContent(artifact));
-        }),
-        ...records.filter((record) => record.type === "repo.git").map((record) => async () => {
-            repositoryChains.set(record.uuid, await resolveRepositoryChain(record));
-        }),
-    ], FILE_CONTENT_DOWNLOAD_CONCURRENCY, (download) => download());
+        const chain = [{ file: local.file, transportVersion: local.contentHash }];
+        repositoryChains.set(record.uuid, chain);
+        return {
+            record,
+            artifact,
+            chain,
+            versions: new Set([local.contentHash]),
+            parent: latest.parentTransportVersion,
+        };
+    });
+    for (let frontier = walking.filter((walk) => walk.parent !== null); frontier.length > 0; frontier = frontier.filter((walk) => walk.parent !== null)) {
+        const parents = await readContent(frontier.map((walk) => {
+            const parent = walk.parent;
+            if (walk.versions.has(parent)) {
+                throw new Error(`cloud repository ${walk.record.uuid} transport chain has a cycle`);
+            }
+            return { ...walk.artifact, contentHash: parent };
+        }));
+        for (const walk of frontier) {
+            const wantedParent = walk.parent;
+            const parent = parents.get(wantedParent);
+            if (!parent) {
+                throw new Error(`cloud repository ${walk.record.uuid} parent transport was not downloaded`);
+            }
+            if (parent.contentHash !== wantedParent) {
+                throw new Error(`cloud repository ${walk.record.uuid} parent transport is corrupt`);
+            }
+            walk.versions.add(parent.contentHash);
+            walk.chain.unshift({ file: parent.file, transportVersion: parent.contentHash });
+            walk.parent = inspectRepositoryTransportFile(parent.file).parentTransportVersion;
+        }
+    }
     const materializeRepository = async (record, destination) => {
         const chain = repositoryChains.get(record.uuid);
         if (!chain)