@amalgm/shell 0.1.17 → 0.1.19

package/dist/user-ground-host.js

@@ -1,11 +1,14 @@
 import { createHash, randomUUID } from "node:crypto";
-import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlinkSync, rmSync, statSync, symlinkSync, watch, writeFileSync, } from "node:fs";
+import { createReadStream, existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlinkSync, rmSync, statSync, symlinkSync, watch, writeFileSync, } from "node:fs";
+import { open } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
-import { CHUNK_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, manifestFromChunks, membershipHash, parseSnapshot, privateEntityResourceId, sameRecords, snapshotFromRecords, stableJson, travelingRecords, unpack, userGroundRecords, } from "@amalgm/live";
+import { CHUNK_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, convergeUserGround, createSuspicionScope, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, download as downloadArtifact, membershipHash, parseSnapshot, pathIsSuspect, privateEntityResourceId, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
 import Database from "better-sqlite3";
-import { atomicWrite, ensurePrivateDir } from "./filesystem.js";
-import { applyRepository, captureRepository, isRepository, } from "./git-repository-host.js";
+import { atomicAssemble, atomicCopy, atomicWrite, ensurePrivateDir } from "./filesystem.js";
+import { applyRepositoryFiles, captureRepository, inspectRepositoryTransportFile, isRepository, } from "./git-repository-host.js";
+import { inspectGitRegistration, } from "./git-registration-host.js";
+import { projectMaterializedGraph } from "./materialized-graph.js";
 import { WireClient, WireRequestError } from "./wire-client.js";
 const PORTABLE_FIELDS = [
     "uuid", "type", "parentUUID", "name", "status", "version",
@@ -24,6 +27,7 @@ export class UserGroundHost {
     cloudState = null;
     converged = false;
     syncing = null;
+    closing = false;
     port;
     constructor(options) {
         this.options = options;
@@ -164,7 +168,7 @@ export class UserGroundHost {
                 workspace: input.workspace,
                 records: input.records,
                 destinationParent: input.destinationParent,
-                readContent: (contentHash) => this.downloadContent(privateEntityResourceId(identity.userId, sha256Hex), contentHash),
+                readContent: (artifact) => this.downloadContent(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifact),
             }),
             coverWorkspace: async () => {
                 this.ensureWatchers(identity);
@@ -178,8 +182,16 @@ export class UserGroundHost {
     async flush() {
         if (!this.converged || !this.activeIdentity || !this.cloudState)
             return;
-        this.watchDirty = false;
+        const identity = this.activeIdentity;
+        for (const watcher of this.watchers.values())
+            watcher.suspicion.suspectAll();
+        this.watchDirty = true;
         await this.syncNow();
+        if (!this.closing)
+            this.ensureWatchers(identity);
+        await this.flushObservedChanges();
+        if (!this.closing)
+            this.ensureWatchers(identity);
         await this.flushObservedChanges();
     }
     async syncNow() {
@@ -193,14 +205,20 @@ export class UserGroundHost {
         await this.syncing;
     }
     async flushObservedChanges() {
-        do {
-            const dirty = this.watchDirty;
+        while (true) {
+            const dirty = this.watchDirty
+                || [...this.watchers.values()].some((watcher) => watcher.suspicion.size > 0);
             this.watchDirty = false;
-            if (dirty)
+            if (dirty) {
                 await this.syncNow();
-            else if (this.syncing)
+                continue;
+            }
+            if (this.syncing) {
                 await this.syncing;
-        } while (this.watchDirty);
+                continue;
+            }
+            return;
+        }
     }
     async activateRuntimeTunnel(gatewayPort, runtimeToken) {
         await this.flush();
@@ -212,12 +230,18 @@ export class UserGroundHost {
         await this.wire.close();
     }
     async close() {
+        // Command shutdown is the last catch-up boundary. A filesystem callback
+        // may still be queued, so watcher silence cannot be used as evidence yet.
+        for (const watcher of this.watchers.values())
+            watcher.suspicion.suspectAll();
+        this.watchDirty = this.watchers.size > 0 || this.watchDirty;
+        this.closing = true;
         if (this.rescanTimer)
             clearTimeout(this.rescanTimer);
         this.rescanTimer = null;
         await this.flushObservedChanges();
         for (const watcher of this.watchers.values())
-            watcher.close();
+            watcher.handle.close();
         this.watchers.clear();
         if (this.rescanTimer)
             clearTimeout(this.rescanTimer);
@@ -296,7 +320,7 @@ export class UserGroundHost {
                     database,
                     cacheDir,
                     cloud,
-                    readContent: (contentHash) => this.downloadContent(cloud.resourceId, contentHash),
+                    readContent: (artifact) => this.downloadContent(cloud.resourceId, cacheDir, artifact),
                 });
                 return localValue(identity, userRoot, database);
             },
@@ -308,113 +332,158 @@ export class UserGroundHost {
     }
     async uploadSnapshotContent(identity, resourceId, records) {
         const cache = this.cacheDir(identity);
-        const contentHeads = [...new Set(records
-                .map((record) => record.type === "repo.git" ? record.transportVersion
-                : ["file.text", "file.binary", "link"].includes(record.type) ? record.payloadVersion : null)
-                .filter((head) => typeof head === "string"))];
-        const small = contentHeads.filter((contentHash) => statSync(join(cache, `${contentHash}.bin`)).size <= CHUNK_BYTES);
-        const large = contentHeads.filter((contentHash) => statSync(join(cache, `${contentHash}.bin`)).size > CHUNK_BYTES);
-        const upload = async (contentHash) => {
-            const bytes = readFileSync(join(cache, `${contentHash}.bin`));
-            if (sha256Hex(bytes) !== contentHash)
-                throw new Error(`local content ${contentHash} is corrupt`);
-            const chunks = [];
-            for (let offset = 0; offset < bytes.length; offset += CHUNK_BYTES) {
-                const part = bytes.subarray(offset, Math.min(offset + CHUNK_BYTES, bytes.length));
-                chunks.push({ sha256: sha256Hex(part), bytes: part.length, data: part });
+        const byHash = new Map();
+        for (const record of records) {
+            const artifact = artifactForRecord(record);
+            if (artifact)
+                byHash.set(artifact.contentHash, artifact);
+        }
+        const artifacts = [...byHash.values()];
+        const small = artifacts.filter((artifact) => statSync(contentCacheFile(cache, artifact.contentHash)).size <= CHUNK_BYTES);
+        const large = artifacts.filter((artifact) => statSync(contentCacheFile(cache, artifact.contentHash)).size > CHUNK_BYTES);
+        const transfer = async (artifact) => {
+            const file = contentCacheFile(cache, artifact.contentHash);
+            const fileBytes = statSync(file).size;
+            const cachedManifest = readCachedManifest(cache, artifact.contentHash);
+            const sourceFile = { handle: null };
+            let manifestPresent = false;
+            try {
+                const receipt = await uploadArtifact(artifact, {
+                    bytes: fileBytes,
+                    ...(cachedManifest ? { manifest: cachedManifest } : {}),
+                    verify: async (contentHash) => await sha256File(file) === contentHash,
+                    read: async (offset, bytes) => {
+                        sourceFile.handle ??= await open(file, "r");
+                        const buffer = Buffer.allocUnsafe(bytes);
+                        const { bytesRead } = await sourceFile.handle.read(buffer, 0, bytes, offset);
+                        return buffer.subarray(0, bytesRead);
+                    },
+                }, {
+                    sha256Hex,
+                    missingChunks: async (_candidate, manifest) => {
+                        const frame = await retryContent(() => this.wire.request({
+                            type: "private.entity-content.inventory",
+                            resource_id: resourceId,
+                            content_hash: artifact.contentHash,
+                            chunks: manifest.chunks,
+                        }, ["private.entity-content.inventory-result"]));
+                        manifestPresent = frame.complete === true;
+                        if (!Array.isArray(frame.missing)) {
+                            throw new Error("content authority returned an invalid inventory");
+                        }
+                        return frame.missing;
+                    },
+                    putChunk: async (_candidate, manifest, index, bytes) => {
+                        const chunk = manifest.chunks[index];
+                        if (!chunk)
+                            throw new Error(`content manifest has no chunk ${index}`);
+                        await retryContent(() => this.wire.requestBinary({
+                            type: "private.entity-content.put",
+                            resource_id: resourceId,
+                            content_hash: artifact.contentHash,
+                            kind: "chunk",
+                            part_index: index,
+                            part_count: manifest.chunks.length,
+                            sha256: chunk.sha256,
+                            bytes: chunk.bytes,
+                        }, bytes, ["private.entity-content.stored"]));
+                    },
+                    putManifest: async (_candidate, manifest) => {
+                        if (manifestPresent)
+                            return;
+                        const bytes = Buffer.from(stableJson(manifest), "utf8");
+                        await retryContent(() => this.wire.requestBinary({
+                            type: "private.entity-content.put",
+                            resource_id: resourceId,
+                            content_hash: artifact.contentHash,
+                            kind: "manifest",
+                            part_index: manifest.chunks.length,
+                            part_count: manifest.chunks.length,
+                            sha256: sha256Hex(bytes),
+                            bytes: bytes.length,
+                        }, bytes, ["private.entity-content.stored"]));
+                    },
+                });
+                atomicWrite(manifestCacheFile(cache, artifact.contentHash), stableJson(receipt.manifest));
             }
-            const checked = manifestFromChunks(contentHash, chunks);
-            if (!checked.ok)
-                throw new Error(checked.error);
-            for (let index = 0; index < chunks.length; index += 1) {
-                const chunk = chunks[index];
-                await retryContent(() => this.wire.request({
-                    type: "private.entity-content.put",
-                    resource_id: resourceId,
-                    content_hash: contentHash,
-                    kind: "chunk",
-                    part_index: index,
-                    part_count: chunks.length,
-                    sha256: chunk.sha256,
-                    bytes: chunk.bytes,
-                    data_b64: chunk.data.toString("base64"),
-                }, ["private.entity-content.stored"]));
+            finally {
+                if (sourceFile.handle)
+                    await sourceFile.handle.close();
             }
-            const manifestBytes = Buffer.from(stableJson(checked.value), "utf8");
-            await retryContent(() => this.wire.request({
-                type: "private.entity-content.put",
-                resource_id: resourceId,
-                content_hash: contentHash,
-                kind: "manifest",
-                part_index: chunks.length,
-                part_count: chunks.length,
-                sha256: sha256Hex(manifestBytes),
-                bytes: manifestBytes.length,
-                data_b64: manifestBytes.toString("base64"),
-            }, ["private.entity-content.stored"]));
         };
-        await boundedForEach(small, SMALL_CONTENT_UPLOAD_CONCURRENCY, upload);
-        for (const contentHash of large) {
-            await upload(contentHash);
+        await boundedForEach(small, SMALL_CONTENT_UPLOAD_CONCURRENCY, transfer);
+        for (const artifact of large) {
+            await transfer(artifact);
         }
     }
-    async downloadContent(resourceId, contentHash) {
-        const manifestFrame = await retryContent(() => this.wire.request({
-            type: "private.entity-content.get",
-            resource_id: resourceId,
-            content_hash: contentHash,
-            kind: "manifest",
-            part_index: 0,
-        }, ["private.entity-content.data"]));
-        const manifestBytes = frameBytes(manifestFrame);
-        const checked = checkContentManifest(JSON.parse(manifestBytes.toString("utf8")), contentHash);
-        if (!checked.ok)
-            throw new Error(checked.error);
-        const parts = [];
-        for (let index = 0; index < checked.value.chunks.length; index += 1) {
-            const chunk = checked.value.chunks[index];
-            const frame = await retryContent(() => this.wire.request({
-                type: "private.entity-content.get",
-                resource_id: resourceId,
-                content_hash: contentHash,
-                kind: "chunk",
-                part_index: index,
-                sha256: chunk.sha256,
-            }, ["private.entity-content.data"]), true);
-            const bytes = frameBytes(frame);
-            if (bytes.length !== chunk.bytes || sha256Hex(bytes) !== chunk.sha256) {
-                throw new Error(`cloud content chunk ${index} is corrupt`);
-            }
-            parts.push(bytes);
-        }
-        const bytes = Buffer.concat(parts);
-        if (bytes.length !== checked.value.bytes || sha256Hex(bytes) !== contentHash) {
-            throw new Error(`cloud content ${contentHash} is corrupt`);
-        }
-        return bytes;
+    async downloadContent(resourceId, cacheDir, artifact) {
+        const receipt = await downloadArtifact(artifact, {
+            sha256Hex,
+            getManifest: async () => {
+                const frame = await retryContent(() => this.wire.request({
+                    type: "private.entity-content.get",
+                    resource_id: resourceId,
+                    content_hash: artifact.contentHash,
+                    kind: "manifest",
+                    part_index: 0,
+                }, ["private.entity-content.data"]));
+                return JSON.parse(frameBytes(frame).toString("utf8"));
+            },
+            hasChunk: async (chunk) => {
+                try {
+                    const bytes = readFileSync(chunkCacheFile(cacheDir, chunk.sha256));
+                    return bytes.length === chunk.bytes && sha256Hex(bytes) === chunk.sha256;
+                }
+                catch {
+                    return false;
+                }
+            },
+            getChunk: async (_candidate, manifest, index) => {
+                const chunk = manifest.chunks[index];
+                if (!chunk)
+                    throw new Error(`content manifest has no chunk ${index}`);
+                const frame = await retryContent(() => this.wire.request({
+                    type: "private.entity-content.get",
+                    resource_id: resourceId,
+                    content_hash: artifact.contentHash,
+                    kind: "chunk",
+                    part_index: index,
+                    sha256: chunk.sha256,
+                }, ["private.entity-content.data"]), true);
+                return frameBytes(frame);
+            },
+            putChunk: async (chunk, bytes) => {
+                atomicWrite(chunkCacheFile(cacheDir, chunk.sha256), bytes);
+            },
+            sealArtifact: async (_candidate, manifest) => {
+                const file = contentCacheFile(cacheDir, artifact.contentHash);
+                const sealed = await atomicAssemble(file, manifest.chunks.map((chunk) => chunkCacheFile(cacheDir, chunk.sha256)));
+                if (sealed.bytes !== manifest.bytes || sealed.sha256 !== manifest.contentHash) {
+                    throw new Error(`cloud content ${artifact.contentHash} is corrupt`);
+                }
+                atomicWrite(manifestCacheFile(cacheDir, artifact.contentHash), stableJson(manifest));
+                return {
+                    local: { file, bytes: sealed.bytes, contentHash: sealed.sha256 },
+                    bytes: sealed.bytes,
+                    contentHash: sealed.sha256,
+                };
+            },
+        });
+        return receipt.local;
     }
     ensureWatchers(identity) {
+        if (this.closing)
+            return;
         const root = this.userRoot(identity);
         const ignoreFile = join(root, ".amalgmignore");
         const policy = createUserGroundEnrollmentPolicy(existsSync(ignoreFile) ? readFileSync(ignoreFile, "utf8") : "");
-        const wanted = new Set();
-        const visit = (directory, enrollment, base = "") => {
-            wanted.add(directory);
-            for (const entry of readdirSync(directory, { withFileTypes: true })) {
-                const rel = base ? `${base}/${entry.name}` : entry.name;
-                if (!entry.isDirectory() || entry.isSymbolicLink() || !enrollment(rel))
-                    continue;
-                const child = join(directory, entry.name);
-                visit(child, enrollment, rel);
-            }
-        };
-        visit(root, policy);
+        const wanted = new Map([[root, policy]]);
         for (const row of readRows(this.databasePath(identity))) {
             if (row.parentUUID !== null || row.absolutePath === root)
                 continue;
             try {
                 if (statSync(row.absolutePath).isDirectory())
-                    visit(row.absolutePath, () => true);
+                    wanted.set(row.absolutePath, () => true);
             }
             catch {
                 // Unresolved ground keeps its rows but cannot hold a watcher handle.
@@ -422,35 +491,62 @@ export class UserGroundHost {
         }
         for (const [directory, watcher] of this.watchers) {
             if (!wanted.has(directory)) {
-                watcher.close();
+                watcher.handle.close();
                 this.watchers.delete(directory);
             }
         }
-        for (const directory of wanted) {
+        for (const [directory, enrollment] of wanted) {
             if (this.watchers.has(directory))
                 continue;
-            const watcher = watch(directory, (_event, file) => {
-                if (String(file || "").split(/[\\/]/).includes(".amalgm"))
+            const suspicion = createSuspicionScope();
+            const handle = watch(directory, { recursive: true }, (_event, file) => {
+                const relativePath = String(file || "").split(sep).join("/");
+                if (relativePath.split("/").includes(".amalgm"))
                     return;
+                if (relativePath && !enrollment(relativePath))
+                    return;
+                const segments = relativePath.split("/");
+                const git = segments.indexOf(".git");
+                if (!relativePath || relativePath === ".amalgmignore"
+                    || (git === 0 && segments[1] === "index")) {
+                    suspicion.suspectAll();
+                }
+                else if (git > 0 && segments[git + 1] === "index") {
+                    suspicion.ring(segments.slice(0, git).join("/"));
+                }
+                else {
+                    suspicion.ring(relativePath);
+                }
                 this.scheduleRescan();
             });
-            watcher.on("error", () => {
-                watcher.close();
+            handle.on("error", () => {
+                suspicion.suspectAll();
+                handle.close();
                 this.watchers.delete(directory);
+                this.scheduleRescan();
             });
-            this.watchers.set(directory, watcher);
+            this.watchers.set(directory, { handle, suspicion });
+            // Coverage began after the preceding observation. One catch-up proves
+            // the unobserved interval before watcher silence can be trusted.
+            suspicion.suspectAll();
+            this.scheduleRescan();
         }
     }
     scheduleRescan() {
+        if (this.closing)
+            return;
         this.watchDirty = true;
         if (this.rescanTimer)
             clearTimeout(this.rescanTimer);
         this.rescanTimer = setTimeout(() => {
             this.rescanTimer = null;
             const identity = this.activeIdentity;
-            if (!identity)
+            if (!identity || this.closing)
                 return;
-            void this.flushObservedChanges().then(() => this.ensureWatchers(identity)).catch(() => {
+            void this.flushObservedChanges().then(() => {
+                if (!this.closing)
+                    this.ensureWatchers(identity);
+            }).catch(() => {
                 // The durable outbox remains claimable by the next event or resume.
             });
         }, 150);
@@ -461,16 +557,25 @@ export class UserGroundHost {
         const state = this.cloudState;
         if (!state)
             throw new Error("cloud state is unavailable for user-ground Watch");
+        const observations = new Map();
+        for (const [directory, watcher] of this.watchers) {
+            observations.set(directory, watcher.suspicion.snapshot());
+        }
         const localRecords = scanAndRegister({
             identity,
             userRoot: this.userRoot(identity),
             database: this.databasePath(identity),
             cacheDir: this.cacheDir(identity),
+            suspicions: new Map([...observations].map(([directory, snapshot]) => [directory, snapshot.paths])),
         });
         const snapshot = snapshotFromRecords(travelingRecords(mergeMaterializedRoots(state.records, localRecords)));
         const checksum = sha256Hex(stableJson(snapshot));
-        if (checksum === state.checksum)
+        if (checksum === state.checksum) {
+            for (const [directory, snapshot] of observations) {
+                this.watchers.get(directory)?.suspicion.settle(snapshot);
+            }
             return;
+        }
         const database = initializeDatabase(this.databasePath(identity));
         try {
             database.prepare(`
@@ -484,6 +589,9 @@ export class UserGroundHost {
             database.close();
         }
         await this.drainOutbox(identity);
+        for (const [directory, snapshot] of observations) {
+            this.watchers.get(directory)?.suspicion.settle(snapshot);
+        }
     }
     async drainOutboxBeforeLookup(identity, resourceId) {
         const pending = readOutbox(this.databasePath(identity));
@@ -532,6 +640,30 @@ export class UserGroundHost {
     }
 }
 const sha256Hex = (input) => createHash("sha256").update(input).digest("hex");
+async function sha256File(file) {
+    const hash = createHash("sha256");
+    for await (const chunk of createReadStream(file))
+        hash.update(chunk);
+    return hash.digest("hex");
+}
+function contentCacheFile(cacheDir, contentHash) {
+    return join(cacheDir, `${contentHash}.bin`);
+}
+function manifestCacheFile(cacheDir, contentHash) {
+    return join(cacheDir, "manifests", `${contentHash}.json`);
+}
+function chunkCacheFile(cacheDir, chunkHash) {
+    return join(cacheDir, "chunks", `${chunkHash}.bin`);
+}
+function readCachedManifest(cacheDir, contentHash) {
+    try {
+        const checked = checkContentManifest(JSON.parse(readFileSync(manifestCacheFile(cacheDir, contentHash), "utf8")), contentHash);
+        return checked.ok ? checked.value : null;
+    }
+    catch {
+        return null;
+    }
+}
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
 function workspaceBindingDir(userRoot, deviceId) {
     return join(liveMachineStateDir(userRoot, deviceId), "bindings");
@@ -693,10 +825,13 @@ function deleteOutbox(file, mutationId) {
         database.close();
     }
 }
+function portableRecord(row) {
+    return Object.fromEntries(PORTABLE_FIELDS.map((field) => [field, row[field]]));
+}
 function portableRecords(file) {
-    return readRows(file).map((row) => Object.fromEntries(PORTABLE_FIELDS.map((field) => [field, row[field]])));
+    return readRows(file).map(portableRecord);
 }
-function persistRows(file, identity, resourceId, rootUUID, rows) {
+function persistRows(file, identity, resourceId, rootUUID, rows, previousRows = []) {
     const database = initializeDatabase(file);
     try {
         const replaceIdentity = database.prepare(`
@@ -707,18 +842,54 @@ function persistRows(file, identity, resourceId, rootUUID, rows) {
         user_email = excluded.user_email,
         device_id = excluded.device_id
     `);
-        const insert = database.prepare(`
+        const upsert = database.prepare(`
       INSERT INTO entities(
         uuid, resource_id, root_uuid, type, parent_uuid, name, status, version, payload_version,
         transport_version, relative_path, absolute_path, device_number, inode
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(uuid) DO UPDATE SET
+        resource_id = excluded.resource_id,
+        root_uuid = excluded.root_uuid,
+        type = excluded.type,
+        parent_uuid = excluded.parent_uuid,
+        name = excluded.name,
+        status = excluded.status,
+        version = excluded.version,
+        payload_version = excluded.payload_version,
+        transport_version = excluded.transport_version,
+        relative_path = excluded.relative_path,
+        absolute_path = excluded.absolute_path,
+        device_number = excluded.device_number,
+        inode = excluded.inode
     `);
+        const remove = database.prepare("DELETE FROM entities WHERE uuid = ? AND resource_id = ? AND root_uuid = ?");
+        const previousByUuid = new Map(previousRows.map((row) => [row.uuid, row]));
+        const currentUuids = new Set(rows.map((row) => row.record.uuid));
+        const changed = rows.filter((row) => {
+            const old = previousByUuid.get(row.record.uuid);
+            return !old
+                || old.resourceId !== resourceId
+                || old.rootUUID !== rootUUID
+                || old.type !== row.record.type
+                || old.parentUUID !== row.record.parentUUID
+                || old.name !== row.record.name
+                || old.status !== row.record.status
+                || old.version !== row.record.version
+                || old.payloadVersion !== row.record.payloadVersion
+                || old.transportVersion !== row.record.transportVersion
+                || old.relativePath !== row.relativePath
+                || old.absolutePath !== row.absolutePath
+                || old.deviceNumber !== row.deviceNumber
+                || old.inode !== row.inode;
+        });
         const replace = database.transaction(() => {
             replaceIdentity.run(identity.userId, identity.userEmail, identity.deviceId);
-            database.prepare("DELETE FROM entities WHERE resource_id = ? AND root_uuid = ?")
-                .run(resourceId, rootUUID);
-            for (const row of rows)
-                insert.run(row.record.uuid, resourceId, rootUUID, row.record.type, row.record.parentUUID, row.record.name, row.record.status, row.record.version, row.record.payloadVersion, row.record.transportVersion, row.relativePath, row.absolutePath, row.deviceNumber, row.inode);
+            for (const old of previousRows) {
+                if (!currentUuids.has(old.uuid))
+                    remove.run(old.uuid, resourceId, rootUUID);
+            }
+            for (const row of changed)
+                upsert.run(row.record.uuid, resourceId, rootUUID, row.record.type, row.record.parentUUID, row.record.name, row.record.status, row.record.version, row.record.payloadVersion, row.record.transportVersion, row.relativePath, row.absolutePath, row.deviceNumber, row.inode);
         });
         replace();
     }
@@ -788,12 +959,17 @@ function slashPath(path) {
     return path.split(sep).join("/");
 }
 function scanAndRegister(input) {
-    const { identity, userRoot, database, cacheDir } = input;
+    const { identity, userRoot, database, cacheDir, suspicions } = input;
     const resourceId = privateEntityResourceId(identity.userId, sha256Hex);
     const existing = readRows(database);
     const ignoreFile = join(userRoot, ".amalgmignore");
     const policy = createUserGroundEnrollmentPolicy(existsSync(ignoreFile) ? readFileSync(ignoreFile, "utf8") : "");
     const bindingDir = workspaceBindingDir(userRoot, identity.deviceId);
+    const suspicionFor = (root) => {
+        if (!suspicions || !suspicions.has(root))
+            return suspicions ? [] : null;
+        return suspicions.get(root);
+    };
     const existingCore = existing.find((row) => row.parentUUID === null && row.absolutePath === userRoot && row.type === "workspace");
     const coreUUID = existingCore?.uuid || randomUUID();
     const core = scanRoot({
@@ -807,9 +983,11 @@ function scanAndRegister(input) {
         cacheDir,
         policy,
         bindingDir,
+        suspects: suspicionFor(userRoot),
+        existingRows: existing.filter((row) => row.rootUUID === coreUUID),
     });
     const targetRoots = new Set(core.referenceWorkspaceIds);
-    for (const row of readRows(database)) {
+    for (const row of existing) {
         if (row.parentUUID === null && row.uuid !== coreUUID)
             targetRoots.add(row.uuid);
     }
@@ -827,7 +1005,8 @@ function scanAndRegister(input) {
             // so a move can be rescued; absence is never permission to delete it.
             continue;
         }
-        const existingRoot = readRows(database).find((row) => row.uuid === workspaceId && row.parentUUID === null);
+        const existingRoot = existing.find((row) => row.uuid === workspaceId && row.parentUUID === null);
+        const existingWorkspaceRows = existing.filter((row) => row.rootUUID === workspaceId);
         scanRoot({
             identity,
             resourceId,
@@ -839,15 +1018,32 @@ function scanAndRegister(input) {
             cacheDir,
             policy: () => true,
             bindingDir,
+            suspects: existingWorkspaceRows.length === 0 ? null : suspicionFor(rootPath),
+            existingRows: existingWorkspaceRows,
         });
     }
     return portableRecords(database);
 }
 function scanRoot(input) {
-    const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, seedByPath = new Map(), repositoryHeads = new Map(), persist = true, } = input;
-    const existingByPath = new Map(readRows(database)
-        .filter((row) => row.resourceId === resourceId && row.rootUUID === rootUUID)
-        .map((row) => [row.relativePath, row]));
+    const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, } = input;
+    const existingRows = (input.existingRows ?? readRows(database))
+        .filter((row) => row.resourceId === resourceId && row.rootUUID === rootUUID);
+    const existingByPath = new Map(existingRows.map((row) => [row.relativePath, row]));
+    if (suspects !== null && suspects.length === 0 && existingRows.length > 0) {
+        return {
+            records: existingRows.map(portableRecord),
+            rows: existingRows.map((row) => ({
+                record: portableRecord(row),
+                relativePath: row.relativePath,
+                absolutePath: row.absolutePath,
+                deviceNumber: row.deviceNumber,
+                inode: row.inode,
+            })),
+            referenceWorkspaceIds: existingRows
+                .filter((row) => row.type === "reference" && row.payloadVersion && UUID.test(row.payloadVersion))
+                .map((row) => row.payloadVersion),
+        };
+    }
     const entries = [];
     const referenceWorkspaceIds = new Set();
     const rootStats = lstatSync(rootPath);
@@ -863,7 +1059,33 @@ function scanRoot(input) {
         deviceNumber: rootStats.dev,
         inode: rootStats.ino,
     });
-    const visit = (directory, parentUUID, parentType, base = "") => {
+    const touchesSuspicion = (path) => suspects === null
+        || pathIsSuspect(path, suspects)
+        || suspects.some((suspect) => suspect.startsWith(`${path}/`));
+    const repositoryEvidencePaths = (repository) => {
+        if (suspects === null)
+            return null;
+        const prefix = slashPath(relative(rootPath, repository));
+        const paths = [];
+        for (const suspect of suspects) {
+            if (suspect === prefix || (prefix && prefix.startsWith(`${suspect}/`)))
+                return null;
+            const local = prefix
+                ? suspect.startsWith(`${prefix}/`) ? suspect.slice(prefix.length + 1) : null
+                : suspect;
+            if (local === null)
+                continue;
+            if (local === ".git/index" || local.startsWith(".git/index/"))
+                return null;
+            if (local === ".git" || local.startsWith(".git/"))
+                continue;
+            paths.push(local);
+        }
+        return [...new Set(paths)].sort();
+    };
+    const visit = (directory, parentUUID, parentType, base = "", activeRepository = parentType === "repo.git"
+        ? { root: directory, evidencePaths: repositoryEvidencePaths(directory), evidence: null }
+        : null) => {
         const children = readdirSync(directory, { withFileTypes: true })
             .sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
         for (const child of children) {
@@ -875,11 +1097,27 @@ function scanRoot(input) {
             const absolutePath = join(directory, child.name);
             const stats = lstatSync(absolutePath);
             const existing = existingByPath.get(relativePath);
-            const seeded = seedByPath.get(relativePath);
-            const uuid = existing?.uuid || seeded?.uuid || randomUUID();
+            const uuid = existing?.uuid || randomUUID();
+            const observe = existing === undefined || touchesSuspicion(relativePath);
+            const repositoryPath = activeRepository
+                ? slashPath(relative(activeRepository.root, absolutePath))
+                : null;
+            if (observe && repositoryPath && activeRepository && !activeRepository.evidence) {
+                activeRepository.evidence = inspectGitRegistration(activeRepository.root, activeRepository.evidencePaths);
+            }
+            const indexed = observe && repositoryPath
+                ? activeRepository?.evidence?.cleanLeaves.get(repositoryPath)
+                : undefined;
             let type;
             let payloadVersion = null;
-            if (stats.isSymbolicLink()) {
+            if (!observe && existing) {
+                type = existing.type;
+                payloadVersion = existing.payloadVersion;
+                if (type === "reference" && payloadVersion && UUID.test(payloadVersion)) {
+                    referenceWorkspaceIds.add(payloadVersion);
+                }
+            }
+            else if (stats.isSymbolicLink()) {
                 const referenceId = referenceWorkspaceId(absolutePath, bindingDir);
                 if (referenceId) {
                     type = "reference";
@@ -888,28 +1126,36 @@ function scanRoot(input) {
                 }
                 else {
                     type = "link";
-                    const bytes = Buffer.from(readlinkSync(absolutePath), "utf8");
-                    payloadVersion = sha256Hex(bytes);
-                    immutableWrite(join(cacheDir, `${payloadVersion}.bin`), bytes);
+                    if (indexed?.type === "link") {
+                        payloadVersion = indexed.payloadVersion;
+                    }
+                    else {
+                        const bytes = Buffer.from(readlinkSync(absolutePath), "utf8");
+                        payloadVersion = sha256Hex(bytes);
+                        if (!activeRepository)
+                            immutableWrite(join(cacheDir, `${payloadVersion}.bin`), bytes);
+                    }
                 }
             }
             else if (stats.isDirectory()) {
                 type = classifyDirectory({ repository: isRepository(absolutePath) });
             }
             else if (stats.isFile()) {
-                const bytes = readFileSync(absolutePath);
-                type = fileType(bytes);
-                payloadVersion = sha256Hex(bytes);
-                immutableWrite(join(cacheDir, `${payloadVersion}.bin`), bytes);
+                if (indexed && indexed.type !== "link") {
+                    type = indexed.type;
+                    payloadVersion = indexed.payloadVersion;
+                }
+                else {
+                    const bytes = readFileSync(absolutePath);
+                    type = fileType(bytes);
+                    payloadVersion = sha256Hex(bytes);
+                    if (!activeRepository)
+                        immutableWrite(join(cacheDir, `${payloadVersion}.bin`), bytes);
+                }
             }
             else {
                 continue;
             }
-            if (seeded && seeded.type !== type
-                && !(["file.text", "file.binary"].includes(seeded.type)
-                    && ["file.text", "file.binary"].includes(type))) {
-                throw new Error(`materialized entity ${relativePath} is ${type}, expected ${seeded.type}`);
-            }
             entries.push({
                 uuid,
                 type,
@@ -923,7 +1169,13 @@ function scanRoot(input) {
                 inode: stats.ino,
             });
             if (stats.isDirectory() && !stats.isSymbolicLink()) {
-                visit(absolutePath, uuid, type, relativePath);
+                visit(absolutePath, uuid, type, relativePath, type === "repo.git"
+                    ? {
+                        root: absolutePath,
+                        evidencePaths: repositoryEvidencePaths(absolutePath),
+                        evidence: null,
+                    }
+                    : activeRepository);
             }
         }
     };
@@ -943,42 +1195,50 @@ function scanRoot(input) {
                 : entry.relativePath,
             uuid: entry.uuid,
             type: entry.type,
+            payloadVersion: entry.payloadVersion,
         }))
             .sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0);
-        const seededHead = repositoryHeads.get(repository.uuid);
-        if (seededHead) {
-            if (sha256Hex(seededHead.bytes) !== seededHead.transportVersion) {
-                throw new Error(`repository transport ${repository.uuid} is corrupt`);
-            }
-            const state = unpack(Buffer.from(seededHead.bytes).toString("utf8"));
-            if (state.stateId !== seededHead.stateId
-                || JSON.stringify(state.identity ?? []) !== JSON.stringify(repoIdentity)) {
-                throw new Error(`repository transport ${repository.uuid} does not match its materialized identity`);
-            }
-            repository.payloadVersion = seededHead.stateId;
-            repository.transportVersion = seededHead.transportVersion;
-            immutableWrite(join(cacheDir, `${seededHead.transportVersion}.bin`), seededHead.bytes);
-            continue;
-        }
         const priorRow = existingByPath.get(repository.relativePath);
         let prior = null;
         if (priorRow?.type === "repo.git" && priorRow.payloadVersion && priorRow.transportVersion) {
             const priorPath = join(cacheDir, `${priorRow.transportVersion}.bin`);
             if (existsSync(priorPath)) {
-                const bytes = readFileSync(priorPath);
-                if (sha256Hex(bytes) === priorRow.transportVersion) {
+                const layout = inspectRepositoryTransportFile(priorPath);
+                if (layout.stateId === priorRow.payloadVersion) {
                     prior = {
                         stateId: priorRow.payloadVersion,
                         transportVersion: priorRow.transportVersion,
-                        bytes,
+                        card: layout.card,
+                        identityHash: layout.identityHash,
                     };
                 }
             }
         }
-        const captured = captureRepository(repository.absolutePath, repoIdentity, prior);
+        const previousRepositories = [...existingByPath.values()]
+            .filter((row) => row.type === "repo.git");
+        const previousOwner = (row) => previousRepositories
+            .filter((candidate) => candidate.uuid !== row.uuid
+            && properDescendantOf(row.relativePath, candidate.relativePath))
+            .sort((left, right) => right.relativePath.length - left.relativePath.length)[0] ?? null;
+        const previousIdentity = prior && priorRow
+            ? [...existingByPath.values()]
+                .filter((row) => previousOwner(row)?.uuid === priorRow.uuid)
+                .map((row) => ({
+                path: priorRow.relativePath
+                    ? row.relativePath.slice(priorRow.relativePath.length + 1)
+                    : row.relativePath,
+                uuid: row.uuid,
+                type: row.type,
+                payloadVersion: row.payloadVersion,
+            }))
+                .sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0)
+            : null;
+        const captured = captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity);
         repository.payloadVersion = captured.stateId;
         repository.transportVersion = captured.transportVersion;
-        immutableWrite(join(cacheDir, `${captured.transportVersion}.bin`), captured.bytes);
+        if (captured.bytes) {
+            immutableWrite(join(cacheDir, `${captured.transportVersion}.bin`), captured.bytes);
+        }
     }
     const children = new Map();
     for (const entry of entries) {
@@ -1016,8 +1276,7 @@ function scanRoot(input) {
             inode: entry.inode,
         };
     });
-    if (persist)
-        persistRows(database, identity, resourceId, rootUUID, rows);
+    persistRows(database, identity, resourceId, rootUUID, rows, existingRows);
     return { records: rows.map((row) => row.record), rows, referenceWorkspaceIds: [...referenceWorkspaceIds] };
 }
 function portablePaths(records) {
@@ -1117,19 +1376,10 @@ function installCloudGround(input) {
                 bindingDir: workspaceBindingDir(userRoot, identity.deviceId),
                 readContent,
             });
-            const rows = materialized.repositories.length > 0
-                ? captureMaterializedGraph({
-                    identity,
-                    resourceId: cloud.resourceId,
-                    root: roots[0],
-                    records,
-                    repositories: materialized.repositories,
-                    rootPath: userRoot,
-                    database,
-                    cacheDir,
-                    bindingDir: workspaceBindingDir(userRoot, identity.deviceId),
-                })
-                : materialized.rows;
+            const rows = projectMaterializedGraph(materialized.rows, materialized.repositories.map((repository) => ({
+                record: repository.record,
+                identity: repository.state.identity,
+            })), sha256Hex);
             persistRows(database, identity, cloud.resourceId, roots[0].uuid, rows);
         }
         catch (error) {
@@ -1147,17 +1397,35 @@ async function materializeRecords(input) {
         if (!record.payloadVersion || !record.transportVersion) {
             throw new Error(`cloud repository ${record.uuid} has no state transport`);
         }
-        const bytes = await readContent(record.transportVersion);
-        if (sha256Hex(bytes) !== record.transportVersion) {
+        const artifact = artifactForRecord(record);
+        if (!artifact)
+            throw new Error(`cloud repository ${record.uuid} has no transfer artifact`);
+        const local = await readContent(artifact);
+        if (local.contentHash !== record.transportVersion) {
             throw new Error(`cloud repository ${record.uuid} transport is corrupt`);
         }
-        const state = unpack(bytes.toString("utf8"));
-        if (state.stateId !== record.payloadVersion) {
+        const chain = [{ file: local.file, transportVersion: local.contentHash }];
+        const versions = new Set([record.transportVersion]);
+        let state = inspectRepositoryTransportFile(local.file);
+        while (state.parentTransportVersion) {
+            if (versions.has(state.parentTransportVersion)) {
+                throw new Error(`cloud repository ${record.uuid} transport chain has a cycle`);
+            }
+            const parentVersion = state.parentTransportVersion;
+            versions.add(parentVersion);
+            const parent = await readContent({ ...artifact, contentHash: parentVersion });
+            if (parent.contentHash !== parentVersion) {
+                throw new Error(`cloud repository ${record.uuid} parent transport is corrupt`);
+            }
+            chain.unshift({ file: parent.file, transportVersion: parent.contentHash });
+            state = inspectRepositoryTransportFile(parent.file);
+        }
+        const latest = inspectRepositoryTransportFile(local.file);
+        if (latest.stateId !== record.payloadVersion) {
             throw new Error(`cloud repository ${record.uuid} transport does not name its declared state`);
         }
-        immutableWrite(join(cacheDir, `${record.transportVersion}.bin`), bytes);
-        const applied = applyRepository(destination, bytes);
-        repositories.push({ record, state: applied, bytes });
+        const applied = await applyRepositoryFiles(destination, chain);
+        repositories.push({ record, state: applied });
     };
     const paths = portablePaths(records);
     const ordered = [...records].sort((left, right) => {
@@ -1203,13 +1471,15 @@ async function materializeRecords(input) {
                 throw new Error(`cloud leaf ${record.uuid} has no content head`);
             if (pathExists(destination))
                 throw new Error(`cloud materialization conflicts with ${relativePath}`);
-            const bytes = await readContent(record.payloadVersion);
+            const artifact = artifactForRecord(record);
+            if (!artifact)
+                throw new Error(`cloud leaf ${record.uuid} has no transfer artifact`);
+            const local = await readContent(artifact);
             ensurePrivateDir(dirname(destination));
             if (record.type === "link")
-                symlinkSync(bytes.toString("utf8"), destination);
+                symlinkSync(readFileSync(local.file, "utf8"), destination);
             else
-                atomicWrite(destination, bytes);
-            immutableWrite(join(cacheDir, `${record.payloadVersion}.bin`), bytes);
+                atomicCopy(local.file, destination);
         }
         else {
             throw new Error(`materialization for ${record.type} requires its dedicated adapter`);
@@ -1229,50 +1499,6 @@ async function materializeRecords(input) {
     });
     return { rows, repositories };
 }
-function captureMaterializedGraph(input) {
-    const { identity, resourceId, root, records, repositories, rootPath, database, cacheDir, bindingDir, } = input;
-    if (root.type !== "workspace" && root.type !== "repo.git") {
-        throw new Error(`cloud root ${root.uuid} cannot be scanned as a workspace`);
-    }
-    const paths = portablePaths(records);
-    const seedByPath = new Map();
-    for (const record of records) {
-        const path = paths.get(record.uuid);
-        if (path)
-            seedByPath.set(path, { uuid: record.uuid, type: record.type });
-    }
-    const repositoryHeads = new Map();
-    for (const repository of repositories) {
-        if (!repository.record.transportVersion || !repository.record.payloadVersion) {
-            throw new Error(`cloud repository ${repository.record.uuid} has no transport identity`);
-        }
-        const base = paths.get(repository.record.uuid);
-        for (const entry of repository.state.identity ?? []) {
-            const path = [base, entry.path].filter(Boolean).join("/");
-            seedByPath.set(path, { uuid: entry.uuid, type: entry.type });
-        }
-        repositoryHeads.set(repository.record.uuid, {
-            stateId: repository.record.payloadVersion,
-            transportVersion: repository.record.transportVersion,
-            bytes: repository.bytes,
-        });
-    }
-    return scanRoot({
-        identity,
-        resourceId,
-        rootUUID: root.uuid,
-        rootName: root.name,
-        rootPath,
-        rootType: root.type,
-        database,
-        cacheDir,
-        policy: () => true,
-        bindingDir,
-        seedByPath,
-        repositoryHeads,
-        persist: false,
-    }).rows;
-}
 async function installCloudWorkspace(input) {
     const { identity, database, cacheDir, bindingDir, resourceId, workspace, records, destinationParent, readContent, } = input;
     const existing = readRows(database).filter((row) => row.rootUUID === workspace.uuid);
@@ -1306,19 +1532,10 @@ async function installCloudWorkspace(input) {
         const materialized = await materializeRecords({
             records, rootPath: destination, cacheDir, bindingDir, readContent,
         });
-        const rows = materialized.repositories.length > 0
-            ? captureMaterializedGraph({
-                identity,
-                resourceId,
-                root: workspace,
-                records,
-                repositories: materialized.repositories,
-                rootPath: destination,
-                database,
-                cacheDir,
-                bindingDir,
-            })
-            : materialized.rows;
+        const rows = projectMaterializedGraph(materialized.rows, materialized.repositories.map((repository) => ({
+            record: repository.record,
+            identity: repository.state.identity,
+        })), sha256Hex);
         symlinkSync(destination, binding, "dir");
         persistRows(database, identity, resourceId, workspace.uuid, rows);
         return {
@@ -1356,7 +1573,10 @@ function readyFromFrame(frame) {
     };
 }
 function frameBytes(frame) {
-    const bytes = Buffer.from(String(frame.data_b64 || ""), "base64");
+    if (!(frame.data_bin instanceof Uint8Array)) {
+        throw new Error("wire content response is not a binary frame");
+    }
+    const bytes = Buffer.from(frame.data_bin);
     if (bytes.length !== Number(frame.bytes) || sha256Hex(bytes) !== String(frame.sha256 || "")) {
         throw new Error("wire content bytes do not match their checksum");
     }