@amalgm/shell 0.1.54 → 0.1.55

package/dist/user-ground-host.js

@@ -3,9 +3,10 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
 import { open } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
-import { CHUNK_BYTES, CONTENT_CONTRACT, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, download as downloadArtifact, encodeEntityRecord, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, privateEntityResourceId, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
+import { CHUNK_BYTES, CONTENT_CONTRACT, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, download as downloadArtifact, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, privateEntityResourceId, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
 import Database from "better-sqlite3";
 import { atomicCopy, atomicWrite, ensurePrivateDir } from "./filesystem.js";
+import { EntityApplyHost } from "./entity-apply-host.js";
 import { EntityRecordSqliteStore } from "./entity-record-store.js";
 import { ContentCacheDownload, captureContentFile, hashContentFile, sealCapturedArtifact, } from "./content-cache-host.js";
 import { decodeContentWireBytes, encodeContentWireBytes } from "./content-wire-codec.js";
@@ -13,7 +14,7 @@ import { exactTextReplay, planGroundDetection, reconcileGroundUUIDs, } from "./d
 import { NamedDetectRuntime } from "./detection/runtime.js";
 import { WORKSPACE_UUID as UUID, createFilesRegisterPorts, ensureWorkspaceBinding, ensureWorkspaceReference, pathExists, pathWithin, referenceWorkspaceId, selectKnownRegistrationId, workspaceBindingDir, } from "./files-register-host.js";
 import { applyRepositoryFiles, captureRepository, inspectRepositoryTransportFile, hasGitMarker, } from "./git-repository-host.js";
-import { inspectGitRegistration, } from "./git-registration-host.js";
+import { canonicalizeGitIdentity, inspectGitRegistration, sameGitIdentity, } from "./git-registration-host.js";
 import { projectMaterializedGraph } from "./materialized-graph.js";
 import { NodeWatchHost, } from "./watching/index.js";
 import { WireClient, WireRequestError } from "./wire-client.js";
@@ -44,9 +45,12 @@ export class UserGroundHost {
     syncing = null;
     recordRail = null;
     recordStore = null;
+    applyRail = null;
+    applyHost = null;
     namedDetect = null;
     coldOperation = false;
     closing = false;
+    groundEffectTail = Promise.resolve();
     port;
     constructor(options) {
         this.options = options;
@@ -88,6 +92,7 @@ export class UserGroundHost {
                     watching: true,
                 };
                 await this.startRecordRail(this.activeIdentity);
+                await this.startApplyRail(this.activeIdentity);
                 this.converged = true;
                 return evidence;
             },
@@ -148,26 +153,38 @@ export class UserGroundHost {
                 return cloud;
             },
             materializeWorkspace: async (input) => {
-                const installed = await installCloudWorkspace({
-                    identity,
-                    userRoot,
-                    database: this.databasePath(identity),
-                    cacheDir: this.cacheDir(identity),
-                    bindingDir,
-                    resourceId: privateEntityResourceId(identity.userId, sha256Hex),
-                    workspace: input.workspace,
-                    reference: input.reference,
-                    references: input.references,
-                    records: input.records,
-                    destinationParent: input.destinationParent,
-                    cloudHead: input.cloudHead,
-                    readContent: (artifact) => this.downloadContent(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifact),
-                    onStage: (stage) => this.options.onWorkspaceAddStage?.({
-                        workspaceId: input.workspace.uuid,
-                        stage,
-                    }),
-                });
-                return installed;
+                const release = await this.acquireGroundEffect();
+                try {
+                    const installed = await installCloudWorkspace({
+                        identity,
+                        userRoot,
+                        database: this.databasePath(identity),
+                        cacheDir: this.cacheDir(identity),
+                        bindingDir,
+                        resourceId: privateEntityResourceId(identity.userId, sha256Hex),
+                        workspace: input.workspace,
+                        reference: input.reference,
+                        references: input.references,
+                        records: input.records,
+                        destinationParent: input.destinationParent,
+                        cloudHead: input.cloudHead,
+                        readContent: (artifact) => this.downloadContent(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifact),
+                        onStage: (stage) => this.options.onWorkspaceAddStage?.({
+                            workspaceId: input.workspace.uuid,
+                            stage,
+                        }),
+                    });
+                    // Add changes both disk and the exact SQLite projection. Open the
+                    // selected root's singular Watch coverage before Detect can observe
+                    // either side of that transaction; the public cover port below
+                    // independently proves the same coverage before clearing intent.
+                    this.ensureWatchers(identity, installed.alreadyMaterialized ? [] : [input.workspace.uuid]);
+                    assertHealthyWatch(this.watchHost.evidence(), input.workspace.uuid);
+                    return installed;
+                }
+                finally {
+                    release();
+                }
             },
             coverWorkspace: async ({ workspaceId, baseline }) => {
                 const pending = readWorkspaceAddIntent(this.databasePath(identity), workspaceId);
@@ -189,37 +206,43 @@ export class UserGroundHost {
         const userRoot = this.userRoot(identity);
         const bindingDir = workspaceBindingDir(userRoot, identity.deviceId);
         for (const intent of readWorkspaceAddIntents(database)) {
-            const selection = workspaceAddSelection(intent);
+            const release = await this.acquireGroundEffect();
             const visibleAcrossBlindInterval = pathExists(intent.destinationPath);
-            const stagedAcrossBlindInterval = pathExists(intent.stagingPath);
-            if (stagedAcrossBlindInterval && readRows(database).some((row) => row.resourceId === intent.resourceId && row.rootUUID === intent.workspaceId)) {
-                // Rows prove that staging was verified before the crash, not that its
-                // bytes remained unchanged while no watcher existed. Return this
-                // private install to the intent-only stage and reproduce it from the
-                // verified immutable cache before making it visible.
-                deleteRootRows(database, intent.resourceId, intent.workspaceId);
+            try {
+                const selection = workspaceAddSelection(intent);
+                const stagedAcrossBlindInterval = pathExists(intent.stagingPath);
+                if (stagedAcrossBlindInterval && readRows(database).some((row) => row.resourceId === intent.resourceId && row.rootUUID === intent.workspaceId)) {
+                    // Rows prove that staging was verified before the crash, not that its
+                    // bytes remained unchanged while no watcher existed. Return this
+                    // private install to the intent-only stage and reproduce it from the
+                    // verified immutable cache before making it visible.
+                    deleteRootRows(database, intent.resourceId, intent.workspaceId);
+                }
+                await installCloudWorkspace({
+                    identity,
+                    userRoot,
+                    database,
+                    cacheDir: this.cacheDir(identity),
+                    bindingDir,
+                    resourceId: intent.resourceId,
+                    workspace: selection.workspace,
+                    reference: selection.reference,
+                    references: intent.references,
+                    records: intent.records,
+                    destinationParent: dirname(intent.destinationPath),
+                    cloudHead: intent.cloudHead,
+                    readContent: (artifact) => this.downloadContent(intent.resourceId, this.cacheDir(identity), artifact),
+                    onStage: (stage) => this.options.onWorkspaceAddStage?.({
+                        workspaceId: intent.workspaceId,
+                        stage,
+                    }),
+                });
+                this.ensureWatchers(identity, visibleAcrossBlindInterval ? [] : [intent.workspaceId]);
+                assertHealthyWatch(this.watchHost.evidence(), intent.workspaceId);
+            }
+            finally {
+                release();
             }
-            await installCloudWorkspace({
-                identity,
-                userRoot,
-                database,
-                cacheDir: this.cacheDir(identity),
-                bindingDir,
-                resourceId: intent.resourceId,
-                workspace: selection.workspace,
-                reference: selection.reference,
-                references: intent.references,
-                records: intent.records,
-                destinationParent: dirname(intent.destinationPath),
-                cloudHead: intent.cloudHead,
-                readContent: (artifact) => this.downloadContent(intent.resourceId, this.cacheDir(identity), artifact),
-                onStage: (stage) => this.options.onWorkspaceAddStage?.({
-                    workspaceId: intent.workspaceId,
-                    stage,
-                }),
-            });
-            this.ensureWatchers(identity, visibleAcrossBlindInterval ? [] : [intent.workspaceId]);
-            assertHealthyWatch(this.watchHost.evidence(), intent.workspaceId);
             if (visibleAcrossBlindInterval) {
                 // Ground that was visible while the runtime was absent may have been
                 // edited after its verified reveal. It is an ordinary startup blind
@@ -258,8 +281,12 @@ export class UserGroundHost {
             throw stageError("Watch coverage", error);
         }
         let rows = readRows(this.databasePath(identity));
-        if (rows.some((row) => row.uuid === workspaceId)
-            && this.cloudState.records.some((record) => record.uuid === workspaceId)) {
+        const hasRegisteredBoundary = (records) => records.some((record) => record.uuid === workspaceId && record.status === "active")
+            && records.some((record) => record.type === "reference"
+                && record.status === "active"
+                && record.payloadVersion === workspaceId);
+        if (hasRegisteredBoundary(rows.map(portableRecord))
+            && hasRegisteredBoundary(this.cloudState.records)) {
             this.finishColdOperation();
             return {
                 registered: true,
@@ -271,11 +298,13 @@ export class UserGroundHost {
         // Register is a cold declaration, not a filesystem edit. It performs one
         // complete reconciliation and publishes one graph head; it must not mint
         // a parallel stream of Detect records for the same initial contents.
-        const observations = this.watchHost.observations();
-        this.watchDirty = false;
+        let releaseGround = null;
         try {
             if (this.syncing)
                 await this.syncing;
+            releaseGround = await this.acquireGroundEffect();
+            const observations = this.watchHost.observations();
+            this.watchDirty = false;
             const scanned = await reconcileGround({
                 identity,
                 userRoot: this.userRoot(identity),
@@ -306,6 +335,7 @@ export class UserGroundHost {
             throw stageError("entity registration or cloud publication", error);
         }
         finally {
+            releaseGround?.();
             this.finishColdOperation();
         }
         try {
@@ -317,11 +347,11 @@ export class UserGroundHost {
             throw stageError("Watch coverage", error);
         }
         rows = readRows(this.databasePath(identity));
-        if (!rows.some((row) => row.uuid === workspaceId)) {
-            throw stageError("entity registration", new Error(`workspace ${workspaceId} was not recorded`));
+        if (!hasRegisteredBoundary(rows.map(portableRecord))) {
+            throw stageError("entity registration", new Error(`workspace ${workspaceId} and its durable reference were not recorded`));
         }
-        if (!this.cloudState.records.some((record) => record.uuid === workspaceId)) {
-            throw stageError("cloud graph commit", new Error(`workspace ${workspaceId} is absent from the committed head`));
+        if (!hasRegisteredBoundary(this.cloudState.records)) {
+            throw stageError("cloud graph commit", new Error(`workspace ${workspaceId} or its durable reference is absent from the committed head`));
         }
         return {
             registered: true,
@@ -357,6 +387,7 @@ export class UserGroundHost {
             }
         } while (!this.watchHost.settled(cutoff));
         await this.recordRail?.flush();
+        await this.applyRail?.flush();
         if (!this.closing)
             this.ensureWatchers(identity);
     }
@@ -410,6 +441,7 @@ export class UserGroundHost {
                 },
                 random: Math.random,
             },
+            onInboxCommitted: () => this.applyRail?.recordsAvailable(),
         });
         this.recordStore = store;
         this.recordRail = rail;
@@ -426,6 +458,62 @@ export class UserGroundHost {
             throw pipelineStageError("entity record subscribe", error);
         }
     }
+    async startApplyRail(identity) {
+        if (!this.cloudState)
+            throw new Error("entity Apply rail requires cloud authority");
+        if (this.applyRail || this.applyHost)
+            throw new Error("entity Apply rail already started");
+        let host;
+        host = new EntityApplyHost({
+            databaseFile: this.databasePath(identity),
+            userRoot: this.userRoot(identity),
+            cacheDir: this.cacheDir(identity),
+            bindingDir: workspaceBindingDir(this.userRoot(identity), identity.deviceId),
+            readContent: (artifact) => this.downloadContent(this.cloudState.resourceId, this.cacheDir(identity), artifact),
+            captureLocal: async (intent, plan, position) => {
+                // Apply never suppresses Watch. This explicit whole-root suspicion is
+                // the synchronous proof boundary used only when Apply observed a
+                // save racing its reveal and must make that save durable first.
+                this.watchHost.suspectAll();
+                this.watchDirty = true;
+                await this.syncNow();
+                if (!host.store.hasLocalWorkAfter(intent.authorityId, intent.entityId, intent.throughSequence)) {
+                    throw new Error(`Apply ${position} capture for ${plan.target.type} ${plan.target.name} (${intent.entityId}) produced no durable local record`);
+                }
+            },
+            acquireGround: () => this.acquireGroundEffect(),
+        });
+        const rail = new EntityApplyRail({
+            store: host.store,
+            effects: host.effects,
+            schedule: {
+                schedule(delayMs, callback) {
+                    const timer = setTimeout(callback, delayMs);
+                    timer.unref();
+                    return { cancel: () => clearTimeout(timer) };
+                },
+                random: Math.random,
+            },
+            sha256Hex,
+            now: Date.now,
+            // Path cones and repository territories are wider than one UUID. Until
+            // the host exposes those locks, one serial machine-effect owner is the
+            // only honest concurrency setting.
+            concurrency: 1,
+        });
+        this.applyHost = host;
+        this.applyRail = rail;
+        try {
+            await rail.start();
+        }
+        catch (error) {
+            await rail.stop();
+            host.close();
+            this.applyRail = null;
+            this.applyHost = null;
+            throw pipelineStageError("entity Apply resume", error);
+        }
+    }
     async activateRuntimeTunnel(gatewayPort, runtimeToken) {
         // Files convergence and Watch/Detect own ground currency. Advertising an
         // HTTP tunnel must never trigger or wait for a second filesystem pass.
@@ -444,8 +532,10 @@ export class UserGroundHost {
             this.watchHost.close({ catchUp: false });
             this.namedDetect?.close();
             this.namedDetect = null;
+            const applyStopped = this.applyRail?.stop();
             const recordStopped = this.recordRail?.stop();
             await this.wire.close();
+            await applyStopped;
             await recordStopped;
             // An upload already inside its retry helper may have reopened the wire
             // after the first close. The stopped rail prevents new queue work; this
@@ -454,8 +544,18 @@ export class UserGroundHost {
             this.recordStore?.close();
             this.recordStore = null;
             this.recordRail = null;
+            this.applyHost?.close();
+            this.applyHost = null;
+            this.applyRail = null;
             return;
         }
+        await this.recordRail?.flush();
+        await this.applyRail?.flush();
+        const applyStopped = this.applyRail?.stop();
+        await applyStopped;
+        this.applyHost?.close();
+        this.applyHost = null;
+        this.applyRail = null;
         // Command shutdown is the last catch-up boundary. A filesystem callback
         // may still be queued, so watcher silence cannot be used as evidence yet.
         this.watchHost.suspectAll();
@@ -798,6 +898,9 @@ export class UserGroundHost {
                 rootId: workspaceId,
                 directory,
                 materialized: true,
+                // Every materialized root has one reserved machine-control subtree.
+                // Apply stages and retains recovery bytes here; Watch never enrolls it.
+                excludedSubtrees: [join(directory, ".amalgm")],
             });
         }
         const health = this.watchHost.reconcile([...wanted.values()], {
@@ -856,7 +959,24 @@ export class UserGroundHost {
             this.scheduleRescan();
         }
     }
+    async acquireGroundEffect() {
+        let release;
+        const held = new Promise((resolve) => { release = resolve; });
+        const preceding = this.groundEffectTail;
+        this.groundEffectTail = preceding.then(() => held);
+        await preceding;
+        return release;
+    }
     async syncLocalChanges(identity) {
+        const release = await this.acquireGroundEffect();
+        try {
+            await this.syncLocalChangesUnlocked(identity);
+        }
+        finally {
+            release();
+        }
+    }
+    async syncLocalChangesUnlocked(identity) {
         if (!this.cloudState)
             throw new Error("cloud state is unavailable for user-ground Watch");
         const observations = this.watchHost.observations();
@@ -885,6 +1005,7 @@ export class UserGroundHost {
                     });
                 }
                 this.watchHost.settle(observations);
+                this.applyRail?.localStateChanged();
                 this.recordRail?.localRecordsAvailable();
                 return;
             }
@@ -935,6 +1056,7 @@ export class UserGroundHost {
         });
         this.namedDetect?.refreshEnrollmentPolicy();
         this.watchHost.settle(observations);
+        this.applyRail?.localStateChanged();
         this.recordRail?.localRecordsAvailable();
     }
     async publishPendingSnapshotsBeforeLookup(identity, resourceId) {
@@ -1482,14 +1604,22 @@ function commitDetectedState(file, input) {
             for (const uuid of input.materializedRemovals ?? [])
                 removeMaterialized.run(uuid);
             const upsertMaterialized = prepareGroundUpsert(database, "entities");
+            // A valid graph may move or re-root several UUIDs through one another's
+            // old address slots. Release every changed UUID's old slot first, then
+            // install the complete new projection in this same transaction; row
+            // iteration order must never decide whether an atomic graph is valid.
             for (const row of input.acceptedMaterializedRows ?? []) {
-                upsertMaterialized.run(...groundRowValues(row));
+                removeMaterialized.run(row.uuid);
+            }
+            for (const row of input.acceptedMaterializedRows ?? []) {
+                runGroundUpsert(database, "entities", upsertMaterialized, row);
             }
             for (const uuid of input.notebookRemovals)
                 remove.run(uuid);
             const upsertNotebook = prepareGroundUpsert(database, "detection_notebook");
-            for (const row of input.acceptedNotebookRows)
-                upsertNotebook.run(...groundRowValues(row));
+            for (const row of input.acceptedNotebookRows) {
+                runGroundUpsert(database, "detection_notebook", upsertNotebook, row);
+            }
         })();
     }
     finally {
@@ -1617,6 +1747,19 @@ function prepareGroundUpsert(database, table) {
       content_verified_at_ms = excluded.content_verified_at_ms
   `);
 }
+function runGroundUpsert(database, table, statement, row) {
+    try {
+        statement.run(...groundRowValues(row));
+    }
+    catch (error) {
+        const occupant = database.prepare(`
+      SELECT uuid, absolute_path AS absolutePath FROM ${table}
+       WHERE resource_id = ? AND root_uuid = ? AND relative_path = ?
+    `).get(row.resourceId, row.rootUUID, row.relativePath);
+        throw new Error(`${table} projection could not place ${row.uuid} at ${row.rootUUID}:${row.relativePath}; `
+            + `occupied by ${occupant?.uuid ?? "no retained row"} (${occupant?.absolutePath ?? "unknown"})`, { cause: error });
+    }
+}
 function persistRows(file, identity, resourceId, rootUUID, rows, previousRows = [], options = {}) {
     const database = initializeDatabase(file);
     try {
@@ -1631,6 +1774,7 @@ function persistRows(file, identity, resourceId, rootUUID, rows, previousRows =
         const upsert = prepareGroundUpsert(database, "entities");
         const upsertNotebook = prepareGroundUpsert(database, "detection_notebook");
         const remove = database.prepare("DELETE FROM entities WHERE uuid = ? AND resource_id = ? AND root_uuid = ?");
+        const releaseChangedSlot = database.prepare("DELETE FROM entities WHERE uuid = ?");
         const removeNotebook = database.prepare("DELETE FROM detection_notebook WHERE uuid = ? AND resource_id = ? AND root_uuid = ?");
         const previousByUuid = new Map(previousRows.map((row) => [row.uuid, row]));
         const materializedIds = new Set(database.prepare("SELECT uuid FROM entities").all()
@@ -1671,10 +1815,13 @@ function persistRows(file, identity, resourceId, rootUUID, rows, previousRows =
                 }
             }
             for (const row of changed)
-                upsert.run(...groundRowValues(capturedGroundRow(resourceId, rootUUID, row)));
+                releaseChangedSlot.run(row.record.uuid);
+            for (const row of changed) {
+                runGroundUpsert(database, "entities", upsert, capturedGroundRow(resourceId, rootUUID, row));
+            }
             if (options.updateNotebook !== false) {
                 for (const row of notebookChanged) {
-                    upsertNotebook.run(...groundRowValues(capturedGroundRow(resourceId, rootUUID, row)));
+                    runGroundUpsert(database, "detection_notebook", upsertNotebook, capturedGroundRow(resourceId, rootUUID, row));
                 }
             }
         });
@@ -2087,6 +2234,11 @@ async function reconcileRoot(input) {
         const children = readdirSync(directory, { withFileTypes: true })
             .sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
         for (const child of children) {
+            // Every materialization boundary reserves its root .amalgm/ directory
+            // for machine intents, staging, and recovery. This is structural host
+            // state, never enrollment policy and never an entity.
+            if (base === "" && child.name === ".amalgm")
+                continue;
             if (isRepositoryMetadataEntry(parentType, child.name))
                 continue;
             const relativePath = base ? `${base}/${child.name}` : child.name;
@@ -2302,7 +2454,7 @@ async function reconcileRoot(input) {
         .sort((left, right) => right.relativePath.length - left.relativePath.length)[0] ?? null;
     try {
         for (const repository of repositories) {
-            const repoIdentity = entries
+            let repoIdentity = entries
                 .filter((entry) => repositoryOwner(entry)?.uuid === repository.uuid)
                 .map((entry) => ({
                 path: repository.relativePath
@@ -2365,7 +2517,30 @@ async function reconcileRoot(input) {
                     .sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0)
                 : null;
             evidence && (evidence.repositoryCaptures += 1);
-            const captured = captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity);
+            let captured = null;
+            let coherent = false;
+            for (let attempt = 0; attempt < 3; attempt += 1) {
+                captured = captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity);
+                const after = canonicalizeGitIdentity(repoIdentity, inspectGitRegistration(repository.absolutePath));
+                if (sameGitIdentity(after, repoIdentity)) {
+                    coherent = true;
+                    break;
+                }
+                repoIdentity = after;
+            }
+            if (!coherent || !captured) {
+                throw new Error(`repository did not settle during capture: ${repository.absolutePath}`);
+            }
+            for (const identityEntry of repoIdentity) {
+                const entry = entries.find((candidate) => repositoryOwner(candidate)?.uuid === repository.uuid
+                    && (repository.relativePath
+                        ? candidate.relativePath.slice(repository.relativePath.length + 1)
+                        : candidate.relativePath) === identityEntry.path);
+                if (!entry)
+                    continue;
+                entry.type = identityEntry.type;
+                entry.payloadVersion = identityEntry.payloadVersion ?? null;
+            }
             repository.payloadVersion = captured.stateId;
             repository.transportVersion = captured.transportVersion;
             if (captured.bytes) {