@amalgm/shell 0.1.33 → 0.1.35

package/dist/user-ground-host.js

@@ -8,7 +8,7 @@ import Database from "better-sqlite3";
 import { atomicCopy, atomicWrite, ensurePrivateDir } from "./filesystem.js";
 import { ContentCacheDownload, hashContentFile } from "./content-cache-host.js";
 import { decodeContentWireBytes, encodeContentWireBytes } from "./content-wire-codec.js";
-import { WORKSPACE_UUID as UUID, createFilesRegisterPorts, pathExists, pathWithin, referenceWorkspaceId, workspaceBindingDir, } from "./files-register-host.js";
+import { WORKSPACE_UUID as UUID, createFilesRegisterPorts, ensureWorkspaceReference, pathExists, pathWithin, referenceWorkspaceId, workspaceBindingDir, } from "./files-register-host.js";
 import { applyRepositoryFiles, captureRepository, inspectRepositoryTransportFile, hasGitMarker, } from "./git-repository-host.js";
 import { inspectGitRegistration, } from "./git-registration-host.js";
 import { projectMaterializedGraph } from "./materialized-graph.js";
@@ -126,17 +126,24 @@ export class UserGroundHost {
                     throw new Error("this user has no cloud entity registry");
                 return cloud;
             },
-            materializeWorkspace: async (input) => installCloudWorkspace({
-                identity,
-                database: this.databasePath(identity),
-                cacheDir: this.cacheDir(identity),
-                bindingDir,
-                resourceId: privateEntityResourceId(identity.userId, sha256Hex),
-                workspace: input.workspace,
-                records: input.records,
-                destinationParent: input.destinationParent,
-                readContent: (artifact) => this.downloadContent(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifact),
-            }),
+            materializeWorkspace: async (input) => {
+                const installed = await installCloudWorkspace({
+                    identity,
+                    userRoot,
+                    database: this.databasePath(identity),
+                    cacheDir: this.cacheDir(identity),
+                    bindingDir,
+                    resourceId: privateEntityResourceId(identity.userId, sha256Hex),
+                    workspace: input.workspace,
+                    reference: input.reference,
+                    references: input.references,
+                    records: input.records,
+                    destinationParent: input.destinationParent,
+                    readContent: (artifact) => this.downloadContent(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifact),
+                });
+                this.scheduleRescan();
+                return installed;
+            },
             coverWorkspace: async ({ workspaceId, baseline }) => {
                 const health = this.ensureWatchers(identity, baseline === "authoritative-install" ? [workspaceId] : []);
                 return { active: true, roots: health.contentHandles };
@@ -282,7 +289,10 @@ export class UserGroundHost {
                     declaration: this.options.declaration ?? shippedUserHomeDeclaration,
                     now: this.options.now?.() ?? new Date(),
                 });
-                scanAndRegister({ identity, userRoot, database, cacheDir });
+                await scanAndRegister({
+                    identity, userRoot, database, cacheDir,
+                    onScan: this.options.onDetectScan,
+                });
                 return localValue(identity, userRoot, database);
             },
             install: async (_who, cloud) => {
@@ -575,11 +585,12 @@ export class UserGroundHost {
         if (!state)
             throw new Error("cloud state is unavailable for user-ground Watch");
         const observations = this.watchHost.observations();
-        const localRecords = scanAndRegister({
+        const localRecords = await scanAndRegister({
             identity,
             userRoot: this.userRoot(identity),
             database: this.databasePath(identity),
             cacheDir: this.cacheDir(identity),
+            onScan: this.options.onDetectScan,
             suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion.paths])),
         });
         const snapshot = snapshotFromRecords(travelingRecords(mergeMaterializedRoots(state.records, localRecords)));
@@ -744,6 +755,10 @@ function initializeDatabase(file) {
       absolute_path TEXT NOT NULL,
       device_number INTEGER NOT NULL,
       inode INTEGER NOT NULL,
+      byte_size INTEGER,
+      modified_time_ms REAL,
+      changed_time_ms REAL,
+      filesystem_mode INTEGER,
       UNIQUE(resource_id, root_uuid, relative_path)
     );
     CREATE TABLE IF NOT EXISTS cloud_outbox (
@@ -757,6 +772,18 @@ function initializeDatabase(file) {
       created_at TEXT NOT NULL
     );
   `);
+    const currentEntityColumns = new Set(database.prepare("PRAGMA table_info(entities)").all()
+        .map(({ name }) => name));
+    const fingerprintColumns = [
+        ["byte_size", "INTEGER"],
+        ["modified_time_ms", "REAL"],
+        ["changed_time_ms", "REAL"],
+        ["filesystem_mode", "INTEGER"],
+    ];
+    for (const [name, type] of fingerprintColumns) {
+        if (!currentEntityColumns.has(name))
+            database.exec(`ALTER TABLE entities ADD COLUMN ${name} ${type}`);
+    }
     return database;
 }
 function readRows(file) {
@@ -769,7 +796,9 @@ function readRows(file) {
              uuid, type, parent_uuid AS parentUUID, name, status, version,
              payload_version AS payloadVersion, transport_version AS transportVersion,
              relative_path AS relativePath, absolute_path AS absolutePath,
-             device_number AS deviceNumber, inode
+             device_number AS deviceNumber, inode,
+             byte_size AS byteSize, modified_time_ms AS modifiedTimeMs,
+             changed_time_ms AS changedTimeMs, filesystem_mode AS filesystemMode
         FROM entities ORDER BY uuid
     `).all();
     }
@@ -823,7 +852,8 @@ function persistRows(file, identity, resourceId, rootUUID, rows, previousRows =
       INSERT INTO entities(
         uuid, resource_id, root_uuid, type, parent_uuid, name, status, version, payload_version,
         transport_version, relative_path, absolute_path, device_number, inode
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        , byte_size, modified_time_ms, changed_time_ms, filesystem_mode
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(uuid) DO UPDATE SET
         resource_id = excluded.resource_id,
         root_uuid = excluded.root_uuid,
@@ -837,7 +867,11 @@ function persistRows(file, identity, resourceId, rootUUID, rows, previousRows =
         relative_path = excluded.relative_path,
         absolute_path = excluded.absolute_path,
         device_number = excluded.device_number,
-        inode = excluded.inode
+        inode = excluded.inode,
+        byte_size = excluded.byte_size,
+        modified_time_ms = excluded.modified_time_ms,
+        changed_time_ms = excluded.changed_time_ms,
+        filesystem_mode = excluded.filesystem_mode
     `);
         const remove = database.prepare("DELETE FROM entities WHERE uuid = ? AND resource_id = ? AND root_uuid = ?");
         const previousByUuid = new Map(previousRows.map((row) => [row.uuid, row]));
@@ -857,7 +891,11 @@ function persistRows(file, identity, resourceId, rootUUID, rows, previousRows =
                 || old.relativePath !== row.relativePath
                 || old.absolutePath !== row.absolutePath
                 || old.deviceNumber !== row.deviceNumber
-                || old.inode !== row.inode;
+                || old.inode !== row.inode
+                || old.byteSize !== row.byteSize
+                || old.modifiedTimeMs !== row.modifiedTimeMs
+                || old.changedTimeMs !== row.changedTimeMs
+                || old.filesystemMode !== row.filesystemMode;
         });
         const replace = database.transaction(() => {
             replaceIdentity.run(identity.userId, identity.userEmail, identity.deviceId);
@@ -866,7 +904,7 @@ function persistRows(file, identity, resourceId, rootUUID, rows, previousRows =
                     remove.run(old.uuid, resourceId, rootUUID);
             }
             for (const row of changed)
-                upsert.run(row.record.uuid, resourceId, rootUUID, row.record.type, row.record.parentUUID, row.record.name, row.record.status, row.record.version, row.record.payloadVersion, row.record.transportVersion, row.relativePath, row.absolutePath, row.deviceNumber, row.inode);
+                upsert.run(row.record.uuid, resourceId, rootUUID, row.record.type, row.record.parentUUID, row.record.name, row.record.status, row.record.version, row.record.payloadVersion, row.record.transportVersion, row.relativePath, row.absolutePath, row.deviceNumber, row.inode, row.byteSize, row.modifiedTimeMs, row.changedTimeMs, row.filesystemMode);
         });
         replace();
     }
@@ -943,6 +981,24 @@ function directoryExists(path) {
         return false;
     }
 }
+function statFingerprint(stats) {
+    return {
+        deviceNumber: stats.dev,
+        inode: stats.ino,
+        byteSize: stats.size,
+        modifiedTimeMs: stats.mtimeMs,
+        changedTimeMs: stats.ctimeMs,
+        filesystemMode: stats.mode,
+    };
+}
+function sameStatFingerprint(row, stats) {
+    return row.deviceNumber === stats.dev
+        && row.inode === stats.ino
+        && row.byteSize === stats.size
+        && row.modifiedTimeMs === stats.mtimeMs
+        && row.changedTimeMs === stats.ctimeMs
+        && row.filesystemMode === stats.mode;
+}
 /** The binding is the durable machine-local declaration of an external root.
  * Registration and Watch both derive their roots from this one inventory, so
  * correctness never depends on receiving the reference-rendering OS event. */
@@ -966,8 +1022,8 @@ function materializedWorkspaceBindings(bindingDir) {
     }
     return bindings.sort((left, right) => left.workspaceId.localeCompare(right.workspaceId));
 }
-function scanAndRegister(input) {
-    const { identity, userRoot, database, cacheDir, suspicions } = input;
+async function scanAndRegister(input) {
+    const { identity, userRoot, database, cacheDir, suspicions, onScan } = input;
     const resourceId = privateEntityResourceId(identity.userId, sha256Hex);
     const existing = readRows(database);
     const ignoreFile = join(userRoot, ".amalgmignore");
@@ -989,7 +1045,31 @@ function scanAndRegister(input) {
     const missingPortableReference = [...boundRoots.keys()]
         .some((workspaceId) => !existingReferenceIds.has(workspaceId));
     const deferredRootIds = new Set();
-    const core = scanRoot({
+    const scan = async (parameters) => {
+        const counters = {
+            entries: 0,
+            metadataReused: 0,
+            contentReads: 0,
+            gitInspections: 0,
+            repositoryCaptures: 0,
+        };
+        const started = performance.now();
+        const result = scanRoot({ ...parameters, evidence: counters });
+        onScan?.({
+            rootId: parameters.rootUUID,
+            directory: parameters.rootPath,
+            scope: parameters.suspects === null || parameters.suspects === undefined
+                ? "complete"
+                : parameters.suspects.length === 0 ? "none" : "paths",
+            ...counters,
+            durationMs: performance.now() - started,
+        });
+        // Each root commits its local projection before Detect yields. Native
+        // callbacks can therefore preserve new suspicion between large roots.
+        await new Promise((resolve) => setImmediate(resolve));
+        return result;
+    };
+    const core = await scan({
         identity,
         resourceId,
         rootUUID: coreUUID,
@@ -1023,7 +1103,7 @@ function scanAndRegister(input) {
         }
         const existingRoot = existing.find((row) => row.uuid === workspaceId && row.parentUUID === null);
         const existingWorkspaceRows = existing.filter((row) => row.rootUUID === workspaceId);
-        const scanned = scanRoot({
+        const scanned = await scan({
             identity,
             resourceId,
             rootUUID: workspaceId,
@@ -1045,7 +1125,7 @@ function scanAndRegister(input) {
         .map(portableRecord);
 }
 function scanRoot(input) {
-    const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, } = input;
+    const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, } = input;
     const existingRows = (input.existingRows ?? readRows(database))
         .filter((row) => row.resourceId === resourceId && row.rootUUID === rootUUID);
     const existingByPath = new Map(existingRows.map((row) => [row.relativePath, row]));
@@ -1063,6 +1143,10 @@ function scanRoot(input) {
                 absolutePath: row.absolutePath,
                 deviceNumber: row.deviceNumber,
                 inode: row.inode,
+                byteSize: row.byteSize,
+                modifiedTimeMs: row.modifiedTimeMs,
+                changedTimeMs: row.changedTimeMs,
+                filesystemMode: row.filesystemMode,
             })),
             referenceWorkspaceIds: existingRows
                 .filter((row) => row.type === "reference" && row.payloadVersion && UUID.test(row.payloadVersion))
@@ -1073,6 +1157,7 @@ function scanRoot(input) {
     const entries = [];
     const referenceWorkspaceIds = new Set();
     const rootStats = lstatSync(rootPath);
+    evidence && (evidence.entries += 1);
     entries.push({
         uuid: rootUUID,
         type: rootType,
@@ -1082,8 +1167,7 @@ function scanRoot(input) {
         absolutePath: rootPath,
         payloadVersion: null,
         transportVersion: null,
-        deviceNumber: rootStats.dev,
-        inode: rootStats.ino,
+        ...statFingerprint(rootStats),
     });
     const touchesSuspicion = (path) => suspects === null
         || pathIsSuspect(path, suspects)
@@ -1122,13 +1206,21 @@ function scanRoot(input) {
                 continue;
             const absolutePath = join(directory, child.name);
             const stats = lstatSync(absolutePath);
+            evidence && (evidence.entries += 1);
             const existing = existingByPath.get(relativePath);
             const uuid = existing?.uuid || randomUUID();
-            const observe = existing === undefined || touchesSuspicion(relativePath);
+            const stableDuringCompleteCatchUp = suspects === null
+                && existing !== undefined
+                && sameStatFingerprint(existing, stats);
+            const observe = existing === undefined
+                || (suspects === null ? !stableDuringCompleteCatchUp : touchesSuspicion(relativePath));
+            if (!observe && suspects === null)
+                evidence && (evidence.metadataReused += 1);
             const repositoryPath = activeRepository
                 ? slashPath(relative(activeRepository.root, absolutePath))
                 : null;
             if (observe && repositoryPath && activeRepository && !activeRepository.evidence) {
+                evidence && (evidence.gitInspections += 1);
                 try {
                     activeRepository.evidence = inspectGitRegistration(activeRepository.root, activeRepository.evidencePaths);
                 }
@@ -1164,6 +1256,7 @@ function scanRoot(input) {
                         payloadVersion = indexed.payloadVersion;
                     }
                     else {
+                        evidence && (evidence.contentReads += 1);
                         const bytes = Buffer.from(readlinkSync(absolutePath), "utf8");
                         payloadVersion = sha256Hex(bytes);
                         if (!activeRepository)
@@ -1185,6 +1278,7 @@ function scanRoot(input) {
                     payloadVersion = indexed.payloadVersion;
                 }
                 else {
+                    evidence && (evidence.contentReads += 1);
                     const bytes = readFileSync(absolutePath);
                     type = fileType(bytes);
                     payloadVersion = sha256Hex(bytes);
@@ -1209,8 +1303,7 @@ function scanRoot(input) {
                 absolutePath,
                 payloadVersion,
                 transportVersion: null,
-                deviceNumber: stats.dev,
-                inode: stats.ino,
+                ...statFingerprint(stats),
             });
             if (stats.isDirectory() && !stats.isSymbolicLink()) {
                 visit(absolutePath, uuid, type, relativePath, type === "repo.git"
@@ -1258,6 +1351,10 @@ function scanRoot(input) {
             absolutePath: entry.absolutePath,
             deviceNumber: entry.deviceNumber,
             inode: entry.inode,
+            byteSize: entry.byteSize,
+            modifiedTimeMs: entry.modifiedTimeMs,
+            changedTimeMs: entry.changedTimeMs,
+            filesystemMode: entry.filesystemMode,
         };
     });
     const persistCurrentRows = () => {
@@ -1333,6 +1430,7 @@ function scanRoot(input) {
                 }))
                     .sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0)
                 : null;
+            evidence && (evidence.repositoryCaptures += 1);
             const captured = captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity);
             repository.payloadVersion = captured.stateId;
             repository.transportVersion = captured.transportVersion;
@@ -1611,14 +1709,111 @@ async function materializeRecords(input) {
             record,
             relativePath,
             absolutePath,
-            deviceNumber: stats.dev,
-            inode: stats.ino,
+            ...statFingerprint(stats),
         };
     });
     return { rows, repositories };
 }
+/** Refresh the always-materialized portable reference inventory selected by
+ * Live. Paths and bindings are local; UUIDs, parents, names, and versions are
+ * the same cloud entities on every machine. */
+function installCloudWorkspaceReferences(input) {
+    const { identity, userRoot, database, bindingDir, resourceId, workspace, references, } = input;
+    const localRows = readRows(database);
+    const rendered = [];
+    try {
+        for (const reference of references) {
+            if (reference.type !== "reference"
+                || reference.status !== "active"
+                || !reference.payloadVersion
+                || reference.parentUUID === null) {
+                throw new Error("cloud registry has an invalid durable reference");
+            }
+            const parent = localRows.find((row) => row.uuid === reference.parentUUID);
+            if (!parent) {
+                throw new Error(`cloud reference ${reference.uuid} parent is not materialized`);
+            }
+            const link = ensureWorkspaceReference({
+                userRoot,
+                bindingDir,
+                workspaceId: reference.payloadVersion,
+                workspaceName: reference.name,
+                exactName: true,
+                requireBinding: reference.payloadVersion === workspace.uuid,
+            });
+            const stats = lstatSync(link.referencePath);
+            rendered.push({
+                created: link.created,
+                referencePath: link.referencePath,
+                rootUUID: parent.rootUUID,
+                row: {
+                    record: reference,
+                    relativePath: slashPath(relative(userRoot, link.referencePath)),
+                    absolutePath: link.referencePath,
+                    ...statFingerprint(stats),
+                },
+            });
+        }
+        const rootIds = new Set(rendered.map(({ rootUUID }) => rootUUID));
+        for (const rootUUID of rootIds) {
+            const priorReferences = localRows.filter((row) => row.rootUUID === rootUUID && row.type === "reference");
+            const currentReferences = rendered.filter((entry) => entry.rootUUID === rootUUID);
+            const parentIds = new Set([
+                ...priorReferences.map((row) => row.parentUUID),
+                ...currentReferences.map(({ row }) => row.record.parentUUID),
+            ].filter((uuid) => uuid !== null));
+            const priorParents = [...parentIds].map((parentId) => {
+                const parent = localRows.find((row) => row.rootUUID === rootUUID && row.uuid === parentId);
+                if (!parent || (parent.type !== "workspace" && parent.type !== "folder")) {
+                    throw new Error(`cloud reference parent ${parentId} is not a local container`);
+                }
+                return parent;
+            });
+            const parents = priorParents.map((parent) => {
+                const parentId = parent.uuid;
+                const stats = lstatSync(parent.absolutePath);
+                const ordinaryChildren = localRows.filter((row) => row.rootUUID === rootUUID
+                    && row.parentUUID === parentId
+                    && row.type !== "reference"
+                    && row.status === "active");
+                const referenceChildren = currentReferences
+                    .filter(({ row }) => row.record.parentUUID === parentId)
+                    .map(({ row }) => row.record);
+                const versionPayload = membershipHash([
+                    ...ordinaryChildren,
+                    ...referenceChildren,
+                ], sha256Hex);
+                return {
+                    record: {
+                        ...portableRecord(parent),
+                        version: canonicalVersion({
+                            type: parent.type,
+                            parentUUID: parent.parentUUID,
+                            name: parent.name,
+                            status: parent.status,
+                            payloadVersion: versionPayload,
+                        }, sha256Hex),
+                    },
+                    relativePath: parent.relativePath,
+                    absolutePath: parent.absolutePath,
+                    ...statFingerprint(stats),
+                };
+            });
+            persistRows(database, identity, resourceId, rootUUID, [...currentReferences.map(({ row }) => row), ...parents], [...priorReferences, ...priorParents]);
+        }
+    }
+    catch (error) {
+        for (const entry of rendered) {
+            if (entry.created && pathExists(entry.referencePath)) {
+                rmSync(entry.referencePath, { force: true });
+            }
+        }
+        throw error;
+    }
+    return references;
+}
 async function installCloudWorkspace(input) {
-    const { identity, database, cacheDir, bindingDir, resourceId, workspace, records, destinationParent, readContent, } = input;
+    const { identity, userRoot, database, cacheDir, bindingDir, resourceId, workspace, reference, references, records, destinationParent, readContent, } = input;
     const existing = readRows(database).filter((row) => row.rootUUID === workspace.uuid);
     if (existing.length > 0) {
         const root = existing.find((row) => row.uuid === workspace.uuid);
@@ -1629,10 +1824,21 @@ async function installCloudWorkspace(input) {
         if (!sameRecords(travelingRecords(localRecords), records)) {
             throw new Error(`local workspace ${workspace.uuid} differs from its cloud graph`);
         }
+        const installedReferences = installCloudWorkspaceReferences({
+            identity,
+            userRoot,
+            database,
+            bindingDir,
+            resourceId,
+            workspace,
+            references,
+        });
         return {
             path: root.absolutePath,
             rows: existing.length,
             records: localRecords,
+            reference,
+            references: installedReferences,
             alreadyMaterialized: true,
         };
     }
@@ -1656,14 +1862,26 @@ async function installCloudWorkspace(input) {
         })), sha256Hex);
         symlinkSync(destination, binding, "dir");
         persistRows(database, identity, resourceId, workspace.uuid, rows);
+        const installedReferences = installCloudWorkspaceReferences({
+            identity,
+            userRoot,
+            database,
+            bindingDir,
+            resourceId,
+            workspace,
+            references,
+        });
         return {
             path: destination,
             rows: rows.length,
             records: rows.map((row) => row.record),
+            reference,
+            references: installedReferences,
             alreadyMaterialized: false,
         };
     }
     catch (error) {
+        deleteRootRows(database, resourceId, workspace.uuid);
         if (pathExists(binding))
             rmSync(binding, { force: true });
         if (pathExists(destination))