@amalgm/shell 0.1.104 → 0.1.106

package/dist/user-ground-host.js

@@ -3,7 +3,7 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
 import { open } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
-import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
+import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
 import Database from "better-sqlite3";
 import { atomicCopy, atomicWrite, ensurePrivateDir, ensureUserDir } from "./filesystem.js";
 import { emitFilesStage, } from "./files-observability.js";
@@ -25,14 +25,14 @@ import { projectMaterializedGraph } from "./materialized-graph.js";
 import { NodeWatchHost, } from "./watching/index.js";
 import { GroundCoordinator } from "./ground-coordination.js";
 import { submitMutationOperation } from "./paged-mutation-submit.js";
+import { adoptSnapshotDeliveryBaseline } from "./snapshot-delivery-baseline.js";
 import { WireClient, WireRequestError } from "./wire-client.js";
 const SMALL_CONTENT_UPLOAD_CONCURRENCY = 16;
-/** Storage-object lanes for one download set. Manifests and blocks are all
- * independent content-addressed objects fetched straight from storage, so
- * twelve thousand small files and one large pack share the same wide bound;
- * direct fetches cost the gateway nothing, and memory stays bounded by
- * lanes × one block. */
+/** Storage-object lanes for one download set. Tiny objects use the wide
+ * request window, while the separate byte budget narrows large chunks so
+ * request concurrency can never silently become an unbounded memory claim. */
 const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = 128;
+const DIRECT_OBJECT_DOWNLOAD_MAX_IN_FLIGHT_BYTES = 8 * CHUNK_BYTES;
 const DETECT_QUIET_MS = 12;
 const DETECT_MAX_DEFERRAL_MS = 75;
 const DETECT_RETRY_MS = 250;
@@ -260,13 +260,31 @@ export class UserGroundHost {
                 const cloud = await this.cloudPort(identity).lookup(resourceId);
                 if (!cloud)
                     throw new Error("this user has no cloud entity registry");
+                // The authority's active snapshot is the compact base; Add's active
+                // materialization head is that snapshot plus the complete committed
+                // tail this machine has now received. Fold the tail before Live
+                // selects the workspace so a destination created after mutations is
+                // exact immediately, rather than first installing stale ground and
+                // racing Apply afterward.
+                await this.recordRail?.flush();
+                const deliveryWatermark = readAuthorityDeliveryCursor(this.databasePath(identity), resourceId);
+                if (deliveryWatermark < cloud.deliveryWatermark) {
+                    throw new Error("local delivery cursor is behind the active snapshot watermark");
+                }
+                const snapshot = snapshotFromRecords(recordsAtDeliveryPrefix(cloud.snapshot.records, readAcceptedResults(this.databasePath(identity), resourceId, cloud.deliveryWatermark, deliveryWatermark)));
+                const current = {
+                    ...cloud,
+                    deliveryWatermark,
+                    checksum: sha256Hex(stableJson(snapshot)),
+                    snapshot,
+                };
                 this.stage({
                     primitive: "download",
                     stage: "registry-lookup",
                     status: "completed",
                     durationMs: performance.now() - started,
                 });
-                return cloud;
+                return current;
             },
             materializeWorkspace: async (input) => {
                 const groundWaitStarted = performance.now();
@@ -299,9 +317,10 @@ export class UserGroundHost {
                         records: input.records,
                         destinationParent: input.destinationParent,
                         cloudHead: input.cloudHead,
+                        deliveryWatermark: input.deliveryWatermark,
                         readContent: (artifacts) => this.downloadContentSet(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifacts.map((artifact) => ({ artifact })), 10, { journey: "add", workspaceId: input.workspace.uuid }),
                         adoptMaterialization: (rootUUID) => {
-                            const settled = this.applyHost?.adoptVerifiedMaterialization(rootUUID) ?? 0;
+                            const settled = this.adoptCloudDeliveryBaseline(identity, input.deliveryWatermark, rootUUID);
                             this.applyRail?.recordsAvailable();
                             if (settled > 0) {
                                 this.stage({
@@ -425,9 +444,10 @@ export class UserGroundHost {
                 records: intent.records,
                 destinationParent: dirname(intent.destinationPath),
                 cloudHead: intent.cloudHead,
+                deliveryWatermark: intent.deliveryWatermark,
                 readContent: (artifacts) => this.downloadContentSet(intent.resourceId, this.cacheDir(identity), artifacts.map((artifact) => ({ artifact })), 10, { journey: "add", workspaceId: intent.workspaceId }),
                 adoptMaterialization: (rootUUID) => {
-                    this.applyHost?.adoptVerifiedMaterialization(rootUUID);
+                    this.adoptCloudDeliveryBaseline(identity, intent.deliveryWatermark, rootUUID);
                     this.applyRail?.recordsAvailable();
                 },
                 onStage: (stage) => this.options.onWorkspaceAddStage?.({
@@ -505,7 +525,8 @@ export class UserGroundHost {
             // the publish path re-proves seal evidence and repairs any missing
             // uploads before this answer is lawful.
             try {
-                await this.publishMaterializedSnapshot(identity, new Set([workspaceId]), { journey: "register", workspaceId });
+                const deliveryWatermark = await this.prepareSnapshotPublication(identity, { journey: "register", workspaceId });
+                await this.publishMaterializedSnapshot(identity, new Set([workspaceId]), deliveryWatermark, { journey: "register", workspaceId });
             }
             finally {
                 this.finishColdOperation();
@@ -520,6 +541,10 @@ export class UserGroundHost {
         // Register is a cold declaration, not a filesystem edit. It performs one
         // complete reconciliation and publishes one graph head; it must not mint
         // a parallel stream of Detect records for the same initial contents.
+        // Freeze the official prefix before taking ground ownership. Apply may
+        // need that same ground, so draining it inside the registration scope
+        // would make the two rails wait on one another.
+        const deliveryWatermark = await this.prepareSnapshotPublication(identity, { journey: "register", workspaceId });
         let releaseGround = null;
         try {
             const groundWaitStarted = performance.now();
@@ -613,7 +638,7 @@ export class UserGroundHost {
             if (!scanned.publicationRootIds) {
                 throw new Error("cold registration reconciliation did not name its publication roots");
             }
-            await this.publishMaterializedSnapshot(identity, new Set(scanned.publicationRootIds), { journey: "register", workspaceId });
+            await this.publishMaterializedSnapshot(identity, new Set(scanned.publicationRootIds), deliveryWatermark, { journey: "register", workspaceId });
             this.watchHost.settle(observations);
         }
         catch (error) {
@@ -711,6 +736,20 @@ export class UserGroundHost {
         if (this.syncing)
             await this.syncing;
     }
+    /** Install the exact snapshot/tail boundary before Receive or Apply can
+     * treat older official history as materialization work. The SQLite cursor,
+     * retained Inbox resolution, and Apply baseline all derive from the same N. */
+    adoptCloudDeliveryBaseline(identity, deliveryWatermark, rootUUID = null) {
+        const database = initializeDatabase(this.databasePath(identity));
+        let resolved = 0;
+        try {
+            resolved = adoptSnapshotDeliveryBaseline(database, privateEntityResourceId(identity.userId, sha256Hex), deliveryWatermark, this.options.now?.().getTime() ?? Date.now());
+        }
+        finally {
+            database.close();
+        }
+        return resolved + (this.applyHost?.adoptVerifiedMaterialization(rootUUID) ?? 0);
+    }
     async startRecordRail(identity) {
         if (!this.cloudState)
             throw new Error("entity Record rail requires cloud authority");
@@ -718,12 +757,13 @@ export class UserGroundHost {
             throw new Error("entity Record rail already started");
         await this.wire.open();
         // One floor, no per-feature gates: this shell and its gateway release
-        // together, and the gateway ships first. 10 = contract-aware cargo
-        // closure; an older gateway drops the sealed-evidence frame the upload
-        // planner depends on, so nothing this shell registers could publish.
-        if (this.wire.protocolVersion < 10) {
+        // together, and the gateway ships first. 11 = snapshot delivery
+        // watermarks; an older gateway cannot atomically bind Add's graph to the
+        // official entity-record prefix it already includes.
+        if (this.wire.protocolVersion < 11) {
             throw new Error("gateway speaks an older wire protocol than this shell");
         }
+        this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
         const store = new EntityRecordSqliteStore(initializeDatabase(this.databasePath(identity)));
         const wireAuthority = createEntityRecordAuthorityPort({
             request: (frame, acceptedTypes, binaryBody) => binaryBody === undefined
@@ -1362,7 +1402,10 @@ export class UserGroundHost {
                     const sealed = await target(artifact).seal();
                     return { local: sealed, bytes: sealed.bytes, contentHash: sealed.contentHash };
                 },
-            }, { concurrency: DIRECT_OBJECT_DOWNLOAD_CONCURRENCY });
+            }, {
+                concurrency: DIRECT_OBJECT_DOWNLOAD_CONCURRENCY,
+                maxInFlightBytes: DIRECT_OBJECT_DOWNLOAD_MAX_IN_FLIGHT_BYTES,
+            });
             const content = new Map();
             for (const receipt of receipts) {
                 declaredBytes += receipt.manifest.bytes;
@@ -1758,6 +1801,7 @@ export class UserGroundHost {
             authorityEpoch: first.authorityEpoch,
             headVersion: first.baseVersion,
             checksum: "",
+            deliveryWatermark: first.deliveryWatermark,
             records: snapshotFromRecords(JSON.parse(first.snapshotJson).records || []).records,
         };
         try {
@@ -1776,13 +1820,10 @@ export class UserGroundHost {
     /** Register is the deliberate cold graph-publication boundary. Ordinary
      * Watch detection stops at a durable exact record and never rebuilds this
      * resource snapshot. */
-    async publishMaterializedSnapshot(identity, publicationRootIds, trace) {
-        // Any durable pending intent resolves before a new snapshot derives: an
-        // acknowledged publication advances the head through its original
-        // mutation id (the authority answers a known mutation id idempotently),
-        // and a conclusive refusal marks the row rejected so the snapshot
-        // derived below may supersede it. A transport failure keeps the row
-        // pending — its outcome is unknown, so nothing may replace it.
+    async prepareSnapshotPublication(identity, trace) {
+        // An older durable publication resolves before the new boundary derives.
+        // Unknown outcomes remain pending; a conclusive refusal may be replaced
+        // by the newer complete snapshot below.
         try {
             await this.publishPendingSnapshots(identity, trace);
         }
@@ -1790,13 +1831,30 @@ export class UserGroundHost {
             if (!conclusivelyRefused(error))
                 throw error;
         }
+        await this.recordRail?.flush();
+        const state = this.cloudState;
+        if (!state)
+            throw new Error("cloud state is unavailable for graph publication");
+        // Freeze N immediately after Receive reaches its explicit boundary. Apply
+        // then settles every Inbox row already known at that boundary. A later
+        // delivery may run concurrently, but the graph still declares the older
+        // N and the authority refuses it if that newer record commits first.
+        const deliveryWatermark = readAuthorityDeliveryCursor(this.databasePath(identity), state.resourceId);
+        if (deliveryWatermark < state.deliveryWatermark) {
+            throw new Error("local delivery cursor is behind the active snapshot watermark");
+        }
+        await this.applyRail?.flush();
+        return deliveryWatermark;
+    }
+    async publishMaterializedSnapshot(identity, publicationRootIds, deliveryWatermark, trace) {
         const state = this.cloudState;
         if (!state)
             throw new Error("cloud state is unavailable for graph publication");
+        const officialRecords = recordsAtDeliveryPrefix(state.records, readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark));
         const localRecords = recordsRetainingMissingEvidence(readGroundRowsForRoots(this.databasePath(identity), "entities", publicationRootIds)
             .map(portableRecord), readGroundRowsForRoots(this.databasePath(identity), "detection_notebook", publicationRootIds)
             .map(portableRecord));
-        const merged = mergeMaterializedRoots(state.records, localRecords);
+        const merged = mergeMaterializedRoots(officialRecords, localRecords);
         const traveling = travelingRecords(merged, sha256Hex);
         const impossibleGitLeaf = traveling.find((record) => (record.type === "file.text" || record.type === "file.binary" || record.type === "link")
             && record.payloadVersion?.startsWith("git:"));
@@ -1824,7 +1882,7 @@ export class UserGroundHost {
         const uploadRecords = cargo
             .filter((entry) => !sealedArtifacts.has(entry.contentHash))
             .map((entry) => entry.record);
-        if (checksum === state.checksum) {
+        if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
             // The graph is already committed; only cargo can be missing. Repair
             // uploads need no journal row — a crash simply re-runs the same
             // repair on the next Register.
@@ -1855,6 +1913,7 @@ export class UserGroundHost {
             baseVersion: state.headVersion,
             snapshotChecksum: checksum,
             snapshotJson,
+            deliveryWatermark,
             uploadRecords,
             replacementRootIds: replacement.rootIds,
             replacementRecords: replacement.records,
@@ -1921,6 +1980,7 @@ export class UserGroundHost {
                     contract: ENTITY_CLOUD_CONTRACT,
                     schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
                     operationKind: replacement.kind,
+                    deliveryWatermark: pending.deliveryWatermark,
                 }, replacement);
             }
             catch (error) {
@@ -1957,6 +2017,7 @@ export class UserGroundHost {
                 authorityEpoch: Number(frame.authority_epoch),
                 headVersion: Number(frame.version),
                 checksum: pending.snapshotChecksum,
+                deliveryWatermark: Number(frame.delivery_watermark),
                 records: snapshot.records,
             };
             deleteJournalEntry(this.databasePath(identity), pending.mutationId);
@@ -2145,6 +2206,10 @@ function initializeDatabase(file) {
       authority_id TEXT PRIMARY KEY,
       received_through INTEGER NOT NULL CHECK(received_through >= 0)
     );
+    CREATE TABLE IF NOT EXISTS authority_snapshot_baselines (
+      authority_id TEXT PRIMARY KEY,
+      delivery_sequence INTEGER NOT NULL CHECK(delivery_sequence >= 0)
+    );
     CREATE TABLE IF NOT EXISTS snapshot_publications (
       sequence INTEGER PRIMARY KEY AUTOINCREMENT,
       mutation_id TEXT NOT NULL UNIQUE,
@@ -2454,9 +2519,13 @@ function parseWorkspaceAddIntent(json) {
     const cloudHead = String(value.cloudHead || "");
     const destinationPath = String(value.destinationPath || "");
     const stagingPath = String(value.stagingPath || "");
+    const deliveryWatermark = Number(value.deliveryWatermark);
     if (!UUID.test(workspaceId) || !resourceId || !/^[0-9a-f]{64}$/i.test(cloudHead)) {
         throw new Error("workspace Add intent identity is invalid");
     }
+    if (!Number.isSafeInteger(deliveryWatermark) || deliveryWatermark < 0) {
+        throw new Error(`workspace Add intent ${workspaceId} has no delivery watermark`);
+    }
     if (!isAbsolute(destinationPath)
         || stagingPath !== workspaceAddStagingPath(destinationPath, workspaceId)) {
         throw new Error(`workspace Add intent ${workspaceId} has invalid placement`);
@@ -2465,6 +2534,7 @@ function parseWorkspaceAddIntent(json) {
         workspaceId,
         resourceId,
         cloudHead: cloudHead.toLowerCase(),
+        deliveryWatermark,
         destinationPath,
         stagingPath,
         records: snapshotFromRecords(Array.isArray(value.records) ? value.records : []).records,
@@ -2576,11 +2646,16 @@ function readSnapshotPublications(file) {
                 || !Array.isArray(publication.replacementRecords)) {
                 throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
             }
+            const deliveryWatermark = Number(publication.deliveryWatermark);
+            if (!Number.isSafeInteger(deliveryWatermark) || deliveryWatermark < 0) {
+                throw new Error(`snapshot publication ${row.mutationId} has no delivery watermark`);
+            }
             return {
                 mutationId: row.mutationId,
                 resourceId: row.resourceId,
                 authorityEpoch: Number(publication.authorityEpoch),
                 baseVersion: Number(publication.baseVersion),
+                deliveryWatermark,
                 snapshotChecksum: String(publication.snapshotChecksum),
                 snapshotJson: String(publication.snapshotJson),
                 uploadRecords: snapshotFromRecords(publication.uploadRecords).records,
@@ -2607,6 +2682,55 @@ function hasRecordsBeyondSnapshot(file) {
         database.close();
     }
 }
+function readAuthorityDeliveryCursor(file, authorityId) {
+    const database = initializeDatabase(file);
+    try {
+        const row = database.prepare(`
+      SELECT received_through AS receivedThrough
+        FROM authority_delivery_cursors WHERE authority_id = ?
+    `).get(authorityId);
+        return row?.receivedThrough ?? 0;
+    }
+    finally {
+        database.close();
+    }
+}
+/** Latest official result for every UUID in the newly received complete
+ * authority prefix. The record carries the whole logical result, so roots
+ * this machine has not materialized still advance honestly in a Register
+ * snapshot without inventing filesystem evidence. */
+function readAcceptedResults(file, authorityId, afterDelivery, throughDelivery) {
+    if (throughDelivery <= afterDelivery)
+        return [];
+    const database = initializeDatabase(file);
+    try {
+        const latest = new Map();
+        for (const { recordJson } of database.prepare(`
+      SELECT record_json AS recordJson FROM record_inbox
+       WHERE authority_id = ?
+         AND delivery_sequence > ? AND delivery_sequence <= ?
+       ORDER BY delivery_sequence
+    `).iterate(authorityId, afterDelivery, throughDelivery)) {
+            const accepted = decodeAcceptedEntityRecord(recordJson);
+            latest.set(accepted.entityId, accepted.change.result);
+        }
+        return [...latest.values()];
+    }
+    finally {
+        database.close();
+    }
+}
+function recordsAtDeliveryPrefix(base, results) {
+    if (results.length === 0)
+        return base;
+    const records = new Map(base.map((record) => [record.uuid, record]));
+    for (const result of results)
+        records.set(result.uuid, result);
+    // Entity channels are independently ordered. Container membership is the
+    // one derived relationship, so derive it once from the final prefix rather
+    // than letting record arrival order choose a container version.
+    return recordsRetainingMissingEvidence([...records.values()], []);
+}
 /** Record is the SQLite transaction that accepts Detect's exact proposals and
  * advances the last-verified notebook. Watch may settle only after this
  * function returns. The record table is the local-only outbox. */
@@ -2709,6 +2833,7 @@ function supersedeSnapshotJournal(file, entry) {
       `).run(entry.mutationId, entry.resourceId, stableJson({
                 authorityEpoch: entry.authorityEpoch,
                 baseVersion: entry.baseVersion,
+                deliveryWatermark: entry.deliveryWatermark,
                 snapshotChecksum: entry.snapshotChecksum,
                 snapshotJson: entry.snapshotJson,
                 uploadRecords: entry.uploadRecords,
@@ -4598,7 +4723,7 @@ function installCloudWorkspaceReferences(input) {
     return references;
 }
 async function installCloudWorkspace(input) {
-    const { identity, userRoot, database, cacheDir, bindingDir, resourceId, workspace, reference, references, records, destinationParent, cloudHead, readContent, onStage, } = input;
+    const { identity, userRoot, database, cacheDir, bindingDir, resourceId, workspace, reference, references, records, destinationParent, cloudHead, deliveryWatermark, readContent, onStage, } = input;
     const allLocalRows = readRows(database);
     const existingBoundary = allLocalRows.find((row) => row.uuid === workspace.uuid);
     const existing = existingBoundary
@@ -4652,6 +4777,7 @@ async function installCloudWorkspace(input) {
             workspace: activeSelection.workspace,
             references: activeReferences,
         });
+        input.adoptMaterialization(workspace.uuid);
         return {
             path: root.absolutePath,
             rows: existing.length,
@@ -4705,6 +4831,7 @@ async function installCloudWorkspace(input) {
             workspaceId: workspace.uuid,
             resourceId,
             cloudHead,
+            deliveryWatermark,
             destinationPath: destination,
             stagingPath,
             records: snapshotFromRecords(records).records,
@@ -4783,6 +4910,7 @@ function readyFromFrame(frame) {
     const cloud = {
         resourceId: String(frame.resource_id || ""),
         version: Number(frame.head_version),
+        deliveryWatermark: Number(frame.delivery_watermark),
         checksum: String(frame.snapshot_checksum || ""),
         snapshot: parseSnapshot(bytes),
     };
@@ -4793,6 +4921,7 @@ function readyFromFrame(frame) {
             authorityEpoch: Number(frame.authority_epoch),
             headVersion: cloud.version,
             checksum: cloud.checksum,
+            deliveryWatermark: cloud.deliveryWatermark,
             records: cloud.snapshot.records,
         },
     };