@amalgm/shell 0.1.136 → 0.1.138

package/PURPOSE.md

@@ -291,9 +291,13 @@ shell only assembles.
     destinations, and an interrupted Add may retain them only with disposable
     chunk evidence beside the ground database. Resume rereads and re-hashes
     every retained chunk against the pinned manifest, downloads every missing
-    or corrupt chunk, proves the ordered whole-artifact hash, and fsyncs each
-    completed file before rows or reveal are trusted. Links and repository
-    transports keep their type-specific cache and adapter rails.
+    or corrupt chunk, and proves the ordered whole-artifact hash. Durability
+    is one explicit barrier over the whole staged tree — every file,
+    duplicate, and repository — that must succeed before entity rows are
+    committed or ground is revealed; no staged file is fsynced on its own. A
+    failed barrier leaves the intent retained, no rows committed, no visible
+    destination, and staging re-proven as untrusted on retry. Links and
+    repository transports keep their type-specific cache and adapter rails.
     One recovered Add intent that cannot currently be satisfied remains
     durable and reports its exact failure, but it cannot hold unrelated machine
     readiness or other Add responsibilities hostage.

package/dist/durability-barrier.d.ts

@@ -0,0 +1,22 @@
+/**
+ * The set-level durability law for Files Add.
+ *
+ * Staged content is verified while it streams and is never trusted until it
+ * is re-proven, so no staged file needs to be durable before the next one is
+ * written. What must hold is one barrier: after every file, duplicate, and
+ * repository has been materialized, the whole staged tree is durable before
+ * entity rows are committed or ground is revealed. A failed barrier fails the
+ * Add with its intent retained — nothing committed, nothing visible — and the
+ * retry re-proves the staging as untrusted substrate.
+ *
+ * Linux settles the filesystem holding the staging root in one syncfs(2) —
+ * coreutils `sync -f` — so twelve thousand small files cost one journal
+ * flush rather than one commit each serialized through the journal. Node
+ * exposes no syncfs, so that is one child process. Elsewhere every regular
+ * file and directory under the root is flushed through its own handle:
+ * libuv's fsync is F_FULLFSYNC on macOS and FlushFileBuffers on Windows,
+ * where NTFS's own log carries the directory entries with their files. A
+ * directory-only fsync never guarantees file contents and is never used.
+ */
+export type DurabilityBarrier = (root: string) => Promise<void>;
+export declare const durabilityBarrier: DurabilityBarrier;

package/dist/durability-barrier.js

@@ -0,0 +1,72 @@
+/**
+ * The set-level durability law for Files Add.
+ *
+ * Staged content is verified while it streams and is never trusted until it
+ * is re-proven, so no staged file needs to be durable before the next one is
+ * written. What must hold is one barrier: after every file, duplicate, and
+ * repository has been materialized, the whole staged tree is durable before
+ * entity rows are committed or ground is revealed. A failed barrier fails the
+ * Add with its intent retained — nothing committed, nothing visible — and the
+ * retry re-proves the staging as untrusted substrate.
+ *
+ * Linux settles the filesystem holding the staging root in one syncfs(2) —
+ * coreutils `sync -f` — so twelve thousand small files cost one journal
+ * flush rather than one commit each serialized through the journal. Node
+ * exposes no syncfs, so that is one child process. Elsewhere every regular
+ * file and directory under the root is flushed through its own handle:
+ * libuv's fsync is F_FULLFSYNC on macOS and FlushFileBuffers on Windows,
+ * where NTFS's own log carries the directory entries with their files. A
+ * directory-only fsync never guarantees file contents and is never used.
+ */
+import { execFile } from "node:child_process";
+import { constants } from "node:fs";
+import { lstat, open, readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { boundedForEach } from "./bounded-for-each.js";
+const execFileAsync = promisify(execFile);
+/** Handles in flight while a tree is flushed entry by entry. */
+const FLUSH_CONCURRENCY = 16;
+async function syncFilesystem(root) {
+    await execFileAsync("sync", ["-f", root]);
+}
+async function flushEntry(path) {
+    const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        await handle.sync();
+    }
+    finally {
+        await handle.close();
+    }
+}
+async function syncTree(root) {
+    const files = [];
+    const directories = [root];
+    for (let index = 0; index < directories.length; index += 1) {
+        const directory = directories[index];
+        for (const entry of await readdir(directory, { withFileTypes: true })) {
+            const path = join(directory, entry.name);
+            if (entry.isDirectory())
+                directories.push(path);
+            else if (entry.isFile())
+                files.push(path);
+        }
+    }
+    const flushed = process.platform === "win32" ? files : [...files, ...directories];
+    await boundedForEach(flushed, FLUSH_CONCURRENCY, flushEntry);
+}
+const platformBarrier = process.platform === "linux" ? syncFilesystem : syncTree;
+export const durabilityBarrier = async (root) => {
+    try {
+        const stats = await lstat(root);
+        if (!stats.isDirectory() || stats.isSymbolicLink()) {
+            throw new Error("staging root is not a physical directory");
+        }
+        await platformBarrier(root);
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`durability barrier failed for ${root}: ${detail}`, { cause: error });
+    }
+};
+//# sourceMappingURL=durability-barrier.js.map

package/dist/durability-barrier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"durability-barrier.js","sourceRoot":"","sources":["../src/durability-barrier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE1C,gEAAgE;AAChE,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAI7B,KAAK,UAAU,cAAc,CAAC,IAAY;IACxC,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAW,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC3C,IAAI,KAAK,CAAC,MAAM,EAAE;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,WAAW,CAAC,CAAC;IAClF,MAAM,cAAc,CAAC,OAAO,EAAE,iBAAiB,EAAE,UAAU,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC;AAEjF,MAAM,CAAC,MAAM,iBAAiB,GAAsB,KAAK,EAAE,IAAI,EAAE,EAAE;IACjE,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,cAAc,EAAE,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,KAAK,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;AACH,CAAC,CAAC"}

package/dist/user-ground-host.d.ts

@@ -3,6 +3,7 @@ import { type EntityType, type FilesAddPorts, type FilesRegisterPorts, type User
 import Database from "better-sqlite3";
 import { type FilesStageObserver } from "./files-observability.js";
 import { type EntityApplyStageEvidence } from "./entity-apply-host.js";
+import { type DurabilityBarrier } from "./durability-barrier.js";
 import { type OpenNativeWatch, type WatchCoverageEvidence, type WatchHealth } from "./watching/index.js";
 interface GroundRow {
     readonly resourceId: string;
@@ -73,6 +74,9 @@ export interface UserGroundHostOptions {
     readonly onWatchHealth?: (health: WatchHealth) => void;
     /** Host injection used by platform certification and failure examinations. */
     readonly openWatch?: OpenNativeWatch;
+    /** The one durability barrier Add takes over its staged tree before rows or
+     * reveal; injected only by failure examinations. */
+    readonly durabilityBarrier?: DurabilityBarrier;
     /** Root-scoped Detect evidence for product diagnostics and examinations. */
     readonly onDetectScan?: (evidence: DetectScanEvidence) => void;
     /** Transient Apply progress; durable responsibility remains in SQLite. */

package/dist/user-ground-host.js

@@ -15,6 +15,7 @@ import { ContentCacheDownloadSet, captureContentFile, contentBlockCollector, has
 import { RegisterUploadFusion } from "./register-upload-fusion.js";
 import { StagedContentSet, addStagingEvidenceDir, clearAddStagingEvidence, } from "./workspace-add-staging.js";
 import { boundedForEach } from "./bounded-for-each.js";
+import { durabilityBarrier as platformDurabilityBarrier, } from "./durability-barrier.js";
 import { DirectContentPlane } from "./content-direct.js";
 import { encodeContentWireBytes } from "./content-wire-codec.js";
 import { TransferLegs } from "./transfer-instrumentation.js";
@@ -44,6 +45,12 @@ const CARGO_SEGMENT_LOAD_CONCURRENCY = 8;
  * few broad ranged reads. Memory is bounded by the byte laws below, never by
  * this count. */
 const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = CONTENT_REFS_BATCH_OBJECTS;
+/** Independent destination writers. Pack reads are intentionally few, but
+ * one pack may contain thousands of unrelated files; those files must not
+ * inherit the pack's request count as their filesystem concurrency. Sixteen
+ * handles are bounded on a 1 GiB runtime and enough to hide metadata latency
+ * behind the volume's byte throughput. */
+const DIRECT_CONTENT_STORE_CONCURRENCY = 16;
 // Opening one protected object transiently retains the fetched envelope,
 // WebCrypto's ciphertext-with-tag input, and the plaintext. Bound plaintext
 // to two full blocks so those three representations stay below 96 MiB even
@@ -388,6 +395,7 @@ export class UserGroundHost {
                         cloudHead: input.cloudHead,
                         deliveryWatermark: input.deliveryWatermark,
                         readContent: (artifacts, staged) => this.downloadContentSet(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifacts.map((artifact) => ({ artifact })), 10, { journey: "add", workspaceId: input.workspace.uuid }, staged),
+                        durabilityBarrier: this.options.durabilityBarrier ?? platformDurabilityBarrier,
                         adoptMaterialization: (rootUUID) => {
                             const settled = this.adoptCloudDeliveryBaseline(identity, input.deliveryWatermark, rootUUID);
                             this.applyRail?.recordsAvailable();
@@ -517,6 +525,7 @@ export class UserGroundHost {
                 cloudHead: intent.cloudHead,
                 deliveryWatermark: intent.deliveryWatermark,
                 readContent: (artifacts, staged) => this.downloadContentSet(intent.resourceId, this.cacheDir(identity), artifacts.map((artifact) => ({ artifact })), 10, { journey: "add", workspaceId: intent.workspaceId }, staged),
+                durabilityBarrier: this.options.durabilityBarrier ?? platformDurabilityBarrier,
                 adoptMaterialization: (rootUUID) => {
                     this.adoptCloudDeliveryBaseline(identity, intent.deliveryWatermark, rootUUID);
                     this.applyRail?.recordsAvailable();
@@ -1962,6 +1971,7 @@ export class UserGroundHost {
                 },
             }, {
                 concurrency: DIRECT_OBJECT_DOWNLOAD_CONCURRENCY,
+                storeConcurrency: DIRECT_CONTENT_STORE_CONCURRENCY,
                 maxInFlightBytes: DIRECT_OBJECT_DOWNLOAD_MAX_IN_FLIGHT_BYTES,
             });
             const content = new Map();
@@ -6073,7 +6083,7 @@ function installCloudWorkspaceReferences(input) {
     return references;
 }
 async function installCloudWorkspace(input) {
-    const { identity, userRoot, database, cacheDir, bindingDir, resourceId, workspace, reference, references, records, destinationParent, cloudHead, deliveryWatermark, readContent, onStage, } = input;
+    const { identity, userRoot, database, cacheDir, bindingDir, resourceId, workspace, reference, references, records, destinationParent, cloudHead, deliveryWatermark, readContent, durabilityBarrier, onStage, } = input;
     const existing = readGroundClosureRows(database, workspace.uuid);
     const existingBoundary = existing.find((row) => row.uuid === workspace.uuid);
     const pending = readWorkspaceAddIntent(database, workspace.uuid);
@@ -6229,11 +6239,19 @@ async function installCloudWorkspace(input) {
             ? join(intent.destinationPath, ...row.relativePath.split("/"))
             : intent.destinationPath,
     }));
+    const graphMs = Math.round(performance.now() - graphStarted);
+    // Every staged byte was proven as it streamed; none of it counts as durable
+    // until the whole tree — files, duplicates, repositories — passes one
+    // barrier. Rows and reveal follow that barrier, never precede it; a failed
+    // barrier leaves the intent retained and the staging untrusted.
+    const barrierStarted = performance.now();
+    await durabilityBarrier(intent.stagingPath);
     // The milestone's duration spans the whole materialization; its legs say
     // where that wall time went after the content set itself.
     await onStage?.("materialization-verified", {
         ...materialized.timing,
-        graphMs: Math.round(performance.now() - graphStarted),
+        graphMs,
+        durabilityBarrierMs: Math.round(performance.now() - barrierStarted),
         repositories: materialized.repositories.length,
     });
     persistRows(database, identity, intent.resourceId, intent.workspaceId, rows);