@superbee/core 0.2.0-pre.1 → 0.2.0-pre.10

package/dist/filesystem-identity.js

@@ -15,6 +15,15 @@
  *   entirely between two probes of one observation can be traversed by the operating system
  *   during a listing or open, but its bytes can never be returned as EXACT, because the leaf
  *   witness at the post-walk must match the entry the requested path reaches.
+ * - A bulk read runs its observations inside one {@link ObservationPass}: the same protocol per
+ *   path, but a directory's listing is taken once and reused while the directory's change
+ *   witness proves its entry set unchanged, and every EXACT verdict that used a reused listing has
+ *   its post-walk repeated at the end of the pass without trusting timestamps. Listing the parent
+ *   per observation would make a pass over a directory of N documents read O(N²) directory
+ *   entries. The speedup assumes local disks, whose directory times advance on every entry change;
+ *   on Linux any other filesystem type lists per use, and elsewhere a mount whose times can lie
+ *   (network or FUSE attribute caches, settable Windows directory times) is covered by the
+ *   end-of-pass re-verification, short of an inode number reused within the pass.
  * - Mutations hold one cross-process lock keyed by a pure fold of the root and `rel`, decide
  *   inside it, create directories one segment at a time with a spelling check, and publish a
  *   first-created leaf with `link` so the host's own equivalence refuses a second spelling the
@@ -37,8 +46,8 @@
  * and the root's spelling is folded into the lock key rather than verified.
  *
  * Every filesystem call goes through a {@link FilesystemIdentityPort} passed to the protocol
- * functions. Production binds {@link nodeFilesystemIdentityPort} once, as a module constant;
- * nothing on a backend instance can substitute it. Normalizing stores that rewrite names on
+ * functions. Production constructs a Node binding with the backend's captured host policy;
+ * the policy supplies classifications, never a replacement identity protocol. Normalizing stores that rewrite names on
  * write (legacy HFS+) are unsupported: an id whose NFD form differs from its own form is written
  * and then refused as an alias on every later observation, never silently aliased.
  */
@@ -47,12 +56,12 @@ import path from "node:path";
 import { createHash, randomUUID } from "node:crypto";
 import { ConcurrentReplacementError, FilesystemIdentityAliasError, InvalidInputError } from "./errors.js";
 import { acquireFilesystemIdentityLock } from "./filesystem-lock.js";
-const WINDOWS_REPLACE_CONFLICT_CODES = new Set(["EACCES", "EBUSY", "EPERM"]);
+import { captureFilesystemHostPolicy } from "./filesystem-host.js";
 /**
- * Windows can transiently deny replacement while another process is closing a read handle or a
+ * A host can transiently deny replacement while another process is closing a read handle or a
  * scanner holds the destination. An in-place rename retry cannot remain bound to the complete
  * parent-and-leaf observation: either can change after any final check. Abort this attempt instead,
- * mapping only the errno values Node uses for a Windows replacement sharing violation to the typed
+ * mapping only a host-classified replacement conflict to the typed
  * contention signal owned by the higher-level fresh read/decide/CAS loop. Every other platform and
  * errno remains the original immediate failure.
  */
@@ -61,8 +70,7 @@ async function replaceExistingLeaf(port, tmp, target, rel) {
         await port.rename(tmp, target);
     }
     catch (err) {
-        const code = err.code ?? "";
-        if ((port.platform ?? process.platform) === "win32" && WINDOWS_REPLACE_CONFLICT_CODES.has(code)) {
+        if (port.isReplacementConflict?.(err)) {
             throw new ConcurrentReplacementError(rel, 1, 10);
         }
         throw err;
@@ -219,6 +227,13 @@ async function probeSegment(port, candidate, rel, segment) {
 function hasExact(listing, segment) {
     return listing.some((entry) => entry.name === segment);
 }
+/** One fresh `entries` call per use: the single-read protocol. */
+function freshListings(port) {
+    return async (dir) => {
+        const listing = await port.entries(dir);
+        return listing === null ? null : { has: (name) => hasExact(listing, name) };
+    };
+}
 /**
  * M-REALIZE: per segment, root to leaf, it must exist and the parent listing must carry its exact
  * spelling. A listing-only alias decision is sound here because the identity lock is held: no
@@ -277,15 +292,15 @@ async function recordWalk(port, rootResolved, segments, rel) {
  * (`handle` given) applies that leaf table against the handle's inode, probing the leaf
  * unconditionally.
  */
-async function confirmedWalk(port, rootResolved, segments, rel, recorded, handle) {
+async function confirmedWalk(port, rootResolved, segments, rel, recorded, handle, listings = freshListings(port)) {
     let parent = rootResolved;
     for (let index = 0; index < segments.length; index++) {
         const segment = segments[index];
         const candidate = path.join(parent, segment);
-        const listing = await port.entries(parent);
+        const listing = await listings(parent);
         if (listing === null)
             return "absent";
-        const listed = hasExact(listing, segment);
+        const listed = listing.has(segment);
         if (handle !== null && index === segments.length - 1) {
             const verdict = classifyLeaf({ listed, probe: await probeSegment(port, candidate, rel, segment), handle });
             if (verdict === "aliased")
@@ -316,14 +331,17 @@ function countRestart(restarts, rel) {
  * from O-PROBE, bounded; the handle is closed before any restart opens another.
  */
 export async function observeExact(port, root, rel, read) {
+    return observeWith(port, path.resolve(root), rel, read, freshListings(port));
+}
+/** The observation protocol of {@link observeExact}, with the parent listings drawn from `listings`. */
+async function observeWith(port, rootResolved, rel, read, listings, onExact) {
     const segments = relSegments(rel);
-    const rootResolved = path.resolve(root);
     const target = path.join(rootResolved, ...segments);
     for (let restarts = 0;;) {
         const recorded = await recordWalk(port, rootResolved, segments, rel);
         if (recorded === null)
             return { state: "absent" };
-        const verified = await confirmedWalk(port, rootResolved, segments, rel, recorded, null);
+        const verified = await confirmedWalk(port, rootResolved, segments, rel, recorded, null, listings);
         if (verified === "absent")
             return { state: "absent" };
         if (verified === "restart") {
@@ -337,10 +355,9 @@ export async function observeExact(port, root, rel, read) {
         catch (err) {
             if (isAbsentPathError(err))
                 return { state: "absent" };
-            // Win32 can report EPERM when a concurrent delete/replace briefly denies an open. That is
-            // an uncertain generation, not a durable permission verdict; re-run the witnessed walk and
+            // A host-classified transient open failure reports an uncertain generation; re-run the witnessed walk and
             // let the existing bound turn persistent contention into ConcurrentReplacementError.
-            if (process.platform === "win32" && err.code === "EPERM") {
+            if (port.isTransientOpenError?.(err)) {
                 restarts = countRestart(restarts, rel);
                 continue;
             }
@@ -352,13 +369,15 @@ export async function observeExact(port, root, rel, read) {
             value = await read(opened.handle, target);
             if (value === null)
                 return { state: "absent" };
-            verdict = await confirmedWalk(port, rootResolved, segments, rel, recorded, opened);
+            verdict = await confirmedWalk(port, rootResolved, segments, rel, recorded, opened, listings);
         }
         finally {
             await port.close(opened.handle);
         }
-        if (verdict === "exact")
+        if (verdict === "exact") {
+            onExact?.(recorded, { dev: opened.dev, ino: opened.ino });
             return { state: "exact", value };
+        }
         if (verdict === "absent")
             return { state: "absent" };
         restarts = countRestart(restarts, rel);
@@ -385,6 +404,154 @@ export async function probeExact(port, root, rel) {
         restarts = countRestart(restarts, rel);
     }
 }
+// ── observation pass ──────────────────────────────────────────────────────────
+/**
+ * How long a directory's modification and status-change times must have been still before a
+ * listing of it may be reused. A change that lands in the same timestamp tick as the directory's
+ * previous change leaves both times unchanged, so a witness is only a change detector once the
+ * clock has moved past that tick. Three seconds covers the coarsest supported granularity (FAT's
+ * two-second mtime) plus margin; finer clocks (APFS, ext4, NTFS) are covered with room to spare.
+ * A time in the future (clock skew) is never settled.
+ */
+const SETTLED_AFTER_NS = 3000000000n;
+function witnessKey(witness) {
+    return `${witness.dev}:${witness.ino}:${witness.mtimeNs}:${witness.ctimeNs}`;
+}
+/**
+ * Open an {@link ObservationPass} over `(port, root)`.
+ *
+ * Every observation runs the exact single-read protocol of {@link observeExact} — recorded walk,
+ * confirmed pre-walk, open, read, confirmed post-walk with the handle still open, bounded
+ * restarts — and differs only in where a walk's parent listing comes from. The first time a
+ * directory is needed the pass takes `witness, entries, witness`; when the two witnesses agree and
+ * the directory's times have settled, that listing is kept. Each later use re-takes the witness
+ * (one `stat`, constant cost) and reuses the kept listing only if the witness is unchanged;
+ * otherwise it lists afresh exactly as a single read would.
+ *
+ * Two independent arguments make a reused listing safe:
+ *
+ * 1. Timestamps (the fast path is right in the moment). Where a directory's times advance on every
+ *    entry change — local APFS, HFS+, ext4, XFS, btrfs, NTFS, FAT — an unchanged settled witness
+ *    means the entry set is the kept listing at the moment of use, so every verdict is the one a
+ *    single read reaches against the same state.
+ * 2. Re-verification (no timestamps trusted). Where the witness can lie — attribute caches on
+ *    network and FUSE mounts, settable directory times on Windows, a clock stepped backwards — an
+ *    observation that used a reused listing may be wrong in the moment, so its verdict is not
+ *    trusted: a refusal or absence reached through a reused listing is re-taken at once by
+ *    {@link observeExact}, and `finish` repeats the post-walk of every EXACT verdict that used one
+ *    against fresh listings (one per directory) and fresh probes. Each segment must still be
+ *    listed under its exact spelling and reach the inode recorded for it, the leaf the inode the
+ *    bytes were read from; anything else is returned for a fresh single read. Since published
+ *    bytes are immutable per inode, a verdict that passes is a state of this identity at `finish`.
+ *    What this cannot see is the read file being deleted and its inode NUMBER reused by a new file
+ *    under the same spelling within the pass, because the handle is closed by then.
+ *
+ * On Linux, a mount whose filesystem type is not a known local one (NFS, SMB, FUSE, 9p, ...) gets
+ * no directory witness at all, so it lists exactly as single reads do.
+ */
+export function openObservationPass(port, root, now = Date.now) {
+    const rootResolved = path.resolve(root);
+    const kept = new Map();
+    const reusedVerdicts = [];
+    const fresh = freshListings(port);
+    const witnessOf = port.directoryWitness?.bind(port);
+    const settled = (witness) => {
+        const nowNs = BigInt(Math.floor(now())) * 1000000n;
+        const latest = witness.mtimeNs > witness.ctimeNs ? witness.mtimeNs : witness.ctimeNs;
+        return nowNs - latest >= SETTLED_AFTER_NS;
+    };
+    /** Listings for one observation; `reused()` says whether any came from the kept set. */
+    const listingsFor = () => {
+        let reused = false;
+        if (witnessOf === undefined)
+            return { source: fresh, reused: () => false };
+        const source = async (dir) => {
+            const keep = kept.get(dir);
+            if (keep !== undefined) {
+                const current = await witnessOf(dir);
+                if (current !== null && witnessKey(current) === keep.witness) {
+                    reused = true;
+                    return keep.names;
+                }
+                kept.delete(dir);
+            }
+            const before = await witnessOf(dir);
+            const listing = await port.entries(dir);
+            if (listing === null)
+                return null;
+            const names = new Set(listing.map((entry) => entry.name));
+            const after = await witnessOf(dir);
+            if (before !== null && after !== null && witnessKey(before) === witnessKey(after) && settled(after)) {
+                kept.set(dir, { witness: witnessKey(after), names });
+            }
+            return names;
+        };
+        return { source, reused: () => reused };
+    };
+    return {
+        async observe(rel, read) {
+            const listings = listingsFor();
+            let exact;
+            let observed;
+            try {
+                observed = await observeWith(port, rootResolved, rel, read, listings.source, (recorded, handle) => {
+                    exact = { rel, segments: relSegments(rel), recorded, handle };
+                });
+            }
+            catch (error) {
+                if (!listings.reused())
+                    throw error;
+                return observeExact(port, rootResolved, rel, read);
+            }
+            if (!listings.reused())
+                return observed;
+            if (observed.state === "absent")
+                return observeExact(port, rootResolved, rel, read);
+            if (exact !== undefined)
+                reusedVerdicts.push(exact);
+            return observed;
+        },
+        async finish() {
+            const retry = new Set();
+            const endListings = new Map();
+            const listedAtEnd = async (dir) => {
+                if (!endListings.has(dir)) {
+                    const listing = await port.entries(dir);
+                    endListings.set(dir, listing === null ? null : new Set(listing.map((entry) => entry.name)));
+                }
+                return endListings.get(dir);
+            };
+            try {
+                for (const verdict of reusedVerdicts) {
+                    if (!(await holdsAtEnd(port, rootResolved, verdict, listedAtEnd)))
+                        retry.add(verdict.rel);
+                }
+            }
+            finally {
+                kept.clear();
+                reusedVerdicts.length = 0;
+            }
+            return [...retry];
+        },
+    };
+}
+/** The post-walk of one EXACT verdict, repeated against end-of-pass listings and fresh probes. */
+async function holdsAtEnd(port, rootResolved, verdict, listedAtEnd) {
+    let parent = rootResolved;
+    for (let index = 0; index < verdict.segments.length; index++) {
+        const segment = verdict.segments[index];
+        const listing = await listedAtEnd(parent);
+        if (listing === null || !listing.has(segment))
+            return false;
+        const candidate = path.join(parent, segment);
+        const probed = await port.probe(candidate);
+        const expected = index === verdict.segments.length - 1 ? verdict.handle : verdict.recorded[index];
+        if (probed === null || probed.kind === "symlink" || !sameWitness(probed, expected))
+            return false;
+        parent = candidate;
+    }
+    return true;
+}
 // ── mutation ──────────────────────────────────────────────────────────────────
 /**
  * Roots (resolved) whose filesystem refused a hard link. Learned lazily on the first creation
@@ -394,7 +561,15 @@ export async function probeExact(port, root, rel) {
  * first-creation writers of a host-equated pair the fold misses are not excluded" applies there
  * and only there.
  */
-const linkUnsupportedRoots = new Set();
+const linkUnsupportedRoots = new WeakMap();
+function unsupportedLinkRoots(port) {
+    let roots = linkUnsupportedRoots.get(port);
+    if (!roots) {
+        roots = new Set();
+        linkUnsupportedRoots.set(port, roots);
+    }
+    return roots;
+}
 /**
  * Process-local queue per identity key, shared by every `FilesystemBackend` instance: the
  * engine constructs a fresh adapter per bundle operation, so an instance-level map would
@@ -508,7 +683,7 @@ async function createExactLeaf(port, tmp, target, rel, leaf, rootResolved) {
         return;
     }
     if (outcome === "unsupported") {
-        linkUnsupportedRoots.add(rootResolved);
+        unsupportedLinkRoots(port).add(rootResolved);
         await port.rename(tmp, target);
         return;
     }
@@ -556,7 +731,7 @@ export async function mutateExact(port, root, rel, body) {
                             await replaceExistingLeaf(port, tmp, target, rel);
                             return;
                         }
-                        if (linkUnsupportedRoots.has(rootResolved)) {
+                        if (unsupportedLinkRoots(port).has(rootResolved)) {
                             await port.rename(tmp, target);
                             return;
                         }
@@ -588,88 +763,149 @@ function kindOf(stats) {
         return "file";
     return "other";
 }
-/** The one production binding: `node:fs`, the temp-file convention, and the identity lock. */
-export const nodeFilesystemIdentityPort = Object.freeze({
-    platform: process.platform,
-    async probe(target) {
-        let lstats;
-        try {
-            lstats = await fs.lstat(target);
-        }
-        catch (err) {
-            if (isAbsentPathError(err))
-                return null;
-            throw err;
-        }
-        return { kind: kindOf(lstats), dev: lstats.dev, ino: lstats.ino };
-    },
-    async entries(dir) {
-        try {
-            const dirents = await fs.readdir(dir, { withFileTypes: true });
-            return dirents.map((dirent) => ({ name: dirent.name, kind: kindOf(dirent) }));
+/**
+ * Linux filesystem types (`statfs` magic numbers) whose directory times are kept by the local
+ * kernel and advance on every entry change. Anything else on Linux — NFS, SMB/CIFS, FUSE, 9p,
+ * Ceph, and every type not named here — gets no directory witness, so an observation pass lists it
+ * exactly as single reads do. Other platforms expose no stable type through Node, so their
+ * witnesses are offered and the pass's end-of-pass re-verification is what covers a remote mount.
+ */
+const LOCAL_LINUX_FILESYSTEMS = new Set([
+    0xef53, // ext2/3/4
+    0x58465342, // xfs
+    0x9123683e, // btrfs
+    0x01021994, // tmpfs
+    0x858458f6, // ramfs
+    0x794c7630, // overlayfs
+    0x2fc12fc1, // zfs
+    0xf2f52010, // f2fs
+    0xca451a4e, // bcachefs
+    0x4d44, // vfat
+    0x2011bab0, // exfat
+    0x7366746e, // ntfs3
+    0x482b, // hfsplus
+]);
+/** Whether a directory witness may be offered on `platform` for a filesystem of `filesystemType`. */
+export function directoryWitnessTrusted(platform, filesystemType) {
+    if (platform !== "linux")
+        return true;
+    return filesystemType !== null && LOCAL_LINUX_FILESYSTEMS.has(filesystemType);
+}
+/** Whether `dir`'s filesystem may offer directory witnesses, asked once per device per port. */
+function witnessTrust() {
+    const devices = new Map();
+    return (dev, dir) => {
+        if (process.platform !== "linux")
+            return Promise.resolve(true);
+        let trusted = devices.get(dev);
+        if (trusted === undefined) {
+            trusted = fs.statfs(dir).then((info) => directoryWitnessTrusted("linux", Number(info.type)), () => false);
+            devices.set(dev, trusted);
         }
-        catch (err) {
-            if (isAbsentPathError(err))
+        return trusted;
+    };
+}
+/** The one production binding: `node:fs`, the temp-file convention, and the identity lock. */
+export function createNodeFilesystemIdentityPort(hostPolicy) {
+    const policy = captureFilesystemHostPolicy(hostPolicy);
+    const witnessTrustedOn = witnessTrust();
+    return Object.freeze({
+        isTransientOpenError: policy.isTransientOpenError,
+        isReplacementConflict: policy.isReplacementConflict,
+        async probe(target) {
+            let lstats;
+            try {
+                lstats = await fs.lstat(target);
+            }
+            catch (err) {
+                if (isAbsentPathError(err))
+                    return null;
+                throw err;
+            }
+            return { kind: kindOf(lstats), dev: lstats.dev, ino: lstats.ino };
+        },
+        async entries(dir) {
+            try {
+                const dirents = await fs.readdir(dir, { withFileTypes: true });
+                return dirents.map((dirent) => ({ name: dirent.name, kind: kindOf(dirent) }));
+            }
+            catch (err) {
+                if (isAbsentPathError(err))
+                    return null;
+                throw err;
+            }
+        },
+        async directoryWitness(dir) {
+            let stats;
+            try {
+                stats = await fs.stat(dir, { bigint: true });
+            }
+            catch (err) {
+                if (isAbsentPathError(err))
+                    return null;
+                throw err;
+            }
+            if (!(await witnessTrustedOn(stats.dev, dir)))
                 return null;
-            throw err;
-        }
-    },
-    async open(target) {
-        const handle = await fs.open(target, "r");
-        try {
-            const stats = await handle.stat();
-            return { handle, dev: stats.dev, ino: stats.ino };
-        }
-        catch (err) {
-            await handle.close().catch(() => { });
-            throw err;
-        }
-    },
-    readAll(handle) {
-        return handle.readFile();
-    },
-    close(handle) {
-        return handle.close();
-    },
-    async stat(target) {
-        return { mtime: (await fs.stat(target)).mtime };
-    },
-    async mkdir(dir) {
-        try {
-            await fs.mkdir(dir);
-            return "created";
-        }
-        catch (err) {
-            if (err.code === "EEXIST")
-                return "exists";
-            throw err;
-        }
-    },
-    async writeTemp(dir, name, bytes) {
-        await fs.writeFile(path.join(dir, name), bytes);
-    },
-    async link(from, to) {
-        try {
-            await fs.link(from, to);
-            return "linked";
-        }
-        catch (err) {
-            const code = err.code;
-            if (code === "EEXIST")
-                return "exists";
-            if (code === "EPERM" || code === "ENOTSUP" || code === "EOPNOTSUPP" || code === "EXDEV")
-                return "unsupported";
-            throw err;
-        }
-    },
-    async rename(from, to) {
-        await fs.rename(from, to);
-    },
-    async unlink(target) {
-        await fs.unlink(target);
-    },
-    claim(key, identity) {
-        return acquireFilesystemIdentityLock(key, `${identity.root}:${identity.rel}`, { portableRoot: identity.root });
-    },
-});
+            return { dev: stats.dev, ino: stats.ino, mtimeNs: stats.mtimeNs, ctimeNs: stats.ctimeNs };
+        },
+        async open(target) {
+            const handle = await fs.open(target, "r");
+            try {
+                const stats = await handle.stat();
+                return { handle, dev: stats.dev, ino: stats.ino };
+            }
+            catch (err) {
+                await handle.close().catch(() => { });
+                throw err;
+            }
+        },
+        readAll(handle) {
+            return handle.readFile();
+        },
+        close(handle) {
+            return handle.close();
+        },
+        async stat(target) {
+            return { mtime: (await fs.stat(target)).mtime };
+        },
+        async mkdir(dir) {
+            try {
+                await fs.mkdir(dir);
+                return "created";
+            }
+            catch (err) {
+                if (err.code === "EEXIST")
+                    return "exists";
+                throw err;
+            }
+        },
+        async writeTemp(dir, name, bytes) {
+            await fs.writeFile(path.join(dir, name), bytes);
+        },
+        async link(from, to) {
+            try {
+                await fs.link(from, to);
+                return "linked";
+            }
+            catch (err) {
+                const code = err.code;
+                if (code === "EEXIST")
+                    return "exists";
+                if (code === "EPERM" || code === "ENOTSUP" || code === "EOPNOTSUPP" || code === "EXDEV")
+                    return "unsupported";
+                throw err;
+            }
+        },
+        async rename(from, to) {
+            await fs.rename(from, to);
+        },
+        async unlink(target) {
+            await fs.unlink(target);
+        },
+        claim(key, identity) {
+            return acquireFilesystemIdentityLock(key, `${identity.root}:${identity.rel}`, { portableRoot: identity.root, hostPolicy: policy });
+        },
+    });
+}
 //# sourceMappingURL=filesystem-identity.js.map