botmux 3.6.0 → 3.7.0

package/dist/utils/file-lock.js

@@ -24,7 +24,7 @@
  * don't need it and the equality check would re-open the stale-break race.)
  */
 import { closeSync, constants, fstatSync, linkSync, lstatSync, openSync, promises as fsp, readFileSync, readdirSync, unlinkSync, writeFileSync, } from 'node:fs';
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { basename, dirname, join } from 'node:path';
 import { readLinuxBootIdentity, readProcessStartIdentity } from '../core/session-marker.js';
 import { logger } from './logger.js';
@@ -42,6 +42,11 @@ const MIN_INVALID_HOLDER_STALE_AGE_MS = 1_000;
 const STALE_CLAIM_PREFIX = '.botmux-stale-claim-';
 const STALE_CLAIM_OWNER_SUFFIX = '.owner-';
 const STALE_CLAIM_OWNER_WIDTH = 12;
+const STALE_CLAIM_OWNER_CANDIDATE_SUFFIX = '.candidate-';
+let fileLockTestHooks;
+export function __testOnly_setFileLockHooks(hooks) {
+    fileLockTestHooks = hooks;
+}
 let selfProcStartResolved = false;
 let selfProcStart;
 let selfBootIdResolved = false;
@@ -157,24 +162,28 @@ async function releaseOwnedLock(lockPath, owned) {
         const current = await fsp.lstat(lockPath);
         if (current.isFile() && !current.isSymbolicLink() && sameInode(current, owned)) {
             await fsp.unlink(lockPath);
+            return true;
         }
     }
     catch (error) {
         if (error.code !== 'ENOENT')
             throw error;
     }
+    return false;
 }
 function releaseOwnedLockSync(lockPath, owned) {
     try {
         const current = lstatSync(lockPath);
         if (current.isFile() && !current.isSymbolicLink() && sameInode(current, owned)) {
             unlinkSync(lockPath);
+            return true;
         }
     }
     catch (error) {
         if (error.code !== 'ENOENT')
             throw error;
     }
+    return false;
 }
 function staleClaimPathFor(lockPath, observed) {
     // link(2) changes ctime, so the generation deliberately uses only identity
@@ -206,17 +215,87 @@ function parseStaleClaimOwnerEpoch(claimPath, name) {
     const epoch = Number(raw);
     return Number.isSafeInteger(epoch) ? epoch : undefined;
 }
+function staleClaimOwnerCandidatePath(ownerPath) {
+    return `${ownerPath}${STALE_CLAIM_OWNER_CANDIDATE_SUFFIX}${process.pid}-${randomUUID()}`;
+}
+function isStaleClaimOwnerCandidate(claimPath, name) {
+    const prefix = `${basename(claimPath)}${STALE_CLAIM_OWNER_SUFFIX}`;
+    if (!name.startsWith(prefix))
+        return false;
+    const suffix = name.slice(prefix.length);
+    return new RegExp(`^\\d{${STALE_CLAIM_OWNER_WIDTH}}\\${STALE_CLAIM_OWNER_CANDIDATE_SUFFIX}` +
+        '\\d+-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$').test(suffix);
+}
+function pinnedHolderMetadataState(before, after) {
+    if (!before.isFile() || !after.isFile() || !sameInode(before, after))
+        return 'untrusted';
+    const sizeChanged = before.size !== after.size;
+    const mtimeChanged = before.mtimeMs !== after.mtimeMs;
+    const ctimeChanged = before.ctimeMs !== after.ctimeMs;
+    const linkCountChanged = before.nlink !== after.nlink;
+    // Compatibility with an older publisher that exposed the O_EXCL inode
+    // before writing its identity. Never trust either snapshot, but let the
+    // caller retry once the payload has become stable.
+    if (before.size === 0 && (sizeChanged || mtimeChanged || ctimeChanged)) {
+        return 'transient';
+    }
+    // Content-relevant metadata changing on a non-empty identity is not a
+    // normal owner lifecycle transition. A size or mtime change fails closed
+    // here. Note we cannot detect a same-size rewrite that also restores mtime
+    // purely from metadata: ctime often does not advance within it at ms
+    // resolution (e.g. ext4), so ctime is not a reliable content-tamper signal.
+    // The legitimate owner protocol never rewrites an epoch in place (O_EXCL
+    // once, then a new epoch pathname), so this residual window is defense in
+    // depth rather than a live hazard.
+    if (sizeChanged || mtimeChanged)
+        return 'untrusted';
+    // link(2)/unlink(2) legitimately change ctime and nlink without touching
+    // bytes. This means the pinned inode lost a publication/cleanup race; do not
+    // use the observation, and do not surface it as a business error.
+    if (linkCountChanged)
+        return 'transient';
+    if (ctimeChanged)
+        return 'untrusted';
+    if (before.nlink < 1 || after.nlink < 1)
+        return 'transient';
+    return 'stable';
+}
+function pinnedHolderIntegrityError() {
+    return new Error('file-lock stale-claim owner changed while reading');
+}
 async function readPinnedHolder(path) {
     const handle = await fsp.open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
     try {
         const before = await handle.stat();
+        if (fileLockTestHooks?.afterPinnedHolderFirstStat) {
+            await fileLockTestHooks.afterPinnedHolderFirstStat(path);
+        }
         const raw = await handle.readFile('utf8');
         const after = await handle.stat();
-        if (!before.isFile() || !sameInode(before, after) || before.size !== after.size ||
-            before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) {
-            throw new Error('file-lock stale-claim owner changed while reading');
+        const pinnedState = pinnedHolderMetadataState(before, after);
+        if (pinnedState === 'transient')
+            return undefined;
+        if (pinnedState === 'untrusted')
+            throw pinnedHolderIntegrityError();
+        let current;
+        try {
+            current = await fsp.lstat(path);
         }
-        return { holder: parseLockHolder(raw), ageMs: Date.now() - after.mtimeMs, stats: after };
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return undefined;
+            throw error;
+        }
+        if (!current.isFile() || current.isSymbolicLink())
+            throw pinnedHolderIntegrityError();
+        if (!sameInode(after, current))
+            return undefined;
+        const pathnameState = pinnedHolderMetadataState(after, current);
+        if (pathnameState === 'transient')
+            return undefined;
+        if (pathnameState === 'untrusted')
+            throw pinnedHolderIntegrityError();
+        return { holder: parseLockHolder(raw), ageMs: Date.now() - current.mtimeMs, stats: current };
     }
     finally {
         await handle.close();
@@ -226,18 +305,76 @@ function readPinnedHolderSync(path) {
     const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
     try {
         const before = fstatSync(fd);
+        fileLockTestHooks?.afterPinnedHolderFirstStatSync?.(path);
         const raw = readFileSync(fd, 'utf8');
         const after = fstatSync(fd);
-        if (!before.isFile() || !sameInode(before, after) || before.size !== after.size ||
-            before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) {
-            throw new Error('file-lock stale-claim owner changed while reading');
+        const pinnedState = pinnedHolderMetadataState(before, after);
+        if (pinnedState === 'transient')
+            return undefined;
+        if (pinnedState === 'untrusted')
+            throw pinnedHolderIntegrityError();
+        let current;
+        try {
+            current = lstatSync(path);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return undefined;
+            throw error;
         }
-        return { holder: parseLockHolder(raw), ageMs: Date.now() - after.mtimeMs, stats: after };
+        if (!current.isFile() || current.isSymbolicLink())
+            throw pinnedHolderIntegrityError();
+        if (!sameInode(after, current))
+            return undefined;
+        const pathnameState = pinnedHolderMetadataState(after, current);
+        if (pathnameState === 'transient')
+            return undefined;
+        if (pathnameState === 'untrusted')
+            throw pinnedHolderIntegrityError();
+        return { holder: parseLockHolder(raw), ageMs: Date.now() - current.mtimeMs, stats: current };
     }
     finally {
         closeSync(fd);
     }
 }
+async function unlinkUnchangedPinnedPath(path, pinned) {
+    let current;
+    try {
+        current = await fsp.lstat(path);
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return false;
+        throw error;
+    }
+    if (!current.isFile() || current.isSymbolicLink())
+        throw pinnedHolderIntegrityError();
+    if (!sameInode(current, pinned))
+        return false;
+    if (pinnedHolderMetadataState(pinned, current) !== 'stable')
+        return false;
+    await fsp.unlink(path);
+    return true;
+}
+function unlinkUnchangedPinnedPathSync(path, pinned) {
+    let current;
+    try {
+        current = lstatSync(path);
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return false;
+        throw error;
+    }
+    if (!current.isFile() || current.isSymbolicLink())
+        throw pinnedHolderIntegrityError();
+    if (!sameInode(current, pinned))
+        return false;
+    if (pinnedHolderMetadataState(pinned, current) !== 'stable')
+        return false;
+    unlinkSync(path);
+    return true;
+}
 async function tryAcquireStaleClaimOwnership(claimPath, claimWasCreated, minStaleAgeMs) {
     let claim;
     try {
@@ -275,6 +412,8 @@ async function tryAcquireStaleClaimOwnership(claimPath, claimWasCreated, minStal
                 return undefined;
             throw error;
         }
+        if (!observedOwner)
+            return undefined;
         const staleAge = observedOwner.holder
             ? minStaleAgeMs
             : Math.max(minStaleAgeMs, MIN_INVALID_HOLDER_STALE_AGE_MS);
@@ -288,9 +427,10 @@ async function tryAcquireStaleClaimOwnership(claimPath, claimWasCreated, minStal
         }
     }
     const ownerPath = staleClaimOwnerPath(claimPath, nextEpoch);
+    const candidatePath = staleClaimOwnerCandidatePath(ownerPath);
     let handle;
     try {
-        handle = await fsp.open(ownerPath, 'wx');
+        handle = await fsp.open(candidatePath, 'wx');
     }
     catch (error) {
         if (error.code === 'EEXIST' ||
@@ -298,31 +438,58 @@ async function tryAcquireStaleClaimOwnership(claimPath, claimWasCreated, minStal
             return undefined;
         throw error;
     }
-    try {
-        // As with the public lock, publish the identity without an async gap.
-        writeFileSync(handle.fd, currentLockHolderPayload());
-        const owner = await handle.stat();
-        const currentClaim = await fsp.lstat(claimPath);
-        if (!owner.isFile() || !sameInode(currentClaim, claim)) {
+    let owned;
+    const abandonCandidate = async () => {
+        try {
             await handle.close();
+        }
+        catch { /* tolerate */ }
+        if (owned) {
+            try {
+                await releaseOwnedLock(ownerPath, owned);
+            }
+            catch { /* tolerate */ }
+            try {
+                await releaseOwnedLock(candidatePath, owned);
+            }
+            catch { /* tolerate */ }
+        }
+        else {
             try {
-                await releaseOwnedLock(ownerPath, owner);
+                await fsp.unlink(candidatePath);
             }
             catch { /* tolerate */ }
+        }
+    };
+    try {
+        writeFileSync(handle.fd, currentLockHolderPayload());
+        owned = await handle.stat();
+        if (!owned.isFile())
+            throw new Error('file-lock stale-claim owner candidate is not a regular file');
+        const currentClaimBeforePublish = await fsp.lstat(claimPath);
+        if (!sameInode(currentClaimBeforePublish, claim)) {
+            await abandonCandidate();
             return undefined;
         }
-        return { ownerPath, handle };
-    }
-    catch (error) {
-        try {
-            await handle.close();
+        if (fileLockTestHooks?.beforeStaleClaimOwnerPublish) {
+            await fileLockTestHooks.beforeStaleClaimOwnerPublish(ownerPath);
         }
-        catch { /* tolerate */ }
-        try {
-            await fsp.unlink(ownerPath);
+        // link(2) is an atomic no-overwrite publication: readers can observe
+        // either no epoch pathname or the complete identity, never an empty file.
+        await fsp.link(candidatePath, ownerPath);
+        const publishedOwner = await fsp.lstat(ownerPath);
+        const currentClaim = await fsp.lstat(claimPath);
+        if (!publishedOwner.isFile() || publishedOwner.isSymbolicLink() ||
+            !sameInode(publishedOwner, owned) || !sameInode(currentClaim, claim)) {
+            await abandonCandidate();
+            return undefined;
         }
-        catch { /* tolerate */ }
-        if (error.code === 'ENOENT')
+        return { ownerPath, candidatePath, owned, handle };
+    }
+    catch (error) {
+        await abandonCandidate();
+        if (error.code === 'EEXIST' ||
+            error.code === 'ENOENT')
             return undefined;
         throw error;
     }
@@ -362,6 +529,8 @@ function tryAcquireStaleClaimOwnershipSync(claimPath, claimWasCreated, minStaleA
                 return undefined;
             throw error;
         }
+        if (!observedOwner)
+            return undefined;
         const staleAge = observedOwner.holder
             ? minStaleAgeMs
             : Math.max(minStaleAgeMs, MIN_INVALID_HOLDER_STALE_AGE_MS);
@@ -375,9 +544,10 @@ function tryAcquireStaleClaimOwnershipSync(claimPath, claimWasCreated, minStaleA
         }
     }
     const ownerPath = staleClaimOwnerPath(claimPath, nextEpoch);
+    const candidatePath = staleClaimOwnerCandidatePath(ownerPath);
     let fd;
     try {
-        fd = openSync(ownerPath, 'wx');
+        fd = openSync(candidatePath, 'wx');
     }
     catch (error) {
         if (error.code === 'EEXIST' ||
@@ -385,30 +555,54 @@ function tryAcquireStaleClaimOwnershipSync(claimPath, claimWasCreated, minStaleA
             return undefined;
         throw error;
     }
-    try {
-        writeFileSync(fd, currentLockHolderPayload());
-        const owner = fstatSync(fd);
-        const currentClaim = lstatSync(claimPath);
-        if (!owner.isFile() || !sameInode(currentClaim, claim)) {
+    let owned;
+    const abandonCandidate = () => {
+        try {
             closeSync(fd);
+        }
+        catch { /* tolerate */ }
+        if (owned) {
             try {
-                releaseOwnedLockSync(ownerPath, owner);
+                releaseOwnedLockSync(ownerPath, owned);
             }
             catch { /* tolerate */ }
+            try {
+                releaseOwnedLockSync(candidatePath, owned);
+            }
+            catch { /* tolerate */ }
+        }
+        else {
+            try {
+                unlinkSync(candidatePath);
+            }
+            catch { /* tolerate */ }
+        }
+    };
+    try {
+        writeFileSync(fd, currentLockHolderPayload());
+        owned = fstatSync(fd);
+        if (!owned.isFile())
+            throw new Error('file-lock stale-claim owner candidate is not a regular file');
+        const currentClaimBeforePublish = lstatSync(claimPath);
+        if (!sameInode(currentClaimBeforePublish, claim)) {
+            abandonCandidate();
             return undefined;
         }
-        return { ownerPath, fd };
+        fileLockTestHooks?.beforeStaleClaimOwnerPublishSync?.(ownerPath);
+        linkSync(candidatePath, ownerPath);
+        const publishedOwner = lstatSync(ownerPath);
+        const currentClaim = lstatSync(claimPath);
+        if (!publishedOwner.isFile() || publishedOwner.isSymbolicLink() ||
+            !sameInode(publishedOwner, owned) || !sameInode(currentClaim, claim)) {
+            abandonCandidate();
+            return undefined;
+        }
+        return { ownerPath, candidatePath, owned, fd };
     }
     catch (error) {
-        try {
-            closeSync(fd);
-        }
-        catch { /* tolerate */ }
-        try {
-            unlinkSync(ownerPath);
-        }
-        catch { /* tolerate */ }
-        if (error.code === 'ENOENT')
+        abandonCandidate();
+        if (error.code === 'EEXIST' ||
+            error.code === 'ENOENT')
             return undefined;
         throw error;
     }
@@ -419,15 +613,20 @@ async function cleanStaleClaimOwnership(claimPath, ownership) {
     }
     catch { /* tolerate */ }
     try {
-        await fsp.unlink(ownership.ownerPath);
+        await releaseOwnedLock(ownership.ownerPath, ownership.owned);
+    }
+    catch { /* tolerate */ }
+    try {
+        await releaseOwnedLock(ownership.candidatePath, ownership.owned);
     }
     catch { /* tolerate */ }
     // Older dead epochs are no longer security-relevant after the claim itself
     // is gone. Clean them best-effort; a loser that was already in flight may
-    // leave another harmless orphan owner file behind.
+    // leave another harmless orphan owner/candidate file behind.
     try {
         const names = await fsp.readdir(dirname(claimPath));
-        await Promise.all(names.filter(name => parseStaleClaimOwnerEpoch(claimPath, name) !== undefined).map(async (name) => {
+        await Promise.all(names.filter(name => parseStaleClaimOwnerEpoch(claimPath, name) !== undefined ||
+            isStaleClaimOwnerCandidate(claimPath, name)).map(async (name) => {
             try {
                 await fsp.unlink(join(dirname(claimPath), name));
             }
@@ -442,12 +641,17 @@ function cleanStaleClaimOwnershipSync(claimPath, ownership) {
     }
     catch { /* tolerate */ }
     try {
-        unlinkSync(ownership.ownerPath);
+        releaseOwnedLockSync(ownership.ownerPath, ownership.owned);
+    }
+    catch { /* tolerate */ }
+    try {
+        releaseOwnedLockSync(ownership.candidatePath, ownership.owned);
     }
     catch { /* tolerate */ }
     try {
         for (const name of readdirSync(dirname(claimPath))) {
-            if (parseStaleClaimOwnerEpoch(claimPath, name) === undefined)
+            if (parseStaleClaimOwnerEpoch(claimPath, name) === undefined &&
+                !isStaleClaimOwnerCandidate(claimPath, name))
                 continue;
             try {
                 unlinkSync(join(dirname(claimPath), name));
@@ -474,12 +678,21 @@ async function resumeStaleBreak(lockPath, claimPath, observed, claimWasCreated,
             return false;
         }
         let current;
+        let publicPathMissing = false;
         try {
             current = await readPinnedHolder(lockPath);
         }
         catch (error) {
             if (error.code !== 'ENOENT')
                 throw error;
+            publicPathMissing = true;
+        }
+        if (!current && !publicPathMissing) {
+            // The pinned inode or its pathname changed during validation. Withdraw
+            // this generation and retry from a fresh public-path observation.
+            await releaseOwnedLock(claimPath, claim);
+            claimRemoved = true;
+            return false;
         }
         if (current && sameInode(current.stats, observed)) {
             const staleAge = current.holder
@@ -494,7 +707,16 @@ async function resumeStaleBreak(lockPath, claimPath, observed, claimWasCreated,
                 claimRemoved = true;
                 return false;
             }
-            await fsp.unlink(lockPath);
+            if (fileLockTestHooks?.beforeStalePublicLockUnlink) {
+                await fileLockTestHooks.beforeStalePublicLockUnlink(lockPath);
+            }
+            if (!(await unlinkUnchangedPinnedPath(lockPath, current.stats))) {
+                // A late holder publication or pathname replacement won the race.
+                // The pinned bytes are no longer authority to remove the public name.
+                await releaseOwnedLock(claimPath, claim);
+                claimRemoved = true;
+                return false;
+            }
             logger.warn(`[file-lock] broke stale lock at ${lockPath} ` +
                 `(${current.holder ? `dead/reused pid ${current.holder.pid}` : 'empty/invalid holder'}, age ${current.ageMs}ms)`);
         }
@@ -520,7 +742,11 @@ async function resumeStaleBreak(lockPath, claimPath, observed, claimWasCreated,
             // unlink. Withdraw its epoch so the same process does not look like a
             // permanently-live crashed breaker after a transient I/O error.
             try {
-                await fsp.unlink(ownership.ownerPath);
+                await releaseOwnedLock(ownership.ownerPath, ownership.owned);
+            }
+            catch { /* tolerate */ }
+            try {
+                await releaseOwnedLock(ownership.candidatePath, ownership.owned);
             }
             catch { /* tolerate */ }
         }
@@ -539,12 +765,19 @@ function resumeStaleBreakSync(lockPath, claimPath, observed, claimWasCreated, mi
             return false;
         }
         let current;
+        let publicPathMissing = false;
         try {
             current = readPinnedHolderSync(lockPath);
         }
         catch (error) {
             if (error.code !== 'ENOENT')
                 throw error;
+            publicPathMissing = true;
+        }
+        if (!current && !publicPathMissing) {
+            releaseOwnedLockSync(claimPath, claim);
+            claimRemoved = true;
+            return false;
         }
         if (current && sameInode(current.stats, observed)) {
             const staleAge = current.holder
@@ -557,7 +790,12 @@ function resumeStaleBreakSync(lockPath, claimPath, observed, claimWasCreated, mi
                 claimRemoved = true;
                 return false;
             }
-            unlinkSync(lockPath);
+            fileLockTestHooks?.beforeStalePublicLockUnlinkSync?.(lockPath);
+            if (!unlinkUnchangedPinnedPathSync(lockPath, current.stats)) {
+                releaseOwnedLockSync(claimPath, claim);
+                claimRemoved = true;
+                return false;
+            }
             logger.warn(`[file-lock] broke stale lock at ${lockPath} ` +
                 `(${current.holder ? `dead/reused pid ${current.holder.pid}` : 'empty/invalid holder'}, age ${current.ageMs}ms)`);
         }
@@ -574,7 +812,11 @@ function resumeStaleBreakSync(lockPath, claimPath, observed, claimWasCreated, mi
             }
             catch { /* tolerate */ }
             try {
-                unlinkSync(ownership.ownerPath);
+                releaseOwnedLockSync(ownership.ownerPath, ownership.owned);
+            }
+            catch { /* tolerate */ }
+            try {
+                releaseOwnedLockSync(ownership.candidatePath, ownership.owned);
             }
             catch { /* tolerate */ }
         }