chainlesschain 0.165.2 → 0.165.3

package/src/lib/background-turn-bootstrap-protocol.js

@@ -11,6 +11,10 @@ export const BACKGROUND_TURN_BOOTSTRAP_READY =
   "background-turn-bootstrap-ready";
 export const BACKGROUND_TURN_BOOTSTRAP_RELEASE =
   "background-turn-bootstrap-release";
+// The formal launch gate permits 120 seconds for a 20-worker hosted runner to
+// become ready. Keep the pre-main barrier alive for that same bounded window;
+// worker/IPC death still fails immediately through the disconnect listener.
+export const BACKGROUND_TURN_BOOTSTRAP_RELEASE_TIMEOUT_MS = 120_000;
 
 export const BACKGROUND_TURN_BOOTSTRAP_ENV = Object.freeze({
   nonce: "CC_BACKGROUND_TURN_BOOTSTRAP_NONCE",

package/src/lib/with-file-lock.js

@@ -14,14 +14,18 @@
  * reclaimed merely because the directory is old, and corrupt ownership fails
  * closed. The default filesystem publishes a fully populated, uniquely staged
  * directory in one rename so a process killed during acquisition cannot expose
- * an ownerless lock. Confirmed-dead owners are reclaimed with an exact-token
- * marker inside the lock directory so a delayed contender cannot delete a
- * replacement lock. Strict callers may use bounded jittered retries and an
+ * an ownerless lock. Confirmed-dead owners are reclaimed with an exact-token,
+ * process-owned claim inside the lock directory so a delayed contender cannot
+ * delete a replacement lock and another contender can take over if the first
+ * reclaimer is killed. Strict callers may use bounded jittered retries and an
  * after-release yield to avoid Windows sharing transients and fixed-interval
  * lock convoys.
  *
  * @param {string} targetPath  the file being guarded (lock is `${targetPath}.lock`)
- * @param {(ctx:{locked:boolean})=>T} fn  critical section; receives whether the lock was held
+ * @param {(ctx:{locked:boolean,publishReleaseAfterPathRemoved:(pendingPath:string)=>boolean})=>T} fn
+ *   critical section; `publishReleaseAfterPathRemoved` lets a transaction
+ *   publish an exact, contender-completable handoff before its final atomic
+ *   rename removes a uniquely named staging path
  * @returns {T} whatever `fn` returns
  */
 export function withFileLock(targetPath, fn, opts = {}) {
@@ -54,7 +58,9 @@ export function withFileLock(targetPath, fn, opts = {}) {
   };
   let held = false;
   let acquisitionError = null;
+  let lastOwnerObservation = null;
   let retryAttempt = 0;
+  let releasePublished = false;
   const deadline = _now() + timeoutMs;
 
   for (;;) {
@@ -88,6 +94,17 @@ export function withFileLock(targetPath, fn, opts = {}) {
       const incumbent = ownerRead.owner;
       acquisitionError = ownerRead.error;
       if (incumbent) {
+        let incumbentAlive = true;
+        try {
+          incumbentAlive = isOwnerAlive(incumbent);
+        } catch {
+          // An unavailable identity probe must retain the lock.
+        }
+        lastOwnerObservation = {
+          pid: incumbent.pid,
+          startedAt: incumbent.startedAt,
+          alive: incumbentAlive,
+        };
         // The owner writes this exact-token marker only after its critical
         // section has finished. A Windows sharing violation can then prevent
         // the final recursive removal while the owner process remains alive,
@@ -102,11 +119,19 @@ export function withFileLock(targetPath, fn, opts = {}) {
           owner,
           isOwnerAlive,
         );
+        lastOwnerObservation.releasePublished = publishedRelease.published;
         if (publishedRelease.completed) continue;
-        if (!publishedRelease.published && !isOwnerAlive(incumbent)) {
-          if (reclaimOwnedDirectory(_fs, lockDir, incumbent, isOwnerAlive)) {
+        if (!incumbentAlive) {
+          // A published handoff can still retain its unique staging path when
+          // the owner is killed between marker publication and the atomic
+          // replacement. No process can commit that staging file after the
+          // exact owner is proven dead, so the lock directory is reclaimable
+          // whether the transaction committed or not. Leave the private
+          // staging file untouched; it cannot affect the guarded target.
+          if (
+            reclaimOwnedDirectory(_fs, lockDir, incumbent, owner, isOwnerAlive)
+          )
             continue;
-          }
         }
       } else if (!failIfUnavailable) {
         // Compatibility for legacy best-effort locks that predate owner.json.
@@ -139,12 +164,21 @@ export function withFileLock(targetPath, fn, opts = {}) {
   }
 
   if (!held && failIfUnavailable) {
+    const ownerDetails = lastOwnerObservation
+      ? `; owner pid=${lastOwnerObservation.pid}, ageMs=${Math.max(
+          0,
+          _now() - lastOwnerObservation.startedAt,
+        )}, alive=${lastOwnerObservation.alive}, releasePublished=${Boolean(lastOwnerObservation.releasePublished)}`
+      : "";
     const error = new Error(
-      `Could not acquire state lock: ${targetPath}`,
+      `Could not acquire state lock: ${targetPath}${ownerDetails}`,
       acquisitionError ? { cause: acquisitionError } : undefined,
     );
     error.code = "STATE_LOCK_UNAVAILABLE";
     error.attempts = retryAttempt + 1;
+    error.lockOwner = lastOwnerObservation
+      ? Object.freeze({ ...lastOwnerObservation })
+      : null;
     throw error;
   }
 
@@ -152,7 +186,32 @@ export function withFileLock(targetPath, fn, opts = {}) {
   let bodyThrew = false;
   let result;
   try {
-    result = fn({ locked: held });
+    result = fn({
+      locked: held,
+      publishReleaseAfterPathRemoved(pendingPath) {
+        if (!held || typeof pendingPath !== "string" || !pendingPath) {
+          return false;
+        }
+        const current = readOwner(_fs, ownerPath);
+        if (!sameOwner(current, owner)) return false;
+        const markerPath = path.join(lockDir, `.release-${owner.token}`);
+        const marker = { ...owner, releaseAfterPathRemoved: pendingPath };
+        try {
+          writeOwnerMarker(_fs, markerPath, marker);
+        } catch (error) {
+          if (error?.code !== "EEXIST") throw error;
+          const existing = readOwner(_fs, markerPath);
+          if (
+            !sameOwner(existing, owner) ||
+            existing?.releaseAfterPathRemoved !== pendingPath
+          ) {
+            return false;
+          }
+        }
+        releasePublished = true;
+        return true;
+      },
+    });
   } catch (error) {
     bodyThrew = true;
     bodyError = error;
@@ -161,7 +220,13 @@ export function withFileLock(targetPath, fn, opts = {}) {
   let releaseError = null;
   if (held) {
     try {
-      released = releaseOwnedDirectory(_fs, lockDir, owner, isOwnerAlive);
+      released = releaseOwnedDirectory(
+        _fs,
+        lockDir,
+        owner,
+        isOwnerAlive,
+        releasePublished,
+      );
     } catch (error) {
       releaseError = error;
     }
@@ -251,6 +316,15 @@ function directoryExists(_fs, target) {
   }
 }
 
+function pathStatus(_fs, target) {
+  try {
+    _fs.statSync(target);
+    return "present";
+  } catch (error) {
+    return error?.code === "ENOENT" ? "absent" : "unknown";
+  }
+}
+
 function isProcessAlive(pid) {
   try {
     process.kill(pid, 0);
@@ -361,21 +435,65 @@ function writeOwnerMarker(_fs, markerPath, owner) {
   });
 }
 
-function reclaimOwnedDirectory(_fs, lockDir, owner, ownerAlive) {
-  const markerPath = path.join(lockDir, `.reclaim-${owner.token}`);
-  try {
-    writeOwnerMarker(_fs, markerPath, owner);
-  } catch (error) {
-    if (error?.code === "ENOENT" || error?.code === "EEXIST") return false;
-    throw error;
+function reclaimOwnedDirectory(_fs, lockDir, incumbent, claimant, ownerAlive) {
+  const claimPath = path.join(lockDir, `.reclaim-${incumbent.token}`);
+  for (;;) {
+    const existingClaim = readOwner(_fs, claimPath);
+    if (existingClaim) {
+      if (sameOwner(existingClaim, claimant)) break;
+      if (!ownerAlive(existingClaim)) {
+        if (removeOwnMarker(_fs, claimPath, existingClaim)) continue;
+      }
+      return false;
+    }
+    try {
+      // The claim belongs to the contender, not the dead incumbent. If this
+      // contender is killed before detaching the directory, another process
+      // can prove that exact claimant dead and safely take over.
+      writeOwnerMarker(_fs, claimPath, claimant);
+      break;
+    } catch (error) {
+      if (error?.code === "ENOENT") {
+        return !directoryExists(_fs, lockDir);
+      }
+      if (error?.code === "EEXIST" || isTransientLockError(error)) {
+        return false;
+      }
+      throw error;
+    }
   }
+
   const current = readOwner(_fs, path.join(lockDir, "owner.json"));
-  if (!sameOwner(current, owner) || ownerAlive(owner)) {
-    removeOwnMarker(_fs, markerPath, owner);
+  if (!sameOwner(current, incumbent) || ownerAlive(incumbent)) {
+    removeOwnMarker(_fs, claimPath, claimant);
     return false;
   }
-  _fs.rmSync(lockDir, { recursive: true, force: true });
-  return true;
+
+  if (typeof _fs.renameSync !== "function") {
+    _fs.rmSync(lockDir, { recursive: true, force: true });
+    return true;
+  }
+
+  const cleanupDir = `${lockDir}.reclaimed-${incumbent.token}-${claimant.token}`;
+  try {
+    // Free the acquisition path atomically before recursive cleanup. A killed
+    // reclaimer can therefore leave only uniquely named debris, never a
+    // claimant marker that permanently fences the shared lock path.
+    _fs.renameSync(lockDir, cleanupDir);
+    try {
+      _fs.rmSync(cleanupDir, { recursive: true, force: true });
+    } catch {
+      /* private reclaim debris is harmless after the lock path is detached */
+    }
+    return true;
+  } catch (error) {
+    removeOwnMarker(_fs, claimPath, claimant);
+    if (error?.code === "ENOENT") {
+      return !directoryExists(_fs, lockDir);
+    }
+    if (isTransientLockError(error)) return false;
+    throw error;
+  }
 }
 
 function reclaimLegacyDirectory(_fs, lockDir, stat) {
@@ -401,9 +519,23 @@ function reclaimLegacyDirectory(_fs, lockDir, stat) {
 
 function completePublishedRelease(_fs, lockDir, owner, claimant, ownerAlive) {
   const markerPath = path.join(lockDir, `.release-${owner.token}`);
-  if (!sameOwner(readOwner(_fs, markerPath), owner)) {
+  const marker = readOwner(_fs, markerPath);
+  if (!sameOwner(marker, owner)) {
     return { published: false, completed: false };
   }
+  // A state transaction can stage its exact replacement, publish this marker,
+  // and only then atomically rename the staging file over the guarded target.
+  // Contenders may complete the release only after that unique staging path is
+  // gone. This closes the otherwise unbounded scheduler-preemption window
+  // between a committed rename and the owner's first release syscall without
+  // ever exposing an in-progress write to a competing critical section.
+  if (
+    typeof marker.releaseAfterPathRemoved === "string" &&
+    marker.releaseAfterPathRemoved &&
+    pathStatus(_fs, marker.releaseAfterPathRemoved) !== "absent"
+  ) {
+    return { published: true, completed: false };
+  }
   const claimPath = path.join(lockDir, `.release-claim-${owner.token}`);
   for (;;) {
     const existingClaim = readOwner(_fs, claimPath);
@@ -461,9 +593,20 @@ function completePublishedRelease(_fs, lockDir, owner, claimant, ownerAlive) {
   }
 }
 
-function releaseOwnedDirectory(_fs, lockDir, owner, ownerAlive) {
+function releaseOwnedDirectory(
+  _fs,
+  lockDir,
+  owner,
+  ownerAlive,
+  releaseWasPublished = false,
+) {
   if (!sameOwner(readOwner(_fs, path.join(lockDir, "owner.json")), owner)) {
-    return false;
+    // Once this exact owner published a guarded handoff, a contender is allowed
+    // to detach the old directory as soon as the final staging path disappears.
+    // A missing or replacement lock therefore proves successful release, not
+    // ownership loss. Without that publication the strict legacy verdict stays
+    // fail-closed.
+    return releaseWasPublished;
   }
   // Atomically move the exact owned directory out of the acquisition path,
   // then remove that uniquely-tokened directory. A replacement owner may
@@ -482,7 +625,7 @@ function releaseOwnedDirectory(_fs, lockDir, owner, ownerAlive) {
       }
       return true;
     } catch (error) {
-      if (error?.code === "ENOENT") return false;
+      if (error?.code === "ENOENT") return releaseWasPublished;
       // Windows sharing transients fall back to the marker-based release.
     }
   }
@@ -490,7 +633,13 @@ function releaseOwnedDirectory(_fs, lockDir, owner, ownerAlive) {
   try {
     writeOwnerMarker(_fs, markerPath, owner);
   } catch (error) {
-    if (error?.code === "ENOENT" || error?.code === "EEXIST") return false;
+    if (error?.code === "ENOENT") return releaseWasPublished;
+    if (error?.code === "EEXIST") {
+      const existing = readOwner(_fs, markerPath);
+      if (!sameOwner(existing, owner)) return false;
+      return completePublishedRelease(_fs, lockDir, owner, owner, ownerAlive)
+        .published;
+    }
     throw error;
   }
   if (!sameOwner(readOwner(_fs, path.join(lockDir, "owner.json")), owner)) {

package/src/workers/background-agent-keeper.js

@@ -9,8 +9,15 @@
  */
 
 import net from "node:net";
-import { readFileSync, unlinkSync, writeSync } from "node:fs";
-import { resolve } from "node:path";
+import {
+  readFileSync,
+  renameSync,
+  rmSync,
+  unlinkSync,
+  writeFileSync,
+  writeSync,
+} from "node:fs";
+import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { createNdjsonReader } from "../lib/background-session-transport.js";
 import {
@@ -28,6 +35,7 @@ import {
   BACKGROUND_AGENT_KEEPER_CLEANUP_CONFIRM_TIMEOUT_MS,
   BACKGROUND_AGENT_KEEPER_HELLO,
   BACKGROUND_AGENT_KEEPER_HEARTBEAT,
+  BACKGROUND_AGENT_KEEPER_IDENTITY_PROBE_DELAY_MS,
   BACKGROUND_AGENT_KEEPER_HEARTBEAT_TIMEOUT_MS,
   BACKGROUND_AGENT_KEEPER_PERSIST_RETRY_TIMEOUT_MS,
   BACKGROUND_AGENT_KEEPER_PROTOCOL_VERSION,
@@ -35,12 +43,15 @@ import {
   BACKGROUND_AGENT_KEEPER_RETIRE,
   BACKGROUND_AGENT_KEEPER_RETIRED,
   BACKGROUND_AGENT_KEEPER_STATE_LOCK_TIMEOUT_MS,
+  backgroundAgentKeeperLaunchClaimPath,
   cleanupBackgroundAgentKeeperPipeDirectory,
+  createBackgroundAgentKeeperLaunchClaim,
   createBackgroundAgentKeeperMessage,
   normalizeBackgroundAgentKeeperHello,
   normalizeBackgroundAgentKeeperTurn,
   sameBackgroundAgentKeeperTurn,
 } from "../lib/background-agent-keeper-protocol.js";
+import { waitForBackgroundAgentLaunchBarrier } from "../lib/background-agent-launch-barrier.js";
 
 const STARTUP_TIMEOUT_MS = 30_000;
 const HEARTBEAT_INTERVAL_MS = 1_000;
@@ -132,18 +143,29 @@ export function runBackgroundAgentKeeperHeartbeat({
   reportPersistenceFailure = reportKeeperPersistenceFailure,
 }) {
   if (finishing) return false;
+  // Startup is deliberately serialized: the keeper publishes its listening
+  // claim, then remains read-only until the worker has claimed and
+  // authenticated. Persisting a pre-auth heartbeat would reintroduce the
+  // first-write lock race the launch barrier prevents.
+  if (!workerSocket || !authenticatedHello) return false;
   const currentTime = now();
+  const workerHeartbeatAge = Number.isFinite(workerHeartbeatAt)
+    ? Math.max(0, currentTime - workerHeartbeatAt)
+    : 0;
   const workerHeartbeatExpired =
     workerSocket &&
     authenticatedHello &&
     Number.isFinite(workerHeartbeatAt) &&
-    currentTime - workerHeartbeatAt >
-      BACKGROUND_AGENT_KEEPER_HEARTBEAT_TIMEOUT_MS;
+    workerHeartbeatAge > BACKGROUND_AGENT_KEEPER_HEARTBEAT_TIMEOUT_MS;
+  const workerIdentityProbeDue =
+    armedTurn &&
+    Number.isFinite(workerHeartbeatAt) &&
+    workerHeartbeatAge > BACKGROUND_AGENT_KEEPER_IDENTITY_PROBE_DELAY_MS;
   if (
     workerSocket &&
     authenticatedHello &&
     (workerHeartbeatExpired ||
-      (armedTurn &&
+      (workerIdentityProbeDue &&
         !workerIdentityAlive(
           authenticatedHello.workerPid,
           authenticatedWorkerStartedAt,
@@ -153,8 +175,9 @@ export function runBackgroundAgentKeeperHeartbeat({
     // after its worker dies if a platform helper retained a duplicate.
     // Socket EOF and PID visibility are therefore not sufficient lifetime
     // signals. The authenticated application heartbeat provides an
-    // independent bounded fence, while the PID/start anchor remains the
-    // immediate path on platforms that report death promptly.
+    // independent bounded fence. Delay the PID/start probe until application
+    // heartbeats are already stale so healthy Windows keepers do not create a
+    // synchronized WMIC/PowerShell probe storm under normal operation.
     finishForWorkerDisconnect();
     workerSocket.destroy();
     return false;
@@ -239,7 +262,9 @@ export function stopBackgroundAgentKeeperTurnTrees(targets, options = {}) {
         leaderAlive ||
         (platform !== "win32" &&
           target.processGroup === true &&
-          processTreeExecutionAlive(target.pid, target.startedAt));
+          processTreeExecutionAlive(target.pid, target.startedAt, {
+            processGroup: true,
+          }));
       if (residualAlive) {
         failures.push(error?.message || String(error));
       }
@@ -248,6 +273,46 @@ export function stopBackgroundAgentKeeperTurnTrees(targets, options = {}) {
   return failures;
 }
 
+export async function waitForBackgroundAgentKeeperTurnTreesToStop(
+  targets,
+  options = {},
+) {
+  const now = options.now || Date.now;
+  const sleep =
+    options.sleep ||
+    ((milliseconds) =>
+      new Promise((resolvePromise) =>
+        setTimeout(resolvePromise, milliseconds),
+      ));
+  const processTreeExecutionAlive =
+    options.isProcessTreeExecutionAlive ||
+    isBackgroundProcessTreeExecutionAlive;
+  const timeoutMs = Math.max(
+    1,
+    Number(options.timeoutMs) ||
+      BACKGROUND_AGENT_KEEPER_CLEANUP_CONFIRM_TIMEOUT_MS,
+  );
+  // Each strict macOS liveness probe invokes ps. Polling every 10 ms across
+  // 20 keepers created a process storm precisely while launchd was trying to
+  // reap the killed groups. A 100 ms cadence remains responsive and bounded
+  // while leaving the host enough scheduling capacity to establish absence.
+  const pollMs = Math.max(1, Number(options.pollMs) || 100);
+  const deadline = now() + timeoutMs;
+  const observeAlive = () =>
+    targets.filter((target) =>
+      processTreeExecutionAlive(target.pid, target.startedAt, {
+        processGroup: target.processGroup !== false,
+      }),
+    );
+
+  let alive = observeAlive();
+  while (alive.length > 0 && now() < deadline) {
+    await sleep(Math.max(1, Math.min(pollMs, deadline - now())));
+    alive = observeAlive();
+  }
+  return alive;
+}
+
 async function cleanupTurn(job, turn, reason) {
   const requestedAt = Date.now();
   const requestedPersistence = await retryKeeperPersistence(() =>
@@ -275,17 +340,7 @@ async function cleanupTurn(job, turn, reason) {
   );
   const failures = stopBackgroundAgentKeeperTurnTrees(targets);
 
-  const deadline =
-    Date.now() + BACKGROUND_AGENT_KEEPER_CLEANUP_CONFIRM_TIMEOUT_MS;
-  let alive = targets.filter((target) =>
-    isBackgroundProcessTreeExecutionAlive(target.pid, target.startedAt),
-  );
-  while (alive.length > 0 && Date.now() < deadline) {
-    await new Promise((resolvePromise) => setTimeout(resolvePromise, 10));
-    alive = targets.filter((target) =>
-      isBackgroundProcessTreeExecutionAlive(target.pid, target.startedAt),
-    );
-  }
+  const alive = await waitForBackgroundAgentKeeperTurnTreesToStop(targets);
   const confirmed = alive.length === 0 && failures.length === 0;
   let finalPersistenceAttempts = 0;
   const finalPersistence = await retryKeeperPersistence(() => {
@@ -322,6 +377,24 @@ async function cleanupTurn(job, turn, reason) {
 
 export async function runBackgroundAgentKeeper(jobFile, options = {}) {
   const job = JSON.parse(readFileSync(jobFile, "utf8"));
+  const launchClaimPath = backgroundAgentKeeperLaunchClaimPath(
+    job.id,
+    job.keeperGeneration,
+    options.backgroundAgentsDirectory || dirname(resolve(jobFile)),
+  );
+  const launchBarrier = await waitForBackgroundAgentLaunchBarrier({
+    id: job.id,
+    workerGeneration: job.workerGeneration,
+    expectedPid: process.pid,
+    pidField: "keeperPid",
+    readState: options.readBackgroundAgentState || readBackgroundAgentState,
+    timeoutMs: Number(options.launchBarrierTimeoutMs) || undefined,
+    sleep: options.launchBarrierSleep,
+  });
+  if (launchBarrier.status !== "ready") {
+    removeJobFile(jobFile);
+    return { status: `launch-${launchBarrier.status}` };
+  }
   removeJobFile(jobFile);
   const startedAt = Date.now();
   const claim = mutateBackgroundAgentState(job.id, (current) => {
@@ -342,6 +415,25 @@ export async function runBackgroundAgentKeeper(jobFile, options = {}) {
     };
   });
   if (!claim.applied) return { status: "unclaimed" };
+  const launchClaim = createBackgroundAgentKeeperLaunchClaim({
+    id: job.id,
+    workerGeneration: job.workerGeneration,
+    keeperGeneration: job.keeperGeneration,
+    keeperPid: process.pid,
+    token: job.token,
+  });
+  const launchClaimCandidate = `${launchClaimPath}.${process.pid}.tmp`;
+  try {
+    writeFileSync(launchClaimCandidate, JSON.stringify(launchClaim), {
+      encoding: "utf8",
+      mode: 0o600,
+      flag: "wx",
+    });
+    renameSync(launchClaimCandidate, launchClaimPath);
+  } catch (error) {
+    rmSync(launchClaimCandidate, { force: true });
+    throw error;
+  }
 
   if (process.platform !== "win32") {
     try {
@@ -686,6 +778,7 @@ export async function runBackgroundAgentKeeper(jobFile, options = {}) {
       // Another keeper can still own a sibling socket in the shared namespace.
     }
   }
+  rmSync(launchClaimPath, { force: true });
   return {
     ...result,
     authenticatedWorkerPid: authenticatedHello?.workerPid || null,