mcp-castor 2026.3.0

package/src/semaphore.js

@@ -0,0 +1,373 @@
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import {
+  TASK_DIR,
+  SLOTS_DIR,
+  MAX_CONCURRENT_TASKS,
+  SLOT_HEARTBEAT_MS,
+  SLOT_WEDGED_MS,
+  SLOT_POLL_MS,
+  IS_WINDOWS,
+} from "./config.js";
+import { wslDistro } from "./wsl_env.js";
+
+/**
+ * Checks whether a given PID is currently alive on the host or across the
+ * WSL/Windows boundary.
+ * @param {number} pid - Target process ID.
+ * @param {string} [platform=process.platform] - OS platform ('win32' | 'linux').
+ * @returns {boolean} true if the PID is alive (or the probe timed out, which
+ *   is treated as alive to avoid falsely declaring a live worker orphaned).
+ */
+export function pidAlive(pid, platform = process.platform) {
+  if (!pid) return false;
+  if (pid === process.pid && platform === process.platform) return true;
+
+  // 1. Same-platform probe
+  if (platform === process.platform) {
+    try {
+      process.kill(pid, 0); // signal 0 = liveness probe, no signal sent
+      return true;
+    } catch (err) {
+      return err.code === "EPERM"; // EPERM = exists, just owned by another user
+    }
+  }
+
+  // 2. Cross-platform probe: Windows host probing a WSL/Linux PID
+  if (IS_WINDOWS && platform === "linux") {
+    try {
+      execFileSync("wsl.exe", ["-d", wslDistro(), "--", "kill", "-0", String(pid)], {
+        timeout: 10000,
+        stdio: "ignore",
+      });
+      return true;
+    } catch (err) {
+      // Conservative: a probe timeout (ETIMEDOUT) indicates heavy WSL/host load,
+      // NOT process death. Never falsely declare a live worker orphaned on timeout.
+      if (err && (err.code === "ETIMEDOUT" || err.signal === "SIGTERM")) return true;
+      return false;
+    }
+  }
+
+  // 3. Cross-platform probe: WSL/Linux guest probing a Windows host PID
+  if (!IS_WINDOWS && platform === "win32") {
+    try {
+      execFileSync("powershell.exe", ["-NoProfile", "-Command", `Get-Process -Id ${pid}`], {
+        timeout: 10000,
+        stdio: "ignore",
+      });
+      return true;
+    } catch (err) {
+      // Conservative: a probe timeout (ETIMEDOUT) indicates heavy host load,
+      // NOT process death. Never falsely declare a live worker orphaned on timeout.
+      if (err && (err.code === "ETIMEDOUT" || err.signal === "SIGTERM")) return true;
+      return false;
+    }
+  }
+
+  // Fallback if platform is unknown: probe local first
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    if (err.code === "EPERM") return true;
+  }
+
+  if (IS_WINDOWS) {
+    try {
+      execFileSync("wsl.exe", ["-d", wslDistro(), "--", "kill", "-0", String(pid)], {
+        timeout: 3000,
+        stdio: "ignore",
+      });
+      return true;
+    } catch {}
+  } else {
+    try {
+      execFileSync("powershell.exe", ["-NoProfile", "-Command", `Get-Process -Id ${pid}`], {
+        timeout: 3000,
+        stdio: "ignore",
+      });
+      return true;
+    } catch {}
+  }
+
+  return false;
+}
+
+export function slotFilePath(i) {
+  return path.join(SLOTS_DIR, `slot_${i}.json`);
+}
+
+export function readLease(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function leaseReclaimable(lease) {
+  if (!lease) return true; // unreadable = crashed mid-write
+  // If the claiming process is dead, reclaim the slot immediately
+  if (!pidAlive(lease.pid, lease.platform)) return true;
+  // If the lease is tied to a task that has reached a terminal state (done: true),
+  // it is an orphaned zombie lease — reclaim it immediately regardless of owner liveness.
+  if (lease.taskId) {
+    try {
+      const taskFile = path.join(TASK_DIR, `${lease.taskId}.json`);
+      if (fs.existsSync(taskFile)) {
+        const diskTask = JSON.parse(fs.readFileSync(taskFile, "utf8"));
+        if (diskTask && diskTask.done) {
+          return true;
+        }
+      }
+    } catch {}
+  }
+  // LIVE PROCESS INVARIANT: A live process's lease running an active task is NEVER reclaimable!
+  // Prevents dual-generation collisions on the MAX_SEQS=1 engine during long tasks.
+  return false;
+}
+
+/**
+ * Acquires a global cross-process task slot lease.
+ * Resolves with {file, refresh} once a slot is held, or null if the task was
+ * cancelled while waiting.
+ *
+ * Single-Tenant Multi-Slot Invariant:
+ * Up to MAX_CONCURRENT_TASKS (default 1) are permitted machine-wide, but ALL
+ * concurrently active slots MUST belong to the SAME tenant (process.pid).
+ * If an alien tenant (lease.pid !== process.pid where lease is live) holds any slot,
+ * this process is blocked and queues at $0 until the alien tenant releases all slots.
+ */
+export async function acquireTaskSlot(taskEntry) {
+  try {
+    fs.mkdirSync(SLOTS_DIR, { recursive: true });
+  } catch {}
+
+  let lastHeartbeatUpdate = Date.now();
+
+  for (;;) {
+    if (taskEntry?.done) return null;
+
+    const now = Date.now();
+    // Keep queued task heartbeat fresh so observers/reaper don't treat it as dead while waiting
+    if (now - lastHeartbeatUpdate > 15_000) {
+      if (taskEntry) {
+        taskEntry.lastHeartbeatAt = now;
+      }
+      lastHeartbeatUpdate = now;
+    }
+
+    // Tenant affinity invariant: check if an alien tenant holds ANY active lease
+    let alienTenantActive = false;
+    for (let i = 0; i < MAX_CONCURRENT_TASKS; i++) {
+      const file = slotFilePath(i);
+      const lease = readLease(file);
+      if (lease && !leaseReclaimable(lease) && lease.pid !== process.pid) {
+        alienTenantActive = true;
+        break;
+      }
+    }
+
+    if (!alienTenantActive) {
+      for (let i = 0; i < MAX_CONCURRENT_TASKS; i++) {
+        const file = slotFilePath(i);
+        const claim = {
+          pid: process.pid,
+          platform: process.platform,
+          taskId: taskEntry?.id ?? null,
+          cwd: taskEntry?.cwd ?? null,
+          at: Date.now(),
+          hb: Date.now(),
+        };
+        try {
+          const fd = fs.openSync(file, "wx"); // atomic claim - only one process wins
+          fs.writeSync(fd, JSON.stringify(claim));
+          fs.closeSync(fd);
+          const refresh = setInterval(() => {
+            try {
+              if (taskEntry && taskEntry.done) {
+                clearInterval(refresh);
+                return;
+              }
+              const cur = readLease(file);
+              // Stop refreshing if lease was deleted, stolen by another process, or assigned to a different task
+              if (!cur || cur.pid !== process.pid || (cur.taskId && taskEntry?.id && cur.taskId !== taskEntry.id)) {
+                clearInterval(refresh);
+                return;
+              }
+              const tmp = `${file}.tmp_${Date.now()}_${process.pid}`;
+              fs.writeFileSync(
+                tmp,
+                JSON.stringify({ ...(cur ?? claim), pid: process.pid, platform: process.platform, hb: Date.now() }),
+                "utf8"
+              );
+              fs.renameSync(tmp, file);
+            } catch (err) {
+              process.stderr.write(`[Semaphore] Heartbeat write failed for ${file}: ${err.message}\n`);
+            }
+          }, SLOT_HEARTBEAT_MS);
+          refresh.unref();
+          return { file, refresh };
+        } catch (err) {
+          if (err.code !== "EEXIST") continue; // transient fs error: try next slot
+          const lease = readLease(file);
+          if (!leaseReclaimable(lease)) continue;
+          // Reclaim: atomic rename
+          const dead = `${file}.dead_${Date.now()}_${process.pid}`;
+          try {
+            fs.renameSync(file, dead);
+            fs.rmSync(dead, { force: true });
+          } catch {}
+        }
+      }
+    }
+
+    await new Promise((r) => setTimeout(r, SLOT_POLL_MS));
+  }
+}
+
+/**
+ * Releases a held task slot lease. Idempotent: a double-release or a release
+ * after natural completion is a no-op. The atomic-heartbeat design (O_EXCL
+ * claim, heartbeat refresh, ownership-guarded unlink) is preserved.
+ *
+ * @param {{file: string, refresh: NodeJS.Timeout, released?: boolean}} slot -
+ *   The slot handle returned by {@link acquireTaskSlot}.
+ * @returns {{released: boolean, reason?: string}}
+ *   - {released:true}  A lease we owned (or a dead/unreadable one) was freed.
+ *   - {released:false, reason:"no_slot"}         No handle passed in.
+ *   - {released:false, reason:"already_released"} This handle was already
+ *     released (idempotent no-op).
+ *   - {released:false, reason:"not_ours"}        Another live instance owns
+ *     the lease; it is not deleted.
+ *   - {released:false, reason:"release_failed"} The unlink threw; the handle
+ *     is not marked released so a later call may retry.
+ */
+export function releaseTaskSlot(slot) {
+  if (!slot) return { released: false, reason: "no_slot" };
+  if (slot.released) return { released: false, reason: "already_released" };
+  clearInterval(slot.refresh);
+  try {
+    const cur = readLease(slot.file);
+    if (!cur || cur.pid === process.pid || !pidAlive(cur.pid)) {
+      fs.rmSync(slot.file, { force: true });
+      slot.released = true;
+      return { released: true };
+    }
+    // Another live instance owns this lease; it is not deleted.
+    slot.released = true;
+    return { released: false, reason: "not_ours" };
+  } catch {
+    // Do NOT mark released on failure so a later call can retry the unlink.
+    return { released: false, reason: "release_failed" };
+  }
+}
+
+/**
+ * Returns active (non-reclaimable) leases across all instances for status reporting.
+ */
+export function listTaskSlots() {
+  const out = [];
+  for (let i = 0; i < MAX_CONCURRENT_TASKS; i++) {
+    const file = slotFilePath(i);
+    const lease = readLease(file);
+    if (lease && !leaseReclaimable(lease)) out.push(lease);
+  }
+  return out;
+}
+
+/**
+ * Returns deterministic slot status distinguishing same-session vs alien-session usage.
+ * Used for anti-panic status telemetry and orchestrator wait advisories.
+ */
+export function getSlotStatus(currentPid = process.pid) {
+  const slots = [];
+  const alienHolders = [];
+  const sameHolders = [];
+  for (let i = 0; i < MAX_CONCURRENT_TASKS; i++) {
+    const file = slotFilePath(i);
+    const lease = readLease(file);
+    if (lease && !leaseReclaimable(lease)) {
+      const entry = { slot: i, ...lease };
+      slots.push(entry);
+      if (lease.pid === currentPid) {
+        sameHolders.push(entry);
+      } else {
+        alienHolders.push(entry);
+      }
+    }
+  }
+  return {
+    slots,
+    totalCapacity: MAX_CONCURRENT_TASKS,
+    alienHolders,
+    sameHolders,
+    isAlienActive: alienHolders.length > 0,
+    isSameActive: sameHolders.length > 0,
+    isFullyOccupied: slots.length >= MAX_CONCURRENT_TASKS,
+  };
+}
+
+/**
+ * Clears reclaimable slot lease locks in SLOTS_DIR.
+ *
+ * Only deletes leases whose owner is this process or whose owner pid is not
+ * alive. A lease that cannot be parsed (truncated mid-write) is treated as
+ * reclaimable, matching the readLease/leaseReclaimable convention.
+ */
+export function clearReclaimableTaskSlots() {
+  try {
+    if (fs.existsSync(SLOTS_DIR)) {
+      for (const f of fs.readdirSync(SLOTS_DIR)) {
+        if (!f.startsWith("slot_") || !f.endsWith(".json")) continue;
+        const lease = readLease(path.join(SLOTS_DIR, f));
+        if (lease && !leaseReclaimable(lease)) continue;
+        fs.rmSync(path.join(SLOTS_DIR, f), { force: true });
+      }
+    }
+  } catch {}
+}
+
+/**
+ * Runs a function within an acquired global task slot.
+ */
+export async function runQueued(fn, taskEntry, onStart) {
+  const slot = await acquireTaskSlot(taskEntry);
+  if (!slot) {
+    return (
+      taskEntry?.result ?? {
+        isError: true,
+        text: "Task cancelled before acquiring an execution slot.",
+      }
+    );
+  }
+  // Record the live slot handle on the task entry so a cancel path (tools.js
+  // `cancel`, the HTTP /task/:id/cancel, or cancelAllTasks) can release the
+  // slot. Cleared in the finally below once the slot is released.
+  if (taskEntry) taskEntry.slot = slot;
+  if (taskEntry?.done || taskEntry?.status === "cancelled") {
+    releaseTaskSlot(slot);
+    if (taskEntry) taskEntry.slot = null;
+    return (
+      taskEntry?.result ?? {
+        isError: true,
+        text: "Task cancelled before execution.",
+      }
+    );
+  }
+  if (taskEntry && !taskEntry.done) {
+    taskEntry.status = "executing";
+    taskEntry.startedAt = Date.now();
+    if (typeof onStart === "function") {
+      onStart(taskEntry);
+    }
+  }
+  try {
+    return await fn();
+  } finally {
+    releaseTaskSlot(slot);
+    if (taskEntry) taskEntry.slot = null;
+  }
+}