gsd-pi 2.77.0-dev.58d3d4d6c → 2.77.0-dev.cfd69e714

package/src/resources/extensions/gsd/sync-lock.ts

@@ -1,11 +1,10 @@
 // GSD Extension — Advisory Sync Lock
 // Prevents concurrent worktree syncs from colliding via a simple file lock.
-// Stale locks (mtime > 60s) are auto-overridden. Lock acquisition waits up
-// to 5 seconds then skips non-fatally.
+// Stale locks (mtime > 60s, owner PID confirmed dead) are overridden. Lock
+// acquisition waits up to 5 seconds then skips non-fatally.
 
-import { existsSync, statSync, unlinkSync } from "node:fs";
-import { join } from "node:path";
-import { atomicWriteSync } from "./atomic-write.js";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, statSync, unlinkSync, writeSync } from "node:fs";
+import { dirname, join } from "node:path";
 
 const STALE_THRESHOLD_MS = 60_000; // 60 seconds
 const DEFAULT_TIMEOUT_MS = 5_000;  // 5 seconds
@@ -23,13 +22,59 @@ function sleepSync(ms: number): void {
   Atomics.wait(SLEEP_VIEW, 0, 0, ms);
 }
 
+/** True if the given PID is alive in the current process namespace. */
+function isPidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "EPERM") return true;
+    return false;
+  }
+}
+
+/**
+ * Atomically create the lock file with `O_EXCL` semantics. Returns true on
+ * exclusive create, false if the file already existed. Any other error
+ * propagates.
+ */
+function tryCreateLockFile(lp: string, payload: string): boolean {
+  // Ensure parent dir exists (`atomicWriteSync` previously did this implicitly).
+  try {
+    mkdirSync(dirname(lp), { recursive: true });
+  } catch {
+    /* best-effort */
+  }
+  let fd: number;
+  try {
+    // "wx" → O_WRONLY | O_CREAT | O_EXCL — atomic create-if-not-exists on POSIX.
+    fd = openSync(lp, "wx");
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === "EEXIST") return false;
+    throw err;
+  }
+  try {
+    writeSync(fd, payload);
+  } finally {
+    closeSync(fd);
+  }
+  return true;
+}
+
 /**
  * Acquire an advisory sync lock for the given basePath.
  * Returns { acquired: true } on success, { acquired: false } after timeout.
  *
- * - Creates lock file at {basePath}/.gsd/sync.lock with JSON { pid, acquired_at }
- * - If lock exists and mtime > 60s (stale), overrides it
- * - If lock exists and not stale, spins up to timeoutMs before giving up
+ * Replaces a non-atomic `existsSync` + `atomicWriteSync` (write-temp+rename,
+ * which is not exclusive-create) sequence that allowed two callers to both
+ * believe they had acquired the lock, corrupting the event log.
+ * (Issue #4980 CRIT-4)
+ *
+ * Stale-lock override now also verifies the recorded owner PID is dead
+ * before stealing — prevents a slow event-loop pause (>60s under heavy I/O)
+ * from making a legitimately-held lock appear stale and get stolen.
+ * (Issue #4980 M-concurrency-3)
  */
 export function acquireSyncLock(
   basePath: string,
@@ -37,45 +82,58 @@ export function acquireSyncLock(
 ): { acquired: boolean } {
   const lp = lockFilePath(basePath);
   const deadline = Date.now() + timeoutMs;
+  const lockData = JSON.stringify(
+    { pid: process.pid, acquired_at: new Date().toISOString() },
+    null,
+    2,
+  );
 
   while (true) {
-    // Check if lock file exists
-    if (existsSync(lp)) {
-      // Check staleness
-      try {
-        const stat = statSync(lp);
-        const age = Date.now() - stat.mtimeMs;
-        if (age > STALE_THRESHOLD_MS) {
-          // Stale lock — override it
-          try { unlinkSync(lp); } catch { /* race: already removed */ }
-        } else {
-          // Lock is held and not stale — wait or give up
-          if (Date.now() >= deadline) {
-            return { acquired: false };
-          }
-          sleepSync(SPIN_INTERVAL_MS);
-          continue;
-        }
-      } catch {
-        // stat failed (file removed between exists check and stat) — try to acquire
+    // First try the fast path: atomic create. No check-then-write race.
+    try {
+      if (tryCreateLockFile(lp, lockData)) {
+        return { acquired: true };
       }
+    } catch {
+      /* unexpected — fall through to retry */
     }
 
-    // Lock file does not exist (or was just removed) — try to write it
+    // File exists. Decide whether to steal (stale + owner dead) or wait.
+    let canSteal = false;
     try {
-      const lockData = {
-        pid: process.pid,
-        acquired_at: new Date().toISOString(),
-      };
-      atomicWriteSync(lp, JSON.stringify(lockData, null, 2));
-      return { acquired: true };
-    } catch {
-      // Write failed (race condition with another process) — retry or give up
-      if (Date.now() >= deadline) {
-        return { acquired: false };
+      const stat = statSync(lp);
+      const age = Date.now() - stat.mtimeMs;
+      if (age > STALE_THRESHOLD_MS) {
+        // Verify the recorded owner PID is dead before stealing.
+        let ownerAlive = false;
+        try {
+          const data = JSON.parse(readFileSync(lp, "utf-8")) as { pid?: number };
+          if (typeof data.pid === "number" && data.pid !== process.pid) {
+            ownerAlive = isPidAlive(data.pid);
+          }
+        } catch {
+          // Lock contents unreadable — be conservative and steal only on age.
+          // A garbage lock file that we cannot parse is safer to remove than
+          // to leave wedging the lock indefinitely.
+        }
+        canSteal = !ownerAlive;
       }
-      sleepSync(SPIN_INTERVAL_MS);
+    } catch {
+      // stat failed (file removed between exists and stat) — retry create.
+      continue;
+    }
+
+    if (canSteal) {
+      try { unlinkSync(lp); } catch { /* race: already removed */ }
+      // Loop back to retry create.
+      continue;
+    }
+
+    // Lock is held and not stale (or owner is alive) — wait or give up.
+    if (Date.now() >= deadline) {
+      return { acquired: false };
     }
+    sleepSync(SPIN_INTERVAL_MS);
   }
 }
 

package/src/resources/extensions/gsd/tests/artifact-retry-cap.test.ts

@@ -0,0 +1,270 @@
+/**
+ * artifact-retry-cap.test.ts — Regression tests for #2007.
+ *
+ * Three interacting bugs caused unbounded artifact-verification retry loops
+ * that burned unlimited budget (202 dispatches observed in production):
+ *
+ * Bug 1: postUnitPreVerification in auto-post-unit.ts had no MAX check before
+ *        returning "retry" when an expected artifact was missing. The attempt
+ *        counter incremented forever.
+ *
+ * Bug 2: runDispatch in auto/phases.ts only pushed to loopState.recentUnits
+ *        when pendingVerificationRetry was falsy, so the sliding-window stuck
+ *        detector never saw artifact-retry dispatches and could not fire.
+ *
+ * Bug 3: MAX_UNIT_DISPATCHES and MAX_LIFETIME_DISPATCHES were exported from
+ *        auto/session.ts but never compared against unitDispatchCount anywhere
+ *        in the codebase — dead constants that provided false confidence.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import ts from "typescript";
+
+const dir = join(import.meta.dirname, "..");
+
+const postUnitSrc = readFileSync(join(dir, "auto-post-unit.ts"), "utf-8");
+const phasesSrc = readFileSync(join(dir, "auto", "phases.ts"), "utf-8");
+const sessionSrc = readFileSync(join(dir, "auto", "session.ts"), "utf-8");
+const autoSrc = readFileSync(join(dir, "auto.ts"), "utf-8");
+
+function extractFunctionBody(source: string, functionName: string): string {
+  const sourceFile = ts.createSourceFile(
+    "auto-post-unit.ts",
+    source,
+    ts.ScriptTarget.Latest,
+    true,
+    ts.ScriptKind.TS,
+  );
+  let body: ts.Block | undefined;
+
+  function visit(node: ts.Node): void {
+    if (
+      ts.isFunctionDeclaration(node) &&
+      node.name?.text === functionName
+    ) {
+      body = node.body;
+      return;
+    }
+    ts.forEachChild(node, visit);
+  }
+
+  visit(sourceFile);
+
+  assert.ok(body, `${functionName} must have a function body`);
+  return source.slice(body.getStart(sourceFile) + 1, body.end - 1);
+}
+
+function extractStuckDetectionSection(source: string): string {
+  const stuckSectionIdx = source.indexOf("Sliding-window stuck detection");
+  assert.ok(stuckSectionIdx !== -1, "stuck-detection section must exist");
+
+  const preDispatchIdx = source.indexOf("// Pre-dispatch hooks", stuckSectionIdx);
+  assert.ok(preDispatchIdx !== -1, "pre-dispatch hooks section must follow stuck detection");
+
+  return source.slice(stuckSectionIdx, preDispatchIdx);
+}
+
+const postUnitPreVerificationBody = extractFunctionBody(
+  postUnitSrc,
+  "postUnitPreVerification",
+);
+
+// ─── Bug 1: artifact retry must be bounded ───────────────────────────────────
+
+test("#2007 bug 1: MAX_ARTIFACT_VERIFICATION_RETRIES constant is defined", () => {
+  assert.ok(
+    postUnitSrc.includes("MAX_ARTIFACT_VERIFICATION_RETRIES"),
+    "auto-post-unit.ts must define MAX_ARTIFACT_VERIFICATION_RETRIES",
+  );
+});
+
+test("#2007 bug 1: attempt is compared against MAX_ARTIFACT_VERIFICATION_RETRIES before returning retry", () => {
+  const retryBlockIdx = postUnitPreVerificationBody.indexOf("const retryKey =");
+  assert.ok(retryBlockIdx !== -1, "retry block must exist in postUnitPreVerification");
+
+  const retryIdx = postUnitPreVerificationBody.indexOf("return \"retry\"", retryBlockIdx);
+  assert.ok(retryIdx !== -1, "return \"retry\" must exist in postUnitPreVerification");
+
+  const maxIdx = postUnitPreVerificationBody.indexOf(
+    "if (attempt > MAX_ARTIFACT_VERIFICATION_RETRIES)",
+    retryBlockIdx,
+  );
+  assert.ok(maxIdx !== -1, "retry block must compare attempt against MAX");
+  assert.ok(
+    maxIdx < retryIdx,
+    "MAX_ARTIFACT_VERIFICATION_RETRIES check must appear before return \"retry\"",
+  );
+});
+
+test("#2007 bug 1: exhaustion path pauses auto-mode instead of silently continuing", () => {
+  const exhaustionIdx = postUnitPreVerificationBody.indexOf("phase: \"artifact-verify-exhausted\"");
+  assert.ok(exhaustionIdx !== -1, "exhaustion branch must log artifact-verify-exhausted");
+
+  const pauseIdx = postUnitPreVerificationBody.indexOf("await pauseAuto", exhaustionIdx);
+  const dispatchedIdx = postUnitPreVerificationBody.indexOf("return \"dispatched\"", exhaustionIdx);
+
+  assert.ok(
+    pauseIdx !== -1 && dispatchedIdx !== -1 && pauseIdx < dispatchedIdx,
+    "exhaustion branch must pause auto-mode before returning \"dispatched\"",
+  );
+});
+
+test("#2007 bug 1: failure context message includes attempt count and max", () => {
+  const failureContextIdx = postUnitPreVerificationBody.indexOf("failureContext:");
+  assert.ok(failureContextIdx !== -1, "failureContext assignment must exist");
+  assert.ok(
+    postUnitPreVerificationBody.includes(
+      "attempt ${attempt}/${MAX_ARTIFACT_VERIFICATION_RETRIES}",
+      failureContextIdx,
+    ),
+    "failure context should include attempt progress (attempt/current-max)",
+  );
+});
+
+// ─── Bug 2: stuck detection must see all dispatches ──────────────────────────
+
+test("#2007 bug 2: recentUnits.push is unconditional — not gated on pendingVerificationRetry", () => {
+  const stuckSection = extractStuckDetectionSection(phasesSrc);
+  const pushIdx = stuckSection.indexOf("recentUnits.push");
+  assert.ok(pushIdx !== -1, "recentUnits.push must exist in phases.ts");
+
+  const pendingCheckIdx = stuckSection.indexOf("!s.pendingVerificationRetry");
+  assert.ok(pendingCheckIdx !== -1, "pendingVerificationRetry guard must exist");
+
+  // The push must come BEFORE the pendingVerificationRetry guard
+  assert.ok(
+    pushIdx < pendingCheckIdx,
+    "recentUnits.push must be unconditional — it must appear before the !pendingVerificationRetry check",
+  );
+});
+
+test("#2007 bug 2: detectStuck is still inside the pendingVerificationRetry guard", () => {
+  // detectStuck should only run when NOT in a retry — to avoid false positives
+  // during legitimate retries, but now the window is always populated.
+  const stuckSection = extractStuckDetectionSection(phasesSrc);
+  const pendingCheckIdx = stuckSection.indexOf("!s.pendingVerificationRetry");
+  assert.ok(
+    pendingCheckIdx !== -1,
+    "pendingVerificationRetry guard must exist in the stuck-detection section",
+  );
+  const detectStuckIdx = stuckSection.indexOf("detectStuck(", pendingCheckIdx);
+
+  assert.ok(
+    detectStuckIdx !== -1 && detectStuckIdx > pendingCheckIdx,
+    "detectStuck call must remain inside the !pendingVerificationRetry block",
+  );
+});
+
+// ─── Bug 3: dead dispatch-limit constants removed ────────────────────────────
+
+test("#2007 bug 3: MAX_UNIT_DISPATCHES is removed from session.ts", () => {
+  assert.ok(
+    !sessionSrc.includes("MAX_UNIT_DISPATCHES"),
+    "MAX_UNIT_DISPATCHES was never enforced and must be removed to prevent false confidence",
+  );
+});
+
+test("#2007 bug 3: MAX_LIFETIME_DISPATCHES is removed from session.ts", () => {
+  assert.ok(
+    !sessionSrc.includes("MAX_LIFETIME_DISPATCHES"),
+    "MAX_LIFETIME_DISPATCHES was never enforced and must be removed to prevent false confidence",
+  );
+});
+
+test("#2007 bug 3: dead constants are not re-exported from auto.ts", () => {
+  assert.ok(
+    !autoSrc.includes("MAX_UNIT_DISPATCHES"),
+    "MAX_UNIT_DISPATCHES must not be re-exported from auto.ts",
+  );
+  assert.ok(
+    !autoSrc.includes("MAX_LIFETIME_DISPATCHES"),
+    "MAX_LIFETIME_DISPATCHES must not be re-exported from auto.ts",
+  );
+});
+
+// ─── No stray dead constants left behind by the fix ──────────────────────────
+
+test("#2007 fix does not introduce a new dead constant (STATE_REBUILD_MIN_INTERVAL_MS)", () => {
+  // Bug 3 was about removing dead constants. A draft of this PR added
+  // STATE_REBUILD_MIN_INTERVAL_MS without referencing it — the same anti-pattern
+  // it set out to remove. Lock that down so it cannot regress.
+  assert.ok(
+    !postUnitSrc.includes("STATE_REBUILD_MIN_INTERVAL_MS"),
+    "STATE_REBUILD_MIN_INTERVAL_MS was added but never referenced — must be removed",
+  );
+});
+
+// ─── Behavioral: retry counter is cleared on success ─────────────────────────
+
+test("#2007 verificationRetryCount is cleared on artifact verification success", () => {
+  // Find the success-clear we just added: when triggerArtifactVerified is
+  // true, the retry counter for the current unit must be deleted so a future
+  // failure of the same unit type+id gets the full retry budget instead of
+  // a stale leftover count.
+  //
+  // We assert on the structural shape because a behavioral test would need
+  // to mock 30+ imports of postUnitPreVerification. The AutoSession-level
+  // test below covers the Map contract.
+  const successClearIdx = postUnitPreVerificationBody.indexOf(
+    "if (triggerArtifactVerified)",
+  );
+  assert.ok(
+    successClearIdx !== -1,
+    "Must guard the retry-count clear behind a triggerArtifactVerified check",
+  );
+  const deleteIdx = postUnitPreVerificationBody.indexOf(
+    "verificationRetryCount.delete",
+    successClearIdx,
+  );
+  assert.ok(
+    deleteIdx !== -1,
+    "verificationRetryCount.delete must be called on the verification-success path",
+  );
+});
+
+// ─── AutoSession.verificationRetryCount Map behavior ─────────────────────────
+
+import { AutoSession } from "../auto/session.ts";
+
+test("AutoSession.verificationRetryCount tracks attempts per retry key", () => {
+  const s = new AutoSession();
+  const key = "execute-task:M01/S01/T01";
+
+  assert.equal(s.verificationRetryCount.get(key), undefined);
+
+  s.verificationRetryCount.set(key, 1);
+  assert.equal(s.verificationRetryCount.get(key), 1);
+
+  s.verificationRetryCount.set(key, 2);
+  assert.equal(s.verificationRetryCount.get(key), 2);
+
+  // Simulate the success-clear path
+  s.verificationRetryCount.delete(key);
+  assert.equal(s.verificationRetryCount.get(key), undefined);
+});
+
+test("AutoSession.verificationRetryCount is cleared on session reset", () => {
+  const s = new AutoSession();
+  s.verificationRetryCount.set("execute-task:M01/S01/T01", 2);
+  s.verificationRetryCount.set("plan-slice:M01/S02", 1);
+
+  s.reset();
+
+  assert.equal(s.verificationRetryCount.size, 0);
+});
+
+test("AutoSession.verificationRetryCount independence across retry keys", () => {
+  // Critical: if retries for unit A fail twice and unit A then succeeds, the
+  // counter for A should be cleared but B's counter must remain untouched.
+  const s = new AutoSession();
+  s.verificationRetryCount.set("execute-task:M01/S01/T01", 2);
+  s.verificationRetryCount.set("execute-task:M01/S01/T02", 1);
+
+  s.verificationRetryCount.delete("execute-task:M01/S01/T01");
+
+  assert.equal(s.verificationRetryCount.get("execute-task:M01/S01/T01"), undefined);
+  assert.equal(s.verificationRetryCount.get("execute-task:M01/S01/T02"), 1);
+});