astrocode-workflow 0.3.1 → 0.3.3

package/src/state/repo-lock.ts

@@ -0,0 +1,574 @@
+// src/state/repo-lock.ts
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+
+const LOCK_VERSION = 2;
+
+// Process-stable identifier for this Node process instance.
+const PROCESS_INSTANCE_ID = crypto.randomUUID();
+
+// Hard guardrails against garbage/corruption.
+const MAX_LOCK_BYTES = 64 * 1024; // 64KB; lock file should be tiny.
+
+// How many times we’ll attempt "atomic-ish replace" before giving up.
+const ATOMIC_REPLACE_RETRIES = 3;
+
+type LockFile = {
+  v: number;
+
+  pid: number;
+  created_at: string;
+  updated_at: string;
+  repo_root: string;
+
+  // Identifies the running process instance (process-stable).
+  instance_id: string;
+
+  // Logical session owner (propagated by opencode).
+  session_id?: string;
+
+  // Fencing token: changes every successful acquire.
+  // Prevents ABA release deleting someone else’s lock.
+  lease_id: string;
+
+  owner?: string; // optional human-readable owner
+};
+
+function nowISO(): string {
+  return new Date().toISOString();
+}
+
+function sleep(ms: number) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+/**
+ * PID existence check:
+ * - EPERM => process exists but we can't signal it (treat as alive)
+ * - ESRCH => process does not exist (dead)
+ */
+function isPidAlive(pid: number): boolean {
+  try {
+    (process as any).kill(pid, 0);
+    return true;
+  } catch (err: any) {
+    const code = err?.code;
+    if (code === "EPERM") return true;
+    if (code === "ESRCH") return false;
+    // Unknown: conservative = don't evict.
+    return true;
+  }
+}
+
+function parseISOToMs(iso: string): number | null {
+  const t = Date.parse(iso);
+  if (Number.isNaN(t)) return null;
+  return t;
+}
+
+function isStaleByAge(existing: LockFile, staleMs: number): boolean {
+  const updatedMs = parseISOToMs(existing.updated_at);
+  if (updatedMs === null) return true;
+  return Date.now() - updatedMs > staleMs;
+}
+
+function safeUnlink(p: string) {
+  try {
+    fs.unlinkSync(p);
+  } catch {
+    // ignore
+  }
+}
+
+/**
+ * Reads & validates lock file defensively.
+ * Returns null on any parse/validation failure.
+ */
+function readLock(lockPath: string): LockFile | null {
+  try {
+    const st = fs.statSync(lockPath);
+    if (!st.isFile()) return null;
+    if (st.size <= 0 || st.size > MAX_LOCK_BYTES) return null;
+
+    const raw = fs.readFileSync(lockPath, "utf8");
+    const parsed = JSON.parse(raw) as LockFile;
+
+    if (!parsed) return null;
+    if (parsed.v !== LOCK_VERSION) return null;
+
+    if (typeof parsed.pid !== "number") return null;
+    if (typeof parsed.created_at !== "string") return null;
+    if (typeof parsed.updated_at !== "string") return null;
+    if (typeof parsed.repo_root !== "string") return null;
+    if (typeof parsed.instance_id !== "string") return null;
+    if (typeof parsed.lease_id !== "string") return null;
+
+    if (parsed.session_id !== undefined && typeof parsed.session_id !== "string") return null;
+    if (parsed.owner !== undefined && typeof parsed.owner !== "string") return null;
+
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Best-effort directory fsync:
+ * Helps durability on crash for some filesystems (mostly POSIX).
+ * On platforms where opening a directory fails, we ignore.
+ */
+function fsyncDirBestEffort(dirPath: string) {
+  try {
+    const fd = fs.openSync(dirPath, "r");
+    try {
+      fs.fsyncSync(fd);
+    } finally {
+      fs.closeSync(fd);
+    }
+  } catch {
+    // ignore (not portable)
+  }
+}
+
+/**
+ * "Atomic-ish" replace:
+ * - Write temp file
+ * - Try rename over target (POSIX generally atomic)
+ * - Windows can fail if target exists/locked; fallback to unlink+rename (not atomic, but best-effort)
+ * - Best-effort directory fsync after rename
+ */
+function writeLockAtomicish(lockPath: string, lock: LockFile) {
+  const dir = path.dirname(lockPath);
+  fs.mkdirSync(dir, { recursive: true });
+
+  const tmp = `${lockPath}.${(process as any).pid}.${Date.now()}.${crypto.randomUUID()}.tmp`;
+  const body = JSON.stringify(lock); // compact JSON to reduce IO
+
+  fs.writeFileSync(tmp, body, "utf8");
+
+  let lastErr: any = null;
+  for (let i = 0; i < ATOMIC_REPLACE_RETRIES; i++) {
+    try {
+      fs.renameSync(tmp, lockPath);
+      fsyncDirBestEffort(dir);
+      return;
+    } catch (err: any) {
+      lastErr = err;
+      const code = err?.code;
+
+      // Common Windows-ish cases where rename over existing fails.
+      if (code === "EEXIST" || code === "EPERM" || code === "ENOTEMPTY") {
+        safeUnlink(lockPath);
+        continue;
+      }
+
+      // If tmp vanished somehow, stop.
+      if (code === "ENOENT") break;
+
+      continue;
+    }
+  }
+
+  safeUnlink(tmp);
+  if (lastErr) throw lastErr;
+  throw new Error(`Failed to replace lock file: ${lockPath}`);
+}
+
+/**
+ * Atomic "create if not exists" using exclusive open.
+ */
+function tryCreateExclusiveFile(filePath: string, contentsUtf8: string): boolean {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+
+  try {
+    const fd = fs.openSync(filePath, "wx");
+    try {
+      fs.writeFileSync(fd, contentsUtf8, "utf8");
+      fs.fsyncSync(fd);
+    } finally {
+      fs.closeSync(fd);
+    }
+    fsyncDirBestEffort(path.dirname(filePath));
+    return true;
+  } catch (err: any) {
+    if (err?.code === "EEXIST") return false;
+    throw err;
+  }
+}
+
+function tryCreateRepoLockExclusive(lockPath: string, lock: LockFile): boolean {
+  return tryCreateExclusiveFile(lockPath, JSON.stringify(lock));
+}
+
+/**
+ * In-process lock cache:
+ * Prevents repeated acquire/release cycles during tool-call storms.
+ */
+type CachedHandle = {
+  key: string;
+  lockPath: string;
+  sessionId?: string;
+  leaseId: string;
+  refCount: number;
+  heartbeatStop: () => void;
+  releaseOnce: () => void;
+};
+
+const ACTIVE_LOCKS = new Map<string, CachedHandle>();
+
+function cacheKey(lockPath: string, sessionId?: string): string {
+  return `${lockPath}::${sessionId ?? ""}`;
+}
+
+/**
+ * Heartbeat loop:
+ * - setTimeout (not setInterval) to avoid backlog drift under load
+ * - Minimizes writes by enforcing minWriteMs
+ * - ABA-safe: only refreshes if lock matches our lease_id and process identity
+ * - Avoids unnecessary writes if lock already has a recent updated_at
+ */
+function startHeartbeat(opts: {
+  lockPath: string;
+  repoRoot: string;
+  sessionId?: string;
+  owner?: string;
+  leaseId: string;
+  heartbeatMs: number;
+  minWriteMs: number;
+}): () => void {
+  let stopped = false;
+  let lastWriteAt = 0;
+  let timer: NodeJS.Timeout | null = null;
+
+  const tick = () => {
+    if (stopped) return;
+
+    const now = Date.now();
+    const shouldAttempt = now - lastWriteAt >= opts.minWriteMs;
+
+    if (shouldAttempt) {
+      const existing = readLock(opts.lockPath);
+
+      if (
+        existing &&
+        existing.lease_id === opts.leaseId &&
+        existing.pid === (process as any).pid &&
+        existing.instance_id === PROCESS_INSTANCE_ID
+      ) {
+        const updatedMs = parseISOToMs(existing.updated_at);
+        const isFresh = updatedMs !== null && now - updatedMs < opts.minWriteMs;
+
+        if (!isFresh) {
+          writeLockAtomicish(opts.lockPath, {
+            ...existing,
+            updated_at: nowISO(),
+            repo_root: opts.repoRoot,
+            session_id: opts.sessionId ?? existing.session_id,
+            owner: opts.owner ?? existing.owner,
+          });
+          lastWriteAt = now;
+        } else {
+          lastWriteAt = now;
+        }
+      }
+    }
+
+    timer = setTimeout(tick, opts.heartbeatMs);
+    (timer as any).unref?.();
+  };
+
+  tick();
+
+  return () => {
+    stopped = true;
+    if (timer) clearTimeout(timer);
+  };
+}
+
+/**
+ * Shutdown cleanup:
+ * Best-effort release on normal termination signals.
+ */
+let EXIT_HOOK_INSTALLED = false;
+function installExitHookOnce() {
+  if (EXIT_HOOK_INSTALLED) return;
+  EXIT_HOOK_INSTALLED = true;
+
+  const cleanup = () => {
+    for (const [key, h] of ACTIVE_LOCKS.entries()) {
+      try {
+        ACTIVE_LOCKS.delete(key);
+        h.heartbeatStop();
+        h.releaseOnce();
+      } catch {
+        // ignore
+      }
+    }
+  };
+
+  (process as any).once("exit", cleanup);
+  (process as any).once("SIGINT", () => {
+    cleanup();
+    (process as any).exit(130);
+  });
+  (process as any).once("SIGTERM", () => {
+    cleanup();
+    (process as any).exit(143);
+  });
+}
+
+/**
+ * Acquire a repo-scoped lock with:
+ * - ✅ process-local caching + refcount (efficient repeated tool calls)
+ * - ✅ heartbeat lease + stale recovery
+ * - ✅ atomic create (`wx`) + portable replace fallback
+ * - ✅ dead PID eviction + stale eviction
+ * - ✅ no live takeover (even same session) to avoid concurrency stomps
+ * - ✅ ABA-safe release via lease_id fencing
+ * - ✅ exponential backoff + jitter to reduce FS churn
+ */
+export async function acquireRepoLock(opts: {
+  lockPath: string;
+  repoRoot: string;
+  sessionId?: string;
+  owner?: string;
+
+  retryMs?: number;      // default 8000
+  pollMs?: number;       // default 20
+  pollMaxMs?: number;    // default 250
+  staleMs?: number;      // default 2 minutes
+  heartbeatMs?: number;  // default 200
+  minWriteMs?: number;   // default 800
+}): Promise<{ release: () => void }> {
+  installExitHookOnce();
+
+  const { lockPath, repoRoot, sessionId, owner } = opts;
+
+  const retryMs = opts.retryMs ?? 8000;
+  const pollBaseMs = opts.pollMs ?? 20;
+  const pollMaxMs = opts.pollMaxMs ?? 250;
+
+  const heartbeatMs = opts.heartbeatMs ?? 200;
+  const minWriteMs = opts.minWriteMs ?? 800;
+
+  // Ensure stale is comfortably above minWriteMs to prevent false-stale under load.
+  const staleMs = Math.max(opts.staleMs ?? 2 * 60 * 1000, minWriteMs * 8);
+
+  // ✅ Fast path: reuse cached handle in the same process/session.
+  const key = cacheKey(lockPath, sessionId);
+  const cached = ACTIVE_LOCKS.get(key);
+  if (cached) {
+    cached.refCount += 1;
+    return {
+      release: () => {
+        cached.refCount -= 1;
+        if (cached.refCount <= 0) {
+          ACTIVE_LOCKS.delete(key);
+          cached.heartbeatStop();
+          cached.releaseOnce();
+        }
+      },
+    };
+  }
+
+  const myPid = ((process as any).pid as number);
+  const startedAt = Date.now();
+  let pollMs = pollBaseMs;
+
+  while (true) {
+    const existing = readLock(lockPath);
+
+    // No lock (or unreadable/invalid) -> try create.
+    if (!existing) {
+      const now = nowISO();
+      const leaseId = crypto.randomUUID();
+
+      const candidate: LockFile = {
+        v: LOCK_VERSION,
+        pid: myPid,
+        created_at: now,
+        updated_at: now,
+        repo_root: repoRoot,
+        instance_id: PROCESS_INSTANCE_ID,
+        session_id: sessionId,
+        lease_id: leaseId,
+        owner,
+      };
+
+      const created = tryCreateRepoLockExclusive(lockPath, candidate);
+      if (created) {
+        const heartbeatStop = startHeartbeat({
+          lockPath,
+          repoRoot,
+          sessionId,
+          owner,
+          leaseId,
+          heartbeatMs,
+          minWriteMs,
+        });
+
+        const releaseOnce = () => {
+          const cur = readLock(lockPath);
+          if (!cur) return;
+
+          // ABA-safe
+          if (cur.lease_id !== leaseId) return;
+
+          // Strict identity: only exact process instance can delete.
+          if (cur.pid !== myPid) return;
+          if (cur.instance_id !== PROCESS_INSTANCE_ID) return;
+
+          safeUnlink(lockPath);
+          fsyncDirBestEffort(path.dirname(lockPath));
+        };
+
+        const handle: CachedHandle = {
+          key,
+          lockPath,
+          sessionId,
+          leaseId,
+          refCount: 1,
+          heartbeatStop,
+          releaseOnce,
+        };
+        ACTIVE_LOCKS.set(key, handle);
+
+        return {
+          release: () => {
+            const h = ACTIVE_LOCKS.get(key);
+            if (!h) return;
+            h.refCount -= 1;
+            if (h.refCount <= 0) {
+              ACTIVE_LOCKS.delete(key);
+              h.heartbeatStop();
+              h.releaseOnce();
+            }
+          },
+        };
+      }
+
+      // Race lost; reset backoff and loop.
+      pollMs = pollBaseMs;
+      continue;
+    }
+
+    // Re-entrant by SAME PROCESS IDENTITY (pid+instance).
+    if (existing.pid === myPid && existing.instance_id === PROCESS_INSTANCE_ID) {
+      const leaseId = crypto.randomUUID();
+
+      writeLockAtomicish(lockPath, {
+        ...existing,
+        v: LOCK_VERSION,
+        updated_at: nowISO(),
+        repo_root: repoRoot,
+        session_id: sessionId ?? existing.session_id,
+        owner: owner ?? existing.owner,
+        lease_id: leaseId,
+      });
+
+      const heartbeatStop = startHeartbeat({
+        lockPath,
+        repoRoot,
+        sessionId: sessionId ?? existing.session_id,
+        owner: owner ?? existing.owner,
+        leaseId,
+        heartbeatMs,
+        minWriteMs,
+      });
+
+      const releaseOnce = () => {
+        const cur = readLock(lockPath);
+        if (!cur) return;
+        if (cur.lease_id !== leaseId) return;
+        if (cur.pid !== myPid) return;
+        if (cur.instance_id !== PROCESS_INSTANCE_ID) return;
+        safeUnlink(lockPath);
+        fsyncDirBestEffort(path.dirname(lockPath));
+      };
+
+      const handle: CachedHandle = {
+        key,
+        lockPath,
+        sessionId,
+        leaseId,
+        refCount: 1,
+        heartbeatStop,
+        releaseOnce,
+      };
+      ACTIVE_LOCKS.set(key, handle);
+
+      return {
+        release: () => {
+          const h = ACTIVE_LOCKS.get(key);
+          if (!h) return;
+          h.refCount -= 1;
+          if (h.refCount <= 0) {
+            ACTIVE_LOCKS.delete(key);
+            h.heartbeatStop();
+            h.releaseOnce();
+          }
+        },
+      };
+    }
+
+    // 🚫 No live takeover (even same session).
+    // We only evict dead/stale locks.
+
+    const pidAlive = isPidAlive(existing.pid);
+    const staleByAge = isStaleByAge(existing, staleMs);
+
+    if (!pidAlive || staleByAge) {
+      safeUnlink(lockPath);
+      fsyncDirBestEffort(path.dirname(lockPath));
+      pollMs = pollBaseMs;
+      continue;
+    }
+
+    // Alive and not us -> bounded wait with exponential backoff + jitter.
+    if (Date.now() - startedAt > retryMs) {
+      const ownerBits = [
+        `pid=${existing.pid}`,
+        existing.session_id ? `session=${existing.session_id}` : null,
+        existing.owner ? `owner=${existing.owner}` : null,
+        `updated_at=${existing.updated_at}`,
+        sessionId && existing.session_id === sessionId ? `(same-session waiting)` : null,
+      ]
+        .filter(Boolean)
+        .join(" ");
+
+      throw new Error(
+        `Astrocode lock is already held (${lockPath}). ${ownerBits}. ` +
+          `Close other opencode processes or wait.`
+      );
+    }
+
+    const jitter = Math.floor(Math.random() * Math.min(12, pollMs));
+    await sleep(pollMs + jitter);
+    pollMs = Math.min(pollMaxMs, Math.floor(pollMs * 1.35));
+  }
+}
+
+/**
+ * Helper wrapper: always releases lock.
+ */
+export async function withRepoLock<T>(opts: {
+  lockPath: string;
+  repoRoot: string;
+  sessionId?: string;
+  owner?: string;
+  fn: () => Promise<T>;
+}): Promise<T> {
+  const handle = await acquireRepoLock({
+    lockPath: opts.lockPath,
+    repoRoot: opts.repoRoot,
+    sessionId: opts.sessionId,
+    owner: opts.owner,
+  });
+
+  try {
+    return await opts.fn();
+  } finally {
+    handle.release();
+  }
+}

package/src/state/workflow-repo-lock.ts

@@ -0,0 +1,74 @@
+// src/state/workflow-repo-lock.ts
+import type { acquireRepoLock } from "./repo-lock";
+
+type RepoLockAcquire = typeof acquireRepoLock;
+
+type Held = {
+  release: () => void;
+  depth: number;
+};
+
+const HELD_BY_KEY = new Map<string, Held>();
+
+function key(lockPath: string, sessionId?: string) {
+  return `${lockPath}::${sessionId ?? ""}`;
+}
+
+/**
+ * Acquire ONCE per workflow/session in this process.
+ * Nested calls reuse the same held lock (no reacquire, no churn).
+ */
+export async function workflowRepoLock<T>(
+  deps: { acquireRepoLock: RepoLockAcquire },
+  opts: {
+    lockPath: string;
+    repoRoot: string;
+    sessionId?: string;
+    owner?: string;
+    fn: () => Promise<T>;
+  }
+): Promise<T> {
+  const k = key(opts.lockPath, opts.sessionId);
+  const existing = HELD_BY_KEY.get(k);
+
+  if (existing) {
+    existing.depth += 1;
+    try {
+      return await opts.fn();
+    } finally {
+      existing.depth -= 1;
+      if (existing.depth <= 0) {
+        HELD_BY_KEY.delete(k);
+        existing.release();
+      }
+    }
+  }
+
+  // IMPORTANT: this is tuned for "hold for whole workflow".
+  const handle = await deps.acquireRepoLock({
+    lockPath: opts.lockPath,
+    repoRoot: opts.repoRoot,
+    sessionId: opts.sessionId,
+    owner: opts.owner,
+
+    retryMs: 30_000,
+    staleMs: 2 * 60_000,
+    heartbeatMs: 200,
+    minWriteMs: 800,
+    pollMs: 20,
+    pollMaxMs: 250,
+  });
+
+  const held: Held = { release: handle.release, depth: 1 };
+  HELD_BY_KEY.set(k, held);
+
+  try {
+    return await opts.fn();
+  } finally {
+    held.depth -= 1;
+    if (held.depth <= 0) {
+      HELD_BY_KEY.delete(k);
+      held.release();
+    }
+  }
+}