astrocode-workflow 0.3.3 → 0.3.5-1

package/dist/src/state/repo-lock.js

@@ -1,29 +1,580 @@
 // src/state/repo-lock.ts
 import fs from "node:fs";
 import path from "node:path";
-export function acquireRepoLock(lockPath) {
-    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
-    let fd;
+import crypto from "node:crypto";
+const LOCK_VERSION = 2;
+// Process-stable identifier for this Node process instance.
+const PROCESS_INSTANCE_ID = crypto.randomUUID();
+// Hard guardrails against garbage/corruption.
+const MAX_LOCK_BYTES = 64 * 1024; // 64KB; lock file should be tiny.
+// How many times we’ll attempt "atomic-ish replace" before giving up.
+const ATOMIC_REPLACE_RETRIES = 3;
+function nowISO() {
+    return new Date().toISOString();
+}
+function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+/**
+ * PID existence check:
+ * - EPERM => process exists but we can't signal it (treat as alive)
+ * - ESRCH => process does not exist (dead)
+ */
+function isPidAlive(pid) {
     try {
-        fd = fs.openSync(lockPath, "wx"); // exclusive create
+        process.kill(pid, 0);
+        return true;
     }
-    catch (e) {
-        const msg = e?.code === "EEXIST"
-            ? `Astrocode lock is already held (${lockPath}). Another opencode process is running in this repo.`
-            : `Failed to acquire lock (${lockPath}): ${e?.message ?? String(e)}`;
-        throw new Error(msg);
+    catch (err) {
+        const code = err?.code;
+        if (code === "EPERM")
+            return true;
+        if (code === "ESRCH")
+            return false;
+        // Unknown: conservative = don't evict.
+        return true;
     }
-    fs.writeFileSync(fd, `${process.pid}\n`, "utf8");
-    return {
-        release: () => {
+}
+function parseISOToMs(iso) {
+    const t = Date.parse(iso);
+    if (Number.isNaN(t))
+        return null;
+    return t;
+}
+function isStaleByAge(existing, staleMs) {
+    const updatedMs = parseISOToMs(existing.updated_at);
+    if (updatedMs === null)
+        return true;
+    return Date.now() - updatedMs > staleMs;
+}
+function safeUnlink(p) {
+    try {
+        fs.unlinkSync(p);
+    }
+    catch {
+        // ignore
+    }
+}
+/**
+ * Reads & validates lock file defensively.
+ * Supports both v2 JSON format and legacy PID-only format for compatibility.
+ * Returns null on any parse/validation failure.
+ */
+function readLock(lockPath) {
+    try {
+        const st = fs.statSync(lockPath);
+        if (!st.isFile())
+            return null;
+        if (st.size <= 0 || st.size > MAX_LOCK_BYTES)
+            return null;
+        const raw = fs.readFileSync(lockPath, "utf8").trim();
+        // Try v2 JSON first
+        try {
+            const parsed = JSON.parse(raw);
+            if (parsed && typeof parsed === "object" && parsed.v === LOCK_VERSION) {
+                if (typeof parsed.pid !== "number")
+                    return null;
+                if (typeof parsed.created_at !== "string")
+                    return null;
+                if (typeof parsed.updated_at !== "string")
+                    return null;
+                if (typeof parsed.repo_root !== "string")
+                    return null;
+                if (typeof parsed.instance_id !== "string")
+                    return null;
+                if (typeof parsed.lease_id !== "string")
+                    return null;
+                if (parsed.session_id !== undefined && typeof parsed.session_id !== "string")
+                    return null;
+                if (parsed.owner !== undefined && typeof parsed.owner !== "string")
+                    return null;
+                return parsed;
+            }
+        }
+        catch {
+            // Not JSON, try legacy format
+        }
+        // Legacy format: just PID as number string
+        const legacyPid = parseInt(raw, 10);
+        if (Number.isNaN(legacyPid) || legacyPid <= 0)
+            return null;
+        // Convert legacy to v2 format
+        const now = nowISO();
+        const leaseId = crypto.randomUUID();
+        return {
+            v: LOCK_VERSION,
+            pid: legacyPid,
+            created_at: now, // Approximate
+            updated_at: now,
+            repo_root: "", // Unknown, will be filled by caller
+            instance_id: PROCESS_INSTANCE_ID, // Assume same instance
+            session_id: undefined,
+            lease_id: leaseId,
+            owner: "legacy-lock",
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Best-effort directory fsync:
+ * Helps durability on crash for some filesystems (mostly POSIX).
+ * On platforms where opening a directory fails, we ignore.
+ */
+function fsyncDirBestEffort(dirPath) {
+    try {
+        const fd = fs.openSync(dirPath, "r");
+        try {
+            fs.fsyncSync(fd);
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+    }
+    catch {
+        // ignore (not portable)
+    }
+}
+/**
+ * "Atomic-ish" replace:
+ * - Write temp file
+ * - Try rename over target (POSIX generally atomic)
+ * - Windows can fail if target exists/locked; fallback to unlink+rename (not atomic, but best-effort)
+ * - Best-effort directory fsync after rename
+ */
+function writeLockAtomicish(lockPath, lock) {
+    const dir = path.dirname(lockPath);
+    fs.mkdirSync(dir, { recursive: true });
+    const tmp = `${lockPath}.${process.pid}.${Date.now()}.${crypto.randomUUID()}.tmp`;
+    const body = JSON.stringify(lock); // compact JSON to reduce IO
+    fs.writeFileSync(tmp, body, "utf8");
+    let lastErr = null;
+    for (let i = 0; i < ATOMIC_REPLACE_RETRIES; i++) {
+        try {
+            fs.renameSync(tmp, lockPath);
+            fsyncDirBestEffort(dir);
+            return;
+        }
+        catch (err) {
+            lastErr = err;
+            const code = err?.code;
+            // Common Windows-ish cases where rename over existing fails.
+            if (code === "EEXIST" || code === "EPERM" || code === "ENOTEMPTY") {
+                safeUnlink(lockPath);
+                continue;
+            }
+            // If tmp vanished somehow, stop.
+            if (code === "ENOENT")
+                break;
+            continue;
+        }
+    }
+    safeUnlink(tmp);
+    if (lastErr)
+        throw lastErr;
+    throw new Error(`Failed to replace lock file: ${lockPath}`);
+}
+/**
+ * Atomic "create if not exists" using exclusive open.
+ */
+function tryCreateExclusiveFile(filePath, contentsUtf8) {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    try {
+        const fd = fs.openSync(filePath, "wx");
+        try {
+            fs.writeFileSync(fd, contentsUtf8, "utf8");
+            fs.fsyncSync(fd);
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+        fsyncDirBestEffort(path.dirname(filePath));
+        return true;
+    }
+    catch (err) {
+        if (err?.code === "EEXIST")
+            return false;
+        throw err;
+    }
+}
+function tryCreateRepoLockExclusive(lockPath, lock) {
+    return tryCreateExclusiveFile(lockPath, JSON.stringify(lock));
+}
+const ACTIVE_LOCKS = new Map();
+function cacheKey(lockPath, sessionId) {
+    return `${lockPath}::${sessionId ?? ""}`;
+}
+/**
+ * Heartbeat loop:
+ * - setTimeout (not setInterval) to avoid backlog drift under load
+ * - Minimizes writes by enforcing minWriteMs
+ * - ABA-safe: only refreshes if lock matches our lease_id and process identity
+ * - Avoids unnecessary writes if lock already has a recent updated_at
+ */
+function startHeartbeat(opts) {
+    let stopped = false;
+    let lastWriteAt = 0;
+    let timer = null;
+    const tick = () => {
+        if (stopped)
+            return;
+        const now = Date.now();
+        const shouldAttempt = now - lastWriteAt >= opts.minWriteMs;
+        if (shouldAttempt) {
             try {
-                fs.closeSync(fd);
+                const existing = readLock(opts.lockPath);
+                if (existing &&
+                    existing.lease_id === opts.leaseId &&
+                    existing.pid === process.pid &&
+                    existing.instance_id === PROCESS_INSTANCE_ID) {
+                    const updatedMs = parseISOToMs(existing.updated_at);
+                    const isFresh = updatedMs !== null && now - updatedMs < opts.minWriteMs;
+                    if (!isFresh) {
+                        writeLockAtomicish(opts.lockPath, {
+                            ...existing,
+                            updated_at: nowISO(),
+                            repo_root: opts.repoRoot,
+                            session_id: opts.sessionId ?? existing.session_id,
+                            owner: opts.owner ?? existing.owner,
+                        });
+                        lastWriteAt = now;
+                    }
+                    else {
+                        lastWriteAt = now;
+                    }
+                }
             }
-            catch { }
+            catch (err) {
+                // Heartbeat write failed - don't propagate, just reschedule
+                // Lock will become stale if heartbeat continues failing
+                // eslint-disable-next-line no-console
+                console.warn("[Astrocode] Heartbeat write error:", err);
+            }
+        }
+        timer = setTimeout(tick, opts.heartbeatMs);
+        timer.unref?.();
+    };
+    tick();
+    return () => {
+        stopped = true;
+        if (timer)
+            clearTimeout(timer);
+    };
+}
+/**
+ * Shutdown cleanup:
+ * Best-effort release on normal termination signals.
+ */
+let EXIT_HOOK_INSTALLED = false;
+function installExitHookOnce() {
+    if (EXIT_HOOK_INSTALLED)
+        return;
+    EXIT_HOOK_INSTALLED = true;
+    const cleanup = () => {
+        for (const [key, h] of ACTIVE_LOCKS.entries()) {
             try {
-                fs.unlinkSync(lockPath);
+                ACTIVE_LOCKS.delete(key);
+                h.heartbeatStop();
+                h.releaseOnce();
+            }
+            catch {
+                // ignore
+            }
+        }
+    };
+    process.once("exit", cleanup);
+    process.once("SIGINT", () => {
+        cleanup();
+        process.exit(130);
+    });
+    process.once("SIGTERM", () => {
+        cleanup();
+        process.exit(143);
+    });
+    process.once("uncaughtException", (err) => {
+        // eslint-disable-next-line no-console
+        console.error("[Astrocode] Uncaught Exception, cleaning up locks:", err);
+        cleanup();
+        process.exit(1);
+    });
+    process.once("unhandledRejection", (reason) => {
+        // eslint-disable-next-line no-console
+        console.error("[Astrocode] Unhandled Rejection, cleaning up locks:", reason);
+        cleanup();
+        process.exit(1);
+    });
+}
+/**
+ * Acquire a repo-scoped lock with:
+ * - ✅ process-local caching + refcount (efficient repeated tool calls)
+ * - ✅ heartbeat lease + stale recovery
+ * - ✅ atomic create (`wx`) + portable replace fallback
+ * - ✅ dead PID eviction + stale eviction
+ * - ✅ no live takeover (even same session) to avoid concurrency stomps
+ * - ✅ ABA-safe release via lease_id fencing
+ * - ✅ exponential backoff + jitter to reduce FS churn
+ */
+export async function acquireRepoLock(opts) {
+    installExitHookOnce();
+    const { lockPath, repoRoot, sessionId, owner } = opts;
+    const retryMs = opts.retryMs ?? 8000;
+    const pollBaseMs = opts.pollMs ?? 20;
+    const pollMaxMs = opts.pollMaxMs ?? 250;
+    const heartbeatMs = opts.heartbeatMs ?? 200;
+    const minWriteMs = opts.minWriteMs ?? 800;
+    // Ensure stale is comfortably above minWriteMs to prevent false-stale under load.
+    const staleMs = Math.max(opts.staleMs ?? 2 * 60 * 1000, minWriteMs * 8);
+    // ✅ Fast path: reuse cached handle in the same process/session.
+    const key = cacheKey(lockPath, sessionId);
+    const cached = ACTIVE_LOCKS.get(key);
+    if (cached) {
+        cached.refCount += 1;
+        return {
+            release: () => {
+                cached.refCount -= 1;
+                if (cached.refCount <= 0) {
+                    ACTIVE_LOCKS.delete(key);
+                    cached.heartbeatStop();
+                    cached.releaseOnce();
+                }
+            },
+        };
+    }
+    const myPid = process.pid;
+    const startedAt = Date.now();
+    let pollMs = pollBaseMs;
+    while (true) {
+        const existing = readLock(lockPath);
+        // No lock (or unreadable/invalid) -> try create.
+        if (!existing) {
+            const now = nowISO();
+            const leaseId = crypto.randomUUID();
+            const candidate = {
+                v: LOCK_VERSION,
+                pid: myPid,
+                created_at: now,
+                updated_at: now,
+                repo_root: repoRoot,
+                instance_id: PROCESS_INSTANCE_ID,
+                session_id: sessionId,
+                lease_id: leaseId,
+                owner,
+            };
+            const created = tryCreateRepoLockExclusive(lockPath, candidate);
+            if (created) {
+                const heartbeatStop = startHeartbeat({
+                    lockPath,
+                    repoRoot,
+                    sessionId,
+                    owner,
+                    leaseId,
+                    heartbeatMs,
+                    minWriteMs,
+                });
+                const releaseOnce = () => {
+                    const cur = readLock(lockPath);
+                    if (!cur)
+                        return;
+                    // ABA-safe
+                    if (cur.lease_id !== leaseId)
+                        return;
+                    // Strict identity: only exact process instance can delete.
+                    if (cur.pid !== myPid)
+                        return;
+                    if (cur.instance_id !== PROCESS_INSTANCE_ID)
+                        return;
+                    safeUnlink(lockPath);
+                    fsyncDirBestEffort(path.dirname(lockPath));
+                };
+                const handle = {
+                    key,
+                    lockPath,
+                    sessionId,
+                    leaseId,
+                    refCount: 1,
+                    heartbeatStop,
+                    releaseOnce,
+                };
+                ACTIVE_LOCKS.set(key, handle);
+                return {
+                    release: () => {
+                        const h = ACTIVE_LOCKS.get(key);
+                        if (!h)
+                            return;
+                        h.refCount -= 1;
+                        if (h.refCount <= 0) {
+                            ACTIVE_LOCKS.delete(key);
+                            h.heartbeatStop();
+                            h.releaseOnce();
+                        }
+                    },
+                };
             }
-            catch { }
-        },
+            // Race lost; reset backoff and loop.
+            pollMs = pollBaseMs;
+            continue;
+        }
+        // Re-entrant by SAME PROCESS IDENTITY (pid+instance), or legacy lock with same PID.
+        if (existing.pid === myPid && (existing.instance_id === PROCESS_INSTANCE_ID || existing.owner === "legacy-lock")) {
+            const leaseId = crypto.randomUUID();
+            writeLockAtomicish(lockPath, {
+                ...existing,
+                v: LOCK_VERSION,
+                updated_at: nowISO(),
+                repo_root: repoRoot,
+                instance_id: PROCESS_INSTANCE_ID, // Upgrade legacy
+                session_id: sessionId ?? existing.session_id,
+                owner: owner ?? existing.owner,
+                lease_id: leaseId,
+            });
+            const heartbeatStop = startHeartbeat({
+                lockPath,
+                repoRoot,
+                sessionId: sessionId ?? existing.session_id,
+                owner: owner ?? existing.owner,
+                leaseId,
+                heartbeatMs,
+                minWriteMs,
+            });
+            const releaseOnce = () => {
+                const cur = readLock(lockPath);
+                if (!cur)
+                    return;
+                if (cur.lease_id !== leaseId)
+                    return;
+                if (cur.pid !== myPid)
+                    return;
+                if (cur.instance_id !== PROCESS_INSTANCE_ID)
+                    return;
+                safeUnlink(lockPath);
+                fsyncDirBestEffort(path.dirname(lockPath));
+            };
+            const handle = {
+                key,
+                lockPath,
+                sessionId,
+                leaseId,
+                refCount: 1,
+                heartbeatStop,
+                releaseOnce,
+            };
+            ACTIVE_LOCKS.set(key, handle);
+            return {
+                release: () => {
+                    const h = ACTIVE_LOCKS.get(key);
+                    if (!h)
+                        return;
+                    h.refCount -= 1;
+                    if (h.refCount <= 0) {
+                        ACTIVE_LOCKS.delete(key);
+                        h.heartbeatStop();
+                        h.releaseOnce();
+                    }
+                },
+            };
+        }
+        // 🚫 No live takeover (even same session).
+        // We only evict dead/stale locks.
+        const pidAlive = isPidAlive(existing.pid);
+        const staleByAge = isStaleByAge(existing, staleMs);
+        if (!pidAlive || staleByAge) {
+            safeUnlink(lockPath);
+            fsyncDirBestEffort(path.dirname(lockPath));
+            pollMs = pollBaseMs;
+            continue;
+        }
+        // Alive and not us -> bounded wait with exponential backoff + jitter.
+        if (Date.now() - startedAt > retryMs) {
+            const ownerBits = [
+                `pid=${existing.pid}`,
+                existing.session_id ? `session=${existing.session_id}` : null,
+                existing.owner ? `owner=${existing.owner}` : null,
+                `updated_at=${existing.updated_at}`,
+                sessionId && existing.session_id === sessionId ? `(same-session waiting)` : null,
+            ]
+                .filter(Boolean)
+                .join(" ");
+            throw new Error(`Astrocode lock is already held (${lockPath}). ${ownerBits}. ` +
+                `Close other opencode processes or wait.`);
+        }
+        const jitter = Math.floor(Math.random() * Math.min(12, pollMs));
+        await sleep(pollMs + jitter);
+        pollMs = Math.min(pollMaxMs, Math.floor(pollMs * 1.35));
+    }
+}
+/**
+ * Helper wrapper: always releases lock.
+ */
+export async function withRepoLock(opts) {
+    const handle = await acquireRepoLock({
+        lockPath: opts.lockPath,
+        repoRoot: opts.repoRoot,
+        sessionId: opts.sessionId,
+        owner: opts.owner,
+    });
+    try {
+        return await opts.fn();
+    }
+    finally {
+        handle.release();
+    }
+}
+/**
+ * Get lock file status and diagnostics.
+ * Returns detailed information about the current lock state.
+ */
+export function getLockStatus(lockPath, staleMs = 30_000) {
+    const existing = readLock(lockPath);
+    if (!existing) {
+        return {
+            exists: false,
+            path: lockPath,
+        };
+    }
+    const updatedMs = parseISOToMs(existing.updated_at);
+    const ageMs = updatedMs !== null ? Date.now() - updatedMs : undefined;
+    const pidAlive = isPidAlive(existing.pid);
+    const isStale = isStaleByAge(existing, staleMs);
+    return {
+        exists: true,
+        path: lockPath,
+        pid: existing.pid,
+        pidAlive,
+        instanceId: existing.instance_id,
+        sessionId: existing.session_id,
+        owner: existing.owner,
+        leaseId: existing.lease_id,
+        createdAt: existing.created_at,
+        updatedAt: existing.updated_at,
+        ageMs,
+        isStale,
+        repoRoot: existing.repo_root,
+        version: existing.v,
     };
 }
+/**
+ * Attempt to remove a lock file if it's safe to do so.
+ * Only removes locks with dead PIDs or stale timestamps.
+ * Returns true if lock was removed, false if lock is still held.
+ */
+export function tryRemoveStaleLock(lockPath, staleMs = 30_000) {
+    const existing = readLock(lockPath);
+    if (!existing) {
+        return { removed: false, reason: "No lock file found" };
+    }
+    const pidAlive = isPidAlive(existing.pid);
+    const isStale = isStaleByAge(existing, staleMs);
+    if (!pidAlive) {
+        safeUnlink(lockPath);
+        fsyncDirBestEffort(path.dirname(lockPath));
+        return { removed: true, reason: `Dead PID ${existing.pid}` };
+    }
+    if (isStale) {
+        safeUnlink(lockPath);
+        fsyncDirBestEffort(path.dirname(lockPath));
+        const ageSeconds = Math.floor((Date.now() - (parseISOToMs(existing.updated_at) ?? 0)) / 1000);
+        return { removed: true, reason: `Stale lock (${ageSeconds}s old, threshold ${staleMs / 1000}s)` };
+    }
+    return { removed: false, reason: `Lock is active (PID ${existing.pid} alive, age ${Math.floor((Date.now() - (parseISOToMs(existing.updated_at) ?? 0)) / 1000)}s)` };
+}

package/dist/src/state/workflow-repo-lock.d.ts

@@ -0,0 +1,16 @@
+import type { acquireRepoLock } from "./repo-lock";
+type RepoLockAcquire = typeof acquireRepoLock;
+/**
+ * Acquire ONCE per workflow/session in this process.
+ * Nested calls reuse the same held lock (no reacquire, no churn).
+ */
+export declare function workflowRepoLock<T>(deps: {
+    acquireRepoLock: RepoLockAcquire;
+}, opts: {
+    lockPath: string;
+    repoRoot: string;
+    sessionId?: string;
+    owner?: string;
+    fn: () => Promise<T>;
+}): Promise<T>;
+export {};

package/dist/src/state/workflow-repo-lock.js

@@ -0,0 +1,50 @@
+const HELD_BY_KEY = new Map();
+function key(lockPath, sessionId) {
+    return `${lockPath}::${sessionId ?? ""}`;
+}
+/**
+ * Acquire ONCE per workflow/session in this process.
+ * Nested calls reuse the same held lock (no reacquire, no churn).
+ */
+export async function workflowRepoLock(deps, opts) {
+    const k = key(opts.lockPath, opts.sessionId);
+    const existing = HELD_BY_KEY.get(k);
+    if (existing) {
+        existing.depth += 1;
+        try {
+            return await opts.fn();
+        }
+        finally {
+            existing.depth -= 1;
+            if (existing.depth <= 0) {
+                HELD_BY_KEY.delete(k);
+                existing.release();
+            }
+        }
+    }
+    // IMPORTANT: this is tuned for "hold for whole workflow".
+    const handle = await deps.acquireRepoLock({
+        lockPath: opts.lockPath,
+        repoRoot: opts.repoRoot,
+        sessionId: opts.sessionId,
+        owner: opts.owner,
+        retryMs: 30_000,
+        staleMs: 30_000, // Reduced from 2 minutes to 30 seconds for faster stale lock recovery
+        heartbeatMs: 200,
+        minWriteMs: 800,
+        pollMs: 20,
+        pollMaxMs: 250,
+    });
+    const held = { release: handle.release, depth: 1 };
+    HELD_BY_KEY.set(k, held);
+    try {
+        return await opts.fn();
+    }
+    finally {
+        held.depth -= 1;
+        if (held.depth <= 0) {
+            HELD_BY_KEY.delete(k);
+            held.release();
+        }
+    }
+}

package/dist/src/tools/index.js

@@ -11,6 +11,7 @@ import { createAstroRepairTool } from "./repair";
 import { createAstroHealthTool } from "./health";
 import { createAstroResetTool } from "./reset";
 import { createAstroMetricsTool } from "./metrics";
+import { createAstroLockStatusTool } from "./lock";
 export function createAstroTools(opts) {
     const { ctx, config, agents, runtime } = opts;
     const { db } = runtime;
@@ -22,6 +23,7 @@ export function createAstroTools(opts) {
     tools.astro_health = createAstroHealthTool({ ctx, config, db });
     tools.astro_reset = createAstroResetTool({ ctx, config, db });
     tools.astro_metrics = createAstroMetricsTool({ ctx, config });
+    tools.astro_lock_status = createAstroLockStatusTool({ ctx });
     // Recovery tool - available even in limited mode to allow DB initialization
     tools.astro_init = createAstroInitTool({ ctx, config, runtime });
     // Database-dependent tools
@@ -83,6 +85,7 @@ export function createAstroTools(opts) {
         ["_astro_health", "astro_health"],
         ["_astro_reset", "astro_reset"],
         ["_astro_metrics", "astro_metrics"],
+        ["_astro_lock_status", "astro_lock_status"],
     ];
     // Only add aliases for tools that exist
     for (const [alias, target] of aliases) {

package/dist/src/tools/lock.d.ts

@@ -0,0 +1,4 @@
+import { type ToolDefinition } from "@opencode-ai/plugin/tool";
+export declare function createAstroLockStatusTool(opts: {
+    ctx: any;
+}): ToolDefinition;