@zq-silk/yui 0.6.0 → 0.6.2

package/dist/repository/projectMaintenanceLock.js

@@ -0,0 +1,315 @@
+import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+/**
+ * Per-Project maintenance fence.
+ *
+ * `project migrate`, Task workspace rebuild, legacy ref archive, and Task
+ * archive cleanup are separate CLI processes that all mutate a Project's
+ * Git repository. Without coordination they can interleave with each other
+ * and with the Controller's worktree preparation. The fence is an exclusive
+ * directory under the Home's `locks/` area (O_EXCL via mkdir), so it works
+ * for managed and external Projects alike and never touches the Project's
+ * own checkout. A crashed holder is reclaimed once its owner process is
+ * gone; the owner file records PID + process start time, so a recycled PID
+ * can never pass for the original holder.
+ *
+ * Maintenance operations are long-lived, so a contended fence fails fast
+ * with a retryable error instead of blocking the CLI for seconds.
+ */
+const PROJECT_MAINTENANCE_LOCK_TIMEOUT_MS = 250;
+const PROJECT_MAINTENANCE_LOCK_RETRY_MS = 10;
+/** A lock older than this with a dead owner is reclaimed. */
+const STALE_PROJECT_MAINTENANCE_LOCK_AGE_MS = 1_000;
+/**
+ * Raised when a Project's maintenance fence is already held. The caller
+ * made no changes and is safe to retry once the holder finishes.
+ */
+export class ProjectMaintenanceLockedError extends Error {
+    projectId;
+    retryable = true;
+    constructor(projectId) {
+        super(`Project maintenance is already in progress for ${projectId}; retry once it finishes.`);
+        this.name = "ProjectMaintenanceLockedError";
+        this.projectId = projectId;
+    }
+}
+/** Directory of one Project's maintenance fence, below the Home's locks area. */
+export function projectMaintenanceLockPath(home, projectId) {
+    return join(home, "locks", "projects", `${projectId}.lock`);
+}
+/**
+ * Acquire one Project's maintenance fence. Returns the release function;
+ * callers MUST release on every exit path (try/finally). A live holder
+ * fails fast with {@link ProjectMaintenanceLockedError}; a stale (dead)
+ * holder is reclaimed.
+ *
+ * There is no in-process reentrancy: every acquisition contends, so two
+ * independent operations in the same process are mutually exclusive just
+ * like two processes. Genuine lexical nesting (an operation that needs the
+ * fence while its caller already holds it) goes through private
+ * already-locked methods, never through a second acquisition.
+ */
+export function acquireProjectMaintenanceLock(home, projectId) {
+    const lock = projectMaintenanceLockPath(home, projectId);
+    mkdirSync(join(home, "locks", "projects"), { recursive: true, mode: 0o700 });
+    const deadline = Date.now() + PROJECT_MAINTENANCE_LOCK_TIMEOUT_MS;
+    while (true) {
+        try {
+            mkdirSync(lock, { mode: 0o700 });
+            const ownerIdentity = writeOwnerIdentity(lock);
+            let released = false;
+            return () => {
+                if (released)
+                    return;
+                released = true;
+                // Release only the exact acquired instance: a lock that was reclaimed
+                // and replaced by a successor after a crash is never deleted by a
+                // stale release handle.
+                releaseOwnedProjectMaintenanceLock(lock, ownerIdentity);
+            };
+        }
+        catch (error) {
+            if (!(error instanceof Error && "code" in error && error.code === "EEXIST"))
+                throw error;
+            reclaimStaleProjectMaintenanceLock(lock);
+            if (Date.now() >= deadline)
+                throw new ProjectMaintenanceLockedError(projectId);
+            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, PROJECT_MAINTENANCE_LOCK_RETRY_MS);
+        }
+    }
+}
+/** Write this process's owner identity and return the exact bytes recorded. */
+function writeOwnerIdentity(lock) {
+    const identity = `${process.pid}:${processStartIdentity() ?? ""}`;
+    writeFileSync(join(lock, "owner"), `${identity}\n`, { mode: 0o600 });
+    return identity;
+}
+/**
+ * Remove the lock directory only when its owner file still records the exact
+ * identity this handle acquired. A replaced (successor) lock is left intact.
+ */
+function releaseOwnedProjectMaintenanceLock(lock, ownerIdentity) {
+    let owner;
+    try {
+        owner = readFileSync(join(lock, "owner"), "utf8").trim();
+    }
+    catch (error) {
+        if (isEnoent(error))
+            return; // already gone; nothing to release.
+        throw error;
+    }
+    if (owner !== ownerIdentity)
+        return;
+    rmSync(lock, { recursive: true, force: true });
+}
+/**
+ * Acquire the maintenance fence for several Projects in a stable (sorted)
+ * order, so multi-Project maintenance can never deadlock against itself.
+ * A failure releases every fence already taken. Every Project contends,
+ * including ones this process already holds: callers that need nesting must
+ * use an already-locked private path instead of acquiring twice.
+ */
+export function acquireProjectMaintenanceLocks(home, projectIds) {
+    const releases = [];
+    try {
+        for (const projectId of [...new Set(projectIds)].sort()) {
+            releases.push(acquireProjectMaintenanceLock(home, projectId));
+        }
+    }
+    catch (error) {
+        for (const release of releases.reverse())
+            release();
+        throw error;
+    }
+    let released = false;
+    return () => {
+        if (released)
+            return;
+        released = true;
+        for (const release of releases.reverse())
+            release();
+    };
+}
+/**
+ * Non-blocking fence check for the Controller: a Project is fenced only
+ * while a live process holds its lock. A stale (dead-owner) lock reads as
+ * unfenced, so a crashed maintenance process never wedges preparation.
+ */
+export function isProjectMaintenanceFenced(home, projectId) {
+    const lock = projectMaintenanceLockPath(home, projectId);
+    if (!existsSync(lock))
+        return false;
+    return lockOwnerIsAlive(lock);
+}
+/**
+ * Reclaim a stale (dead-owner) lock through a crash-safe compare-and-delete
+ * critical section. Two contenders can both observe the same dead owner;
+ * without serialization the second could rmSync the successor lock the first
+ * created after reclaiming, admitting two holders. A reclaim lock (itself
+ * reclaimable) serializes reclaimers, and under it we re-read the owner and
+ * delete only when it is still the exact stale instance observed above.
+ */
+function reclaimStaleProjectMaintenanceLock(lock) {
+    let expectedOwner;
+    try {
+        expectedOwner = readFileSync(join(lock, "owner"), "utf8");
+    }
+    catch (error) {
+        if (!isEnoent(error))
+            throw error;
+        // Ownerless lock: a crash between mkdir and owner publication. Reclaim
+        // only when the directory is old enough that the creator is not still
+        // initializing; a fresh lock may have its owner file written imminently.
+        try {
+            if (Date.now() - statSync(lock).mtimeMs < STALE_PROJECT_MAINTENANCE_LOCK_AGE_MS)
+                return;
+        }
+        catch (statError) {
+            if (isEnoent(statError))
+                return;
+            throw statError;
+        }
+        expectedOwner = null;
+    }
+    try {
+        if (Date.now() - statSync(lock).mtimeMs < STALE_PROJECT_MAINTENANCE_LOCK_AGE_MS)
+            return;
+    }
+    catch (error) {
+        if (isEnoent(error))
+            return;
+        throw error;
+    }
+    if (expectedOwner !== null && lockOwnerIsAlive(lock))
+        return;
+    const reclaimLock = `${lock}.reclaim`;
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+        try {
+            mkdirSync(reclaimLock, { mode: 0o700 });
+        }
+        catch (error) {
+            if (!isEexist(error))
+                throw error;
+            // Another reclaimer holds the critical section. Reclaim its lock only
+            // when orphaned (dead owner and old enough), then retry once; a live
+            // reclaimer will finish and the caller's next O_EXCL mkdir settles it.
+            if (reclaimOrphanedReclaimLock(reclaimLock))
+                continue;
+            return;
+        }
+        try {
+            writeOwnerIdentity(reclaimLock);
+            // Compare-and-delete under the lock: re-read and remove only the exact
+            // stale instance observed above. A successor that replaced the lock has
+            // different owner bytes (and a fresh mtime) and is never clobbered.
+            let stat;
+            try {
+                stat = statSync(lock);
+            }
+            catch (error) {
+                if (isEnoent(error))
+                    return;
+                throw error;
+            }
+            if (Date.now() - stat.mtimeMs < STALE_PROJECT_MAINTENANCE_LOCK_AGE_MS)
+                return;
+            let currentOwner;
+            try {
+                currentOwner = readFileSync(join(lock, "owner"), "utf8");
+            }
+            catch (error) {
+                if (!isEnoent(error))
+                    throw error;
+                // Still ownerless: only reclaim when the initial read was also
+                // ownerless. A non-null expectedOwner means another process wrote
+                // the owner file between the two reads — leave its lock intact.
+                if (expectedOwner !== null)
+                    return;
+                currentOwner = null;
+            }
+            if (currentOwner !== expectedOwner)
+                return;
+            if (currentOwner !== null && lockOwnerIsAlive(lock))
+                return;
+            rmSync(lock, { recursive: true, force: true });
+        }
+        finally {
+            rmSync(reclaimLock, { recursive: true, force: true });
+        }
+        return;
+    }
+}
+/**
+ * Reclaim a reclaim-lock directory only when it is provably orphaned: its
+ * owner is dead (or unrecorded) AND it is older than the age bound, so a
+ * lock whose owner just mkdir'ed but has not written its owner is not stolen.
+ * Uses the same PID + start-time identity check as the main fence, so a
+ * recycled PID with a different start identity does not keep an orphaned
+ * reclaim lock alive. An ownerless (ENOENT owner) lock is still reclaimed;
+ * an unreadable owner fails closed (treated as live).
+ * Returns true when it removed the lock (caller retries), false otherwise.
+ */
+function reclaimOrphanedReclaimLock(reclaimLock) {
+    try {
+        if (Date.now() - statSync(reclaimLock).mtimeMs < STALE_PROJECT_MAINTENANCE_LOCK_AGE_MS)
+            return false;
+        if (lockOwnerIsAlive(reclaimLock))
+            return false;
+        rmSync(reclaimLock, { recursive: true, force: true });
+        return true;
+    }
+    catch (error) {
+        return isEnoent(error);
+    }
+}
+function lockOwnerIsAlive(lock) {
+    let owner;
+    try {
+        owner = readFileSync(join(lock, "owner"), "utf8").trim();
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return false;
+        // An unreadable owner file fails closed: treat the fence as held.
+        return true;
+    }
+    const separator = owner.indexOf(":");
+    const pid = Number.parseInt(separator < 0 ? owner : owner.slice(0, separator), 10);
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    const recordedIdentity = separator < 0 ? "" : owner.slice(separator + 1);
+    // PID + start-time identity: a recycled PID can never match the holder.
+    const currentIdentity = processStartIdentity(pid);
+    if (currentIdentity !== undefined) {
+        return recordedIdentity !== "" && currentIdentity === recordedIdentity;
+    }
+    // Off /proc (non-Linux): fall back to liveness.
+    return processIsAlive(pid);
+}
+/** Linux process start time (clock ticks since boot) for a PID; undefined off /proc. */
+function processStartIdentity(pid = process.pid) {
+    try {
+        const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
+        const close = stat.lastIndexOf(")");
+        const fields = stat.slice(close + 2).split(" ");
+        return fields[19] ?? "0"; // field 22: starttime
+    }
+    catch {
+        return undefined;
+    }
+}
+function processIsAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error instanceof Error && "code" in error && error.code === "EPERM";
+    }
+}
+function isEnoent(error) {
+    return error instanceof Error && "code" in error && error.code === "ENOENT";
+}
+function isEexist(error) {
+    return error instanceof Error && "code" in error && error.code === "EEXIST";
+}

package/dist/repository/taskWorkspaceCoordinator.js

@@ -2,6 +2,7 @@ import { isDeepStrictEqual } from "node:util";
 import { retireConfirmedAbsentInactiveTaskRolePlaceholders } from "../executor/agentExecutor.js";
 import { hasRuntimeLifecycleWork, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
 import { managedWorkspaceKey } from "../worktree/managedWorkspace.js";
+import { acquireProjectMaintenanceLocks } from "./projectMaintenanceLock.js";
 import { WorkspaceCleanupBlockedError } from "./taskWorkspacePreparer.js";
 export { WorkspaceCleanupBlockedError } from "./taskWorkspacePreparer.js";
 /**
@@ -71,15 +72,32 @@ export class TaskWorkspaceCoordinator {
         }
         if (item.workspaceDisposition === disposition)
             return "missing";
-        const state = await this.preparer.inspectWorkItemWorkspace(item.taskId, item.id);
-        if (state === "dirty")
-            return "dirty";
-        this.#assertWorkItemRuntimeQuiescent(item);
-        await this.#stopLiveRoles(item.taskId, this.#workItemRoleNames(item));
-        const laneCleanup = await this.preparer.cleanupExecutionLaneWorkspacesForWorkItem(item.taskId, item.id);
-        if (laneCleanup === "dirty")
-            return "dirty";
-        return this.preparer.cleanupWorkItemWorkspace(item.taskId, item.id, disposition);
+        // Hold the per-Project maintenance fence so a concurrent migrate/rebuild/
+        // archive cannot interleave with worktree removal.
+        const workspace = this.store.getWorkItemWorkspace(item.taskId, item.id);
+        const projectIds = workspace === null
+            ? []
+            : workspace.entries
+                .filter(({ access }) => access === "write")
+                .map(({ projectId }) => projectId);
+        const releaseMaintenance = projectIds.length === 0
+            ? () => { }
+            : acquireProjectMaintenanceLocks(this.preparer.home, projectIds);
+        try {
+            const state = await this.preparer.inspectWorkItemWorkspace(item.taskId, item.id);
+            if (state === "dirty")
+                return "dirty";
+            this.#assertWorkItemRuntimeQuiescent(item);
+            this.#assertNoActiveWorkItemDurableJobs(item);
+            await this.#stopLiveRoles(item.taskId, this.#workItemRoleNames(item));
+            const laneCleanup = await this.preparer.cleanupExecutionLaneWorkspacesForWorkItem(item.taskId, item.id);
+            if (laneCleanup === "dirty")
+                return "dirty";
+            return this.preparer.cleanupWorkItemWorkspace(item.taskId, item.id, disposition);
+        }
+        finally {
+            releaseMaintenance();
+        }
     }
     async cleanupWorkItemRuntime(taskId, workItemId) {
         const item = this.store.getWorkItem(taskId, workItemId);
@@ -96,16 +114,33 @@ export class TaskWorkspaceCoordinator {
         if (round.status !== "completed" && round.status !== "failed") {
             throw new Error(`ReviewRound must be terminal before cleanup: ${round.id}.`);
         }
-        const state = await this.preparer.inspectReviewRoundWorkspace(taskId, reviewRoundId);
-        if (state === "dirty")
-            return "dirty";
-        await this.#stopLiveRoles(taskId, this.#reviewRoundRoleNames(round));
-        const laneCleanup = await this.preparer.cleanupExecutionLaneWorkspacesForReviewRound(taskId, reviewRoundId);
-        if (laneCleanup === "dirty")
-            return "dirty";
-        return this.preparer.cleanupReviewRoundWorkspace(taskId, reviewRoundId);
+        // Hold the per-Project maintenance fence so a concurrent migrate/rebuild/
+        // archive cannot interleave with worktree removal.
+        const workspace = this.store.getReviewRoundWorkspace(taskId, reviewRoundId);
+        const projectIds = workspace === null
+            ? []
+            : workspace.entries
+                .filter(({ access }) => access === "write")
+                .map(({ projectId }) => projectId);
+        const releaseMaintenance = projectIds.length === 0
+            ? () => { }
+            : acquireProjectMaintenanceLocks(this.preparer.home, projectIds);
+        try {
+            const state = await this.preparer.inspectReviewRoundWorkspace(taskId, reviewRoundId);
+            if (state === "dirty")
+                return "dirty";
+            await this.#stopLiveRoles(taskId, this.#reviewRoundRoleNames(round));
+            const laneCleanup = await this.preparer.cleanupExecutionLaneWorkspacesForReviewRound(taskId, reviewRoundId);
+            if (laneCleanup === "dirty")
+                return "dirty";
+            return this.preparer.cleanupReviewRoundWorkspace(taskId, reviewRoundId);
+        }
+        finally {
+            releaseMaintenance();
+        }
     }
     async cleanupTaskForArchive(taskId, disposition) {
+        let releaseMaintenance;
         try {
             const task = this.store.getTask(taskId);
             if (task === null)
@@ -116,6 +151,15 @@ export class TaskWorkspaceCoordinator {
             const managedWorkspaces = [...this.store.listManagedWorkspaces(task.id)]
                 .sort((left, right) => managedWorkspaceKey(left.owner)
                 .localeCompare(managedWorkspaceKey(right.owner)));
+            // Archive cleanup removes worktrees from every Project the Task uses:
+            // hold each Project's maintenance fence so the Controller defers
+            // preparation and no migrate/rebuild/archive interleaves.
+            const projectIds = new Set(task.projectBindings.map(({ projectId }) => projectId));
+            for (const workspace of managedWorkspaces) {
+                for (const entry of workspace.entries)
+                    projectIds.add(entry.projectId);
+            }
+            releaseMaintenance = acquireProjectMaintenanceLocks(this.preparer.home, projectIds);
             const laneWorkspaces = managedWorkspaces.filter(({ owner }) => owner.type === "execution-lane");
             const workspaces = managedWorkspaces
                 .filter(({ owner }) => owner.type === "work-item");
@@ -207,6 +251,7 @@ export class TaskWorkspaceCoordinator {
             if (roleNames.length > 0) {
                 await this.runtime.stopTaskRoleSessions(taskId, roleNames);
             }
+            await this.runtime.assertTaskPhysicalResourcesReleased?.(task.id);
             this.#assertTaskArchiveSnapshot(snapshot);
             for (const workspace of laneWorkspaces) {
                 this.#assertTaskArchiveLifecycle(task);
@@ -245,6 +290,9 @@ export class TaskWorkspaceCoordinator {
                     })
             };
         }
+        finally {
+            releaseMaintenance?.();
+        }
     }
     #assertTaskArchiveSnapshot(snapshot) {
         this.#assertTaskArchiveLifecycle(snapshot.task);
@@ -274,6 +322,28 @@ export class TaskWorkspaceCoordinator {
             throw new WorkspaceCleanupBlockedError("active-run", `work-item:${item.taskId}/${item.id}`, true, `Work item still has an active Run: ${item.taskId}/${item.id}.`);
         }
     }
+    /**
+     * rr5/f4: A WorkItem workspace must not be removed while a DurableJob it
+     * owns could still be using it. Queued, running, and unacknowledged
+     * unknown-needs-attention jobs are unsettled — the runner (or its corpse)
+     * may still hold the worktree. Acknowledged unknown jobs are settled: a
+     * human/Leader has taken responsibility for the outcome.
+     */
+    #assertNoActiveWorkItemDurableJobs(item) {
+        const jobs = this.store.listDurableJobs?.(item.taskId);
+        if (jobs === undefined)
+            return;
+        const blocking = jobs.filter((job) => (job.owner.kind === "work-item"
+            && job.owner.workItemId === item.id
+            && (job.status === "queued"
+                || job.status === "running"
+                || (job.status === "unknown-needs-attention" && job.acknowledgedAt === undefined))));
+        if (blocking.length > 0) {
+            throw new WorkspaceCleanupBlockedError("active-durable-job", `work-item:${item.taskId}/${item.id}`, true, `Work item ${item.id} still has ${blocking.length} active DurableJob(s): `
+                + `${blocking.map((job) => `${job.id}/${job.status}`).join(", ")}. `
+                + "Cancel or acknowledge them before cleanup.");
+        }
+    }
     #workItemRoleNames(item) {
         return [
             ...(item.assignee === undefined ? [] : [item.assignee]),