harnery 0.43.0 → 0.44.0

package/src/core/workflow/workspaces/leases.ts

@@ -39,7 +39,8 @@ export type LeaseRecoveryStep =
   | "claim_created"
   | "stale_pinned"
   | "current_removed"
-  | "new_current_linked";
+  | "new_current_linked"
+  | "recovery_retired";
 
 export interface AcquireNoClobberLeaseInput {
   path: string;
@@ -51,6 +52,8 @@ export interface AcquireNoClobberLeaseInput {
   now?: () => number;
   pid?: number;
   host?: string;
+  /** Probe dependency for deterministic liveness-failure tests. */
+  probePid?: (pid: number) => void;
   /** Fault-injection hook immediately before the lease directory is created. */
   onLeaseDirectoryCreate?: () => void;
   /** Fault-injection hook after contention is observed and before owner inspection. */
@@ -137,8 +140,10 @@ export function acquireNoClobberLease(input: AcquireNoClobberLeaseInput): NoClob
     if (observed === undefined) {
       throw new Error(`lease ${input.scope} changed repeatedly during contention`);
     }
-    if (leaseIsLive(observed.owner, now(), input.staleAfterMs)) {
-      throw new Error(`lease ${input.scope} is held by a live or unexpired owner`);
+    if (leaseIsLive(observed.owner, now(), input.staleAfterMs, input.probePid)) {
+      throw new Error(
+        `lease ${input.scope} is held by a live or unexpired owner, or its liveness is unknown`,
+      );
     }
     if (
       observed.owner.scope !== input.scope ||
@@ -156,7 +161,7 @@ export function acquireNoClobberLease(input: AcquireNoClobberLeaseInput): NoClob
       linkSync(currentPath, pinPath);
       assertSameInodeAndBytes(currentPath, pinPath, observed.bytes);
     } catch {
-      abortUnchangedRecoveryClaim(leaseDir, recoveryPath, pinPath, owner);
+      abortUnchangedRecoveryClaim(leaseDir, recoveryPath, owner);
       recoveryClaimed = false;
       throw new Error(`lease ${input.scope} recovery raced with another contender`);
     }
@@ -174,10 +179,9 @@ export function acquireNoClobberLease(input: AcquireNoClobberLeaseInput): NoClob
     input.onRecoveryStep?.("new_current_linked");
 
     removePinnedOwner(leaseDir, pinPath, observed.owner);
-    unlinkSync(pinPath);
-    unlinkSync(join(recoveryPath, "claim.json"));
-    rmdirSync(recoveryPath);
-    fsyncDirectory(leaseDir);
+    retireRecoveryClaim(leaseDir, recoveryPath, owner, () =>
+      input.onRecoveryStep?.("recovery_retired"),
+    );
     return {
       ...leaseHandle(leaseDir, ownerPath, ownerBytes, owner),
       recovered_owner: observed.owner,
@@ -266,7 +270,7 @@ function reclaimAbandonedRecovery(
   now: number,
 ): void {
   const claim = readRecoveryClaim(recoveryPath);
-  if (leaseIsLive(claim.claimant, now, input.staleAfterMs)) {
+  if (leaseIsLive(claim.claimant, now, input.staleAfterMs, input.probePid)) {
     throw new Error(`lease ${input.scope} recovery is already in progress`);
   }
   if (
@@ -326,16 +330,34 @@ function reclaimAbandonedRecovery(
 function abortUnchangedRecoveryClaim(
   leaseDir: string,
   recoveryPath: string,
-  pinPath: string,
   claimant: NoClobberLeaseOwner,
 ): void {
   const claim = readRecoveryClaim(recoveryPath);
   if (claim.claimant.owner_id !== claimant.owner_id) {
     throw new Error(`lease ${claimant.scope} recovery ownership changed`);
   }
-  safeUnlink(pinPath);
-  unlinkSync(join(recoveryPath, "claim.json"));
-  rmdirSync(recoveryPath);
+  retireRecoveryClaim(leaseDir, recoveryPath, claimant);
+}
+
+/**
+ * Move this claimant's recovery directory to a private name before emptying
+ * it. Renaming a directory onto an existing empty directory replaces it, so
+ * emptying `recovery` in place would let a contender's claim land there and
+ * fail this claimant's final rmdir after it already holds the lease.
+ */
+function retireRecoveryClaim(
+  leaseDir: string,
+  recoveryPath: string,
+  claimant: NoClobberLeaseOwner,
+  onRetired?: () => void,
+): void {
+  const retired = join(leaseDir, `recovery-retired-${claimant.owner_id}`);
+  renameSync(recoveryPath, retired);
+  fsyncDirectory(leaseDir);
+  onRetired?.();
+  safeUnlink(join(retired, "stale-pin"));
+  unlinkSync(join(retired, "claim.json"));
+  rmdirSync(retired);
   fsyncDirectory(leaseDir);
 }
 
@@ -466,7 +488,14 @@ function isStringRecord(value: unknown): value is Record<string, string> {
   );
 }
 
-function leaseIsLive(owner: NoClobberLeaseOwner, now: number, staleAfterMs: number): boolean {
+function leaseIsLive(
+  owner: NoClobberLeaseOwner,
+  now: number,
+  staleAfterMs: number,
+  probePid: (pid: number) => void = (pid) => {
+    process.kill(pid, 0);
+  },
+): boolean {
   const created = Date.parse(owner.created_at);
   const expires = Date.parse(owner.expires_at);
   if (
@@ -478,12 +507,14 @@ function leaseIsLive(owner: NoClobberLeaseOwner, now: number, staleAfterMs: numb
   ) {
     return true;
   }
-  if (owner.host !== hostname()) return false;
+  // Expiry only makes a same-host owner eligible for a process check. Neither
+  // a remote owner nor an unreadable probe proves that its work has stopped.
+  if (owner.host !== hostname()) return true;
   try {
-    process.kill(owner.pid, 0);
+    probePid(owner.pid);
     return true;
   } catch (error) {
-    return (error as NodeJS.ErrnoException).code === "EPERM";
+    return (error as NodeJS.ErrnoException).code !== "ESRCH";
   }
 }
 

package/src/core/workflow/workspaces/local-git.ts

@@ -9,7 +9,9 @@ import {
 } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { stateDirMode, stateFileMode } from "../../storage/modes.ts";
+import { acquireWorkflowResumeLease } from "../run-state.ts";
 import { WorkspaceAttestationError } from "./attestation-error.ts";
+import { cleanupWorkspace } from "./cleanup.ts";
 import {
   git,
   gitMaybe,
@@ -22,6 +24,11 @@ import {
   worktreeInventory,
 } from "./git.ts";
 import { acquireNoClobberLease } from "./leases.ts";
+import {
+  claimOverlapsWorkspace,
+  reserveWorkspaceReclaim,
+  type WorkspaceReclaimReservation,
+} from "./ownership.ts";
 import {
   assertContainedExisting,
   assertPathIdentity,
@@ -36,6 +43,16 @@ import {
   type ValidatedRoot,
   validateConfiguredRoot,
 } from "./paths.ts";
+import {
+  prepareReclaim,
+  type ReclaimLeaseHolder,
+  type ReclaimMode,
+  type ReclaimPreparation,
+  WorkflowReclaimError,
+  type WorkspaceReclaimOptions,
+  type WorkspaceReclaimPreview,
+  type WorkspaceReclaimResult,
+} from "./reclaim.ts";
 import { readWorkflowLiveness } from "./recovery.ts";
 import {
   appendWorkspaceEvent,
@@ -43,6 +60,7 @@ import {
   readWorkflowSupplement,
   readWorkspaceBinding,
   readWorkspaceClaim,
+  readWorkspaceClaims,
   readWorkspaceEvents,
   stableDigest,
   type WorkspaceClaim,
@@ -86,11 +104,18 @@ export interface LocalGitWorktreeProviderOptions {
   descriptorPathsSupported?: () => boolean;
 }
 
+export interface LocalGitWorktreeProvider extends WorkspaceProvider {
+  reclaim(
+    binding: WorkspaceBinding,
+    options: WorkspaceReclaimOptions,
+  ): Promise<WorkspaceReclaimResult>;
+}
+
 export function createLocalGitWorktreeProvider(
   options: LocalGitWorktreeProviderOptions,
-): WorkspaceProvider {
+): LocalGitWorktreeProvider {
   const coordRoot = resolve(options.coordRoot);
-  return {
+  const provider: LocalGitWorktreeProvider = {
     probe: (input) => probe(input, options.descriptorPathsSupported),
     allocate: (request) => allocate(coordRoot, request),
     readBinding: async (binding) => {
@@ -103,7 +128,9 @@ export function createLocalGitWorktreeProvider(
     previewIntegration: (input) => previewIntegration(coordRoot, input),
     applyAuthorizedIntegration: (input) => applyAuthorizedIntegration(coordRoot, input),
     cleanup: (binding, intent) => cleanup(coordRoot, binding, intent),
+    reclaim: (binding, reclaimOptions) => reclaim(coordRoot, provider, binding, reclaimOptions),
   };
+  return provider;
 }
 
 export async function probe(
@@ -783,6 +810,258 @@ function cancelUnderLease(
   return { ...base, status: "cancelled" };
 }
 
+function reclaimPreview(
+  coordRoot: string,
+  binding: WorkspaceBinding,
+  mode: ReclaimMode,
+): WorkspaceReclaimPreview {
+  const preview: WorkspaceReclaimPreview = {
+    action: "preview",
+    run_id: binding.run_id,
+    mode,
+    liveness: readWorkflowLiveness(coordRoot, binding.run_id),
+    ownership: "unproven",
+    dirty_paths: null,
+    lease_holder: reclaimLeaseHolders(coordRoot, binding),
+  };
+  try {
+    const current = attestSync(coordRoot, binding);
+    if (current.status === "ok") {
+      preview.ownership = "ok";
+      preview.dirty_paths = [...(current.repository?.dirty_paths ?? [])];
+    } else {
+      preview.ownership_detail =
+        current.provider_drift.join("; ") || "ownership attestation failed";
+    }
+  } catch (error) {
+    preview.ownership_detail = (error as Error).message;
+  }
+  return preview;
+}
+
+function assertReclaimLiveness(coordRoot: string, binding: WorkspaceBinding): void {
+  const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
+  if (liveness === "inactive") return;
+  throw new WorkflowReclaimError(
+    liveness === "active" ? "workflow_reclaim_run_live" : "workflow_reclaim_liveness_unknown",
+    liveness === "active"
+      ? `run ${binding.run_id} is active; its workspace was left untouched`
+      : `run ${binding.run_id} has missing, corrupt, or ambiguous liveness evidence; its workspace was left untouched`,
+  );
+}
+
+function assertReclaimOwnership(coordRoot: string, binding: WorkspaceBinding): WorkspaceClaim {
+  try {
+    const claim = requiredClaim(coordRoot, binding);
+    const current = attestSync(coordRoot, binding);
+    if (current.status !== "ok") {
+      throw new Error(current.provider_drift.join("; ") || "ownership attestation failed");
+    }
+    return claim;
+  } catch (error) {
+    throw new WorkflowReclaimError("workflow_reclaim_ownership_unproven", (error as Error).message);
+  }
+}
+
+function assertPeerWorkspaceAuthority(coordRoot: string, claim: WorkspaceClaim): void {
+  try {
+    for (const peer of readWorkspaceClaims(coordRoot, PROVIDER_ID)) {
+      if (peer.binding_id === claim.binding_id) continue;
+      const binding = readWorkspaceBinding(coordRoot, PROVIDER_ID, peer.binding_id);
+      const events = readWorkspaceEvents(coordRoot, PROVIDER_ID, peer.binding_id);
+      if (binding) requiredClaim(coordRoot, binding);
+      else if (events.some((event) => event.event === "bound")) {
+        throw new Error("retained peer binding authority is missing");
+      }
+      if (!claimOverlapsWorkspace(peer, claim.workspace_root)) continue;
+      // Process termination, archival and an expired reservation do not release
+      // a resource. Require the provider's release record AND current absence.
+      if (binding && events.some((event) => event.event === "released")) {
+        const current = attestSync(coordRoot, binding);
+        if (current.status === "ok" && current.resource_state === "released") continue;
+      }
+      throw new Error(`retained peer workspace ${peer.binding_id} overlaps the reclaim path`);
+    }
+  } catch (error) {
+    throw new WorkflowReclaimError("workflow_reclaim_ownership_unproven", (error as Error).message);
+  }
+}
+
+function reclaimAuthorityGuard(
+  coordRoot: string,
+  binding: WorkspaceBinding,
+  claim: WorkspaceClaim,
+  reservation: WorkspaceReclaimReservation,
+): void {
+  try {
+    reservation.assertCurrent();
+    assertReclaimLiveness(coordRoot, binding);
+    assertReclaimOwnership(coordRoot, binding);
+    assertPeerWorkspaceAuthority(coordRoot, claim);
+  } catch (error) {
+    if (error instanceof WorkflowReclaimError) throw error;
+    throw new WorkflowReclaimError("workflow_reclaim_ownership_unproven", (error as Error).message);
+  }
+}
+
+async function reclaim(
+  coordRoot: string,
+  provider: WorkspaceProvider,
+  binding: WorkspaceBinding,
+  options: WorkspaceReclaimOptions,
+): Promise<WorkspaceReclaimResult> {
+  if (options.mode !== "salvage" && options.mode !== "discard") {
+    throw new Error("unsupported workspace reclaim mode");
+  }
+  if (!options.yes) return reclaimPreview(coordRoot, binding, options.mode);
+
+  // Liveness comes before terminal-proof cleanup helpers. A live run may not
+  // have a proof yet, and a missing proof must never be discovered after reset.
+  assertReclaimLiveness(coordRoot, binding);
+  const claim = assertReclaimOwnership(coordRoot, binding);
+  let reservation: WorkspaceReclaimReservation;
+  try {
+    reservation = reserveWorkspaceReclaim(coordRoot, claim);
+  } catch (error) {
+    throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
+  }
+  try {
+    reclaimAuthorityGuard(coordRoot, binding, claim, reservation);
+    let releaseResume: () => void;
+    try {
+      releaseResume = acquireWorkflowResumeLease(coordRoot, binding.run_id);
+    } catch (error) {
+      throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
+    }
+    try {
+      let releaseBinding: () => void;
+      try {
+        releaseBinding = acquireLease(
+          coordRoot,
+          workspaceLockPath(coordRoot, PROVIDER_ID, binding.binding_id),
+          claim,
+          "binding",
+        );
+      } catch (error) {
+        throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
+      }
+      try {
+        let releaseRepository: () => void;
+        try {
+          releaseRepository = acquireRepositoryLease(coordRoot, claim);
+        } catch (error) {
+          throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
+        }
+        try {
+          assertReclaimLiveness(coordRoot, binding);
+          assertReclaimOwnership(coordRoot, binding);
+          const preview = reclaimPreview(coordRoot, binding, options.mode);
+          let preparation: ReclaimPreparation | undefined;
+          // Keep ordinary cleanup's terminal-evidence checks, intent and receipts.
+          // Its provider callback uses the leases already held here instead of
+          // trying to acquire them again. Preparation occurs only after all of
+          // cleanup's path, ownership, ref and registration checks have succeeded.
+          const cleanupResult = await cleanupWorkspace({
+            coordRoot,
+            runId: binding.run_id,
+            provider: {
+              ...provider,
+              cleanup: async (candidate, intent) => {
+                assertReclaimLiveness(coordRoot, candidate);
+                assertReclaimOwnership(coordRoot, candidate);
+                return cleanupUnderLease(coordRoot, candidate, claim, intent, {
+                  mode: options.mode,
+                  beforeEffect: () =>
+                    reclaimAuthorityGuard(coordRoot, candidate, claim, reservation),
+                  prepared: (value) => {
+                    preparation = value;
+                  },
+                });
+              },
+            },
+          });
+          if (
+            !preparation &&
+            cleanupResult.status !== "released" &&
+            cleanupResult.status !== "already_released"
+          ) {
+            throw new WorkflowReclaimError(
+              "workflow_reclaim_ownership_unproven",
+              ("reason" in cleanupResult ? cleanupResult.reason : undefined) ??
+                "workspace cleanup preconditions could not be proved",
+            );
+          }
+          return {
+            ...preview,
+            action: "reclaimed",
+            lease_holder: [],
+            preparation: preparation ?? {
+              action: "already_gone",
+              detail: "the workspace is already released",
+            },
+            cleanup: cleanupResult,
+          };
+        } finally {
+          releaseRepository();
+        }
+      } finally {
+        releaseBinding();
+      }
+    } finally {
+      releaseResume();
+    }
+  } finally {
+    reservation.release();
+  }
+}
+
+function reclaimLeaseHolders(coordRoot: string, binding: WorkspaceBinding): ReclaimLeaseHolder[] {
+  const paths: Array<[ReclaimLeaseHolder["scope"], string]> = [
+    ["binding", workspaceLockPath(coordRoot, PROVIDER_ID, binding.binding_id)],
+  ];
+  if (binding.repository) {
+    const key = stableDigest({ common: binding.repository.common_dir.identity });
+    paths.push([
+      "repository",
+      join(coordRoot, ".harnery", "workspaces", PROVIDER_ID, ".leases", `repository-${key}.lock`),
+    ]);
+  }
+  const holders: ReclaimLeaseHolder[] = [];
+  try {
+    const owner = JSON.parse(
+      readFileSync(join(coordRoot, ".harnery", "workflows", binding.run_id, "resume.lock"), "utf8"),
+    ) as Record<string, unknown>;
+    holders.push({
+      scope: "resume",
+      pid: typeof owner.pid === "number" ? owner.pid : undefined,
+      host: typeof owner.host === "string" ? owner.host : undefined,
+    });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+      holders.push({ scope: "resume", detail: "lease holder could not be read" });
+    }
+  }
+  for (const [scope, path] of paths) {
+    try {
+      const owner = JSON.parse(readFileSync(join(path, "current"), "utf8")) as Record<
+        string,
+        unknown
+      >;
+      holders.push({
+        scope,
+        owner_id: typeof owner.owner_id === "string" ? owner.owner_id : undefined,
+        pid: typeof owner.pid === "number" ? owner.pid : undefined,
+        host: typeof owner.host === "string" ? owner.host : undefined,
+      });
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+        holders.push({ scope, detail: "lease holder could not be read" });
+      }
+    }
+  }
+  return holders;
+}
+
 async function cleanup(
   coordRoot: string,
   binding: WorkspaceBinding,
@@ -799,7 +1078,17 @@ async function cleanup(
   try {
     const releaseRepository = acquireRepositoryLease(coordRoot, claim);
     try {
-      return cleanupUnderLease(coordRoot, binding, claim, intent);
+      const reservation = reserveWorkspaceReclaim(coordRoot, claim);
+      try {
+        const authorityGuard = () => {
+          reservation.assertCurrent();
+          assertPeerWorkspaceAuthority(coordRoot, claim);
+        };
+        authorityGuard();
+        return cleanupUnderLease(coordRoot, binding, claim, intent, undefined, authorityGuard);
+      } finally {
+        reservation.release();
+      }
     } finally {
       releaseRepository();
     }
@@ -813,6 +1102,12 @@ function cleanupUnderLease(
   binding: WorkspaceBinding,
   claim: WorkspaceClaim,
   intent: WorkspaceCleanupIntent,
+  reclaimRequest?: {
+    mode: ReclaimMode;
+    beforeEffect: () => void;
+    prepared: (value: ReclaimPreparation) => void;
+  },
+  beforeRemoval?: () => void,
 ): WorkspaceCleanupResult {
   const repository = binding.repository;
   if (
@@ -871,11 +1166,11 @@ function cleanupUnderLease(
     return { ...base, status: "blocked", reason: "ownership attestation failed" };
   }
   if (
-    (before.repository?.dirty_paths.length ?? 0) > 0 ||
+    (!reclaimRequest && (before.repository?.dirty_paths.length ?? 0) > 0) ||
     (before.repository?.conflicts.length ?? 0) > 0 ||
     (before.repository?.operation_in_progress.length ?? 0) > 0
   ) {
-    appendWorkspaceEvent(coordRoot, claim, "preserved_dirty");
+    if (!reclaimRequest) appendWorkspaceEvent(coordRoot, claim, "preserved_dirty");
     return {
       ...base,
       status: "preserved_dirty",
@@ -949,6 +1244,19 @@ function cleanupUnderLease(
     };
   }
 
+  if (reclaimRequest) {
+    assertReclaimLiveness(coordRoot, binding);
+    assertReclaimOwnership(coordRoot, binding);
+    revalidateClaimResources(claim, root);
+    reclaimRequest.prepared(
+      prepareReclaim({
+        worktreePath: binding.workspace_root,
+        mode: reclaimRequest.mode,
+        runId: binding.run_id,
+        beforeEffect: reclaimRequest.beforeEffect,
+      }),
+    );
+  }
   appendWorkspaceEvent(coordRoot, claim, "cleanup_pending", {
     worktree_path: binding.workspace_root,
     gitdir: repository.gitdir,
@@ -958,6 +1266,8 @@ function cleanupUnderLease(
     target_ref_oid: targetRef.out,
   });
   if (before.workspace_exists) {
+    reclaimRequest?.beforeEffect();
+    beforeRemoval?.();
     revalidateClaimResources(claim, root);
     assertContainedExisting(root, binding.workspace_root, "workspace cleanup path");
     git(repository.source_root.realpath, ["worktree", "remove", binding.workspace_root]);