harnery 0.43.0 → 0.45.0

package/src/core/workflow/workspaces/local-git.ts

@@ -9,7 +9,10 @@ import {
 } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { stateDirMode, stateFileMode } from "../../storage/modes.ts";
+import { beginWorkflowRecovery, validateWorkflowRecoveryAuthorization } from "../recovery-audit.ts";
+import { acquireWorkflowResumeLease } from "../run-state.ts";
 import { WorkspaceAttestationError } from "./attestation-error.ts";
+import { cleanupWorkspace } from "./cleanup.ts";
 import {
   git,
   gitMaybe,
@@ -22,6 +25,11 @@ import {
   worktreeInventory,
 } from "./git.ts";
 import { acquireNoClobberLease } from "./leases.ts";
+import {
+  claimOverlapsWorkspace,
+  reserveWorkspaceReclaim,
+  type WorkspaceReclaimReservation,
+} from "./ownership.ts";
 import {
   assertContainedExisting,
   assertPathIdentity,
@@ -36,13 +44,24 @@ import {
   type ValidatedRoot,
   validateConfiguredRoot,
 } from "./paths.ts";
-import { readWorkflowLiveness } from "./recovery.ts";
+import {
+  prepareReclaim,
+  type ReclaimLeaseHolder,
+  type ReclaimMode,
+  type ReclaimPreparation,
+  WorkflowReclaimError,
+  type WorkspaceReclaimOptions,
+  type WorkspaceReclaimPreview,
+  type WorkspaceReclaimResult,
+} from "./reclaim.ts";
+import { hasActiveWorkflowEvidence, readWorkflowLiveness } from "./recovery.ts";
 import {
   appendWorkspaceEvent,
   fileSha256,
   readWorkflowSupplement,
   readWorkspaceBinding,
   readWorkspaceClaim,
+  readWorkspaceClaims,
   readWorkspaceEvents,
   stableDigest,
   type WorkspaceClaim,
@@ -86,11 +105,18 @@ export interface LocalGitWorktreeProviderOptions {
   descriptorPathsSupported?: () => boolean;
 }
 
+export interface LocalGitWorktreeProvider extends WorkspaceProvider {
+  reclaim(
+    binding: WorkspaceBinding,
+    options: WorkspaceReclaimOptions,
+  ): Promise<WorkspaceReclaimResult>;
+}
+
 export function createLocalGitWorktreeProvider(
   options: LocalGitWorktreeProviderOptions,
-): WorkspaceProvider {
+): LocalGitWorktreeProvider {
   const coordRoot = resolve(options.coordRoot);
-  return {
+  const provider: LocalGitWorktreeProvider = {
     probe: (input) => probe(input, options.descriptorPathsSupported),
     allocate: (request) => allocate(coordRoot, request),
     readBinding: async (binding) => {
@@ -103,7 +129,9 @@ export function createLocalGitWorktreeProvider(
     previewIntegration: (input) => previewIntegration(coordRoot, input),
     applyAuthorizedIntegration: (input) => applyAuthorizedIntegration(coordRoot, input),
     cleanup: (binding, intent) => cleanup(coordRoot, binding, intent),
+    reclaim: (binding, reclaimOptions) => reclaim(coordRoot, provider, binding, reclaimOptions),
   };
+  return provider;
 }
 
 export async function probe(
@@ -783,6 +811,325 @@ function cancelUnderLease(
   return { ...base, status: "cancelled" };
 }
 
+function reclaimPreview(
+  coordRoot: string,
+  binding: WorkspaceBinding,
+  mode: ReclaimMode,
+): WorkspaceReclaimPreview {
+  const preview: WorkspaceReclaimPreview = {
+    action: "preview",
+    run_id: binding.run_id,
+    mode,
+    liveness: readWorkflowLiveness(coordRoot, binding.run_id),
+    evidence_sha256: reclaimEvidenceDigest(coordRoot, binding),
+    ownership: "unproven",
+    dirty_paths: null,
+    lease_holder: reclaimLeaseHolders(coordRoot, binding),
+  };
+  try {
+    const current = attestSync(coordRoot, binding);
+    if (current.status === "ok") {
+      preview.ownership = "ok";
+      preview.dirty_paths = [...(current.repository?.dirty_paths ?? [])];
+    } else {
+      preview.ownership_detail =
+        current.provider_drift.join("; ") || "ownership attestation failed";
+    }
+  } catch (error) {
+    preview.ownership_detail = (error as Error).message;
+  }
+  return preview;
+}
+
+function reclaimEvidenceDigest(coordRoot: string, binding: WorkspaceBinding): string {
+  const evidence = (name: string): string | null => {
+    try {
+      return readFileSync(join(coordRoot, ".harnery", "workflows", binding.run_id, name), "utf8");
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+      throw error;
+    }
+  };
+  return stableDigest({
+    binding,
+    proof: evidence("proof.json"),
+    transcript: evidence("transcript.jsonl"),
+  });
+}
+
+function assertReclaimLiveness(
+  coordRoot: string,
+  binding: WorkspaceBinding,
+  recovery?: WorkspaceReclaimOptions["unknownLiveness"],
+): void {
+  const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
+  if (liveness === "active" || (recovery && hasActiveWorkflowEvidence(coordRoot, binding.run_id))) {
+    throw new WorkflowReclaimError(
+      "workflow_reclaim_run_live",
+      `run ${binding.run_id} has active evidence; its workspace was left untouched`,
+    );
+  }
+  if (recovery && reclaimEvidenceDigest(coordRoot, binding) !== recovery.evidenceSha256) {
+    throw new WorkflowReclaimError(
+      "workflow_reclaim_recovery_mismatch",
+      "workflow reclaim requires the exact current preview evidence digest",
+    );
+  }
+  if (liveness === "inactive") return;
+  if (liveness === "unknown" && recovery) return;
+  throw new WorkflowReclaimError(
+    "workflow_reclaim_liveness_unknown",
+    `run ${binding.run_id} has missing, corrupt, or ambiguous liveness evidence; its workspace was left untouched`,
+  );
+}
+
+function assertReclaimOwnership(coordRoot: string, binding: WorkspaceBinding): WorkspaceClaim {
+  try {
+    const claim = requiredClaim(coordRoot, binding);
+    const current = attestSync(coordRoot, binding);
+    if (current.status !== "ok") {
+      throw new Error(current.provider_drift.join("; ") || "ownership attestation failed");
+    }
+    return claim;
+  } catch (error) {
+    throw new WorkflowReclaimError("workflow_reclaim_ownership_unproven", (error as Error).message);
+  }
+}
+
+function assertPeerWorkspaceAuthority(coordRoot: string, claim: WorkspaceClaim): void {
+  try {
+    for (const peer of readWorkspaceClaims(coordRoot, PROVIDER_ID)) {
+      if (peer.binding_id === claim.binding_id) continue;
+      const binding = readWorkspaceBinding(coordRoot, PROVIDER_ID, peer.binding_id);
+      const events = readWorkspaceEvents(coordRoot, PROVIDER_ID, peer.binding_id);
+      if (binding) requiredClaim(coordRoot, binding);
+      else if (events.some((event) => event.event === "bound")) {
+        throw new Error("retained peer binding authority is missing");
+      }
+      if (!claimOverlapsWorkspace(peer, claim.workspace_root)) continue;
+      // Process termination, archival and an expired reservation do not release
+      // a resource. Require the provider's release record AND current absence.
+      if (binding && events.some((event) => event.event === "released")) {
+        const current = attestSync(coordRoot, binding);
+        if (current.status === "ok" && current.resource_state === "released") continue;
+      }
+      throw new Error(`retained peer workspace ${peer.binding_id} overlaps the reclaim path`);
+    }
+  } catch (error) {
+    throw new WorkflowReclaimError("workflow_reclaim_ownership_unproven", (error as Error).message);
+  }
+}
+
+function reclaimAuthorityGuard(
+  coordRoot: string,
+  binding: WorkspaceBinding,
+  claim: WorkspaceClaim,
+  reservation: WorkspaceReclaimReservation,
+  recovery?: WorkspaceReclaimOptions["unknownLiveness"],
+): void {
+  try {
+    reservation.assertCurrent();
+    assertReclaimLiveness(coordRoot, binding, recovery);
+    assertReclaimOwnership(coordRoot, binding);
+    assertPeerWorkspaceAuthority(coordRoot, claim);
+  } catch (error) {
+    if (error instanceof WorkflowReclaimError) throw error;
+    throw new WorkflowReclaimError("workflow_reclaim_ownership_unproven", (error as Error).message);
+  }
+}
+
+async function reclaim(
+  coordRoot: string,
+  provider: WorkspaceProvider,
+  binding: WorkspaceBinding,
+  options: WorkspaceReclaimOptions,
+): Promise<WorkspaceReclaimResult> {
+  if (options.mode !== "salvage" && options.mode !== "discard") {
+    throw new Error("unsupported workspace reclaim mode");
+  }
+  if (!options.yes) return reclaimPreview(coordRoot, binding, options.mode);
+  if (options.unknownLiveness) validateWorkflowRecoveryAuthorization(options.unknownLiveness);
+
+  // Liveness comes before terminal-proof cleanup helpers. A live run may not
+  // have a proof yet, and a missing proof must never be discovered after reset.
+  assertReclaimLiveness(coordRoot, binding, options.unknownLiveness);
+  const claim = assertReclaimOwnership(coordRoot, binding);
+  let reservation: WorkspaceReclaimReservation;
+  try {
+    reservation = reserveWorkspaceReclaim(coordRoot, claim);
+  } catch (error) {
+    throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
+  }
+  try {
+    reclaimAuthorityGuard(coordRoot, binding, claim, reservation, options.unknownLiveness);
+    let releaseResume: () => void;
+    try {
+      releaseResume = acquireWorkflowResumeLease(coordRoot, binding.run_id);
+    } catch (error) {
+      throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
+    }
+    try {
+      let releaseBinding: () => void;
+      try {
+        releaseBinding = acquireLease(
+          coordRoot,
+          workspaceLockPath(coordRoot, PROVIDER_ID, binding.binding_id),
+          claim,
+          "binding",
+        );
+      } catch (error) {
+        throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
+      }
+      try {
+        let releaseRepository: () => void;
+        try {
+          releaseRepository = acquireRepositoryLease(coordRoot, claim);
+        } catch (error) {
+          throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
+        }
+        try {
+          assertReclaimLiveness(coordRoot, binding, options.unknownLiveness);
+          assertReclaimOwnership(coordRoot, binding);
+          const preview = reclaimPreview(coordRoot, binding, options.mode);
+          const audit = options.unknownLiveness
+            ? beginWorkflowRecovery(coordRoot, {
+                run_id: binding.run_id,
+                action: "reclaim_unknown_liveness",
+                actor: options.unknownLiveness.actor,
+                reason: options.unknownLiveness.reason,
+                target_sha256: options.unknownLiveness.evidenceSha256,
+                target: {
+                  binding_id: binding.binding_id,
+                  workspace_root: binding.workspace_root,
+                  mode: options.mode,
+                  liveness: preview.liveness,
+                },
+              })
+            : undefined;
+          let preparation: ReclaimPreparation | undefined;
+          // Keep ordinary cleanup's terminal-evidence checks, intent and receipts.
+          // Its provider callback uses the leases already held here instead of
+          // trying to acquire them again. Preparation occurs only after all of
+          // cleanup's path, ownership, ref and registration checks have succeeded.
+          const cleanupResult = await cleanupWorkspace({
+            coordRoot,
+            runId: binding.run_id,
+            provider: {
+              ...provider,
+              cleanup: async (candidate, intent) => {
+                assertReclaimLiveness(coordRoot, candidate, options.unknownLiveness);
+                assertReclaimOwnership(coordRoot, candidate);
+                return cleanupUnderLease(coordRoot, candidate, claim, intent, {
+                  mode: options.mode,
+                  recovery: options.unknownLiveness,
+                  beforeEffect: () => {
+                    audit?.assertCurrent();
+                    reclaimAuthorityGuard(
+                      coordRoot,
+                      candidate,
+                      claim,
+                      reservation,
+                      options.unknownLiveness,
+                    );
+                  },
+                  beforeRefEffect: () => {
+                    audit?.assertCurrent();
+                    reservation.assertCurrent();
+                    assertReclaimLiveness(coordRoot, candidate, options.unknownLiveness);
+                    requiredClaim(coordRoot, candidate);
+                    assertPeerWorkspaceAuthority(coordRoot, claim);
+                  },
+                  prepared: (value) => {
+                    preparation = value;
+                  },
+                });
+              },
+            },
+          });
+          if (
+            !preparation &&
+            cleanupResult.status !== "released" &&
+            cleanupResult.status !== "already_released"
+          ) {
+            throw new WorkflowReclaimError(
+              "workflow_reclaim_ownership_unproven",
+              ("reason" in cleanupResult ? cleanupResult.reason : undefined) ??
+                "workspace cleanup preconditions could not be proved",
+            );
+          }
+          if (cleanupResult.status === "released" || cleanupResult.status === "already_released")
+            audit?.complete();
+          return {
+            ...preview,
+            action: "reclaimed",
+            lease_holder: [],
+            preparation: preparation ?? {
+              action: "already_gone",
+              detail: "the workspace is already released",
+            },
+            cleanup: cleanupResult,
+          };
+        } finally {
+          releaseRepository();
+        }
+      } finally {
+        releaseBinding();
+      }
+    } finally {
+      releaseResume();
+    }
+  } finally {
+    reservation.release();
+  }
+}
+
+function reclaimLeaseHolders(coordRoot: string, binding: WorkspaceBinding): ReclaimLeaseHolder[] {
+  const paths: Array<[ReclaimLeaseHolder["scope"], string]> = [
+    ["binding", workspaceLockPath(coordRoot, PROVIDER_ID, binding.binding_id)],
+  ];
+  if (binding.repository) {
+    const key = stableDigest({ common: binding.repository.common_dir.identity });
+    paths.push([
+      "repository",
+      join(coordRoot, ".harnery", "workspaces", PROVIDER_ID, ".leases", `repository-${key}.lock`),
+    ]);
+  }
+  const holders: ReclaimLeaseHolder[] = [];
+  try {
+    const owner = JSON.parse(
+      readFileSync(join(coordRoot, ".harnery", "workflows", binding.run_id, "resume.lock"), "utf8"),
+    ) as Record<string, unknown>;
+    holders.push({
+      scope: "resume",
+      pid: typeof owner.pid === "number" ? owner.pid : undefined,
+      host: typeof owner.host === "string" ? owner.host : undefined,
+    });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+      holders.push({ scope: "resume", detail: "lease holder could not be read" });
+    }
+  }
+  for (const [scope, path] of paths) {
+    try {
+      const owner = JSON.parse(readFileSync(join(path, "current"), "utf8")) as Record<
+        string,
+        unknown
+      >;
+      holders.push({
+        scope,
+        owner_id: typeof owner.owner_id === "string" ? owner.owner_id : undefined,
+        pid: typeof owner.pid === "number" ? owner.pid : undefined,
+        host: typeof owner.host === "string" ? owner.host : undefined,
+      });
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+        holders.push({ scope, detail: "lease holder could not be read" });
+      }
+    }
+  }
+  return holders;
+}
+
 async function cleanup(
   coordRoot: string,
   binding: WorkspaceBinding,
@@ -799,7 +1146,17 @@ async function cleanup(
   try {
     const releaseRepository = acquireRepositoryLease(coordRoot, claim);
     try {
-      return cleanupUnderLease(coordRoot, binding, claim, intent);
+      const reservation = reserveWorkspaceReclaim(coordRoot, claim);
+      try {
+        const authorityGuard = () => {
+          reservation.assertCurrent();
+          assertPeerWorkspaceAuthority(coordRoot, claim);
+        };
+        authorityGuard();
+        return cleanupUnderLease(coordRoot, binding, claim, intent, undefined, authorityGuard);
+      } finally {
+        reservation.release();
+      }
     } finally {
       releaseRepository();
     }
@@ -813,6 +1170,14 @@ function cleanupUnderLease(
   binding: WorkspaceBinding,
   claim: WorkspaceClaim,
   intent: WorkspaceCleanupIntent,
+  reclaimRequest?: {
+    mode: ReclaimMode;
+    recovery?: WorkspaceReclaimOptions["unknownLiveness"];
+    beforeEffect: () => void;
+    beforeRefEffect: () => void;
+    prepared: (value: ReclaimPreparation) => void;
+  },
+  beforeRemoval?: () => void,
 ): WorkspaceCleanupResult {
   const repository = binding.repository;
   if (
@@ -857,7 +1222,7 @@ function cleanupUnderLease(
     };
   }
   const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
-  if (liveness !== "inactive") {
+  if (liveness !== "inactive" && !(liveness === "unknown" && reclaimRequest?.recovery)) {
     return {
       ...base,
       status: "blocked",
@@ -871,11 +1236,11 @@ function cleanupUnderLease(
     return { ...base, status: "blocked", reason: "ownership attestation failed" };
   }
   if (
-    (before.repository?.dirty_paths.length ?? 0) > 0 ||
+    (!reclaimRequest && (before.repository?.dirty_paths.length ?? 0) > 0) ||
     (before.repository?.conflicts.length ?? 0) > 0 ||
     (before.repository?.operation_in_progress.length ?? 0) > 0
   ) {
-    appendWorkspaceEvent(coordRoot, claim, "preserved_dirty");
+    if (!reclaimRequest) appendWorkspaceEvent(coordRoot, claim, "preserved_dirty");
     return {
       ...base,
       status: "preserved_dirty",
@@ -949,6 +1314,19 @@ function cleanupUnderLease(
     };
   }
 
+  if (reclaimRequest) {
+    assertReclaimLiveness(coordRoot, binding, reclaimRequest.recovery);
+    assertReclaimOwnership(coordRoot, binding);
+    revalidateClaimResources(claim, root);
+    reclaimRequest.prepared(
+      prepareReclaim({
+        worktreePath: binding.workspace_root,
+        mode: reclaimRequest.mode,
+        runId: binding.run_id,
+        beforeEffect: reclaimRequest.beforeEffect,
+      }),
+    );
+  }
   appendWorkspaceEvent(coordRoot, claim, "cleanup_pending", {
     worktree_path: binding.workspace_root,
     gitdir: repository.gitdir,
@@ -958,6 +1336,8 @@ function cleanupUnderLease(
     target_ref_oid: targetRef.out,
   });
   if (before.workspace_exists) {
+    reclaimRequest?.beforeEffect();
+    beforeRemoval?.();
     revalidateClaimResources(claim, root);
     assertContainedExisting(root, binding.workspace_root, "workspace cleanup path");
     git(repository.source_root.realpath, ["worktree", "remove", binding.workspace_root]);
@@ -983,6 +1363,7 @@ function cleanupUnderLease(
       return { ...base, status: "blocked", reason: "workspace ref changed during cleanup" };
     }
     revalidateClaimResources(claim, root);
+    reclaimRequest?.beforeRefEffect();
     const deleted = gitMaybe(repository.source_root.realpath, [
       "update-ref",
       "-d",