@ricsam/r5d-worker 0.0.123 → 0.0.124

package/dist/mjs/workspace-git-sync.mjs

@@ -3,9 +3,9 @@ import path from "node:path";
 import { gitCredentialUsernameConfigKey, gitTransportSecurityArgs, workerGitProcessEnvironment } from "./git-process-environment.mjs";
 import {
   materializeWorkspaceProjectionTree,
-  mergeWorkspaceProjectionMount,
   workspaceMergeProjectionSupport
 } from "./workspace-merge-projection.mjs";
+import { workspaceFilesystemExecutor } from "./workspace-filesystem-executor.mjs";
 import {
   captureWorkingTreeMirrorPlan,
   fsyncWorkingTreePaths,
@@ -23,6 +23,22 @@ const WORKSPACE_GIT_INTEGRATED_REF = "refs/r5d/workspace-local/integrated";
 const WORKSPACE_GIT_HYDRATED_RECEIPT = "r5d/workspace-hydrated-head";
 const WORKSPACE_GIT_HYDRATION_TRANSACTION = "r5d/workspace-hydration-transaction";
 const WORKSPACE_GIT_CHECKOUT_DURABILITY = "r5d/workspace-checkout-durability";
+function workspaceGitMountData(mount) {
+  return {
+    id: mount.id,
+    hydrationIncarnationKey: mount.hydrationIncarnationKey,
+    sourcePath: mount.sourcePath,
+    durabilityRootPath: mount.durabilityRootPath,
+    workspaceRelativePath: mount.workspaceRelativePath,
+    sourceMode: mount.sourceMode,
+    hydrateDeletionMode: mount.hydrateDeletionMode,
+    ...mount.preserveLocalOnInitialOuterAbsence === void 0 ? {} : { preserveLocalOnInitialOuterAbsence: mount.preserveLocalOnInitialOuterAbsence },
+    ...mount.preserveLocalOnHydrationBasisChange === void 0 ? {} : { preserveLocalOnHydrationBasisChange: mount.preserveLocalOnHydrationBasisChange },
+    ...mount.initialPublicationSourceRelativePath === void 0 ? {} : { initialPublicationSourceRelativePath: mount.initialPublicationSourceRelativePath },
+    ...mount.deleteWhenSourceMissing === void 0 ? {} : { deleteWhenSourceMissing: mount.deleteWhenSourceMissing },
+    ...mount.lastProjectedMutationToken === void 0 ? {} : { lastProjectedMutationToken: mount.lastProjectedMutationToken }
+  };
+}
 function workspaceGitMountsForRemediation(mounts) {
   return mounts.map((mount) => ({ ...mount, busy: mount.busyForRecovery ?? mount.busy }));
 }
@@ -114,9 +130,6 @@ async function revParseAsync(workspacePath, revision) {
   const result = await gitResultAsync(workspacePath, ["rev-parse", "--verify", revision]);
   return result.exitCode === 0 ? result.stdout : null;
 }
-function yieldToWorkerEventLoop() {
-  return new Promise((resolve) => setImmediate(resolve));
-}
 function workspaceCommitsExist(workspacePath, heads) {
   const uniqueHeads = [...new Set(heads)];
   if (uniqueHeads.length === 0) return true;
@@ -381,20 +394,35 @@ function readHydratedWorkspaceReceipt(workspacePath) {
   }
   return unverified.receipt;
 }
+async function readHydratedWorkspaceReceiptUnverifiedAsync(workspacePath) {
+  const receiptPath = hydratedWorkspaceReceiptPath(workspacePath);
+  let status;
+  try {
+    status = await fs.promises.lstat(receiptPath);
+  } catch (error) {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  }
+  if (!status.isFile() || status.isSymbolicLink()) {
+    throw new Error(`Workspace hydration receipt is not a regular file: ${receiptPath}`);
+  }
+  const content = await fs.promises.readFile(receiptPath, "utf8");
+  return { ...parseWorkspaceHydrationReceiptContentUnverified(receiptPath, content), receiptPath };
+}
 async function readHydratedWorkspaceReceiptAsync(workspacePath) {
-  const unverified = readHydratedWorkspaceReceiptUnverified(workspacePath);
+  const unverified = await readHydratedWorkspaceReceiptUnverifiedAsync(workspacePath);
   if (!unverified) return null;
   if (!await workspaceCommitsExistAsync(workspacePath, unverified.headsToVerify)) {
     throw new Error(`Workspace hydration receipt does not resolve to a local commit: ${unverified.receiptPath}`);
   }
   return unverified.receipt;
 }
-function hydratedWorkspaceReceiptFileIsV3(workspacePath) {
+async function hydratedWorkspaceReceiptFileIsV3(workspacePath) {
   const receiptPath = hydratedWorkspaceReceiptPath(workspacePath);
-  const status = lstatIfExists(receiptPath);
-  if (!status?.isFile() || status.isSymbolicLink()) return false;
   try {
-    return JSON.parse(fs.readFileSync(receiptPath, "utf8")).version === 3;
+    const status = await fs.promises.lstat(receiptPath);
+    if (!status.isFile() || status.isSymbolicLink()) return false;
+    return JSON.parse(await fs.promises.readFile(receiptPath, "utf8")).version === 3;
   } catch {
     return false;
   }
@@ -491,24 +519,6 @@ function runWorkspaceRefUpdateBatch(workspacePath, instructions, action) {
     throw new Error(`${action}: ${updated.stderr.toString().trim() || `git exited ${updated.exitCode}`}`);
   }
 }
-async function runWorkspaceRefUpdateBatchAsync(workspacePath, instructions, action) {
-  if (instructions.length === 0) return;
-  const subprocess = Bun.spawn(gitCommandArgs(["update-ref", "--stdin"]), {
-    cwd: workspacePath,
-    stdin: Buffer.from([...instructions, ""].join("\n")),
-    stdout: "pipe",
-    stderr: "pipe",
-    env: workerGitProcessEnvironment()
-  });
-  const [, stderr, exitCode] = await Promise.all([
-    new Response(subprocess.stdout).text(),
-    new Response(subprocess.stderr).text(),
-    subprocess.exited
-  ]);
-  if (exitCode !== 0) {
-    throw new Error(`${action}: ${stderr.trim() || `git exited ${exitCode}`}`);
-  }
-}
 function workspaceBasisRefTransaction(receipt, stableRefs) {
   const desired = desiredWorkspaceBasisRefs(receipt);
   const transaction = ["start"];
@@ -531,24 +541,6 @@ function updateWorkspaceBasisRefs(workspacePath, receipt) {
     "remove staged workspace hydration basis refs"
   );
 }
-async function updateWorkspaceBasisRefsAsync(workspacePath, receipt) {
-  const stableRefs = (await gitAsync(workspacePath, ["for-each-ref", "--format=%(refname)", WORKSPACE_GIT_BASIS_REF_PREFIX], "list workspace basis refs")).split("\n").filter(Boolean).sort();
-  await runWorkspaceRefUpdateBatchAsync(
-    workspacePath,
-    workspaceBasisRefTransaction(receipt, stableRefs),
-    "Synchronize workspace hydration basis refs"
-  );
-  const stagedRefs = (await gitAsync(
-    workspacePath,
-    ["for-each-ref", "--format=%(refname)", WORKSPACE_GIT_BASIS_STAGING_REF_PREFIX],
-    "list staged workspace basis refs"
-  )).split("\n").filter(Boolean).sort();
-  await runWorkspaceRefUpdateBatchAsync(
-    workspacePath,
-    stagedRefs.map((ref) => `delete ${ref}`),
-    "remove staged workspace hydration basis refs"
-  );
-}
 function assertCanonicalHydrationReceiptHeads(receipt) {
   for (const mount of receipt.mounts) {
     if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(mount.head)) {
@@ -562,20 +554,15 @@ function throwMissingHydrationReceiptCommit(workspacePath, receipt) {
 }
 function updateHydratedWorkspaceReceipt(workspacePath, receipt) {
   assertCanonicalHydrationReceiptHeads(receipt);
-  if (!workspaceCommitsExist(workspacePath, receipt.mounts.map(({ head }) => head))) {
+  if (!workspaceCommitsExist(
+    workspacePath,
+    receipt.mounts.map(({ head }) => head)
+  )) {
     throwMissingHydrationReceiptCommit(workspacePath, receipt);
   }
   readHydratedWorkspaceReceipt(workspacePath);
   writeHydratedWorkspaceReceiptCore(workspacePath, receipt);
 }
-async function updateHydratedWorkspaceReceiptAsync(workspacePath, receipt) {
-  assertCanonicalHydrationReceiptHeads(receipt);
-  if (!await workspaceCommitsExistAsync(workspacePath, receipt.mounts.map(({ head }) => head))) {
-    throwMissingHydrationReceiptCommit(workspacePath, receipt);
-  }
-  await readHydratedWorkspaceReceiptAsync(workspacePath);
-  await writeHydratedWorkspaceReceiptCoreAsync(workspacePath, receipt);
-}
 function prepareHydratedWorkspaceReceiptWrite(workspacePath) {
   const gitDirectory = path.join(workspacePath, ".git");
   const gitDirectoryStatus = fs.lstatSync(gitDirectory);
@@ -630,17 +617,6 @@ function writeHydratedWorkspaceReceiptCore(workspacePath, receipt) {
   publishHydratedWorkspaceReceiptFile(workspacePath, receipt, paths);
   updateWorkspaceBasisRefs(workspacePath, receipt);
 }
-async function writeHydratedWorkspaceReceiptCoreAsync(workspacePath, receipt) {
-  const paths = prepareHydratedWorkspaceReceiptWrite(workspacePath);
-  const stagingPrefix = `${WORKSPACE_GIT_BASIS_STAGING_REF_PREFIX}${crypto.randomUUID()}/`;
-  await runWorkspaceRefUpdateBatchAsync(
-    workspacePath,
-    stagedWorkspaceBasisRefInstructions(receipt, stagingPrefix),
-    "stage workspace hydration basis refs"
-  );
-  publishHydratedWorkspaceReceiptFile(workspacePath, receipt, paths);
-  await updateWorkspaceBasisRefsAsync(workspacePath, receipt);
-}
 function hydrationTransactionPath(workspacePath) {
   return path.join(workspacePath, ".git", ...WORKSPACE_GIT_HYDRATION_TRANSACTION.split("/"));
 }
@@ -735,8 +711,21 @@ function readWorkspaceCheckoutDurabilityRecord(workspacePath) {
   return unverified.record;
 }
 async function readWorkspaceCheckoutDurabilityRecordAsync(workspacePath) {
-  const unverified = readWorkspaceCheckoutDurabilityRecordUnverified(workspacePath);
-  if (!unverified) return null;
+  const recordPath = workspaceCheckoutDurabilityPath(workspacePath);
+  let status;
+  try {
+    status = await fs.promises.lstat(recordPath);
+  } catch (error) {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  }
+  if (!status.isFile() || status.isSymbolicLink()) {
+    throw new Error(`Workspace checkout durability record is not a regular file: ${recordPath}`);
+  }
+  const unverified = {
+    record: parseWorkspaceCheckoutDurabilityRecordUnverified(recordPath, await fs.promises.readFile(recordPath, "utf8")),
+    recordPath
+  };
   if (unverified.record.head && !await tryGitAsync(workspacePath, ["cat-file", "-e", `${unverified.record.head}^{commit}`])) {
     throw new Error(`Workspace checkout durability record names a missing commit: ${unverified.recordPath}`);
   }
@@ -763,13 +752,6 @@ function updateWorkspaceCheckoutDurabilityRecord(workspacePath, record) {
   readWorkspaceCheckoutDurabilityRecord(workspacePath);
   writeWorkspaceCheckoutDurabilityRecordFile(workspacePath, record);
 }
-async function updateWorkspaceCheckoutDurabilityRecordAsync(workspacePath, record) {
-  if (record.head && !await tryGitAsync(workspacePath, ["cat-file", "-e", `${record.head}^{commit}`])) {
-    throw new Error(`Cannot record workspace checkout durability for a missing commit: ${record.head}`);
-  }
-  await readWorkspaceCheckoutDurabilityRecordAsync(workspacePath);
-  writeWorkspaceCheckoutDurabilityRecordFile(workspacePath, record);
-}
 function fsyncWorkspaceCheckoutTree(workspacePath) {
   const status = fs.lstatSync(workspacePath);
   if (!status.isDirectory() || status.isSymbolicLink()) {
@@ -781,40 +763,6 @@ function fsyncWorkspaceCheckoutTree(workspacePath) {
   }
   fsyncDirectory(workspacePath);
 }
-const WORKSPACE_FSYNC_YIELD_INTERVAL = 64;
-async function fsyncWorkspaceCheckoutTreeYielding(workspacePath) {
-  const status = fs.lstatSync(workspacePath);
-  if (!status.isDirectory() || status.isSymbolicLink()) {
-    throw new Error(`Workspace clone root is not a regular directory: ${workspacePath}`);
-  }
-  let entriesSinceYield = 0;
-  const syncEntry = async (entryPath) => {
-    const entryStatus = fs.lstatSync(entryPath);
-    if (entryStatus.isSymbolicLink()) return;
-    if (entryStatus.isDirectory()) {
-      for (const name of fs.readdirSync(entryPath)) await syncEntry(path.join(entryPath, name));
-      fsyncDirectory(entryPath);
-      return;
-    }
-    if (!entryStatus.isFile()) return;
-    const descriptor = fs.openSync(entryPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
-    try {
-      fs.fsyncSync(descriptor);
-    } finally {
-      fs.closeSync(descriptor);
-    }
-    entriesSinceYield += 1;
-    if (entriesSinceYield >= WORKSPACE_FSYNC_YIELD_INTERVAL) {
-      entriesSinceYield = 0;
-      await yieldToWorkerEventLoop();
-    }
-  };
-  for (const name of fs.readdirSync(workspacePath)) {
-    if (name === ".git") continue;
-    await syncEntry(path.join(workspacePath, name));
-  }
-  fsyncDirectory(workspacePath);
-}
 function workspaceRebaseInProgress(workspacePath) {
   const gitDirectory = path.join(workspacePath, ".git");
   return fs.existsSync(path.join(gitDirectory, "rebase-merge")) || fs.existsSync(path.join(gitDirectory, "rebase-apply"));
@@ -883,22 +831,36 @@ async function workspaceCheckoutDurabilityIsCurrentAsync(workspacePath) {
   const record = await readWorkspaceCheckoutDurabilityRecordAsync(workspacePath);
   return Boolean(record?.state === "durable" && record.head === await revParseAsync(workspacePath, "HEAD"));
 }
-function beginWorkspaceCheckoutTransition(workspacePath) {
+function runCheckoutTransitionBeginJob(input) {
+  const workspacePath = path.resolve(input.workspacePath);
   recoverWorkspaceCheckoutDurability(workspacePath);
   const head = revParse(workspacePath, "HEAD");
   updateWorkspaceCheckoutDurabilityRecord(workspacePath, { version: 1, state: "transition", head });
+  return null;
+}
+function runCheckoutTransitionCompleteJob(input) {
+  const workspacePath = path.resolve(input.workspacePath);
+  fsyncWorkspaceCheckoutTree(workspacePath);
+  updateWorkspaceCheckoutDurabilityRecord(workspacePath, { version: 1, state: "durable", head: revParse(workspacePath, "HEAD") });
+  return null;
+}
+function runCheckoutDurabilityRecoveryJob(input) {
+  recoverWorkspaceCheckoutDurability(path.resolve(input.workspacePath), input.preserveResolutionInProgress);
+  return null;
+}
+function runOuterCheckoutFsyncJob(input) {
+  fsyncWorkspaceCheckoutTree(path.resolve(input.workspacePath));
+  return null;
+}
+async function beginWorkspaceCheckoutTransition(workspacePath) {
+  await workspaceFilesystemExecutor().run("checkout_transition_begin", { workspacePath });
 }
 async function completeWorkspaceCheckoutTransition(workspacePath) {
-  await fsyncWorkspaceCheckoutTreeYielding(workspacePath);
-  await updateWorkspaceCheckoutDurabilityRecordAsync(workspacePath, {
-    version: 1,
-    state: "durable",
-    head: await revParseAsync(workspacePath, "HEAD")
-  });
+  await workspaceFilesystemExecutor().run("checkout_transition_complete", { workspacePath });
 }
-function throwAfterWorkspaceCheckoutRecovery(workspacePath, error) {
+async function throwAfterWorkspaceCheckoutRecovery(workspacePath, error) {
   try {
-    recoverWorkspaceCheckoutDurability(workspacePath);
+    await workspaceFilesystemExecutor().run("checkout_durability_recovery", { workspacePath, preserveResolutionInProgress: false });
   } catch (recoveryError) {
     throw new AggregateError(
       [error, recoveryError],
@@ -908,12 +870,12 @@ function throwAfterWorkspaceCheckoutRecovery(workspacePath, error) {
   throw error;
 }
 async function runWorkspaceCheckoutTransition(workspacePath, mutate) {
-  beginWorkspaceCheckoutTransition(workspacePath);
+  await beginWorkspaceCheckoutTransition(workspacePath);
   try {
     await mutate();
     await completeWorkspaceCheckoutTransition(workspacePath);
   } catch (error) {
-    throwAfterWorkspaceCheckoutRecovery(workspacePath, error);
+    await throwAfterWorkspaceCheckoutRecovery(workspacePath, error);
   }
 }
 function parseHydrationTransactionManifest(transactionPath) {
@@ -1295,8 +1257,16 @@ async function configureWorkspaceRepository(input) {
     await gitAsync(input.workspacePath, ["remote", "add", "origin", input.remoteUrl], "configure workspace origin");
   }
   await tryGitAsync(input.workspacePath, ["config", "--local", "--unset-all", "remote.origin.pushurl"]);
-  await gitAsync(input.workspacePath, ["config", "--local", "--replace-all", "credential.helper", ""], "reset workspace credential helpers");
-  await gitAsync(input.workspacePath, ["config", "--local", "credential.useHttpPath", "false"], "configure workspace credential path matching");
+  await gitAsync(
+    input.workspacePath,
+    ["config", "--local", "--replace-all", "credential.helper", ""],
+    "reset workspace credential helpers"
+  );
+  await gitAsync(
+    input.workspacePath,
+    ["config", "--local", "credential.useHttpPath", "false"],
+    "configure workspace credential path matching"
+  );
   if (input.credentialUsername) {
     await gitAsync(
       input.workspacePath,
@@ -1394,19 +1364,24 @@ async function ensureWorkspaceGitClone(input) {
       input.credentialHelper,
       input.credentialUsername
     );
-    const clone = Bun.spawnSync(cloneCommand, {
+    const clone = Bun.spawn(cloneCommand, {
+      stdin: "ignore",
       stdout: "pipe",
       stderr: "pipe",
       env: workerGitProcessEnvironment()
     });
-    if (clone.exitCode !== 0) {
+    const [, , cloneExitCode] = await Promise.all([new Response(clone.stdout).text(), new Response(clone.stderr).text(), clone.exited]);
+    if (cloneExitCode !== 0) {
       fs.rmSync(workspacePath, { recursive: true, force: true });
       fs.mkdirSync(workspacePath, { recursive: true });
       git(workspacePath, ["init", `--initial-branch=${WORKSPACE_GIT_BRANCH}`], "initialize workspace clone");
     }
   }
   await configureWorkspaceRepository({ ...input, workspacePath });
-  recoverWorkspaceCheckoutDurability(workspacePath, input.preserveResolutionInProgress);
+  await workspaceFilesystemExecutor().run("checkout_durability_recovery", {
+    workspacePath,
+    preserveResolutionInProgress: input.preserveResolutionInProgress === true
+  });
   const previousRemoteHead = await revParseAsync(workspacePath, `refs/remotes/origin/${WORKSPACE_GIT_BRANCH}`);
   const localHeadBeforeFetch = await revParseAsync(workspacePath, "HEAD");
   if (!await revParseAsync(workspacePath, WORKSPACE_GIT_INTEGRATED_REF) && previousRemoteHead && localHeadBeforeFetch && await tryGitAsync(workspacePath, ["merge-base", "--is-ancestor", previousRemoteHead, localHeadBeforeFetch])) {
@@ -1685,7 +1660,7 @@ async function hydrateWorkspaceGitMountsTransactionally(input) {
     };
   };
   const noopCandidate = computeSelection();
-  if (noopCandidate.hydrationMounts.length === 0 && noopCandidate.advancedDurabilityMounts.length === 0 && noopCandidate.targetReceipt && lstatIfExists(hydrationTransactionPath(workspacePath)) === null && workspaceHydrationReceiptsEqual(receiptBeforeTransaction, noopCandidate.targetReceipt) && hydratedWorkspaceReceiptFileIsV3(workspacePath) && await workspaceBasisRefsAreCurrentAsync(workspacePath, noopCandidate.targetReceipt)) {
+  if (noopCandidate.hydrationMounts.length === 0 && noopCandidate.advancedDurabilityMounts.length === 0 && noopCandidate.targetReceipt && lstatIfExists(hydrationTransactionPath(workspacePath)) === null && workspaceHydrationReceiptsEqual(receiptBeforeTransaction, noopCandidate.targetReceipt) && await hydratedWorkspaceReceiptFileIsV3(workspacePath) && await workspaceBasisRefsAreCurrentAsync(workspacePath, noopCandidate.targetReceipt)) {
     return { hydratedMountIds: [], skippedMountIds: noopCandidate.skippedMountIds };
   }
   const hooks = input.hydrationHooks;
@@ -1698,47 +1673,92 @@ async function hydrateWorkspaceGitMountsTransactionally(input) {
   try {
     const selection = computeSelection();
     const { hydrationMounts, advancedDurabilityMounts, targetReceipt } = selection;
-    const { manifest, durabilityScopes } = beginHydrationTransaction(workspacePath, hydrationMounts, targetReceipt);
+    const heldMountIds = [...new Set([...hydrationMounts, ...advancedDurabilityMounts].map(({ id }) => id))];
+    const releaseHold = hooks?.holdMounts(heldMountIds, `hydration transaction ${context.transactionId}`);
     try {
-      const hydration = hydrateWorkspaceGitMountsRaw(workspacePath, hydrationMounts, { ignoreBusy: true });
-      const expectedHydratedIds = hydrationMounts.map(({ id }) => id).sort();
-      if (hydration.skippedMountIds.length > 0 || hydration.hydratedMountIds.length !== expectedHydratedIds.length || hydration.hydratedMountIds.some((id, index) => id !== expectedHydratedIds[index])) {
-        throw new Error("Workspace hydration did not include every required mount");
-      }
-      fsyncHydratedWorkspaceMounts(advancedDurabilityMounts, durabilityScopes);
-      markHydrationTransactionDurable(workspacePath);
-      if (targetReceipt) updateHydratedWorkspaceReceipt(workspacePath, targetReceipt);
-      removeHydrationTransaction(workspacePath);
-      return {
-        hydratedMountIds: hydration.hydratedMountIds,
-        skippedMountIds: selection.skippedMountIds
-      };
-    } catch (error) {
-      const transactionPath = hydrationTransactionPath(workspacePath);
-      if (lstatIfExists(transactionPath) && hydrationTransactionCommitted(transactionPath)) {
-        throw error;
-      }
-      try {
-        restoreHydrationTransaction(workspacePath, manifest);
-        removeHydrationTransaction(workspacePath);
-      } catch (restoreError) {
-        throw new AggregateError(
-          [error, restoreError],
-          `Workspace hydration failed and its durable snapshot could not be restored: ${error instanceof Error ? error.message : String(error)}`
-        );
-      }
-      throw error;
+      const hydration = await workspaceFilesystemExecutor().run("hydration_transaction", {
+        workspacePath,
+        hydrationMounts: hydrationMounts.map(workspaceGitMountData),
+        advancedDurabilityMounts: advancedDurabilityMounts.map(workspaceGitMountData),
+        targetReceipt
+      });
+      return { hydratedMountIds: hydration.hydratedMountIds, skippedMountIds: selection.skippedMountIds };
+    } finally {
+      releaseHold?.();
     }
   } finally {
     hooks?.afterTransaction(context);
   }
 }
+function runHydrationTransactionJob(input) {
+  const workspacePath = path.resolve(input.workspacePath);
+  const { hydrationMounts, advancedDurabilityMounts, targetReceipt } = input;
+  const { manifest, durabilityScopes } = beginHydrationTransaction(workspacePath, hydrationMounts, targetReceipt);
+  try {
+    const hydration = hydrateWorkspaceGitMountsRaw(workspacePath, hydrationMounts, { ignoreBusy: true });
+    const expectedHydratedIds = hydrationMounts.map(({ id }) => id).sort();
+    if (hydration.skippedMountIds.length > 0 || hydration.hydratedMountIds.length !== expectedHydratedIds.length || hydration.hydratedMountIds.some((id, index) => id !== expectedHydratedIds[index])) {
+      throw new Error("Workspace hydration did not include every required mount");
+    }
+    fsyncHydratedWorkspaceMounts(advancedDurabilityMounts, durabilityScopes);
+    markHydrationTransactionDurable(workspacePath);
+    if (targetReceipt) updateHydratedWorkspaceReceipt(workspacePath, targetReceipt);
+    removeHydrationTransaction(workspacePath);
+    return { hydratedMountIds: hydration.hydratedMountIds };
+  } catch (error) {
+    const transactionPath = hydrationTransactionPath(workspacePath);
+    if (lstatIfExists(transactionPath) && hydrationTransactionCommitted(transactionPath)) {
+      throw error;
+    }
+    try {
+      restoreHydrationTransaction(workspacePath, manifest);
+      removeHydrationTransaction(workspacePath);
+    } catch (restoreError) {
+      throw new AggregateError(
+        [error, restoreError],
+        `Workspace hydration failed and its durable snapshot could not be restored: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+    throw error;
+  }
+}
+function runHydrationRawJob(input) {
+  return {
+    hydratedMountIds: hydrateWorkspaceGitMountsRaw(path.resolve(input.workspacePath), input.mounts, { ignoreBusy: true }).hydratedMountIds
+  };
+}
+function runHydrationRecoveryJob(input) {
+  const workspacePath = path.resolve(input.workspacePath);
+  recoverWorkspaceCheckoutDurability(workspacePath, input.preserveResolutionInProgress);
+  recoverPendingHydrationTransaction(workspacePath);
+  repairWorkspaceBasisRefs(workspacePath, readHydratedWorkspaceReceipt(workspacePath));
+  return null;
+}
 async function hydrateWorkspaceGitMounts(workspacePath, mounts, options = {}) {
-  const gitDirectory = path.join(path.resolve(workspacePath), ".git");
+  const resolvedWorkspacePath = path.resolve(workspacePath);
+  const gitDirectory = path.join(resolvedWorkspacePath, ".git");
   if (!fs.existsSync(gitDirectory)) {
-    return hydrateWorkspaceGitMountsRaw(workspacePath, mounts, options);
+    const skippedMountIds = options.ignoreBusy ? [] : mounts.filter((mount) => mount.busy?.()).map(({ id }) => id).sort();
+    const skipped = new Set(skippedMountIds);
+    const selected = mounts.filter(({ id }) => !skipped.has(id));
+    const releaseHold = options.hydrationHooks?.holdMounts(
+      selected.map(({ id }) => id),
+      "pre-clone hydration"
+    );
+    try {
+      const hydration = await workspaceFilesystemExecutor().run("hydration_raw", {
+        workspacePath: resolvedWorkspacePath,
+        mounts: selected.map(workspaceGitMountData)
+      });
+      return { hydratedMountIds: hydration.hydratedMountIds, skippedMountIds };
+    } finally {
+      releaseHold?.();
+    }
   }
-  recoverWorkspaceCheckoutDurability(path.resolve(workspacePath));
+  await workspaceFilesystemExecutor().run("checkout_durability_recovery", {
+    workspacePath: resolvedWorkspacePath,
+    preserveResolutionInProgress: false
+  });
   return hydrateWorkspaceGitMountsTransactionally({
     workspacePath,
     mounts,
@@ -1749,13 +1769,12 @@ async function hydrateWorkspaceGitMounts(workspacePath, mounts, options = {}) {
 async function recoverWorkspaceGitHydration(workspacePath, mounts, options = {}) {
   workspacePath = path.resolve(workspacePath);
   validateMounts(workspacePath, mounts);
-  recoverWorkspaceCheckoutDurability(workspacePath, options.preserveResolutionInProgress);
-  recoverPendingHydrationTransaction(workspacePath);
+  await workspaceFilesystemExecutor().run("hydration_recovery", {
+    workspacePath,
+    preserveResolutionInProgress: options.preserveResolutionInProgress === true
+  });
   const currentHead = await revParseAsync(workspacePath, "HEAD");
   const currentReceipt = await readHydratedWorkspaceReceiptAsync(workspacePath);
-  if (!await workspaceBasisRefsAreCurrentAsync(workspacePath, currentReceipt ?? { version: 3, mounts: [] })) {
-    repairWorkspaceBasisRefs(workspacePath, currentReceipt);
-  }
   if (!currentHead) return;
   const outerHeadContainsMount = (mount) => tryGitAsync(workspacePath, ["cat-file", "-e", `HEAD:${normalizedWorkspaceMountPath(mount.workspaceRelativePath)}`]);
   const configuredBasisMounts = configuredWorkspaceHydrationBasisMounts(workspacePath, mounts, options.deferMountIds);
@@ -1796,13 +1815,17 @@ async function resetWorkspaceGit(input) {
     throw new Error(`Workspace reset cannot run while mounts are busy: ${busyResetMountIds.sort().join(", ")}`);
   }
   const initial = await ensureWorkspaceGitClone({ ...input, workspacePath });
-  recoverPendingHydrationTransaction(workspacePath);
+  await workspaceFilesystemExecutor().run("hydration_recovery", { workspacePath, preserveResolutionInProgress: false });
   const status = gitResult(workspacePath, ["status", "--porcelain=v1", "-z"]);
   const discardedPaths = status.exitCode === 0 ? status.stdout.split("\0").filter(Boolean).map((entry) => entry.slice(3)).sort() : [];
   const remoteHead = await fetchWorkspaceHead(workspacePath, input.remoteUrl, input.credentialHelper, input.credentialUsername);
   if (remoteHead) {
     await runWorkspaceCheckoutTransition(workspacePath, async () => {
-      await gitAsync(workspacePath, ["checkout", "--no-recurse-submodules", "-B", WORKSPACE_GIT_BRANCH, remoteHead], "reset workspace main");
+      await gitAsync(
+        workspacePath,
+        ["checkout", "--no-recurse-submodules", "-B", WORKSPACE_GIT_BRANCH, remoteHead],
+        "reset workspace main"
+      );
       await gitAsync(workspacePath, ["clean", "-fd", "--", "."], "remove untracked workspace changes");
     });
   } else if (revParse(workspacePath, "HEAD")) {
@@ -2013,7 +2036,7 @@ function snapshotConflict(input) {
 }
 async function rebaseWorkspace(input) {
   const remoteRevision = `refs/remotes/origin/${WORKSPACE_GIT_BRANCH}`;
-  beginWorkspaceCheckoutTransition(input.workspacePath);
+  await beginWorkspaceCheckoutTransition(input.workspacePath);
   const rebase = await gitResultAsync(
     input.workspacePath,
     input.truncatedHistoryBase ? ["rebase", "--onto", remoteRevision, input.truncatedHistoryBase] : ["rebase", remoteRevision]
@@ -2183,18 +2206,24 @@ async function runWorkspaceGitSynchronization(input, remoteHeadFetches) {
         cycleSkippedMountIds.add(mount.id);
         continue;
       }
-      const merged = mergeWorkspaceProjectionMount({
-        workspacePath,
-        mount: {
-          id: mount.id,
-          sourcePath: mount.sourcePath,
-          workspaceRelativePath: normalizedWorkspaceMountPath(mount.workspaceRelativePath),
-          sourceMode: mount.sourceMode
-        },
-        basisHead,
-        currentHead: projectionBasisHead,
-        attemptId
-      });
+      const releaseMergeHold = input.hydrationHooks?.holdMounts([mount.id], `merge projection of mount ${mount.id}`);
+      let merged;
+      try {
+        merged = await workspaceFilesystemExecutor().run("projection_merge", {
+          workspacePath,
+          mount: {
+            id: mount.id,
+            sourcePath: mount.sourcePath,
+            workspaceRelativePath: normalizedWorkspaceMountPath(mount.workspaceRelativePath),
+            sourceMode: mount.sourceMode
+          },
+          basisHead,
+          currentHead: projectionBasisHead,
+          attemptId
+        });
+      } finally {
+        releaseMergeHold?.();
+      }
       if (merged.kind === "conflict") {
         const refs = snapshotConflict({ workspacePath, attemptId, localHead: merged.oursCommit, remoteHead: projectionBasisHead });
         return {
@@ -2217,7 +2246,6 @@ async function runWorkspaceGitSynchronization(input, remoteHeadFetches) {
       }
       mergedProjectionTrees.set(mount.id, merged.resultTree);
       mergedProjectionMounts.push(mount);
-      await yieldToWorkerEventLoop();
     }
   }
   if (projectionWarning) process.stderr.write(`[r5d-worker] ${projectionWarning}
@@ -2300,7 +2328,7 @@ async function runWorkspaceGitSynchronization(input, remoteHeadFetches) {
   const projectedFastMounts = [];
   const currentProjectionMounts = [];
   if (!input.skipMountMirror) {
-    beginWorkspaceCheckoutTransition(workspacePath);
+    await beginWorkspaceCheckoutTransition(workspacePath);
     try {
       const selectedActiveIds = new Set(selected.active.map(({ id }) => id));
       const fastActiveMounts = fastProjectionMounts.filter(({ id }) => selectedActiveIds.has(id));
@@ -2314,30 +2342,34 @@ async function runWorkspaceGitSynchronization(input, remoteHeadFetches) {
           currentProjectionMounts.push(mount);
           continue;
         }
-        projectedGitlinks.push(...mirrorMountsToWorkspace(workspacePath, [mount]));
+        const releaseMirrorHold = input.hydrationHooks?.holdMounts([mount.id], `projection of mount ${mount.id}`);
+        try {
+          const mirrored = await workspaceFilesystemExecutor().run("projection_mirror", {
+            workspacePath,
+            mount: workspaceGitMountData(mount)
+          });
+          projectedGitlinks.push({ mount, gitlinks: mirrored.gitlinks });
+        } finally {
+          releaseMirrorHold?.();
+        }
         projectedFastMounts.push(mount);
-        await yieldToWorkerEventLoop();
       }
-      removeWorkspaceMounts(workspacePath, fastTombstones);
-      projectedFastMounts.push(...fastTombstones);
-      for (const mount of mergedProjectionMounts) {
-        materializeWorkspaceProjectionTree({
-          workspacePath,
+      await workspaceFilesystemExecutor().run("projection_outer_apply", {
+        workspacePath,
+        removeMounts: fastTombstones.map(workspaceGitMountData),
+        materialize: mergedProjectionMounts.map((mount) => ({
           workspaceRelativePath: normalizedWorkspaceMountPath(mount.workspaceRelativePath),
           resultTree: mergedProjectionTrees.get(mount.id)
-        });
-        await yieldToWorkerEventLoop();
-      }
-      restoreDeferredWorkspaceMountsFromHead(
-        workspacePath,
-        input.mounts.filter(
+        })),
+        restoreFromHead: input.mounts.filter(
           (mount) => deferredMountIds.has(mount.id) || mount.preserveLocalOnHydrationBasisChange === true && cycleSkippedMountIds.has(mount.id)
-        )
-      );
+        ).map(workspaceGitMountData)
+      });
+      projectedFastMounts.push(...fastTombstones);
       await stageAndCommitWorkspace();
-      await fsyncWorkspaceCheckoutTreeYielding(workspacePath);
+      await workspaceFilesystemExecutor().run("outer_checkout_fsync", { workspacePath });
     } catch (error) {
-      throwAfterWorkspaceCheckoutRecovery(workspacePath, error);
+      await throwAfterWorkspaceCheckoutRecovery(workspacePath, error);
     }
   } else {
     await stageAndCommitWorkspace();
@@ -2353,12 +2385,12 @@ async function runWorkspaceGitSynchronization(input, remoteHeadFetches) {
       allProjectionReceiptMounts
     );
     const currentProjectionReceipt = await readHydratedWorkspaceReceiptAsync(workspacePath);
-    if (!workspaceHydrationReceiptsEqual(currentProjectionReceipt, targetProjectionReceipt) || !hydratedWorkspaceReceiptFileIsV3(workspacePath) || !await workspaceBasisRefsAreCurrentAsync(workspacePath, targetProjectionReceipt)) {
-      for (const mount of projectedFastMounts) {
-        fsyncHydratedWorkspaceMount(mount);
-        await yieldToWorkerEventLoop();
-      }
-      await updateHydratedWorkspaceReceiptAsync(workspacePath, targetProjectionReceipt);
+    if (!workspaceHydrationReceiptsEqual(currentProjectionReceipt, targetProjectionReceipt) || !await hydratedWorkspaceReceiptFileIsV3(workspacePath) || !await workspaceBasisRefsAreCurrentAsync(workspacePath, targetProjectionReceipt)) {
+      await workspaceFilesystemExecutor().run("projection_receipt", {
+        workspacePath,
+        fsyncMounts: projectedFastMounts.map(workspaceGitMountData),
+        receipt: targetProjectionReceipt
+      });
     }
     await completeWorkspaceCheckoutTransition(workspacePath);
   }
@@ -2525,6 +2557,25 @@ ${push.stdout}`)) {
   }
   throw new Error(`Workspace push did not converge after ${maxPushAttempts} attempts`);
 }
+function runProjectionMirrorJob(input) {
+  const projected = mirrorMountsToWorkspace(path.resolve(input.workspacePath), [input.mount]);
+  return { gitlinks: projected[0]?.gitlinks ?? [] };
+}
+function runProjectionOuterApplyJob(input) {
+  const workspacePath = path.resolve(input.workspacePath);
+  removeWorkspaceMounts(workspacePath, input.removeMounts);
+  for (const { workspaceRelativePath, resultTree } of input.materialize) {
+    materializeWorkspaceProjectionTree({ workspacePath, workspaceRelativePath, resultTree });
+  }
+  restoreDeferredWorkspaceMountsFromHead(workspacePath, input.restoreFromHead);
+  return null;
+}
+function runProjectionReceiptJob(input) {
+  const workspacePath = path.resolve(input.workspacePath);
+  for (const mount of input.fsyncMounts) fsyncHydratedWorkspaceMount(mount);
+  updateHydratedWorkspaceReceipt(workspacePath, input.receipt);
+  return null;
+}
 const workspaceGitSyncTestHarness = {
   commandArgs: gitCommandArgs,
   workspaceCloneCommandArgs,
@@ -2545,9 +2596,20 @@ export {
   hydrateWorkspaceGitMounts,
   recoverWorkspaceGitHydration,
   resetWorkspaceGit,
+  runCheckoutDurabilityRecoveryJob,
+  runCheckoutTransitionBeginJob,
+  runCheckoutTransitionCompleteJob,
+  runHydrationRawJob,
+  runHydrationRecoveryJob,
+  runHydrationTransactionJob,
+  runOuterCheckoutFsyncJob,
+  runProjectionMirrorJob,
+  runProjectionOuterApplyJob,
+  runProjectionReceiptJob,
   synchronizeWorkspaceGit,
   workspaceGitHydrationIsCurrent,
   workspaceGitHydrationIsRecoverable,
+  workspaceGitMountData,
   workspaceGitMountsForRemediation,
   workspaceGitSyncTestHarness
 };