@usine/executor 0.3.0 → 0.4.0

package/dist/remote/remote.js

@@ -4,13 +4,14 @@ import { InfraFailure } from "@usine/core/attempt/failure";
 import { RepositoryName, safeSegment } from "@usine/core/repository";
 import { harvestFacts, harvestPostedComments, } from "@usine/core/step/harvest";
 import { AttemptRuntime } from "@usine/durable/attempt-runtime";
-import { Deferred, Effect, Exit, Option, Schema, Scope } from "effect";
+import { Deferred, Effect, Exit, Option, Schema, Scope, SynchronizedRef } from "effect";
 import { durableSessionFor } from "../durable-session.js";
 import { CheckoutRequest, Executor, ExecutorError, ExecutorOperationError, WorkerCapabilityError, } from "../executor.js";
 import { verifyHarnessReadiness } from "../harness-preflight.js";
+import { childEnvironment, commitIdentity } from "../child-environment.js";
 import { keyedLockFor } from "../keyed-lock.js";
-import { withGitHooks } from "../pre-push-hook.js";
-import { cacheDirectoryName, provisionWorktree, reattachWorktree, recoverWorktree, } from "../provision.js";
+import { cacheDirectoryName, keptCheckoutRecovery, linkedWorktree, provisionWorktree, reattachWorktree, recoverWorktree, recoveryFields, } from "../provision.js";
+import { attemptContainerName, containerCheckoutMarker, containersFor, } from "./container.js";
 import { workerLeftoversFor } from "./leftovers.js";
 import { workerWatchesFor } from "./liveness.js";
 import { abandonedWorktreeSweepFor } from "./sweep.js";
@@ -23,10 +24,31 @@ mkdir -p "$(dirname "$1")"
 printf '%s' "$2" > "$1"
 chmod 755 "$1"
 `.trim();
+/**
+ * A script that creates the checkout `$1` as its own repository at commit `$4`: objects come from
+ * the cached clone `$2` through an alternate, and missing blobs from the partial-clone remote
+ * `$3`. The checkout writes only its own directory, so the cached clone can be read-only.
+ */
+const STANDALONE_CHECKOUT_SCRIPT = String.raw `
+set -e
+git init -q "$1"
+printf '%s\n' "$2/objects" > "$1/.git/objects/info/alternates"
+git -C "$1" config core.repositoryformatversion 1
+git -C "$1" config extensions.partialClone origin
+git -C "$1" remote add origin "$3"
+git -C "$1" config remote.origin.promisor true
+git -C "$1" config remote.origin.partialclonefilter blob:none
+git -C "$1" checkout -q --detach "$4"
+`.trim();
+/** A script that removes the checkout `$1`, then its container marker `$2`. */
+const REMOVE_CHECKOUT_SCRIPT = 'rm -rf "$1" && rm -f "$2"';
+/** A script that removes everything inside the directory `$1` and keeps the directory. */
+const EMPTY_DIRECTORY_SCRIPT = 'find "$1" -mindepth 1 -maxdepth 1 -exec rm -rf {} +';
 /**
  * Returns the executor for Workers accessed over SSH. Each Attempt's Pi parent is a durable task
  * in this process, and each child runs on the machine as one SSH command with a pid file next
- * to the worktree. `worker` provides the machine, preflight, and repository access.
+ * to the worktree, or inside the Attempt's container. `worker` provides the machine, preflight,
+ * isolation, and repository access.
  */
 export const remoteExecutorFor = (worker) => Effect.gen(function* () {
     const { commands, label } = worker;
@@ -38,9 +60,22 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
     const session = yield* durableSessionFor({ interruptGrace: worker.interruptGrace });
     const runtime = yield* AttemptRuntime;
     const workerWatches = yield* workerWatchesFor(commands);
-    const stopChildren = Effect.fn(span("stopChildren"))(function* (operation, machine, worktree) {
+    const containers = containersFor(commands);
+    const attemptMachine = worker.attemptMachine ??
+        ((name, _attemptId) => worker.machineFor(name));
+    const leftoversOf = (attempt) => worker.leftovers === undefined
+        ? workerLeftovers(attempt.worker).pipe(Effect.map((found) => Option.some(found)))
+        : worker.leftovers(attempt, workerLeftovers(attempt.worker));
+    const recovered = worker.recovered ?? (() => Effect.void);
+    const checkoutHeld = worker.checkoutHeld ?? (() => Effect.void);
+    /** Returns `removal` that records the checkout of `attemptId` as gone once it succeeds. */
+    const removing = (attemptId, removal) => Effect.tap(removal, () => checkoutHeld(attemptId, false));
+    const cachePath = (machine, repository) => `${machine.workerDirectory}/repos/${cacheDirectoryName(repository)}`;
+    const stopChildren = Effect.fn(span("stopChildren"))(function* (operation, machine, worktree, attemptId, isolation) {
         const host = machine.shell.destination;
-        yield* Effect.annotateCurrentSpan({ host, worktree });
+        yield* Effect.annotateCurrentSpan({ host, worktree, isolation: isolation._tag });
+        if (isolation._tag === "Container")
+            return yield* containers.stopChildren(operation, machine.shell, attemptContainerName(attemptId), worktree);
         const outcome = yield* commands.run(operation, machine.shell, "sh", [
             "-c",
             CHILD_KILL_SCRIPT,
@@ -53,74 +88,212 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
             children: outcome.trim().split("\n"),
         });
     });
-    /** Runs git on the machine with the Attempt's repository access, refreshing the credential first if it's about to expire. */
-    const accessGit = (operation, access) => (args) => access.refresh.pipe(Effect.andThen(commands.run(operation, access.shell, "git", args, {
-        environment: access.gitEnvironment,
-        scrub: access.scrub,
-    })));
-    const harvestCommand = (operation, worktree, access) => (command, args) => access.refresh
-        .pipe(Effect.andThen(commands.run(operation, access.shell, command, args, {
+    /**
+     * Returns how commands run in `attemptId`'s existing checkout at `worktree`: in its container
+     * if the container marker, which no container can reach, records one, or as the SSH user if
+     * it records none. Nothing in the checkout decides, since an Attempt can change it. Fails if
+     * the Worker's `isolation` can't run the checkout that way: a container's checkout on a
+     * machine without Docker, or a checkout from before Docker, as the SSH user who can now use
+     * Docker.
+     */
+    const isolationOfCheckout = Effect.fn(span("isolationOfCheckout"))(function* (operation, machine, attemptId, worktree, isolation) {
+        if (!worker.runsContainers)
+            return isolation;
+        const host = machine.shell.destination;
+        yield* Effect.annotateCurrentSpan({ host, worktree, isolation: isolation._tag });
+        const contained = yield* containers.marked(operation, machine.shell, containerCheckoutMarker(machine.workerDirectory, attemptId));
+        if (contained === (isolation._tag === "Container"))
+            return isolation;
+        const message = contained
+            ? `The checkout ${worktree} was mounted in an attempt container, and the machine no longer has Docker, so usine runs no command in it as the SSH user. Install Docker, or salvage and remove the checkout by hand.`
+            : `The checkout ${worktree} is from before the machine had Docker, and the SSH user can use Docker, so usine runs no command in it as that user. Salvage and remove the checkout by hand.`;
+        yield* Effect.logError("Refused to run commands in the checkout", {
+            attemptId,
+            host,
+            worktree,
+            contained,
+            message,
+        });
+        return yield* new ExecutorOperationError({
+            operation,
+            message,
+            cause: new Error(contained ? "container marker without Docker" : "no container marker"),
+        });
+    });
+    /** Returns the container of an Attempt, which mounts its checkout and the paths of `access`. */
+    const containerSpec = (machine, settings, attempt, worktree, access, harnesses) => ({
+        name: attemptContainerName(attempt.attemptId),
+        settings,
+        worker: attempt.worker,
+        attemptId: Option.some(attempt.attemptId),
+        checkout: Option.some({ path: worktree, parent: `${machine.workerDirectory}/attempts` }),
+        readOnly: [cachePath(machine, attempt.repository), ...access.readOnlyPaths],
+        writable: access.writablePaths,
+        harnesses,
+        environment: access.environment,
+        secrets: access.secrets,
+    });
+    /**
+     * Returns where the Attempt's commands in its checkout run. In a container, the container
+     * starts on first use in `scope` and is removed when `scope` closes.
+     */
+    const checkoutHostFor = Effect.fn(span("checkoutHost"))(function* (operation, scope, machine, attempt, worktree, access, isolation, harnesses) {
+        if (isolation._tag === "SharedUser") {
+            const shell = Effect.succeed(access.shell);
+            const shared = {
+                isolation,
+                shell,
+                salvageShell: shell,
+                recordOutOfMemory: () => Effect.void,
+            };
+            return shared;
+        }
+        const started = yield* SynchronizedRef.make(Option.none());
+        const spec = containerSpec(machine, isolation.settings, attempt, worktree, access, harnesses);
+        const marker = containerCheckoutMarker(machine.workerDirectory, attempt.attemptId);
+        const memory = yield* containers.trackMemory(machine.shell, spec);
+        const shell = SynchronizedRef.modifyEffect(started, (current) => Option.match(current, {
+            onSome: (shell) => Effect.succeed([shell, current]),
+            onNone: () => containers.mark(operation, machine.shell, marker).pipe(Effect.andThen(containers.start(operation, machine.shell, spec)), 
+            // Added after the start's removal, so it runs before it.
+            Effect.tap(() => Effect.addFinalizer(() => memory.recordKills())), Scope.provide(scope), Effect.map((shell) => [shell, Option.some(shell)])),
+        }));
+        const contained = {
+            isolation,
+            shell,
+            salvageShell: yield* Effect.cached(Effect.tap(shell, () => memory.ensureRunning)),
+            recordOutOfMemory: memory.recordTo,
+        };
+        return contained;
+    });
+    /**
+     * Runs git on the machine with the Attempt's repository access and `author` as the author and
+     * committer, refreshing the credential first if it's about to expire.
+     */
+    const accessGit = (operation, access, author, shell = Effect.succeed(access.shell)) => (args) => {
+        const environment = { ...access.gitEnvironment, ...commitIdentity(author) };
+        return access.refresh.pipe(Effect.andThen(shell), Effect.flatMap((on) => commands.run(operation, on, "git", args, { environment, scrub: access.scrub })));
+    };
+    const harvestCommand = (operation, worktree, access, shell) => (command, args) => access.refresh
+        .pipe(Effect.andThen(commands.run(operation, shell, command, args, {
         cwd: worktree,
         environment: access.environment,
         scrub: access.scrub,
     })))
         .pipe(Effect.map((stdout) => (stdout === "" ? [] : stdout.replace(/\n$/u, "").split("\n"))));
-    const recover = Effect.fn(span("recover"))(function* (attempt, record) {
-        yield* Effect.annotateCurrentSpan({ ...attempt });
+    /** Returns whether the checkout at `worktree` exists on `machine`. */
+    const checkoutExists = (machine, worktree) => Effect.gen(function* () {
+        const inspected = yield* commands.inspect("cleanup", machine.shell, "test", [
+            "-d",
+            worktree,
+        ]);
+        if (inspected.exitCode === 1)
+            return false;
+        if (inspected.exitCode !== 0)
+            return yield* executorError("Can't inspect the recovery checkout", new Error(inspected.stderr));
+        return true;
+    });
+    /**
+     * Removes a container's checkout at `worktree` from the machine, then its container marker.
+     * It holds no registration in the cached clone.
+     */
+    const removeStandalone = (operation, machine, attemptId, worktree) => Effect.asVoid(commands.run(operation, machine.shell, "sh", [
+        "-c",
+        REMOVE_CHECKOUT_SCRIPT,
+        "sh",
+        worktree,
+        containerCheckoutMarker(machine.workerDirectory, attemptId),
+    ]));
+    /**
+     * Stops the children in `attempt`'s checkout at `worktree`, salvages it with a new
+     * credential, records the salvage through `record`, and removes the checkout once its work is
+     * pushed. See `recoverWorktree`.
+     */
+    const reconcileCheckout = Effect.fn(span("reconcileCheckout"))(function* (attempt, machine, worktree, record, commitAuthor) {
+        yield* Effect.annotateCurrentSpan({ attemptId: attempt.attemptId, worker: attempt.worker });
+        const isolation = yield* isolationOfCheckout("cleanup", machine, attempt.attemptId, worktree, yield* worker.isolation(machine));
+        const reconcile = (git, removeCheckout) => recoverWorktree({ attempt, host: machine.shell.destination, worktree, git, removeCheckout }, record);
+        const removeCheckout = (git) => removing(attempt.attemptId, isolation._tag === "Container"
+            ? removeStandalone("cleanup", machine, attempt.attemptId, worktree)
+            : Effect.asVoid(git([
+                "--git-dir",
+                cachePath(machine, attempt.repository),
+                "worktree",
+                "remove",
+                "--force",
+                worktree,
+            ])));
+        if (attempt.succeeded || attempt.permissions === "read") {
+            yield* stopChildren("cleanup", machine, worktree, attempt.attemptId, isolation);
+            const git = (args) => commands.run("cleanup", machine.shell, "git", args);
+            yield* reconcile(git, removeCheckout(git));
+            if (isolation._tag === "Container")
+                yield* containers.remove(machine.shell, attemptContainerName(attempt.attemptId));
+            return;
+        }
+        const author = yield* commitAuthor;
+        const grant = yield* worker.grantFor(attempt.repository);
+        yield* Effect.scoped(Effect.gen(function* () {
+            const access = yield* grant.acquire(attempt.attemptId, attempt.worker, attempt.permissions, machine, "cleanup");
+            yield* stopChildren("cleanup", machine, worktree, attempt.attemptId, isolation);
+            const checkout = yield* checkoutHostFor("cleanup", yield* Effect.scope, machine, attempt, worktree, access, isolation, []);
+            const git = accessGit("cleanup", access, author, checkout.shell);
+            yield* reconcile(git, removeCheckout(git));
+        }));
+    });
+    const recover = Effect.fn(span("recover"))(function* (attempt, record, commitAuthor) {
+        yield* Effect.annotateCurrentSpan(recoveryFields(attempt));
         yield* session.stopForRecovery(attempt);
-        const { machine, checkouts } = yield* workerLeftovers(attempt.worker);
+        const located = yield* leftoversOf(attempt);
+        if (Option.isNone(located)) {
+            yield* Effect.logInfo("The Attempt's machine is gone; nothing is left to reconcile", {
+                attemptId: attempt.attemptId,
+                worker: attempt.worker,
+            });
+            yield* session.recoverPostedComments(attempt, Option.none(), record);
+            return;
+        }
+        const { machine, checkouts } = located.value;
         const worktree = `${machine.workerDirectory}/attempts/${safeSegment(attempt.attemptId)}`;
-        const inspectCheckout = Effect.gen(function* () {
-            const inspected = yield* commands.inspect("cleanup", machine.shell, "test", [
-                "-d",
-                worktree,
-            ]);
-            if (inspected.exitCode === 1)
-                return false;
-            if (inspected.exitCode !== 0)
-                return yield* executorError("Can't inspect the recovery checkout", new Error(inspected.stderr));
-            return true;
-        });
+        yield* Effect.addFinalizer((exit) => recovered(attempt, Exit.isSuccess(exit)));
         // Checks the cached listing first to avoid an SSH round trip per persisted Attempt.
-        const present = checkouts.has(worktree) && (yield* inspectCheckout);
+        const present = checkouts.has(worktree) && (yield* checkoutExists(machine, worktree));
+        yield* checkoutHeld(attempt.attemptId, present);
         yield* session.recoverPostedComments(attempt, present
             ? Option.some((spec, windows) => Effect.scoped(Effect.gen(function* () {
                 const grant = yield* worker.grantFor(attempt.repository);
                 const access = yield* grant.acquire(attempt.attemptId, attempt.worker, attempt.permissions, machine, "cleanup");
-                return yield* harvestPostedComments(spec, harvestCommand("cleanup", worktree, access), windows);
+                const checkout = yield* checkoutHostFor("cleanup", yield* Effect.scope, machine, attempt, worktree, access, yield* isolationOfCheckout("cleanup", machine, attempt.attemptId, worktree, yield* worker.isolation(machine)), []);
+                return yield* harvestPostedComments(spec, harvestCommand("cleanup", worktree, access, yield* checkout.shell), windows);
             })))
             : Option.none(), record);
         if (!present)
             return;
-        const reconcile = (git) => recoverWorktree({
-            attempt,
-            host: machine.shell.destination,
-            worktree,
-            cache: `${machine.workerDirectory}/repos/${cacheDirectoryName(attempt.repository)}`,
-            git,
-        }, record.salvage);
-        if (attempt.succeeded || attempt.permissions === "read") {
-            yield* stopChildren("cleanup", machine, worktree);
-            yield* reconcile((args) => commands.run("cleanup", machine.shell, "git", args));
+        yield* reconcileCheckout(attempt, machine, worktree, record.salvage, commitAuthor);
+    }, Effect.scoped);
+    /**
+     * Reconciles the checkout at `worktree` on `machine` that the launch of `request` kept while
+     * its parent task's stop was unconfirmed. Does nothing if the checkout is gone.
+     */
+    const reconcileKept = Effect.fn(span("reconcileKept"))(function* (request, machine, worktree, record) {
+        const fields = { attemptId: request.attemptId, worker: request.worker, worktree };
+        yield* Effect.annotateCurrentSpan(fields);
+        if (!(yield* checkoutExists(machine, worktree))) {
+            yield* Effect.logInfo("The kept checkout is gone; nothing to reconcile", fields);
             return;
         }
-        const grant = yield* worker.grantFor(attempt.repository);
-        yield* Effect.scoped(Effect.gen(function* () {
-            const access = yield* grant.acquire(attempt.attemptId, attempt.worker, attempt.permissions, machine, "cleanup");
-            yield* stopChildren("cleanup", machine, worktree);
-            yield* reconcile(accessGit("cleanup", access));
-        }));
+        yield* reconcileCheckout(keptCheckoutRecovery(request), machine, worktree, record, Effect.succeed(request.commitAuthor));
     });
-    const harvest = (workspace, access, windows) => harvestFacts(workspace.spec, harvestCommand("launch", workspace.worktree, access), windows).pipe(Effect.mapError((error) => new InfraFailure({
+    const harvest = (workspace, access, shell, windows) => harvestFacts(workspace.spec, harvestCommand("launch", workspace.worktree, access, shell), windows).pipe(Effect.mapError((error) => new InfraFailure({
         message: `Can't harvest the attempt's checkout: ${error.message}`,
         cause: error,
     })));
     /**
-     * Checks that a reattached Attempt's parent task was created for this worktree and the
-     * worktree still exists, then stops children left from before the restart. Returns the
-     * parent's RunSpec.
+     * Checks that a reattached Attempt's parent task was created for this worktree, the worktree
+     * still exists, and `isolation` can run commands in it, then stops children left from before
+     * the restart. Returns the parent's RunSpec.
      */
-    const takeOver = Effect.fn(span("takeOver"))(function* (request, machine, worktree) {
+    const takeOver = Effect.fn(span("takeOver"))(function* (request, machine, worktree, isolation) {
         const { cwd, spec } = yield* session.storedRequest(request.attemptId);
         if (cwd !== worktree)
             return yield* executorError(`The reattached attempt's checkout isn't on the machine of this ${label} worker`, new Error(cwd));
@@ -129,37 +302,85 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
             return yield* executorError("The reattached attempt's checkout no longer exists", new Error(worktree));
         if (inspected.exitCode !== 0)
             return yield* executorError("Can't inspect the reattached attempt's checkout", new Error(inspected.stderr.trim()));
-        yield* stopChildren("prepare", machine, worktree);
+        yield* isolationOfCheckout("prepare", machine, request.attemptId, worktree, isolation);
+        yield* stopChildren("prepare", machine, worktree, request.attemptId, isolation);
         return spec;
     });
+    /**
+     * Returns the checkout operations of an Attempt in its own container: a standalone repository
+     * that borrows the cached clone's objects, with every git command run in the container.
+     */
+    const containerCheckout = (request, machine, access, worktree, cache, url, checkout) => {
+        const inContainer = (operation, command, args) => access.refresh.pipe(
+        // The only cleanup command in the checkout is the salvage's git.
+        Effect.andThen(operation === "cleanup" ? checkout.salvageShell : checkout.shell), Effect.flatMap((shell) => commands.run(operation, shell, command, args, {
+            environment: { ...access.gitEnvironment, ...commitIdentity(request.commitAuthor) },
+            scrub: access.scrub,
+        })));
+        return {
+            add: (commit) => commands
+                .run("prepare", machine.shell, "mkdir", ["-p", worktree])
+                .pipe(Effect.andThen(inContainer("prepare", "sh", [
+                "-c",
+                STANDALONE_CHECKOUT_SCRIPT,
+                "sh",
+                worktree,
+                cache,
+                url,
+                commit,
+            ])), Effect.asVoid),
+            // The container mounts the directory itself, so only its contents go.
+            discard: Effect.asVoid(commands.run("prepare", machine.shell, "sh", [
+                "-c",
+                EMPTY_DIRECTORY_SCRIPT,
+                "sh",
+                worktree,
+            ])),
+            git: (operation, args) => inContainer(operation, "git", args),
+            writeExecutable: (target, content) => Effect.asVoid(inContainer("prepare", "sh", ["-c", WRITE_EXECUTABLE_SCRIPT, "sh", target, content])),
+            remove: (cleanup) => cleanup("worktree-directory", removing(request.attemptId, removeStandalone("cleanup", machine, request.attemptId, worktree))),
+        };
+    };
     /** Returns the `WorkspaceOps` for the machine; git runs with the Attempt's repository access. */
-    const workspaceOps = (request, machine, access, layout) => ({
-        ensureRoots: Effect.asVoid(commands.run("prepare", machine.shell, "mkdir", [
-            "-p",
-            `${machine.workerDirectory}/repos`,
-            `${machine.workerDirectory}/attempts`,
-        ])),
-        removePath: (operation, target) => Effect.asVoid(commands.run(operation, machine.shell, "rm", ["-rf", target])),
-        cacheExists: Effect.gen(function* () {
-            const inspected = yield* commands.inspect("prepare", machine.shell, "test", [
-                "-d",
-                layout.cache,
-            ]);
-            if (inspected.exitCode === 0)
-                return true;
-            if (inspected.exitCode === 1)
-                return false;
-            return yield* executorError(`Can't inspect the cached clone on worker "${request.worker}"`, new Error(inspected.stderr.trim()));
-        }),
-        promoteStaging: Effect.asVoid(commands.run("prepare", machine.shell, "mv", [layout.staging, layout.cache])),
-        git: (operation, args) => accessGit(operation, access)(args),
-        writeExecutable: (target, content) => Effect.asVoid(commands.run("prepare", machine.shell, "sh", [
-            "-c",
-            WRITE_EXECUTABLE_SCRIPT,
-            "sh",
-            target,
-            content,
-        ])),
+    const workspaceOps = (request, machine, access, layout, checkout) => {
+        const host = {
+            removePath: (operation, target) => Effect.asVoid(commands.run(operation, machine.shell, "rm", ["-rf", target])),
+            cacheExists: Effect.gen(function* () {
+                const inspected = yield* commands.inspect("prepare", machine.shell, "test", [
+                    "-d",
+                    layout.cache,
+                ]);
+                if (inspected.exitCode === 0)
+                    return true;
+                if (inspected.exitCode === 1)
+                    return false;
+                return yield* executorError(`Can't inspect the cached clone on worker "${request.worker}"`, new Error(inspected.stderr.trim()));
+            }),
+            git: (operation, args) => accessGit(operation, access, request.commitAuthor)(args),
+        };
+        return {
+            ...host,
+            ensureRoots: Effect.asVoid(commands.run("prepare", machine.shell, "mkdir", [
+                "-p",
+                `${machine.workerDirectory}/repos`,
+                `${machine.workerDirectory}/attempts`,
+            ])),
+            promoteStaging: Effect.asVoid(commands.run("prepare", machine.shell, "mv", [layout.staging, layout.cache])),
+            checkout: checkout.isolation._tag === "Container"
+                ? containerCheckout(request, machine, access, layout.worktree, layout.cache, worker.repositoryUrl(machine, request.repository), checkout)
+                : linkedCheckout(request.attemptId, linkedWorktree(host, layout, (target, content) => Effect.asVoid(commands.run("prepare", machine.shell, "sh", [
+                    "-c",
+                    WRITE_EXECUTABLE_SCRIPT,
+                    "sh",
+                    target,
+                    content,
+                ])))),
+        };
+    };
+    /** Returns `ops` whose removal records the checkout as gone once its directory is removed. */
+    const linkedCheckout = (attemptId, ops) => ({
+        ...ops,
+        remove: (cleanup) => ops.remove((resource, effect) => cleanup(resource, resource === "worktree-directory" ? removing(attemptId, effect) : effect)),
     });
     const prepare = Effect.fn(span("prepare"))(function* (request) {
         yield* Effect.annotateCurrentSpan({
@@ -187,13 +408,17 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
             });
         }
         const grant = yield* worker.grantFor(request.repository);
-        const machine = yield* worker.machineFor(request.worker);
-        yield* verifyHarnessReadiness(request.worker, [request.agent.brain], worker.harnessProbe(machine), logins);
+        const machine = yield* attemptMachine(request.worker, request.attemptId);
+        const isolation = yield* worker.isolation(machine);
+        const probeBrain = (shell) => verifyHarnessReadiness(request.worker, machine.provisioned, [request.agent.brain], worker.harnessProbe(machine, shell), logins);
+        // An Attempt container doesn't exist yet, so its probe runs once the checkout is prepared.
+        if (isolation._tag === "SharedUser")
+            yield* probeBrain(machine.shell);
         const host = machine.shell.destination;
-        const cache = `${machine.workerDirectory}/repos/${cacheDirectoryName(request.repository)}`;
+        const cache = cachePath(machine, request.repository);
         const worktree = `${machine.workerDirectory}/attempts/${safeSegment(request.attemptId)}`;
         const staging = `${cache}.cloning`;
-        yield* Effect.annotateCurrentSpan({ host });
+        yield* Effect.annotateCurrentSpan({ host, isolation: isolation._tag });
         yield* Effect.logInfo(request.reattach
             ? `Reattaching the Attempt to its ${label} checkout`
             : `Preparing ${label} Attempt workspace`, {
@@ -203,13 +428,18 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
             repository: request.repository,
             loop: request.loop,
             permissions: request.permissions,
+            isolation: isolation._tag,
             worktree,
         });
         yield* sweepAbandonedWorktrees(request.attemptId, request.worker, machine.shell, machine.workerDirectory);
         // A reattachment reuses its checkout before any credential is granted.
         const reattached = request.reattach
-            ? Option.some(yield* takeOver(request, machine, worktree))
+            ? Option.some(yield* takeOver(request, machine, worktree, isolation))
             : Option.none();
+        const harness = Option.match(reattached, {
+            onNone: () => request.agent.brain.harness,
+            onSome: (spec) => spec.agent.brain.harness,
+        });
         // A checkout kept after a failed remote stop never closes its scope, so its salvage result
         // is completed here instead of by the checkout finalizer.
         const salvaged = yield* Deferred.make();
@@ -220,7 +450,7 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
                 host,
             });
             // Closes the checkout scope only if stopping the remote children succeeds.
-            yield* stopChildren("cleanup", machine, resource.provisioned.workspace.worktree);
+            yield* stopChildren("cleanup", machine, resource.provisioned.workspace.worktree, request.attemptId, isolation);
             yield* Scope.close(resource.scope, exit);
             const salvage = yield* Effect.exit(resource.provisioned.salvaged);
             yield* Deferred.done(salvaged, salvage);
@@ -231,11 +461,16 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
             worktree,
             message: error.message,
         })), Effect.ignore, Effect.andThen(Deferred.succeed(salvaged, Option.none()))));
+        const prepareScope = yield* Effect.scope;
         const acquire = Effect.gen(function* () {
             // Credentials are scoped to the Attempt, not the checkout, so they are released exactly once
             // even when checkout cleanup is skipped.
             const access = yield* grant.acquire(request.attemptId, request.worker, request.permissions, machine, "prepare");
-            const ops = workspaceOps(request, machine, access, { cache, staging });
+            // Forked before the checkout's release is registered, so the container is removed after
+            // the release salvages the checkout in it.
+            const containerScope = yield* Scope.fork(prepareScope);
+            const checkoutHost = yield* checkoutHostFor("prepare", containerScope, machine, request, worktree, access, isolation, [harness]);
+            const ops = workspaceOps(request, machine, access, { cache, staging, worktree }, checkoutHost);
             const checkout = Option.match(reattached, {
                 onNone: () => provisionWorktree({
                     request,
@@ -244,7 +479,7 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
                     layout: { cache, staging, worktree },
                     ops,
                 }),
-                onSome: (spec) => reattachWorktree({ request, host, layout: { cache, worktree }, ops, spec }),
+                onSome: (spec) => reattachWorktree({ request, host, layout: { worktree }, ops, spec }),
             });
             const { provisioned } = yield* Effect.acquireRelease(Effect.uninterruptibleMask((restore) => Effect.gen(function* () {
                 const scope = yield* Scope.make();
@@ -253,11 +488,14 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
                 const provisioned = yield* restore(checkout).pipe(Scope.provide(scope), Effect.onError((cause) => Scope.close(scope, Exit.failCause(cause))));
                 return { provisioned, scope };
             })), releaseWorkspace, { interruptible: true });
-            return { provisioned, access };
+            return { provisioned, access, checkoutHost };
         });
-        const { provisioned, access } = yield* (request.reattach ? acquire : fetches.withLock(request.repository)(acquire)).pipe(Effect.mapError((cause) => isExecutorError(cause)
+        const { provisioned, access, checkoutHost } = yield* (request.reattach ? acquire : fetches.withLock(request.repository)(acquire)).pipe(Effect.mapError((cause) => isExecutorError(cause)
             ? cause
             : executorError(`Can't prepare the ${label} attempt workspace`, cause)));
+        yield* checkoutHeld(request.attemptId, true);
+        if (isolation._tag === "Container")
+            yield* probeBrain(yield* checkoutHost.shell);
         const prepared = provisioned.workspace;
         yield* Effect.logInfo(`${label} Attempt workspace ready`, {
             attemptId: request.attemptId,
@@ -267,6 +505,7 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
             baseCommit: prepared.spec.baseCommit,
             resume: Option.map(prepared.spec.resume, ({ ref }) => ref),
             reattach: request.reattach,
+            isolation: isolation._tag,
             worktree,
         });
         return {
@@ -275,6 +514,11 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
                 // Uses the RunSpec's brain, including on reattachment. Each child reads the kind when it
                 // starts, so a preflight during the Attempt takes effect.
                 const gatewayCredential = worker.gatewayCredential(machine, prepared.spec.agent.brain.harness);
+                yield* checkoutHost.recordOutOfMemory(io.outOfMemory);
+                const shell = yield* checkoutHost.shell.pipe(Effect.mapError((error) => new InfraFailure({
+                    message: `Can't start the attempt's container: ${error.message}`,
+                    cause: error,
+                })));
                 yield* workerWatches.watch({
                     attemptId: request.attemptId,
                     worker: request.worker,
@@ -282,12 +526,15 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
                     report: (notice) => runtime.notify(request.attemptId, notice),
                 });
                 return yield* session.launch(prepared, io, {
-                    host: access.shell,
-                    environment: withGitHooks(access.environment, provisioned.gitHooks),
+                    host: shell,
+                    machine: machine.provisioned,
+                    environment: childEnvironment(access.environment, request.commitAuthor, provisioned.gitHooks),
                     gatewayCredential,
-                    harvest: (windows) => harvest(prepared, access, windows),
-                    harvestComments: (windows) => harvestPostedComments(prepared.spec, harvestCommand("launch", prepared.worktree, access), windows),
+                    harvest: (windows) => harvest(prepared, access, shell, windows),
+                    harvestComments: (windows) => harvestPostedComments(prepared.spec, harvestCommand("launch", prepared.worktree, access, shell), windows),
                     keepForReattach: provisioned.keepForReattach,
+                    keepUntilStopped: provisioned.keepUntilStopped,
+                    reconcileKept: (record) => reconcileKept(request, machine, worktree, record),
                 });
             })),
             salvaged: Deferred.await(salvaged),
@@ -303,6 +550,7 @@ export const remoteExecutorFor = (worker) => Effect.gen(function* () {
         outcome: session.outcome,
         preflight: worker.preflight,
         prepare,
+        sweep: worker.sweep ?? Effect.void,
     });
 });
 //# sourceMappingURL=remote.js.map