@usine/orchestrator 0.3.0 → 0.4.0

package/dist/scheduler.js

@@ -1,26 +1,27 @@
 import { resolveParentEffort } from "@usine/core/agent";
 import { AgentFailure, InfraFailure } from "@usine/core/attempt/failure";
 import { salvageRefFor } from "@usine/core/attempt/salvage";
+import { canonicalRepositoryName } from "@usine/core/repository";
 import { BotLogin, WorkerName } from "@usine/core/worker";
-import { CheckoutRequest, Executor, ExecutorError, ExecutorOperationError } from "@usine/executor";
+import { CheckoutRequest, Executor, ExecutorError, ExecutorOperationError, fullPreflight, } from "@usine/executor";
 import { StepExecutor } from "@usine/engine/step-executor";
-import { GitHubApp } from "@usine/github/github-app";
-import { Array as EffectArray, Context, DateTime, Deferred, Effect, Exit, Layer, Match, Option, Order, Ref, Schema, Semaphore, TxPriorityQueue, TxRef, } from "effect";
+import { GitHubApp, isRefusal } from "@usine/github/github-app";
+import { Array as EffectArray, Context, DateTime, Deferred, Duration, Effect, Exit, FiberMap, Layer, Match, Option, Ref, Result, Schema, Semaphore, TxPriorityQueue, TxRef, } from "effect";
 import { agentsWithNewBrains, workersChange, } from "./current-instance/changes.js";
 import { CurrentInstance, reloadPreflightFailed, restartRequired, } from "./current-instance/current-instance.js";
-import { recordPreflight, unchecked, WorkerStatus } from "./worker-status.js";
+import { payloadLabels } from "./board/items.js";
+import { aheadCount, leaseOrder, passedCount, pipelinePosition, } from "./lease-order.js";
+import { recordPreflight, unchecked, WorkerStatus, } from "./worker-status.js";
 export class UnsupportedWorkerError extends Schema.TaggedError("@usine/orchestrator/UnsupportedWorkerError")("UnsupportedWorkerError", {
     worker: WorkerName,
     kind: Schema.String,
 }) {
 }
-// Reservations for a specific Worker (Attempts already running there) go first; otherwise, the Run
-// whose Event arrived first goes first.
-const reservationOrder = Order.combineAll([
-    Order.mapInput(Order.Boolean, (reservation) => Option.isNone(reservation.worker)),
-    Order.mapInput(Order.Number, (reservation) => reservation.eventTime),
-    Order.mapInput(Order.Number, (reservation) => reservation.order),
-]);
+const positionFields = (position) => Match.valueTags(position, {
+    Priority: ({ priority }) => ({ positionSource: "priority", position: priority }),
+    Lane: ({ lane, number }) => ({ positionSource: "lane", position: number, lane }),
+    Unpositioned: () => ({ positionSource: "none" }),
+});
 const slotPoolOf = (workers, previous) => {
     const declared = new Set(workers.map(({ name }) => name));
     return new Map([
@@ -33,18 +34,36 @@ const slotPoolOf = (workers, previous) => {
             .map(([name, { leased }]) => [name, { capacity: 0, leased }]),
     ]);
 };
+/** Returns the Workers that take new Attempts: those `usine.yaml` declares with slots. */
+const openWorkers = (pool) => [...pool].filter(([, { capacity }]) => capacity > 0).map(([worker]) => worker);
+/** Returns the Runs of `queued`, which is in lease order, with the Runs ahead of each. */
+const queuedRuns = (queued, pool) => {
+    const open = openWorkers(pool);
+    return queued.map((reservation, index) => ({
+        runId: reservation.runId,
+        loop: reservation.loop,
+        itemKey: reservation.itemKey,
+        position: reservation.position,
+        ahead: aheadCount(queued.slice(0, index), reservation, open),
+    }));
+};
 const isFree = (pool, worker) => {
     const slots = pool.get(worker);
     return slots !== undefined && slots.leased < slots.capacity;
 };
 /**
  * Returns the Worker with a free slot for `reservation`: its specified Worker, or else the first
- * free one.
+ * free one that isn't `paused` from creating machines.
  */
-const workerFor = (pool, reservation) => Option.match(reservation.worker, {
-    onNone: () => Option.fromNullishOr([...pool.keys()].find((worker) => isFree(pool, worker))),
+const workerFor = (pool, paused, reservation) => Option.match(reservation.worker, {
+    onNone: () => Option.fromNullishOr([...pool.keys()].find((worker) => isFree(pool, worker) && paused.get(worker)?.blocked !== true)),
     onSome: (worker) => (isFree(pool, worker) ? Option.some(worker) : Option.none()),
 });
+/** The first pause after a machine creation fails on a Worker. It doubles up to `MAX_PAUSE`. */
+const FIRST_PAUSE = Duration.seconds(30);
+const MAX_PAUSE = Duration.minutes(10);
+/** Returns the pause after the `failures`th consecutive failed machine creation. */
+const pauseAfter = (failures) => Duration.min(Duration.times(FIRST_PAUSE, 2 ** Math.min(failures - 1, 10)), MAX_PAUSE);
 const withLeased = (pool, worker, change) => {
     const slots = pool.get(worker);
     if (slots === undefined)
@@ -90,13 +109,18 @@ const validateWorkers = (kind, workers) => {
 const brainLabel = ({ harness, model, effort }) => `${harness}/${model}/${effort}`;
 /** The login that exe.dev Workers write to GitHub as, through exe.dev's integration (ADR 0011). */
 const EXE_DEV_BOT_LOGIN = BotLogin.make("exe-dev-github-integration[bot]");
-const decodeBotLogin = Schema.decodeUnknownOption(BotLogin);
+/** Returns the Attempt's failure for an unreadable bot account. A refusal isn't retried. */
+const botAccountFailure = (error) => {
+    const message = `Can't read the GitHub App's bot account, which authors the attempt's commits: ${error.message}`;
+    return isRefusal(error)
+        ? new AgentFailure({ message })
+        : new InfraFailure({ message, cause: error });
+};
 /** The Worker set of an executor that runs any Worker of its kind by name. */
 const anyWorker = { add: () => Effect.void, remove: () => Effect.void };
 /**
- * Schedules Attempts FIFO over the Workers and adapts the Executor to the engine's `StepExecutor`
- * port. Reservations for a specific Worker go first (#177), and a removed Worker stays until its
- * last Attempt ends.
+ * Schedules Attempts over the Workers in `leaseOrder` and adapts the Executor to the engine's
+ * `StepExecutor` port. A removed Worker stays until its last Attempt ends.
  */
 export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectContext(Effect.gen(function* () {
     const executor = yield* Executor;
@@ -104,7 +128,7 @@ export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectC
     const app = yield* GitHubApp;
     const workers = yield* validateWorkers(executor.kind, config.workers);
     const pool = yield* TxRef.make(slotPoolOf(workers, new Map()));
-    const pending = yield* TxPriorityQueue.empty(reservationOrder);
+    const pending = yield* TxPriorityQueue.empty(leaseOrder);
     const nextOrder = yield* Ref.make(0);
     const preflights = yield* Ref.make(new Map());
     const executorWorkers = options.workers ?? anyWorker;
@@ -112,13 +136,22 @@ export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectC
     // addition.
     const membership = yield* Semaphore.make(1);
     const adding = yield* Ref.make(new Set());
+    const machines = options.machines;
+    const paused = yield* TxRef.make(new Map());
+    // The machine of each Run's lease, which `execute` gives to the Run's Attempt.
+    const runMachines = yield* Ref.make(new Map());
+    // The machine creation of each reservation, by its order.
+    const creations = yield* FiberMap.make();
+    const provisions = (worker) => machines === undefined ? Effect.succeed(false) : machines.provisions(worker);
     const recordPreflightExit = Effect.fn("WorkerScheduler.recordPreflight")(function* (worker, roster, exit) {
-        const failure = Exit.findErrorOption(exit);
+        const outcome = Exit.isSuccess(exit)
+            ? Option.some(Result.succeed(exit.value))
+            : Option.map(Exit.findErrorOption(exit), (error) => Result.fail(error));
         // An interrupted or defective preflight gives no information about the Worker.
-        if (Exit.isFailure(exit) && Option.isNone(failure))
+        if (Option.isNone(outcome))
             return;
         const checkedAt = yield* DateTime.now;
-        yield* Ref.update(preflights, (current) => new Map(current).set(worker, recordPreflight(current.get(worker) ?? unchecked(worker), roster, checkedAt, failure)));
+        yield* Ref.update(preflights, (current) => new Map(current).set(worker, recordPreflight(current.get(worker) ?? unchecked(worker), roster, checkedAt, outcome.value)));
     });
     /**
      * Runs `before`, then the Worker's preflight, and records both outcomes as the Worker's
@@ -230,42 +263,193 @@ export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectC
         Retired: ({ running, slots }) => Effect.logInfo("An Attempt ended on a Worker that runs more Attempts than it has slots; its slot retires", { worker, running, slots }),
         Dropped: () => Effect.logInfo("The last Attempt on a removed Worker ended; the Worker leaves the scheduler", { worker }).pipe(Effect.andThen(membership.withPermit(removeUnused(worker)))),
     })));
-    const makeLease = Effect.fn("WorkerScheduler.makeLease")(function* (worker) {
+    /** Deletes the machine that `machine` holds, if any, before the slot frees. */
+    const releaseMachine = (worker, machine) => Ref.getAndSet(machine, Option.none()).pipe(Effect.flatMap((hold) => Option.match(hold, {
+        onNone: () => Effect.void,
+        onSome: (held) => machines === undefined
+            ? Effect.void
+            : Effect.logDebug("Deleting the machine of the released slot", {
+                worker,
+                hold: held,
+            }).pipe(Effect.andThen(machines.release(held))),
+    })));
+    const makeLease = Effect.fn("WorkerScheduler.makeLease")(function* (worker, hold = Option.none(), runId = Option.none()) {
         const released = yield* Ref.make(false);
+        const machine = yield* Ref.make(hold);
+        if (Option.isSome(runId))
+            yield* Ref.update(runMachines, (current) => new Map(current).set(runId.value, machine));
+        const forget = Option.match(runId, {
+            onNone: () => Effect.void,
+            onSome: (run) => Ref.update(runMachines, (current) => {
+                if (current.get(run) !== machine)
+                    return current;
+                const next = new Map(current);
+                next.delete(run);
+                return next;
+            }),
+        });
         return {
             worker,
-            release: Ref.modify(released, (done) => [!done, true]).pipe(Effect.flatMap((first) => (first ? releaseSlot(worker) : Effect.void))),
+            release: Ref.modify(released, (done) => [!done, true]).pipe(Effect.flatMap((first) => first
+                ? forget.pipe(Effect.andThen(releaseMachine(worker, machine)), Effect.andThen(releaseSlot(worker)))
+                : Effect.void)),
         };
     });
     // Retries until a free slot can take a waiting reservation, then takes both. A reservation
     // whose Worker has no free slot doesn't block the ones behind it.
     const nextAssignment = Effect.tx(Effect.gen(function* () {
         const current = yield* TxRef.get(pool);
-        const next = EffectArray.findFirst(yield* TxPriorityQueue.toArray(pending), (reservation) => Option.map(workerFor(current, reservation), (worker) => ({ worker, reservation })));
+        const pauses = yield* TxRef.get(paused);
+        const waiting = yield* TxPriorityQueue.toArray(pending);
+        const next = EffectArray.findFirst(waiting, (reservation) => Option.map(workerFor(current, pauses, reservation), (worker) => ({
+            worker,
+            reservation,
+        })));
         if (Option.isNone(next))
             return yield* Effect.txRetry;
         const { worker, reservation } = next.value;
-        yield* TxPriorityQueue.removeIf(pending, (waiting) => waiting === reservation);
+        yield* TxPriorityQueue.removeIf(pending, (queued) => queued === reservation);
         yield* TxRef.set(pool, withLeased(current, worker, 1));
-        return next.value;
+        return { worker, reservation, passed: passedCount(waiting, reservation) };
     }));
+    /** Pauses machine creation on `worker` after `failure`, and resumes it when the pause ends. */
+    const pauseWorker = Effect.fn("WorkerScheduler.pauseWorker")(function* (worker, failure) {
+        const failedAt = yield* DateTime.now;
+        const pause = yield* Effect.tx(TxRef.modify(paused, (current) => {
+            const failures = (current.get(worker)?.failures ?? 0) + 1;
+            const delay = pauseAfter(failures);
+            const next = {
+                worker,
+                reason: failure._tag === "MachineUnavailableError" ? failure.reason : "check",
+                message: failure.message,
+                failures,
+                failedAt,
+                retryAt: DateTime.addDuration(failedAt, delay),
+                blocked: true,
+            };
+            return [next, new Map(current).set(worker, next)];
+        }));
+        yield* Effect.logWarning("A provisioned Worker could not create a machine; its Runs keep waiting, and it tries again after a pause", {
+            worker,
+            reason: pause.reason,
+            message: pause.message,
+            failures: pause.failures,
+            retryAt: DateTime.formatIso(pause.retryAt),
+        });
+        yield* Effect.sleep(DateTime.distance(failedAt, pause.retryAt)).pipe(Effect.andThen(Effect.tx(TxRef.update(paused, (current) => {
+            const latest = current.get(worker);
+            return latest === undefined || latest.failures !== pause.failures
+                ? current
+                : new Map(current).set(worker, { ...latest, blocked: false });
+        }))), Effect.andThen(Effect.logInfo("A provisioned Worker's pause ended; it creates machines again", {
+            worker,
+            failures: pause.failures,
+        })), Effect.forkScoped);
+    });
+    /**
+     * Creates the machine of `reservation`'s next Attempt on `worker`, whose slot it holds, and
+     * then assigns the lease with the machine. On failure, frees the slot, pauses the Worker,
+     * and puts the reservation back in the queue, so the Run keeps waiting.
+     */
+    const provision = Effect.fn("WorkerScheduler.provision")(function* (worker, reservation, passed) {
+        const fields = {
+            runId: reservation.runId,
+            loop: reservation.loop,
+            itemKey: reservation.itemKey,
+            worker,
+        };
+        yield* Effect.annotateCurrentSpan(fields);
+        if (machines === undefined)
+            return yield* Effect.die(new Error("no machines to create"));
+        const { agents } = yield* instances.get;
+        const roster = EffectArray.dedupe(agents.map(({ agent }) => agent.brain));
+        yield* Effect.logInfo("Worker slot taken in lease order; creating the Attempt's machine", {
+            ...fields,
+            ...positionFields(reservation.position),
+            passed,
+        });
+        const created = yield* machines.create(worker, reservation.runId, roster).pipe(Effect.result, Effect.onInterrupt(() => Effect.logInfo("The Run stopped waiting while its machine was created; the slot is free again", fields).pipe(Effect.andThen(releaseSlot(worker)))));
+        yield* Effect.uninterruptible(Effect.gen(function* () {
+            if (Result.isFailure(created)) {
+                const failure = created.failure;
+                yield* releaseSlot(worker);
+                if (failure._tag === "WorkerCapabilityError")
+                    yield* recordPreflightExit(worker, roster, Exit.fail(failure));
+                yield* pauseWorker(worker, failure);
+                const requeued = yield* Ref.modify(reservation.state, (state) => state._tag === "Creating"
+                    ? [true, { _tag: "Pending" }]
+                    : [false, state]);
+                if (requeued)
+                    yield* Effect.tx(TxPriorityQueue.offer(pending, reservation));
+                return;
+            }
+            const grant = created.success;
+            yield* Effect.tx(TxRef.update(paused, (current) => {
+                if (!current.has(worker))
+                    return current;
+                const next = new Map(current);
+                next.delete(worker);
+                return next;
+            }));
+            if (Option.isSome(grant.checked)) {
+                const { brains, containerLimits } = grant.checked.value;
+                yield* recordPreflightExit(worker, brains, Exit.succeed({ ...fullPreflight, containerLimits }));
+            }
+            const lease = yield* makeLease(worker, Option.some(grant.hold), Option.some(reservation.runId));
+            const assign = yield* Ref.modify(reservation.state, (state) => state._tag === "Creating"
+                ? [true, { _tag: "Assigned", lease }]
+                : [false, state]);
+            if (!assign) {
+                yield* Effect.logInfo("The Run stopped waiting while its machine was created; the machine is deleted and the slot is free again", { ...fields, machine: grant.id });
+                return yield* lease.release;
+            }
+            yield* Effect.logInfo("Worker slot leased with a new machine", {
+                ...fields,
+                machine: grant.id,
+                provider: grant.provider,
+            });
+            yield* Deferred.succeed(reservation.deferred, lease);
+        }));
+    });
     const dispatch = Effect.forever(Effect.gen(function* () {
-        const { worker, reservation } = yield* nextAssignment;
-        const lease = yield* makeLease(worker);
+        const { worker, reservation, passed } = yield* nextAssignment;
+        // A reattached Attempt keeps its machine, so only a new Attempt gets one.
+        if (Option.isNone(reservation.worker) && (yield* provisions(worker))) {
+            const creating = yield* Ref.modify(reservation.state, (state) => state._tag === "Pending"
+                ? [true, { _tag: "Creating", worker }]
+                : [false, state]);
+            if (creating)
+                yield* FiberMap.run(creations, reservation.order, provision(worker, reservation, passed));
+            else
+                yield* releaseSlot(worker);
+            return;
+        }
+        const lease = yield* makeLease(worker, Option.none(), Option.some(reservation.runId));
         const assign = yield* Ref.modify(reservation.state, (state) => state._tag === "Pending"
             ? [true, { _tag: "Assigned", lease }]
             : [false, state]);
+        const fields = {
+            runId: reservation.runId,
+            loop: reservation.loop,
+            itemKey: reservation.itemKey,
+            worker,
+        };
         if (assign) {
-            yield* Effect.logDebug("Worker slot assigned", { worker });
+            yield* Effect.logInfo("Worker slot leased in lease order", {
+                ...fields,
+                ...positionFields(reservation.position),
+                passed,
+            });
             yield* Deferred.succeed(reservation.deferred, lease);
         }
         else {
+            yield* Effect.logDebug("The Run stopped waiting before its lease; the Worker slot is free again", fields);
             yield* lease.release;
         }
     }));
     yield* Effect.forkScoped(dispatch);
-    const occupy = Effect.fn("WorkerScheduler.occupy")(function* (worker) {
-        yield* Effect.annotateCurrentSpan({ worker });
+    const occupy = Effect.fn("WorkerScheduler.occupy")(function* (worker, attemptId) {
+        yield* Effect.annotateCurrentSpan({ worker, attemptId });
         // The Attempt runs on the Worker even without a free slot, so count it now. A slot over
         // capacity retires when an Attempt there ends.
         const slots = yield* TxRef.modify(pool, (current) => [current.get(worker), withLeased(current, worker, 1)]);
@@ -278,51 +462,117 @@ export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectC
             yield* Effect.logInfo("An Attempt that already runs on the Worker occupies a slot", fields);
         else
             yield* Effect.logWarning("An Attempt that already runs on the Worker occupies a slot beyond its max_concurrent_attempts; the Worker takes no new Attempt until enough of its Attempts end", fields);
-        return yield* makeLease(worker);
+        const hold = machines !== undefined && (yield* provisions(worker))
+            ? yield* machines.adopt(worker, attemptId)
+            : Option.none();
+        return yield* makeLease(worker, hold);
     });
-    const reserve = Effect.fn("WorkerScheduler.reserve")(function* (submission, worker) {
-        const run = { loop: submission.loop, itemKey: submission.itemKey };
+    /**
+     * Returns the pipeline position of the Run of `submission`, from the current Loop file and
+     * the lanes of the Run's repository.
+     */
+    const positionOf = Effect.fn("WorkerScheduler.position")(function* (runId, submission) {
+        yield* Effect.annotateCurrentSpan({ runId, loop: submission.loop });
+        const { config, loops } = yield* instances.get;
+        const loop = loops.find(({ name }) => name === submission.loop);
+        const priority = Option.flatMap(Option.fromNullishOr(loop), ({ priority }) => priority);
+        const lanes = config.repos.find(({ name }) => canonicalRepositoryName(name) === canonicalRepositoryName(submission.repository))?.lanes ?? [];
+        const stream = loop?.trigger.on;
+        // Only issue and pull request Events record Item labels.
+        if (Option.isSome(priority) ||
+            lanes.length === 0 ||
+            stream === undefined ||
+            stream === "schedule") {
+            return pipelinePosition(priority, lanes, []);
+        }
+        const labels = yield* payloadLabels(stream, submission.event.payload).pipe(Effect.catch((error) => Effect.logWarning("The Run's Event has no labels to read; the Run waits for a worker slot without a lane position", {
+            runId,
+            loop: submission.loop,
+            eventId: submission.event.id,
+            errorTag: error._tag,
+            message: error.message,
+        }).pipe(Effect.as([]))));
+        return pipelinePosition(priority, lanes, labels);
+    });
+    const reserve = Effect.fn("WorkerScheduler.reserve")(function* (runId, submission, worker) {
+        const run = { runId, loop: submission.loop, itemKey: submission.itemKey };
         yield* Effect.annotateCurrentSpan(run);
         if (Option.isSome(worker))
             yield* Effect.annotateCurrentSpan({ worker: worker.value });
         const deferred = yield* Deferred.make();
         const state = yield* Ref.make({ _tag: "Pending" });
         const reservation = {
+            ...run,
             deferred,
             worker,
+            position: yield* positionOf(runId, submission),
             eventTime: DateTime.toEpochMillis(submission.event.receivedAt),
             order: yield* Ref.getAndUpdate(nextOrder, (value) => value + 1),
             state,
         };
-        yield* TxPriorityQueue.offer(pending, reservation);
-        const fields = { ...run, queued: yield* TxPriorityQueue.size(pending) };
+        const [queue, current] = yield* Effect.tx(TxPriorityQueue.offer(pending, reservation).pipe(Effect.andThen(Effect.all([TxPriorityQueue.toArray(pending), TxRef.get(pool)]))));
+        const fields = {
+            ...run,
+            ...positionFields(reservation.position),
+            queued: queue.length,
+            ahead: aheadCount(queue.slice(0, queue.indexOf(reservation)), reservation, openWorkers(current)),
+        };
         if (Option.isNone(worker))
             yield* Effect.logInfo("Run waits for a worker slot", fields);
-        else if (((yield* TxRef.get(pool)).get(worker.value)?.capacity ?? 0) > 0)
+        else if ((current.get(worker.value)?.capacity ?? 0) > 0)
             yield* Effect.logInfo("Run waits for a slot on the Worker its reattached Attempt runs on", { ...fields, worker: worker.value });
         else
             yield* Effect.logWarning("Run waits for a slot on the Worker its reattached Attempt runs on, which usine.yaml does not declare; it waits until a reload declares that Worker again or the Run is canceled", { ...fields, worker: worker.value });
         return yield* Deferred.await(deferred).pipe(Effect.onInterrupt(() => Ref.modify(state, (current) => current._tag === "Assigned"
             ? [Option.some(current.lease), { _tag: "Canceled" }]
             : [Option.none(), { _tag: "Canceled" }]).pipe(
+        // Stops a machine creation for the Run, which deletes the machine and frees the slot.
+        Effect.tap(() => FiberMap.remove(creations, reservation.order)), 
         // Remove a canceled reservation from the queue, so one whose Worker never frees a
         // slot doesn't stay there.
         Effect.tap(() => TxPriorityQueue.removeIf(pending, (waiting) => waiting === reservation)), Effect.flatMap(Option.match({ onNone: () => Effect.void, onSome: (lease) => lease.release })))));
     });
-    // If the login can't be read, harvesting asks GitHub; the Attempt never fails for this.
-    const botLoginFor = Effect.fn("WorkerScheduler.botLogin")(function* (attemptId) {
-        if (executor.kind === "exe.dev")
-            return Option.some(EXE_DEV_BOT_LOGIN);
-        const installation = yield* app.installation().pipe(Effect.map(Option.some), Effect.catch((error) => Effect.logWarning("Could not read the App's installation; harvesting will ask GitHub for the login of the Attempt's credential", { attemptId, errorTag: error._tag, message: error.message }).pipe(Effect.as(Option.none()))));
-        if (Option.isNone(installation))
-            return Option.none();
-        const { appSlug } = installation.value;
-        // GitHub's REST API names an App's bot `<slug>[bot]`; its GraphQL API drops the suffix.
-        const login = decodeBotLogin(`${appSlug}[bot]`);
-        if (Option.isNone(login))
-            yield* Effect.logWarning("The App's slug does not form a bot login; harvesting will ask GitHub for the login of the Attempt's credential", { attemptId, appSlug });
-        return login;
+    /**
+     * Gives a provisioned Worker's machine to the Attempt of `execution`: the machine its Run's
+     * lease created, or for a reattachment the machine it ran on before the restart.
+     */
+    const machineForAttempt = Effect.fn("WorkerScheduler.machineForAttempt")(function* (execution) {
+        const worker = execution.attempt.worker;
+        if (machines === undefined || !(yield* provisions(worker)))
+            return;
+        const attemptId = execution.attempt.id;
+        const fields = { runId: execution.run.id, attemptId, worker };
+        const machine = (yield* Ref.get(runMachines)).get(execution.run.id);
+        if (execution.reattach) {
+            const adopted = yield* machines.adopt(worker, attemptId);
+            if (machine !== undefined && Option.isNone(yield* Ref.get(machine)))
+                yield* Ref.set(machine, adopted);
+            return yield* Effect.logDebug("The reattached Attempt keeps its machine", {
+                ...fields,
+                found: Option.isSome(adopted),
+            });
+        }
+        const hold = machine === undefined ? Option.none() : yield* Ref.get(machine);
+        if (Option.isNone(hold))
+            return yield* new InfraFailure({
+                message: `Worker "${worker}" created no machine for the attempt`,
+                cause: new Error("no machine in the Run's lease"),
+            });
+        yield* machines.assign(hold.value, attemptId).pipe(Effect.mapError((error) => new InfraFailure({
+            message: `Can't give the machine to the attempt: ${error.message}`,
+            cause: error,
+        })));
     });
+    // Every Worker kind commits as the App's bot account; only exe.dev writes to GitHub through
+    // another bot.
+    const writerFor = Effect.fn("WorkerScheduler.writer")(function* (attemptId) {
+        yield* Effect.annotateCurrentSpan({ attemptId });
+        const commitAuthor = yield* app.botAccount();
+        const botLogin = executor.kind === "exe.dev" ? EXE_DEV_BOT_LOGIN : commitAuthor.login;
+        return { commitAuthor, botLogin };
+    }, (effect, attemptId) => effect.pipe(Effect.tapError((error) => Effect.logWarning(isRefusal(error)
+        ? "Could not read the App's bot account, which authors the Attempt's commits; the Attempt fails without an automatic retry"
+        : "Could not read the App's bot account, which authors the Attempt's commits; the Attempt fails with an infrastructure failure, which is retried", { attemptId, errorTag: error._tag, message: error.message })), Effect.mapError(botAccountFailure)));
     const execute = Effect.fn("WorkerScheduler.execute")(function* (execution, io) {
         yield* Effect.annotateCurrentSpan({
             attemptId: execution.attempt.id,
@@ -333,7 +583,8 @@ export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectC
             reattach: execution.reattach,
         });
         const { settings } = (yield* instances.get).config;
-        const botLogin = yield* botLoginFor(execution.attempt.id);
+        const { commitAuthor, botLogin } = yield* writerFor(execution.attempt.id);
+        yield* machineForAttempt(execution);
         const prepared = yield* executor
             .prepare(new CheckoutRequest({
             attemptId: execution.attempt.id,
@@ -353,7 +604,8 @@ export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectC
             salvageRef: salvageRefFor(execution.run.id, execution.attempt.number),
             resume: execution.resume,
             reattach: execution.reattach,
-            botLogin,
+            botLogin: Option.some(botLogin),
+            commitAuthor,
         }))
             .pipe(Effect.tapError((error) => error._tag === "CredentialUnavailableError"
             ? Effect.logWarning(error.reason === "refused"
@@ -371,10 +623,13 @@ export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectC
         yield* io.checkoutAcquired;
         return yield* prepared.launch(io);
     }, Effect.scoped);
-    const interrupt = (attemptId) => executor.cancel(attemptId).pipe(Effect.mapError(asAttemptFailure));
+    const interrupt = (attemptId, reason) => executor.cancel(attemptId, reason).pipe(Effect.mapError(asAttemptFailure));
     const status = WorkerStatus.of({
         slots: TxRef.get(pool).pipe(Effect.map((current) => [...current].map(([worker, { capacity, leased }]) => ({ worker, capacity, leased })))),
+        queue: Effect.tx(Effect.all([TxPriorityQueue.toArray(pending), TxRef.get(pool)])).pipe(Effect.map(([queued, current]) => queuedRuns(queued, current))),
         preflights: Ref.get(preflights).pipe(Effect.map((current) => [...current.values()])),
+        machines: machines === undefined ? Effect.succeed([]) : machines.machines,
+        provisioning: TxRef.get(paused).pipe(Effect.map((current) => [...current.values()])),
     });
     const stepExecutor = StepExecutor.of({
         workers: TxRef.get(pool).pipe(Effect.map((current) => [...current.keys()])),
@@ -384,9 +639,11 @@ export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectC
         outcome: executor.outcome,
         reserve,
         occupy,
+        sweep: Effect.andThen(executor.sweep, machines === undefined ? Effect.void : machines.sweep),
+        discard: options.discard ?? (() => Effect.void),
         intervene: executor.intervene,
         recover: (attempt, record) => executor
-            .recover(attempt, record)
+            .recover(attempt, record, app.botAccount().pipe(Effect.mapError(botAccountFailure)))
             .pipe(Effect.catchIf(Schema.is(ExecutorError), (error) => Effect.fail(asAttemptFailure(error)))),
     });
     return Context.make(StepExecutor, stepExecutor).pipe(Context.add(WorkerStatus, status));