@usine/orchestrator 0.2.1 → 0.4.0

package/dist/scheduler.js

@@ -1,108 +1,651 @@
 import { resolveParentEffort } from "@usine/core/agent";
 import { AgentFailure, InfraFailure } from "@usine/core/attempt/failure";
-import { WorkerName } from "@usine/core/worker";
-import { CheckoutRequest, Executor, ExecutorError, } from "@usine/executor";
+import { salvageRefFor } from "@usine/core/attempt/salvage";
+import { canonicalRepositoryName } from "@usine/core/repository";
+import { BotLogin, WorkerName } from "@usine/core/worker";
+import { CheckoutRequest, Executor, ExecutorError, ExecutorOperationError, fullPreflight, } from "@usine/executor";
 import { StepExecutor } from "@usine/engine/step-executor";
-import { DateTime, Deferred, Effect, Layer, Option, Order, Queue, Ref, Schema, TxPriorityQueue, } from "effect";
+import { GitHubApp, isRefusal } from "@usine/github/github-app";
+import { Array as EffectArray, Context, DateTime, Deferred, Duration, Effect, Exit, FiberMap, Layer, Match, Option, Ref, Result, Schema, Semaphore, TxPriorityQueue, TxRef, } from "effect";
+import { agentsWithNewBrains, workersChange, } from "./current-instance/changes.js";
+import { CurrentInstance, reloadPreflightFailed, restartRequired, } from "./current-instance/current-instance.js";
+import { payloadLabels } from "./board/items.js";
+import { aheadCount, leaseOrder, passedCount, pipelinePosition, } from "./lease-order.js";
+import { recordPreflight, unchecked, WorkerStatus, } from "./worker-status.js";
 export class UnsupportedWorkerError extends Schema.TaggedError("@usine/orchestrator/UnsupportedWorkerError")("UnsupportedWorkerError", {
     worker: WorkerName,
     kind: Schema.String,
 }) {
 }
-const reservationOrder = Order.combine(Order.mapInput(Order.Number, (reservation) => reservation.eventTime), Order.mapInput(Order.Number, (reservation) => reservation.order));
-const asAttemptFailure = (error) => error._tag === "WorkerCapabilityError"
-    ? new AgentFailure({ message: error.message })
-    : new InfraFailure({
-        message: error._tag === "ExecutorOperationError"
-            ? `Executor ${error.operation} failed: ${error.message}`
-            : error.message,
-        cause: error,
-    });
-const makeLease = Effect.fn("WorkerScheduler.makeLease")(function* (worker, available) {
-    const released = yield* Ref.make(false);
-    return {
-        worker,
-        release: Ref.modify(released, (done) => [!done, true]).pipe(Effect.flatMap((first) => (first ? Queue.offer(available, worker) : Effect.void)), Effect.asVoid),
-    };
+const positionFields = (position) => Match.valueTags(position, {
+    Priority: ({ priority }) => ({ positionSource: "priority", position: priority }),
+    Lane: ({ lane, number }) => ({ positionSource: "lane", position: number, lane }),
+    Unpositioned: () => ({ positionSource: "none" }),
+});
+const slotPoolOf = (workers, previous) => {
+    const declared = new Set(workers.map(({ name }) => name));
+    return new Map([
+        ...workers.map(({ name, maxConcurrentAttempts }) => [
+            name,
+            { capacity: maxConcurrentAttempts, leased: previous.get(name)?.leased ?? 0 },
+        ]),
+        ...[...previous]
+            .filter(([name, { leased }]) => !declared.has(name) && leased > 0)
+            .map(([name, { leased }]) => [name, { capacity: 0, leased }]),
+    ]);
+};
+/** Returns the Workers that take new Attempts: those `usine.yaml` declares with slots. */
+const openWorkers = (pool) => [...pool].filter(([, { capacity }]) => capacity > 0).map(([worker]) => worker);
+/** Returns the Runs of `queued`, which is in lease order, with the Runs ahead of each. */
+const queuedRuns = (queued, pool) => {
+    const open = openWorkers(pool);
+    return queued.map((reservation, index) => ({
+        runId: reservation.runId,
+        loop: reservation.loop,
+        itemKey: reservation.itemKey,
+        position: reservation.position,
+        ahead: aheadCount(queued.slice(0, index), reservation, open),
+    }));
+};
+const isFree = (pool, worker) => {
+    const slots = pool.get(worker);
+    return slots !== undefined && slots.leased < slots.capacity;
+};
+/**
+ * Returns the Worker with a free slot for `reservation`: its specified Worker, or else the first
+ * free one that isn't `paused` from creating machines.
+ */
+const workerFor = (pool, paused, reservation) => Option.match(reservation.worker, {
+    onNone: () => Option.fromNullishOr([...pool.keys()].find((worker) => isFree(pool, worker) && paused.get(worker)?.blocked !== true)),
+    onSome: (worker) => (isFree(pool, worker) ? Option.some(worker) : Option.none()),
 });
+/** The first pause after a machine creation fails on a Worker. It doubles up to `MAX_PAUSE`. */
+const FIRST_PAUSE = Duration.seconds(30);
+const MAX_PAUSE = Duration.minutes(10);
+/** Returns the pause after the `failures`th consecutive failed machine creation. */
+const pauseAfter = (failures) => Duration.min(Duration.times(FIRST_PAUSE, 2 ** Math.min(failures - 1, 10)), MAX_PAUSE);
+const withLeased = (pool, worker, change) => {
+    const slots = pool.get(worker);
+    if (slots === undefined)
+        return pool;
+    return new Map(pool).set(worker, { ...slots, leased: slots.leased + change });
+};
+const releasedFrom = (pool, worker) => {
+    const slots = pool.get(worker);
+    if (slots === undefined)
+        return [{ _tag: "Dropped" }, pool];
+    if (slots.capacity === 0 && slots.leased <= 1) {
+        const next = new Map(pool);
+        next.delete(worker);
+        return [{ _tag: "Dropped" }, next];
+    }
+    const next = withLeased(pool, worker, -1);
+    return slots.leased > slots.capacity
+        ? [{ _tag: "Retired", running: slots.leased - 1, slots: slots.capacity }, next]
+        : [{ _tag: "Freed" }, next];
+};
+// Only errors that a retry can avoid become the `InfraFailure` that the engine retries.
+const asAttemptFailure = (error) => Match.valueTags(error, {
+    WorkerCapabilityError: ({ message }) => new AgentFailure({ message }),
+    CredentialUnavailableError: ({ reason, message }) => reason === "refused"
+        ? new AgentFailure({ message })
+        : new InfraFailure({ message, cause: error }),
+    ExecutorOperationError: ({ operation, message }) => new InfraFailure({ message: `Executor ${operation} failed: ${message}`, cause: error }),
+});
+// The operation error has no Worker field, and reports must include the Worker.
+const namingWorker = (worker) => (error) => error._tag === "ExecutorOperationError"
+    ? new ExecutorOperationError({
+        operation: error.operation,
+        message: `worker "${worker}": ${error.message}`,
+        cause: error.cause,
+    })
+    : error;
 const validateWorkers = (kind, workers) => {
     const unsupported = workers.find((worker) => worker.kind !== kind);
     return unsupported === undefined
         ? Effect.succeed(workers)
         : new UnsupportedWorkerError({ worker: unsupported.name, kind: unsupported.kind });
 };
-/** Schedules Attempts FIFO over the configured Workers and adapts the Executor to the engine's StepExecutor port. */
-export const stepExecutorLayer = (config, brains) => Layer.effect(StepExecutor, Effect.gen(function* () {
+const brainLabel = ({ harness, model, effort }) => `${harness}/${model}/${effort}`;
+/** The login that exe.dev Workers write to GitHub as, through exe.dev's integration (ADR 0011). */
+const EXE_DEV_BOT_LOGIN = BotLogin.make("exe-dev-github-integration[bot]");
+/** Returns the Attempt's failure for an unreadable bot account. A refusal isn't retried. */
+const botAccountFailure = (error) => {
+    const message = `Can't read the GitHub App's bot account, which authors the attempt's commits: ${error.message}`;
+    return isRefusal(error)
+        ? new AgentFailure({ message })
+        : new InfraFailure({ message, cause: error });
+};
+/** The Worker set of an executor that runs any Worker of its kind by name. */
+const anyWorker = { add: () => Effect.void, remove: () => Effect.void };
+/**
+ * Schedules Attempts over the Workers in `leaseOrder` and adapts the Executor to the engine's
+ * `StepExecutor` port. A removed Worker stays until its last Attempt ends.
+ */
+export const stepExecutorLayer = (config, brains, options = {}) => Layer.effectContext(Effect.gen(function* () {
     const executor = yield* Executor;
+    const instances = yield* CurrentInstance;
+    const app = yield* GitHubApp;
     const workers = yield* validateWorkers(executor.kind, config.workers);
-    yield* Effect.forEach(workers, ({ name }) => executor.preflight(name, brains), {
+    const pool = yield* TxRef.make(slotPoolOf(workers, new Map()));
+    const pending = yield* TxPriorityQueue.empty(leaseOrder);
+    const nextOrder = yield* Ref.make(0);
+    const preflights = yield* Ref.make(new Map());
+    const executorWorkers = options.workers ?? anyWorker;
+    // Every change to the executor's Workers takes this permit, so a removal never undoes an
+    // addition.
+    const membership = yield* Semaphore.make(1);
+    const adding = yield* Ref.make(new Set());
+    const machines = options.machines;
+    const paused = yield* TxRef.make(new Map());
+    // The machine of each Run's lease, which `execute` gives to the Run's Attempt.
+    const runMachines = yield* Ref.make(new Map());
+    // The machine creation of each reservation, by its order.
+    const creations = yield* FiberMap.make();
+    const provisions = (worker) => machines === undefined ? Effect.succeed(false) : machines.provisions(worker);
+    const recordPreflightExit = Effect.fn("WorkerScheduler.recordPreflight")(function* (worker, roster, exit) {
+        const outcome = Exit.isSuccess(exit)
+            ? Option.some(Result.succeed(exit.value))
+            : Option.map(Exit.findErrorOption(exit), (error) => Result.fail(error));
+        // An interrupted or defective preflight gives no information about the Worker.
+        if (Option.isNone(outcome))
+            return;
+        const checkedAt = yield* DateTime.now;
+        yield* Ref.update(preflights, (current) => new Map(current).set(worker, recordPreflight(current.get(worker) ?? unchecked(worker), roster, checkedAt, outcome.value)));
+    });
+    /**
+     * Runs `before`, then the Worker's preflight, and records both outcomes as the Worker's
+     * preflight.
+     */
+    const preflightWorker = (worker, roster, before = Effect.void) => before.pipe(Effect.andThen(executor.preflight(worker.name, roster, worker.maxConcurrentAttempts)), Effect.mapError(namingWorker(worker.name)), Effect.onExit((exit) => recordPreflightExit(worker.name, roster, exit)));
+    /**
+     * Removes `worker` from the executor unless the scheduler has it or a reload is adding it.
+     * Callers must hold `membership`.
+     */
+    const removeUnused = Effect.fn("WorkerScheduler.removeFromExecutor")(function* (worker) {
+        yield* Effect.annotateCurrentSpan({ worker });
+        const scheduled = (yield* TxRef.get(pool)).has(worker);
+        const added = (yield* Ref.get(adding)).has(worker);
+        if (scheduled || added) {
+            yield* Effect.logDebug("The Worker stays with the executor, because the scheduler has it or a reload is adding it", { worker, scheduled, added });
+            return;
+        }
+        yield* executorWorkers.remove(worker);
+    });
+    /**
+     * Ends a reload's addition of `added` and removes each Worker that the scheduler doesn't
+     * have.
+     */
+    const settleAdded = (added, removed) => membership.withPermit(Ref.update(adding, (current) => new Set([...current].filter((name) => !added.has(name)))).pipe(Effect.andThen(Effect.forEach(removed, removeUnused, { discard: true }))));
+    const unservedChange = (worker) => worker.kind === executor.kind
+        ? Option.none()
+        : Option.some({
+            block: "workers",
+            field: `workers.${worker.name}.kind`,
+            reason: `worker "${worker.name}" has kind ${worker.kind}, and the running workers have kind ${executor.kind}`,
+        });
+    const applyWorkers = Effect.fn("WorkerScheduler.applyWorkers")(function* (change, declared) {
+        const before = yield* TxRef.modify(pool, (current) => [
+            current,
+            slotPoolOf(declared, current),
+        ]);
+        const running = (worker) => before.get(worker)?.leased ?? 0;
+        yield* Effect.forEach(change.resized, ({ worker, from }) => Effect.logInfo("Worker slots resized by a reload; running Attempts continue", {
+            worker: worker.name,
+            slotsBefore: from,
+            slotsAfter: worker.maxConcurrentAttempts,
+            running: running(worker.name),
+        }));
+        yield* Effect.forEach(change.added, (worker) => Effect.logInfo("Worker added by a reload; it takes Attempts now", {
+            worker: worker.name,
+            slots: worker.maxConcurrentAttempts,
+        }));
+        yield* Effect.forEach(change.removed, ({ name }) => running(name) === 0
+            ? Effect.logInfo("Worker removed by a reload; it runs no Attempt, so it leaves the scheduler now", { worker: name })
+            : Effect.logInfo("Worker removed by a reload; it takes no new Attempt, and its running Attempts finish", { worker: name, running: running(name) }));
+        yield* settleAdded(new Set(change.added.map(({ name }) => name)), change.removed.map(({ name }) => name));
+    });
+    const prepareReload = Effect.fn("WorkerScheduler.prepareReload")(function* (running, loaded) {
+        const change = workersChange(running.config.workers, loaded.config.workers);
+        const restart = [
+            ...change.restart,
+            ...EffectArray.getSomes(change.added.map(unservedChange)),
+        ];
+        if (EffectArray.isReadonlyArrayNonEmpty(restart)) {
+            return yield* restartRequired(restart);
+        }
+        const agents = agentsWithNewBrains(running, loaded);
+        const agentNames = agents.map(({ name }) => name);
+        const newBrains = EffectArray.dedupe(agents.map(({ agent }) => agent.brain));
+        const roster = EffectArray.dedupe(loaded.agents.map(({ agent }) => agent.brain));
+        const wholeRoster = new Set([
+            ...change.added.map(({ name }) => name),
+            ...change.resized
+                .filter(({ worker, from }) => worker.maxConcurrentAttempts > from)
+                .map(({ worker }) => worker.name),
+        ]);
+        const checks = loaded.config.workers.flatMap((worker) => wholeRoster.has(worker.name)
+            ? [{ worker, brains: roster }]
+            : newBrains.length > 0
+                ? [{ worker, brains: newBrains }]
+                : []);
+        yield* Effect.annotateCurrentSpan({ agents: agentNames, workers: [...wholeRoster] });
+        if (checks.length === 0) {
+            yield* Effect.logDebug("The reload adds no Worker, slot, or brain, so it runs no Worker preflight");
+            return applyWorkers(change, loaded.config.workers);
+        }
+        const added = new Set(change.added.map(({ name }) => name));
+        yield* membership.withPermit(Ref.update(adding, (current) => new Set([...current, ...added])));
+        yield* Effect.logInfo("Preflighting the Workers, slots, and brains the reload adds", {
+            agents: agentNames,
+            brains: newBrains.map(brainLabel),
+            workers: [...wholeRoster],
+        });
+        // Add the Worker to the executor first, so its preflight can connect to its machine.
+        yield* Effect.forEach(checks, ({ worker, brains }) => added.has(worker.name)
+            ? preflightWorker(worker, brains, executorWorkers.add(worker))
+            : preflightWorker(worker, brains), { concurrency: "unbounded", discard: true }).pipe(Effect.mapError((cause) => reloadPreflightFailed(agentNames, [...wholeRoster], cause)), Effect.onExit((exit) => Exit.isFailure(exit) ? settleAdded(added, [...added]) : Effect.void));
+        return applyWorkers(change, loaded.config.workers);
+    });
+    yield* instances.registerWorkers(prepareReload);
+    // A reload can change the roster after the caller read `brains` and before the registration.
+    const { agents } = yield* instances.get;
+    const startupRoster = EffectArray.dedupe([
+        ...brains,
+        ...agents.map(({ agent }) => agent.brain),
+    ]);
+    yield* Effect.forEach(workers, (worker) => preflightWorker(worker, startupRoster), {
         concurrency: "unbounded",
         discard: true,
+    }).pipe(Effect.withSpan("WorkerScheduler.preflight"));
+    const releaseSlot = (worker) => TxRef.modify(pool, (current) => releasedFrom(current, worker)).pipe(Effect.flatMap((release) => Match.valueTags(release, {
+        Freed: () => Effect.logDebug("Worker slot released", { worker }),
+        Retired: ({ running, slots }) => Effect.logInfo("An Attempt ended on a Worker that runs more Attempts than it has slots; its slot retires", { worker, running, slots }),
+        Dropped: () => Effect.logInfo("The last Attempt on a removed Worker ended; the Worker leaves the scheduler", { worker }).pipe(Effect.andThen(membership.withPermit(removeUnused(worker)))),
+    })));
+    /** Deletes the machine that `machine` holds, if any, before the slot frees. */
+    const releaseMachine = (worker, machine) => Ref.getAndSet(machine, Option.none()).pipe(Effect.flatMap((hold) => Option.match(hold, {
+        onNone: () => Effect.void,
+        onSome: (held) => machines === undefined
+            ? Effect.void
+            : Effect.logDebug("Deleting the machine of the released slot", {
+                worker,
+                hold: held,
+            }).pipe(Effect.andThen(machines.release(held))),
+    })));
+    const makeLease = Effect.fn("WorkerScheduler.makeLease")(function* (worker, hold = Option.none(), runId = Option.none()) {
+        const released = yield* Ref.make(false);
+        const machine = yield* Ref.make(hold);
+        if (Option.isSome(runId))
+            yield* Ref.update(runMachines, (current) => new Map(current).set(runId.value, machine));
+        const forget = Option.match(runId, {
+            onNone: () => Effect.void,
+            onSome: (run) => Ref.update(runMachines, (current) => {
+                if (current.get(run) !== machine)
+                    return current;
+                const next = new Map(current);
+                next.delete(run);
+                return next;
+            }),
+        });
+        return {
+            worker,
+            release: Ref.modify(released, (done) => [!done, true]).pipe(Effect.flatMap((first) => first
+                ? forget.pipe(Effect.andThen(releaseMachine(worker, machine)), Effect.andThen(releaseSlot(worker)))
+                : Effect.void)),
+        };
+    });
+    // Retries until a free slot can take a waiting reservation, then takes both. A reservation
+    // whose Worker has no free slot doesn't block the ones behind it.
+    const nextAssignment = Effect.tx(Effect.gen(function* () {
+        const current = yield* TxRef.get(pool);
+        const pauses = yield* TxRef.get(paused);
+        const waiting = yield* TxPriorityQueue.toArray(pending);
+        const next = EffectArray.findFirst(waiting, (reservation) => Option.map(workerFor(current, pauses, reservation), (worker) => ({
+            worker,
+            reservation,
+        })));
+        if (Option.isNone(next))
+            return yield* Effect.txRetry;
+        const { worker, reservation } = next.value;
+        yield* TxPriorityQueue.removeIf(pending, (queued) => queued === reservation);
+        yield* TxRef.set(pool, withLeased(current, worker, 1));
+        return { worker, reservation, passed: passedCount(waiting, reservation) };
+    }));
+    /** Pauses machine creation on `worker` after `failure`, and resumes it when the pause ends. */
+    const pauseWorker = Effect.fn("WorkerScheduler.pauseWorker")(function* (worker, failure) {
+        const failedAt = yield* DateTime.now;
+        const pause = yield* Effect.tx(TxRef.modify(paused, (current) => {
+            const failures = (current.get(worker)?.failures ?? 0) + 1;
+            const delay = pauseAfter(failures);
+            const next = {
+                worker,
+                reason: failure._tag === "MachineUnavailableError" ? failure.reason : "check",
+                message: failure.message,
+                failures,
+                failedAt,
+                retryAt: DateTime.addDuration(failedAt, delay),
+                blocked: true,
+            };
+            return [next, new Map(current).set(worker, next)];
+        }));
+        yield* Effect.logWarning("A provisioned Worker could not create a machine; its Runs keep waiting, and it tries again after a pause", {
+            worker,
+            reason: pause.reason,
+            message: pause.message,
+            failures: pause.failures,
+            retryAt: DateTime.formatIso(pause.retryAt),
+        });
+        yield* Effect.sleep(DateTime.distance(failedAt, pause.retryAt)).pipe(Effect.andThen(Effect.tx(TxRef.update(paused, (current) => {
+            const latest = current.get(worker);
+            return latest === undefined || latest.failures !== pause.failures
+                ? current
+                : new Map(current).set(worker, { ...latest, blocked: false });
+        }))), Effect.andThen(Effect.logInfo("A provisioned Worker's pause ended; it creates machines again", {
+            worker,
+            failures: pause.failures,
+        })), Effect.forkScoped);
+    });
+    /**
+     * Creates the machine of `reservation`'s next Attempt on `worker`, whose slot it holds, and
+     * then assigns the lease with the machine. On failure, frees the slot, pauses the Worker,
+     * and puts the reservation back in the queue, so the Run keeps waiting.
+     */
+    const provision = Effect.fn("WorkerScheduler.provision")(function* (worker, reservation, passed) {
+        const fields = {
+            runId: reservation.runId,
+            loop: reservation.loop,
+            itemKey: reservation.itemKey,
+            worker,
+        };
+        yield* Effect.annotateCurrentSpan(fields);
+        if (machines === undefined)
+            return yield* Effect.die(new Error("no machines to create"));
+        const { agents } = yield* instances.get;
+        const roster = EffectArray.dedupe(agents.map(({ agent }) => agent.brain));
+        yield* Effect.logInfo("Worker slot taken in lease order; creating the Attempt's machine", {
+            ...fields,
+            ...positionFields(reservation.position),
+            passed,
+        });
+        const created = yield* machines.create(worker, reservation.runId, roster).pipe(Effect.result, Effect.onInterrupt(() => Effect.logInfo("The Run stopped waiting while its machine was created; the slot is free again", fields).pipe(Effect.andThen(releaseSlot(worker)))));
+        yield* Effect.uninterruptible(Effect.gen(function* () {
+            if (Result.isFailure(created)) {
+                const failure = created.failure;
+                yield* releaseSlot(worker);
+                if (failure._tag === "WorkerCapabilityError")
+                    yield* recordPreflightExit(worker, roster, Exit.fail(failure));
+                yield* pauseWorker(worker, failure);
+                const requeued = yield* Ref.modify(reservation.state, (state) => state._tag === "Creating"
+                    ? [true, { _tag: "Pending" }]
+                    : [false, state]);
+                if (requeued)
+                    yield* Effect.tx(TxPriorityQueue.offer(pending, reservation));
+                return;
+            }
+            const grant = created.success;
+            yield* Effect.tx(TxRef.update(paused, (current) => {
+                if (!current.has(worker))
+                    return current;
+                const next = new Map(current);
+                next.delete(worker);
+                return next;
+            }));
+            if (Option.isSome(grant.checked)) {
+                const { brains, containerLimits } = grant.checked.value;
+                yield* recordPreflightExit(worker, brains, Exit.succeed({ ...fullPreflight, containerLimits }));
+            }
+            const lease = yield* makeLease(worker, Option.some(grant.hold), Option.some(reservation.runId));
+            const assign = yield* Ref.modify(reservation.state, (state) => state._tag === "Creating"
+                ? [true, { _tag: "Assigned", lease }]
+                : [false, state]);
+            if (!assign) {
+                yield* Effect.logInfo("The Run stopped waiting while its machine was created; the machine is deleted and the slot is free again", { ...fields, machine: grant.id });
+                return yield* lease.release;
+            }
+            yield* Effect.logInfo("Worker slot leased with a new machine", {
+                ...fields,
+                machine: grant.id,
+                provider: grant.provider,
+            });
+            yield* Deferred.succeed(reservation.deferred, lease);
+        }));
     });
-    const available = yield* Queue.make();
-    yield* Queue.offerAll(available, workers.flatMap(({ maxConcurrentAttempts, name }) => Array.from({ length: maxConcurrentAttempts }, () => name)));
-    const pending = yield* TxPriorityQueue.empty(reservationOrder);
-    const nextOrder = yield* Ref.make(0);
     const dispatch = Effect.forever(Effect.gen(function* () {
-        const worker = yield* Queue.take(available);
-        const reservation = yield* TxPriorityQueue.take(pending);
-        const lease = yield* makeLease(worker, available);
+        const { worker, reservation, passed } = yield* nextAssignment;
+        // A reattached Attempt keeps its machine, so only a new Attempt gets one.
+        if (Option.isNone(reservation.worker) && (yield* provisions(worker))) {
+            const creating = yield* Ref.modify(reservation.state, (state) => state._tag === "Pending"
+                ? [true, { _tag: "Creating", worker }]
+                : [false, state]);
+            if (creating)
+                yield* FiberMap.run(creations, reservation.order, provision(worker, reservation, passed));
+            else
+                yield* releaseSlot(worker);
+            return;
+        }
+        const lease = yield* makeLease(worker, Option.none(), Option.some(reservation.runId));
         const assign = yield* Ref.modify(reservation.state, (state) => state._tag === "Pending"
             ? [true, { _tag: "Assigned", lease }]
             : [false, state]);
+        const fields = {
+            runId: reservation.runId,
+            loop: reservation.loop,
+            itemKey: reservation.itemKey,
+            worker,
+        };
         if (assign) {
-            yield* Effect.logDebug("Worker slot assigned", { worker });
+            yield* Effect.logInfo("Worker slot leased in lease order", {
+                ...fields,
+                ...positionFields(reservation.position),
+                passed,
+            });
             yield* Deferred.succeed(reservation.deferred, lease);
         }
         else {
+            yield* Effect.logDebug("The Run stopped waiting before its lease; the Worker slot is free again", fields);
             yield* lease.release;
         }
     }));
     yield* Effect.forkScoped(dispatch);
-    const reserve = Effect.fn("WorkerScheduler.reserve")(function* (submission) {
+    const occupy = Effect.fn("WorkerScheduler.occupy")(function* (worker, attemptId) {
+        yield* Effect.annotateCurrentSpan({ worker, attemptId });
+        // The Attempt runs on the Worker even without a free slot, so count it now. A slot over
+        // capacity retires when an Attempt there ends.
+        const slots = yield* TxRef.modify(pool, (current) => [current.get(worker), withLeased(current, worker, 1)]);
+        if (slots === undefined) {
+            yield* Effect.logWarning("An Attempt already runs on a Worker the scheduler does not have; it occupies no slot, and that Worker takes no new Attempt", { worker });
+            return { worker, release: Effect.void };
+        }
+        const fields = { worker, running: slots.leased + 1, slots: slots.capacity };
+        if (slots.leased < slots.capacity)
+            yield* Effect.logInfo("An Attempt that already runs on the Worker occupies a slot", fields);
+        else
+            yield* Effect.logWarning("An Attempt that already runs on the Worker occupies a slot beyond its max_concurrent_attempts; the Worker takes no new Attempt until enough of its Attempts end", fields);
+        const hold = machines !== undefined && (yield* provisions(worker))
+            ? yield* machines.adopt(worker, attemptId)
+            : Option.none();
+        return yield* makeLease(worker, hold);
+    });
+    /**
+     * Returns the pipeline position of the Run of `submission`, from the current Loop file and
+     * the lanes of the Run's repository.
+     */
+    const positionOf = Effect.fn("WorkerScheduler.position")(function* (runId, submission) {
+        yield* Effect.annotateCurrentSpan({ runId, loop: submission.loop });
+        const { config, loops } = yield* instances.get;
+        const loop = loops.find(({ name }) => name === submission.loop);
+        const priority = Option.flatMap(Option.fromNullishOr(loop), ({ priority }) => priority);
+        const lanes = config.repos.find(({ name }) => canonicalRepositoryName(name) === canonicalRepositoryName(submission.repository))?.lanes ?? [];
+        const stream = loop?.trigger.on;
+        // Only issue and pull request Events record Item labels.
+        if (Option.isSome(priority) ||
+            lanes.length === 0 ||
+            stream === undefined ||
+            stream === "schedule") {
+            return pipelinePosition(priority, lanes, []);
+        }
+        const labels = yield* payloadLabels(stream, submission.event.payload).pipe(Effect.catch((error) => Effect.logWarning("The Run's Event has no labels to read; the Run waits for a worker slot without a lane position", {
+            runId,
+            loop: submission.loop,
+            eventId: submission.event.id,
+            errorTag: error._tag,
+            message: error.message,
+        }).pipe(Effect.as([]))));
+        return pipelinePosition(priority, lanes, labels);
+    });
+    const reserve = Effect.fn("WorkerScheduler.reserve")(function* (runId, submission, worker) {
+        const run = { runId, loop: submission.loop, itemKey: submission.itemKey };
+        yield* Effect.annotateCurrentSpan(run);
+        if (Option.isSome(worker))
+            yield* Effect.annotateCurrentSpan({ worker: worker.value });
         const deferred = yield* Deferred.make();
         const state = yield* Ref.make({ _tag: "Pending" });
         const reservation = {
+            ...run,
             deferred,
+            worker,
+            position: yield* positionOf(runId, submission),
             eventTime: DateTime.toEpochMillis(submission.event.receivedAt),
             order: yield* Ref.getAndUpdate(nextOrder, (value) => value + 1),
             state,
         };
-        yield* TxPriorityQueue.offer(pending, reservation);
-        yield* Effect.logInfo("Run waits for a worker slot", {
-            loop: submission.loop,
-            itemKey: submission.itemKey,
-            queued: yield* TxPriorityQueue.size(pending),
-        });
+        const [queue, current] = yield* Effect.tx(TxPriorityQueue.offer(pending, reservation).pipe(Effect.andThen(Effect.all([TxPriorityQueue.toArray(pending), TxRef.get(pool)]))));
+        const fields = {
+            ...run,
+            ...positionFields(reservation.position),
+            queued: queue.length,
+            ahead: aheadCount(queue.slice(0, queue.indexOf(reservation)), reservation, openWorkers(current)),
+        };
+        if (Option.isNone(worker))
+            yield* Effect.logInfo("Run waits for a worker slot", fields);
+        else if ((current.get(worker.value)?.capacity ?? 0) > 0)
+            yield* Effect.logInfo("Run waits for a slot on the Worker its reattached Attempt runs on", { ...fields, worker: worker.value });
+        else
+            yield* Effect.logWarning("Run waits for a slot on the Worker its reattached Attempt runs on, which usine.yaml does not declare; it waits until a reload declares that Worker again or the Run is canceled", { ...fields, worker: worker.value });
         return yield* Deferred.await(deferred).pipe(Effect.onInterrupt(() => Ref.modify(state, (current) => current._tag === "Assigned"
             ? [Option.some(current.lease), { _tag: "Canceled" }]
-            : [Option.none(), { _tag: "Canceled" }]).pipe(Effect.flatMap(Option.match({ onNone: () => Effect.void, onSome: (lease) => lease.release })))));
+            : [Option.none(), { _tag: "Canceled" }]).pipe(
+        // Stops a machine creation for the Run, which deletes the machine and frees the slot.
+        Effect.tap(() => FiberMap.remove(creations, reservation.order)), 
+        // Remove a canceled reservation from the queue, so one whose Worker never frees a
+        // slot doesn't stay there.
+        Effect.tap(() => TxPriorityQueue.removeIf(pending, (waiting) => waiting === reservation)), Effect.flatMap(Option.match({ onNone: () => Effect.void, onSome: (lease) => lease.release })))));
+    });
+    /**
+     * Gives a provisioned Worker's machine to the Attempt of `execution`: the machine its Run's
+     * lease created, or for a reattachment the machine it ran on before the restart.
+     */
+    const machineForAttempt = Effect.fn("WorkerScheduler.machineForAttempt")(function* (execution) {
+        const worker = execution.attempt.worker;
+        if (machines === undefined || !(yield* provisions(worker)))
+            return;
+        const attemptId = execution.attempt.id;
+        const fields = { runId: execution.run.id, attemptId, worker };
+        const machine = (yield* Ref.get(runMachines)).get(execution.run.id);
+        if (execution.reattach) {
+            const adopted = yield* machines.adopt(worker, attemptId);
+            if (machine !== undefined && Option.isNone(yield* Ref.get(machine)))
+                yield* Ref.set(machine, adopted);
+            return yield* Effect.logDebug("The reattached Attempt keeps its machine", {
+                ...fields,
+                found: Option.isSome(adopted),
+            });
+        }
+        const hold = machine === undefined ? Option.none() : yield* Ref.get(machine);
+        if (Option.isNone(hold))
+            return yield* new InfraFailure({
+                message: `Worker "${worker}" created no machine for the attempt`,
+                cause: new Error("no machine in the Run's lease"),
+            });
+        yield* machines.assign(hold.value, attemptId).pipe(Effect.mapError((error) => new InfraFailure({
+            message: `Can't give the machine to the attempt: ${error.message}`,
+            cause: error,
+        })));
     });
+    // Every Worker kind commits as the App's bot account; only exe.dev writes to GitHub through
+    // another bot.
+    const writerFor = Effect.fn("WorkerScheduler.writer")(function* (attemptId) {
+        yield* Effect.annotateCurrentSpan({ attemptId });
+        const commitAuthor = yield* app.botAccount();
+        const botLogin = executor.kind === "exe.dev" ? EXE_DEV_BOT_LOGIN : commitAuthor.login;
+        return { commitAuthor, botLogin };
+    }, (effect, attemptId) => effect.pipe(Effect.tapError((error) => Effect.logWarning(isRefusal(error)
+        ? "Could not read the App's bot account, which authors the Attempt's commits; the Attempt fails without an automatic retry"
+        : "Could not read the App's bot account, which authors the Attempt's commits; the Attempt fails with an infrastructure failure, which is retried", { attemptId, errorTag: error._tag, message: error.message })), Effect.mapError(botAccountFailure)));
     const execute = Effect.fn("WorkerScheduler.execute")(function* (execution, io) {
         yield* Effect.annotateCurrentSpan({
             attemptId: execution.attempt.id,
             runId: execution.run.id,
             worker: execution.attempt.worker,
+            loop: execution.run.loop,
+            permissions: execution.permissions,
+            reattach: execution.reattach,
         });
+        const { settings } = (yield* instances.get).config;
+        const { commitAuthor, botLogin } = yield* writerFor(execution.attempt.id);
+        yield* machineForAttempt(execution);
         const prepared = yield* executor
             .prepare(new CheckoutRequest({
             attemptId: execution.attempt.id,
             worker: execution.attempt.worker,
             repository: execution.repository,
+            loop: execution.run.loop,
+            permissions: execution.permissions,
             revision: execution.revision,
             payload: execution.event.payload,
             task: execution.task,
             agent: execution.agent,
             parentModel: execution.parentModel,
-            maxChildren: config.settings.maxChildren,
-            maxAttemptMinutes: config.settings.maxAttemptMinutes,
-            maxAttemptTokens: config.settings.maxAttemptTokens,
-            parentEffort: resolveParentEffort(execution.agent, config.settings.parentEffort),
+            maxChildren: settings.maxChildren,
+            maxAttemptMinutes: settings.maxAttemptMinutes,
+            maxAttemptTokens: settings.maxAttemptTokens,
+            parentEffort: resolveParentEffort(execution.agent, settings.parentEffort),
+            salvageRef: salvageRefFor(execution.run.id, execution.attempt.number),
+            resume: execution.resume,
+            reattach: execution.reattach,
+            botLogin: Option.some(botLogin),
+            commitAuthor,
         }))
-            .pipe(Effect.mapError(asAttemptFailure));
+            .pipe(Effect.tapError((error) => error._tag === "CredentialUnavailableError"
+            ? Effect.logWarning(error.reason === "refused"
+                ? "The Attempt's repository access was refused; the Attempt fails without an automatic retry"
+                : "The Attempt's repository access could not be obtained now; the Attempt fails with an infrastructure failure, which is retried", {
+                attemptId: execution.attempt.id,
+                worker: execution.attempt.worker,
+                repository: error.repository,
+                reason: error.reason,
+                message: error.message,
+            })
+            : Effect.void), Effect.mapError(asAttemptFailure));
+        // The engine reconciles a checkout kept for a reattachment until preparation acquires it;
+        // from here the executor releases it.
+        yield* io.checkoutAcquired;
         return yield* prepared.launch(io);
     }, Effect.scoped);
-    const interrupt = (attemptId) => executor.cancel(attemptId).pipe(Effect.mapError(asAttemptFailure));
-    return StepExecutor.of({ execute, interrupt, reserve, intervene: executor.intervene });
+    const interrupt = (attemptId, reason) => executor.cancel(attemptId, reason).pipe(Effect.mapError(asAttemptFailure));
+    const status = WorkerStatus.of({
+        slots: TxRef.get(pool).pipe(Effect.map((current) => [...current].map(([worker, { capacity, leased }]) => ({ worker, capacity, leased })))),
+        queue: Effect.tx(Effect.all([TxPriorityQueue.toArray(pending), TxRef.get(pool)])).pipe(Effect.map(([queued, current]) => queuedRuns(queued, current))),
+        preflights: Ref.get(preflights).pipe(Effect.map((current) => [...current.values()])),
+        machines: machines === undefined ? Effect.succeed([]) : machines.machines,
+        provisioning: TxRef.get(paused).pipe(Effect.map((current) => [...current.values()])),
+    });
+    const stepExecutor = StepExecutor.of({
+        workers: TxRef.get(pool).pipe(Effect.map((current) => [...current.keys()])),
+        execute,
+        interrupt,
+        isLive: executor.isLive,
+        outcome: executor.outcome,
+        reserve,
+        occupy,
+        sweep: Effect.andThen(executor.sweep, machines === undefined ? Effect.void : machines.sweep),
+        discard: options.discard ?? (() => Effect.void),
+        intervene: executor.intervene,
+        recover: (attempt, record) => executor
+            .recover(attempt, record, app.botAccount().pipe(Effect.mapError(botAccountFailure)))
+            .pipe(Effect.catchIf(Schema.is(ExecutorError), (error) => Effect.fail(asAttemptFailure(error)))),
+    });
+    return Context.make(StepExecutor, stepExecutor).pipe(Context.add(WorkerStatus, status));
 }));
 //# sourceMappingURL=scheduler.js.map