@usine/orchestrator 0.2.1 → 0.4.0

package/dist/operator/operator.js

@@ -1,35 +1,71 @@
 import { transcriptEvent } from "./projection.js";
 import { TranscriptSessionsQuery } from "@usine/engine/transcript";
 import { GitHubLabelName } from "@usine/config/label";
-import { resolveParentModel } from "@usine/core/agent";
-import { AttemptContext } from "@usine/core/attempt";
+import { Agent, resolveParentModel } from "@usine/core/agent";
+import { AttemptContext, AttemptId } from "@usine/core/attempt";
 import { Intervention } from "@usine/core/attempt/channel";
 import { Event, DedupKey, EventId } from "@usine/core/event";
 import { CycleBudgetAskId } from "@usine/core/cycle-budget";
+import { BoardMoveRecord, BoardMoves } from "@usine/engine/board-moves";
 import { CycleBudgets } from "@usine/engine/cycle-budget";
 import { LoopName } from "@usine/core/loop";
 import { RunId } from "@usine/core/run";
-import { RunEngine, RunOperationRejectedError, } from "@usine/engine/engine";
+import { ApprovalSource } from "@usine/core/run/approval";
+import * as RunState from "@usine/core/run/state";
+import { canonicalRepositoryName } from "@usine/core/repository";
+import { RetryRequest, RunAgainRequest, RunEngine, RunOperationRejectedError, UnconfirmedStops, } from "@usine/engine/engine";
 import { ItemProjectionReader } from "@usine/engine/item-projection";
 import { LoopControl } from "@usine/engine/loop-control";
-import { RunReader } from "@usine/engine/run-reader";
+import { RunReader, } from "@usine/engine/run-reader";
 import { RepositoryLabelProjectionReader } from "@usine/engine/repository-label-projection";
 import { GitHubItemWriter } from "@usine/github/item-writer";
 import { ChangedIssue, ChangedPullRequest } from "@usine/github/polling";
-import { Context, DateTime, Effect, Exit, Layer, Option, Schema, SynchronizedRef } from "effect";
+import { Array as EffectArray, Context, DateTime, Effect, Exit, Layer, Match, Option, Schema, SynchronizedRef, } from "effect";
+import { runNeedsAttention, settledItemState } from "../board/items.js";
 import { labelMutationForBoardMove, projectBoard } from "../board/projection.js";
+import { LoopMatches } from "../loop-matches.js";
 import { GitHubItemKey, PollingTriggers } from "../polling-triggers.js";
 import { ScheduleTriggers } from "../schedule-triggers.js";
-import { ApprovalNeed, CycleBudgetNeed, BoardProjectionNeed, PollFailureNeed, PreflightFindingNeed, RunFailureNeed, ScheduleFailureNeed, } from "./need.js";
-import { BoardItemNotFoundError as BoardItemNotFound, BoardLaneNotFoundError as BoardLaneNotFound, BoardLaneReadOnlyError as BoardLaneReadOnly, BoardProjectNotFoundError as BoardProjectNotFound, LabelView, } from "./board.js";
+import { cycleLimitOf } from "../trigger-submission.js";
+import { ApprovalNeed, ConfigFileNeed, CycleBudgetNeed, BoardProjectionNeed, PollFailureNeed, PreflightFindingNeed, RunFailureNeed, ScheduleFailureNeed, UnconfiguredWorkerNeed, UnconfirmedStopNeed, } from "./need.js";
+import { BoardItemNotFoundError as BoardItemNotFound, BoardLaneNotFoundError as BoardLaneNotFound, BoardLaneReadOnlyError as BoardLaneReadOnly, BoardProjectNotFoundError as BoardProjectNotFound, BoardTaskView, BoardView, LabelView, ReadOnlyBoardLaneView, WritableBoardLaneView, } from "./board.js";
 import { ColdTranscriptRequest, } from "./transcript.js";
 import { LoopView, ProjectView, ReadyOverview } from "./overview.js";
-import { RunDetailView, EditAndRetry, SessionIntervention, RosterAgentNotFoundError, } from "./run.js";
-import { readTranscript, runView } from "./projection.js";
-/** How many settled Runs the overview keeps beside the open ones. */
+import { RunDetailView, EditAndRetry, LoopNotFoundError, SessionIntervention, RosterAgentNotFoundError, UnconfirmedStopView, } from "./run.js";
+import { readRunningEvents, readTranscript, readUnrecordedEnds, runDuration, runView, workerAccesses, } from "./projection.js";
+import { RunUsageReader, taskUsage } from "./usage.js";
+import { RunGitHubChanges, } from "./github-changes.js";
+import { TaskReader } from "./task.js";
+import { OperatorDesk } from "./desk.js";
+import { OperatorStatistics } from "./statistics/statistics.js";
+import { machineView } from "./machines.js";
+import { WorkerStatus } from "../worker-status.js";
+/** The number of closed Runs the overview keeps next to the open ones. */
 const OVERVIEW_CLOSED_RUNS = 100;
-/** How many newest Items a Board shows; older ones drop off. */
+/** The number of newest Items a Board shows. */
 const OVERVIEW_BOARD_ITEMS = 500;
+const sameAgent = Schema.toEquivalence(Agent);
+/**
+ * Returns `overview` with the usage of every card that contains a `changed` Run recomputed from
+ * `usage`.
+ */
+const withRunUsage = (overview, changed, usage) => new ReadyOverview({
+    ...overview,
+    projects: overview.projects.map((project) => new ProjectView({
+        ...project,
+        board: new BoardView({
+            ...project.board,
+            lanes: project.board.lanes.map((lane) => {
+                const items = lane.items.map((task) => task.runs.some(({ id }) => changed.has(id))
+                    ? new BoardTaskView({ ...task, usage: taskUsage(task.runs, usage) })
+                    : task);
+                return lane._tag === "WritableBoardLane"
+                    ? new WritableBoardLaneView({ ...lane, items })
+                    : new ReadOnlyBoardLaneView({ ...lane, items });
+            }),
+        }),
+    })),
+});
 const makeOperatorObservation = Effect.gen(function* () {
     const budgets = yield* CycleBudgets;
     const reader = yield* RunReader;
@@ -38,44 +74,90 @@ const makeOperatorObservation = Effect.gen(function* () {
     const repositoryLabels = yield* RepositoryLabelProjectionReader;
     const polling = yield* PollingTriggers;
     const schedules = yield* ScheduleTriggers;
+    const loopMatches = yield* LoopMatches;
     const cache = yield* SynchronizedRef.make(Option.none());
-    const projectOverview = Effect.fn("Operator.projectOverview")(function* (instance, pollFailures, preflightFindings, scheduleFailures, revision) {
-        const [snapshots, states, cycleAsks] = yield* Effect.all([
+    const usageReader = yield* RunUsageReader;
+    const stops = yield* UnconfirmedStops;
+    const workerStatus = yield* WorkerStatus;
+    const projectOverview = Effect.fn("Operator.projectOverview")(function* (instance, pollFailures, preflightFindings, scheduleFailures, revision, unconfirmedStops) {
+        const [snapshots, states, cycleAsks, workerAttempts] = yield* Effect.all([
             reader.list({ closedLimit: OVERVIEW_CLOSED_RUNS }),
             controls.states,
             budgets.asks(),
+            reader.attemptsByWorker(),
         ], { concurrency: "unbounded" });
         const paused = new Set(states.filter((state) => state.paused).map((state) => state.loop));
-        const views = snapshots.map((snapshot) => runView(snapshot, paused));
+        const access = workerAccesses(instance.config.workers);
+        const views = snapshots.map((snapshot) => runView(snapshot, paused, access));
         const runsByRepository = new Map();
         for (const view of views) {
             const projectRuns = runsByRepository.get(view.repository) ?? [];
             projectRuns.push(view);
             runsByRepository.set(view.repository, projectRuns);
         }
-        const projects = yield* Effect.forEach(instance.config.repos, (repository) => Effect.gen(function* () {
+        const projected = yield* Effect.forEach(instance.config.repos, (repository) => Effect.gen(function* () {
             const { name } = repository;
             const projectFindings = preflightFindings.filter(({ repository: findingRepository }) => findingRepository === name);
             const [boardItems, labelProjection] = yield* Effect.all([
                 itemReader.list(name, { limit: OVERVIEW_BOARD_ITEMS }),
                 repositoryLabels.get(name),
             ], { concurrency: "unbounded" });
-            const history = yield* reader.forItems(boardItems.map(({ itemKey }) => itemKey));
+            const [history, matches] = yield* Effect.all([
+                reader.forItems(boardItems.map(({ itemKey }) => itemKey)),
+                loopMatches.matching(instance, repository, boardItems),
+            ], { concurrency: "unbounded" });
+            const itemLoops = new Map();
+            for (const { loop, item } of matches) {
+                itemLoops.set(item.itemKey, [...(itemLoops.get(item.itemKey) ?? []), loop.name]);
+            }
             const merged = new Map((runsByRepository.get(name) ?? []).map((run) => [run.id, run]));
             for (const saved of history) {
-                const view = runView(saved, paused);
+                const view = runView(saved, paused, access);
                 merged.set(view.id, view);
             }
             const projectRuns = [...merged.values()].sort((a, b) => b.openedAt.localeCompare(a.openedAt) || b.id.localeCompare(a.id));
+            // The reader keeps ended Attempts' reports, so only running Attempts are read again. A
+            // running Run's Event gives its card's lane while its Item has no lane label.
+            const [usages, runEvents] = yield* Effect.all([
+                Effect.forEach(history, (summary) => usageReader
+                    .read(summary)
+                    .pipe(Effect.map((usage) => [summary.run.id, usage])), { concurrency: 4 }),
+                readRunningEvents(reader, history),
+            ], { concurrency: "unbounded" });
+            const runUsage = new Map(usages);
+            yield* Effect.logDebug("Read the usage of the Board's Runs", {
+                repository: name,
+                runs: runUsage.size,
+            });
             const catalog = Option.match(labelProjection, {
                 onNone: () => [],
                 onSome: ({ labels }) => labels,
             });
-            const board = yield* projectBoard(repository, catalog.map(({ name }) => name), boardItems, projectRuns, cycleAsks.map(({ itemKey }) => itemKey));
+            const board = yield* projectBoard(repository, catalog.map(({ name }) => name), boardItems, projectRuns, cycleAsks.map(({ itemKey }) => itemKey), itemLoops, runUsage, runEvents);
+            const working = history.filter(({ attempts }) => attempts.some(({ outcome }) => Option.isNone(outcome)));
+            return { name, projectFindings, projectRuns, board, catalog, runUsage, working };
+        }), { concurrency: 4 });
+        const allRuns = new Map(views.map((run) => [run.id, run]));
+        for (const { projectRuns } of projected)
+            for (const run of projectRuns)
+                allRuns.set(run.id, run);
+        // A Run of a repository that usine.yaml no longer lists has no Project to mark.
+        const managed = new Set(instance.config.repos.map(({ name }) => canonicalRepositoryName(name)));
+        const managedRuns = [];
+        for (const run of allRuns.values()) {
+            if (managed.has(canonicalRepositoryName(run.repository)))
+                managedRuns.push(run);
+            else if (run.state === "Waiting" || run.canResolveFailure)
+                yield* Effect.logDebug("Run's repository is not in usine.yaml; left out of Needs attention", { runId: run.id, repository: run.repository, state: run.state });
+        }
+        const failedItems = new Set(managedRuns.flatMap((run) => run.state !== "Waiting" && run.canResolveFailure ? [run.itemKey] : []));
+        // The Items of failed Runs that are closed, merged, or gone, with their state.
+        const settledItems = new Map(EffectArray.getSomes(yield* Effect.forEach(failedItems, (key) => settledItemState(key).pipe(Effect.provideService(ItemProjectionReader, itemReader), Effect.map(Option.map((state) => [key, state]))), { concurrency: 4 })));
+        const projects = projected.map(({ name, projectFindings, projectRuns, board, catalog }) => {
             const status = cycleAsks.some((ask) => ask.repository === name) ||
                 board.failures.length > 0 ||
                 projectFindings.length > 0 ||
-                projectRuns.some((run) => run.state === "Waiting" || run.state === "Failed")
+                projectRuns.some((run) => runNeedsAttention(run, settledItems.has(run.itemKey)))
                 ? "needs-you"
                 : projectRuns.some((run) => run.canCancel)
                     ? "active"
@@ -92,7 +174,11 @@ const makeOperatorObservation = Effect.gen(function* () {
                 })),
                 preflightFindings: projectFindings,
             });
-        }), { concurrency: 4 });
+        });
+        const usage = {
+            runs: new Map(projected.flatMap(({ runUsage }) => [...runUsage])),
+            working: projected.flatMap(({ working }) => working),
+        };
         const loops = instance.loops.map((loop) => new LoopView({
             name: loop.name,
             trigger: loop.trigger.on,
@@ -115,23 +201,31 @@ const makeOperatorObservation = Effect.gen(function* () {
                 }),
             }));
         }
-        const allRuns = new Map(views.map((run) => [run.id, run]));
-        for (const project of projects)
-            for (const run of project.runs)
-                allRuns.set(run.id, run);
-        for (const run of allRuns.values()) {
+        const cardedItems = new Map(projects.flatMap(({ board }) => board.lanes.flatMap(({ items }) => items.flatMap((task) => task.items.map((item) => [item.itemKey, item])))));
+        for (const run of managedRuns) {
             if (run.state === "Waiting") {
                 needsYou.push(new ApprovalNeed({
                     id: `approval:${run.id}`,
                     repository: run.repository,
                     runId: run.id,
                     loop: run.loop,
+                    item: cardedItems.get(run.itemKey) ?? null,
                 }));
             }
-            else if (run.state === "Failed") {
+            else if (run.canResolveFailure) {
                 const attempt = run.attempts.at(-1);
                 if (attempt?.outcome?._tag !== "Failed") {
-                    return yield* Effect.die(new Error(`Failed Run ${run.id} has no failed Attempt`));
+                    return yield* Effect.die(new Error(`Failed run ${run.id} has no failed attempt`));
+                }
+                const settled = settledItems.get(run.itemKey);
+                if (settled !== undefined) {
+                    yield* Effect.logDebug("Failed Run's Item is closed; left out of Needs attention and the Project status", {
+                        runId: run.id,
+                        itemKey: run.itemKey,
+                        itemState: settled,
+                        failure: attempt.outcome.failure._tag,
+                    });
+                    continue;
                 }
                 needsYou.push(new RunFailureNeed({
                     id: `run-failure:${run.id}`,
@@ -168,16 +262,78 @@ const makeOperatorObservation = Effect.gen(function* () {
                 failure,
             }));
         });
-        const projected = new ReadyOverview({ projects, loops, needsYou });
+        for (const { file, error } of instance.invalidAgents) {
+            needsYou.push(new ConfigFileNeed({ id: `config-file:${file}`, kind: "agent", error }));
+        }
+        for (const { file, error } of instance.quarantinedLoops) {
+            needsYou.push(new ConfigFileNeed({ id: `config-file:${file}`, kind: "loop", error }));
+        }
+        const configuredWorkers = new Set(instance.config.workers.map(({ name }) => name));
+        for (const { worker, attempts, running } of workerAttempts) {
+            if (configuredWorkers.has(worker))
+                continue;
+            needsYou.push(new UnconfiguredWorkerNeed({
+                id: `unconfigured-worker:${worker}`,
+                worker,
+                attempts,
+                running,
+            }));
+        }
+        for (const { attemptId, runId, worker } of unconfirmedStops) {
+            const run = allRuns.get(runId) ?? runView(yield* reader.inspect(runId), paused, access);
+            const attempt = run.attempts.find(({ id }) => id === attemptId);
+            if (attempt === undefined) {
+                yield* Effect.logWarning("The Run has no Attempt with the unconfirmed stop; no need shown", {
+                    runId,
+                    attemptId,
+                });
+                continue;
+            }
+            needsYou.push(new UnconfirmedStopNeed({
+                id: `unconfirmed-stop:${attemptId}`,
+                repository: run.repository,
+                runId,
+                loop: run.loop,
+                attemptNumber: attempt.number,
+                worker,
+            }));
+        }
+        const overview = new ReadyOverview({ projects, loops, needsYou });
         yield* Effect.logDebug("Refreshed operator overview", {
             revision,
             runs: views.length,
             needsYou: needsYou.length,
+            runsAtWork: usage.working.length,
+        });
+        return { overview, usage };
+    });
+    // Each session keeps its highest report, so a Run's usage changes only with more tokens or
+    // another session.
+    const refreshWorkingUsage = Effect.fn("Operator.refreshWorkingUsage")(function* (cached) {
+        if (cached.usage.working.length === 0)
+            return cached;
+        const read = yield* Effect.forEach(cached.usage.working, (summary) => usageReader.read(summary).pipe(Effect.map((usage) => [summary.run.id, usage])), { concurrency: 4 });
+        const changed = new Set(read.flatMap(([runId, usage]) => {
+            const before = cached.usage.runs.get(runId);
+            return before?.tokens === usage.tokens && before.sessions.length === usage.sessions.length
+                ? []
+                : [runId];
+        }));
+        if (changed.size === 0)
+            return cached;
+        const runs = new Map([...cached.usage.runs, ...read]);
+        yield* Effect.logDebug("Refreshed the usage of Runs at work on the cached overview", {
+            runsAtWork: read.length,
+            changed: changed.size,
         });
-        return projected;
+        return {
+            ...cached,
+            overview: withRunUsage(cached.overview, changed, runs),
+            usage: { ...cached.usage, runs },
+        };
     });
     const overview = Effect.fn("Operator.overview")((instance) => SynchronizedRef.modifyEffect(cache, (cached) => Effect.gen(function* () {
-        const [revision, latestPoll, latestSchedule] = yield* Effect.all([reader.revision(), polling.latest, schedules.latest], { concurrency: "unbounded" });
+        const [revision, latestPoll, latestSchedule, unconfirmedStops] = yield* Effect.all([reader.revision(), polling.latest, schedules.latest, stops.list], { concurrency: "unbounded" });
         const pollFailures = Option.match(latestPoll, {
             onNone: () => [],
             onSome: ({ failures }) => failures,
@@ -190,52 +346,119 @@ const makeOperatorObservation = Effect.gen(function* () {
             onNone: () => [],
             onSome: ({ failures }) => failures,
         });
-        // Without the invalidation singleton staleness cannot be detected, so the
-        // cache is bypassed.
+        // Without the invalidation singleton, staleness can't be detected, so the cache is
+        // bypassed.
         const token = Option.map(revision, (value) => JSON.stringify([
             value,
-            instance.config.repos.map(({ name, lanes }) => [name, lanes]),
-            instance.loops.map(({ name, agent, trigger }) => [name, agent, trigger.on]),
+            instance.config.repos.map(({ name, tags, lanes }) => [name, tags, lanes]),
+            [...workerAccesses(instance.config.workers)],
+            instance.loops.map(({ name, agent, trigger, selector }) => [
+                name,
+                agent,
+                trigger,
+                selector,
+            ]),
+            [...instance.invalidAgents, ...instance.quarantinedLoops].map(({ file, error }) => [
+                file,
+                error.stage,
+                error.message,
+            ]),
             pollFailures,
             preflightFindings,
             scheduleFailures,
+            unconfirmedStops.map(({ attemptId }) => attemptId),
         ]));
         if (Option.isNone(token)) {
             yield* Effect.logWarning("Projection revision singleton is missing; the overview cache is bypassed");
-            const projected = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision);
+            const { overview: projected } = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision, unconfirmedStops);
             return [projected, Option.none()];
         }
         if (Option.isSome(cached) && cached.value.token === token.value) {
             yield* Effect.logDebug("Reusing unchanged operator overview", {
                 revision,
             });
-            return [cached.value.overview, cached];
+            const current = yield* refreshWorkingUsage(cached.value);
+            return [current.overview, Option.some(current)];
         }
-        const projected = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision);
-        return [projected, Option.some({ token: token.value, overview: projected })];
+        const projected = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision, unconfirmedStops);
+        return [projected.overview, Option.some({ token: token.value, ...projected })];
     })));
-    const detail = Effect.fn("Operator.detail")(function* (runId, request = new ColdTranscriptRequest()) {
+    /**
+     * Returns the unconfirmed stop on the Loop and Item of the open Run of `snapshot`, which keeps
+     * the Run from starting an Attempt, or null.
+     */
+    const blockingStop = Effect.fn("Operator.blockingStop")(function* (snapshot, unconfirmedStops) {
+        if (!RunState.isOpen(snapshot.run.state))
+            return null;
+        const stop = unconfirmedStops.find(({ loop, itemKey }) => loop === snapshot.run.loop && itemKey === snapshot.itemKey);
+        if (stop === undefined)
+            return null;
+        const fields = {
+            runId: snapshot.run.id,
+            unstoppedRunId: stop.runId,
+            attemptId: stop.attemptId,
+        };
+        yield* Effect.annotateCurrentSpan(fields);
+        const attempts = stop.runId === snapshot.run.id
+            ? snapshot.attempts
+            : (yield* reader.inspect(stop.runId)).attempts;
+        const attempt = attempts.find(({ id }) => id === stop.attemptId);
+        if (attempt === undefined) {
+            yield* Effect.logWarning("The Run with the unconfirmed stop has no such Attempt; the Run view shows no wait", fields);
+            return null;
+        }
+        return new UnconfirmedStopView({
+            runId: stop.runId,
+            attemptNumber: attempt.number,
+            worker: stop.worker,
+        });
+    });
+    const detail = Effect.fn("Operator.detail")(function* (instance, runId, request = new ColdTranscriptRequest()) {
         yield* Effect.annotateCurrentSpan({ runId });
-        const [snapshot, states] = yield* Effect.all([reader.inspect(runId), controls.states], { concurrency: "unbounded" });
+        const [snapshot, states, unconfirmedStops, inFlight] = yield* Effect.all([reader.inspect(runId), controls.states, stops.list, workerStatus.machines], { concurrency: "unbounded" });
+        const openAttempt = snapshot.attempts.find((attempt) => Option.isNone(attempt.outcome));
+        // The machines are oldest first, so the last is the newest Attempt's.
+        const machine = [...inFlight]
+            .reverse()
+            .find((candidate) => candidate.state !== "deleting" &&
+            (Option.contains(candidate.runId, runId) ||
+                (openAttempt !== undefined && Option.contains(candidate.attemptId, openAttempt.id))));
         const paused = new Set(states.filter((state) => state.paused).map((state) => state.loop));
-        const [transcript, sessionState] = yield* Effect.all([
+        const [transcript, sessionState, usage, unrecordedEnds, settled, unconfirmedStop] = yield* Effect.all([
             readTranscript(reader, snapshot, request),
             Effect.forEach(snapshot.attempts.filter((attempt) => Option.isNone(attempt.outcome)), (attempt) => reader.transcript(new TranscriptSessionsQuery({ attemptId: attempt.id })), { concurrency: "unbounded" }),
+            usageReader.read(snapshot),
+            readUnrecordedEnds(reader, snapshot),
+            settledItemState(snapshot.itemKey).pipe(Effect.provideService(ItemProjectionReader, itemReader)),
+            blockingStop(snapshot, unconfirmedStops),
         ], { concurrency: "unbounded" });
+        const currentAgent = instance.agents.find(({ name }) => name === snapshot.agent.name);
         return new RunDetailView({
-            run: runView(snapshot, paused),
+            run: runView(snapshot, paused, workerAccesses(instance.config.workers)),
+            agentDefinitionChanged: currentAgent !== undefined && !sameAgent(snapshot.agent, currentAgent.agent),
             eventDedupKey: snapshot.event.dedupKey,
             eventPayload: snapshot.event.payload,
             transcript: transcript.events,
             sessionState: sessionState.flat().map(transcriptEvent),
             earlierTranscript: transcript.earlier,
+            worker: snapshot.attempts.at(-1)?.worker ?? null,
+            settledItemState: Option.getOrNull(settled),
+            duration: runDuration(snapshot.attempts, unrecordedEnds),
+            usage,
+            unconfirmedStop,
+            machine: machine === undefined ? null : machineView(machine),
         });
     });
-    return { detail, overview };
+    const tasks = yield* TaskReader;
+    const { desk } = yield* OperatorDesk;
+    const { statistics, runStatistics } = yield* OperatorStatistics;
+    return { detail, overview, task: tasks.read, desk, statistics, runStatistics };
 });
-/** The read-only operator surface. It needs no trigger execution to build. */
+/** The read-only operator surface, which needs no trigger execution. */
 export class OperatorObservation extends Context.Service()("@usine/orchestrator/OperatorObservation") {
-    static layer = Layer.effect(OperatorObservation, makeOperatorObservation);
+    static layer = Layer.effect(OperatorObservation, makeOperatorObservation).pipe(Layer.provide(Layer.mergeAll(TaskReader.layer, OperatorDesk.layer, LoopMatches.layer, RunUsageReader.layer, 
+    // Shares the overview's usage reader, so both read an ended Attempt's reports once.
+    OperatorStatistics.layer.pipe(Layer.provide(RunUsageReader.layer)))));
 }
 const makeOperator = Effect.gen(function* () {
     const budgets = yield* CycleBudgets;
@@ -245,8 +468,11 @@ const makeOperator = Effect.gen(function* () {
     const items = yield* ItemProjectionReader;
     const repositoryLabels = yield* RepositoryLabelProjectionReader;
     const github = yield* GitHubItemWriter;
+    const reader = yield* RunReader;
+    const boardMoves = yield* BoardMoves;
     const polling = yield* PollingTriggers;
-    const approve = Effect.fn("Operator.approve")(function* (runId) {
+    const githubChanges = yield* RunGitHubChanges;
+    const approve = Effect.fn("Operator.approve")(function* (instance, runId) {
         yield* Effect.annotateCurrentSpan({ runId });
         const current = yield* runs.inspect(runId);
         if (current.run.state._tag !== "Waiting") {
@@ -267,12 +493,16 @@ const makeOperator = Effect.gen(function* () {
             dedupKey,
             payload: { runId, decision: "approved" },
             receivedAt,
-        }));
+        }), new ApprovalSource({ channel: "ui", approver: instance.config.operator }));
         yield* Effect.logInfo("Operator approved Run", { runId });
     });
-    const retry = Effect.fn("Operator.retry")(function* (runId) {
+    const resolveFailure = Effect.fn("Operator.resolveFailure")(function* (instance, runId, attemptId) {
+        yield* Effect.annotateCurrentSpan({ runId, attemptId });
+        yield* runs.resolveFailure(runId, attemptId, instance.config.operator);
+    });
+    const retry = Effect.fn("Operator.retry")(function* (instance, runId) {
         yield* Effect.annotateCurrentSpan({ runId });
-        yield* runs.retry(runId);
+        yield* runs.retry(runId, new RetryRequest({ requestedBy: instance.config.operator, amendment: Option.none() }));
         yield* Effect.logInfo("Operator retried Run", { runId });
     });
     const editAndRetry = Effect.fn("Operator.editAndRetry")(function* (instance, runId, edit) {
@@ -281,16 +511,63 @@ const makeOperator = Effect.gen(function* () {
         if (file === undefined)
             return yield* new RosterAgentNotFoundError({
                 agent: edit.agent,
-                message: `Agent ${edit.agent} is not in the roster`,
+                message: `Agent "${edit.agent}" is not in the roster`,
             });
         const agent = file.agent;
-        yield* runs.retry(runId, Option.some(new AttemptContext({
-            task: edit.task,
-            agent,
-            parentModel: resolveParentModel(agent, instance.config.settings.parentModel),
-        })));
+        yield* runs.retry(runId, new RetryRequest({
+            requestedBy: instance.config.operator,
+            amendment: Option.some(new AttemptContext({
+                task: edit.task,
+                agent,
+                parentModel: resolveParentModel(agent, instance.config.settings.parentModel),
+            })),
+        }));
         yield* Effect.logInfo("Operator amended and retried Run", { runId, agent: agent.name });
     });
+    const runAgain = Effect.fn("Operator.runAgain")(function* (instance, runId, edit) {
+        yield* Effect.annotateCurrentSpan({ runId, agent: edit.agent });
+        const current = yield* runs.inspect(runId);
+        const loop = instance.loops.find(({ name }) => name === current.run.loop);
+        if (loop === undefined) {
+            yield* Effect.logInfo("Run again rejected; the Run's Loop is not loaded", {
+                runId,
+                loop: current.run.loop,
+            });
+            return yield* new LoopNotFoundError({
+                loop: current.run.loop,
+                message: `Run ${runId} cannot run again: loop "${current.run.loop}" is not loaded.`,
+            });
+        }
+        const file = instance.agents.find(({ name }) => name === edit.agent);
+        if (file === undefined) {
+            yield* Effect.logInfo("Run again rejected; the Agent is not in the roster", {
+                runId,
+                agent: edit.agent,
+            });
+            return yield* new RosterAgentNotFoundError({
+                agent: edit.agent,
+                message: `Agent "${edit.agent}" is not in the roster`,
+            });
+        }
+        const agent = file.agent;
+        const result = yield* runs.runAgain(runId, new RunAgainRequest({
+            step: loop.step,
+            approval: loop.approval,
+            cycleLimit: cycleLimitOf(loop),
+            permissions: loop.permissions,
+            context: new AttemptContext({
+                task: edit.task,
+                agent,
+                parentModel: resolveParentModel(agent, instance.config.settings.parentModel),
+            }),
+            requestedBy: instance.config.operator,
+        }));
+        yield* Effect.logInfo("Operator ran a Run again", Match.valueTags(result, {
+            Claimed: ({ runId: openedRunId }) => ({ runId, agent: agent.name, openedRunId }),
+            CycleBlocked: ({ ask }) => ({ runId, agent: agent.name, askId: ask.id }),
+        }));
+        return result;
+    });
     const intervene = Effect.fn("Operator.intervene")(function* (instance, runId, input) {
         yield* Effect.annotateCurrentSpan({
             runId,
@@ -330,10 +607,16 @@ const makeOperator = Effect.gen(function* () {
             items.list(move.repository, { limit: OVERVIEW_BOARD_ITEMS }),
             repositoryLabels.get(move.repository),
         ], { concurrency: "unbounded" });
+        // Place each card as the Board does, so the move records the lane that the card left.
+        const summaries = yield* reader.forItems(storedItems.map(({ itemKey }) => itemKey));
+        const runEvents = yield* readRunningEvents(reader, summaries);
+        const access = workerAccesses(instance.config.workers);
         const board = yield* projectBoard(repository, Option.match(labelProjection, {
             onNone: () => [],
             onSome: ({ labels }) => labels.map(({ name }) => name),
-        }), storedItems, []);
+        }), storedItems, 
+        // Placement reads no pause state, so these views leave every Loop unpaused.
+        summaries.map((summary) => runView(summary, new Set(), access)), [], new Map(), new Map(), runEvents);
         const source = board.lanes.find(({ items: laneItems }) => laneItems.some(({ items }) => items.some(({ number }) => number === move.item)));
         const movedItem = storedItems.find(({ number }) => number === move.item);
         if (source === undefined || movedItem === undefined) {
@@ -347,7 +630,18 @@ const makeOperator = Effect.gen(function* () {
             return yield* new BoardLaneReadOnly({ repository: move.repository, lane: move.toLane });
         }
         const mutation = labelMutationForBoardMove(board, target);
-        const changed = { number: move.item, updatedAt: yield* DateTime.now };
+        const movedAt = yield* DateTime.now;
+        // Recorded before the write, so the Run reader never attributes the swap to an Agent.
+        yield* boardMoves.record(new BoardMoveRecord({
+            repository: move.repository,
+            itemKey: movedItem.itemKey,
+            at: movedAt,
+            fromLane: source.name,
+            toLane: target.name,
+            added: mutation.add,
+            removed: mutation.remove,
+        }));
+        const changed = { number: move.item, updatedAt: movedAt };
         const item = movedItem.source === "pr" ? new ChangedPullRequest(changed) : new ChangedIssue(changed);
         const applied = yield* github.updateLabels(move.repository, move.item, mutation).pipe(
         // A failed or interrupted request may have applied part of the label
@@ -380,23 +674,36 @@ const makeOperator = Effect.gen(function* () {
         ...observation,
         approve,
         cancel,
+        discardLeftovers: (worker) => runs.discardLeftovers(worker).pipe(Effect.asVoid),
         moveBoardItem,
+        githubChanges: githubChanges.read,
         pause,
         retry,
+        resolveFailure,
         editAndRetry,
+        runAgain,
         intervene,
         unpause,
         resetCycleBudget,
     };
 });
-/** The operator's view of the factory and actions for approving, retrying, canceling, and directing work. */
+/**
+ * The operator's factory view and actions for approving, retrying, canceling, and directing work.
+ */
 export class Operator extends Context.Service()("@usine/orchestrator/Operator") {
-    static layer = Layer.effect(Operator, makeOperator).pipe(Layer.provide(OperatorObservation.layer));
+    static layer = Layer.effect(Operator, makeOperator).pipe(Layer.provide(Layer.merge(OperatorObservation.layer, RunGitHubChanges.layer)));
 }
 export { ProjectStatus, ProjectView, LoopView, ReadyOverview, NeedsSetupOverview, FactoryOverview, } from "./overview.js";
-export { RunStateTag, RunView, RunDetailView, EditAndRetry, SessionIntervention, RosterAgentNotFoundError, } from "./run.js";
-export { InfraFailureView, DefectFailureView, AgentFailureView, RunawayFailureView, ChildSessionBudgetFailureView, FailureView, SucceededAttemptView, FailedAttemptView, CanceledAttemptView, AttemptOutcomeView, AttemptView, } from "./attempt.js";
-export { BoardLaneId, BoardLaneOrigin, BoardItemLink, BoardTaskView, WritableBoardLaneView, ReadOnlyBoardLaneView, BoardLaneView, BoardProjectionError, BoardView, LabelView, BoardMove, BoardProjectNotFoundError, BoardItemNotFoundError, BoardLaneNotFoundError, BoardLaneReadOnlyError, } from "./board.js";
-export { ApprovalNeed, CycleBudgetNeed, RunFailureNeed, PollFailureNeed, PreflightFindingNeed, BoardProjectionNeed, ScheduleFailureNeed, NeedView, } from "./need.js";
+export { LabelAddedView, LabelRemovedView, CommentedView, ReviewedView, BoardMovedView, GitHubChangeView, AttemptGitHubChangesView, RunGitHubChangesView, } from "./github-changes.js";
+export { ApprovalView, RunStateTag, RunView, RunDurationView, RunDetailView, UnconfirmedStopView, EditAndRetry, LoopNotFoundError, SessionIntervention, RosterAgentNotFoundError, } from "./run.js";
+export { UsageCostView, SessionUsageView, PoolUsageView, RunUsageView, TaskPoolUsageView, TaskUsageView, } from "./usage.js";
+export { InfraFailureView, DefectFailureView, AgentFailureView, BlockedFailureView, QuestionFailureView, RunawayFailureView, ChildSessionBudgetFailureView, FailureView, SucceededAttemptView, FailedAttemptView, CanceledAttemptView, AttemptOutcomeView, LocalWorkerAccess, SshWorkerAccess, ProvisionedWorkerAccess, SandboxWorkerAccess, WorkerAccess, AttemptView, } from "./attempt.js";
+export { MachineView, ProvisioningPauseView, HetznerProvisionerView, BoatProvisionerView, ProvisionerView, } from "./machines.js";
+export { BoardLaneId, BoardLaneOrigin, BoardItemLink, BoardPullRequestView, BoardTaskView, WritableBoardLaneView, ReadOnlyBoardLaneView, BoardLaneView, BoardProjectionError, BoardView, LabelView, BoardMove, BoardProjectNotFoundError, BoardItemNotFoundError, BoardLaneNotFoundError, BoardLaneReadOnlyError, } from "./board.js";
+export { TaskItemView, TaskLaneView, TaskCommentView, TaskView, } from "./task.js";
+export { ApprovalNeed, CycleBudgetNeed, RunFailureNeed, PollFailureNeed, PreflightFindingNeed, BoardProjectionNeed, ScheduleFailureNeed, ConfigFileNeed, UnconfiguredWorkerNeed, UnconfirmedStopNeed, NeedView, } from "./need.js";
+export { LoopSelectorView, LoopFileView, QuarantinedLoopView, AgentFileView, InvalidAgentView, GatewayView, WorkerSlotsView, PassedCheckView, DegradedCheckView, DeferredCheckView, FailedCheckView, PreflightCheckView, ContainerView, HarnessPreflightView, WorkerConfigurationView, RemovedWorkerView, LabelStateView, LabelLaneSourceView, StateLaneSourceView, LaneSourceView, LaneConfigurationView, RepositoryConfigurationView, SettingsView, ConfigurationView, OperatorConfiguration, } from "./configuration.js";
+export { WorkerSilentView, ProviderRetryingView, ConnectivityWarningView, RunningAttemptView, SuppressedForOperatorView, ClosedUnchangedView, RunFailedView, LoopPausedView, WaitingForStopView, CycleBudgetUsedUpView, AwaitingApprovalView, QueuedForWorkerView, AwaitingPollView, StallReasonView, StalledItemView, ClosedRunView, WorkerView, RepositoryPollView, DeskView, } from "./desk.js";
+export { StatisticsWindow, StatisticsBucket, TaskDoneView, TaskPeriodView, LoopShareView, LoopRunTimeView, TaskTimeView, TaskCostView, TaskTokensView, UntimedTasksView, StatisticsView, DurationSummaryView, OutcomeCountsView, PoolTokensView, CostSummaryView, UsageSummaryView, LoopStatisticsView, AgentStatisticsView, BrainStatisticsView, ThroughputPeriodView, WorkerQueueWaitView, AttemptCountView, WorkerUtilizationView, NeedsAttentionPeriodView, NeedsAttentionView, RunStatisticsView, } from "./statistics/view.js";
 export { FailedTranscriptView, TranscriptPayload, TranscriptEventView, TranscriptPageCursor, ColdTranscriptRequest, TailTranscriptRequest, EarlierTranscriptRequest, TranscriptRequest, TranscriptAttemptNotFoundError, } from "./transcript.js";
 //# sourceMappingURL=operator.js.map