@usine/engine 0.3.0 → 0.5.0

package/dist/store/store.js

@@ -1,75 +1,138 @@
-import { Attempt, AttemptId, AttemptContext } from "@usine/core/attempt";
+import { Attempt, AttemptId, FailureResolution, } from "@usine/core/attempt";
 import * as AttemptChannel from "@usine/core/attempt/channel";
 import { AttemptFailure, InfraFailure } from "@usine/core/attempt/failure";
-import { Event } from "@usine/core/event";
+import { StartedByEditAndRetry, StartedByRetry } from "@usine/core/attempt/start";
+import { Event, EventId } from "@usine/core/event";
 import { ItemKey } from "@usine/core/item";
 import { LoopName } from "@usine/core/loop";
+import { CycleGeneration } from "@usine/core/cycle-budget";
 import { RunId, Run } from "@usine/core/run";
+import { RunAgain } from "@usine/core/run/again";
+import { ApprovalSource } from "@usine/core/run/approval";
 import { WorkerName } from "@usine/core/worker";
 import * as RunState from "@usine/core/run/state";
 import { StepResult } from "@usine/core/step/result";
-import { Cause, Context, DateTime, Effect, Exit, Layer, Match, Option, Result, Schema, } from "effect";
+import { Context, Crypto, DateTime, Effect, Layer, Match, Option, Result, Schema } from "effect";
 import { RunDatabase } from "../database/database.js";
-import { Claimed, CycleBlocked, Ignored, Queued, Submission, SubmissionResult, } from "../engine/submission.js";
-import { RunOperationRejectedError } from "../engine/error.js";
+import { Claimed, CycleBlocked, Ignored, Queued, reopeningOf, runIdFor, Submission, SubmissionResult, Suppressed, } from "../engine/submission.js";
+import { RunNotFoundError, RunOperationRejectedError, RunRetryBlockedError, } from "../engine/error.js";
 import { RunSnapshot } from "../run-reader/model.js";
+import { runAgainSubmission, } from "../engine/run-again.js";
+import { plainRetry } from "../engine/retry.js";
 import { StepExecution } from "../step-executor.js";
 import { TranscriptEntry, TranscriptQuery } from "../transcript.js";
+import { attemptWindowQueries } from "./attempt-windows.js";
+import { leftoverQueries } from "./leftovers.js";
 import { persistence, submissionFromRow } from "./codec.js";
 import { queriesFor } from "./queries.js";
-import { beginAttempt as decideBeginAttempt, cancelRun, claimRun, finishAttempt as decideFinishAttempt, reconcileAttempt, retryRun, } from "../engine/lifecycle.js";
-export class StoredSubmission extends Schema.Class("@usine/engine/internal/StoredSubmission")({
-    submission: Submission,
-    runId: RunId,
-}) {
-}
-export class RecoverableRun extends Schema.Class("@usine/engine/internal/RecoverableRun")({
-    submission: Submission,
-    runId: RunId,
-    state: RunState.RunState,
-}) {
-}
+import { beginAttempt as decideBeginAttempt, cameBack, cancelRun, claimRun, finishAttempt as decideFinishAttempt, operatorStop, resightEvent, retryRun, suppressionDecision, } from "../engine/lifecycle.js";
+const OptionalRunId = Schema.Option(RunId).pipe(Schema.withConstructorDefault(Effect.succeed(Option.none())));
+/** The result of recording an Event, with the Run it opened and the unstarted Run it superseded. */
 export class Claim extends Schema.Class("@usine/engine/internal/Claim")({
-    submission: Submission,
     result: SubmissionResult,
-    supersededRun: Schema.Option(RunId),
+    /** The Run that this claim opened, not yet driven; `None` for a joined Run. */
+    openedRun: OptionalRunId,
+    supersededRun: OptionalRunId,
 }) {
 }
 export class AttemptStarted extends Schema.TaggedClass("@usine/engine/internal/AttemptStarted")("AttemptStarted", {
     execution: StepExecution,
 }) {
 }
-export class AttemptRecoveryFailed extends Schema.TaggedClass("@usine/engine/internal/AttemptRecoveryFailed")("AttemptRecoveryFailed", {
-    failure: InfraFailure,
-}) {
+/** A Run that failed because failing the interrupted Attempt used its last automatic retry. */
+export class AttemptRecoveryFailed extends Schema.TaggedClass("@usine/engine/internal/AttemptRecoveryFailed")("AttemptRecoveryFailed", {}) {
 }
-/**
- * The Run closed (canceled, superseded, or revoked) while it waited for a
- * worker slot. Routine once capacity queues exist, never an error.
- */
-export class RunClosed extends Schema.TaggedClass("@usine/engine/internal/RunClosed")("RunClosed", {
-    state: Schema.String,
-}) {
+/** A Run that closed (canceled, superseded, or revoked) while waiting for a Worker slot; not an error. */
+export class RunClosed extends Schema.TaggedClass("@usine/engine/internal/RunClosed")("RunClosed", {}) {
 }
 const makeRunStore = Effect.gen(function* () {
     const database = yield* RunDatabase;
+    const crypto = yield* Crypto.Crypto;
     const queries = queriesFor(database);
-    const claimedRun = (submission, runId, supersededRun = Option.none()) => new Claim({
-        submission,
-        result: new Claimed({ eventId: submission.event.id, runId }),
-        supersededRun,
-    });
-    const openRun = Effect.fn("RunStore.openRun")(function* (tx, submission, runId, supersededRun = Option.none()) {
+    /** Opens the Submission's Run as `runId` unless the Item's cycle budget is used up. */
+    const admitRun = Effect.fn("RunStore.admitRun")(function* (tx, submission, runId) {
         const blocked = yield* tx.cycles.admit(submission);
         if (Option.isSome(blocked))
-            return new Claim({
-                submission,
-                supersededRun,
-                result: new CycleBlocked({ eventId: submission.event.id, ask: blocked.value }),
-            });
+            return new CycleBlocked({ eventId: submission.event.id, ask: blocked.value });
         yield* tx.createRun(submission, submission.event.id, runId);
         yield* tx.cycles.reserve(submission, runId);
-        return claimedRun(submission, runId, supersededRun);
+        return new Claimed({ eventId: submission.event.id, runId });
+    });
+    const openRun = Effect.fn("RunStore.openRun")(function* (tx, submission, runId, supersededRun = Option.none()) {
+        const result = yield* admitRun(tx, submission, runId);
+        return new Claim({
+            result,
+            openedRun: result._tag === "Claimed" ? Option.some(runId) : Option.none(),
+            supersededRun,
+        });
+    });
+    const runIdOf = (submission) => runIdFor(submission).pipe(Effect.provideService(Crypto.Crypto, crypto));
+    /** Returns the Event's Run or ignore reason, or `None` if the Event isn't claimed yet. */
+    const settlementOf = Effect.fn("RunStore.settlementOf")(function* (tx, eventId) {
+        const owner = (yield* tx.runForEvent(eventId))[0];
+        if (owner !== undefined) {
+            const run = (yield* tx.snapshot(owner.runId)).run;
+            return Option.some({ _tag: "Owned", run });
+        }
+        const ignored = (yield* tx.ignoredEvent(eventId))[0];
+        return ignored === undefined
+            ? Option.none()
+            : Option.some({ _tag: "Ignored", reason: ignored.reason });
+    });
+    /**
+     * Claims a recorded Event that has no Run and wasn't ignored: keeps it waiting during a
+     * suppression, queues it, or opens its Run as `runId`.
+     */
+    const claimEvent = Effect.fn("RunStore.claimEvent")(function* (tx, recorded, runId) {
+        if (!(yield* tx.cycles.isCurrentEvent(recorded))) {
+            return new Claim({
+                result: new Ignored({ eventId: recorded.event.id, reason: "superseded" }),
+            });
+        }
+        const suppression = yield* tx.loopItems.suppression(recorded.loop, recorded.itemKey);
+        if (Option.isSome(suppression)) {
+            const decision = suppressionDecision(suppression.value, recorded.event.id);
+            const fields = {
+                eventId: recorded.event.id,
+                suppressedBy: suppression.value.runId,
+                loop: recorded.loop,
+                itemKey: recorded.itemKey,
+            };
+            switch (decision._tag) {
+                case "SuppressFrom":
+                    yield* tx.loopItems.suppressFrom(recorded.loop, recorded.itemKey, recorded.event.id);
+                    yield* Effect.logInfo("First Item state after the Agent stopped for the operator; it waits for the operator", fields);
+                    return new Claim({
+                        result: new Suppressed({ eventId: recorded.event.id, runId: suppression.value.runId }),
+                    });
+                case "Suppress":
+                    yield* Effect.logInfo("Item unchanged since the Agent stopped for the operator; no Run opens", fields);
+                    return new Claim({
+                        result: new Suppressed({ eventId: recorded.event.id, runId: suppression.value.runId }),
+                    });
+                case "Lift":
+                    yield* tx.loopItems.lift(recorded.loop, recorded.itemKey);
+                    yield* Effect.logInfo("Item changed after the Agent stopped for the operator; the Loop acts on its new state", fields);
+            }
+        }
+        const blocking = yield* tx.blockingRun(recorded.loop, recorded.itemKey);
+        const blockedBy = blocking[0];
+        const decision = claimRun(blockedBy === undefined
+            ? Option.none()
+            : Option.some((yield* tx.snapshot(blockedBy.runId)).run), recorded.event.id);
+        yield* Effect.logInfo("Event claim decided", {
+            eventId: recorded.event.id,
+            decision: decision._tag,
+        });
+        switch (decision._tag) {
+            case "QueueEvent":
+                return new Claim({ result: new Queued({ eventId: recorded.event.id }) });
+            case "SupersedeRun":
+                yield* tx.updateState(decision.runId, decision.state);
+                return yield* openRun(tx, recorded, runId, Option.some(decision.runId));
+            case "OpenRun":
+                return yield* openRun(tx, recorded, runId);
+        }
     });
     const recordAndClaim = Effect.fn("RunStore.recordAndClaim")(function* (submission, requestedRunId) {
         yield* Effect.annotateCurrentSpan({
@@ -81,72 +144,57 @@ const makeRunStore = Effect.gen(function* () {
         return yield* database.transaction((transaction) => Effect.gen(function* () {
             const tx = queriesFor(transaction);
             const recorded = yield* tx.recordTriggerEvent(submission);
-            const existing = yield* tx.runForEvent(recorded.event.id);
-            if (existing[0] !== undefined) {
-                yield* Effect.logInfo("Event joined its existing Run", {
-                    eventId: recorded.event.id,
-                    runId: existing[0].runId,
-                });
-                return claimedRun(recorded, existing[0].runId);
-            }
-            const ignored = yield* tx.ignoredEvent(recorded.event.id);
-            if (ignored[0] !== undefined) {
-                yield* Effect.logInfo("Event remains ignored", {
-                    eventId: recorded.event.id,
-                    reason: ignored[0].reason,
-                });
-                return new Claim({
-                    submission: recorded,
-                    result: new Ignored({
-                        eventId: recorded.event.id,
-                        reason: ignored[0].reason,
-                    }),
-                    supersededRun: Option.none(),
-                });
+            const previous = yield* tx.loopItems.sighted(recorded.loop, recorded.itemKey, recorded.event.id);
+            // After a reopening, later sightings of the trigger's Event join the newest reopening.
+            const current = Option.getOrElse(yield* tx.newestReopening(recorded.event.id), () => ({
+                submission: recorded,
+                reopening: 0,
+            }));
+            const eventId = current.submission.event.id;
+            const settled = yield* settlementOf(tx, eventId);
+            if (Option.isNone(settled)) {
+                return yield* claimEvent(tx, current.submission, current.reopening === 0 ? requestedRunId : yield* runIdOf(current.submission));
             }
-            if (!(yield* tx.cycles.isCurrentEvent(recorded))) {
-                return new Claim({
-                    submission: recorded,
-                    supersededRun: Option.none(),
-                    result: new Ignored({ eventId: recorded.event.id, reason: "superseded" }),
-                });
-            }
-            const blocking = yield* tx.blockingRun(recorded.loop, recorded.itemKey);
-            const blockedBy = blocking[0];
-            const decision = claimRun(blockedBy === undefined
-                ? Option.none()
-                : Option.some((yield* tx.snapshot(blockedBy.runId)).run), recorded.event.id);
-            yield* Effect.logInfo("Event claim decided", {
-                eventId: recorded.event.id,
-                decision: decision._tag,
-            });
+            const decision = resightEvent(settled.value, cameBack(previous, recorded.event.id));
             switch (decision._tag) {
-                case "QueueEvent":
-                    return new Claim({
-                        submission: recorded,
-                        result: new Queued({ eventId: recorded.event.id }),
-                        supersededRun: Option.none(),
+                case "JoinRun":
+                    yield* Effect.logInfo("Event joined its existing Run", {
+                        eventId,
+                        runId: decision.run.id,
+                        state: decision.run.state._tag,
                     });
-                case "SupersedeRun":
-                    yield* tx.updateState(decision.runId, decision.state);
-                    yield* tx.requestInterrupt(decision.runId);
-                    return yield* openRun(tx, recorded, requestedRunId, Option.some(decision.runId));
-                case "OpenRun":
-                    return yield* openRun(tx, recorded, requestedRunId);
+                    return new Claim({ result: new Claimed({ eventId, runId: decision.run.id }) });
+                case "StayIgnored":
+                    yield* Effect.logInfo("Event remains ignored", { eventId, reason: decision.reason });
+                    return new Claim({ result: new Ignored({ eventId, reason: decision.reason }) });
+                case "ReopenEvent": {
+                    const reopening = current.reopening + 1;
+                    const reopened = yield* tx.recordTriggerEvent(reopeningOf(submission, recorded.event.dedupKey, reopening), Option.some({ eventId: recorded.event.id, reopening }));
+                    yield* Effect.logInfo("The Item came back to the state of an Event no Attempt worked on; it opens again", {
+                        eventId,
+                        settled: settled.value._tag,
+                        reopenedEventId: reopened.event.id,
+                        reopening,
+                    });
+                    return yield* claimEvent(tx, reopened, yield* runIdOf(reopened));
+                }
             }
         }));
-    }, persistence("submit Event"));
+    }, persistence("submit event"));
     const parkForApproval = Effect.fn("RunStore.parkForApproval")(function* (runId) {
-        yield* database.transaction((transaction) => Effect.gen(function* () {
+        yield* Effect.annotateCurrentSpan({ runId });
+        return yield* database.transaction((transaction) => Effect.gen(function* () {
             const tx = queriesFor(transaction);
             const current = yield* tx.snapshot(runId);
-            if (current.run.state._tag === "Pending") {
-                yield* tx.updateState(runId, RunState.awaitApproval(current.run.state));
-            }
+            if (current.run.state._tag !== "Pending")
+                return current.run.state;
+            const waiting = RunState.awaitApproval(current.run.state);
+            yield* tx.updateState(runId, waiting);
+            return waiting;
         }));
     }, persistence("park run for approval"));
-    // All callers supply a transaction: Attempt, Run, Step and outbox writes
-    // either commit together or roll back together (ADRs 0010 and 0020).
+    // Runs in the caller's transaction, so Attempt, Run, Step, and outbox writes commit or roll
+    // back together (ADR 0010, ADR 0020).
     const applyFinish = Effect.fn("RunStore.applyFinish")(function* (tx, runId, decision) {
         yield* Effect.annotateCurrentSpan({ runId });
         if (decision._tag === "Unchanged") {
@@ -165,17 +213,52 @@ const makeRunStore = Effect.gen(function* () {
             Failed: ({ failure }) => ({ ...fields, failure: failure._tag }),
             Succeeded: () => fields,
         }));
-        yield* tx.closeAttempt(attempt.id, outcome);
+        const endedAt = yield* DateTime.now;
+        yield* tx.closeAttempt(attempt.id, outcome, endedAt);
         if (decision._tag === "RetryAttempt")
             return;
         if (decision._tag === "RunSucceeded") {
             yield* tx.updateStepResult(runId, attempt.stepName, decision.result);
         }
-        yield* tx.updateState(runId, decision.state);
-        yield* tx.requestWorkflowCommand(runId, "Continue");
+        yield* tx.updateState(runId, decision.state, endedAt);
     });
-    const beginAttempt = Effect.fn("RunStore.beginAttempt")(function* (runId, worker) {
-        yield* Effect.annotateCurrentSpan({ runId });
+    // The Event queued during the Attempt is usually the Agent's own question or blocker comment,
+    // so it becomes the suppressed state.
+    const suppressForOperator = Effect.fn("RunStore.suppressForOperator")(function* (tx, current, stop) {
+        const runId = current.run.id;
+        const queued = Option.map(yield* tx.nextQueued(runId), ({ submission }) => submission.event.id);
+        const fields = { runId, loop: current.run.loop, itemKey: current.itemKey, stop };
+        if (yield* tx.loopItems.startSuppression(current.run.loop, current.itemKey, runId, current.event.id, queued)) {
+            yield* Effect.logInfo("Agent stopped for the operator; its Loop suppresses the Item", {
+                ...fields,
+                suppressedEventId: Option.getOrElse(queued, () => "none yet"),
+            });
+            return;
+        }
+        yield* Effect.logInfo("Agent stopped for the operator, but nothing is suppressed; the Loop last saw the Item not matching", fields);
+    });
+    const executionOf = (current, { attempt, state, resume, reattach, }) => new StepExecution({
+        attempt,
+        event: current.event,
+        run: new Run({
+            id: current.run.id,
+            eventId: current.run.eventId,
+            loop: current.run.loop,
+            state,
+            openedAt: current.run.openedAt,
+        }),
+        step: current.step,
+        repository: current.repository,
+        revision: current.revision,
+        task: current.task,
+        agent: current.agent,
+        parentModel: current.parentModel,
+        permissions: current.permissions,
+        resume,
+        reattach,
+    });
+    const beginAttempt = Effect.fn("RunStore.beginAttempt")(function* (runId, worker, live) {
+        yield* Effect.annotateCurrentSpan({ runId, worker });
         return yield* database.transaction((transaction) => Effect.gen(function* () {
             const tx = queriesFor(transaction);
             const current = yield* tx.snapshot(runId);
@@ -183,14 +266,48 @@ const makeRunStore = Effect.gen(function* () {
                 message: "The orchestrator stopped during the attempt",
                 cause: new Error("attempt interrupted by orchestrator restart"),
             });
-            const decision = decideBeginAttempt(current, interruption);
+            const requested = yield* tx.nextStart(runId);
+            const decision = decideBeginAttempt(current, interruption, live, requested);
             if (decision._tag === "RunClosed") {
                 yield* Effect.logInfo("Attempt start skipped", { runId, state: decision.state._tag });
-                return new RunClosed({ state: decision.state._tag });
+                return new RunClosed();
             }
             if (decision._tag === "RunFailed") {
                 yield* applyFinish(tx, runId, decision);
-                return new AttemptRecoveryFailed({ failure: interruption });
+                return new AttemptRecoveryFailed();
+            }
+            if (decision._tag === "ReattachAttempt") {
+                const reattachedAt = yield* DateTime.now;
+                const attempt = new Attempt({
+                    ...decision.attempt,
+                    reattachedAt: Option.some(reattachedAt),
+                });
+                yield* tx.recordReattach(attempt.id, reattachedAt);
+                yield* Effect.annotateCurrentSpan({ attemptId: attempt.id });
+                yield* Effect.logInfo("Attempt reattached after an orchestrator restart", {
+                    runId,
+                    attemptId: attempt.id,
+                    number: attempt.number,
+                    worker: attempt.worker,
+                });
+                if (attempt.worker !== worker) {
+                    // The Run reserves a slot on the Attempt's own Worker (#177), so a lease on another
+                    // Worker is a `StepExecutor` defect.
+                    yield* Effect.logError("Reattached Attempt keeps its own Worker, not the reserved one", {
+                        runId,
+                        attemptId: attempt.id,
+                        worker: attempt.worker,
+                        reservedWorker: worker,
+                    });
+                }
+                return new AttemptStarted({
+                    execution: executionOf(current, {
+                        attempt,
+                        state: decision.state,
+                        resume: decision.resume,
+                        reattach: true,
+                    }),
+                });
             }
             if (Option.isSome(decision.interrupted)) {
                 yield* applyFinish(tx, runId, decision.interrupted.value);
@@ -213,35 +330,85 @@ const makeRunStore = Effect.gen(function* () {
                 parentModel: current.parentModel,
                 startedAt,
                 outcome: Option.none(),
+                endedAt: Option.none(),
+                salvage: Option.none(),
+                start: Option.some(decision.start),
+                reattachedAt: Option.none(),
             });
             yield* tx.addAttempt(attempt);
+            if (Option.isSome(requested))
+                yield* tx.setNextStart(runId, Option.none());
             yield* Effect.logInfo("Attempt started", {
                 runId,
                 attemptId,
                 number: decision.number,
                 worker,
+                start: decision.start._tag,
+                resumesFrom: Option.map(decision.resume, ({ ref }) => ref),
             });
             return new AttemptStarted({
-                execution: new StepExecution({
+                execution: executionOf(current, {
                     attempt,
-                    event: current.event,
-                    run: new Run({
-                        id: current.run.id,
-                        eventId: current.run.eventId,
-                        loop: current.run.loop,
-                        state: decision.state,
-                        openedAt: current.run.openedAt,
-                    }),
-                    step: current.step,
-                    repository: current.repository,
-                    revision: current.revision,
-                    task: current.task,
-                    agent: current.agent,
-                    parentModel: current.parentModel,
+                    state: decision.state,
+                    resume: decision.resume,
+                    reattach: false,
                 }),
             });
         }));
     }, persistence("begin attempt"));
+    const recordSalvage = Effect.fn("RunStore.recordSalvage")(function* (attemptId, salvage) {
+        yield* Effect.annotateCurrentSpan({ attemptId, salvage: salvage._tag });
+        yield* database.transaction((transaction) => queriesFor(transaction).recordSalvage(attemptId, salvage));
+        yield* Effect.logInfo("Attempt salvage recorded", Match.valueTags(salvage, {
+            Pushed: ({ ref, commit }) => ({ attemptId, ref, commit }),
+            PartlyPushed: ({ ref, commit, path }) => ({ attemptId, ref, commit, path }),
+            Retained: ({ path, commit }) => ({ attemptId, path, commit }),
+            Rejected: ({ path, commit }) => ({ attemptId, path, commit }),
+        }));
+    }, persistence("record attempt salvage"));
+    const recordComments = Effect.fn("RunStore.recordComments")(function* (attemptId, comments) {
+        yield* Effect.annotateCurrentSpan({ attemptId, comments: comments.length });
+        yield* database.transaction((transaction) => queriesFor(transaction).recordComments(attemptId, comments));
+        yield* Effect.logInfo("Recorded the comments an Attempt posted before it ended without success", {
+            attemptId,
+            comments: comments.map(({ kind, url }) => `${kind} ${url.href}`),
+        });
+    }, persistence("record attempt comments"));
+    const recordOutOfMemory = Effect.fn("RunStore.recordOutOfMemory")(function* (attemptId, kills) {
+        yield* Effect.annotateCurrentSpan({
+            attemptId,
+            container: kills.container,
+            kills: kills.kills,
+        });
+        const attempt = yield* database.transaction((transaction) => queriesFor(transaction).recordOutOfMemory(attemptId, kills));
+        yield* Effect.logInfo("Recorded the processes the kernel killed in the Attempt's container", {
+            attemptId,
+            container: kills.container,
+            kills: kills.kills,
+            memoryMib: kills.memoryMib,
+            attemptKills: attempt.kills,
+        });
+    }, persistence("record attempt out-of-memory kills"));
+    const attemptWindows = Effect.fn("RunStore.attemptWindows")(function* (attemptId) {
+        yield* Effect.annotateCurrentSpan({ attemptId });
+        const windows = yield* attemptWindowQueries(database).attemptWindows(attemptId);
+        yield* Effect.logDebug("Read the Attempt's window and the Attempts of the same Agent that overlap it on its Item", {
+            attemptId,
+            recorded: Option.isSome(windows.own),
+            ended: Option.exists(windows.own, ({ endedAt }) => Option.isSome(endedAt)),
+            overlapping: windows.overlapping.map((window) => window.attemptId),
+        });
+        return windows;
+    }, persistence("read attempt windows"));
+    const discardLeftovers = Effect.fn("RunStore.discardLeftovers")(function* (worker) {
+        yield* Effect.annotateCurrentSpan({ worker });
+        const discardedAt = yield* DateTime.now;
+        return yield* database.transaction((transaction) => leftoverQueries(transaction).discardLeftovers(worker, discardedAt));
+    }, persistence("discard attempt leftovers"));
+    const openAttempts = Effect.fn("RunStore.openAttempts")(function* (worker) {
+        yield* Effect.annotateCurrentSpan({ worker });
+        return yield* leftoverQueries(database).openAttempts(worker);
+    }, persistence("count open attempts"));
     const finishAttempt = Effect.fn("RunStore.finishAttempt")(function* (attempt, result) {
         yield* Effect.annotateCurrentSpan({ runId: attempt.runId, attemptId: attempt.id });
         return yield* database.transaction((transaction) => Effect.gen(function* () {
@@ -249,14 +416,55 @@ const makeRunStore = Effect.gen(function* () {
             const current = yield* tx.snapshot(attempt.runId);
             const decision = decideFinishAttempt(current, attempt.id, result);
             yield* applyFinish(tx, attempt.runId, decision);
+            const stop = operatorStop(decision);
+            if (Option.isSome(stop))
+                yield* suppressForOperator(tx, current, stop.value);
             return decision;
         }));
     }, persistence("finish attempt"));
-    const approve = Effect.fn("RunStore.approve")(function* (runId, approval) {
-        yield* database.transaction((transaction) => Effect.gen(function* () {
+    const resolveFailure = Effect.fn("RunStore.resolveFailure")(function* (runId, attemptId, operator) {
+        yield* Effect.annotateCurrentSpan({ runId, attemptId });
+        return yield* database.transaction((transaction) => Effect.gen(function* () {
+            const tx = queriesFor(transaction);
+            const current = yield* tx.snapshot(runId);
+            const latest = current.attempts.at(-1);
+            if (!RunState.operatorActions(current.run.state).canResolveFailure ||
+                latest?.id !== attemptId) {
+                yield* Effect.logInfo("Failure resolution rejected; Run or Attempt changed", {
+                    runId,
+                    attemptId,
+                    state: current.run.state._tag,
+                    latestAttemptId: latest?.id,
+                });
+                return yield* new RunOperationRejectedError({
+                    runId,
+                    operation: "resolveFailure",
+                    state: current.run.state._tag,
+                });
+            }
+            if (Option.contains(current.resolvedFailureAttemptId, attemptId)) {
+                yield* Effect.logInfo("Run failure already resolved", { runId, attemptId });
+                return false;
+            }
+            const resolution = new FailureResolution({ resolvedAt: yield* DateTime.now, operator });
+            yield* tx.resolveFailure(runId, attemptId, resolution);
+            const lifted = yield* tx.loopItems.liftRun(runId);
+            yield* Effect.logInfo("Run failure resolved", {
+                runId,
+                attemptId,
+                liftedSuppression: lifted,
+                resolvedAt: DateTime.formatIso(resolution.resolvedAt),
+                operator: Option.getOrNull(operator),
+            });
+            return lifted;
+        }));
+    }, persistence("resolve run failure"));
+    const approve = Effect.fn("RunStore.approve")(function* (runId, approval, source) {
+        yield* Effect.annotateCurrentSpan({ runId, eventId: approval.id, channel: source.channel });
+        return yield* database.transaction((transaction) => Effect.gen(function* () {
             const tx = queriesFor(transaction);
             if (yield* tx.approvalExists(runId))
-                return;
+                return false;
             const current = yield* tx.snapshot(runId);
             if (current.run.state._tag !== "Waiting") {
                 return yield* new RunOperationRejectedError({
@@ -265,23 +473,40 @@ const makeRunStore = Effect.gen(function* () {
                     state: current.run.state._tag,
                 });
             }
-            yield* tx.recordApproval(runId, approval);
+            yield* tx.recordApproval(runId, approval, source);
             yield* tx.updateState(runId, RunState.approve(current.run.state));
-            yield* tx.requestWorkflowCommand(runId, "Approve");
+            return true;
         }));
     }, persistence("approve run"));
-    const retry = Effect.fn("RunStore.retry")(function* (runId, amendment = Option.none()) {
+    const retry = Effect.fn("RunStore.retry")(function* (runId, request = plainRetry) {
+        const { amendment, requestedBy } = request;
         yield* Effect.annotateCurrentSpan({ runId });
-        yield* database.transaction((transaction) => Effect.gen(function* () {
+        return yield* database.transaction((transaction) => Effect.gen(function* () {
             const tx = queriesFor(transaction);
             const current = yield* tx.snapshot(runId);
-            const decision = retryRun(current.run.state, yield* tx.workflowCommandsFor(runId));
+            const decision = retryRun(current);
             yield* Effect.logInfo("Operator retry decided", {
                 runId,
                 state: current.run.state._tag,
                 decision: decision._tag,
             });
             if (decision._tag === "Resume") {
+                // A later Event can have opened another Run of this Loop on the Item; reopening this one
+                // would run both.
+                const blocking = (yield* tx.blockingRun(current.run.loop, current.itemKey))[0];
+                if (blocking !== undefined) {
+                    yield* Effect.logInfo("Operator retry rejected; another Run is open on the Item", {
+                        runId,
+                        openRunId: blocking.runId,
+                        loop: current.run.loop,
+                        itemKey: current.itemKey,
+                    });
+                    return yield* new RunRetryBlockedError({
+                        runId,
+                        openRunId: blocking.runId,
+                        message: `Cannot retry run ${runId}: run ${blocking.runId} of loop "${current.run.loop}" is open on the same item. Cancel run ${blocking.runId}, or try again after it closes.`,
+                    });
+                }
                 if (Option.isSome(amendment)) {
                     yield* tx.amend(runId, amendment.value);
                     yield* Effect.logInfo("Run effective task and Agent amended", {
@@ -289,12 +514,18 @@ const makeRunStore = Effect.gen(function* () {
                         agent: amendment.value.agent.name,
                     });
                 }
+                // A retry lifts the suppression, so Events queue behind the running Run again.
+                if (yield* tx.loopItems.liftRun(runId)) {
+                    yield* Effect.logInfo("Operator retry lifts the suppression of the Item", { runId });
+                }
+                yield* tx.setNextStart(runId, Option.some(Option.isSome(amendment)
+                    ? new StartedByEditAndRetry({ operator: requestedBy })
+                    : new StartedByRetry({ operator: requestedBy })));
                 yield* tx.updateState(runId, decision.state);
-                yield* tx.requestWorkflowCommand(runId, "Resume");
-                return;
+                return true;
             }
             if (decision._tag === "ResumePending" && Option.isNone(amendment))
-                return;
+                return false;
             return yield* new RunOperationRejectedError({
                 runId,
                 operation: "retry",
@@ -302,6 +533,83 @@ const makeRunStore = Effect.gen(function* () {
             });
         }));
     }, persistence("retry run"));
+    const runAgain = Effect.fn("RunStore.runAgain")(function* (runId, request) {
+        yield* Effect.annotateCurrentSpan({ runId, agent: request.context.agent.name });
+        const receivedAt = yield* DateTime.now;
+        return yield* database.transaction((transaction) => Effect.gen(function* () {
+            const tx = queriesFor(transaction);
+            const current = yield* tx.snapshot(runId);
+            const { loop } = current.run;
+            const { itemKey } = current;
+            if (!RunState.operatorActions(current.run.state).canRunAgain) {
+                yield* Effect.logInfo("Run again rejected; only a succeeded or canceled Run runs again", {
+                    runId,
+                    state: current.run.state._tag,
+                });
+                return yield* new RunOperationRejectedError({
+                    runId,
+                    operation: "runAgain",
+                    state: current.run.state._tag,
+                });
+            }
+            const blocking = (yield* tx.blockingRun(loop, itemKey))[0];
+            if (blocking !== undefined) {
+                yield* Effect.logInfo("Run again rejected; another Run is open on the Item", {
+                    runId,
+                    openRunId: blocking.runId,
+                    loop,
+                    itemKey,
+                });
+                return yield* new RunRetryBlockedError({
+                    runId,
+                    openRunId: blocking.runId,
+                    message: `Run ${runId} cannot run again: run ${blocking.runId} of loop "${loop}" is open on the same item. Cancel run ${blocking.runId}, or try again after it closes.`,
+                });
+            }
+            const trigger = Option.getOrElse(yield* tx.reopenedEvent(current.event.id), () => ({
+                eventId: current.event.id,
+                dedupKey: current.event.dedupKey,
+            }));
+            const reopening = Option.match(yield* tx.newestReopening(trigger.eventId), {
+                onNone: () => 1,
+                onSome: (newest) => newest.reopening + 1,
+            });
+            const generation = Option.match(yield* tx.cycles.get(itemKey), {
+                onNone: () => CycleGeneration.make(0),
+                onSome: (budget) => budget.generation,
+            });
+            const again = runAgainSubmission(current, request, {
+                trigger: trigger.dedupKey,
+                reopening,
+                generation,
+                receivedAt,
+            });
+            // Not recorded as a sighting, so a later poll of the unchanged Item finds the new Run
+            // through the reopening.
+            const recorded = yield* tx.recordTriggerEvent(again, Option.some({ eventId: trigger.eventId, reopening }));
+            yield* tx.recordRunAgain(recorded.event.id, new RunAgain({ from: runId, requestedBy: request.requestedBy }));
+            const result = yield* admitRun(tx, recorded, yield* runIdOf(recorded));
+            yield* Effect.logInfo("Run again decided", Match.valueTags(result, {
+                Claimed: ({ runId: openedRunId }) => ({
+                    runId,
+                    eventId: recorded.event.id,
+                    reopening,
+                    decision: "OpenRun",
+                    openedRunId,
+                    requestedBy: Option.getOrElse(request.requestedBy, () => "none"),
+                }),
+                CycleBlocked: ({ ask }) => ({
+                    runId,
+                    eventId: recorded.event.id,
+                    reopening,
+                    decision: "CycleBlocked",
+                    askId: ask.id,
+                    requestedBy: Option.getOrElse(request.requestedBy, () => "none"),
+                }),
+            }));
+            return result;
+        }));
+    }, persistence("run again"));
     const cancel = Effect.fn("RunStore.cancel")(function* (runId) {
         yield* Effect.annotateCurrentSpan({ runId });
         return yield* database.transaction((transaction) => Effect.gen(function* () {
@@ -315,15 +623,23 @@ const makeRunStore = Effect.gen(function* () {
             });
             if (decision._tag === "AlreadyClosed")
                 return decision;
+            if (decision._tag === "CancelRejected")
+                return yield* new RunOperationRejectedError({
+                    runId,
+                    operation: "cancel",
+                    state: current.run.state._tag,
+                });
+            const canceledAt = yield* DateTime.now;
             if (Option.isSome(decision.attempt)) {
-                yield* tx.closeAttempt(decision.attempt.value.id, decision.attempt.value.outcome);
+                const { id, outcome } = decision.attempt.value;
+                yield* tx.closeAttempt(id, outcome, canceledAt);
             }
-            yield* tx.updateState(runId, decision.state);
-            yield* tx.requestInterrupt(runId);
-            yield* tx.requestWorkflowCommand(runId, "Continue");
+            yield* tx.updateState(runId, decision.state, canceledAt);
             const canceled = {
                 _tag: "Canceled",
                 openAttempt: Option.map(decision.attempt, ({ id }) => id),
+                loop: current.run.loop,
+                itemKey: current.itemKey,
             };
             return canceled;
         }));
@@ -331,23 +647,34 @@ const makeRunStore = Effect.gen(function* () {
     const revoke = Effect.fn("RunStore.revoke")(function* (loop, itemKey) {
         return yield* database.transaction((transaction) => Effect.gen(function* () {
             const tx = queriesFor(transaction);
+            yield* tx.loopItems.unmatched(loop, itemKey);
             const storedRuns = yield* tx.revokeUnstarted(loop, itemKey);
             for (const row of storedRuns) {
                 const current = yield* tx.snapshot(row.runId);
                 if (RunState.isUnstarted(current.run.state)) {
                     yield* tx.updateState(row.runId, RunState.revoke(current.run.state));
-                    yield* tx.requestInterrupt(row.runId);
                 }
             }
             return storedRuns.map((row) => row.runId);
         }));
     }, persistence("revoke run"));
+    const revokeQueued = Effect.fn("RunStore.revokeQueued")(function* (queued) {
+        const { loop, itemKey, event } = queued.submission;
+        yield* Effect.annotateCurrentSpan({ loop, itemKey, eventId: event.id });
+        // Not recorded as a sighting, because the trigger can still submit this Event from a fetch
+        // made before the closing Run changed the Item.
+        return yield* queries.revokeQueued(loop, itemKey, queued.eventIds);
+    }, persistence("revoke queued events"));
     const claimQueued = Effect.fn("RunStore.claimQueued")(function* (submission, runId) {
         return yield* database.transaction((transaction) => Effect.gen(function* () {
             const tx = queriesFor(transaction);
             const existing = yield* tx.runForEvent(submission.event.id);
             if (existing[0] !== undefined) {
-                return Option.some(new StoredSubmission({ submission, runId: existing[0].runId }));
+                yield* Effect.logInfo("Queued Event already opened its Run", {
+                    eventId: submission.event.id,
+                    runId: existing[0].runId,
+                });
+                return Option.none();
             }
             if ((yield* tx.ignoredEvent(submission.event.id)).length > 0) {
                 return Option.none();
@@ -355,21 +682,14 @@ const makeRunStore = Effect.gen(function* () {
             if ((yield* tx.blockingRun(submission.loop, submission.itemKey)).length > 0) {
                 return Option.none();
             }
+            if (Option.isSome(yield* tx.loopItems.suppression(submission.loop, submission.itemKey))) {
+                return Option.none();
+            }
             if (!(yield* tx.cycles.isCurrentEvent(submission)))
                 return Option.none();
-            const claimed = yield* openRun(tx, submission, runId);
-            return claimed.result._tag === "Claimed"
-                ? Option.some(new StoredSubmission({ submission, runId }))
-                : Option.none();
-        }));
-    }, persistence("claim queued Event"));
-    const recoverable = Effect.fn("RunStore.recoverable")(function* () {
-        return (yield* queries.recoverable()).map((row) => new RecoverableRun({
-            submission: submissionFromRow(row),
-            runId: row.runId,
-            state: row.state,
+            return (yield* openRun(tx, submission, runId)).openedRun;
         }));
-    }, persistence("list recoverable runs"));
+    }, persistence("claim queued event"));
     const appendTranscript = Effect.fn("RunStore.appendTranscript")(function* (direction, envelope) {
         const receivedAt = yield* DateTime.now;
         yield* database.transaction((transaction) => Effect.gen(function* () {
@@ -380,65 +700,52 @@ const makeRunStore = Effect.gen(function* () {
             if (direction === "outbound" && envelope.payload._tag === "InterventionApplied") {
                 yield* tx.markIntervened(envelope.attemptId);
             }
-            yield* tx.appendTranscript(direction, envelope, receivedAt);
+            if (!(yield* tx.appendTranscript(direction, envelope, receivedAt)))
+                yield* Effect.logDebug("Transcript envelope already recorded; keeping the first copy", {
+                    attemptId: envelope.attemptId,
+                    direction,
+                    sequence: envelope.sequence,
+                });
         }));
-    }, persistence("append Attempt transcript"));
+    }, persistence("append attempt transcript"));
     const transcript = Effect.fn("RunStore.transcript")(function* (query) {
         return yield* queries.transcript(query);
-    }, persistence("read Attempt transcript"));
-    const reconcileCompletion = Effect.fn("RunStore.reconcileCompletion")(function* (runId, result) {
-        yield* Effect.annotateCurrentSpan({ runId });
-        if (result._tag !== "Complete") {
-            yield* Effect.logDebug("Workflow reconciliation skipped", { runId, reason: "Suspended" });
-            return;
-        }
-        yield* database.transaction((transaction) => Effect.gen(function* () {
-            const tx = queriesFor(transaction);
-            const current = yield* tx.snapshot(runId);
-            // Workflow defects and interruption do not become typed Attempt failures.
-            // Keep the projection available to the existing restart recovery path.
-            const completion = Exit.isSuccess(result.exit)
-                ? Option.some(Result.succeed(result.exit.value))
-                : Option.map(Cause.findErrorOption(result.exit.cause), Result.fail);
-            if (Option.isNone(completion)) {
-                yield* Effect.logInfo("Workflow reconciliation skipped", {
-                    runId,
-                    reason: "NoTypedOutcome",
-                });
-                return;
-            }
-            yield* applyFinish(tx, runId, reconcileAttempt(current, completion.value));
-        }));
-    }, persistence("reconcile workflow completion"));
+    }, persistence("read attempt transcript"));
     return {
         approve,
         appendTranscript,
         beginAttempt,
         cancel,
         claimQueued,
-        completeWorkflowCommand: (runId, command) => queries
-            .completeWorkflowCommand(runId, command)
-            .pipe(persistence("complete workflow command")),
         finishAttempt,
-        nextQueued: (runId) => queries.nextQueued(runId).pipe(persistence("find queued Event")),
+        nextQueued: (runId) => queries.nextQueued(runId).pipe(persistence("find queued event")),
         parkForApproval,
-        pendingWorkflowCommands: queries
-            .pendingWorkflowCommands()
-            .pipe(persistence("list pending workflow commands")),
+        pendingContinuations: queries
+            .pendingContinuations()
+            .pipe(persistence("list pending continuations")),
+        runIdsToDrive: queries.runIdsToDrive().pipe(persistence("list the runs to drive")),
         recordAndClaim,
-        reconcileCompletion,
-        recoverable: recoverable(),
+        salvageCandidates: queries.salvageCandidates().pipe(persistence("list salvage candidates")),
+        salvageCandidate: (attemptId) => queries.salvageCandidate(attemptId).pipe(persistence("read a salvage candidate")),
+        recordSalvage,
+        recordComments,
+        recordOutOfMemory,
+        attemptWindows,
+        discardLeftovers,
+        openAttempts,
+        submission: (runId) => queries.eventRouteByRun(runId).pipe(Effect.flatMap((routes) => routes[0] === undefined
+            ? Effect.fail(new RunNotFoundError({ runId }))
+            : Effect.succeed(submissionFromRow(routes[0]))), persistence("read run submission")),
+        resolveFailure,
         retry,
         revoke,
+        revokeQueued,
+        runAgain,
         snapshot: (runId) => queries.snapshot(runId).pipe(persistence("read run")),
         transcript,
     };
 });
-/**
- * The durable Run projection store. Another backend implements this interface
- * with its own `Layer.effect(RunStore, ...)`; methods speak domain vocabulary
- * and fail only with `RunStoreError`.
- */
+/** The durable Run store; its methods fail only with `RunStoreError`. */
 export class RunStore extends Context.Service()("@usine/engine/internal/RunStore") {
     static layer = Layer.effect(RunStore, makeRunStore);
 }