@effect-agent/platform-cloudflare 0.1.0-beta.111 → 0.1.0-beta.113

package/src/Alarm.ts

@@ -9,8 +9,10 @@ import {
   Fiber,
   Layer,
   Option,
+  PubSub,
   Random,
   Ref,
+  Result,
   Schema,
   Scope,
   Semaphore,
@@ -20,11 +22,25 @@ import {
 import { type DurableBindingFailure } from "effect-agent/agent-registration";
 import {
   DurableAgentRuntime,
+  DurableRuntimeConfig,
+  isolateRecovery,
+  RecoveryBlocked,
+  RecoveryFailure,
   type DurableWorkerFailure,
   type RecoveryReport,
+  RecoverySweepResult,
 } from "effect-agent/durable-agent-runtime";
 import { ThreadId, SubmissionId } from "effect-agent/identifiers";
-import { SubmissionLedger, type SubmissionSnapshot } from "effect-agent/submission-ledger";
+import {
+  OperationAuthorizationRequest,
+  OperationAuthorizer,
+  type OperationDenied,
+} from "effect-agent/operation-authorizer";
+import {
+  AbortIntentRequest,
+  SubmissionLedger,
+  type SubmissionWorkItem,
+} from "effect-agent/submission-ledger";
 import {
   ThreadProjectionMaintenance,
   drainDue,
@@ -187,7 +203,7 @@ export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
 )({
   /** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
   phase: Schema.Literals(["caught-up", "actionable"]),
-  /** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
+  /** Recovery decisions and Thread faults observed during this event. */
   recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
   /** Head Attempts settled during the event. Joined input may settle with each head. */
   settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
@@ -211,6 +227,8 @@ export type ThreadMaintenanceFailpointLocation =
   | "maintenance:select:after"
   | "maintenance:binding-retry:before"
   | "maintenance:binding-retry:after"
+  | "maintenance:recovery-status:before"
+  | "maintenance:recovery-status:after"
   | "maintenance:retry:before"
   | "maintenance:retry:after"
   | "maintenance:checkpoint:before"
@@ -268,6 +286,65 @@ export class ThreadPublication extends Context.Service<
   });
 }
 
+/** Event-local accounting shared by the existing maintenance pumps and native scheduler. */
+export class ThreadMaintenanceActivity extends Context.Service<
+  ThreadMaintenanceActivity,
+  {
+    /**
+     * Account for one finite selection/dispatch, through joined completion or interruption.
+     * Register before reading/claiming work. After dispatch closes the body is not started.
+     * Host waves retain their declared allowance from this call; message Claims keep the driver's
+     * own deadline. Waiting for notifications belongs outside this bracket.
+     */
+    readonly run: <R>(
+      wave: Effect.Effect<void, DurableAlarmError, R>,
+    ) => Effect.Effect<void, DurableAlarmError, R>;
+    /** Acknowledge all initial subscriptions/scans. Composite hosts use `all` below. */
+    readonly ready: Effect.Effect<void>;
+    /**
+     * Acquire an independent scoped subscription before the initial scan and `ready`.
+     * The returned wait observes coalesced native scan hints, including those published before
+     * its first execution. Each hint requests a current local due-work check.
+     */
+    readonly subscribeChanges: Effect.Effect<Effect.Effect<void>, never, Scope.Scope>;
+  }
+>()("@effect-agent/platform-cloudflare/ThreadMaintenanceActivity") {
+  /** Join child pumps; acknowledge the parent only after every child is ready or has exited. */
+  static readonly all = Effect.fnUntraced(function* <A, E, R>(
+    lanes: ReadonlyArray<Effect.Effect<A, E, R>>,
+  ): Effect.fn.Return<ReadonlyArray<A>, E, R | ThreadMaintenanceActivity> {
+    const activity = yield* ThreadMaintenanceActivity;
+    let remaining = lanes.length;
+
+    if (remaining === 0) return yield* activity.ready.pipe(Effect.as([]));
+
+    return yield* Effect.forEach(
+      lanes,
+      (lane) => {
+        let ready = false;
+
+        const child: ThreadMaintenanceActivity["Service"] = {
+          ...activity,
+          ready: Effect.suspend(() => {
+            if (ready) return Effect.void;
+            ready = true;
+
+            return --remaining === 0 ? activity.ready : Effect.void;
+          }).pipe(Effect.uninterruptible),
+        };
+
+        // A failed initial setup is accounted for without swallowing its Cause. Callers
+        // may collect Exits when sibling pumps must keep their independent opportunities.
+        return lane.pipe(
+          Effect.provideService(ThreadMaintenanceActivity, child),
+          Effect.onExit(() => child.ready),
+        );
+      },
+      { concurrency: "unbounded" },
+    );
+  });
+}
+
 /**
  * Host-assembled native message recovery. The driver bounds each actual Claim and persists its
  * timeout/retry before this pump returns. Do not add a second timer starting at batch selection:
@@ -277,7 +354,7 @@ export const ThreadMessageDelivery = Context.Reference<{
   readonly drainUntil: (
     dispatchClosed: Effect.Effect<void>,
     dispatchUntil: DateTime.Utc,
-  ) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
+  ) => Effect.Effect<void, DurableAlarmError, Scope.Scope | ThreadMaintenanceActivity>;
   readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
 }>("@effect-agent/platform-cloudflare/ThreadMessageDelivery", {
   defaultValue: () => ({
@@ -288,14 +365,16 @@ export const ThreadMessageDelivery = Context.Reference<{
 
 /**
  * Application obligations sharing this Object's alarm. Admit one initial external wave even on
- * a caught-up pass, then respond to wakes until dispatchClosed. This closes only admission
- * of NEW external waves; native work can continue while admitted waves finish. Keep local
- * admission/hub subscriptions in the event Scope until maintenance tears it down.
+ * a caught-up pass, then respond to wakes and ThreadMaintenanceActivity.subscribeChanges until
+ * dispatchClosed. Yield the event's ThreadMaintenanceActivity to bracket each finite lane with
+ * run and acknowledge initial setup with ready.
+ * Passive subscriptions do not count as active work. Closure stops both new waves and native
+ * Attempts; local admission/control subscriptions belong to the event Scope until teardown.
  * No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
  *
  * Declare a finite whole-wave allowance (1..300000ms): maximum for parallel lanes, sum for
  * sequential operations. Admit a wave only if its allowance fits before dispatchUntil. Later
- * arrivals cannot renew the retirement window. Maintenance bounds the join after dispatchClosed,
+ * arrivals cannot renew an admitted wave's allowance. Maintenance bounds each registered wave,
  * interrupts and joins event Scope, then reads local deadlines under the mutation gate.
  *
  * Setup and pendingDeadline are bounded local operations. Persist claims/envelopes before
@@ -309,7 +388,7 @@ export const ThreadHostMaintenance = Context.Reference<{
   readonly drainUntil: (
     dispatchClosed: Effect.Effect<void>,
     dispatchUntil: DateTime.Utc,
-  ) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
+  ) => Effect.Effect<void, DurableAlarmError, Scope.Scope | ThreadMaintenanceActivity>;
   readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
 }>("@effect-agent/platform-cloudflare/ThreadHostMaintenance", {
   defaultValue: () => ({
@@ -354,14 +433,64 @@ class BindingRetry extends Schema.Class<BindingRetry>("BindingRetry")({
   reportedAt: Schema.Finite,
 }) {}
 
+/**
+ * A durable observation of blocked recovery, independent of the execution journal. This is
+ * neither a Settlement nor proof that external effects did not happen. A successful recovery
+ * sweep clears it; repair must preserve canonical history and the original admission identity.
+ */
+export class ThreadRecoveryFault extends Schema.Class<ThreadRecoveryFault>(
+  "@effect-agent/platform-cloudflare/ThreadRecoveryFault",
+)({
+  schemaVersion: Schema.Literal(1),
+  threadId: ThreadId,
+  firstFailedAt: Schema.Finite,
+  lastFailedAt: Schema.Finite,
+  /** Earliest automatic recovery retry; new admissions do not erase this deadline. */
+  retryAt: Schema.Finite,
+  /** Saturates at 2^31 - 1; one observation per Thread per recovery sweep. */
+  attempts: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(2_147_483_647)),
+  failure: RecoveryFailure,
+}) {}
+
+const recoveryFaultKey = (threadId: ThreadId) =>
+  `effect-agent:thread-recovery-fault:v1:${threadId}`;
+
+const decodeRecoveryFaultValue = Schema.decodeUnknownSync(ThreadRecoveryFault);
+
+const decodeRecoveryFault = (threadId: ThreadId, encoded: unknown) => {
+  const fault = decodeRecoveryFaultValue(encoded);
+
+  if (fault.threadId !== threadId) throw new Error("Recovery status does not match its Thread key");
+
+  return fault;
+};
+
+const encodeRecoveryFault = Schema.encodeSync(ThreadRecoveryFault);
+
 interface NativePassResult {
   readonly phase: "caught-up" | "actionable";
-  readonly recovered: number;
   readonly settled: number;
   readonly nonterminal: number;
   readonly nextAttemptAt: number | undefined;
 }
 
+/** Event-local observations only; durable ingress keeps racing mutations dirty. */
+interface NativeRecovery {
+  readonly queue: Deferred.Deferred<ReadonlyArray<ThreadId>>;
+  readonly pending: Set<ThreadId>;
+  readonly loaded: Set<ThreadId>;
+  readonly reports: Map<SubmissionId, RecoveryReport>;
+  readonly faults: Map<ThreadId, ThreadRecoveryFault>;
+  observation?: {
+    readonly generation: bigint;
+    readonly activeAtStart: number;
+  };
+  started: boolean;
+  needsCheckpoint: boolean;
+  recovered: number;
+  repaired: boolean;
+}
+
 interface MaintenanceObservation {
   generation?: bigint;
   nativeOnly: boolean;
@@ -384,6 +513,8 @@ class ThreadMaintenanceState extends Schema.Class<ThreadMaintenanceState>(
   nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
   /** One physical-owner cursor; old single-lane records need no conversion. */
   lastServedThreadId: Schema.optionalKey(ThreadId),
+  /** Rotate old recovery independently of dispatch, including after eviction or timeout. */
+  lastRecoveredThreadId: Schema.optionalKey(ThreadId),
   bindingRetries: Schema.optionalKey(Schema.Array(BindingRetry)),
   /** Absent on older records. A newer mutation makes this retry obsolete. */
   retry: Schema.optionalKey(MaintenanceRetry),
@@ -425,7 +556,7 @@ const ensureTransactionAlarmBy = async (
 };
 
 const stableExternalWait = (
-  snapshot: SubmissionSnapshot,
+  snapshot: SubmissionWorkItem,
   reports: ReadonlyMap<string, RecoveryReport>,
 ): boolean => {
   const report = reports.get(snapshot.submissionId);
@@ -553,10 +684,10 @@ export type MaintenancePassFailure =
  *    recovery, ledger scans or canonical-history reads.
  * 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
  *    producer keeps its newer generation dirty. Native retries retain their durable backoff.
- * 3. After the initial native opportunity, stop admitting new external waves and let the
- *    active waves finish. While they remain in flight, native wakes and bounded scans can
- *    advance more heads. All Attempts share the event's original ten-minute yield deadline.
- * 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
+ * 3. Keep native and delivery admission open together while finite waves remain active, so
+ *    fresh replies and abort controls can progress during unrelated cleanup. Close atomically
+ *    at quiescence, or at the original ten-minute yield deadline, before retiring listeners.
+ * 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill waves are
  *    bounded independently; incoming native work never restarts or cancels their attempts.
  *    Auxiliary failures are reported after the current native opportunity.
  * 5. Close every event resource before the final gated deadline snapshot and alarm decision.
@@ -572,6 +703,15 @@ export class ThreadMaintenance extends Context.Service<
      * generation has an alarm. It never scans the ledger or canonical history.
      */
     readonly ensureAlarm: Effect.Effect<void, MaintenancePassFailure>;
+    /**
+     * Authorize `explain`, then read one bounded local record without reading execution history.
+     * None means no recorded fault, not proof of health or settlement. The host authenticates
+     * callers and verifies local Thread membership before exposing this service across RPC.
+     * Provide OperationAuthorizer when constructing this Layer, as for DurableAgentRuntime.
+     */
+    readonly recoveryStatus: (
+      threadId: ThreadId,
+    ) => Effect.Effect<Option.Option<ThreadRecoveryFault>, DurableAlarmError | OperationDenied>;
     /**
      * Serialize the pre-arm boundary with pass acknowledgement, advance the durable dirty
      * generation and arm the alarm in one transaction BEFORE running the caller's mutation.
@@ -589,6 +729,7 @@ export class ThreadMaintenance extends Context.Service<
     | ThreadPublication
     | ThreadProjectionMaintenance
     | DurableAgentRuntime
+    | DurableRuntimeConfig
     | SubmissionLedger
     | WakeScheduler
     | DurableAlarmService
@@ -599,6 +740,7 @@ export class ThreadMaintenance extends Context.Service<
   > = Layer.effect(ThreadMaintenance)(
     Effect.gen(function* () {
       const runtime = yield* DurableAgentRuntime;
+      const recoveryConfig = yield* DurableRuntimeConfig;
       const ledger = yield* SubmissionLedger;
       const wakes = yield* WakeScheduler;
       const alarm = yield* DurableAlarmService;
@@ -611,6 +753,7 @@ export class ThreadMaintenance extends Context.Service<
       const projection = yield* ThreadProjectionMaintenance;
       const messages = yield* ThreadMessageDelivery;
       const host = yield* ThreadHostMaintenance;
+      const authorizer = yield* OperationAuthorizer;
 
       // A broken disposable index still needs a retry alarm and must not prevent startup.
       const projectionDeadline = projection.pendingDeadline.pipe(
@@ -635,6 +778,97 @@ export class ThreadMaintenance extends Context.Service<
 
       const runTransaction = yield* makeStorageOperation;
 
+      const recoveryStatus = Effect.fn("ThreadMaintenance.recoveryStatus")(function* (
+        threadId: ThreadId,
+      ) {
+        yield* authorizer.authorize(
+          OperationAuthorizationRequest.make({ operation: "explain", threadId }),
+        );
+
+        return yield* runTransaction("read Thread recovery status", async () => {
+          const encoded = await ctx.storage.get(recoveryFaultKey(threadId));
+
+          return encoded === undefined
+            ? Option.none()
+            : Option.some(decodeRecoveryFault(threadId, encoded));
+        });
+      });
+
+      const recordRecoveryStatus = Effect.fn("ThreadMaintenance.recordRecoveryStatus")(function* (
+        result: RecoverySweepResult,
+      ) {
+        const threads = new Map<ThreadId, RecoveryFailure | undefined>();
+
+        for (const report of result.reports) threads.set(report.threadId, undefined);
+        for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
+        if (threads.size === 0) return new Map<ThreadId, ThreadRecoveryFault>();
+        const now = yield* Clock.currentTimeMillis;
+
+        yield* failpoint.hit("maintenance:recovery-status:before");
+
+        const retained = yield* runTransaction("record Thread recovery status", () =>
+          ctx.storage.transaction(async (transaction) => {
+            const newlyBlocked: Array<ThreadRecoveryFault> = [];
+            const faults = new Map<ThreadId, ThreadRecoveryFault>();
+
+            for (const [threadId, failure] of threads) {
+              const key = recoveryFaultKey(threadId);
+              const encoded = await transaction.get(key);
+
+              const previous =
+                encoded === undefined ? undefined : decodeRecoveryFault(threadId, encoded);
+
+              if (failure === undefined) {
+                if (previous !== undefined) await transaction.delete(key);
+                continue;
+              }
+
+              const fault = ThreadRecoveryFault.make({
+                schemaVersion: 1,
+                threadId,
+                firstFailedAt: previous?.firstFailedAt ?? now,
+                lastFailedAt: now,
+                attempts: Math.min(2_147_483_647, (previous?.attempts ?? 0) + 1),
+                retryAt: now + Math.min(60_000, 5_000 * 2 ** Math.min(30, previous?.attempts ?? 0)),
+                failure,
+              });
+
+              await transaction.put(key, encodeRecoveryFault(fault));
+              faults.set(threadId, fault);
+              if (previous === undefined) newlyBlocked.push(fault);
+            }
+
+            return { newlyBlocked, faults };
+          }),
+        );
+
+        yield* failpoint.hit("maintenance:recovery-status:after");
+        for (const fault of retained.newlyBlocked)
+          yield* Effect.logError(
+            "Native Thread recovery blocked; accepted work remains pending",
+            fault.failure.reason === "defect"
+              ? Cause.die(fault.failure)
+              : Cause.fail(fault.failure),
+          ).pipe(Effect.annotateLogs({ threadId: fault.threadId }));
+
+        return retained.faults;
+      });
+
+      const recoverThread = Effect.fn("ThreadMaintenance.recoverThread")(function* (
+        threadId: ThreadId,
+        recovery: NativeRecovery,
+      ) {
+        const result = yield* runtime.runRecovery({ threadId });
+        // Visibility is committed before a claim or any fallible auxiliary join.
+        const faults = yield* recordRecoveryStatus(result);
+
+        for (const report of result.reports) recovery.reports.set(report.submissionId, report);
+        recovery.faults.delete(threadId);
+        for (const [id, fault] of faults) recovery.faults.set(id, fault);
+        recovery.recovered += result.reports.length + result.blocked.length;
+        recovery.repaired ||= result.reports.some((report) => report.disposition === "repaired");
+      });
+
       const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
         yield* failpoint.hit("maintenance:ensure:before");
         const now = yield* Clock.currentTimeMillis;
@@ -821,6 +1055,8 @@ export class ThreadMaintenance extends Context.Service<
         started: Effect.Success<ReturnType<typeof beginNative>>,
         yieldAfter: DateTime.Utc,
         observed: MaintenanceObservation,
+        recovery: NativeRecovery,
+        dispatch = true,
       ): Effect.fn.Return<NativePassResult, MaintenancePassFailure> {
         const deadline = yield* publication.pendingDeadline;
 
@@ -856,33 +1092,99 @@ export class ThreadMaintenance extends Context.Service<
 
           return {
             phase: "caught-up",
-            recovered: 0,
             settled: 0,
             nonterminal: started.nonterminal,
             nextAttemptAt,
           };
         }
-        // Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
+        // Select from control state before reading execution history. A recovering or faulted
+        // Thread cannot enter dispatch; old cleanup has its own scoped opportunity below.
         observed.nativeOnly = true;
-        const recovered: ReadonlyArray<RecoveryReport> = yield* runtime.runRecovery;
-        const reports = new Map(recovered.map((report) => [report.submissionId, report]));
+
+        // Every checkpoint keeps the producer overlap that belongs to this recovery wave.
+        const observation = (recovery.observation ??= {
+          generation: started.generation,
+          activeAtStart: started.activeAtStart,
+        });
+
         const current = yield* Stream.runCollect(ledger.scanNonterminal);
-        const heads = new Map<ThreadId, SubmissionSnapshot>();
+        const selectionTime = yield* Clock.currentTimeMillis;
+
+        yield* runTransaction("read Thread recovery deadlines", async () => {
+          for (const threadId of new Set(current.map((row) => row.threadId))) {
+            if (recovery.loaded.has(threadId)) continue;
+            const encoded = await ctx.storage.get(recoveryFaultKey(threadId));
+
+            if (encoded !== undefined)
+              recovery.faults.set(threadId, decodeRecoveryFault(threadId, encoded));
+            recovery.loaded.add(threadId);
+          }
+        });
+        const reports = recovery.reports;
+        const recoveryFaults = recovery.faults;
+
+        const waiting = (row: SubmissionWorkItem) =>
+          !recovery.pending.has(row.threadId) &&
+          !recoveryFaults.has(row.threadId) &&
+          stableExternalWait(row, reports);
+
+        const heads = new Map<ThreadId, SubmissionWorkItem>();
 
         for (const row of current) {
-          // Parked uncertainty keeps its settlement obligation, but later input can run.
-          // Accepted aborts and every other wait remain subject to the lane's FIFO barrier.
-          if (row.state === "unknown" && stableExternalWait(row, reports)) continue;
+          // Only recovered uncertainty may release later input; accepted aborts retain FIFO.
+          if (row.state === "unknown" && waiting(row)) continue;
           if (!heads.has(row.threadId)) heads.set(row.threadId, row);
         }
+        const stopping = new Set<ThreadId>();
+
+        for (const head of heads.values()) {
+          if (
+            head.state !== "ready" ||
+            recovery.pending.has(head.threadId) ||
+            recoveryFaults.has(head.threadId)
+          )
+            continue;
+
+          // An accepted abort is cleanup even when its input was never claimed. This
+          // control-only read must not decode the execution journal or a recovery snapshot.
+          const intent = yield* isolateRecovery(
+            ledger.readAbortIntent(AbortIntentRequest.make({ submissionId: head.submissionId })),
+            {
+              timeout: recoveryConfig.recoveryTimeout,
+              phase: () => "recovery",
+              operation: "read abort intent",
+            },
+          );
+
+          if (Result.isSuccess(intent)) {
+            if (intent.success !== undefined) stopping.add(head.threadId);
+          } else {
+            const faults = yield* recordRecoveryStatus(
+              RecoverySweepResult.make({
+                reports: [],
+                blocked: [
+                  RecoveryBlocked.make({ threadId: head.threadId, failure: intent.failure }),
+                ],
+              }),
+            );
+
+            for (const [threadId, fault] of faults) recoveryFaults.set(threadId, fault);
+            recovery.recovered++;
+          }
+        }
 
         const eligible = [...heads.values()]
-          .filter((head) => !stableExternalWait(head, reports))
+          .filter(
+            (head) =>
+              !recovery.pending.has(head.threadId) &&
+              !stopping.has(head.threadId) &&
+              !recoveryFaults.has(head.threadId) &&
+              !waiting(head) &&
+              (head.state === "ready" || reports.has(head.submissionId)),
+          )
           .map((head) => head.threadId)
           .sort();
 
-        const selectionTime = yield* Clock.currentTimeMillis;
-
         yield* failpoint.hit("maintenance:select:before");
 
         const selection = yield* runTransaction("select maintenance lane", () =>
@@ -900,11 +1202,12 @@ export class ThreadMaintenance extends Context.Service<
                 ),
             );
 
-            const next =
-              runnable.find(
-                (threadId) =>
-                  state.lastServedThreadId === undefined || threadId > state.lastServedThreadId,
-              ) ?? runnable[0];
+            const next = dispatch
+              ? (runnable.find(
+                  (threadId) =>
+                    state.lastServedThreadId === undefined || threadId > state.lastServedThreadId,
+                ) ?? runnable[0])
+              : undefined;
 
             if (next !== undefined) {
               await transaction.put(
@@ -915,22 +1218,54 @@ export class ThreadMaintenance extends Context.Service<
               );
             }
 
-            return { selected: next, retries };
+            const backlog = recovery.started
+              ? []
+              : [...heads.keys()].filter((threadId) => {
+                  const fault = recoveryFaults.get(threadId);
+
+                  return (
+                    !eligible.includes(threadId) &&
+                    (fault === undefined || fault.retryAt <= selectionTime)
+                  );
+                });
+
+            // One finite wave, one Thread at a time, with the runtime's per-Thread deadline.
+            // This cursor ensures an event deadline/eviction cannot always restart at the front.
+            const after = backlog.filter(
+              (threadId) =>
+                state.lastRecoveredThreadId === undefined || threadId > state.lastRecoveredThreadId,
+            );
+
+            const before = backlog.filter(
+              (threadId) =>
+                state.lastRecoveredThreadId !== undefined &&
+                threadId <= state.lastRecoveredThreadId,
+            );
+
+            return { selected: next, retries, backlog: [...after, ...before] };
           }),
         );
 
         yield* failpoint.hit("maintenance:select:after");
+        if (!recovery.started) {
+          recovery.started = true;
+          recovery.needsCheckpoint = selection.backlog.length > 0;
+          for (const threadId of selection.backlog) recovery.pending.add(threadId);
+          yield* Deferred.succeed(recovery.queue, selection.backlog);
+        }
 
         const selected =
           selection.selected === undefined ? undefined : heads.get(selection.selected);
 
+        if (selected !== undefined) yield* recoverThread(selected.threadId, recovery);
+
         let retries = selection.retries;
         let bindingFailure: DurableBindingFailure | undefined;
 
         // One runnable FIFO head per native opportunity. An absent agent waits for a deployment,
         // including for children; other local lanes and host deliveries remain independently due.
         const settlement =
-          selected === undefined
+          selected === undefined || recoveryFaults.has(selected.threadId)
             ? Option.none()
             : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(
                 Effect.catchTag("BindingUnavailable", (failure) => {
@@ -999,11 +1334,10 @@ export class ThreadMaintenance extends Context.Service<
         const waitingHeads = new Map<ThreadId, boolean>();
 
         const autonomous = remaining.some((snapshot) => {
-          if (snapshot.state === "unknown" && stableExternalWait(snapshot, reports)) return false;
+          if (snapshot.state === "unknown" && waiting(snapshot)) return false;
           const headWaiting = waitingHeads.get(snapshot.threadId);
 
-          if (headWaiting === undefined)
-            waitingHeads.set(snapshot.threadId, stableExternalWait(snapshot, reports));
+          if (headWaiting === undefined) waitingHeads.set(snapshot.threadId, waiting(snapshot));
           // FIFO followers cannot execute through a stable external wait. Only plain queued
           // input is dormant here; admission repairs and accepted aborts still need a pass.
           if (
@@ -1013,26 +1347,28 @@ export class ThreadMaintenance extends Context.Service<
           )
             return false;
 
-          return !stableExternalWait(snapshot, reports);
+          return !waiting(snapshot);
         });
 
-        const progressed =
-          Option.isSome(settlement) ||
-          recovered.some((report) => report.disposition === "repaired");
+        const progressed = Option.isSome(settlement) || recovery.repaired;
 
         const now = yield* Clock.currentTimeMillis;
         const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
 
-        const nextEligible = eligible.map(
-          (threadId) =>
-            retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)
-              ?.notBefore ?? now,
-        );
+        const nextEligible = [...recoveryFaults.values()]
+          .map((fault) => fault.retryAt)
+          .concat(
+            eligible.map(
+              (threadId) =>
+                retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)
+                  ?.notBefore ?? now,
+            ),
+          );
 
-        const bindingDelay =
+        const retryDelay =
           nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
 
-        const delay = Math.max(ordinaryDelay, bindingDelay);
+        const delay = Math.max(ordinaryDelay, retryDelay);
 
         // Checkpoint native progress without changing the physical alarm. Auxiliary
         // delivery remains live; later mutations still advance the shared generation.
@@ -1044,11 +1380,14 @@ export class ThreadMaintenance extends Context.Service<
               const { state } = await readMaintenanceState(transaction);
 
               const processed =
-                autonomous || started.activeAtStart > 0 || active > 0
+                autonomous ||
+                observation.activeAtStart > 0 ||
+                started.activeAtStart > 0 ||
+                active > 0
                   ? state.processed
-                  : state.processed > started.generation
+                  : state.processed > observation.generation
                     ? state.processed
-                    : started.generation;
+                    : observation.generation;
 
               const next = ThreadMaintenanceState.make({
                 ...Struct.omit(state, ["retry"]),
@@ -1087,7 +1426,6 @@ export class ThreadMaintenance extends Context.Service<
 
         return {
           phase: "actionable",
-          recovered: recovered.length,
           settled: Option.isSome(settlement) ? 1 : 0,
           nonterminal: remaining.length,
           nextAttemptAt,
@@ -1106,7 +1444,9 @@ export class ThreadMaintenance extends Context.Service<
           Effect.catch(() => Effect.never),
         );
 
-        let started = yield* beginNative(observed);
+        // A wake may observe temporary backoff while this event's recovery is still running.
+        // Its completion must retain the original actionable observation for acknowledgement.
+        const started = yield* beginNative(observed);
 
         // This scope owns auxiliary dispatch and listeners, independently of native progress.
         // Close it before final alarm rearming, including on failure or event interruption.
@@ -1116,11 +1456,127 @@ export class ThreadMaintenance extends Context.Service<
 
         const dispatchClosed = yield* Deferred.make<void>();
         const stopDispatch = Deferred.await(dispatchClosed);
+        // Coalesce local scan hints while a lane is busy; one hint requests a current read.
+        const checks = yield* PubSub.sliding<void>(1);
+
+        yield* Effect.addFinalizer(() => PubSub.shutdown(checks));
+        let dispatchOpen = true;
+        let activityChanged = yield* Deferred.make<void>();
+        const signalActivity = Effect.suspend(() => Deferred.succeed(activityChanged, undefined));
+
+        const closeDispatch = Effect.sync(() => {
+          dispatchOpen = false;
+        }).pipe(Effect.andThen(Deferred.succeed(dispatchClosed, undefined)));
+
+        const makeActivity = Effect.fnUntraced(function* (allowance?: number) {
+          const initialized = yield* Deferred.make<void>();
+          let active = 0;
+          let failed = false;
+
+          const ready = Deferred.succeed(initialized, undefined).pipe(
+            Effect.andThen(signalActivity),
+            Effect.asVoid,
+            Effect.uninterruptible,
+          );
+
+          const activity: ThreadMaintenanceActivity["Service"] = {
+            ready,
+            subscribeChanges: PubSub.subscribe(checks).pipe(Effect.map(PubSub.take)),
+            run: (wave) =>
+              Effect.acquireUseRelease(
+                Effect.sync(() => {
+                  if (!dispatchOpen) return false;
+                  active++;
+
+                  return true;
+                }),
+                (admitted) =>
+                  !admitted
+                    ? Effect.void
+                    : (allowance === undefined
+                        ? wave
+                        : wave.pipe(
+                            Effect.timeoutOrElse({
+                              duration: allowance,
+                              orElse: () =>
+                                DurableAlarmError.make({
+                                  operation: "host dispatch allowance",
+                                  message:
+                                    "The admitted host wave exceeded its allowance; durable work remains pending",
+                                }),
+                            }),
+                          )
+                      ).pipe(
+                        Effect.onExit((exit) =>
+                          Effect.sync(() => {
+                            if (Exit.isFailure(exit)) failed = true;
+                          }),
+                        ),
+                      ),
+                (admitted) =>
+                  Effect.sync(() => {
+                    if (admitted) active--;
+                  }).pipe(Effect.andThen(signalActivity)),
+              ),
+          };
+
+          return {
+            activity,
+            initialized: Deferred.await(initialized),
+            quiet: () => active === 0 && Deferred.isDoneUnsafe(initialized),
+            failed: () => failed,
+          };
+        });
+
+        const messageActivity = yield* makeActivity();
+        const hostActivity = yield* makeActivity(host.dispatchTimeoutMillis);
+
+        const recovery: NativeRecovery = {
+          queue: yield* Deferred.make<ReadonlyArray<ThreadId>>(),
+          pending: new Set(),
+          loaded: new Set(),
+          reports: new Map(),
+          faults: new Map(),
+          started: false,
+          needsCheckpoint: false,
+          recovered: 0,
+          repaired: false,
+        };
+
+        const recoveryFiber = yield* Effect.forkIn(
+          Effect.gen(function* () {
+            for (const threadId of yield* Deferred.await(recovery.queue)) {
+              yield* failpoint.hit("maintenance:select:before");
+              yield* runTransaction("select old recovery lane", () =>
+                ctx.storage.transaction(async (transaction) => {
+                  const { state } = await readMaintenanceState(transaction);
+
+                  await transaction.put(
+                    MAINTENANCE_STATE_KEY,
+                    encodeMaintenanceState(
+                      ThreadMaintenanceState.make({ ...state, lastRecoveredThreadId: threadId }),
+                    ),
+                  );
+                }),
+              );
+              yield* failpoint.hit("maintenance:select:after");
+              yield* recoverThread(threadId, recovery);
+              recovery.pending.delete(threadId);
+              yield* wakes.notify(threadId);
+            }
+          }),
+          auxiliaryScope,
+        );
 
         // Fork setup too: an ordinary auxiliary setup failure is reported after native work,
         // rather than gating its opportunity. Event interruption still closes every fiber.
         const deliveryFiber = yield* Effect.forkIn(
-          Scope.provide(auxiliaryScope)(messages.drainUntil(stopDispatch, dispatchUntil)),
+          Scope.provide(auxiliaryScope)(
+            messages.drainUntil(stopDispatch, dispatchUntil).pipe(
+              Effect.provideService(ThreadMaintenanceActivity, messageActivity.activity),
+              Effect.onExit(() => messageActivity.activity.ready),
+            ),
+          ),
           auxiliaryScope,
         );
 
@@ -1137,8 +1593,29 @@ export class ThreadMaintenance extends Context.Service<
                   }),
                 ),
               );
-              yield* host.drainUntil(stopDispatch, dispatchUntil);
-            }),
+
+              // Initial setup must also be finite. Once it is accounted for, each actual
+              // wave has its own unchanged allowance; passive listeners have no timer.
+              const initialization = hostActivity.initialized.pipe(
+                Effect.timeoutOrElse({
+                  duration: host.dispatchTimeoutMillis,
+                  orElse: () =>
+                    DurableAlarmError.make({
+                      operation: "host initialization",
+                      message:
+                        "The host did not account for initial maintenance before its allowance; durable work remains pending",
+                    }),
+                }),
+                Effect.andThen(Effect.never),
+              );
+
+              yield* Effect.raceFirst(
+                host
+                  .drainUntil(stopDispatch, dispatchUntil)
+                  .pipe(Effect.provideService(ThreadMaintenanceActivity, hostActivity.activity)),
+                initialization,
+              );
+            }).pipe(Effect.onExit(() => hostActivity.activity.ready)),
           ),
           auxiliaryScope,
         );
@@ -1148,45 +1625,99 @@ export class ThreadMaintenance extends Context.Service<
           drainDue.pipe(
             Effect.provideService(ThreadProjectionMaintenance, projection),
             Effect.timeoutOption(config.projectionDispatchTimeoutMillis),
+            Effect.onExit(() => signalActivity),
           ),
           auxiliaryScope,
         );
 
-        let result = yield* advance(started, yieldAfter, observed);
+        let result = yield* advance(started, yieldAfter, observed, recovery);
+
+        // This event owns one finite old-recovery wave, including an empty caught-up wave.
+        recovery.started = true;
+        yield* Deferred.succeed(recovery.queue, []);
         let phase = result.phase;
-        let recovered = result.recovered;
         let settled = result.settled;
 
         observed.nativeOnly = false;
 
-        // Close admission of new delivery waves once, then keep advancing native
-        // work while the already-admitted waves finish. Neither lane restarts the
-        // other's work or receives a fresh event budget.
-        yield* Deferred.succeed(dispatchClosed, undefined);
+        // Pump termination and quiescence are distinct: listeners await closure, while their
+        // finite waves report activity. A held sibling never retires another lane's controls.
+        const hostJoin = yield* Effect.forkIn(
+          Effect.gen(function* () {
+            yield* stopDispatch;
 
-        const remaining = Math.max(
-          1,
-          DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis),
-        );
+            const remaining = Math.max(
+              1,
+              DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis),
+            );
 
-        const hostJoin = yield* Effect.forkIn(
-          Fiber.join(hostFiber).pipe(
-            Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)),
-            Effect.tap((outcome) =>
-              Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) }),
-            ),
-          ),
+            const outcome = yield* Fiber.join(hostFiber).pipe(
+              Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)),
+            );
+
+            yield* Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) });
+          }),
           auxiliaryScope,
         );
 
-        const retired = yield* Effect.forkChild(Fiber.joinAll([deliveryFiber, hostJoin, backfill]));
+        const retired = yield* Effect.forkChild(
+          Fiber.joinAll([deliveryFiber, hostJoin, backfill, recoveryFiber]),
+        );
+
+        while (true) {
+          const recoveryFinished = recoveryFiber.pollUnsafe() !== undefined;
+          const retiredExit = retired.pollUnsafe();
+
+          if (recoveryFinished) {
+            if (recovery.needsCheckpoint) {
+              yield* Fiber.join(recoveryFiber);
+              // Fold the original recovery observation into its checkpoint before closing.
+              result = yield* advance(started, yieldAfter, observed, recovery, settled === 0);
+              if (result.phase === "actionable") phase = "actionable";
+              settled += result.settled;
+              recovery.needsCheckpoint = false;
+              yield* PubSub.publish(checks, undefined);
+            }
+          }
+          if (retiredExit !== undefined && Exit.isFailure(retiredExit)) break;
 
-        const auxiliaryPending = () =>
-          deliveryFiber.pollUnsafe() === undefined ||
-          hostFiber.pollUnsafe() === undefined ||
-          backfill.pollUnsafe() === undefined;
+          if (
+            recoveryFinished &&
+            backfill.pollUnsafe() !== undefined &&
+            messageActivity.quiet() &&
+            hostActivity.quiet()
+          ) {
+            const now = yield* Clock.currentTimeMillis;
+            const messageDeadline = yield* messages.pendingDeadline;
+            const hostDeadline = yield* host.pendingDeadline;
+
+            const messageDue =
+              deliveryFiber.pollUnsafe() === undefined &&
+              !messageActivity.failed() &&
+              Option.isSome(messageDeadline) &&
+              messageDeadline.value <= now;
+
+            const hostDue =
+              hostFiber.pollUnsafe() === undefined &&
+              !hostActivity.failed() &&
+              now + host.dispatchTimeoutMillis <= DateTime.toEpochMillis(dispatchUntil) &&
+              Option.isSome(hostDeadline) &&
+              hostDeadline.value <= now;
+
+            // No asynchronous operation separates the activity recheck from closure. A
+            // racing registration either keeps this window open or cannot start its body.
+            const closed = yield* Effect.sync(() => {
+              if (messageDue || hostDue || !messageActivity.quiet() || !hostActivity.quiet())
+                return false;
+
+              dispatchOpen = false;
+
+              return true;
+            });
 
-        while (retired.pollUnsafe() === undefined && auxiliaryPending()) {
+            if (closed) break;
+            yield* PubSub.publish(checks, undefined);
+          }
           const now = yield* Clock.currentTimeMillis;
           const until = DateTime.toEpochMillis(yieldAfter);
 
@@ -1198,24 +1729,50 @@ export class ThreadMaintenance extends Context.Service<
             until,
           );
 
+          // A completion racing this iteration stays armed until the loop handles it.
+          const recoveryDone = recoveryFinished ? Effect.never : Fiber.await(recoveryFiber);
+          const retirementDone = retiredExit === undefined ? Fiber.await(retired) : Effect.never;
+          const changed = activityChanged;
+
           const ready = yield* Effect.raceFirst(
-            Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as(true)),
-            Fiber.await(retired).pipe(Effect.as(false)),
+            Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(
+              Effect.as("native" as const),
+            ),
+            Effect.raceFirst(
+              recoveryDone.pipe(Effect.as("recovery" as const)),
+              Effect.raceFirst(
+                retirementDone.pipe(Effect.as("retired" as const)),
+                Deferred.await(changed).pipe(Effect.as("activity" as const)),
+              ),
+            ),
           );
 
-          if (!ready || retired.pollUnsafe() !== undefined || !auxiliaryPending()) break;
+          if (ready === "recovery" || ready === "retired") continue;
+          if (ready === "activity") activityChanged = yield* Deferred.make<void>();
           if ((yield* Clock.currentTimeMillis) >= until) break;
+          if (ready === "native") yield* PubSub.publish(checks, undefined);
 
-          started = yield* beginNative(observed);
-          result = yield* advance(started, yieldAfter, observed);
+          const awakened = yield* beginNative(observed);
+
+          result = yield* advance(awakened, yieldAfter, observed, recovery);
           if (result.phase === "actionable") phase = "actionable";
-          recovered += result.recovered;
           settled += result.settled;
           observed.nativeOnly = false;
+          if (result.settled > 0) yield* PubSub.publish(checks, undefined);
         }
+        // The original native yield deadline closes all new waves even if old recovery
+        // is still pending. No recovery or delivery receives a renewed event budget.
+        yield* closeDispatch;
         // Preserve driver-owned Claim deadlines and failures, then close every
         // listener before the one final alarm decision.
         yield* Fiber.join(retired);
+        if (recovery.needsCheckpoint) {
+          // The native yield deadline ended dispatch before old recovery finished. Fold its
+          // control state into acknowledgement without starting an Attempt after retirement.
+          result = yield* advance(started, yieldAfter, observed, recovery, false);
+          if (result.phase === "actionable") phase = "actionable";
+          settled += result.settled;
+        }
         yield* Scope.close(auxiliaryScope, Exit.void);
         yield* failpoint.hit("maintenance:finish:before");
 
@@ -1256,7 +1813,7 @@ export class ThreadMaintenance extends Context.Service<
 
         const report = MaintenancePassReport.make({
           phase,
-          recovered,
+          recovered: recovery.recovered,
           settled,
           nonterminal: result.nonterminal,
           alarm: disposition,
@@ -1309,6 +1866,7 @@ export class ThreadMaintenance extends Context.Service<
           }),
         ),
         ensureAlarm: mutations.withSnapshot(() => ensureAlarm()),
+        recoveryStatus,
         withMutation: (body) =>
           mutations.withMutation(
             body.pipe(