@effect-agent/platform-cloudflare 0.1.0-beta.111 → 0.1.0-beta.112

package/dist/Alarm.d.mts

@@ -2,13 +2,15 @@ import { DurableObjectContext } from "./CloudflareBindings.mjs";
 import { s as CloudflareDurableRuntimeConfig } from "./CloudflareConfig-DKXGo8L3.mjs";
 import { Cause, Context, DateTime, Effect, Layer, Option, Schema, Scope } from "effect";
 import { DurableBindingFailure } from "effect-agent/agent-registration";
-import { DurableAgentRuntime, DurableWorkerFailure } from "effect-agent/durable-agent-runtime";
+import { DurableAgentRuntime, DurableRuntimeConfig, DurableWorkerFailure, RecoveryFailure } from "effect-agent/durable-agent-runtime";
+import { ThreadId } from "effect-agent/identifiers";
+import { OperationDenied } from "effect-agent/operation-authorizer";
 import { SubmissionLedger } from "effect-agent/submission-ledger";
 import { ThreadProjectionError, ThreadProjectionMaintenance } from "effect-agent/thread-projection-maintenance";
 import { WakeScheduler } from "effect-agent/wake-scheduler";
 import { SqlClient } from "effect/unstable/sql/SqlClient";
 declare namespace Alarm_d_exports {
-  export { DurableAlarmError, DurableAlarmService, MaintenancePassFailure, MaintenancePassReport, ThreadHostMaintenance, ThreadMaintenance, ThreadMaintenanceFailpoint, ThreadMaintenanceFailpointHandler, ThreadMaintenanceFailpointLocation, ThreadMessageDelivery, ThreadMutationGate, ThreadPublication, ThreadPublicationService, publishCommitted };
+  export { DurableAlarmError, DurableAlarmService, MaintenancePassFailure, MaintenancePassReport, ThreadHostMaintenance, ThreadMaintenance, ThreadMaintenanceFailpoint, ThreadMaintenanceFailpointHandler, ThreadMaintenanceFailpointLocation, ThreadMessageDelivery, ThreadMutationGate, ThreadPublication, ThreadPublicationService, ThreadRecoveryFault, publishCommitted };
 }
 declare const DurableAlarmError_base: Schema.Class<DurableAlarmError, Schema.TaggedStruct<"DurableAlarmError", {
   readonly operation: Schema.String;
@@ -64,7 +66,7 @@ export declare class DurableAlarmService extends DurableAlarmService_base {
 declare const MaintenancePassReport_base: Schema.Class<MaintenancePassReport, Schema.Struct<{
   /** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
   readonly phase: Schema.Literals<readonly ["caught-up", "actionable"]>;
-  /** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
+  /** Recovery decisions and Thread faults observed during this event. */
   readonly recovered: Schema.Int;
   /** Head Attempts settled during the event. Joined input may settle with each head. */
   readonly settled: Schema.Int;
@@ -76,7 +78,7 @@ declare const MaintenancePassReport_base: Schema.Class<MaintenancePassReport, Sc
 /** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
 export declare class MaintenancePassReport extends MaintenancePassReport_base {}
 /** Fault boundaries around every maintenance-owned durable mutation. */
-export type ThreadMaintenanceFailpointLocation = "maintenance:dirty:before" | "maintenance:dirty:after" | "maintenance:mutation:armed" | "maintenance:mutation:finished" | "maintenance:ensure:before" | "maintenance:ensure:after" | "maintenance:begin:before" | "maintenance:begin:after" | "maintenance:select:before" | "maintenance:select:after" | "maintenance:binding-retry:before" | "maintenance:binding-retry:after" | "maintenance:retry:before" | "maintenance:retry:after" | "maintenance:checkpoint:before" | "maintenance:checkpoint:after" | "maintenance:finish:before" | "maintenance:finish:after";
+export type ThreadMaintenanceFailpointLocation = "maintenance:dirty:before" | "maintenance:dirty:after" | "maintenance:mutation:armed" | "maintenance:mutation:finished" | "maintenance:ensure:before" | "maintenance:ensure:after" | "maintenance:begin:before" | "maintenance:begin:after" | "maintenance:select:before" | "maintenance:select:after" | "maintenance:binding-retry:before" | "maintenance:binding-retry:after" | "maintenance:recovery-status:before" | "maintenance:recovery-status:after" | "maintenance:retry:before" | "maintenance:retry:after" | "maintenance:checkpoint:before" | "maintenance:checkpoint:after" | "maintenance:finish:before" | "maintenance:finish:after";
 export type ThreadMaintenanceFailpointHandler = (location: ThreadMaintenanceFailpointLocation) => Effect.Effect<void>;
 declare const ThreadMaintenanceFailpoint_base: Context.ServiceClass<ThreadMaintenanceFailpoint, "@effect-agent/platform-cloudflare/ThreadMaintenanceFailpoint", {
   readonly hit: ThreadMaintenanceFailpointHandler;
@@ -146,6 +148,23 @@ export declare const ThreadHostMaintenance: Context.Reference<{
 }>;
 /** @internal A committed source operation must not become a failed operation because delivery failed. */
 export declare const publishCommitted: Effect.Effect<void, never, ThreadPublication>;
+declare const ThreadRecoveryFault_base: Schema.Class<ThreadRecoveryFault, Schema.Struct<{
+  readonly schemaVersion: Schema.Literal<1>;
+  readonly threadId: Schema.brand<Schema.NonEmptyString, "@effect-agent/core/ThreadId">;
+  readonly firstFailedAt: Schema.Finite;
+  readonly lastFailedAt: Schema.Finite;
+  /** Earliest automatic recovery retry; new admissions do not erase this deadline. */
+  readonly retryAt: Schema.Finite;
+  /** Saturates at 2^31 - 1; one observation per Thread per recovery sweep. */
+  readonly attempts: Schema.Int;
+  readonly failure: typeof RecoveryFailure;
+}>, {}>;
+/**
+ * A durable observation of blocked recovery, independent of the execution journal. This is
+ * neither a Settlement nor proof that external effects did not happen. A successful recovery
+ * sweep clears it; repair must preserve canonical history and the original admission identity.
+ */
+export declare class ThreadRecoveryFault extends ThreadRecoveryFault_base {}
 declare const ThreadMutationGate_base: Context.ServiceClass<ThreadMutationGate, "@effect-agent/platform-cloudflare/internal/ThreadMutationGate", {
   readonly withMutation: <A, E, R>(body: Effect.Effect<A, E, R>,
   /**
@@ -175,6 +194,13 @@ declare const ThreadMaintenance_base: Context.ServiceClass<ThreadMaintenance, "@
    * generation has an alarm. It never scans the ledger or canonical history.
    */
   readonly ensureAlarm: Effect.Effect<void, MaintenancePassFailure>;
+  /**
+   * Authorize `explain`, then read one bounded local record without reading execution history.
+   * None means no recorded fault, not proof of health or settlement. The host authenticates
+   * callers and verifies local Thread membership before exposing this service across RPC.
+   * Provide OperationAuthorizer when constructing this Layer, as for DurableAgentRuntime.
+   */
+  readonly recoveryStatus: (threadId: ThreadId) => Effect.Effect<Option.Option<ThreadRecoveryFault>, DurableAlarmError | OperationDenied>;
   /**
    * Serialize the pre-arm boundary with pass acknowledgement, advance the durable dirty
    * generation and arm the alarm in one transaction BEFORE running the caller's mutation.
@@ -191,9 +217,9 @@ declare const ThreadMaintenance_base: Context.ServiceClass<ThreadMaintenance, "@
  *    recovery, ledger scans or canonical-history reads.
  * 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
  *    producer keeps its newer generation dirty. Native retries retain their durable backoff.
- * 3. After the initial native opportunity, stop admitting new external waves and let the
- *    active waves finish. While they remain in flight, native wakes and bounded scans can
- *    advance more heads. All Attempts share the event's original ten-minute yield deadline.
+ * 3. Keep delivery admission open while old recovery is pending, so fresh native replies can
+ *    publish in this event. Close once, at latest at the original ten-minute yield deadline;
+ *    native wakes can still advance heads while admitted deliveries retire.
  * 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
  *    bounded independently; incoming native work never restarts or cancels their attempts.
  *    Auxiliary failures are reported after the current native opportunity.
@@ -201,7 +227,7 @@ declare const ThreadMaintenance_base: Context.ServiceClass<ThreadMaintenance, "@
  *    The whole event retains one fourteen-minute cooperative timeout.
  */
 export declare class ThreadMaintenance extends ThreadMaintenance_base {
-  static readonly layer: Layer.Layer<ThreadMaintenance, never, ThreadMutationGate | ThreadPublication | ThreadProjectionMaintenance | DurableAgentRuntime | SubmissionLedger | WakeScheduler | DurableAlarmService | ThreadMaintenanceFailpoint | CloudflareDurableRuntimeConfig | DurableObjectContext | SqlClient>;
+  static readonly layer: Layer.Layer<ThreadMaintenance, never, ThreadMutationGate | ThreadPublication | ThreadProjectionMaintenance | DurableAgentRuntime | DurableRuntimeConfig | SubmissionLedger | WakeScheduler | DurableAlarmService | ThreadMaintenanceFailpoint | CloudflareDurableRuntimeConfig | DurableObjectContext | SqlClient>;
 }
 //#endregion
 export { Alarm_d_exports as t };

package/dist/Alarm.mjs

@@ -2,11 +2,12 @@ import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
 import { DurableObjectContext } from "./CloudflareBindings.mjs";
 import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.mjs";
 import { r as safeCauseMessage } from "./boundary-BguazVkh.mjs";
-import { Cause, Clock, Context, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Random, Ref, Schema, Scope, Semaphore, Stream, Struct } from "effect";
+import { Cause, Clock, Context, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Random, Ref, Result, Schema, Scope, Semaphore, Stream, Struct } from "effect";
 import "effect-agent/agent-registration";
-import { DurableAgentRuntime } from "effect-agent/durable-agent-runtime";
+import { DurableAgentRuntime, DurableRuntimeConfig, RecoveryBlocked, RecoveryFailure, RecoverySweepResult, isolateRecovery } from "effect-agent/durable-agent-runtime";
 import { SubmissionId, ThreadId } from "effect-agent/identifiers";
-import { SubmissionLedger } from "effect-agent/submission-ledger";
+import { OperationAuthorizationRequest, OperationAuthorizer } from "effect-agent/operation-authorizer";
+import { AbortIntentRequest, SubmissionLedger } from "effect-agent/submission-ledger";
 import { ThreadProjectionMaintenance, drainDue } from "effect-agent/thread-projection-maintenance";
 import { WakeScheduler } from "effect-agent/wake-scheduler";
 import { SqlClient } from "effect/unstable/sql/SqlClient";
@@ -21,6 +22,7 @@ var Alarm_exports = /* @__PURE__ */ __exportAll({
 	ThreadMessageDelivery: () => ThreadMessageDelivery,
 	ThreadMutationGate: () => ThreadMutationGate,
 	ThreadPublication: () => ThreadPublication,
+	ThreadRecoveryFault: () => ThreadRecoveryFault,
 	publishCommitted: () => publishCommitted
 });
 /**
@@ -87,7 +89,7 @@ var DurableAlarmService = class DurableAlarmService extends Context.Service()("@
 var MaintenancePassReport = class extends Schema.Class("@effect-agent/platform-cloudflare/MaintenancePassReport")({
 	/** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
 	phase: Schema.Literals(["caught-up", "actionable"]),
-	/** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
+	/** Recovery decisions and Thread faults observed during this event. */
 	recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
 	/** Head Attempts settled during the event. Joined input may settle with each head. */
 	settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
@@ -155,6 +157,30 @@ var BindingRetry = class extends Schema.Class("BindingRetry")({
 	notBefore: Schema.Finite,
 	reportedAt: Schema.Finite
 }) {};
+/**
+* A durable observation of blocked recovery, independent of the execution journal. This is
+* neither a Settlement nor proof that external effects did not happen. A successful recovery
+* sweep clears it; repair must preserve canonical history and the original admission identity.
+*/
+var ThreadRecoveryFault = class extends Schema.Class("@effect-agent/platform-cloudflare/ThreadRecoveryFault")({
+	schemaVersion: Schema.Literal(1),
+	threadId: ThreadId,
+	firstFailedAt: Schema.Finite,
+	lastFailedAt: Schema.Finite,
+	/** Earliest automatic recovery retry; new admissions do not erase this deadline. */
+	retryAt: Schema.Finite,
+	/** Saturates at 2^31 - 1; one observation per Thread per recovery sweep. */
+	attempts: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(2147483647)),
+	failure: RecoveryFailure
+}) {};
+const recoveryFaultKey = (threadId) => `effect-agent:thread-recovery-fault:v1:${threadId}`;
+const decodeRecoveryFaultValue = Schema.decodeUnknownSync(ThreadRecoveryFault);
+const decodeRecoveryFault = (threadId, encoded) => {
+	const fault = decodeRecoveryFaultValue(encoded);
+	if (fault.threadId !== threadId) throw new Error("Recovery status does not match its Thread key");
+	return fault;
+};
+const encodeRecoveryFault = Schema.encodeSync(ThreadRecoveryFault);
 var MaintenanceRetry = class extends Schema.Class("MaintenanceRetry")({
 	generation: MaintenanceGeneration,
 	notBefore: Schema.Finite,
@@ -169,6 +195,8 @@ var ThreadMaintenanceState = class extends Schema.Class("@effect-agent/platform-
 	nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
 	/** One physical-owner cursor; old single-lane records need no conversion. */
 	lastServedThreadId: Schema.optionalKey(ThreadId),
+	/** Rotate old recovery independently of dispatch, including after eviction or timeout. */
+	lastRecoveredThreadId: Schema.optionalKey(ThreadId),
 	bindingRetries: Schema.optionalKey(Schema.Array(BindingRetry)),
 	/** Absent on older records. A newer mutation makes this retry obsolete. */
 	retry: Schema.optionalKey(MaintenanceRetry)
@@ -258,9 +286,9 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
 *    recovery, ledger scans or canonical-history reads.
 * 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
 *    producer keeps its newer generation dirty. Native retries retain their durable backoff.
-* 3. After the initial native opportunity, stop admitting new external waves and let the
-*    active waves finish. While they remain in flight, native wakes and bounded scans can
-*    advance more heads. All Attempts share the event's original ten-minute yield deadline.
+* 3. Keep delivery admission open while old recovery is pending, so fresh native replies can
+*    publish in this event. Close once, at latest at the original ten-minute yield deadline;
+*    native wakes can still advance heads while admitted deliveries retire.
 * 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
 *    bounded independently; incoming native work never restarts or cancels their attempts.
 *    Auxiliary failures are reported after the current native opportunity.
@@ -270,6 +298,7 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
 var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
 	static layer = Layer.effect(ThreadMaintenance)(Effect.gen(function* () {
 		const runtime = yield* DurableAgentRuntime;
+		const recoveryConfig = yield* DurableRuntimeConfig;
 		const ledger = yield* SubmissionLedger;
 		const wakes = yield* WakeScheduler;
 		const alarm = yield* DurableAlarmService;
@@ -281,6 +310,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		const projection = yield* ThreadProjectionMaintenance;
 		const messages = yield* ThreadMessageDelivery;
 		const host = yield* ThreadHostMaintenance;
+		const authorizer = yield* OperationAuthorizer;
 		const projectionDeadline = projection.pendingDeadline.pipe(Effect.catchCauseIf((cause) => !Cause.hasInterrupts(cause), (cause) => Effect.logError("Thread projection deadline unavailable", cause).pipe(Effect.as(Option.some(0)))));
 		const pendingDeadline = Effect.gen(function* () {
 			return earliestDeadline(earliestDeadline(yield* publication.pendingDeadline, yield* messages.pendingDeadline), earliestDeadline(yield* projectionDeadline, yield* host.pendingDeadline));
@@ -288,6 +318,65 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		const maintenancePassGate = yield* Semaphore.make(1);
 		const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
 		const runTransaction = yield* makeStorageOperation;
+		const recoveryStatus = Effect.fn("ThreadMaintenance.recoveryStatus")(function* (threadId) {
+			yield* authorizer.authorize(OperationAuthorizationRequest.make({
+				operation: "explain",
+				threadId
+			}));
+			return yield* runTransaction("read Thread recovery status", async () => {
+				const encoded = await ctx.storage.get(recoveryFaultKey(threadId));
+				return encoded === void 0 ? Option.none() : Option.some(decodeRecoveryFault(threadId, encoded));
+			});
+		});
+		const recordRecoveryStatus = Effect.fn("ThreadMaintenance.recordRecoveryStatus")(function* (result) {
+			const threads = /* @__PURE__ */ new Map();
+			for (const report of result.reports) threads.set(report.threadId, void 0);
+			for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
+			if (threads.size === 0) return /* @__PURE__ */ new Map();
+			const now = yield* Clock.currentTimeMillis;
+			yield* failpoint.hit("maintenance:recovery-status:before");
+			const retained = yield* runTransaction("record Thread recovery status", () => ctx.storage.transaction(async (transaction) => {
+				const newlyBlocked = [];
+				const faults = /* @__PURE__ */ new Map();
+				for (const [threadId, failure] of threads) {
+					const key = recoveryFaultKey(threadId);
+					const encoded = await transaction.get(key);
+					const previous = encoded === void 0 ? void 0 : decodeRecoveryFault(threadId, encoded);
+					if (failure === void 0) {
+						if (previous !== void 0) await transaction.delete(key);
+						continue;
+					}
+					const fault = ThreadRecoveryFault.make({
+						schemaVersion: 1,
+						threadId,
+						firstFailedAt: previous?.firstFailedAt ?? now,
+						lastFailedAt: now,
+						attempts: Math.min(2147483647, (previous?.attempts ?? 0) + 1),
+						retryAt: now + Math.min(6e4, 5e3 * 2 ** Math.min(30, previous?.attempts ?? 0)),
+						failure
+					});
+					await transaction.put(key, encodeRecoveryFault(fault));
+					faults.set(threadId, fault);
+					if (previous === void 0) newlyBlocked.push(fault);
+				}
+				return {
+					newlyBlocked,
+					faults
+				};
+			}));
+			yield* failpoint.hit("maintenance:recovery-status:after");
+			for (const fault of retained.newlyBlocked) yield* Effect.logError("Native Thread recovery blocked; accepted work remains pending", fault.failure.reason === "defect" ? Cause.die(fault.failure) : Cause.fail(fault.failure)).pipe(Effect.annotateLogs({ threadId: fault.threadId }));
+			return retained.faults;
+		});
+		const recoverThread = Effect.fn("ThreadMaintenance.recoverThread")(function* (threadId, recovery) {
+			const result = yield* runtime.runRecovery({ threadId });
+			const faults = yield* recordRecoveryStatus(result);
+			for (const report of result.reports) recovery.reports.set(report.submissionId, report);
+			recovery.faults.delete(threadId);
+			for (const [id, fault] of faults) recovery.faults.set(id, fault);
+			recovery.recovered += result.reports.length + result.blocked.length;
+			recovery.repaired ||= result.reports.some((report) => report.disposition === "repaired");
+		});
 		const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
 			yield* failpoint.hit("maintenance:ensure:before");
 			const now = yield* Clock.currentTimeMillis;
@@ -368,7 +457,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				};
 			}));
 		});
-		const advance = Effect.fn("ThreadMaintenance.advance")(function* (started, yieldAfter, observed) {
+		const advance = Effect.fn("ThreadMaintenance.advance")(function* (started, yieldAfter, observed, recovery, dispatch = true) {
 			const deadline = yield* publication.pendingDeadline;
 			if (started._tag === "Actionable" || Option.isSome(deadline) && deadline.value <= (yield* Clock.currentTimeMillis)) yield* publication.drain;
 			const pending = yield* publication.pendingDeadline;
@@ -382,43 +471,91 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				}));
 				return {
 					phase: "caught-up",
-					recovered: 0,
 					settled: 0,
 					nonterminal: started.nonterminal,
 					nextAttemptAt
 				};
 			}
 			observed.nativeOnly = true;
-			const recovered = yield* runtime.runRecovery;
-			const reports = new Map(recovered.map((report) => [report.submissionId, report]));
+			const observation = recovery.observation ??= {
+				generation: started.generation,
+				activeAtStart: started.activeAtStart
+			};
 			const current = yield* Stream.runCollect(ledger.scanNonterminal);
+			const selectionTime = yield* Clock.currentTimeMillis;
+			yield* runTransaction("read Thread recovery deadlines", async () => {
+				for (const threadId of new Set(current.map((row) => row.threadId))) {
+					if (recovery.loaded.has(threadId)) continue;
+					const encoded = await ctx.storage.get(recoveryFaultKey(threadId));
+					if (encoded !== void 0) recovery.faults.set(threadId, decodeRecoveryFault(threadId, encoded));
+					recovery.loaded.add(threadId);
+				}
+			});
+			const reports = recovery.reports;
+			const recoveryFaults = recovery.faults;
+			const waiting = (row) => !recovery.pending.has(row.threadId) && !recoveryFaults.has(row.threadId) && stableExternalWait(row, reports);
 			const heads = /* @__PURE__ */ new Map();
 			for (const row of current) {
-				if (row.state === "unknown" && stableExternalWait(row, reports)) continue;
+				if (row.state === "unknown" && waiting(row)) continue;
 				if (!heads.has(row.threadId)) heads.set(row.threadId, row);
 			}
-			const eligible = [...heads.values()].filter((head) => !stableExternalWait(head, reports)).map((head) => head.threadId).sort();
-			const selectionTime = yield* Clock.currentTimeMillis;
+			const stopping = /* @__PURE__ */ new Set();
+			for (const head of heads.values()) {
+				if (head.state !== "ready" || recovery.pending.has(head.threadId) || recoveryFaults.has(head.threadId)) continue;
+				const intent = yield* isolateRecovery(ledger.readAbortIntent(AbortIntentRequest.make({ submissionId: head.submissionId })), {
+					timeout: recoveryConfig.recoveryTimeout,
+					phase: () => "recovery",
+					operation: "read abort intent"
+				});
+				if (Result.isSuccess(intent)) {
+					if (intent.success !== void 0) stopping.add(head.threadId);
+				} else {
+					const faults = yield* recordRecoveryStatus(RecoverySweepResult.make({
+						reports: [],
+						blocked: [RecoveryBlocked.make({
+							threadId: head.threadId,
+							failure: intent.failure
+						})]
+					}));
+					for (const [threadId, fault] of faults) recoveryFaults.set(threadId, fault);
+					recovery.recovered++;
+				}
+			}
+			const eligible = [...heads.values()].filter((head) => !recovery.pending.has(head.threadId) && !stopping.has(head.threadId) && !recoveryFaults.has(head.threadId) && !waiting(head) && (head.state === "ready" || reports.has(head.submissionId))).map((head) => head.threadId).sort();
 			yield* failpoint.hit("maintenance:select:before");
 			const selection = yield* runTransaction("select maintenance lane", () => ctx.storage.transaction(async (transaction) => {
 				const { state } = await readMaintenanceState(transaction);
 				const retries = (state.bindingRetries ?? []).filter((retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId);
 				const runnable = eligible.filter((threadId) => !retries.some((retry) => retry.threadId === threadId && retry.notBefore > selectionTime));
-				const next = runnable.find((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId) ?? runnable[0];
+				const next = dispatch ? runnable.find((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId) ?? runnable[0] : void 0;
 				if (next !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
 					...state,
 					lastServedThreadId: next
 				})));
+				const backlog = recovery.started ? [] : [...heads.keys()].filter((threadId) => {
+					const fault = recoveryFaults.get(threadId);
+					return !eligible.includes(threadId) && (fault === void 0 || fault.retryAt <= selectionTime);
+				});
+				const after = backlog.filter((threadId) => state.lastRecoveredThreadId === void 0 || threadId > state.lastRecoveredThreadId);
+				const before = backlog.filter((threadId) => state.lastRecoveredThreadId !== void 0 && threadId <= state.lastRecoveredThreadId);
 				return {
 					selected: next,
-					retries
+					retries,
+					backlog: [...after, ...before]
 				};
 			}));
 			yield* failpoint.hit("maintenance:select:after");
+			if (!recovery.started) {
+				recovery.started = true;
+				recovery.needsCheckpoint = selection.backlog.length > 0;
+				for (const threadId of selection.backlog) recovery.pending.add(threadId);
+				yield* Deferred.succeed(recovery.queue, selection.backlog);
+			}
 			const selected = selection.selected === void 0 ? void 0 : heads.get(selection.selected);
+			if (selected !== void 0) yield* recoverThread(selected.threadId, recovery);
 			let retries = selection.retries;
 			let bindingFailure;
-			const settlement = selected === void 0 ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.catchTag("BindingUnavailable", (failure) => {
+			const settlement = selected === void 0 || recoveryFaults.has(selected.threadId) ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.catchTag("BindingUnavailable", (failure) => {
 				bindingFailure = failure;
 				return Effect.succeed(Option.none());
 			}));
@@ -456,22 +593,22 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
 			const waitingHeads = /* @__PURE__ */ new Map();
 			const autonomous = remaining.some((snapshot) => {
-				if (snapshot.state === "unknown" && stableExternalWait(snapshot, reports)) return false;
+				if (snapshot.state === "unknown" && waiting(snapshot)) return false;
 				const headWaiting = waitingHeads.get(snapshot.threadId);
-				if (headWaiting === void 0) waitingHeads.set(snapshot.threadId, stableExternalWait(snapshot, reports));
+				if (headWaiting === void 0) waitingHeads.set(snapshot.threadId, waiting(snapshot));
 				if (headWaiting === true && snapshot.state === "ready" && reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput") return false;
-				return !stableExternalWait(snapshot, reports);
+				return !waiting(snapshot);
 			});
-			const progressed = Option.isSome(settlement) || recovered.some((report) => report.disposition === "repaired");
+			const progressed = Option.isSome(settlement) || recovery.repaired;
 			const now = yield* Clock.currentTimeMillis;
 			const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
-			const nextEligible = eligible.map((threadId) => retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)?.notBefore ?? now);
-			const bindingDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
-			const delay = Math.max(ordinaryDelay, bindingDelay);
+			const nextEligible = [...recoveryFaults.values()].map((fault) => fault.retryAt).concat(eligible.map((threadId) => retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)?.notBefore ?? now));
+			const retryDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
+			const delay = Math.max(ordinaryDelay, retryDelay);
 			yield* failpoint.hit("maintenance:checkpoint:before");
 			const nextAttemptAt = yield* mutations.withSnapshot((active) => runTransaction("checkpoint native maintenance", () => ctx.storage.transaction(async (transaction) => {
 				const { state } = await readMaintenanceState(transaction);
-				const processed = autonomous || started.activeAtStart > 0 || active > 0 ? state.processed : state.processed > started.generation ? state.processed : started.generation;
+				const processed = autonomous || observation.activeAtStart > 0 || started.activeAtStart > 0 || active > 0 ? state.processed : state.processed > observation.generation ? state.processed : observation.generation;
 				const next = ThreadMaintenanceState.make({
 					...Struct.omit(state, ["retry"]),
 					processed,
@@ -491,7 +628,6 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			yield* failpoint.hit("maintenance:checkpoint:after");
 			return {
 				phase: "actionable",
-				recovered: recovered.length,
 				settled: Option.isSome(settlement) ? 1 : 0,
 				nonterminal: remaining.length,
 				nextAttemptAt
@@ -499,8 +635,35 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		});
 		const pass = Effect.fn("ThreadMaintenance.pass")(function* (yieldAfter, dispatchUntil, observed) {
 			const notified = (yield* Stream.toPull(wakes.wakes)).pipe(Effect.asVoid, Effect.catch(() => Effect.never));
-			let started = yield* beginNative(observed);
+			const started = yield* beginNative(observed);
 			const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) => Scope.close(scope, exit));
+			const recovery = {
+				queue: yield* Deferred.make(),
+				pending: /* @__PURE__ */ new Set(),
+				loaded: /* @__PURE__ */ new Set(),
+				reports: /* @__PURE__ */ new Map(),
+				faults: /* @__PURE__ */ new Map(),
+				started: false,
+				needsCheckpoint: false,
+				recovered: 0,
+				repaired: false
+			};
+			const recoveryFiber = yield* Effect.forkIn(Effect.gen(function* () {
+				for (const threadId of yield* Deferred.await(recovery.queue)) {
+					yield* failpoint.hit("maintenance:select:before");
+					yield* runTransaction("select old recovery lane", () => ctx.storage.transaction(async (transaction) => {
+						const { state } = await readMaintenanceState(transaction);
+						await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
+							...state,
+							lastRecoveredThreadId: threadId
+						})));
+					}));
+					yield* failpoint.hit("maintenance:select:after");
+					yield* recoverThread(threadId, recovery);
+					recovery.pending.delete(threadId);
+					yield* wakes.notify(threadId);
+				}
+			}), auxiliaryScope);
 			const dispatchClosed = yield* Deferred.make();
 			const stopDispatch = Deferred.await(dispatchClosed);
 			const deliveryFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(messages.drainUntil(stopDispatch, dispatchUntil)), auxiliaryScope);
@@ -513,35 +676,61 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				yield* host.drainUntil(stopDispatch, dispatchUntil);
 			})), auxiliaryScope);
 			const backfill = yield* Effect.forkIn(drainDue.pipe(Effect.provideService(ThreadProjectionMaintenance, projection), Effect.timeoutOption(config.projectionDispatchTimeoutMillis)), auxiliaryScope);
-			let result = yield* advance(started, yieldAfter, observed);
+			let result = yield* advance(started, yieldAfter, observed, recovery);
+			recovery.started = true;
+			yield* Deferred.succeed(recovery.queue, []);
 			let phase = result.phase;
-			let recovered = result.recovered;
 			let settled = result.settled;
 			observed.nativeOnly = false;
-			yield* Deferred.succeed(dispatchClosed, void 0);
-			const remaining = Math.max(1, DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis));
-			const hostJoin = yield* Effect.forkIn(Fiber.join(hostFiber).pipe(Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)), Effect.tap((outcome) => Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) }))), auxiliaryScope);
+			const hostJoin = yield* Effect.forkIn(Effect.gen(function* () {
+				yield* stopDispatch;
+				const remaining = Math.max(1, DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis));
+				const outcome = yield* Fiber.join(hostFiber).pipe(Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)));
+				yield* Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) });
+			}), auxiliaryScope);
 			const retired = yield* Effect.forkChild(Fiber.joinAll([
 				deliveryFiber,
 				hostJoin,
-				backfill
+				backfill,
+				recoveryFiber
 			]));
-			const auxiliaryPending = () => deliveryFiber.pollUnsafe() === void 0 || hostFiber.pollUnsafe() === void 0 || backfill.pollUnsafe() === void 0;
-			while (retired.pollUnsafe() === void 0 && auxiliaryPending()) {
+			const auxiliaryPending = () => deliveryFiber.pollUnsafe() === void 0 || hostFiber.pollUnsafe() === void 0 || backfill.pollUnsafe() === void 0 || recoveryFiber.pollUnsafe() === void 0;
+			while (true) {
+				const recoveryFinished = recoveryFiber.pollUnsafe() !== void 0;
+				if (recoveryFinished) {
+					if (recovery.needsCheckpoint) {
+						yield* Fiber.join(recoveryFiber);
+						result = yield* advance(started, yieldAfter, observed, recovery, settled === 0);
+						if (result.phase === "actionable") phase = "actionable";
+						settled += result.settled;
+						recovery.needsCheckpoint = false;
+					}
+					yield* Deferred.succeed(dispatchClosed, void 0);
+				}
+				if (retired.pollUnsafe() !== void 0 || !auxiliaryPending()) break;
 				const now = yield* Clock.currentTimeMillis;
 				const until = DateTime.toEpochMillis(yieldAfter);
 				if (now >= until) break;
 				const next = Math.min(result.nextAttemptAt ?? Infinity, now + config.wakeScanInterval, until);
-				if (!(yield* Effect.raceFirst(Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as(true)), Fiber.await(retired).pipe(Effect.as(false)))) || retired.pollUnsafe() !== void 0 || !auxiliaryPending()) break;
+				const recoveryDone = recoveryFinished ? Effect.never : Fiber.await(recoveryFiber);
+				const ready = yield* Effect.raceFirst(Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as("native")), Effect.raceFirst(recoveryDone.pipe(Effect.as("recovery")), Fiber.await(retired).pipe(Effect.as("retired"))));
+				if (ready === "recovery") continue;
+				if (ready === "retired" || retired.pollUnsafe() !== void 0) break;
+				if (!auxiliaryPending()) continue;
 				if ((yield* Clock.currentTimeMillis) >= until) break;
-				started = yield* beginNative(observed);
-				result = yield* advance(started, yieldAfter, observed);
+				const awakened = yield* beginNative(observed);
+				result = yield* advance(awakened, yieldAfter, observed, recovery);
 				if (result.phase === "actionable") phase = "actionable";
-				recovered += result.recovered;
 				settled += result.settled;
 				observed.nativeOnly = false;
 			}
+			yield* Deferred.succeed(dispatchClosed, void 0);
 			yield* Fiber.join(retired);
+			if (recovery.needsCheckpoint) {
+				result = yield* advance(started, yieldAfter, observed, recovery, false);
+				if (result.phase === "actionable") phase = "actionable";
+				settled += result.settled;
+			}
 			yield* Scope.close(auxiliaryScope, Exit.void);
 			yield* failpoint.hit("maintenance:finish:before");
 			const disposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
@@ -562,7 +751,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			yield* failpoint.hit("maintenance:finish:after");
 			const report = MaintenancePassReport.make({
 				phase,
-				recovered,
+				recovered: recovery.recovered,
 				settled,
 				nonterminal: result.nonterminal,
 				alarm: disposition
@@ -591,11 +780,12 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				})
 			})),
 			ensureAlarm: mutations.withSnapshot(() => ensureAlarm()),
+			recoveryStatus,
 			withMutation: (body) => mutations.withMutation(body.pipe(Effect.tap(() => publishCommitted.pipe(Effect.provideService(ThreadPublication, publication)))))
 		});
 	}));
 };
 //#endregion
-export { DurableAlarmError, DurableAlarmService, MaintenancePassReport, ThreadHostMaintenance, ThreadMaintenance, ThreadMaintenanceFailpoint, ThreadMessageDelivery, ThreadMutationGate, ThreadPublication, publishCommitted, Alarm_exports as t };
+export { DurableAlarmError, DurableAlarmService, MaintenancePassReport, ThreadHostMaintenance, ThreadMaintenance, ThreadMaintenanceFailpoint, ThreadMessageDelivery, ThreadMutationGate, ThreadPublication, ThreadRecoveryFault, publishCommitted, Alarm_exports as t };
 
 //# sourceMappingURL=Alarm.mjs.map