@effect-agent/platform-cloudflare 0.1.0-beta.111 → 0.1.0-beta.113

package/dist/Alarm.mjs

@@ -2,11 +2,12 @@ import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
 import { DurableObjectContext } from "./CloudflareBindings.mjs";
 import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.mjs";
 import { r as safeCauseMessage } from "./boundary-BguazVkh.mjs";
-import { Cause, Clock, Context, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Random, Ref, Schema, Scope, Semaphore, Stream, Struct } from "effect";
+import { Cause, Clock, Context, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, PubSub, Random, Ref, Result, Schema, Scope, Semaphore, Stream, Struct } from "effect";
 import "effect-agent/agent-registration";
-import { DurableAgentRuntime } from "effect-agent/durable-agent-runtime";
+import { DurableAgentRuntime, DurableRuntimeConfig, RecoveryBlocked, RecoveryFailure, RecoverySweepResult, isolateRecovery } from "effect-agent/durable-agent-runtime";
 import { SubmissionId, ThreadId } from "effect-agent/identifiers";
-import { SubmissionLedger } from "effect-agent/submission-ledger";
+import { OperationAuthorizationRequest, OperationAuthorizer } from "effect-agent/operation-authorizer";
+import { AbortIntentRequest, SubmissionLedger } from "effect-agent/submission-ledger";
 import { ThreadProjectionMaintenance, drainDue } from "effect-agent/thread-projection-maintenance";
 import { WakeScheduler } from "effect-agent/wake-scheduler";
 import { SqlClient } from "effect/unstable/sql/SqlClient";
@@ -17,10 +18,12 @@ var Alarm_exports = /* @__PURE__ */ __exportAll({
 	MaintenancePassReport: () => MaintenancePassReport,
 	ThreadHostMaintenance: () => ThreadHostMaintenance,
 	ThreadMaintenance: () => ThreadMaintenance,
+	ThreadMaintenanceActivity: () => ThreadMaintenanceActivity,
 	ThreadMaintenanceFailpoint: () => ThreadMaintenanceFailpoint,
 	ThreadMessageDelivery: () => ThreadMessageDelivery,
 	ThreadMutationGate: () => ThreadMutationGate,
 	ThreadPublication: () => ThreadPublication,
+	ThreadRecoveryFault: () => ThreadRecoveryFault,
 	publishCommitted: () => publishCommitted
 });
 /**
@@ -87,7 +90,7 @@ var DurableAlarmService = class DurableAlarmService extends Context.Service()("@
 var MaintenancePassReport = class extends Schema.Class("@effect-agent/platform-cloudflare/MaintenancePassReport")({
 	/** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
 	phase: Schema.Literals(["caught-up", "actionable"]),
-	/** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
+	/** Recovery decisions and Thread faults observed during this event. */
 	recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
 	/** Head Attempts settled during the event. Joined input may settle with each head. */
 	settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
@@ -109,6 +112,27 @@ var ThreadPublication = class extends Context.Service()("@effect-agent/platform-
 		pendingDeadline: Effect.succeed(Option.none())
 	});
 };
+/** Event-local accounting shared by the existing maintenance pumps and native scheduler. */
+var ThreadMaintenanceActivity = class ThreadMaintenanceActivity extends Context.Service()("@effect-agent/platform-cloudflare/ThreadMaintenanceActivity") {
+	/** Join child pumps; acknowledge the parent only after every child is ready or has exited. */
+	static all = Effect.fnUntraced(function* (lanes) {
+		const activity = yield* ThreadMaintenanceActivity;
+		let remaining = lanes.length;
+		if (remaining === 0) return yield* activity.ready.pipe(Effect.as([]));
+		return yield* Effect.forEach(lanes, (lane) => {
+			let ready = false;
+			const child = {
+				...activity,
+				ready: Effect.suspend(() => {
+					if (ready) return Effect.void;
+					ready = true;
+					return --remaining === 0 ? activity.ready : Effect.void;
+				}).pipe(Effect.uninterruptible)
+			};
+			return lane.pipe(Effect.provideService(ThreadMaintenanceActivity, child), Effect.onExit(() => child.ready));
+		}, { concurrency: "unbounded" });
+	});
+};
 /**
 * Host-assembled native message recovery. The driver bounds each actual Claim and persists its
 * timeout/retry before this pump returns. Do not add a second timer starting at batch selection:
@@ -120,14 +144,16 @@ const ThreadMessageDelivery = Context.Reference("@effect-agent/platform-cloudfla
 }) });
 /**
 * Application obligations sharing this Object's alarm. Admit one initial external wave even on
-* a caught-up pass, then respond to wakes until dispatchClosed. This closes only admission
-* of NEW external waves; native work can continue while admitted waves finish. Keep local
-* admission/hub subscriptions in the event Scope until maintenance tears it down.
+* a caught-up pass, then respond to wakes and ThreadMaintenanceActivity.subscribeChanges until
+* dispatchClosed. Yield the event's ThreadMaintenanceActivity to bracket each finite lane with
+* run and acknowledge initial setup with ready.
+* Passive subscriptions do not count as active work. Closure stops both new waves and native
+* Attempts; local admission/control subscriptions belong to the event Scope until teardown.
 * No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
 *
 * Declare a finite whole-wave allowance (1..300000ms): maximum for parallel lanes, sum for
 * sequential operations. Admit a wave only if its allowance fits before dispatchUntil. Later
-* arrivals cannot renew the retirement window. Maintenance bounds the join after dispatchClosed,
+* arrivals cannot renew an admitted wave's allowance. Maintenance bounds each registered wave,
 * interrupts and joins event Scope, then reads local deadlines under the mutation gate.
 *
 * Setup and pendingDeadline are bounded local operations. Persist claims/envelopes before
@@ -155,6 +181,30 @@ var BindingRetry = class extends Schema.Class("BindingRetry")({
 	notBefore: Schema.Finite,
 	reportedAt: Schema.Finite
 }) {};
+/**
+* A durable observation of blocked recovery, independent of the execution journal. This is
+* neither a Settlement nor proof that external effects did not happen. A successful recovery
+* sweep clears it; repair must preserve canonical history and the original admission identity.
+*/
+var ThreadRecoveryFault = class extends Schema.Class("@effect-agent/platform-cloudflare/ThreadRecoveryFault")({
+	schemaVersion: Schema.Literal(1),
+	threadId: ThreadId,
+	firstFailedAt: Schema.Finite,
+	lastFailedAt: Schema.Finite,
+	/** Earliest automatic recovery retry; new admissions do not erase this deadline. */
+	retryAt: Schema.Finite,
+	/** Saturates at 2^31 - 1; one observation per Thread per recovery sweep. */
+	attempts: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(2147483647)),
+	failure: RecoveryFailure
+}) {};
+const recoveryFaultKey = (threadId) => `effect-agent:thread-recovery-fault:v1:${threadId}`;
+const decodeRecoveryFaultValue = Schema.decodeUnknownSync(ThreadRecoveryFault);
+const decodeRecoveryFault = (threadId, encoded) => {
+	const fault = decodeRecoveryFaultValue(encoded);
+	if (fault.threadId !== threadId) throw new Error("Recovery status does not match its Thread key");
+	return fault;
+};
+const encodeRecoveryFault = Schema.encodeSync(ThreadRecoveryFault);
 var MaintenanceRetry = class extends Schema.Class("MaintenanceRetry")({
 	generation: MaintenanceGeneration,
 	notBefore: Schema.Finite,
@@ -169,6 +219,8 @@ var ThreadMaintenanceState = class extends Schema.Class("@effect-agent/platform-
 	nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
 	/** One physical-owner cursor; old single-lane records need no conversion. */
 	lastServedThreadId: Schema.optionalKey(ThreadId),
+	/** Rotate old recovery independently of dispatch, including after eviction or timeout. */
+	lastRecoveredThreadId: Schema.optionalKey(ThreadId),
 	bindingRetries: Schema.optionalKey(Schema.Array(BindingRetry)),
 	/** Absent on older records. A newer mutation makes this retry obsolete. */
 	retry: Schema.optionalKey(MaintenanceRetry)
@@ -258,10 +310,10 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
 *    recovery, ledger scans or canonical-history reads.
 * 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
 *    producer keeps its newer generation dirty. Native retries retain their durable backoff.
-* 3. After the initial native opportunity, stop admitting new external waves and let the
-*    active waves finish. While they remain in flight, native wakes and bounded scans can
-*    advance more heads. All Attempts share the event's original ten-minute yield deadline.
-* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
+* 3. Keep native and delivery admission open together while finite waves remain active, so
+*    fresh replies and abort controls can progress during unrelated cleanup. Close atomically
+*    at quiescence, or at the original ten-minute yield deadline, before retiring listeners.
+* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill waves are
 *    bounded independently; incoming native work never restarts or cancels their attempts.
 *    Auxiliary failures are reported after the current native opportunity.
 * 5. Close every event resource before the final gated deadline snapshot and alarm decision.
@@ -270,6 +322,7 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
 var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
 	static layer = Layer.effect(ThreadMaintenance)(Effect.gen(function* () {
 		const runtime = yield* DurableAgentRuntime;
+		const recoveryConfig = yield* DurableRuntimeConfig;
 		const ledger = yield* SubmissionLedger;
 		const wakes = yield* WakeScheduler;
 		const alarm = yield* DurableAlarmService;
@@ -281,6 +334,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		const projection = yield* ThreadProjectionMaintenance;
 		const messages = yield* ThreadMessageDelivery;
 		const host = yield* ThreadHostMaintenance;
+		const authorizer = yield* OperationAuthorizer;
 		const projectionDeadline = projection.pendingDeadline.pipe(Effect.catchCauseIf((cause) => !Cause.hasInterrupts(cause), (cause) => Effect.logError("Thread projection deadline unavailable", cause).pipe(Effect.as(Option.some(0)))));
 		const pendingDeadline = Effect.gen(function* () {
 			return earliestDeadline(earliestDeadline(yield* publication.pendingDeadline, yield* messages.pendingDeadline), earliestDeadline(yield* projectionDeadline, yield* host.pendingDeadline));
@@ -288,6 +342,65 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		const maintenancePassGate = yield* Semaphore.make(1);
 		const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
 		const runTransaction = yield* makeStorageOperation;
+		const recoveryStatus = Effect.fn("ThreadMaintenance.recoveryStatus")(function* (threadId) {
+			yield* authorizer.authorize(OperationAuthorizationRequest.make({
+				operation: "explain",
+				threadId
+			}));
+			return yield* runTransaction("read Thread recovery status", async () => {
+				const encoded = await ctx.storage.get(recoveryFaultKey(threadId));
+				return encoded === void 0 ? Option.none() : Option.some(decodeRecoveryFault(threadId, encoded));
+			});
+		});
+		const recordRecoveryStatus = Effect.fn("ThreadMaintenance.recordRecoveryStatus")(function* (result) {
+			const threads = /* @__PURE__ */ new Map();
+			for (const report of result.reports) threads.set(report.threadId, void 0);
+			for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
+			if (threads.size === 0) return /* @__PURE__ */ new Map();
+			const now = yield* Clock.currentTimeMillis;
+			yield* failpoint.hit("maintenance:recovery-status:before");
+			const retained = yield* runTransaction("record Thread recovery status", () => ctx.storage.transaction(async (transaction) => {
+				const newlyBlocked = [];
+				const faults = /* @__PURE__ */ new Map();
+				for (const [threadId, failure] of threads) {
+					const key = recoveryFaultKey(threadId);
+					const encoded = await transaction.get(key);
+					const previous = encoded === void 0 ? void 0 : decodeRecoveryFault(threadId, encoded);
+					if (failure === void 0) {
+						if (previous !== void 0) await transaction.delete(key);
+						continue;
+					}
+					const fault = ThreadRecoveryFault.make({
+						schemaVersion: 1,
+						threadId,
+						firstFailedAt: previous?.firstFailedAt ?? now,
+						lastFailedAt: now,
+						attempts: Math.min(2147483647, (previous?.attempts ?? 0) + 1),
+						retryAt: now + Math.min(6e4, 5e3 * 2 ** Math.min(30, previous?.attempts ?? 0)),
+						failure
+					});
+					await transaction.put(key, encodeRecoveryFault(fault));
+					faults.set(threadId, fault);
+					if (previous === void 0) newlyBlocked.push(fault);
+				}
+				return {
+					newlyBlocked,
+					faults
+				};
+			}));
+			yield* failpoint.hit("maintenance:recovery-status:after");
+			for (const fault of retained.newlyBlocked) yield* Effect.logError("Native Thread recovery blocked; accepted work remains pending", fault.failure.reason === "defect" ? Cause.die(fault.failure) : Cause.fail(fault.failure)).pipe(Effect.annotateLogs({ threadId: fault.threadId }));
+			return retained.faults;
+		});
+		const recoverThread = Effect.fn("ThreadMaintenance.recoverThread")(function* (threadId, recovery) {
+			const result = yield* runtime.runRecovery({ threadId });
+			const faults = yield* recordRecoveryStatus(result);
+			for (const report of result.reports) recovery.reports.set(report.submissionId, report);
+			recovery.faults.delete(threadId);
+			for (const [id, fault] of faults) recovery.faults.set(id, fault);
+			recovery.recovered += result.reports.length + result.blocked.length;
+			recovery.repaired ||= result.reports.some((report) => report.disposition === "repaired");
+		});
 		const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
 			yield* failpoint.hit("maintenance:ensure:before");
 			const now = yield* Clock.currentTimeMillis;
@@ -368,7 +481,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				};
 			}));
 		});
-		const advance = Effect.fn("ThreadMaintenance.advance")(function* (started, yieldAfter, observed) {
+		const advance = Effect.fn("ThreadMaintenance.advance")(function* (started, yieldAfter, observed, recovery, dispatch = true) {
 			const deadline = yield* publication.pendingDeadline;
 			if (started._tag === "Actionable" || Option.isSome(deadline) && deadline.value <= (yield* Clock.currentTimeMillis)) yield* publication.drain;
 			const pending = yield* publication.pendingDeadline;
@@ -382,43 +495,91 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				}));
 				return {
 					phase: "caught-up",
-					recovered: 0,
 					settled: 0,
 					nonterminal: started.nonterminal,
 					nextAttemptAt
 				};
 			}
 			observed.nativeOnly = true;
-			const recovered = yield* runtime.runRecovery;
-			const reports = new Map(recovered.map((report) => [report.submissionId, report]));
+			const observation = recovery.observation ??= {
+				generation: started.generation,
+				activeAtStart: started.activeAtStart
+			};
 			const current = yield* Stream.runCollect(ledger.scanNonterminal);
+			const selectionTime = yield* Clock.currentTimeMillis;
+			yield* runTransaction("read Thread recovery deadlines", async () => {
+				for (const threadId of new Set(current.map((row) => row.threadId))) {
+					if (recovery.loaded.has(threadId)) continue;
+					const encoded = await ctx.storage.get(recoveryFaultKey(threadId));
+					if (encoded !== void 0) recovery.faults.set(threadId, decodeRecoveryFault(threadId, encoded));
+					recovery.loaded.add(threadId);
+				}
+			});
+			const reports = recovery.reports;
+			const recoveryFaults = recovery.faults;
+			const waiting = (row) => !recovery.pending.has(row.threadId) && !recoveryFaults.has(row.threadId) && stableExternalWait(row, reports);
 			const heads = /* @__PURE__ */ new Map();
 			for (const row of current) {
-				if (row.state === "unknown" && stableExternalWait(row, reports)) continue;
+				if (row.state === "unknown" && waiting(row)) continue;
 				if (!heads.has(row.threadId)) heads.set(row.threadId, row);
 			}
-			const eligible = [...heads.values()].filter((head) => !stableExternalWait(head, reports)).map((head) => head.threadId).sort();
-			const selectionTime = yield* Clock.currentTimeMillis;
+			const stopping = /* @__PURE__ */ new Set();
+			for (const head of heads.values()) {
+				if (head.state !== "ready" || recovery.pending.has(head.threadId) || recoveryFaults.has(head.threadId)) continue;
+				const intent = yield* isolateRecovery(ledger.readAbortIntent(AbortIntentRequest.make({ submissionId: head.submissionId })), {
+					timeout: recoveryConfig.recoveryTimeout,
+					phase: () => "recovery",
+					operation: "read abort intent"
+				});
+				if (Result.isSuccess(intent)) {
+					if (intent.success !== void 0) stopping.add(head.threadId);
+				} else {
+					const faults = yield* recordRecoveryStatus(RecoverySweepResult.make({
+						reports: [],
+						blocked: [RecoveryBlocked.make({
+							threadId: head.threadId,
+							failure: intent.failure
+						})]
+					}));
+					for (const [threadId, fault] of faults) recoveryFaults.set(threadId, fault);
+					recovery.recovered++;
+				}
+			}
+			const eligible = [...heads.values()].filter((head) => !recovery.pending.has(head.threadId) && !stopping.has(head.threadId) && !recoveryFaults.has(head.threadId) && !waiting(head) && (head.state === "ready" || reports.has(head.submissionId))).map((head) => head.threadId).sort();
 			yield* failpoint.hit("maintenance:select:before");
 			const selection = yield* runTransaction("select maintenance lane", () => ctx.storage.transaction(async (transaction) => {
 				const { state } = await readMaintenanceState(transaction);
 				const retries = (state.bindingRetries ?? []).filter((retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId);
 				const runnable = eligible.filter((threadId) => !retries.some((retry) => retry.threadId === threadId && retry.notBefore > selectionTime));
-				const next = runnable.find((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId) ?? runnable[0];
+				const next = dispatch ? runnable.find((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId) ?? runnable[0] : void 0;
 				if (next !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
 					...state,
 					lastServedThreadId: next
 				})));
+				const backlog = recovery.started ? [] : [...heads.keys()].filter((threadId) => {
+					const fault = recoveryFaults.get(threadId);
+					return !eligible.includes(threadId) && (fault === void 0 || fault.retryAt <= selectionTime);
+				});
+				const after = backlog.filter((threadId) => state.lastRecoveredThreadId === void 0 || threadId > state.lastRecoveredThreadId);
+				const before = backlog.filter((threadId) => state.lastRecoveredThreadId !== void 0 && threadId <= state.lastRecoveredThreadId);
 				return {
 					selected: next,
-					retries
+					retries,
+					backlog: [...after, ...before]
 				};
 			}));
 			yield* failpoint.hit("maintenance:select:after");
+			if (!recovery.started) {
+				recovery.started = true;
+				recovery.needsCheckpoint = selection.backlog.length > 0;
+				for (const threadId of selection.backlog) recovery.pending.add(threadId);
+				yield* Deferred.succeed(recovery.queue, selection.backlog);
+			}
 			const selected = selection.selected === void 0 ? void 0 : heads.get(selection.selected);
+			if (selected !== void 0) yield* recoverThread(selected.threadId, recovery);
 			let retries = selection.retries;
 			let bindingFailure;
-			const settlement = selected === void 0 ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.catchTag("BindingUnavailable", (failure) => {
+			const settlement = selected === void 0 || recoveryFaults.has(selected.threadId) ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.catchTag("BindingUnavailable", (failure) => {
 				bindingFailure = failure;
 				return Effect.succeed(Option.none());
 			}));
@@ -456,22 +617,22 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
 			const waitingHeads = /* @__PURE__ */ new Map();
 			const autonomous = remaining.some((snapshot) => {
-				if (snapshot.state === "unknown" && stableExternalWait(snapshot, reports)) return false;
+				if (snapshot.state === "unknown" && waiting(snapshot)) return false;
 				const headWaiting = waitingHeads.get(snapshot.threadId);
-				if (headWaiting === void 0) waitingHeads.set(snapshot.threadId, stableExternalWait(snapshot, reports));
+				if (headWaiting === void 0) waitingHeads.set(snapshot.threadId, waiting(snapshot));
 				if (headWaiting === true && snapshot.state === "ready" && reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput") return false;
-				return !stableExternalWait(snapshot, reports);
+				return !waiting(snapshot);
 			});
-			const progressed = Option.isSome(settlement) || recovered.some((report) => report.disposition === "repaired");
+			const progressed = Option.isSome(settlement) || recovery.repaired;
 			const now = yield* Clock.currentTimeMillis;
 			const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
-			const nextEligible = eligible.map((threadId) => retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)?.notBefore ?? now);
-			const bindingDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
-			const delay = Math.max(ordinaryDelay, bindingDelay);
+			const nextEligible = [...recoveryFaults.values()].map((fault) => fault.retryAt).concat(eligible.map((threadId) => retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)?.notBefore ?? now));
+			const retryDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
+			const delay = Math.max(ordinaryDelay, retryDelay);
 			yield* failpoint.hit("maintenance:checkpoint:before");
 			const nextAttemptAt = yield* mutations.withSnapshot((active) => runTransaction("checkpoint native maintenance", () => ctx.storage.transaction(async (transaction) => {
 				const { state } = await readMaintenanceState(transaction);
-				const processed = autonomous || started.activeAtStart > 0 || active > 0 ? state.processed : state.processed > started.generation ? state.processed : started.generation;
+				const processed = autonomous || observation.activeAtStart > 0 || started.activeAtStart > 0 || active > 0 ? state.processed : state.processed > observation.generation ? state.processed : observation.generation;
 				const next = ThreadMaintenanceState.make({
 					...Struct.omit(state, ["retry"]),
 					processed,
@@ -491,7 +652,6 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			yield* failpoint.hit("maintenance:checkpoint:after");
 			return {
 				phase: "actionable",
-				recovered: recovered.length,
 				settled: Option.isSome(settlement) ? 1 : 0,
 				nonterminal: remaining.length,
 				nextAttemptAt
@@ -499,49 +659,164 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		});
 		const pass = Effect.fn("ThreadMaintenance.pass")(function* (yieldAfter, dispatchUntil, observed) {
 			const notified = (yield* Stream.toPull(wakes.wakes)).pipe(Effect.asVoid, Effect.catch(() => Effect.never));
-			let started = yield* beginNative(observed);
+			const started = yield* beginNative(observed);
 			const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) => Scope.close(scope, exit));
 			const dispatchClosed = yield* Deferred.make();
 			const stopDispatch = Deferred.await(dispatchClosed);
-			const deliveryFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(messages.drainUntil(stopDispatch, dispatchUntil)), auxiliaryScope);
+			const checks = yield* PubSub.sliding(1);
+			yield* Effect.addFinalizer(() => PubSub.shutdown(checks));
+			let dispatchOpen = true;
+			let activityChanged = yield* Deferred.make();
+			const signalActivity = Effect.suspend(() => Deferred.succeed(activityChanged, void 0));
+			const closeDispatch = Effect.sync(() => {
+				dispatchOpen = false;
+			}).pipe(Effect.andThen(Deferred.succeed(dispatchClosed, void 0)));
+			const makeActivity = Effect.fnUntraced(function* (allowance) {
+				const initialized = yield* Deferred.make();
+				let active = 0;
+				let failed = false;
+				return {
+					activity: {
+						ready: Deferred.succeed(initialized, void 0).pipe(Effect.andThen(signalActivity), Effect.asVoid, Effect.uninterruptible),
+						subscribeChanges: PubSub.subscribe(checks).pipe(Effect.map(PubSub.take)),
+						run: (wave) => Effect.acquireUseRelease(Effect.sync(() => {
+							if (!dispatchOpen) return false;
+							active++;
+							return true;
+						}), (admitted) => !admitted ? Effect.void : (allowance === void 0 ? wave : wave.pipe(Effect.timeoutOrElse({
+							duration: allowance,
+							orElse: () => DurableAlarmError.make({
+								operation: "host dispatch allowance",
+								message: "The admitted host wave exceeded its allowance; durable work remains pending"
+							})
+						}))).pipe(Effect.onExit((exit) => Effect.sync(() => {
+							if (Exit.isFailure(exit)) failed = true;
+						}))), (admitted) => Effect.sync(() => {
+							if (admitted) active--;
+						}).pipe(Effect.andThen(signalActivity)))
+					},
+					initialized: Deferred.await(initialized),
+					quiet: () => active === 0 && Deferred.isDoneUnsafe(initialized),
+					failed: () => failed
+				};
+			});
+			const messageActivity = yield* makeActivity();
+			const hostActivity = yield* makeActivity(host.dispatchTimeoutMillis);
+			const recovery = {
+				queue: yield* Deferred.make(),
+				pending: /* @__PURE__ */ new Set(),
+				loaded: /* @__PURE__ */ new Set(),
+				reports: /* @__PURE__ */ new Map(),
+				faults: /* @__PURE__ */ new Map(),
+				started: false,
+				needsCheckpoint: false,
+				recovered: 0,
+				repaired: false
+			};
+			const recoveryFiber = yield* Effect.forkIn(Effect.gen(function* () {
+				for (const threadId of yield* Deferred.await(recovery.queue)) {
+					yield* failpoint.hit("maintenance:select:before");
+					yield* runTransaction("select old recovery lane", () => ctx.storage.transaction(async (transaction) => {
+						const { state } = await readMaintenanceState(transaction);
+						await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
+							...state,
+							lastRecoveredThreadId: threadId
+						})));
+					}));
+					yield* failpoint.hit("maintenance:select:after");
+					yield* recoverThread(threadId, recovery);
+					recovery.pending.delete(threadId);
+					yield* wakes.notify(threadId);
+				}
+			}), auxiliaryScope);
+			const deliveryFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(messages.drainUntil(stopDispatch, dispatchUntil).pipe(Effect.provideService(ThreadMaintenanceActivity, messageActivity.activity), Effect.onExit(() => messageActivity.activity.ready))), auxiliaryScope);
 			const hostFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(Effect.gen(function* () {
 				yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(Effect.mapError((cause) => DurableAlarmError.make({
 					operation: "host dispatch allowance",
 					message: "Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
 					cause
 				})));
-				yield* host.drainUntil(stopDispatch, dispatchUntil);
-			})), auxiliaryScope);
-			const backfill = yield* Effect.forkIn(drainDue.pipe(Effect.provideService(ThreadProjectionMaintenance, projection), Effect.timeoutOption(config.projectionDispatchTimeoutMillis)), auxiliaryScope);
-			let result = yield* advance(started, yieldAfter, observed);
+				const initialization = hostActivity.initialized.pipe(Effect.timeoutOrElse({
+					duration: host.dispatchTimeoutMillis,
+					orElse: () => DurableAlarmError.make({
+						operation: "host initialization",
+						message: "The host did not account for initial maintenance before its allowance; durable work remains pending"
+					})
+				}), Effect.andThen(Effect.never));
+				yield* Effect.raceFirst(host.drainUntil(stopDispatch, dispatchUntil).pipe(Effect.provideService(ThreadMaintenanceActivity, hostActivity.activity)), initialization);
+			}).pipe(Effect.onExit(() => hostActivity.activity.ready))), auxiliaryScope);
+			const backfill = yield* Effect.forkIn(drainDue.pipe(Effect.provideService(ThreadProjectionMaintenance, projection), Effect.timeoutOption(config.projectionDispatchTimeoutMillis), Effect.onExit(() => signalActivity)), auxiliaryScope);
+			let result = yield* advance(started, yieldAfter, observed, recovery);
+			recovery.started = true;
+			yield* Deferred.succeed(recovery.queue, []);
 			let phase = result.phase;
-			let recovered = result.recovered;
 			let settled = result.settled;
 			observed.nativeOnly = false;
-			yield* Deferred.succeed(dispatchClosed, void 0);
-			const remaining = Math.max(1, DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis));
-			const hostJoin = yield* Effect.forkIn(Fiber.join(hostFiber).pipe(Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)), Effect.tap((outcome) => Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) }))), auxiliaryScope);
+			const hostJoin = yield* Effect.forkIn(Effect.gen(function* () {
+				yield* stopDispatch;
+				const remaining = Math.max(1, DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis));
+				const outcome = yield* Fiber.join(hostFiber).pipe(Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)));
+				yield* Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) });
+			}), auxiliaryScope);
 			const retired = yield* Effect.forkChild(Fiber.joinAll([
 				deliveryFiber,
 				hostJoin,
-				backfill
+				backfill,
+				recoveryFiber
 			]));
-			const auxiliaryPending = () => deliveryFiber.pollUnsafe() === void 0 || hostFiber.pollUnsafe() === void 0 || backfill.pollUnsafe() === void 0;
-			while (retired.pollUnsafe() === void 0 && auxiliaryPending()) {
+			while (true) {
+				const recoveryFinished = recoveryFiber.pollUnsafe() !== void 0;
+				const retiredExit = retired.pollUnsafe();
+				if (recoveryFinished) {
+					if (recovery.needsCheckpoint) {
+						yield* Fiber.join(recoveryFiber);
+						result = yield* advance(started, yieldAfter, observed, recovery, settled === 0);
+						if (result.phase === "actionable") phase = "actionable";
+						settled += result.settled;
+						recovery.needsCheckpoint = false;
+						yield* PubSub.publish(checks, void 0);
+					}
+				}
+				if (retiredExit !== void 0 && Exit.isFailure(retiredExit)) break;
+				if (recoveryFinished && backfill.pollUnsafe() !== void 0 && messageActivity.quiet() && hostActivity.quiet()) {
+					const now = yield* Clock.currentTimeMillis;
+					const messageDeadline = yield* messages.pendingDeadline;
+					const hostDeadline = yield* host.pendingDeadline;
+					const messageDue = deliveryFiber.pollUnsafe() === void 0 && !messageActivity.failed() && Option.isSome(messageDeadline) && messageDeadline.value <= now;
+					const hostDue = hostFiber.pollUnsafe() === void 0 && !hostActivity.failed() && now + host.dispatchTimeoutMillis <= DateTime.toEpochMillis(dispatchUntil) && Option.isSome(hostDeadline) && hostDeadline.value <= now;
+					if (yield* Effect.sync(() => {
+						if (messageDue || hostDue || !messageActivity.quiet() || !hostActivity.quiet()) return false;
+						dispatchOpen = false;
+						return true;
+					})) break;
+					yield* PubSub.publish(checks, void 0);
+				}
 				const now = yield* Clock.currentTimeMillis;
 				const until = DateTime.toEpochMillis(yieldAfter);
 				if (now >= until) break;
 				const next = Math.min(result.nextAttemptAt ?? Infinity, now + config.wakeScanInterval, until);
-				if (!(yield* Effect.raceFirst(Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as(true)), Fiber.await(retired).pipe(Effect.as(false)))) || retired.pollUnsafe() !== void 0 || !auxiliaryPending()) break;
+				const recoveryDone = recoveryFinished ? Effect.never : Fiber.await(recoveryFiber);
+				const retirementDone = retiredExit === void 0 ? Fiber.await(retired) : Effect.never;
+				const changed = activityChanged;
+				const ready = yield* Effect.raceFirst(Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as("native")), Effect.raceFirst(recoveryDone.pipe(Effect.as("recovery")), Effect.raceFirst(retirementDone.pipe(Effect.as("retired")), Deferred.await(changed).pipe(Effect.as("activity")))));
+				if (ready === "recovery" || ready === "retired") continue;
+				if (ready === "activity") activityChanged = yield* Deferred.make();
 				if ((yield* Clock.currentTimeMillis) >= until) break;
-				started = yield* beginNative(observed);
-				result = yield* advance(started, yieldAfter, observed);
+				if (ready === "native") yield* PubSub.publish(checks, void 0);
+				const awakened = yield* beginNative(observed);
+				result = yield* advance(awakened, yieldAfter, observed, recovery);
 				if (result.phase === "actionable") phase = "actionable";
-				recovered += result.recovered;
 				settled += result.settled;
 				observed.nativeOnly = false;
+				if (result.settled > 0) yield* PubSub.publish(checks, void 0);
 			}
+			yield* closeDispatch;
 			yield* Fiber.join(retired);
+			if (recovery.needsCheckpoint) {
+				result = yield* advance(started, yieldAfter, observed, recovery, false);
+				if (result.phase === "actionable") phase = "actionable";
+				settled += result.settled;
+			}
 			yield* Scope.close(auxiliaryScope, Exit.void);
 			yield* failpoint.hit("maintenance:finish:before");
 			const disposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
@@ -562,7 +837,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			yield* failpoint.hit("maintenance:finish:after");
 			const report = MaintenancePassReport.make({
 				phase,
-				recovered,
+				recovered: recovery.recovered,
 				settled,
 				nonterminal: result.nonterminal,
 				alarm: disposition
@@ -591,11 +866,12 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				})
 			})),
 			ensureAlarm: mutations.withSnapshot(() => ensureAlarm()),
+			recoveryStatus,
 			withMutation: (body) => mutations.withMutation(body.pipe(Effect.tap(() => publishCommitted.pipe(Effect.provideService(ThreadPublication, publication)))))
 		});
 	}));
 };
 //#endregion
-export { DurableAlarmError, DurableAlarmService, MaintenancePassReport, ThreadHostMaintenance, ThreadMaintenance, ThreadMaintenanceFailpoint, ThreadMessageDelivery, ThreadMutationGate, ThreadPublication, publishCommitted, Alarm_exports as t };
+export { DurableAlarmError, DurableAlarmService, MaintenancePassReport, ThreadHostMaintenance, ThreadMaintenance, ThreadMaintenanceActivity, ThreadMaintenanceFailpoint, ThreadMessageDelivery, ThreadMutationGate, ThreadPublication, ThreadRecoveryFault, publishCommitted, Alarm_exports as t };
 
 //# sourceMappingURL=Alarm.mjs.map