@effect-agent/platform-cloudflare 0.1.0-beta.95 → 0.1.0-beta.96

package/dist/Alarm.d.mts

@@ -1,5 +1,5 @@
 import { DurableObjectContext } from "./CloudflareBindings.mjs";
-import { s as CloudflareDurableRuntimeConfig } from "./CloudflareConfig-D472JOQA.mjs";
+import { s as CloudflareDurableRuntimeConfig } from "./CloudflareConfig-DKXGo8L3.mjs";
 import { Cause, Context, DateTime, Effect, Layer, Option, Schema, Scope } from "effect";
 import { DurableBindingFailure } from "effect-agent/agent-registration";
 import { DurableAgentRuntime, DurableWorkerFailure } from "effect-agent/durable-agent-runtime";
@@ -75,7 +75,7 @@ declare const MaintenancePassReport_base: Schema.Class<MaintenancePassReport, Sc
 /** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
 declare class MaintenancePassReport extends MaintenancePassReport_base {}
 /** Fault boundaries around every maintenance-owned durable mutation. */
-type ThreadMaintenanceFailpointLocation = "maintenance:dirty:before" | "maintenance:dirty:after" | "maintenance:mutation:armed" | "maintenance:mutation:finished" | "maintenance:ensure:before" | "maintenance:ensure:after" | "maintenance:begin:before" | "maintenance:begin:after" | "maintenance:select:before" | "maintenance:select:after" | "maintenance:finish:before" | "maintenance:finish:after";
+type ThreadMaintenanceFailpointLocation = "maintenance:dirty:before" | "maintenance:dirty:after" | "maintenance:mutation:armed" | "maintenance:mutation:finished" | "maintenance:ensure:before" | "maintenance:ensure:after" | "maintenance:begin:before" | "maintenance:begin:after" | "maintenance:select:before" | "maintenance:select:after" | "maintenance:binding-retry:before" | "maintenance:binding-retry:after" | "maintenance:retry:before" | "maintenance:retry:after" | "maintenance:finish:before" | "maintenance:finish:after";
 type ThreadMaintenanceFailpointHandler = (location: ThreadMaintenanceFailpointLocation) => Effect.Effect<void>;
 declare const ThreadMaintenanceFailpoint_base: Context.ServiceClass<ThreadMaintenanceFailpoint, "@effect-agent/platform-cloudflare/ThreadMaintenanceFailpoint", {
   readonly hit: ThreadMaintenanceFailpointHandler;
@@ -167,7 +167,7 @@ declare class ThreadMutationGate extends ThreadMutationGate_base {
 }
 type MaintenancePassFailure = DurableWorkerFailure | DurableBindingFailure | DurableAlarmError | ThreadProjectionError;
 declare const ThreadMaintenance_base: Context.ServiceClass<ThreadMaintenance, "@effect-agent/platform-cloudflare/ThreadMaintenance", {
-  /** One idempotent maintenance pass; failures propagate so workerd retries the alarm. */
+  /** One idempotent pass; failures propagate after durably scheduling bounded recovery. */
   readonly pass: Effect.Effect<MaintenancePassReport, MaintenancePassFailure>;
   /**
    * Constructor gate: initialize/inspect only the O(1) maintenance record and ensure a dirty

package/dist/Alarm.mjs

@@ -2,10 +2,10 @@ import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
 import { DurableObjectContext } from "./CloudflareBindings.mjs";
 import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.mjs";
 import { r as safeCauseMessage } from "./boundary-BguazVkh.mjs";
-import { Cause, Clock, Context, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Random, Ref, Schema, Scope, Semaphore, Stream } from "effect";
+import { Cause, Clock, Context, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Random, Ref, Schema, Scope, Semaphore, Stream, Struct } from "effect";
 import "effect-agent/agent-registration";
 import { DurableAgentRuntime } from "effect-agent/durable-agent-runtime";
-import { ThreadId } from "effect-agent/identifiers";
+import { SubmissionId, ThreadId } from "effect-agent/identifiers";
 import { SubmissionLedger } from "effect-agent/submission-ledger";
 import { ThreadProjectionMaintenance, drainDue } from "effect-agent/thread-projection-maintenance";
 import { SqlClient } from "effect/unstable/sql/SqlClient";
@@ -147,6 +147,19 @@ const publishCommitted = Effect.gen(function* () {
 	yield* publication.invalidate.pipe(Effect.andThen(publication.drain));
 }).pipe(Effect.catchCause((cause) => Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.logError("Thread publication deferred after source commit", cause)));
 const MaintenanceGeneration = Schema.BigIntFromString.check(Schema.isGreaterThanOrEqualToBigInt(0n));
+var BindingRetry = class extends Schema.Class("BindingRetry")({
+	threadId: ThreadId,
+	submissionId: SubmissionId,
+	attempts: Schema.Natural,
+	notBefore: Schema.Finite,
+	reportedAt: Schema.Finite
+}) {};
+var MaintenanceRetry = class extends Schema.Class("MaintenanceRetry")({
+	generation: MaintenanceGeneration,
+	notBefore: Schema.Finite,
+	nativeOnly: Schema.Boolean,
+	stalls: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(30))
+}) {};
 /** Versioned, platform-private maintenance state stored through Durable Object KV. */
 var ThreadMaintenanceState = class extends Schema.Class("@effect-agent/platform-cloudflare/ThreadMaintenanceState")({
 	schemaVersion: Schema.Literal(1),
@@ -154,7 +167,10 @@ var ThreadMaintenanceState = class extends Schema.Class("@effect-agent/platform-
 	processed: MaintenanceGeneration,
 	nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
 	/** One physical-owner cursor; old single-lane records need no conversion. */
-	lastServedThreadId: Schema.optionalKey(ThreadId)
+	lastServedThreadId: Schema.optionalKey(ThreadId),
+	bindingRetries: Schema.optionalKey(Schema.Array(BindingRetry)),
+	/** Absent on older records. A newer mutation makes this retry obsolete. */
+	retry: Schema.optionalKey(MaintenanceRetry)
 }) {};
 const MAINTENANCE_STATE_KEY = "effect-agent:thread-maintenance:v1";
 const decodeMaintenanceState = Schema.decodeUnknownSync(ThreadMaintenanceState);
@@ -258,11 +274,6 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		const config = yield* CloudflareDurableRuntimeConfig;
 		const { ctx } = yield* DurableObjectContext;
 		const failpoint = yield* ThreadMaintenanceFailpoint;
-		/**
-		* Consecutive no-progress passes — an in-memory CACHE, not state: a fresh incarnation
-		* restarts at zero and merely re-arms sooner than a long-lived one would have.
-		*/
-		const stalls = yield* Ref.make(0);
 		const mutations = yield* ThreadMutationGate;
 		const publication = yield* ThreadPublication;
 		const projection = yield* ThreadProjectionMaintenance;
@@ -278,48 +289,74 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
 			yield* failpoint.hit("maintenance:ensure:before");
 			const now = yield* Clock.currentTimeMillis;
-			yield* runTransaction("ensure maintenance alarm", () => ctx.storage.transaction(async (transaction) => {
+			const retry = yield* runTransaction("ensure maintenance alarm", () => ctx.storage.transaction(async (transaction) => {
 				const { state, initialized } = await readMaintenanceState(transaction);
 				if (!initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
-				if (state.dirty > state.processed) await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
+				if (state.dirty > state.processed) await ensureTransactionAlarmBy(transaction, state.retry?.generation === state.dirty ? Math.max(now + minimumAlarmDelay, state.retry.notBefore) : now + config.wakeScanInterval);
+				return state.retry?.generation === state.dirty ? state.retry : void 0;
 			}));
 			const deadline = yield* pendingDeadline;
-			if (Option.isSome(deadline)) yield* runTransaction("ensure publication alarm", () => ctx.storage.transaction((transaction) => ensureTransactionAlarmBy(transaction, Math.max(now + minimumAlarmDelay, deadline.value))));
+			if (Option.isSome(deadline)) yield* runTransaction("ensure publication alarm", () => ctx.storage.transaction((transaction) => ensureTransactionAlarmBy(transaction, Math.max(now + minimumAlarmDelay, deadline.value <= now && retry !== void 0 && !retry.nativeOnly ? Math.max(deadline.value, retry.notBefore) : deadline.value))));
 			yield* failpoint.hit("maintenance:ensure:after");
 		});
-		const beginPass = Effect.fn("ThreadMaintenance.beginPass")(function* () {
+		const beginPass = Effect.fn("ThreadMaintenance.beginPass")(function* (observed) {
 			yield* failpoint.hit("maintenance:begin:before");
 			const now = yield* Clock.currentTimeMillis;
 			const result = yield* runTransaction("begin maintenance pass", () => ctx.storage.transaction(async (transaction) => {
 				const { state, initialized } = await readMaintenanceState(transaction);
+				observed.generation = state.dirty;
 				if (!initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
-				if (state.processed >= state.dirty) {
-					await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
-					return {
-						_tag: "CaughtUp",
-						nonterminal: state.nonterminal
-					};
-				}
+				const retryAt = state.retry?.generation === state.dirty ? state.retry.notBefore : 0;
 				await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
+				if (state.processed >= state.dirty || retryAt > now) return {
+					_tag: "CaughtUp",
+					nonterminal: state.nonterminal
+				};
 				return {
 					_tag: "Actionable",
 					generation: state.dirty,
-					nonterminal: state.nonterminal
+					nonterminal: state.nonterminal,
+					stalls: state.retry?.generation === state.dirty ? state.retry.stalls : 0
 				};
 			}));
 			yield* failpoint.hit("maintenance:begin:after");
 			return result;
 		});
-		const rearmDelay = Effect.fn("ThreadMaintenance.rearmDelay")(function* (progressed) {
-			const priorStalls = yield* Ref.getAndUpdate(stalls, (count) => progressed ? 0 : count + 1);
-			if (progressed) return config.alarmBackoffBase;
-			const exponent = Math.min(priorStalls, 30);
-			const backoff = Math.min(config.alarmBackoffCap, config.alarmBackoffBase * 2 ** exponent);
-			const jitter = yield* Random.next;
-			const jittered = Math.ceil(backoff / 2 + backoff / 2 * jitter);
-			return Math.min(jittered, config.wakeScanInterval);
+		const backoffDelay = (priorStalls, jitter) => {
+			const backoff = Math.min(config.alarmBackoffCap, config.alarmBackoffBase * 2 ** Math.min(priorStalls, 30));
+			return Math.ceil(backoff / 2 + backoff / 2 * jitter);
+		};
+		const rearmDelay = Effect.fn("ThreadMaintenance.rearmDelay")(function* (progressed, priorStalls) {
+			return progressed ? config.alarmBackoffBase : backoffDelay(priorStalls, yield* Random.next);
 		});
-		const pass = Effect.fn("ThreadMaintenance.pass")(function* (yieldAfter, dispatchUntil) {
+		const rearmFailure = Effect.fn("ThreadMaintenance.rearmFailure")(function* (observed) {
+			const { generation, nativeOnly } = observed;
+			if (generation === void 0) return;
+			yield* failpoint.hit("maintenance:retry:before");
+			yield* mutations.withSnapshot((active) => Effect.gen(function* () {
+				const deadline = yield* pendingDeadline.pipe(Effect.catchCause(() => Effect.succeed(Option.none())));
+				const now = yield* Clock.currentTimeMillis;
+				const jitter = yield* Random.next;
+				yield* runTransaction("back off failed maintenance", () => ctx.storage.transaction(async (transaction) => {
+					const { state } = await readMaintenanceState(transaction);
+					const previous = state.retry?.generation === generation ? state.retry : void 0;
+					const retry = MaintenanceRetry.make({
+						generation,
+						notBefore: Math.max(previous?.notBefore ?? 0, now + backoffDelay(previous?.stalls ?? 0, jitter)),
+						nativeOnly,
+						stalls: Math.min(30, (previous?.stalls ?? 0) + 1)
+					});
+					await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
+						...state,
+						retry
+					})));
+					const nativeDeadline = active > 0 || state.dirty !== generation ? now + minimumAlarmDelay : retry.notBefore;
+					await transaction.setAlarm(Math.max(now + minimumAlarmDelay, Option.isSome(deadline) && (nativeOnly || deadline.value > now) ? Math.min(nativeDeadline, deadline.value) : nativeDeadline));
+				}));
+			}));
+			yield* failpoint.hit("maintenance:retry:after");
+		});
+		const pass = Effect.fn("ThreadMaintenance.pass")(function* (yieldAfter, dispatchUntil, observed) {
 			const annotate = (report) => Effect.annotateCurrentSpan({
 				phase: report.phase,
 				recovered: report.recovered,
@@ -328,7 +365,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				alarm: report.alarm
 			}).pipe(Effect.as(report));
 			const started = yield* mutations.withSnapshot((activeAtStart) => Effect.gen(function* () {
-				const generation = yield* beginPass();
+				const generation = yield* beginPass(observed);
 				if (generation._tag === "Actionable" && activeAtStart === 0) yield* publication.prepareGeneration(generation.generation);
 				return {
 					...generation,
@@ -370,7 +407,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 					const now = yield* Clock.currentTimeMillis;
 					return yield* runTransaction("finish publication pass", () => ctx.storage.transaction(async (transaction) => {
 						const { state } = await readMaintenanceState(transaction);
-						const nativeDeadline = active > 0 || state.dirty > state.processed ? now + config.wakeScanInterval : Infinity;
+						const nativeDeadline = active > 0 || state.dirty > state.processed ? state.retry?.generation === state.dirty && active === 0 ? Math.max(now + minimumAlarmDelay, state.retry.notBefore) : now + config.wakeScanInterval : Infinity;
 						const next = Option.isSome(latest) ? Math.min(nativeDeadline, latest.value) : nativeDeadline;
 						if (Number.isFinite(next)) {
 							await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
@@ -389,27 +426,69 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 					alarm: disposition
 				}));
 			}
+			observed.nativeOnly = true;
 			const recovered = yield* runtime.runRecovery;
 			const reports = new Map(recovered.map((report) => [report.submissionId, report]));
 			const current = yield* Stream.runCollect(ledger.scanNonterminal);
 			const heads = /* @__PURE__ */ new Map();
 			for (const row of current) if (!heads.has(row.threadId)) heads.set(row.threadId, row);
 			const eligible = [...heads.values()].filter((head) => !stableExternalWait(head, reports)).map((head) => head.threadId).sort();
-			let selected = eligible[0];
-			if (heads.size > 1 && selected !== void 0) {
-				yield* failpoint.hit("maintenance:select:before");
-				selected = yield* runTransaction("select maintenance lane", () => ctx.storage.transaction(async (transaction) => {
-					const { state } = await readMaintenanceState(transaction);
-					const next = eligible.find((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId) ?? eligible[0];
-					if (next !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
-						...state,
-						lastServedThreadId: next
-					})));
-					return next;
-				}));
-				yield* failpoint.hit("maintenance:select:after");
+			const selectionTime = yield* Clock.currentTimeMillis;
+			yield* failpoint.hit("maintenance:select:before");
+			const selection = yield* runTransaction("select maintenance lane", () => ctx.storage.transaction(async (transaction) => {
+				const { state } = await readMaintenanceState(transaction);
+				const retries = (state.bindingRetries ?? []).filter((retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId);
+				const runnable = eligible.filter((threadId) => !retries.some((retry) => retry.threadId === threadId && retry.notBefore > selectionTime));
+				const next = runnable.find((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId) ?? runnable[0];
+				if (next !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
+					...state,
+					lastServedThreadId: next
+				})));
+				return {
+					selected: next,
+					retries
+				};
+			}));
+			yield* failpoint.hit("maintenance:select:after");
+			const selected = selection.selected === void 0 ? void 0 : heads.get(selection.selected);
+			let retries = selection.retries;
+			let bindingFailure;
+			const settlement = selected === void 0 ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.catchTag(["BindingUnavailable", "BindingDigestMismatch"], (failure) => {
+				bindingFailure = failure;
+				return Effect.succeed(Option.none());
+			}));
+			if (selected !== void 0) {
+				const previous = retries.find((retry) => retry.submissionId === selected.submissionId);
+				let retry;
+				let reportBindingFailure = false;
+				if (bindingFailure !== void 0) {
+					const now = yield* Clock.currentTimeMillis;
+					const attempts = Math.min(30, (previous?.attempts ?? 0) + 1);
+					reportBindingFailure = previous === void 0 || now - previous.reportedAt >= 9e5;
+					retry = BindingRetry.make({
+						threadId: selected.threadId,
+						submissionId: selected.submissionId,
+						attempts,
+						notBefore: now + Math.min(6e4, 5e3 * 2 ** (attempts - 1)),
+						reportedAt: reportBindingFailure ? now : previous?.reportedAt ?? now
+					});
+				}
+				if (retry !== void 0 || previous !== void 0) {
+					yield* failpoint.hit("maintenance:binding-retry:before");
+					retries = yield* runTransaction("record submission binding retry", () => ctx.storage.transaction(async (transaction) => {
+						const { state } = await readMaintenanceState(transaction);
+						const bindingRetries = [...(state.bindingRetries ?? []).filter((entry) => entry.submissionId !== selected.submissionId), ...retry === void 0 ? [] : [retry]];
+						await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
+							...state,
+							bindingRetries
+						})));
+						return bindingRetries;
+					}));
+					yield* failpoint.hit("maintenance:binding-retry:after");
+				}
+				if (bindingFailure !== void 0) yield* reportBindingFailure ? Effect.logError("Thread awaits a compatible binding; original work remains pending", Cause.fail(bindingFailure)) : Effect.logDebug("Thread binding retry remains pending", Cause.fail(bindingFailure));
 			}
-			const settlement = selected === void 0 ? Option.none() : yield* runtime.processThreadHead(selected, { yieldAfter });
+			observed.nativeOnly = false;
 			yield* finishAuxiliary;
 			const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
 			const waitingHeads = /* @__PURE__ */ new Map();
@@ -420,8 +499,11 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				return !stableExternalWait(snapshot, reports);
 			});
 			const progressed = Option.isSome(settlement) || recovered.some((report) => report.disposition === "repaired");
-			const delay = autonomous ? yield* rearmDelay(progressed) : 0;
 			const now = yield* Clock.currentTimeMillis;
+			const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
+			const nextEligible = eligible.map((threadId) => retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)?.notBefore ?? now);
+			const bindingDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
+			const delay = Math.max(ordinaryDelay, bindingDelay);
 			yield* failpoint.hit("maintenance:finish:before");
 			const alarmDisposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
 				const publicationDeadline = yield* pendingDeadline;
@@ -429,13 +511,21 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 					const { state } = await readMaintenanceState(transaction);
 					const processed = autonomous || started.activeAtStart > 0 || active > 0 ? state.processed : state.processed > started.generation ? state.processed : started.generation;
 					const next = ThreadMaintenanceState.make({
-						...state,
+						...Struct.omit(state, ["retry"]),
 						processed,
-						nonterminal: remaining.length
+						nonterminal: remaining.length,
+						bindingRetries: (state.bindingRetries ?? []).filter((retry) => remaining.some((row) => row.submissionId === retry.submissionId)),
+						...autonomous && !progressed ? { retry: MaintenanceRetry.make({
+							generation: started.generation,
+							notBefore: now + delay,
+							nativeOnly: true,
+							stalls: Math.min(30, started.stalls + 1)
+						}) } : {}
 					});
 					await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
 					if (autonomous) {
-						await transaction.setAlarm(Option.isSome(publicationDeadline) ? Math.max(now + minimumAlarmDelay, Math.min(now + delay, publicationDeadline.value)) : now + delay);
+						const nativeDeadline = started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation ? now + minimumAlarmDelay : now + delay;
+						await transaction.setAlarm(Option.isSome(publicationDeadline) ? Math.max(now + minimumAlarmDelay, Math.min(nativeDeadline, publicationDeadline.value)) : nativeDeadline);
 						return "rearmed";
 					}
 					if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
@@ -451,7 +541,6 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				}));
 			}));
 			yield* failpoint.hit("maintenance:finish:after");
-			if (alarmDisposition === "cleared") yield* Ref.set(stalls, 0);
 			return yield* annotate(MaintenancePassReport.make({
 				phase: "actionable",
 				recovered: recovered.length,
@@ -465,7 +554,8 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				const now = yield* Clock.currentTimeMillis;
 				const yieldAfter = DateTime.makeUnsafe(now + 6e5);
 				const dispatchUntil = DateTime.makeUnsafe(now + 84e4);
-				return yield* alarm.withWakesDeferred(maintenancePassGate.withPermit(Effect.scoped(pass(yieldAfter, dispatchUntil))));
+				const observed = { nativeOnly: false };
+				return yield* alarm.withWakesDeferred(maintenancePassGate.withPermit(Effect.scoped(pass(yieldAfter, dispatchUntil, observed)).pipe(Effect.onErrorIf(() => true, () => rearmFailure(observed)))));
 			}).pipe(Effect.timeoutOrElse({
 				duration: "14 minutes",
 				orElse: () => DurableAlarmError.make({