@effect-agent/platform-cloudflare 0.1.0-beta.125 → 0.1.0-beta.129

package/dist/Alarm.d.mts

@@ -226,7 +226,8 @@ declare const ThreadMaintenance_base: Context.ServiceClass<ThreadMaintenance, "@
  * 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
  *    producer keeps its generation dirty and immediately eligible. Quiescent native retries
  *    retain their durable backoff.
- * 3. Keep native and delivery admission open together while finite waves remain active, so
+ * 3. Admit at most two independent Thread Attempts, one active head per Thread.
+ *    Keep native and delivery admission open together while finite waves remain active, so
  *    fresh replies and abort controls can progress during unrelated cleanup. Close atomically
  *    at quiescence, or at the original ten-minute yield deadline, then join admitted waves.
  * 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill waves are

package/dist/Alarm.mjs

@@ -10,6 +10,7 @@ import { OperationAuthorizationRequest, OperationAuthorizer } from "effect-agent
 import { AbortIntentRequest, SubmissionLedger } from "effect-agent/submission-ledger";
 import { ThreadProjectionMaintenance, drainDue } from "effect-agent/thread-projection-maintenance";
 import { WakeScheduler } from "effect-agent/wake-scheduler";
+import { DurableObjectStorage } from "effect-cf";
 import { SqlClient } from "effect/unstable/sql/SqlClient";
 //#region src/Alarm.ts
 var Alarm_exports = /* @__PURE__ */ __exportAll({
@@ -47,13 +48,14 @@ const alarmFailure = (operation) => (cause) => DurableAlarmError.make({
 	message: safeCauseMessage(cause, "The Cloudflare alarm API failed without a diagnostic"),
 	cause
 });
-const makeStorageOperation = Effect.map(SqlClient, (sql) => (operation, execute) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => {
-	const body = Effect.uninterruptible(Effect.tryPromise({
-		try: execute,
-		catch: alarmFailure(operation)
-	}));
+const makeStorageEffect = Effect.map(SqlClient, (sql) => (operation, execute) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => {
+	const body = Effect.uninterruptible(execute);
 	return current._tag === "Some" ? body : Effect.scoped(Effect.andThen(sql.reserve.pipe(Effect.mapError(alarmFailure(operation))), body));
 }));
+const makeStorageOperation = Effect.map(makeStorageEffect, (run) => (operation, execute) => run(operation, Effect.tryPromise({
+	try: execute,
+	catch: alarmFailure(operation)
+})));
 /** `ctx.storage` alarm slot as an Effect service; storage is truth, never a memory field. */
 var DurableAlarmService = class DurableAlarmService extends Context.Service()("@effect-agent/platform-cloudflare/DurableAlarmService") {
 	static layer = Layer.effect(DurableAlarmService)(Effect.gen(function* () {
@@ -164,6 +166,7 @@ const decodeRecoveryFault = (threadId, encoded) => {
 	return fault;
 };
 const encodeRecoveryFault = Schema.encodeSync(ThreadRecoveryFault);
+const nativeDispatchConcurrency = 2;
 var MaintenanceRetry = class extends Schema.Class("MaintenanceRetry")({
 	generation: MaintenanceGeneration,
 	notBefore: Schema.Finite,
@@ -270,7 +273,8 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
 * 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
 *    producer keeps its generation dirty and immediately eligible. Quiescent native retries
 *    retain their durable backoff.
-* 3. Keep native and delivery admission open together while finite waves remain active, so
+* 3. Admit at most two independent Thread Attempts, one active head per Thread.
+*    Keep native and delivery admission open together while finite waves remain active, so
 *    fresh replies and abort controls can progress during unrelated cleanup. Close atomically
 *    at quiescence, or at the original ten-minute yield deadline, then join admitted waves.
 * 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill waves are
@@ -288,6 +292,8 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 		const alarm = yield* DurableAlarmService;
 		const config = yield* CloudflareDurableRuntimeConfig;
 		const { ctx } = yield* DurableObjectContext;
+		const storage = DurableObjectStorage.fromDurableObjectStorage(ctx.storage);
+		const runStorage = yield* makeStorageEffect;
 		const failpoint = yield* ThreadMaintenanceFailpoint;
 		const mutations = yield* ThreadMutationGate;
 		const publication = yield* ThreadPublication;
@@ -448,7 +454,45 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				};
 			}));
 		});
-		const advance = Effect.fn("ThreadMaintenance.advance")(function* (started, yieldAfter, observed, recovery, dispatch = true) {
+		const attemptSelected = Effect.fn("ThreadMaintenance.attemptSelected")(function* (selected, previousRetries, yieldAfter, recovery) {
+			yield* recoverThread(selected.threadId, recovery);
+			let bindingFailure;
+			const settlement = recovery.faults.has(selected.threadId) ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.catchTag("BindingUnavailable", (failure) => {
+				bindingFailure = failure;
+				return Effect.succeed(Option.none());
+			}));
+			const previous = previousRetries.find((retry) => retry.submissionId === selected.submissionId);
+			let retry;
+			let reportBindingFailure = false;
+			if (bindingFailure !== void 0) {
+				const now = yield* Clock.currentTimeMillis;
+				const attempts = Math.min(30, (previous?.attempts ?? 0) + 1);
+				reportBindingFailure = previous === void 0 || now - previous.reportedAt >= 9e5;
+				retry = BindingRetry.make({
+					threadId: selected.threadId,
+					submissionId: selected.submissionId,
+					attempts,
+					notBefore: now + Math.min(6e4, 5e3 * 2 ** (attempts - 1)),
+					reportedAt: reportBindingFailure ? now : previous?.reportedAt ?? now
+				});
+			}
+			if (retry !== void 0 || previous !== void 0) {
+				yield* failpoint.hit("maintenance:binding-retry:before");
+				yield* runStorage("record submission binding retry", storage.transaction((transaction) => Effect.gen(function* () {
+					const encoded = yield* transaction.get(MAINTENANCE_STATE_KEY);
+					const state = encoded === void 0 ? initialMaintenanceState() : yield* Schema.decodeUnknownEffect(ThreadMaintenanceState)(encoded);
+					const bindingRetries = [...(state.bindingRetries ?? []).filter((entry) => entry.submissionId !== selected.submissionId), ...retry === void 0 ? [] : [retry]];
+					yield* transaction.put(MAINTENANCE_STATE_KEY, yield* Schema.encodeEffect(ThreadMaintenanceState)(ThreadMaintenanceState.make({
+						...state,
+						bindingRetries
+					})));
+				})).pipe(Effect.mapError(alarmFailure("record submission binding retry"))));
+				yield* failpoint.hit("maintenance:binding-retry:after");
+			}
+			if (bindingFailure !== void 0) yield* reportBindingFailure ? Effect.logError("Thread awaits a current agent binding; original work remains pending", Cause.fail(bindingFailure)) : Effect.logDebug("Thread binding retry remains pending", Cause.fail(bindingFailure));
+			return Option.isSome(settlement) ? 1 : 0;
+		});
+		const advance = Effect.fn("ThreadMaintenance.advance")(function* (started, yieldAfter, observed, recovery, native, dispatch = true, reserved = new Set(native.active.keys())) {
 			const deadline = yield* publication.pendingDeadline;
 			if (started._tag === "Actionable" || Option.isSome(deadline) && deadline.value <= (yield* Clock.currentTimeMillis)) yield* publication.drain;
 			const pending = yield* publication.pendingDeadline;
@@ -462,11 +506,20 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				}));
 				return {
 					phase: "caught-up",
-					settled: 0,
 					nonterminal: started.nonterminal,
 					nextAttemptAt
 				};
 			}
+			if (native.generation !== started.generation) {
+				native.deferred.clear();
+				native.generation = started.generation;
+			}
+			if (native.active.size > 0 && native.scanGeneration === started.generation) return {
+				phase: "actionable",
+				nonterminal: started.nonterminal,
+				nextAttemptAt: void 0
+			};
+			native.scanGeneration = started.activeAtStart === 0 ? started.generation : void 0;
 			observed.nativeOnly = true;
 			const observation = recovery.observation ??= {
 				generation: started.generation,
@@ -492,7 +545,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			}
 			const stopping = /* @__PURE__ */ new Set();
 			for (const head of heads.values()) {
-				if (head.state !== "ready" || recovery.pending.has(head.threadId) || recoveryFaults.has(head.threadId)) continue;
+				if (head.state !== "ready" || native.active.has(head.threadId) || recovery.pending.has(head.threadId) || recoveryFaults.has(head.threadId)) continue;
 				const intent = yield* isolateRecovery(ledger.readAbortIntent(AbortIntentRequest.make({ submissionId: head.submissionId })), {
 					timeout: recoveryConfig.recoveryTimeout,
 					phase: () => "recovery",
@@ -517,8 +570,10 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			const selection = yield* runTransaction("select maintenance lane", () => ctx.storage.transaction(async (transaction) => {
 				const { state } = await readMaintenanceState(transaction);
 				const retries = (state.bindingRetries ?? []).filter((retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId);
-				const runnable = eligible.filter((threadId) => !retries.some((retry) => retry.threadId === threadId && retry.notBefore > selectionTime));
-				const next = dispatch ? runnable.find((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId) ?? runnable[0] : void 0;
+				const runnable = eligible.filter((threadId) => !reserved.has(threadId) && !native.active.has(threadId) && !native.deferred.has(threadId) && !retries.some((retry) => retry.threadId === threadId && retry.notBefore > selectionTime));
+				const pivot = Math.max(0, runnable.findIndex((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId));
+				const selected = dispatch ? [...runnable.slice(pivot), ...runnable.slice(0, pivot)].slice(0, nativeDispatchConcurrency - reserved.size) : [];
+				const next = selected.at(-1);
 				if (next !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
 					...state,
 					lastServedThreadId: next
@@ -530,7 +585,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				const after = backlog.filter((threadId) => state.lastRecoveredThreadId === void 0 || threadId > state.lastRecoveredThreadId);
 				const before = backlog.filter((threadId) => state.lastRecoveredThreadId !== void 0 && threadId <= state.lastRecoveredThreadId);
 				return {
-					selected: next,
+					selected,
 					retries,
 					backlog: [...after, ...before]
 				};
@@ -542,45 +597,19 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				for (const threadId of selection.backlog) recovery.pending.add(threadId);
 				yield* Deferred.succeed(recovery.queue, selection.backlog);
 			}
-			const selected = selection.selected === void 0 ? void 0 : heads.get(selection.selected);
-			if (selected !== void 0) yield* recoverThread(selected.threadId, recovery);
-			let retries = selection.retries;
-			let bindingFailure;
-			const settlement = selected === void 0 || recoveryFaults.has(selected.threadId) ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.catchTag("BindingUnavailable", (failure) => {
-				bindingFailure = failure;
-				return Effect.succeed(Option.none());
-			}));
-			if (selected !== void 0) {
-				const previous = retries.find((retry) => retry.submissionId === selected.submissionId);
-				let retry;
-				let reportBindingFailure = false;
-				if (bindingFailure !== void 0) {
-					const now = yield* Clock.currentTimeMillis;
-					const attempts = Math.min(30, (previous?.attempts ?? 0) + 1);
-					reportBindingFailure = previous === void 0 || now - previous.reportedAt >= 9e5;
-					retry = BindingRetry.make({
-						threadId: selected.threadId,
-						submissionId: selected.submissionId,
-						attempts,
-						notBefore: now + Math.min(6e4, 5e3 * 2 ** (attempts - 1)),
-						reportedAt: reportBindingFailure ? now : previous?.reportedAt ?? now
-					});
-				}
-				if (retry !== void 0 || previous !== void 0) {
-					yield* failpoint.hit("maintenance:binding-retry:before");
-					retries = yield* runTransaction("record submission binding retry", () => ctx.storage.transaction(async (transaction) => {
-						const { state } = await readMaintenanceState(transaction);
-						const bindingRetries = [...(state.bindingRetries ?? []).filter((entry) => entry.submissionId !== selected.submissionId), ...retry === void 0 ? [] : [retry]];
-						await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
-							...state,
-							bindingRetries
-						})));
-						return bindingRetries;
-					}));
-					yield* failpoint.hit("maintenance:binding-retry:after");
-				}
-				if (bindingFailure !== void 0) yield* reportBindingFailure ? Effect.logError("Thread awaits a current agent binding; original work remains pending", Cause.fail(bindingFailure)) : Effect.logDebug("Thread binding retry remains pending", Cause.fail(bindingFailure));
+			for (const threadId of selection.selected) {
+				const selected = heads.get(threadId);
+				if (selected === void 0) continue;
+				native.dispatched++;
+				native.active.set(threadId, yield* Effect.forkIn(attemptSelected(selected, selection.retries, yieldAfter, recovery), native.scope));
 			}
+			if (native.active.size > 0) return {
+				phase: "actionable",
+				nonterminal: current.length,
+				nextAttemptAt: void 0,
+				dispatched: selection.selected.length > 0
+			};
+			const retries = selection.retries;
 			const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
 			const waitingHeads = /* @__PURE__ */ new Map();
 			const autonomous = remaining.some((snapshot) => {
@@ -590,7 +619,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				if (headWaiting === true && snapshot.state === "ready" && reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput") return false;
 				return !waiting(snapshot);
 			});
-			const progressed = Option.isSome(settlement) || recovery.repaired;
+			const progressed = native.progressed || recovery.repaired;
 			const now = yield* Clock.currentTimeMillis;
 			const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
 			const nextEligible = [...recoveryFaults.values()].map((fault) => fault.retryAt).concat(eligible.map((threadId) => retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)?.notBefore ?? now));
@@ -619,9 +648,11 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				return mutationOverlap || next.dirty > next.processed ? now + minimumAlarmDelay : void 0;
 			})));
 			yield* failpoint.hit("maintenance:checkpoint:after");
+			native.deferred.clear();
+			native.progressed = false;
+			native.needsCheckpoint = false;
 			return {
 				phase: "actionable",
-				settled: Option.isSome(settlement) ? 1 : 0,
 				nonterminal: remaining.length,
 				nextAttemptAt
 			};
@@ -671,15 +702,22 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			const backfill = yield* fork(drainDue.pipe(Effect.provideService(ThreadProjectionMaintenance, projection), Effect.timeoutOption(config.projectionDispatchTimeoutMillis)));
 			let backfillObserved = false;
 			let recoveryObserved = false;
-			let native = yield* fork(advance(started, yieldAfter, observed, recovery));
+			const dispatch = {
+				scope: auxiliaryScope,
+				active: /* @__PURE__ */ new Map(),
+				deferred: /* @__PURE__ */ new Set(),
+				dispatched: 0,
+				settled: 0,
+				progressed: false,
+				needsCheckpoint: false
+			};
+			let native = yield* fork(advance(started, yieldAfter, observed, recovery, dispatch));
 			let result = {
 				phase: "caught-up",
-				settled: 0,
 				nonterminal: started.nonterminal,
 				nextAttemptAt: void 0
 			};
 			let phase = result.phase;
-			let settled = 0;
 			let nativeCheck = false;
 			const until = DateTime.toEpochMillis(yieldAfter);
 			const dispatchEnd = DateTime.toEpochMillis(dispatchUntil);
@@ -691,16 +729,33 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 					result = yield* Fiber.join(native);
 					native = void 0;
 					if (result.phase === "actionable") phase = "actionable";
-					settled += result.settled;
-					observed.nativeOnly = false;
+					observed.nativeOnly = dispatch.active.size > 0;
+					if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency) {
+						dispatch.scanGeneration = void 0;
+						nativeCheck = true;
+					}
 					recovery.started = true;
 					yield* Deferred.succeed(recovery.queue, []);
 					checkLanes();
 				}
+				for (const [threadId, attempt] of dispatch.active) {
+					const exit = attempt.pollUnsafe();
+					if (exit === void 0) continue;
+					dispatch.active.delete(threadId);
+					dispatch.scanGeneration = void 0;
+					if (Exit.isFailure(exit)) return yield* Effect.failCause(exit.cause);
+					dispatch.settled += exit.value;
+					dispatch.progressed ||= exit.value > 0;
+					dispatch.needsCheckpoint = true;
+					if (exit.value === 0) dispatch.deferred.add(threadId);
+					nativeCheck = true;
+					checkLanes();
+				}
 				if (!recoveryObserved && recoveryFiber.pollUnsafe() !== void 0) {
 					yield* Fiber.join(recoveryFiber);
 					recoveryObserved = true;
-					nativeCheck = recovery.needsCheckpoint;
+					dispatch.scanGeneration = void 0;
+					nativeCheck ||= recovery.needsCheckpoint;
 				}
 				if (!backfillObserved) {
 					const exit = backfill.pollUnsafe();
@@ -726,7 +781,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 						messageExhausted = true;
 					}
 				}
-				if (failure !== void 0 && native === void 0) return yield* Effect.failCause(failure);
+				if (failure !== void 0 && native === void 0 && dispatch.active.size === 0) return yield* Effect.failCause(failure);
 				const now = yield* Clock.currentTimeMillis;
 				if (now >= until) break;
 				for (const lane of lanes) {
@@ -779,19 +834,22 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 						else delivery = yield* fork(wave.run);
 					}
 				}
-				if (native === void 0 && recoveryObserved && !recovery.needsCheckpoint && backfillObserved && delivery === void 0 && lanes.every((lane) => lane.fiber === void 0)) break;
-				if (native === void 0 && nativeCheck) {
+				if (native === void 0 && dispatch.active.size === 0 && recoveryObserved && !recovery.needsCheckpoint && backfillObserved && delivery === void 0 && lanes.every((lane) => lane.fiber === void 0)) break;
+				if (native === void 0 && nativeCheck && dispatch.active.size < nativeDispatchConcurrency) {
 					nativeCheck = false;
-					const checkpoint = recoveryObserved && recovery.needsCheckpoint;
+					const nativeCheckpoint = dispatch.needsCheckpoint && dispatch.active.size === 0;
+					const checkpoint = recoveryObserved && recovery.needsCheckpoint || nativeCheckpoint;
 					if (checkpoint) recovery.needsCheckpoint = false;
+					const reserved = new Set(dispatch.active.keys());
 					native = yield* fork(Effect.gen(function* () {
 						const awakened = checkpoint ? started : yield* beginNative(observed);
-						return yield* advance(awakened, yieldAfter, observed, recovery, !checkpoint || settled === 0);
+						return yield* advance(awakened, yieldAfter, observed, recovery, dispatch, failure === void 0 && !nativeCheckpoint && (!checkpoint || dispatch.settled === 0), reserved);
 					}));
 				}
 				const next = Math.min(native === void 0 ? result.nextAttemptAt ?? Infinity : Infinity, now + config.wakeScanInterval, until);
 				const completing = [
 					...native === void 0 ? [] : [native],
+					...dispatch.active.values(),
 					...recoveryObserved ? [] : [recoveryFiber],
 					...backfillObserved ? [] : [backfill],
 					...delivery === void 0 ? [] : [delivery],
@@ -805,11 +863,17 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			if (native !== void 0) {
 				result = yield* Fiber.join(native);
 				if (result.phase === "actionable") phase = "actionable";
-				settled += result.settled;
-				observed.nativeOnly = false;
+				observed.nativeOnly = dispatch.active.size > 0;
 				recovery.started = true;
 				yield* Deferred.succeed(recovery.queue, []);
 			}
+			for (const attempt of dispatch.active.values()) {
+				const settled = yield* Fiber.join(attempt);
+				dispatch.settled += settled;
+				dispatch.progressed ||= settled > 0;
+			}
+			dispatch.active.clear();
+			observed.nativeOnly = false;
 			yield* Fiber.joinAll([
 				recoveryFiber,
 				backfill,
@@ -817,10 +881,9 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 				...lanes.flatMap((lane) => lane.fiber === void 0 ? [] : [lane.fiber])
 			]);
 			if (failure !== void 0) return yield* Effect.failCause(failure);
-			if (recovery.needsCheckpoint) {
-				result = yield* advance(started, yieldAfter, observed, recovery, false);
+			if (recovery.needsCheckpoint || dispatch.dispatched > 0) {
+				result = yield* advance(started, yieldAfter, observed, recovery, dispatch, false);
 				if (result.phase === "actionable") phase = "actionable";
-				settled += result.settled;
 			}
 			yield* Scope.close(auxiliaryScope, Exit.void);
 			yield* failpoint.hit("maintenance:finish:before");
@@ -843,7 +906,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
 			const report = MaintenancePassReport.make({
 				phase,
 				recovered: recovery.recovered,
-				settled,
+				settled: dispatch.settled,
 				nonterminal: result.nonterminal,
 				alarm: disposition
 			});