@effect-agent/platform-cloudflare 0.1.0-beta.104 → 0.1.0-beta.107

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/Alarm.ts CHANGED
@@ -30,6 +30,7 @@ import {
30
30
  drainDue,
31
31
  type ThreadProjectionError,
32
32
  } from "effect-agent/thread-projection-maintenance";
33
+ import { WakeScheduler } from "effect-agent/wake-scheduler";
33
34
  import { SqlClient } from "effect/unstable/sql/SqlClient";
34
35
 
35
36
  import { DurableObjectContext } from "./CloudflareBindings.ts";
@@ -188,7 +189,7 @@ export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
188
189
  phase: Schema.Literals(["caught-up", "actionable"]),
189
190
  /** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
190
191
  recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
191
- /** Whether the head Attempt settled. Joined input may settle with that head. */
192
+ /** Head Attempts settled during the event. Joined input may settle with each head. */
192
193
  settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
193
194
  /** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
194
195
  nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
@@ -212,6 +213,8 @@ export type ThreadMaintenanceFailpointLocation =
212
213
  | "maintenance:binding-retry:after"
213
214
  | "maintenance:retry:before"
214
215
  | "maintenance:retry:after"
216
+ | "maintenance:checkpoint:before"
217
+ | "maintenance:checkpoint:after"
215
218
  | "maintenance:finish:before"
216
219
  | "maintenance:finish:after";
217
220
 
@@ -272,7 +275,7 @@ export class ThreadPublication extends Context.Service<
272
275
  */
273
276
  export const ThreadMessageDelivery = Context.Reference<{
274
277
  readonly drainUntil: (
275
- sourceFinished: Effect.Effect<void>,
278
+ dispatchClosed: Effect.Effect<void>,
276
279
  dispatchUntil: DateTime.Utc,
277
280
  ) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
278
281
  readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
@@ -285,14 +288,14 @@ export const ThreadMessageDelivery = Context.Reference<{
285
288
 
286
289
  /**
287
290
  * Application obligations sharing this Object's alarm. Admit one initial external wave even on
288
- * a caught-up pass, then respond to wakes while native work runs. sourceFinished closes only
289
- * admission of NEW external waves. Keep local admission/hub subscriptions in the supplied event
290
- * Scope until maintenance tears it down; do not scope them to sourceFinished or to this Effect.
291
+ * a caught-up pass, then respond to wakes until dispatchClosed. This closes only admission
292
+ * of NEW external waves; native work can continue while admitted waves finish. Keep local
293
+ * admission/hub subscriptions in the event Scope until maintenance tears it down.
291
294
  * No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
292
295
  *
293
296
  * Declare a finite whole-wave allowance (1..300000ms): maximum for parallel lanes, sum for
294
297
  * sequential operations. Admit a wave only if its allowance fits before dispatchUntil. Later
295
- * arrivals cannot renew the retirement window. Maintenance bounds the join after sourceFinished,
298
+ * arrivals cannot renew the retirement window. Maintenance bounds the join after dispatchClosed,
296
299
  * interrupts and joins event Scope, then reads local deadlines under the mutation gate.
297
300
  *
298
301
  * Setup and pendingDeadline are bounded local operations. Persist claims/envelopes before
@@ -304,7 +307,7 @@ export const ThreadMessageDelivery = Context.Reference<{
304
307
  export const ThreadHostMaintenance = Context.Reference<{
305
308
  readonly dispatchTimeoutMillis: number;
306
309
  readonly drainUntil: (
307
- sourceFinished: Effect.Effect<void>,
310
+ dispatchClosed: Effect.Effect<void>,
308
311
  dispatchUntil: DateTime.Utc,
309
312
  ) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
310
313
  readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
@@ -351,6 +354,14 @@ class BindingRetry extends Schema.Class<BindingRetry>("BindingRetry")({
351
354
  reportedAt: Schema.Finite,
352
355
  }) {}
353
356
 
357
+ interface NativePassResult {
358
+ readonly phase: "caught-up" | "actionable";
359
+ readonly recovered: number;
360
+ readonly settled: number;
361
+ readonly nonterminal: number;
362
+ readonly nextAttemptAt: number | undefined;
363
+ }
364
+
354
365
  interface MaintenanceObservation {
355
366
  generation?: bigint;
356
367
  nativeOnly: boolean;
@@ -536,20 +547,20 @@ export type MaintenancePassFailure =
536
547
  /**
537
548
  * Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
538
549
  *
539
- * `pass` = generation snapshot/pre-arm → recovery → one head Attempt → generation acknowledgement:
550
+ * One physical event owns native scheduling, auxiliary delivery and final alarm rearming.
540
551
  *
541
- * 1. One storage transaction reads dirty/processed and re-arms before work. A caught-up forced
542
- * alarm takes an O(1) path without recovery, ledger scans, or canonical-history reads.
543
- * 2. Recovery strictly precedes a new claim. One head Attempt advances the lane and requests
544
- * a safe yield after ten minutes. The whole event has a fourteen-minute cooperative timeout.
545
- * 3. Auxiliary dispatch and listeners belong to the event Scope. After native completion,
546
- * stop new external waves, join native delivery through its driver-owned Claim deadline,
547
- * and bound host/backfill work by explicit allowances before closing and joining Scope.
548
- * Ordinary auxiliary setup failures are reported after the one native opportunity.
549
- * 4. The final transaction acknowledges only the generation observed at pass start. A racing
550
- * mutation therefore remains `dirty > processed` and retains its atomically-established alarm.
551
- * 5. Stable external waits acknowledge and clear. Autonomous retry, indeterminate, and lease
552
- * recovery states leave their generation dirty and retain bounded backoff rearming.
552
+ * 1. Prearm before any work. A caught-up native step uses the O(1) generation record without
553
+ * recovery, ledger scans or canonical-history reads.
554
+ * 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
555
+ * producer keeps its newer generation dirty. Native retries retain their durable backoff.
556
+ * 3. After the initial native opportunity, stop admitting new external waves and let the
557
+ * active waves finish. While they remain in flight, native wakes and bounded scans can
558
+ * advance more heads. All Attempts share the event's original ten-minute yield deadline.
559
+ * 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
560
+ * bounded independently; incoming native work never restarts or cancels their attempts.
561
+ * Auxiliary failures are reported after the current native opportunity.
562
+ * 5. Close every event resource before the final gated deadline snapshot and alarm decision.
563
+ * The whole event retains one fourteen-minute cooperative timeout.
553
564
  */
554
565
  export class ThreadMaintenance extends Context.Service<
555
566
  ThreadMaintenance,
@@ -579,6 +590,7 @@ export class ThreadMaintenance extends Context.Service<
579
590
  | ThreadProjectionMaintenance
580
591
  | DurableAgentRuntime
581
592
  | SubmissionLedger
593
+ | WakeScheduler
582
594
  | DurableAlarmService
583
595
  | ThreadMaintenanceFailpoint
584
596
  | CloudflareDurableRuntimeConfig
@@ -588,6 +600,7 @@ export class ThreadMaintenance extends Context.Service<
588
600
  Effect.gen(function* () {
589
601
  const runtime = yield* DurableAgentRuntime;
590
602
  const ledger = yield* SubmissionLedger;
603
+ const wakes = yield* WakeScheduler;
591
604
  const alarm = yield* DurableAlarmService;
592
605
  const config = yield* CloudflareDurableRuntimeConfig;
593
606
  const { ctx } = yield* DurableObjectContext;
@@ -745,7 +758,10 @@ export class ThreadMaintenance extends Context.Service<
745
758
  ctx.storage.transaction(async (transaction) => {
746
759
  const { state } = await readMaintenanceState(transaction);
747
760
 
748
- const previous = state.retry?.generation === generation ? state.retry : undefined;
761
+ const previous =
762
+ state.retry?.generation === generation && state.retry.nativeOnly === nativeOnly
763
+ ? state.retry
764
+ : undefined;
749
765
 
750
766
  const retry = MaintenanceRetry.make({
751
767
  generation,
@@ -784,21 +800,10 @@ export class ThreadMaintenance extends Context.Service<
784
800
  yield* failpoint.hit("maintenance:retry:after");
785
801
  });
786
802
 
787
- const pass = Effect.fn("ThreadMaintenance.pass")(function* (
788
- yieldAfter: DateTime.Utc,
789
- dispatchUntil: DateTime.Utc,
803
+ const beginNative = Effect.fn("ThreadMaintenance.beginNative")(function* (
790
804
  observed: MaintenanceObservation,
791
- ): Effect.fn.Return<MaintenancePassReport, MaintenancePassFailure, Scope.Scope> {
792
- const annotate = (report: MaintenancePassReport) =>
793
- Effect.annotateCurrentSpan({
794
- phase: report.phase,
795
- recovered: report.recovered,
796
- settled: report.settled,
797
- nonterminal: report.nonterminal,
798
- alarm: report.alarm,
799
- }).pipe(Effect.as(report));
800
-
801
- const started = yield* mutations.withSnapshot((activeAtStart) =>
805
+ ) {
806
+ return yield* mutations.withSnapshot((activeAtStart) =>
802
807
  Effect.gen(function* () {
803
808
  const generation = yield* beginPass(observed);
804
809
 
@@ -810,76 +815,13 @@ export class ThreadMaintenance extends Context.Service<
810
815
  return { ...generation, activeAtStart };
811
816
  }),
812
817
  );
818
+ });
813
819
 
814
- // This scope owns auxiliary dispatch and listeners, independently of source completion.
815
- // Close it before acknowledgement, including on failure or event interruption.
816
- const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) =>
817
- Scope.close(scope, exit),
818
- );
819
-
820
- const sourceFinished = yield* Deferred.make<void>();
821
- const finished = Deferred.await(sourceFinished);
822
-
823
- // Fork setup too: an ordinary auxiliary setup failure is reported after native work,
824
- // rather than gating its opportunity. Event interruption still closes every fiber.
825
- const deliveryFiber = yield* Effect.forkIn(
826
- Scope.provide(auxiliaryScope)(messages.drainUntil(finished, dispatchUntil)),
827
- auxiliaryScope,
828
- );
829
-
830
- const hostFiber = yield* Effect.forkIn(
831
- Scope.provide(auxiliaryScope)(
832
- Effect.gen(function* () {
833
- yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(
834
- Effect.mapError((cause) =>
835
- DurableAlarmError.make({
836
- operation: "host dispatch allowance",
837
- message:
838
- "Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
839
- cause,
840
- }),
841
- ),
842
- );
843
- yield* host.drainUntil(finished, dispatchUntil);
844
- }),
845
- ),
846
- auxiliaryScope,
847
- );
848
-
849
- // Backfill is one disposable wave; its timer starts beside native execution.
850
- const backfill = yield* Effect.forkIn(
851
- drainDue.pipe(
852
- Effect.provideService(ThreadProjectionMaintenance, projection),
853
- Effect.timeoutOption(config.projectionDispatchTimeoutMillis),
854
- ),
855
- auxiliaryScope,
856
- );
857
-
858
- const finishAuxiliary = Effect.gen(function* () {
859
- yield* Deferred.succeed(sourceFinished, undefined);
860
-
861
- const remaining = Math.max(
862
- 1,
863
- DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis),
864
- );
865
-
866
- const hostJoin = yield* Effect.forkIn(
867
- Fiber.join(hostFiber).pipe(
868
- Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)),
869
- Effect.tap((result) =>
870
- Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(result) }),
871
- ),
872
- ),
873
- auxiliaryScope,
874
- );
875
-
876
- // Native transport uses the driver's actual Claim deadline, through Retry persistence.
877
- // Only our host/backfill timers suppress their own interruption. Earlier failed exits,
878
- // including pure self-interruption, remain failures while siblings are still blocked.
879
- yield* Fiber.joinAll([deliveryFiber, hostJoin, backfill]);
880
- yield* Scope.close(auxiliaryScope, Exit.void);
881
- });
882
-
820
+ const advance = Effect.fn("ThreadMaintenance.advance")(function* (
821
+ started: Effect.Success<ReturnType<typeof beginNative>>,
822
+ yieldAfter: DateTime.Utc,
823
+ observed: MaintenanceObservation,
824
+ ): Effect.fn.Return<NativePassResult, MaintenancePassFailure> {
883
825
  const deadline = yield* publication.pendingDeadline;
884
826
 
885
827
  if (
@@ -891,55 +833,34 @@ export class ThreadMaintenance extends Context.Service<
891
833
  const pending = yield* publication.pendingDeadline;
892
834
 
893
835
  if (started._tag === "CaughtUp" || Option.isSome(pending)) {
894
- yield* finishAuxiliary;
895
- yield* failpoint.hit("maintenance:finish:before");
896
-
897
- const disposition = yield* mutations.withSnapshot((active) =>
836
+ const nextAttemptAt = yield* mutations.withSnapshot((active) =>
898
837
  Effect.gen(function* () {
899
- // Re-read under the producer gate: a concurrent append/host mutation cannot be
900
- // cleared using a stale empty deadline. Dirty generations bound all producer races.
901
- const latest = yield* pendingDeadline;
902
838
  const now = yield* Clock.currentTimeMillis;
903
839
 
904
- return yield* runTransaction("finish publication pass", () =>
905
- ctx.storage.transaction(async (transaction) => {
906
- const { state } = await readMaintenanceState(transaction);
907
-
908
- const nativeDeadline =
909
- active > 0 || state.dirty > state.processed
910
- ? state.retry?.generation === state.dirty && active === 0
911
- ? Math.max(now + minimumAlarmDelay, state.retry.notBefore)
912
- : now + config.wakeScanInterval
913
- : Infinity;
914
-
915
- const next = Option.isSome(latest)
916
- ? Math.min(nativeDeadline, latest.value)
917
- : nativeDeadline;
840
+ const { state } = yield* runTransaction("read native maintenance deadline", () =>
841
+ ctx.storage.transaction((transaction) => readMaintenanceState(transaction)),
842
+ );
918
843
 
919
- if (Number.isFinite(next)) {
920
- await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
844
+ const native =
845
+ active > 0 || state.dirty > state.processed
846
+ ? state.retry?.generation === state.dirty && active === 0
847
+ ? state.retry.notBefore
848
+ : now + minimumAlarmDelay
849
+ : Infinity;
921
850
 
922
- return "rearmed" as const;
923
- }
924
- await transaction.deleteAlarm();
851
+ const next = Option.isSome(pending) ? pending.value : native;
925
852
 
926
- return "cleared" as const;
927
- }),
928
- );
853
+ return Number.isFinite(next) ? Math.max(now + minimumAlarmDelay, next) : undefined;
929
854
  }),
930
855
  );
931
856
 
932
- yield* failpoint.hit("maintenance:finish:after");
933
-
934
- return yield* annotate(
935
- MaintenancePassReport.make({
936
- phase: "caught-up",
937
- recovered: 0,
938
- settled: 0,
939
- nonterminal: started.nonterminal,
940
- alarm: disposition,
941
- }),
942
- );
857
+ return {
858
+ phase: "caught-up",
859
+ recovered: 0,
860
+ settled: 0,
861
+ nonterminal: started.nonterminal,
862
+ nextAttemptAt,
863
+ };
943
864
  }
944
865
  // Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
945
866
  observed.nativeOnly = true;
@@ -1006,7 +927,7 @@ export class ThreadMaintenance extends Context.Service<
1006
927
  let retries = selection.retries;
1007
928
  let bindingFailure: DurableBindingFailure | undefined;
1008
929
 
1009
- // One runnable FIFO head per event. An absent agent is a durable wait for a deployment,
930
+ // One runnable FIFO head per native opportunity. An absent agent waits for a deployment,
1010
931
  // including for children; other local lanes and host deliveries remain independently due.
1011
932
  const settlement =
1012
933
  selected === undefined
@@ -1041,7 +962,7 @@ export class ThreadMaintenance extends Context.Service<
1041
962
  if (retry !== undefined || previous !== undefined) {
1042
963
  // The Attempt released its Claim. Commit its binding wait (or clear) once,
1043
964
  // before joining fallible auxiliary work. This local fact neither acknowledges
1044
- // a generation nor changes the shared alarm; those require event retirement.
965
+ // a generation nor changes the shared alarm.
1045
966
  yield* failpoint.hit("maintenance:binding-retry:before");
1046
967
  retries = yield* runTransaction("record submission binding retry", () =>
1047
968
  ctx.storage.transaction(async (transaction) => {
@@ -1074,8 +995,6 @@ export class ThreadMaintenance extends Context.Service<
1074
995
  }
1075
996
  }
1076
997
 
1077
- observed.nativeOnly = false;
1078
- yield* finishAuxiliary;
1079
998
  const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
1080
999
  const waitingHeads = new Map<ThreadId, boolean>();
1081
1000
 
@@ -1115,87 +1034,213 @@ export class ThreadMaintenance extends Context.Service<
1115
1034
 
1116
1035
  const delay = Math.max(ordinaryDelay, bindingDelay);
1117
1036
 
1037
+ // Checkpoint native progress without changing the physical alarm. Auxiliary
1038
+ // delivery remains live; later mutations still advance the shared generation.
1039
+ yield* failpoint.hit("maintenance:checkpoint:before");
1040
+
1041
+ const nextAttemptAt = yield* mutations.withSnapshot((active) =>
1042
+ runTransaction("checkpoint native maintenance", () =>
1043
+ ctx.storage.transaction(async (transaction) => {
1044
+ const { state } = await readMaintenanceState(transaction);
1045
+
1046
+ const processed =
1047
+ autonomous || started.activeAtStart > 0 || active > 0
1048
+ ? state.processed
1049
+ : state.processed > started.generation
1050
+ ? state.processed
1051
+ : started.generation;
1052
+
1053
+ const next = ThreadMaintenanceState.make({
1054
+ ...Struct.omit(state, ["retry"]),
1055
+ processed,
1056
+ nonterminal: remaining.length,
1057
+ bindingRetries: (state.bindingRetries ?? []).filter((retry) =>
1058
+ remaining.some((row) => row.submissionId === retry.submissionId),
1059
+ ),
1060
+ ...(autonomous && !progressed
1061
+ ? {
1062
+ retry: MaintenanceRetry.make({
1063
+ generation: started.generation,
1064
+ notBefore: now + delay,
1065
+ nativeOnly: true,
1066
+ stalls: Math.min(30, started.stalls + 1),
1067
+ }),
1068
+ }
1069
+ : {}),
1070
+ });
1071
+
1072
+ await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
1073
+ if (autonomous) {
1074
+ return started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation
1075
+ ? now + minimumAlarmDelay
1076
+ : now + delay;
1077
+ }
1078
+
1079
+ return started.activeAtStart > 0 || active > 0 || next.dirty > next.processed
1080
+ ? now + minimumAlarmDelay
1081
+ : undefined;
1082
+ }),
1083
+ ),
1084
+ );
1085
+
1086
+ yield* failpoint.hit("maintenance:checkpoint:after");
1087
+
1088
+ return {
1089
+ phase: "actionable",
1090
+ recovered: recovered.length,
1091
+ settled: Option.isSome(settlement) ? 1 : 0,
1092
+ nonterminal: remaining.length,
1093
+ nextAttemptAt,
1094
+ };
1095
+ });
1096
+
1097
+ const pass = Effect.fn("ThreadMaintenance.pass")(function* (
1098
+ yieldAfter: DateTime.Utc,
1099
+ dispatchUntil: DateTime.Utc,
1100
+ observed: MaintenanceObservation,
1101
+ ): Effect.fn.Return<MaintenancePassReport, MaintenancePassFailure, Scope.Scope> {
1102
+ // Subscribe before the first native snapshot. Hints accelerate rechecks; the
1103
+ // bounded scan and durable generation still recover dropped notifications.
1104
+ const notified = (yield* Stream.toPull(wakes.wakes)).pipe(
1105
+ Effect.asVoid,
1106
+ Effect.catch(() => Effect.never),
1107
+ );
1108
+
1109
+ let started = yield* beginNative(observed);
1110
+
1111
+ // This scope owns auxiliary dispatch and listeners, independently of native progress.
1112
+ // Close it before final alarm rearming, including on failure or event interruption.
1113
+ const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) =>
1114
+ Scope.close(scope, exit),
1115
+ );
1116
+
1117
+ const dispatchClosed = yield* Deferred.make<void>();
1118
+ const stopDispatch = Deferred.await(dispatchClosed);
1119
+
1120
+ // Fork setup too: an ordinary auxiliary setup failure is reported after native work,
1121
+ // rather than gating its opportunity. Event interruption still closes every fiber.
1122
+ const deliveryFiber = yield* Effect.forkIn(
1123
+ Scope.provide(auxiliaryScope)(messages.drainUntil(stopDispatch, dispatchUntil)),
1124
+ auxiliaryScope,
1125
+ );
1126
+
1127
+ const hostFiber = yield* Effect.forkIn(
1128
+ Scope.provide(auxiliaryScope)(
1129
+ Effect.gen(function* () {
1130
+ yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(
1131
+ Effect.mapError((cause) =>
1132
+ DurableAlarmError.make({
1133
+ operation: "host dispatch allowance",
1134
+ message:
1135
+ "Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
1136
+ cause,
1137
+ }),
1138
+ ),
1139
+ );
1140
+ yield* host.drainUntil(stopDispatch, dispatchUntil);
1141
+ }),
1142
+ ),
1143
+ auxiliaryScope,
1144
+ );
1145
+
1146
+ // Backfill is one disposable wave; its timer starts beside native execution.
1147
+ const backfill = yield* Effect.forkIn(
1148
+ drainDue.pipe(
1149
+ Effect.provideService(ThreadProjectionMaintenance, projection),
1150
+ Effect.timeoutOption(config.projectionDispatchTimeoutMillis),
1151
+ ),
1152
+ auxiliaryScope,
1153
+ );
1154
+
1155
+ let result = yield* advance(started, yieldAfter, observed);
1156
+ let phase = result.phase;
1157
+ let recovered = result.recovered;
1158
+ let settled = result.settled;
1159
+
1160
+ observed.nativeOnly = false;
1161
+
1162
+ // Close admission of new delivery waves once, then keep advancing native
1163
+ // work while the already-admitted waves finish. Neither lane restarts the
1164
+ // other's work or receives a fresh event budget.
1165
+ yield* Deferred.succeed(dispatchClosed, undefined);
1166
+
1167
+ const remaining = Math.max(
1168
+ 1,
1169
+ DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis),
1170
+ );
1171
+
1172
+ const hostJoin = yield* Effect.forkIn(
1173
+ Fiber.join(hostFiber).pipe(
1174
+ Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)),
1175
+ Effect.tap((outcome) =>
1176
+ Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) }),
1177
+ ),
1178
+ ),
1179
+ auxiliaryScope,
1180
+ );
1181
+
1182
+ const retired = yield* Effect.forkChild(Fiber.joinAll([deliveryFiber, hostJoin, backfill]));
1183
+
1184
+ const auxiliaryPending = () =>
1185
+ deliveryFiber.pollUnsafe() === undefined ||
1186
+ hostFiber.pollUnsafe() === undefined ||
1187
+ backfill.pollUnsafe() === undefined;
1188
+
1189
+ while (retired.pollUnsafe() === undefined && auxiliaryPending()) {
1190
+ const now = yield* Clock.currentTimeMillis;
1191
+ const until = DateTime.toEpochMillis(yieldAfter);
1192
+
1193
+ if (now >= until) break;
1194
+
1195
+ const next = Math.min(
1196
+ result.nextAttemptAt ?? Infinity,
1197
+ now + config.wakeScanInterval,
1198
+ until,
1199
+ );
1200
+
1201
+ const ready = yield* Effect.raceFirst(
1202
+ Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as(true)),
1203
+ Fiber.await(retired).pipe(Effect.as(false)),
1204
+ );
1205
+
1206
+ if (!ready || retired.pollUnsafe() !== undefined || !auxiliaryPending()) break;
1207
+ if ((yield* Clock.currentTimeMillis) >= until) break;
1208
+
1209
+ started = yield* beginNative(observed);
1210
+ result = yield* advance(started, yieldAfter, observed);
1211
+ if (result.phase === "actionable") phase = "actionable";
1212
+ recovered += result.recovered;
1213
+ settled += result.settled;
1214
+ observed.nativeOnly = false;
1215
+ }
1216
+ // Preserve driver-owned Claim deadlines and failures, then close every
1217
+ // listener before the one final alarm decision.
1218
+ yield* Fiber.join(retired);
1219
+ yield* Scope.close(auxiliaryScope, Exit.void);
1118
1220
  yield* failpoint.hit("maintenance:finish:before");
1119
1221
 
1120
- const alarmDisposition = yield* mutations.withSnapshot((active) =>
1222
+ const disposition = yield* mutations.withSnapshot((active) =>
1121
1223
  Effect.gen(function* () {
1122
- const publicationDeadline = yield* pendingDeadline;
1224
+ const latest = yield* pendingDeadline;
1225
+ const now = yield* Clock.currentTimeMillis;
1123
1226
 
1124
- return yield* runTransaction("finish maintenance pass", () =>
1227
+ return yield* runTransaction("finish maintenance event", () =>
1125
1228
  ctx.storage.transaction(async (transaction) => {
1126
1229
  const { state } = await readMaintenanceState(transaction);
1127
1230
 
1128
- // Autonomous work and in-flight mutations intentionally leave the observed
1129
- // generation dirty. Otherwise acknowledge only the pass-start generation.
1130
- const processed =
1131
- autonomous || started.activeAtStart > 0 || active > 0
1132
- ? state.processed
1133
- : state.processed > started.generation
1134
- ? state.processed
1135
- : started.generation;
1136
-
1137
- const next = ThreadMaintenanceState.make({
1138
- ...Struct.omit(state, ["retry"]),
1139
- processed,
1140
- nonterminal: remaining.length,
1141
- bindingRetries: (state.bindingRetries ?? []).filter((retry) =>
1142
- remaining.some((row) => row.submissionId === retry.submissionId),
1143
- ),
1144
- ...(autonomous && !progressed
1145
- ? {
1146
- retry: MaintenanceRetry.make({
1147
- generation: started.generation,
1148
- notBefore: now + delay,
1149
- nativeOnly: true,
1150
- stalls: Math.min(30, started.stalls + 1),
1151
- }),
1152
- }
1153
- : {}),
1154
- });
1155
-
1156
- await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
1157
- if (autonomous) {
1158
- // Replace the crash-fallback slot with this pass's bounded backoff. The target
1159
- // is never earlier than the begin-pass fallback, so workerd does not cancel
1160
- // this running alarm handler before its report/span can complete.
1161
- const nativeDeadline =
1162
- started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation
1163
- ? now + minimumAlarmDelay
1164
- : now + delay;
1165
-
1166
- await transaction.setAlarm(
1167
- Option.isSome(publicationDeadline)
1168
- ? Math.max(
1169
- now + minimumAlarmDelay,
1170
- Math.min(nativeDeadline, publicationDeadline.value),
1171
- )
1172
- : nativeDeadline,
1173
- );
1231
+ const native =
1232
+ active > 0 || state.dirty > state.processed
1233
+ ? state.retry?.generation === state.dirty && active === 0
1234
+ ? Math.max(now + minimumAlarmDelay, state.retry.notBefore)
1235
+ : state.dirty === observed.generation && active === 0
1236
+ ? (result.nextAttemptAt ?? now + config.wakeScanInterval)
1237
+ : now + minimumAlarmDelay
1238
+ : Infinity;
1174
1239
 
1175
- return "rearmed" as const;
1176
- }
1177
- if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
1178
- // A mutation overlapped this pass's observation window or raced
1179
- // acknowledgement. It stays dirty and its pre-armed bounded alarm survives;
1180
- // unseen effects are never acknowledged. Do not accelerate that future alarm
1181
- // from inside the current handler: workerd cancels a running handler when it
1182
- // writes an earlier slot.
1183
- await ensureTransactionAlarmBy(
1184
- transaction,
1185
- Option.isSome(publicationDeadline)
1186
- ? Math.max(
1187
- now + minimumAlarmDelay,
1188
- Math.min(now + config.wakeScanInterval, publicationDeadline.value),
1189
- )
1190
- : now + config.wakeScanInterval,
1191
- );
1240
+ const next = Option.isSome(latest) ? Math.min(native, latest.value) : native;
1192
1241
 
1193
- return "rearmed" as const;
1194
- }
1195
- if (Option.isSome(publicationDeadline)) {
1196
- await transaction.setAlarm(
1197
- Math.max(now + minimumAlarmDelay, publicationDeadline.value),
1198
- );
1242
+ if (Number.isFinite(next)) {
1243
+ await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
1199
1244
 
1200
1245
  return "rearmed" as const;
1201
1246
  }
@@ -1209,15 +1254,23 @@ export class ThreadMaintenance extends Context.Service<
1209
1254
 
1210
1255
  yield* failpoint.hit("maintenance:finish:after");
1211
1256
 
1212
- return yield* annotate(
1213
- MaintenancePassReport.make({
1214
- phase: "actionable",
1215
- recovered: recovered.length,
1216
- settled: Option.isSome(settlement) ? 1 : 0,
1217
- nonterminal: remaining.length,
1218
- alarm: alarmDisposition,
1219
- }),
1220
- );
1257
+ const report = MaintenancePassReport.make({
1258
+ phase,
1259
+ recovered,
1260
+ settled,
1261
+ nonterminal: result.nonterminal,
1262
+ alarm: disposition,
1263
+ });
1264
+
1265
+ yield* Effect.annotateCurrentSpan({
1266
+ phase: report.phase,
1267
+ recovered: report.recovered,
1268
+ settled: report.settled,
1269
+ nonterminal: report.nonterminal,
1270
+ alarm: report.alarm,
1271
+ });
1272
+
1273
+ return report;
1221
1274
  });
1222
1275
 
1223
1276
  return ThreadMaintenance.of({