@effect-agent/platform-cloudflare 0.1.0-beta.13 → 0.1.0-beta.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@effect-agent/platform-cloudflare",
3
- "version": "0.1.0-beta.13",
3
+ "version": "0.1.0-beta.15",
4
4
  "exports": {
5
5
  ".": {
6
6
  "types": "./dist/index.d.mts",
@@ -8,10 +8,10 @@
8
8
  }
9
9
  },
10
10
  "dependencies": {
11
- "@effect-agent/core": "0.1.0-beta.13",
12
- "@effect-agent/sandbox": "0.1.0-beta.13",
13
- "@effect-agent/session": "0.1.0-beta.13",
14
- "@effect-agent/storage-cloudflare": "0.1.0-beta.13",
11
+ "@effect-agent/core": "0.1.0-beta.15",
12
+ "@effect-agent/sandbox": "0.1.0-beta.15",
13
+ "@effect-agent/session": "0.1.0-beta.15",
14
+ "@effect-agent/storage-cloudflare": "0.1.0-beta.15",
15
15
  "@effect/platform-browser": "4.0.0-beta.107",
16
16
  "@effect/sql-sqlite-do": "4.0.0-beta.107",
17
17
  "effect": "4.0.0-beta.107"
@@ -42,9 +42,9 @@
42
42
  "devDependencies": {
43
43
  "@cloudflare/vitest-pool-workers": "0.21.3",
44
44
  "@cloudflare/workers-types": "5.20260813.1",
45
- "@effect-agent/capabilities": "0.1.0-beta.6",
46
- "@effect-agent/engine": "0.1.0-beta.6",
47
- "@effect-agent/testing": "0.1.0-beta.6",
45
+ "@effect-agent/capabilities": "0.1.0-beta.14",
46
+ "@effect-agent/engine": "0.1.0-beta.14",
47
+ "@effect-agent/testing": "0.1.0-beta.14",
48
48
  "@effect/vitest": "4.0.0-beta.107",
49
49
  "effect-cf": "0.27.0",
50
50
  "esbuild": "0.28.1",
package/src/alarm.ts CHANGED
@@ -5,8 +5,20 @@ import {
5
5
  type DurableBindingFailure,
6
6
  type DurableWorkerFailure,
7
7
  type RecoveryReport,
8
+ type SubmissionSnapshot,
8
9
  } from "@effect-agent/session";
9
- import { Clock, Context, Effect, Layer, Option, Random, Ref, Schema, Stream } from "effect";
10
+ import {
11
+ Clock,
12
+ Context,
13
+ Effect,
14
+ Layer,
15
+ Option,
16
+ Random,
17
+ Ref,
18
+ Schema,
19
+ Semaphore,
20
+ Stream,
21
+ } from "effect";
10
22
 
11
23
  import { ConversationObjectIdentity, DurableObjectContext } from "./bindings.ts";
12
24
  import { CloudflareDurableRuntimeConfig } from "./config.ts";
@@ -17,11 +29,10 @@ import { CloudflareDurableRuntimeConfig } from "./config.ts";
17
29
  * and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
18
30
  * the slot always holds the EARLIEST deadline any caller asked for.
19
31
  *
20
- * The alarm invariant (plan §1.4): committed nonterminal work implies a committed alarm.
21
- * It is established by pre-arming every mutating entry point and every pass arms the alarm
22
- * BEFORE its durable mutations — so an eviction at any failpoint leaves a persisted alarm
23
- * that workerd re-delivers to a fresh incarnation WITHOUT any incoming request. Spurious
24
- * alarms are harmless by design: a pass over an all-settled lane simply deletes the slot.
32
+ * The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
33
+ * maintenance generation and a committed alarm. Stable externally-driven waits may be
34
+ * nonterminal without retaining an alarm; their resolving mutation advances the generation and
35
+ * restores the alarm atomically.
25
36
  */
26
37
 
27
38
  /** The Durable Object alarm API failed; surfaces on host entry points as a typed refusal. */
@@ -63,17 +74,16 @@ export class DurableAlarmService extends Context.Service<
63
74
  * caused, and routing open uncertain-class Tool Calls into spurious Unknown Outcomes.
64
75
  * Deferral is contract-safe: wakes are droppable hints, every mutating entry point
65
76
  * pre-arms BEFORE its first durable mutation (the alarm invariant never rests on this
66
- * call), and the deferred wake is flushed when the pass completes.
77
+ * call). The pass's durable generation check observes any racing mutation, so the
78
+ * in-memory hint does not need to be flushed after a stable wait is acknowledged.
67
79
  */
68
80
  readonly scheduleNow: Effect.Effect<void, DurableAlarmError>;
69
81
  /**
70
- * Run one maintenance pass with wake deferral (see `scheduleNow`): `scheduleNow` calls
71
- * while `body` executes coalesce into one flag, flushed as an immediate re-arm after the
72
- * pass — failures of the flush are logged and swallowed (hints are droppable; the pass's
73
- * own re-arm policy already bounded the next delivery).
82
+ * Run one maintenance pass with wake deferral (see `scheduleNow`). Calls made while `body`
83
+ * executes are droppable promptness hints; correctness rests on the durable generation.
74
84
  */
75
85
  readonly withWakesDeferred: <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
76
- /** Clear the slot; only the maintenance pass does this, and only when all work settled. */
86
+ /** Clear the slot; correctness-sensitive clears live in maintenance generation transactions. */
77
87
  readonly cancel: Effect.Effect<void, DurableAlarmError>;
78
88
  }
79
89
  >()("@effect-agent/platform-cloudflare/DurableAlarmService") {
@@ -87,7 +97,6 @@ export class DurableAlarmService extends Context.Service<
87
97
  * on top of the already-committed pre-armed alarm.
88
98
  */
89
99
  const runningPasses = yield* Ref.make(0);
90
- const wakeDeferred = yield* Ref.make(false);
91
100
  const scheduled = Effect.tryPromise({
92
101
  try: () => ctx.storage.getAlarm(),
93
102
  catch: alarmFailure("get alarm"),
@@ -113,33 +122,12 @@ export class DurableAlarmService extends Context.Service<
113
122
  Effect.flatMap((now) => ensureScheduledBy(now)),
114
123
  );
115
124
  const scheduleNow = Ref.get(runningPasses).pipe(
116
- Effect.flatMap((passes) => (passes > 0 ? Ref.set(wakeDeferred, true) : armNow)),
117
- );
118
- /**
119
- * Flush after the LAST concurrent pass: the re-arm lands at the very end of the alarm
120
- * handler, where a superseding cancellation only re-delivers to the already-idempotent
121
- * pass. Runs inside `Effect.ensuring`, so flush failures are logged, never raised.
122
- */
123
- const flushDeferredWake = Ref.get(runningPasses).pipe(
124
- Effect.flatMap((passes) =>
125
- passes > 0
126
- ? Effect.void
127
- : Ref.getAndSet(wakeDeferred, false).pipe(
128
- Effect.flatMap((wanted) => (wanted ? armNow : Effect.void)),
129
- ),
130
- ),
131
- Effect.catch((error) =>
132
- Effect.logWarning("DurableAlarmService: deferred wake flush failed", error),
133
- ),
125
+ Effect.flatMap((passes) => (passes > 0 ? Effect.void : armNow)),
134
126
  );
135
127
  const withWakesDeferred = <A, E, R>(body: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
136
128
  Ref.update(runningPasses, (passes) => passes + 1).pipe(
137
129
  Effect.andThen(body),
138
- Effect.ensuring(
139
- Ref.update(runningPasses, (passes) => passes - 1).pipe(
140
- Effect.andThen(flushDeferredWake),
141
- ),
142
- ),
130
+ Effect.ensuring(Ref.update(runningPasses, (passes) => passes - 1)),
143
131
  );
144
132
  const cancel = Effect.tryPromise({
145
133
  try: () => ctx.storage.deleteAlarm(),
@@ -161,41 +149,130 @@ export class DurableAlarmService extends Context.Service<
161
149
  export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
162
150
  "@effect-agent/platform-cloudflare/MaintenancePassReport",
163
151
  )({
152
+ /** `caught-up` is generation-only; `actionable` ran recovery and one bounded drain. */
153
+ phase: Schema.Literals(["caught-up", "actionable"]),
164
154
  /** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
165
155
  recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
166
156
  /** Settlements the drain pass finalized. */
167
157
  settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
168
158
  /** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
169
159
  nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
170
- /** `rearmed` while nonterminal work remains, `cleared` once everything settled. */
160
+ /** `rearmed` for dirty/autonomous work, `cleared` for stable waits or settlement. */
171
161
  alarm: Schema.Literals(["rearmed", "cleared"]),
172
162
  }) {}
173
163
 
164
+ /** Fault boundaries around every maintenance-owned durable mutation. */
165
+ export type ConversationMaintenanceFailpointLocation =
166
+ | "maintenance:dirty:before"
167
+ | "maintenance:dirty:after"
168
+ | "maintenance:mutation:armed"
169
+ | "maintenance:mutation:finished"
170
+ | "maintenance:ensure:before"
171
+ | "maintenance:ensure:after"
172
+ | "maintenance:begin:before"
173
+ | "maintenance:begin:after"
174
+ | "maintenance:finish:before"
175
+ | "maintenance:finish:after";
176
+
177
+ export type ConversationMaintenanceFailpointHandler = (
178
+ location: ConversationMaintenanceFailpointLocation,
179
+ ) => Effect.Effect<void>;
180
+
181
+ /** Test-only fault authority; production uses the inert layer. */
182
+ export class ConversationMaintenanceFailpoint extends Context.Service<
183
+ ConversationMaintenanceFailpoint,
184
+ {
185
+ readonly hit: ConversationMaintenanceFailpointHandler;
186
+ }
187
+ >()("@effect-agent/platform-cloudflare/ConversationMaintenanceFailpoint") {
188
+ static readonly layer = Layer.succeed(this)({ hit: () => Effect.void });
189
+ }
190
+
191
+ const MaintenanceGeneration = Schema.BigIntFromString.check(
192
+ Schema.isGreaterThanOrEqualToBigInt(0n),
193
+ );
194
+
195
+ /** Versioned, platform-private maintenance state stored through Durable Object KV. */
196
+ class ConversationMaintenanceState extends Schema.Class<ConversationMaintenanceState>(
197
+ "@effect-agent/platform-cloudflare/ConversationMaintenanceState",
198
+ )({
199
+ schemaVersion: Schema.Literal(1),
200
+ dirty: MaintenanceGeneration,
201
+ processed: MaintenanceGeneration,
202
+ nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
203
+ }) {}
204
+
205
+ const MAINTENANCE_STATE_KEY = "effect-agent:conversation-maintenance:v1";
206
+ const decodeMaintenanceState = Schema.decodeUnknownSync(ConversationMaintenanceState);
207
+ const encodeMaintenanceState = Schema.encodeSync(ConversationMaintenanceState);
208
+
209
+ const initialMaintenanceState = (): ConversationMaintenanceState =>
210
+ ConversationMaintenanceState.make({
211
+ schemaVersion: 1,
212
+ // Bootstrap Objects created by the pre-generation release without scanning the ledger in
213
+ // the constructor. One useful pass classifies and acknowledges any existing obligation.
214
+ dirty: 1n,
215
+ processed: 0n,
216
+ nonterminal: 0,
217
+ });
218
+
219
+ const readMaintenanceState = async (
220
+ transaction: DurableObjectTransaction,
221
+ ): Promise<{ readonly state: ConversationMaintenanceState; readonly initialized: boolean }> => {
222
+ const encoded = await transaction.get(MAINTENANCE_STATE_KEY);
223
+ return encoded === undefined
224
+ ? { state: initialMaintenanceState(), initialized: false }
225
+ : { state: decodeMaintenanceState(encoded), initialized: true };
226
+ };
227
+
228
+ const ensureTransactionAlarmBy = async (
229
+ transaction: DurableObjectTransaction,
230
+ deadline: number,
231
+ ): Promise<void> => {
232
+ const scheduled = await transaction.getAlarm();
233
+ if (scheduled === null || scheduled > deadline) {
234
+ await transaction.setAlarm(deadline);
235
+ }
236
+ };
237
+
238
+ const stableExternalWait = (
239
+ snapshot: SubmissionSnapshot,
240
+ reports: ReadonlyMap<string, RecoveryReport>,
241
+ ): boolean => {
242
+ switch (snapshot.state) {
243
+ case "suspended":
244
+ case "unknown":
245
+ case "joined":
246
+ return true;
247
+ case "admitted":
248
+ return reports.get(snapshot.submissionId)?.decision._tag === "AwaitParentEstablishment";
249
+ case "input-applied":
250
+ case "joining":
251
+ case "ready":
252
+ case "running":
253
+ case "settled":
254
+ case "terminalizing":
255
+ return false;
256
+ }
257
+ };
258
+
174
259
  export type MaintenancePassFailure =
175
260
  | DurableWorkerFailure
176
261
  | DurableBindingFailure
177
262
  | DurableAlarmError;
178
263
 
179
264
  /**
180
- * The idempotent maintenance pass and the alarm-invariant helpers (plan §1.4, D-P6-1/2).
181
- *
182
- * `pass` = pre-arm → `runRecovery` → `processConversationResolved` → re-arm-or-clear:
265
+ * Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
183
266
  *
184
- * 1. **Pre-arm**: the alarm is re-armed at `now + wakeScanInterval` BEFORE any work, so an
185
- * eviction (or thrown failure → workerd alarm retry) at any point of the pass leaves a
186
- * committed alarm and the fresh incarnation converges without an incoming request.
187
- * 2. **Reconcile before new work** (exit gate): every pass classifies and repairs every
188
- * nonterminal Submission before the drain claims anything; the pure classifier and the
189
- * repair executors are idempotent, so at-least-once alarm delivery re-runs them safely.
190
- * 3. **Drain**: one bounded `processConversationResolved` pass over this Object's lane — the
191
- * D-P6-1 shape; no infinite `runResolvedWorker` loop ever pins the Object.
192
- * 4. **Re-arm policy**: nonterminal work re-arms at `now + min(backoff-with-jitter,
193
- * wakeScanInterval)` — the scan interval bounds every wait (lease expiry of a dead
194
- * incarnation included, since claims retry each pass) and the backoff (reset on progress,
195
- * grown otherwise) keeps stuck lanes from busy-spinning; all settled clears the slot.
267
+ * `pass` = generation snapshot/pre-arm recovery bounded drain generation acknowledgement:
196
268
  *
197
- * Every step is a durability-protocol step that already tolerates re-execution, which is
198
- * what makes double-fired alarms harmless (the alarm.test.ts gate).
269
+ * 1. One storage transaction reads dirty/processed and re-arms before work. A caught-up forced
270
+ * alarm takes an O(1) path without recovery, ledger scans, or canonical-history reads.
271
+ * 2. Recovery still strictly precedes a new claim, and one bounded drain advances the lane.
272
+ * 3. The final transaction acknowledges only the generation observed at pass start. A racing
273
+ * mutation therefore remains `dirty > processed` and retains its atomically-established alarm.
274
+ * 4. Stable external waits acknowledge and clear. Autonomous retry, indeterminate, and lease
275
+ * recovery states leave their generation dirty and retain bounded backoff rearming.
199
276
  */
200
277
  export class ConversationMaintenance extends Context.Service<
201
278
  ConversationMaintenance,
@@ -203,17 +280,18 @@ export class ConversationMaintenance extends Context.Service<
203
280
  /** One idempotent maintenance pass; failures propagate so workerd retries the alarm. */
204
281
  readonly pass: Effect.Effect<MaintenancePassReport, MaintenancePassFailure>;
205
282
  /**
206
- * Defensive half of the alarm invariant, run by the constructor gate: if any local
207
- * Submission is nonterminal and no alarm is scheduled, arm one within the scan interval.
208
- * Local-only (one ledger scan + the alarm slot) — never a recovery pass, never transport.
283
+ * Constructor gate: initialize/inspect only the O(1) maintenance record and ensure a dirty
284
+ * generation has an alarm. It never scans the ledger or canonical history.
209
285
  */
210
286
  readonly ensureAlarm: Effect.Effect<void, MaintenancePassFailure>;
211
287
  /**
212
- * The pre-arm every mutating entry point runs BEFORE its first durable mutation: with
213
- * the alarm committed first, a committed admission (or any other committed nonterminal
214
- * transition) can never be observed without the alarm that will finish it.
288
+ * Serialize the pre-arm boundary with pass acknowledgement, advance the durable dirty
289
+ * generation and arm the alarm in one transaction BEFORE running the caller's mutation.
290
+ * A pass cannot acknowledge while that mutation remains in flight.
215
291
  */
216
- readonly preArm: Effect.Effect<void, DurableAlarmError>;
292
+ readonly withMutation: <A, E, R>(
293
+ body: Effect.Effect<A, E, R>,
294
+ ) => Effect.Effect<A, E | DurableAlarmError, R>;
217
295
  }
218
296
  >()("@effect-agent/platform-cloudflare/ConversationMaintenance") {
219
297
  static readonly layer: Layer.Layer<
@@ -223,8 +301,10 @@ export class ConversationMaintenance extends Context.Service<
223
301
  | AgentBindingResolver
224
302
  | SubmissionLedger
225
303
  | DurableAlarmService
304
+ | ConversationMaintenanceFailpoint
226
305
  | CloudflareDurableRuntimeConfig
227
306
  | ConversationObjectIdentity
307
+ | DurableObjectContext
228
308
  > = Layer.effect(ConversationMaintenance)(
229
309
  Effect.gen(function* () {
230
310
  const runtime = yield* DurableAgentRuntime;
@@ -233,21 +313,114 @@ export class ConversationMaintenance extends Context.Service<
233
313
  const alarm = yield* DurableAlarmService;
234
314
  const config = yield* CloudflareDurableRuntimeConfig;
235
315
  const identity = yield* ConversationObjectIdentity;
316
+ const { ctx } = yield* DurableObjectContext;
317
+ const failpoint = yield* ConversationMaintenanceFailpoint;
236
318
 
237
319
  /**
238
320
  * Consecutive no-progress passes — an in-memory CACHE, not state: a fresh incarnation
239
321
  * restarts at zero and merely re-arms sooner than a long-lived one would have.
240
322
  */
241
323
  const stalls = yield* Ref.make(0);
324
+ /**
325
+ * Incarnation-local mutation count guarded with the generation transactions below. It is
326
+ * deliberately not durable: after eviction every begun mutation has stopped, while its
327
+ * pre-armed dirty generation remains durable for recovery. The short gate never spans the
328
+ * caller's mutation or cross-Object I/O.
329
+ */
330
+ const activeMutations = yield* Ref.make(0);
331
+ const generationGate = yield* Semaphore.make(1);
332
+ // At-least-once deliveries are idempotent, but overlapping pass bodies could otherwise
333
+ // acknowledge state while a sibling pass is still mutating it. Port/RPC mutations do not
334
+ // take this permit, so cross-Object I/O cannot deadlock the maintenance serialization.
335
+ const maintenancePassGate = yield* Semaphore.make(1);
336
+ const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
242
337
 
243
- const preArm = Clock.currentTimeMillis.pipe(
244
- Effect.flatMap((now) => alarm.ensureScheduledBy(now + config.wakeScanInterval)),
245
- );
338
+ const runTransaction = <A>(
339
+ operation: string,
340
+ transaction: () => Promise<A>,
341
+ ): Effect.Effect<A, DurableAlarmError> =>
342
+ Effect.tryPromise({
343
+ try: transaction,
344
+ catch: alarmFailure(operation),
345
+ });
246
346
 
247
- const countNonterminal = Stream.runCollect(ledger.scanNonterminal).pipe(
248
- Effect.map((snapshots) => snapshots.length),
347
+ const beginMutation = Effect.fn("ConversationMaintenance.beginMutation")(function* () {
348
+ yield* failpoint.hit("maintenance:dirty:before");
349
+ const now = yield* Clock.currentTimeMillis;
350
+ yield* runTransaction("advance maintenance generation", () =>
351
+ ctx.storage.transaction(async (transaction) => {
352
+ const { state } = await readMaintenanceState(transaction);
353
+ const next = ConversationMaintenanceState.make({
354
+ ...state,
355
+ dirty: state.dirty + 1n,
356
+ });
357
+ await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
358
+ // The earliest configured retry bounds a newly actionable mutation without relying
359
+ // on its best-effort immediate wake hint.
360
+ await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
361
+ }),
362
+ );
363
+ yield* failpoint.hit("maintenance:dirty:after");
364
+ yield* Ref.update(activeMutations, (active) => active + 1);
365
+ });
366
+
367
+ const endMutation = generationGate.withPermit(
368
+ Ref.update(activeMutations, (active) => Math.max(0, active - 1)),
249
369
  );
250
370
 
371
+ const withMutation = <A, E, R>(
372
+ body: Effect.Effect<A, E, R>,
373
+ ): Effect.Effect<A, E | DurableAlarmError, R> =>
374
+ Effect.acquireUseRelease(
375
+ generationGate.withPermit(beginMutation()),
376
+ () =>
377
+ failpoint.hit("maintenance:mutation:armed").pipe(
378
+ Effect.andThen(body),
379
+ Effect.tap(() => failpoint.hit("maintenance:mutation:finished")),
380
+ ),
381
+ () => endMutation,
382
+ );
383
+
384
+ const ensureAlarm = Effect.fn("ConversationMaintenance.ensureAlarm")(function* () {
385
+ yield* failpoint.hit("maintenance:ensure:before");
386
+ const now = yield* Clock.currentTimeMillis;
387
+ yield* runTransaction("ensure maintenance alarm", () =>
388
+ ctx.storage.transaction(async (transaction) => {
389
+ const { state, initialized } = await readMaintenanceState(transaction);
390
+ if (!initialized) {
391
+ await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
392
+ }
393
+ if (state.dirty > state.processed) {
394
+ await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
395
+ }
396
+ }),
397
+ );
398
+ yield* failpoint.hit("maintenance:ensure:after");
399
+ });
400
+
401
+ const beginPass = Effect.fn("ConversationMaintenance.beginPass")(function* () {
402
+ yield* failpoint.hit("maintenance:begin:before");
403
+ const now = yield* Clock.currentTimeMillis;
404
+ const result = yield* runTransaction("begin maintenance pass", () =>
405
+ ctx.storage.transaction(async (transaction) => {
406
+ const { state, initialized } = await readMaintenanceState(transaction);
407
+ if (!initialized) {
408
+ await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
409
+ }
410
+ if (state.processed >= state.dirty) {
411
+ await transaction.deleteAlarm();
412
+ return { _tag: "CaughtUp" as const, nonterminal: state.nonterminal };
413
+ }
414
+ // Pre-arm the earliest retry before recovery. A successful finish may move this slot
415
+ // LATER to its bounded backoff, which does not cancel the running handler.
416
+ await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
417
+ return { _tag: "Actionable" as const, generation: state.dirty };
418
+ }),
419
+ );
420
+ yield* failpoint.hit("maintenance:begin:after");
421
+ return result;
422
+ });
423
+
251
424
  const rearmDelay = Effect.fn("ConversationMaintenance.rearmDelay")(function* (
252
425
  progressed: boolean,
253
426
  ) {
@@ -268,66 +441,110 @@ export class ConversationMaintenance extends Context.Service<
268
441
  MaintenancePassReport,
269
442
  MaintenancePassFailure
270
443
  > {
271
- // Step 1 — pre-arm: an abort or throw anywhere below leaves a committed alarm.
272
- yield* preArm;
444
+ const annotate = (report: MaintenancePassReport) =>
445
+ Effect.annotateCurrentSpan({
446
+ phase: report.phase,
447
+ recovered: report.recovered,
448
+ settled: report.settled,
449
+ nonterminal: report.nonterminal,
450
+ alarm: report.alarm,
451
+ }).pipe(Effect.as(report));
452
+
453
+ const started = yield* generationGate.withPermit(
454
+ Effect.gen(function* () {
455
+ const activeAtStart = yield* Ref.get(activeMutations);
456
+ const generation = yield* beginPass();
457
+ return { ...generation, activeAtStart };
458
+ }),
459
+ );
460
+ if (started._tag === "CaughtUp") {
461
+ return yield* annotate(
462
+ MaintenancePassReport.make({
463
+ phase: "caught-up",
464
+ recovered: 0,
465
+ settled: 0,
466
+ nonterminal: started.nonterminal,
467
+ alarm: "cleared",
468
+ }),
469
+ );
470
+ }
273
471
  // Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
274
472
  const recovered: ReadonlyArray<RecoveryReport> = yield* runtime.runRecovery;
275
473
  // Step 3 — one bounded drain pass over this Object's own lane.
276
474
  const settlements = yield* runtime
277
475
  .processConversationResolved(identity.conversationId)
278
476
  .pipe(Effect.provideService(AgentBindingResolver, resolver));
279
- // Step 4 re-arm or clear.
280
- const nonterminal = yield* countNonterminal;
281
- if (nonterminal === 0) {
282
- yield* alarm.cancel;
283
- yield* Ref.set(stalls, 0);
284
- // Close the cancel/admission interleaving window: a submission committing between
285
- // the count and the cancel (Durable Object events interleave at storage-operation
286
- // boundaries) must not be left without its alarm. The recount re-arms if anything
287
- // appeared; the submit entry's own `scheduleNow` covers commits after the recount.
288
- const appeared = yield* countNonterminal;
289
- if (appeared > 0) {
290
- yield* preArm;
291
- return MaintenancePassReport.make({
292
- recovered: recovered.length,
293
- settled: settlements.length,
294
- nonterminal: appeared,
295
- alarm: "rearmed",
296
- });
297
- }
298
- return MaintenancePassReport.make({
299
- recovered: recovered.length,
300
- settled: settlements.length,
301
- nonterminal,
302
- alarm: "cleared",
303
- });
304
- }
477
+ // Observe residual state before acknowledging this exact pass-start generation.
478
+ const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
479
+ const reports = new Map(recovered.map((report) => [report.submissionId, report]));
480
+ const autonomous = remaining.some((snapshot) => !stableExternalWait(snapshot, reports));
305
481
  const progressed =
306
482
  settlements.length > 0 || recovered.some((report) => report.disposition === "repaired");
307
- const delay = yield* rearmDelay(progressed);
483
+ const delay = autonomous ? yield* rearmDelay(progressed) : 0;
308
484
  const now = yield* Clock.currentTimeMillis;
309
- yield* alarm.ensureScheduledBy(now + delay);
310
- return MaintenancePassReport.make({
311
- recovered: recovered.length,
312
- settled: settlements.length,
313
- nonterminal,
314
- alarm: "rearmed",
315
- });
316
- });
317
-
318
- const ensureAlarm = Effect.fn("ConversationMaintenance.ensureAlarm")(function* () {
319
- const nonterminal = yield* countNonterminal;
320
- if (nonterminal === 0) return;
321
- yield* preArm;
485
+ yield* failpoint.hit("maintenance:finish:before");
486
+ const alarmDisposition = yield* generationGate.withPermit(
487
+ Effect.gen(function* () {
488
+ const active = yield* Ref.get(activeMutations);
489
+ return yield* runTransaction("finish maintenance pass", () =>
490
+ ctx.storage.transaction(async (transaction) => {
491
+ const { state } = await readMaintenanceState(transaction);
492
+ // Autonomous work and in-flight mutations intentionally leave the observed
493
+ // generation dirty. Otherwise acknowledge only the pass-start generation.
494
+ const processed =
495
+ autonomous || started.activeAtStart > 0 || active > 0
496
+ ? state.processed
497
+ : state.processed > started.generation
498
+ ? state.processed
499
+ : started.generation;
500
+ const next = ConversationMaintenanceState.make({
501
+ ...state,
502
+ processed,
503
+ nonterminal: remaining.length,
504
+ });
505
+ await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
506
+ if (autonomous) {
507
+ // Replace the crash-fallback slot with this pass's bounded backoff. The target
508
+ // is never earlier than the begin-pass fallback, so workerd does not cancel
509
+ // this running alarm handler before its report/span can complete.
510
+ await transaction.setAlarm(now + delay);
511
+ return "rearmed" as const;
512
+ }
513
+ if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
514
+ // A mutation overlapped this pass's observation window or raced
515
+ // acknowledgement. It stays dirty and its pre-armed bounded alarm survives;
516
+ // unseen effects are never acknowledged. Do not accelerate that future alarm
517
+ // from inside the current handler: workerd cancels a running handler when it
518
+ // writes an earlier slot.
519
+ await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
520
+ return "rearmed" as const;
521
+ }
522
+ await transaction.deleteAlarm();
523
+ return "cleared" as const;
524
+ }),
525
+ );
526
+ }),
527
+ );
528
+ yield* failpoint.hit("maintenance:finish:after");
529
+ if (alarmDisposition === "cleared") {
530
+ yield* Ref.set(stalls, 0);
531
+ }
532
+ return yield* annotate(
533
+ MaintenancePassReport.make({
534
+ phase: "actionable",
535
+ recovered: recovered.length,
536
+ settled: settlements.length,
537
+ nonterminal: remaining.length,
538
+ alarm: alarmDisposition,
539
+ }),
540
+ );
322
541
  });
323
542
 
324
543
  return ConversationMaintenance.of({
325
- // Wake deferral (see `DurableAlarmService.scheduleNow`): an immediate wake written
326
- // while THIS handler runs would make workerd cancel it mid-Attempt; deferring keeps
327
- // the running pass alive and flushes the hint as the pass's final re-arm.
328
- pass: alarm.withWakesDeferred(pass()),
544
+ // A mid-pass immediate hint is droppable; durable dirty state decides the final alarm.
545
+ pass: alarm.withWakesDeferred(maintenancePassGate.withPermit(pass())),
329
546
  ensureAlarm: ensureAlarm(),
330
- preArm,
547
+ withMutation,
331
548
  });
332
549
  }),
333
550
  );