@effect-agent/platform-cloudflare 0.1.0-beta.13 → 0.1.0-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +107 -104
- package/dist/index.mjs +208 -90
- package/dist/index.mjs.map +1 -1
- package/package.json +8 -8
- package/src/alarm.ts +330 -113
- package/src/conversation-object.ts +34 -35
- package/src/index.ts +3 -2
- package/src/layers.ts +21 -5
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@effect-agent/platform-cloudflare",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.15",
|
|
4
4
|
"exports": {
|
|
5
5
|
".": {
|
|
6
6
|
"types": "./dist/index.d.mts",
|
|
@@ -8,10 +8,10 @@
|
|
|
8
8
|
}
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
|
-
"@effect-agent/core": "0.1.0-beta.
|
|
12
|
-
"@effect-agent/sandbox": "0.1.0-beta.
|
|
13
|
-
"@effect-agent/session": "0.1.0-beta.
|
|
14
|
-
"@effect-agent/storage-cloudflare": "0.1.0-beta.
|
|
11
|
+
"@effect-agent/core": "0.1.0-beta.15",
|
|
12
|
+
"@effect-agent/sandbox": "0.1.0-beta.15",
|
|
13
|
+
"@effect-agent/session": "0.1.0-beta.15",
|
|
14
|
+
"@effect-agent/storage-cloudflare": "0.1.0-beta.15",
|
|
15
15
|
"@effect/platform-browser": "4.0.0-beta.107",
|
|
16
16
|
"@effect/sql-sqlite-do": "4.0.0-beta.107",
|
|
17
17
|
"effect": "4.0.0-beta.107"
|
|
@@ -42,9 +42,9 @@
|
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@cloudflare/vitest-pool-workers": "0.21.3",
|
|
44
44
|
"@cloudflare/workers-types": "5.20260813.1",
|
|
45
|
-
"@effect-agent/capabilities": "0.1.0-beta.
|
|
46
|
-
"@effect-agent/engine": "0.1.0-beta.
|
|
47
|
-
"@effect-agent/testing": "0.1.0-beta.
|
|
45
|
+
"@effect-agent/capabilities": "0.1.0-beta.14",
|
|
46
|
+
"@effect-agent/engine": "0.1.0-beta.14",
|
|
47
|
+
"@effect-agent/testing": "0.1.0-beta.14",
|
|
48
48
|
"@effect/vitest": "4.0.0-beta.107",
|
|
49
49
|
"effect-cf": "0.27.0",
|
|
50
50
|
"esbuild": "0.28.1",
|
package/src/alarm.ts
CHANGED
|
@@ -5,8 +5,20 @@ import {
|
|
|
5
5
|
type DurableBindingFailure,
|
|
6
6
|
type DurableWorkerFailure,
|
|
7
7
|
type RecoveryReport,
|
|
8
|
+
type SubmissionSnapshot,
|
|
8
9
|
} from "@effect-agent/session";
|
|
9
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
Clock,
|
|
12
|
+
Context,
|
|
13
|
+
Effect,
|
|
14
|
+
Layer,
|
|
15
|
+
Option,
|
|
16
|
+
Random,
|
|
17
|
+
Ref,
|
|
18
|
+
Schema,
|
|
19
|
+
Semaphore,
|
|
20
|
+
Stream,
|
|
21
|
+
} from "effect";
|
|
10
22
|
|
|
11
23
|
import { ConversationObjectIdentity, DurableObjectContext } from "./bindings.ts";
|
|
12
24
|
import { CloudflareDurableRuntimeConfig } from "./config.ts";
|
|
@@ -17,11 +29,10 @@ import { CloudflareDurableRuntimeConfig } from "./config.ts";
|
|
|
17
29
|
* and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
|
|
18
30
|
* the slot always holds the EARLIEST deadline any caller asked for.
|
|
19
31
|
*
|
|
20
|
-
* The alarm invariant (plan §1.4): committed
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
* alarms are harmless by design: a pass over an all-settled lane simply deletes the slot.
|
|
32
|
+
* The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
|
|
33
|
+
* maintenance generation and a committed alarm. Stable externally-driven waits may be
|
|
34
|
+
* nonterminal without retaining an alarm; their resolving mutation advances the generation and
|
|
35
|
+
* restores the alarm atomically.
|
|
25
36
|
*/
|
|
26
37
|
|
|
27
38
|
/** The Durable Object alarm API failed; surfaces on host entry points as a typed refusal. */
|
|
@@ -63,17 +74,16 @@ export class DurableAlarmService extends Context.Service<
|
|
|
63
74
|
* caused, and routing open uncertain-class Tool Calls into spurious Unknown Outcomes.
|
|
64
75
|
* Deferral is contract-safe: wakes are droppable hints, every mutating entry point
|
|
65
76
|
* pre-arms BEFORE its first durable mutation (the alarm invariant never rests on this
|
|
66
|
-
* call)
|
|
77
|
+
* call). The pass's durable generation check observes any racing mutation, so the
|
|
78
|
+
* in-memory hint does not need to be flushed after a stable wait is acknowledged.
|
|
67
79
|
*/
|
|
68
80
|
readonly scheduleNow: Effect.Effect<void, DurableAlarmError>;
|
|
69
81
|
/**
|
|
70
|
-
* Run one maintenance pass with wake deferral (see `scheduleNow`)
|
|
71
|
-
*
|
|
72
|
-
* pass — failures of the flush are logged and swallowed (hints are droppable; the pass's
|
|
73
|
-
* own re-arm policy already bounded the next delivery).
|
|
82
|
+
* Run one maintenance pass with wake deferral (see `scheduleNow`). Calls made while `body`
|
|
83
|
+
* executes are droppable promptness hints; correctness rests on the durable generation.
|
|
74
84
|
*/
|
|
75
85
|
readonly withWakesDeferred: <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
76
|
-
/** Clear the slot;
|
|
86
|
+
/** Clear the slot; correctness-sensitive clears live in maintenance generation transactions. */
|
|
77
87
|
readonly cancel: Effect.Effect<void, DurableAlarmError>;
|
|
78
88
|
}
|
|
79
89
|
>()("@effect-agent/platform-cloudflare/DurableAlarmService") {
|
|
@@ -87,7 +97,6 @@ export class DurableAlarmService extends Context.Service<
|
|
|
87
97
|
* on top of the already-committed pre-armed alarm.
|
|
88
98
|
*/
|
|
89
99
|
const runningPasses = yield* Ref.make(0);
|
|
90
|
-
const wakeDeferred = yield* Ref.make(false);
|
|
91
100
|
const scheduled = Effect.tryPromise({
|
|
92
101
|
try: () => ctx.storage.getAlarm(),
|
|
93
102
|
catch: alarmFailure("get alarm"),
|
|
@@ -113,33 +122,12 @@ export class DurableAlarmService extends Context.Service<
|
|
|
113
122
|
Effect.flatMap((now) => ensureScheduledBy(now)),
|
|
114
123
|
);
|
|
115
124
|
const scheduleNow = Ref.get(runningPasses).pipe(
|
|
116
|
-
Effect.flatMap((passes) => (passes > 0 ?
|
|
117
|
-
);
|
|
118
|
-
/**
|
|
119
|
-
* Flush after the LAST concurrent pass: the re-arm lands at the very end of the alarm
|
|
120
|
-
* handler, where a superseding cancellation only re-delivers to the already-idempotent
|
|
121
|
-
* pass. Runs inside `Effect.ensuring`, so flush failures are logged, never raised.
|
|
122
|
-
*/
|
|
123
|
-
const flushDeferredWake = Ref.get(runningPasses).pipe(
|
|
124
|
-
Effect.flatMap((passes) =>
|
|
125
|
-
passes > 0
|
|
126
|
-
? Effect.void
|
|
127
|
-
: Ref.getAndSet(wakeDeferred, false).pipe(
|
|
128
|
-
Effect.flatMap((wanted) => (wanted ? armNow : Effect.void)),
|
|
129
|
-
),
|
|
130
|
-
),
|
|
131
|
-
Effect.catch((error) =>
|
|
132
|
-
Effect.logWarning("DurableAlarmService: deferred wake flush failed", error),
|
|
133
|
-
),
|
|
125
|
+
Effect.flatMap((passes) => (passes > 0 ? Effect.void : armNow)),
|
|
134
126
|
);
|
|
135
127
|
const withWakesDeferred = <A, E, R>(body: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
|
|
136
128
|
Ref.update(runningPasses, (passes) => passes + 1).pipe(
|
|
137
129
|
Effect.andThen(body),
|
|
138
|
-
Effect.ensuring(
|
|
139
|
-
Ref.update(runningPasses, (passes) => passes - 1).pipe(
|
|
140
|
-
Effect.andThen(flushDeferredWake),
|
|
141
|
-
),
|
|
142
|
-
),
|
|
130
|
+
Effect.ensuring(Ref.update(runningPasses, (passes) => passes - 1)),
|
|
143
131
|
);
|
|
144
132
|
const cancel = Effect.tryPromise({
|
|
145
133
|
try: () => ctx.storage.deleteAlarm(),
|
|
@@ -161,41 +149,130 @@ export class DurableAlarmService extends Context.Service<
|
|
|
161
149
|
export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
|
|
162
150
|
"@effect-agent/platform-cloudflare/MaintenancePassReport",
|
|
163
151
|
)({
|
|
152
|
+
/** `caught-up` is generation-only; `actionable` ran recovery and one bounded drain. */
|
|
153
|
+
phase: Schema.Literals(["caught-up", "actionable"]),
|
|
164
154
|
/** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
|
|
165
155
|
recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
166
156
|
/** Settlements the drain pass finalized. */
|
|
167
157
|
settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
168
158
|
/** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
|
|
169
159
|
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
170
|
-
/** `rearmed`
|
|
160
|
+
/** `rearmed` for dirty/autonomous work, `cleared` for stable waits or settlement. */
|
|
171
161
|
alarm: Schema.Literals(["rearmed", "cleared"]),
|
|
172
162
|
}) {}
|
|
173
163
|
|
|
164
|
+
/** Fault boundaries around every maintenance-owned durable mutation. */
|
|
165
|
+
export type ConversationMaintenanceFailpointLocation =
|
|
166
|
+
| "maintenance:dirty:before"
|
|
167
|
+
| "maintenance:dirty:after"
|
|
168
|
+
| "maintenance:mutation:armed"
|
|
169
|
+
| "maintenance:mutation:finished"
|
|
170
|
+
| "maintenance:ensure:before"
|
|
171
|
+
| "maintenance:ensure:after"
|
|
172
|
+
| "maintenance:begin:before"
|
|
173
|
+
| "maintenance:begin:after"
|
|
174
|
+
| "maintenance:finish:before"
|
|
175
|
+
| "maintenance:finish:after";
|
|
176
|
+
|
|
177
|
+
export type ConversationMaintenanceFailpointHandler = (
|
|
178
|
+
location: ConversationMaintenanceFailpointLocation,
|
|
179
|
+
) => Effect.Effect<void>;
|
|
180
|
+
|
|
181
|
+
/** Test-only fault authority; production uses the inert layer. */
|
|
182
|
+
export class ConversationMaintenanceFailpoint extends Context.Service<
|
|
183
|
+
ConversationMaintenanceFailpoint,
|
|
184
|
+
{
|
|
185
|
+
readonly hit: ConversationMaintenanceFailpointHandler;
|
|
186
|
+
}
|
|
187
|
+
>()("@effect-agent/platform-cloudflare/ConversationMaintenanceFailpoint") {
|
|
188
|
+
static readonly layer = Layer.succeed(this)({ hit: () => Effect.void });
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const MaintenanceGeneration = Schema.BigIntFromString.check(
|
|
192
|
+
Schema.isGreaterThanOrEqualToBigInt(0n),
|
|
193
|
+
);
|
|
194
|
+
|
|
195
|
+
/** Versioned, platform-private maintenance state stored through Durable Object KV. */
|
|
196
|
+
class ConversationMaintenanceState extends Schema.Class<ConversationMaintenanceState>(
|
|
197
|
+
"@effect-agent/platform-cloudflare/ConversationMaintenanceState",
|
|
198
|
+
)({
|
|
199
|
+
schemaVersion: Schema.Literal(1),
|
|
200
|
+
dirty: MaintenanceGeneration,
|
|
201
|
+
processed: MaintenanceGeneration,
|
|
202
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
203
|
+
}) {}
|
|
204
|
+
|
|
205
|
+
const MAINTENANCE_STATE_KEY = "effect-agent:conversation-maintenance:v1";
|
|
206
|
+
const decodeMaintenanceState = Schema.decodeUnknownSync(ConversationMaintenanceState);
|
|
207
|
+
const encodeMaintenanceState = Schema.encodeSync(ConversationMaintenanceState);
|
|
208
|
+
|
|
209
|
+
const initialMaintenanceState = (): ConversationMaintenanceState =>
|
|
210
|
+
ConversationMaintenanceState.make({
|
|
211
|
+
schemaVersion: 1,
|
|
212
|
+
// Bootstrap Objects created by the pre-generation release without scanning the ledger in
|
|
213
|
+
// the constructor. One useful pass classifies and acknowledges any existing obligation.
|
|
214
|
+
dirty: 1n,
|
|
215
|
+
processed: 0n,
|
|
216
|
+
nonterminal: 0,
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
const readMaintenanceState = async (
|
|
220
|
+
transaction: DurableObjectTransaction,
|
|
221
|
+
): Promise<{ readonly state: ConversationMaintenanceState; readonly initialized: boolean }> => {
|
|
222
|
+
const encoded = await transaction.get(MAINTENANCE_STATE_KEY);
|
|
223
|
+
return encoded === undefined
|
|
224
|
+
? { state: initialMaintenanceState(), initialized: false }
|
|
225
|
+
: { state: decodeMaintenanceState(encoded), initialized: true };
|
|
226
|
+
};
|
|
227
|
+
|
|
228
|
+
const ensureTransactionAlarmBy = async (
|
|
229
|
+
transaction: DurableObjectTransaction,
|
|
230
|
+
deadline: number,
|
|
231
|
+
): Promise<void> => {
|
|
232
|
+
const scheduled = await transaction.getAlarm();
|
|
233
|
+
if (scheduled === null || scheduled > deadline) {
|
|
234
|
+
await transaction.setAlarm(deadline);
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
const stableExternalWait = (
|
|
239
|
+
snapshot: SubmissionSnapshot,
|
|
240
|
+
reports: ReadonlyMap<string, RecoveryReport>,
|
|
241
|
+
): boolean => {
|
|
242
|
+
switch (snapshot.state) {
|
|
243
|
+
case "suspended":
|
|
244
|
+
case "unknown":
|
|
245
|
+
case "joined":
|
|
246
|
+
return true;
|
|
247
|
+
case "admitted":
|
|
248
|
+
return reports.get(snapshot.submissionId)?.decision._tag === "AwaitParentEstablishment";
|
|
249
|
+
case "input-applied":
|
|
250
|
+
case "joining":
|
|
251
|
+
case "ready":
|
|
252
|
+
case "running":
|
|
253
|
+
case "settled":
|
|
254
|
+
case "terminalizing":
|
|
255
|
+
return false;
|
|
256
|
+
}
|
|
257
|
+
};
|
|
258
|
+
|
|
174
259
|
export type MaintenancePassFailure =
|
|
175
260
|
| DurableWorkerFailure
|
|
176
261
|
| DurableBindingFailure
|
|
177
262
|
| DurableAlarmError;
|
|
178
263
|
|
|
179
264
|
/**
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
* `pass` = pre-arm → `runRecovery` → `processConversationResolved` → re-arm-or-clear:
|
|
265
|
+
* Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
|
|
183
266
|
*
|
|
184
|
-
*
|
|
185
|
-
* eviction (or thrown failure → workerd alarm retry) at any point of the pass leaves a
|
|
186
|
-
* committed alarm and the fresh incarnation converges without an incoming request.
|
|
187
|
-
* 2. **Reconcile before new work** (exit gate): every pass classifies and repairs every
|
|
188
|
-
* nonterminal Submission before the drain claims anything; the pure classifier and the
|
|
189
|
-
* repair executors are idempotent, so at-least-once alarm delivery re-runs them safely.
|
|
190
|
-
* 3. **Drain**: one bounded `processConversationResolved` pass over this Object's lane — the
|
|
191
|
-
* D-P6-1 shape; no infinite `runResolvedWorker` loop ever pins the Object.
|
|
192
|
-
* 4. **Re-arm policy**: nonterminal work re-arms at `now + min(backoff-with-jitter,
|
|
193
|
-
* wakeScanInterval)` — the scan interval bounds every wait (lease expiry of a dead
|
|
194
|
-
* incarnation included, since claims retry each pass) and the backoff (reset on progress,
|
|
195
|
-
* grown otherwise) keeps stuck lanes from busy-spinning; all settled clears the slot.
|
|
267
|
+
* `pass` = generation snapshot/pre-arm → recovery → bounded drain → generation acknowledgement:
|
|
196
268
|
*
|
|
197
|
-
*
|
|
198
|
-
*
|
|
269
|
+
* 1. One storage transaction reads dirty/processed and re-arms before work. A caught-up forced
|
|
270
|
+
* alarm takes an O(1) path without recovery, ledger scans, or canonical-history reads.
|
|
271
|
+
* 2. Recovery still strictly precedes a new claim, and one bounded drain advances the lane.
|
|
272
|
+
* 3. The final transaction acknowledges only the generation observed at pass start. A racing
|
|
273
|
+
* mutation therefore remains `dirty > processed` and retains its atomically-established alarm.
|
|
274
|
+
* 4. Stable external waits acknowledge and clear. Autonomous retry, indeterminate, and lease
|
|
275
|
+
* recovery states leave their generation dirty and retain bounded backoff rearming.
|
|
199
276
|
*/
|
|
200
277
|
export class ConversationMaintenance extends Context.Service<
|
|
201
278
|
ConversationMaintenance,
|
|
@@ -203,17 +280,18 @@ export class ConversationMaintenance extends Context.Service<
|
|
|
203
280
|
/** One idempotent maintenance pass; failures propagate so workerd retries the alarm. */
|
|
204
281
|
readonly pass: Effect.Effect<MaintenancePassReport, MaintenancePassFailure>;
|
|
205
282
|
/**
|
|
206
|
-
*
|
|
207
|
-
*
|
|
208
|
-
* Local-only (one ledger scan + the alarm slot) — never a recovery pass, never transport.
|
|
283
|
+
* Constructor gate: initialize/inspect only the O(1) maintenance record and ensure a dirty
|
|
284
|
+
* generation has an alarm. It never scans the ledger or canonical history.
|
|
209
285
|
*/
|
|
210
286
|
readonly ensureAlarm: Effect.Effect<void, MaintenancePassFailure>;
|
|
211
287
|
/**
|
|
212
|
-
*
|
|
213
|
-
* the alarm
|
|
214
|
-
*
|
|
288
|
+
* Serialize the pre-arm boundary with pass acknowledgement, advance the durable dirty
|
|
289
|
+
* generation and arm the alarm in one transaction BEFORE running the caller's mutation.
|
|
290
|
+
* A pass cannot acknowledge while that mutation remains in flight.
|
|
215
291
|
*/
|
|
216
|
-
readonly
|
|
292
|
+
readonly withMutation: <A, E, R>(
|
|
293
|
+
body: Effect.Effect<A, E, R>,
|
|
294
|
+
) => Effect.Effect<A, E | DurableAlarmError, R>;
|
|
217
295
|
}
|
|
218
296
|
>()("@effect-agent/platform-cloudflare/ConversationMaintenance") {
|
|
219
297
|
static readonly layer: Layer.Layer<
|
|
@@ -223,8 +301,10 @@ export class ConversationMaintenance extends Context.Service<
|
|
|
223
301
|
| AgentBindingResolver
|
|
224
302
|
| SubmissionLedger
|
|
225
303
|
| DurableAlarmService
|
|
304
|
+
| ConversationMaintenanceFailpoint
|
|
226
305
|
| CloudflareDurableRuntimeConfig
|
|
227
306
|
| ConversationObjectIdentity
|
|
307
|
+
| DurableObjectContext
|
|
228
308
|
> = Layer.effect(ConversationMaintenance)(
|
|
229
309
|
Effect.gen(function* () {
|
|
230
310
|
const runtime = yield* DurableAgentRuntime;
|
|
@@ -233,21 +313,114 @@ export class ConversationMaintenance extends Context.Service<
|
|
|
233
313
|
const alarm = yield* DurableAlarmService;
|
|
234
314
|
const config = yield* CloudflareDurableRuntimeConfig;
|
|
235
315
|
const identity = yield* ConversationObjectIdentity;
|
|
316
|
+
const { ctx } = yield* DurableObjectContext;
|
|
317
|
+
const failpoint = yield* ConversationMaintenanceFailpoint;
|
|
236
318
|
|
|
237
319
|
/**
|
|
238
320
|
* Consecutive no-progress passes — an in-memory CACHE, not state: a fresh incarnation
|
|
239
321
|
* restarts at zero and merely re-arms sooner than a long-lived one would have.
|
|
240
322
|
*/
|
|
241
323
|
const stalls = yield* Ref.make(0);
|
|
324
|
+
/**
|
|
325
|
+
* Incarnation-local mutation count guarded with the generation transactions below. It is
|
|
326
|
+
* deliberately not durable: after eviction every begun mutation has stopped, while its
|
|
327
|
+
* pre-armed dirty generation remains durable for recovery. The short gate never spans the
|
|
328
|
+
* caller's mutation or cross-Object I/O.
|
|
329
|
+
*/
|
|
330
|
+
const activeMutations = yield* Ref.make(0);
|
|
331
|
+
const generationGate = yield* Semaphore.make(1);
|
|
332
|
+
// At-least-once deliveries are idempotent, but overlapping pass bodies could otherwise
|
|
333
|
+
// acknowledge state while a sibling pass is still mutating it. Port/RPC mutations do not
|
|
334
|
+
// take this permit, so cross-Object I/O cannot deadlock the maintenance serialization.
|
|
335
|
+
const maintenancePassGate = yield* Semaphore.make(1);
|
|
336
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
242
337
|
|
|
243
|
-
const
|
|
244
|
-
|
|
245
|
-
|
|
338
|
+
const runTransaction = <A>(
|
|
339
|
+
operation: string,
|
|
340
|
+
transaction: () => Promise<A>,
|
|
341
|
+
): Effect.Effect<A, DurableAlarmError> =>
|
|
342
|
+
Effect.tryPromise({
|
|
343
|
+
try: transaction,
|
|
344
|
+
catch: alarmFailure(operation),
|
|
345
|
+
});
|
|
246
346
|
|
|
247
|
-
const
|
|
248
|
-
|
|
347
|
+
const beginMutation = Effect.fn("ConversationMaintenance.beginMutation")(function* () {
|
|
348
|
+
yield* failpoint.hit("maintenance:dirty:before");
|
|
349
|
+
const now = yield* Clock.currentTimeMillis;
|
|
350
|
+
yield* runTransaction("advance maintenance generation", () =>
|
|
351
|
+
ctx.storage.transaction(async (transaction) => {
|
|
352
|
+
const { state } = await readMaintenanceState(transaction);
|
|
353
|
+
const next = ConversationMaintenanceState.make({
|
|
354
|
+
...state,
|
|
355
|
+
dirty: state.dirty + 1n,
|
|
356
|
+
});
|
|
357
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
358
|
+
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
359
|
+
// on its best-effort immediate wake hint.
|
|
360
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
361
|
+
}),
|
|
362
|
+
);
|
|
363
|
+
yield* failpoint.hit("maintenance:dirty:after");
|
|
364
|
+
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
const endMutation = generationGate.withPermit(
|
|
368
|
+
Ref.update(activeMutations, (active) => Math.max(0, active - 1)),
|
|
249
369
|
);
|
|
250
370
|
|
|
371
|
+
const withMutation = <A, E, R>(
|
|
372
|
+
body: Effect.Effect<A, E, R>,
|
|
373
|
+
): Effect.Effect<A, E | DurableAlarmError, R> =>
|
|
374
|
+
Effect.acquireUseRelease(
|
|
375
|
+
generationGate.withPermit(beginMutation()),
|
|
376
|
+
() =>
|
|
377
|
+
failpoint.hit("maintenance:mutation:armed").pipe(
|
|
378
|
+
Effect.andThen(body),
|
|
379
|
+
Effect.tap(() => failpoint.hit("maintenance:mutation:finished")),
|
|
380
|
+
),
|
|
381
|
+
() => endMutation,
|
|
382
|
+
);
|
|
383
|
+
|
|
384
|
+
const ensureAlarm = Effect.fn("ConversationMaintenance.ensureAlarm")(function* () {
|
|
385
|
+
yield* failpoint.hit("maintenance:ensure:before");
|
|
386
|
+
const now = yield* Clock.currentTimeMillis;
|
|
387
|
+
yield* runTransaction("ensure maintenance alarm", () =>
|
|
388
|
+
ctx.storage.transaction(async (transaction) => {
|
|
389
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
390
|
+
if (!initialized) {
|
|
391
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
392
|
+
}
|
|
393
|
+
if (state.dirty > state.processed) {
|
|
394
|
+
await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
|
|
395
|
+
}
|
|
396
|
+
}),
|
|
397
|
+
);
|
|
398
|
+
yield* failpoint.hit("maintenance:ensure:after");
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
const beginPass = Effect.fn("ConversationMaintenance.beginPass")(function* () {
|
|
402
|
+
yield* failpoint.hit("maintenance:begin:before");
|
|
403
|
+
const now = yield* Clock.currentTimeMillis;
|
|
404
|
+
const result = yield* runTransaction("begin maintenance pass", () =>
|
|
405
|
+
ctx.storage.transaction(async (transaction) => {
|
|
406
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
407
|
+
if (!initialized) {
|
|
408
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
409
|
+
}
|
|
410
|
+
if (state.processed >= state.dirty) {
|
|
411
|
+
await transaction.deleteAlarm();
|
|
412
|
+
return { _tag: "CaughtUp" as const, nonterminal: state.nonterminal };
|
|
413
|
+
}
|
|
414
|
+
// Pre-arm the earliest retry before recovery. A successful finish may move this slot
|
|
415
|
+
// LATER to its bounded backoff, which does not cancel the running handler.
|
|
416
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
417
|
+
return { _tag: "Actionable" as const, generation: state.dirty };
|
|
418
|
+
}),
|
|
419
|
+
);
|
|
420
|
+
yield* failpoint.hit("maintenance:begin:after");
|
|
421
|
+
return result;
|
|
422
|
+
});
|
|
423
|
+
|
|
251
424
|
const rearmDelay = Effect.fn("ConversationMaintenance.rearmDelay")(function* (
|
|
252
425
|
progressed: boolean,
|
|
253
426
|
) {
|
|
@@ -268,66 +441,110 @@ export class ConversationMaintenance extends Context.Service<
|
|
|
268
441
|
MaintenancePassReport,
|
|
269
442
|
MaintenancePassFailure
|
|
270
443
|
> {
|
|
271
|
-
|
|
272
|
-
|
|
444
|
+
const annotate = (report: MaintenancePassReport) =>
|
|
445
|
+
Effect.annotateCurrentSpan({
|
|
446
|
+
phase: report.phase,
|
|
447
|
+
recovered: report.recovered,
|
|
448
|
+
settled: report.settled,
|
|
449
|
+
nonterminal: report.nonterminal,
|
|
450
|
+
alarm: report.alarm,
|
|
451
|
+
}).pipe(Effect.as(report));
|
|
452
|
+
|
|
453
|
+
const started = yield* generationGate.withPermit(
|
|
454
|
+
Effect.gen(function* () {
|
|
455
|
+
const activeAtStart = yield* Ref.get(activeMutations);
|
|
456
|
+
const generation = yield* beginPass();
|
|
457
|
+
return { ...generation, activeAtStart };
|
|
458
|
+
}),
|
|
459
|
+
);
|
|
460
|
+
if (started._tag === "CaughtUp") {
|
|
461
|
+
return yield* annotate(
|
|
462
|
+
MaintenancePassReport.make({
|
|
463
|
+
phase: "caught-up",
|
|
464
|
+
recovered: 0,
|
|
465
|
+
settled: 0,
|
|
466
|
+
nonterminal: started.nonterminal,
|
|
467
|
+
alarm: "cleared",
|
|
468
|
+
}),
|
|
469
|
+
);
|
|
470
|
+
}
|
|
273
471
|
// Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
|
|
274
472
|
const recovered: ReadonlyArray<RecoveryReport> = yield* runtime.runRecovery;
|
|
275
473
|
// Step 3 — one bounded drain pass over this Object's own lane.
|
|
276
474
|
const settlements = yield* runtime
|
|
277
475
|
.processConversationResolved(identity.conversationId)
|
|
278
476
|
.pipe(Effect.provideService(AgentBindingResolver, resolver));
|
|
279
|
-
//
|
|
280
|
-
const
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
yield* Ref.set(stalls, 0);
|
|
284
|
-
// Close the cancel/admission interleaving window: a submission committing between
|
|
285
|
-
// the count and the cancel (Durable Object events interleave at storage-operation
|
|
286
|
-
// boundaries) must not be left without its alarm. The recount re-arms if anything
|
|
287
|
-
// appeared; the submit entry's own `scheduleNow` covers commits after the recount.
|
|
288
|
-
const appeared = yield* countNonterminal;
|
|
289
|
-
if (appeared > 0) {
|
|
290
|
-
yield* preArm;
|
|
291
|
-
return MaintenancePassReport.make({
|
|
292
|
-
recovered: recovered.length,
|
|
293
|
-
settled: settlements.length,
|
|
294
|
-
nonterminal: appeared,
|
|
295
|
-
alarm: "rearmed",
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
return MaintenancePassReport.make({
|
|
299
|
-
recovered: recovered.length,
|
|
300
|
-
settled: settlements.length,
|
|
301
|
-
nonterminal,
|
|
302
|
-
alarm: "cleared",
|
|
303
|
-
});
|
|
304
|
-
}
|
|
477
|
+
// Observe residual state before acknowledging this exact pass-start generation.
|
|
478
|
+
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
479
|
+
const reports = new Map(recovered.map((report) => [report.submissionId, report]));
|
|
480
|
+
const autonomous = remaining.some((snapshot) => !stableExternalWait(snapshot, reports));
|
|
305
481
|
const progressed =
|
|
306
482
|
settlements.length > 0 || recovered.some((report) => report.disposition === "repaired");
|
|
307
|
-
const delay = yield* rearmDelay(progressed);
|
|
483
|
+
const delay = autonomous ? yield* rearmDelay(progressed) : 0;
|
|
308
484
|
const now = yield* Clock.currentTimeMillis;
|
|
309
|
-
yield*
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
485
|
+
yield* failpoint.hit("maintenance:finish:before");
|
|
486
|
+
const alarmDisposition = yield* generationGate.withPermit(
|
|
487
|
+
Effect.gen(function* () {
|
|
488
|
+
const active = yield* Ref.get(activeMutations);
|
|
489
|
+
return yield* runTransaction("finish maintenance pass", () =>
|
|
490
|
+
ctx.storage.transaction(async (transaction) => {
|
|
491
|
+
const { state } = await readMaintenanceState(transaction);
|
|
492
|
+
// Autonomous work and in-flight mutations intentionally leave the observed
|
|
493
|
+
// generation dirty. Otherwise acknowledge only the pass-start generation.
|
|
494
|
+
const processed =
|
|
495
|
+
autonomous || started.activeAtStart > 0 || active > 0
|
|
496
|
+
? state.processed
|
|
497
|
+
: state.processed > started.generation
|
|
498
|
+
? state.processed
|
|
499
|
+
: started.generation;
|
|
500
|
+
const next = ConversationMaintenanceState.make({
|
|
501
|
+
...state,
|
|
502
|
+
processed,
|
|
503
|
+
nonterminal: remaining.length,
|
|
504
|
+
});
|
|
505
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
506
|
+
if (autonomous) {
|
|
507
|
+
// Replace the crash-fallback slot with this pass's bounded backoff. The target
|
|
508
|
+
// is never earlier than the begin-pass fallback, so workerd does not cancel
|
|
509
|
+
// this running alarm handler before its report/span can complete.
|
|
510
|
+
await transaction.setAlarm(now + delay);
|
|
511
|
+
return "rearmed" as const;
|
|
512
|
+
}
|
|
513
|
+
if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
|
|
514
|
+
// A mutation overlapped this pass's observation window or raced
|
|
515
|
+
// acknowledgement. It stays dirty and its pre-armed bounded alarm survives;
|
|
516
|
+
// unseen effects are never acknowledged. Do not accelerate that future alarm
|
|
517
|
+
// from inside the current handler: workerd cancels a running handler when it
|
|
518
|
+
// writes an earlier slot.
|
|
519
|
+
await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
|
|
520
|
+
return "rearmed" as const;
|
|
521
|
+
}
|
|
522
|
+
await transaction.deleteAlarm();
|
|
523
|
+
return "cleared" as const;
|
|
524
|
+
}),
|
|
525
|
+
);
|
|
526
|
+
}),
|
|
527
|
+
);
|
|
528
|
+
yield* failpoint.hit("maintenance:finish:after");
|
|
529
|
+
if (alarmDisposition === "cleared") {
|
|
530
|
+
yield* Ref.set(stalls, 0);
|
|
531
|
+
}
|
|
532
|
+
return yield* annotate(
|
|
533
|
+
MaintenancePassReport.make({
|
|
534
|
+
phase: "actionable",
|
|
535
|
+
recovered: recovered.length,
|
|
536
|
+
settled: settlements.length,
|
|
537
|
+
nonterminal: remaining.length,
|
|
538
|
+
alarm: alarmDisposition,
|
|
539
|
+
}),
|
|
540
|
+
);
|
|
322
541
|
});
|
|
323
542
|
|
|
324
543
|
return ConversationMaintenance.of({
|
|
325
|
-
//
|
|
326
|
-
|
|
327
|
-
// the running pass alive and flushes the hint as the pass's final re-arm.
|
|
328
|
-
pass: alarm.withWakesDeferred(pass()),
|
|
544
|
+
// A mid-pass immediate hint is droppable; durable dirty state decides the final alarm.
|
|
545
|
+
pass: alarm.withWakesDeferred(maintenancePassGate.withPermit(pass())),
|
|
329
546
|
ensureAlarm: ensureAlarm(),
|
|
330
|
-
|
|
547
|
+
withMutation,
|
|
331
548
|
});
|
|
332
549
|
}),
|
|
333
550
|
);
|