@effect-agent/platform-cloudflare 0.1.0-beta.38 → 0.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bindings-BrVEOkfe.d.mts +102 -0
- package/dist/browser-quick-action.d.mts +18 -1
- package/dist/browser-quick-action.mjs +14 -1
- package/dist/browser-quick-action.mjs.map +1 -1
- package/dist/browser-rest-capture.d.mts +17 -2
- package/dist/browser-rest-capture.mjs +12 -1
- package/dist/browser-rest-capture.mjs.map +1 -1
- package/dist/browser-rest-crawl.mjs +2 -2
- package/dist/browser-rest-crawl.mjs.map +1 -1
- package/dist/index.d.mts +245 -237
- package/dist/index.mjs +168 -164
- package/dist/index.mjs.map +1 -1
- package/dist/interactive-browser.d.mts +160 -2
- package/dist/interactive-browser.mjs +1166 -2
- package/dist/interactive-browser.mjs.map +1 -0
- package/dist/{scheduling-B-OFqoS9.mjs → prepared-admission-BKp_Upw2.mjs} +134 -433
- package/dist/prepared-admission-BKp_Upw2.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/dist/scheduling.d.mts +47 -2
- package/dist/scheduling.mjs +339 -1
- package/dist/scheduling.mjs.map +1 -0
- package/dist/subscriptions.d.mts +52 -0
- package/dist/subscriptions.mjs +346 -0
- package/dist/subscriptions.mjs.map +1 -0
- package/package.json +20 -13
- package/src/alarm.ts +264 -265
- package/src/bindings.ts +34 -30
- package/src/browser-quick-action.ts +31 -0
- package/src/browser-rest-capture.ts +27 -0
- package/src/client.ts +111 -105
- package/src/code-mode-executor.ts +19 -0
- package/src/config.ts +7 -7
- package/src/index.ts +10 -11
- package/src/interactive-browser.ts +29 -1
- package/src/layers.ts +178 -257
- package/src/prepared-admission.ts +90 -0
- package/src/scheduling.ts +19 -55
- package/src/subscriptions.ts +661 -0
- package/src/{conversation-object.ts → thread-object.ts} +167 -136
- package/src/transport.ts +15 -15
- package/src/wake-scheduler.ts +17 -22
- package/dist/interactive-browser-BRo-Bfvb.d.mts +0 -144
- package/dist/interactive-browser-Cy5q-ldl.mjs +0 -1154
- package/dist/interactive-browser-Cy5q-ldl.mjs.map +0 -1
- package/dist/scheduling-B-OFqoS9.mjs.map +0 -1
- package/dist/scheduling-BJs_kHTx.d.mts +0 -142
package/src/alarm.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
2
|
+
type ResolvedBinding,
|
|
3
3
|
DurableAgentRuntime,
|
|
4
4
|
SubmissionLedger,
|
|
5
5
|
type DurableBindingFailure,
|
|
6
6
|
type DurableWorkerFailure,
|
|
7
7
|
type RecoveryReport,
|
|
8
8
|
type SubmissionSnapshot,
|
|
9
|
-
} from "@effect-agent/
|
|
9
|
+
} from "@effect-agent/thread";
|
|
10
10
|
import {
|
|
11
11
|
Clock,
|
|
12
12
|
Context,
|
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
Stream,
|
|
21
21
|
} from "effect";
|
|
22
22
|
|
|
23
|
-
import {
|
|
23
|
+
import { ThreadObjectIdentity, DurableObjectContext } from "./bindings.ts";
|
|
24
24
|
import { safeCauseMessage } from "./boundary.ts";
|
|
25
25
|
import { CloudflareDurableRuntimeConfig } from "./config.ts";
|
|
26
26
|
|
|
@@ -163,7 +163,7 @@ export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
|
|
|
163
163
|
}) {}
|
|
164
164
|
|
|
165
165
|
/** Fault boundaries around every maintenance-owned durable mutation. */
|
|
166
|
-
export type
|
|
166
|
+
export type ThreadMaintenanceFailpointLocation =
|
|
167
167
|
| "maintenance:dirty:before"
|
|
168
168
|
| "maintenance:dirty:after"
|
|
169
169
|
| "maintenance:mutation:armed"
|
|
@@ -175,17 +175,17 @@ export type ConversationMaintenanceFailpointLocation =
|
|
|
175
175
|
| "maintenance:finish:before"
|
|
176
176
|
| "maintenance:finish:after";
|
|
177
177
|
|
|
178
|
-
export type
|
|
179
|
-
location:
|
|
178
|
+
export type ThreadMaintenanceFailpointHandler = (
|
|
179
|
+
location: ThreadMaintenanceFailpointLocation,
|
|
180
180
|
) => Effect.Effect<void>;
|
|
181
181
|
|
|
182
182
|
/** Test-only fault authority; production uses the inert layer. */
|
|
183
|
-
export class
|
|
184
|
-
|
|
183
|
+
export class ThreadMaintenanceFailpoint extends Context.Service<
|
|
184
|
+
ThreadMaintenanceFailpoint,
|
|
185
185
|
{
|
|
186
|
-
readonly hit:
|
|
186
|
+
readonly hit: ThreadMaintenanceFailpointHandler;
|
|
187
187
|
}
|
|
188
|
-
>()("@effect-agent/platform-cloudflare/
|
|
188
|
+
>()("@effect-agent/platform-cloudflare/ThreadMaintenanceFailpoint") {
|
|
189
189
|
static readonly layer = Layer.succeed(this)({ hit: () => Effect.void });
|
|
190
190
|
}
|
|
191
191
|
|
|
@@ -194,8 +194,8 @@ const MaintenanceGeneration = Schema.BigIntFromString.check(
|
|
|
194
194
|
);
|
|
195
195
|
|
|
196
196
|
/** Versioned, platform-private maintenance state stored through Durable Object KV. */
|
|
197
|
-
class
|
|
198
|
-
"@effect-agent/platform-cloudflare/
|
|
197
|
+
class ThreadMaintenanceState extends Schema.Class<ThreadMaintenanceState>(
|
|
198
|
+
"@effect-agent/platform-cloudflare/ThreadMaintenanceState",
|
|
199
199
|
)({
|
|
200
200
|
schemaVersion: Schema.Literal(1),
|
|
201
201
|
dirty: MaintenanceGeneration,
|
|
@@ -203,12 +203,12 @@ class ConversationMaintenanceState extends Schema.Class<ConversationMaintenanceS
|
|
|
203
203
|
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
204
204
|
}) {}
|
|
205
205
|
|
|
206
|
-
const MAINTENANCE_STATE_KEY = "effect-agent:
|
|
207
|
-
const decodeMaintenanceState = Schema.decodeUnknownSync(
|
|
208
|
-
const encodeMaintenanceState = Schema.encodeSync(
|
|
206
|
+
const MAINTENANCE_STATE_KEY = "effect-agent:thread-maintenance:v1";
|
|
207
|
+
const decodeMaintenanceState = Schema.decodeUnknownSync(ThreadMaintenanceState);
|
|
208
|
+
const encodeMaintenanceState = Schema.encodeSync(ThreadMaintenanceState);
|
|
209
209
|
|
|
210
|
-
const initialMaintenanceState = ():
|
|
211
|
-
|
|
210
|
+
const initialMaintenanceState = (): ThreadMaintenanceState =>
|
|
211
|
+
ThreadMaintenanceState.make({
|
|
212
212
|
schemaVersion: 1,
|
|
213
213
|
// Bootstrap Objects created by the pre-generation release without scanning the ledger in
|
|
214
214
|
// the constructor. One useful pass classifies and acknowledges any existing obligation.
|
|
@@ -219,7 +219,7 @@ const initialMaintenanceState = (): ConversationMaintenanceState =>
|
|
|
219
219
|
|
|
220
220
|
const readMaintenanceState = async (
|
|
221
221
|
transaction: DurableObjectTransaction,
|
|
222
|
-
): Promise<{ readonly state:
|
|
222
|
+
): Promise<{ readonly state: ThreadMaintenanceState; readonly initialized: boolean }> => {
|
|
223
223
|
const encoded = await transaction.get(MAINTENANCE_STATE_KEY);
|
|
224
224
|
return encoded === undefined
|
|
225
225
|
? { state: initialMaintenanceState(), initialized: false }
|
|
@@ -279,8 +279,8 @@ export type MaintenancePassFailure =
|
|
|
279
279
|
* 4. Stable external waits acknowledge and clear. Autonomous retry, indeterminate, and lease
|
|
280
280
|
* recovery states leave their generation dirty and retain bounded backoff rearming.
|
|
281
281
|
*/
|
|
282
|
-
export class
|
|
283
|
-
|
|
282
|
+
export class ThreadMaintenance extends Context.Service<
|
|
283
|
+
ThreadMaintenance,
|
|
284
284
|
{
|
|
285
285
|
/** One idempotent maintenance pass; failures propagate so workerd retries the alarm. */
|
|
286
286
|
readonly pass: Effect.Effect<MaintenancePassReport, MaintenancePassFailure>;
|
|
@@ -298,272 +298,271 @@ export class ConversationMaintenance extends Context.Service<
|
|
|
298
298
|
body: Effect.Effect<A, E, R>,
|
|
299
299
|
) => Effect.Effect<A, E | DurableAlarmError, R>;
|
|
300
300
|
}
|
|
301
|
-
>()("@effect-agent/platform-cloudflare/
|
|
302
|
-
static readonly layer
|
|
303
|
-
|
|
301
|
+
>()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
|
|
302
|
+
static readonly layer = (
|
|
303
|
+
bindings: ReadonlyArray<ResolvedBinding>,
|
|
304
|
+
): Layer.Layer<
|
|
305
|
+
ThreadMaintenance,
|
|
304
306
|
never,
|
|
305
307
|
| DurableAgentRuntime
|
|
306
|
-
| AgentBindingResolver
|
|
307
308
|
| SubmissionLedger
|
|
308
309
|
| DurableAlarmService
|
|
309
|
-
|
|
|
310
|
+
| ThreadMaintenanceFailpoint
|
|
310
311
|
| CloudflareDurableRuntimeConfig
|
|
311
|
-
|
|
|
312
|
+
| ThreadObjectIdentity
|
|
312
313
|
| DurableObjectContext
|
|
313
|
-
>
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
314
|
+
> =>
|
|
315
|
+
Layer.effect(ThreadMaintenance)(
|
|
316
|
+
Effect.gen(function* () {
|
|
317
|
+
const runtime = yield* DurableAgentRuntime;
|
|
318
|
+
const ledger = yield* SubmissionLedger;
|
|
319
|
+
const alarm = yield* DurableAlarmService;
|
|
320
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
321
|
+
const identity = yield* ThreadObjectIdentity;
|
|
322
|
+
const { ctx } = yield* DurableObjectContext;
|
|
323
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Consecutive no-progress passes — an in-memory CACHE, not state: a fresh incarnation
|
|
327
|
+
* restarts at zero and merely re-arms sooner than a long-lived one would have.
|
|
328
|
+
*/
|
|
329
|
+
const stalls = yield* Ref.make(0);
|
|
330
|
+
/**
|
|
331
|
+
* Incarnation-local mutation count guarded with the generation transactions below. It is
|
|
332
|
+
* deliberately not durable: after eviction every begun mutation has stopped, while its
|
|
333
|
+
* pre-armed dirty generation remains durable for recovery. The short gate never spans the
|
|
334
|
+
* caller's mutation or cross-Object I/O.
|
|
335
|
+
*/
|
|
336
|
+
const activeMutations = yield* Ref.make(0);
|
|
337
|
+
const generationGate = yield* Semaphore.make(1);
|
|
338
|
+
// At-least-once deliveries are idempotent, but overlapping pass bodies could otherwise
|
|
339
|
+
// acknowledge state while a sibling pass is still mutating it. Port/RPC mutations do not
|
|
340
|
+
// take this permit, so cross-Object I/O cannot deadlock the maintenance serialization.
|
|
341
|
+
const maintenancePassGate = yield* Semaphore.make(1);
|
|
342
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
323
343
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
* pre-armed dirty generation remains durable for recovery. The short gate never spans the
|
|
333
|
-
* caller's mutation or cross-Object I/O.
|
|
334
|
-
*/
|
|
335
|
-
const activeMutations = yield* Ref.make(0);
|
|
336
|
-
const generationGate = yield* Semaphore.make(1);
|
|
337
|
-
// At-least-once deliveries are idempotent, but overlapping pass bodies could otherwise
|
|
338
|
-
// acknowledge state while a sibling pass is still mutating it. Port/RPC mutations do not
|
|
339
|
-
// take this permit, so cross-Object I/O cannot deadlock the maintenance serialization.
|
|
340
|
-
const maintenancePassGate = yield* Semaphore.make(1);
|
|
341
|
-
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
344
|
+
const runTransaction = <A>(
|
|
345
|
+
operation: string,
|
|
346
|
+
transaction: () => Promise<A>,
|
|
347
|
+
): Effect.Effect<A, DurableAlarmError> =>
|
|
348
|
+
Effect.tryPromise({
|
|
349
|
+
try: transaction,
|
|
350
|
+
catch: alarmFailure(operation),
|
|
351
|
+
});
|
|
342
352
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
353
|
+
const beginMutation = Effect.fn("ThreadMaintenance.beginMutation")(function* () {
|
|
354
|
+
yield* failpoint.hit("maintenance:dirty:before");
|
|
355
|
+
const now = yield* Clock.currentTimeMillis;
|
|
356
|
+
yield* runTransaction("advance maintenance generation", () =>
|
|
357
|
+
ctx.storage.transaction(async (transaction) => {
|
|
358
|
+
const { state } = await readMaintenanceState(transaction);
|
|
359
|
+
const next = ThreadMaintenanceState.make({
|
|
360
|
+
...state,
|
|
361
|
+
dirty: state.dirty + 1n,
|
|
362
|
+
});
|
|
363
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
364
|
+
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
365
|
+
// on its best-effort immediate wake hint.
|
|
366
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
367
|
+
}),
|
|
368
|
+
);
|
|
369
|
+
yield* failpoint.hit("maintenance:dirty:after");
|
|
370
|
+
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
350
371
|
});
|
|
351
372
|
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
const now = yield* Clock.currentTimeMillis;
|
|
355
|
-
yield* runTransaction("advance maintenance generation", () =>
|
|
356
|
-
ctx.storage.transaction(async (transaction) => {
|
|
357
|
-
const { state } = await readMaintenanceState(transaction);
|
|
358
|
-
const next = ConversationMaintenanceState.make({
|
|
359
|
-
...state,
|
|
360
|
-
dirty: state.dirty + 1n,
|
|
361
|
-
});
|
|
362
|
-
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
363
|
-
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
364
|
-
// on its best-effort immediate wake hint.
|
|
365
|
-
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
366
|
-
}),
|
|
373
|
+
const endMutation = generationGate.withPermit(
|
|
374
|
+
Ref.update(activeMutations, (active) => Math.max(0, active - 1)),
|
|
367
375
|
);
|
|
368
|
-
yield* failpoint.hit("maintenance:dirty:after");
|
|
369
|
-
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
370
|
-
});
|
|
371
|
-
|
|
372
|
-
const endMutation = generationGate.withPermit(
|
|
373
|
-
Ref.update(activeMutations, (active) => Math.max(0, active - 1)),
|
|
374
|
-
);
|
|
375
376
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
377
|
+
const withMutation = <A, E, R>(
|
|
378
|
+
body: Effect.Effect<A, E, R>,
|
|
379
|
+
): Effect.Effect<A, E | DurableAlarmError, R> =>
|
|
380
|
+
Effect.acquireUseRelease(
|
|
381
|
+
generationGate.withPermit(beginMutation()),
|
|
382
|
+
() =>
|
|
383
|
+
failpoint.hit("maintenance:mutation:armed").pipe(
|
|
384
|
+
Effect.andThen(body),
|
|
385
|
+
Effect.tap(() => failpoint.hit("maintenance:mutation:finished")),
|
|
386
|
+
),
|
|
387
|
+
() => endMutation,
|
|
388
|
+
);
|
|
388
389
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
390
|
+
const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
|
|
391
|
+
yield* failpoint.hit("maintenance:ensure:before");
|
|
392
|
+
const now = yield* Clock.currentTimeMillis;
|
|
393
|
+
yield* runTransaction("ensure maintenance alarm", () =>
|
|
394
|
+
ctx.storage.transaction(async (transaction) => {
|
|
395
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
396
|
+
if (!initialized) {
|
|
397
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
398
|
+
}
|
|
399
|
+
if (state.dirty > state.processed) {
|
|
400
|
+
await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
|
|
401
|
+
}
|
|
402
|
+
}),
|
|
403
|
+
);
|
|
404
|
+
yield* failpoint.hit("maintenance:ensure:after");
|
|
405
|
+
});
|
|
405
406
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
407
|
+
const beginPass = Effect.fn("ThreadMaintenance.beginPass")(function* () {
|
|
408
|
+
yield* failpoint.hit("maintenance:begin:before");
|
|
409
|
+
const now = yield* Clock.currentTimeMillis;
|
|
410
|
+
const result = yield* runTransaction("begin maintenance pass", () =>
|
|
411
|
+
ctx.storage.transaction(async (transaction) => {
|
|
412
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
413
|
+
if (!initialized) {
|
|
414
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
415
|
+
}
|
|
416
|
+
if (state.processed >= state.dirty) {
|
|
417
|
+
await transaction.deleteAlarm();
|
|
418
|
+
return { _tag: "CaughtUp" as const, nonterminal: state.nonterminal };
|
|
419
|
+
}
|
|
420
|
+
// Pre-arm the earliest retry before recovery. A successful finish may move this slot
|
|
421
|
+
// LATER to its bounded backoff, which does not cancel the running handler.
|
|
422
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
423
|
+
return { _tag: "Actionable" as const, generation: state.dirty };
|
|
424
|
+
}),
|
|
425
|
+
);
|
|
426
|
+
yield* failpoint.hit("maintenance:begin:after");
|
|
427
|
+
return result;
|
|
428
|
+
});
|
|
428
429
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
430
|
+
const rearmDelay = Effect.fn("ThreadMaintenance.rearmDelay")(function* (
|
|
431
|
+
progressed: boolean,
|
|
432
|
+
) {
|
|
433
|
+
const priorStalls = yield* Ref.getAndUpdate(stalls, (count) =>
|
|
434
|
+
progressed ? 0 : count + 1,
|
|
435
|
+
);
|
|
436
|
+
if (progressed) return config.alarmBackoffBase;
|
|
437
|
+
const exponent = Math.min(priorStalls, 30);
|
|
438
|
+
const backoff = Math.min(config.alarmBackoffCap, config.alarmBackoffBase * 2 ** exponent);
|
|
439
|
+
const jitter = yield* Random.next;
|
|
440
|
+
// Full jitter over [backoff/2, backoff]: desynchronizes retry storms without ever
|
|
441
|
+
// waiting longer than the deterministic bound.
|
|
442
|
+
const jittered = Math.ceil(backoff / 2 + (backoff / 2) * jitter);
|
|
443
|
+
return Math.min(jittered, config.wakeScanInterval);
|
|
444
|
+
});
|
|
444
445
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
446
|
+
const pass = Effect.fn("ThreadMaintenance.pass")(function* (): Effect.fn.Return<
|
|
447
|
+
MaintenancePassReport,
|
|
448
|
+
MaintenancePassFailure
|
|
449
|
+
> {
|
|
450
|
+
const annotate = (report: MaintenancePassReport) =>
|
|
451
|
+
Effect.annotateCurrentSpan({
|
|
452
|
+
phase: report.phase,
|
|
453
|
+
recovered: report.recovered,
|
|
454
|
+
settled: report.settled,
|
|
455
|
+
nonterminal: report.nonterminal,
|
|
456
|
+
alarm: report.alarm,
|
|
457
|
+
}).pipe(Effect.as(report));
|
|
457
458
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
459
|
+
const started = yield* generationGate.withPermit(
|
|
460
|
+
Effect.gen(function* () {
|
|
461
|
+
const activeAtStart = yield* Ref.get(activeMutations);
|
|
462
|
+
const generation = yield* beginPass();
|
|
463
|
+
return { ...generation, activeAtStart };
|
|
464
|
+
}),
|
|
465
|
+
);
|
|
466
|
+
if (started._tag === "CaughtUp") {
|
|
467
|
+
return yield* annotate(
|
|
468
|
+
MaintenancePassReport.make({
|
|
469
|
+
phase: "caught-up",
|
|
470
|
+
recovered: 0,
|
|
471
|
+
settled: 0,
|
|
472
|
+
nonterminal: started.nonterminal,
|
|
473
|
+
alarm: "cleared",
|
|
474
|
+
}),
|
|
475
|
+
);
|
|
476
|
+
}
|
|
477
|
+
// Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
|
|
478
|
+
const recovered: ReadonlyArray<RecoveryReport> = yield* runtime.runRecovery;
|
|
479
|
+
// Step 3 — one bounded drain pass over this Object's own lane.
|
|
480
|
+
const settlements = yield* runtime.processThreadResolved(identity.threadId, bindings);
|
|
481
|
+
// Observe residual state before acknowledging this exact pass-start generation.
|
|
482
|
+
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
483
|
+
const reports = new Map(recovered.map((report) => [report.submissionId, report]));
|
|
484
|
+
const head = remaining[0];
|
|
485
|
+
const headWaiting = head !== undefined && stableExternalWait(head, reports);
|
|
486
|
+
const autonomous = remaining.some((snapshot, index) => {
|
|
487
|
+
// FIFO followers cannot execute through a stable external wait. Only plain queued
|
|
488
|
+
// input is dormant here; admission repairs and accepted aborts still need a pass.
|
|
489
|
+
if (
|
|
490
|
+
index > 0 &&
|
|
491
|
+
headWaiting &&
|
|
492
|
+
snapshot.state === "ready" &&
|
|
493
|
+
reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput"
|
|
494
|
+
)
|
|
495
|
+
return false;
|
|
496
|
+
return !stableExternalWait(snapshot, reports);
|
|
497
|
+
});
|
|
498
|
+
const progressed =
|
|
499
|
+
settlements.length > 0 || recovered.some((report) => report.disposition === "repaired");
|
|
500
|
+
const delay = autonomous ? yield* rearmDelay(progressed) : 0;
|
|
501
|
+
const now = yield* Clock.currentTimeMillis;
|
|
502
|
+
yield* failpoint.hit("maintenance:finish:before");
|
|
503
|
+
const alarmDisposition = yield* generationGate.withPermit(
|
|
504
|
+
Effect.gen(function* () {
|
|
505
|
+
const active = yield* Ref.get(activeMutations);
|
|
506
|
+
return yield* runTransaction("finish maintenance pass", () =>
|
|
507
|
+
ctx.storage.transaction(async (transaction) => {
|
|
508
|
+
const { state } = await readMaintenanceState(transaction);
|
|
509
|
+
// Autonomous work and in-flight mutations intentionally leave the observed
|
|
510
|
+
// generation dirty. Otherwise acknowledge only the pass-start generation.
|
|
511
|
+
const processed =
|
|
512
|
+
autonomous || started.activeAtStart > 0 || active > 0
|
|
513
|
+
? state.processed
|
|
514
|
+
: state.processed > started.generation
|
|
515
|
+
? state.processed
|
|
516
|
+
: started.generation;
|
|
517
|
+
const next = ThreadMaintenanceState.make({
|
|
518
|
+
...state,
|
|
519
|
+
processed,
|
|
520
|
+
nonterminal: remaining.length,
|
|
521
|
+
});
|
|
522
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
523
|
+
if (autonomous) {
|
|
524
|
+
// Replace the crash-fallback slot with this pass's bounded backoff. The target
|
|
525
|
+
// is never earlier than the begin-pass fallback, so workerd does not cancel
|
|
526
|
+
// this running alarm handler before its report/span can complete.
|
|
527
|
+
await transaction.setAlarm(now + delay);
|
|
528
|
+
return "rearmed" as const;
|
|
529
|
+
}
|
|
530
|
+
if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
|
|
531
|
+
// A mutation overlapped this pass's observation window or raced
|
|
532
|
+
// acknowledgement. It stays dirty and its pre-armed bounded alarm survives;
|
|
533
|
+
// unseen effects are never acknowledged. Do not accelerate that future alarm
|
|
534
|
+
// from inside the current handler: workerd cancels a running handler when it
|
|
535
|
+
// writes an earlier slot.
|
|
536
|
+
await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
|
|
537
|
+
return "rearmed" as const;
|
|
538
|
+
}
|
|
539
|
+
await transaction.deleteAlarm();
|
|
540
|
+
return "cleared" as const;
|
|
541
|
+
}),
|
|
542
|
+
);
|
|
543
|
+
}),
|
|
544
|
+
);
|
|
545
|
+
yield* failpoint.hit("maintenance:finish:after");
|
|
546
|
+
if (alarmDisposition === "cleared") {
|
|
547
|
+
yield* Ref.set(stalls, 0);
|
|
548
|
+
}
|
|
466
549
|
return yield* annotate(
|
|
467
550
|
MaintenancePassReport.make({
|
|
468
|
-
phase: "
|
|
469
|
-
recovered:
|
|
470
|
-
settled:
|
|
471
|
-
nonterminal:
|
|
472
|
-
alarm:
|
|
551
|
+
phase: "actionable",
|
|
552
|
+
recovered: recovered.length,
|
|
553
|
+
settled: settlements.length,
|
|
554
|
+
nonterminal: remaining.length,
|
|
555
|
+
alarm: alarmDisposition,
|
|
473
556
|
}),
|
|
474
557
|
);
|
|
475
|
-
}
|
|
476
|
-
// Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
|
|
477
|
-
const recovered: ReadonlyArray<RecoveryReport> = yield* runtime.runRecovery;
|
|
478
|
-
// Step 3 — one bounded drain pass over this Object's own lane.
|
|
479
|
-
const settlements = yield* runtime
|
|
480
|
-
.processConversationResolved(identity.conversationId)
|
|
481
|
-
.pipe(Effect.provideService(AgentBindingResolver, resolver));
|
|
482
|
-
// Observe residual state before acknowledging this exact pass-start generation.
|
|
483
|
-
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
484
|
-
const reports = new Map(recovered.map((report) => [report.submissionId, report]));
|
|
485
|
-
const head = remaining[0];
|
|
486
|
-
const headWaiting = head !== undefined && stableExternalWait(head, reports);
|
|
487
|
-
const autonomous = remaining.some((snapshot, index) => {
|
|
488
|
-
// FIFO followers cannot execute through a stable external wait. Only plain queued
|
|
489
|
-
// input is dormant here; admission repairs and accepted aborts still need a pass.
|
|
490
|
-
if (
|
|
491
|
-
index > 0 &&
|
|
492
|
-
headWaiting &&
|
|
493
|
-
snapshot.state === "ready" &&
|
|
494
|
-
reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput"
|
|
495
|
-
)
|
|
496
|
-
return false;
|
|
497
|
-
return !stableExternalWait(snapshot, reports);
|
|
498
558
|
});
|
|
499
|
-
const progressed =
|
|
500
|
-
settlements.length > 0 || recovered.some((report) => report.disposition === "repaired");
|
|
501
|
-
const delay = autonomous ? yield* rearmDelay(progressed) : 0;
|
|
502
|
-
const now = yield* Clock.currentTimeMillis;
|
|
503
|
-
yield* failpoint.hit("maintenance:finish:before");
|
|
504
|
-
const alarmDisposition = yield* generationGate.withPermit(
|
|
505
|
-
Effect.gen(function* () {
|
|
506
|
-
const active = yield* Ref.get(activeMutations);
|
|
507
|
-
return yield* runTransaction("finish maintenance pass", () =>
|
|
508
|
-
ctx.storage.transaction(async (transaction) => {
|
|
509
|
-
const { state } = await readMaintenanceState(transaction);
|
|
510
|
-
// Autonomous work and in-flight mutations intentionally leave the observed
|
|
511
|
-
// generation dirty. Otherwise acknowledge only the pass-start generation.
|
|
512
|
-
const processed =
|
|
513
|
-
autonomous || started.activeAtStart > 0 || active > 0
|
|
514
|
-
? state.processed
|
|
515
|
-
: state.processed > started.generation
|
|
516
|
-
? state.processed
|
|
517
|
-
: started.generation;
|
|
518
|
-
const next = ConversationMaintenanceState.make({
|
|
519
|
-
...state,
|
|
520
|
-
processed,
|
|
521
|
-
nonterminal: remaining.length,
|
|
522
|
-
});
|
|
523
|
-
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
524
|
-
if (autonomous) {
|
|
525
|
-
// Replace the crash-fallback slot with this pass's bounded backoff. The target
|
|
526
|
-
// is never earlier than the begin-pass fallback, so workerd does not cancel
|
|
527
|
-
// this running alarm handler before its report/span can complete.
|
|
528
|
-
await transaction.setAlarm(now + delay);
|
|
529
|
-
return "rearmed" as const;
|
|
530
|
-
}
|
|
531
|
-
if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
|
|
532
|
-
// A mutation overlapped this pass's observation window or raced
|
|
533
|
-
// acknowledgement. It stays dirty and its pre-armed bounded alarm survives;
|
|
534
|
-
// unseen effects are never acknowledged. Do not accelerate that future alarm
|
|
535
|
-
// from inside the current handler: workerd cancels a running handler when it
|
|
536
|
-
// writes an earlier slot.
|
|
537
|
-
await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
|
|
538
|
-
return "rearmed" as const;
|
|
539
|
-
}
|
|
540
|
-
await transaction.deleteAlarm();
|
|
541
|
-
return "cleared" as const;
|
|
542
|
-
}),
|
|
543
|
-
);
|
|
544
|
-
}),
|
|
545
|
-
);
|
|
546
|
-
yield* failpoint.hit("maintenance:finish:after");
|
|
547
|
-
if (alarmDisposition === "cleared") {
|
|
548
|
-
yield* Ref.set(stalls, 0);
|
|
549
|
-
}
|
|
550
|
-
return yield* annotate(
|
|
551
|
-
MaintenancePassReport.make({
|
|
552
|
-
phase: "actionable",
|
|
553
|
-
recovered: recovered.length,
|
|
554
|
-
settled: settlements.length,
|
|
555
|
-
nonterminal: remaining.length,
|
|
556
|
-
alarm: alarmDisposition,
|
|
557
|
-
}),
|
|
558
|
-
);
|
|
559
|
-
});
|
|
560
559
|
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
560
|
+
return ThreadMaintenance.of({
|
|
561
|
+
// A mid-pass immediate hint is droppable; durable dirty state decides the final alarm.
|
|
562
|
+
pass: alarm.withWakesDeferred(maintenancePassGate.withPermit(pass())),
|
|
563
|
+
ensureAlarm: ensureAlarm(),
|
|
564
|
+
withMutation,
|
|
565
|
+
});
|
|
566
|
+
}),
|
|
567
|
+
);
|
|
569
568
|
}
|