@effect-agent/platform-cloudflare 0.1.0-beta.42 → 0.1.0-beta.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-quick-action.mjs.map +1 -1
- package/dist/browser-rest-capture.mjs.map +1 -1
- package/dist/browser-rest-crawl.mjs.map +1 -1
- package/dist/browser-session-lifecycle-DqntvG-Y.d.mts +21 -0
- package/dist/browser-session-lifecycle-ZGgb3pnK.mjs +84 -0
- package/dist/browser-session-lifecycle-ZGgb3pnK.mjs.map +1 -0
- package/dist/index.d.mts +47 -34
- package/dist/index.mjs +3 -3
- package/dist/index.mjs.map +1 -1
- package/dist/interactive-browser.d.mts +1 -18
- package/dist/interactive-browser.mjs +2 -81
- package/dist/interactive-browser.mjs.map +1 -1
- package/dist/{prepared-admission-BKp_Upw2.mjs → prepared-admission-BhW2eT_a.mjs} +3 -3
- package/dist/prepared-admission-BhW2eT_a.mjs.map +1 -0
- package/dist/protected-browser.d.mts +62 -0
- package/dist/protected-browser.mjs +714 -0
- package/dist/protected-browser.mjs.map +1 -0
- package/dist/scheduling.mjs +1 -1
- package/dist/scheduling.mjs.map +1 -1
- package/dist/subscriptions.mjs +1 -1
- package/dist/subscriptions.mjs.map +1 -1
- package/package.json +1 -81
- package/src/alarm.ts +277 -242
- package/src/bindings.ts +5 -0
- package/src/boundary.ts +4 -0
- package/src/browser-quick-action.ts +52 -0
- package/src/browser-rest-capture.ts +30 -0
- package/src/browser-rest-crawl.ts +43 -0
- package/src/browser-session-lifecycle.ts +18 -0
- package/src/client.ts +40 -0
- package/src/code-mode-executor.ts +50 -0
- package/src/interactive-browser.ts +176 -0
- package/src/layers.ts +13 -3
- package/src/memory.ts +42 -3
- package/src/prepared-admission.ts +1 -0
- package/src/progress-wait.ts +14 -0
- package/src/protected-browser/binding.ts +185 -0
- package/src/protected-browser/inspect-frame.ts +82 -0
- package/src/protected-browser/native.ts +384 -0
- package/src/protected-browser/policy.ts +594 -0
- package/src/protected-browser.ts +2 -0
- package/src/scheduling.ts +34 -0
- package/src/subscriptions.ts +50 -0
- package/src/thread-object.ts +55 -1
- package/src/transport.ts +1 -0
- package/src/wake-scheduler.ts +1 -0
- package/dist/prepared-admission-BKp_Upw2.mjs.map +0 -1
package/src/alarm.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import {
|
|
2
|
-
type ResolvedBinding,
|
|
3
2
|
DurableAgentRuntime,
|
|
4
3
|
SubmissionLedger,
|
|
5
4
|
type DurableBindingFailure,
|
|
@@ -98,6 +97,7 @@ export class DurableAlarmService extends Context.Service<
|
|
|
98
97
|
* on top of the already-committed pre-armed alarm.
|
|
99
98
|
*/
|
|
100
99
|
const runningPasses = yield* Ref.make(0);
|
|
100
|
+
|
|
101
101
|
const scheduled = Effect.tryPromise({
|
|
102
102
|
try: () => ctx.storage.getAlarm(),
|
|
103
103
|
catch: alarmFailure("get alarm"),
|
|
@@ -106,11 +106,13 @@ export class DurableAlarmService extends Context.Service<
|
|
|
106
106
|
deadline === null ? Option.none<number>() : Option.some(deadline),
|
|
107
107
|
),
|
|
108
108
|
);
|
|
109
|
+
|
|
109
110
|
const scheduleAt = (epochMillis: number) =>
|
|
110
111
|
Effect.tryPromise({
|
|
111
112
|
try: () => ctx.storage.setAlarm(epochMillis),
|
|
112
113
|
catch: alarmFailure("set alarm"),
|
|
113
114
|
});
|
|
115
|
+
|
|
114
116
|
const ensureScheduledBy = (epochMillis: number) =>
|
|
115
117
|
scheduled.pipe(
|
|
116
118
|
Effect.flatMap((existing) =>
|
|
@@ -119,21 +121,26 @@ export class DurableAlarmService extends Context.Service<
|
|
|
119
121
|
: scheduleAt(epochMillis),
|
|
120
122
|
),
|
|
121
123
|
);
|
|
124
|
+
|
|
122
125
|
const armNow = Clock.currentTimeMillis.pipe(
|
|
123
126
|
Effect.flatMap((now) => ensureScheduledBy(now)),
|
|
124
127
|
);
|
|
128
|
+
|
|
125
129
|
const scheduleNow = Ref.get(runningPasses).pipe(
|
|
126
130
|
Effect.flatMap((passes) => (passes > 0 ? Effect.void : armNow)),
|
|
127
131
|
);
|
|
132
|
+
|
|
128
133
|
const withWakesDeferred = <A, E, R>(body: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
|
|
129
134
|
Ref.update(runningPasses, (passes) => passes + 1).pipe(
|
|
130
135
|
Effect.andThen(body),
|
|
131
136
|
Effect.ensuring(Ref.update(runningPasses, (passes) => passes - 1)),
|
|
132
137
|
);
|
|
138
|
+
|
|
133
139
|
const cancel = Effect.tryPromise({
|
|
134
140
|
try: () => ctx.storage.deleteAlarm(),
|
|
135
141
|
catch: alarmFailure("delete alarm"),
|
|
136
142
|
});
|
|
143
|
+
|
|
137
144
|
return DurableAlarmService.of({
|
|
138
145
|
scheduled,
|
|
139
146
|
scheduleAt,
|
|
@@ -221,6 +228,7 @@ const readMaintenanceState = async (
|
|
|
221
228
|
transaction: DurableObjectTransaction,
|
|
222
229
|
): Promise<{ readonly state: ThreadMaintenanceState; readonly initialized: boolean }> => {
|
|
223
230
|
const encoded = await transaction.get(MAINTENANCE_STATE_KEY);
|
|
231
|
+
|
|
224
232
|
return encoded === undefined
|
|
225
233
|
? { state: initialMaintenanceState(), initialized: false }
|
|
226
234
|
: { state: decodeMaintenanceState(encoded), initialized: true };
|
|
@@ -231,6 +239,7 @@ const ensureTransactionAlarmBy = async (
|
|
|
231
239
|
deadline: number,
|
|
232
240
|
): Promise<void> => {
|
|
233
241
|
const scheduled = await transaction.getAlarm();
|
|
242
|
+
|
|
234
243
|
if (scheduled === null || scheduled > deadline) {
|
|
235
244
|
await transaction.setAlarm(deadline);
|
|
236
245
|
}
|
|
@@ -241,6 +250,7 @@ const stableExternalWait = (
|
|
|
241
250
|
reports: ReadonlyMap<string, RecoveryReport>,
|
|
242
251
|
): boolean => {
|
|
243
252
|
const decision = reports.get(snapshot.submissionId)?.decision._tag;
|
|
253
|
+
|
|
244
254
|
// An accepted abort still owes cleanup/settlement even if its claim was deferred this pass.
|
|
245
255
|
if (decision === "SettleAborted") return false;
|
|
246
256
|
switch (snapshot.state) {
|
|
@@ -299,9 +309,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
299
309
|
) => Effect.Effect<A, E | DurableAlarmError, R>;
|
|
300
310
|
}
|
|
301
311
|
>()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
|
|
302
|
-
static readonly layer
|
|
303
|
-
bindings: ReadonlyArray<ResolvedBinding>,
|
|
304
|
-
): Layer.Layer<
|
|
312
|
+
static readonly layer: Layer.Layer<
|
|
305
313
|
ThreadMaintenance,
|
|
306
314
|
never,
|
|
307
315
|
| DurableAgentRuntime
|
|
@@ -311,258 +319,285 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
311
319
|
| CloudflareDurableRuntimeConfig
|
|
312
320
|
| ThreadObjectIdentity
|
|
313
321
|
| DurableObjectContext
|
|
314
|
-
>
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
322
|
+
> = Layer.effect(ThreadMaintenance)(
|
|
323
|
+
Effect.gen(function* () {
|
|
324
|
+
const runtime = yield* DurableAgentRuntime;
|
|
325
|
+
const ledger = yield* SubmissionLedger;
|
|
326
|
+
const alarm = yield* DurableAlarmService;
|
|
327
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
328
|
+
const identity = yield* ThreadObjectIdentity;
|
|
329
|
+
const { ctx } = yield* DurableObjectContext;
|
|
330
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
324
331
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
const runTransaction = <A>(
|
|
345
|
-
operation: string,
|
|
346
|
-
transaction: () => Promise<A>,
|
|
347
|
-
): Effect.Effect<A, DurableAlarmError> =>
|
|
348
|
-
Effect.tryPromise({
|
|
349
|
-
try: transaction,
|
|
350
|
-
catch: alarmFailure(operation),
|
|
351
|
-
});
|
|
332
|
+
/**
|
|
333
|
+
* Consecutive no-progress passes — an in-memory CACHE, not state: a fresh incarnation
|
|
334
|
+
* restarts at zero and merely re-arms sooner than a long-lived one would have.
|
|
335
|
+
*/
|
|
336
|
+
const stalls = yield* Ref.make(0);
|
|
337
|
+
/**
|
|
338
|
+
* Incarnation-local mutation count guarded with the generation transactions below. It is
|
|
339
|
+
* deliberately not durable: after eviction every begun mutation has stopped, while its
|
|
340
|
+
* pre-armed dirty generation remains durable for recovery. The short gate never spans the
|
|
341
|
+
* caller's mutation or cross-Object I/O.
|
|
342
|
+
*/
|
|
343
|
+
const activeMutations = yield* Ref.make(0);
|
|
344
|
+
const generationGate = yield* Semaphore.make(1);
|
|
345
|
+
// At-least-once deliveries are idempotent, but overlapping pass bodies could otherwise
|
|
346
|
+
// acknowledge state while a sibling pass is still mutating it. Port/RPC mutations do not
|
|
347
|
+
// take this permit, so cross-Object I/O cannot deadlock the maintenance serialization.
|
|
348
|
+
const maintenancePassGate = yield* Semaphore.make(1);
|
|
349
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
352
350
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
...state,
|
|
361
|
-
dirty: state.dirty + 1n,
|
|
362
|
-
});
|
|
363
|
-
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
364
|
-
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
365
|
-
// on its best-effort immediate wake hint.
|
|
366
|
-
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
367
|
-
}),
|
|
368
|
-
);
|
|
369
|
-
yield* failpoint.hit("maintenance:dirty:after");
|
|
370
|
-
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
351
|
+
const runTransaction = <A>(
|
|
352
|
+
operation: string,
|
|
353
|
+
transaction: () => Promise<A>,
|
|
354
|
+
): Effect.Effect<A, DurableAlarmError> =>
|
|
355
|
+
Effect.tryPromise({
|
|
356
|
+
try: transaction,
|
|
357
|
+
catch: alarmFailure(operation),
|
|
371
358
|
});
|
|
372
359
|
|
|
373
|
-
|
|
374
|
-
|
|
360
|
+
const beginMutation = Effect.fn("ThreadMaintenance.beginMutation")(function* () {
|
|
361
|
+
yield* failpoint.hit("maintenance:dirty:before");
|
|
362
|
+
const now = yield* Clock.currentTimeMillis;
|
|
363
|
+
|
|
364
|
+
yield* runTransaction("advance maintenance generation", () =>
|
|
365
|
+
ctx.storage.transaction(async (transaction) => {
|
|
366
|
+
const { state } = await readMaintenanceState(transaction);
|
|
367
|
+
|
|
368
|
+
const next = ThreadMaintenanceState.make({
|
|
369
|
+
...state,
|
|
370
|
+
dirty: state.dirty + 1n,
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
374
|
+
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
375
|
+
// on its best-effort immediate wake hint.
|
|
376
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
377
|
+
}),
|
|
375
378
|
);
|
|
379
|
+
yield* failpoint.hit("maintenance:dirty:after");
|
|
380
|
+
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
381
|
+
});
|
|
376
382
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
Effect.acquireUseRelease(
|
|
381
|
-
generationGate.withPermit(beginMutation()),
|
|
382
|
-
() =>
|
|
383
|
-
failpoint.hit("maintenance:mutation:armed").pipe(
|
|
384
|
-
Effect.andThen(body),
|
|
385
|
-
Effect.tap(() => failpoint.hit("maintenance:mutation:finished")),
|
|
386
|
-
),
|
|
387
|
-
() => endMutation,
|
|
388
|
-
);
|
|
383
|
+
const endMutation = generationGate.withPermit(
|
|
384
|
+
Ref.update(activeMutations, (active) => Math.max(0, active - 1)),
|
|
385
|
+
);
|
|
389
386
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
}),
|
|
403
|
-
);
|
|
404
|
-
yield* failpoint.hit("maintenance:ensure:after");
|
|
405
|
-
});
|
|
387
|
+
const withMutation = <A, E, R>(
|
|
388
|
+
body: Effect.Effect<A, E, R>,
|
|
389
|
+
): Effect.Effect<A, E | DurableAlarmError, R> =>
|
|
390
|
+
Effect.acquireUseRelease(
|
|
391
|
+
generationGate.withPermit(beginMutation()),
|
|
392
|
+
() =>
|
|
393
|
+
failpoint.hit("maintenance:mutation:armed").pipe(
|
|
394
|
+
Effect.andThen(body),
|
|
395
|
+
Effect.tap(() => failpoint.hit("maintenance:mutation:finished")),
|
|
396
|
+
),
|
|
397
|
+
() => endMutation,
|
|
398
|
+
);
|
|
406
399
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
const result = yield* runTransaction("begin maintenance pass", () =>
|
|
411
|
-
ctx.storage.transaction(async (transaction) => {
|
|
412
|
-
const { state, initialized } = await readMaintenanceState(transaction);
|
|
413
|
-
if (!initialized) {
|
|
414
|
-
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
415
|
-
}
|
|
416
|
-
if (state.processed >= state.dirty) {
|
|
417
|
-
await transaction.deleteAlarm();
|
|
418
|
-
return { _tag: "CaughtUp" as const, nonterminal: state.nonterminal };
|
|
419
|
-
}
|
|
420
|
-
// Pre-arm the earliest retry before recovery. A successful finish may move this slot
|
|
421
|
-
// LATER to its bounded backoff, which does not cancel the running handler.
|
|
422
|
-
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
423
|
-
return { _tag: "Actionable" as const, generation: state.dirty };
|
|
424
|
-
}),
|
|
425
|
-
);
|
|
426
|
-
yield* failpoint.hit("maintenance:begin:after");
|
|
427
|
-
return result;
|
|
428
|
-
});
|
|
400
|
+
const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
|
|
401
|
+
yield* failpoint.hit("maintenance:ensure:before");
|
|
402
|
+
const now = yield* Clock.currentTimeMillis;
|
|
429
403
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
const priorStalls = yield* Ref.getAndUpdate(stalls, (count) =>
|
|
434
|
-
progressed ? 0 : count + 1,
|
|
435
|
-
);
|
|
436
|
-
if (progressed) return config.alarmBackoffBase;
|
|
437
|
-
const exponent = Math.min(priorStalls, 30);
|
|
438
|
-
const backoff = Math.min(config.alarmBackoffCap, config.alarmBackoffBase * 2 ** exponent);
|
|
439
|
-
const jitter = yield* Random.next;
|
|
440
|
-
// Full jitter over [backoff/2, backoff]: desynchronizes retry storms without ever
|
|
441
|
-
// waiting longer than the deterministic bound.
|
|
442
|
-
const jittered = Math.ceil(backoff / 2 + (backoff / 2) * jitter);
|
|
443
|
-
return Math.min(jittered, config.wakeScanInterval);
|
|
444
|
-
});
|
|
404
|
+
yield* runTransaction("ensure maintenance alarm", () =>
|
|
405
|
+
ctx.storage.transaction(async (transaction) => {
|
|
406
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
445
407
|
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
);
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
// is never earlier than the begin-pass fallback, so workerd does not cancel
|
|
526
|
-
// this running alarm handler before its report/span can complete.
|
|
527
|
-
await transaction.setAlarm(now + delay);
|
|
528
|
-
return "rearmed" as const;
|
|
529
|
-
}
|
|
530
|
-
if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
|
|
531
|
-
// A mutation overlapped this pass's observation window or raced
|
|
532
|
-
// acknowledgement. It stays dirty and its pre-armed bounded alarm survives;
|
|
533
|
-
// unseen effects are never acknowledged. Do not accelerate that future alarm
|
|
534
|
-
// from inside the current handler: workerd cancels a running handler when it
|
|
535
|
-
// writes an earlier slot.
|
|
536
|
-
await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
|
|
537
|
-
return "rearmed" as const;
|
|
538
|
-
}
|
|
539
|
-
await transaction.deleteAlarm();
|
|
540
|
-
return "cleared" as const;
|
|
541
|
-
}),
|
|
542
|
-
);
|
|
543
|
-
}),
|
|
544
|
-
);
|
|
545
|
-
yield* failpoint.hit("maintenance:finish:after");
|
|
546
|
-
if (alarmDisposition === "cleared") {
|
|
547
|
-
yield* Ref.set(stalls, 0);
|
|
548
|
-
}
|
|
408
|
+
if (!initialized) {
|
|
409
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
410
|
+
}
|
|
411
|
+
if (state.dirty > state.processed) {
|
|
412
|
+
await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
|
|
413
|
+
}
|
|
414
|
+
}),
|
|
415
|
+
);
|
|
416
|
+
yield* failpoint.hit("maintenance:ensure:after");
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
const beginPass = Effect.fn("ThreadMaintenance.beginPass")(function* () {
|
|
420
|
+
yield* failpoint.hit("maintenance:begin:before");
|
|
421
|
+
const now = yield* Clock.currentTimeMillis;
|
|
422
|
+
|
|
423
|
+
const result = yield* runTransaction("begin maintenance pass", () =>
|
|
424
|
+
ctx.storage.transaction(async (transaction) => {
|
|
425
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
426
|
+
|
|
427
|
+
if (!initialized) {
|
|
428
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
429
|
+
}
|
|
430
|
+
if (state.processed >= state.dirty) {
|
|
431
|
+
await transaction.deleteAlarm();
|
|
432
|
+
|
|
433
|
+
return { _tag: "CaughtUp" as const, nonterminal: state.nonterminal };
|
|
434
|
+
}
|
|
435
|
+
// Pre-arm the earliest retry before recovery. A successful finish may move this slot
|
|
436
|
+
// LATER to its bounded backoff, which does not cancel the running handler.
|
|
437
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
438
|
+
|
|
439
|
+
return { _tag: "Actionable" as const, generation: state.dirty };
|
|
440
|
+
}),
|
|
441
|
+
);
|
|
442
|
+
|
|
443
|
+
yield* failpoint.hit("maintenance:begin:after");
|
|
444
|
+
|
|
445
|
+
return result;
|
|
446
|
+
});
|
|
447
|
+
|
|
448
|
+
const rearmDelay = Effect.fn("ThreadMaintenance.rearmDelay")(function* (progressed: boolean) {
|
|
449
|
+
const priorStalls = yield* Ref.getAndUpdate(stalls, (count) =>
|
|
450
|
+
progressed ? 0 : count + 1,
|
|
451
|
+
);
|
|
452
|
+
|
|
453
|
+
if (progressed) return config.alarmBackoffBase;
|
|
454
|
+
const exponent = Math.min(priorStalls, 30);
|
|
455
|
+
const backoff = Math.min(config.alarmBackoffCap, config.alarmBackoffBase * 2 ** exponent);
|
|
456
|
+
const jitter = yield* Random.next;
|
|
457
|
+
// Full jitter over [backoff/2, backoff]: desynchronizes retry storms without ever
|
|
458
|
+
// waiting longer than the deterministic bound.
|
|
459
|
+
const jittered = Math.ceil(backoff / 2 + (backoff / 2) * jitter);
|
|
460
|
+
|
|
461
|
+
return Math.min(jittered, config.wakeScanInterval);
|
|
462
|
+
});
|
|
463
|
+
|
|
464
|
+
const pass = Effect.fn("ThreadMaintenance.pass")(function* (): Effect.fn.Return<
|
|
465
|
+
MaintenancePassReport,
|
|
466
|
+
MaintenancePassFailure
|
|
467
|
+
> {
|
|
468
|
+
const annotate = (report: MaintenancePassReport) =>
|
|
469
|
+
Effect.annotateCurrentSpan({
|
|
470
|
+
phase: report.phase,
|
|
471
|
+
recovered: report.recovered,
|
|
472
|
+
settled: report.settled,
|
|
473
|
+
nonterminal: report.nonterminal,
|
|
474
|
+
alarm: report.alarm,
|
|
475
|
+
}).pipe(Effect.as(report));
|
|
476
|
+
|
|
477
|
+
const started = yield* generationGate.withPermit(
|
|
478
|
+
Effect.gen(function* () {
|
|
479
|
+
const activeAtStart = yield* Ref.get(activeMutations);
|
|
480
|
+
const generation = yield* beginPass();
|
|
481
|
+
|
|
482
|
+
return { ...generation, activeAtStart };
|
|
483
|
+
}),
|
|
484
|
+
);
|
|
485
|
+
|
|
486
|
+
if (started._tag === "CaughtUp") {
|
|
549
487
|
return yield* annotate(
|
|
550
488
|
MaintenancePassReport.make({
|
|
551
|
-
phase: "
|
|
552
|
-
recovered:
|
|
553
|
-
settled:
|
|
554
|
-
nonterminal:
|
|
555
|
-
alarm:
|
|
489
|
+
phase: "caught-up",
|
|
490
|
+
recovered: 0,
|
|
491
|
+
settled: 0,
|
|
492
|
+
nonterminal: started.nonterminal,
|
|
493
|
+
alarm: "cleared",
|
|
556
494
|
}),
|
|
557
495
|
);
|
|
558
|
-
}
|
|
496
|
+
}
|
|
497
|
+
// Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
|
|
498
|
+
const recovered: ReadonlyArray<RecoveryReport> = yield* runtime.runRecovery;
|
|
499
|
+
// Step 3 — one bounded drain pass over this Object's own lane.
|
|
500
|
+
const settlements = yield* runtime.processThreadResolved(identity.threadId);
|
|
501
|
+
// Observe residual state before acknowledging this exact pass-start generation.
|
|
502
|
+
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
503
|
+
const reports = new Map(recovered.map((report) => [report.submissionId, report]));
|
|
504
|
+
const head = remaining[0];
|
|
505
|
+
const headWaiting = head !== undefined && stableExternalWait(head, reports);
|
|
559
506
|
|
|
560
|
-
|
|
561
|
-
//
|
|
562
|
-
pass
|
|
563
|
-
|
|
564
|
-
|
|
507
|
+
const autonomous = remaining.some((snapshot, index) => {
|
|
508
|
+
// FIFO followers cannot execute through a stable external wait. Only plain queued
|
|
509
|
+
// input is dormant here; admission repairs and accepted aborts still need a pass.
|
|
510
|
+
if (
|
|
511
|
+
index > 0 &&
|
|
512
|
+
headWaiting &&
|
|
513
|
+
snapshot.state === "ready" &&
|
|
514
|
+
reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput"
|
|
515
|
+
)
|
|
516
|
+
return false;
|
|
517
|
+
|
|
518
|
+
return !stableExternalWait(snapshot, reports);
|
|
565
519
|
});
|
|
566
|
-
|
|
567
|
-
|
|
520
|
+
|
|
521
|
+
const progressed =
|
|
522
|
+
settlements.length > 0 || recovered.some((report) => report.disposition === "repaired");
|
|
523
|
+
|
|
524
|
+
const delay = autonomous ? yield* rearmDelay(progressed) : 0;
|
|
525
|
+
const now = yield* Clock.currentTimeMillis;
|
|
526
|
+
|
|
527
|
+
yield* failpoint.hit("maintenance:finish:before");
|
|
528
|
+
|
|
529
|
+
const alarmDisposition = yield* generationGate.withPermit(
|
|
530
|
+
Effect.gen(function* () {
|
|
531
|
+
const active = yield* Ref.get(activeMutations);
|
|
532
|
+
|
|
533
|
+
return yield* runTransaction("finish maintenance pass", () =>
|
|
534
|
+
ctx.storage.transaction(async (transaction) => {
|
|
535
|
+
const { state } = await readMaintenanceState(transaction);
|
|
536
|
+
|
|
537
|
+
// Autonomous work and in-flight mutations intentionally leave the observed
|
|
538
|
+
// generation dirty. Otherwise acknowledge only the pass-start generation.
|
|
539
|
+
const processed =
|
|
540
|
+
autonomous || started.activeAtStart > 0 || active > 0
|
|
541
|
+
? state.processed
|
|
542
|
+
: state.processed > started.generation
|
|
543
|
+
? state.processed
|
|
544
|
+
: started.generation;
|
|
545
|
+
|
|
546
|
+
const next = ThreadMaintenanceState.make({
|
|
547
|
+
...state,
|
|
548
|
+
processed,
|
|
549
|
+
nonterminal: remaining.length,
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
553
|
+
if (autonomous) {
|
|
554
|
+
// Replace the crash-fallback slot with this pass's bounded backoff. The target
|
|
555
|
+
// is never earlier than the begin-pass fallback, so workerd does not cancel
|
|
556
|
+
// this running alarm handler before its report/span can complete.
|
|
557
|
+
await transaction.setAlarm(now + delay);
|
|
558
|
+
|
|
559
|
+
return "rearmed" as const;
|
|
560
|
+
}
|
|
561
|
+
if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
|
|
562
|
+
// A mutation overlapped this pass's observation window or raced
|
|
563
|
+
// acknowledgement. It stays dirty and its pre-armed bounded alarm survives;
|
|
564
|
+
// unseen effects are never acknowledged. Do not accelerate that future alarm
|
|
565
|
+
// from inside the current handler: workerd cancels a running handler when it
|
|
566
|
+
// writes an earlier slot.
|
|
567
|
+
await ensureTransactionAlarmBy(transaction, now + config.wakeScanInterval);
|
|
568
|
+
|
|
569
|
+
return "rearmed" as const;
|
|
570
|
+
}
|
|
571
|
+
await transaction.deleteAlarm();
|
|
572
|
+
|
|
573
|
+
return "cleared" as const;
|
|
574
|
+
}),
|
|
575
|
+
);
|
|
576
|
+
}),
|
|
577
|
+
);
|
|
578
|
+
|
|
579
|
+
yield* failpoint.hit("maintenance:finish:after");
|
|
580
|
+
if (alarmDisposition === "cleared") {
|
|
581
|
+
yield* Ref.set(stalls, 0);
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
return yield* annotate(
|
|
585
|
+
MaintenancePassReport.make({
|
|
586
|
+
phase: "actionable",
|
|
587
|
+
recovered: recovered.length,
|
|
588
|
+
settled: settlements.length,
|
|
589
|
+
nonterminal: remaining.length,
|
|
590
|
+
alarm: alarmDisposition,
|
|
591
|
+
}),
|
|
592
|
+
);
|
|
593
|
+
});
|
|
594
|
+
|
|
595
|
+
return ThreadMaintenance.of({
|
|
596
|
+
// A mid-pass immediate hint is droppable; durable dirty state decides the final alarm.
|
|
597
|
+
pass: alarm.withWakesDeferred(maintenancePassGate.withPermit(pass())),
|
|
598
|
+
ensureAlarm: ensureAlarm(),
|
|
599
|
+
withMutation,
|
|
600
|
+
});
|
|
601
|
+
}),
|
|
602
|
+
);
|
|
568
603
|
}
|
package/src/bindings.ts
CHANGED
|
@@ -91,12 +91,15 @@ export const threadNamespaceFromEnv = Effect.fn("threadNamespaceFromEnv")(functi
|
|
|
91
91
|
message: "The Worker environment is not an object; no bindings are available.",
|
|
92
92
|
});
|
|
93
93
|
}
|
|
94
|
+
|
|
94
95
|
const candidate = yield* Effect.try({
|
|
95
96
|
try: () => {
|
|
96
97
|
const value: unknown = Reflect.get(env, binding);
|
|
98
|
+
|
|
97
99
|
if (!Predicate.isObjectKeyword(value)) return undefined;
|
|
98
100
|
const idFromName: unknown = Reflect.get(value, "idFromName");
|
|
99
101
|
const get: unknown = Reflect.get(value, "get");
|
|
102
|
+
|
|
100
103
|
return typeof idFromName === "function" && typeof get === "function" ? value : undefined;
|
|
101
104
|
},
|
|
102
105
|
catch: () =>
|
|
@@ -105,11 +108,13 @@ export const threadNamespaceFromEnv = Effect.fn("threadNamespaceFromEnv")(functi
|
|
|
105
108
|
message: `env.${binding} could not be inspected as a DurableObjectNamespace binding.`,
|
|
106
109
|
}),
|
|
107
110
|
});
|
|
111
|
+
|
|
108
112
|
if (candidate !== undefined) {
|
|
109
113
|
// The structural probe above is the entire runtime contract this package relies on;
|
|
110
114
|
// the assertion records that `idFromName`/`get` name a DurableObjectNamespace.
|
|
111
115
|
return candidate as unknown as DurableObjectNamespace<ThreadObjectRpc>;
|
|
112
116
|
}
|
|
117
|
+
|
|
113
118
|
return yield* CloudflareBindingError.make({
|
|
114
119
|
binding,
|
|
115
120
|
message:
|