@effect-agent/platform-cloudflare 0.1.0-beta.113 → 0.1.0-beta.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Alarm.d.mts +32 -48
- package/dist/Alarm.mjs +173 -175
- package/dist/Alarm.mjs.map +1 -1
- package/dist/CloudflareThreadClient.d.mts +45 -30
- package/dist/ProtectedBrowser.mjs +2 -1
- package/dist/ProtectedBrowser.mjs.map +1 -1
- package/dist/{ThreadObject-BPt5uByD.mjs → ThreadObject-CEWwifrL.mjs} +10 -29
- package/dist/ThreadObject-CEWwifrL.mjs.map +1 -0
- package/dist/{ThreadObject-DAuaRyGl.d.mts → ThreadObject-HwODpdru.d.mts} +22 -22
- package/dist/ThreadObject.d.mts +1 -1
- package/dist/ThreadObject.mjs +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +291 -350
- package/src/ThreadObject.ts +4 -0
- package/src/internal/layers.ts +8 -0
- package/src/internal/message-delivery.ts +4 -64
- package/src/protected-browser/inspect-frame.ts +2 -1
- package/dist/ThreadObject-BPt5uByD.mjs.map +0 -1
package/src/Alarm.ts
CHANGED
|
@@ -9,7 +9,6 @@ import {
|
|
|
9
9
|
Fiber,
|
|
10
10
|
Layer,
|
|
11
11
|
Option,
|
|
12
|
-
PubSub,
|
|
13
12
|
Random,
|
|
14
13
|
Ref,
|
|
15
14
|
Result,
|
|
@@ -286,116 +285,56 @@ export class ThreadPublication extends Context.Service<
|
|
|
286
285
|
});
|
|
287
286
|
}
|
|
288
287
|
|
|
289
|
-
/** Event-local accounting shared by the existing maintenance pumps and native scheduler. */
|
|
290
|
-
export class ThreadMaintenanceActivity extends Context.Service<
|
|
291
|
-
ThreadMaintenanceActivity,
|
|
292
|
-
{
|
|
293
|
-
/**
|
|
294
|
-
* Account for one finite selection/dispatch, through joined completion or interruption.
|
|
295
|
-
* Register before reading/claiming work. After dispatch closes the body is not started.
|
|
296
|
-
* Host waves retain their declared allowance from this call; message Claims keep the driver's
|
|
297
|
-
* own deadline. Waiting for notifications belongs outside this bracket.
|
|
298
|
-
*/
|
|
299
|
-
readonly run: <R>(
|
|
300
|
-
wave: Effect.Effect<void, DurableAlarmError, R>,
|
|
301
|
-
) => Effect.Effect<void, DurableAlarmError, R>;
|
|
302
|
-
/** Acknowledge all initial subscriptions/scans. Composite hosts use `all` below. */
|
|
303
|
-
readonly ready: Effect.Effect<void>;
|
|
304
|
-
/**
|
|
305
|
-
* Acquire an independent scoped subscription before the initial scan and `ready`.
|
|
306
|
-
* The returned wait observes coalesced native scan hints, including those published before
|
|
307
|
-
* its first execution. Each hint requests a current local due-work check.
|
|
308
|
-
*/
|
|
309
|
-
readonly subscribeChanges: Effect.Effect<Effect.Effect<void>, never, Scope.Scope>;
|
|
310
|
-
}
|
|
311
|
-
>()("@effect-agent/platform-cloudflare/ThreadMaintenanceActivity") {
|
|
312
|
-
/** Join child pumps; acknowledge the parent only after every child is ready or has exited. */
|
|
313
|
-
static readonly all = Effect.fnUntraced(function* <A, E, R>(
|
|
314
|
-
lanes: ReadonlyArray<Effect.Effect<A, E, R>>,
|
|
315
|
-
): Effect.fn.Return<ReadonlyArray<A>, E, R | ThreadMaintenanceActivity> {
|
|
316
|
-
const activity = yield* ThreadMaintenanceActivity;
|
|
317
|
-
let remaining = lanes.length;
|
|
318
|
-
|
|
319
|
-
if (remaining === 0) return yield* activity.ready.pipe(Effect.as([]));
|
|
320
|
-
|
|
321
|
-
return yield* Effect.forEach(
|
|
322
|
-
lanes,
|
|
323
|
-
(lane) => {
|
|
324
|
-
let ready = false;
|
|
325
|
-
|
|
326
|
-
const child: ThreadMaintenanceActivity["Service"] = {
|
|
327
|
-
...activity,
|
|
328
|
-
ready: Effect.suspend(() => {
|
|
329
|
-
if (ready) return Effect.void;
|
|
330
|
-
ready = true;
|
|
331
|
-
|
|
332
|
-
return --remaining === 0 ? activity.ready : Effect.void;
|
|
333
|
-
}).pipe(Effect.uninterruptible),
|
|
334
|
-
};
|
|
335
|
-
|
|
336
|
-
// A failed initial setup is accounted for without swallowing its Cause. Callers
|
|
337
|
-
// may collect Exits when sibling pumps must keep their independent opportunities.
|
|
338
|
-
return lane.pipe(
|
|
339
|
-
Effect.provideService(ThreadMaintenanceActivity, child),
|
|
340
|
-
Effect.onExit(() => child.ready),
|
|
341
|
-
);
|
|
342
|
-
},
|
|
343
|
-
{ concurrency: "unbounded" },
|
|
344
|
-
);
|
|
345
|
-
});
|
|
346
|
-
}
|
|
347
|
-
|
|
348
288
|
/**
|
|
349
|
-
* Host-assembled native message recovery.
|
|
350
|
-
*
|
|
351
|
-
*
|
|
289
|
+
* Host-assembled native message recovery. Preparation is a bounded local selection; the
|
|
290
|
+
* driver bounds each actual Claim and persists its timeout/retry before `run` returns.
|
|
291
|
+
* The alarm must not add a timer starting at selection: Claim setup and retry commits belong
|
|
292
|
+
* to the driver. The prepared allowance decides only whether a wave fits this event.
|
|
352
293
|
*/
|
|
353
294
|
export const ThreadMessageDelivery = Context.Reference<{
|
|
354
|
-
readonly
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
295
|
+
readonly prepare: Effect.Effect<
|
|
296
|
+
{
|
|
297
|
+
readonly timeoutMillis: number;
|
|
298
|
+
readonly run: Effect.Effect<void, DurableAlarmError>;
|
|
299
|
+
},
|
|
300
|
+
DurableAlarmError
|
|
301
|
+
>;
|
|
358
302
|
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
359
303
|
}>("@effect-agent/platform-cloudflare/ThreadMessageDelivery", {
|
|
360
304
|
defaultValue: () => ({
|
|
361
|
-
|
|
305
|
+
prepare: Effect.succeed({ timeoutMillis: 1, run: Effect.void }),
|
|
362
306
|
pendingDeadline: Effect.succeed(Option.none()),
|
|
363
307
|
}),
|
|
364
308
|
});
|
|
365
309
|
|
|
366
310
|
/**
|
|
367
|
-
*
|
|
368
|
-
*
|
|
369
|
-
*
|
|
370
|
-
*
|
|
371
|
-
* Passive subscriptions do not count as active work. Closure stops both new waves and native
|
|
372
|
-
* Attempts; local admission/control subscriptions belong to the event Scope until teardown.
|
|
373
|
-
* No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
|
|
311
|
+
* One independently schedulable application obligation. `run` selects and joins one finite
|
|
312
|
+
* wave, including its retry/receipt commits. The alarm gives every lane an initial opportunity,
|
|
313
|
+
* then checks its local pending deadline on completion, native progress, wakes and bounded scans.
|
|
314
|
+
* No subscriptions, readiness acknowledgements, deadline sleeps or dispatch loops are needed.
|
|
374
315
|
*
|
|
375
|
-
* Declare a
|
|
376
|
-
*
|
|
377
|
-
*
|
|
378
|
-
*
|
|
316
|
+
* Declare a whole-wave allowance (1..300000ms), including selection, local commits and cleanup. A wave
|
|
317
|
+
* starts only if that allowance fits the event and later arrivals cannot renew it. A failed
|
|
318
|
+
* lane is not retried within the event. The alarm joins admitted work and closes its Scope
|
|
319
|
+
* before reading durable deadlines under the shared ThreadMutationGate.
|
|
379
320
|
*
|
|
380
|
-
*
|
|
381
|
-
* dispatch;
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
* alarm slot. Required native publication gates belong to ThreadPublication.
|
|
321
|
+
* `pendingDeadline` is bounded local control state, never a retained-history scan. Persist
|
|
322
|
+
* envelopes/claims before dispatch; cancellation is local, not remote rollback. Network waits
|
|
323
|
+
* and finalizers must be interruptible; short atomic commits may be uninterruptible. Mutations
|
|
324
|
+
* use the shared gate and producers notify the existing WakeScheduler after commit. Hooks never
|
|
325
|
+
* write the raw alarm slot. Required native publication gates belong to ThreadPublication.
|
|
385
326
|
*/
|
|
386
|
-
export
|
|
327
|
+
export interface ThreadHostMaintenanceLane {
|
|
387
328
|
readonly dispatchTimeoutMillis: number;
|
|
388
|
-
readonly
|
|
389
|
-
dispatchClosed: Effect.Effect<void>,
|
|
390
|
-
dispatchUntil: DateTime.Utc,
|
|
391
|
-
) => Effect.Effect<void, DurableAlarmError, Scope.Scope | ThreadMaintenanceActivity>;
|
|
329
|
+
readonly run: Effect.Effect<void, DurableAlarmError, Scope.Scope>;
|
|
392
330
|
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** Independent lanes share the existing alarm; compose hosts by concatenating their lanes. */
|
|
334
|
+
export const ThreadHostMaintenance = Context.Reference<{
|
|
335
|
+
readonly lanes: ReadonlyArray<ThreadHostMaintenanceLane>;
|
|
393
336
|
}>("@effect-agent/platform-cloudflare/ThreadHostMaintenance", {
|
|
394
|
-
defaultValue: () => ({
|
|
395
|
-
dispatchTimeoutMillis: 1,
|
|
396
|
-
drainUntil: () => Effect.void,
|
|
397
|
-
pendingDeadline: Effect.succeed(Option.none()),
|
|
398
|
-
}),
|
|
337
|
+
defaultValue: () => ({ lanes: [] }),
|
|
399
338
|
});
|
|
400
339
|
|
|
401
340
|
const earliestDeadline = (
|
|
@@ -683,10 +622,11 @@ export type MaintenancePassFailure =
|
|
|
683
622
|
* 1. Prearm before any work. A caught-up native step uses the O(1) generation record without
|
|
684
623
|
* recovery, ledger scans or canonical-history reads.
|
|
685
624
|
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
686
|
-
* producer keeps its
|
|
625
|
+
* producer keeps its generation dirty and immediately eligible. Quiescent native retries
|
|
626
|
+
* retain their durable backoff.
|
|
687
627
|
* 3. Keep native and delivery admission open together while finite waves remain active, so
|
|
688
628
|
* fresh replies and abort controls can progress during unrelated cleanup. Close atomically
|
|
689
|
-
* at quiescence, or at the original ten-minute yield deadline,
|
|
629
|
+
* at quiescence, or at the original ten-minute yield deadline, then join admitted waves.
|
|
690
630
|
* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill waves are
|
|
691
631
|
* bounded independently; incoming native work never restarts or cancels their attempts.
|
|
692
632
|
* Auxiliary failures are reported after the current native opportunity.
|
|
@@ -769,7 +709,13 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
769
709
|
const pendingDeadline = Effect.gen(function* () {
|
|
770
710
|
return earliestDeadline(
|
|
771
711
|
earliestDeadline(yield* publication.pendingDeadline, yield* messages.pendingDeadline),
|
|
772
|
-
earliestDeadline(
|
|
712
|
+
earliestDeadline(
|
|
713
|
+
yield* projectionDeadline,
|
|
714
|
+
(yield* Effect.forEach(host.lanes, (lane) => lane.pendingDeadline)).reduce(
|
|
715
|
+
earliestDeadline,
|
|
716
|
+
Option.none<number>(),
|
|
717
|
+
),
|
|
718
|
+
),
|
|
773
719
|
);
|
|
774
720
|
});
|
|
775
721
|
|
|
@@ -1379,16 +1325,20 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1379
1325
|
ctx.storage.transaction(async (transaction) => {
|
|
1380
1326
|
const { state } = await readMaintenanceState(transaction);
|
|
1381
1327
|
|
|
1328
|
+
const mutationOverlap =
|
|
1329
|
+
observation.activeAtStart > 0 || started.activeAtStart > 0 || active > 0;
|
|
1330
|
+
|
|
1382
1331
|
const processed =
|
|
1383
|
-
autonomous ||
|
|
1384
|
-
observation.activeAtStart > 0 ||
|
|
1385
|
-
started.activeAtStart > 0 ||
|
|
1386
|
-
active > 0
|
|
1332
|
+
autonomous || mutationOverlap
|
|
1387
1333
|
? state.processed
|
|
1388
1334
|
: state.processed > observation.generation
|
|
1389
1335
|
? state.processed
|
|
1390
1336
|
: observation.generation;
|
|
1391
1337
|
|
|
1338
|
+
// Enrollment precedes the mutation body. A retry from an overlapping snapshot
|
|
1339
|
+
// must not defer work that becomes ready later in that same generation.
|
|
1340
|
+
const canBackoff = !mutationOverlap && state.dirty === started.generation;
|
|
1341
|
+
|
|
1392
1342
|
const next = ThreadMaintenanceState.make({
|
|
1393
1343
|
...Struct.omit(state, ["retry"]),
|
|
1394
1344
|
processed,
|
|
@@ -1396,7 +1346,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1396
1346
|
bindingRetries: (state.bindingRetries ?? []).filter((retry) =>
|
|
1397
1347
|
remaining.some((row) => row.submissionId === retry.submissionId),
|
|
1398
1348
|
),
|
|
1399
|
-
...(autonomous && !progressed
|
|
1349
|
+
...(autonomous && !progressed && canBackoff
|
|
1400
1350
|
? {
|
|
1401
1351
|
retry: MaintenanceRetry.make({
|
|
1402
1352
|
generation: started.generation,
|
|
@@ -1410,12 +1360,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1410
1360
|
|
|
1411
1361
|
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
1412
1362
|
if (autonomous) {
|
|
1413
|
-
return
|
|
1414
|
-
? now + minimumAlarmDelay
|
|
1415
|
-
: now + delay;
|
|
1363
|
+
return now + (canBackoff ? delay : minimumAlarmDelay);
|
|
1416
1364
|
}
|
|
1417
1365
|
|
|
1418
|
-
return
|
|
1366
|
+
return mutationOverlap || next.dirty > next.processed
|
|
1419
1367
|
? now + minimumAlarmDelay
|
|
1420
1368
|
: undefined;
|
|
1421
1369
|
}),
|
|
@@ -1448,88 +1396,27 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1448
1396
|
// Its completion must retain the original actionable observation for acknowledgement.
|
|
1449
1397
|
const started = yield* beginNative(observed);
|
|
1450
1398
|
|
|
1451
|
-
// This scope owns
|
|
1399
|
+
// This scope owns every admitted finite wave, including native advancement.
|
|
1452
1400
|
// Close it before final alarm rearming, including on failure or event interruption.
|
|
1453
1401
|
const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) =>
|
|
1454
1402
|
Scope.close(scope, exit),
|
|
1455
1403
|
);
|
|
1456
1404
|
|
|
1457
|
-
const
|
|
1458
|
-
const stopDispatch = Deferred.await(dispatchClosed);
|
|
1459
|
-
// Coalesce local scan hints while a lane is busy; one hint requests a current read.
|
|
1460
|
-
const checks = yield* PubSub.sliding<void>(1);
|
|
1461
|
-
|
|
1462
|
-
yield* Effect.addFinalizer(() => PubSub.shutdown(checks));
|
|
1463
|
-
let dispatchOpen = true;
|
|
1464
|
-
let activityChanged = yield* Deferred.make<void>();
|
|
1465
|
-
const signalActivity = Effect.suspend(() => Deferred.succeed(activityChanged, undefined));
|
|
1466
|
-
|
|
1467
|
-
const closeDispatch = Effect.sync(() => {
|
|
1468
|
-
dispatchOpen = false;
|
|
1469
|
-
}).pipe(Effect.andThen(Deferred.succeed(dispatchClosed, undefined)));
|
|
1470
|
-
|
|
1471
|
-
const makeActivity = Effect.fnUntraced(function* (allowance?: number) {
|
|
1472
|
-
const initialized = yield* Deferred.make<void>();
|
|
1473
|
-
let active = 0;
|
|
1474
|
-
let failed = false;
|
|
1475
|
-
|
|
1476
|
-
const ready = Deferred.succeed(initialized, undefined).pipe(
|
|
1477
|
-
Effect.andThen(signalActivity),
|
|
1478
|
-
Effect.asVoid,
|
|
1479
|
-
Effect.uninterruptible,
|
|
1480
|
-
);
|
|
1405
|
+
const fork = <A, E>(work: Effect.Effect<A, E>) => Effect.forkIn(work, auxiliaryScope);
|
|
1481
1406
|
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1407
|
+
// Each lane has at most one finite wave. A completion is retained until the single
|
|
1408
|
+
// scheduling loop observes it; a busy sibling cannot consume another lane's hint.
|
|
1409
|
+
const lanes = host.lanes.map((lane) => ({
|
|
1410
|
+
...lane,
|
|
1411
|
+
initial: true,
|
|
1412
|
+
check: true,
|
|
1413
|
+
exhausted: false,
|
|
1414
|
+
fiber: undefined as Fiber.Fiber<boolean, DurableAlarmError> | undefined,
|
|
1415
|
+
}));
|
|
1490
1416
|
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
!admitted
|
|
1495
|
-
? Effect.void
|
|
1496
|
-
: (allowance === undefined
|
|
1497
|
-
? wave
|
|
1498
|
-
: wave.pipe(
|
|
1499
|
-
Effect.timeoutOrElse({
|
|
1500
|
-
duration: allowance,
|
|
1501
|
-
orElse: () =>
|
|
1502
|
-
DurableAlarmError.make({
|
|
1503
|
-
operation: "host dispatch allowance",
|
|
1504
|
-
message:
|
|
1505
|
-
"The admitted host wave exceeded its allowance; durable work remains pending",
|
|
1506
|
-
}),
|
|
1507
|
-
}),
|
|
1508
|
-
)
|
|
1509
|
-
).pipe(
|
|
1510
|
-
Effect.onExit((exit) =>
|
|
1511
|
-
Effect.sync(() => {
|
|
1512
|
-
if (Exit.isFailure(exit)) failed = true;
|
|
1513
|
-
}),
|
|
1514
|
-
),
|
|
1515
|
-
),
|
|
1516
|
-
(admitted) =>
|
|
1517
|
-
Effect.sync(() => {
|
|
1518
|
-
if (admitted) active--;
|
|
1519
|
-
}).pipe(Effect.andThen(signalActivity)),
|
|
1520
|
-
),
|
|
1521
|
-
};
|
|
1522
|
-
|
|
1523
|
-
return {
|
|
1524
|
-
activity,
|
|
1525
|
-
initialized: Deferred.await(initialized),
|
|
1526
|
-
quiet: () => active === 0 && Deferred.isDoneUnsafe(initialized),
|
|
1527
|
-
failed: () => failed,
|
|
1528
|
-
};
|
|
1529
|
-
});
|
|
1530
|
-
|
|
1531
|
-
const messageActivity = yield* makeActivity();
|
|
1532
|
-
const hostActivity = yield* makeActivity(host.dispatchTimeoutMillis);
|
|
1417
|
+
let messageExhausted = false;
|
|
1418
|
+
let delivery: Fiber.Fiber<void, DurableAlarmError> | undefined;
|
|
1419
|
+
let failure: Cause.Cause<DurableAlarmError | ThreadProjectionError> | undefined;
|
|
1533
1420
|
|
|
1534
1421
|
const recovery: NativeRecovery = {
|
|
1535
1422
|
queue: yield* Deferred.make<ReadonlyArray<ThreadId>>(),
|
|
@@ -1543,7 +1430,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1543
1430
|
repaired: false,
|
|
1544
1431
|
};
|
|
1545
1432
|
|
|
1546
|
-
const recoveryFiber = yield*
|
|
1433
|
+
const recoveryFiber = yield* fork(
|
|
1547
1434
|
Effect.gen(function* () {
|
|
1548
1435
|
for (const threadId of yield* Deferred.await(recovery.queue)) {
|
|
1549
1436
|
yield* failpoint.hit("maintenance:select:before");
|
|
@@ -1565,210 +1452,264 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1565
1452
|
yield* wakes.notify(threadId);
|
|
1566
1453
|
}
|
|
1567
1454
|
}),
|
|
1568
|
-
auxiliaryScope,
|
|
1569
|
-
);
|
|
1570
|
-
|
|
1571
|
-
// Fork setup too: an ordinary auxiliary setup failure is reported after native work,
|
|
1572
|
-
// rather than gating its opportunity. Event interruption still closes every fiber.
|
|
1573
|
-
const deliveryFiber = yield* Effect.forkIn(
|
|
1574
|
-
Scope.provide(auxiliaryScope)(
|
|
1575
|
-
messages.drainUntil(stopDispatch, dispatchUntil).pipe(
|
|
1576
|
-
Effect.provideService(ThreadMaintenanceActivity, messageActivity.activity),
|
|
1577
|
-
Effect.onExit(() => messageActivity.activity.ready),
|
|
1578
|
-
),
|
|
1579
|
-
),
|
|
1580
|
-
auxiliaryScope,
|
|
1581
|
-
);
|
|
1582
|
-
|
|
1583
|
-
const hostFiber = yield* Effect.forkIn(
|
|
1584
|
-
Scope.provide(auxiliaryScope)(
|
|
1585
|
-
Effect.gen(function* () {
|
|
1586
|
-
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(
|
|
1587
|
-
Effect.mapError((cause) =>
|
|
1588
|
-
DurableAlarmError.make({
|
|
1589
|
-
operation: "host dispatch allowance",
|
|
1590
|
-
message:
|
|
1591
|
-
"Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
|
|
1592
|
-
cause,
|
|
1593
|
-
}),
|
|
1594
|
-
),
|
|
1595
|
-
);
|
|
1596
|
-
|
|
1597
|
-
// Initial setup must also be finite. Once it is accounted for, each actual
|
|
1598
|
-
// wave has its own unchanged allowance; passive listeners have no timer.
|
|
1599
|
-
const initialization = hostActivity.initialized.pipe(
|
|
1600
|
-
Effect.timeoutOrElse({
|
|
1601
|
-
duration: host.dispatchTimeoutMillis,
|
|
1602
|
-
orElse: () =>
|
|
1603
|
-
DurableAlarmError.make({
|
|
1604
|
-
operation: "host initialization",
|
|
1605
|
-
message:
|
|
1606
|
-
"The host did not account for initial maintenance before its allowance; durable work remains pending",
|
|
1607
|
-
}),
|
|
1608
|
-
}),
|
|
1609
|
-
Effect.andThen(Effect.never),
|
|
1610
|
-
);
|
|
1611
|
-
|
|
1612
|
-
yield* Effect.raceFirst(
|
|
1613
|
-
host
|
|
1614
|
-
.drainUntil(stopDispatch, dispatchUntil)
|
|
1615
|
-
.pipe(Effect.provideService(ThreadMaintenanceActivity, hostActivity.activity)),
|
|
1616
|
-
initialization,
|
|
1617
|
-
);
|
|
1618
|
-
}).pipe(Effect.onExit(() => hostActivity.activity.ready)),
|
|
1619
|
-
),
|
|
1620
|
-
auxiliaryScope,
|
|
1621
1455
|
);
|
|
1622
1456
|
|
|
1623
|
-
// Backfill is one disposable wave;
|
|
1624
|
-
const backfill = yield*
|
|
1457
|
+
// Backfill is one disposable wave; native and host work keep their own opportunities.
|
|
1458
|
+
const backfill = yield* fork(
|
|
1625
1459
|
drainDue.pipe(
|
|
1626
1460
|
Effect.provideService(ThreadProjectionMaintenance, projection),
|
|
1627
1461
|
Effect.timeoutOption(config.projectionDispatchTimeoutMillis),
|
|
1628
|
-
Effect.onExit(() => signalActivity),
|
|
1629
1462
|
),
|
|
1630
|
-
auxiliaryScope,
|
|
1631
1463
|
);
|
|
1632
1464
|
|
|
1633
|
-
let
|
|
1465
|
+
let backfillObserved = false;
|
|
1466
|
+
let recoveryObserved = false;
|
|
1467
|
+
|
|
1468
|
+
let native: Fiber.Fiber<NativePassResult, MaintenancePassFailure> | undefined = yield* fork(
|
|
1469
|
+
advance(started, yieldAfter, observed, recovery),
|
|
1470
|
+
);
|
|
1471
|
+
|
|
1472
|
+
let result: NativePassResult = {
|
|
1473
|
+
phase: "caught-up",
|
|
1474
|
+
settled: 0,
|
|
1475
|
+
nonterminal: started.nonterminal,
|
|
1476
|
+
nextAttemptAt: undefined,
|
|
1477
|
+
};
|
|
1634
1478
|
|
|
1635
|
-
// This event owns one finite old-recovery wave, including an empty caught-up wave.
|
|
1636
|
-
recovery.started = true;
|
|
1637
|
-
yield* Deferred.succeed(recovery.queue, []);
|
|
1638
1479
|
let phase = result.phase;
|
|
1639
|
-
let settled =
|
|
1480
|
+
let settled = 0;
|
|
1481
|
+
let nativeCheck = false;
|
|
1482
|
+
const until = DateTime.toEpochMillis(yieldAfter);
|
|
1483
|
+
const dispatchEnd = DateTime.toEpochMillis(dispatchUntil);
|
|
1640
1484
|
|
|
1641
|
-
|
|
1485
|
+
const checkLanes = () => {
|
|
1486
|
+
for (const lane of lanes) lane.check = true;
|
|
1487
|
+
};
|
|
1642
1488
|
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1489
|
+
while (true) {
|
|
1490
|
+
if (native?.pollUnsafe() !== undefined) {
|
|
1491
|
+
result = yield* Fiber.join(native);
|
|
1492
|
+
native = undefined;
|
|
1493
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
1494
|
+
settled += result.settled;
|
|
1495
|
+
observed.nativeOnly = false;
|
|
1496
|
+
// An empty initial scan still opens exactly one old-recovery opportunity.
|
|
1497
|
+
recovery.started = true;
|
|
1498
|
+
yield* Deferred.succeed(recovery.queue, []);
|
|
1499
|
+
checkLanes();
|
|
1500
|
+
}
|
|
1501
|
+
if (!recoveryObserved && recoveryFiber.pollUnsafe() !== undefined) {
|
|
1502
|
+
yield* Fiber.join(recoveryFiber);
|
|
1503
|
+
recoveryObserved = true;
|
|
1504
|
+
nativeCheck = recovery.needsCheckpoint;
|
|
1505
|
+
}
|
|
1506
|
+
if (!backfillObserved) {
|
|
1507
|
+
const exit = backfill.pollUnsafe();
|
|
1648
1508
|
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1509
|
+
if (exit !== undefined) {
|
|
1510
|
+
backfillObserved = true;
|
|
1511
|
+
if (Exit.isFailure(exit)) failure ??= exit.cause;
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
for (const lane of lanes) {
|
|
1515
|
+
const exit = lane.fiber?.pollUnsafe();
|
|
1516
|
+
|
|
1517
|
+
if (exit === undefined) continue;
|
|
1518
|
+
lane.fiber = undefined;
|
|
1519
|
+
if (Exit.isFailure(exit)) {
|
|
1520
|
+
failure ??= exit.cause;
|
|
1521
|
+
lane.exhausted = true;
|
|
1522
|
+
} else {
|
|
1523
|
+
lane.check ||= exit.value;
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
const deliveryExit = delivery?.pollUnsafe();
|
|
1653
1527
|
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
)
|
|
1528
|
+
if (deliveryExit !== undefined) {
|
|
1529
|
+
delivery = undefined;
|
|
1530
|
+
if (Exit.isFailure(deliveryExit)) {
|
|
1531
|
+
failure ??= deliveryExit.cause;
|
|
1532
|
+
messageExhausted = true;
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1657
1535
|
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1536
|
+
// Preserve the initial native opportunity even when auxiliary setup fails.
|
|
1537
|
+
if (failure !== undefined && native === undefined)
|
|
1538
|
+
return yield* Effect.failCause(failure);
|
|
1539
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1662
1540
|
|
|
1663
|
-
|
|
1664
|
-
Fiber.joinAll([deliveryFiber, hostJoin, backfill, recoveryFiber]),
|
|
1665
|
-
);
|
|
1541
|
+
if (now >= until) break;
|
|
1666
1542
|
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1543
|
+
for (const lane of lanes) {
|
|
1544
|
+
if (lane.fiber !== undefined || lane.exhausted || !lane.check) continue;
|
|
1545
|
+
lane.check = false;
|
|
1546
|
+
lane.fiber = yield* fork(
|
|
1547
|
+
Effect.gen(function* () {
|
|
1548
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(
|
|
1549
|
+
lane.dispatchTimeoutMillis,
|
|
1550
|
+
).pipe(
|
|
1551
|
+
Effect.mapError((cause) =>
|
|
1552
|
+
DurableAlarmError.make({
|
|
1553
|
+
operation: "host dispatch allowance",
|
|
1554
|
+
message:
|
|
1555
|
+
"Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
|
|
1556
|
+
cause,
|
|
1557
|
+
}),
|
|
1558
|
+
),
|
|
1559
|
+
);
|
|
1560
|
+
if ((yield* Clock.currentTimeMillis) + lane.dispatchTimeoutMillis > dispatchEnd) {
|
|
1561
|
+
lane.exhausted = true;
|
|
1562
|
+
|
|
1563
|
+
return false;
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
return yield* Effect.gen(function* () {
|
|
1567
|
+
if (!lane.initial) {
|
|
1568
|
+
const deadline = yield* lane.pendingDeadline;
|
|
1569
|
+
|
|
1570
|
+
if (
|
|
1571
|
+
Option.isNone(deadline) ||
|
|
1572
|
+
deadline.value > (yield* Clock.currentTimeMillis)
|
|
1573
|
+
)
|
|
1574
|
+
return false;
|
|
1575
|
+
}
|
|
1576
|
+
const selectedAt = yield* Clock.currentTimeMillis;
|
|
1577
|
+
|
|
1578
|
+
if (
|
|
1579
|
+
selectedAt >= until ||
|
|
1580
|
+
selectedAt + lane.dispatchTimeoutMillis > dispatchEnd
|
|
1581
|
+
) {
|
|
1582
|
+
lane.exhausted = true;
|
|
1583
|
+
|
|
1584
|
+
return false;
|
|
1585
|
+
}
|
|
1586
|
+
lane.initial = false;
|
|
1587
|
+
yield* lane.run;
|
|
1588
|
+
|
|
1589
|
+
return true;
|
|
1590
|
+
}).pipe(
|
|
1591
|
+
// Cleanup shares this wave's original allowance.
|
|
1592
|
+
Effect.scoped,
|
|
1593
|
+
Effect.timeoutOrElse({
|
|
1594
|
+
duration: lane.dispatchTimeoutMillis,
|
|
1595
|
+
orElse: () =>
|
|
1596
|
+
DurableAlarmError.make({
|
|
1597
|
+
operation: "host dispatch allowance",
|
|
1598
|
+
message:
|
|
1599
|
+
"The admitted host wave exceeded its allowance; durable work remains pending",
|
|
1600
|
+
}),
|
|
1601
|
+
}),
|
|
1602
|
+
);
|
|
1603
|
+
}),
|
|
1604
|
+
);
|
|
1605
|
+
}
|
|
1606
|
+
if (delivery === undefined && !messageExhausted) {
|
|
1607
|
+
// Selection is bounded local work. Fork only an actual due wave, so an empty
|
|
1608
|
+
// deadline check cannot extend retirement or consume another scheduling turn.
|
|
1609
|
+
const selected = yield* Effect.gen(function* () {
|
|
1610
|
+
const deadline = yield* messages.pendingDeadline;
|
|
1611
|
+
|
|
1612
|
+
return Option.isSome(deadline) && deadline.value <= (yield* Clock.currentTimeMillis)
|
|
1613
|
+
? Option.some(yield* messages.prepare)
|
|
1614
|
+
: Option.none();
|
|
1615
|
+
}).pipe(Effect.exit);
|
|
1616
|
+
|
|
1617
|
+
if (Exit.isFailure(selected)) {
|
|
1618
|
+
failure ??= selected.cause;
|
|
1619
|
+
messageExhausted = true;
|
|
1620
|
+
} else if (Option.isSome(selected.value)) {
|
|
1621
|
+
const wave = selected.value.value;
|
|
1622
|
+
const selectedAt = yield* Clock.currentTimeMillis;
|
|
1623
|
+
|
|
1624
|
+
if (selectedAt >= until || selectedAt + wave.timeoutMillis > dispatchEnd) {
|
|
1625
|
+
messageExhausted = true;
|
|
1626
|
+
} else {
|
|
1627
|
+
// No second timeout: the driver owes the Claim's timeout/retry commit.
|
|
1628
|
+
delivery = yield* fork(wave.run);
|
|
1629
|
+
}
|
|
1680
1630
|
}
|
|
1681
1631
|
}
|
|
1682
|
-
if (retiredExit !== undefined && Exit.isFailure(retiredExit)) break;
|
|
1683
1632
|
|
|
1633
|
+
// Retire a quiet event before consuming hints queued during its native Attempt.
|
|
1634
|
+
// Recovery still gets its checkpoint and, if needed, initial dispatch opportunity.
|
|
1684
1635
|
if (
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
dispatchOpen = false;
|
|
1714
|
-
|
|
1715
|
-
return true;
|
|
1716
|
-
});
|
|
1717
|
-
|
|
1718
|
-
if (closed) break;
|
|
1719
|
-
yield* PubSub.publish(checks, undefined);
|
|
1636
|
+
native === undefined &&
|
|
1637
|
+
recoveryObserved &&
|
|
1638
|
+
!recovery.needsCheckpoint &&
|
|
1639
|
+
backfillObserved &&
|
|
1640
|
+
delivery === undefined &&
|
|
1641
|
+
lanes.every((lane) => lane.fiber === undefined)
|
|
1642
|
+
)
|
|
1643
|
+
break;
|
|
1644
|
+
|
|
1645
|
+
if (native === undefined && nativeCheck) {
|
|
1646
|
+
nativeCheck = false;
|
|
1647
|
+
const checkpoint = recoveryObserved && recovery.needsCheckpoint;
|
|
1648
|
+
|
|
1649
|
+
if (checkpoint) recovery.needsCheckpoint = false;
|
|
1650
|
+
native = yield* fork(
|
|
1651
|
+
Effect.gen(function* () {
|
|
1652
|
+
const awakened = checkpoint ? started : yield* beginNative(observed);
|
|
1653
|
+
|
|
1654
|
+
return yield* advance(
|
|
1655
|
+
awakened,
|
|
1656
|
+
yieldAfter,
|
|
1657
|
+
observed,
|
|
1658
|
+
recovery,
|
|
1659
|
+
!checkpoint || settled === 0,
|
|
1660
|
+
);
|
|
1661
|
+
}),
|
|
1662
|
+
);
|
|
1720
1663
|
}
|
|
1721
|
-
const now = yield* Clock.currentTimeMillis;
|
|
1722
|
-
const until = DateTime.toEpochMillis(yieldAfter);
|
|
1723
|
-
|
|
1724
|
-
if (now >= until) break;
|
|
1725
1664
|
|
|
1726
1665
|
const next = Math.min(
|
|
1727
|
-
result.nextAttemptAt ?? Infinity,
|
|
1666
|
+
native === undefined ? (result.nextAttemptAt ?? Infinity) : Infinity,
|
|
1728
1667
|
now + config.wakeScanInterval,
|
|
1729
1668
|
until,
|
|
1730
1669
|
);
|
|
1731
1670
|
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
retirementDone.pipe(Effect.as("retired" as const)),
|
|
1745
|
-
Deferred.await(changed).pipe(Effect.as("activity" as const)),
|
|
1671
|
+
const completing = [
|
|
1672
|
+
...(native === undefined ? [] : [native]),
|
|
1673
|
+
...(recoveryObserved ? [] : [recoveryFiber]),
|
|
1674
|
+
...(backfillObserved ? [] : [backfill]),
|
|
1675
|
+
...(delivery === undefined ? [] : [delivery]),
|
|
1676
|
+
...lanes.flatMap((lane) => (lane.fiber === undefined ? [] : [lane.fiber])),
|
|
1677
|
+
];
|
|
1678
|
+
|
|
1679
|
+
const ready = yield* Effect.raceAllFirst([
|
|
1680
|
+
...completing.map((fiber) =>
|
|
1681
|
+
Fiber.await<unknown, MaintenancePassFailure>(fiber).pipe(
|
|
1682
|
+
Effect.as("completed" as const),
|
|
1746
1683
|
),
|
|
1747
1684
|
),
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
if ((yield* Clock.currentTimeMillis) >= until) break;
|
|
1753
|
-
if (ready === "native") yield* PubSub.publish(checks, undefined);
|
|
1754
|
-
|
|
1755
|
-
const awakened = yield* beginNative(observed);
|
|
1685
|
+
Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(
|
|
1686
|
+
Effect.as("scan" as const),
|
|
1687
|
+
),
|
|
1688
|
+
]);
|
|
1756
1689
|
|
|
1757
|
-
|
|
1690
|
+
if (ready === "scan") {
|
|
1691
|
+
nativeCheck = true;
|
|
1692
|
+
checkLanes();
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
// Dispatch is closed. Join each admitted wave without renewing its allowance.
|
|
1696
|
+
// Message Claims retain their driver's own timer and local retry commit.
|
|
1697
|
+
if (native !== undefined) {
|
|
1698
|
+
result = yield* Fiber.join(native);
|
|
1758
1699
|
if (result.phase === "actionable") phase = "actionable";
|
|
1759
1700
|
settled += result.settled;
|
|
1760
1701
|
observed.nativeOnly = false;
|
|
1761
|
-
|
|
1702
|
+
recovery.started = true;
|
|
1703
|
+
yield* Deferred.succeed(recovery.queue, []);
|
|
1762
1704
|
}
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1705
|
+
yield* Fiber.joinAll([
|
|
1706
|
+
recoveryFiber,
|
|
1707
|
+
backfill,
|
|
1708
|
+
...(delivery === undefined ? [] : [delivery]),
|
|
1709
|
+
...lanes.flatMap((lane) => (lane.fiber === undefined ? [] : [lane.fiber])),
|
|
1710
|
+
]);
|
|
1711
|
+
if (failure !== undefined) return yield* Effect.failCause(failure);
|
|
1769
1712
|
if (recovery.needsCheckpoint) {
|
|
1770
|
-
// The native yield deadline ended dispatch before old recovery finished. Fold its
|
|
1771
|
-
// control state into acknowledgement without starting an Attempt after retirement.
|
|
1772
1713
|
result = yield* advance(started, yieldAfter, observed, recovery, false);
|
|
1773
1714
|
if (result.phase === "actionable") phase = "actionable";
|
|
1774
1715
|
settled += result.settled;
|