@effect-agent/platform-cloudflare 0.1.0-beta.104 → 0.1.0-beta.107
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Alarm.d.mts +23 -22
- package/dist/Alarm.mjs +131 -108
- package/dist/Alarm.mjs.map +1 -1
- package/dist/CloudflareThreadClient.d.mts +82 -52
- package/dist/{InteractiveBrowser-BR16UPbF.d.mts → InteractiveBrowser-B3S8if51.d.mts} +3 -2
- package/dist/{InteractiveBrowser-BCS9g0tn.mjs → InteractiveBrowser-xVNX7ARl.mjs} +96 -4
- package/dist/InteractiveBrowser-xVNX7ARl.mjs.map +1 -0
- package/dist/InteractiveBrowser.d.mts +1 -1
- package/dist/InteractiveBrowser.mjs +1 -1
- package/dist/ProtectedBrowser.d.mts +1 -1
- package/dist/ProtectedBrowser.mjs +1 -1
- package/dist/{ThreadObject-B66d5rzM.mjs → ThreadObject-BGla_Obk.mjs} +4 -4
- package/dist/ThreadObject-BGla_Obk.mjs.map +1 -0
- package/dist/{ThreadObject-DYMeJA39.d.mts → ThreadObject-CnqCuWlA.d.mts} +38 -36
- package/dist/ThreadObject.d.mts +1 -1
- package/dist/ThreadObject.mjs +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +282 -229
- package/src/InteractiveBrowser.ts +53 -2
- package/src/internal/browser-file-selection.ts +126 -0
- package/src/internal/message-delivery.ts +4 -4
- package/dist/InteractiveBrowser-BCS9g0tn.mjs.map +0 -1
- package/dist/ThreadObject-B66d5rzM.mjs.map +0 -1
package/src/Alarm.ts
CHANGED
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
drainDue,
|
|
31
31
|
type ThreadProjectionError,
|
|
32
32
|
} from "effect-agent/thread-projection-maintenance";
|
|
33
|
+
import { WakeScheduler } from "effect-agent/wake-scheduler";
|
|
33
34
|
import { SqlClient } from "effect/unstable/sql/SqlClient";
|
|
34
35
|
|
|
35
36
|
import { DurableObjectContext } from "./CloudflareBindings.ts";
|
|
@@ -188,7 +189,7 @@ export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
|
|
|
188
189
|
phase: Schema.Literals(["caught-up", "actionable"]),
|
|
189
190
|
/** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
|
|
190
191
|
recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
191
|
-
/**
|
|
192
|
+
/** Head Attempts settled during the event. Joined input may settle with each head. */
|
|
192
193
|
settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
193
194
|
/** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
|
|
194
195
|
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
@@ -212,6 +213,8 @@ export type ThreadMaintenanceFailpointLocation =
|
|
|
212
213
|
| "maintenance:binding-retry:after"
|
|
213
214
|
| "maintenance:retry:before"
|
|
214
215
|
| "maintenance:retry:after"
|
|
216
|
+
| "maintenance:checkpoint:before"
|
|
217
|
+
| "maintenance:checkpoint:after"
|
|
215
218
|
| "maintenance:finish:before"
|
|
216
219
|
| "maintenance:finish:after";
|
|
217
220
|
|
|
@@ -272,7 +275,7 @@ export class ThreadPublication extends Context.Service<
|
|
|
272
275
|
*/
|
|
273
276
|
export const ThreadMessageDelivery = Context.Reference<{
|
|
274
277
|
readonly drainUntil: (
|
|
275
|
-
|
|
278
|
+
dispatchClosed: Effect.Effect<void>,
|
|
276
279
|
dispatchUntil: DateTime.Utc,
|
|
277
280
|
) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
|
|
278
281
|
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
@@ -285,14 +288,14 @@ export const ThreadMessageDelivery = Context.Reference<{
|
|
|
285
288
|
|
|
286
289
|
/**
|
|
287
290
|
* Application obligations sharing this Object's alarm. Admit one initial external wave even on
|
|
288
|
-
* a caught-up pass, then respond to wakes
|
|
289
|
-
*
|
|
290
|
-
* Scope until maintenance tears it down
|
|
291
|
+
* a caught-up pass, then respond to wakes until dispatchClosed. This closes only admission
|
|
292
|
+
* of NEW external waves; native work can continue while admitted waves finish. Keep local
|
|
293
|
+
* admission/hub subscriptions in the event Scope until maintenance tears it down.
|
|
291
294
|
* No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
|
|
292
295
|
*
|
|
293
296
|
* Declare a finite whole-wave allowance (1..300000ms): maximum for parallel lanes, sum for
|
|
294
297
|
* sequential operations. Admit a wave only if its allowance fits before dispatchUntil. Later
|
|
295
|
-
* arrivals cannot renew the retirement window. Maintenance bounds the join after
|
|
298
|
+
* arrivals cannot renew the retirement window. Maintenance bounds the join after dispatchClosed,
|
|
296
299
|
* interrupts and joins event Scope, then reads local deadlines under the mutation gate.
|
|
297
300
|
*
|
|
298
301
|
* Setup and pendingDeadline are bounded local operations. Persist claims/envelopes before
|
|
@@ -304,7 +307,7 @@ export const ThreadMessageDelivery = Context.Reference<{
|
|
|
304
307
|
export const ThreadHostMaintenance = Context.Reference<{
|
|
305
308
|
readonly dispatchTimeoutMillis: number;
|
|
306
309
|
readonly drainUntil: (
|
|
307
|
-
|
|
310
|
+
dispatchClosed: Effect.Effect<void>,
|
|
308
311
|
dispatchUntil: DateTime.Utc,
|
|
309
312
|
) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
|
|
310
313
|
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
@@ -351,6 +354,14 @@ class BindingRetry extends Schema.Class<BindingRetry>("BindingRetry")({
|
|
|
351
354
|
reportedAt: Schema.Finite,
|
|
352
355
|
}) {}
|
|
353
356
|
|
|
357
|
+
interface NativePassResult {
|
|
358
|
+
readonly phase: "caught-up" | "actionable";
|
|
359
|
+
readonly recovered: number;
|
|
360
|
+
readonly settled: number;
|
|
361
|
+
readonly nonterminal: number;
|
|
362
|
+
readonly nextAttemptAt: number | undefined;
|
|
363
|
+
}
|
|
364
|
+
|
|
354
365
|
interface MaintenanceObservation {
|
|
355
366
|
generation?: bigint;
|
|
356
367
|
nativeOnly: boolean;
|
|
@@ -536,20 +547,20 @@ export type MaintenancePassFailure =
|
|
|
536
547
|
/**
|
|
537
548
|
* Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
|
|
538
549
|
*
|
|
539
|
-
*
|
|
550
|
+
* One physical event owns native scheduling, auxiliary delivery and final alarm rearming.
|
|
540
551
|
*
|
|
541
|
-
* 1.
|
|
542
|
-
*
|
|
543
|
-
* 2.
|
|
544
|
-
*
|
|
545
|
-
* 3.
|
|
546
|
-
*
|
|
547
|
-
*
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
* 5.
|
|
552
|
-
*
|
|
552
|
+
* 1. Prearm before any work. A caught-up native step uses the O(1) generation record without
|
|
553
|
+
* recovery, ledger scans or canonical-history reads.
|
|
554
|
+
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
555
|
+
* producer keeps its newer generation dirty. Native retries retain their durable backoff.
|
|
556
|
+
* 3. After the initial native opportunity, stop admitting new external waves and let the
|
|
557
|
+
* active waves finish. While they remain in flight, native wakes and bounded scans can
|
|
558
|
+
* advance more heads. All Attempts share the event's original ten-minute yield deadline.
|
|
559
|
+
* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
|
|
560
|
+
* bounded independently; incoming native work never restarts or cancels their attempts.
|
|
561
|
+
* Auxiliary failures are reported after the current native opportunity.
|
|
562
|
+
* 5. Close every event resource before the final gated deadline snapshot and alarm decision.
|
|
563
|
+
* The whole event retains one fourteen-minute cooperative timeout.
|
|
553
564
|
*/
|
|
554
565
|
export class ThreadMaintenance extends Context.Service<
|
|
555
566
|
ThreadMaintenance,
|
|
@@ -579,6 +590,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
579
590
|
| ThreadProjectionMaintenance
|
|
580
591
|
| DurableAgentRuntime
|
|
581
592
|
| SubmissionLedger
|
|
593
|
+
| WakeScheduler
|
|
582
594
|
| DurableAlarmService
|
|
583
595
|
| ThreadMaintenanceFailpoint
|
|
584
596
|
| CloudflareDurableRuntimeConfig
|
|
@@ -588,6 +600,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
588
600
|
Effect.gen(function* () {
|
|
589
601
|
const runtime = yield* DurableAgentRuntime;
|
|
590
602
|
const ledger = yield* SubmissionLedger;
|
|
603
|
+
const wakes = yield* WakeScheduler;
|
|
591
604
|
const alarm = yield* DurableAlarmService;
|
|
592
605
|
const config = yield* CloudflareDurableRuntimeConfig;
|
|
593
606
|
const { ctx } = yield* DurableObjectContext;
|
|
@@ -745,7 +758,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
745
758
|
ctx.storage.transaction(async (transaction) => {
|
|
746
759
|
const { state } = await readMaintenanceState(transaction);
|
|
747
760
|
|
|
748
|
-
const previous =
|
|
761
|
+
const previous =
|
|
762
|
+
state.retry?.generation === generation && state.retry.nativeOnly === nativeOnly
|
|
763
|
+
? state.retry
|
|
764
|
+
: undefined;
|
|
749
765
|
|
|
750
766
|
const retry = MaintenanceRetry.make({
|
|
751
767
|
generation,
|
|
@@ -784,21 +800,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
784
800
|
yield* failpoint.hit("maintenance:retry:after");
|
|
785
801
|
});
|
|
786
802
|
|
|
787
|
-
const
|
|
788
|
-
yieldAfter: DateTime.Utc,
|
|
789
|
-
dispatchUntil: DateTime.Utc,
|
|
803
|
+
const beginNative = Effect.fn("ThreadMaintenance.beginNative")(function* (
|
|
790
804
|
observed: MaintenanceObservation,
|
|
791
|
-
)
|
|
792
|
-
|
|
793
|
-
Effect.annotateCurrentSpan({
|
|
794
|
-
phase: report.phase,
|
|
795
|
-
recovered: report.recovered,
|
|
796
|
-
settled: report.settled,
|
|
797
|
-
nonterminal: report.nonterminal,
|
|
798
|
-
alarm: report.alarm,
|
|
799
|
-
}).pipe(Effect.as(report));
|
|
800
|
-
|
|
801
|
-
const started = yield* mutations.withSnapshot((activeAtStart) =>
|
|
805
|
+
) {
|
|
806
|
+
return yield* mutations.withSnapshot((activeAtStart) =>
|
|
802
807
|
Effect.gen(function* () {
|
|
803
808
|
const generation = yield* beginPass(observed);
|
|
804
809
|
|
|
@@ -810,76 +815,13 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
810
815
|
return { ...generation, activeAtStart };
|
|
811
816
|
}),
|
|
812
817
|
);
|
|
818
|
+
});
|
|
813
819
|
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
const sourceFinished = yield* Deferred.make<void>();
|
|
821
|
-
const finished = Deferred.await(sourceFinished);
|
|
822
|
-
|
|
823
|
-
// Fork setup too: an ordinary auxiliary setup failure is reported after native work,
|
|
824
|
-
// rather than gating its opportunity. Event interruption still closes every fiber.
|
|
825
|
-
const deliveryFiber = yield* Effect.forkIn(
|
|
826
|
-
Scope.provide(auxiliaryScope)(messages.drainUntil(finished, dispatchUntil)),
|
|
827
|
-
auxiliaryScope,
|
|
828
|
-
);
|
|
829
|
-
|
|
830
|
-
const hostFiber = yield* Effect.forkIn(
|
|
831
|
-
Scope.provide(auxiliaryScope)(
|
|
832
|
-
Effect.gen(function* () {
|
|
833
|
-
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(
|
|
834
|
-
Effect.mapError((cause) =>
|
|
835
|
-
DurableAlarmError.make({
|
|
836
|
-
operation: "host dispatch allowance",
|
|
837
|
-
message:
|
|
838
|
-
"Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
|
|
839
|
-
cause,
|
|
840
|
-
}),
|
|
841
|
-
),
|
|
842
|
-
);
|
|
843
|
-
yield* host.drainUntil(finished, dispatchUntil);
|
|
844
|
-
}),
|
|
845
|
-
),
|
|
846
|
-
auxiliaryScope,
|
|
847
|
-
);
|
|
848
|
-
|
|
849
|
-
// Backfill is one disposable wave; its timer starts beside native execution.
|
|
850
|
-
const backfill = yield* Effect.forkIn(
|
|
851
|
-
drainDue.pipe(
|
|
852
|
-
Effect.provideService(ThreadProjectionMaintenance, projection),
|
|
853
|
-
Effect.timeoutOption(config.projectionDispatchTimeoutMillis),
|
|
854
|
-
),
|
|
855
|
-
auxiliaryScope,
|
|
856
|
-
);
|
|
857
|
-
|
|
858
|
-
const finishAuxiliary = Effect.gen(function* () {
|
|
859
|
-
yield* Deferred.succeed(sourceFinished, undefined);
|
|
860
|
-
|
|
861
|
-
const remaining = Math.max(
|
|
862
|
-
1,
|
|
863
|
-
DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis),
|
|
864
|
-
);
|
|
865
|
-
|
|
866
|
-
const hostJoin = yield* Effect.forkIn(
|
|
867
|
-
Fiber.join(hostFiber).pipe(
|
|
868
|
-
Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)),
|
|
869
|
-
Effect.tap((result) =>
|
|
870
|
-
Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(result) }),
|
|
871
|
-
),
|
|
872
|
-
),
|
|
873
|
-
auxiliaryScope,
|
|
874
|
-
);
|
|
875
|
-
|
|
876
|
-
// Native transport uses the driver's actual Claim deadline, through Retry persistence.
|
|
877
|
-
// Only our host/backfill timers suppress their own interruption. Earlier failed exits,
|
|
878
|
-
// including pure self-interruption, remain failures while siblings are still blocked.
|
|
879
|
-
yield* Fiber.joinAll([deliveryFiber, hostJoin, backfill]);
|
|
880
|
-
yield* Scope.close(auxiliaryScope, Exit.void);
|
|
881
|
-
});
|
|
882
|
-
|
|
820
|
+
const advance = Effect.fn("ThreadMaintenance.advance")(function* (
|
|
821
|
+
started: Effect.Success<ReturnType<typeof beginNative>>,
|
|
822
|
+
yieldAfter: DateTime.Utc,
|
|
823
|
+
observed: MaintenanceObservation,
|
|
824
|
+
): Effect.fn.Return<NativePassResult, MaintenancePassFailure> {
|
|
883
825
|
const deadline = yield* publication.pendingDeadline;
|
|
884
826
|
|
|
885
827
|
if (
|
|
@@ -891,55 +833,34 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
891
833
|
const pending = yield* publication.pendingDeadline;
|
|
892
834
|
|
|
893
835
|
if (started._tag === "CaughtUp" || Option.isSome(pending)) {
|
|
894
|
-
yield*
|
|
895
|
-
yield* failpoint.hit("maintenance:finish:before");
|
|
896
|
-
|
|
897
|
-
const disposition = yield* mutations.withSnapshot((active) =>
|
|
836
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) =>
|
|
898
837
|
Effect.gen(function* () {
|
|
899
|
-
// Re-read under the producer gate: a concurrent append/host mutation cannot be
|
|
900
|
-
// cleared using a stale empty deadline. Dirty generations bound all producer races.
|
|
901
|
-
const latest = yield* pendingDeadline;
|
|
902
838
|
const now = yield* Clock.currentTimeMillis;
|
|
903
839
|
|
|
904
|
-
|
|
905
|
-
ctx.storage.transaction(
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
const nativeDeadline =
|
|
909
|
-
active > 0 || state.dirty > state.processed
|
|
910
|
-
? state.retry?.generation === state.dirty && active === 0
|
|
911
|
-
? Math.max(now + minimumAlarmDelay, state.retry.notBefore)
|
|
912
|
-
: now + config.wakeScanInterval
|
|
913
|
-
: Infinity;
|
|
914
|
-
|
|
915
|
-
const next = Option.isSome(latest)
|
|
916
|
-
? Math.min(nativeDeadline, latest.value)
|
|
917
|
-
: nativeDeadline;
|
|
840
|
+
const { state } = yield* runTransaction("read native maintenance deadline", () =>
|
|
841
|
+
ctx.storage.transaction((transaction) => readMaintenanceState(transaction)),
|
|
842
|
+
);
|
|
918
843
|
|
|
919
|
-
|
|
920
|
-
|
|
844
|
+
const native =
|
|
845
|
+
active > 0 || state.dirty > state.processed
|
|
846
|
+
? state.retry?.generation === state.dirty && active === 0
|
|
847
|
+
? state.retry.notBefore
|
|
848
|
+
: now + minimumAlarmDelay
|
|
849
|
+
: Infinity;
|
|
921
850
|
|
|
922
|
-
|
|
923
|
-
}
|
|
924
|
-
await transaction.deleteAlarm();
|
|
851
|
+
const next = Option.isSome(pending) ? pending.value : native;
|
|
925
852
|
|
|
926
|
-
|
|
927
|
-
}),
|
|
928
|
-
);
|
|
853
|
+
return Number.isFinite(next) ? Math.max(now + minimumAlarmDelay, next) : undefined;
|
|
929
854
|
}),
|
|
930
855
|
);
|
|
931
856
|
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
nonterminal: started.nonterminal,
|
|
940
|
-
alarm: disposition,
|
|
941
|
-
}),
|
|
942
|
-
);
|
|
857
|
+
return {
|
|
858
|
+
phase: "caught-up",
|
|
859
|
+
recovered: 0,
|
|
860
|
+
settled: 0,
|
|
861
|
+
nonterminal: started.nonterminal,
|
|
862
|
+
nextAttemptAt,
|
|
863
|
+
};
|
|
943
864
|
}
|
|
944
865
|
// Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
|
|
945
866
|
observed.nativeOnly = true;
|
|
@@ -1006,7 +927,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1006
927
|
let retries = selection.retries;
|
|
1007
928
|
let bindingFailure: DurableBindingFailure | undefined;
|
|
1008
929
|
|
|
1009
|
-
// One runnable FIFO head per
|
|
930
|
+
// One runnable FIFO head per native opportunity. An absent agent waits for a deployment,
|
|
1010
931
|
// including for children; other local lanes and host deliveries remain independently due.
|
|
1011
932
|
const settlement =
|
|
1012
933
|
selected === undefined
|
|
@@ -1041,7 +962,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1041
962
|
if (retry !== undefined || previous !== undefined) {
|
|
1042
963
|
// The Attempt released its Claim. Commit its binding wait (or clear) once,
|
|
1043
964
|
// before joining fallible auxiliary work. This local fact neither acknowledges
|
|
1044
|
-
// a generation nor changes the shared alarm
|
|
965
|
+
// a generation nor changes the shared alarm.
|
|
1045
966
|
yield* failpoint.hit("maintenance:binding-retry:before");
|
|
1046
967
|
retries = yield* runTransaction("record submission binding retry", () =>
|
|
1047
968
|
ctx.storage.transaction(async (transaction) => {
|
|
@@ -1074,8 +995,6 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1074
995
|
}
|
|
1075
996
|
}
|
|
1076
997
|
|
|
1077
|
-
observed.nativeOnly = false;
|
|
1078
|
-
yield* finishAuxiliary;
|
|
1079
998
|
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
1080
999
|
const waitingHeads = new Map<ThreadId, boolean>();
|
|
1081
1000
|
|
|
@@ -1115,87 +1034,213 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1115
1034
|
|
|
1116
1035
|
const delay = Math.max(ordinaryDelay, bindingDelay);
|
|
1117
1036
|
|
|
1037
|
+
// Checkpoint native progress without changing the physical alarm. Auxiliary
|
|
1038
|
+
// delivery remains live; later mutations still advance the shared generation.
|
|
1039
|
+
yield* failpoint.hit("maintenance:checkpoint:before");
|
|
1040
|
+
|
|
1041
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) =>
|
|
1042
|
+
runTransaction("checkpoint native maintenance", () =>
|
|
1043
|
+
ctx.storage.transaction(async (transaction) => {
|
|
1044
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1045
|
+
|
|
1046
|
+
const processed =
|
|
1047
|
+
autonomous || started.activeAtStart > 0 || active > 0
|
|
1048
|
+
? state.processed
|
|
1049
|
+
: state.processed > started.generation
|
|
1050
|
+
? state.processed
|
|
1051
|
+
: started.generation;
|
|
1052
|
+
|
|
1053
|
+
const next = ThreadMaintenanceState.make({
|
|
1054
|
+
...Struct.omit(state, ["retry"]),
|
|
1055
|
+
processed,
|
|
1056
|
+
nonterminal: remaining.length,
|
|
1057
|
+
bindingRetries: (state.bindingRetries ?? []).filter((retry) =>
|
|
1058
|
+
remaining.some((row) => row.submissionId === retry.submissionId),
|
|
1059
|
+
),
|
|
1060
|
+
...(autonomous && !progressed
|
|
1061
|
+
? {
|
|
1062
|
+
retry: MaintenanceRetry.make({
|
|
1063
|
+
generation: started.generation,
|
|
1064
|
+
notBefore: now + delay,
|
|
1065
|
+
nativeOnly: true,
|
|
1066
|
+
stalls: Math.min(30, started.stalls + 1),
|
|
1067
|
+
}),
|
|
1068
|
+
}
|
|
1069
|
+
: {}),
|
|
1070
|
+
});
|
|
1071
|
+
|
|
1072
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
1073
|
+
if (autonomous) {
|
|
1074
|
+
return started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation
|
|
1075
|
+
? now + minimumAlarmDelay
|
|
1076
|
+
: now + delay;
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
return started.activeAtStart > 0 || active > 0 || next.dirty > next.processed
|
|
1080
|
+
? now + minimumAlarmDelay
|
|
1081
|
+
: undefined;
|
|
1082
|
+
}),
|
|
1083
|
+
),
|
|
1084
|
+
);
|
|
1085
|
+
|
|
1086
|
+
yield* failpoint.hit("maintenance:checkpoint:after");
|
|
1087
|
+
|
|
1088
|
+
return {
|
|
1089
|
+
phase: "actionable",
|
|
1090
|
+
recovered: recovered.length,
|
|
1091
|
+
settled: Option.isSome(settlement) ? 1 : 0,
|
|
1092
|
+
nonterminal: remaining.length,
|
|
1093
|
+
nextAttemptAt,
|
|
1094
|
+
};
|
|
1095
|
+
});
|
|
1096
|
+
|
|
1097
|
+
const pass = Effect.fn("ThreadMaintenance.pass")(function* (
|
|
1098
|
+
yieldAfter: DateTime.Utc,
|
|
1099
|
+
dispatchUntil: DateTime.Utc,
|
|
1100
|
+
observed: MaintenanceObservation,
|
|
1101
|
+
): Effect.fn.Return<MaintenancePassReport, MaintenancePassFailure, Scope.Scope> {
|
|
1102
|
+
// Subscribe before the first native snapshot. Hints accelerate rechecks; the
|
|
1103
|
+
// bounded scan and durable generation still recover dropped notifications.
|
|
1104
|
+
const notified = (yield* Stream.toPull(wakes.wakes)).pipe(
|
|
1105
|
+
Effect.asVoid,
|
|
1106
|
+
Effect.catch(() => Effect.never),
|
|
1107
|
+
);
|
|
1108
|
+
|
|
1109
|
+
let started = yield* beginNative(observed);
|
|
1110
|
+
|
|
1111
|
+
// This scope owns auxiliary dispatch and listeners, independently of native progress.
|
|
1112
|
+
// Close it before final alarm rearming, including on failure or event interruption.
|
|
1113
|
+
const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) =>
|
|
1114
|
+
Scope.close(scope, exit),
|
|
1115
|
+
);
|
|
1116
|
+
|
|
1117
|
+
const dispatchClosed = yield* Deferred.make<void>();
|
|
1118
|
+
const stopDispatch = Deferred.await(dispatchClosed);
|
|
1119
|
+
|
|
1120
|
+
// Fork setup too: an ordinary auxiliary setup failure is reported after native work,
|
|
1121
|
+
// rather than gating its opportunity. Event interruption still closes every fiber.
|
|
1122
|
+
const deliveryFiber = yield* Effect.forkIn(
|
|
1123
|
+
Scope.provide(auxiliaryScope)(messages.drainUntil(stopDispatch, dispatchUntil)),
|
|
1124
|
+
auxiliaryScope,
|
|
1125
|
+
);
|
|
1126
|
+
|
|
1127
|
+
const hostFiber = yield* Effect.forkIn(
|
|
1128
|
+
Scope.provide(auxiliaryScope)(
|
|
1129
|
+
Effect.gen(function* () {
|
|
1130
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(
|
|
1131
|
+
Effect.mapError((cause) =>
|
|
1132
|
+
DurableAlarmError.make({
|
|
1133
|
+
operation: "host dispatch allowance",
|
|
1134
|
+
message:
|
|
1135
|
+
"Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
|
|
1136
|
+
cause,
|
|
1137
|
+
}),
|
|
1138
|
+
),
|
|
1139
|
+
);
|
|
1140
|
+
yield* host.drainUntil(stopDispatch, dispatchUntil);
|
|
1141
|
+
}),
|
|
1142
|
+
),
|
|
1143
|
+
auxiliaryScope,
|
|
1144
|
+
);
|
|
1145
|
+
|
|
1146
|
+
// Backfill is one disposable wave; its timer starts beside native execution.
|
|
1147
|
+
const backfill = yield* Effect.forkIn(
|
|
1148
|
+
drainDue.pipe(
|
|
1149
|
+
Effect.provideService(ThreadProjectionMaintenance, projection),
|
|
1150
|
+
Effect.timeoutOption(config.projectionDispatchTimeoutMillis),
|
|
1151
|
+
),
|
|
1152
|
+
auxiliaryScope,
|
|
1153
|
+
);
|
|
1154
|
+
|
|
1155
|
+
let result = yield* advance(started, yieldAfter, observed);
|
|
1156
|
+
let phase = result.phase;
|
|
1157
|
+
let recovered = result.recovered;
|
|
1158
|
+
let settled = result.settled;
|
|
1159
|
+
|
|
1160
|
+
observed.nativeOnly = false;
|
|
1161
|
+
|
|
1162
|
+
// Close admission of new delivery waves once, then keep advancing native
|
|
1163
|
+
// work while the already-admitted waves finish. Neither lane restarts the
|
|
1164
|
+
// other's work or receives a fresh event budget.
|
|
1165
|
+
yield* Deferred.succeed(dispatchClosed, undefined);
|
|
1166
|
+
|
|
1167
|
+
const remaining = Math.max(
|
|
1168
|
+
1,
|
|
1169
|
+
DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis),
|
|
1170
|
+
);
|
|
1171
|
+
|
|
1172
|
+
const hostJoin = yield* Effect.forkIn(
|
|
1173
|
+
Fiber.join(hostFiber).pipe(
|
|
1174
|
+
Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)),
|
|
1175
|
+
Effect.tap((outcome) =>
|
|
1176
|
+
Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) }),
|
|
1177
|
+
),
|
|
1178
|
+
),
|
|
1179
|
+
auxiliaryScope,
|
|
1180
|
+
);
|
|
1181
|
+
|
|
1182
|
+
const retired = yield* Effect.forkChild(Fiber.joinAll([deliveryFiber, hostJoin, backfill]));
|
|
1183
|
+
|
|
1184
|
+
const auxiliaryPending = () =>
|
|
1185
|
+
deliveryFiber.pollUnsafe() === undefined ||
|
|
1186
|
+
hostFiber.pollUnsafe() === undefined ||
|
|
1187
|
+
backfill.pollUnsafe() === undefined;
|
|
1188
|
+
|
|
1189
|
+
while (retired.pollUnsafe() === undefined && auxiliaryPending()) {
|
|
1190
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1191
|
+
const until = DateTime.toEpochMillis(yieldAfter);
|
|
1192
|
+
|
|
1193
|
+
if (now >= until) break;
|
|
1194
|
+
|
|
1195
|
+
const next = Math.min(
|
|
1196
|
+
result.nextAttemptAt ?? Infinity,
|
|
1197
|
+
now + config.wakeScanInterval,
|
|
1198
|
+
until,
|
|
1199
|
+
);
|
|
1200
|
+
|
|
1201
|
+
const ready = yield* Effect.raceFirst(
|
|
1202
|
+
Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as(true)),
|
|
1203
|
+
Fiber.await(retired).pipe(Effect.as(false)),
|
|
1204
|
+
);
|
|
1205
|
+
|
|
1206
|
+
if (!ready || retired.pollUnsafe() !== undefined || !auxiliaryPending()) break;
|
|
1207
|
+
if ((yield* Clock.currentTimeMillis) >= until) break;
|
|
1208
|
+
|
|
1209
|
+
started = yield* beginNative(observed);
|
|
1210
|
+
result = yield* advance(started, yieldAfter, observed);
|
|
1211
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
1212
|
+
recovered += result.recovered;
|
|
1213
|
+
settled += result.settled;
|
|
1214
|
+
observed.nativeOnly = false;
|
|
1215
|
+
}
|
|
1216
|
+
// Preserve driver-owned Claim deadlines and failures, then close every
|
|
1217
|
+
// listener before the one final alarm decision.
|
|
1218
|
+
yield* Fiber.join(retired);
|
|
1219
|
+
yield* Scope.close(auxiliaryScope, Exit.void);
|
|
1118
1220
|
yield* failpoint.hit("maintenance:finish:before");
|
|
1119
1221
|
|
|
1120
|
-
const
|
|
1222
|
+
const disposition = yield* mutations.withSnapshot((active) =>
|
|
1121
1223
|
Effect.gen(function* () {
|
|
1122
|
-
const
|
|
1224
|
+
const latest = yield* pendingDeadline;
|
|
1225
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1123
1226
|
|
|
1124
|
-
return yield* runTransaction("finish maintenance
|
|
1227
|
+
return yield* runTransaction("finish maintenance event", () =>
|
|
1125
1228
|
ctx.storage.transaction(async (transaction) => {
|
|
1126
1229
|
const { state } = await readMaintenanceState(transaction);
|
|
1127
1230
|
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
const next = ThreadMaintenanceState.make({
|
|
1138
|
-
...Struct.omit(state, ["retry"]),
|
|
1139
|
-
processed,
|
|
1140
|
-
nonterminal: remaining.length,
|
|
1141
|
-
bindingRetries: (state.bindingRetries ?? []).filter((retry) =>
|
|
1142
|
-
remaining.some((row) => row.submissionId === retry.submissionId),
|
|
1143
|
-
),
|
|
1144
|
-
...(autonomous && !progressed
|
|
1145
|
-
? {
|
|
1146
|
-
retry: MaintenanceRetry.make({
|
|
1147
|
-
generation: started.generation,
|
|
1148
|
-
notBefore: now + delay,
|
|
1149
|
-
nativeOnly: true,
|
|
1150
|
-
stalls: Math.min(30, started.stalls + 1),
|
|
1151
|
-
}),
|
|
1152
|
-
}
|
|
1153
|
-
: {}),
|
|
1154
|
-
});
|
|
1155
|
-
|
|
1156
|
-
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
1157
|
-
if (autonomous) {
|
|
1158
|
-
// Replace the crash-fallback slot with this pass's bounded backoff. The target
|
|
1159
|
-
// is never earlier than the begin-pass fallback, so workerd does not cancel
|
|
1160
|
-
// this running alarm handler before its report/span can complete.
|
|
1161
|
-
const nativeDeadline =
|
|
1162
|
-
started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation
|
|
1163
|
-
? now + minimumAlarmDelay
|
|
1164
|
-
: now + delay;
|
|
1165
|
-
|
|
1166
|
-
await transaction.setAlarm(
|
|
1167
|
-
Option.isSome(publicationDeadline)
|
|
1168
|
-
? Math.max(
|
|
1169
|
-
now + minimumAlarmDelay,
|
|
1170
|
-
Math.min(nativeDeadline, publicationDeadline.value),
|
|
1171
|
-
)
|
|
1172
|
-
: nativeDeadline,
|
|
1173
|
-
);
|
|
1231
|
+
const native =
|
|
1232
|
+
active > 0 || state.dirty > state.processed
|
|
1233
|
+
? state.retry?.generation === state.dirty && active === 0
|
|
1234
|
+
? Math.max(now + minimumAlarmDelay, state.retry.notBefore)
|
|
1235
|
+
: state.dirty === observed.generation && active === 0
|
|
1236
|
+
? (result.nextAttemptAt ?? now + config.wakeScanInterval)
|
|
1237
|
+
: now + minimumAlarmDelay
|
|
1238
|
+
: Infinity;
|
|
1174
1239
|
|
|
1175
|
-
|
|
1176
|
-
}
|
|
1177
|
-
if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
|
|
1178
|
-
// A mutation overlapped this pass's observation window or raced
|
|
1179
|
-
// acknowledgement. It stays dirty and its pre-armed bounded alarm survives;
|
|
1180
|
-
// unseen effects are never acknowledged. Do not accelerate that future alarm
|
|
1181
|
-
// from inside the current handler: workerd cancels a running handler when it
|
|
1182
|
-
// writes an earlier slot.
|
|
1183
|
-
await ensureTransactionAlarmBy(
|
|
1184
|
-
transaction,
|
|
1185
|
-
Option.isSome(publicationDeadline)
|
|
1186
|
-
? Math.max(
|
|
1187
|
-
now + minimumAlarmDelay,
|
|
1188
|
-
Math.min(now + config.wakeScanInterval, publicationDeadline.value),
|
|
1189
|
-
)
|
|
1190
|
-
: now + config.wakeScanInterval,
|
|
1191
|
-
);
|
|
1240
|
+
const next = Option.isSome(latest) ? Math.min(native, latest.value) : native;
|
|
1192
1241
|
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
if (Option.isSome(publicationDeadline)) {
|
|
1196
|
-
await transaction.setAlarm(
|
|
1197
|
-
Math.max(now + minimumAlarmDelay, publicationDeadline.value),
|
|
1198
|
-
);
|
|
1242
|
+
if (Number.isFinite(next)) {
|
|
1243
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
|
|
1199
1244
|
|
|
1200
1245
|
return "rearmed" as const;
|
|
1201
1246
|
}
|
|
@@ -1209,15 +1254,23 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1209
1254
|
|
|
1210
1255
|
yield* failpoint.hit("maintenance:finish:after");
|
|
1211
1256
|
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1257
|
+
const report = MaintenancePassReport.make({
|
|
1258
|
+
phase,
|
|
1259
|
+
recovered,
|
|
1260
|
+
settled,
|
|
1261
|
+
nonterminal: result.nonterminal,
|
|
1262
|
+
alarm: disposition,
|
|
1263
|
+
});
|
|
1264
|
+
|
|
1265
|
+
yield* Effect.annotateCurrentSpan({
|
|
1266
|
+
phase: report.phase,
|
|
1267
|
+
recovered: report.recovered,
|
|
1268
|
+
settled: report.settled,
|
|
1269
|
+
nonterminal: report.nonterminal,
|
|
1270
|
+
alarm: report.alarm,
|
|
1271
|
+
});
|
|
1272
|
+
|
|
1273
|
+
return report;
|
|
1221
1274
|
});
|
|
1222
1275
|
|
|
1223
1276
|
return ThreadMaintenance.of({
|