@effect-agent/platform-cloudflare 0.1.0-beta.103 → 0.1.0-beta.106
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Alarm.d.mts +23 -22
- package/dist/Alarm.mjs +131 -108
- package/dist/Alarm.mjs.map +1 -1
- package/dist/CloudflareThreadClient.d.mts +101 -71
- package/dist/{ThreadObject-B66d5rzM.mjs → ThreadObject-BGla_Obk.mjs} +4 -4
- package/dist/ThreadObject-BGla_Obk.mjs.map +1 -0
- package/dist/{ThreadObject-CMQiW2uX.d.mts → ThreadObject-CLBMWEPn.d.mts} +33 -31
- package/dist/ThreadObject.d.mts +1 -1
- package/dist/ThreadObject.mjs +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +282 -229
- package/src/internal/message-delivery.ts +4 -4
- package/dist/ThreadObject-B66d5rzM.mjs.map +0 -1
package/dist/Alarm.d.mts
CHANGED
|
@@ -5,6 +5,7 @@ import { DurableBindingFailure } from "effect-agent/agent-registration";
|
|
|
5
5
|
import { DurableAgentRuntime, DurableWorkerFailure } from "effect-agent/durable-agent-runtime";
|
|
6
6
|
import { SubmissionLedger } from "effect-agent/submission-ledger";
|
|
7
7
|
import { ThreadProjectionError, ThreadProjectionMaintenance } from "effect-agent/thread-projection-maintenance";
|
|
8
|
+
import { WakeScheduler } from "effect-agent/wake-scheduler";
|
|
8
9
|
import { SqlClient } from "effect/unstable/sql/SqlClient";
|
|
9
10
|
declare namespace Alarm_d_exports {
|
|
10
11
|
export { DurableAlarmError, DurableAlarmService, MaintenancePassFailure, MaintenancePassReport, ThreadHostMaintenance, ThreadMaintenance, ThreadMaintenanceFailpoint, ThreadMaintenanceFailpointHandler, ThreadMaintenanceFailpointLocation, ThreadMessageDelivery, ThreadMutationGate, ThreadPublication, ThreadPublicationService, publishCommitted };
|
|
@@ -65,7 +66,7 @@ declare const MaintenancePassReport_base: Schema.Class<MaintenancePassReport, Sc
|
|
|
65
66
|
readonly phase: Schema.Literals<readonly ["caught-up", "actionable"]>;
|
|
66
67
|
/** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
|
|
67
68
|
readonly recovered: Schema.Int;
|
|
68
|
-
/**
|
|
69
|
+
/** Head Attempts settled during the event. Joined input may settle with each head. */
|
|
69
70
|
readonly settled: Schema.Int;
|
|
70
71
|
/** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
|
|
71
72
|
readonly nonterminal: Schema.Int;
|
|
@@ -75,7 +76,7 @@ declare const MaintenancePassReport_base: Schema.Class<MaintenancePassReport, Sc
|
|
|
75
76
|
/** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
|
|
76
77
|
export declare class MaintenancePassReport extends MaintenancePassReport_base {}
|
|
77
78
|
/** Fault boundaries around every maintenance-owned durable mutation. */
|
|
78
|
-
export type ThreadMaintenanceFailpointLocation = "maintenance:dirty:before" | "maintenance:dirty:after" | "maintenance:mutation:armed" | "maintenance:mutation:finished" | "maintenance:ensure:before" | "maintenance:ensure:after" | "maintenance:begin:before" | "maintenance:begin:after" | "maintenance:select:before" | "maintenance:select:after" | "maintenance:binding-retry:before" | "maintenance:binding-retry:after" | "maintenance:retry:before" | "maintenance:retry:after" | "maintenance:finish:before" | "maintenance:finish:after";
|
|
79
|
+
export type ThreadMaintenanceFailpointLocation = "maintenance:dirty:before" | "maintenance:dirty:after" | "maintenance:mutation:armed" | "maintenance:mutation:finished" | "maintenance:ensure:before" | "maintenance:ensure:after" | "maintenance:begin:before" | "maintenance:begin:after" | "maintenance:select:before" | "maintenance:select:after" | "maintenance:binding-retry:before" | "maintenance:binding-retry:after" | "maintenance:retry:before" | "maintenance:retry:after" | "maintenance:checkpoint:before" | "maintenance:checkpoint:after" | "maintenance:finish:before" | "maintenance:finish:after";
|
|
79
80
|
export type ThreadMaintenanceFailpointHandler = (location: ThreadMaintenanceFailpointLocation) => Effect.Effect<void>;
|
|
80
81
|
declare const ThreadMaintenanceFailpoint_base: Context.ServiceClass<ThreadMaintenanceFailpoint, "@effect-agent/platform-cloudflare/ThreadMaintenanceFailpoint", {
|
|
81
82
|
readonly hit: ThreadMaintenanceFailpointHandler;
|
|
@@ -117,19 +118,19 @@ export declare class ThreadPublication extends ThreadPublication_base {
|
|
|
117
118
|
* local Claim setup may take time, and expiration still owes the driver's local retry commit.
|
|
118
119
|
*/
|
|
119
120
|
export declare const ThreadMessageDelivery: Context.Reference<{
|
|
120
|
-
readonly drainUntil: (
|
|
121
|
+
readonly drainUntil: (dispatchClosed: Effect.Effect<void>, dispatchUntil: DateTime.Utc) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
|
|
121
122
|
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
122
123
|
}>;
|
|
123
124
|
/**
|
|
124
125
|
* Application obligations sharing this Object's alarm. Admit one initial external wave even on
|
|
125
|
-
* a caught-up pass, then respond to wakes
|
|
126
|
-
*
|
|
127
|
-
* Scope until maintenance tears it down
|
|
126
|
+
* a caught-up pass, then respond to wakes until dispatchClosed. This closes only admission
|
|
127
|
+
* of NEW external waves; native work can continue while admitted waves finish. Keep local
|
|
128
|
+
* admission/hub subscriptions in the event Scope until maintenance tears it down.
|
|
128
129
|
* No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
|
|
129
130
|
*
|
|
130
131
|
* Declare a finite whole-wave allowance (1..300000ms): maximum for parallel lanes, sum for
|
|
131
132
|
* sequential operations. Admit a wave only if its allowance fits before dispatchUntil. Later
|
|
132
|
-
* arrivals cannot renew the retirement window. Maintenance bounds the join after
|
|
133
|
+
* arrivals cannot renew the retirement window. Maintenance bounds the join after dispatchClosed,
|
|
133
134
|
* interrupts and joins event Scope, then reads local deadlines under the mutation gate.
|
|
134
135
|
*
|
|
135
136
|
* Setup and pendingDeadline are bounded local operations. Persist claims/envelopes before
|
|
@@ -140,7 +141,7 @@ export declare const ThreadMessageDelivery: Context.Reference<{
|
|
|
140
141
|
*/
|
|
141
142
|
export declare const ThreadHostMaintenance: Context.Reference<{
|
|
142
143
|
readonly dispatchTimeoutMillis: number;
|
|
143
|
-
readonly drainUntil: (
|
|
144
|
+
readonly drainUntil: (dispatchClosed: Effect.Effect<void>, dispatchUntil: DateTime.Utc) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
|
|
144
145
|
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
145
146
|
}>;
|
|
146
147
|
/** @internal A committed source operation must not become a failed operation because delivery failed. */
|
|
@@ -184,23 +185,23 @@ declare const ThreadMaintenance_base: Context.ServiceClass<ThreadMaintenance, "@
|
|
|
184
185
|
/**
|
|
185
186
|
* Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
|
|
186
187
|
*
|
|
187
|
-
*
|
|
188
|
+
* One physical event owns native scheduling, auxiliary delivery and final alarm rearming.
|
|
188
189
|
*
|
|
189
|
-
* 1.
|
|
190
|
-
*
|
|
191
|
-
* 2.
|
|
192
|
-
*
|
|
193
|
-
* 3.
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
*
|
|
199
|
-
* 5.
|
|
200
|
-
*
|
|
190
|
+
* 1. Prearm before any work. A caught-up native step uses the O(1) generation record without
|
|
191
|
+
* recovery, ledger scans or canonical-history reads.
|
|
192
|
+
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
193
|
+
* producer keeps its newer generation dirty. Native retries retain their durable backoff.
|
|
194
|
+
* 3. After the initial native opportunity, stop admitting new external waves and let the
|
|
195
|
+
* active waves finish. While they remain in flight, native wakes and bounded scans can
|
|
196
|
+
* advance more heads. All Attempts share the event's original ten-minute yield deadline.
|
|
197
|
+
* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
|
|
198
|
+
* bounded independently; incoming native work never restarts or cancels their attempts.
|
|
199
|
+
* Auxiliary failures are reported after the current native opportunity.
|
|
200
|
+
* 5. Close every event resource before the final gated deadline snapshot and alarm decision.
|
|
201
|
+
* The whole event retains one fourteen-minute cooperative timeout.
|
|
201
202
|
*/
|
|
202
203
|
export declare class ThreadMaintenance extends ThreadMaintenance_base {
|
|
203
|
-
static readonly layer: Layer.Layer<ThreadMaintenance, never, ThreadMutationGate | ThreadPublication | ThreadProjectionMaintenance | DurableAgentRuntime | SubmissionLedger | DurableAlarmService | ThreadMaintenanceFailpoint | CloudflareDurableRuntimeConfig | DurableObjectContext | SqlClient>;
|
|
204
|
+
static readonly layer: Layer.Layer<ThreadMaintenance, never, ThreadMutationGate | ThreadPublication | ThreadProjectionMaintenance | DurableAgentRuntime | SubmissionLedger | WakeScheduler | DurableAlarmService | ThreadMaintenanceFailpoint | CloudflareDurableRuntimeConfig | DurableObjectContext | SqlClient>;
|
|
204
205
|
}
|
|
205
206
|
//#endregion
|
|
206
207
|
export { Alarm_d_exports as t };
|
package/dist/Alarm.mjs
CHANGED
|
@@ -8,6 +8,7 @@ import { DurableAgentRuntime } from "effect-agent/durable-agent-runtime";
|
|
|
8
8
|
import { SubmissionId, ThreadId } from "effect-agent/identifiers";
|
|
9
9
|
import { SubmissionLedger } from "effect-agent/submission-ledger";
|
|
10
10
|
import { ThreadProjectionMaintenance, drainDue } from "effect-agent/thread-projection-maintenance";
|
|
11
|
+
import { WakeScheduler } from "effect-agent/wake-scheduler";
|
|
11
12
|
import { SqlClient } from "effect/unstable/sql/SqlClient";
|
|
12
13
|
//#region src/Alarm.ts
|
|
13
14
|
var Alarm_exports = /* @__PURE__ */ __exportAll({
|
|
@@ -88,7 +89,7 @@ var MaintenancePassReport = class extends Schema.Class("@effect-agent/platform-c
|
|
|
88
89
|
phase: Schema.Literals(["caught-up", "actionable"]),
|
|
89
90
|
/** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
|
|
90
91
|
recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
91
|
-
/**
|
|
92
|
+
/** Head Attempts settled during the event. Joined input may settle with each head. */
|
|
92
93
|
settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
93
94
|
/** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
|
|
94
95
|
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
@@ -119,14 +120,14 @@ const ThreadMessageDelivery = Context.Reference("@effect-agent/platform-cloudfla
|
|
|
119
120
|
}) });
|
|
120
121
|
/**
|
|
121
122
|
* Application obligations sharing this Object's alarm. Admit one initial external wave even on
|
|
122
|
-
* a caught-up pass, then respond to wakes
|
|
123
|
-
*
|
|
124
|
-
* Scope until maintenance tears it down
|
|
123
|
+
* a caught-up pass, then respond to wakes until dispatchClosed. This closes only admission
|
|
124
|
+
* of NEW external waves; native work can continue while admitted waves finish. Keep local
|
|
125
|
+
* admission/hub subscriptions in the event Scope until maintenance tears it down.
|
|
125
126
|
* No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
|
|
126
127
|
*
|
|
127
128
|
* Declare a finite whole-wave allowance (1..300000ms): maximum for parallel lanes, sum for
|
|
128
129
|
* sequential operations. Admit a wave only if its allowance fits before dispatchUntil. Later
|
|
129
|
-
* arrivals cannot renew the retirement window. Maintenance bounds the join after
|
|
130
|
+
* arrivals cannot renew the retirement window. Maintenance bounds the join after dispatchClosed,
|
|
130
131
|
* interrupts and joins event Scope, then reads local deadlines under the mutation gate.
|
|
131
132
|
*
|
|
132
133
|
* Setup and pendingDeadline are bounded local operations. Persist claims/envelopes before
|
|
@@ -251,25 +252,26 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
|
|
|
251
252
|
/**
|
|
252
253
|
* Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
|
|
253
254
|
*
|
|
254
|
-
*
|
|
255
|
+
* One physical event owns native scheduling, auxiliary delivery and final alarm rearming.
|
|
255
256
|
*
|
|
256
|
-
* 1.
|
|
257
|
-
*
|
|
258
|
-
* 2.
|
|
259
|
-
*
|
|
260
|
-
* 3.
|
|
261
|
-
*
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
* 5.
|
|
267
|
-
*
|
|
257
|
+
* 1. Prearm before any work. A caught-up native step uses the O(1) generation record without
|
|
258
|
+
* recovery, ledger scans or canonical-history reads.
|
|
259
|
+
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
260
|
+
* producer keeps its newer generation dirty. Native retries retain their durable backoff.
|
|
261
|
+
* 3. After the initial native opportunity, stop admitting new external waves and let the
|
|
262
|
+
* active waves finish. While they remain in flight, native wakes and bounded scans can
|
|
263
|
+
* advance more heads. All Attempts share the event's original ten-minute yield deadline.
|
|
264
|
+
* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
|
|
265
|
+
* bounded independently; incoming native work never restarts or cancels their attempts.
|
|
266
|
+
* Auxiliary failures are reported after the current native opportunity.
|
|
267
|
+
* 5. Close every event resource before the final gated deadline snapshot and alarm decision.
|
|
268
|
+
* The whole event retains one fourteen-minute cooperative timeout.
|
|
268
269
|
*/
|
|
269
270
|
var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
|
|
270
271
|
static layer = Layer.effect(ThreadMaintenance)(Effect.gen(function* () {
|
|
271
272
|
const runtime = yield* DurableAgentRuntime;
|
|
272
273
|
const ledger = yield* SubmissionLedger;
|
|
274
|
+
const wakes = yield* WakeScheduler;
|
|
273
275
|
const alarm = yield* DurableAlarmService;
|
|
274
276
|
const config = yield* CloudflareDurableRuntimeConfig;
|
|
275
277
|
const { ctx } = yield* DurableObjectContext;
|
|
@@ -339,7 +341,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
339
341
|
const jitter = yield* Random.next;
|
|
340
342
|
yield* runTransaction("back off failed maintenance", () => ctx.storage.transaction(async (transaction) => {
|
|
341
343
|
const { state } = await readMaintenanceState(transaction);
|
|
342
|
-
const previous = state.retry?.generation === generation ? state.retry : void 0;
|
|
344
|
+
const previous = state.retry?.generation === generation && state.retry.nativeOnly === nativeOnly ? state.retry : void 0;
|
|
343
345
|
const retry = MaintenanceRetry.make({
|
|
344
346
|
generation,
|
|
345
347
|
notBefore: Math.max(previous?.notBefore ?? 0, now + backoffDelay(previous?.stalls ?? 0, jitter)),
|
|
@@ -356,15 +358,8 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
356
358
|
}));
|
|
357
359
|
yield* failpoint.hit("maintenance:retry:after");
|
|
358
360
|
});
|
|
359
|
-
const
|
|
360
|
-
|
|
361
|
-
phase: report.phase,
|
|
362
|
-
recovered: report.recovered,
|
|
363
|
-
settled: report.settled,
|
|
364
|
-
nonterminal: report.nonterminal,
|
|
365
|
-
alarm: report.alarm
|
|
366
|
-
}).pipe(Effect.as(report));
|
|
367
|
-
const started = yield* mutations.withSnapshot((activeAtStart) => Effect.gen(function* () {
|
|
361
|
+
const beginNative = Effect.fn("ThreadMaintenance.beginNative")(function* (observed) {
|
|
362
|
+
return yield* mutations.withSnapshot((activeAtStart) => Effect.gen(function* () {
|
|
368
363
|
const generation = yield* beginPass(observed);
|
|
369
364
|
if (generation._tag === "Actionable" && activeAtStart === 0) yield* publication.prepareGeneration(generation.generation);
|
|
370
365
|
return {
|
|
@@ -372,59 +367,26 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
372
367
|
activeAtStart
|
|
373
368
|
};
|
|
374
369
|
}));
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
const finished = Deferred.await(sourceFinished);
|
|
378
|
-
const deliveryFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(messages.drainUntil(finished, dispatchUntil)), auxiliaryScope);
|
|
379
|
-
const hostFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(Effect.gen(function* () {
|
|
380
|
-
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(Effect.mapError((cause) => DurableAlarmError.make({
|
|
381
|
-
operation: "host dispatch allowance",
|
|
382
|
-
message: "Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
|
|
383
|
-
cause
|
|
384
|
-
})));
|
|
385
|
-
yield* host.drainUntil(finished, dispatchUntil);
|
|
386
|
-
})), auxiliaryScope);
|
|
387
|
-
const backfill = yield* Effect.forkIn(drainDue.pipe(Effect.provideService(ThreadProjectionMaintenance, projection), Effect.timeoutOption(config.projectionDispatchTimeoutMillis)), auxiliaryScope);
|
|
388
|
-
const finishAuxiliary = Effect.gen(function* () {
|
|
389
|
-
yield* Deferred.succeed(sourceFinished, void 0);
|
|
390
|
-
const remaining = Math.max(1, DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis));
|
|
391
|
-
const hostJoin = yield* Effect.forkIn(Fiber.join(hostFiber).pipe(Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)), Effect.tap((result) => Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(result) }))), auxiliaryScope);
|
|
392
|
-
yield* Fiber.joinAll([
|
|
393
|
-
deliveryFiber,
|
|
394
|
-
hostJoin,
|
|
395
|
-
backfill
|
|
396
|
-
]);
|
|
397
|
-
yield* Scope.close(auxiliaryScope, Exit.void);
|
|
398
|
-
});
|
|
370
|
+
});
|
|
371
|
+
const advance = Effect.fn("ThreadMaintenance.advance")(function* (started, yieldAfter, observed) {
|
|
399
372
|
const deadline = yield* publication.pendingDeadline;
|
|
400
373
|
if (started._tag === "Actionable" || Option.isSome(deadline) && deadline.value <= (yield* Clock.currentTimeMillis)) yield* publication.drain;
|
|
401
374
|
const pending = yield* publication.pendingDeadline;
|
|
402
375
|
if (started._tag === "CaughtUp" || Option.isSome(pending)) {
|
|
403
|
-
yield*
|
|
404
|
-
yield* failpoint.hit("maintenance:finish:before");
|
|
405
|
-
const disposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
406
|
-
const latest = yield* pendingDeadline;
|
|
376
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
407
377
|
const now = yield* Clock.currentTimeMillis;
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
if (Number.isFinite(next)) {
|
|
413
|
-
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
|
|
414
|
-
return "rearmed";
|
|
415
|
-
}
|
|
416
|
-
await transaction.deleteAlarm();
|
|
417
|
-
return "cleared";
|
|
418
|
-
}));
|
|
378
|
+
const { state } = yield* runTransaction("read native maintenance deadline", () => ctx.storage.transaction((transaction) => readMaintenanceState(transaction)));
|
|
379
|
+
const native = active > 0 || state.dirty > state.processed ? state.retry?.generation === state.dirty && active === 0 ? state.retry.notBefore : now + minimumAlarmDelay : Infinity;
|
|
380
|
+
const next = Option.isSome(pending) ? pending.value : native;
|
|
381
|
+
return Number.isFinite(next) ? Math.max(now + minimumAlarmDelay, next) : void 0;
|
|
419
382
|
}));
|
|
420
|
-
|
|
421
|
-
return yield* annotate(MaintenancePassReport.make({
|
|
383
|
+
return {
|
|
422
384
|
phase: "caught-up",
|
|
423
385
|
recovered: 0,
|
|
424
386
|
settled: 0,
|
|
425
387
|
nonterminal: started.nonterminal,
|
|
426
|
-
|
|
427
|
-
}
|
|
388
|
+
nextAttemptAt
|
|
389
|
+
};
|
|
428
390
|
}
|
|
429
391
|
observed.nativeOnly = true;
|
|
430
392
|
const recovered = yield* runtime.runRecovery;
|
|
@@ -491,8 +453,6 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
491
453
|
}
|
|
492
454
|
if (bindingFailure !== void 0) yield* reportBindingFailure ? Effect.logError("Thread awaits a current agent binding; original work remains pending", Cause.fail(bindingFailure)) : Effect.logDebug("Thread binding retry remains pending", Cause.fail(bindingFailure));
|
|
493
455
|
}
|
|
494
|
-
observed.nativeOnly = false;
|
|
495
|
-
yield* finishAuxiliary;
|
|
496
456
|
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
497
457
|
const waitingHeads = /* @__PURE__ */ new Map();
|
|
498
458
|
const autonomous = remaining.some((snapshot) => {
|
|
@@ -508,36 +468,91 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
508
468
|
const nextEligible = eligible.map((threadId) => retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)?.notBefore ?? now);
|
|
509
469
|
const bindingDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
|
|
510
470
|
const delay = Math.max(ordinaryDelay, bindingDelay);
|
|
471
|
+
yield* failpoint.hit("maintenance:checkpoint:before");
|
|
472
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) => runTransaction("checkpoint native maintenance", () => ctx.storage.transaction(async (transaction) => {
|
|
473
|
+
const { state } = await readMaintenanceState(transaction);
|
|
474
|
+
const processed = autonomous || started.activeAtStart > 0 || active > 0 ? state.processed : state.processed > started.generation ? state.processed : started.generation;
|
|
475
|
+
const next = ThreadMaintenanceState.make({
|
|
476
|
+
...Struct.omit(state, ["retry"]),
|
|
477
|
+
processed,
|
|
478
|
+
nonterminal: remaining.length,
|
|
479
|
+
bindingRetries: (state.bindingRetries ?? []).filter((retry) => remaining.some((row) => row.submissionId === retry.submissionId)),
|
|
480
|
+
...autonomous && !progressed ? { retry: MaintenanceRetry.make({
|
|
481
|
+
generation: started.generation,
|
|
482
|
+
notBefore: now + delay,
|
|
483
|
+
nativeOnly: true,
|
|
484
|
+
stalls: Math.min(30, started.stalls + 1)
|
|
485
|
+
}) } : {}
|
|
486
|
+
});
|
|
487
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
488
|
+
if (autonomous) return started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation ? now + minimumAlarmDelay : now + delay;
|
|
489
|
+
return started.activeAtStart > 0 || active > 0 || next.dirty > next.processed ? now + minimumAlarmDelay : void 0;
|
|
490
|
+
})));
|
|
491
|
+
yield* failpoint.hit("maintenance:checkpoint:after");
|
|
492
|
+
return {
|
|
493
|
+
phase: "actionable",
|
|
494
|
+
recovered: recovered.length,
|
|
495
|
+
settled: Option.isSome(settlement) ? 1 : 0,
|
|
496
|
+
nonterminal: remaining.length,
|
|
497
|
+
nextAttemptAt
|
|
498
|
+
};
|
|
499
|
+
});
|
|
500
|
+
const pass = Effect.fn("ThreadMaintenance.pass")(function* (yieldAfter, dispatchUntil, observed) {
|
|
501
|
+
const notified = (yield* Stream.toPull(wakes.wakes)).pipe(Effect.asVoid, Effect.catch(() => Effect.never));
|
|
502
|
+
let started = yield* beginNative(observed);
|
|
503
|
+
const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) => Scope.close(scope, exit));
|
|
504
|
+
const dispatchClosed = yield* Deferred.make();
|
|
505
|
+
const stopDispatch = Deferred.await(dispatchClosed);
|
|
506
|
+
const deliveryFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(messages.drainUntil(stopDispatch, dispatchUntil)), auxiliaryScope);
|
|
507
|
+
const hostFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(Effect.gen(function* () {
|
|
508
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(Effect.mapError((cause) => DurableAlarmError.make({
|
|
509
|
+
operation: "host dispatch allowance",
|
|
510
|
+
message: "Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
|
|
511
|
+
cause
|
|
512
|
+
})));
|
|
513
|
+
yield* host.drainUntil(stopDispatch, dispatchUntil);
|
|
514
|
+
})), auxiliaryScope);
|
|
515
|
+
const backfill = yield* Effect.forkIn(drainDue.pipe(Effect.provideService(ThreadProjectionMaintenance, projection), Effect.timeoutOption(config.projectionDispatchTimeoutMillis)), auxiliaryScope);
|
|
516
|
+
let result = yield* advance(started, yieldAfter, observed);
|
|
517
|
+
let phase = result.phase;
|
|
518
|
+
let recovered = result.recovered;
|
|
519
|
+
let settled = result.settled;
|
|
520
|
+
observed.nativeOnly = false;
|
|
521
|
+
yield* Deferred.succeed(dispatchClosed, void 0);
|
|
522
|
+
const remaining = Math.max(1, DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis));
|
|
523
|
+
const hostJoin = yield* Effect.forkIn(Fiber.join(hostFiber).pipe(Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)), Effect.tap((outcome) => Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) }))), auxiliaryScope);
|
|
524
|
+
const retired = yield* Effect.forkChild(Fiber.joinAll([
|
|
525
|
+
deliveryFiber,
|
|
526
|
+
hostJoin,
|
|
527
|
+
backfill
|
|
528
|
+
]));
|
|
529
|
+
const auxiliaryPending = () => deliveryFiber.pollUnsafe() === void 0 || hostFiber.pollUnsafe() === void 0 || backfill.pollUnsafe() === void 0;
|
|
530
|
+
while (retired.pollUnsafe() === void 0 && auxiliaryPending()) {
|
|
531
|
+
const now = yield* Clock.currentTimeMillis;
|
|
532
|
+
const until = DateTime.toEpochMillis(yieldAfter);
|
|
533
|
+
if (now >= until) break;
|
|
534
|
+
const next = Math.min(result.nextAttemptAt ?? Infinity, now + config.wakeScanInterval, until);
|
|
535
|
+
if (!(yield* Effect.raceFirst(Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as(true)), Fiber.await(retired).pipe(Effect.as(false)))) || retired.pollUnsafe() !== void 0 || !auxiliaryPending()) break;
|
|
536
|
+
if ((yield* Clock.currentTimeMillis) >= until) break;
|
|
537
|
+
started = yield* beginNative(observed);
|
|
538
|
+
result = yield* advance(started, yieldAfter, observed);
|
|
539
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
540
|
+
recovered += result.recovered;
|
|
541
|
+
settled += result.settled;
|
|
542
|
+
observed.nativeOnly = false;
|
|
543
|
+
}
|
|
544
|
+
yield* Fiber.join(retired);
|
|
545
|
+
yield* Scope.close(auxiliaryScope, Exit.void);
|
|
511
546
|
yield* failpoint.hit("maintenance:finish:before");
|
|
512
|
-
const
|
|
513
|
-
const
|
|
514
|
-
|
|
547
|
+
const disposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
548
|
+
const latest = yield* pendingDeadline;
|
|
549
|
+
const now = yield* Clock.currentTimeMillis;
|
|
550
|
+
return yield* runTransaction("finish maintenance event", () => ctx.storage.transaction(async (transaction) => {
|
|
515
551
|
const { state } = await readMaintenanceState(transaction);
|
|
516
|
-
const
|
|
517
|
-
const next =
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
nonterminal: remaining.length,
|
|
521
|
-
bindingRetries: (state.bindingRetries ?? []).filter((retry) => remaining.some((row) => row.submissionId === retry.submissionId)),
|
|
522
|
-
...autonomous && !progressed ? { retry: MaintenanceRetry.make({
|
|
523
|
-
generation: started.generation,
|
|
524
|
-
notBefore: now + delay,
|
|
525
|
-
nativeOnly: true,
|
|
526
|
-
stalls: Math.min(30, started.stalls + 1)
|
|
527
|
-
}) } : {}
|
|
528
|
-
});
|
|
529
|
-
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
530
|
-
if (autonomous) {
|
|
531
|
-
const nativeDeadline = started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation ? now + minimumAlarmDelay : now + delay;
|
|
532
|
-
await transaction.setAlarm(Option.isSome(publicationDeadline) ? Math.max(now + minimumAlarmDelay, Math.min(nativeDeadline, publicationDeadline.value)) : nativeDeadline);
|
|
533
|
-
return "rearmed";
|
|
534
|
-
}
|
|
535
|
-
if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
|
|
536
|
-
await ensureTransactionAlarmBy(transaction, Option.isSome(publicationDeadline) ? Math.max(now + minimumAlarmDelay, Math.min(now + config.wakeScanInterval, publicationDeadline.value)) : now + config.wakeScanInterval);
|
|
537
|
-
return "rearmed";
|
|
538
|
-
}
|
|
539
|
-
if (Option.isSome(publicationDeadline)) {
|
|
540
|
-
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, publicationDeadline.value));
|
|
552
|
+
const native = active > 0 || state.dirty > state.processed ? state.retry?.generation === state.dirty && active === 0 ? Math.max(now + minimumAlarmDelay, state.retry.notBefore) : state.dirty === observed.generation && active === 0 ? result.nextAttemptAt ?? now + config.wakeScanInterval : now + minimumAlarmDelay : Infinity;
|
|
553
|
+
const next = Option.isSome(latest) ? Math.min(native, latest.value) : native;
|
|
554
|
+
if (Number.isFinite(next)) {
|
|
555
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
|
|
541
556
|
return "rearmed";
|
|
542
557
|
}
|
|
543
558
|
await transaction.deleteAlarm();
|
|
@@ -545,13 +560,21 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
545
560
|
}));
|
|
546
561
|
}));
|
|
547
562
|
yield* failpoint.hit("maintenance:finish:after");
|
|
548
|
-
|
|
549
|
-
phase
|
|
550
|
-
recovered
|
|
551
|
-
settled
|
|
552
|
-
nonterminal:
|
|
553
|
-
alarm:
|
|
554
|
-
})
|
|
563
|
+
const report = MaintenancePassReport.make({
|
|
564
|
+
phase,
|
|
565
|
+
recovered,
|
|
566
|
+
settled,
|
|
567
|
+
nonterminal: result.nonterminal,
|
|
568
|
+
alarm: disposition
|
|
569
|
+
});
|
|
570
|
+
yield* Effect.annotateCurrentSpan({
|
|
571
|
+
phase: report.phase,
|
|
572
|
+
recovered: report.recovered,
|
|
573
|
+
settled: report.settled,
|
|
574
|
+
nonterminal: report.nonterminal,
|
|
575
|
+
alarm: report.alarm
|
|
576
|
+
});
|
|
577
|
+
return report;
|
|
555
578
|
});
|
|
556
579
|
return ThreadMaintenance.of({
|
|
557
580
|
pass: Effect.gen(function* () {
|