@yielded/agent-platform-cloudflare 0.0.0-stage → 0.1.0-beta.167
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/Alarm-B0ODr1NE.mjs +1738 -0
- package/dist/Alarm-B0ODr1NE.mjs.map +1 -0
- package/dist/Alarm.d.mts +267 -0
- package/dist/Alarm.mjs +2 -0
- package/dist/BrowserCredentials.d.mts +91 -0
- package/dist/BrowserCredentials.mjs +120 -0
- package/dist/BrowserCredentials.mjs.map +1 -0
- package/dist/BrowserRestCapture.d.mts +34 -0
- package/dist/BrowserRestCapture.mjs +238 -0
- package/dist/BrowserRestCapture.mjs.map +1 -0
- package/dist/BrowserRestCrawl.d.mts +16 -0
- package/dist/BrowserRestCrawl.mjs +244 -0
- package/dist/BrowserRestCrawl.mjs.map +1 -0
- package/dist/BrowserSession-B969Xg-2.d.mts +132 -0
- package/dist/BrowserSession-RfIzxGCH.mjs +515 -0
- package/dist/BrowserSession-RfIzxGCH.mjs.map +1 -0
- package/dist/BrowserSession.d.mts +3 -0
- package/dist/BrowserSession.mjs +4 -0
- package/dist/CloudflareAiGateway.d.mts +64 -0
- package/dist/CloudflareAiGateway.mjs +70 -0
- package/dist/CloudflareAiGateway.mjs.map +1 -0
- package/dist/CloudflareBindings.d.mts +115 -0
- package/dist/CloudflareBindings.mjs +113 -0
- package/dist/CloudflareBindings.mjs.map +1 -0
- package/dist/CloudflareBrowser-5an6m-Qb.mjs +495 -0
- package/dist/CloudflareBrowser-5an6m-Qb.mjs.map +1 -0
- package/dist/CloudflareBrowser-gu7gma0Y.d.mts +80 -0
- package/dist/CloudflareBrowser.d.mts +2 -0
- package/dist/CloudflareBrowser.mjs +2 -0
- package/dist/CloudflareCodeMode.d.mts +44 -0
- package/dist/CloudflareCodeMode.mjs +616 -0
- package/dist/CloudflareCodeMode.mjs.map +1 -0
- package/dist/CloudflareConfig-qDrRLofL.d.mts +100 -0
- package/dist/CloudflareConfig.d.mts +2 -0
- package/dist/CloudflareConfig.mjs +122 -0
- package/dist/CloudflareConfig.mjs.map +1 -0
- package/dist/CloudflareMemory.d.mts +142 -0
- package/dist/CloudflareMemory.mjs +202 -0
- package/dist/CloudflareMemory.mjs.map +1 -0
- package/dist/CloudflareScheduling.d.mts +52 -0
- package/dist/CloudflareScheduling.mjs +396 -0
- package/dist/CloudflareScheduling.mjs.map +1 -0
- package/dist/CloudflareSubscriptions.d.mts +87 -0
- package/dist/CloudflareSubscriptions.mjs +529 -0
- package/dist/CloudflareSubscriptions.mjs.map +1 -0
- package/dist/CloudflareThreadClient.d.mts +1796 -0
- package/dist/CloudflareThreadClient.mjs +382 -0
- package/dist/CloudflareThreadClient.mjs.map +1 -0
- package/dist/InteractiveBrowser-BbG3kGA9.mjs +1934 -0
- package/dist/InteractiveBrowser-BbG3kGA9.mjs.map +1 -0
- package/dist/InteractiveBrowser-wV2Bti5e.d.mts +228 -0
- package/dist/InteractiveBrowser.d.mts +2 -0
- package/dist/InteractiveBrowser.mjs +2 -0
- package/dist/ThreadObject-BcfXbzSO.d.mts +867 -0
- package/dist/ThreadObject-DHjrnlX4.mjs +937 -0
- package/dist/ThreadObject-DHjrnlX4.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +2 -0
- package/dist/ThreadObject.mjs +3 -0
- package/dist/WakeScheduler.d.mts +23 -0
- package/dist/WakeScheduler.mjs +57 -0
- package/dist/WakeScheduler.mjs.map +1 -0
- package/dist/boundary-BguazVkh.mjs +42 -0
- package/dist/boundary-BguazVkh.mjs.map +1 -0
- package/dist/index.d.mts +15 -0
- package/dist/index.mjs +15 -0
- package/dist/prepared-admission-DMX5p9dC.mjs +73 -0
- package/dist/prepared-admission-DMX5p9dC.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -6
- package/src/Alarm.ts +2961 -0
- package/src/BrowserCredentials.ts +146 -0
- package/src/BrowserRestCapture.ts +477 -0
- package/src/BrowserRestCrawl.ts +540 -0
- package/src/BrowserSession.ts +667 -0
- package/src/CloudflareAiGateway.ts +170 -0
- package/src/CloudflareBindings.ts +199 -0
- package/src/CloudflareBrowser.ts +15 -0
- package/src/CloudflareCodeMode.ts +878 -0
- package/src/CloudflareConfig.ts +138 -0
- package/src/CloudflareMemory.ts +420 -0
- package/src/CloudflareScheduling.ts +745 -0
- package/src/CloudflareSubscriptions.ts +1044 -0
- package/src/CloudflareThreadClient.ts +809 -0
- package/src/InteractiveBrowser.ts +3275 -0
- package/src/ThreadObject.ts +1198 -0
- package/src/WakeScheduler.ts +91 -0
- package/src/index.ts +14 -0
- package/src/internal/boundary.ts +55 -0
- package/src/internal/browser-binding.ts +322 -0
- package/src/internal/browser-credentials.ts +362 -0
- package/src/internal/browser-failure.ts +92 -0
- package/src/internal/browser-file-selection.ts +126 -0
- package/src/internal/browser-quick-action.ts +857 -0
- package/src/internal/browser-readonly-live-view.ts +111 -0
- package/src/internal/browser-session-lifecycle.ts +193 -0
- package/src/internal/due-queue.ts +493 -0
- package/src/internal/layers.ts +1068 -0
- package/src/internal/message-delivery.ts +120 -0
- package/src/internal/prepared-admission.ts +116 -0
- package/src/internal/progress-wait.ts +121 -0
- package/src/internal/transport.ts +56 -0
- package/README.md +0 -3
package/src/Alarm.ts
ADDED
|
@@ -0,0 +1,2961 @@
|
|
|
1
|
+
import { type DurableBindingFailure } from "@yielded/agent/agent-registration";
|
|
2
|
+
import {
|
|
3
|
+
DurableAgentRuntime,
|
|
4
|
+
DurableRuntimeConfig,
|
|
5
|
+
isolateRecovery,
|
|
6
|
+
RecoveryBlocked,
|
|
7
|
+
RecoveryFailure,
|
|
8
|
+
type DurableWorkerFailure,
|
|
9
|
+
type RecoveryReport,
|
|
10
|
+
RecoverySweepResult,
|
|
11
|
+
} from "@yielded/agent/durable-agent-runtime";
|
|
12
|
+
import { ThreadId, SubmissionId } from "@yielded/agent/identifiers";
|
|
13
|
+
import {
|
|
14
|
+
AbortIntentRequest,
|
|
15
|
+
SubmissionLedger,
|
|
16
|
+
type SubmissionWorkItem,
|
|
17
|
+
} from "@yielded/agent/submission-ledger";
|
|
18
|
+
import {
|
|
19
|
+
ThreadProjectionMaintenance,
|
|
20
|
+
type ThreadProjectionError,
|
|
21
|
+
} from "@yielded/agent/thread-projection-maintenance";
|
|
22
|
+
import { WakeScheduler } from "@yielded/agent/wake-scheduler";
|
|
23
|
+
import {
|
|
24
|
+
Cause,
|
|
25
|
+
Clock,
|
|
26
|
+
Context,
|
|
27
|
+
DateTime,
|
|
28
|
+
Deferred,
|
|
29
|
+
Effect,
|
|
30
|
+
ErrorReporter,
|
|
31
|
+
Exit,
|
|
32
|
+
Fiber,
|
|
33
|
+
Layer,
|
|
34
|
+
Option,
|
|
35
|
+
Random,
|
|
36
|
+
Ref,
|
|
37
|
+
Result,
|
|
38
|
+
Schema,
|
|
39
|
+
Scope,
|
|
40
|
+
Semaphore,
|
|
41
|
+
Stream,
|
|
42
|
+
Struct,
|
|
43
|
+
} from "effect";
|
|
44
|
+
import { DurableObjectStorage } from "effect-cf";
|
|
45
|
+
import { SqlClient } from "effect/sql/SqlClient";
|
|
46
|
+
import type { SqlError } from "effect/sql/SqlError";
|
|
47
|
+
|
|
48
|
+
import { DurableObjectContext } from "./CloudflareBindings.ts";
|
|
49
|
+
import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.ts";
|
|
50
|
+
import { safeCauseMessage } from "./internal/boundary.ts";
|
|
51
|
+
import * as DueQueue from "./internal/due-queue.ts";
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The single multiplexed Durable Object alarm (decision D-P6-2). A Durable Object has ONE
|
|
55
|
+
* alarm slot; due work (lease expiry, settlement
|
|
56
|
+
* and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
|
|
57
|
+
* the slot always holds the EARLIEST deadline any caller asked for.
|
|
58
|
+
*
|
|
59
|
+
* The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
|
|
60
|
+
* maintenance generation and a committed alarm. Stable externally-driven waits may be
|
|
61
|
+
* nonterminal without retaining an alarm; their resolving mutation advances the generation and
|
|
62
|
+
* restores the alarm atomically.
|
|
63
|
+
*/
|
|
64
|
+
|
|
65
|
+
/** The Durable Object alarm API failed; surfaces on host entry points as a typed refusal. */
|
|
66
|
+
export class DurableAlarmError extends Schema.TaggedError<DurableAlarmError>()(
|
|
67
|
+
"DurableAlarmError",
|
|
68
|
+
{
|
|
69
|
+
operation: Schema.String,
|
|
70
|
+
message: Schema.String,
|
|
71
|
+
cause: Schema.optionalKey(Schema.Defect()),
|
|
72
|
+
},
|
|
73
|
+
) {}
|
|
74
|
+
|
|
75
|
+
/** Content-free operational notice: pending work now retries at the hourly ceiling. */
|
|
76
|
+
class MaintenanceRetryParked extends Schema.TaggedError<MaintenanceRetryParked>()(
|
|
77
|
+
"MaintenanceRetryParked",
|
|
78
|
+
{ lane: Schema.String, retryAfterMillis: Schema.Number },
|
|
79
|
+
) {}
|
|
80
|
+
|
|
81
|
+
const alarmFailure =
|
|
82
|
+
(operation: string) =>
|
|
83
|
+
(cause: unknown): DurableAlarmError =>
|
|
84
|
+
DurableAlarmError.make({
|
|
85
|
+
operation,
|
|
86
|
+
message: safeCauseMessage(cause, "The Cloudflare alarm API failed without a diagnostic"),
|
|
87
|
+
cause,
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
// SQL and raw KV/alarm operations share one physical SQLite transaction. Reserve its
|
|
91
|
+
// connection for each short storage operation, never around a mutation or snapshot body.
|
|
92
|
+
const makeStorageEffect = Effect.gen(function* () {
|
|
93
|
+
const sql = yield* SqlClient;
|
|
94
|
+
const { ctx } = yield* DurableObjectContext;
|
|
95
|
+
const invalidate = Effect.sync(() => DueQueue.invalidate(ctx.storage));
|
|
96
|
+
|
|
97
|
+
return <A, R>(operation: string, execute: Effect.Effect<A, DurableAlarmError, R>) =>
|
|
98
|
+
Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => {
|
|
99
|
+
const body = Effect.uninterruptible(execute);
|
|
100
|
+
|
|
101
|
+
// Explicit source transactions own the view through flush and rollback.
|
|
102
|
+
// Unwrapped host SQL keeps eager writes and discards speculative cache rows.
|
|
103
|
+
if (current._tag === "Some") {
|
|
104
|
+
if (DueQueue.buffering(ctx.storage)) return body;
|
|
105
|
+
|
|
106
|
+
return body.pipe(Effect.ensuring(invalidate));
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return Effect.scoped(
|
|
110
|
+
Effect.andThen(
|
|
111
|
+
sql.reserve.pipe(Effect.mapError(alarmFailure(operation))),
|
|
112
|
+
body.pipe(Effect.onExit((exit) => (Exit.isFailure(exit) ? invalidate : Effect.void))),
|
|
113
|
+
),
|
|
114
|
+
);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
const makeStorageOperation = Effect.map(
|
|
119
|
+
makeStorageEffect,
|
|
120
|
+
(run) =>
|
|
121
|
+
<A>(operation: string, execute: () => Promise<A>) =>
|
|
122
|
+
run(operation, Effect.tryPromise({ try: execute, catch: alarmFailure(operation) })),
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
/** Native `ctx.storage` alarm slot owned by ThreadMaintenance; storage is truth. */
|
|
126
|
+
export class DurableAlarmService extends Context.Service<
|
|
127
|
+
DurableAlarmService,
|
|
128
|
+
{
|
|
129
|
+
/** The scheduled deadline in epoch milliseconds, if any. */
|
|
130
|
+
readonly scheduled: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
131
|
+
/** Replace the slot with this deadline. */
|
|
132
|
+
readonly scheduleAt: (epochMillis: number) => Effect.Effect<void, DurableAlarmError>;
|
|
133
|
+
/** Keep the EARLIER of the existing deadline and this one (the multiplexing rule). */
|
|
134
|
+
readonly ensureScheduledBy: (epochMillis: number) => Effect.Effect<void, DurableAlarmError>;
|
|
135
|
+
/**
|
|
136
|
+
* Arm an immediate alarm (the durable, coalescing local wake) — DEFERRED while a
|
|
137
|
+
* maintenance pass is executing. Workerd cancels an in-flight alarm handler when a new
|
|
138
|
+
* EARLIER deadline is written during its execution (`requestScheduledAlarm`), and the
|
|
139
|
+
* maintenance pass runs INSIDE the alarm handler: an immediate wake landing mid-pass
|
|
140
|
+
* (a routed port mutation, a sibling's `wake()`, the coordinator's own local notify)
|
|
141
|
+
* would kill the running Attempt — manufacturing an ownership loss no real eviction
|
|
142
|
+
* caused, and routing open uncertain-class Tool Calls into spurious Unknown Outcomes.
|
|
143
|
+
* Deferral is contract-safe: wakes are droppable hints, every mutating entry point
|
|
144
|
+
* pre-arms BEFORE its first durable mutation (the alarm invariant never rests on this
|
|
145
|
+
* call). The pass's durable generation check observes any racing mutation, so the
|
|
146
|
+
* in-memory hint does not need to be flushed after a stable wait is acknowledged.
|
|
147
|
+
*/
|
|
148
|
+
readonly scheduleNow: Effect.Effect<void, DurableAlarmError>;
|
|
149
|
+
/**
|
|
150
|
+
* Run one maintenance pass with wake deferral (see `scheduleNow`). Calls made while `body`
|
|
151
|
+
* executes are droppable promptness hints; correctness rests on the durable generation.
|
|
152
|
+
*/
|
|
153
|
+
readonly withWakesDeferred: <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
154
|
+
/** Clear the slot; correctness-sensitive clears live in maintenance generation transactions. */
|
|
155
|
+
readonly cancel: Effect.Effect<void, DurableAlarmError>;
|
|
156
|
+
}
|
|
157
|
+
>()("@effect-agent/platform-cloudflare/DurableAlarmService") {
|
|
158
|
+
static readonly layer: Layer.Layer<DurableAlarmService, never, DurableObjectContext | SqlClient> =
|
|
159
|
+
Layer.effect(DurableAlarmService)(
|
|
160
|
+
Effect.gen(function* () {
|
|
161
|
+
const { ctx } = yield* DurableObjectContext;
|
|
162
|
+
/**
|
|
163
|
+
* In-memory pass bookkeeping — a pure CACHE, never state: a fresh incarnation has no
|
|
164
|
+
* running pass, and a deferred wake lost to eviction was only ever a promptness hint
|
|
165
|
+
* on top of the already-committed pre-armed alarm.
|
|
166
|
+
*/
|
|
167
|
+
const runningPasses = yield* Ref.make(0);
|
|
168
|
+
|
|
169
|
+
const storageOperation = yield* makeStorageOperation;
|
|
170
|
+
|
|
171
|
+
const scheduled = storageOperation("get alarm", () => ctx.storage.getAlarm()).pipe(
|
|
172
|
+
Effect.map((deadline) =>
|
|
173
|
+
deadline === null ? Option.none<number>() : Option.some(deadline),
|
|
174
|
+
),
|
|
175
|
+
);
|
|
176
|
+
|
|
177
|
+
const scheduleAt = (epochMillis: number) =>
|
|
178
|
+
storageOperation("set alarm", () => ctx.storage.setAlarm(epochMillis));
|
|
179
|
+
|
|
180
|
+
const ensureScheduledBy = (epochMillis: number) =>
|
|
181
|
+
storageOperation("ensure alarm", () =>
|
|
182
|
+
ctx.storage.transaction(async (transaction) => {
|
|
183
|
+
const existing = await transaction.getAlarm();
|
|
184
|
+
|
|
185
|
+
if (existing === null || existing > epochMillis) {
|
|
186
|
+
await transaction.setAlarm(epochMillis);
|
|
187
|
+
}
|
|
188
|
+
}),
|
|
189
|
+
);
|
|
190
|
+
|
|
191
|
+
const armNow = Clock.currentTimeMillis.pipe(
|
|
192
|
+
Effect.flatMap((now) =>
|
|
193
|
+
storageOperation("wake maintenance lanes", () =>
|
|
194
|
+
ctx.storage.transaction(async (transaction) => {
|
|
195
|
+
const next = DueQueue.next(DueQueue.make(ctx.storage).read());
|
|
196
|
+
|
|
197
|
+
if (Number.isFinite(next))
|
|
198
|
+
await ensureTransactionAlarmBy(transaction, Math.max(now, next));
|
|
199
|
+
}),
|
|
200
|
+
),
|
|
201
|
+
),
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
const scheduleNow = Ref.get(runningPasses).pipe(
|
|
205
|
+
Effect.flatMap((passes) => (passes > 0 ? Effect.void : armNow)),
|
|
206
|
+
);
|
|
207
|
+
|
|
208
|
+
const withWakesDeferred = <A, E, R>(body: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
|
|
209
|
+
Ref.update(runningPasses, (passes) => passes + 1).pipe(
|
|
210
|
+
Effect.andThen(body),
|
|
211
|
+
Effect.ensuring(Ref.update(runningPasses, (passes) => passes - 1)),
|
|
212
|
+
);
|
|
213
|
+
|
|
214
|
+
const cancel = storageOperation("delete alarm", () => ctx.storage.deleteAlarm());
|
|
215
|
+
|
|
216
|
+
return DurableAlarmService.of({
|
|
217
|
+
scheduled,
|
|
218
|
+
scheduleAt,
|
|
219
|
+
ensureScheduledBy,
|
|
220
|
+
scheduleNow,
|
|
221
|
+
withWakesDeferred,
|
|
222
|
+
cancel,
|
|
223
|
+
});
|
|
224
|
+
}),
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
|
|
229
|
+
export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
|
|
230
|
+
"@effect-agent/platform-cloudflare/MaintenancePassReport",
|
|
231
|
+
)({
|
|
232
|
+
/** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
|
|
233
|
+
phase: Schema.Literals(["caught-up", "actionable"]),
|
|
234
|
+
/** Recovery decisions and Thread faults observed during this event. */
|
|
235
|
+
recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
236
|
+
/** Head Attempts settled during the event. Joined input may settle with each head. */
|
|
237
|
+
settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
238
|
+
/** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
|
|
239
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
240
|
+
/** `rearmed` for dirty/autonomous work, `cleared` for stable waits or settlement. */
|
|
241
|
+
alarm: Schema.Literals(["rearmed", "cleared"]),
|
|
242
|
+
}) {}
|
|
243
|
+
|
|
244
|
+
/** Fault boundaries around every maintenance-owned durable mutation. */
|
|
245
|
+
export type ThreadMaintenanceFailpointLocation =
|
|
246
|
+
| "maintenance:dirty:before"
|
|
247
|
+
| "maintenance:dirty:after"
|
|
248
|
+
| "maintenance:mutation:armed"
|
|
249
|
+
| "maintenance:mutation:finished"
|
|
250
|
+
| "maintenance:ensure:before"
|
|
251
|
+
| "maintenance:ensure:after"
|
|
252
|
+
| "maintenance:begin:before"
|
|
253
|
+
| "maintenance:begin:after"
|
|
254
|
+
| "maintenance:select:before"
|
|
255
|
+
| "maintenance:select:after"
|
|
256
|
+
| "maintenance:binding-retry:before"
|
|
257
|
+
| "maintenance:binding-retry:after"
|
|
258
|
+
| "maintenance:recovery-status:before"
|
|
259
|
+
| "maintenance:recovery-status:after"
|
|
260
|
+
| "maintenance:retry:before"
|
|
261
|
+
| "maintenance:retry:after"
|
|
262
|
+
| "maintenance:checkpoint:before"
|
|
263
|
+
| "maintenance:checkpoint:after"
|
|
264
|
+
| "maintenance:finish:before"
|
|
265
|
+
| "maintenance:finish:after";
|
|
266
|
+
|
|
267
|
+
export type ThreadMaintenanceFailpointHandler = (
|
|
268
|
+
location: ThreadMaintenanceFailpointLocation,
|
|
269
|
+
) => Effect.Effect<void>;
|
|
270
|
+
|
|
271
|
+
/** Test-only fault authority; production uses the inert layer. */
|
|
272
|
+
export class ThreadMaintenanceFailpoint extends Context.Service<
|
|
273
|
+
ThreadMaintenanceFailpoint,
|
|
274
|
+
{
|
|
275
|
+
readonly hit: ThreadMaintenanceFailpointHandler;
|
|
276
|
+
}
|
|
277
|
+
>()("@effect-agent/platform-cloudflare/ThreadMaintenanceFailpoint") {
|
|
278
|
+
static readonly layer = Layer.succeed(this)({ hit: () => Effect.void });
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Durable host publication of canonical records and ledger approval/abort/resolution intents.
|
|
283
|
+
* The host owns schema-versioned cursors, destination idempotency and acknowledgement. Delivery
|
|
284
|
+
* is at least once. Hooks must not write the alarm slot or mutate the supplied raw source ports.
|
|
285
|
+
*
|
|
286
|
+
* `invalidate` and `prepareGeneration` must be bounded local operations.
|
|
287
|
+
* `prepareGeneration` durably invalidates a scan only when its generation changes; repeated
|
|
288
|
+
* calls must preserve partial scan progress. It runs with no source mutation in flight.
|
|
289
|
+
* Use this gate only for publication required before dependent native execution. Independent
|
|
290
|
+
* UI relays and outboxes belong to ThreadHostMaintenance.
|
|
291
|
+
* `drain` performs bounded delivery and returns its next deadline (None when caught up),
|
|
292
|
+
* after persisting receipts and retries. A pending deadline
|
|
293
|
+
* defers runtime recovery/Attempts, allowing committed host publications to drain first.
|
|
294
|
+
* Unexpected hook failures leave the prearmed generation for retry. Hooks acquire per-call
|
|
295
|
+
* resources with Effect.scoped; Layer construction owns incarnation resources (eviction need
|
|
296
|
+
* not run finalizers). Do not hold a local hook behind network I/O or call back into producers.
|
|
297
|
+
*/
|
|
298
|
+
export interface ThreadPublicationService {
|
|
299
|
+
readonly invalidate: Effect.Effect<void, DurableAlarmError>;
|
|
300
|
+
readonly prepareGeneration: (generation: bigint) => Effect.Effect<void, DurableAlarmError>;
|
|
301
|
+
readonly drain: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
const emptyPublication: ThreadPublicationService = {
|
|
305
|
+
invalidate: Effect.void,
|
|
306
|
+
prepareGeneration: () => Effect.void,
|
|
307
|
+
drain: Effect.succeed(Option.none()),
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
/** Opt in with `ThreadObject.layer(registrations, { publication: Layer.effect(ThreadPublication)(...) })`. */
|
|
311
|
+
export class ThreadPublication extends Context.Service<
|
|
312
|
+
ThreadPublication,
|
|
313
|
+
ThreadPublicationService
|
|
314
|
+
>()("@effect-agent/platform-cloudflare/ThreadPublication") {
|
|
315
|
+
static readonly layer = Layer.succeed(this)(emptyPublication);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// Only the actual empty default can waive required publication. Missing or replaced
|
|
319
|
+
// services retain the barrier, including hosts that rebuild maintenance Layers.
|
|
320
|
+
const requiresPublication = Effect.map(
|
|
321
|
+
Effect.serviceOption(ThreadPublication),
|
|
322
|
+
(service) => Option.isNone(service) || service.value !== emptyPublication,
|
|
323
|
+
);
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Host-assembled native message recovery. Preparation is a bounded local selection; the
|
|
327
|
+
* driver bounds each actual Claim and persists its timeout/retry before `run` returns.
|
|
328
|
+
* The alarm must not add a timer starting at selection: Claim setup and retry commits belong
|
|
329
|
+
* to the driver. The prepared allowance decides only whether a wave fits this event.
|
|
330
|
+
*/
|
|
331
|
+
export const ThreadMessageDelivery = Context.Reference<{
|
|
332
|
+
readonly prepare: Effect.Effect<
|
|
333
|
+
{
|
|
334
|
+
readonly timeoutMillis: number;
|
|
335
|
+
readonly run: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
336
|
+
},
|
|
337
|
+
DurableAlarmError
|
|
338
|
+
>;
|
|
339
|
+
}>("@effect-agent/platform-cloudflare/ThreadMessageDelivery", {
|
|
340
|
+
defaultValue: () => ({
|
|
341
|
+
prepare: Effect.succeed({ timeoutMillis: 1, run: Effect.succeed(Option.none()) }),
|
|
342
|
+
}),
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* One explicitly scheduled application obligation. IDs are stable and unique within the
|
|
347
|
+
* physical Object; the `@yielded/agent:` prefix is reserved. There is no initial host wave.
|
|
348
|
+
* Producers enroll only affected IDs through ThreadMutationGate.withMutation({ lanes })
|
|
349
|
+
* or schedule(id, dueAt). A lane returns its next epoch-millisecond deadline (None = idle)
|
|
350
|
+
* with its finite wave, after persisting claims, receipts and retries. No deadline callback
|
|
351
|
+
* or polling is used. A racing producer keeps its newer revision due.
|
|
352
|
+
*
|
|
353
|
+
* The 1..300000ms allowance includes selection, dispatch, commits and scoped cleanup.
|
|
354
|
+
* Failed waves retain independent backoff and are not retried in the same event. Delivery
|
|
355
|
+
* remains at least once: persist exact envelopes and deduplicate by domain receipt identity.
|
|
356
|
+
* Hooks never write the alarm slot. Required execution gates belong to ThreadPublication.
|
|
357
|
+
*/
|
|
358
|
+
export interface ThreadHostMaintenanceLane {
|
|
359
|
+
readonly id: string;
|
|
360
|
+
/** Omission runs concurrently with native work. After-native lanes get one due wave
|
|
361
|
+
* after all admitted native Attempts and their scoped cleanup, even on native failure.
|
|
362
|
+
* They share the event deadline and remain inside the maintenance pass permit.
|
|
363
|
+
* Newly enrolled concurrent work yields this phase after scoped cleanup; unfinished
|
|
364
|
+
* waves retain their due revision and domain receipts for the next alarm.
|
|
365
|
+
*/
|
|
366
|
+
readonly phase?: "concurrent" | "after-native";
|
|
367
|
+
readonly dispatchTimeoutMillis: number;
|
|
368
|
+
readonly run: Effect.Effect<Option.Option<number>, DurableAlarmError, Scope.Scope>;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/** Independent lanes share the existing alarm; compose hosts by concatenating their lanes. */
|
|
372
|
+
export const ThreadHostMaintenance = Context.Reference<{
|
|
373
|
+
readonly lanes: ReadonlyArray<ThreadHostMaintenanceLane>;
|
|
374
|
+
}>("@effect-agent/platform-cloudflare/ThreadHostMaintenance", {
|
|
375
|
+
defaultValue: () => ({ lanes: [] }),
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
/** @internal Framework handlers are separate from host registrations and their namespace. */
|
|
379
|
+
export const ThreadNativeMaintenance = Context.Reference<{
|
|
380
|
+
readonly lanes: ReadonlyArray<ThreadHostMaintenanceLane>;
|
|
381
|
+
}>("@effect-agent/platform-cloudflare/ThreadNativeMaintenance", {
|
|
382
|
+
defaultValue: () => ({ lanes: [] }),
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
/** @internal Required host publication only; native lifecycle facts use a maintenance lane. */
|
|
386
|
+
export const publishCommitted = Effect.gen(function* () {
|
|
387
|
+
const publication = yield* ThreadPublication;
|
|
388
|
+
|
|
389
|
+
yield* publication.invalidate;
|
|
390
|
+
yield* publication.drain;
|
|
391
|
+
}).pipe(
|
|
392
|
+
Effect.catchCause((cause) =>
|
|
393
|
+
Cause.hasInterrupts(cause)
|
|
394
|
+
? Effect.interrupt
|
|
395
|
+
: Effect.logError("Thread publication deferred after source commit", cause),
|
|
396
|
+
),
|
|
397
|
+
);
|
|
398
|
+
|
|
399
|
+
const MaintenanceGeneration = Schema.BigIntFromString.check(
|
|
400
|
+
Schema.isGreaterThanOrEqualToBigInt(0n),
|
|
401
|
+
);
|
|
402
|
+
|
|
403
|
+
/** A committed recovery transition for one affected Submission; never a Settlement. */
|
|
404
|
+
export class ThreadRecoveryFaultEvent extends Schema.Class<ThreadRecoveryFaultEvent>(
|
|
405
|
+
"@effect-agent/platform-cloudflare/ThreadRecoveryFaultEvent",
|
|
406
|
+
)({
|
|
407
|
+
schemaVersion: Schema.Literal(1),
|
|
408
|
+
/** Monotonic within the physical Object; retained unchanged on delivery retries. */
|
|
409
|
+
sequence: MaintenanceGeneration,
|
|
410
|
+
transition: Schema.Literals(["created", "changed", "cleared"]),
|
|
411
|
+
threadId: ThreadId,
|
|
412
|
+
submissionId: SubmissionId,
|
|
413
|
+
occurredAt: Schema.Finite,
|
|
414
|
+
firstFailedAt: Schema.Finite,
|
|
415
|
+
/** The current failure, or the last failure for a cleared event. */
|
|
416
|
+
failure: RecoveryFailure,
|
|
417
|
+
}) {}
|
|
418
|
+
|
|
419
|
+
const recoveryEventKey = (sequence: bigint) => `effect-agent:thread-recovery-event:v1:${sequence}`;
|
|
420
|
+
const encodeRecoveryEvent = Schema.encodeSync(ThreadRecoveryFaultEvent);
|
|
421
|
+
const decodeRecoveryEvent = Schema.decodeUnknownSync(ThreadRecoveryFaultEvent);
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Ordered, at-least-once delivery through the existing host maintenance lane. Return only
|
|
425
|
+
* after durably applying the event or retaining it in a host outbox. Deduplicate by physical
|
|
426
|
+
* Object and sequence; callbacks may repeat after interruption. This is a trusted host hook,
|
|
427
|
+
* not pre-authorized UI data: the host owns recipient authorization and safe presentation.
|
|
428
|
+
* Capture services in the Layer. Do not call producers or write the raw alarm slot.
|
|
429
|
+
*/
|
|
430
|
+
export class ThreadRecoveryEvents extends Context.Service<
|
|
431
|
+
ThreadRecoveryEvents,
|
|
432
|
+
{ readonly publish: (event: ThreadRecoveryFaultEvent) => Effect.Effect<void, DurableAlarmError> }
|
|
433
|
+
>()("@effect-agent/platform-cloudflare/ThreadRecoveryEvents") {}
|
|
434
|
+
|
|
435
|
+
class BindingWait extends Schema.Class<BindingWait>("BindingWait")({
|
|
436
|
+
threadId: ThreadId,
|
|
437
|
+
submissionId: SubmissionId,
|
|
438
|
+
reportedAt: Schema.Finite,
|
|
439
|
+
}) {}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* A durable observation of blocked recovery, independent of the execution journal. This is
|
|
443
|
+
* neither a Settlement nor proof that external effects did not happen. A successful recovery
|
|
444
|
+
* sweep clears it; repair must preserve canonical history and the original admission identity.
|
|
445
|
+
*/
|
|
446
|
+
class ThreadRecoveryFault extends Schema.Class<ThreadRecoveryFault>(
|
|
447
|
+
"@effect-agent/platform-cloudflare/ThreadRecoveryFault",
|
|
448
|
+
)({
|
|
449
|
+
schemaVersion: Schema.Literal(1),
|
|
450
|
+
threadId: ThreadId,
|
|
451
|
+
firstFailedAt: Schema.Finite,
|
|
452
|
+
lastFailedAt: Schema.Finite,
|
|
453
|
+
/** Earliest automatic recovery retry; new admissions do not erase this deadline. */
|
|
454
|
+
retryAt: Schema.Finite,
|
|
455
|
+
/** Saturates at 2^31 - 1; one observation per Thread per recovery sweep. */
|
|
456
|
+
attempts: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(2_147_483_647)),
|
|
457
|
+
failure: RecoveryFailure,
|
|
458
|
+
/** Individually retained recipients survive settlement until their clear commits. */
|
|
459
|
+
recipientCount: Schema.optionalKey(Schema.Natural),
|
|
460
|
+
}) {}
|
|
461
|
+
|
|
462
|
+
const recoveryFaultKey = (threadId: ThreadId) =>
|
|
463
|
+
`effect-agent:thread-recovery-fault:v1:${threadId}`;
|
|
464
|
+
|
|
465
|
+
const decodeRecoveryFaultValue = Schema.decodeUnknownSync(ThreadRecoveryFault);
|
|
466
|
+
|
|
467
|
+
const decodeRecoveryFault = (threadId: ThreadId, encoded: unknown) => {
|
|
468
|
+
const fault = decodeRecoveryFaultValue(encoded);
|
|
469
|
+
|
|
470
|
+
if (fault.threadId !== threadId) throw new Error("Recovery status does not match its Thread key");
|
|
471
|
+
|
|
472
|
+
return fault;
|
|
473
|
+
};
|
|
474
|
+
|
|
475
|
+
const encodeRecoveryFault = Schema.encodeSync(ThreadRecoveryFault);
|
|
476
|
+
const sameRecoveryFailure = Schema.toEquivalence(RecoveryFailure);
|
|
477
|
+
const decodeRecoveryRecipient = Schema.decodeUnknownSync(SubmissionId);
|
|
478
|
+
const encodeRecoveryRecipient = Schema.encodeSync(SubmissionId);
|
|
479
|
+
|
|
480
|
+
const recoveryRecipientKey = (threadId: ThreadId, index: number) =>
|
|
481
|
+
`effect-agent:recovery-recipient:v1:${threadId.length}:${threadId}:${index}`;
|
|
482
|
+
|
|
483
|
+
const recoveryRecipientKeys = (fault: ThreadRecoveryFault, start: number) =>
|
|
484
|
+
Array.from({ length: Math.min(128, (fault.recipientCount ?? 0) - start) }, (_, offset) =>
|
|
485
|
+
recoveryRecipientKey(fault.threadId, start + offset),
|
|
486
|
+
);
|
|
487
|
+
|
|
488
|
+
const readRecoveryRecipients = async (
|
|
489
|
+
storage: Pick<DurableObjectTransaction, "get">,
|
|
490
|
+
fault: ThreadRecoveryFault,
|
|
491
|
+
): Promise<Array<SubmissionId>> => {
|
|
492
|
+
const ids: Array<SubmissionId> = [];
|
|
493
|
+
|
|
494
|
+
for (let start = 0; start < (fault.recipientCount ?? 0); start += 128) {
|
|
495
|
+
const keys = recoveryRecipientKeys(fault, start);
|
|
496
|
+
const values = await storage.get(keys);
|
|
497
|
+
|
|
498
|
+
for (const key of keys) ids.push(decodeRecoveryRecipient(values.get(key)));
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
return ids;
|
|
502
|
+
};
|
|
503
|
+
|
|
504
|
+
const appendRecoveryRecipients = async (
|
|
505
|
+
transaction: DurableObjectTransaction,
|
|
506
|
+
threadId: ThreadId,
|
|
507
|
+
count: number,
|
|
508
|
+
added: ReadonlyArray<SubmissionId>,
|
|
509
|
+
) => {
|
|
510
|
+
for (let start = 0; start < added.length; start += 128)
|
|
511
|
+
await transaction.put(
|
|
512
|
+
Object.fromEntries(
|
|
513
|
+
added
|
|
514
|
+
.slice(start, start + 128)
|
|
515
|
+
.map((id, offset) => [
|
|
516
|
+
recoveryRecipientKey(threadId, count + start + offset),
|
|
517
|
+
encodeRecoveryRecipient(id),
|
|
518
|
+
]),
|
|
519
|
+
),
|
|
520
|
+
);
|
|
521
|
+
};
|
|
522
|
+
|
|
523
|
+
const recoveryEvent = (
|
|
524
|
+
fault: ThreadRecoveryFault,
|
|
525
|
+
submissionId: SubmissionId,
|
|
526
|
+
transition: ThreadRecoveryFaultEvent["transition"],
|
|
527
|
+
occurredAt: number,
|
|
528
|
+
): Omit<ThreadRecoveryFaultEvent, "sequence"> => ({
|
|
529
|
+
schemaVersion: 1,
|
|
530
|
+
transition,
|
|
531
|
+
threadId: fault.threadId,
|
|
532
|
+
submissionId,
|
|
533
|
+
occurredAt,
|
|
534
|
+
firstFailedAt: fault.firstFailedAt,
|
|
535
|
+
failure: fault.failure,
|
|
536
|
+
});
|
|
537
|
+
|
|
538
|
+
interface NativePassResult {
|
|
539
|
+
readonly phase: "caught-up" | "actionable";
|
|
540
|
+
readonly nonterminal: number;
|
|
541
|
+
readonly nextAttemptAt: number | undefined;
|
|
542
|
+
readonly dispatched?: boolean;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/** Independent Threads share two bounded native slots; one active head per Thread. */
|
|
546
|
+
interface NativeDispatch {
|
|
547
|
+
readonly scope: Scope.Scope;
|
|
548
|
+
readonly active: Map<ThreadId, Fiber.Fiber<number, MaintenancePassFailure>>;
|
|
549
|
+
readonly deferred: Set<ThreadId>;
|
|
550
|
+
dispatched: number;
|
|
551
|
+
settled: number;
|
|
552
|
+
progressed: boolean;
|
|
553
|
+
needsCheckpoint: boolean;
|
|
554
|
+
generation?: bigint;
|
|
555
|
+
sourceRevision?: number;
|
|
556
|
+
scanGeneration?: bigint;
|
|
557
|
+
scanRevision?: number;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
const nativeDispatchConcurrency = 2;
|
|
561
|
+
/** Post-native delivery shares a finite connection budget across independent host lanes. */
|
|
562
|
+
const afterNativeDispatchConcurrency = 2;
|
|
563
|
+
|
|
564
|
+
/** Event-local observations only; durable ingress keeps racing mutations dirty. */
|
|
565
|
+
interface NativeRecovery {
|
|
566
|
+
readonly queue: Deferred.Deferred<ReadonlyArray<ThreadId>>;
|
|
567
|
+
readonly pending: Set<ThreadId>;
|
|
568
|
+
readonly loaded: Set<ThreadId>;
|
|
569
|
+
readonly reports: Map<SubmissionId, RecoveryReport>;
|
|
570
|
+
readonly faults: Map<ThreadId, ThreadRecoveryFault>;
|
|
571
|
+
readonly recipients: Map<ThreadId, ReadonlySet<SubmissionId>>;
|
|
572
|
+
observation?: {
|
|
573
|
+
readonly generation: bigint;
|
|
574
|
+
readonly activeAtStart: number;
|
|
575
|
+
};
|
|
576
|
+
started: boolean;
|
|
577
|
+
needsCheckpoint: boolean;
|
|
578
|
+
recovered: number;
|
|
579
|
+
repaired: boolean;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
interface MaintenanceObservation {
|
|
583
|
+
readonly queue: Map<string, DueQueue.DueLane>;
|
|
584
|
+
readonly charged: Map<string, number>;
|
|
585
|
+
/** Initially available or serviced revisions; active producers are only failure observations. */
|
|
586
|
+
readonly available: Map<string, number>;
|
|
587
|
+
generation?: bigint;
|
|
588
|
+
nativeOnly: boolean;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
class MaintenanceRetry extends Schema.Class<MaintenanceRetry>("MaintenanceRetry")({
|
|
592
|
+
generation: MaintenanceGeneration,
|
|
593
|
+
notBefore: Schema.Finite,
|
|
594
|
+
nativeOnly: Schema.Boolean,
|
|
595
|
+
stalls: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(30)),
|
|
596
|
+
}) {}
|
|
597
|
+
|
|
598
|
+
/** Versioned, platform-private maintenance state stored through Durable Object KV. */
|
|
599
|
+
class ThreadMaintenanceState extends Schema.Class<ThreadMaintenanceState>(
|
|
600
|
+
"@effect-agent/platform-cloudflare/ThreadMaintenanceState",
|
|
601
|
+
)({
|
|
602
|
+
schemaVersion: Schema.Literal(1),
|
|
603
|
+
dirty: MaintenanceGeneration,
|
|
604
|
+
processed: MaintenanceGeneration,
|
|
605
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
606
|
+
/** One physical-owner cursor; old single-lane records need no conversion. */
|
|
607
|
+
lastServedThreadId: Schema.optionalKey(ThreadId),
|
|
608
|
+
/** Rotate old recovery independently of dispatch, including after eviction or timeout. */
|
|
609
|
+
lastRecoveredThreadId: Schema.optionalKey(ThreadId),
|
|
610
|
+
/** Rotate bounded post-native admission across full events and Object eviction. */
|
|
611
|
+
lastAfterNativeLaneId: Schema.optionalKey(DueQueue.LaneId),
|
|
612
|
+
bindingRegistryKey: Schema.optionalKey(Schema.String),
|
|
613
|
+
bindingWaits: Schema.optionalKey(Schema.Array(BindingWait)),
|
|
614
|
+
recoveryEventSequence: Schema.optionalKey(MaintenanceGeneration),
|
|
615
|
+
recoveryEventAcknowledged: Schema.optionalKey(MaintenanceGeneration),
|
|
616
|
+
/** Absent on older records. A newer mutation makes this retry obsolete. */
|
|
617
|
+
retry: Schema.optionalKey(MaintenanceRetry),
|
|
618
|
+
}) {}
|
|
619
|
+
|
|
620
|
+
const MAINTENANCE_STATE_KEY = "effect-agent:thread-maintenance:v1";
|
|
621
|
+
const decodeMaintenanceState = Schema.decodeUnknownSync(ThreadMaintenanceState);
|
|
622
|
+
const encodeMaintenanceState = Schema.encodeSync(ThreadMaintenanceState);
|
|
623
|
+
|
|
624
|
+
const hasRecoveryEvents = (state: ThreadMaintenanceState) =>
|
|
625
|
+
(state.recoveryEventSequence ?? 0n) > (state.recoveryEventAcknowledged ?? 0n);
|
|
626
|
+
|
|
627
|
+
const initialMaintenanceState = (): ThreadMaintenanceState =>
|
|
628
|
+
ThreadMaintenanceState.make({
|
|
629
|
+
schemaVersion: 1,
|
|
630
|
+
// Bootstrap Objects created by the pre-generation release without scanning the ledger in
|
|
631
|
+
// the constructor. One useful pass classifies and acknowledges any existing obligation.
|
|
632
|
+
dirty: 1n,
|
|
633
|
+
processed: 0n,
|
|
634
|
+
nonterminal: 0,
|
|
635
|
+
});
|
|
636
|
+
|
|
637
|
+
const readMaintenanceState = async (
|
|
638
|
+
transaction: Pick<DurableObjectTransaction, "get">,
|
|
639
|
+
): Promise<{ readonly state: ThreadMaintenanceState; readonly initialized: boolean }> => {
|
|
640
|
+
const encoded = await transaction.get(MAINTENANCE_STATE_KEY);
|
|
641
|
+
|
|
642
|
+
return encoded === undefined
|
|
643
|
+
? { state: initialMaintenanceState(), initialized: false }
|
|
644
|
+
: { state: decodeMaintenanceState(encoded), initialized: true };
|
|
645
|
+
};
|
|
646
|
+
|
|
647
|
+
const ensureTransactionAlarmBy = async (
|
|
648
|
+
transaction: Pick<DurableObjectTransaction, "getAlarm" | "setAlarm">,
|
|
649
|
+
deadline: number,
|
|
650
|
+
): Promise<void> => {
|
|
651
|
+
const scheduled = await transaction.getAlarm();
|
|
652
|
+
|
|
653
|
+
if (scheduled === null || scheduled > deadline) {
|
|
654
|
+
await transaction.setAlarm(deadline);
|
|
655
|
+
}
|
|
656
|
+
};
|
|
657
|
+
|
|
658
|
+
const stableExternalWait = (
|
|
659
|
+
snapshot: SubmissionWorkItem,
|
|
660
|
+
reports: ReadonlyMap<string, RecoveryReport>,
|
|
661
|
+
): boolean => {
|
|
662
|
+
const report = reports.get(snapshot.submissionId);
|
|
663
|
+
const decision = report?.decision._tag;
|
|
664
|
+
|
|
665
|
+
// An accepted abort still owes cleanup/settlement even if its claim was deferred this pass.
|
|
666
|
+
if (decision === "SettleAborted") return false;
|
|
667
|
+
switch (snapshot.state) {
|
|
668
|
+
case "suspended":
|
|
669
|
+
case "joined":
|
|
670
|
+
return true;
|
|
671
|
+
case "unknown":
|
|
672
|
+
return report?.disposition === "unknown";
|
|
673
|
+
case "admitted":
|
|
674
|
+
return reports.get(snapshot.submissionId)?.decision._tag === "AwaitParentEstablishment";
|
|
675
|
+
case "input-applied":
|
|
676
|
+
case "joining":
|
|
677
|
+
case "ready":
|
|
678
|
+
case "running":
|
|
679
|
+
case "settled":
|
|
680
|
+
case "terminalizing":
|
|
681
|
+
return false;
|
|
682
|
+
}
|
|
683
|
+
};
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* Shared prearm/acknowledgement boundary for ingress and runtime-owned producers.
|
|
687
|
+
* `ThreadObject.layer` provides this same instance in its Services. Rebuilt runtime/maintenance
|
|
688
|
+
* Layers must reuse that instance; a second gate cannot observe the native producers' activity.
|
|
689
|
+
*/
|
|
690
|
+
const CurrentMutationLanes = Context.Reference<ReadonlyArray<string>>(
|
|
691
|
+
"@effect-agent/platform-cloudflare/CurrentMutationLanes",
|
|
692
|
+
{
|
|
693
|
+
defaultValue: () => [],
|
|
694
|
+
},
|
|
695
|
+
);
|
|
696
|
+
|
|
697
|
+
// Source changes made by this pass are certified after its Attempts join. External
|
|
698
|
+
// producers still advance the dirty generation and fence that certification.
|
|
699
|
+
const CurrentNativeSource = Context.Reference<boolean>(
|
|
700
|
+
"@effect-agent/platform-cloudflare/CurrentNativeSource",
|
|
701
|
+
{
|
|
702
|
+
defaultValue: () => false,
|
|
703
|
+
},
|
|
704
|
+
);
|
|
705
|
+
|
|
706
|
+
export class ThreadMutationGate extends Context.Service<
|
|
707
|
+
ThreadMutationGate,
|
|
708
|
+
{
|
|
709
|
+
/** Commit source facts and their scheduling intent together. The flush runs before
|
|
710
|
+
* native commit; failed or interrupted transactions discard their queue view. */
|
|
711
|
+
readonly withTransaction: <A, E, R>(
|
|
712
|
+
body: Effect.Effect<A, E, R>,
|
|
713
|
+
) => Effect.Effect<A, E | SqlError, R>;
|
|
714
|
+
readonly withMutation: <A, E, R>(
|
|
715
|
+
body: Effect.Effect<A, E, R>,
|
|
716
|
+
/**
|
|
717
|
+
* Call outside SQL transactions so prearming commits before the body runs.
|
|
718
|
+
* Native admission, approval, abort and unknown resolution keep the default true.
|
|
719
|
+
* Host producers use false and name only the lanes for which they create work.
|
|
720
|
+
* Receipt-only bookkeeping with no new obligation names no lanes. Enrollment preserves
|
|
721
|
+
* a pending retry budget; idle completion or actual source progress replenishes it.
|
|
722
|
+
* Actual new work must record that progress in its source transaction, so a commit
|
|
723
|
+
* after parking cannot be stranded by eviction before this body returns.
|
|
724
|
+
*/
|
|
725
|
+
options?: {
|
|
726
|
+
readonly invalidatesRecovery?: boolean;
|
|
727
|
+
readonly lanes?: ReadonlyArray<string>;
|
|
728
|
+
},
|
|
729
|
+
) => Effect.Effect<A, E | DurableAlarmError, R>;
|
|
730
|
+
/** Enroll a known lane without accelerating its retry floor. A strictly increasing
|
|
731
|
+
* source cursor replenishes its budget; equal/older notices are replayed hints.
|
|
732
|
+
* Cursors identify committed source facts, never clocks, attempts or retry counters.
|
|
733
|
+
* Local sources call inside their source transaction. Remote sources retain the cursor
|
|
734
|
+
* notice with their work and retry delivery here; this call atomically accepts that
|
|
735
|
+
* notice and its alarm. Prearming alone cannot fence a later remote commit. */
|
|
736
|
+
readonly schedule: (
|
|
737
|
+
id: string,
|
|
738
|
+
dueAt: number,
|
|
739
|
+
progressCursor?: bigint,
|
|
740
|
+
) => Effect.Effect<void, DurableAlarmError>;
|
|
741
|
+
/** Record an actual new fact inside its local source SQL transaction. The source
|
|
742
|
+
* transaction owns duplicate detection. Claims, lease renewals and retries are not progress.
|
|
743
|
+
* Also enrolls the lanes selected by this producer's enclosing withMutation boundary. */
|
|
744
|
+
readonly recordProgress: (
|
|
745
|
+
lanes: ReadonlyArray<string>,
|
|
746
|
+
) => Effect.Effect<void, DurableAlarmError>;
|
|
747
|
+
/** In-incarnation notification; the durable due queue is the recovery authority. */
|
|
748
|
+
readonly activeLanes: Effect.Effect<ReadonlySet<string>>;
|
|
749
|
+
readonly revision: Effect.Effect<number>;
|
|
750
|
+
readonly awaitChange: (revision: number) => Effect.Effect<void>;
|
|
751
|
+
readonly withSnapshot: <A, E, R>(
|
|
752
|
+
body: (active: number) => Effect.Effect<A, E, R>,
|
|
753
|
+
) => Effect.Effect<A, E, R>;
|
|
754
|
+
}
|
|
755
|
+
>()("@effect-agent/platform-cloudflare/internal/ThreadMutationGate") {
|
|
756
|
+
static readonly layer = Layer.effect(this)(
|
|
757
|
+
Effect.gen(function* () {
|
|
758
|
+
const { ctx } = yield* DurableObjectContext;
|
|
759
|
+
const sql = yield* SqlClient;
|
|
760
|
+
const dueQueue = DueQueue.make(ctx.storage);
|
|
761
|
+
|
|
762
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
763
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
764
|
+
// A fresh incarnation has no live mutations; durable generations survive eviction.
|
|
765
|
+
const activeMutations = yield* Ref.make(0);
|
|
766
|
+
const activeLanes = new Map<string, number>();
|
|
767
|
+
const generationGate = yield* Semaphore.make(1);
|
|
768
|
+
let revision = 0;
|
|
769
|
+
let changed = yield* Deferred.make<void>();
|
|
770
|
+
|
|
771
|
+
const notify = Effect.gen(function* () {
|
|
772
|
+
const previous = changed;
|
|
773
|
+
|
|
774
|
+
changed = yield* Deferred.make<void>();
|
|
775
|
+
revision++;
|
|
776
|
+
yield* Deferred.succeed(previous, undefined);
|
|
777
|
+
});
|
|
778
|
+
|
|
779
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
780
|
+
|
|
781
|
+
const runTransaction = yield* makeStorageOperation;
|
|
782
|
+
|
|
783
|
+
const validateSourceBoundary = Effect.flatMap(
|
|
784
|
+
Effect.serviceOption(sql.transactionService),
|
|
785
|
+
(current) =>
|
|
786
|
+
current._tag === "Some" &&
|
|
787
|
+
DueQueue.buffering(ctx.storage) &&
|
|
788
|
+
!DueQueue.ownsTransaction(ctx.storage, current.value[1])
|
|
789
|
+
? Effect.fail(
|
|
790
|
+
DurableAlarmError.make({
|
|
791
|
+
operation: "enroll maintenance source",
|
|
792
|
+
message: "Nested source scheduling requires ThreadMutationGate.withTransaction",
|
|
793
|
+
}),
|
|
794
|
+
)
|
|
795
|
+
: Effect.void,
|
|
796
|
+
);
|
|
797
|
+
|
|
798
|
+
yield* runTransaction("initialize maintenance due queue", async () => {
|
|
799
|
+
dueQueue.initialize();
|
|
800
|
+
dueQueue.register(DueQueue.Native);
|
|
801
|
+
});
|
|
802
|
+
|
|
803
|
+
const schedule = Effect.fn("ThreadMutationGate.schedule")(function* (
|
|
804
|
+
id: string,
|
|
805
|
+
dueAt: number,
|
|
806
|
+
progressCursor?: bigint,
|
|
807
|
+
) {
|
|
808
|
+
yield* validateSourceBoundary;
|
|
809
|
+
const now = yield* Clock.currentTimeMillis;
|
|
810
|
+
const current = yield* Effect.serviceOption(sql.transactionService);
|
|
811
|
+
|
|
812
|
+
const enroll = async (
|
|
813
|
+
transaction: Pick<DurableObjectTransaction, "get" | "put" | "getAlarm" | "setAlarm">,
|
|
814
|
+
) => {
|
|
815
|
+
if (progressCursor === undefined) dueQueue.dirty(id, dueAt);
|
|
816
|
+
else if (
|
|
817
|
+
dueQueue.progress(id, dueAt, progressCursor) &&
|
|
818
|
+
(id === DueQueue.Native || id === DueQueue.Publication)
|
|
819
|
+
) {
|
|
820
|
+
// Required publication is progress for its dependent native execution too.
|
|
821
|
+
if (id === DueQueue.Publication) dueQueue.dirty(DueQueue.Native, dueAt, true);
|
|
822
|
+
const { state } = await readMaintenanceState(transaction);
|
|
823
|
+
|
|
824
|
+
await transaction.put(
|
|
825
|
+
MAINTENANCE_STATE_KEY,
|
|
826
|
+
encodeMaintenanceState(
|
|
827
|
+
ThreadMaintenanceState.make({
|
|
828
|
+
...Struct.omit(state, ["retry"]),
|
|
829
|
+
dirty: state.dirty + 1n,
|
|
830
|
+
}),
|
|
831
|
+
),
|
|
832
|
+
);
|
|
833
|
+
}
|
|
834
|
+
const next = DueQueue.next(dueQueue.read());
|
|
835
|
+
|
|
836
|
+
if (Number.isFinite(next))
|
|
837
|
+
await ensureTransactionAlarmBy(transaction, Math.max(next, now + minimumAlarmDelay));
|
|
838
|
+
};
|
|
839
|
+
|
|
840
|
+
yield* runTransaction("schedule maintenance lane", () =>
|
|
841
|
+
Option.isSome(current) ? enroll(ctx.storage) : dueQueue.transaction(enroll),
|
|
842
|
+
);
|
|
843
|
+
yield* notify;
|
|
844
|
+
});
|
|
845
|
+
|
|
846
|
+
const beginMutation = Effect.fn("ThreadMaintenance.beginMutation")(function* (
|
|
847
|
+
invalidatesRecovery: boolean,
|
|
848
|
+
lanes: ReadonlyArray<string>,
|
|
849
|
+
) {
|
|
850
|
+
yield* failpoint.hit("maintenance:dirty:before");
|
|
851
|
+
const now = yield* Clock.currentTimeMillis;
|
|
852
|
+
const nativeSource = yield* CurrentNativeSource;
|
|
853
|
+
const publicationRequired = yield* requiresPublication;
|
|
854
|
+
|
|
855
|
+
if (invalidatesRecovery || lanes.length > 0)
|
|
856
|
+
yield* runTransaction("advance maintenance generation", () =>
|
|
857
|
+
dueQueue.transaction(async (transaction) => {
|
|
858
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
859
|
+
|
|
860
|
+
if (invalidatesRecovery) {
|
|
861
|
+
dueQueue.dirty(DueQueue.Native, now);
|
|
862
|
+
if (publicationRequired) dueQueue.dirty(DueQueue.Publication, now);
|
|
863
|
+
}
|
|
864
|
+
// Enrollment is a crash fallback, not evidence that the body committed.
|
|
865
|
+
for (const id of lanes) dueQueue.dirty(id, now, false);
|
|
866
|
+
|
|
867
|
+
const next = ThreadMaintenanceState.make({
|
|
868
|
+
...state,
|
|
869
|
+
dirty: state.dirty + (invalidatesRecovery && !nativeSource ? 1n : 0n),
|
|
870
|
+
});
|
|
871
|
+
|
|
872
|
+
if (invalidatesRecovery || !initialized)
|
|
873
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
874
|
+
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
875
|
+
// on its best-effort immediate wake hint.
|
|
876
|
+
const deadline = DueQueue.next(dueQueue.read());
|
|
877
|
+
|
|
878
|
+
if (Number.isFinite(deadline))
|
|
879
|
+
await ensureTransactionAlarmBy(
|
|
880
|
+
transaction,
|
|
881
|
+
Math.max(deadline, now + minimumAlarmDelay),
|
|
882
|
+
);
|
|
883
|
+
}),
|
|
884
|
+
);
|
|
885
|
+
yield* failpoint.hit("maintenance:dirty:after");
|
|
886
|
+
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
887
|
+
|
|
888
|
+
const enrolled = new Set([
|
|
889
|
+
...lanes,
|
|
890
|
+
...(invalidatesRecovery
|
|
891
|
+
? [DueQueue.Native, ...(publicationRequired ? [DueQueue.Publication] : [])]
|
|
892
|
+
: []),
|
|
893
|
+
]);
|
|
894
|
+
|
|
895
|
+
for (const id of enrolled) activeLanes.set(id, (activeLanes.get(id) ?? 0) + 1);
|
|
896
|
+
|
|
897
|
+
return enrolled;
|
|
898
|
+
});
|
|
899
|
+
|
|
900
|
+
const releaseLanes = (enrolled: Set<string>, committed = false) =>
|
|
901
|
+
Effect.sync(() => {
|
|
902
|
+
for (const id of enrolled) {
|
|
903
|
+
if (committed && (id === DueQueue.Native || id === DueQueue.Publication)) continue;
|
|
904
|
+
const remaining = (activeLanes.get(id) ?? 1) - 1;
|
|
905
|
+
|
|
906
|
+
if (remaining === 0) activeLanes.delete(id);
|
|
907
|
+
else activeLanes.set(id, remaining);
|
|
908
|
+
enrolled.delete(id);
|
|
909
|
+
}
|
|
910
|
+
});
|
|
911
|
+
|
|
912
|
+
const endMutation = (enrolled: Set<string>) =>
|
|
913
|
+
generationGate.withPermit(
|
|
914
|
+
releaseLanes(enrolled).pipe(
|
|
915
|
+
Effect.andThen(Ref.update(activeMutations, (active) => Math.max(0, active - 1))),
|
|
916
|
+
),
|
|
917
|
+
);
|
|
918
|
+
|
|
919
|
+
const withMutation = <A, E, R>(
|
|
920
|
+
body: Effect.Effect<A, E, R>,
|
|
921
|
+
options?: {
|
|
922
|
+
readonly invalidatesRecovery?: boolean;
|
|
923
|
+
readonly lanes?: ReadonlyArray<string>;
|
|
924
|
+
},
|
|
925
|
+
): Effect.Effect<A, E | DurableAlarmError, R> =>
|
|
926
|
+
Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) =>
|
|
927
|
+
// Reject before the generation gate: a prearmer may already be waiting
|
|
928
|
+
// for this caller's SQL connection while holding that gate.
|
|
929
|
+
Option.isSome(current)
|
|
930
|
+
? Effect.fail(
|
|
931
|
+
DurableAlarmError.make({
|
|
932
|
+
operation: "prearm maintenance mutation",
|
|
933
|
+
message:
|
|
934
|
+
"Run withMutation outside the source SQL transaction so prearming commits first",
|
|
935
|
+
}),
|
|
936
|
+
)
|
|
937
|
+
: Effect.acquireUseRelease(
|
|
938
|
+
generationGate.withPermit(
|
|
939
|
+
beginMutation(options?.invalidatesRecovery ?? true, options?.lanes ?? []),
|
|
940
|
+
),
|
|
941
|
+
(enrolled) =>
|
|
942
|
+
failpoint.hit("maintenance:mutation:armed").pipe(
|
|
943
|
+
Effect.andThen(
|
|
944
|
+
Effect.flatMap(CurrentMutationLanes, (current) =>
|
|
945
|
+
body.pipe(
|
|
946
|
+
Effect.provideService(CurrentMutationLanes, [
|
|
947
|
+
...new Set([...current, ...enrolled]),
|
|
948
|
+
]),
|
|
949
|
+
),
|
|
950
|
+
),
|
|
951
|
+
),
|
|
952
|
+
// The committed outbox may be claimed by the alarm even while the caller
|
|
953
|
+
// is suspended after commit. This synchronous decrement does not acquire the
|
|
954
|
+
// native snapshot gate: a snapshot may conservatively retain its active count.
|
|
955
|
+
// Native certification keeps its broader guard until the mutation is released.
|
|
956
|
+
Effect.tap(() => releaseLanes(enrolled, true).pipe(Effect.andThen(notify))),
|
|
957
|
+
Effect.tap(() => failpoint.hit("maintenance:mutation:finished")),
|
|
958
|
+
),
|
|
959
|
+
(enrolled) => endMutation(enrolled).pipe(Effect.andThen(notify)),
|
|
960
|
+
),
|
|
961
|
+
);
|
|
962
|
+
|
|
963
|
+
const recordProgress = Effect.fn("ThreadMutationGate.recordProgress")(function* (
|
|
964
|
+
lanes: ReadonlyArray<string>,
|
|
965
|
+
) {
|
|
966
|
+
yield* validateSourceBoundary;
|
|
967
|
+
if (Option.isNone(yield* Effect.serviceOption(sql.transactionService)))
|
|
968
|
+
return yield* DurableAlarmError.make({
|
|
969
|
+
operation: "record source progress",
|
|
970
|
+
message: "Source progress requires its authoritative SQL transaction",
|
|
971
|
+
});
|
|
972
|
+
const affected = new Set([...lanes, ...(yield* CurrentMutationLanes)]);
|
|
973
|
+
|
|
974
|
+
if (!(yield* requiresPublication)) affected.delete(DueQueue.Publication);
|
|
975
|
+
|
|
976
|
+
if (affected.has(DueQueue.Publication)) affected.add(DueQueue.Native);
|
|
977
|
+
|
|
978
|
+
if (affected.size === 0) return;
|
|
979
|
+
const now = yield* Clock.currentTimeMillis;
|
|
980
|
+
const nativeSource = yield* CurrentNativeSource;
|
|
981
|
+
|
|
982
|
+
yield* runTransaction("record maintenance producer progress", async () => {
|
|
983
|
+
for (const id of affected) dueQueue.dirty(id, now, true);
|
|
984
|
+
if (affected.has(DueQueue.Native)) {
|
|
985
|
+
const { state } = await readMaintenanceState(ctx.storage);
|
|
986
|
+
|
|
987
|
+
await ctx.storage.put(
|
|
988
|
+
MAINTENANCE_STATE_KEY,
|
|
989
|
+
encodeMaintenanceState(
|
|
990
|
+
ThreadMaintenanceState.make({
|
|
991
|
+
...Struct.omit(state, ["retry"]),
|
|
992
|
+
dirty: state.dirty + (nativeSource ? 0n : 1n),
|
|
993
|
+
}),
|
|
994
|
+
),
|
|
995
|
+
);
|
|
996
|
+
}
|
|
997
|
+
await ensureTransactionAlarmBy(ctx.storage, now + minimumAlarmDelay);
|
|
998
|
+
});
|
|
999
|
+
yield* notify;
|
|
1000
|
+
});
|
|
1001
|
+
|
|
1002
|
+
return ThreadMutationGate.of({
|
|
1003
|
+
withTransaction: <A, E, R>(body: Effect.Effect<A, E, R>) =>
|
|
1004
|
+
dueQueue.withTransaction(body).pipe(Effect.provideService(SqlClient, sql)),
|
|
1005
|
+
withMutation,
|
|
1006
|
+
schedule,
|
|
1007
|
+
recordProgress,
|
|
1008
|
+
activeLanes: Effect.sync(() => new Set(activeLanes.keys())),
|
|
1009
|
+
revision: Effect.sync(() => revision),
|
|
1010
|
+
awaitChange: (observed) =>
|
|
1011
|
+
Effect.suspend(() => (observed === revision ? Deferred.await(changed) : Effect.void)),
|
|
1012
|
+
withSnapshot: (body) =>
|
|
1013
|
+
generationGate.withPermit(Effect.flatMap(Ref.get(activeMutations), body)),
|
|
1014
|
+
});
|
|
1015
|
+
}),
|
|
1016
|
+
);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
export type MaintenancePassFailure =
|
|
1020
|
+
| DurableWorkerFailure
|
|
1021
|
+
| DurableBindingFailure
|
|
1022
|
+
| DurableAlarmError
|
|
1023
|
+
| ThreadProjectionError;
|
|
1024
|
+
|
|
1025
|
+
/**
|
|
1026
|
+
* Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
|
|
1027
|
+
*
|
|
1028
|
+
* One physical event owns native scheduling, auxiliary delivery and final alarm rearming.
|
|
1029
|
+
*
|
|
1030
|
+
* 1. Prearm before any work. A caught-up native step uses the O(1) generation record without
|
|
1031
|
+
* recovery, ledger scans or canonical-history reads.
|
|
1032
|
+
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
1033
|
+
* producer keeps its generation dirty and immediately eligible. Quiescent native retries
|
|
1034
|
+
* retain their durable backoff.
|
|
1035
|
+
* 3. Admit at most two independent Thread Attempts, one active head per Thread.
|
|
1036
|
+
* Keep native and delivery admission open together while finite waves remain active, so
|
|
1037
|
+
* fresh replies and abort controls can progress during unrelated cleanup. Close atomically
|
|
1038
|
+
* at quiescence, or at the original ten-minute yield deadline, then join admitted waves.
|
|
1039
|
+
* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill waves are
|
|
1040
|
+
* bounded independently; incoming native work never restarts or cancels their attempts.
|
|
1041
|
+
* Auxiliary failures are reported after the current native opportunity.
|
|
1042
|
+
* 5. Close every event resource before the final gated deadline snapshot and alarm decision.
|
|
1043
|
+
* The whole event retains one fourteen-minute cooperative timeout.
|
|
1044
|
+
*/
|
|
1045
|
+
export class ThreadMaintenance extends Context.Service<
|
|
1046
|
+
ThreadMaintenance,
|
|
1047
|
+
{
|
|
1048
|
+
/** One idempotent pass; failures propagate after durably scheduling bounded recovery. */
|
|
1049
|
+
readonly pass: Effect.Effect<MaintenancePassReport, MaintenancePassFailure>;
|
|
1050
|
+
/**
|
|
1051
|
+
* Constructor gate: initialize/inspect only the O(1) maintenance record and ensure a dirty
|
|
1052
|
+
* generation has an alarm. It never scans the ledger or canonical history.
|
|
1053
|
+
*/
|
|
1054
|
+
readonly ensureAlarm: Effect.Effect<void, MaintenancePassFailure>;
|
|
1055
|
+
/**
|
|
1056
|
+
* Serialize the pre-arm boundary with pass acknowledgement, advance the durable dirty
|
|
1057
|
+
* generation and arm the alarm in one transaction BEFORE running the caller's mutation.
|
|
1058
|
+
* A pass cannot acknowledge while that mutation remains in flight.
|
|
1059
|
+
*/
|
|
1060
|
+
readonly withMutation: <A, E, R>(
|
|
1061
|
+
body: Effect.Effect<A, E, R>,
|
|
1062
|
+
) => Effect.Effect<A, E | DurableAlarmError, R>;
|
|
1063
|
+
}
|
|
1064
|
+
>()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
|
|
1065
|
+
static readonly layer: Layer.Layer<
|
|
1066
|
+
ThreadMaintenance,
|
|
1067
|
+
DurableAlarmError,
|
|
1068
|
+
| ThreadMutationGate
|
|
1069
|
+
| ThreadPublication
|
|
1070
|
+
| ThreadProjectionMaintenance
|
|
1071
|
+
| DurableAgentRuntime
|
|
1072
|
+
| DurableRuntimeConfig
|
|
1073
|
+
| SubmissionLedger
|
|
1074
|
+
| WakeScheduler
|
|
1075
|
+
| DurableAlarmService
|
|
1076
|
+
| ThreadMaintenanceFailpoint
|
|
1077
|
+
| CloudflareDurableRuntimeConfig
|
|
1078
|
+
| DurableObjectContext
|
|
1079
|
+
| SqlClient
|
|
1080
|
+
> = Layer.effect(ThreadMaintenance)(
|
|
1081
|
+
Effect.gen(function* () {
|
|
1082
|
+
const runtime = yield* DurableAgentRuntime;
|
|
1083
|
+
const recoveryConfig = yield* DurableRuntimeConfig;
|
|
1084
|
+
const ledger = yield* SubmissionLedger;
|
|
1085
|
+
const wakes = yield* WakeScheduler;
|
|
1086
|
+
const alarm = yield* DurableAlarmService;
|
|
1087
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
1088
|
+
const { ctx } = yield* DurableObjectContext;
|
|
1089
|
+
const dueQueue = DueQueue.make(ctx.storage);
|
|
1090
|
+
const storage = DurableObjectStorage.fromDurableObjectStorage(ctx.storage);
|
|
1091
|
+
const runStorage = yield* makeStorageEffect;
|
|
1092
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
1093
|
+
|
|
1094
|
+
const mutations = yield* ThreadMutationGate;
|
|
1095
|
+
const publication = yield* ThreadPublication;
|
|
1096
|
+
|
|
1097
|
+
const publicationContext = yield* Effect.context<
|
|
1098
|
+
ThreadPublication | DurableObjectContext | SqlClient
|
|
1099
|
+
>();
|
|
1100
|
+
|
|
1101
|
+
const projection = yield* ThreadProjectionMaintenance;
|
|
1102
|
+
const messages = yield* ThreadMessageDelivery;
|
|
1103
|
+
const host = yield* ThreadHostMaintenance;
|
|
1104
|
+
const framework = yield* ThreadNativeMaintenance;
|
|
1105
|
+
|
|
1106
|
+
for (const lane of host.lanes)
|
|
1107
|
+
yield* Schema.decodeEffect(DueQueue.HostLaneId)(lane.id).pipe(
|
|
1108
|
+
Effect.mapError(alarmFailure("maintenance lane ID")),
|
|
1109
|
+
);
|
|
1110
|
+
|
|
1111
|
+
const recoveryEvents = Option.getOrElse(
|
|
1112
|
+
yield* Effect.serviceOption(ThreadRecoveryEvents),
|
|
1113
|
+
() => ThreadRecoveryEvents.of({ publish: () => Effect.void }),
|
|
1114
|
+
);
|
|
1115
|
+
|
|
1116
|
+
// Hydrated by the existing maintenance-record reads, never by a status scan. All
|
|
1117
|
+
// queue mutations update this hint under the same storage reservation. Eviction
|
|
1118
|
+
// reconstructs it in ensureAlarm/beginPass; the durable queue remains authoritative.
|
|
1119
|
+
let recoveryEventsPending = false;
|
|
1120
|
+
|
|
1121
|
+
const maintenancePassGate = yield* Semaphore.make(1);
|
|
1122
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
1123
|
+
const runTransaction = yield* makeStorageOperation;
|
|
1124
|
+
|
|
1125
|
+
const queueSnapshot = runTransaction("read maintenance due queue", async () =>
|
|
1126
|
+
dueQueue.read(),
|
|
1127
|
+
);
|
|
1128
|
+
|
|
1129
|
+
const reportParked = Effect.gen(function* () {
|
|
1130
|
+
if (!(yield* queueSnapshot).some((row) => row.state === "parked" && row.reported === 0))
|
|
1131
|
+
return;
|
|
1132
|
+
|
|
1133
|
+
const rows = yield* runTransaction("mark parked maintenance reports", () =>
|
|
1134
|
+
dueQueue.transaction(async () => dueQueue.takeParkedReports()),
|
|
1135
|
+
);
|
|
1136
|
+
|
|
1137
|
+
for (const row of rows) {
|
|
1138
|
+
const lane = [
|
|
1139
|
+
DueQueue.Native,
|
|
1140
|
+
DueQueue.Publication,
|
|
1141
|
+
DueQueue.Projection,
|
|
1142
|
+
DueQueue.Messages,
|
|
1143
|
+
DueQueue.RecoveryEvents,
|
|
1144
|
+
DueQueue.Lifecycle,
|
|
1145
|
+
].includes(row.id)
|
|
1146
|
+
? row.id.slice("effect-agent:".length)
|
|
1147
|
+
: "host";
|
|
1148
|
+
|
|
1149
|
+
yield* ErrorReporter.report(
|
|
1150
|
+
Cause.fail(
|
|
1151
|
+
MaintenanceRetryParked.make({
|
|
1152
|
+
lane,
|
|
1153
|
+
retryAfterMillis: DueQueue.ParkedRetryMillis,
|
|
1154
|
+
}),
|
|
1155
|
+
),
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
1158
|
+
}).pipe(Effect.catchCause((cause) => ErrorReporter.report(cause)));
|
|
1159
|
+
|
|
1160
|
+
yield* runTransaction("register native maintenance lanes", () =>
|
|
1161
|
+
dueQueue.transaction(async () => {
|
|
1162
|
+
for (const id of [
|
|
1163
|
+
DueQueue.Publication,
|
|
1164
|
+
DueQueue.Projection,
|
|
1165
|
+
DueQueue.Messages,
|
|
1166
|
+
...framework.lanes.map((lane) => lane.id),
|
|
1167
|
+
])
|
|
1168
|
+
dueQueue.register(id);
|
|
1169
|
+
|
|
1170
|
+
// Earlier releases split this one outbox across start and after-native lanes.
|
|
1171
|
+
// Retire only scheduling metadata, retaining revision fencing and domain retries.
|
|
1172
|
+
if (framework.lanes.some((lane) => lane.id === DueQueue.Lifecycle)) {
|
|
1173
|
+
const start = dueQueue.read().find((row) => row.id === "effect-agent:lifecycle-start");
|
|
1174
|
+
|
|
1175
|
+
if (start !== undefined && start.dueAt !== null) {
|
|
1176
|
+
dueQueue.dirty(DueQueue.Lifecycle, start.dueAt);
|
|
1177
|
+
dueQueue.complete(start, null);
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
}),
|
|
1181
|
+
);
|
|
1182
|
+
|
|
1183
|
+
const runQueued = <E, R>(
|
|
1184
|
+
selected: DueQueue.DueLane,
|
|
1185
|
+
work: Effect.Effect<Option.Option<number>, E, R>,
|
|
1186
|
+
observed: MaintenanceObservation,
|
|
1187
|
+
yielded?: () => boolean,
|
|
1188
|
+
) => {
|
|
1189
|
+
let row = selected;
|
|
1190
|
+
let charged = false;
|
|
1191
|
+
|
|
1192
|
+
return Effect.gen(function* () {
|
|
1193
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1194
|
+
|
|
1195
|
+
const claimed = yield* runTransaction("charge maintenance attempt", () =>
|
|
1196
|
+
dueQueue.transaction(async (transaction) => {
|
|
1197
|
+
const claimed = dueQueue.claim(selected, now);
|
|
1198
|
+
const next = DueQueue.next(dueQueue.read());
|
|
1199
|
+
|
|
1200
|
+
if (Number.isFinite(next))
|
|
1201
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
|
|
1202
|
+
else await transaction.deleteAlarm();
|
|
1203
|
+
|
|
1204
|
+
return claimed;
|
|
1205
|
+
}),
|
|
1206
|
+
);
|
|
1207
|
+
|
|
1208
|
+
if (claimed === undefined) return Option.none<number>();
|
|
1209
|
+
row = claimed;
|
|
1210
|
+
charged = true;
|
|
1211
|
+
observed.charged.set(row.id, row.revision);
|
|
1212
|
+
// Include mid-pass enrollments in failure rearming if their checkpoint is
|
|
1213
|
+
// interrupted. CAS still protects completed waves and racing producers.
|
|
1214
|
+
observed.queue.set(row.id, row);
|
|
1215
|
+
observed.available.set(row.id, row.revision);
|
|
1216
|
+
|
|
1217
|
+
return yield* work;
|
|
1218
|
+
}).pipe(
|
|
1219
|
+
Effect.onExit((exit) => {
|
|
1220
|
+
if (!charged) return Effect.void;
|
|
1221
|
+
if (
|
|
1222
|
+
yielded?.() === true &&
|
|
1223
|
+
Exit.isFailure(exit) &&
|
|
1224
|
+
Cause.hasInterruptsOnly(exit.cause)
|
|
1225
|
+
) {
|
|
1226
|
+
// A promptness yield is neither completion nor a failed delivery. Keep the
|
|
1227
|
+
// exact due revision; pass-level failure rearming must not back off this wave.
|
|
1228
|
+
observed.queue.delete(row.id);
|
|
1229
|
+
|
|
1230
|
+
return Effect.void;
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
return mutations
|
|
1234
|
+
.withSnapshot(() =>
|
|
1235
|
+
Effect.gen(function* () {
|
|
1236
|
+
const active = (yield* mutations.activeLanes).has(row.id);
|
|
1237
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1238
|
+
const failed = Exit.isFailure(exit);
|
|
1239
|
+
|
|
1240
|
+
const next = failed
|
|
1241
|
+
? now + backoffDelay(row.stalls, yield* Random.next)
|
|
1242
|
+
: Option.getOrNull(exit.value);
|
|
1243
|
+
|
|
1244
|
+
const checkpointed = yield* runTransaction("checkpoint maintenance lane", () =>
|
|
1245
|
+
dueQueue.transaction(async () => {
|
|
1246
|
+
// A producer can be between enrollment and its source commit. Never acknowledge
|
|
1247
|
+
// that observation; its completion will notify the event or retain the alarm.
|
|
1248
|
+
if (active) return;
|
|
1249
|
+
const current = dueQueue.read().find((lane) => lane.id === row.id);
|
|
1250
|
+
|
|
1251
|
+
if (current?.revision !== row.revision) return;
|
|
1252
|
+
|
|
1253
|
+
return dueQueue.complete(row, next)?.revision;
|
|
1254
|
+
}),
|
|
1255
|
+
);
|
|
1256
|
+
|
|
1257
|
+
// This pass's own checkpoint is not fresh producer enrollment. Record it
|
|
1258
|
+
// only after commit, while the snapshot gate still excludes a new producer.
|
|
1259
|
+
if (checkpointed !== undefined) observed.available.set(row.id, checkpointed);
|
|
1260
|
+
}),
|
|
1261
|
+
)
|
|
1262
|
+
.pipe(
|
|
1263
|
+
// Waiting for a producer or the SQL connection belongs to the wave budget.
|
|
1264
|
+
// An interrupted checkpoint leaves its revision due for event rearming;
|
|
1265
|
+
// the short storage transaction itself remains atomic.
|
|
1266
|
+
Effect.interruptible,
|
|
1267
|
+
);
|
|
1268
|
+
}),
|
|
1269
|
+
);
|
|
1270
|
+
};
|
|
1271
|
+
|
|
1272
|
+
const appendRecoveryEvents = async (
|
|
1273
|
+
transaction: DurableObjectTransaction,
|
|
1274
|
+
events: ReadonlyArray<Omit<ThreadRecoveryFaultEvent, "sequence">>,
|
|
1275
|
+
) => {
|
|
1276
|
+
if (events.length === 0) return;
|
|
1277
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1278
|
+
let sequence = state.recoveryEventSequence ?? 0n;
|
|
1279
|
+
|
|
1280
|
+
for (const event of events) {
|
|
1281
|
+
sequence++;
|
|
1282
|
+
await transaction.put(
|
|
1283
|
+
recoveryEventKey(sequence),
|
|
1284
|
+
encodeRecoveryEvent(ThreadRecoveryFaultEvent.make({ ...event, sequence })),
|
|
1285
|
+
);
|
|
1286
|
+
}
|
|
1287
|
+
await transaction.put(
|
|
1288
|
+
MAINTENANCE_STATE_KEY,
|
|
1289
|
+
encodeMaintenanceState(
|
|
1290
|
+
ThreadMaintenanceState.make({
|
|
1291
|
+
...state,
|
|
1292
|
+
recoveryEventSequence: sequence,
|
|
1293
|
+
}),
|
|
1294
|
+
),
|
|
1295
|
+
);
|
|
1296
|
+
dueQueue.dirty(DueQueue.RecoveryEvents, 0, true);
|
|
1297
|
+
recoveryEventsPending = true;
|
|
1298
|
+
};
|
|
1299
|
+
|
|
1300
|
+
const recoveryEventLane: ThreadHostMaintenanceLane = {
|
|
1301
|
+
id: DueQueue.RecoveryEvents,
|
|
1302
|
+
dispatchTimeoutMillis: 30_000,
|
|
1303
|
+
run: Effect.gen(function* () {
|
|
1304
|
+
if (!recoveryEventsPending) return Option.none<number>();
|
|
1305
|
+
|
|
1306
|
+
const range = yield* runTransaction("select recovery events", () =>
|
|
1307
|
+
dueQueue.transaction(async (transaction) => {
|
|
1308
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1309
|
+
|
|
1310
|
+
return {
|
|
1311
|
+
next: (state.recoveryEventAcknowledged ?? 0n) + 1n,
|
|
1312
|
+
through: state.recoveryEventSequence ?? 0n,
|
|
1313
|
+
};
|
|
1314
|
+
}),
|
|
1315
|
+
);
|
|
1316
|
+
|
|
1317
|
+
for (let sequence = range.next; sequence <= range.through; sequence++) {
|
|
1318
|
+
const event = yield* runTransaction("read recovery event", async () => {
|
|
1319
|
+
const event = decodeRecoveryEvent(await ctx.storage.get(recoveryEventKey(sequence)));
|
|
1320
|
+
|
|
1321
|
+
if (event.sequence !== sequence)
|
|
1322
|
+
throw new Error("Recovery event does not match its sequence key");
|
|
1323
|
+
|
|
1324
|
+
return event;
|
|
1325
|
+
});
|
|
1326
|
+
|
|
1327
|
+
// No storage reservation or source mutation gate is held during host delivery.
|
|
1328
|
+
yield* recoveryEvents.publish(event);
|
|
1329
|
+
const acknowledgedAt = yield* Clock.currentTimeMillis;
|
|
1330
|
+
|
|
1331
|
+
yield* runTransaction("acknowledge recovery event", () =>
|
|
1332
|
+
dueQueue.transaction(async (transaction) => {
|
|
1333
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1334
|
+
|
|
1335
|
+
const next = ThreadMaintenanceState.make({
|
|
1336
|
+
...state,
|
|
1337
|
+
recoveryEventAcknowledged: sequence,
|
|
1338
|
+
});
|
|
1339
|
+
|
|
1340
|
+
await transaction.delete(recoveryEventKey(sequence));
|
|
1341
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
1342
|
+
recoveryEventsPending = hasRecoveryEvents(next);
|
|
1343
|
+
dueQueue.progress(DueQueue.RecoveryEvents, acknowledgedAt, sequence);
|
|
1344
|
+
const lane = dueQueue.read().find((row) => row.id === DueQueue.RecoveryEvents);
|
|
1345
|
+
|
|
1346
|
+
if (lane !== undefined)
|
|
1347
|
+
dueQueue.complete(lane, recoveryEventsPending ? acknowledgedAt : null);
|
|
1348
|
+
}),
|
|
1349
|
+
);
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
return recoveryEventsPending ? Option.some(0) : Option.none<number>();
|
|
1353
|
+
}),
|
|
1354
|
+
};
|
|
1355
|
+
|
|
1356
|
+
const recordRecoveryFaults = Effect.fn("ThreadMaintenance.recordRecoveryFaults")(function* (
|
|
1357
|
+
result: RecoverySweepResult,
|
|
1358
|
+
recovery: NativeRecovery,
|
|
1359
|
+
current?: ReadonlyArray<SubmissionWorkItem>,
|
|
1360
|
+
) {
|
|
1361
|
+
const threads = new Map<ThreadId, RecoveryFailure | undefined>();
|
|
1362
|
+
|
|
1363
|
+
for (const report of result.reports) threads.set(report.threadId, undefined);
|
|
1364
|
+
for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
|
|
1365
|
+
if (threads.size === 0) return new Map<ThreadId, ThreadRecoveryFault>();
|
|
1366
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1367
|
+
|
|
1368
|
+
// Fault-only discovery uses control state even when retained payloads cannot decode.
|
|
1369
|
+
const submissions =
|
|
1370
|
+
current ??
|
|
1371
|
+
(result.blocked.length === 0 ? [] : yield* Stream.runCollect(ledger.scanNonterminal));
|
|
1372
|
+
|
|
1373
|
+
yield* failpoint.hit("maintenance:recovery-status:before");
|
|
1374
|
+
|
|
1375
|
+
const retained = yield* runTransaction("record Thread recovery faults", () =>
|
|
1376
|
+
dueQueue.transaction(async (transaction) => {
|
|
1377
|
+
const events: Array<Omit<ThreadRecoveryFaultEvent, "sequence">> = [];
|
|
1378
|
+
const newlyBlocked: Array<ThreadRecoveryFault> = [];
|
|
1379
|
+
const faults = new Map<ThreadId, ThreadRecoveryFault>();
|
|
1380
|
+
const recipients = new Map<ThreadId, ReadonlySet<SubmissionId>>();
|
|
1381
|
+
|
|
1382
|
+
for (const [threadId, failure] of threads) {
|
|
1383
|
+
const key = recoveryFaultKey(threadId);
|
|
1384
|
+
const encoded = await transaction.get(key);
|
|
1385
|
+
|
|
1386
|
+
const previous =
|
|
1387
|
+
encoded === undefined ? undefined : decodeRecoveryFault(threadId, encoded);
|
|
1388
|
+
|
|
1389
|
+
const previousIds =
|
|
1390
|
+
previous === undefined ? [] : await readRecoveryRecipients(transaction, previous);
|
|
1391
|
+
|
|
1392
|
+
const known = new Set(previousIds);
|
|
1393
|
+
|
|
1394
|
+
recipients.set(threadId, known);
|
|
1395
|
+
|
|
1396
|
+
if (failure === undefined) {
|
|
1397
|
+
if (previous !== undefined) {
|
|
1398
|
+
await transaction.delete(key);
|
|
1399
|
+
for (let start = 0; start < (previous.recipientCount ?? 0); start += 128)
|
|
1400
|
+
await transaction.delete(recoveryRecipientKeys(previous, start));
|
|
1401
|
+
for (const id of previous.recipientCount === undefined
|
|
1402
|
+
? result.reports
|
|
1403
|
+
.filter((report) => report.threadId === threadId)
|
|
1404
|
+
.map((report) => report.submissionId)
|
|
1405
|
+
: previousIds)
|
|
1406
|
+
events.push(recoveryEvent(previous, id, "cleared", now));
|
|
1407
|
+
}
|
|
1408
|
+
continue;
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
const added = submissions
|
|
1412
|
+
.filter((row) => row.threadId === threadId && !known.has(row.submissionId))
|
|
1413
|
+
.map((row) => row.submissionId);
|
|
1414
|
+
|
|
1415
|
+
await appendRecoveryRecipients(transaction, threadId, previousIds.length, added);
|
|
1416
|
+
|
|
1417
|
+
const fault = ThreadRecoveryFault.make({
|
|
1418
|
+
schemaVersion: 1,
|
|
1419
|
+
threadId,
|
|
1420
|
+
firstFailedAt: previous?.firstFailedAt ?? now,
|
|
1421
|
+
lastFailedAt: now,
|
|
1422
|
+
attempts: Math.min(2_147_483_647, (previous?.attempts ?? 0) + 1),
|
|
1423
|
+
retryAt: now + Math.min(60_000, 5_000 * 2 ** Math.min(30, previous?.attempts ?? 0)),
|
|
1424
|
+
failure,
|
|
1425
|
+
recipientCount: previousIds.length + added.length,
|
|
1426
|
+
});
|
|
1427
|
+
|
|
1428
|
+
await transaction.put(key, encodeRecoveryFault(fault));
|
|
1429
|
+
if (previous !== undefined && !sameRecoveryFailure(previous.failure, failure))
|
|
1430
|
+
for (const id of previousIds) events.push(recoveryEvent(fault, id, "changed", now));
|
|
1431
|
+
for (const id of added) {
|
|
1432
|
+
known.add(id);
|
|
1433
|
+
events.push(recoveryEvent(fault, id, "created", now));
|
|
1434
|
+
}
|
|
1435
|
+
faults.set(threadId, fault);
|
|
1436
|
+
if (previous === undefined) newlyBlocked.push(fault);
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
await appendRecoveryEvents(transaction, events);
|
|
1440
|
+
|
|
1441
|
+
return {
|
|
1442
|
+
changed: new Set(events.map((event) => event.threadId)),
|
|
1443
|
+
newlyBlocked,
|
|
1444
|
+
faults,
|
|
1445
|
+
recipients,
|
|
1446
|
+
};
|
|
1447
|
+
}),
|
|
1448
|
+
);
|
|
1449
|
+
|
|
1450
|
+
yield* failpoint.hit("maintenance:recovery-status:after");
|
|
1451
|
+
for (const [threadId, ids] of retained.recipients) recovery.recipients.set(threadId, ids);
|
|
1452
|
+
// A healthy unchanged head must not wake its own maintenance loop.
|
|
1453
|
+
for (const threadId of retained.changed) yield* wakes.notify(threadId);
|
|
1454
|
+
for (const fault of retained.newlyBlocked)
|
|
1455
|
+
yield* Effect.logError(
|
|
1456
|
+
"Native Thread recovery blocked; accepted work remains pending",
|
|
1457
|
+
fault.failure.reason === "defect"
|
|
1458
|
+
? Cause.die(fault.failure)
|
|
1459
|
+
: Cause.fail(fault.failure),
|
|
1460
|
+
).pipe(Effect.annotateLogs({ threadId: fault.threadId }));
|
|
1461
|
+
|
|
1462
|
+
return retained.faults;
|
|
1463
|
+
});
|
|
1464
|
+
|
|
1465
|
+
const recoverThread = Effect.fn("ThreadMaintenance.recoverThread")(function* (
|
|
1466
|
+
threadId: ThreadId,
|
|
1467
|
+
recovery: NativeRecovery,
|
|
1468
|
+
) {
|
|
1469
|
+
const result = yield* runtime
|
|
1470
|
+
.runRecovery({ threadId })
|
|
1471
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
1472
|
+
|
|
1473
|
+
// Visibility is committed before a claim or any fallible auxiliary join.
|
|
1474
|
+
const faults = yield* recordRecoveryFaults(result, recovery);
|
|
1475
|
+
|
|
1476
|
+
for (const report of result.reports) recovery.reports.set(report.submissionId, report);
|
|
1477
|
+
recovery.faults.delete(threadId);
|
|
1478
|
+
for (const [id, fault] of faults) recovery.faults.set(id, fault);
|
|
1479
|
+
recovery.recovered += result.reports.length + result.blocked.length;
|
|
1480
|
+
recovery.repaired ||= result.reports.some((report) => report.disposition === "repaired");
|
|
1481
|
+
});
|
|
1482
|
+
|
|
1483
|
+
// Registry changes, including upgrades from timed binding retries, create one native
|
|
1484
|
+
// opportunity. Unchanged registries leave parked work dormant across Object eviction.
|
|
1485
|
+
const readCurrentMaintenanceState = async (transaction: DurableObjectTransaction) => {
|
|
1486
|
+
const current = await readMaintenanceState(transaction);
|
|
1487
|
+
|
|
1488
|
+
if (current.state.bindingRegistryKey === runtime.bindingRegistryKey) return current;
|
|
1489
|
+
|
|
1490
|
+
const state = ThreadMaintenanceState.make({
|
|
1491
|
+
...Struct.omit(current.state, ["retry"]),
|
|
1492
|
+
bindingRegistryKey: runtime.bindingRegistryKey,
|
|
1493
|
+
bindingWaits: [],
|
|
1494
|
+
dirty: current.state.dirty + 1n,
|
|
1495
|
+
});
|
|
1496
|
+
|
|
1497
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
1498
|
+
dueQueue.dirty(DueQueue.Native, 0, true);
|
|
1499
|
+
|
|
1500
|
+
return { state, initialized: true };
|
|
1501
|
+
};
|
|
1502
|
+
|
|
1503
|
+
const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
|
|
1504
|
+
yield* failpoint.hit("maintenance:ensure:before");
|
|
1505
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1506
|
+
|
|
1507
|
+
yield* runTransaction("ensure maintenance alarm", () =>
|
|
1508
|
+
dueQueue.transaction(async (transaction) => {
|
|
1509
|
+
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1510
|
+
|
|
1511
|
+
recoveryEventsPending = hasRecoveryEvents(state);
|
|
1512
|
+
const rows = dueQueue.read();
|
|
1513
|
+
|
|
1514
|
+
if (recoveryEventsPending && !rows.some((row) => row.id === DueQueue.RecoveryEvents))
|
|
1515
|
+
dueQueue.dirty(DueQueue.RecoveryEvents, now);
|
|
1516
|
+
if (!initialized)
|
|
1517
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
1518
|
+
|
|
1519
|
+
const native =
|
|
1520
|
+
state.dirty > state.processed
|
|
1521
|
+
? state.retry?.generation === state.dirty
|
|
1522
|
+
? state.retry.notBefore
|
|
1523
|
+
: (rows.find((row) => row.id === DueQueue.Native)?.dueAt ??
|
|
1524
|
+
now + minimumAlarmDelay)
|
|
1525
|
+
: null;
|
|
1526
|
+
|
|
1527
|
+
dueQueue.checkpointNative(native);
|
|
1528
|
+
const next = DueQueue.next(dueQueue.read());
|
|
1529
|
+
|
|
1530
|
+
if (Number.isFinite(next))
|
|
1531
|
+
await ensureTransactionAlarmBy(transaction, Math.max(now + minimumAlarmDelay, next));
|
|
1532
|
+
}),
|
|
1533
|
+
);
|
|
1534
|
+
yield* failpoint.hit("maintenance:ensure:after");
|
|
1535
|
+
});
|
|
1536
|
+
|
|
1537
|
+
const beginPass = Effect.fn("ThreadMaintenance.beginPass")(function* (
|
|
1538
|
+
observed: MaintenanceObservation,
|
|
1539
|
+
) {
|
|
1540
|
+
yield* failpoint.hit("maintenance:begin:before");
|
|
1541
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1542
|
+
|
|
1543
|
+
const result = yield* runTransaction("begin maintenance pass", () =>
|
|
1544
|
+
dueQueue.transaction(async (transaction) => {
|
|
1545
|
+
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1546
|
+
|
|
1547
|
+
recoveryEventsPending = hasRecoveryEvents(state);
|
|
1548
|
+
|
|
1549
|
+
observed.generation = state.dirty;
|
|
1550
|
+
if (!initialized) {
|
|
1551
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
1552
|
+
}
|
|
1553
|
+
const retryAt = state.retry?.generation === state.dirty ? state.retry.notBefore : 0;
|
|
1554
|
+
|
|
1555
|
+
const rows = dueQueue.read();
|
|
1556
|
+
const native = rows.find((row) => row.id === DueQueue.Native);
|
|
1557
|
+
const publication = rows.find((row) => row.id === DueQueue.Publication);
|
|
1558
|
+
|
|
1559
|
+
if (
|
|
1560
|
+
state.processed >= state.dirty ||
|
|
1561
|
+
retryAt > now ||
|
|
1562
|
+
native === undefined ||
|
|
1563
|
+
native.dueAt === null ||
|
|
1564
|
+
native.notBefore > now ||
|
|
1565
|
+
publication?.state === "parked"
|
|
1566
|
+
) {
|
|
1567
|
+
return {
|
|
1568
|
+
_tag: "CaughtUp" as const,
|
|
1569
|
+
nonterminal: state.nonterminal,
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
// One debit per native progress revision in this event. Source progress may
|
|
1574
|
+
// legitimately admit another wave; a generation change alone cannot.
|
|
1575
|
+
const alreadyCharged = observed.charged.get(DueQueue.Native) === native.revision;
|
|
1576
|
+
const charged = alreadyCharged ? native : dueQueue.claim(native, now, true);
|
|
1577
|
+
|
|
1578
|
+
if (charged === undefined)
|
|
1579
|
+
return { _tag: "CaughtUp" as const, nonterminal: state.nonterminal };
|
|
1580
|
+
observed.queue.set(DueQueue.Native, charged);
|
|
1581
|
+
observed.charged.set(DueQueue.Native, charged.revision);
|
|
1582
|
+
const deadline = DueQueue.next(dueQueue.read());
|
|
1583
|
+
|
|
1584
|
+
if (Number.isFinite(deadline))
|
|
1585
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, deadline));
|
|
1586
|
+
else await transaction.deleteAlarm();
|
|
1587
|
+
|
|
1588
|
+
return {
|
|
1589
|
+
_tag: "Actionable" as const,
|
|
1590
|
+
generation: state.dirty,
|
|
1591
|
+
sourceRevision: charged.revision,
|
|
1592
|
+
nonterminal: state.nonterminal,
|
|
1593
|
+
stalls: charged.stalls,
|
|
1594
|
+
};
|
|
1595
|
+
}),
|
|
1596
|
+
);
|
|
1597
|
+
|
|
1598
|
+
yield* failpoint.hit("maintenance:begin:after");
|
|
1599
|
+
|
|
1600
|
+
return result;
|
|
1601
|
+
});
|
|
1602
|
+
|
|
1603
|
+
const backoffDelay = (priorStalls: number, jitter: number) => {
|
|
1604
|
+
const backoff = Math.min(
|
|
1605
|
+
config.alarmBackoffCap,
|
|
1606
|
+
config.alarmBackoffBase * 2 ** Math.min(priorStalls, 30),
|
|
1607
|
+
);
|
|
1608
|
+
|
|
1609
|
+
// Jitter over [backoff/2, backoff] spreads retries without exceeding the cap.
|
|
1610
|
+
return Math.ceil(backoff / 2 + (backoff / 2) * jitter);
|
|
1611
|
+
};
|
|
1612
|
+
|
|
1613
|
+
const rearmDelay = Effect.fn("ThreadMaintenance.rearmDelay")(function* (
|
|
1614
|
+
progressed: boolean,
|
|
1615
|
+
priorStalls: number,
|
|
1616
|
+
) {
|
|
1617
|
+
return progressed ? config.alarmBackoffBase : backoffDelay(priorStalls, yield* Random.next);
|
|
1618
|
+
});
|
|
1619
|
+
|
|
1620
|
+
const rearmFailure = Effect.fn("ThreadMaintenance.rearmFailure")(function* (
|
|
1621
|
+
observed: MaintenanceObservation,
|
|
1622
|
+
) {
|
|
1623
|
+
const { generation, nativeOnly } = observed;
|
|
1624
|
+
|
|
1625
|
+
if (generation === undefined) return;
|
|
1626
|
+
yield* failpoint.hit("maintenance:retry:before");
|
|
1627
|
+
yield* mutations.withSnapshot((active) =>
|
|
1628
|
+
Effect.gen(function* () {
|
|
1629
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1630
|
+
|
|
1631
|
+
const jitter = yield* Random.next;
|
|
1632
|
+
|
|
1633
|
+
yield* runTransaction("back off failed maintenance", () =>
|
|
1634
|
+
dueQueue.transaction(async (transaction) => {
|
|
1635
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1636
|
+
|
|
1637
|
+
const previous =
|
|
1638
|
+
state.retry?.generation === generation && state.retry.nativeOnly === nativeOnly
|
|
1639
|
+
? state.retry
|
|
1640
|
+
: undefined;
|
|
1641
|
+
|
|
1642
|
+
const retry =
|
|
1643
|
+
state.retry?.generation === generation && state.retry.nativeOnly
|
|
1644
|
+
? state.retry
|
|
1645
|
+
: MaintenanceRetry.make({
|
|
1646
|
+
generation,
|
|
1647
|
+
notBefore: Math.max(
|
|
1648
|
+
previous?.notBefore ?? 0,
|
|
1649
|
+
now + backoffDelay(previous?.stalls ?? 0, jitter),
|
|
1650
|
+
),
|
|
1651
|
+
nativeOnly,
|
|
1652
|
+
stalls: Math.min(30, (previous?.stalls ?? 0) + 1),
|
|
1653
|
+
});
|
|
1654
|
+
|
|
1655
|
+
await transaction.put(
|
|
1656
|
+
MAINTENANCE_STATE_KEY,
|
|
1657
|
+
encodeMaintenanceState(ThreadMaintenanceState.make({ ...state, retry })),
|
|
1658
|
+
);
|
|
1659
|
+
|
|
1660
|
+
// Never postpone a producer that raced the failed observation. Host work
|
|
1661
|
+
// retains its own deadline; an early delivery skips native recovery below.
|
|
1662
|
+
const nativeDeadline =
|
|
1663
|
+
active > 0 || state.dirty !== generation
|
|
1664
|
+
? now + minimumAlarmDelay
|
|
1665
|
+
: retry.notBefore;
|
|
1666
|
+
|
|
1667
|
+
dueQueue.checkpointNative(
|
|
1668
|
+
state.dirty > state.processed || active > 0 ? nativeDeadline : null,
|
|
1669
|
+
);
|
|
1670
|
+
// Defer only the unacknowledged observations from this failed event. A
|
|
1671
|
+
// completed wave or racing enrollment has a different revision.
|
|
1672
|
+
for (const row of observed.queue.values()) {
|
|
1673
|
+
if (row.id !== DueQueue.Native && row.dueAt !== null && row.dueAt <= now) {
|
|
1674
|
+
const charged =
|
|
1675
|
+
observed.charged.get(row.id) === row.revision
|
|
1676
|
+
? row
|
|
1677
|
+
: dueQueue.claim(row, now);
|
|
1678
|
+
|
|
1679
|
+
if (charged !== undefined)
|
|
1680
|
+
dueQueue.complete(charged, now + backoffDelay(charged.stalls, jitter));
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
const next = DueQueue.next(dueQueue.read());
|
|
1684
|
+
|
|
1685
|
+
if (Number.isFinite(next))
|
|
1686
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
|
|
1687
|
+
else await transaction.deleteAlarm();
|
|
1688
|
+
}),
|
|
1689
|
+
);
|
|
1690
|
+
}),
|
|
1691
|
+
);
|
|
1692
|
+
yield* failpoint.hit("maintenance:retry:after");
|
|
1693
|
+
});
|
|
1694
|
+
|
|
1695
|
+
const beginNative = Effect.fn("ThreadMaintenance.beginNative")(function* (
|
|
1696
|
+
observed: MaintenanceObservation,
|
|
1697
|
+
) {
|
|
1698
|
+
return yield* mutations.withSnapshot((activeAtStart) =>
|
|
1699
|
+
Effect.gen(function* () {
|
|
1700
|
+
const generation = yield* beginPass(observed);
|
|
1701
|
+
|
|
1702
|
+
if (activeAtStart === 0) {
|
|
1703
|
+
const native = (yield* queueSnapshot).find((row) => row.id === DueQueue.Native);
|
|
1704
|
+
|
|
1705
|
+
if (native !== undefined) observed.available.set(native.id, native.revision);
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
return { ...generation, activeAtStart };
|
|
1709
|
+
}),
|
|
1710
|
+
);
|
|
1711
|
+
});
|
|
1712
|
+
|
|
1713
|
+
const attemptSelected = Effect.fn("ThreadMaintenance.attemptSelected")(function* (
|
|
1714
|
+
selected: SubmissionWorkItem,
|
|
1715
|
+
previousWaits: ReadonlyArray<BindingWait>,
|
|
1716
|
+
yieldAfter: DateTime.Utc,
|
|
1717
|
+
recovery: NativeRecovery,
|
|
1718
|
+
): Effect.fn.Return<number, MaintenancePassFailure> {
|
|
1719
|
+
yield* recoverThread(selected.threadId, recovery);
|
|
1720
|
+
|
|
1721
|
+
let bindingFailure: DurableBindingFailure | undefined;
|
|
1722
|
+
|
|
1723
|
+
// A missing binding releases its slot after retaining this exact Submission
|
|
1724
|
+
// and committing its registry wait below.
|
|
1725
|
+
const settlement = recovery.faults.has(selected.threadId)
|
|
1726
|
+
? Option.none()
|
|
1727
|
+
: yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(
|
|
1728
|
+
Effect.provideService(CurrentNativeSource, true),
|
|
1729
|
+
Effect.catchTag("BindingUnavailable", (failure) => {
|
|
1730
|
+
bindingFailure = failure;
|
|
1731
|
+
|
|
1732
|
+
return Effect.succeed(Option.none());
|
|
1733
|
+
}),
|
|
1734
|
+
);
|
|
1735
|
+
|
|
1736
|
+
const previous = previousWaits.find((wait) => wait.submissionId === selected.submissionId);
|
|
1737
|
+
|
|
1738
|
+
const wait =
|
|
1739
|
+
bindingFailure === undefined
|
|
1740
|
+
? undefined
|
|
1741
|
+
: BindingWait.make({
|
|
1742
|
+
threadId: selected.threadId,
|
|
1743
|
+
submissionId: selected.submissionId,
|
|
1744
|
+
reportedAt: previous?.reportedAt ?? (yield* Clock.currentTimeMillis),
|
|
1745
|
+
});
|
|
1746
|
+
|
|
1747
|
+
// Report before recording reportedAt. Interruption before the write may repeat a
|
|
1748
|
+
// report, but a durable wait must never suppress a report that was not delivered.
|
|
1749
|
+
if (bindingFailure !== undefined && previous === undefined) {
|
|
1750
|
+
yield* Effect.logError(
|
|
1751
|
+
"Thread parked until its agent binding registry changes; original work remains pending",
|
|
1752
|
+
Cause.fail(bindingFailure),
|
|
1753
|
+
).pipe(
|
|
1754
|
+
Effect.annotateLogs({
|
|
1755
|
+
threadId: selected.threadId,
|
|
1756
|
+
submissionId: selected.submissionId,
|
|
1757
|
+
}),
|
|
1758
|
+
);
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
if (wait !== undefined || previous !== undefined) {
|
|
1762
|
+
// The Attempt released its Claim. Commit its binding wait (or clear) once,
|
|
1763
|
+
// before joining fallible auxiliary work. This local fact neither acknowledges
|
|
1764
|
+
// a generation nor changes the shared alarm.
|
|
1765
|
+
yield* failpoint.hit("maintenance:binding-retry:before");
|
|
1766
|
+
yield* runStorage(
|
|
1767
|
+
"record submission binding wait",
|
|
1768
|
+
storage
|
|
1769
|
+
.transaction((transaction) =>
|
|
1770
|
+
Effect.gen(function* () {
|
|
1771
|
+
const encoded = yield* transaction.get(MAINTENANCE_STATE_KEY);
|
|
1772
|
+
|
|
1773
|
+
const state =
|
|
1774
|
+
encoded === undefined
|
|
1775
|
+
? initialMaintenanceState()
|
|
1776
|
+
: yield* Schema.decodeUnknownEffect(ThreadMaintenanceState)(encoded);
|
|
1777
|
+
|
|
1778
|
+
const bindingWaits = [
|
|
1779
|
+
...(state.bindingWaits ?? []).filter(
|
|
1780
|
+
(entry) => entry.submissionId !== selected.submissionId,
|
|
1781
|
+
),
|
|
1782
|
+
...(wait === undefined ? [] : [wait]),
|
|
1783
|
+
];
|
|
1784
|
+
|
|
1785
|
+
yield* transaction.put(
|
|
1786
|
+
MAINTENANCE_STATE_KEY,
|
|
1787
|
+
yield* Schema.encodeEffect(ThreadMaintenanceState)(
|
|
1788
|
+
ThreadMaintenanceState.make({ ...state, bindingWaits }),
|
|
1789
|
+
),
|
|
1790
|
+
);
|
|
1791
|
+
}),
|
|
1792
|
+
)
|
|
1793
|
+
.pipe(Effect.mapError(alarmFailure("record submission binding wait"))),
|
|
1794
|
+
);
|
|
1795
|
+
yield* failpoint.hit("maintenance:binding-retry:after");
|
|
1796
|
+
}
|
|
1797
|
+
|
|
1798
|
+
return Option.isSome(settlement) ? 1 : 0;
|
|
1799
|
+
});
|
|
1800
|
+
|
|
1801
|
+
const advance = Effect.fn("ThreadMaintenance.advance")(function* (
|
|
1802
|
+
started: Effect.Success<ReturnType<typeof beginNative>>,
|
|
1803
|
+
yieldAfter: DateTime.Utc,
|
|
1804
|
+
observed: MaintenanceObservation,
|
|
1805
|
+
recovery: NativeRecovery,
|
|
1806
|
+
native: NativeDispatch,
|
|
1807
|
+
dispatch = true,
|
|
1808
|
+
reserved: ReadonlySet<ThreadId> = new Set(native.active.keys()),
|
|
1809
|
+
): Effect.fn.Return<NativePassResult, MaintenancePassFailure> {
|
|
1810
|
+
let publicationRow = (yield* queueSnapshot).find((row) => row.id === DueQueue.Publication);
|
|
1811
|
+
|
|
1812
|
+
if (
|
|
1813
|
+
publicationRow !== undefined &&
|
|
1814
|
+
publicationRow.dueAt !== null &&
|
|
1815
|
+
publicationRow.dueAt <= (yield* Clock.currentTimeMillis)
|
|
1816
|
+
) {
|
|
1817
|
+
yield* runQueued(
|
|
1818
|
+
publicationRow,
|
|
1819
|
+
mutations
|
|
1820
|
+
.withSnapshot((active) =>
|
|
1821
|
+
started._tag === "Actionable" && active === 0
|
|
1822
|
+
? publication.prepareGeneration(started.generation)
|
|
1823
|
+
: Effect.void,
|
|
1824
|
+
)
|
|
1825
|
+
.pipe(Effect.andThen(publication.drain)),
|
|
1826
|
+
observed,
|
|
1827
|
+
);
|
|
1828
|
+
publicationRow = (yield* queueSnapshot).find((row) => row.id === DueQueue.Publication);
|
|
1829
|
+
}
|
|
1830
|
+
const pending = Option.fromNullishOr(publicationRow?.dueAt);
|
|
1831
|
+
|
|
1832
|
+
if (
|
|
1833
|
+
started._tag === "CaughtUp" ||
|
|
1834
|
+
Option.isSome(pending) ||
|
|
1835
|
+
publicationRow?.state === "parked"
|
|
1836
|
+
) {
|
|
1837
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) =>
|
|
1838
|
+
Effect.gen(function* () {
|
|
1839
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1840
|
+
|
|
1841
|
+
const { state } = yield* runTransaction("read native maintenance deadline", () =>
|
|
1842
|
+
dueQueue.transaction((transaction) => readMaintenanceState(transaction)),
|
|
1843
|
+
);
|
|
1844
|
+
|
|
1845
|
+
const native =
|
|
1846
|
+
active > 0 || state.dirty > state.processed
|
|
1847
|
+
? state.retry?.generation === state.dirty && active === 0
|
|
1848
|
+
? state.retry.notBefore
|
|
1849
|
+
: now + minimumAlarmDelay
|
|
1850
|
+
: Infinity;
|
|
1851
|
+
|
|
1852
|
+
const next = Option.isSome(pending) ? pending.value : native;
|
|
1853
|
+
|
|
1854
|
+
return Number.isFinite(next) ? Math.max(now + minimumAlarmDelay, next) : undefined;
|
|
1855
|
+
}),
|
|
1856
|
+
);
|
|
1857
|
+
|
|
1858
|
+
return {
|
|
1859
|
+
phase: "caught-up",
|
|
1860
|
+
nonterminal: started.nonterminal,
|
|
1861
|
+
nextAttemptAt,
|
|
1862
|
+
};
|
|
1863
|
+
}
|
|
1864
|
+
if (
|
|
1865
|
+
native.generation !== started.generation ||
|
|
1866
|
+
native.sourceRevision !== started.sourceRevision
|
|
1867
|
+
) {
|
|
1868
|
+
native.deferred.clear();
|
|
1869
|
+
native.generation = started.generation;
|
|
1870
|
+
native.sourceRevision = started.sourceRevision;
|
|
1871
|
+
}
|
|
1872
|
+
// A stable generation with no freed slot needs no second ledger/history scan.
|
|
1873
|
+
// Enrollment precedes admission: never cache a scan overlapping a producer body,
|
|
1874
|
+
// which may make a head ready later without another generation increment.
|
|
1875
|
+
if (
|
|
1876
|
+
native.active.size > 0 &&
|
|
1877
|
+
native.scanGeneration === started.generation &&
|
|
1878
|
+
native.scanRevision === started.sourceRevision
|
|
1879
|
+
) {
|
|
1880
|
+
return {
|
|
1881
|
+
phase: "actionable",
|
|
1882
|
+
nonterminal: started.nonterminal,
|
|
1883
|
+
nextAttemptAt: undefined,
|
|
1884
|
+
};
|
|
1885
|
+
}
|
|
1886
|
+
native.scanGeneration = started.activeAtStart === 0 ? started.generation : undefined;
|
|
1887
|
+
native.scanRevision = started.activeAtStart === 0 ? started.sourceRevision : undefined;
|
|
1888
|
+
|
|
1889
|
+
// Select from control state before reading execution history. A recovering or faulted
|
|
1890
|
+
// Thread cannot enter dispatch; old cleanup has its own scoped opportunity below.
|
|
1891
|
+
observed.nativeOnly = true;
|
|
1892
|
+
|
|
1893
|
+
// Every checkpoint keeps the producer overlap that belongs to this recovery wave.
|
|
1894
|
+
const observation = (recovery.observation ??= {
|
|
1895
|
+
generation: started.generation,
|
|
1896
|
+
activeAtStart: started.activeAtStart,
|
|
1897
|
+
});
|
|
1898
|
+
|
|
1899
|
+
const current = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
1900
|
+
const selectionTime = yield* Clock.currentTimeMillis;
|
|
1901
|
+
const submissionsByThread = new Map<ThreadId, Array<SubmissionId>>();
|
|
1902
|
+
|
|
1903
|
+
for (const row of current) {
|
|
1904
|
+
const ids = submissionsByThread.get(row.threadId) ?? [];
|
|
1905
|
+
|
|
1906
|
+
ids.push(row.submissionId);
|
|
1907
|
+
submissionsByThread.set(row.threadId, ids);
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
yield* runTransaction("read Thread recovery deadlines", async () => {
|
|
1911
|
+
for (const [threadId, ids] of submissionsByThread) {
|
|
1912
|
+
if (!recovery.loaded.has(threadId)) {
|
|
1913
|
+
const encoded = await ctx.storage.get(recoveryFaultKey(threadId));
|
|
1914
|
+
|
|
1915
|
+
if (encoded !== undefined) {
|
|
1916
|
+
const fault = decodeRecoveryFault(threadId, encoded);
|
|
1917
|
+
|
|
1918
|
+
recovery.faults.set(threadId, fault);
|
|
1919
|
+
recovery.recipients.set(
|
|
1920
|
+
threadId,
|
|
1921
|
+
new Set(await readRecoveryRecipients(ctx.storage, fault)),
|
|
1922
|
+
);
|
|
1923
|
+
}
|
|
1924
|
+
recovery.loaded.add(threadId);
|
|
1925
|
+
}
|
|
1926
|
+
const fault = recovery.faults.get(threadId);
|
|
1927
|
+
|
|
1928
|
+
if (
|
|
1929
|
+
fault === undefined ||
|
|
1930
|
+
ids.every((id) => recovery.recipients.get(threadId)?.has(id))
|
|
1931
|
+
)
|
|
1932
|
+
continue;
|
|
1933
|
+
|
|
1934
|
+
// Admissions during backoff need visibility without accelerating recovery.
|
|
1935
|
+
// Preserve recipients even if recovery later settles them before clearing.
|
|
1936
|
+
await dueQueue.transaction(async (transaction) => {
|
|
1937
|
+
const encoded = await transaction.get(recoveryFaultKey(threadId));
|
|
1938
|
+
|
|
1939
|
+
if (encoded === undefined) return;
|
|
1940
|
+
const previous = decodeRecoveryFault(threadId, encoded);
|
|
1941
|
+
const previousIds = await readRecoveryRecipients(transaction, previous);
|
|
1942
|
+
const known = new Set(previousIds);
|
|
1943
|
+
const added = ids.filter((id) => !known.has(id));
|
|
1944
|
+
|
|
1945
|
+
if (added.length === 0) return;
|
|
1946
|
+
|
|
1947
|
+
const updated = ThreadRecoveryFault.make({
|
|
1948
|
+
...previous,
|
|
1949
|
+
recipientCount: previousIds.length + added.length,
|
|
1950
|
+
});
|
|
1951
|
+
|
|
1952
|
+
await appendRecoveryRecipients(transaction, threadId, previousIds.length, added);
|
|
1953
|
+
await transaction.put(recoveryFaultKey(threadId), encodeRecoveryFault(updated));
|
|
1954
|
+
await appendRecoveryEvents(
|
|
1955
|
+
transaction,
|
|
1956
|
+
added.map((id) => recoveryEvent(updated, id, "created", selectionTime)),
|
|
1957
|
+
);
|
|
1958
|
+
recovery.faults.set(threadId, updated);
|
|
1959
|
+
recovery.recipients.set(threadId, new Set([...previousIds, ...added]));
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
1962
|
+
});
|
|
1963
|
+
const reports = recovery.reports;
|
|
1964
|
+
const recoveryFaults = recovery.faults;
|
|
1965
|
+
|
|
1966
|
+
const bindingWaits = yield* runTransaction(
|
|
1967
|
+
"read binding waits",
|
|
1968
|
+
async () => (await readMaintenanceState(ctx.storage)).state.bindingWaits ?? [],
|
|
1969
|
+
);
|
|
1970
|
+
|
|
1971
|
+
const parked = new Set(bindingWaits.map((wait) => wait.submissionId));
|
|
1972
|
+
|
|
1973
|
+
const waiting = (row: SubmissionWorkItem) =>
|
|
1974
|
+
!recovery.pending.has(row.threadId) &&
|
|
1975
|
+
!recoveryFaults.has(row.threadId) &&
|
|
1976
|
+
(parked.has(row.submissionId) || stableExternalWait(row, reports));
|
|
1977
|
+
|
|
1978
|
+
const heads = new Map<ThreadId, SubmissionWorkItem>();
|
|
1979
|
+
|
|
1980
|
+
for (const row of current) {
|
|
1981
|
+
// Only recovered uncertainty may release later input; accepted aborts retain FIFO.
|
|
1982
|
+
if (row.state === "unknown" && waiting(row)) continue;
|
|
1983
|
+
if (!heads.has(row.threadId)) heads.set(row.threadId, row);
|
|
1984
|
+
}
|
|
1985
|
+
const stopping = new Set<ThreadId>();
|
|
1986
|
+
|
|
1987
|
+
for (const head of heads.values()) {
|
|
1988
|
+
if (
|
|
1989
|
+
head.state !== "ready" ||
|
|
1990
|
+
native.active.has(head.threadId) ||
|
|
1991
|
+
recovery.pending.has(head.threadId) ||
|
|
1992
|
+
recoveryFaults.has(head.threadId)
|
|
1993
|
+
)
|
|
1994
|
+
continue;
|
|
1995
|
+
|
|
1996
|
+
// An accepted abort is cleanup even when its input was never claimed. This
|
|
1997
|
+
// control-only read must not decode the execution journal or a recovery snapshot.
|
|
1998
|
+
const intent = yield* isolateRecovery(
|
|
1999
|
+
ledger.readAbortIntent(AbortIntentRequest.make({ submissionId: head.submissionId })),
|
|
2000
|
+
{
|
|
2001
|
+
timeout: recoveryConfig.recoveryTimeout,
|
|
2002
|
+
phase: () => "recovery",
|
|
2003
|
+
operation: "read abort intent",
|
|
2004
|
+
},
|
|
2005
|
+
);
|
|
2006
|
+
|
|
2007
|
+
if (Result.isSuccess(intent)) {
|
|
2008
|
+
if (intent.success !== undefined) stopping.add(head.threadId);
|
|
2009
|
+
} else {
|
|
2010
|
+
const faults = yield* recordRecoveryFaults(
|
|
2011
|
+
RecoverySweepResult.make({
|
|
2012
|
+
reports: [],
|
|
2013
|
+
blocked: [
|
|
2014
|
+
RecoveryBlocked.make({ threadId: head.threadId, failure: intent.failure }),
|
|
2015
|
+
],
|
|
2016
|
+
}),
|
|
2017
|
+
recovery,
|
|
2018
|
+
current,
|
|
2019
|
+
);
|
|
2020
|
+
|
|
2021
|
+
for (const [threadId, fault] of faults) recoveryFaults.set(threadId, fault);
|
|
2022
|
+
recovery.recovered++;
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
|
|
2026
|
+
const eligible = [...heads.values()]
|
|
2027
|
+
.filter(
|
|
2028
|
+
(head) =>
|
|
2029
|
+
!recovery.pending.has(head.threadId) &&
|
|
2030
|
+
!stopping.has(head.threadId) &&
|
|
2031
|
+
!recoveryFaults.has(head.threadId) &&
|
|
2032
|
+
!waiting(head) &&
|
|
2033
|
+
(head.state === "ready" || reports.has(head.submissionId)),
|
|
2034
|
+
)
|
|
2035
|
+
.map((head) => head.threadId)
|
|
2036
|
+
.sort();
|
|
2037
|
+
|
|
2038
|
+
yield* failpoint.hit("maintenance:select:before");
|
|
2039
|
+
|
|
2040
|
+
const selection = yield* runTransaction("select maintenance lane", () =>
|
|
2041
|
+
dueQueue.transaction(async (transaction) => {
|
|
2042
|
+
const { state } = await readMaintenanceState(transaction);
|
|
2043
|
+
|
|
2044
|
+
const waits = (state.bindingWaits ?? []).filter(
|
|
2045
|
+
(retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId,
|
|
2046
|
+
);
|
|
2047
|
+
|
|
2048
|
+
const runnable = eligible.filter(
|
|
2049
|
+
(threadId) =>
|
|
2050
|
+
!reserved.has(threadId) &&
|
|
2051
|
+
!native.active.has(threadId) &&
|
|
2052
|
+
!native.deferred.has(threadId) &&
|
|
2053
|
+
!waits.some((wait) => wait.threadId === threadId),
|
|
2054
|
+
);
|
|
2055
|
+
|
|
2056
|
+
const pivot = Math.max(
|
|
2057
|
+
0,
|
|
2058
|
+
runnable.findIndex(
|
|
2059
|
+
(threadId) =>
|
|
2060
|
+
state.lastServedThreadId === undefined || threadId > state.lastServedThreadId,
|
|
2061
|
+
),
|
|
2062
|
+
);
|
|
2063
|
+
|
|
2064
|
+
const selected = dispatch
|
|
2065
|
+
? [...runnable.slice(pivot), ...runnable.slice(0, pivot)].slice(
|
|
2066
|
+
0,
|
|
2067
|
+
nativeDispatchConcurrency - reserved.size,
|
|
2068
|
+
)
|
|
2069
|
+
: [];
|
|
2070
|
+
|
|
2071
|
+
const next = selected.at(-1);
|
|
2072
|
+
|
|
2073
|
+
if (next !== undefined) {
|
|
2074
|
+
await transaction.put(
|
|
2075
|
+
MAINTENANCE_STATE_KEY,
|
|
2076
|
+
encodeMaintenanceState(
|
|
2077
|
+
ThreadMaintenanceState.make({ ...state, lastServedThreadId: next }),
|
|
2078
|
+
),
|
|
2079
|
+
);
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
const backlog = recovery.started
|
|
2083
|
+
? []
|
|
2084
|
+
: [...heads.keys()].filter((threadId) => {
|
|
2085
|
+
const fault = recoveryFaults.get(threadId);
|
|
2086
|
+
|
|
2087
|
+
return (
|
|
2088
|
+
!eligible.includes(threadId) &&
|
|
2089
|
+
(fault === undefined || fault.retryAt <= selectionTime)
|
|
2090
|
+
);
|
|
2091
|
+
});
|
|
2092
|
+
|
|
2093
|
+
// One finite wave, one Thread at a time, with the runtime's per-Thread deadline.
|
|
2094
|
+
// This cursor ensures an event deadline/eviction cannot always restart at the front.
|
|
2095
|
+
const after = backlog.filter(
|
|
2096
|
+
(threadId) =>
|
|
2097
|
+
state.lastRecoveredThreadId === undefined || threadId > state.lastRecoveredThreadId,
|
|
2098
|
+
);
|
|
2099
|
+
|
|
2100
|
+
const before = backlog.filter(
|
|
2101
|
+
(threadId) =>
|
|
2102
|
+
state.lastRecoveredThreadId !== undefined &&
|
|
2103
|
+
threadId <= state.lastRecoveredThreadId,
|
|
2104
|
+
);
|
|
2105
|
+
|
|
2106
|
+
return { selected, waits, backlog: [...after, ...before] };
|
|
2107
|
+
}),
|
|
2108
|
+
);
|
|
2109
|
+
|
|
2110
|
+
yield* failpoint.hit("maintenance:select:after");
|
|
2111
|
+
if (!recovery.started) {
|
|
2112
|
+
recovery.started = true;
|
|
2113
|
+
recovery.needsCheckpoint = selection.backlog.length > 0;
|
|
2114
|
+
for (const threadId of selection.backlog) recovery.pending.add(threadId);
|
|
2115
|
+
yield* Deferred.succeed(recovery.queue, selection.backlog);
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
for (const threadId of selection.selected) {
|
|
2119
|
+
const selected = heads.get(threadId);
|
|
2120
|
+
|
|
2121
|
+
if (selected === undefined) continue;
|
|
2122
|
+
// Reserve before recovery or claim acquisition. The durable Claim remains the
|
|
2123
|
+
// authority; this event-local reservation only prevents duplicate dispatch.
|
|
2124
|
+
native.dispatched++;
|
|
2125
|
+
native.active.set(
|
|
2126
|
+
threadId,
|
|
2127
|
+
yield* Effect.forkIn(
|
|
2128
|
+
attemptSelected(selected, selection.waits, yieldAfter, recovery),
|
|
2129
|
+
native.scope,
|
|
2130
|
+
),
|
|
2131
|
+
);
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
if (native.active.size > 0) {
|
|
2135
|
+
// Active Attempts own their claims and may still append/settle. Only acknowledge
|
|
2136
|
+
// a generation after joining every admitted native Attempt.
|
|
2137
|
+
return {
|
|
2138
|
+
phase: "actionable",
|
|
2139
|
+
nonterminal: current.length,
|
|
2140
|
+
nextAttemptAt: undefined,
|
|
2141
|
+
dispatched: selection.selected.length > 0,
|
|
2142
|
+
};
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
const scannedGeneration = yield* runTransaction(
|
|
2146
|
+
"observe native source generation",
|
|
2147
|
+
async () => (await readMaintenanceState(ctx.storage)).state.dirty,
|
|
2148
|
+
);
|
|
2149
|
+
|
|
2150
|
+
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
2151
|
+
const waitingHeads = new Map<ThreadId, boolean>();
|
|
2152
|
+
|
|
2153
|
+
const autonomous = remaining.some((snapshot) => {
|
|
2154
|
+
if (snapshot.state === "unknown" && waiting(snapshot)) return false;
|
|
2155
|
+
const headWaiting = waitingHeads.get(snapshot.threadId);
|
|
2156
|
+
|
|
2157
|
+
if (headWaiting === undefined) waitingHeads.set(snapshot.threadId, waiting(snapshot));
|
|
2158
|
+
// FIFO followers cannot execute through a stable external wait. Only plain queued
|
|
2159
|
+
// input is dormant here; admission repairs and accepted aborts still need a pass.
|
|
2160
|
+
if (
|
|
2161
|
+
headWaiting === true &&
|
|
2162
|
+
snapshot.state === "ready" &&
|
|
2163
|
+
reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput"
|
|
2164
|
+
)
|
|
2165
|
+
return false;
|
|
2166
|
+
|
|
2167
|
+
return !waiting(snapshot);
|
|
2168
|
+
});
|
|
2169
|
+
|
|
2170
|
+
const progressed = native.progressed || recovery.repaired;
|
|
2171
|
+
|
|
2172
|
+
const now = yield* Clock.currentTimeMillis;
|
|
2173
|
+
const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
|
|
2174
|
+
|
|
2175
|
+
const nextEligible = [...recoveryFaults.values()]
|
|
2176
|
+
.map((fault) => fault.retryAt)
|
|
2177
|
+
.concat(
|
|
2178
|
+
eligible
|
|
2179
|
+
.filter((threadId) => !selection.waits.some((wait) => wait.threadId === threadId))
|
|
2180
|
+
.map(() => now),
|
|
2181
|
+
);
|
|
2182
|
+
|
|
2183
|
+
const retryDelay =
|
|
2184
|
+
nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
|
|
2185
|
+
|
|
2186
|
+
const delay = Math.max(ordinaryDelay, retryDelay);
|
|
2187
|
+
|
|
2188
|
+
// Checkpoint native progress without changing the physical alarm. Auxiliary
|
|
2189
|
+
// delivery remains live; later mutations still advance the shared generation.
|
|
2190
|
+
yield* failpoint.hit("maintenance:checkpoint:before");
|
|
2191
|
+
|
|
2192
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) =>
|
|
2193
|
+
runTransaction("checkpoint native maintenance", () =>
|
|
2194
|
+
dueQueue.transaction(async (transaction) => {
|
|
2195
|
+
const { state } = await readMaintenanceState(transaction);
|
|
2196
|
+
|
|
2197
|
+
const mutationOverlap =
|
|
2198
|
+
observation.activeAtStart > 0 || started.activeAtStart > 0 || active > 0;
|
|
2199
|
+
|
|
2200
|
+
// An empty control scan needs no older recovery report. Certify its fresh
|
|
2201
|
+
// snapshot only when no producer overlapped it or advanced the generation.
|
|
2202
|
+
const generation =
|
|
2203
|
+
remaining.length === 0 && !mutationOverlap && state.dirty === scannedGeneration
|
|
2204
|
+
? scannedGeneration
|
|
2205
|
+
: observation.generation;
|
|
2206
|
+
|
|
2207
|
+
const processed =
|
|
2208
|
+
autonomous || mutationOverlap
|
|
2209
|
+
? state.processed
|
|
2210
|
+
: state.processed > generation
|
|
2211
|
+
? state.processed
|
|
2212
|
+
: generation;
|
|
2213
|
+
|
|
2214
|
+
// Enrollment precedes the mutation body. A retry from an overlapping snapshot
|
|
2215
|
+
// must not defer work that becomes ready later in that same generation.
|
|
2216
|
+
const canBackoff = !mutationOverlap && state.dirty === started.generation;
|
|
2217
|
+
|
|
2218
|
+
const next = ThreadMaintenanceState.make({
|
|
2219
|
+
...Struct.omit(state, ["retry"]),
|
|
2220
|
+
processed,
|
|
2221
|
+
nonterminal: remaining.length,
|
|
2222
|
+
bindingWaits: (state.bindingWaits ?? []).filter((retry) =>
|
|
2223
|
+
remaining.some((row) => row.submissionId === retry.submissionId),
|
|
2224
|
+
),
|
|
2225
|
+
...(autonomous && !progressed && canBackoff
|
|
2226
|
+
? {
|
|
2227
|
+
retry: MaintenanceRetry.make({
|
|
2228
|
+
generation: started.generation,
|
|
2229
|
+
notBefore: now + delay,
|
|
2230
|
+
nativeOnly: true,
|
|
2231
|
+
stalls: Math.min(30, started.stalls + 1),
|
|
2232
|
+
}),
|
|
2233
|
+
}
|
|
2234
|
+
: {}),
|
|
2235
|
+
});
|
|
2236
|
+
|
|
2237
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
2238
|
+
|
|
2239
|
+
const dueAt = autonomous
|
|
2240
|
+
? now + (canBackoff ? delay : minimumAlarmDelay)
|
|
2241
|
+
: mutationOverlap || next.dirty > next.processed
|
|
2242
|
+
? now + minimumAlarmDelay
|
|
2243
|
+
: null;
|
|
2244
|
+
|
|
2245
|
+
dueQueue.checkpointNative(dueAt);
|
|
2246
|
+
|
|
2247
|
+
return dueAt ?? undefined;
|
|
2248
|
+
}),
|
|
2249
|
+
),
|
|
2250
|
+
);
|
|
2251
|
+
|
|
2252
|
+
yield* failpoint.hit("maintenance:checkpoint:after");
|
|
2253
|
+
// Once this empty recovery wave is checkpointed, its producer overlap must not
|
|
2254
|
+
// prevent a later fresh snapshot from acknowledging native quiescence.
|
|
2255
|
+
if (remaining.length === 0 && recovery.pending.size === 0) delete recovery.observation;
|
|
2256
|
+
native.deferred.clear();
|
|
2257
|
+
native.progressed = false;
|
|
2258
|
+
native.needsCheckpoint = false;
|
|
2259
|
+
|
|
2260
|
+
return {
|
|
2261
|
+
phase: "actionable",
|
|
2262
|
+
nonterminal: remaining.length,
|
|
2263
|
+
nextAttemptAt,
|
|
2264
|
+
};
|
|
2265
|
+
});
|
|
2266
|
+
|
|
2267
|
+
const validateLanes = Effect.fn("ThreadMaintenance.validateLanes")(function* (
|
|
2268
|
+
lanes: ReadonlyArray<ThreadHostMaintenanceLane>,
|
|
2269
|
+
queued: ReadonlyArray<DueQueue.DueLane>,
|
|
2270
|
+
) {
|
|
2271
|
+
const ids = new Set([
|
|
2272
|
+
DueQueue.Native,
|
|
2273
|
+
DueQueue.Publication,
|
|
2274
|
+
DueQueue.Projection,
|
|
2275
|
+
DueQueue.Messages,
|
|
2276
|
+
]);
|
|
2277
|
+
|
|
2278
|
+
for (const lane of lanes) {
|
|
2279
|
+
yield* Schema.decodeEffect(DueQueue.LaneId)(lane.id).pipe(
|
|
2280
|
+
Effect.mapError(alarmFailure("maintenance lane ID")),
|
|
2281
|
+
);
|
|
2282
|
+
if (ids.has(lane.id))
|
|
2283
|
+
return yield* DurableAlarmError.make({
|
|
2284
|
+
operation: "maintenance lane ID",
|
|
2285
|
+
message: `Duplicate maintenance lane: ${lane.id}`,
|
|
2286
|
+
});
|
|
2287
|
+
ids.add(lane.id);
|
|
2288
|
+
}
|
|
2289
|
+
for (const row of queued) {
|
|
2290
|
+
if (row.dueAt !== null && !ids.has(row.id))
|
|
2291
|
+
return yield* DurableAlarmError.make({
|
|
2292
|
+
operation: "maintenance lane ID",
|
|
2293
|
+
message: `No handler registered for scheduled lane: ${row.id}`,
|
|
2294
|
+
});
|
|
2295
|
+
}
|
|
2296
|
+
});
|
|
2297
|
+
|
|
2298
|
+
const dispatch = Effect.fn("ThreadMaintenance.dispatch")(function* (
|
|
2299
|
+
yieldAfter: DateTime.Utc,
|
|
2300
|
+
dispatchUntil: DateTime.Utc,
|
|
2301
|
+
observed: MaintenanceObservation,
|
|
2302
|
+
) {
|
|
2303
|
+
// Subscribe before the first snapshot. Durable enrollment recovers eviction;
|
|
2304
|
+
// producer completion and wake hints make work visible during this incarnation.
|
|
2305
|
+
const notified = (yield* Stream.toPull(wakes.wakes)).pipe(
|
|
2306
|
+
Effect.asVoid,
|
|
2307
|
+
Effect.catch(() => Effect.never),
|
|
2308
|
+
);
|
|
2309
|
+
|
|
2310
|
+
// A wake may observe temporary backoff while this event's recovery is still running.
|
|
2311
|
+
// Its completion must retain the original actionable observation for acknowledgement.
|
|
2312
|
+
const started = yield* beginNative(observed);
|
|
2313
|
+
|
|
2314
|
+
yield* mutations.withSnapshot(() =>
|
|
2315
|
+
Effect.gen(function* () {
|
|
2316
|
+
const queued = yield* queueSnapshot;
|
|
2317
|
+
const producing = yield* mutations.activeLanes;
|
|
2318
|
+
|
|
2319
|
+
for (const row of queued) {
|
|
2320
|
+
observed.queue.set(row.id, row);
|
|
2321
|
+
if (!producing.has(row.id)) observed.available.set(row.id, row.revision);
|
|
2322
|
+
}
|
|
2323
|
+
}),
|
|
2324
|
+
);
|
|
2325
|
+
|
|
2326
|
+
// This scope owns every admitted finite wave, including native advancement.
|
|
2327
|
+
// Close it before final alarm rearming, including on failure or event interruption.
|
|
2328
|
+
const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) =>
|
|
2329
|
+
Scope.close(scope, exit),
|
|
2330
|
+
);
|
|
2331
|
+
|
|
2332
|
+
const fork = <A, E>(work: Effect.Effect<A, E>) => Effect.forkIn(work, auxiliaryScope);
|
|
2333
|
+
|
|
2334
|
+
// Each lane has at most one finite wave. A completion is retained until the single
|
|
2335
|
+
// scheduling loop observes it; a busy sibling cannot consume another lane's hint.
|
|
2336
|
+
const lanes = [recoveryEventLane, ...framework.lanes, ...host.lanes].map((lane) => ({
|
|
2337
|
+
...lane,
|
|
2338
|
+
exhausted: false,
|
|
2339
|
+
fiber: undefined as Fiber.Fiber<Option.Option<number>, DurableAlarmError> | undefined,
|
|
2340
|
+
}));
|
|
2341
|
+
|
|
2342
|
+
let messageExhausted = false;
|
|
2343
|
+
let delivery: Fiber.Fiber<void, DurableAlarmError> | undefined;
|
|
2344
|
+
let failure: Cause.Cause<MaintenancePassFailure> | undefined;
|
|
2345
|
+
|
|
2346
|
+
const recovery: NativeRecovery = {
|
|
2347
|
+
queue: yield* Deferred.make<ReadonlyArray<ThreadId>>(),
|
|
2348
|
+
pending: new Set(),
|
|
2349
|
+
loaded: new Set(),
|
|
2350
|
+
reports: new Map(),
|
|
2351
|
+
faults: new Map(),
|
|
2352
|
+
recipients: new Map(),
|
|
2353
|
+
started: false,
|
|
2354
|
+
needsCheckpoint: false,
|
|
2355
|
+
recovered: 0,
|
|
2356
|
+
repaired: false,
|
|
2357
|
+
};
|
|
2358
|
+
|
|
2359
|
+
const recoveryFiber = yield* fork(
|
|
2360
|
+
Effect.gen(function* () {
|
|
2361
|
+
for (const threadId of yield* Deferred.await(recovery.queue)) {
|
|
2362
|
+
yield* failpoint.hit("maintenance:select:before");
|
|
2363
|
+
yield* runTransaction("select old recovery lane", () =>
|
|
2364
|
+
dueQueue.transaction(async (transaction) => {
|
|
2365
|
+
const { state } = await readMaintenanceState(transaction);
|
|
2366
|
+
|
|
2367
|
+
await transaction.put(
|
|
2368
|
+
MAINTENANCE_STATE_KEY,
|
|
2369
|
+
encodeMaintenanceState(
|
|
2370
|
+
ThreadMaintenanceState.make({
|
|
2371
|
+
...state,
|
|
2372
|
+
lastRecoveredThreadId: threadId,
|
|
2373
|
+
}),
|
|
2374
|
+
),
|
|
2375
|
+
);
|
|
2376
|
+
}),
|
|
2377
|
+
);
|
|
2378
|
+
yield* failpoint.hit("maintenance:select:after");
|
|
2379
|
+
yield* recoverThread(threadId, recovery);
|
|
2380
|
+
recovery.pending.delete(threadId);
|
|
2381
|
+
if (!recovery.faults.has(threadId)) yield* wakes.notify(threadId);
|
|
2382
|
+
}
|
|
2383
|
+
}),
|
|
2384
|
+
);
|
|
2385
|
+
|
|
2386
|
+
// Backfill is one disposable wave; native and host work keep their own opportunities.
|
|
2387
|
+
const backfill = yield* fork(
|
|
2388
|
+
Effect.gen(function* () {
|
|
2389
|
+
if ((yield* mutations.activeLanes).has(DueQueue.Projection)) return;
|
|
2390
|
+
const row = (yield* queueSnapshot).find((row) => row.id === DueQueue.Projection);
|
|
2391
|
+
|
|
2392
|
+
if (
|
|
2393
|
+
row?.dueAt === null ||
|
|
2394
|
+
row === undefined ||
|
|
2395
|
+
row.dueAt > (yield* Clock.currentTimeMillis)
|
|
2396
|
+
)
|
|
2397
|
+
return;
|
|
2398
|
+
|
|
2399
|
+
return yield* runQueued(
|
|
2400
|
+
row,
|
|
2401
|
+
projection.drain.pipe(Effect.andThen(projection.pendingDeadline)),
|
|
2402
|
+
observed,
|
|
2403
|
+
).pipe(Effect.timeoutOption(config.projectionDispatchTimeoutMillis));
|
|
2404
|
+
}),
|
|
2405
|
+
);
|
|
2406
|
+
|
|
2407
|
+
let backfillObserved = false;
|
|
2408
|
+
let recoveryObserved = false;
|
|
2409
|
+
|
|
2410
|
+
const dispatch: NativeDispatch = {
|
|
2411
|
+
scope: auxiliaryScope,
|
|
2412
|
+
active: new Map(),
|
|
2413
|
+
deferred: new Set(),
|
|
2414
|
+
dispatched: 0,
|
|
2415
|
+
settled: 0,
|
|
2416
|
+
progressed: false,
|
|
2417
|
+
needsCheckpoint: false,
|
|
2418
|
+
};
|
|
2419
|
+
|
|
2420
|
+
let native: Fiber.Fiber<NativePassResult, MaintenancePassFailure> | undefined = yield* fork(
|
|
2421
|
+
advance(started, yieldAfter, observed, recovery, dispatch),
|
|
2422
|
+
);
|
|
2423
|
+
|
|
2424
|
+
let result: NativePassResult = {
|
|
2425
|
+
phase: "caught-up",
|
|
2426
|
+
nonterminal: started.nonterminal,
|
|
2427
|
+
nextAttemptAt: undefined,
|
|
2428
|
+
};
|
|
2429
|
+
|
|
2430
|
+
let phase = result.phase;
|
|
2431
|
+
let nativeCheck = false;
|
|
2432
|
+
const until = DateTime.toEpochMillis(yieldAfter);
|
|
2433
|
+
const dispatchEnd = DateTime.toEpochMillis(dispatchUntil);
|
|
2434
|
+
|
|
2435
|
+
while (true) {
|
|
2436
|
+
if (native?.pollUnsafe() !== undefined) {
|
|
2437
|
+
result = yield* Fiber.join(native);
|
|
2438
|
+
native = undefined;
|
|
2439
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
2440
|
+
observed.nativeOnly = dispatch.active.size > 0;
|
|
2441
|
+
if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency) {
|
|
2442
|
+
dispatch.scanGeneration = undefined;
|
|
2443
|
+
nativeCheck = true;
|
|
2444
|
+
}
|
|
2445
|
+
// An empty initial scan still opens exactly one old-recovery opportunity.
|
|
2446
|
+
recovery.started = true;
|
|
2447
|
+
yield* Deferred.succeed(recovery.queue, []);
|
|
2448
|
+
}
|
|
2449
|
+
for (const [threadId, attempt] of dispatch.active) {
|
|
2450
|
+
const exit = attempt.pollUnsafe();
|
|
2451
|
+
|
|
2452
|
+
if (exit === undefined) continue;
|
|
2453
|
+
dispatch.active.delete(threadId);
|
|
2454
|
+
dispatch.scanGeneration = undefined;
|
|
2455
|
+
if (Exit.isFailure(exit)) return yield* Effect.failCause(exit.cause);
|
|
2456
|
+
dispatch.settled += exit.value;
|
|
2457
|
+
dispatch.progressed ||= exit.value > 0;
|
|
2458
|
+
dispatch.needsCheckpoint = true;
|
|
2459
|
+
if (exit.value === 0) dispatch.deferred.add(threadId);
|
|
2460
|
+
nativeCheck = true;
|
|
2461
|
+
}
|
|
2462
|
+
if (!recoveryObserved && recoveryFiber.pollUnsafe() !== undefined) {
|
|
2463
|
+
yield* Fiber.join(recoveryFiber);
|
|
2464
|
+
recoveryObserved = true;
|
|
2465
|
+
dispatch.scanGeneration = undefined;
|
|
2466
|
+
nativeCheck ||= recovery.needsCheckpoint;
|
|
2467
|
+
}
|
|
2468
|
+
if (!backfillObserved) {
|
|
2469
|
+
const exit = backfill.pollUnsafe();
|
|
2470
|
+
|
|
2471
|
+
if (exit !== undefined) {
|
|
2472
|
+
backfillObserved = true;
|
|
2473
|
+
if (Exit.isFailure(exit)) failure ??= exit.cause;
|
|
2474
|
+
}
|
|
2475
|
+
}
|
|
2476
|
+
for (const lane of lanes) {
|
|
2477
|
+
const exit = lane.fiber?.pollUnsafe();
|
|
2478
|
+
|
|
2479
|
+
if (exit === undefined) continue;
|
|
2480
|
+
lane.fiber = undefined;
|
|
2481
|
+
if (Exit.isFailure(exit)) {
|
|
2482
|
+
failure ??= exit.cause;
|
|
2483
|
+
lane.exhausted = true;
|
|
2484
|
+
}
|
|
2485
|
+
}
|
|
2486
|
+
const deliveryExit = delivery?.pollUnsafe();
|
|
2487
|
+
|
|
2488
|
+
if (deliveryExit !== undefined) {
|
|
2489
|
+
delivery = undefined;
|
|
2490
|
+
if (Exit.isFailure(deliveryExit)) {
|
|
2491
|
+
failure ??= deliveryExit.cause;
|
|
2492
|
+
messageExhausted = true;
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
// A failed auxiliary lane is exhausted for this event. Its error must not
|
|
2497
|
+
// retire healthy admission, delivery or native work that still has an allowance.
|
|
2498
|
+
const now = yield* Clock.currentTimeMillis;
|
|
2499
|
+
|
|
2500
|
+
if (now >= until) break;
|
|
2501
|
+
|
|
2502
|
+
const queueRevision = yield* mutations.revision;
|
|
2503
|
+
const queued = yield* queueSnapshot;
|
|
2504
|
+
const producing = yield* mutations.activeLanes;
|
|
2505
|
+
|
|
2506
|
+
yield* validateLanes(lanes, queued);
|
|
2507
|
+
for (const lane of lanes) {
|
|
2508
|
+
if (
|
|
2509
|
+
lane.phase === "after-native" ||
|
|
2510
|
+
lane.fiber !== undefined ||
|
|
2511
|
+
lane.exhausted ||
|
|
2512
|
+
producing.has(lane.id)
|
|
2513
|
+
)
|
|
2514
|
+
continue;
|
|
2515
|
+
const row = queued.find((row) => row.id === lane.id);
|
|
2516
|
+
|
|
2517
|
+
if (row === undefined || row.dueAt === null || row.dueAt > now) continue;
|
|
2518
|
+
const dueAt = row.dueAt;
|
|
2519
|
+
|
|
2520
|
+
lane.fiber = yield* fork(
|
|
2521
|
+
Effect.gen(function* () {
|
|
2522
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(
|
|
2523
|
+
lane.dispatchTimeoutMillis,
|
|
2524
|
+
).pipe(Effect.mapError(alarmFailure("host dispatch allowance")));
|
|
2525
|
+
const selectedAt = yield* Clock.currentTimeMillis;
|
|
2526
|
+
|
|
2527
|
+
if (selectedAt >= until || selectedAt + lane.dispatchTimeoutMillis > dispatchEnd) {
|
|
2528
|
+
lane.exhausted = true;
|
|
2529
|
+
|
|
2530
|
+
return Option.some(dueAt);
|
|
2531
|
+
}
|
|
2532
|
+
|
|
2533
|
+
return yield* runQueued(row, lane.run.pipe(Effect.scoped), observed).pipe(
|
|
2534
|
+
Effect.timeoutOrElse({
|
|
2535
|
+
duration: lane.dispatchTimeoutMillis,
|
|
2536
|
+
orElse: () =>
|
|
2537
|
+
DurableAlarmError.make({
|
|
2538
|
+
operation: "host dispatch allowance",
|
|
2539
|
+
message:
|
|
2540
|
+
"The admitted host wave exceeded its allowance; durable work remains pending",
|
|
2541
|
+
}),
|
|
2542
|
+
}),
|
|
2543
|
+
);
|
|
2544
|
+
}),
|
|
2545
|
+
);
|
|
2546
|
+
}
|
|
2547
|
+
if (delivery === undefined && !messageExhausted && !producing.has(DueQueue.Messages)) {
|
|
2548
|
+
// Selection is bounded local work. Fork only an actual due wave, so an empty
|
|
2549
|
+
// deadline check cannot extend retirement or consume another scheduling turn.
|
|
2550
|
+
const row = queued.find((row) => row.id === DueQueue.Messages);
|
|
2551
|
+
|
|
2552
|
+
if (row !== undefined && row.dueAt !== null && row.dueAt <= now) {
|
|
2553
|
+
// Selection can fail too, so charge its attempt before preparing the wave.
|
|
2554
|
+
delivery = yield* fork(
|
|
2555
|
+
runQueued(
|
|
2556
|
+
row,
|
|
2557
|
+
Effect.gen(function* () {
|
|
2558
|
+
const wave = yield* messages.prepare;
|
|
2559
|
+
const selectedAt = yield* Clock.currentTimeMillis;
|
|
2560
|
+
|
|
2561
|
+
if (selectedAt >= until || selectedAt + wave.timeoutMillis > dispatchEnd) {
|
|
2562
|
+
messageExhausted = true;
|
|
2563
|
+
|
|
2564
|
+
return Option.fromNullishOr(row.dueAt);
|
|
2565
|
+
}
|
|
2566
|
+
|
|
2567
|
+
// No second timeout: the driver owes the Claim's timeout/retry commit.
|
|
2568
|
+
return yield* wave.run;
|
|
2569
|
+
}),
|
|
2570
|
+
observed,
|
|
2571
|
+
).pipe(Effect.asVoid),
|
|
2572
|
+
);
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2576
|
+
// Retire a quiet event before consuming hints queued during its native Attempt.
|
|
2577
|
+
// Recovery still gets its checkpoint and, if needed, initial dispatch opportunity.
|
|
2578
|
+
if (
|
|
2579
|
+
native === undefined &&
|
|
2580
|
+
dispatch.active.size === 0 &&
|
|
2581
|
+
recoveryObserved &&
|
|
2582
|
+
!recovery.needsCheckpoint &&
|
|
2583
|
+
backfillObserved &&
|
|
2584
|
+
delivery === undefined &&
|
|
2585
|
+
lanes.every((lane) => lane.fiber === undefined)
|
|
2586
|
+
)
|
|
2587
|
+
break;
|
|
2588
|
+
|
|
2589
|
+
if (
|
|
2590
|
+
native === undefined &&
|
|
2591
|
+
nativeCheck &&
|
|
2592
|
+
dispatch.active.size < nativeDispatchConcurrency
|
|
2593
|
+
) {
|
|
2594
|
+
nativeCheck = false;
|
|
2595
|
+
const nativeCheckpoint = dispatch.needsCheckpoint && dispatch.active.size === 0;
|
|
2596
|
+
const checkpoint = (recoveryObserved && recovery.needsCheckpoint) || nativeCheckpoint;
|
|
2597
|
+
|
|
2598
|
+
if (checkpoint) recovery.needsCheckpoint = false;
|
|
2599
|
+
// Keep this opportunity's reservations across beginNative's storage awaits.
|
|
2600
|
+
// A finishing Attempt cannot turn a refill into another same-Thread Attempt.
|
|
2601
|
+
const reserved = new Set(dispatch.active.keys());
|
|
2602
|
+
|
|
2603
|
+
native = yield* fork(
|
|
2604
|
+
Effect.gen(function* () {
|
|
2605
|
+
const awakened = checkpoint ? started : yield* beginNative(observed);
|
|
2606
|
+
|
|
2607
|
+
return yield* advance(
|
|
2608
|
+
awakened,
|
|
2609
|
+
yieldAfter,
|
|
2610
|
+
observed,
|
|
2611
|
+
recovery,
|
|
2612
|
+
dispatch,
|
|
2613
|
+
!nativeCheckpoint && (!checkpoint || dispatch.settled === 0),
|
|
2614
|
+
reserved,
|
|
2615
|
+
);
|
|
2616
|
+
}),
|
|
2617
|
+
);
|
|
2618
|
+
}
|
|
2619
|
+
|
|
2620
|
+
const next = Math.min(
|
|
2621
|
+
native === undefined ? (result.nextAttemptAt ?? Infinity) : Infinity,
|
|
2622
|
+
...queued.flatMap((row) =>
|
|
2623
|
+
row.id !== DueQueue.Native && row.dueAt !== null && row.dueAt > now
|
|
2624
|
+
? [row.dueAt]
|
|
2625
|
+
: [],
|
|
2626
|
+
),
|
|
2627
|
+
until,
|
|
2628
|
+
);
|
|
2629
|
+
|
|
2630
|
+
const completing = [
|
|
2631
|
+
...(native === undefined ? [] : [native]),
|
|
2632
|
+
...dispatch.active.values(),
|
|
2633
|
+
...(recoveryObserved ? [] : [recoveryFiber]),
|
|
2634
|
+
...(backfillObserved ? [] : [backfill]),
|
|
2635
|
+
...(delivery === undefined ? [] : [delivery]),
|
|
2636
|
+
...lanes.flatMap((lane) => (lane.fiber === undefined ? [] : [lane.fiber])),
|
|
2637
|
+
];
|
|
2638
|
+
|
|
2639
|
+
const ready = yield* Effect.raceAllFirst([
|
|
2640
|
+
...completing.map((fiber) =>
|
|
2641
|
+
Fiber.await<unknown, MaintenancePassFailure>(fiber).pipe(
|
|
2642
|
+
Effect.as("completed" as const),
|
|
2643
|
+
),
|
|
2644
|
+
),
|
|
2645
|
+
Effect.raceAllFirst([
|
|
2646
|
+
notified,
|
|
2647
|
+
mutations.awaitChange(queueRevision),
|
|
2648
|
+
Effect.sleep(Math.max(0, next - now)),
|
|
2649
|
+
]).pipe(Effect.as("scan" as const)),
|
|
2650
|
+
]);
|
|
2651
|
+
|
|
2652
|
+
if (ready === "scan") {
|
|
2653
|
+
nativeCheck = true;
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
// Dispatch is closed. Join each admitted wave without renewing its allowance.
|
|
2657
|
+
// Message Claims retain their driver's own timer and local retry commit.
|
|
2658
|
+
if (native !== undefined) {
|
|
2659
|
+
result = yield* Fiber.join(native);
|
|
2660
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
2661
|
+
observed.nativeOnly = dispatch.active.size > 0;
|
|
2662
|
+
recovery.started = true;
|
|
2663
|
+
yield* Deferred.succeed(recovery.queue, []);
|
|
2664
|
+
}
|
|
2665
|
+
for (const attempt of dispatch.active.values()) {
|
|
2666
|
+
const settled = yield* Fiber.join(attempt);
|
|
2667
|
+
|
|
2668
|
+
dispatch.settled += settled;
|
|
2669
|
+
dispatch.progressed ||= settled > 0;
|
|
2670
|
+
}
|
|
2671
|
+
dispatch.active.clear();
|
|
2672
|
+
observed.nativeOnly = false;
|
|
2673
|
+
for (const fiber of [
|
|
2674
|
+
recoveryFiber,
|
|
2675
|
+
backfill,
|
|
2676
|
+
...(delivery === undefined ? [] : [delivery]),
|
|
2677
|
+
...lanes.flatMap((lane) => (lane.fiber === undefined ? [] : [lane.fiber])),
|
|
2678
|
+
]) {
|
|
2679
|
+
// Await exits without failing fast: every admitted wave owns its original
|
|
2680
|
+
// bounded allowance and cleanup even if another wave has already failed.
|
|
2681
|
+
const exit = yield* Fiber.await<unknown, MaintenancePassFailure>(fiber);
|
|
2682
|
+
|
|
2683
|
+
if (Exit.isFailure(exit)) failure ??= exit.cause;
|
|
2684
|
+
}
|
|
2685
|
+
if (failure !== undefined) return yield* Effect.failCause(failure);
|
|
2686
|
+
if (recovery.needsCheckpoint || dispatch.dispatched > 0) {
|
|
2687
|
+
result = yield* advance(started, yieldAfter, observed, recovery, dispatch, false);
|
|
2688
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
2689
|
+
}
|
|
2690
|
+
yield* Scope.close(auxiliaryScope, Exit.void);
|
|
2691
|
+
|
|
2692
|
+
return { result, phase, recovered: recovery.recovered, settled: dispatch.settled };
|
|
2693
|
+
});
|
|
2694
|
+
|
|
2695
|
+
const pass = Effect.fn("ThreadMaintenance.pass")(function* (
|
|
2696
|
+
yieldAfter: DateTime.Utc,
|
|
2697
|
+
dispatchUntil: DateTime.Utc,
|
|
2698
|
+
observed: MaintenanceObservation,
|
|
2699
|
+
): Effect.fn.Return<MaintenancePassReport, MaintenancePassFailure, Scope.Scope> {
|
|
2700
|
+
const concurrent = yield* Effect.exit(
|
|
2701
|
+
Effect.scoped(dispatch(yieldAfter, dispatchUntil, observed)),
|
|
2702
|
+
);
|
|
2703
|
+
|
|
2704
|
+
// Closing the concurrent scope first releases every native Attempt and its resources.
|
|
2705
|
+
// Select fresh revisions: finalizers may have retained the last host obligation.
|
|
2706
|
+
const afterNative = yield* Effect.gen(function* () {
|
|
2707
|
+
const lanes = [recoveryEventLane, ...framework.lanes, ...host.lanes];
|
|
2708
|
+
const selected = lanes.filter((lane) => lane.phase === "after-native");
|
|
2709
|
+
|
|
2710
|
+
if (selected.length === 0) return [];
|
|
2711
|
+
const queued = yield* queueSnapshot;
|
|
2712
|
+
const producing = yield* mutations.activeLanes;
|
|
2713
|
+
|
|
2714
|
+
// Finished native work may have produced new source facts. They are already
|
|
2715
|
+
// present at this phase boundary; only arrivals during the host wave preempt it.
|
|
2716
|
+
for (const row of queued)
|
|
2717
|
+
if (!producing.has(row.id)) observed.available.set(row.id, row.revision);
|
|
2718
|
+
const dispatchEnd = DateTime.toEpochMillis(dispatchUntil);
|
|
2719
|
+
const afterNativeIds = new Set(selected.map((lane) => lane.id));
|
|
2720
|
+
const completed: Array<Exit.Exit<void, DurableAlarmError>> = [];
|
|
2721
|
+
let yielded = false;
|
|
2722
|
+
|
|
2723
|
+
// Subscribe by revision before reading the queue, so an enrollment racing the
|
|
2724
|
+
// snapshot cannot be lost. Only producer notifications rescan scheduling state;
|
|
2725
|
+
// there is no timer or application deadline read. New arrivals renew no budget.
|
|
2726
|
+
const yieldToConcurrent = Effect.gen(function* () {
|
|
2727
|
+
while (true) {
|
|
2728
|
+
const revision = yield* mutations.revision;
|
|
2729
|
+
const now = yield* Clock.currentTimeMillis;
|
|
2730
|
+
|
|
2731
|
+
const current = yield* queueSnapshot;
|
|
2732
|
+
const active = yield* mutations.activeLanes;
|
|
2733
|
+
|
|
2734
|
+
if (
|
|
2735
|
+
current.some(
|
|
2736
|
+
(row) =>
|
|
2737
|
+
!afterNativeIds.has(row.id) &&
|
|
2738
|
+
!active.has(row.id) &&
|
|
2739
|
+
row.dueAt !== null &&
|
|
2740
|
+
row.dueAt <= now &&
|
|
2741
|
+
row.revision !== observed.available.get(row.id),
|
|
2742
|
+
)
|
|
2743
|
+
) {
|
|
2744
|
+
yielded = true;
|
|
2745
|
+
|
|
2746
|
+
return completed;
|
|
2747
|
+
}
|
|
2748
|
+
yield* mutations.awaitChange(revision);
|
|
2749
|
+
}
|
|
2750
|
+
});
|
|
2751
|
+
|
|
2752
|
+
yield* validateLanes(lanes, queued);
|
|
2753
|
+
|
|
2754
|
+
const cursor = yield* mutations.withSnapshot(() =>
|
|
2755
|
+
runTransaction("read post-native dispatch cursor", () =>
|
|
2756
|
+
dueQueue.transaction(
|
|
2757
|
+
async (transaction) =>
|
|
2758
|
+
(await readMaintenanceState(transaction)).state.lastAfterNativeLaneId,
|
|
2759
|
+
),
|
|
2760
|
+
),
|
|
2761
|
+
);
|
|
2762
|
+
|
|
2763
|
+
const ordered = selected.toSorted((left, right) =>
|
|
2764
|
+
left.id < right.id ? -1 : left.id > right.id ? 1 : 0,
|
|
2765
|
+
);
|
|
2766
|
+
|
|
2767
|
+
const after = cursor === undefined ? 0 : ordered.findIndex((lane) => lane.id > cursor);
|
|
2768
|
+
|
|
2769
|
+
const rotated =
|
|
2770
|
+
after <= 0 ? ordered : [...ordered.slice(after), ...ordered.slice(0, after)];
|
|
2771
|
+
|
|
2772
|
+
const waves = Effect.forEach(
|
|
2773
|
+
rotated,
|
|
2774
|
+
(lane) =>
|
|
2775
|
+
Effect.gen(function* () {
|
|
2776
|
+
const row = queued.find((row) => row.id === lane.id);
|
|
2777
|
+
const now = yield* Clock.currentTimeMillis;
|
|
2778
|
+
|
|
2779
|
+
if (
|
|
2780
|
+
row === undefined ||
|
|
2781
|
+
row.dueAt === null ||
|
|
2782
|
+
row.dueAt > now ||
|
|
2783
|
+
producing.has(lane.id)
|
|
2784
|
+
)
|
|
2785
|
+
return;
|
|
2786
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(
|
|
2787
|
+
lane.dispatchTimeoutMillis,
|
|
2788
|
+
).pipe(Effect.mapError(alarmFailure("host dispatch allowance")));
|
|
2789
|
+
if (now + lane.dispatchTimeoutMillis > dispatchEnd) return;
|
|
2790
|
+
|
|
2791
|
+
// Persist admission, not completion: slow or failing early lanes must not
|
|
2792
|
+
// monopolize the next event's remaining allowance. Domain receipts stay local
|
|
2793
|
+
// to the lane and its due-queue revision still fences acknowledgement.
|
|
2794
|
+
const admit = mutations.withSnapshot(() =>
|
|
2795
|
+
runTransaction("advance post-native dispatch cursor", () =>
|
|
2796
|
+
dueQueue.transaction(async (transaction) => {
|
|
2797
|
+
const { state } = await readMaintenanceState(transaction);
|
|
2798
|
+
|
|
2799
|
+
await transaction.put(
|
|
2800
|
+
MAINTENANCE_STATE_KEY,
|
|
2801
|
+
encodeMaintenanceState(
|
|
2802
|
+
ThreadMaintenanceState.make({ ...state, lastAfterNativeLaneId: lane.id }),
|
|
2803
|
+
),
|
|
2804
|
+
);
|
|
2805
|
+
}),
|
|
2806
|
+
),
|
|
2807
|
+
);
|
|
2808
|
+
|
|
2809
|
+
yield* runQueued(
|
|
2810
|
+
row,
|
|
2811
|
+
admit.pipe(Effect.andThen(lane.run.pipe(Effect.scoped))),
|
|
2812
|
+
observed,
|
|
2813
|
+
() => yielded,
|
|
2814
|
+
).pipe(
|
|
2815
|
+
Effect.timeoutOrElse({
|
|
2816
|
+
duration: lane.dispatchTimeoutMillis,
|
|
2817
|
+
orElse: () =>
|
|
2818
|
+
DurableAlarmError.make({
|
|
2819
|
+
operation: "host dispatch allowance",
|
|
2820
|
+
message:
|
|
2821
|
+
"The admitted host wave exceeded its allowance; durable work remains pending",
|
|
2822
|
+
}),
|
|
2823
|
+
}),
|
|
2824
|
+
);
|
|
2825
|
+
}).pipe(
|
|
2826
|
+
Effect.exit,
|
|
2827
|
+
Effect.tap((exit) =>
|
|
2828
|
+
Effect.sync(() => {
|
|
2829
|
+
if (!yielded || Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause))
|
|
2830
|
+
completed.push(exit);
|
|
2831
|
+
}),
|
|
2832
|
+
),
|
|
2833
|
+
),
|
|
2834
|
+
{ concurrency: afterNativeDispatchConcurrency },
|
|
2835
|
+
).pipe(Effect.as(completed));
|
|
2836
|
+
|
|
2837
|
+
// raceFirst waits for interrupted wave scopes to close before the alarm can
|
|
2838
|
+
// retire and admit native work in its next event. Finished receipts stay finished.
|
|
2839
|
+
return yield* Effect.raceFirst(waves, yieldToConcurrent);
|
|
2840
|
+
}).pipe(Effect.exit);
|
|
2841
|
+
|
|
2842
|
+
let failure = Exit.isFailure(concurrent) ? concurrent.cause : Cause.empty;
|
|
2843
|
+
|
|
2844
|
+
if (Exit.isFailure(afterNative)) failure = Cause.combine(failure, afterNative.cause);
|
|
2845
|
+
else
|
|
2846
|
+
for (const exit of afterNative.value) {
|
|
2847
|
+
if (Exit.isFailure(exit)) failure = Cause.combine(failure, exit.cause);
|
|
2848
|
+
}
|
|
2849
|
+
if (Exit.isFailure(concurrent) || failure.reasons.length > 0)
|
|
2850
|
+
return yield* Effect.failCause(failure);
|
|
2851
|
+
const { result, phase, recovered, settled } = concurrent.value;
|
|
2852
|
+
|
|
2853
|
+
yield* failpoint.hit("maintenance:finish:before");
|
|
2854
|
+
|
|
2855
|
+
const disposition = yield* mutations.withSnapshot((active) =>
|
|
2856
|
+
Effect.gen(function* () {
|
|
2857
|
+
const now = yield* Clock.currentTimeMillis;
|
|
2858
|
+
|
|
2859
|
+
return yield* runTransaction("finish maintenance event", () =>
|
|
2860
|
+
dueQueue.transaction(async (transaction) => {
|
|
2861
|
+
const { state } = await readMaintenanceState(transaction);
|
|
2862
|
+
|
|
2863
|
+
const native =
|
|
2864
|
+
active > 0 || state.dirty > state.processed
|
|
2865
|
+
? state.retry?.generation === state.dirty && active === 0
|
|
2866
|
+
? Math.max(now + minimumAlarmDelay, state.retry.notBefore)
|
|
2867
|
+
: state.dirty === observed.generation && active === 0
|
|
2868
|
+
? (result.nextAttemptAt ?? now + minimumAlarmDelay)
|
|
2869
|
+
: now + minimumAlarmDelay
|
|
2870
|
+
: Infinity;
|
|
2871
|
+
|
|
2872
|
+
dueQueue.checkpointNative(Number.isFinite(native) ? native : null);
|
|
2873
|
+
const next = DueQueue.next(dueQueue.read());
|
|
2874
|
+
|
|
2875
|
+
if (Number.isFinite(next)) {
|
|
2876
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
|
|
2877
|
+
|
|
2878
|
+
return "rearmed" as const;
|
|
2879
|
+
}
|
|
2880
|
+
await transaction.deleteAlarm();
|
|
2881
|
+
|
|
2882
|
+
return "cleared" as const;
|
|
2883
|
+
}),
|
|
2884
|
+
);
|
|
2885
|
+
}),
|
|
2886
|
+
);
|
|
2887
|
+
|
|
2888
|
+
yield* failpoint.hit("maintenance:finish:after");
|
|
2889
|
+
|
|
2890
|
+
const report = MaintenancePassReport.make({
|
|
2891
|
+
phase,
|
|
2892
|
+
recovered,
|
|
2893
|
+
settled,
|
|
2894
|
+
nonterminal: result.nonterminal,
|
|
2895
|
+
alarm: disposition,
|
|
2896
|
+
});
|
|
2897
|
+
|
|
2898
|
+
yield* Effect.annotateCurrentSpan({
|
|
2899
|
+
phase: report.phase,
|
|
2900
|
+
recovered: report.recovered,
|
|
2901
|
+
settled: report.settled,
|
|
2902
|
+
nonterminal: report.nonterminal,
|
|
2903
|
+
alarm: report.alarm,
|
|
2904
|
+
});
|
|
2905
|
+
|
|
2906
|
+
return report;
|
|
2907
|
+
});
|
|
2908
|
+
|
|
2909
|
+
return ThreadMaintenance.of({
|
|
2910
|
+
// A mid-pass immediate hint is droppable; durable dirty state decides the final alarm.
|
|
2911
|
+
pass: Effect.gen(function* () {
|
|
2912
|
+
const now = yield* Clock.currentTimeMillis;
|
|
2913
|
+
const yieldAfter = DateTime.makeUnsafe(now + 10 * 60_000);
|
|
2914
|
+
const dispatchUntil = DateTime.makeUnsafe(now + 14 * 60_000);
|
|
2915
|
+
|
|
2916
|
+
const observed: MaintenanceObservation = {
|
|
2917
|
+
nativeOnly: false,
|
|
2918
|
+
queue: new Map(),
|
|
2919
|
+
charged: new Map(),
|
|
2920
|
+
available: new Map(),
|
|
2921
|
+
};
|
|
2922
|
+
|
|
2923
|
+
return yield* alarm.withWakesDeferred(
|
|
2924
|
+
maintenancePassGate.withPermit(
|
|
2925
|
+
dueQueue.withView(Effect.scoped(pass(yieldAfter, dispatchUntil, observed))).pipe(
|
|
2926
|
+
// Close event-owned auxiliary work and release Attempt ownership before
|
|
2927
|
+
// failure rearming, while still holding the pass permit.
|
|
2928
|
+
Effect.onErrorIf(
|
|
2929
|
+
() => true,
|
|
2930
|
+
() => rearmFailure(observed),
|
|
2931
|
+
),
|
|
2932
|
+
),
|
|
2933
|
+
),
|
|
2934
|
+
);
|
|
2935
|
+
}).pipe(
|
|
2936
|
+
// Include permit waiting, recovery and acknowledgement in the event deadline.
|
|
2937
|
+
// Interruption releases Attempt ownership, leaving the prearmed dirty generation
|
|
2938
|
+
// for recovery. It never changes the logical Run duration or settles a policy failure.
|
|
2939
|
+
// This cooperative timer cannot preempt synchronous CPU work or stuck finalizers.
|
|
2940
|
+
Effect.timeoutOrElse({
|
|
2941
|
+
duration: "14 minutes",
|
|
2942
|
+
orElse: () =>
|
|
2943
|
+
DurableAlarmError.make({
|
|
2944
|
+
operation: "maintenance pass deadline",
|
|
2945
|
+
message:
|
|
2946
|
+
"The maintenance event exceeded its 14 minute deadline; durable recovery remains pending",
|
|
2947
|
+
}),
|
|
2948
|
+
}),
|
|
2949
|
+
Effect.ensuring(reportParked),
|
|
2950
|
+
),
|
|
2951
|
+
ensureAlarm: mutations
|
|
2952
|
+
.withSnapshot(() => ensureAlarm())
|
|
2953
|
+
.pipe(Effect.tap(() => reportParked)),
|
|
2954
|
+
withMutation: (body) =>
|
|
2955
|
+
mutations.withMutation(
|
|
2956
|
+
body.pipe(Effect.tap(() => publishCommitted.pipe(Effect.provide(publicationContext)))),
|
|
2957
|
+
),
|
|
2958
|
+
});
|
|
2959
|
+
}),
|
|
2960
|
+
);
|
|
2961
|
+
}
|