@effect-agent/platform-cloudflare 0.1.0-beta.11 → 0.1.0-beta.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Alarm.d.mts +208 -0
- package/dist/Alarm.mjs +601 -0
- package/dist/Alarm.mjs.map +1 -0
- package/dist/BrowserRestCapture.d.mts +34 -0
- package/dist/BrowserRestCapture.mjs +238 -0
- package/dist/BrowserRestCapture.mjs.map +1 -0
- package/dist/BrowserRestCrawl.d.mts +16 -0
- package/dist/BrowserRestCrawl.mjs +244 -0
- package/dist/BrowserRestCrawl.mjs.map +1 -0
- package/dist/CloudflareAiGateway.d.mts +64 -0
- package/dist/CloudflareAiGateway.mjs +70 -0
- package/dist/CloudflareAiGateway.mjs.map +1 -0
- package/dist/CloudflareBindings.d.mts +115 -0
- package/dist/CloudflareBindings.mjs +113 -0
- package/dist/CloudflareBindings.mjs.map +1 -0
- package/dist/CloudflareBrowser-DV6kya1H.mjs +495 -0
- package/dist/CloudflareBrowser-DV6kya1H.mjs.map +1 -0
- package/dist/CloudflareBrowser-Doe1M7R5.d.mts +80 -0
- package/dist/CloudflareBrowser.d.mts +2 -0
- package/dist/CloudflareBrowser.mjs +2 -0
- package/dist/CloudflareCodeMode.d.mts +44 -0
- package/dist/CloudflareCodeMode.mjs +616 -0
- package/dist/CloudflareCodeMode.mjs.map +1 -0
- package/dist/CloudflareConfig-DKXGo8L3.d.mts +106 -0
- package/dist/CloudflareConfig.d.mts +2 -0
- package/dist/CloudflareConfig.mjs +128 -0
- package/dist/CloudflareConfig.mjs.map +1 -0
- package/dist/CloudflareMemory.d.mts +142 -0
- package/dist/CloudflareMemory.mjs +202 -0
- package/dist/CloudflareMemory.mjs.map +1 -0
- package/dist/CloudflareScheduling.d.mts +52 -0
- package/dist/CloudflareScheduling.mjs +396 -0
- package/dist/CloudflareScheduling.mjs.map +1 -0
- package/dist/CloudflareSubscriptions.d.mts +87 -0
- package/dist/CloudflareSubscriptions.mjs +530 -0
- package/dist/CloudflareSubscriptions.mjs.map +1 -0
- package/dist/CloudflareThreadClient.d.mts +1675 -0
- package/dist/CloudflareThreadClient.mjs +382 -0
- package/dist/CloudflareThreadClient.mjs.map +1 -0
- package/dist/InteractiveBrowser-CKp6pqmw.d.mts +171 -0
- package/dist/InteractiveBrowser-l6J4eP6T.mjs +1427 -0
- package/dist/InteractiveBrowser-l6J4eP6T.mjs.map +1 -0
- package/dist/InteractiveBrowser.d.mts +2 -0
- package/dist/InteractiveBrowser.mjs +2 -0
- package/dist/ProtectedBrowser.d.mts +123 -0
- package/dist/ProtectedBrowser.mjs +1161 -0
- package/dist/ProtectedBrowser.mjs.map +1 -0
- package/dist/ThreadObject-BGla_Obk.mjs +858 -0
- package/dist/ThreadObject-BGla_Obk.mjs.map +1 -0
- package/dist/ThreadObject-MSlpVBUD.d.mts +829 -0
- package/dist/ThreadObject.d.mts +2 -0
- package/dist/ThreadObject.mjs +3 -0
- package/dist/WakeScheduler.d.mts +23 -0
- package/dist/WakeScheduler.mjs +57 -0
- package/dist/WakeScheduler.mjs.map +1 -0
- package/dist/boundary-BguazVkh.mjs +42 -0
- package/dist/boundary-BguazVkh.mjs.map +1 -0
- package/dist/index.d.mts +13 -1548
- package/dist/index.mjs +13 -1636
- package/dist/prepared-admission-DU0_T-ev.mjs +73 -0
- package/dist/prepared-admission-DU0_T-ev.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -53
- package/src/Alarm.ts +1323 -0
- package/src/BrowserRestCapture.ts +477 -0
- package/src/BrowserRestCrawl.ts +540 -0
- package/src/CloudflareAiGateway.ts +170 -0
- package/src/CloudflareBindings.ts +199 -0
- package/src/CloudflareBrowser.ts +15 -0
- package/src/{code-mode-executor.ts → CloudflareCodeMode.ts} +372 -204
- package/src/{config.ts → CloudflareConfig.ts} +19 -12
- package/src/CloudflareMemory.ts +420 -0
- package/src/CloudflareScheduling.ts +747 -0
- package/src/CloudflareSubscriptions.ts +1041 -0
- package/src/CloudflareThreadClient.ts +809 -0
- package/src/InteractiveBrowser.ts +2612 -0
- package/src/ProtectedBrowser.ts +9 -0
- package/src/ThreadObject.ts +1184 -0
- package/src/{wake-scheduler.ts → WakeScheduler.ts} +32 -26
- package/src/index.ts +12 -23
- package/src/internal/boundary.ts +55 -0
- package/src/internal/browser-binding.ts +82 -0
- package/src/internal/browser-failure.ts +92 -0
- package/src/internal/browser-file-selection.ts +126 -0
- package/src/internal/browser-quick-action.ts +857 -0
- package/src/internal/browser-session-lifecycle.ts +193 -0
- package/src/internal/layers.ts +765 -0
- package/src/internal/message-delivery.ts +181 -0
- package/src/internal/prepared-admission.ts +116 -0
- package/src/internal/progress-wait.ts +121 -0
- package/src/internal/transport.ts +44 -0
- package/src/protected-browser/binding.ts +354 -0
- package/src/protected-browser/host.ts +379 -0
- package/src/protected-browser/inspect-frame.ts +125 -0
- package/src/protected-browser/native.ts +473 -0
- package/src/protected-browser/policy.ts +857 -0
- package/dist/index.mjs.map +0 -1
- package/src/alarm.ts +0 -334
- package/src/bindings.ts +0 -145
- package/src/client.ts +0 -646
- package/src/conversation-object.ts +0 -768
- package/src/layers.ts +0 -376
- package/src/transport.ts +0 -38
package/src/Alarm.ts
ADDED
|
@@ -0,0 +1,1323 @@
|
|
|
1
|
+
import {
|
|
2
|
+
Cause,
|
|
3
|
+
Clock,
|
|
4
|
+
Context,
|
|
5
|
+
DateTime,
|
|
6
|
+
Deferred,
|
|
7
|
+
Effect,
|
|
8
|
+
Exit,
|
|
9
|
+
Fiber,
|
|
10
|
+
Layer,
|
|
11
|
+
Option,
|
|
12
|
+
Random,
|
|
13
|
+
Ref,
|
|
14
|
+
Schema,
|
|
15
|
+
Scope,
|
|
16
|
+
Semaphore,
|
|
17
|
+
Stream,
|
|
18
|
+
Struct,
|
|
19
|
+
} from "effect";
|
|
20
|
+
import { type DurableBindingFailure } from "effect-agent/agent-registration";
|
|
21
|
+
import {
|
|
22
|
+
DurableAgentRuntime,
|
|
23
|
+
type DurableWorkerFailure,
|
|
24
|
+
type RecoveryReport,
|
|
25
|
+
} from "effect-agent/durable-agent-runtime";
|
|
26
|
+
import { ThreadId, SubmissionId } from "effect-agent/identifiers";
|
|
27
|
+
import { SubmissionLedger, type SubmissionSnapshot } from "effect-agent/submission-ledger";
|
|
28
|
+
import {
|
|
29
|
+
ThreadProjectionMaintenance,
|
|
30
|
+
drainDue,
|
|
31
|
+
type ThreadProjectionError,
|
|
32
|
+
} from "effect-agent/thread-projection-maintenance";
|
|
33
|
+
import { WakeScheduler } from "effect-agent/wake-scheduler";
|
|
34
|
+
import { SqlClient } from "effect/unstable/sql/SqlClient";
|
|
35
|
+
|
|
36
|
+
import { DurableObjectContext } from "./CloudflareBindings.ts";
|
|
37
|
+
import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.ts";
|
|
38
|
+
import { safeCauseMessage } from "./internal/boundary.ts";
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The single multiplexed Durable Object alarm (decision D-P6-2). A Durable Object has ONE
|
|
42
|
+
* alarm slot; every cadence the Node host ran on fibers (wake scan, lease expiry, settlement
|
|
43
|
+
* and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
|
|
44
|
+
* the slot always holds the EARLIEST deadline any caller asked for.
|
|
45
|
+
*
|
|
46
|
+
* The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
|
|
47
|
+
* maintenance generation and a committed alarm. Stable externally-driven waits may be
|
|
48
|
+
* nonterminal without retaining an alarm; their resolving mutation advances the generation and
|
|
49
|
+
* restores the alarm atomically.
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
/** The Durable Object alarm API failed; surfaces on host entry points as a typed refusal. */
|
|
53
|
+
export class DurableAlarmError extends Schema.TaggedError<DurableAlarmError>()(
|
|
54
|
+
"DurableAlarmError",
|
|
55
|
+
{
|
|
56
|
+
operation: Schema.String,
|
|
57
|
+
message: Schema.String,
|
|
58
|
+
cause: Schema.optionalKey(Schema.Defect()),
|
|
59
|
+
},
|
|
60
|
+
) {}
|
|
61
|
+
|
|
62
|
+
const alarmFailure =
|
|
63
|
+
(operation: string) =>
|
|
64
|
+
(cause: unknown): DurableAlarmError =>
|
|
65
|
+
DurableAlarmError.make({
|
|
66
|
+
operation,
|
|
67
|
+
message: safeCauseMessage(cause, "The Cloudflare alarm API failed without a diagnostic"),
|
|
68
|
+
cause,
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// SQL and raw KV/alarm operations share one physical SQLite transaction. Reserve its
|
|
72
|
+
// connection for each short storage operation, never around a mutation or snapshot body.
|
|
73
|
+
const makeStorageOperation = Effect.map(
|
|
74
|
+
SqlClient,
|
|
75
|
+
(sql) =>
|
|
76
|
+
<A>(operation: string, execute: () => Promise<A>) =>
|
|
77
|
+
Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => {
|
|
78
|
+
const body = Effect.uninterruptible(
|
|
79
|
+
Effect.tryPromise({ try: execute, catch: alarmFailure(operation) }),
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
return current._tag === "Some"
|
|
83
|
+
? body
|
|
84
|
+
: Effect.scoped(
|
|
85
|
+
Effect.andThen(sql.reserve.pipe(Effect.mapError(alarmFailure(operation))), body),
|
|
86
|
+
);
|
|
87
|
+
}),
|
|
88
|
+
);
|
|
89
|
+
|
|
90
|
+
/** `ctx.storage` alarm slot as an Effect service; storage is truth, never a memory field. */
|
|
91
|
+
export class DurableAlarmService extends Context.Service<
|
|
92
|
+
DurableAlarmService,
|
|
93
|
+
{
|
|
94
|
+
/** The scheduled deadline in epoch milliseconds, if any. */
|
|
95
|
+
readonly scheduled: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
96
|
+
/** Replace the slot with this deadline. */
|
|
97
|
+
readonly scheduleAt: (epochMillis: number) => Effect.Effect<void, DurableAlarmError>;
|
|
98
|
+
/** Keep the EARLIER of the existing deadline and this one (the multiplexing rule). */
|
|
99
|
+
readonly ensureScheduledBy: (epochMillis: number) => Effect.Effect<void, DurableAlarmError>;
|
|
100
|
+
/**
|
|
101
|
+
* Arm an immediate alarm (the durable, coalescing local wake) — DEFERRED while a
|
|
102
|
+
* maintenance pass is executing. Workerd cancels an in-flight alarm handler when a new
|
|
103
|
+
* EARLIER deadline is written during its execution (`requestScheduledAlarm`), and the
|
|
104
|
+
* maintenance pass runs INSIDE the alarm handler: an immediate wake landing mid-pass
|
|
105
|
+
* (a routed port mutation, a sibling's `wake()`, the coordinator's own local notify)
|
|
106
|
+
* would kill the running Attempt — manufacturing an ownership loss no real eviction
|
|
107
|
+
* caused, and routing open uncertain-class Tool Calls into spurious Unknown Outcomes.
|
|
108
|
+
* Deferral is contract-safe: wakes are droppable hints, every mutating entry point
|
|
109
|
+
* pre-arms BEFORE its first durable mutation (the alarm invariant never rests on this
|
|
110
|
+
* call). The pass's durable generation check observes any racing mutation, so the
|
|
111
|
+
* in-memory hint does not need to be flushed after a stable wait is acknowledged.
|
|
112
|
+
*/
|
|
113
|
+
readonly scheduleNow: Effect.Effect<void, DurableAlarmError>;
|
|
114
|
+
/**
|
|
115
|
+
* Run one maintenance pass with wake deferral (see `scheduleNow`). Calls made while `body`
|
|
116
|
+
* executes are droppable promptness hints; correctness rests on the durable generation.
|
|
117
|
+
*/
|
|
118
|
+
readonly withWakesDeferred: <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
119
|
+
/** Clear the slot; correctness-sensitive clears live in maintenance generation transactions. */
|
|
120
|
+
readonly cancel: Effect.Effect<void, DurableAlarmError>;
|
|
121
|
+
}
|
|
122
|
+
>()("@effect-agent/platform-cloudflare/DurableAlarmService") {
|
|
123
|
+
static readonly layer: Layer.Layer<DurableAlarmService, never, DurableObjectContext | SqlClient> =
|
|
124
|
+
Layer.effect(DurableAlarmService)(
|
|
125
|
+
Effect.gen(function* () {
|
|
126
|
+
const { ctx } = yield* DurableObjectContext;
|
|
127
|
+
/**
|
|
128
|
+
* In-memory pass bookkeeping — a pure CACHE, never state: a fresh incarnation has no
|
|
129
|
+
* running pass, and a deferred wake lost to eviction was only ever a promptness hint
|
|
130
|
+
* on top of the already-committed pre-armed alarm.
|
|
131
|
+
*/
|
|
132
|
+
const runningPasses = yield* Ref.make(0);
|
|
133
|
+
|
|
134
|
+
const storageOperation = yield* makeStorageOperation;
|
|
135
|
+
|
|
136
|
+
const scheduled = storageOperation("get alarm", () => ctx.storage.getAlarm()).pipe(
|
|
137
|
+
Effect.map((deadline) =>
|
|
138
|
+
deadline === null ? Option.none<number>() : Option.some(deadline),
|
|
139
|
+
),
|
|
140
|
+
);
|
|
141
|
+
|
|
142
|
+
const scheduleAt = (epochMillis: number) =>
|
|
143
|
+
storageOperation("set alarm", () => ctx.storage.setAlarm(epochMillis));
|
|
144
|
+
|
|
145
|
+
const ensureScheduledBy = (epochMillis: number) =>
|
|
146
|
+
storageOperation("ensure alarm", () =>
|
|
147
|
+
ctx.storage.transaction(async (transaction) => {
|
|
148
|
+
const existing = await transaction.getAlarm();
|
|
149
|
+
|
|
150
|
+
if (existing === null || existing > epochMillis) {
|
|
151
|
+
await transaction.setAlarm(epochMillis);
|
|
152
|
+
}
|
|
153
|
+
}),
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
const armNow = Clock.currentTimeMillis.pipe(
|
|
157
|
+
Effect.flatMap((now) => ensureScheduledBy(now)),
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
const scheduleNow = Ref.get(runningPasses).pipe(
|
|
161
|
+
Effect.flatMap((passes) => (passes > 0 ? Effect.void : armNow)),
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
const withWakesDeferred = <A, E, R>(body: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
|
|
165
|
+
Ref.update(runningPasses, (passes) => passes + 1).pipe(
|
|
166
|
+
Effect.andThen(body),
|
|
167
|
+
Effect.ensuring(Ref.update(runningPasses, (passes) => passes - 1)),
|
|
168
|
+
);
|
|
169
|
+
|
|
170
|
+
const cancel = storageOperation("delete alarm", () => ctx.storage.deleteAlarm());
|
|
171
|
+
|
|
172
|
+
return DurableAlarmService.of({
|
|
173
|
+
scheduled,
|
|
174
|
+
scheduleAt,
|
|
175
|
+
ensureScheduledBy,
|
|
176
|
+
scheduleNow,
|
|
177
|
+
withWakesDeferred,
|
|
178
|
+
cancel,
|
|
179
|
+
});
|
|
180
|
+
}),
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
|
|
185
|
+
export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
|
|
186
|
+
"@effect-agent/platform-cloudflare/MaintenancePassReport",
|
|
187
|
+
)({
|
|
188
|
+
/** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
|
|
189
|
+
phase: Schema.Literals(["caught-up", "actionable"]),
|
|
190
|
+
/** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
|
|
191
|
+
recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
192
|
+
/** Head Attempts settled during the event. Joined input may settle with each head. */
|
|
193
|
+
settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
194
|
+
/** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
|
|
195
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
196
|
+
/** `rearmed` for dirty/autonomous work, `cleared` for stable waits or settlement. */
|
|
197
|
+
alarm: Schema.Literals(["rearmed", "cleared"]),
|
|
198
|
+
}) {}
|
|
199
|
+
|
|
200
|
+
/** Fault boundaries around every maintenance-owned durable mutation. */
|
|
201
|
+
export type ThreadMaintenanceFailpointLocation =
|
|
202
|
+
| "maintenance:dirty:before"
|
|
203
|
+
| "maintenance:dirty:after"
|
|
204
|
+
| "maintenance:mutation:armed"
|
|
205
|
+
| "maintenance:mutation:finished"
|
|
206
|
+
| "maintenance:ensure:before"
|
|
207
|
+
| "maintenance:ensure:after"
|
|
208
|
+
| "maintenance:begin:before"
|
|
209
|
+
| "maintenance:begin:after"
|
|
210
|
+
| "maintenance:select:before"
|
|
211
|
+
| "maintenance:select:after"
|
|
212
|
+
| "maintenance:binding-retry:before"
|
|
213
|
+
| "maintenance:binding-retry:after"
|
|
214
|
+
| "maintenance:retry:before"
|
|
215
|
+
| "maintenance:retry:after"
|
|
216
|
+
| "maintenance:checkpoint:before"
|
|
217
|
+
| "maintenance:checkpoint:after"
|
|
218
|
+
| "maintenance:finish:before"
|
|
219
|
+
| "maintenance:finish:after";
|
|
220
|
+
|
|
221
|
+
export type ThreadMaintenanceFailpointHandler = (
|
|
222
|
+
location: ThreadMaintenanceFailpointLocation,
|
|
223
|
+
) => Effect.Effect<void>;
|
|
224
|
+
|
|
225
|
+
/** Test-only fault authority; production uses the inert layer. */
|
|
226
|
+
export class ThreadMaintenanceFailpoint extends Context.Service<
|
|
227
|
+
ThreadMaintenanceFailpoint,
|
|
228
|
+
{
|
|
229
|
+
readonly hit: ThreadMaintenanceFailpointHandler;
|
|
230
|
+
}
|
|
231
|
+
>()("@effect-agent/platform-cloudflare/ThreadMaintenanceFailpoint") {
|
|
232
|
+
static readonly layer = Layer.succeed(this)({ hit: () => Effect.void });
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Durable host publication of canonical records and ledger approval/abort/resolution intents.
|
|
237
|
+
* The host owns schema-versioned cursors, destination idempotency and acknowledgement. Delivery
|
|
238
|
+
* is at least once. Hooks must not write the alarm slot or mutate the supplied raw source ports.
|
|
239
|
+
*
|
|
240
|
+
* `invalidate`, `prepareGeneration` and `pendingDeadline` must be bounded local operations.
|
|
241
|
+
* `prepareGeneration` durably invalidates a scan only when its generation changes; repeated
|
|
242
|
+
* calls must preserve partial scan progress. It runs with no source mutation in flight.
|
|
243
|
+
* Use this gate only for publication required before dependent native execution. Independent
|
|
244
|
+
* UI relays and outboxes belong to ThreadHostMaintenance.
|
|
245
|
+
* `drain` performs bounded delivery and persists retries before returning. A pending deadline
|
|
246
|
+
* defers runtime recovery/Attempts, allowing committed host publications to drain first.
|
|
247
|
+
* Unexpected hook failures leave the prearmed generation for retry. Hooks acquire per-call
|
|
248
|
+
* resources with Effect.scoped; Layer construction owns incarnation resources (eviction need
|
|
249
|
+
* not run finalizers). Do not hold a local hook behind network I/O or call back into producers.
|
|
250
|
+
*/
|
|
251
|
+
export interface ThreadPublicationService {
|
|
252
|
+
readonly invalidate: Effect.Effect<void, DurableAlarmError>;
|
|
253
|
+
readonly prepareGeneration: (generation: bigint) => Effect.Effect<void, DurableAlarmError>;
|
|
254
|
+
readonly drain: Effect.Effect<void, DurableAlarmError>;
|
|
255
|
+
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Opt in with `ThreadObject.layer(registrations, { publication: Layer.effect(ThreadPublication)(...) })`. */
|
|
259
|
+
export class ThreadPublication extends Context.Service<
|
|
260
|
+
ThreadPublication,
|
|
261
|
+
ThreadPublicationService
|
|
262
|
+
>()("@effect-agent/platform-cloudflare/ThreadPublication") {
|
|
263
|
+
static readonly layer = Layer.succeed(this)({
|
|
264
|
+
invalidate: Effect.void,
|
|
265
|
+
prepareGeneration: () => Effect.void,
|
|
266
|
+
drain: Effect.void,
|
|
267
|
+
pendingDeadline: Effect.succeed(Option.none()),
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Host-assembled native message recovery. The driver bounds each actual Claim and persists its
|
|
273
|
+
* timeout/retry before this pump returns. Do not add a second timer starting at batch selection:
|
|
274
|
+
* local Claim setup may take time, and expiration still owes the driver's local retry commit.
|
|
275
|
+
*/
|
|
276
|
+
export const ThreadMessageDelivery = Context.Reference<{
|
|
277
|
+
readonly drainUntil: (
|
|
278
|
+
dispatchClosed: Effect.Effect<void>,
|
|
279
|
+
dispatchUntil: DateTime.Utc,
|
|
280
|
+
) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
|
|
281
|
+
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
282
|
+
}>("@effect-agent/platform-cloudflare/ThreadMessageDelivery", {
|
|
283
|
+
defaultValue: () => ({
|
|
284
|
+
drainUntil: () => Effect.void,
|
|
285
|
+
pendingDeadline: Effect.succeed(Option.none()),
|
|
286
|
+
}),
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Application obligations sharing this Object's alarm. Admit one initial external wave even on
|
|
291
|
+
* a caught-up pass, then respond to wakes until dispatchClosed. This closes only admission
|
|
292
|
+
* of NEW external waves; native work can continue while admitted waves finish. Keep local
|
|
293
|
+
* admission/hub subscriptions in the event Scope until maintenance tears it down.
|
|
294
|
+
* No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
|
|
295
|
+
*
|
|
296
|
+
* Declare a finite whole-wave allowance (1..300000ms): maximum for parallel lanes, sum for
|
|
297
|
+
* sequential operations. Admit a wave only if its allowance fits before dispatchUntil. Later
|
|
298
|
+
* arrivals cannot renew the retirement window. Maintenance bounds the join after dispatchClosed,
|
|
299
|
+
* interrupts and joins event Scope, then reads local deadlines under the mutation gate.
|
|
300
|
+
*
|
|
301
|
+
* Setup and pendingDeadline are bounded local operations. Persist claims/envelopes before
|
|
302
|
+
* dispatch; interrupted waits leave exact retries/receipts recoverable. Cancellation is local,
|
|
303
|
+
* not remote rollback. Retry accounting is unchanged. Network waits/finalizers must be
|
|
304
|
+
* interruptible; short local atomic commits may be uninterruptible. Hooks never write the raw
|
|
305
|
+
* alarm slot. Required native publication gates belong to ThreadPublication.
|
|
306
|
+
*/
|
|
307
|
+
export const ThreadHostMaintenance = Context.Reference<{
|
|
308
|
+
readonly dispatchTimeoutMillis: number;
|
|
309
|
+
readonly drainUntil: (
|
|
310
|
+
dispatchClosed: Effect.Effect<void>,
|
|
311
|
+
dispatchUntil: DateTime.Utc,
|
|
312
|
+
) => Effect.Effect<void, DurableAlarmError, Scope.Scope>;
|
|
313
|
+
readonly pendingDeadline: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
314
|
+
}>("@effect-agent/platform-cloudflare/ThreadHostMaintenance", {
|
|
315
|
+
defaultValue: () => ({
|
|
316
|
+
dispatchTimeoutMillis: 1,
|
|
317
|
+
drainUntil: () => Effect.void,
|
|
318
|
+
pendingDeadline: Effect.succeed(Option.none()),
|
|
319
|
+
}),
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
const earliestDeadline = (
|
|
323
|
+
left: Option.Option<number>,
|
|
324
|
+
right: Option.Option<number>,
|
|
325
|
+
): Option.Option<number> =>
|
|
326
|
+
Option.isSome(left)
|
|
327
|
+
? Option.isSome(right)
|
|
328
|
+
? Option.some(Math.min(left.value, right.value))
|
|
329
|
+
: left
|
|
330
|
+
: right;
|
|
331
|
+
|
|
332
|
+
/** @internal A committed source operation must not become a failed operation because delivery failed. */
|
|
333
|
+
export const publishCommitted = Effect.gen(function* () {
|
|
334
|
+
const publication = yield* ThreadPublication;
|
|
335
|
+
|
|
336
|
+
yield* publication.invalidate.pipe(Effect.andThen(publication.drain));
|
|
337
|
+
}).pipe(
|
|
338
|
+
Effect.catchCause((cause) =>
|
|
339
|
+
Cause.hasInterrupts(cause)
|
|
340
|
+
? Effect.interrupt
|
|
341
|
+
: Effect.logError("Thread publication deferred after source commit", cause),
|
|
342
|
+
),
|
|
343
|
+
);
|
|
344
|
+
|
|
345
|
+
const MaintenanceGeneration = Schema.BigIntFromString.check(
|
|
346
|
+
Schema.isGreaterThanOrEqualToBigInt(0n),
|
|
347
|
+
);
|
|
348
|
+
|
|
349
|
+
class BindingRetry extends Schema.Class<BindingRetry>("BindingRetry")({
|
|
350
|
+
threadId: ThreadId,
|
|
351
|
+
submissionId: SubmissionId,
|
|
352
|
+
attempts: Schema.Natural,
|
|
353
|
+
notBefore: Schema.Finite,
|
|
354
|
+
reportedAt: Schema.Finite,
|
|
355
|
+
}) {}
|
|
356
|
+
|
|
357
|
+
interface NativePassResult {
|
|
358
|
+
readonly phase: "caught-up" | "actionable";
|
|
359
|
+
readonly recovered: number;
|
|
360
|
+
readonly settled: number;
|
|
361
|
+
readonly nonterminal: number;
|
|
362
|
+
readonly nextAttemptAt: number | undefined;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
interface MaintenanceObservation {
|
|
366
|
+
generation?: bigint;
|
|
367
|
+
nativeOnly: boolean;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
class MaintenanceRetry extends Schema.Class<MaintenanceRetry>("MaintenanceRetry")({
|
|
371
|
+
generation: MaintenanceGeneration,
|
|
372
|
+
notBefore: Schema.Finite,
|
|
373
|
+
nativeOnly: Schema.Boolean,
|
|
374
|
+
stalls: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(30)),
|
|
375
|
+
}) {}
|
|
376
|
+
|
|
377
|
+
/** Versioned, platform-private maintenance state stored through Durable Object KV. */
|
|
378
|
+
class ThreadMaintenanceState extends Schema.Class<ThreadMaintenanceState>(
|
|
379
|
+
"@effect-agent/platform-cloudflare/ThreadMaintenanceState",
|
|
380
|
+
)({
|
|
381
|
+
schemaVersion: Schema.Literal(1),
|
|
382
|
+
dirty: MaintenanceGeneration,
|
|
383
|
+
processed: MaintenanceGeneration,
|
|
384
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
385
|
+
/** One physical-owner cursor; old single-lane records need no conversion. */
|
|
386
|
+
lastServedThreadId: Schema.optionalKey(ThreadId),
|
|
387
|
+
bindingRetries: Schema.optionalKey(Schema.Array(BindingRetry)),
|
|
388
|
+
/** Absent on older records. A newer mutation makes this retry obsolete. */
|
|
389
|
+
retry: Schema.optionalKey(MaintenanceRetry),
|
|
390
|
+
}) {}
|
|
391
|
+
|
|
392
|
+
const MAINTENANCE_STATE_KEY = "effect-agent:thread-maintenance:v1";
|
|
393
|
+
const decodeMaintenanceState = Schema.decodeUnknownSync(ThreadMaintenanceState);
|
|
394
|
+
const encodeMaintenanceState = Schema.encodeSync(ThreadMaintenanceState);
|
|
395
|
+
|
|
396
|
+
const initialMaintenanceState = (): ThreadMaintenanceState =>
|
|
397
|
+
ThreadMaintenanceState.make({
|
|
398
|
+
schemaVersion: 1,
|
|
399
|
+
// Bootstrap Objects created by the pre-generation release without scanning the ledger in
|
|
400
|
+
// the constructor. One useful pass classifies and acknowledges any existing obligation.
|
|
401
|
+
dirty: 1n,
|
|
402
|
+
processed: 0n,
|
|
403
|
+
nonterminal: 0,
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
const readMaintenanceState = async (
|
|
407
|
+
transaction: DurableObjectTransaction,
|
|
408
|
+
): Promise<{ readonly state: ThreadMaintenanceState; readonly initialized: boolean }> => {
|
|
409
|
+
const encoded = await transaction.get(MAINTENANCE_STATE_KEY);
|
|
410
|
+
|
|
411
|
+
return encoded === undefined
|
|
412
|
+
? { state: initialMaintenanceState(), initialized: false }
|
|
413
|
+
: { state: decodeMaintenanceState(encoded), initialized: true };
|
|
414
|
+
};
|
|
415
|
+
|
|
416
|
+
const ensureTransactionAlarmBy = async (
|
|
417
|
+
transaction: DurableObjectTransaction,
|
|
418
|
+
deadline: number,
|
|
419
|
+
): Promise<void> => {
|
|
420
|
+
const scheduled = await transaction.getAlarm();
|
|
421
|
+
|
|
422
|
+
if (scheduled === null || scheduled > deadline) {
|
|
423
|
+
await transaction.setAlarm(deadline);
|
|
424
|
+
}
|
|
425
|
+
};
|
|
426
|
+
|
|
427
|
+
const stableExternalWait = (
|
|
428
|
+
snapshot: SubmissionSnapshot,
|
|
429
|
+
reports: ReadonlyMap<string, RecoveryReport>,
|
|
430
|
+
): boolean => {
|
|
431
|
+
const report = reports.get(snapshot.submissionId);
|
|
432
|
+
const decision = report?.decision._tag;
|
|
433
|
+
|
|
434
|
+
// An accepted abort still owes cleanup/settlement even if its claim was deferred this pass.
|
|
435
|
+
if (decision === "SettleAborted") return false;
|
|
436
|
+
switch (snapshot.state) {
|
|
437
|
+
case "suspended":
|
|
438
|
+
case "joined":
|
|
439
|
+
return true;
|
|
440
|
+
case "unknown":
|
|
441
|
+
return report?.disposition === "unknown";
|
|
442
|
+
case "admitted":
|
|
443
|
+
return reports.get(snapshot.submissionId)?.decision._tag === "AwaitParentEstablishment";
|
|
444
|
+
case "input-applied":
|
|
445
|
+
case "joining":
|
|
446
|
+
case "ready":
|
|
447
|
+
case "running":
|
|
448
|
+
case "settled":
|
|
449
|
+
case "terminalizing":
|
|
450
|
+
return false;
|
|
451
|
+
}
|
|
452
|
+
};
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Shared prearm/acknowledgement boundary for ingress and runtime-owned producers.
|
|
456
|
+
* `ThreadObject.layer` provides this same instance in its Services. Rebuilt runtime/maintenance
|
|
457
|
+
* Layers must reuse that instance; a second gate cannot observe the native producers' activity.
|
|
458
|
+
*/
|
|
459
|
+
export class ThreadMutationGate extends Context.Service<
|
|
460
|
+
ThreadMutationGate,
|
|
461
|
+
{
|
|
462
|
+
readonly withMutation: <A, E, R>(
|
|
463
|
+
body: Effect.Effect<A, E, R>,
|
|
464
|
+
/**
|
|
465
|
+
* Native admission, approval, abort and unknown resolution keep the default true.
|
|
466
|
+
* Projection/relay/reply receipt-only bookkeeping must use false: its local durable
|
|
467
|
+
* pendingDeadline owns scheduling without creating native recovery debt.
|
|
468
|
+
*/
|
|
469
|
+
options?: { readonly invalidatesRecovery: boolean },
|
|
470
|
+
) => Effect.Effect<A, E | DurableAlarmError, R>;
|
|
471
|
+
readonly withSnapshot: <A, E, R>(
|
|
472
|
+
body: (active: number) => Effect.Effect<A, E, R>,
|
|
473
|
+
) => Effect.Effect<A, E, R>;
|
|
474
|
+
}
|
|
475
|
+
>()("@effect-agent/platform-cloudflare/internal/ThreadMutationGate") {
|
|
476
|
+
static readonly layer = Layer.effect(this)(
|
|
477
|
+
Effect.gen(function* () {
|
|
478
|
+
const { ctx } = yield* DurableObjectContext;
|
|
479
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
480
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
481
|
+
// A fresh incarnation has no live mutations; durable generations survive eviction.
|
|
482
|
+
const activeMutations = yield* Ref.make(0);
|
|
483
|
+
const generationGate = yield* Semaphore.make(1);
|
|
484
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
485
|
+
|
|
486
|
+
const runTransaction = yield* makeStorageOperation;
|
|
487
|
+
|
|
488
|
+
const beginMutation = Effect.fn("ThreadMaintenance.beginMutation")(function* (
|
|
489
|
+
invalidatesRecovery: boolean,
|
|
490
|
+
) {
|
|
491
|
+
yield* failpoint.hit("maintenance:dirty:before");
|
|
492
|
+
const now = yield* Clock.currentTimeMillis;
|
|
493
|
+
|
|
494
|
+
yield* runTransaction("advance maintenance generation", () =>
|
|
495
|
+
ctx.storage.transaction(async (transaction) => {
|
|
496
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
497
|
+
|
|
498
|
+
const next = ThreadMaintenanceState.make({
|
|
499
|
+
...state,
|
|
500
|
+
dirty: state.dirty + (invalidatesRecovery ? 1n : 0n),
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
if (invalidatesRecovery || !initialized)
|
|
504
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
505
|
+
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
506
|
+
// on its best-effort immediate wake hint.
|
|
507
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
508
|
+
}),
|
|
509
|
+
);
|
|
510
|
+
yield* failpoint.hit("maintenance:dirty:after");
|
|
511
|
+
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
512
|
+
});
|
|
513
|
+
|
|
514
|
+
const endMutation = generationGate.withPermit(
|
|
515
|
+
Ref.update(activeMutations, (active) => Math.max(0, active - 1)),
|
|
516
|
+
);
|
|
517
|
+
|
|
518
|
+
const withMutation = <A, E, R>(
|
|
519
|
+
body: Effect.Effect<A, E, R>,
|
|
520
|
+
options?: { readonly invalidatesRecovery: boolean },
|
|
521
|
+
): Effect.Effect<A, E | DurableAlarmError, R> =>
|
|
522
|
+
Effect.acquireUseRelease(
|
|
523
|
+
generationGate.withPermit(beginMutation(options?.invalidatesRecovery ?? true)),
|
|
524
|
+
() =>
|
|
525
|
+
failpoint.hit("maintenance:mutation:armed").pipe(
|
|
526
|
+
Effect.andThen(body),
|
|
527
|
+
Effect.tap(() => failpoint.hit("maintenance:mutation:finished")),
|
|
528
|
+
),
|
|
529
|
+
() => endMutation,
|
|
530
|
+
);
|
|
531
|
+
|
|
532
|
+
return ThreadMutationGate.of({
|
|
533
|
+
withMutation,
|
|
534
|
+
withSnapshot: (body) =>
|
|
535
|
+
generationGate.withPermit(Effect.flatMap(Ref.get(activeMutations), body)),
|
|
536
|
+
});
|
|
537
|
+
}),
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
export type MaintenancePassFailure =
|
|
542
|
+
| DurableWorkerFailure
|
|
543
|
+
| DurableBindingFailure
|
|
544
|
+
| DurableAlarmError
|
|
545
|
+
| ThreadProjectionError;
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
|
|
549
|
+
*
|
|
550
|
+
* One physical event owns native scheduling, auxiliary delivery and final alarm rearming.
|
|
551
|
+
*
|
|
552
|
+
* 1. Prearm before any work. A caught-up native step uses the O(1) generation record without
|
|
553
|
+
* recovery, ledger scans or canonical-history reads.
|
|
554
|
+
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
555
|
+
* producer keeps its newer generation dirty. Native retries retain their durable backoff.
|
|
556
|
+
* 3. After the initial native opportunity, stop admitting new external waves and let the
|
|
557
|
+
* active waves finish. While they remain in flight, native wakes and bounded scans can
|
|
558
|
+
* advance more heads. All Attempts share the event's original ten-minute yield deadline.
|
|
559
|
+
* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill joins are
|
|
560
|
+
* bounded independently; incoming native work never restarts or cancels their attempts.
|
|
561
|
+
* Auxiliary failures are reported after the current native opportunity.
|
|
562
|
+
* 5. Close every event resource before the final gated deadline snapshot and alarm decision.
|
|
563
|
+
* The whole event retains one fourteen-minute cooperative timeout.
|
|
564
|
+
*/
|
|
565
|
+
export class ThreadMaintenance extends Context.Service<
|
|
566
|
+
ThreadMaintenance,
|
|
567
|
+
{
|
|
568
|
+
/** One idempotent pass; failures propagate after durably scheduling bounded recovery. */
|
|
569
|
+
readonly pass: Effect.Effect<MaintenancePassReport, MaintenancePassFailure>;
|
|
570
|
+
/**
|
|
571
|
+
* Constructor gate: initialize/inspect only the O(1) maintenance record and ensure a dirty
|
|
572
|
+
* generation has an alarm. It never scans the ledger or canonical history.
|
|
573
|
+
*/
|
|
574
|
+
readonly ensureAlarm: Effect.Effect<void, MaintenancePassFailure>;
|
|
575
|
+
/**
|
|
576
|
+
* Serialize the pre-arm boundary with pass acknowledgement, advance the durable dirty
|
|
577
|
+
* generation and arm the alarm in one transaction BEFORE running the caller's mutation.
|
|
578
|
+
* A pass cannot acknowledge while that mutation remains in flight.
|
|
579
|
+
*/
|
|
580
|
+
readonly withMutation: <A, E, R>(
|
|
581
|
+
body: Effect.Effect<A, E, R>,
|
|
582
|
+
) => Effect.Effect<A, E | DurableAlarmError, R>;
|
|
583
|
+
}
|
|
584
|
+
>()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
|
|
585
|
+
static readonly layer: Layer.Layer<
|
|
586
|
+
ThreadMaintenance,
|
|
587
|
+
never,
|
|
588
|
+
| ThreadMutationGate
|
|
589
|
+
| ThreadPublication
|
|
590
|
+
| ThreadProjectionMaintenance
|
|
591
|
+
| DurableAgentRuntime
|
|
592
|
+
| SubmissionLedger
|
|
593
|
+
| WakeScheduler
|
|
594
|
+
| DurableAlarmService
|
|
595
|
+
| ThreadMaintenanceFailpoint
|
|
596
|
+
| CloudflareDurableRuntimeConfig
|
|
597
|
+
| DurableObjectContext
|
|
598
|
+
| SqlClient
|
|
599
|
+
> = Layer.effect(ThreadMaintenance)(
|
|
600
|
+
Effect.gen(function* () {
|
|
601
|
+
const runtime = yield* DurableAgentRuntime;
|
|
602
|
+
const ledger = yield* SubmissionLedger;
|
|
603
|
+
const wakes = yield* WakeScheduler;
|
|
604
|
+
const alarm = yield* DurableAlarmService;
|
|
605
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
606
|
+
const { ctx } = yield* DurableObjectContext;
|
|
607
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
608
|
+
|
|
609
|
+
const mutations = yield* ThreadMutationGate;
|
|
610
|
+
const publication = yield* ThreadPublication;
|
|
611
|
+
const projection = yield* ThreadProjectionMaintenance;
|
|
612
|
+
const messages = yield* ThreadMessageDelivery;
|
|
613
|
+
const host = yield* ThreadHostMaintenance;
|
|
614
|
+
|
|
615
|
+
// A broken disposable index still needs a retry alarm and must not prevent startup.
|
|
616
|
+
const projectionDeadline = projection.pendingDeadline.pipe(
|
|
617
|
+
Effect.catchCauseIf(
|
|
618
|
+
(cause) => !Cause.hasInterrupts(cause),
|
|
619
|
+
(cause) =>
|
|
620
|
+
Effect.logError("Thread projection deadline unavailable", cause).pipe(
|
|
621
|
+
Effect.as(Option.some(0)),
|
|
622
|
+
),
|
|
623
|
+
),
|
|
624
|
+
);
|
|
625
|
+
|
|
626
|
+
const pendingDeadline = Effect.gen(function* () {
|
|
627
|
+
return earliestDeadline(
|
|
628
|
+
earliestDeadline(yield* publication.pendingDeadline, yield* messages.pendingDeadline),
|
|
629
|
+
earliestDeadline(yield* projectionDeadline, yield* host.pendingDeadline),
|
|
630
|
+
);
|
|
631
|
+
});
|
|
632
|
+
|
|
633
|
+
const maintenancePassGate = yield* Semaphore.make(1);
|
|
634
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
635
|
+
|
|
636
|
+
const runTransaction = yield* makeStorageOperation;
|
|
637
|
+
|
|
638
|
+
const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
|
|
639
|
+
yield* failpoint.hit("maintenance:ensure:before");
|
|
640
|
+
const now = yield* Clock.currentTimeMillis;
|
|
641
|
+
|
|
642
|
+
const retry = yield* runTransaction("ensure maintenance alarm", () =>
|
|
643
|
+
ctx.storage.transaction(async (transaction) => {
|
|
644
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
645
|
+
|
|
646
|
+
if (!initialized) {
|
|
647
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
648
|
+
}
|
|
649
|
+
if (state.dirty > state.processed) {
|
|
650
|
+
await ensureTransactionAlarmBy(
|
|
651
|
+
transaction,
|
|
652
|
+
state.retry?.generation === state.dirty
|
|
653
|
+
? Math.max(now + minimumAlarmDelay, state.retry.notBefore)
|
|
654
|
+
: now + config.wakeScanInterval,
|
|
655
|
+
);
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
return state.retry?.generation === state.dirty ? state.retry : undefined;
|
|
659
|
+
}),
|
|
660
|
+
);
|
|
661
|
+
|
|
662
|
+
const deadline = yield* pendingDeadline;
|
|
663
|
+
|
|
664
|
+
if (Option.isSome(deadline)) {
|
|
665
|
+
yield* runTransaction("ensure publication alarm", () =>
|
|
666
|
+
ctx.storage.transaction((transaction) =>
|
|
667
|
+
ensureTransactionAlarmBy(
|
|
668
|
+
transaction,
|
|
669
|
+
Math.max(
|
|
670
|
+
now + minimumAlarmDelay,
|
|
671
|
+
deadline.value <= now && retry !== undefined && !retry.nativeOnly
|
|
672
|
+
? Math.max(deadline.value, retry.notBefore)
|
|
673
|
+
: deadline.value,
|
|
674
|
+
),
|
|
675
|
+
),
|
|
676
|
+
),
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
yield* failpoint.hit("maintenance:ensure:after");
|
|
680
|
+
});
|
|
681
|
+
|
|
682
|
+
const beginPass = Effect.fn("ThreadMaintenance.beginPass")(function* (observed: {
|
|
683
|
+
generation?: bigint;
|
|
684
|
+
}) {
|
|
685
|
+
yield* failpoint.hit("maintenance:begin:before");
|
|
686
|
+
const now = yield* Clock.currentTimeMillis;
|
|
687
|
+
|
|
688
|
+
const result = yield* runTransaction("begin maintenance pass", () =>
|
|
689
|
+
ctx.storage.transaction(async (transaction) => {
|
|
690
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
691
|
+
|
|
692
|
+
observed.generation = state.dirty;
|
|
693
|
+
if (!initialized) {
|
|
694
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
695
|
+
}
|
|
696
|
+
const retryAt = state.retry?.generation === state.dirty ? state.retry.notBefore : 0;
|
|
697
|
+
|
|
698
|
+
// Prearm before recovery or any host hook, including publication-only passes.
|
|
699
|
+
// A completed pass may move this crash fallback later to its bounded retry.
|
|
700
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
701
|
+
if (state.processed >= state.dirty || retryAt > now) {
|
|
702
|
+
return {
|
|
703
|
+
_tag: "CaughtUp" as const,
|
|
704
|
+
nonterminal: state.nonterminal,
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
return {
|
|
709
|
+
_tag: "Actionable" as const,
|
|
710
|
+
generation: state.dirty,
|
|
711
|
+
nonterminal: state.nonterminal,
|
|
712
|
+
stalls: state.retry?.generation === state.dirty ? state.retry.stalls : 0,
|
|
713
|
+
};
|
|
714
|
+
}),
|
|
715
|
+
);
|
|
716
|
+
|
|
717
|
+
yield* failpoint.hit("maintenance:begin:after");
|
|
718
|
+
|
|
719
|
+
return result;
|
|
720
|
+
});
|
|
721
|
+
|
|
722
|
+
const backoffDelay = (priorStalls: number, jitter: number) => {
|
|
723
|
+
const backoff = Math.min(
|
|
724
|
+
config.alarmBackoffCap,
|
|
725
|
+
config.alarmBackoffBase * 2 ** Math.min(priorStalls, 30),
|
|
726
|
+
);
|
|
727
|
+
|
|
728
|
+
// Jitter over [backoff/2, backoff] spreads retries without exceeding the cap.
|
|
729
|
+
return Math.ceil(backoff / 2 + (backoff / 2) * jitter);
|
|
730
|
+
};
|
|
731
|
+
|
|
732
|
+
const rearmDelay = Effect.fn("ThreadMaintenance.rearmDelay")(function* (
|
|
733
|
+
progressed: boolean,
|
|
734
|
+
priorStalls: number,
|
|
735
|
+
) {
|
|
736
|
+
return progressed ? config.alarmBackoffBase : backoffDelay(priorStalls, yield* Random.next);
|
|
737
|
+
});
|
|
738
|
+
|
|
739
|
+
const rearmFailure = Effect.fn("ThreadMaintenance.rearmFailure")(function* (
|
|
740
|
+
observed: MaintenanceObservation,
|
|
741
|
+
) {
|
|
742
|
+
const { generation, nativeOnly } = observed;
|
|
743
|
+
|
|
744
|
+
if (generation === undefined) return;
|
|
745
|
+
yield* failpoint.hit("maintenance:retry:before");
|
|
746
|
+
yield* mutations.withSnapshot((active) =>
|
|
747
|
+
Effect.gen(function* () {
|
|
748
|
+
// A failed deadline read must not prevent committing the native retry.
|
|
749
|
+
const deadline = yield* pendingDeadline.pipe(
|
|
750
|
+
Effect.catchCause(() => Effect.succeed(Option.none<number>())),
|
|
751
|
+
);
|
|
752
|
+
|
|
753
|
+
const now = yield* Clock.currentTimeMillis;
|
|
754
|
+
|
|
755
|
+
const jitter = yield* Random.next;
|
|
756
|
+
|
|
757
|
+
yield* runTransaction("back off failed maintenance", () =>
|
|
758
|
+
ctx.storage.transaction(async (transaction) => {
|
|
759
|
+
const { state } = await readMaintenanceState(transaction);
|
|
760
|
+
|
|
761
|
+
const previous =
|
|
762
|
+
state.retry?.generation === generation && state.retry.nativeOnly === nativeOnly
|
|
763
|
+
? state.retry
|
|
764
|
+
: undefined;
|
|
765
|
+
|
|
766
|
+
const retry = MaintenanceRetry.make({
|
|
767
|
+
generation,
|
|
768
|
+
notBefore: Math.max(
|
|
769
|
+
previous?.notBefore ?? 0,
|
|
770
|
+
now + backoffDelay(previous?.stalls ?? 0, jitter),
|
|
771
|
+
),
|
|
772
|
+
nativeOnly,
|
|
773
|
+
stalls: Math.min(30, (previous?.stalls ?? 0) + 1),
|
|
774
|
+
});
|
|
775
|
+
|
|
776
|
+
await transaction.put(
|
|
777
|
+
MAINTENANCE_STATE_KEY,
|
|
778
|
+
encodeMaintenanceState(ThreadMaintenanceState.make({ ...state, retry })),
|
|
779
|
+
);
|
|
780
|
+
|
|
781
|
+
// Never postpone a producer that raced the failed observation. Host work
|
|
782
|
+
// retains its own deadline; an early delivery skips native recovery below.
|
|
783
|
+
const nativeDeadline =
|
|
784
|
+
active > 0 || state.dirty !== generation
|
|
785
|
+
? now + minimumAlarmDelay
|
|
786
|
+
: retry.notBefore;
|
|
787
|
+
|
|
788
|
+
await transaction.setAlarm(
|
|
789
|
+
Math.max(
|
|
790
|
+
now + minimumAlarmDelay,
|
|
791
|
+
Option.isSome(deadline) && (nativeOnly || deadline.value > now)
|
|
792
|
+
? Math.min(nativeDeadline, deadline.value)
|
|
793
|
+
: nativeDeadline,
|
|
794
|
+
),
|
|
795
|
+
);
|
|
796
|
+
}),
|
|
797
|
+
);
|
|
798
|
+
}),
|
|
799
|
+
);
|
|
800
|
+
yield* failpoint.hit("maintenance:retry:after");
|
|
801
|
+
});
|
|
802
|
+
|
|
803
|
+
const beginNative = Effect.fn("ThreadMaintenance.beginNative")(function* (
|
|
804
|
+
observed: MaintenanceObservation,
|
|
805
|
+
) {
|
|
806
|
+
return yield* mutations.withSnapshot((activeAtStart) =>
|
|
807
|
+
Effect.gen(function* () {
|
|
808
|
+
const generation = yield* beginPass(observed);
|
|
809
|
+
|
|
810
|
+
if (generation._tag === "Actionable" && activeAtStart === 0) {
|
|
811
|
+
// The gate excludes a producer starting between the snapshot and certification.
|
|
812
|
+
yield* publication.prepareGeneration(generation.generation);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
return { ...generation, activeAtStart };
|
|
816
|
+
}),
|
|
817
|
+
);
|
|
818
|
+
});
|
|
819
|
+
|
|
820
|
+
const advance = Effect.fn("ThreadMaintenance.advance")(function* (
|
|
821
|
+
started: Effect.Success<ReturnType<typeof beginNative>>,
|
|
822
|
+
yieldAfter: DateTime.Utc,
|
|
823
|
+
observed: MaintenanceObservation,
|
|
824
|
+
): Effect.fn.Return<NativePassResult, MaintenancePassFailure> {
|
|
825
|
+
const deadline = yield* publication.pendingDeadline;
|
|
826
|
+
|
|
827
|
+
if (
|
|
828
|
+
started._tag === "Actionable" ||
|
|
829
|
+
(Option.isSome(deadline) && deadline.value <= (yield* Clock.currentTimeMillis))
|
|
830
|
+
) {
|
|
831
|
+
yield* publication.drain;
|
|
832
|
+
}
|
|
833
|
+
const pending = yield* publication.pendingDeadline;
|
|
834
|
+
|
|
835
|
+
if (started._tag === "CaughtUp" || Option.isSome(pending)) {
|
|
836
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) =>
|
|
837
|
+
Effect.gen(function* () {
|
|
838
|
+
const now = yield* Clock.currentTimeMillis;
|
|
839
|
+
|
|
840
|
+
const { state } = yield* runTransaction("read native maintenance deadline", () =>
|
|
841
|
+
ctx.storage.transaction((transaction) => readMaintenanceState(transaction)),
|
|
842
|
+
);
|
|
843
|
+
|
|
844
|
+
const native =
|
|
845
|
+
active > 0 || state.dirty > state.processed
|
|
846
|
+
? state.retry?.generation === state.dirty && active === 0
|
|
847
|
+
? state.retry.notBefore
|
|
848
|
+
: now + minimumAlarmDelay
|
|
849
|
+
: Infinity;
|
|
850
|
+
|
|
851
|
+
const next = Option.isSome(pending) ? pending.value : native;
|
|
852
|
+
|
|
853
|
+
return Number.isFinite(next) ? Math.max(now + minimumAlarmDelay, next) : undefined;
|
|
854
|
+
}),
|
|
855
|
+
);
|
|
856
|
+
|
|
857
|
+
return {
|
|
858
|
+
phase: "caught-up",
|
|
859
|
+
recovered: 0,
|
|
860
|
+
settled: 0,
|
|
861
|
+
nonterminal: started.nonterminal,
|
|
862
|
+
nextAttemptAt,
|
|
863
|
+
};
|
|
864
|
+
}
|
|
865
|
+
// Step 2 — reconciliation strictly precedes new work in this pass (exit gate).
|
|
866
|
+
observed.nativeOnly = true;
|
|
867
|
+
const recovered: ReadonlyArray<RecoveryReport> = yield* runtime.runRecovery;
|
|
868
|
+
const reports = new Map(recovered.map((report) => [report.submissionId, report]));
|
|
869
|
+
const current = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
870
|
+
const heads = new Map<ThreadId, SubmissionSnapshot>();
|
|
871
|
+
|
|
872
|
+
for (const row of current) {
|
|
873
|
+
// Parked uncertainty keeps its settlement obligation, but later input can run.
|
|
874
|
+
// Accepted aborts and every other wait remain subject to the lane's FIFO barrier.
|
|
875
|
+
if (row.state === "unknown" && stableExternalWait(row, reports)) continue;
|
|
876
|
+
if (!heads.has(row.threadId)) heads.set(row.threadId, row);
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
const eligible = [...heads.values()]
|
|
880
|
+
.filter((head) => !stableExternalWait(head, reports))
|
|
881
|
+
.map((head) => head.threadId)
|
|
882
|
+
.sort();
|
|
883
|
+
|
|
884
|
+
const selectionTime = yield* Clock.currentTimeMillis;
|
|
885
|
+
|
|
886
|
+
yield* failpoint.hit("maintenance:select:before");
|
|
887
|
+
|
|
888
|
+
const selection = yield* runTransaction("select maintenance lane", () =>
|
|
889
|
+
ctx.storage.transaction(async (transaction) => {
|
|
890
|
+
const { state } = await readMaintenanceState(transaction);
|
|
891
|
+
|
|
892
|
+
const retries = (state.bindingRetries ?? []).filter(
|
|
893
|
+
(retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId,
|
|
894
|
+
);
|
|
895
|
+
|
|
896
|
+
const runnable = eligible.filter(
|
|
897
|
+
(threadId) =>
|
|
898
|
+
!retries.some(
|
|
899
|
+
(retry) => retry.threadId === threadId && retry.notBefore > selectionTime,
|
|
900
|
+
),
|
|
901
|
+
);
|
|
902
|
+
|
|
903
|
+
const next =
|
|
904
|
+
runnable.find(
|
|
905
|
+
(threadId) =>
|
|
906
|
+
state.lastServedThreadId === undefined || threadId > state.lastServedThreadId,
|
|
907
|
+
) ?? runnable[0];
|
|
908
|
+
|
|
909
|
+
if (next !== undefined) {
|
|
910
|
+
await transaction.put(
|
|
911
|
+
MAINTENANCE_STATE_KEY,
|
|
912
|
+
encodeMaintenanceState(
|
|
913
|
+
ThreadMaintenanceState.make({ ...state, lastServedThreadId: next }),
|
|
914
|
+
),
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
return { selected: next, retries };
|
|
919
|
+
}),
|
|
920
|
+
);
|
|
921
|
+
|
|
922
|
+
yield* failpoint.hit("maintenance:select:after");
|
|
923
|
+
|
|
924
|
+
const selected =
|
|
925
|
+
selection.selected === undefined ? undefined : heads.get(selection.selected);
|
|
926
|
+
|
|
927
|
+
let retries = selection.retries;
|
|
928
|
+
let bindingFailure: DurableBindingFailure | undefined;
|
|
929
|
+
|
|
930
|
+
// One runnable FIFO head per native opportunity. An absent agent waits for a deployment,
|
|
931
|
+
// including for children; other local lanes and host deliveries remain independently due.
|
|
932
|
+
const settlement =
|
|
933
|
+
selected === undefined
|
|
934
|
+
? Option.none()
|
|
935
|
+
: yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(
|
|
936
|
+
Effect.catchTag("BindingUnavailable", (failure) => {
|
|
937
|
+
bindingFailure = failure;
|
|
938
|
+
|
|
939
|
+
return Effect.succeed(Option.none());
|
|
940
|
+
}),
|
|
941
|
+
);
|
|
942
|
+
|
|
943
|
+
if (selected !== undefined) {
|
|
944
|
+
const previous = retries.find((retry) => retry.submissionId === selected.submissionId);
|
|
945
|
+
let retry: BindingRetry | undefined;
|
|
946
|
+
let reportBindingFailure = false;
|
|
947
|
+
|
|
948
|
+
if (bindingFailure !== undefined) {
|
|
949
|
+
const now = yield* Clock.currentTimeMillis;
|
|
950
|
+
const attempts = Math.min(30, (previous?.attempts ?? 0) + 1);
|
|
951
|
+
|
|
952
|
+
reportBindingFailure =
|
|
953
|
+
previous === undefined || now - previous.reportedAt >= 15 * 60_000;
|
|
954
|
+
retry = BindingRetry.make({
|
|
955
|
+
threadId: selected.threadId,
|
|
956
|
+
submissionId: selected.submissionId,
|
|
957
|
+
attempts,
|
|
958
|
+
notBefore: now + Math.min(60_000, 5_000 * 2 ** (attempts - 1)),
|
|
959
|
+
reportedAt: reportBindingFailure ? now : (previous?.reportedAt ?? now),
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
if (retry !== undefined || previous !== undefined) {
|
|
963
|
+
// The Attempt released its Claim. Commit its binding wait (or clear) once,
|
|
964
|
+
// before joining fallible auxiliary work. This local fact neither acknowledges
|
|
965
|
+
// a generation nor changes the shared alarm.
|
|
966
|
+
yield* failpoint.hit("maintenance:binding-retry:before");
|
|
967
|
+
retries = yield* runTransaction("record submission binding retry", () =>
|
|
968
|
+
ctx.storage.transaction(async (transaction) => {
|
|
969
|
+
const { state } = await readMaintenanceState(transaction);
|
|
970
|
+
|
|
971
|
+
const bindingRetries = [
|
|
972
|
+
...(state.bindingRetries ?? []).filter(
|
|
973
|
+
(entry) => entry.submissionId !== selected.submissionId,
|
|
974
|
+
),
|
|
975
|
+
...(retry === undefined ? [] : [retry]),
|
|
976
|
+
];
|
|
977
|
+
|
|
978
|
+
await transaction.put(
|
|
979
|
+
MAINTENANCE_STATE_KEY,
|
|
980
|
+
encodeMaintenanceState(ThreadMaintenanceState.make({ ...state, bindingRetries })),
|
|
981
|
+
);
|
|
982
|
+
|
|
983
|
+
return bindingRetries;
|
|
984
|
+
}),
|
|
985
|
+
);
|
|
986
|
+
yield* failpoint.hit("maintenance:binding-retry:after");
|
|
987
|
+
}
|
|
988
|
+
if (bindingFailure !== undefined) {
|
|
989
|
+
yield* reportBindingFailure
|
|
990
|
+
? Effect.logError(
|
|
991
|
+
"Thread awaits a current agent binding; original work remains pending",
|
|
992
|
+
Cause.fail(bindingFailure),
|
|
993
|
+
)
|
|
994
|
+
: Effect.logDebug("Thread binding retry remains pending", Cause.fail(bindingFailure));
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
999
|
+
const waitingHeads = new Map<ThreadId, boolean>();
|
|
1000
|
+
|
|
1001
|
+
const autonomous = remaining.some((snapshot) => {
|
|
1002
|
+
if (snapshot.state === "unknown" && stableExternalWait(snapshot, reports)) return false;
|
|
1003
|
+
const headWaiting = waitingHeads.get(snapshot.threadId);
|
|
1004
|
+
|
|
1005
|
+
if (headWaiting === undefined)
|
|
1006
|
+
waitingHeads.set(snapshot.threadId, stableExternalWait(snapshot, reports));
|
|
1007
|
+
// FIFO followers cannot execute through a stable external wait. Only plain queued
|
|
1008
|
+
// input is dormant here; admission repairs and accepted aborts still need a pass.
|
|
1009
|
+
if (
|
|
1010
|
+
headWaiting === true &&
|
|
1011
|
+
snapshot.state === "ready" &&
|
|
1012
|
+
reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput"
|
|
1013
|
+
)
|
|
1014
|
+
return false;
|
|
1015
|
+
|
|
1016
|
+
return !stableExternalWait(snapshot, reports);
|
|
1017
|
+
});
|
|
1018
|
+
|
|
1019
|
+
const progressed =
|
|
1020
|
+
Option.isSome(settlement) ||
|
|
1021
|
+
recovered.some((report) => report.disposition === "repaired");
|
|
1022
|
+
|
|
1023
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1024
|
+
const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
|
|
1025
|
+
|
|
1026
|
+
const nextEligible = eligible.map(
|
|
1027
|
+
(threadId) =>
|
|
1028
|
+
retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)
|
|
1029
|
+
?.notBefore ?? now,
|
|
1030
|
+
);
|
|
1031
|
+
|
|
1032
|
+
const bindingDelay =
|
|
1033
|
+
nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
|
|
1034
|
+
|
|
1035
|
+
const delay = Math.max(ordinaryDelay, bindingDelay);
|
|
1036
|
+
|
|
1037
|
+
// Checkpoint native progress without changing the physical alarm. Auxiliary
|
|
1038
|
+
// delivery remains live; later mutations still advance the shared generation.
|
|
1039
|
+
yield* failpoint.hit("maintenance:checkpoint:before");
|
|
1040
|
+
|
|
1041
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) =>
|
|
1042
|
+
runTransaction("checkpoint native maintenance", () =>
|
|
1043
|
+
ctx.storage.transaction(async (transaction) => {
|
|
1044
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1045
|
+
|
|
1046
|
+
const processed =
|
|
1047
|
+
autonomous || started.activeAtStart > 0 || active > 0
|
|
1048
|
+
? state.processed
|
|
1049
|
+
: state.processed > started.generation
|
|
1050
|
+
? state.processed
|
|
1051
|
+
: started.generation;
|
|
1052
|
+
|
|
1053
|
+
const next = ThreadMaintenanceState.make({
|
|
1054
|
+
...Struct.omit(state, ["retry"]),
|
|
1055
|
+
processed,
|
|
1056
|
+
nonterminal: remaining.length,
|
|
1057
|
+
bindingRetries: (state.bindingRetries ?? []).filter((retry) =>
|
|
1058
|
+
remaining.some((row) => row.submissionId === retry.submissionId),
|
|
1059
|
+
),
|
|
1060
|
+
...(autonomous && !progressed
|
|
1061
|
+
? {
|
|
1062
|
+
retry: MaintenanceRetry.make({
|
|
1063
|
+
generation: started.generation,
|
|
1064
|
+
notBefore: now + delay,
|
|
1065
|
+
nativeOnly: true,
|
|
1066
|
+
stalls: Math.min(30, started.stalls + 1),
|
|
1067
|
+
}),
|
|
1068
|
+
}
|
|
1069
|
+
: {}),
|
|
1070
|
+
});
|
|
1071
|
+
|
|
1072
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
1073
|
+
if (autonomous) {
|
|
1074
|
+
return started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation
|
|
1075
|
+
? now + minimumAlarmDelay
|
|
1076
|
+
: now + delay;
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
return started.activeAtStart > 0 || active > 0 || next.dirty > next.processed
|
|
1080
|
+
? now + minimumAlarmDelay
|
|
1081
|
+
: undefined;
|
|
1082
|
+
}),
|
|
1083
|
+
),
|
|
1084
|
+
);
|
|
1085
|
+
|
|
1086
|
+
yield* failpoint.hit("maintenance:checkpoint:after");
|
|
1087
|
+
|
|
1088
|
+
return {
|
|
1089
|
+
phase: "actionable",
|
|
1090
|
+
recovered: recovered.length,
|
|
1091
|
+
settled: Option.isSome(settlement) ? 1 : 0,
|
|
1092
|
+
nonterminal: remaining.length,
|
|
1093
|
+
nextAttemptAt,
|
|
1094
|
+
};
|
|
1095
|
+
});
|
|
1096
|
+
|
|
1097
|
+
const pass = Effect.fn("ThreadMaintenance.pass")(function* (
|
|
1098
|
+
yieldAfter: DateTime.Utc,
|
|
1099
|
+
dispatchUntil: DateTime.Utc,
|
|
1100
|
+
observed: MaintenanceObservation,
|
|
1101
|
+
): Effect.fn.Return<MaintenancePassReport, MaintenancePassFailure, Scope.Scope> {
|
|
1102
|
+
// Subscribe before the first native snapshot. Hints accelerate rechecks; the
|
|
1103
|
+
// bounded scan and durable generation still recover dropped notifications.
|
|
1104
|
+
const notified = (yield* Stream.toPull(wakes.wakes)).pipe(
|
|
1105
|
+
Effect.asVoid,
|
|
1106
|
+
Effect.catch(() => Effect.never),
|
|
1107
|
+
);
|
|
1108
|
+
|
|
1109
|
+
let started = yield* beginNative(observed);
|
|
1110
|
+
|
|
1111
|
+
// This scope owns auxiliary dispatch and listeners, independently of native progress.
|
|
1112
|
+
// Close it before final alarm rearming, including on failure or event interruption.
|
|
1113
|
+
const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) =>
|
|
1114
|
+
Scope.close(scope, exit),
|
|
1115
|
+
);
|
|
1116
|
+
|
|
1117
|
+
const dispatchClosed = yield* Deferred.make<void>();
|
|
1118
|
+
const stopDispatch = Deferred.await(dispatchClosed);
|
|
1119
|
+
|
|
1120
|
+
// Fork setup too: an ordinary auxiliary setup failure is reported after native work,
|
|
1121
|
+
// rather than gating its opportunity. Event interruption still closes every fiber.
|
|
1122
|
+
const deliveryFiber = yield* Effect.forkIn(
|
|
1123
|
+
Scope.provide(auxiliaryScope)(messages.drainUntil(stopDispatch, dispatchUntil)),
|
|
1124
|
+
auxiliaryScope,
|
|
1125
|
+
);
|
|
1126
|
+
|
|
1127
|
+
const hostFiber = yield* Effect.forkIn(
|
|
1128
|
+
Scope.provide(auxiliaryScope)(
|
|
1129
|
+
Effect.gen(function* () {
|
|
1130
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(
|
|
1131
|
+
Effect.mapError((cause) =>
|
|
1132
|
+
DurableAlarmError.make({
|
|
1133
|
+
operation: "host dispatch allowance",
|
|
1134
|
+
message:
|
|
1135
|
+
"Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
|
|
1136
|
+
cause,
|
|
1137
|
+
}),
|
|
1138
|
+
),
|
|
1139
|
+
);
|
|
1140
|
+
yield* host.drainUntil(stopDispatch, dispatchUntil);
|
|
1141
|
+
}),
|
|
1142
|
+
),
|
|
1143
|
+
auxiliaryScope,
|
|
1144
|
+
);
|
|
1145
|
+
|
|
1146
|
+
// Backfill is one disposable wave; its timer starts beside native execution.
|
|
1147
|
+
const backfill = yield* Effect.forkIn(
|
|
1148
|
+
drainDue.pipe(
|
|
1149
|
+
Effect.provideService(ThreadProjectionMaintenance, projection),
|
|
1150
|
+
Effect.timeoutOption(config.projectionDispatchTimeoutMillis),
|
|
1151
|
+
),
|
|
1152
|
+
auxiliaryScope,
|
|
1153
|
+
);
|
|
1154
|
+
|
|
1155
|
+
let result = yield* advance(started, yieldAfter, observed);
|
|
1156
|
+
let phase = result.phase;
|
|
1157
|
+
let recovered = result.recovered;
|
|
1158
|
+
let settled = result.settled;
|
|
1159
|
+
|
|
1160
|
+
observed.nativeOnly = false;
|
|
1161
|
+
|
|
1162
|
+
// Close admission of new delivery waves once, then keep advancing native
|
|
1163
|
+
// work while the already-admitted waves finish. Neither lane restarts the
|
|
1164
|
+
// other's work or receives a fresh event budget.
|
|
1165
|
+
yield* Deferred.succeed(dispatchClosed, undefined);
|
|
1166
|
+
|
|
1167
|
+
const remaining = Math.max(
|
|
1168
|
+
1,
|
|
1169
|
+
DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis),
|
|
1170
|
+
);
|
|
1171
|
+
|
|
1172
|
+
const hostJoin = yield* Effect.forkIn(
|
|
1173
|
+
Fiber.join(hostFiber).pipe(
|
|
1174
|
+
Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)),
|
|
1175
|
+
Effect.tap((outcome) =>
|
|
1176
|
+
Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(outcome) }),
|
|
1177
|
+
),
|
|
1178
|
+
),
|
|
1179
|
+
auxiliaryScope,
|
|
1180
|
+
);
|
|
1181
|
+
|
|
1182
|
+
const retired = yield* Effect.forkChild(Fiber.joinAll([deliveryFiber, hostJoin, backfill]));
|
|
1183
|
+
|
|
1184
|
+
const auxiliaryPending = () =>
|
|
1185
|
+
deliveryFiber.pollUnsafe() === undefined ||
|
|
1186
|
+
hostFiber.pollUnsafe() === undefined ||
|
|
1187
|
+
backfill.pollUnsafe() === undefined;
|
|
1188
|
+
|
|
1189
|
+
while (retired.pollUnsafe() === undefined && auxiliaryPending()) {
|
|
1190
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1191
|
+
const until = DateTime.toEpochMillis(yieldAfter);
|
|
1192
|
+
|
|
1193
|
+
if (now >= until) break;
|
|
1194
|
+
|
|
1195
|
+
const next = Math.min(
|
|
1196
|
+
result.nextAttemptAt ?? Infinity,
|
|
1197
|
+
now + config.wakeScanInterval,
|
|
1198
|
+
until,
|
|
1199
|
+
);
|
|
1200
|
+
|
|
1201
|
+
const ready = yield* Effect.raceFirst(
|
|
1202
|
+
Effect.raceFirst(notified, Effect.sleep(Math.max(0, next - now))).pipe(Effect.as(true)),
|
|
1203
|
+
Fiber.await(retired).pipe(Effect.as(false)),
|
|
1204
|
+
);
|
|
1205
|
+
|
|
1206
|
+
if (!ready || retired.pollUnsafe() !== undefined || !auxiliaryPending()) break;
|
|
1207
|
+
if ((yield* Clock.currentTimeMillis) >= until) break;
|
|
1208
|
+
|
|
1209
|
+
started = yield* beginNative(observed);
|
|
1210
|
+
result = yield* advance(started, yieldAfter, observed);
|
|
1211
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
1212
|
+
recovered += result.recovered;
|
|
1213
|
+
settled += result.settled;
|
|
1214
|
+
observed.nativeOnly = false;
|
|
1215
|
+
}
|
|
1216
|
+
// Preserve driver-owned Claim deadlines and failures, then close every
|
|
1217
|
+
// listener before the one final alarm decision.
|
|
1218
|
+
yield* Fiber.join(retired);
|
|
1219
|
+
yield* Scope.close(auxiliaryScope, Exit.void);
|
|
1220
|
+
yield* failpoint.hit("maintenance:finish:before");
|
|
1221
|
+
|
|
1222
|
+
const disposition = yield* mutations.withSnapshot((active) =>
|
|
1223
|
+
Effect.gen(function* () {
|
|
1224
|
+
const latest = yield* pendingDeadline;
|
|
1225
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1226
|
+
|
|
1227
|
+
return yield* runTransaction("finish maintenance event", () =>
|
|
1228
|
+
ctx.storage.transaction(async (transaction) => {
|
|
1229
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1230
|
+
|
|
1231
|
+
const native =
|
|
1232
|
+
active > 0 || state.dirty > state.processed
|
|
1233
|
+
? state.retry?.generation === state.dirty && active === 0
|
|
1234
|
+
? Math.max(now + minimumAlarmDelay, state.retry.notBefore)
|
|
1235
|
+
: state.dirty === observed.generation && active === 0
|
|
1236
|
+
? (result.nextAttemptAt ?? now + config.wakeScanInterval)
|
|
1237
|
+
: now + minimumAlarmDelay
|
|
1238
|
+
: Infinity;
|
|
1239
|
+
|
|
1240
|
+
const next = Option.isSome(latest) ? Math.min(native, latest.value) : native;
|
|
1241
|
+
|
|
1242
|
+
if (Number.isFinite(next)) {
|
|
1243
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
|
|
1244
|
+
|
|
1245
|
+
return "rearmed" as const;
|
|
1246
|
+
}
|
|
1247
|
+
await transaction.deleteAlarm();
|
|
1248
|
+
|
|
1249
|
+
return "cleared" as const;
|
|
1250
|
+
}),
|
|
1251
|
+
);
|
|
1252
|
+
}),
|
|
1253
|
+
);
|
|
1254
|
+
|
|
1255
|
+
yield* failpoint.hit("maintenance:finish:after");
|
|
1256
|
+
|
|
1257
|
+
const report = MaintenancePassReport.make({
|
|
1258
|
+
phase,
|
|
1259
|
+
recovered,
|
|
1260
|
+
settled,
|
|
1261
|
+
nonterminal: result.nonterminal,
|
|
1262
|
+
alarm: disposition,
|
|
1263
|
+
});
|
|
1264
|
+
|
|
1265
|
+
yield* Effect.annotateCurrentSpan({
|
|
1266
|
+
phase: report.phase,
|
|
1267
|
+
recovered: report.recovered,
|
|
1268
|
+
settled: report.settled,
|
|
1269
|
+
nonterminal: report.nonterminal,
|
|
1270
|
+
alarm: report.alarm,
|
|
1271
|
+
});
|
|
1272
|
+
|
|
1273
|
+
return report;
|
|
1274
|
+
});
|
|
1275
|
+
|
|
1276
|
+
return ThreadMaintenance.of({
|
|
1277
|
+
// A mid-pass immediate hint is droppable; durable dirty state decides the final alarm.
|
|
1278
|
+
pass: Effect.gen(function* () {
|
|
1279
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1280
|
+
const yieldAfter = DateTime.makeUnsafe(now + 10 * 60_000);
|
|
1281
|
+
const dispatchUntil = DateTime.makeUnsafe(now + 14 * 60_000);
|
|
1282
|
+
const observed: MaintenanceObservation = { nativeOnly: false };
|
|
1283
|
+
|
|
1284
|
+
return yield* alarm.withWakesDeferred(
|
|
1285
|
+
maintenancePassGate.withPermit(
|
|
1286
|
+
Effect.scoped(pass(yieldAfter, dispatchUntil, observed)).pipe(
|
|
1287
|
+
// Close event-owned auxiliary work and release Attempt ownership before
|
|
1288
|
+
// failure rearming, while still holding the pass permit.
|
|
1289
|
+
Effect.onErrorIf(
|
|
1290
|
+
() => true,
|
|
1291
|
+
() => rearmFailure(observed),
|
|
1292
|
+
),
|
|
1293
|
+
),
|
|
1294
|
+
),
|
|
1295
|
+
);
|
|
1296
|
+
}).pipe(
|
|
1297
|
+
// Include permit waiting, recovery and acknowledgement in the event deadline.
|
|
1298
|
+
// Interruption releases Attempt ownership, leaving the prearmed dirty generation
|
|
1299
|
+
// for recovery. It never changes the logical Run duration or settles a policy failure.
|
|
1300
|
+
// This cooperative timer cannot preempt synchronous CPU work or stuck finalizers.
|
|
1301
|
+
Effect.timeoutOrElse({
|
|
1302
|
+
duration: "14 minutes",
|
|
1303
|
+
orElse: () =>
|
|
1304
|
+
DurableAlarmError.make({
|
|
1305
|
+
operation: "maintenance pass deadline",
|
|
1306
|
+
message:
|
|
1307
|
+
"The maintenance event exceeded its 14 minute deadline; durable recovery remains pending",
|
|
1308
|
+
}),
|
|
1309
|
+
}),
|
|
1310
|
+
),
|
|
1311
|
+
ensureAlarm: mutations.withSnapshot(() => ensureAlarm()),
|
|
1312
|
+
withMutation: (body) =>
|
|
1313
|
+
mutations.withMutation(
|
|
1314
|
+
body.pipe(
|
|
1315
|
+
Effect.tap(() =>
|
|
1316
|
+
publishCommitted.pipe(Effect.provideService(ThreadPublication, publication)),
|
|
1317
|
+
),
|
|
1318
|
+
),
|
|
1319
|
+
),
|
|
1320
|
+
});
|
|
1321
|
+
}),
|
|
1322
|
+
);
|
|
1323
|
+
}
|