@yielded/agent-platform-cloudflare 0.1.0-beta.167 → 0.1.0-beta.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{Alarm-B0ODr1NE.mjs → Alarm-BMEAZo3p.mjs} +261 -159
- package/dist/Alarm-BMEAZo3p.mjs.map +1 -0
- package/dist/Alarm.d.mts +29 -36
- package/dist/Alarm.mjs +2 -2
- package/dist/BrowserCredentials-C5IWfi7f.d.mts +91 -0
- package/dist/BrowserCredentials.d.mts +2 -91
- package/dist/BrowserRestCapture.mjs +3 -3
- package/dist/BrowserRestCapture.mjs.map +1 -1
- package/dist/BrowserRestCrawl.mjs +6 -6
- package/dist/BrowserRestCrawl.mjs.map +1 -1
- package/dist/{BrowserSession-RfIzxGCH.mjs → BrowserSession-BfNmyayH.mjs} +5 -5
- package/dist/BrowserSession-BfNmyayH.mjs.map +1 -0
- package/dist/{BrowserSession-B969Xg-2.d.mts → BrowserSession-Bx0V1-6_.d.mts} +6 -3
- package/dist/BrowserSession.d.mts +1 -1
- package/dist/BrowserSession.mjs +2 -2
- package/dist/BrowserUse-Bc2JdLW2.mjs +1528 -0
- package/dist/BrowserUse-Bc2JdLW2.mjs.map +1 -0
- package/dist/BrowserUse.d.mts +83 -0
- package/dist/BrowserUse.mjs +2 -0
- package/dist/CloudflareBindings.d.mts +8 -1
- package/dist/CloudflareBindings.mjs +2 -2
- package/dist/CloudflareBindings.mjs.map +1 -1
- package/dist/{CloudflareBrowser-5an6m-Qb.mjs → CloudflareBrowser-Vt8qTdYW.mjs} +11 -13
- package/dist/CloudflareBrowser-Vt8qTdYW.mjs.map +1 -0
- package/dist/CloudflareBrowser.mjs +1 -1
- package/dist/CloudflareCodeMode.mjs +1 -1
- package/dist/CloudflareCodeMode.mjs.map +1 -1
- package/dist/{CloudflareConfig-qDrRLofL.d.mts → CloudflareConfig-C9MvPbX8.d.mts} +9 -11
- package/dist/CloudflareConfig.d.mts +1 -1
- package/dist/CloudflareConfig.mjs +9 -12
- package/dist/CloudflareConfig.mjs.map +1 -1
- package/dist/CloudflareMemory.d.mts +3 -25
- package/dist/CloudflareMemory.mjs +33 -31
- package/dist/CloudflareMemory.mjs.map +1 -1
- package/dist/CloudflareScheduling.mjs +8 -15
- package/dist/CloudflareScheduling.mjs.map +1 -1
- package/dist/CloudflareSubscriptions.mjs +9 -16
- package/dist/CloudflareSubscriptions.mjs.map +1 -1
- package/dist/CloudflareThreadClient-CLZpN9zU.mjs +588 -0
- package/dist/CloudflareThreadClient-CLZpN9zU.mjs.map +1 -0
- package/dist/CloudflareThreadClient-QsVzLGGZ.d.mts +2033 -0
- package/dist/CloudflareThreadClient.d.mts +2 -1796
- package/dist/CloudflareThreadClient.mjs +2 -382
- package/dist/{InteractiveBrowser-BbG3kGA9.mjs → InteractiveBrowser-LdKzQeLR.mjs} +18 -19
- package/dist/InteractiveBrowser-LdKzQeLR.mjs.map +1 -0
- package/dist/InteractiveBrowser.mjs +1 -1
- package/dist/{ThreadObject-BcfXbzSO.d.mts → ThreadObject-7s5XnlH-.d.mts} +103 -79
- package/dist/{ThreadObject-DHjrnlX4.mjs → ThreadObject-CgXZMlr4.mjs} +265 -66
- package/dist/ThreadObject-CgXZMlr4.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +2 -2
- package/dist/ThreadObject.mjs +2 -3
- package/dist/WakeScheduler.d.mts +6 -5
- package/dist/WakeScheduler.mjs +15 -11
- package/dist/WakeScheduler.mjs.map +1 -1
- package/dist/index.d.mts +7 -6
- package/dist/index.mjs +7 -6
- package/dist/page-function-CmUCcyNP.mjs +11 -0
- package/dist/page-function-CmUCcyNP.mjs.map +1 -0
- package/dist/{prepared-admission-DMX5p9dC.mjs → prepared-admission-B61A7y6O.mjs} +3 -3
- package/dist/{prepared-admission-DMX5p9dC.mjs.map → prepared-admission-B61A7y6O.mjs.map} +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +512 -279
- package/src/BrowserRestCapture.ts +2 -2
- package/src/BrowserRestCrawl.ts +10 -14
- package/src/BrowserSession.ts +10 -2
- package/src/BrowserUse.ts +1591 -0
- package/src/CloudflareBindings.ts +13 -4
- package/src/CloudflareCodeMode.ts +1 -1
- package/src/CloudflareConfig.ts +9 -12
- package/src/CloudflareMemory.ts +21 -28
- package/src/CloudflareScheduling.ts +7 -15
- package/src/CloudflareSubscriptions.ts +11 -20
- package/src/CloudflareThreadClient.ts +289 -9
- package/src/InteractiveBrowser.ts +325 -323
- package/src/ThreadObject.ts +172 -52
- package/src/WakeScheduler.ts +29 -18
- package/src/index.ts +1 -0
- package/src/internal/browser-dom.ts +886 -0
- package/src/internal/browser-file-selection.ts +8 -1
- package/src/internal/browser-jev-dom.ts +362 -0
- package/src/internal/browser-quick-action.ts +7 -8
- package/src/internal/browser-session-lifecycle.ts +2 -5
- package/src/internal/crypto.ts +41 -0
- package/src/internal/due-queue.ts +5 -69
- package/src/internal/layers.ts +100 -17
- package/src/internal/live-text-protocol.ts +51 -0
- package/src/internal/live-text.ts +256 -0
- package/src/internal/page-function.ts +12 -0
- package/src/internal/prepared-admission.ts +1 -1
- package/src/internal/progress-wait.ts +19 -24
- package/src/internal/transport.ts +1 -1
- package/dist/Alarm-B0ODr1NE.mjs.map +0 -1
- package/dist/BrowserSession-RfIzxGCH.mjs.map +0 -1
- package/dist/CloudflareBrowser-5an6m-Qb.mjs.map +0 -1
- package/dist/CloudflareThreadClient.mjs.map +0 -1
- package/dist/InteractiveBrowser-BbG3kGA9.mjs.map +0 -1
- package/dist/ThreadObject-DHjrnlX4.mjs.map +0 -1
package/src/Alarm.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CurrentNativeSource } from "@yielded/agent-storage-cloudflare/port-routing";
|
|
1
2
|
import { type DurableBindingFailure } from "@yielded/agent/agent-registration";
|
|
2
3
|
import {
|
|
3
4
|
DurableAgentRuntime,
|
|
@@ -35,15 +36,16 @@ import {
|
|
|
35
36
|
Random,
|
|
36
37
|
Ref,
|
|
37
38
|
Result,
|
|
39
|
+
Scheduler,
|
|
38
40
|
Schema,
|
|
39
41
|
Scope,
|
|
40
42
|
Semaphore,
|
|
41
43
|
Stream,
|
|
42
44
|
Struct,
|
|
43
45
|
} from "effect";
|
|
44
|
-
import { DurableObjectStorage } from "effect-cf";
|
|
46
|
+
import { DurableObjectAlarm, DurableObjectStorage } from "effect-cf";
|
|
45
47
|
import { SqlClient } from "effect/sql/SqlClient";
|
|
46
|
-
import
|
|
48
|
+
import { SqlError, UnknownError } from "effect/sql/SqlError";
|
|
47
49
|
|
|
48
50
|
import { DurableObjectContext } from "./CloudflareBindings.ts";
|
|
49
51
|
import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.ts";
|
|
@@ -51,10 +53,8 @@ import { safeCauseMessage } from "./internal/boundary.ts";
|
|
|
51
53
|
import * as DueQueue from "./internal/due-queue.ts";
|
|
52
54
|
|
|
53
55
|
/**
|
|
54
|
-
* The
|
|
55
|
-
* alarm slot
|
|
56
|
-
* and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
|
|
57
|
-
* the slot always holds the EARLIEST deadline any caller asked for.
|
|
56
|
+
* The maintenance queue owns lease expiry, settlement, retries and dirty generations.
|
|
57
|
+
* Its manual effect-cf alarm shares the native slot with other registered alarms.
|
|
58
58
|
*
|
|
59
59
|
* The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
|
|
60
60
|
* maintenance generation and a committed alarm. Stable externally-driven waits may be
|
|
@@ -122,110 +122,268 @@ const makeStorageOperation = Effect.map(
|
|
|
122
122
|
run(operation, Effect.tryPromise({ try: execute, catch: alarmFailure(operation) })),
|
|
123
123
|
);
|
|
124
124
|
|
|
125
|
-
/**
|
|
125
|
+
/** One manual alarm for the entire consumer-owned maintenance queue. */
|
|
126
|
+
export class ThreadMaintenanceAlarm extends DurableObjectAlarm.Tag<ThreadMaintenanceAlarm>()(
|
|
127
|
+
"@yielded/agent/thread-maintenance",
|
|
128
|
+
{ "@yielded/agent/thread-maintenance": { payload: Schema.Null, lifecycle: "manual" } },
|
|
129
|
+
) {}
|
|
130
|
+
|
|
131
|
+
const maintenanceAlarmRef = { tag: "@yielded/agent/thread-maintenance", id: "queue" } as const;
|
|
132
|
+
|
|
133
|
+
const maintenanceAlarmInput = (deadline: number) => ({
|
|
134
|
+
...maintenanceAlarmRef,
|
|
135
|
+
payload: null,
|
|
136
|
+
runAt: DateTime.makeUnsafe(deadline),
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
/** Named maintenance checkpoint; the library owns the shared native alarm slot. */
|
|
126
140
|
export class DurableAlarmService extends Context.Service<
|
|
127
141
|
DurableAlarmService,
|
|
128
142
|
{
|
|
129
|
-
/** The scheduled deadline in epoch milliseconds, if any. */
|
|
130
143
|
readonly scheduled: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
131
|
-
/** Replace the slot with this deadline. */
|
|
132
144
|
readonly scheduleAt: (epochMillis: number) => Effect.Effect<void, DurableAlarmError>;
|
|
133
|
-
/** Keep the EARLIER of the existing deadline and this one (the multiplexing rule). */
|
|
134
145
|
readonly ensureScheduledBy: (epochMillis: number) => Effect.Effect<void, DurableAlarmError>;
|
|
135
|
-
/**
|
|
136
|
-
* Arm an immediate alarm (the durable, coalescing local wake) — DEFERRED while a
|
|
137
|
-
* maintenance pass is executing. Workerd cancels an in-flight alarm handler when a new
|
|
138
|
-
* EARLIER deadline is written during its execution (`requestScheduledAlarm`), and the
|
|
139
|
-
* maintenance pass runs INSIDE the alarm handler: an immediate wake landing mid-pass
|
|
140
|
-
* (a routed port mutation, a sibling's `wake()`, the coordinator's own local notify)
|
|
141
|
-
* would kill the running Attempt — manufacturing an ownership loss no real eviction
|
|
142
|
-
* caused, and routing open uncertain-class Tool Calls into spurious Unknown Outcomes.
|
|
143
|
-
* Deferral is contract-safe: wakes are droppable hints, every mutating entry point
|
|
144
|
-
* pre-arms BEFORE its first durable mutation (the alarm invariant never rests on this
|
|
145
|
-
* call). The pass's durable generation check observes any racing mutation, so the
|
|
146
|
-
* in-memory hint does not need to be flushed after a stable wait is acknowledged.
|
|
147
|
-
*/
|
|
146
|
+
/** Queue-aware promptness hint; idle queues stay idle and retry floors survive. */
|
|
148
147
|
readonly scheduleNow: Effect.Effect<void, DurableAlarmError>;
|
|
149
|
-
/**
|
|
150
|
-
|
|
151
|
-
* executes are droppable promptness hints; correctness rests on the durable generation.
|
|
152
|
-
*/
|
|
153
|
-
readonly withWakesDeferred: <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
154
|
-
/** Clear the slot; correctness-sensitive clears live in maintenance generation transactions. */
|
|
148
|
+
/** Coalesce droppable inline hints without delaying committed deadlines or their delivery. */
|
|
149
|
+
readonly withHintsDeferred: <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
155
150
|
readonly cancel: Effect.Effect<void, DurableAlarmError>;
|
|
156
151
|
}
|
|
157
152
|
>()("@effect-agent/platform-cloudflare/DurableAlarmService") {
|
|
158
|
-
static readonly layer
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
153
|
+
static readonly layer = Layer.effect(DurableAlarmService)(
|
|
154
|
+
Effect.gen(function* () {
|
|
155
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
156
|
+
const { ctx } = yield* DurableObjectContext;
|
|
157
|
+
const run = yield* makeStorageEffect;
|
|
158
|
+
const inlineWork = yield* Ref.make(0);
|
|
159
|
+
|
|
160
|
+
const wrap = <A, E, R>(operation: string, effect: Effect.Effect<A, E, R>) =>
|
|
161
|
+
run(operation, effect.pipe(Effect.mapError(alarmFailure(operation))));
|
|
162
|
+
|
|
163
|
+
return DurableAlarmService.of({
|
|
164
|
+
scheduled: wrap(
|
|
165
|
+
"read maintenance checkpoint",
|
|
166
|
+
alarms.getAlarmStatus(maintenanceAlarmRef),
|
|
167
|
+
).pipe(
|
|
168
|
+
Effect.map((at) =>
|
|
169
|
+
at === undefined ? Option.none() : Option.some(DateTime.toEpochMillis(at.runAt)),
|
|
174
170
|
),
|
|
175
|
-
)
|
|
171
|
+
),
|
|
172
|
+
scheduleAt: (at) =>
|
|
173
|
+
wrap("replace maintenance checkpoint", alarms.scheduleAlarm(maintenanceAlarmInput(at))),
|
|
174
|
+
ensureScheduledBy: (at) =>
|
|
175
|
+
wrap(
|
|
176
|
+
"advance maintenance checkpoint",
|
|
177
|
+
alarms.scheduleAlarmEarlier(maintenanceAlarmInput(at)),
|
|
178
|
+
),
|
|
179
|
+
scheduleNow: Ref.get(inlineWork).pipe(
|
|
180
|
+
Effect.flatMap((active) =>
|
|
181
|
+
active > 0
|
|
182
|
+
? Effect.void
|
|
183
|
+
: wrap(
|
|
184
|
+
"hint maintenance checkpoint",
|
|
185
|
+
alarms.transaction((tx) =>
|
|
186
|
+
Effect.gen(function* () {
|
|
187
|
+
const now = yield* Clock.currentTimeMillis;
|
|
188
|
+
const next = DueQueue.next(DueQueue.make(ctx.storage).read());
|
|
189
|
+
|
|
190
|
+
if (Number.isFinite(next))
|
|
191
|
+
yield* tx.scheduleAlarmEarlier(maintenanceAlarmInput(Math.max(now, next)));
|
|
192
|
+
}),
|
|
193
|
+
),
|
|
194
|
+
),
|
|
195
|
+
),
|
|
196
|
+
),
|
|
197
|
+
withHintsDeferred: (body) =>
|
|
198
|
+
Effect.acquireUseRelease(
|
|
199
|
+
Ref.update(inlineWork, (active) => active + 1),
|
|
200
|
+
() => body,
|
|
201
|
+
() => Ref.update(inlineWork, (active) => active - 1),
|
|
202
|
+
),
|
|
203
|
+
cancel: wrap("cancel maintenance checkpoint", alarms.cancelAlarm(maintenanceAlarmRef)),
|
|
204
|
+
});
|
|
205
|
+
}),
|
|
206
|
+
);
|
|
207
|
+
}
|
|
176
208
|
|
|
177
|
-
|
|
178
|
-
|
|
209
|
+
type CheckpointIntent = {
|
|
210
|
+
readonly at: (deadline: number) => void;
|
|
211
|
+
readonly earlier: (deadline: number) => void;
|
|
212
|
+
readonly cancel: () => void;
|
|
213
|
+
};
|
|
179
214
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
215
|
+
type MaintenanceStorage = Pick<DurableObjectTransaction, "get" | "put" | "delete">;
|
|
216
|
+
type MaintenanceCheckpoint = {
|
|
217
|
+
readonly mutations: Parameters<
|
|
218
|
+
Parameters<ThreadMaintenanceAlarm["Service"]["transaction"]>[0]
|
|
219
|
+
>[0];
|
|
220
|
+
readonly requireActive: Effect.Effect<void, DurableAlarmError>;
|
|
221
|
+
};
|
|
184
222
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
223
|
+
const CurrentCheckpoint = Context.Reference<MaintenanceCheckpoint | undefined>(
|
|
224
|
+
"@yielded/agent/cloudflare/CurrentCheckpoint",
|
|
225
|
+
{ defaultValue: () => undefined },
|
|
226
|
+
);
|
|
227
|
+
|
|
228
|
+
/** One source transaction owns queue intent and the library checkpoint through commit. */
|
|
229
|
+
const makeAlarmOperation = Effect.gen(function* () {
|
|
230
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
231
|
+
const sql = yield* SqlClient;
|
|
232
|
+
const { ctx } = yield* DurableObjectContext;
|
|
233
|
+
const queue = DueQueue.make(ctx.storage);
|
|
234
|
+
|
|
235
|
+
const transaction: SqlClient["withTransaction"] = (body) =>
|
|
236
|
+
alarms
|
|
237
|
+
.transaction(
|
|
238
|
+
(mutations) =>
|
|
239
|
+
Effect.withFiber((owner) => {
|
|
240
|
+
let active = true;
|
|
241
|
+
|
|
242
|
+
const requireActive = Effect.withFiber((fiber) =>
|
|
243
|
+
active && fiber === owner
|
|
244
|
+
? Effect.void
|
|
245
|
+
: Effect.fail(
|
|
246
|
+
DurableAlarmError.make({
|
|
247
|
+
operation: "enroll maintenance source",
|
|
248
|
+
message: "Source intent requires its active transaction callback's fiber",
|
|
249
|
+
}),
|
|
250
|
+
),
|
|
251
|
+
);
|
|
252
|
+
|
|
253
|
+
return body.pipe(
|
|
254
|
+
Effect.provideService(CurrentCheckpoint, { mutations, requireActive }),
|
|
255
|
+
Effect.ensuring(
|
|
256
|
+
Effect.sync(() => {
|
|
257
|
+
active = false;
|
|
258
|
+
}),
|
|
259
|
+
),
|
|
260
|
+
);
|
|
261
|
+
}),
|
|
262
|
+
{ sqlClient: sql },
|
|
263
|
+
)
|
|
264
|
+
.pipe(
|
|
265
|
+
Effect.mapError((cause) =>
|
|
266
|
+
cause instanceof DurableObjectStorage.StorageOperationError
|
|
267
|
+
? SqlError.make({
|
|
268
|
+
reason: UnknownError.make({
|
|
269
|
+
operation: "commit maintenance checkpoint",
|
|
270
|
+
message: "Source facts and maintenance checkpoint could not commit",
|
|
271
|
+
cause,
|
|
272
|
+
}),
|
|
273
|
+
})
|
|
274
|
+
: cause,
|
|
275
|
+
),
|
|
276
|
+
);
|
|
277
|
+
|
|
278
|
+
const withTransaction = <A, E, R>(body: Effect.Effect<A, E, R>) =>
|
|
279
|
+
Effect.flatMap(Effect.serviceOption(sql.transactionService), (ambient) => {
|
|
280
|
+
if (ambient._tag === "Some" && !DueQueue.ownsTransaction(ctx.storage, ambient.value[1]))
|
|
281
|
+
return Effect.fail(
|
|
282
|
+
SqlError.make({
|
|
283
|
+
reason: UnknownError.make({
|
|
284
|
+
operation: "begin maintenance source transaction",
|
|
285
|
+
message: "Use ThreadMutationGate.withTransaction at the outermost source transaction",
|
|
286
|
+
cause: new Error("Unwrapped SQL transaction cannot own buffered maintenance intent"),
|
|
188
287
|
}),
|
|
189
|
-
)
|
|
288
|
+
}),
|
|
289
|
+
);
|
|
190
290
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
ctx.storage.transaction(async (transaction) => {
|
|
195
|
-
const next = DueQueue.next(DueQueue.make(ctx.storage).read());
|
|
291
|
+
return Effect.uninterruptibleMask((restore) =>
|
|
292
|
+
Effect.suspend(() => {
|
|
293
|
+
let frame: ReturnType<typeof queue.begin> | undefined;
|
|
196
294
|
|
|
197
|
-
|
|
198
|
-
|
|
295
|
+
return transaction(
|
|
296
|
+
Effect.flatMap(Effect.serviceOption(sql.transactionService), (transaction) =>
|
|
297
|
+
Effect.sync(() => {
|
|
298
|
+
if (transaction._tag === "None") throw new Error("SQL transaction unavailable");
|
|
299
|
+
frame = queue.begin(transaction.value[1]);
|
|
199
300
|
}),
|
|
301
|
+
).pipe(
|
|
302
|
+
Effect.andThen(restore(body)),
|
|
303
|
+
Effect.tap(() =>
|
|
304
|
+
Effect.try({
|
|
305
|
+
try: () => {
|
|
306
|
+
if (frame !== undefined) queue.flush(frame);
|
|
307
|
+
},
|
|
308
|
+
catch: (cause) =>
|
|
309
|
+
SqlError.make({
|
|
310
|
+
reason: UnknownError.make({
|
|
311
|
+
cause,
|
|
312
|
+
operation: "flush maintenance due queue",
|
|
313
|
+
message: "Maintenance intent could not commit",
|
|
314
|
+
}),
|
|
315
|
+
}),
|
|
316
|
+
}),
|
|
317
|
+
),
|
|
200
318
|
),
|
|
201
|
-
)
|
|
202
|
-
|
|
319
|
+
).pipe(
|
|
320
|
+
Effect.onExit((exit) =>
|
|
321
|
+
Effect.sync(() => {
|
|
322
|
+
if (frame !== undefined) queue.finish(frame, Exit.isSuccess(exit));
|
|
323
|
+
}),
|
|
324
|
+
),
|
|
325
|
+
// A released permit can resume another transaction. Do not auto-yield
|
|
326
|
+
// between the driver's settlement and removing this transaction's frame.
|
|
327
|
+
Effect.provideService(Scheduler.PreventSchedulerYield, true),
|
|
328
|
+
);
|
|
329
|
+
}),
|
|
330
|
+
);
|
|
331
|
+
}).pipe(Effect.provideService(SqlClient, sql));
|
|
203
332
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
333
|
+
const run = <A>(
|
|
334
|
+
operation: string,
|
|
335
|
+
execute: (transaction: MaintenanceStorage, checkpoint: CheckpointIntent) => Promise<A>,
|
|
336
|
+
) => {
|
|
337
|
+
const commit = Effect.flatMap(CurrentCheckpoint, (tx) => {
|
|
338
|
+
if (tx === undefined)
|
|
339
|
+
return Effect.die(new Error("Maintenance checkpoint transaction unavailable"));
|
|
207
340
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
341
|
+
return tx.requireActive.pipe(
|
|
342
|
+
Effect.andThen(
|
|
343
|
+
Effect.suspend(() => {
|
|
344
|
+
let checkpoint: ReturnType<DurableObjectAlarm.AlarmScheduler["scheduleAlarm"]> =
|
|
345
|
+
Effect.void;
|
|
346
|
+
|
|
347
|
+
const intent: CheckpointIntent = {
|
|
348
|
+
at: (at) => {
|
|
349
|
+
checkpoint = tx.mutations.scheduleAlarm(maintenanceAlarmInput(at));
|
|
350
|
+
},
|
|
351
|
+
earlier: (at) => {
|
|
352
|
+
checkpoint = tx.mutations.scheduleAlarmEarlier(maintenanceAlarmInput(at));
|
|
353
|
+
},
|
|
354
|
+
cancel: () => {
|
|
355
|
+
checkpoint = tx.mutations.cancelAlarm(maintenanceAlarmRef);
|
|
356
|
+
},
|
|
357
|
+
};
|
|
213
358
|
|
|
214
|
-
|
|
359
|
+
// The SQL/alarm callback already owns the native transaction; KV uses the same storage.
|
|
360
|
+
return Effect.tryPromise({
|
|
361
|
+
try: () => execute(ctx.storage, intent),
|
|
362
|
+
catch: alarmFailure(operation),
|
|
363
|
+
}).pipe(Effect.tap(() => checkpoint));
|
|
364
|
+
}),
|
|
365
|
+
),
|
|
366
|
+
);
|
|
367
|
+
});
|
|
215
368
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
369
|
+
return Effect.gen(function* () {
|
|
370
|
+
const current = yield* CurrentCheckpoint;
|
|
371
|
+
|
|
372
|
+
if (current !== undefined) return yield* commit;
|
|
373
|
+
const ambient = yield* Effect.serviceOption(sql.transactionService);
|
|
374
|
+
|
|
375
|
+
// Preserve eager writes and invalidation for an unwrapped host SQL transaction.
|
|
376
|
+
return yield* ambient._tag === "Some" && !DueQueue.buffering(ctx.storage)
|
|
377
|
+
? transaction(commit).pipe(
|
|
378
|
+
Effect.ensuring(Effect.sync(() => DueQueue.invalidate(ctx.storage))),
|
|
379
|
+
)
|
|
380
|
+
: withTransaction(commit);
|
|
381
|
+
}).pipe(Effect.uninterruptible, Effect.mapError(alarmFailure(operation)));
|
|
382
|
+
};
|
|
383
|
+
|
|
384
|
+
return { run, withTransaction };
|
|
385
|
+
});
|
|
227
386
|
|
|
228
|
-
/** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
|
|
229
387
|
export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
|
|
230
388
|
"@effect-agent/platform-cloudflare/MaintenancePassReport",
|
|
231
389
|
)({
|
|
@@ -378,6 +536,8 @@ export const ThreadHostMaintenance = Context.Reference<{
|
|
|
378
536
|
/** @internal Framework handlers are separate from host registrations and their namespace. */
|
|
379
537
|
export const ThreadNativeMaintenance = Context.Reference<{
|
|
380
538
|
readonly lanes: ReadonlyArray<ThreadHostMaintenanceLane>;
|
|
539
|
+
/** Exact runtime built over a single-Thread placement; never apply to a replacement. */
|
|
540
|
+
readonly singleThreadRuntime?: DurableAgentRuntime["Service"];
|
|
381
541
|
}>("@effect-agent/platform-cloudflare/ThreadNativeMaintenance", {
|
|
382
542
|
defaultValue: () => ({ lanes: [] }),
|
|
383
543
|
});
|
|
@@ -426,6 +586,8 @@ const decodeRecoveryEvent = Schema.decodeUnknownSync(ThreadRecoveryFaultEvent);
|
|
|
426
586
|
* Object and sequence; callbacks may repeat after interruption. This is a trusted host hook,
|
|
427
587
|
* not pre-authorized UI data: the host owns recipient authorization and safe presentation.
|
|
428
588
|
* Capture services in the Layer. Do not call producers or write the raw alarm slot.
|
|
589
|
+
* Without this service, maintenance acknowledges and discards transitions. Install the
|
|
590
|
+
* handler before the first maintenance pass if the application needs that event stream.
|
|
429
591
|
*/
|
|
430
592
|
export class ThreadRecoveryEvents extends Context.Service<
|
|
431
593
|
ThreadRecoveryEvents,
|
|
@@ -502,7 +664,7 @@ const readRecoveryRecipients = async (
|
|
|
502
664
|
};
|
|
503
665
|
|
|
504
666
|
const appendRecoveryRecipients = async (
|
|
505
|
-
transaction:
|
|
667
|
+
transaction: MaintenanceStorage,
|
|
506
668
|
threadId: ThreadId,
|
|
507
669
|
count: number,
|
|
508
670
|
added: ReadonlyArray<SubmissionId>,
|
|
@@ -542,7 +704,7 @@ interface NativePassResult {
|
|
|
542
704
|
readonly dispatched?: boolean;
|
|
543
705
|
}
|
|
544
706
|
|
|
545
|
-
/** Independent Threads share
|
|
707
|
+
/** Independent Threads share bounded native slots; one active head per Thread. */
|
|
546
708
|
interface NativeDispatch {
|
|
547
709
|
readonly scope: Scope.Scope;
|
|
548
710
|
readonly active: Map<ThreadId, Fiber.Fiber<number, MaintenancePassFailure>>;
|
|
@@ -557,7 +719,6 @@ interface NativeDispatch {
|
|
|
557
719
|
scanRevision?: number;
|
|
558
720
|
}
|
|
559
721
|
|
|
560
|
-
const nativeDispatchConcurrency = 2;
|
|
561
722
|
/** Post-native delivery shares a finite connection budget across independent host lanes. */
|
|
562
723
|
const afterNativeDispatchConcurrency = 2;
|
|
563
724
|
|
|
@@ -577,8 +738,17 @@ interface NativeRecovery {
|
|
|
577
738
|
needsCheckpoint: boolean;
|
|
578
739
|
recovered: number;
|
|
579
740
|
repaired: boolean;
|
|
741
|
+
discoveryPending: boolean;
|
|
742
|
+
retryAtMillis?: number;
|
|
580
743
|
}
|
|
581
744
|
|
|
745
|
+
const NativeWorkCursor = Schema.Struct({
|
|
746
|
+
threadId: ThreadId,
|
|
747
|
+
cursor: Schema.NonEmptyString.check(Schema.isMaxLength(8192)),
|
|
748
|
+
});
|
|
749
|
+
|
|
750
|
+
const workCursorKey = (threadId: ThreadId) => `effect-agent:work-recovery-cursor:v1:${threadId}`;
|
|
751
|
+
|
|
582
752
|
interface MaintenanceObservation {
|
|
583
753
|
readonly queue: Map<string, DueQueue.DueLane>;
|
|
584
754
|
readonly charged: Map<string, number>;
|
|
@@ -644,17 +814,6 @@ const readMaintenanceState = async (
|
|
|
644
814
|
: { state: decodeMaintenanceState(encoded), initialized: true };
|
|
645
815
|
};
|
|
646
816
|
|
|
647
|
-
const ensureTransactionAlarmBy = async (
|
|
648
|
-
transaction: Pick<DurableObjectTransaction, "getAlarm" | "setAlarm">,
|
|
649
|
-
deadline: number,
|
|
650
|
-
): Promise<void> => {
|
|
651
|
-
const scheduled = await transaction.getAlarm();
|
|
652
|
-
|
|
653
|
-
if (scheduled === null || scheduled > deadline) {
|
|
654
|
-
await transaction.setAlarm(deadline);
|
|
655
|
-
}
|
|
656
|
-
};
|
|
657
|
-
|
|
658
817
|
const stableExternalWait = (
|
|
659
818
|
snapshot: SubmissionWorkItem,
|
|
660
819
|
reports: ReadonlyMap<string, RecoveryReport>,
|
|
@@ -675,9 +834,7 @@ const stableExternalWait = (
|
|
|
675
834
|
case "input-applied":
|
|
676
835
|
case "joining":
|
|
677
836
|
case "ready":
|
|
678
|
-
case "running":
|
|
679
837
|
case "settled":
|
|
680
|
-
case "terminalizing":
|
|
681
838
|
return false;
|
|
682
839
|
}
|
|
683
840
|
};
|
|
@@ -696,18 +853,12 @@ const CurrentMutationLanes = Context.Reference<ReadonlyArray<string>>(
|
|
|
696
853
|
|
|
697
854
|
// Source changes made by this pass are certified after its Attempts join. External
|
|
698
855
|
// producers still advance the dirty generation and fence that certification.
|
|
699
|
-
const CurrentNativeSource = Context.Reference<boolean>(
|
|
700
|
-
"@effect-agent/platform-cloudflare/CurrentNativeSource",
|
|
701
|
-
{
|
|
702
|
-
defaultValue: () => false,
|
|
703
|
-
},
|
|
704
|
-
);
|
|
705
|
-
|
|
706
856
|
export class ThreadMutationGate extends Context.Service<
|
|
707
857
|
ThreadMutationGate,
|
|
708
858
|
{
|
|
709
|
-
/** Commit source facts and their scheduling intent together.
|
|
710
|
-
*
|
|
859
|
+
/** Commit source facts and their scheduling intent together. Mutate from this callback's
|
|
860
|
+
* fiber, or use an explicit child transaction. The flush runs before native commit;
|
|
861
|
+
* failed or interrupted transactions discard their queue view. */
|
|
711
862
|
readonly withTransaction: <A, E, R>(
|
|
712
863
|
body: Effect.Effect<A, E, R>,
|
|
713
864
|
) => Effect.Effect<A, E | SqlError, R>;
|
|
@@ -779,6 +930,7 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
779
930
|
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
780
931
|
|
|
781
932
|
const runTransaction = yield* makeStorageOperation;
|
|
933
|
+
const { run: runAlarmTransaction, withTransaction } = yield* makeAlarmOperation;
|
|
782
934
|
|
|
783
935
|
const validateSourceBoundary = Effect.flatMap(
|
|
784
936
|
Effect.serviceOption(sql.transactionService),
|
|
@@ -800,18 +952,15 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
800
952
|
dueQueue.register(DueQueue.Native);
|
|
801
953
|
});
|
|
802
954
|
|
|
803
|
-
const schedule = Effect.
|
|
955
|
+
const schedule = Effect.fnUntraced(function* (
|
|
804
956
|
id: string,
|
|
805
957
|
dueAt: number,
|
|
806
958
|
progressCursor?: bigint,
|
|
807
959
|
) {
|
|
808
960
|
yield* validateSourceBoundary;
|
|
809
961
|
const now = yield* Clock.currentTimeMillis;
|
|
810
|
-
const current = yield* Effect.serviceOption(sql.transactionService);
|
|
811
962
|
|
|
812
|
-
const enroll = async (
|
|
813
|
-
transaction: Pick<DurableObjectTransaction, "get" | "put" | "getAlarm" | "setAlarm">,
|
|
814
|
-
) => {
|
|
963
|
+
const enroll = async (transaction: MaintenanceStorage, checkpoint: CheckpointIntent) => {
|
|
815
964
|
if (progressCursor === undefined) dueQueue.dirty(id, dueAt);
|
|
816
965
|
else if (
|
|
817
966
|
dueQueue.progress(id, dueAt, progressCursor) &&
|
|
@@ -833,17 +982,14 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
833
982
|
}
|
|
834
983
|
const next = DueQueue.next(dueQueue.read());
|
|
835
984
|
|
|
836
|
-
if (Number.isFinite(next))
|
|
837
|
-
await ensureTransactionAlarmBy(transaction, Math.max(next, now + minimumAlarmDelay));
|
|
985
|
+
if (Number.isFinite(next)) checkpoint.earlier(Math.max(next, now + minimumAlarmDelay));
|
|
838
986
|
};
|
|
839
987
|
|
|
840
|
-
yield*
|
|
841
|
-
Option.isSome(current) ? enroll(ctx.storage) : dueQueue.transaction(enroll),
|
|
842
|
-
);
|
|
988
|
+
yield* runAlarmTransaction("schedule maintenance lane", enroll);
|
|
843
989
|
yield* notify;
|
|
844
990
|
});
|
|
845
991
|
|
|
846
|
-
const beginMutation = Effect.
|
|
992
|
+
const beginMutation = Effect.fnUntraced(function* (
|
|
847
993
|
invalidatesRecovery: boolean,
|
|
848
994
|
lanes: ReadonlyArray<string>,
|
|
849
995
|
) {
|
|
@@ -853,8 +999,9 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
853
999
|
const publicationRequired = yield* requiresPublication;
|
|
854
1000
|
|
|
855
1001
|
if (invalidatesRecovery || lanes.length > 0)
|
|
856
|
-
yield*
|
|
857
|
-
|
|
1002
|
+
yield* runAlarmTransaction(
|
|
1003
|
+
"advance maintenance generation",
|
|
1004
|
+
async (transaction, checkpoint) => {
|
|
858
1005
|
const { state, initialized } = await readMaintenanceState(transaction);
|
|
859
1006
|
|
|
860
1007
|
if (invalidatesRecovery) {
|
|
@@ -864,23 +1011,23 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
864
1011
|
// Enrollment is a crash fallback, not evidence that the body committed.
|
|
865
1012
|
for (const id of lanes) dueQueue.dirty(id, now, false);
|
|
866
1013
|
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
1014
|
+
if (!initialized || (invalidatesRecovery && !nativeSource))
|
|
1015
|
+
await transaction.put(
|
|
1016
|
+
MAINTENANCE_STATE_KEY,
|
|
1017
|
+
encodeMaintenanceState(
|
|
1018
|
+
ThreadMaintenanceState.make({
|
|
1019
|
+
...state,
|
|
1020
|
+
dirty: state.dirty + (invalidatesRecovery && !nativeSource ? 1n : 0n),
|
|
1021
|
+
}),
|
|
1022
|
+
),
|
|
1023
|
+
);
|
|
874
1024
|
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
875
1025
|
// on its best-effort immediate wake hint.
|
|
876
1026
|
const deadline = DueQueue.next(dueQueue.read());
|
|
877
1027
|
|
|
878
1028
|
if (Number.isFinite(deadline))
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
Math.max(deadline, now + minimumAlarmDelay),
|
|
882
|
-
);
|
|
883
|
-
}),
|
|
1029
|
+
checkpoint.earlier(Math.max(deadline, now + minimumAlarmDelay));
|
|
1030
|
+
},
|
|
884
1031
|
);
|
|
885
1032
|
yield* failpoint.hit("maintenance:dirty:after");
|
|
886
1033
|
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
@@ -960,9 +1107,7 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
960
1107
|
),
|
|
961
1108
|
);
|
|
962
1109
|
|
|
963
|
-
const recordProgress = Effect.
|
|
964
|
-
lanes: ReadonlyArray<string>,
|
|
965
|
-
) {
|
|
1110
|
+
const recordProgress = Effect.fnUntraced(function* (lanes: ReadonlyArray<string>) {
|
|
966
1111
|
yield* validateSourceBoundary;
|
|
967
1112
|
if (Option.isNone(yield* Effect.serviceOption(sql.transactionService)))
|
|
968
1113
|
return yield* DurableAlarmError.make({
|
|
@@ -979,29 +1124,34 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
979
1124
|
const now = yield* Clock.currentTimeMillis;
|
|
980
1125
|
const nativeSource = yield* CurrentNativeSource;
|
|
981
1126
|
|
|
982
|
-
yield*
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
const
|
|
1127
|
+
yield* runAlarmTransaction(
|
|
1128
|
+
"record maintenance producer progress",
|
|
1129
|
+
async (transaction, checkpoint) => {
|
|
1130
|
+
for (const id of affected) dueQueue.dirty(id, now, true);
|
|
1131
|
+
if (affected.has(DueQueue.Native)) {
|
|
1132
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
986
1133
|
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
1134
|
+
// Native progress already belongs to this generation. Persist only
|
|
1135
|
+
// initialization, external progress, or the removal of a retry budget.
|
|
1136
|
+
if (!initialized || !nativeSource || state.retry !== undefined)
|
|
1137
|
+
await transaction.put(
|
|
1138
|
+
MAINTENANCE_STATE_KEY,
|
|
1139
|
+
encodeMaintenanceState(
|
|
1140
|
+
ThreadMaintenanceState.make({
|
|
1141
|
+
...Struct.omit(state, ["retry"]),
|
|
1142
|
+
dirty: state.dirty + (nativeSource ? 0n : 1n),
|
|
1143
|
+
}),
|
|
1144
|
+
),
|
|
1145
|
+
);
|
|
1146
|
+
}
|
|
1147
|
+
checkpoint.earlier(now + minimumAlarmDelay);
|
|
1148
|
+
},
|
|
1149
|
+
);
|
|
999
1150
|
yield* notify;
|
|
1000
1151
|
});
|
|
1001
1152
|
|
|
1002
1153
|
return ThreadMutationGate.of({
|
|
1003
|
-
withTransaction
|
|
1004
|
-
dueQueue.withTransaction(body).pipe(Effect.provideService(SqlClient, sql)),
|
|
1154
|
+
withTransaction,
|
|
1005
1155
|
withMutation,
|
|
1006
1156
|
schedule,
|
|
1007
1157
|
recordProgress,
|
|
@@ -1032,7 +1182,8 @@ export type MaintenancePassFailure =
|
|
|
1032
1182
|
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
1033
1183
|
* producer keeps its generation dirty and immediately eligible. Quiescent native retries
|
|
1034
1184
|
* retain their durable backoff.
|
|
1035
|
-
* 3. Admit
|
|
1185
|
+
* 3. Admit one Attempt for a single-Thread runtime, or at most two independent Thread
|
|
1186
|
+
* Attempts for a shared host, with one active head per Thread.
|
|
1036
1187
|
* Keep native and delivery admission open together while finite waves remain active, so
|
|
1037
1188
|
* fresh replies and abort controls can progress during unrelated cleanup. Close atomically
|
|
1038
1189
|
* at quiescence, or at the original ten-minute yield deadline, then join admitted waves.
|
|
@@ -1073,6 +1224,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1073
1224
|
| SubmissionLedger
|
|
1074
1225
|
| WakeScheduler
|
|
1075
1226
|
| DurableAlarmService
|
|
1227
|
+
| ThreadMaintenanceAlarm
|
|
1076
1228
|
| ThreadMaintenanceFailpoint
|
|
1077
1229
|
| CloudflareDurableRuntimeConfig
|
|
1078
1230
|
| DurableObjectContext
|
|
@@ -1083,7 +1235,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1083
1235
|
const recoveryConfig = yield* DurableRuntimeConfig;
|
|
1084
1236
|
const ledger = yield* SubmissionLedger;
|
|
1085
1237
|
const wakes = yield* WakeScheduler;
|
|
1086
|
-
const
|
|
1238
|
+
const hints = yield* DurableAlarmService;
|
|
1239
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
1087
1240
|
const config = yield* CloudflareDurableRuntimeConfig;
|
|
1088
1241
|
const { ctx } = yield* DurableObjectContext;
|
|
1089
1242
|
const dueQueue = DueQueue.make(ctx.storage);
|
|
@@ -1103,6 +1256,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1103
1256
|
const host = yield* ThreadHostMaintenance;
|
|
1104
1257
|
const framework = yield* ThreadNativeMaintenance;
|
|
1105
1258
|
|
|
1259
|
+
const nativeDispatchConcurrency = () => (framework.singleThreadRuntime === runtime ? 1 : 2);
|
|
1260
|
+
|
|
1106
1261
|
for (const lane of host.lanes)
|
|
1107
1262
|
yield* Schema.decodeEffect(DueQueue.HostLaneId)(lane.id).pipe(
|
|
1108
1263
|
Effect.mapError(alarmFailure("maintenance lane ID")),
|
|
@@ -1121,6 +1276,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1121
1276
|
const maintenancePassGate = yield* Semaphore.make(1);
|
|
1122
1277
|
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
1123
1278
|
const runTransaction = yield* makeStorageOperation;
|
|
1279
|
+
const { run: runAlarmTransaction } = yield* makeAlarmOperation;
|
|
1124
1280
|
|
|
1125
1281
|
const queueSnapshot = runTransaction("read maintenance due queue", async () =>
|
|
1126
1282
|
dueQueue.read(),
|
|
@@ -1192,17 +1348,17 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1192
1348
|
return Effect.gen(function* () {
|
|
1193
1349
|
const now = yield* Clock.currentTimeMillis;
|
|
1194
1350
|
|
|
1195
|
-
const claimed = yield*
|
|
1196
|
-
|
|
1351
|
+
const claimed = yield* runAlarmTransaction(
|
|
1352
|
+
"charge maintenance attempt",
|
|
1353
|
+
async (_transaction, checkpoint) => {
|
|
1197
1354
|
const claimed = dueQueue.claim(selected, now);
|
|
1198
1355
|
const next = DueQueue.next(dueQueue.read());
|
|
1199
1356
|
|
|
1200
|
-
if (Number.isFinite(next))
|
|
1201
|
-
|
|
1202
|
-
else await transaction.deleteAlarm();
|
|
1357
|
+
if (Number.isFinite(next)) checkpoint.at(Math.max(now + minimumAlarmDelay, next));
|
|
1358
|
+
else checkpoint.cancel();
|
|
1203
1359
|
|
|
1204
1360
|
return claimed;
|
|
1205
|
-
}
|
|
1361
|
+
},
|
|
1206
1362
|
);
|
|
1207
1363
|
|
|
1208
1364
|
if (claimed === undefined) return Option.none<number>();
|
|
@@ -1270,7 +1426,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1270
1426
|
};
|
|
1271
1427
|
|
|
1272
1428
|
const appendRecoveryEvents = async (
|
|
1273
|
-
transaction:
|
|
1429
|
+
transaction: MaintenanceStorage,
|
|
1274
1430
|
events: ReadonlyArray<Omit<ThreadRecoveryFaultEvent, "sequence">>,
|
|
1275
1431
|
) => {
|
|
1276
1432
|
if (events.length === 0) return;
|
|
@@ -1353,14 +1509,17 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1353
1509
|
}),
|
|
1354
1510
|
};
|
|
1355
1511
|
|
|
1356
|
-
const recordRecoveryFaults = Effect.
|
|
1512
|
+
const recordRecoveryFaults = Effect.fnUntraced(function* (
|
|
1357
1513
|
result: RecoverySweepResult,
|
|
1358
1514
|
recovery: NativeRecovery,
|
|
1359
1515
|
current?: ReadonlyArray<SubmissionWorkItem>,
|
|
1516
|
+
selectedThread?: ThreadId,
|
|
1360
1517
|
) {
|
|
1361
1518
|
const threads = new Map<ThreadId, RecoveryFailure | undefined>();
|
|
1362
1519
|
|
|
1520
|
+
if (selectedThread !== undefined) threads.set(selectedThread, undefined);
|
|
1363
1521
|
for (const report of result.reports) threads.set(report.threadId, undefined);
|
|
1522
|
+
for (const report of result.workReports ?? []) threads.set(report.threadId, undefined);
|
|
1364
1523
|
for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
|
|
1365
1524
|
if (threads.size === 0) return new Map<ThreadId, ThreadRecoveryFault>();
|
|
1366
1525
|
const now = yield* Clock.currentTimeMillis;
|
|
@@ -1462,27 +1621,59 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1462
1621
|
return retained.faults;
|
|
1463
1622
|
});
|
|
1464
1623
|
|
|
1465
|
-
const recoverThread = Effect.
|
|
1624
|
+
const recoverThread = Effect.fnUntraced(function* (
|
|
1466
1625
|
threadId: ThreadId,
|
|
1467
1626
|
recovery: NativeRecovery,
|
|
1468
1627
|
) {
|
|
1469
|
-
|
|
1470
|
-
.
|
|
1471
|
-
|
|
1628
|
+
let cursor = yield* runTransaction("read work recovery cursor", async () => {
|
|
1629
|
+
const retained = await ctx.storage.get(workCursorKey(threadId));
|
|
1630
|
+
|
|
1631
|
+
if (retained === undefined) return undefined;
|
|
1632
|
+
const decoded = Schema.decodeUnknownSync(NativeWorkCursor)(retained);
|
|
1472
1633
|
|
|
1473
|
-
|
|
1474
|
-
|
|
1634
|
+
if (decoded.threadId !== threadId)
|
|
1635
|
+
throw new Error("Work cursor belongs to another Thread");
|
|
1475
1636
|
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1637
|
+
return decoded.cursor;
|
|
1638
|
+
});
|
|
1639
|
+
|
|
1640
|
+
do {
|
|
1641
|
+
const result = yield* runtime
|
|
1642
|
+
.runRecovery({ threadId, ...(cursor === undefined ? {} : { cursor }) })
|
|
1643
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
1644
|
+
|
|
1645
|
+
// Persist each bounded pass before a claim; eviction resumes past consumed metadata.
|
|
1646
|
+
const faults = yield* recordRecoveryFaults(result, recovery, undefined, threadId);
|
|
1647
|
+
|
|
1648
|
+
yield* runTransaction("checkpoint work recovery cursor", async () => {
|
|
1649
|
+
if (result.cursor === undefined) await ctx.storage.delete(workCursorKey(threadId));
|
|
1650
|
+
else
|
|
1651
|
+
await ctx.storage.put(
|
|
1652
|
+
workCursorKey(threadId),
|
|
1653
|
+
Schema.encodeSync(NativeWorkCursor)({ threadId, cursor: result.cursor }),
|
|
1654
|
+
);
|
|
1655
|
+
});
|
|
1656
|
+
for (const report of result.reports) recovery.reports.set(report.submissionId, report);
|
|
1657
|
+
recovery.faults.delete(threadId);
|
|
1658
|
+
for (const [id, fault] of faults) recovery.faults.set(id, fault);
|
|
1659
|
+
recovery.recovered +=
|
|
1660
|
+
(result.workReports?.length ?? result.reports.length) + result.blocked.length;
|
|
1661
|
+
recovery.repaired ||= (result.workReports ?? result.reports).some(
|
|
1662
|
+
(report) => report.disposition === "repaired",
|
|
1663
|
+
);
|
|
1664
|
+
for (const report of result.workReports ?? [])
|
|
1665
|
+
if (report.retryAtMillis !== undefined)
|
|
1666
|
+
recovery.retryAtMillis = Math.min(
|
|
1667
|
+
recovery.retryAtMillis ?? Infinity,
|
|
1668
|
+
report.retryAtMillis,
|
|
1669
|
+
);
|
|
1670
|
+
cursor = result.blocked.length > 0 ? undefined : result.cursor;
|
|
1671
|
+
} while (cursor !== undefined);
|
|
1481
1672
|
});
|
|
1482
1673
|
|
|
1483
1674
|
// Registry changes, including upgrades from timed binding retries, create one native
|
|
1484
1675
|
// opportunity. Unchanged registries leave parked work dormant across Object eviction.
|
|
1485
|
-
const readCurrentMaintenanceState = async (transaction:
|
|
1676
|
+
const readCurrentMaintenanceState = async (transaction: MaintenanceStorage) => {
|
|
1486
1677
|
const current = await readMaintenanceState(transaction);
|
|
1487
1678
|
|
|
1488
1679
|
if (current.state.bindingRegistryKey === runtime.bindingRegistryKey) return current;
|
|
@@ -1500,48 +1691,43 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1500
1691
|
return { state, initialized: true };
|
|
1501
1692
|
};
|
|
1502
1693
|
|
|
1503
|
-
const ensureAlarm = Effect.
|
|
1694
|
+
const ensureAlarm = Effect.fnUntraced(function* () {
|
|
1504
1695
|
yield* failpoint.hit("maintenance:ensure:before");
|
|
1505
1696
|
const now = yield* Clock.currentTimeMillis;
|
|
1506
1697
|
|
|
1507
|
-
yield*
|
|
1508
|
-
|
|
1509
|
-
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1698
|
+
yield* runAlarmTransaction("ensure maintenance alarm", async (transaction, checkpoint) => {
|
|
1699
|
+
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1510
1700
|
|
|
1511
|
-
|
|
1512
|
-
|
|
1701
|
+
recoveryEventsPending = hasRecoveryEvents(state);
|
|
1702
|
+
const rows = dueQueue.read();
|
|
1513
1703
|
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1704
|
+
if (recoveryEventsPending && !rows.some((row) => row.id === DueQueue.RecoveryEvents))
|
|
1705
|
+
dueQueue.dirty(DueQueue.RecoveryEvents, now);
|
|
1706
|
+
if (!initialized)
|
|
1707
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
1518
1708
|
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
: null;
|
|
1709
|
+
const native =
|
|
1710
|
+
state.dirty > state.processed
|
|
1711
|
+
? state.retry?.generation === state.dirty
|
|
1712
|
+
? state.retry.notBefore
|
|
1713
|
+
: (rows.find((row) => row.id === DueQueue.Native)?.dueAt ?? now + minimumAlarmDelay)
|
|
1714
|
+
: null;
|
|
1526
1715
|
|
|
1527
|
-
|
|
1528
|
-
|
|
1716
|
+
dueQueue.checkpointNative(native);
|
|
1717
|
+
const next = DueQueue.next(dueQueue.read());
|
|
1529
1718
|
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
}),
|
|
1533
|
-
);
|
|
1719
|
+
if (Number.isFinite(next)) checkpoint.earlier(Math.max(now + minimumAlarmDelay, next));
|
|
1720
|
+
});
|
|
1534
1721
|
yield* failpoint.hit("maintenance:ensure:after");
|
|
1535
1722
|
});
|
|
1536
1723
|
|
|
1537
|
-
const beginPass = Effect.
|
|
1538
|
-
observed: MaintenanceObservation,
|
|
1539
|
-
) {
|
|
1724
|
+
const beginPass = Effect.fnUntraced(function* (observed: MaintenanceObservation) {
|
|
1540
1725
|
yield* failpoint.hit("maintenance:begin:before");
|
|
1541
1726
|
const now = yield* Clock.currentTimeMillis;
|
|
1542
1727
|
|
|
1543
|
-
const result = yield*
|
|
1544
|
-
|
|
1728
|
+
const result = yield* runAlarmTransaction(
|
|
1729
|
+
"begin maintenance pass",
|
|
1730
|
+
async (transaction, checkpoint) => {
|
|
1545
1731
|
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1546
1732
|
|
|
1547
1733
|
recoveryEventsPending = hasRecoveryEvents(state);
|
|
@@ -1582,8 +1768,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1582
1768
|
const deadline = DueQueue.next(dueQueue.read());
|
|
1583
1769
|
|
|
1584
1770
|
if (Number.isFinite(deadline))
|
|
1585
|
-
|
|
1586
|
-
else
|
|
1771
|
+
checkpoint.at(Math.max(now + minimumAlarmDelay, deadline));
|
|
1772
|
+
else checkpoint.cancel();
|
|
1587
1773
|
|
|
1588
1774
|
return {
|
|
1589
1775
|
_tag: "Actionable" as const,
|
|
@@ -1592,7 +1778,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1592
1778
|
nonterminal: state.nonterminal,
|
|
1593
1779
|
stalls: charged.stalls,
|
|
1594
1780
|
};
|
|
1595
|
-
}
|
|
1781
|
+
},
|
|
1596
1782
|
);
|
|
1597
1783
|
|
|
1598
1784
|
yield* failpoint.hit("maintenance:begin:after");
|
|
@@ -1610,16 +1796,11 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1610
1796
|
return Math.ceil(backoff / 2 + (backoff / 2) * jitter);
|
|
1611
1797
|
};
|
|
1612
1798
|
|
|
1613
|
-
const rearmDelay = Effect.
|
|
1614
|
-
progressed: boolean,
|
|
1615
|
-
priorStalls: number,
|
|
1616
|
-
) {
|
|
1799
|
+
const rearmDelay = Effect.fnUntraced(function* (progressed: boolean, priorStalls: number) {
|
|
1617
1800
|
return progressed ? config.alarmBackoffBase : backoffDelay(priorStalls, yield* Random.next);
|
|
1618
1801
|
});
|
|
1619
1802
|
|
|
1620
|
-
const rearmFailure = Effect.
|
|
1621
|
-
observed: MaintenanceObservation,
|
|
1622
|
-
) {
|
|
1803
|
+
const rearmFailure = Effect.fnUntraced(function* (observed: MaintenanceObservation) {
|
|
1623
1804
|
const { generation, nativeOnly } = observed;
|
|
1624
1805
|
|
|
1625
1806
|
if (generation === undefined) return;
|
|
@@ -1630,8 +1811,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1630
1811
|
|
|
1631
1812
|
const jitter = yield* Random.next;
|
|
1632
1813
|
|
|
1633
|
-
yield*
|
|
1634
|
-
|
|
1814
|
+
yield* runAlarmTransaction(
|
|
1815
|
+
"back off failed maintenance",
|
|
1816
|
+
async (transaction, checkpoint) => {
|
|
1635
1817
|
const { state } = await readMaintenanceState(transaction);
|
|
1636
1818
|
|
|
1637
1819
|
const previous =
|
|
@@ -1682,20 +1864,17 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1682
1864
|
}
|
|
1683
1865
|
const next = DueQueue.next(dueQueue.read());
|
|
1684
1866
|
|
|
1685
|
-
if (Number.isFinite(next))
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
}),
|
|
1867
|
+
if (Number.isFinite(next)) checkpoint.at(Math.max(now + minimumAlarmDelay, next));
|
|
1868
|
+
else checkpoint.cancel();
|
|
1869
|
+
},
|
|
1689
1870
|
);
|
|
1690
1871
|
}),
|
|
1691
1872
|
);
|
|
1692
1873
|
yield* failpoint.hit("maintenance:retry:after");
|
|
1693
1874
|
});
|
|
1694
1875
|
|
|
1695
|
-
const beginNative =
|
|
1696
|
-
|
|
1697
|
-
) {
|
|
1698
|
-
return yield* mutations.withSnapshot((activeAtStart) =>
|
|
1876
|
+
const beginNative = (observed: MaintenanceObservation) =>
|
|
1877
|
+
mutations.withSnapshot((activeAtStart) =>
|
|
1699
1878
|
Effect.gen(function* () {
|
|
1700
1879
|
const generation = yield* beginPass(observed);
|
|
1701
1880
|
|
|
@@ -1708,9 +1887,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1708
1887
|
return { ...generation, activeAtStart };
|
|
1709
1888
|
}),
|
|
1710
1889
|
);
|
|
1711
|
-
});
|
|
1712
1890
|
|
|
1713
|
-
const attemptSelected = Effect.
|
|
1891
|
+
const attemptSelected = Effect.fnUntraced(function* (
|
|
1714
1892
|
selected: SubmissionWorkItem,
|
|
1715
1893
|
previousWaits: ReadonlyArray<BindingWait>,
|
|
1716
1894
|
yieldAfter: DateTime.Utc,
|
|
@@ -1798,7 +1976,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1798
1976
|
return Option.isSome(settlement) ? 1 : 0;
|
|
1799
1977
|
});
|
|
1800
1978
|
|
|
1801
|
-
const advance = Effect.
|
|
1979
|
+
const advance = Effect.fnUntraced(function* (
|
|
1802
1980
|
started: Effect.Success<ReturnType<typeof beginNative>>,
|
|
1803
1981
|
yieldAfter: DateTime.Utc,
|
|
1804
1982
|
observed: MaintenanceObservation,
|
|
@@ -1896,7 +2074,32 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1896
2074
|
activeAtStart: started.activeAtStart,
|
|
1897
2075
|
});
|
|
1898
2076
|
|
|
2077
|
+
const { scannedGeneration, afterWorkThread } = yield* runTransaction(
|
|
2078
|
+
"read work discovery position",
|
|
2079
|
+
async () => {
|
|
2080
|
+
const { state } = await readMaintenanceState(ctx.storage);
|
|
2081
|
+
|
|
2082
|
+
return {
|
|
2083
|
+
scannedGeneration: state.dirty,
|
|
2084
|
+
afterWorkThread: state.lastRecoveredThreadId,
|
|
2085
|
+
};
|
|
2086
|
+
},
|
|
2087
|
+
);
|
|
2088
|
+
|
|
1899
2089
|
const current = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
2090
|
+
|
|
2091
|
+
let workOwners = yield* runtime
|
|
2092
|
+
.discoverWorkThreads({
|
|
2093
|
+
limit: 32,
|
|
2094
|
+
...(afterWorkThread === undefined ? {} : { afterThreadId: afterWorkThread }),
|
|
2095
|
+
})
|
|
2096
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
2097
|
+
|
|
2098
|
+
if (workOwners.threadIds.length === 0 && afterWorkThread !== undefined)
|
|
2099
|
+
workOwners = yield* runtime
|
|
2100
|
+
.discoverWorkThreads({ limit: 32 })
|
|
2101
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
2102
|
+
recovery.discoveryPending ||= workOwners.afterThreadId !== undefined;
|
|
1900
2103
|
const selectionTime = yield* Clock.currentTimeMillis;
|
|
1901
2104
|
const submissionsByThread = new Map<ThreadId, Array<SubmissionId>>();
|
|
1902
2105
|
|
|
@@ -1906,6 +2109,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1906
2109
|
ids.push(row.submissionId);
|
|
1907
2110
|
submissionsByThread.set(row.threadId, ids);
|
|
1908
2111
|
}
|
|
2112
|
+
// Canonical obligations survive their last Submission. Reload their retained backoff
|
|
2113
|
+
// before selecting recovery, including after eviction or a newer host mutation.
|
|
2114
|
+
for (const threadId of workOwners.threadIds)
|
|
2115
|
+
if (!submissionsByThread.has(threadId)) submissionsByThread.set(threadId, []);
|
|
1909
2116
|
|
|
1910
2117
|
yield* runTransaction("read Thread recovery deadlines", async () => {
|
|
1911
2118
|
for (const [threadId, ids] of submissionsByThread) {
|
|
@@ -2064,7 +2271,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2064
2271
|
const selected = dispatch
|
|
2065
2272
|
? [...runnable.slice(pivot), ...runnable.slice(0, pivot)].slice(
|
|
2066
2273
|
0,
|
|
2067
|
-
nativeDispatchConcurrency - reserved.size,
|
|
2274
|
+
nativeDispatchConcurrency() - reserved.size,
|
|
2068
2275
|
)
|
|
2069
2276
|
: [];
|
|
2070
2277
|
|
|
@@ -2081,7 +2288,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2081
2288
|
|
|
2082
2289
|
const backlog = recovery.started
|
|
2083
2290
|
? []
|
|
2084
|
-
: [...heads.keys()].filter((threadId) => {
|
|
2291
|
+
: [...new Set([...heads.keys(), ...workOwners.threadIds])].filter((threadId) => {
|
|
2085
2292
|
const fault = recoveryFaults.get(threadId);
|
|
2086
2293
|
|
|
2087
2294
|
return (
|
|
@@ -2103,7 +2310,12 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2103
2310
|
threadId <= state.lastRecoveredThreadId,
|
|
2104
2311
|
);
|
|
2105
2312
|
|
|
2106
|
-
return {
|
|
2313
|
+
return {
|
|
2314
|
+
selected,
|
|
2315
|
+
waits,
|
|
2316
|
+
backlog: [...after, ...before].slice(0, 32),
|
|
2317
|
+
discoveryPending: backlog.length > 32,
|
|
2318
|
+
};
|
|
2107
2319
|
}),
|
|
2108
2320
|
);
|
|
2109
2321
|
|
|
@@ -2111,6 +2323,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2111
2323
|
if (!recovery.started) {
|
|
2112
2324
|
recovery.started = true;
|
|
2113
2325
|
recovery.needsCheckpoint = selection.backlog.length > 0;
|
|
2326
|
+
recovery.discoveryPending ||= selection.discoveryPending;
|
|
2114
2327
|
for (const threadId of selection.backlog) recovery.pending.add(threadId);
|
|
2115
2328
|
yield* Deferred.succeed(recovery.queue, selection.backlog);
|
|
2116
2329
|
}
|
|
@@ -2142,30 +2355,32 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2142
2355
|
};
|
|
2143
2356
|
}
|
|
2144
2357
|
|
|
2145
|
-
const scannedGeneration = yield* runTransaction(
|
|
2146
|
-
"observe native source generation",
|
|
2147
|
-
async () => (await readMaintenanceState(ctx.storage)).state.dirty,
|
|
2148
|
-
);
|
|
2149
|
-
|
|
2150
2358
|
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
2151
2359
|
const waitingHeads = new Map<ThreadId, boolean>();
|
|
2152
2360
|
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2361
|
+
// Selected recovery must finish before its generation can be acknowledged: terminal
|
|
2362
|
+
// owners can supply a retry deadline only after their evidence read completes.
|
|
2363
|
+
const autonomous =
|
|
2364
|
+
recovery.pending.size > 0 ||
|
|
2365
|
+
recovery.discoveryPending ||
|
|
2366
|
+
recoveryFaults.size > 0 ||
|
|
2367
|
+
recovery.retryAtMillis !== undefined ||
|
|
2368
|
+
remaining.some((snapshot) => {
|
|
2369
|
+
if (snapshot.state === "unknown" && waiting(snapshot)) return false;
|
|
2370
|
+
const headWaiting = waitingHeads.get(snapshot.threadId);
|
|
2371
|
+
|
|
2372
|
+
if (headWaiting === undefined) waitingHeads.set(snapshot.threadId, waiting(snapshot));
|
|
2373
|
+
// FIFO followers cannot execute through a stable external wait. Only plain queued
|
|
2374
|
+
// input is dormant here; admission repairs and accepted aborts still need a pass.
|
|
2375
|
+
if (
|
|
2376
|
+
headWaiting === true &&
|
|
2377
|
+
snapshot.state === "ready" &&
|
|
2378
|
+
reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput"
|
|
2379
|
+
)
|
|
2380
|
+
return false;
|
|
2166
2381
|
|
|
2167
|
-
|
|
2168
|
-
|
|
2382
|
+
return !waiting(snapshot);
|
|
2383
|
+
});
|
|
2169
2384
|
|
|
2170
2385
|
const progressed = native.progressed || recovery.repaired;
|
|
2171
2386
|
|
|
@@ -2180,6 +2395,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2180
2395
|
.map(() => now),
|
|
2181
2396
|
);
|
|
2182
2397
|
|
|
2398
|
+
if (recovery.retryAtMillis !== undefined) nextEligible.push(recovery.retryAtMillis);
|
|
2399
|
+
|
|
2183
2400
|
const retryDelay =
|
|
2184
2401
|
nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
|
|
2185
2402
|
|
|
@@ -2197,10 +2414,14 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2197
2414
|
const mutationOverlap =
|
|
2198
2415
|
observation.activeAtStart > 0 || started.activeAtStart > 0 || active > 0;
|
|
2199
2416
|
|
|
2200
|
-
//
|
|
2201
|
-
//
|
|
2417
|
+
// Only a scan without ledger work or canonical obligations can certify a newer
|
|
2418
|
+
// generation without another recovery wave. Read its generation before discovery
|
|
2419
|
+
// so progress committed during the scan retains its own recovery opportunity.
|
|
2202
2420
|
const generation =
|
|
2203
|
-
remaining.length === 0 &&
|
|
2421
|
+
remaining.length === 0 &&
|
|
2422
|
+
workOwners.threadIds.length === 0 &&
|
|
2423
|
+
!mutationOverlap &&
|
|
2424
|
+
state.dirty === scannedGeneration
|
|
2204
2425
|
? scannedGeneration
|
|
2205
2426
|
: observation.generation;
|
|
2206
2427
|
|
|
@@ -2250,9 +2471,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2250
2471
|
);
|
|
2251
2472
|
|
|
2252
2473
|
yield* failpoint.hit("maintenance:checkpoint:after");
|
|
2253
|
-
//
|
|
2254
|
-
//
|
|
2255
|
-
if (remaining.length === 0 && recovery.pending.size === 0)
|
|
2474
|
+
// A fresh empty scan may clear earlier producer overlap, but this pass has only
|
|
2475
|
+
// recovered its original generation. Retain that boundary for later canonical work.
|
|
2476
|
+
if (remaining.length === 0 && recovery.pending.size === 0)
|
|
2477
|
+
recovery.observation = { generation: observation.generation, activeAtStart: 0 };
|
|
2256
2478
|
native.deferred.clear();
|
|
2257
2479
|
native.progressed = false;
|
|
2258
2480
|
native.needsCheckpoint = false;
|
|
@@ -2264,7 +2486,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2264
2486
|
};
|
|
2265
2487
|
});
|
|
2266
2488
|
|
|
2267
|
-
const validateLanes = Effect.
|
|
2489
|
+
const validateLanes = Effect.fnUntraced(function* (
|
|
2268
2490
|
lanes: ReadonlyArray<ThreadHostMaintenanceLane>,
|
|
2269
2491
|
queued: ReadonlyArray<DueQueue.DueLane>,
|
|
2270
2492
|
) {
|
|
@@ -2295,7 +2517,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2295
2517
|
}
|
|
2296
2518
|
});
|
|
2297
2519
|
|
|
2298
|
-
const dispatch = Effect.
|
|
2520
|
+
const dispatch = Effect.fnUntraced(function* (
|
|
2299
2521
|
yieldAfter: DateTime.Utc,
|
|
2300
2522
|
dispatchUntil: DateTime.Utc,
|
|
2301
2523
|
observed: MaintenanceObservation,
|
|
@@ -2354,6 +2576,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2354
2576
|
needsCheckpoint: false,
|
|
2355
2577
|
recovered: 0,
|
|
2356
2578
|
repaired: false,
|
|
2579
|
+
discoveryPending: false,
|
|
2357
2580
|
};
|
|
2358
2581
|
|
|
2359
2582
|
const recoveryFiber = yield* fork(
|
|
@@ -2438,7 +2661,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2438
2661
|
native = undefined;
|
|
2439
2662
|
if (result.phase === "actionable") phase = "actionable";
|
|
2440
2663
|
observed.nativeOnly = dispatch.active.size > 0;
|
|
2441
|
-
if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency) {
|
|
2664
|
+
if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency()) {
|
|
2442
2665
|
dispatch.scanGeneration = undefined;
|
|
2443
2666
|
nativeCheck = true;
|
|
2444
2667
|
}
|
|
@@ -2589,7 +2812,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2589
2812
|
if (
|
|
2590
2813
|
native === undefined &&
|
|
2591
2814
|
nativeCheck &&
|
|
2592
|
-
dispatch.active.size < nativeDispatchConcurrency
|
|
2815
|
+
dispatch.active.size < nativeDispatchConcurrency()
|
|
2593
2816
|
) {
|
|
2594
2817
|
nativeCheck = false;
|
|
2595
2818
|
const nativeCheckpoint = dispatch.needsCheckpoint && dispatch.active.size === 0;
|
|
@@ -2692,7 +2915,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2692
2915
|
return { result, phase, recovered: recovery.recovered, settled: dispatch.settled };
|
|
2693
2916
|
});
|
|
2694
2917
|
|
|
2695
|
-
const pass = Effect.
|
|
2918
|
+
const pass = Effect.fnUntraced(function* (
|
|
2696
2919
|
yieldAfter: DateTime.Utc,
|
|
2697
2920
|
dispatchUntil: DateTime.Utc,
|
|
2698
2921
|
observed: MaintenanceObservation,
|
|
@@ -2856,8 +3079,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2856
3079
|
Effect.gen(function* () {
|
|
2857
3080
|
const now = yield* Clock.currentTimeMillis;
|
|
2858
3081
|
|
|
2859
|
-
return yield*
|
|
2860
|
-
|
|
3082
|
+
return yield* runAlarmTransaction(
|
|
3083
|
+
"finish maintenance event",
|
|
3084
|
+
async (transaction, checkpoint) => {
|
|
2861
3085
|
const { state } = await readMaintenanceState(transaction);
|
|
2862
3086
|
|
|
2863
3087
|
const native =
|
|
@@ -2873,14 +3097,14 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2873
3097
|
const next = DueQueue.next(dueQueue.read());
|
|
2874
3098
|
|
|
2875
3099
|
if (Number.isFinite(next)) {
|
|
2876
|
-
|
|
3100
|
+
checkpoint.at(Math.max(now + minimumAlarmDelay, next));
|
|
2877
3101
|
|
|
2878
3102
|
return "rearmed" as const;
|
|
2879
3103
|
}
|
|
2880
|
-
|
|
3104
|
+
checkpoint.cancel();
|
|
2881
3105
|
|
|
2882
3106
|
return "cleared" as const;
|
|
2883
|
-
}
|
|
3107
|
+
},
|
|
2884
3108
|
);
|
|
2885
3109
|
}),
|
|
2886
3110
|
);
|
|
@@ -2920,18 +3144,27 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2920
3144
|
available: new Map(),
|
|
2921
3145
|
};
|
|
2922
3146
|
|
|
2923
|
-
return yield*
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
(
|
|
2930
|
-
|
|
3147
|
+
return yield* alarms
|
|
3148
|
+
.deferWakes(
|
|
3149
|
+
maintenancePassGate.withPermit(
|
|
3150
|
+
dueQueue.withView(Effect.scoped(pass(yieldAfter, dispatchUntil, observed))).pipe(
|
|
3151
|
+
// Close event-owned auxiliary work and release Attempt ownership before
|
|
3152
|
+
// failure rearming, while still holding the pass permit.
|
|
3153
|
+
Effect.onErrorIf(
|
|
3154
|
+
() => true,
|
|
3155
|
+
() => rearmFailure(observed),
|
|
3156
|
+
),
|
|
2931
3157
|
),
|
|
2932
3158
|
),
|
|
2933
|
-
)
|
|
2934
|
-
|
|
3159
|
+
)
|
|
3160
|
+
.pipe(
|
|
3161
|
+
hints.withHintsDeferred,
|
|
3162
|
+
Effect.catchTags({
|
|
3163
|
+
StorageOperationError: (error) => alarmFailure("defer maintenance wakes")(error),
|
|
3164
|
+
InvalidScheduleConfigurationError: (error) =>
|
|
3165
|
+
alarmFailure("defer maintenance wakes")(error),
|
|
3166
|
+
}),
|
|
3167
|
+
);
|
|
2935
3168
|
}).pipe(
|
|
2936
3169
|
// Include permit waiting, recovery and acknowledgement in the event deadline.
|
|
2937
3170
|
// Interruption releases Attempt ownership, leaving the prearmed dirty generation
|