@yielded/agent-platform-cloudflare 0.1.0-beta.166 → 0.1.0-beta.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{Alarm-B0ODr1NE.mjs → Alarm-DxODgAZZ.mjs} +259 -159
- package/dist/Alarm-DxODgAZZ.mjs.map +1 -0
- package/dist/Alarm.d.mts +27 -36
- package/dist/Alarm.mjs +2 -2
- package/dist/BrowserCredentials-C5IWfi7f.d.mts +91 -0
- package/dist/BrowserCredentials.d.mts +2 -91
- package/dist/BrowserRestCapture.mjs +3 -3
- package/dist/BrowserRestCapture.mjs.map +1 -1
- package/dist/BrowserRestCrawl.mjs +6 -6
- package/dist/BrowserRestCrawl.mjs.map +1 -1
- package/dist/{BrowserSession-RfIzxGCH.mjs → BrowserSession-BfNmyayH.mjs} +5 -5
- package/dist/BrowserSession-BfNmyayH.mjs.map +1 -0
- package/dist/{BrowserSession-B969Xg-2.d.mts → BrowserSession-Bx0V1-6_.d.mts} +6 -3
- package/dist/BrowserSession.d.mts +1 -1
- package/dist/BrowserSession.mjs +2 -2
- package/dist/BrowserUse-AstK0jKX.mjs +1467 -0
- package/dist/BrowserUse-AstK0jKX.mjs.map +1 -0
- package/dist/BrowserUse.d.mts +83 -0
- package/dist/BrowserUse.mjs +2 -0
- package/dist/CloudflareBindings.d.mts +8 -1
- package/dist/CloudflareBindings.mjs +2 -2
- package/dist/CloudflareBindings.mjs.map +1 -1
- package/dist/{CloudflareBrowser-5an6m-Qb.mjs → CloudflareBrowser-Vt8qTdYW.mjs} +11 -13
- package/dist/CloudflareBrowser-Vt8qTdYW.mjs.map +1 -0
- package/dist/CloudflareBrowser.mjs +1 -1
- package/dist/CloudflareCodeMode.mjs +1 -1
- package/dist/CloudflareCodeMode.mjs.map +1 -1
- package/dist/{CloudflareConfig-qDrRLofL.d.mts → CloudflareConfig-C9MvPbX8.d.mts} +9 -11
- package/dist/CloudflareConfig.d.mts +1 -1
- package/dist/CloudflareConfig.mjs +9 -12
- package/dist/CloudflareConfig.mjs.map +1 -1
- package/dist/CloudflareMemory.d.mts +3 -25
- package/dist/CloudflareMemory.mjs +33 -31
- package/dist/CloudflareMemory.mjs.map +1 -1
- package/dist/CloudflareScheduling.mjs +8 -15
- package/dist/CloudflareScheduling.mjs.map +1 -1
- package/dist/CloudflareSubscriptions.mjs +9 -16
- package/dist/CloudflareSubscriptions.mjs.map +1 -1
- package/dist/CloudflareThreadClient-CZm0TvUe.mjs +588 -0
- package/dist/CloudflareThreadClient-CZm0TvUe.mjs.map +1 -0
- package/dist/CloudflareThreadClient-QsVzLGGZ.d.mts +2033 -0
- package/dist/CloudflareThreadClient.d.mts +2 -1796
- package/dist/CloudflareThreadClient.mjs +2 -382
- package/dist/{InteractiveBrowser-BbG3kGA9.mjs → InteractiveBrowser-LdKzQeLR.mjs} +18 -19
- package/dist/InteractiveBrowser-LdKzQeLR.mjs.map +1 -0
- package/dist/InteractiveBrowser.mjs +1 -1
- package/dist/{ThreadObject-BcfXbzSO.d.mts → ThreadObject-7s5XnlH-.d.mts} +103 -79
- package/dist/{ThreadObject-DHjrnlX4.mjs → ThreadObject-C0vrcDrw.mjs} +265 -66
- package/dist/ThreadObject-C0vrcDrw.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +2 -2
- package/dist/ThreadObject.mjs +2 -3
- package/dist/WakeScheduler.d.mts +6 -5
- package/dist/WakeScheduler.mjs +15 -11
- package/dist/WakeScheduler.mjs.map +1 -1
- package/dist/index.d.mts +7 -6
- package/dist/index.mjs +7 -6
- package/dist/page-function-CmUCcyNP.mjs +11 -0
- package/dist/page-function-CmUCcyNP.mjs.map +1 -0
- package/dist/{prepared-admission-DMX5p9dC.mjs → prepared-admission-3UUq0zth.mjs} +3 -3
- package/dist/{prepared-admission-DMX5p9dC.mjs.map → prepared-admission-3UUq0zth.mjs.map} +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +510 -279
- package/src/BrowserRestCapture.ts +2 -2
- package/src/BrowserRestCrawl.ts +10 -14
- package/src/BrowserSession.ts +10 -2
- package/src/BrowserUse.ts +1541 -0
- package/src/CloudflareBindings.ts +13 -4
- package/src/CloudflareCodeMode.ts +1 -1
- package/src/CloudflareConfig.ts +9 -12
- package/src/CloudflareMemory.ts +21 -28
- package/src/CloudflareScheduling.ts +7 -15
- package/src/CloudflareSubscriptions.ts +11 -20
- package/src/CloudflareThreadClient.ts +289 -9
- package/src/InteractiveBrowser.ts +325 -323
- package/src/ThreadObject.ts +172 -52
- package/src/WakeScheduler.ts +29 -18
- package/src/index.ts +1 -0
- package/src/internal/browser-dom.ts +845 -0
- package/src/internal/browser-file-selection.ts +8 -1
- package/src/internal/browser-jev-dom.ts +344 -0
- package/src/internal/browser-quick-action.ts +7 -8
- package/src/internal/browser-session-lifecycle.ts +2 -5
- package/src/internal/crypto.ts +41 -0
- package/src/internal/due-queue.ts +5 -69
- package/src/internal/layers.ts +100 -17
- package/src/internal/live-text-protocol.ts +51 -0
- package/src/internal/live-text.ts +256 -0
- package/src/internal/page-function.ts +12 -0
- package/src/internal/prepared-admission.ts +1 -1
- package/src/internal/progress-wait.ts +19 -24
- package/src/internal/transport.ts +1 -1
- package/dist/Alarm-B0ODr1NE.mjs.map +0 -1
- package/dist/BrowserSession-RfIzxGCH.mjs.map +0 -1
- package/dist/CloudflareBrowser-5an6m-Qb.mjs.map +0 -1
- package/dist/CloudflareThreadClient.mjs.map +0 -1
- package/dist/InteractiveBrowser-BbG3kGA9.mjs.map +0 -1
- package/dist/ThreadObject-DHjrnlX4.mjs.map +0 -1
package/src/Alarm.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CurrentNativeSource } from "@yielded/agent-storage-cloudflare/port-routing";
|
|
1
2
|
import { type DurableBindingFailure } from "@yielded/agent/agent-registration";
|
|
2
3
|
import {
|
|
3
4
|
DurableAgentRuntime,
|
|
@@ -35,15 +36,16 @@ import {
|
|
|
35
36
|
Random,
|
|
36
37
|
Ref,
|
|
37
38
|
Result,
|
|
39
|
+
Scheduler,
|
|
38
40
|
Schema,
|
|
39
41
|
Scope,
|
|
40
42
|
Semaphore,
|
|
41
43
|
Stream,
|
|
42
44
|
Struct,
|
|
43
45
|
} from "effect";
|
|
44
|
-
import { DurableObjectStorage } from "effect-cf";
|
|
46
|
+
import { DurableObjectAlarm, DurableObjectStorage } from "effect-cf";
|
|
45
47
|
import { SqlClient } from "effect/sql/SqlClient";
|
|
46
|
-
import
|
|
48
|
+
import { SqlError, UnknownError } from "effect/sql/SqlError";
|
|
47
49
|
|
|
48
50
|
import { DurableObjectContext } from "./CloudflareBindings.ts";
|
|
49
51
|
import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.ts";
|
|
@@ -51,10 +53,8 @@ import { safeCauseMessage } from "./internal/boundary.ts";
|
|
|
51
53
|
import * as DueQueue from "./internal/due-queue.ts";
|
|
52
54
|
|
|
53
55
|
/**
|
|
54
|
-
* The
|
|
55
|
-
* alarm slot
|
|
56
|
-
* and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
|
|
57
|
-
* the slot always holds the EARLIEST deadline any caller asked for.
|
|
56
|
+
* The maintenance queue owns lease expiry, settlement, retries and dirty generations.
|
|
57
|
+
* Its manual effect-cf alarm shares the native slot with other registered alarms.
|
|
58
58
|
*
|
|
59
59
|
* The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
|
|
60
60
|
* maintenance generation and a committed alarm. Stable externally-driven waits may be
|
|
@@ -122,110 +122,268 @@ const makeStorageOperation = Effect.map(
|
|
|
122
122
|
run(operation, Effect.tryPromise({ try: execute, catch: alarmFailure(operation) })),
|
|
123
123
|
);
|
|
124
124
|
|
|
125
|
-
/**
|
|
125
|
+
/** One manual alarm for the entire consumer-owned maintenance queue. */
|
|
126
|
+
export class ThreadMaintenanceAlarm extends DurableObjectAlarm.Tag<ThreadMaintenanceAlarm>()(
|
|
127
|
+
"@yielded/agent/thread-maintenance",
|
|
128
|
+
{ "@yielded/agent/thread-maintenance": { payload: Schema.Null, lifecycle: "manual" } },
|
|
129
|
+
) {}
|
|
130
|
+
|
|
131
|
+
const maintenanceAlarmRef = { tag: "@yielded/agent/thread-maintenance", id: "queue" } as const;
|
|
132
|
+
|
|
133
|
+
const maintenanceAlarmInput = (deadline: number) => ({
|
|
134
|
+
...maintenanceAlarmRef,
|
|
135
|
+
payload: null,
|
|
136
|
+
runAt: DateTime.makeUnsafe(deadline),
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
/** Named maintenance checkpoint; the library owns the shared native alarm slot. */
|
|
126
140
|
export class DurableAlarmService extends Context.Service<
|
|
127
141
|
DurableAlarmService,
|
|
128
142
|
{
|
|
129
|
-
/** The scheduled deadline in epoch milliseconds, if any. */
|
|
130
143
|
readonly scheduled: Effect.Effect<Option.Option<number>, DurableAlarmError>;
|
|
131
|
-
/** Replace the slot with this deadline. */
|
|
132
144
|
readonly scheduleAt: (epochMillis: number) => Effect.Effect<void, DurableAlarmError>;
|
|
133
|
-
/** Keep the EARLIER of the existing deadline and this one (the multiplexing rule). */
|
|
134
145
|
readonly ensureScheduledBy: (epochMillis: number) => Effect.Effect<void, DurableAlarmError>;
|
|
135
|
-
/**
|
|
136
|
-
* Arm an immediate alarm (the durable, coalescing local wake) — DEFERRED while a
|
|
137
|
-
* maintenance pass is executing. Workerd cancels an in-flight alarm handler when a new
|
|
138
|
-
* EARLIER deadline is written during its execution (`requestScheduledAlarm`), and the
|
|
139
|
-
* maintenance pass runs INSIDE the alarm handler: an immediate wake landing mid-pass
|
|
140
|
-
* (a routed port mutation, a sibling's `wake()`, the coordinator's own local notify)
|
|
141
|
-
* would kill the running Attempt — manufacturing an ownership loss no real eviction
|
|
142
|
-
* caused, and routing open uncertain-class Tool Calls into spurious Unknown Outcomes.
|
|
143
|
-
* Deferral is contract-safe: wakes are droppable hints, every mutating entry point
|
|
144
|
-
* pre-arms BEFORE its first durable mutation (the alarm invariant never rests on this
|
|
145
|
-
* call). The pass's durable generation check observes any racing mutation, so the
|
|
146
|
-
* in-memory hint does not need to be flushed after a stable wait is acknowledged.
|
|
147
|
-
*/
|
|
146
|
+
/** Queue-aware promptness hint; idle queues stay idle and retry floors survive. */
|
|
148
147
|
readonly scheduleNow: Effect.Effect<void, DurableAlarmError>;
|
|
149
|
-
/**
|
|
150
|
-
|
|
151
|
-
* executes are droppable promptness hints; correctness rests on the durable generation.
|
|
152
|
-
*/
|
|
153
|
-
readonly withWakesDeferred: <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
154
|
-
/** Clear the slot; correctness-sensitive clears live in maintenance generation transactions. */
|
|
148
|
+
/** Coalesce droppable inline hints without delaying committed deadlines or their delivery. */
|
|
149
|
+
readonly withHintsDeferred: <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
155
150
|
readonly cancel: Effect.Effect<void, DurableAlarmError>;
|
|
156
151
|
}
|
|
157
152
|
>()("@effect-agent/platform-cloudflare/DurableAlarmService") {
|
|
158
|
-
static readonly layer
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
153
|
+
static readonly layer = Layer.effect(DurableAlarmService)(
|
|
154
|
+
Effect.gen(function* () {
|
|
155
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
156
|
+
const { ctx } = yield* DurableObjectContext;
|
|
157
|
+
const run = yield* makeStorageEffect;
|
|
158
|
+
const inlineWork = yield* Ref.make(0);
|
|
159
|
+
|
|
160
|
+
const wrap = <A, E, R>(operation: string, effect: Effect.Effect<A, E, R>) =>
|
|
161
|
+
run(operation, effect.pipe(Effect.mapError(alarmFailure(operation))));
|
|
162
|
+
|
|
163
|
+
return DurableAlarmService.of({
|
|
164
|
+
scheduled: wrap(
|
|
165
|
+
"read maintenance checkpoint",
|
|
166
|
+
alarms.getAlarmStatus(maintenanceAlarmRef),
|
|
167
|
+
).pipe(
|
|
168
|
+
Effect.map((at) =>
|
|
169
|
+
at === undefined ? Option.none() : Option.some(DateTime.toEpochMillis(at.runAt)),
|
|
174
170
|
),
|
|
175
|
-
)
|
|
171
|
+
),
|
|
172
|
+
scheduleAt: (at) =>
|
|
173
|
+
wrap("replace maintenance checkpoint", alarms.scheduleAlarm(maintenanceAlarmInput(at))),
|
|
174
|
+
ensureScheduledBy: (at) =>
|
|
175
|
+
wrap(
|
|
176
|
+
"advance maintenance checkpoint",
|
|
177
|
+
alarms.scheduleAlarmEarlier(maintenanceAlarmInput(at)),
|
|
178
|
+
),
|
|
179
|
+
scheduleNow: Ref.get(inlineWork).pipe(
|
|
180
|
+
Effect.flatMap((active) =>
|
|
181
|
+
active > 0
|
|
182
|
+
? Effect.void
|
|
183
|
+
: wrap(
|
|
184
|
+
"hint maintenance checkpoint",
|
|
185
|
+
alarms.transaction((tx) =>
|
|
186
|
+
Effect.gen(function* () {
|
|
187
|
+
const now = yield* Clock.currentTimeMillis;
|
|
188
|
+
const next = DueQueue.next(DueQueue.make(ctx.storage).read());
|
|
189
|
+
|
|
190
|
+
if (Number.isFinite(next))
|
|
191
|
+
yield* tx.scheduleAlarmEarlier(maintenanceAlarmInput(Math.max(now, next)));
|
|
192
|
+
}),
|
|
193
|
+
),
|
|
194
|
+
),
|
|
195
|
+
),
|
|
196
|
+
),
|
|
197
|
+
withHintsDeferred: (body) =>
|
|
198
|
+
Effect.acquireUseRelease(
|
|
199
|
+
Ref.update(inlineWork, (active) => active + 1),
|
|
200
|
+
() => body,
|
|
201
|
+
() => Ref.update(inlineWork, (active) => active - 1),
|
|
202
|
+
),
|
|
203
|
+
cancel: wrap("cancel maintenance checkpoint", alarms.cancelAlarm(maintenanceAlarmRef)),
|
|
204
|
+
});
|
|
205
|
+
}),
|
|
206
|
+
);
|
|
207
|
+
}
|
|
176
208
|
|
|
177
|
-
|
|
178
|
-
|
|
209
|
+
type CheckpointIntent = {
|
|
210
|
+
readonly at: (deadline: number) => void;
|
|
211
|
+
readonly earlier: (deadline: number) => void;
|
|
212
|
+
readonly cancel: () => void;
|
|
213
|
+
};
|
|
179
214
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
215
|
+
type MaintenanceStorage = Pick<DurableObjectTransaction, "get" | "put" | "delete">;
|
|
216
|
+
type MaintenanceCheckpoint = {
|
|
217
|
+
readonly mutations: Parameters<
|
|
218
|
+
Parameters<ThreadMaintenanceAlarm["Service"]["transaction"]>[0]
|
|
219
|
+
>[0];
|
|
220
|
+
readonly requireActive: Effect.Effect<void, DurableAlarmError>;
|
|
221
|
+
};
|
|
184
222
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
223
|
+
const CurrentCheckpoint = Context.Reference<MaintenanceCheckpoint | undefined>(
|
|
224
|
+
"@yielded/agent/cloudflare/CurrentCheckpoint",
|
|
225
|
+
{ defaultValue: () => undefined },
|
|
226
|
+
);
|
|
227
|
+
|
|
228
|
+
/** One source transaction owns queue intent and the library checkpoint through commit. */
|
|
229
|
+
const makeAlarmOperation = Effect.gen(function* () {
|
|
230
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
231
|
+
const sql = yield* SqlClient;
|
|
232
|
+
const { ctx } = yield* DurableObjectContext;
|
|
233
|
+
const queue = DueQueue.make(ctx.storage);
|
|
234
|
+
|
|
235
|
+
const transaction: SqlClient["withTransaction"] = (body) =>
|
|
236
|
+
alarms
|
|
237
|
+
.transaction(
|
|
238
|
+
(mutations) =>
|
|
239
|
+
Effect.withFiber((owner) => {
|
|
240
|
+
let active = true;
|
|
241
|
+
|
|
242
|
+
const requireActive = Effect.withFiber((fiber) =>
|
|
243
|
+
active && fiber === owner
|
|
244
|
+
? Effect.void
|
|
245
|
+
: Effect.fail(
|
|
246
|
+
DurableAlarmError.make({
|
|
247
|
+
operation: "enroll maintenance source",
|
|
248
|
+
message: "Source intent requires its active transaction callback's fiber",
|
|
249
|
+
}),
|
|
250
|
+
),
|
|
251
|
+
);
|
|
252
|
+
|
|
253
|
+
return body.pipe(
|
|
254
|
+
Effect.provideService(CurrentCheckpoint, { mutations, requireActive }),
|
|
255
|
+
Effect.ensuring(
|
|
256
|
+
Effect.sync(() => {
|
|
257
|
+
active = false;
|
|
258
|
+
}),
|
|
259
|
+
),
|
|
260
|
+
);
|
|
261
|
+
}),
|
|
262
|
+
{ sqlClient: sql },
|
|
263
|
+
)
|
|
264
|
+
.pipe(
|
|
265
|
+
Effect.mapError((cause) =>
|
|
266
|
+
cause instanceof DurableObjectStorage.StorageOperationError
|
|
267
|
+
? SqlError.make({
|
|
268
|
+
reason: UnknownError.make({
|
|
269
|
+
operation: "commit maintenance checkpoint",
|
|
270
|
+
message: "Source facts and maintenance checkpoint could not commit",
|
|
271
|
+
cause,
|
|
272
|
+
}),
|
|
273
|
+
})
|
|
274
|
+
: cause,
|
|
275
|
+
),
|
|
276
|
+
);
|
|
277
|
+
|
|
278
|
+
const withTransaction = <A, E, R>(body: Effect.Effect<A, E, R>) =>
|
|
279
|
+
Effect.flatMap(Effect.serviceOption(sql.transactionService), (ambient) => {
|
|
280
|
+
if (ambient._tag === "Some" && !DueQueue.ownsTransaction(ctx.storage, ambient.value[1]))
|
|
281
|
+
return Effect.fail(
|
|
282
|
+
SqlError.make({
|
|
283
|
+
reason: UnknownError.make({
|
|
284
|
+
operation: "begin maintenance source transaction",
|
|
285
|
+
message: "Use ThreadMutationGate.withTransaction at the outermost source transaction",
|
|
286
|
+
cause: new Error("Unwrapped SQL transaction cannot own buffered maintenance intent"),
|
|
188
287
|
}),
|
|
189
|
-
)
|
|
288
|
+
}),
|
|
289
|
+
);
|
|
190
290
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
ctx.storage.transaction(async (transaction) => {
|
|
195
|
-
const next = DueQueue.next(DueQueue.make(ctx.storage).read());
|
|
291
|
+
return Effect.uninterruptibleMask((restore) =>
|
|
292
|
+
Effect.suspend(() => {
|
|
293
|
+
let frame: ReturnType<typeof queue.begin> | undefined;
|
|
196
294
|
|
|
197
|
-
|
|
198
|
-
|
|
295
|
+
return transaction(
|
|
296
|
+
Effect.flatMap(Effect.serviceOption(sql.transactionService), (transaction) =>
|
|
297
|
+
Effect.sync(() => {
|
|
298
|
+
if (transaction._tag === "None") throw new Error("SQL transaction unavailable");
|
|
299
|
+
frame = queue.begin(transaction.value[1]);
|
|
199
300
|
}),
|
|
301
|
+
).pipe(
|
|
302
|
+
Effect.andThen(restore(body)),
|
|
303
|
+
Effect.tap(() =>
|
|
304
|
+
Effect.try({
|
|
305
|
+
try: () => {
|
|
306
|
+
if (frame !== undefined) queue.flush(frame);
|
|
307
|
+
},
|
|
308
|
+
catch: (cause) =>
|
|
309
|
+
SqlError.make({
|
|
310
|
+
reason: UnknownError.make({
|
|
311
|
+
cause,
|
|
312
|
+
operation: "flush maintenance due queue",
|
|
313
|
+
message: "Maintenance intent could not commit",
|
|
314
|
+
}),
|
|
315
|
+
}),
|
|
316
|
+
}),
|
|
317
|
+
),
|
|
200
318
|
),
|
|
201
|
-
)
|
|
202
|
-
|
|
319
|
+
).pipe(
|
|
320
|
+
Effect.onExit((exit) =>
|
|
321
|
+
Effect.sync(() => {
|
|
322
|
+
if (frame !== undefined) queue.finish(frame, Exit.isSuccess(exit));
|
|
323
|
+
}),
|
|
324
|
+
),
|
|
325
|
+
// A released permit can resume another transaction. Do not auto-yield
|
|
326
|
+
// between the driver's settlement and removing this transaction's frame.
|
|
327
|
+
Effect.provideService(Scheduler.PreventSchedulerYield, true),
|
|
328
|
+
);
|
|
329
|
+
}),
|
|
330
|
+
);
|
|
331
|
+
}).pipe(Effect.provideService(SqlClient, sql));
|
|
203
332
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
333
|
+
const run = <A>(
|
|
334
|
+
operation: string,
|
|
335
|
+
execute: (transaction: MaintenanceStorage, checkpoint: CheckpointIntent) => Promise<A>,
|
|
336
|
+
) => {
|
|
337
|
+
const commit = Effect.flatMap(CurrentCheckpoint, (tx) => {
|
|
338
|
+
if (tx === undefined)
|
|
339
|
+
return Effect.die(new Error("Maintenance checkpoint transaction unavailable"));
|
|
207
340
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
341
|
+
return tx.requireActive.pipe(
|
|
342
|
+
Effect.andThen(
|
|
343
|
+
Effect.suspend(() => {
|
|
344
|
+
let checkpoint: ReturnType<DurableObjectAlarm.AlarmScheduler["scheduleAlarm"]> =
|
|
345
|
+
Effect.void;
|
|
346
|
+
|
|
347
|
+
const intent: CheckpointIntent = {
|
|
348
|
+
at: (at) => {
|
|
349
|
+
checkpoint = tx.mutations.scheduleAlarm(maintenanceAlarmInput(at));
|
|
350
|
+
},
|
|
351
|
+
earlier: (at) => {
|
|
352
|
+
checkpoint = tx.mutations.scheduleAlarmEarlier(maintenanceAlarmInput(at));
|
|
353
|
+
},
|
|
354
|
+
cancel: () => {
|
|
355
|
+
checkpoint = tx.mutations.cancelAlarm(maintenanceAlarmRef);
|
|
356
|
+
},
|
|
357
|
+
};
|
|
213
358
|
|
|
214
|
-
|
|
359
|
+
// The SQL/alarm callback already owns the native transaction; KV uses the same storage.
|
|
360
|
+
return Effect.tryPromise({
|
|
361
|
+
try: () => execute(ctx.storage, intent),
|
|
362
|
+
catch: alarmFailure(operation),
|
|
363
|
+
}).pipe(Effect.tap(() => checkpoint));
|
|
364
|
+
}),
|
|
365
|
+
),
|
|
366
|
+
);
|
|
367
|
+
});
|
|
215
368
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
369
|
+
return Effect.gen(function* () {
|
|
370
|
+
const current = yield* CurrentCheckpoint;
|
|
371
|
+
|
|
372
|
+
if (current !== undefined) return yield* commit;
|
|
373
|
+
const ambient = yield* Effect.serviceOption(sql.transactionService);
|
|
374
|
+
|
|
375
|
+
// Preserve eager writes and invalidation for an unwrapped host SQL transaction.
|
|
376
|
+
return yield* ambient._tag === "Some" && !DueQueue.buffering(ctx.storage)
|
|
377
|
+
? transaction(commit).pipe(
|
|
378
|
+
Effect.ensuring(Effect.sync(() => DueQueue.invalidate(ctx.storage))),
|
|
379
|
+
)
|
|
380
|
+
: withTransaction(commit);
|
|
381
|
+
}).pipe(Effect.uninterruptible, Effect.mapError(alarmFailure(operation)));
|
|
382
|
+
};
|
|
383
|
+
|
|
384
|
+
return { run, withTransaction };
|
|
385
|
+
});
|
|
227
386
|
|
|
228
|
-
/** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
|
|
229
387
|
export class MaintenancePassReport extends Schema.Class<MaintenancePassReport>(
|
|
230
388
|
"@effect-agent/platform-cloudflare/MaintenancePassReport",
|
|
231
389
|
)({
|
|
@@ -378,6 +536,8 @@ export const ThreadHostMaintenance = Context.Reference<{
|
|
|
378
536
|
/** @internal Framework handlers are separate from host registrations and their namespace. */
|
|
379
537
|
export const ThreadNativeMaintenance = Context.Reference<{
|
|
380
538
|
readonly lanes: ReadonlyArray<ThreadHostMaintenanceLane>;
|
|
539
|
+
/** Exact runtime built over a single-Thread placement; never apply to a replacement. */
|
|
540
|
+
readonly singleThreadRuntime?: DurableAgentRuntime["Service"];
|
|
381
541
|
}>("@effect-agent/platform-cloudflare/ThreadNativeMaintenance", {
|
|
382
542
|
defaultValue: () => ({ lanes: [] }),
|
|
383
543
|
});
|
|
@@ -502,7 +662,7 @@ const readRecoveryRecipients = async (
|
|
|
502
662
|
};
|
|
503
663
|
|
|
504
664
|
const appendRecoveryRecipients = async (
|
|
505
|
-
transaction:
|
|
665
|
+
transaction: MaintenanceStorage,
|
|
506
666
|
threadId: ThreadId,
|
|
507
667
|
count: number,
|
|
508
668
|
added: ReadonlyArray<SubmissionId>,
|
|
@@ -542,7 +702,7 @@ interface NativePassResult {
|
|
|
542
702
|
readonly dispatched?: boolean;
|
|
543
703
|
}
|
|
544
704
|
|
|
545
|
-
/** Independent Threads share
|
|
705
|
+
/** Independent Threads share bounded native slots; one active head per Thread. */
|
|
546
706
|
interface NativeDispatch {
|
|
547
707
|
readonly scope: Scope.Scope;
|
|
548
708
|
readonly active: Map<ThreadId, Fiber.Fiber<number, MaintenancePassFailure>>;
|
|
@@ -557,7 +717,6 @@ interface NativeDispatch {
|
|
|
557
717
|
scanRevision?: number;
|
|
558
718
|
}
|
|
559
719
|
|
|
560
|
-
const nativeDispatchConcurrency = 2;
|
|
561
720
|
/** Post-native delivery shares a finite connection budget across independent host lanes. */
|
|
562
721
|
const afterNativeDispatchConcurrency = 2;
|
|
563
722
|
|
|
@@ -577,8 +736,17 @@ interface NativeRecovery {
|
|
|
577
736
|
needsCheckpoint: boolean;
|
|
578
737
|
recovered: number;
|
|
579
738
|
repaired: boolean;
|
|
739
|
+
discoveryPending: boolean;
|
|
740
|
+
retryAtMillis?: number;
|
|
580
741
|
}
|
|
581
742
|
|
|
743
|
+
const NativeWorkCursor = Schema.Struct({
|
|
744
|
+
threadId: ThreadId,
|
|
745
|
+
cursor: Schema.NonEmptyString.check(Schema.isMaxLength(8192)),
|
|
746
|
+
});
|
|
747
|
+
|
|
748
|
+
const workCursorKey = (threadId: ThreadId) => `effect-agent:work-recovery-cursor:v1:${threadId}`;
|
|
749
|
+
|
|
582
750
|
interface MaintenanceObservation {
|
|
583
751
|
readonly queue: Map<string, DueQueue.DueLane>;
|
|
584
752
|
readonly charged: Map<string, number>;
|
|
@@ -644,17 +812,6 @@ const readMaintenanceState = async (
|
|
|
644
812
|
: { state: decodeMaintenanceState(encoded), initialized: true };
|
|
645
813
|
};
|
|
646
814
|
|
|
647
|
-
const ensureTransactionAlarmBy = async (
|
|
648
|
-
transaction: Pick<DurableObjectTransaction, "getAlarm" | "setAlarm">,
|
|
649
|
-
deadline: number,
|
|
650
|
-
): Promise<void> => {
|
|
651
|
-
const scheduled = await transaction.getAlarm();
|
|
652
|
-
|
|
653
|
-
if (scheduled === null || scheduled > deadline) {
|
|
654
|
-
await transaction.setAlarm(deadline);
|
|
655
|
-
}
|
|
656
|
-
};
|
|
657
|
-
|
|
658
815
|
const stableExternalWait = (
|
|
659
816
|
snapshot: SubmissionWorkItem,
|
|
660
817
|
reports: ReadonlyMap<string, RecoveryReport>,
|
|
@@ -675,9 +832,7 @@ const stableExternalWait = (
|
|
|
675
832
|
case "input-applied":
|
|
676
833
|
case "joining":
|
|
677
834
|
case "ready":
|
|
678
|
-
case "running":
|
|
679
835
|
case "settled":
|
|
680
|
-
case "terminalizing":
|
|
681
836
|
return false;
|
|
682
837
|
}
|
|
683
838
|
};
|
|
@@ -696,18 +851,12 @@ const CurrentMutationLanes = Context.Reference<ReadonlyArray<string>>(
|
|
|
696
851
|
|
|
697
852
|
// Source changes made by this pass are certified after its Attempts join. External
|
|
698
853
|
// producers still advance the dirty generation and fence that certification.
|
|
699
|
-
const CurrentNativeSource = Context.Reference<boolean>(
|
|
700
|
-
"@effect-agent/platform-cloudflare/CurrentNativeSource",
|
|
701
|
-
{
|
|
702
|
-
defaultValue: () => false,
|
|
703
|
-
},
|
|
704
|
-
);
|
|
705
|
-
|
|
706
854
|
export class ThreadMutationGate extends Context.Service<
|
|
707
855
|
ThreadMutationGate,
|
|
708
856
|
{
|
|
709
|
-
/** Commit source facts and their scheduling intent together.
|
|
710
|
-
*
|
|
857
|
+
/** Commit source facts and their scheduling intent together. Mutate from this callback's
|
|
858
|
+
* fiber, or use an explicit child transaction. The flush runs before native commit;
|
|
859
|
+
* failed or interrupted transactions discard their queue view. */
|
|
711
860
|
readonly withTransaction: <A, E, R>(
|
|
712
861
|
body: Effect.Effect<A, E, R>,
|
|
713
862
|
) => Effect.Effect<A, E | SqlError, R>;
|
|
@@ -779,6 +928,7 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
779
928
|
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
780
929
|
|
|
781
930
|
const runTransaction = yield* makeStorageOperation;
|
|
931
|
+
const { run: runAlarmTransaction, withTransaction } = yield* makeAlarmOperation;
|
|
782
932
|
|
|
783
933
|
const validateSourceBoundary = Effect.flatMap(
|
|
784
934
|
Effect.serviceOption(sql.transactionService),
|
|
@@ -800,18 +950,15 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
800
950
|
dueQueue.register(DueQueue.Native);
|
|
801
951
|
});
|
|
802
952
|
|
|
803
|
-
const schedule = Effect.
|
|
953
|
+
const schedule = Effect.fnUntraced(function* (
|
|
804
954
|
id: string,
|
|
805
955
|
dueAt: number,
|
|
806
956
|
progressCursor?: bigint,
|
|
807
957
|
) {
|
|
808
958
|
yield* validateSourceBoundary;
|
|
809
959
|
const now = yield* Clock.currentTimeMillis;
|
|
810
|
-
const current = yield* Effect.serviceOption(sql.transactionService);
|
|
811
960
|
|
|
812
|
-
const enroll = async (
|
|
813
|
-
transaction: Pick<DurableObjectTransaction, "get" | "put" | "getAlarm" | "setAlarm">,
|
|
814
|
-
) => {
|
|
961
|
+
const enroll = async (transaction: MaintenanceStorage, checkpoint: CheckpointIntent) => {
|
|
815
962
|
if (progressCursor === undefined) dueQueue.dirty(id, dueAt);
|
|
816
963
|
else if (
|
|
817
964
|
dueQueue.progress(id, dueAt, progressCursor) &&
|
|
@@ -833,17 +980,14 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
833
980
|
}
|
|
834
981
|
const next = DueQueue.next(dueQueue.read());
|
|
835
982
|
|
|
836
|
-
if (Number.isFinite(next))
|
|
837
|
-
await ensureTransactionAlarmBy(transaction, Math.max(next, now + minimumAlarmDelay));
|
|
983
|
+
if (Number.isFinite(next)) checkpoint.earlier(Math.max(next, now + minimumAlarmDelay));
|
|
838
984
|
};
|
|
839
985
|
|
|
840
|
-
yield*
|
|
841
|
-
Option.isSome(current) ? enroll(ctx.storage) : dueQueue.transaction(enroll),
|
|
842
|
-
);
|
|
986
|
+
yield* runAlarmTransaction("schedule maintenance lane", enroll);
|
|
843
987
|
yield* notify;
|
|
844
988
|
});
|
|
845
989
|
|
|
846
|
-
const beginMutation = Effect.
|
|
990
|
+
const beginMutation = Effect.fnUntraced(function* (
|
|
847
991
|
invalidatesRecovery: boolean,
|
|
848
992
|
lanes: ReadonlyArray<string>,
|
|
849
993
|
) {
|
|
@@ -853,8 +997,9 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
853
997
|
const publicationRequired = yield* requiresPublication;
|
|
854
998
|
|
|
855
999
|
if (invalidatesRecovery || lanes.length > 0)
|
|
856
|
-
yield*
|
|
857
|
-
|
|
1000
|
+
yield* runAlarmTransaction(
|
|
1001
|
+
"advance maintenance generation",
|
|
1002
|
+
async (transaction, checkpoint) => {
|
|
858
1003
|
const { state, initialized } = await readMaintenanceState(transaction);
|
|
859
1004
|
|
|
860
1005
|
if (invalidatesRecovery) {
|
|
@@ -864,23 +1009,23 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
864
1009
|
// Enrollment is a crash fallback, not evidence that the body committed.
|
|
865
1010
|
for (const id of lanes) dueQueue.dirty(id, now, false);
|
|
866
1011
|
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
1012
|
+
if (!initialized || (invalidatesRecovery && !nativeSource))
|
|
1013
|
+
await transaction.put(
|
|
1014
|
+
MAINTENANCE_STATE_KEY,
|
|
1015
|
+
encodeMaintenanceState(
|
|
1016
|
+
ThreadMaintenanceState.make({
|
|
1017
|
+
...state,
|
|
1018
|
+
dirty: state.dirty + (invalidatesRecovery && !nativeSource ? 1n : 0n),
|
|
1019
|
+
}),
|
|
1020
|
+
),
|
|
1021
|
+
);
|
|
874
1022
|
// The earliest configured retry bounds a newly actionable mutation without relying
|
|
875
1023
|
// on its best-effort immediate wake hint.
|
|
876
1024
|
const deadline = DueQueue.next(dueQueue.read());
|
|
877
1025
|
|
|
878
1026
|
if (Number.isFinite(deadline))
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
Math.max(deadline, now + minimumAlarmDelay),
|
|
882
|
-
);
|
|
883
|
-
}),
|
|
1027
|
+
checkpoint.earlier(Math.max(deadline, now + minimumAlarmDelay));
|
|
1028
|
+
},
|
|
884
1029
|
);
|
|
885
1030
|
yield* failpoint.hit("maintenance:dirty:after");
|
|
886
1031
|
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
@@ -960,9 +1105,7 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
960
1105
|
),
|
|
961
1106
|
);
|
|
962
1107
|
|
|
963
|
-
const recordProgress = Effect.
|
|
964
|
-
lanes: ReadonlyArray<string>,
|
|
965
|
-
) {
|
|
1108
|
+
const recordProgress = Effect.fnUntraced(function* (lanes: ReadonlyArray<string>) {
|
|
966
1109
|
yield* validateSourceBoundary;
|
|
967
1110
|
if (Option.isNone(yield* Effect.serviceOption(sql.transactionService)))
|
|
968
1111
|
return yield* DurableAlarmError.make({
|
|
@@ -979,29 +1122,34 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
979
1122
|
const now = yield* Clock.currentTimeMillis;
|
|
980
1123
|
const nativeSource = yield* CurrentNativeSource;
|
|
981
1124
|
|
|
982
|
-
yield*
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
const
|
|
1125
|
+
yield* runAlarmTransaction(
|
|
1126
|
+
"record maintenance producer progress",
|
|
1127
|
+
async (transaction, checkpoint) => {
|
|
1128
|
+
for (const id of affected) dueQueue.dirty(id, now, true);
|
|
1129
|
+
if (affected.has(DueQueue.Native)) {
|
|
1130
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
986
1131
|
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
1132
|
+
// Native progress already belongs to this generation. Persist only
|
|
1133
|
+
// initialization, external progress, or the removal of a retry budget.
|
|
1134
|
+
if (!initialized || !nativeSource || state.retry !== undefined)
|
|
1135
|
+
await transaction.put(
|
|
1136
|
+
MAINTENANCE_STATE_KEY,
|
|
1137
|
+
encodeMaintenanceState(
|
|
1138
|
+
ThreadMaintenanceState.make({
|
|
1139
|
+
...Struct.omit(state, ["retry"]),
|
|
1140
|
+
dirty: state.dirty + (nativeSource ? 0n : 1n),
|
|
1141
|
+
}),
|
|
1142
|
+
),
|
|
1143
|
+
);
|
|
1144
|
+
}
|
|
1145
|
+
checkpoint.earlier(now + minimumAlarmDelay);
|
|
1146
|
+
},
|
|
1147
|
+
);
|
|
999
1148
|
yield* notify;
|
|
1000
1149
|
});
|
|
1001
1150
|
|
|
1002
1151
|
return ThreadMutationGate.of({
|
|
1003
|
-
withTransaction
|
|
1004
|
-
dueQueue.withTransaction(body).pipe(Effect.provideService(SqlClient, sql)),
|
|
1152
|
+
withTransaction,
|
|
1005
1153
|
withMutation,
|
|
1006
1154
|
schedule,
|
|
1007
1155
|
recordProgress,
|
|
@@ -1032,7 +1180,8 @@ export type MaintenancePassFailure =
|
|
|
1032
1180
|
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
1033
1181
|
* producer keeps its generation dirty and immediately eligible. Quiescent native retries
|
|
1034
1182
|
* retain their durable backoff.
|
|
1035
|
-
* 3. Admit
|
|
1183
|
+
* 3. Admit one Attempt for a single-Thread runtime, or at most two independent Thread
|
|
1184
|
+
* Attempts for a shared host, with one active head per Thread.
|
|
1036
1185
|
* Keep native and delivery admission open together while finite waves remain active, so
|
|
1037
1186
|
* fresh replies and abort controls can progress during unrelated cleanup. Close atomically
|
|
1038
1187
|
* at quiescence, or at the original ten-minute yield deadline, then join admitted waves.
|
|
@@ -1073,6 +1222,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1073
1222
|
| SubmissionLedger
|
|
1074
1223
|
| WakeScheduler
|
|
1075
1224
|
| DurableAlarmService
|
|
1225
|
+
| ThreadMaintenanceAlarm
|
|
1076
1226
|
| ThreadMaintenanceFailpoint
|
|
1077
1227
|
| CloudflareDurableRuntimeConfig
|
|
1078
1228
|
| DurableObjectContext
|
|
@@ -1083,7 +1233,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1083
1233
|
const recoveryConfig = yield* DurableRuntimeConfig;
|
|
1084
1234
|
const ledger = yield* SubmissionLedger;
|
|
1085
1235
|
const wakes = yield* WakeScheduler;
|
|
1086
|
-
const
|
|
1236
|
+
const hints = yield* DurableAlarmService;
|
|
1237
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
1087
1238
|
const config = yield* CloudflareDurableRuntimeConfig;
|
|
1088
1239
|
const { ctx } = yield* DurableObjectContext;
|
|
1089
1240
|
const dueQueue = DueQueue.make(ctx.storage);
|
|
@@ -1103,6 +1254,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1103
1254
|
const host = yield* ThreadHostMaintenance;
|
|
1104
1255
|
const framework = yield* ThreadNativeMaintenance;
|
|
1105
1256
|
|
|
1257
|
+
const nativeDispatchConcurrency = () => (framework.singleThreadRuntime === runtime ? 1 : 2);
|
|
1258
|
+
|
|
1106
1259
|
for (const lane of host.lanes)
|
|
1107
1260
|
yield* Schema.decodeEffect(DueQueue.HostLaneId)(lane.id).pipe(
|
|
1108
1261
|
Effect.mapError(alarmFailure("maintenance lane ID")),
|
|
@@ -1121,6 +1274,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1121
1274
|
const maintenancePassGate = yield* Semaphore.make(1);
|
|
1122
1275
|
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
1123
1276
|
const runTransaction = yield* makeStorageOperation;
|
|
1277
|
+
const { run: runAlarmTransaction } = yield* makeAlarmOperation;
|
|
1124
1278
|
|
|
1125
1279
|
const queueSnapshot = runTransaction("read maintenance due queue", async () =>
|
|
1126
1280
|
dueQueue.read(),
|
|
@@ -1192,17 +1346,17 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1192
1346
|
return Effect.gen(function* () {
|
|
1193
1347
|
const now = yield* Clock.currentTimeMillis;
|
|
1194
1348
|
|
|
1195
|
-
const claimed = yield*
|
|
1196
|
-
|
|
1349
|
+
const claimed = yield* runAlarmTransaction(
|
|
1350
|
+
"charge maintenance attempt",
|
|
1351
|
+
async (_transaction, checkpoint) => {
|
|
1197
1352
|
const claimed = dueQueue.claim(selected, now);
|
|
1198
1353
|
const next = DueQueue.next(dueQueue.read());
|
|
1199
1354
|
|
|
1200
|
-
if (Number.isFinite(next))
|
|
1201
|
-
|
|
1202
|
-
else await transaction.deleteAlarm();
|
|
1355
|
+
if (Number.isFinite(next)) checkpoint.at(Math.max(now + minimumAlarmDelay, next));
|
|
1356
|
+
else checkpoint.cancel();
|
|
1203
1357
|
|
|
1204
1358
|
return claimed;
|
|
1205
|
-
}
|
|
1359
|
+
},
|
|
1206
1360
|
);
|
|
1207
1361
|
|
|
1208
1362
|
if (claimed === undefined) return Option.none<number>();
|
|
@@ -1270,7 +1424,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1270
1424
|
};
|
|
1271
1425
|
|
|
1272
1426
|
const appendRecoveryEvents = async (
|
|
1273
|
-
transaction:
|
|
1427
|
+
transaction: MaintenanceStorage,
|
|
1274
1428
|
events: ReadonlyArray<Omit<ThreadRecoveryFaultEvent, "sequence">>,
|
|
1275
1429
|
) => {
|
|
1276
1430
|
if (events.length === 0) return;
|
|
@@ -1353,14 +1507,17 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1353
1507
|
}),
|
|
1354
1508
|
};
|
|
1355
1509
|
|
|
1356
|
-
const recordRecoveryFaults = Effect.
|
|
1510
|
+
const recordRecoveryFaults = Effect.fnUntraced(function* (
|
|
1357
1511
|
result: RecoverySweepResult,
|
|
1358
1512
|
recovery: NativeRecovery,
|
|
1359
1513
|
current?: ReadonlyArray<SubmissionWorkItem>,
|
|
1514
|
+
selectedThread?: ThreadId,
|
|
1360
1515
|
) {
|
|
1361
1516
|
const threads = new Map<ThreadId, RecoveryFailure | undefined>();
|
|
1362
1517
|
|
|
1518
|
+
if (selectedThread !== undefined) threads.set(selectedThread, undefined);
|
|
1363
1519
|
for (const report of result.reports) threads.set(report.threadId, undefined);
|
|
1520
|
+
for (const report of result.workReports ?? []) threads.set(report.threadId, undefined);
|
|
1364
1521
|
for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
|
|
1365
1522
|
if (threads.size === 0) return new Map<ThreadId, ThreadRecoveryFault>();
|
|
1366
1523
|
const now = yield* Clock.currentTimeMillis;
|
|
@@ -1462,27 +1619,59 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1462
1619
|
return retained.faults;
|
|
1463
1620
|
});
|
|
1464
1621
|
|
|
1465
|
-
const recoverThread = Effect.
|
|
1622
|
+
const recoverThread = Effect.fnUntraced(function* (
|
|
1466
1623
|
threadId: ThreadId,
|
|
1467
1624
|
recovery: NativeRecovery,
|
|
1468
1625
|
) {
|
|
1469
|
-
|
|
1470
|
-
.
|
|
1471
|
-
|
|
1626
|
+
let cursor = yield* runTransaction("read work recovery cursor", async () => {
|
|
1627
|
+
const retained = await ctx.storage.get(workCursorKey(threadId));
|
|
1628
|
+
|
|
1629
|
+
if (retained === undefined) return undefined;
|
|
1630
|
+
const decoded = Schema.decodeUnknownSync(NativeWorkCursor)(retained);
|
|
1472
1631
|
|
|
1473
|
-
|
|
1474
|
-
|
|
1632
|
+
if (decoded.threadId !== threadId)
|
|
1633
|
+
throw new Error("Work cursor belongs to another Thread");
|
|
1475
1634
|
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1635
|
+
return decoded.cursor;
|
|
1636
|
+
});
|
|
1637
|
+
|
|
1638
|
+
do {
|
|
1639
|
+
const result = yield* runtime
|
|
1640
|
+
.runRecovery({ threadId, ...(cursor === undefined ? {} : { cursor }) })
|
|
1641
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
1642
|
+
|
|
1643
|
+
// Persist each bounded pass before a claim; eviction resumes past consumed metadata.
|
|
1644
|
+
const faults = yield* recordRecoveryFaults(result, recovery, undefined, threadId);
|
|
1645
|
+
|
|
1646
|
+
yield* runTransaction("checkpoint work recovery cursor", async () => {
|
|
1647
|
+
if (result.cursor === undefined) await ctx.storage.delete(workCursorKey(threadId));
|
|
1648
|
+
else
|
|
1649
|
+
await ctx.storage.put(
|
|
1650
|
+
workCursorKey(threadId),
|
|
1651
|
+
Schema.encodeSync(NativeWorkCursor)({ threadId, cursor: result.cursor }),
|
|
1652
|
+
);
|
|
1653
|
+
});
|
|
1654
|
+
for (const report of result.reports) recovery.reports.set(report.submissionId, report);
|
|
1655
|
+
recovery.faults.delete(threadId);
|
|
1656
|
+
for (const [id, fault] of faults) recovery.faults.set(id, fault);
|
|
1657
|
+
recovery.recovered +=
|
|
1658
|
+
(result.workReports?.length ?? result.reports.length) + result.blocked.length;
|
|
1659
|
+
recovery.repaired ||= (result.workReports ?? result.reports).some(
|
|
1660
|
+
(report) => report.disposition === "repaired",
|
|
1661
|
+
);
|
|
1662
|
+
for (const report of result.workReports ?? [])
|
|
1663
|
+
if (report.retryAtMillis !== undefined)
|
|
1664
|
+
recovery.retryAtMillis = Math.min(
|
|
1665
|
+
recovery.retryAtMillis ?? Infinity,
|
|
1666
|
+
report.retryAtMillis,
|
|
1667
|
+
);
|
|
1668
|
+
cursor = result.blocked.length > 0 ? undefined : result.cursor;
|
|
1669
|
+
} while (cursor !== undefined);
|
|
1481
1670
|
});
|
|
1482
1671
|
|
|
1483
1672
|
// Registry changes, including upgrades from timed binding retries, create one native
|
|
1484
1673
|
// opportunity. Unchanged registries leave parked work dormant across Object eviction.
|
|
1485
|
-
const readCurrentMaintenanceState = async (transaction:
|
|
1674
|
+
const readCurrentMaintenanceState = async (transaction: MaintenanceStorage) => {
|
|
1486
1675
|
const current = await readMaintenanceState(transaction);
|
|
1487
1676
|
|
|
1488
1677
|
if (current.state.bindingRegistryKey === runtime.bindingRegistryKey) return current;
|
|
@@ -1500,48 +1689,43 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1500
1689
|
return { state, initialized: true };
|
|
1501
1690
|
};
|
|
1502
1691
|
|
|
1503
|
-
const ensureAlarm = Effect.
|
|
1692
|
+
const ensureAlarm = Effect.fnUntraced(function* () {
|
|
1504
1693
|
yield* failpoint.hit("maintenance:ensure:before");
|
|
1505
1694
|
const now = yield* Clock.currentTimeMillis;
|
|
1506
1695
|
|
|
1507
|
-
yield*
|
|
1508
|
-
|
|
1509
|
-
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1696
|
+
yield* runAlarmTransaction("ensure maintenance alarm", async (transaction, checkpoint) => {
|
|
1697
|
+
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1510
1698
|
|
|
1511
|
-
|
|
1512
|
-
|
|
1699
|
+
recoveryEventsPending = hasRecoveryEvents(state);
|
|
1700
|
+
const rows = dueQueue.read();
|
|
1513
1701
|
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1702
|
+
if (recoveryEventsPending && !rows.some((row) => row.id === DueQueue.RecoveryEvents))
|
|
1703
|
+
dueQueue.dirty(DueQueue.RecoveryEvents, now);
|
|
1704
|
+
if (!initialized)
|
|
1705
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
1518
1706
|
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
: null;
|
|
1707
|
+
const native =
|
|
1708
|
+
state.dirty > state.processed
|
|
1709
|
+
? state.retry?.generation === state.dirty
|
|
1710
|
+
? state.retry.notBefore
|
|
1711
|
+
: (rows.find((row) => row.id === DueQueue.Native)?.dueAt ?? now + minimumAlarmDelay)
|
|
1712
|
+
: null;
|
|
1526
1713
|
|
|
1527
|
-
|
|
1528
|
-
|
|
1714
|
+
dueQueue.checkpointNative(native);
|
|
1715
|
+
const next = DueQueue.next(dueQueue.read());
|
|
1529
1716
|
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
}),
|
|
1533
|
-
);
|
|
1717
|
+
if (Number.isFinite(next)) checkpoint.earlier(Math.max(now + minimumAlarmDelay, next));
|
|
1718
|
+
});
|
|
1534
1719
|
yield* failpoint.hit("maintenance:ensure:after");
|
|
1535
1720
|
});
|
|
1536
1721
|
|
|
1537
|
-
const beginPass = Effect.
|
|
1538
|
-
observed: MaintenanceObservation,
|
|
1539
|
-
) {
|
|
1722
|
+
const beginPass = Effect.fnUntraced(function* (observed: MaintenanceObservation) {
|
|
1540
1723
|
yield* failpoint.hit("maintenance:begin:before");
|
|
1541
1724
|
const now = yield* Clock.currentTimeMillis;
|
|
1542
1725
|
|
|
1543
|
-
const result = yield*
|
|
1544
|
-
|
|
1726
|
+
const result = yield* runAlarmTransaction(
|
|
1727
|
+
"begin maintenance pass",
|
|
1728
|
+
async (transaction, checkpoint) => {
|
|
1545
1729
|
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1546
1730
|
|
|
1547
1731
|
recoveryEventsPending = hasRecoveryEvents(state);
|
|
@@ -1582,8 +1766,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1582
1766
|
const deadline = DueQueue.next(dueQueue.read());
|
|
1583
1767
|
|
|
1584
1768
|
if (Number.isFinite(deadline))
|
|
1585
|
-
|
|
1586
|
-
else
|
|
1769
|
+
checkpoint.at(Math.max(now + minimumAlarmDelay, deadline));
|
|
1770
|
+
else checkpoint.cancel();
|
|
1587
1771
|
|
|
1588
1772
|
return {
|
|
1589
1773
|
_tag: "Actionable" as const,
|
|
@@ -1592,7 +1776,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1592
1776
|
nonterminal: state.nonterminal,
|
|
1593
1777
|
stalls: charged.stalls,
|
|
1594
1778
|
};
|
|
1595
|
-
}
|
|
1779
|
+
},
|
|
1596
1780
|
);
|
|
1597
1781
|
|
|
1598
1782
|
yield* failpoint.hit("maintenance:begin:after");
|
|
@@ -1610,16 +1794,11 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1610
1794
|
return Math.ceil(backoff / 2 + (backoff / 2) * jitter);
|
|
1611
1795
|
};
|
|
1612
1796
|
|
|
1613
|
-
const rearmDelay = Effect.
|
|
1614
|
-
progressed: boolean,
|
|
1615
|
-
priorStalls: number,
|
|
1616
|
-
) {
|
|
1797
|
+
const rearmDelay = Effect.fnUntraced(function* (progressed: boolean, priorStalls: number) {
|
|
1617
1798
|
return progressed ? config.alarmBackoffBase : backoffDelay(priorStalls, yield* Random.next);
|
|
1618
1799
|
});
|
|
1619
1800
|
|
|
1620
|
-
const rearmFailure = Effect.
|
|
1621
|
-
observed: MaintenanceObservation,
|
|
1622
|
-
) {
|
|
1801
|
+
const rearmFailure = Effect.fnUntraced(function* (observed: MaintenanceObservation) {
|
|
1623
1802
|
const { generation, nativeOnly } = observed;
|
|
1624
1803
|
|
|
1625
1804
|
if (generation === undefined) return;
|
|
@@ -1630,8 +1809,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1630
1809
|
|
|
1631
1810
|
const jitter = yield* Random.next;
|
|
1632
1811
|
|
|
1633
|
-
yield*
|
|
1634
|
-
|
|
1812
|
+
yield* runAlarmTransaction(
|
|
1813
|
+
"back off failed maintenance",
|
|
1814
|
+
async (transaction, checkpoint) => {
|
|
1635
1815
|
const { state } = await readMaintenanceState(transaction);
|
|
1636
1816
|
|
|
1637
1817
|
const previous =
|
|
@@ -1682,20 +1862,17 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1682
1862
|
}
|
|
1683
1863
|
const next = DueQueue.next(dueQueue.read());
|
|
1684
1864
|
|
|
1685
|
-
if (Number.isFinite(next))
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
}),
|
|
1865
|
+
if (Number.isFinite(next)) checkpoint.at(Math.max(now + minimumAlarmDelay, next));
|
|
1866
|
+
else checkpoint.cancel();
|
|
1867
|
+
},
|
|
1689
1868
|
);
|
|
1690
1869
|
}),
|
|
1691
1870
|
);
|
|
1692
1871
|
yield* failpoint.hit("maintenance:retry:after");
|
|
1693
1872
|
});
|
|
1694
1873
|
|
|
1695
|
-
const beginNative =
|
|
1696
|
-
|
|
1697
|
-
) {
|
|
1698
|
-
return yield* mutations.withSnapshot((activeAtStart) =>
|
|
1874
|
+
const beginNative = (observed: MaintenanceObservation) =>
|
|
1875
|
+
mutations.withSnapshot((activeAtStart) =>
|
|
1699
1876
|
Effect.gen(function* () {
|
|
1700
1877
|
const generation = yield* beginPass(observed);
|
|
1701
1878
|
|
|
@@ -1708,9 +1885,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1708
1885
|
return { ...generation, activeAtStart };
|
|
1709
1886
|
}),
|
|
1710
1887
|
);
|
|
1711
|
-
});
|
|
1712
1888
|
|
|
1713
|
-
const attemptSelected = Effect.
|
|
1889
|
+
const attemptSelected = Effect.fnUntraced(function* (
|
|
1714
1890
|
selected: SubmissionWorkItem,
|
|
1715
1891
|
previousWaits: ReadonlyArray<BindingWait>,
|
|
1716
1892
|
yieldAfter: DateTime.Utc,
|
|
@@ -1798,7 +1974,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1798
1974
|
return Option.isSome(settlement) ? 1 : 0;
|
|
1799
1975
|
});
|
|
1800
1976
|
|
|
1801
|
-
const advance = Effect.
|
|
1977
|
+
const advance = Effect.fnUntraced(function* (
|
|
1802
1978
|
started: Effect.Success<ReturnType<typeof beginNative>>,
|
|
1803
1979
|
yieldAfter: DateTime.Utc,
|
|
1804
1980
|
observed: MaintenanceObservation,
|
|
@@ -1896,7 +2072,32 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1896
2072
|
activeAtStart: started.activeAtStart,
|
|
1897
2073
|
});
|
|
1898
2074
|
|
|
2075
|
+
const { scannedGeneration, afterWorkThread } = yield* runTransaction(
|
|
2076
|
+
"read work discovery position",
|
|
2077
|
+
async () => {
|
|
2078
|
+
const { state } = await readMaintenanceState(ctx.storage);
|
|
2079
|
+
|
|
2080
|
+
return {
|
|
2081
|
+
scannedGeneration: state.dirty,
|
|
2082
|
+
afterWorkThread: state.lastRecoveredThreadId,
|
|
2083
|
+
};
|
|
2084
|
+
},
|
|
2085
|
+
);
|
|
2086
|
+
|
|
1899
2087
|
const current = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
2088
|
+
|
|
2089
|
+
let workOwners = yield* runtime
|
|
2090
|
+
.discoverWorkThreads({
|
|
2091
|
+
limit: 32,
|
|
2092
|
+
...(afterWorkThread === undefined ? {} : { afterThreadId: afterWorkThread }),
|
|
2093
|
+
})
|
|
2094
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
2095
|
+
|
|
2096
|
+
if (workOwners.threadIds.length === 0 && afterWorkThread !== undefined)
|
|
2097
|
+
workOwners = yield* runtime
|
|
2098
|
+
.discoverWorkThreads({ limit: 32 })
|
|
2099
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
2100
|
+
recovery.discoveryPending ||= workOwners.afterThreadId !== undefined;
|
|
1900
2101
|
const selectionTime = yield* Clock.currentTimeMillis;
|
|
1901
2102
|
const submissionsByThread = new Map<ThreadId, Array<SubmissionId>>();
|
|
1902
2103
|
|
|
@@ -1906,6 +2107,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1906
2107
|
ids.push(row.submissionId);
|
|
1907
2108
|
submissionsByThread.set(row.threadId, ids);
|
|
1908
2109
|
}
|
|
2110
|
+
// Canonical obligations survive their last Submission. Reload their retained backoff
|
|
2111
|
+
// before selecting recovery, including after eviction or a newer host mutation.
|
|
2112
|
+
for (const threadId of workOwners.threadIds)
|
|
2113
|
+
if (!submissionsByThread.has(threadId)) submissionsByThread.set(threadId, []);
|
|
1909
2114
|
|
|
1910
2115
|
yield* runTransaction("read Thread recovery deadlines", async () => {
|
|
1911
2116
|
for (const [threadId, ids] of submissionsByThread) {
|
|
@@ -2064,7 +2269,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2064
2269
|
const selected = dispatch
|
|
2065
2270
|
? [...runnable.slice(pivot), ...runnable.slice(0, pivot)].slice(
|
|
2066
2271
|
0,
|
|
2067
|
-
nativeDispatchConcurrency - reserved.size,
|
|
2272
|
+
nativeDispatchConcurrency() - reserved.size,
|
|
2068
2273
|
)
|
|
2069
2274
|
: [];
|
|
2070
2275
|
|
|
@@ -2081,7 +2286,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2081
2286
|
|
|
2082
2287
|
const backlog = recovery.started
|
|
2083
2288
|
? []
|
|
2084
|
-
: [...heads.keys()].filter((threadId) => {
|
|
2289
|
+
: [...new Set([...heads.keys(), ...workOwners.threadIds])].filter((threadId) => {
|
|
2085
2290
|
const fault = recoveryFaults.get(threadId);
|
|
2086
2291
|
|
|
2087
2292
|
return (
|
|
@@ -2103,7 +2308,12 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2103
2308
|
threadId <= state.lastRecoveredThreadId,
|
|
2104
2309
|
);
|
|
2105
2310
|
|
|
2106
|
-
return {
|
|
2311
|
+
return {
|
|
2312
|
+
selected,
|
|
2313
|
+
waits,
|
|
2314
|
+
backlog: [...after, ...before].slice(0, 32),
|
|
2315
|
+
discoveryPending: backlog.length > 32,
|
|
2316
|
+
};
|
|
2107
2317
|
}),
|
|
2108
2318
|
);
|
|
2109
2319
|
|
|
@@ -2111,6 +2321,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2111
2321
|
if (!recovery.started) {
|
|
2112
2322
|
recovery.started = true;
|
|
2113
2323
|
recovery.needsCheckpoint = selection.backlog.length > 0;
|
|
2324
|
+
recovery.discoveryPending ||= selection.discoveryPending;
|
|
2114
2325
|
for (const threadId of selection.backlog) recovery.pending.add(threadId);
|
|
2115
2326
|
yield* Deferred.succeed(recovery.queue, selection.backlog);
|
|
2116
2327
|
}
|
|
@@ -2142,30 +2353,32 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2142
2353
|
};
|
|
2143
2354
|
}
|
|
2144
2355
|
|
|
2145
|
-
const scannedGeneration = yield* runTransaction(
|
|
2146
|
-
"observe native source generation",
|
|
2147
|
-
async () => (await readMaintenanceState(ctx.storage)).state.dirty,
|
|
2148
|
-
);
|
|
2149
|
-
|
|
2150
2356
|
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
2151
2357
|
const waitingHeads = new Map<ThreadId, boolean>();
|
|
2152
2358
|
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2359
|
+
// Selected recovery must finish before its generation can be acknowledged: terminal
|
|
2360
|
+
// owners can supply a retry deadline only after their evidence read completes.
|
|
2361
|
+
const autonomous =
|
|
2362
|
+
recovery.pending.size > 0 ||
|
|
2363
|
+
recovery.discoveryPending ||
|
|
2364
|
+
recoveryFaults.size > 0 ||
|
|
2365
|
+
recovery.retryAtMillis !== undefined ||
|
|
2366
|
+
remaining.some((snapshot) => {
|
|
2367
|
+
if (snapshot.state === "unknown" && waiting(snapshot)) return false;
|
|
2368
|
+
const headWaiting = waitingHeads.get(snapshot.threadId);
|
|
2369
|
+
|
|
2370
|
+
if (headWaiting === undefined) waitingHeads.set(snapshot.threadId, waiting(snapshot));
|
|
2371
|
+
// FIFO followers cannot execute through a stable external wait. Only plain queued
|
|
2372
|
+
// input is dormant here; admission repairs and accepted aborts still need a pass.
|
|
2373
|
+
if (
|
|
2374
|
+
headWaiting === true &&
|
|
2375
|
+
snapshot.state === "ready" &&
|
|
2376
|
+
reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput"
|
|
2377
|
+
)
|
|
2378
|
+
return false;
|
|
2166
2379
|
|
|
2167
|
-
|
|
2168
|
-
|
|
2380
|
+
return !waiting(snapshot);
|
|
2381
|
+
});
|
|
2169
2382
|
|
|
2170
2383
|
const progressed = native.progressed || recovery.repaired;
|
|
2171
2384
|
|
|
@@ -2180,6 +2393,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2180
2393
|
.map(() => now),
|
|
2181
2394
|
);
|
|
2182
2395
|
|
|
2396
|
+
if (recovery.retryAtMillis !== undefined) nextEligible.push(recovery.retryAtMillis);
|
|
2397
|
+
|
|
2183
2398
|
const retryDelay =
|
|
2184
2399
|
nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
|
|
2185
2400
|
|
|
@@ -2197,10 +2412,14 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2197
2412
|
const mutationOverlap =
|
|
2198
2413
|
observation.activeAtStart > 0 || started.activeAtStart > 0 || active > 0;
|
|
2199
2414
|
|
|
2200
|
-
//
|
|
2201
|
-
//
|
|
2415
|
+
// Only a scan without ledger work or canonical obligations can certify a newer
|
|
2416
|
+
// generation without another recovery wave. Read its generation before discovery
|
|
2417
|
+
// so progress committed during the scan retains its own recovery opportunity.
|
|
2202
2418
|
const generation =
|
|
2203
|
-
remaining.length === 0 &&
|
|
2419
|
+
remaining.length === 0 &&
|
|
2420
|
+
workOwners.threadIds.length === 0 &&
|
|
2421
|
+
!mutationOverlap &&
|
|
2422
|
+
state.dirty === scannedGeneration
|
|
2204
2423
|
? scannedGeneration
|
|
2205
2424
|
: observation.generation;
|
|
2206
2425
|
|
|
@@ -2250,9 +2469,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2250
2469
|
);
|
|
2251
2470
|
|
|
2252
2471
|
yield* failpoint.hit("maintenance:checkpoint:after");
|
|
2253
|
-
//
|
|
2254
|
-
//
|
|
2255
|
-
if (remaining.length === 0 && recovery.pending.size === 0)
|
|
2472
|
+
// A fresh empty scan may clear earlier producer overlap, but this pass has only
|
|
2473
|
+
// recovered its original generation. Retain that boundary for later canonical work.
|
|
2474
|
+
if (remaining.length === 0 && recovery.pending.size === 0)
|
|
2475
|
+
recovery.observation = { generation: observation.generation, activeAtStart: 0 };
|
|
2256
2476
|
native.deferred.clear();
|
|
2257
2477
|
native.progressed = false;
|
|
2258
2478
|
native.needsCheckpoint = false;
|
|
@@ -2264,7 +2484,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2264
2484
|
};
|
|
2265
2485
|
});
|
|
2266
2486
|
|
|
2267
|
-
const validateLanes = Effect.
|
|
2487
|
+
const validateLanes = Effect.fnUntraced(function* (
|
|
2268
2488
|
lanes: ReadonlyArray<ThreadHostMaintenanceLane>,
|
|
2269
2489
|
queued: ReadonlyArray<DueQueue.DueLane>,
|
|
2270
2490
|
) {
|
|
@@ -2295,7 +2515,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2295
2515
|
}
|
|
2296
2516
|
});
|
|
2297
2517
|
|
|
2298
|
-
const dispatch = Effect.
|
|
2518
|
+
const dispatch = Effect.fnUntraced(function* (
|
|
2299
2519
|
yieldAfter: DateTime.Utc,
|
|
2300
2520
|
dispatchUntil: DateTime.Utc,
|
|
2301
2521
|
observed: MaintenanceObservation,
|
|
@@ -2354,6 +2574,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2354
2574
|
needsCheckpoint: false,
|
|
2355
2575
|
recovered: 0,
|
|
2356
2576
|
repaired: false,
|
|
2577
|
+
discoveryPending: false,
|
|
2357
2578
|
};
|
|
2358
2579
|
|
|
2359
2580
|
const recoveryFiber = yield* fork(
|
|
@@ -2438,7 +2659,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2438
2659
|
native = undefined;
|
|
2439
2660
|
if (result.phase === "actionable") phase = "actionable";
|
|
2440
2661
|
observed.nativeOnly = dispatch.active.size > 0;
|
|
2441
|
-
if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency) {
|
|
2662
|
+
if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency()) {
|
|
2442
2663
|
dispatch.scanGeneration = undefined;
|
|
2443
2664
|
nativeCheck = true;
|
|
2444
2665
|
}
|
|
@@ -2589,7 +2810,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2589
2810
|
if (
|
|
2590
2811
|
native === undefined &&
|
|
2591
2812
|
nativeCheck &&
|
|
2592
|
-
dispatch.active.size < nativeDispatchConcurrency
|
|
2813
|
+
dispatch.active.size < nativeDispatchConcurrency()
|
|
2593
2814
|
) {
|
|
2594
2815
|
nativeCheck = false;
|
|
2595
2816
|
const nativeCheckpoint = dispatch.needsCheckpoint && dispatch.active.size === 0;
|
|
@@ -2692,7 +2913,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2692
2913
|
return { result, phase, recovered: recovery.recovered, settled: dispatch.settled };
|
|
2693
2914
|
});
|
|
2694
2915
|
|
|
2695
|
-
const pass = Effect.
|
|
2916
|
+
const pass = Effect.fnUntraced(function* (
|
|
2696
2917
|
yieldAfter: DateTime.Utc,
|
|
2697
2918
|
dispatchUntil: DateTime.Utc,
|
|
2698
2919
|
observed: MaintenanceObservation,
|
|
@@ -2856,8 +3077,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2856
3077
|
Effect.gen(function* () {
|
|
2857
3078
|
const now = yield* Clock.currentTimeMillis;
|
|
2858
3079
|
|
|
2859
|
-
return yield*
|
|
2860
|
-
|
|
3080
|
+
return yield* runAlarmTransaction(
|
|
3081
|
+
"finish maintenance event",
|
|
3082
|
+
async (transaction, checkpoint) => {
|
|
2861
3083
|
const { state } = await readMaintenanceState(transaction);
|
|
2862
3084
|
|
|
2863
3085
|
const native =
|
|
@@ -2873,14 +3095,14 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2873
3095
|
const next = DueQueue.next(dueQueue.read());
|
|
2874
3096
|
|
|
2875
3097
|
if (Number.isFinite(next)) {
|
|
2876
|
-
|
|
3098
|
+
checkpoint.at(Math.max(now + minimumAlarmDelay, next));
|
|
2877
3099
|
|
|
2878
3100
|
return "rearmed" as const;
|
|
2879
3101
|
}
|
|
2880
|
-
|
|
3102
|
+
checkpoint.cancel();
|
|
2881
3103
|
|
|
2882
3104
|
return "cleared" as const;
|
|
2883
|
-
}
|
|
3105
|
+
},
|
|
2884
3106
|
);
|
|
2885
3107
|
}),
|
|
2886
3108
|
);
|
|
@@ -2920,18 +3142,27 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2920
3142
|
available: new Map(),
|
|
2921
3143
|
};
|
|
2922
3144
|
|
|
2923
|
-
return yield*
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
(
|
|
2930
|
-
|
|
3145
|
+
return yield* alarms
|
|
3146
|
+
.deferWakes(
|
|
3147
|
+
maintenancePassGate.withPermit(
|
|
3148
|
+
dueQueue.withView(Effect.scoped(pass(yieldAfter, dispatchUntil, observed))).pipe(
|
|
3149
|
+
// Close event-owned auxiliary work and release Attempt ownership before
|
|
3150
|
+
// failure rearming, while still holding the pass permit.
|
|
3151
|
+
Effect.onErrorIf(
|
|
3152
|
+
() => true,
|
|
3153
|
+
() => rearmFailure(observed),
|
|
3154
|
+
),
|
|
2931
3155
|
),
|
|
2932
3156
|
),
|
|
2933
|
-
)
|
|
2934
|
-
|
|
3157
|
+
)
|
|
3158
|
+
.pipe(
|
|
3159
|
+
hints.withHintsDeferred,
|
|
3160
|
+
Effect.catchTags({
|
|
3161
|
+
StorageOperationError: (error) => alarmFailure("defer maintenance wakes")(error),
|
|
3162
|
+
InvalidScheduleConfigurationError: (error) =>
|
|
3163
|
+
alarmFailure("defer maintenance wakes")(error),
|
|
3164
|
+
}),
|
|
3165
|
+
);
|
|
2935
3166
|
}).pipe(
|
|
2936
3167
|
// Include permit waiting, recovery and acknowledgement in the event deadline.
|
|
2937
3168
|
// Interruption releases Attempt ownership, leaving the prearmed dirty generation
|