@yielded/agent-platform-cloudflare 0.1.0-beta.174 → 0.1.0-beta.176
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{Alarm-4J9B8Fg8.mjs → Alarm-7uipp6LW.mjs} +155 -117
- package/dist/Alarm-7uipp6LW.mjs.map +1 -0
- package/dist/Alarm.d.mts +2 -0
- package/dist/Alarm.mjs +1 -1
- package/dist/CloudflareScheduling.mjs +2 -2
- package/dist/CloudflareSubscriptions.mjs +2 -2
- package/dist/{CloudflareThreadClient-lTm6baDP.d.mts → CloudflareThreadClient-HO_3rkM7.d.mts} +362 -362
- package/dist/{CloudflareThreadClient-DU548EEj.mjs → CloudflareThreadClient-bA3lGA_2.mjs} +4 -4
- package/dist/CloudflareThreadClient-bA3lGA_2.mjs.map +1 -0
- package/dist/CloudflareThreadClient.d.mts +1 -1
- package/dist/CloudflareThreadClient.mjs +1 -1
- package/dist/{ThreadObject-BLYgzcdT.d.mts → ThreadObject-CxNlhoPL.d.mts} +13 -13
- package/dist/{ThreadObject-C2LF3btz.mjs → ThreadObject-SNcCsrS2.mjs} +52 -50
- package/dist/ThreadObject-SNcCsrS2.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +1 -1
- package/dist/ThreadObject.mjs +1 -1
- package/dist/WakeScheduler.mjs +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +3 -3
- package/dist/{prepared-admission-BoUPHSNQ.mjs → prepared-admission-BvMiFR41.mjs} +2 -2
- package/dist/{prepared-admission-BoUPHSNQ.mjs.map → prepared-admission-BvMiFR41.mjs.map} +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +295 -201
- package/src/CloudflareThreadClient.ts +3 -2
- package/src/ThreadObject.ts +17 -22
- package/src/internal/layers.ts +124 -82
- package/src/internal/message-delivery.ts +61 -37
- package/dist/Alarm-4J9B8Fg8.mjs.map +0 -1
- package/dist/CloudflareThreadClient-DU548EEj.mjs.map +0 -1
- package/dist/ThreadObject-C2LF3btz.mjs.map +0 -1
package/src/Alarm.ts
CHANGED
|
@@ -217,7 +217,7 @@ type MaintenanceCheckpoint = {
|
|
|
217
217
|
readonly mutations: Parameters<
|
|
218
218
|
Parameters<ThreadMaintenanceAlarm["Service"]["transaction"]>[0]
|
|
219
219
|
>[0];
|
|
220
|
-
readonly
|
|
220
|
+
readonly isActive: (fiber: Fiber.Fiber<unknown, unknown>) => boolean;
|
|
221
221
|
};
|
|
222
222
|
|
|
223
223
|
const CurrentCheckpoint = Context.Reference<MaintenanceCheckpoint | undefined>(
|
|
@@ -232,31 +232,27 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
232
232
|
const { ctx } = yield* DurableObjectContext;
|
|
233
233
|
const queue = DueQueue.make(ctx.storage);
|
|
234
234
|
|
|
235
|
-
const transaction
|
|
235
|
+
const transaction = <A, E, R>(
|
|
236
|
+
body: (owner: Fiber.Fiber<unknown, unknown>) => Effect.Effect<A, E, R>,
|
|
237
|
+
): Effect.Effect<A, E | SqlError, R> =>
|
|
236
238
|
alarms
|
|
237
239
|
.transaction(
|
|
238
240
|
(mutations) =>
|
|
239
241
|
Effect.withFiber((owner) => {
|
|
240
242
|
let active = true;
|
|
241
243
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
operation: "enroll maintenance source",
|
|
248
|
-
message: "Source intent requires its active transaction callback's fiber",
|
|
249
|
-
}),
|
|
250
|
-
),
|
|
251
|
-
);
|
|
252
|
-
|
|
253
|
-
return body.pipe(
|
|
254
|
-
Effect.provideService(CurrentCheckpoint, { mutations, requireActive }),
|
|
255
|
-
Effect.ensuring(
|
|
256
|
-
Effect.sync(() => {
|
|
257
|
-
active = false;
|
|
244
|
+
return Effect.onExitPrimitive(
|
|
245
|
+
body(owner).pipe(
|
|
246
|
+
Effect.provideService(CurrentCheckpoint, {
|
|
247
|
+
mutations,
|
|
248
|
+
isActive: (fiber) => active && fiber === owner,
|
|
258
249
|
}),
|
|
259
250
|
),
|
|
251
|
+
() => {
|
|
252
|
+
active = false;
|
|
253
|
+
|
|
254
|
+
return undefined;
|
|
255
|
+
},
|
|
260
256
|
);
|
|
261
257
|
}),
|
|
262
258
|
{ sqlClient: sql },
|
|
@@ -276,7 +272,9 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
276
272
|
);
|
|
277
273
|
|
|
278
274
|
const withTransaction = <A, E, R>(body: Effect.Effect<A, E, R>) =>
|
|
279
|
-
Effect.
|
|
275
|
+
Effect.withFiber((fiber) => {
|
|
276
|
+
const ambient = Context.getOption(fiber.context, sql.transactionService);
|
|
277
|
+
|
|
280
278
|
if (ambient._tag === "Some" && !DueQueue.ownsTransaction(ctx.storage, ambient.value[1]))
|
|
281
279
|
return Effect.fail(
|
|
282
280
|
SqlError.make({
|
|
@@ -287,53 +285,53 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
287
285
|
}),
|
|
288
286
|
}),
|
|
289
287
|
);
|
|
288
|
+
let frame: ReturnType<typeof queue.begin> | undefined;
|
|
290
289
|
|
|
291
290
|
return Effect.uninterruptibleMask((restore) =>
|
|
292
|
-
Effect.
|
|
293
|
-
|
|
291
|
+
Effect.onExitPrimitive(
|
|
292
|
+
transaction((owner) => {
|
|
293
|
+
const current = Context.getOption(owner.context, sql.transactionService);
|
|
294
294
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
295
|
+
if (current._tag === "None")
|
|
296
|
+
return SqlError.make({
|
|
297
|
+
reason: UnknownError.make({
|
|
298
|
+
operation: "begin maintenance source transaction",
|
|
299
|
+
message: "SQL transaction unavailable",
|
|
300
|
+
cause: "SQL transaction unavailable",
|
|
301
|
+
}),
|
|
302
|
+
});
|
|
303
|
+
const started = queue.begin(current.value[1]);
|
|
304
|
+
|
|
305
|
+
frame = started;
|
|
306
|
+
|
|
307
|
+
return Effect.flatMap(restore(body), (value) =>
|
|
308
|
+
Effect.try({
|
|
309
|
+
try: () => {
|
|
310
|
+
queue.flush(started);
|
|
311
|
+
|
|
312
|
+
return value;
|
|
313
|
+
},
|
|
314
|
+
catch: (cause) =>
|
|
315
|
+
SqlError.make({
|
|
300
316
|
reason: UnknownError.make({
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
317
|
+
cause,
|
|
318
|
+
operation: "flush maintenance due queue",
|
|
319
|
+
message: "Maintenance intent could not commit",
|
|
304
320
|
}),
|
|
305
|
-
})
|
|
306
|
-
frame = queue.begin(transaction.value[1]);
|
|
307
|
-
}),
|
|
308
|
-
).pipe(
|
|
309
|
-
Effect.andThen(restore(body)),
|
|
310
|
-
Effect.tap(() =>
|
|
311
|
-
Effect.try({
|
|
312
|
-
try: () => {
|
|
313
|
-
if (frame !== undefined) queue.flush(frame);
|
|
314
|
-
},
|
|
315
|
-
catch: (cause) =>
|
|
316
|
-
SqlError.make({
|
|
317
|
-
reason: UnknownError.make({
|
|
318
|
-
cause,
|
|
319
|
-
operation: "flush maintenance due queue",
|
|
320
|
-
message: "Maintenance intent could not commit",
|
|
321
|
-
}),
|
|
322
|
-
}),
|
|
323
|
-
}),
|
|
324
|
-
),
|
|
325
|
-
),
|
|
326
|
-
).pipe(
|
|
327
|
-
Effect.onExit((exit) =>
|
|
328
|
-
Effect.sync(() => {
|
|
329
|
-
if (frame !== undefined) queue.finish(frame, Exit.isSuccess(exit));
|
|
321
|
+
}),
|
|
330
322
|
}),
|
|
331
|
-
)
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
323
|
+
);
|
|
324
|
+
}),
|
|
325
|
+
(exit) => {
|
|
326
|
+
if (frame !== undefined) queue.finish(frame, Exit.isSuccess(exit));
|
|
327
|
+
|
|
328
|
+
return undefined;
|
|
329
|
+
},
|
|
330
|
+
).pipe(
|
|
331
|
+
// A released permit can resume another transaction. Do not auto-yield
|
|
332
|
+
// between the driver's settlement and removing this transaction's frame.
|
|
333
|
+
Effect.provideService(Scheduler.PreventSchedulerYield, true),
|
|
334
|
+
),
|
|
337
335
|
);
|
|
338
336
|
}).pipe(Effect.provideService(SqlClient, sql));
|
|
339
337
|
|
|
@@ -352,32 +350,33 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
352
350
|
}),
|
|
353
351
|
);
|
|
354
352
|
|
|
355
|
-
|
|
356
|
-
Effect.
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
353
|
+
if (!tx.isActive(fiber))
|
|
354
|
+
return Effect.fail(
|
|
355
|
+
DurableAlarmError.make({
|
|
356
|
+
operation: "enroll maintenance source",
|
|
357
|
+
message: "Source intent requires its active transaction callback's fiber",
|
|
358
|
+
}),
|
|
359
|
+
);
|
|
360
360
|
|
|
361
|
-
|
|
362
|
-
at: (at) => {
|
|
363
|
-
checkpoint = tx.mutations.scheduleAlarm(maintenanceAlarmInput(at));
|
|
364
|
-
},
|
|
365
|
-
earlier: (at) => {
|
|
366
|
-
checkpoint = tx.mutations.scheduleAlarmEarlier(maintenanceAlarmInput(at));
|
|
367
|
-
},
|
|
368
|
-
cancel: () => {
|
|
369
|
-
checkpoint = tx.mutations.cancelAlarm(maintenanceAlarmRef);
|
|
370
|
-
},
|
|
371
|
-
};
|
|
361
|
+
let checkpoint: ReturnType<DurableObjectAlarm.AlarmScheduler["scheduleAlarm"]> = Effect.void;
|
|
372
362
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
363
|
+
const intent: CheckpointIntent = {
|
|
364
|
+
at: (at) => {
|
|
365
|
+
checkpoint = tx.mutations.scheduleAlarm(maintenanceAlarmInput(at));
|
|
366
|
+
},
|
|
367
|
+
earlier: (at) => {
|
|
368
|
+
checkpoint = tx.mutations.scheduleAlarmEarlier(maintenanceAlarmInput(at));
|
|
369
|
+
},
|
|
370
|
+
cancel: () => {
|
|
371
|
+
checkpoint = tx.mutations.cancelAlarm(maintenanceAlarmRef);
|
|
372
|
+
},
|
|
373
|
+
};
|
|
374
|
+
|
|
375
|
+
// The SQL/alarm callback already owns the native transaction; KV uses the same storage.
|
|
376
|
+
return Effect.tryPromise({
|
|
377
|
+
try: () => execute(ctx.storage, intent),
|
|
378
|
+
catch: alarmFailure(operation),
|
|
379
|
+
}).pipe(Effect.tap(() => checkpoint));
|
|
381
380
|
});
|
|
382
381
|
|
|
383
382
|
return Effect.withFiber((fiber) => {
|
|
@@ -388,7 +387,7 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
388
387
|
|
|
389
388
|
// Preserve eager writes and invalidation for an unwrapped host SQL transaction.
|
|
390
389
|
return ambient._tag === "Some" && !DueQueue.buffering(ctx.storage)
|
|
391
|
-
? transaction(commit).pipe(
|
|
390
|
+
? transaction(() => commit).pipe(
|
|
392
391
|
Effect.ensuring(Effect.sync(() => DueQueue.invalidate(ctx.storage))),
|
|
393
392
|
)
|
|
394
393
|
: withTransaction(commit);
|
|
@@ -734,6 +733,25 @@ interface NativeDispatch {
|
|
|
734
733
|
scanRevision?: number;
|
|
735
734
|
}
|
|
736
735
|
|
|
736
|
+
/** Observe already-owned waves without creating one waiter fiber for each wave. */
|
|
737
|
+
const awaitAnyCompletion = Effect.fnUntraced(function* (
|
|
738
|
+
fibers: ReadonlyArray<Fiber.Fiber<unknown, MaintenancePassFailure>>,
|
|
739
|
+
revision: number,
|
|
740
|
+
) {
|
|
741
|
+
const mutations = yield* ThreadMutationGate;
|
|
742
|
+
|
|
743
|
+
return yield* Effect.acquireUseRelease(
|
|
744
|
+
Effect.sync<Array<() => void>>(() => []),
|
|
745
|
+
(observers) =>
|
|
746
|
+
Effect.callback<"completed" | "scan">((resume) => {
|
|
747
|
+
observers.push(mutations.observeChange(revision, () => resume(Effect.succeed("scan"))));
|
|
748
|
+
for (const fiber of fibers)
|
|
749
|
+
observers.push(fiber.addObserver(() => resume(Effect.succeed("completed"))));
|
|
750
|
+
}),
|
|
751
|
+
(observers) => Effect.sync(() => observers.forEach((remove) => remove())),
|
|
752
|
+
);
|
|
753
|
+
});
|
|
754
|
+
|
|
737
755
|
/** Post-native delivery shares a finite connection budget across independent host lanes. */
|
|
738
756
|
const afterNativeDispatchConcurrency = 2;
|
|
739
757
|
|
|
@@ -914,6 +932,8 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
914
932
|
readonly activeLanes: Effect.Effect<ReadonlySet<string>>;
|
|
915
933
|
readonly revision: Effect.Effect<number>;
|
|
916
934
|
readonly awaitChange: (revision: number) => Effect.Effect<void>;
|
|
935
|
+
/** Register an incarnation-local observer; its caller must detach it on completion. */
|
|
936
|
+
readonly observeChange: (revision: number, notify: () => void) => () => void;
|
|
917
937
|
readonly withSnapshot: <A, E, R>(
|
|
918
938
|
body: (active: number) => Effect.Effect<A, E, R>,
|
|
919
939
|
) => Effect.Effect<A, E, R>;
|
|
@@ -934,14 +954,27 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
934
954
|
const activeLanes = new Map<string, number>();
|
|
935
955
|
const generationGate = yield* Semaphore.make(1);
|
|
936
956
|
let revision = 0;
|
|
937
|
-
|
|
957
|
+
const listeners = new Set<() => void>();
|
|
938
958
|
|
|
939
|
-
const
|
|
940
|
-
|
|
959
|
+
const observeChange = (observed: number, listener: () => void): (() => void) => {
|
|
960
|
+
if (observed !== revision) {
|
|
961
|
+
listener();
|
|
962
|
+
|
|
963
|
+
return () => {};
|
|
964
|
+
}
|
|
965
|
+
listeners.add(listener);
|
|
941
966
|
|
|
942
|
-
|
|
967
|
+
return () => {
|
|
968
|
+
listeners.delete(listener);
|
|
969
|
+
};
|
|
970
|
+
};
|
|
971
|
+
|
|
972
|
+
const notify = Effect.sync(() => {
|
|
943
973
|
revision++;
|
|
944
|
-
|
|
974
|
+
const pending = [...listeners];
|
|
975
|
+
|
|
976
|
+
listeners.clear();
|
|
977
|
+
for (const listener of pending) listener();
|
|
945
978
|
});
|
|
946
979
|
|
|
947
980
|
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
@@ -1182,8 +1215,13 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
1182
1215
|
recordProgress,
|
|
1183
1216
|
activeLanes: Effect.sync(() => new Set(activeLanes.keys())),
|
|
1184
1217
|
revision: Effect.sync(() => revision),
|
|
1218
|
+
observeChange,
|
|
1185
1219
|
awaitChange: (observed) =>
|
|
1186
|
-
Effect.
|
|
1220
|
+
Effect.callback((resume) => {
|
|
1221
|
+
const remove = observeChange(observed, () => resume(Effect.void));
|
|
1222
|
+
|
|
1223
|
+
return Effect.sync(remove);
|
|
1224
|
+
}),
|
|
1187
1225
|
withSnapshot: (body) =>
|
|
1188
1226
|
generationGate.withPermit(Effect.flatMap(Ref.get(activeMutations), body)),
|
|
1189
1227
|
});
|
|
@@ -1259,6 +1297,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1259
1297
|
const runtime = yield* DurableAgentRuntime;
|
|
1260
1298
|
const recoveryConfig = yield* DurableRuntimeConfig;
|
|
1261
1299
|
const ledger = yield* SubmissionLedger;
|
|
1300
|
+
const listNonterminal = ledger.listNonterminal ?? Stream.runCollect(ledger.scanNonterminal);
|
|
1262
1301
|
const wakes = yield* WakeScheduler;
|
|
1263
1302
|
const hints = yield* DurableAlarmService;
|
|
1264
1303
|
const alarms = yield* ThreadMaintenanceAlarm;
|
|
@@ -1280,6 +1319,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1280
1319
|
const messages = yield* ThreadMessageDelivery;
|
|
1281
1320
|
const host = yield* ThreadHostMaintenance;
|
|
1282
1321
|
const framework = yield* ThreadNativeMaintenance;
|
|
1322
|
+
const placement = yield* Effect.serviceOption(ThreadObjectPlacement);
|
|
1323
|
+
const nativeThread = Option.isSome(placement) ? placement.value.threadId : undefined;
|
|
1283
1324
|
|
|
1284
1325
|
const nativeDispatchConcurrency = () => (framework.singleThreadRuntime === runtime ? 1 : 2);
|
|
1285
1326
|
|
|
@@ -1564,9 +1605,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1564
1605
|
const now = yield* Clock.currentTimeMillis;
|
|
1565
1606
|
|
|
1566
1607
|
// Fault-only discovery uses control state even when retained payloads cannot decode.
|
|
1567
|
-
const submissions =
|
|
1568
|
-
current ??
|
|
1569
|
-
(result.blocked.length === 0 ? [] : yield* Stream.runCollect(ledger.scanNonterminal));
|
|
1608
|
+
const submissions = current ?? (result.blocked.length === 0 ? [] : yield* listNonterminal);
|
|
1570
1609
|
|
|
1571
1610
|
yield* failpoint.hit("maintenance:recovery-status:before");
|
|
1572
1611
|
|
|
@@ -1681,7 +1720,11 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1681
1720
|
|
|
1682
1721
|
do {
|
|
1683
1722
|
const result = yield* runtime
|
|
1684
|
-
.runRecovery({
|
|
1723
|
+
.runRecovery({
|
|
1724
|
+
threadId,
|
|
1725
|
+
deferUnclaimedAdmissions: true,
|
|
1726
|
+
...(cursor === undefined ? {} : { cursor }),
|
|
1727
|
+
})
|
|
1685
1728
|
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
1686
1729
|
|
|
1687
1730
|
// Persist each bounded pass before a claim; eviction resumes past consumed metadata.
|
|
@@ -2124,19 +2167,31 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2124
2167
|
|
|
2125
2168
|
return {
|
|
2126
2169
|
scannedGeneration: state.dirty,
|
|
2127
|
-
|
|
2170
|
+
// Single-Thread placement would immediately wrap past its own cursor.
|
|
2171
|
+
afterWorkThread:
|
|
2172
|
+
framework.singleThreadRuntime === runtime &&
|
|
2173
|
+
state.lastRecoveredThreadId === nativeThread
|
|
2174
|
+
? undefined
|
|
2175
|
+
: state.lastRecoveredThreadId,
|
|
2128
2176
|
};
|
|
2129
2177
|
},
|
|
2130
2178
|
);
|
|
2131
2179
|
|
|
2132
|
-
const current = yield*
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
.
|
|
2180
|
+
const current = yield* listNonterminal;
|
|
2181
|
+
|
|
2182
|
+
// A current admission proves this native placement is materialized. Empty Objects
|
|
2183
|
+
// still discover retained canonical obligations without inventing a recovery owner.
|
|
2184
|
+
let workOwners =
|
|
2185
|
+
framework.singleThreadRuntime === runtime &&
|
|
2186
|
+
nativeThread !== undefined &&
|
|
2187
|
+
current.some((submission) => submission.threadId === nativeThread)
|
|
2188
|
+
? { threadIds: [nativeThread], afterThreadId: undefined }
|
|
2189
|
+
: yield* runtime
|
|
2190
|
+
.discoverWorkThreads({
|
|
2191
|
+
limit: 32,
|
|
2192
|
+
...(afterWorkThread === undefined ? {} : { afterThreadId: afterWorkThread }),
|
|
2193
|
+
})
|
|
2194
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
2140
2195
|
|
|
2141
2196
|
if (workOwners.threadIds.length === 0 && afterWorkThread !== undefined)
|
|
2142
2197
|
workOwners = yield* runtime
|
|
@@ -2398,7 +2453,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2398
2453
|
};
|
|
2399
2454
|
}
|
|
2400
2455
|
|
|
2401
|
-
const remaining = yield*
|
|
2456
|
+
const remaining = yield* listNonterminal;
|
|
2402
2457
|
const waitingHeads = new Map<ThreadId, boolean>();
|
|
2403
2458
|
|
|
2404
2459
|
// Selected recovery must finish before its generation can be acknowledged: terminal
|
|
@@ -2565,12 +2620,22 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2565
2620
|
dispatchUntil: DateTime.Utc,
|
|
2566
2621
|
observed: MaintenanceObservation,
|
|
2567
2622
|
) {
|
|
2623
|
+
const leanSelection =
|
|
2624
|
+
framework.singleThreadRuntime === runtime &&
|
|
2625
|
+
publication === emptyPublication &&
|
|
2626
|
+
host.lanes.length === 0 &&
|
|
2627
|
+
framework.lanes.length === 0;
|
|
2628
|
+
|
|
2568
2629
|
// Subscribe before the first snapshot. Durable enrollment recovers eviction;
|
|
2569
2630
|
// producer completion and wake hints make work visible during this incarnation.
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2631
|
+
// Native source mutations already notify the revision waiter below. A replacement
|
|
2632
|
+
// or shared runtime keeps the generic wake stream; hints confer no authority.
|
|
2633
|
+
const notified = leanSelection
|
|
2634
|
+
? Effect.never
|
|
2635
|
+
: (yield* Stream.toPull(wakes.wakes)).pipe(
|
|
2636
|
+
Effect.asVoid,
|
|
2637
|
+
Effect.catch(() => Effect.never),
|
|
2638
|
+
);
|
|
2574
2639
|
|
|
2575
2640
|
// A wake may observe temporary backoff while this event's recovery is still running.
|
|
2576
2641
|
// Its completion must retain the original actionable observation for acknowledgement.
|
|
@@ -2622,56 +2687,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2622
2687
|
discoveryPending: false,
|
|
2623
2688
|
};
|
|
2624
2689
|
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
for (const threadId of yield* Deferred.await(recovery.queue)) {
|
|
2628
|
-
yield* failpoint.hit("maintenance:select:before");
|
|
2629
|
-
yield* runTransaction("select old recovery lane", () =>
|
|
2630
|
-
dueQueue.transaction(async (transaction) => {
|
|
2631
|
-
const { state } = await readMaintenanceState(transaction);
|
|
2632
|
-
|
|
2633
|
-
await transaction.put(
|
|
2634
|
-
MAINTENANCE_STATE_KEY,
|
|
2635
|
-
encodeMaintenanceState(
|
|
2636
|
-
ThreadMaintenanceState.make({
|
|
2637
|
-
...state,
|
|
2638
|
-
lastRecoveredThreadId: threadId,
|
|
2639
|
-
}),
|
|
2640
|
-
),
|
|
2641
|
-
);
|
|
2642
|
-
}),
|
|
2643
|
-
);
|
|
2644
|
-
yield* failpoint.hit("maintenance:select:after");
|
|
2645
|
-
yield* recoverThread(threadId, recovery);
|
|
2646
|
-
recovery.pending.delete(threadId);
|
|
2647
|
-
if (!recovery.faults.has(threadId)) yield* wakes.notify(threadId);
|
|
2648
|
-
}
|
|
2649
|
-
}),
|
|
2650
|
-
);
|
|
2651
|
-
|
|
2652
|
-
// Backfill is one disposable wave; native and host work keep their own opportunities.
|
|
2653
|
-
const backfill = yield* fork(
|
|
2654
|
-
Effect.gen(function* () {
|
|
2655
|
-
if ((yield* mutations.activeLanes).has(DueQueue.Projection)) return;
|
|
2656
|
-
const row = (yield* queueSnapshot).find((row) => row.id === DueQueue.Projection);
|
|
2657
|
-
|
|
2658
|
-
if (
|
|
2659
|
-
row?.dueAt === null ||
|
|
2660
|
-
row === undefined ||
|
|
2661
|
-
row.dueAt > (yield* Clock.currentTimeMillis)
|
|
2662
|
-
)
|
|
2663
|
-
return;
|
|
2664
|
-
|
|
2665
|
-
return yield* runQueued(
|
|
2666
|
-
row,
|
|
2667
|
-
projection.drain.pipe(Effect.andThen(projection.pendingDeadline)),
|
|
2668
|
-
observed,
|
|
2669
|
-
).pipe(Effect.timeoutOption(config.projectionDispatchTimeoutMillis));
|
|
2670
|
-
}),
|
|
2671
|
-
);
|
|
2672
|
-
|
|
2673
|
-
let backfillObserved = false;
|
|
2674
|
-
let recoveryObserved = false;
|
|
2690
|
+
let recoveryFiber: Fiber.Fiber<void, MaintenancePassFailure> | undefined;
|
|
2691
|
+
let backfill: Fiber.Fiber<void, DurableAlarmError | ThreadProjectionError> | undefined;
|
|
2675
2692
|
|
|
2676
2693
|
const dispatch: NativeDispatch = {
|
|
2677
2694
|
scope: auxiliaryScope,
|
|
@@ -2683,18 +2700,79 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2683
2700
|
needsCheckpoint: false,
|
|
2684
2701
|
};
|
|
2685
2702
|
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2703
|
+
// Selection is bounded local work, not an independently owned wave. It opens
|
|
2704
|
+
// actual Attempt scopes, then returns; empty lanes create no fibers.
|
|
2705
|
+
let native: Fiber.Fiber<NativePassResult, MaintenancePassFailure> | undefined;
|
|
2706
|
+
let result: NativePassResult;
|
|
2707
|
+
let backlog: ReadonlyArray<ThreadId> | undefined;
|
|
2689
2708
|
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2709
|
+
if (leanSelection) {
|
|
2710
|
+
result = yield* advance(started, yieldAfter, observed, recovery, dispatch);
|
|
2711
|
+
recovery.started = true;
|
|
2712
|
+
yield* Deferred.succeed(recovery.queue, []);
|
|
2713
|
+
backlog = yield* Deferred.await(recovery.queue);
|
|
2714
|
+
} else {
|
|
2715
|
+
// Required publication and arbitrary host lanes keep independent opportunities
|
|
2716
|
+
// while selection is suspended in an application callback.
|
|
2717
|
+
native = yield* fork(advance(started, yieldAfter, observed, recovery, dispatch));
|
|
2718
|
+
result = {
|
|
2719
|
+
phase: "caught-up",
|
|
2720
|
+
nonterminal: started.nonterminal,
|
|
2721
|
+
nextAttemptAt: undefined,
|
|
2722
|
+
};
|
|
2723
|
+
}
|
|
2724
|
+
|
|
2725
|
+
if (backlog === undefined || backlog.length > 0) {
|
|
2726
|
+
recoveryFiber = yield* fork(
|
|
2727
|
+
Effect.gen(function* () {
|
|
2728
|
+
for (const threadId of backlog ?? (yield* Deferred.await(recovery.queue))) {
|
|
2729
|
+
yield* failpoint.hit("maintenance:select:before");
|
|
2730
|
+
yield* runTransaction("select old recovery lane", () =>
|
|
2731
|
+
dueQueue.transaction(async (transaction) => {
|
|
2732
|
+
const { state } = await readMaintenanceState(transaction);
|
|
2733
|
+
|
|
2734
|
+
await transaction.put(
|
|
2735
|
+
MAINTENANCE_STATE_KEY,
|
|
2736
|
+
encodeMaintenanceState(
|
|
2737
|
+
ThreadMaintenanceState.make({
|
|
2738
|
+
...state,
|
|
2739
|
+
lastRecoveredThreadId: threadId,
|
|
2740
|
+
}),
|
|
2741
|
+
),
|
|
2742
|
+
);
|
|
2743
|
+
}),
|
|
2744
|
+
);
|
|
2745
|
+
yield* failpoint.hit("maintenance:select:after");
|
|
2746
|
+
yield* recoverThread(threadId, recovery);
|
|
2747
|
+
recovery.pending.delete(threadId);
|
|
2748
|
+
if (!recovery.faults.has(threadId)) yield* wakes.notify(threadId);
|
|
2749
|
+
}
|
|
2750
|
+
}),
|
|
2751
|
+
);
|
|
2752
|
+
}
|
|
2753
|
+
const projectionRow = (yield* queueSnapshot).find((row) => row.id === DueQueue.Projection);
|
|
2754
|
+
|
|
2755
|
+
if (
|
|
2756
|
+
projectionRow !== undefined &&
|
|
2757
|
+
projectionRow.dueAt !== null &&
|
|
2758
|
+
projectionRow.dueAt <= (yield* Clock.currentTimeMillis) &&
|
|
2759
|
+
!(yield* mutations.activeLanes).has(DueQueue.Projection)
|
|
2760
|
+
) {
|
|
2761
|
+
backfill = yield* fork(
|
|
2762
|
+
runQueued(
|
|
2763
|
+
projectionRow,
|
|
2764
|
+
projection.drain.pipe(Effect.andThen(projection.pendingDeadline)),
|
|
2765
|
+
observed,
|
|
2766
|
+
).pipe(Effect.timeoutOption(config.projectionDispatchTimeoutMillis), Effect.asVoid),
|
|
2767
|
+
);
|
|
2768
|
+
}
|
|
2695
2769
|
|
|
2696
2770
|
let phase = result.phase;
|
|
2697
|
-
|
|
2771
|
+
|
|
2772
|
+
let nativeCheck =
|
|
2773
|
+
result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency();
|
|
2774
|
+
|
|
2775
|
+
observed.nativeOnly = dispatch.active.size > 0;
|
|
2698
2776
|
const until = DateTime.toEpochMillis(yieldAfter);
|
|
2699
2777
|
const dispatchEnd = DateTime.toEpochMillis(dispatchUntil);
|
|
2700
2778
|
|
|
@@ -2725,19 +2803,17 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2725
2803
|
if (exit.value === 0) dispatch.deferred.add(threadId);
|
|
2726
2804
|
nativeCheck = true;
|
|
2727
2805
|
}
|
|
2728
|
-
if (
|
|
2806
|
+
if (recoveryFiber?.pollUnsafe() !== undefined) {
|
|
2729
2807
|
yield* Fiber.join(recoveryFiber);
|
|
2730
|
-
|
|
2808
|
+
recoveryFiber = undefined;
|
|
2731
2809
|
dispatch.scanGeneration = undefined;
|
|
2732
2810
|
nativeCheck ||= recovery.needsCheckpoint;
|
|
2733
2811
|
}
|
|
2734
|
-
|
|
2735
|
-
const exit = backfill.pollUnsafe();
|
|
2812
|
+
const backfillExit = backfill?.pollUnsafe();
|
|
2736
2813
|
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
}
|
|
2814
|
+
if (backfillExit !== undefined) {
|
|
2815
|
+
backfill = undefined;
|
|
2816
|
+
if (Exit.isFailure(backfillExit)) failure ??= backfillExit.cause;
|
|
2741
2817
|
}
|
|
2742
2818
|
for (const lane of lanes) {
|
|
2743
2819
|
const exit = lane.fiber?.pollUnsafe();
|
|
@@ -2844,9 +2920,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2844
2920
|
if (
|
|
2845
2921
|
native === undefined &&
|
|
2846
2922
|
dispatch.active.size === 0 &&
|
|
2847
|
-
|
|
2923
|
+
recoveryFiber === undefined &&
|
|
2848
2924
|
!recovery.needsCheckpoint &&
|
|
2849
|
-
|
|
2925
|
+
backfill === undefined &&
|
|
2850
2926
|
delivery === undefined &&
|
|
2851
2927
|
lanes.every((lane) => lane.fiber === undefined)
|
|
2852
2928
|
)
|
|
@@ -2859,7 +2935,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2859
2935
|
) {
|
|
2860
2936
|
nativeCheck = false;
|
|
2861
2937
|
const nativeCheckpoint = dispatch.needsCheckpoint && dispatch.active.size === 0;
|
|
2862
|
-
|
|
2938
|
+
|
|
2939
|
+
const checkpoint =
|
|
2940
|
+
(recoveryFiber === undefined && recovery.needsCheckpoint) || nativeCheckpoint;
|
|
2863
2941
|
|
|
2864
2942
|
if (checkpoint) recovery.needsCheckpoint = false;
|
|
2865
2943
|
// Keep this opportunity's reservations across beginNative's storage awaits.
|
|
@@ -2894,26 +2972,39 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2894
2972
|
);
|
|
2895
2973
|
|
|
2896
2974
|
const completing = [
|
|
2897
|
-
|
|
2975
|
+
native,
|
|
2898
2976
|
...dispatch.active.values(),
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
...lanes.
|
|
2903
|
-
];
|
|
2904
|
-
|
|
2905
|
-
const
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2977
|
+
recoveryFiber,
|
|
2978
|
+
backfill,
|
|
2979
|
+
delivery,
|
|
2980
|
+
...lanes.map((lane) => lane.fiber),
|
|
2981
|
+
].filter((fiber) => fiber !== undefined);
|
|
2982
|
+
|
|
2983
|
+
const changed = awaitAnyCompletion(completing, queueRevision);
|
|
2984
|
+
|
|
2985
|
+
// An occupied native-only slot cannot admit another Attempt. With no auxiliary
|
|
2986
|
+
// wake before yield, the timer would only switch to joining that same Attempt.
|
|
2987
|
+
// Incoming work still wakes this callback; all new waves recheck the deadline.
|
|
2988
|
+
const joiningNative =
|
|
2989
|
+
leanSelection &&
|
|
2990
|
+
native === undefined &&
|
|
2991
|
+
dispatch.active.size === 1 &&
|
|
2992
|
+
completing.length === 1 &&
|
|
2993
|
+
!queued.some(
|
|
2994
|
+
(row) =>
|
|
2995
|
+
row.id !== DueQueue.Native &&
|
|
2996
|
+
row.dueAt !== null &&
|
|
2997
|
+
row.dueAt > now &&
|
|
2998
|
+
row.dueAt < until,
|
|
2999
|
+
);
|
|
3000
|
+
|
|
3001
|
+
const ready = yield* joiningNative
|
|
3002
|
+
? changed
|
|
3003
|
+
: Effect.raceAllFirst([
|
|
3004
|
+
changed,
|
|
3005
|
+
...(leanSelection ? [] : [notified.pipe(Effect.as("scan" as const))]),
|
|
3006
|
+
Effect.sleep(Math.max(0, next - now)).pipe(Effect.as("scan" as const)),
|
|
3007
|
+
]);
|
|
2917
3008
|
|
|
2918
3009
|
if (ready === "scan") {
|
|
2919
3010
|
nativeCheck = true;
|
|
@@ -2939,9 +3030,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2939
3030
|
for (const fiber of [
|
|
2940
3031
|
recoveryFiber,
|
|
2941
3032
|
backfill,
|
|
2942
|
-
|
|
2943
|
-
...lanes.
|
|
3033
|
+
delivery,
|
|
3034
|
+
...lanes.map((lane) => lane.fiber),
|
|
2944
3035
|
]) {
|
|
3036
|
+
if (fiber === undefined) continue;
|
|
2945
3037
|
// Await exits without failing fast: every admitted wave owns its original
|
|
2946
3038
|
// bounded allowance and cleanup even if another wave has already failed.
|
|
2947
3039
|
const exit = yield* Fiber.await<unknown, MaintenancePassFailure>(fiber);
|
|
@@ -2964,7 +3056,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2964
3056
|
observed: MaintenanceObservation,
|
|
2965
3057
|
): Effect.fn.Return<MaintenancePassReport, MaintenancePassFailure, Scope.Scope> {
|
|
2966
3058
|
const concurrent = yield* Effect.exit(
|
|
2967
|
-
Effect.scoped(dispatch(yieldAfter, dispatchUntil, observed))
|
|
3059
|
+
Effect.scoped(dispatch(yieldAfter, dispatchUntil, observed)).pipe(
|
|
3060
|
+
Effect.provideService(ThreadMutationGate, mutations),
|
|
3061
|
+
),
|
|
2968
3062
|
);
|
|
2969
3063
|
|
|
2970
3064
|
// Closing the concurrent scope first releases every native Attempt and its resources.
|