@yielded/agent-platform-cloudflare 0.1.0-beta.173 → 0.1.0-beta.175
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{Alarm-BMEAZo3p.mjs → Alarm-B44efvB1.mjs} +162 -103
- package/dist/Alarm-B44efvB1.mjs.map +1 -0
- package/dist/Alarm.d.mts +2 -0
- package/dist/Alarm.mjs +1 -1
- package/dist/BrowserRestCapture.mjs.map +1 -1
- package/dist/BrowserRestCrawl.mjs.map +1 -1
- package/dist/{BrowserSession-Bx0V1-6_.d.mts → BrowserSession-Cojp7nth.d.mts} +3 -2
- package/dist/{BrowserSession-BfNmyayH.mjs → BrowserSession-aovMjfL4.mjs} +14 -14
- package/dist/BrowserSession-aovMjfL4.mjs.map +1 -0
- package/dist/BrowserSession.d.mts +1 -1
- package/dist/BrowserSession.mjs +2 -2
- package/dist/BrowserUse.d.mts +40 -64
- package/dist/BrowserUse.mjs +223 -2
- package/dist/BrowserUse.mjs.map +1 -0
- package/dist/CloudflareBrowser-Vt8qTdYW.mjs.map +1 -1
- package/dist/CloudflareCodeMode.mjs.map +1 -1
- package/dist/CloudflareScheduling.mjs +9 -9
- package/dist/CloudflareScheduling.mjs.map +1 -1
- package/dist/CloudflareSubscriptions.mjs +15 -15
- package/dist/CloudflareSubscriptions.mjs.map +1 -1
- package/dist/{CloudflareThreadClient-DgZVrZzF.mjs → CloudflareThreadClient-Ai3MTs3X.mjs} +43 -40
- package/dist/CloudflareThreadClient-Ai3MTs3X.mjs.map +1 -0
- package/dist/CloudflareThreadClient.mjs +1 -1
- package/dist/{InteractiveBrowser-LdKzQeLR.mjs → InteractiveBrowser-BHNlwjZL.mjs} +40 -31
- package/dist/InteractiveBrowser-BHNlwjZL.mjs.map +1 -0
- package/dist/InteractiveBrowser.mjs +1 -1
- package/dist/{ThreadObject-CRbLbdXM.d.mts → ThreadObject-CxNlhoPL.d.mts} +3 -3
- package/dist/{ThreadObject-CGIaz6Jv.mjs → ThreadObject-DGvlHWYi.mjs} +50 -52
- package/dist/ThreadObject-DGvlHWYi.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +1 -1
- package/dist/ThreadObject.mjs +1 -1
- package/dist/WakeScheduler.d.mts +1 -1
- package/dist/WakeScheduler.mjs +8 -3
- package/dist/WakeScheduler.mjs.map +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +5 -5
- package/dist/{prepared-admission-CSdzFM_v.mjs → prepared-admission-DYdffQ8K.mjs} +2 -2
- package/dist/{prepared-admission-CSdzFM_v.mjs.map → prepared-admission-DYdffQ8K.mjs.map} +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +293 -155
- package/src/BrowserRestCapture.ts +1 -0
- package/src/BrowserRestCrawl.ts +1 -0
- package/src/BrowserSession.ts +3 -2
- package/src/BrowserUse.ts +287 -1504
- package/src/CloudflareCodeMode.ts +6 -0
- package/src/CloudflareScheduling.ts +103 -103
- package/src/CloudflareSubscriptions.ts +192 -199
- package/src/CloudflareThreadClient.ts +258 -269
- package/src/InteractiveBrowser.ts +43 -17
- package/src/ThreadObject.ts +51 -45
- package/src/WakeScheduler.ts +12 -9
- package/src/internal/browser-binding.ts +8 -0
- package/src/internal/browser-credentials.ts +18 -13
- package/src/internal/browser-file-selection.ts +13 -2
- package/src/internal/browser-quick-action.ts +1 -0
- package/src/internal/browser-session-lifecycle.ts +32 -36
- package/src/internal/due-queue.ts +38 -19
- package/src/internal/layers.ts +108 -87
- package/src/internal/message-delivery.ts +61 -37
- package/dist/Alarm-BMEAZo3p.mjs.map +0 -1
- package/dist/BrowserSession-BfNmyayH.mjs.map +0 -1
- package/dist/BrowserUse-Bc2JdLW2.mjs +0 -1528
- package/dist/BrowserUse-Bc2JdLW2.mjs.map +0 -1
- package/dist/CloudflareThreadClient-DgZVrZzF.mjs.map +0 -1
- package/dist/InteractiveBrowser-LdKzQeLR.mjs.map +0 -1
- package/dist/ThreadObject-CGIaz6Jv.mjs.map +0 -1
- package/dist/page-function-CmUCcyNP.mjs +0 -11
- package/dist/page-function-CmUCcyNP.mjs.map +0 -1
- package/src/internal/browser-dom.ts +0 -886
- package/src/internal/browser-jev-dom.ts +0 -362
package/src/Alarm.ts
CHANGED
|
@@ -47,7 +47,7 @@ import { DurableObjectAlarm, DurableObjectStorage } from "effect-cf";
|
|
|
47
47
|
import { SqlClient } from "effect/sql/SqlClient";
|
|
48
48
|
import { SqlError, UnknownError } from "effect/sql/SqlError";
|
|
49
49
|
|
|
50
|
-
import { DurableObjectContext } from "./CloudflareBindings.ts";
|
|
50
|
+
import { DurableObjectContext, ThreadObjectPlacement } from "./CloudflareBindings.ts";
|
|
51
51
|
import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.ts";
|
|
52
52
|
import { safeCauseMessage } from "./internal/boundary.ts";
|
|
53
53
|
import * as DueQueue from "./internal/due-queue.ts";
|
|
@@ -283,7 +283,7 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
283
283
|
reason: UnknownError.make({
|
|
284
284
|
operation: "begin maintenance source transaction",
|
|
285
285
|
message: "Use ThreadMutationGate.withTransaction at the outermost source transaction",
|
|
286
|
-
cause:
|
|
286
|
+
cause: "Unwrapped SQL transaction cannot own buffered maintenance intent",
|
|
287
287
|
}),
|
|
288
288
|
}),
|
|
289
289
|
);
|
|
@@ -294,8 +294,15 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
294
294
|
|
|
295
295
|
return transaction(
|
|
296
296
|
Effect.flatMap(Effect.serviceOption(sql.transactionService), (transaction) =>
|
|
297
|
-
Effect.
|
|
298
|
-
if (transaction._tag === "None")
|
|
297
|
+
Effect.gen(function* () {
|
|
298
|
+
if (transaction._tag === "None")
|
|
299
|
+
return yield* SqlError.make({
|
|
300
|
+
reason: UnknownError.make({
|
|
301
|
+
operation: "begin maintenance source transaction",
|
|
302
|
+
message: "SQL transaction unavailable",
|
|
303
|
+
cause: "SQL transaction unavailable",
|
|
304
|
+
}),
|
|
305
|
+
});
|
|
299
306
|
frame = queue.begin(transaction.value[1]);
|
|
300
307
|
}),
|
|
301
308
|
).pipe(
|
|
@@ -334,9 +341,16 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
334
341
|
operation: string,
|
|
335
342
|
execute: (transaction: MaintenanceStorage, checkpoint: CheckpointIntent) => Promise<A>,
|
|
336
343
|
) => {
|
|
337
|
-
const commit = Effect.
|
|
344
|
+
const commit = Effect.withFiber((fiber) => {
|
|
345
|
+
const tx = fiber.getRef(CurrentCheckpoint);
|
|
346
|
+
|
|
338
347
|
if (tx === undefined)
|
|
339
|
-
return Effect.die(
|
|
348
|
+
return Effect.die(
|
|
349
|
+
DurableAlarmError.make({
|
|
350
|
+
operation: "checkpoint",
|
|
351
|
+
message: "Maintenance checkpoint transaction unavailable",
|
|
352
|
+
}),
|
|
353
|
+
);
|
|
340
354
|
|
|
341
355
|
return tx.requireActive.pipe(
|
|
342
356
|
Effect.andThen(
|
|
@@ -366,14 +380,14 @@ const makeAlarmOperation = Effect.gen(function* () {
|
|
|
366
380
|
);
|
|
367
381
|
});
|
|
368
382
|
|
|
369
|
-
return Effect.
|
|
370
|
-
const current =
|
|
383
|
+
return Effect.withFiber((fiber) => {
|
|
384
|
+
const current = fiber.getRef(CurrentCheckpoint);
|
|
371
385
|
|
|
372
|
-
if (current !== undefined) return
|
|
373
|
-
const ambient =
|
|
386
|
+
if (current !== undefined) return commit;
|
|
387
|
+
const ambient = Context.getOption(fiber.context, sql.transactionService);
|
|
374
388
|
|
|
375
389
|
// Preserve eager writes and invalidation for an unwrapped host SQL transaction.
|
|
376
|
-
return
|
|
390
|
+
return ambient._tag === "Some" && !DueQueue.buffering(ctx.storage)
|
|
377
391
|
? transaction(commit).pipe(
|
|
378
392
|
Effect.ensuring(Effect.sync(() => DueQueue.invalidate(ctx.storage))),
|
|
379
393
|
)
|
|
@@ -629,6 +643,7 @@ const decodeRecoveryFaultValue = Schema.decodeUnknownSync(ThreadRecoveryFault);
|
|
|
629
643
|
const decodeRecoveryFault = (threadId: ThreadId, encoded: unknown) => {
|
|
630
644
|
const fault = decodeRecoveryFaultValue(encoded);
|
|
631
645
|
|
|
646
|
+
// oxlint-disable-next-line effect/no-untyped-throw, effect/no-native-error -- Native Durable Object transaction decoder must abort before corrupt persisted data can be committed.
|
|
632
647
|
if (fault.threadId !== threadId) throw new Error("Recovery status does not match its Thread key");
|
|
633
648
|
|
|
634
649
|
return fault;
|
|
@@ -719,6 +734,25 @@ interface NativeDispatch {
|
|
|
719
734
|
scanRevision?: number;
|
|
720
735
|
}
|
|
721
736
|
|
|
737
|
+
/** Observe already-owned waves without creating one waiter fiber for each wave. */
|
|
738
|
+
const awaitAnyCompletion = Effect.fnUntraced(function* (
|
|
739
|
+
fibers: ReadonlyArray<Fiber.Fiber<unknown, MaintenancePassFailure>>,
|
|
740
|
+
revision: number,
|
|
741
|
+
) {
|
|
742
|
+
const mutations = yield* ThreadMutationGate;
|
|
743
|
+
|
|
744
|
+
return yield* Effect.acquireUseRelease(
|
|
745
|
+
Effect.sync<Array<() => void>>(() => []),
|
|
746
|
+
(observers) =>
|
|
747
|
+
Effect.callback<"completed" | "scan">((resume) => {
|
|
748
|
+
observers.push(mutations.observeChange(revision, () => resume(Effect.succeed("scan"))));
|
|
749
|
+
for (const fiber of fibers)
|
|
750
|
+
observers.push(fiber.addObserver(() => resume(Effect.succeed("completed"))));
|
|
751
|
+
}),
|
|
752
|
+
(observers) => Effect.sync(() => observers.forEach((remove) => remove())),
|
|
753
|
+
);
|
|
754
|
+
});
|
|
755
|
+
|
|
722
756
|
/** Post-native delivery shares a finite connection budget across independent host lanes. */
|
|
723
757
|
const afterNativeDispatchConcurrency = 2;
|
|
724
758
|
|
|
@@ -899,6 +933,8 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
899
933
|
readonly activeLanes: Effect.Effect<ReadonlySet<string>>;
|
|
900
934
|
readonly revision: Effect.Effect<number>;
|
|
901
935
|
readonly awaitChange: (revision: number) => Effect.Effect<void>;
|
|
936
|
+
/** Register an incarnation-local observer; its caller must detach it on completion. */
|
|
937
|
+
readonly observeChange: (revision: number, notify: () => void) => () => void;
|
|
902
938
|
readonly withSnapshot: <A, E, R>(
|
|
903
939
|
body: (active: number) => Effect.Effect<A, E, R>,
|
|
904
940
|
) => Effect.Effect<A, E, R>;
|
|
@@ -909,6 +945,8 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
909
945
|
const { ctx } = yield* DurableObjectContext;
|
|
910
946
|
const sql = yield* SqlClient;
|
|
911
947
|
const dueQueue = DueQueue.make(ctx.storage);
|
|
948
|
+
const placement = yield* Effect.serviceOption(ThreadObjectPlacement);
|
|
949
|
+
const singleThread = Option.isSome(placement) && placement.value.threadId !== undefined;
|
|
912
950
|
|
|
913
951
|
const config = yield* CloudflareDurableRuntimeConfig;
|
|
914
952
|
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
@@ -917,14 +955,27 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
917
955
|
const activeLanes = new Map<string, number>();
|
|
918
956
|
const generationGate = yield* Semaphore.make(1);
|
|
919
957
|
let revision = 0;
|
|
920
|
-
|
|
958
|
+
const listeners = new Set<() => void>();
|
|
959
|
+
|
|
960
|
+
const observeChange = (observed: number, listener: () => void): (() => void) => {
|
|
961
|
+
if (observed !== revision) {
|
|
962
|
+
listener();
|
|
921
963
|
|
|
922
|
-
|
|
923
|
-
|
|
964
|
+
return () => {};
|
|
965
|
+
}
|
|
966
|
+
listeners.add(listener);
|
|
924
967
|
|
|
925
|
-
|
|
968
|
+
return () => {
|
|
969
|
+
listeners.delete(listener);
|
|
970
|
+
};
|
|
971
|
+
};
|
|
972
|
+
|
|
973
|
+
const notify = Effect.sync(() => {
|
|
926
974
|
revision++;
|
|
927
|
-
|
|
975
|
+
const pending = [...listeners];
|
|
976
|
+
|
|
977
|
+
listeners.clear();
|
|
978
|
+
for (const listener of pending) listener();
|
|
928
979
|
});
|
|
929
980
|
|
|
930
981
|
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
@@ -1127,7 +1178,12 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
1127
1178
|
yield* runAlarmTransaction(
|
|
1128
1179
|
"record maintenance producer progress",
|
|
1129
1180
|
async (transaction, checkpoint) => {
|
|
1130
|
-
for (const id of affected)
|
|
1181
|
+
for (const id of affected) {
|
|
1182
|
+
// Shared hosts need every revision to discover newly actionable sibling Threads.
|
|
1183
|
+
if (singleThread && nativeSource && id === DueQueue.Native)
|
|
1184
|
+
dueQueue.progressNative(now);
|
|
1185
|
+
else dueQueue.dirty(id, now, true);
|
|
1186
|
+
}
|
|
1131
1187
|
if (affected.has(DueQueue.Native)) {
|
|
1132
1188
|
const { state, initialized } = await readMaintenanceState(transaction);
|
|
1133
1189
|
|
|
@@ -1147,7 +1203,10 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
1147
1203
|
checkpoint.earlier(now + minimumAlarmDelay);
|
|
1148
1204
|
},
|
|
1149
1205
|
);
|
|
1150
|
-
|
|
1206
|
+
// Maintenance already awaits this native fiber's completion. With one Thread,
|
|
1207
|
+
// native-only progress cannot free another dispatch slot or enroll an auxiliary lane.
|
|
1208
|
+
if (!(singleThread && nativeSource && affected.size === 1 && affected.has(DueQueue.Native)))
|
|
1209
|
+
yield* notify;
|
|
1151
1210
|
});
|
|
1152
1211
|
|
|
1153
1212
|
return ThreadMutationGate.of({
|
|
@@ -1157,8 +1216,13 @@ export class ThreadMutationGate extends Context.Service<
|
|
|
1157
1216
|
recordProgress,
|
|
1158
1217
|
activeLanes: Effect.sync(() => new Set(activeLanes.keys())),
|
|
1159
1218
|
revision: Effect.sync(() => revision),
|
|
1219
|
+
observeChange,
|
|
1160
1220
|
awaitChange: (observed) =>
|
|
1161
|
-
Effect.
|
|
1221
|
+
Effect.callback((resume) => {
|
|
1222
|
+
const remove = observeChange(observed, () => resume(Effect.void));
|
|
1223
|
+
|
|
1224
|
+
return Effect.sync(remove);
|
|
1225
|
+
}),
|
|
1162
1226
|
withSnapshot: (body) =>
|
|
1163
1227
|
generationGate.withPermit(Effect.flatMap(Ref.get(activeMutations), body)),
|
|
1164
1228
|
});
|
|
@@ -1234,6 +1298,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1234
1298
|
const runtime = yield* DurableAgentRuntime;
|
|
1235
1299
|
const recoveryConfig = yield* DurableRuntimeConfig;
|
|
1236
1300
|
const ledger = yield* SubmissionLedger;
|
|
1301
|
+
const listNonterminal = ledger.listNonterminal ?? Stream.runCollect(ledger.scanNonterminal);
|
|
1237
1302
|
const wakes = yield* WakeScheduler;
|
|
1238
1303
|
const hints = yield* DurableAlarmService;
|
|
1239
1304
|
const alarms = yield* ThreadMaintenanceAlarm;
|
|
@@ -1255,6 +1320,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1255
1320
|
const messages = yield* ThreadMessageDelivery;
|
|
1256
1321
|
const host = yield* ThreadHostMaintenance;
|
|
1257
1322
|
const framework = yield* ThreadNativeMaintenance;
|
|
1323
|
+
const placement = yield* Effect.serviceOption(ThreadObjectPlacement);
|
|
1324
|
+
const nativeThread = Option.isSome(placement) ? placement.value.threadId : undefined;
|
|
1258
1325
|
|
|
1259
1326
|
const nativeDispatchConcurrency = () => (framework.singleThreadRuntime === runtime ? 1 : 2);
|
|
1260
1327
|
|
|
@@ -1471,14 +1538,15 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1471
1538
|
);
|
|
1472
1539
|
|
|
1473
1540
|
for (let sequence = range.next; sequence <= range.through; sequence++) {
|
|
1474
|
-
const event = yield* runTransaction("read recovery event", async () =>
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
if (event.sequence !== sequence)
|
|
1478
|
-
throw new Error("Recovery event does not match its sequence key");
|
|
1541
|
+
const event = yield* runTransaction("read recovery event", async () =>
|
|
1542
|
+
decodeRecoveryEvent(await ctx.storage.get(recoveryEventKey(sequence))),
|
|
1543
|
+
);
|
|
1479
1544
|
|
|
1480
|
-
|
|
1481
|
-
|
|
1545
|
+
if (event.sequence !== sequence)
|
|
1546
|
+
return yield* DurableAlarmError.make({
|
|
1547
|
+
operation: "read recovery event",
|
|
1548
|
+
message: "Recovery event does not match its sequence key",
|
|
1549
|
+
});
|
|
1482
1550
|
|
|
1483
1551
|
// No storage reservation or source mutation gate is held during host delivery.
|
|
1484
1552
|
yield* recoveryEvents.publish(event);
|
|
@@ -1521,13 +1589,24 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1521
1589
|
for (const report of result.reports) threads.set(report.threadId, undefined);
|
|
1522
1590
|
for (const report of result.workReports ?? []) threads.set(report.threadId, undefined);
|
|
1523
1591
|
for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
|
|
1524
|
-
|
|
1592
|
+
|
|
1593
|
+
// This pass owns fault writes. Known absence needs no second transaction;
|
|
1594
|
+
// blocked results and retained faults still commit their transitions.
|
|
1595
|
+
if (
|
|
1596
|
+
threads.size === 0 ||
|
|
1597
|
+
[...threads].every(
|
|
1598
|
+
([threadId, failure]) =>
|
|
1599
|
+
failure === undefined &&
|
|
1600
|
+
recovery.loaded.has(threadId) &&
|
|
1601
|
+
!recovery.faults.has(threadId),
|
|
1602
|
+
)
|
|
1603
|
+
)
|
|
1604
|
+
return new Map<ThreadId, ThreadRecoveryFault>();
|
|
1605
|
+
|
|
1525
1606
|
const now = yield* Clock.currentTimeMillis;
|
|
1526
1607
|
|
|
1527
1608
|
// Fault-only discovery uses control state even when retained payloads cannot decode.
|
|
1528
|
-
const submissions =
|
|
1529
|
-
current ??
|
|
1530
|
-
(result.blocked.length === 0 ? [] : yield* Stream.runCollect(ledger.scanNonterminal));
|
|
1609
|
+
const submissions = current ?? (result.blocked.length === 0 ? [] : yield* listNonterminal);
|
|
1531
1610
|
|
|
1532
1611
|
yield* failpoint.hit("maintenance:recovery-status:before");
|
|
1533
1612
|
|
|
@@ -1625,34 +1704,42 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
1625
1704
|
threadId: ThreadId,
|
|
1626
1705
|
recovery: NativeRecovery,
|
|
1627
1706
|
) {
|
|
1628
|
-
|
|
1629
|
-
const
|
|
1630
|
-
|
|
1631
|
-
if (retained === undefined) return undefined;
|
|
1632
|
-
const decoded = Schema.decodeUnknownSync(NativeWorkCursor)(retained);
|
|
1707
|
+
const retained = yield* runTransaction("read work recovery cursor", async () => {
|
|
1708
|
+
const value = await ctx.storage.get(workCursorKey(threadId));
|
|
1633
1709
|
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
return decoded.cursor;
|
|
1710
|
+
return value === undefined
|
|
1711
|
+
? undefined
|
|
1712
|
+
: Schema.decodeUnknownSync(NativeWorkCursor)(value);
|
|
1638
1713
|
});
|
|
1639
1714
|
|
|
1715
|
+
if (retained !== undefined && retained.threadId !== threadId)
|
|
1716
|
+
return yield* DurableAlarmError.make({
|
|
1717
|
+
operation: "read work recovery cursor",
|
|
1718
|
+
message: "Work cursor belongs to another Thread",
|
|
1719
|
+
});
|
|
1720
|
+
let cursor = retained?.cursor;
|
|
1721
|
+
|
|
1640
1722
|
do {
|
|
1641
1723
|
const result = yield* runtime
|
|
1642
|
-
.runRecovery({
|
|
1724
|
+
.runRecovery({
|
|
1725
|
+
threadId,
|
|
1726
|
+
deferUnclaimedAdmissions: true,
|
|
1727
|
+
...(cursor === undefined ? {} : { cursor }),
|
|
1728
|
+
})
|
|
1643
1729
|
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
1644
1730
|
|
|
1645
1731
|
// Persist each bounded pass before a claim; eviction resumes past consumed metadata.
|
|
1646
1732
|
const faults = yield* recordRecoveryFaults(result, recovery, undefined, threadId);
|
|
1647
1733
|
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1734
|
+
if (cursor !== undefined || result.cursor !== undefined)
|
|
1735
|
+
yield* runTransaction("checkpoint work recovery cursor", async () => {
|
|
1736
|
+
if (result.cursor === undefined) await ctx.storage.delete(workCursorKey(threadId));
|
|
1737
|
+
else
|
|
1738
|
+
await ctx.storage.put(
|
|
1739
|
+
workCursorKey(threadId),
|
|
1740
|
+
Schema.encodeSync(NativeWorkCursor)({ threadId, cursor: result.cursor }),
|
|
1741
|
+
);
|
|
1742
|
+
});
|
|
1656
1743
|
for (const report of result.reports) recovery.reports.set(report.submissionId, report);
|
|
1657
1744
|
recovery.faults.delete(threadId);
|
|
1658
1745
|
for (const [id, fault] of faults) recovery.faults.set(id, fault);
|
|
@@ -2081,19 +2168,31 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2081
2168
|
|
|
2082
2169
|
return {
|
|
2083
2170
|
scannedGeneration: state.dirty,
|
|
2084
|
-
|
|
2171
|
+
// Single-Thread placement would immediately wrap past its own cursor.
|
|
2172
|
+
afterWorkThread:
|
|
2173
|
+
framework.singleThreadRuntime === runtime &&
|
|
2174
|
+
state.lastRecoveredThreadId === nativeThread
|
|
2175
|
+
? undefined
|
|
2176
|
+
: state.lastRecoveredThreadId,
|
|
2085
2177
|
};
|
|
2086
2178
|
},
|
|
2087
2179
|
);
|
|
2088
2180
|
|
|
2089
|
-
const current = yield*
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
.
|
|
2181
|
+
const current = yield* listNonterminal;
|
|
2182
|
+
|
|
2183
|
+
// A current admission proves this native placement is materialized. Empty Objects
|
|
2184
|
+
// still discover retained canonical obligations without inventing a recovery owner.
|
|
2185
|
+
let workOwners =
|
|
2186
|
+
framework.singleThreadRuntime === runtime &&
|
|
2187
|
+
nativeThread !== undefined &&
|
|
2188
|
+
current.some((submission) => submission.threadId === nativeThread)
|
|
2189
|
+
? { threadIds: [nativeThread], afterThreadId: undefined }
|
|
2190
|
+
: yield* runtime
|
|
2191
|
+
.discoverWorkThreads({
|
|
2192
|
+
limit: 32,
|
|
2193
|
+
...(afterWorkThread === undefined ? {} : { afterThreadId: afterWorkThread }),
|
|
2194
|
+
})
|
|
2195
|
+
.pipe(Effect.provideService(CurrentNativeSource, true));
|
|
2097
2196
|
|
|
2098
2197
|
if (workOwners.threadIds.length === 0 && afterWorkThread !== undefined)
|
|
2099
2198
|
workOwners = yield* runtime
|
|
@@ -2277,7 +2376,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2277
2376
|
|
|
2278
2377
|
const next = selected.at(-1);
|
|
2279
2378
|
|
|
2280
|
-
if (next !== undefined) {
|
|
2379
|
+
if (next !== undefined && next !== state.lastServedThreadId) {
|
|
2281
2380
|
await transaction.put(
|
|
2282
2381
|
MAINTENANCE_STATE_KEY,
|
|
2283
2382
|
encodeMaintenanceState(
|
|
@@ -2355,7 +2454,7 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2355
2454
|
};
|
|
2356
2455
|
}
|
|
2357
2456
|
|
|
2358
|
-
const remaining = yield*
|
|
2457
|
+
const remaining = yield* listNonterminal;
|
|
2359
2458
|
const waitingHeads = new Map<ThreadId, boolean>();
|
|
2360
2459
|
|
|
2361
2460
|
// Selected recovery must finish before its generation can be acknowledged: terminal
|
|
@@ -2522,12 +2621,22 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2522
2621
|
dispatchUntil: DateTime.Utc,
|
|
2523
2622
|
observed: MaintenanceObservation,
|
|
2524
2623
|
) {
|
|
2624
|
+
const leanSelection =
|
|
2625
|
+
framework.singleThreadRuntime === runtime &&
|
|
2626
|
+
publication === emptyPublication &&
|
|
2627
|
+
host.lanes.length === 0 &&
|
|
2628
|
+
framework.lanes.length === 0;
|
|
2629
|
+
|
|
2525
2630
|
// Subscribe before the first snapshot. Durable enrollment recovers eviction;
|
|
2526
2631
|
// producer completion and wake hints make work visible during this incarnation.
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2632
|
+
// Native source mutations already notify the revision waiter below. A replacement
|
|
2633
|
+
// or shared runtime keeps the generic wake stream; hints confer no authority.
|
|
2634
|
+
const notified = leanSelection
|
|
2635
|
+
? Effect.never
|
|
2636
|
+
: (yield* Stream.toPull(wakes.wakes)).pipe(
|
|
2637
|
+
Effect.asVoid,
|
|
2638
|
+
Effect.catch(() => Effect.never),
|
|
2639
|
+
);
|
|
2531
2640
|
|
|
2532
2641
|
// A wake may observe temporary backoff while this event's recovery is still running.
|
|
2533
2642
|
// Its completion must retain the original actionable observation for acknowledgement.
|
|
@@ -2579,56 +2688,8 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2579
2688
|
discoveryPending: false,
|
|
2580
2689
|
};
|
|
2581
2690
|
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
for (const threadId of yield* Deferred.await(recovery.queue)) {
|
|
2585
|
-
yield* failpoint.hit("maintenance:select:before");
|
|
2586
|
-
yield* runTransaction("select old recovery lane", () =>
|
|
2587
|
-
dueQueue.transaction(async (transaction) => {
|
|
2588
|
-
const { state } = await readMaintenanceState(transaction);
|
|
2589
|
-
|
|
2590
|
-
await transaction.put(
|
|
2591
|
-
MAINTENANCE_STATE_KEY,
|
|
2592
|
-
encodeMaintenanceState(
|
|
2593
|
-
ThreadMaintenanceState.make({
|
|
2594
|
-
...state,
|
|
2595
|
-
lastRecoveredThreadId: threadId,
|
|
2596
|
-
}),
|
|
2597
|
-
),
|
|
2598
|
-
);
|
|
2599
|
-
}),
|
|
2600
|
-
);
|
|
2601
|
-
yield* failpoint.hit("maintenance:select:after");
|
|
2602
|
-
yield* recoverThread(threadId, recovery);
|
|
2603
|
-
recovery.pending.delete(threadId);
|
|
2604
|
-
if (!recovery.faults.has(threadId)) yield* wakes.notify(threadId);
|
|
2605
|
-
}
|
|
2606
|
-
}),
|
|
2607
|
-
);
|
|
2608
|
-
|
|
2609
|
-
// Backfill is one disposable wave; native and host work keep their own opportunities.
|
|
2610
|
-
const backfill = yield* fork(
|
|
2611
|
-
Effect.gen(function* () {
|
|
2612
|
-
if ((yield* mutations.activeLanes).has(DueQueue.Projection)) return;
|
|
2613
|
-
const row = (yield* queueSnapshot).find((row) => row.id === DueQueue.Projection);
|
|
2614
|
-
|
|
2615
|
-
if (
|
|
2616
|
-
row?.dueAt === null ||
|
|
2617
|
-
row === undefined ||
|
|
2618
|
-
row.dueAt > (yield* Clock.currentTimeMillis)
|
|
2619
|
-
)
|
|
2620
|
-
return;
|
|
2621
|
-
|
|
2622
|
-
return yield* runQueued(
|
|
2623
|
-
row,
|
|
2624
|
-
projection.drain.pipe(Effect.andThen(projection.pendingDeadline)),
|
|
2625
|
-
observed,
|
|
2626
|
-
).pipe(Effect.timeoutOption(config.projectionDispatchTimeoutMillis));
|
|
2627
|
-
}),
|
|
2628
|
-
);
|
|
2629
|
-
|
|
2630
|
-
let backfillObserved = false;
|
|
2631
|
-
let recoveryObserved = false;
|
|
2691
|
+
let recoveryFiber: Fiber.Fiber<void, MaintenancePassFailure> | undefined;
|
|
2692
|
+
let backfill: Fiber.Fiber<void, DurableAlarmError | ThreadProjectionError> | undefined;
|
|
2632
2693
|
|
|
2633
2694
|
const dispatch: NativeDispatch = {
|
|
2634
2695
|
scope: auxiliaryScope,
|
|
@@ -2640,18 +2701,79 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2640
2701
|
needsCheckpoint: false,
|
|
2641
2702
|
};
|
|
2642
2703
|
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2704
|
+
// Selection is bounded local work, not an independently owned wave. It opens
|
|
2705
|
+
// actual Attempt scopes, then returns; empty lanes create no fibers.
|
|
2706
|
+
let native: Fiber.Fiber<NativePassResult, MaintenancePassFailure> | undefined;
|
|
2707
|
+
let result: NativePassResult;
|
|
2708
|
+
let backlog: ReadonlyArray<ThreadId> | undefined;
|
|
2646
2709
|
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2710
|
+
if (leanSelection) {
|
|
2711
|
+
result = yield* advance(started, yieldAfter, observed, recovery, dispatch);
|
|
2712
|
+
recovery.started = true;
|
|
2713
|
+
yield* Deferred.succeed(recovery.queue, []);
|
|
2714
|
+
backlog = yield* Deferred.await(recovery.queue);
|
|
2715
|
+
} else {
|
|
2716
|
+
// Required publication and arbitrary host lanes keep independent opportunities
|
|
2717
|
+
// while selection is suspended in an application callback.
|
|
2718
|
+
native = yield* fork(advance(started, yieldAfter, observed, recovery, dispatch));
|
|
2719
|
+
result = {
|
|
2720
|
+
phase: "caught-up",
|
|
2721
|
+
nonterminal: started.nonterminal,
|
|
2722
|
+
nextAttemptAt: undefined,
|
|
2723
|
+
};
|
|
2724
|
+
}
|
|
2725
|
+
|
|
2726
|
+
if (backlog === undefined || backlog.length > 0) {
|
|
2727
|
+
recoveryFiber = yield* fork(
|
|
2728
|
+
Effect.gen(function* () {
|
|
2729
|
+
for (const threadId of backlog ?? (yield* Deferred.await(recovery.queue))) {
|
|
2730
|
+
yield* failpoint.hit("maintenance:select:before");
|
|
2731
|
+
yield* runTransaction("select old recovery lane", () =>
|
|
2732
|
+
dueQueue.transaction(async (transaction) => {
|
|
2733
|
+
const { state } = await readMaintenanceState(transaction);
|
|
2734
|
+
|
|
2735
|
+
await transaction.put(
|
|
2736
|
+
MAINTENANCE_STATE_KEY,
|
|
2737
|
+
encodeMaintenanceState(
|
|
2738
|
+
ThreadMaintenanceState.make({
|
|
2739
|
+
...state,
|
|
2740
|
+
lastRecoveredThreadId: threadId,
|
|
2741
|
+
}),
|
|
2742
|
+
),
|
|
2743
|
+
);
|
|
2744
|
+
}),
|
|
2745
|
+
);
|
|
2746
|
+
yield* failpoint.hit("maintenance:select:after");
|
|
2747
|
+
yield* recoverThread(threadId, recovery);
|
|
2748
|
+
recovery.pending.delete(threadId);
|
|
2749
|
+
if (!recovery.faults.has(threadId)) yield* wakes.notify(threadId);
|
|
2750
|
+
}
|
|
2751
|
+
}),
|
|
2752
|
+
);
|
|
2753
|
+
}
|
|
2754
|
+
const projectionRow = (yield* queueSnapshot).find((row) => row.id === DueQueue.Projection);
|
|
2755
|
+
|
|
2756
|
+
if (
|
|
2757
|
+
projectionRow !== undefined &&
|
|
2758
|
+
projectionRow.dueAt !== null &&
|
|
2759
|
+
projectionRow.dueAt <= (yield* Clock.currentTimeMillis) &&
|
|
2760
|
+
!(yield* mutations.activeLanes).has(DueQueue.Projection)
|
|
2761
|
+
) {
|
|
2762
|
+
backfill = yield* fork(
|
|
2763
|
+
runQueued(
|
|
2764
|
+
projectionRow,
|
|
2765
|
+
projection.drain.pipe(Effect.andThen(projection.pendingDeadline)),
|
|
2766
|
+
observed,
|
|
2767
|
+
).pipe(Effect.timeoutOption(config.projectionDispatchTimeoutMillis), Effect.asVoid),
|
|
2768
|
+
);
|
|
2769
|
+
}
|
|
2652
2770
|
|
|
2653
2771
|
let phase = result.phase;
|
|
2654
|
-
|
|
2772
|
+
|
|
2773
|
+
let nativeCheck =
|
|
2774
|
+
result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency();
|
|
2775
|
+
|
|
2776
|
+
observed.nativeOnly = dispatch.active.size > 0;
|
|
2655
2777
|
const until = DateTime.toEpochMillis(yieldAfter);
|
|
2656
2778
|
const dispatchEnd = DateTime.toEpochMillis(dispatchUntil);
|
|
2657
2779
|
|
|
@@ -2682,19 +2804,17 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2682
2804
|
if (exit.value === 0) dispatch.deferred.add(threadId);
|
|
2683
2805
|
nativeCheck = true;
|
|
2684
2806
|
}
|
|
2685
|
-
if (
|
|
2807
|
+
if (recoveryFiber?.pollUnsafe() !== undefined) {
|
|
2686
2808
|
yield* Fiber.join(recoveryFiber);
|
|
2687
|
-
|
|
2809
|
+
recoveryFiber = undefined;
|
|
2688
2810
|
dispatch.scanGeneration = undefined;
|
|
2689
2811
|
nativeCheck ||= recovery.needsCheckpoint;
|
|
2690
2812
|
}
|
|
2691
|
-
|
|
2692
|
-
const exit = backfill.pollUnsafe();
|
|
2813
|
+
const backfillExit = backfill?.pollUnsafe();
|
|
2693
2814
|
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
}
|
|
2815
|
+
if (backfillExit !== undefined) {
|
|
2816
|
+
backfill = undefined;
|
|
2817
|
+
if (Exit.isFailure(backfillExit)) failure ??= backfillExit.cause;
|
|
2698
2818
|
}
|
|
2699
2819
|
for (const lane of lanes) {
|
|
2700
2820
|
const exit = lane.fiber?.pollUnsafe();
|
|
@@ -2801,9 +2921,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2801
2921
|
if (
|
|
2802
2922
|
native === undefined &&
|
|
2803
2923
|
dispatch.active.size === 0 &&
|
|
2804
|
-
|
|
2924
|
+
recoveryFiber === undefined &&
|
|
2805
2925
|
!recovery.needsCheckpoint &&
|
|
2806
|
-
|
|
2926
|
+
backfill === undefined &&
|
|
2807
2927
|
delivery === undefined &&
|
|
2808
2928
|
lanes.every((lane) => lane.fiber === undefined)
|
|
2809
2929
|
)
|
|
@@ -2816,7 +2936,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2816
2936
|
) {
|
|
2817
2937
|
nativeCheck = false;
|
|
2818
2938
|
const nativeCheckpoint = dispatch.needsCheckpoint && dispatch.active.size === 0;
|
|
2819
|
-
|
|
2939
|
+
|
|
2940
|
+
const checkpoint =
|
|
2941
|
+
(recoveryFiber === undefined && recovery.needsCheckpoint) || nativeCheckpoint;
|
|
2820
2942
|
|
|
2821
2943
|
if (checkpoint) recovery.needsCheckpoint = false;
|
|
2822
2944
|
// Keep this opportunity's reservations across beginNative's storage awaits.
|
|
@@ -2851,26 +2973,39 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2851
2973
|
);
|
|
2852
2974
|
|
|
2853
2975
|
const completing = [
|
|
2854
|
-
|
|
2976
|
+
native,
|
|
2855
2977
|
...dispatch.active.values(),
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
...lanes.
|
|
2860
|
-
];
|
|
2861
|
-
|
|
2862
|
-
const
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2978
|
+
recoveryFiber,
|
|
2979
|
+
backfill,
|
|
2980
|
+
delivery,
|
|
2981
|
+
...lanes.map((lane) => lane.fiber),
|
|
2982
|
+
].filter((fiber) => fiber !== undefined);
|
|
2983
|
+
|
|
2984
|
+
const changed = awaitAnyCompletion(completing, queueRevision);
|
|
2985
|
+
|
|
2986
|
+
// An occupied native-only slot cannot admit another Attempt. With no auxiliary
|
|
2987
|
+
// wake before yield, the timer would only switch to joining that same Attempt.
|
|
2988
|
+
// Incoming work still wakes this callback; all new waves recheck the deadline.
|
|
2989
|
+
const joiningNative =
|
|
2990
|
+
leanSelection &&
|
|
2991
|
+
native === undefined &&
|
|
2992
|
+
dispatch.active.size === 1 &&
|
|
2993
|
+
completing.length === 1 &&
|
|
2994
|
+
!queued.some(
|
|
2995
|
+
(row) =>
|
|
2996
|
+
row.id !== DueQueue.Native &&
|
|
2997
|
+
row.dueAt !== null &&
|
|
2998
|
+
row.dueAt > now &&
|
|
2999
|
+
row.dueAt < until,
|
|
3000
|
+
);
|
|
3001
|
+
|
|
3002
|
+
const ready = yield* joiningNative
|
|
3003
|
+
? changed
|
|
3004
|
+
: Effect.raceAllFirst([
|
|
3005
|
+
changed,
|
|
3006
|
+
...(leanSelection ? [] : [notified.pipe(Effect.as("scan" as const))]),
|
|
3007
|
+
Effect.sleep(Math.max(0, next - now)).pipe(Effect.as("scan" as const)),
|
|
3008
|
+
]);
|
|
2874
3009
|
|
|
2875
3010
|
if (ready === "scan") {
|
|
2876
3011
|
nativeCheck = true;
|
|
@@ -2896,9 +3031,10 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2896
3031
|
for (const fiber of [
|
|
2897
3032
|
recoveryFiber,
|
|
2898
3033
|
backfill,
|
|
2899
|
-
|
|
2900
|
-
...lanes.
|
|
3034
|
+
delivery,
|
|
3035
|
+
...lanes.map((lane) => lane.fiber),
|
|
2901
3036
|
]) {
|
|
3037
|
+
if (fiber === undefined) continue;
|
|
2902
3038
|
// Await exits without failing fast: every admitted wave owns its original
|
|
2903
3039
|
// bounded allowance and cleanup even if another wave has already failed.
|
|
2904
3040
|
const exit = yield* Fiber.await<unknown, MaintenancePassFailure>(fiber);
|
|
@@ -2921,7 +3057,9 @@ export class ThreadMaintenance extends Context.Service<
|
|
|
2921
3057
|
observed: MaintenanceObservation,
|
|
2922
3058
|
): Effect.fn.Return<MaintenancePassReport, MaintenancePassFailure, Scope.Scope> {
|
|
2923
3059
|
const concurrent = yield* Effect.exit(
|
|
2924
|
-
Effect.scoped(dispatch(yieldAfter, dispatchUntil, observed))
|
|
3060
|
+
Effect.scoped(dispatch(yieldAfter, dispatchUntil, observed)).pipe(
|
|
3061
|
+
Effect.provideService(ThreadMutationGate, mutations),
|
|
3062
|
+
),
|
|
2925
3063
|
);
|
|
2926
3064
|
|
|
2927
3065
|
// Closing the concurrent scope first releases every native Attempt and its resources.
|