@yielded/agent-platform-cloudflare 0.1.0-beta.166 → 0.1.0-beta.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{Alarm-B0ODr1NE.mjs → Alarm-DxODgAZZ.mjs} +259 -159
- package/dist/Alarm-DxODgAZZ.mjs.map +1 -0
- package/dist/Alarm.d.mts +27 -36
- package/dist/Alarm.mjs +2 -2
- package/dist/BrowserCredentials-C5IWfi7f.d.mts +91 -0
- package/dist/BrowserCredentials.d.mts +2 -91
- package/dist/BrowserRestCapture.mjs +3 -3
- package/dist/BrowserRestCapture.mjs.map +1 -1
- package/dist/BrowserRestCrawl.mjs +6 -6
- package/dist/BrowserRestCrawl.mjs.map +1 -1
- package/dist/{BrowserSession-RfIzxGCH.mjs → BrowserSession-BfNmyayH.mjs} +5 -5
- package/dist/BrowserSession-BfNmyayH.mjs.map +1 -0
- package/dist/{BrowserSession-B969Xg-2.d.mts → BrowserSession-Bx0V1-6_.d.mts} +6 -3
- package/dist/BrowserSession.d.mts +1 -1
- package/dist/BrowserSession.mjs +2 -2
- package/dist/BrowserUse-AstK0jKX.mjs +1467 -0
- package/dist/BrowserUse-AstK0jKX.mjs.map +1 -0
- package/dist/BrowserUse.d.mts +83 -0
- package/dist/BrowserUse.mjs +2 -0
- package/dist/CloudflareBindings.d.mts +8 -1
- package/dist/CloudflareBindings.mjs +2 -2
- package/dist/CloudflareBindings.mjs.map +1 -1
- package/dist/{CloudflareBrowser-5an6m-Qb.mjs → CloudflareBrowser-Vt8qTdYW.mjs} +11 -13
- package/dist/CloudflareBrowser-Vt8qTdYW.mjs.map +1 -0
- package/dist/CloudflareBrowser.mjs +1 -1
- package/dist/CloudflareCodeMode.mjs +1 -1
- package/dist/CloudflareCodeMode.mjs.map +1 -1
- package/dist/{CloudflareConfig-qDrRLofL.d.mts → CloudflareConfig-C9MvPbX8.d.mts} +9 -11
- package/dist/CloudflareConfig.d.mts +1 -1
- package/dist/CloudflareConfig.mjs +9 -12
- package/dist/CloudflareConfig.mjs.map +1 -1
- package/dist/CloudflareMemory.d.mts +3 -25
- package/dist/CloudflareMemory.mjs +33 -31
- package/dist/CloudflareMemory.mjs.map +1 -1
- package/dist/CloudflareScheduling.mjs +8 -15
- package/dist/CloudflareScheduling.mjs.map +1 -1
- package/dist/CloudflareSubscriptions.mjs +9 -16
- package/dist/CloudflareSubscriptions.mjs.map +1 -1
- package/dist/CloudflareThreadClient-CZm0TvUe.mjs +588 -0
- package/dist/CloudflareThreadClient-CZm0TvUe.mjs.map +1 -0
- package/dist/CloudflareThreadClient-QsVzLGGZ.d.mts +2033 -0
- package/dist/CloudflareThreadClient.d.mts +2 -1796
- package/dist/CloudflareThreadClient.mjs +2 -382
- package/dist/{InteractiveBrowser-BbG3kGA9.mjs → InteractiveBrowser-LdKzQeLR.mjs} +18 -19
- package/dist/InteractiveBrowser-LdKzQeLR.mjs.map +1 -0
- package/dist/InteractiveBrowser.mjs +1 -1
- package/dist/{ThreadObject-BcfXbzSO.d.mts → ThreadObject-7s5XnlH-.d.mts} +103 -79
- package/dist/{ThreadObject-DHjrnlX4.mjs → ThreadObject-C0vrcDrw.mjs} +265 -66
- package/dist/ThreadObject-C0vrcDrw.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +2 -2
- package/dist/ThreadObject.mjs +2 -3
- package/dist/WakeScheduler.d.mts +6 -5
- package/dist/WakeScheduler.mjs +15 -11
- package/dist/WakeScheduler.mjs.map +1 -1
- package/dist/index.d.mts +7 -6
- package/dist/index.mjs +7 -6
- package/dist/page-function-CmUCcyNP.mjs +11 -0
- package/dist/page-function-CmUCcyNP.mjs.map +1 -0
- package/dist/{prepared-admission-DMX5p9dC.mjs → prepared-admission-3UUq0zth.mjs} +3 -3
- package/dist/{prepared-admission-DMX5p9dC.mjs.map → prepared-admission-3UUq0zth.mjs.map} +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +510 -279
- package/src/BrowserRestCapture.ts +2 -2
- package/src/BrowserRestCrawl.ts +10 -14
- package/src/BrowserSession.ts +10 -2
- package/src/BrowserUse.ts +1541 -0
- package/src/CloudflareBindings.ts +13 -4
- package/src/CloudflareCodeMode.ts +1 -1
- package/src/CloudflareConfig.ts +9 -12
- package/src/CloudflareMemory.ts +21 -28
- package/src/CloudflareScheduling.ts +7 -15
- package/src/CloudflareSubscriptions.ts +11 -20
- package/src/CloudflareThreadClient.ts +289 -9
- package/src/InteractiveBrowser.ts +325 -323
- package/src/ThreadObject.ts +172 -52
- package/src/WakeScheduler.ts +29 -18
- package/src/index.ts +1 -0
- package/src/internal/browser-dom.ts +845 -0
- package/src/internal/browser-file-selection.ts +8 -1
- package/src/internal/browser-jev-dom.ts +344 -0
- package/src/internal/browser-quick-action.ts +7 -8
- package/src/internal/browser-session-lifecycle.ts +2 -5
- package/src/internal/crypto.ts +41 -0
- package/src/internal/due-queue.ts +5 -69
- package/src/internal/layers.ts +100 -17
- package/src/internal/live-text-protocol.ts +51 -0
- package/src/internal/live-text.ts +256 -0
- package/src/internal/page-function.ts +12 -0
- package/src/internal/prepared-admission.ts +1 -1
- package/src/internal/progress-wait.ts +19 -24
- package/src/internal/transport.ts +1 -1
- package/dist/Alarm-B0ODr1NE.mjs.map +0 -1
- package/dist/BrowserSession-RfIzxGCH.mjs.map +0 -1
- package/dist/CloudflareBrowser-5an6m-Qb.mjs.map +0 -1
- package/dist/CloudflareThreadClient.mjs.map +0 -1
- package/dist/InteractiveBrowser-BbG3kGA9.mjs.map +0 -1
- package/dist/ThreadObject-DHjrnlX4.mjs.map +0 -1
|
@@ -2,6 +2,7 @@ import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
|
2
2
|
import { DurableObjectContext } from "./CloudflareBindings.mjs";
|
|
3
3
|
import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.mjs";
|
|
4
4
|
import { r as safeCauseMessage } from "./boundary-BguazVkh.mjs";
|
|
5
|
+
import { CurrentNativeSource } from "@yielded/agent-storage-cloudflare/port-routing";
|
|
5
6
|
import "@yielded/agent/agent-registration";
|
|
6
7
|
import { DurableAgentRuntime, DurableRuntimeConfig, RecoveryBlocked, RecoveryFailure, RecoverySweepResult, isolateRecovery } from "@yielded/agent/durable-agent-runtime";
|
|
7
8
|
import { SubmissionId, ThreadId } from "@yielded/agent/identifiers";
|
|
@@ -9,7 +10,7 @@ import { AbortIntentRequest, SubmissionLedger } from "@yielded/agent/submission-
|
|
|
9
10
|
import { ThreadProjectionMaintenance } from "@yielded/agent/thread-projection-maintenance";
|
|
10
11
|
import { WakeScheduler } from "@yielded/agent/wake-scheduler";
|
|
11
12
|
import { Cause, Clock, Context, DateTime, Deferred, Effect, ErrorReporter, Exit, Fiber, Layer, Option, Random, Ref, Result, Scheduler, Schema, Scope, Semaphore, Stream, Struct } from "effect";
|
|
12
|
-
import { DurableObjectStorage } from "effect-cf";
|
|
13
|
+
import { DurableObjectAlarm, DurableObjectStorage } from "effect-cf";
|
|
13
14
|
import { SqlClient } from "effect/sql/SqlClient";
|
|
14
15
|
import { SqlError, UnknownError } from "effect/sql/SqlError";
|
|
15
16
|
//#region src/internal/due-queue.ts
|
|
@@ -134,34 +135,6 @@ const make = (storage) => {
|
|
|
134
135
|
const changes = [...current.changes.values()].filter(({ before, after }) => before === void 0 || !equivalent(before, after));
|
|
135
136
|
for (let offset = 0; offset < changes.length; offset += 11) write(changes.slice(offset, offset + 11));
|
|
136
137
|
};
|
|
137
|
-
/** Own the flush inside the source transaction without changing the SQL service.
|
|
138
|
-
* Acquire its shared connection permit through Effect requirements. Cleanup follows
|
|
139
|
-
* native settlement, including commit rejection or interruption. */
|
|
140
|
-
const withTransaction = (body) => Effect.flatMap(SqlClient, (client) => Effect.flatMap(Effect.serviceOption(client.transactionService), (ambient) => {
|
|
141
|
-
if (ambient._tag === "Some" && !ownsTransaction(storage, ambient.value[1])) return Effect.fail(SqlError.make({ reason: UnknownError.make({
|
|
142
|
-
operation: "begin maintenance source transaction",
|
|
143
|
-
message: "Use ThreadMutationGate.withTransaction at the outermost source transaction",
|
|
144
|
-
cause: /* @__PURE__ */ new Error("Unwrapped SQL transaction cannot own buffered maintenance intent")
|
|
145
|
-
}) }));
|
|
146
|
-
return Effect.uninterruptibleMask((restore) => Effect.suspend(() => {
|
|
147
|
-
let frame;
|
|
148
|
-
return client.withTransaction(Effect.flatMap(Effect.serviceOption(client.transactionService), (transaction) => Effect.sync(() => {
|
|
149
|
-
if (transaction._tag === "None") throw new Error("SQL transaction unavailable");
|
|
150
|
-
frame = begin(transaction.value[1]);
|
|
151
|
-
})).pipe(Effect.andThen(restore(body)), Effect.tap(() => Effect.try({
|
|
152
|
-
try: () => {
|
|
153
|
-
if (frame !== void 0) flush(frame);
|
|
154
|
-
},
|
|
155
|
-
catch: (cause) => SqlError.make({ reason: UnknownError.make({
|
|
156
|
-
cause,
|
|
157
|
-
operation: "flush maintenance due queue",
|
|
158
|
-
message: "Maintenance intent could not commit"
|
|
159
|
-
}) })
|
|
160
|
-
})))).pipe(Effect.onExit((exit) => Effect.sync(() => {
|
|
161
|
-
if (frame !== void 0) finish(frame, Exit.isSuccess(exit));
|
|
162
|
-
})), Effect.provideService(Scheduler.PreventSchedulerYield, true));
|
|
163
|
-
}));
|
|
164
|
-
}));
|
|
165
138
|
const transaction = async (body) => {
|
|
166
139
|
let frame;
|
|
167
140
|
try {
|
|
@@ -308,7 +281,9 @@ const make = (storage) => {
|
|
|
308
281
|
return {
|
|
309
282
|
initialize,
|
|
310
283
|
withView,
|
|
311
|
-
|
|
284
|
+
begin,
|
|
285
|
+
flush,
|
|
286
|
+
finish,
|
|
312
287
|
transaction,
|
|
313
288
|
read,
|
|
314
289
|
dirty,
|
|
@@ -329,6 +304,7 @@ var Alarm_exports = /* @__PURE__ */ __exportAll({
|
|
|
329
304
|
MaintenancePassReport: () => MaintenancePassReport,
|
|
330
305
|
ThreadHostMaintenance: () => ThreadHostMaintenance,
|
|
331
306
|
ThreadMaintenance: () => ThreadMaintenance,
|
|
307
|
+
ThreadMaintenanceAlarm: () => ThreadMaintenanceAlarm,
|
|
332
308
|
ThreadMaintenanceFailpoint: () => ThreadMaintenanceFailpoint,
|
|
333
309
|
ThreadMessageDelivery: () => ThreadMessageDelivery,
|
|
334
310
|
ThreadMutationGate: () => ThreadMutationGate,
|
|
@@ -339,10 +315,8 @@ var Alarm_exports = /* @__PURE__ */ __exportAll({
|
|
|
339
315
|
publishCommitted: () => publishCommitted
|
|
340
316
|
});
|
|
341
317
|
/**
|
|
342
|
-
* The
|
|
343
|
-
* alarm slot
|
|
344
|
-
* and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
|
|
345
|
-
* the slot always holds the EARLIEST deadline any caller asked for.
|
|
318
|
+
* The maintenance queue owns lease expiry, settlement, retries and dirty generations.
|
|
319
|
+
* Its manual effect-cf alarm shares the native slot with other registered alarms.
|
|
346
320
|
*
|
|
347
321
|
* The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
|
|
348
322
|
* maintenance generation and a committed alarm. Stable externally-driven waits may be
|
|
@@ -382,41 +356,123 @@ const makeStorageOperation = Effect.map(makeStorageEffect, (run) => (operation,
|
|
|
382
356
|
try: execute,
|
|
383
357
|
catch: alarmFailure(operation)
|
|
384
358
|
})));
|
|
385
|
-
/**
|
|
359
|
+
/** One manual alarm for the entire consumer-owned maintenance queue. */
|
|
360
|
+
var ThreadMaintenanceAlarm = class extends DurableObjectAlarm.Tag()("@yielded/agent/thread-maintenance", { "@yielded/agent/thread-maintenance": {
|
|
361
|
+
payload: Schema.Null,
|
|
362
|
+
lifecycle: "manual"
|
|
363
|
+
} }) {};
|
|
364
|
+
const maintenanceAlarmRef = {
|
|
365
|
+
tag: "@yielded/agent/thread-maintenance",
|
|
366
|
+
id: "queue"
|
|
367
|
+
};
|
|
368
|
+
const maintenanceAlarmInput = (deadline) => ({
|
|
369
|
+
...maintenanceAlarmRef,
|
|
370
|
+
payload: null,
|
|
371
|
+
runAt: DateTime.makeUnsafe(deadline)
|
|
372
|
+
});
|
|
373
|
+
/** Named maintenance checkpoint; the library owns the shared native alarm slot. */
|
|
386
374
|
var DurableAlarmService = class DurableAlarmService extends Context.Service()("@effect-agent/platform-cloudflare/DurableAlarmService") {
|
|
387
375
|
static layer = Layer.effect(DurableAlarmService)(Effect.gen(function* () {
|
|
376
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
388
377
|
const { ctx } = yield* DurableObjectContext;
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
* on top of the already-committed pre-armed alarm.
|
|
393
|
-
*/
|
|
394
|
-
const runningPasses = yield* Ref.make(0);
|
|
395
|
-
const storageOperation = yield* makeStorageOperation;
|
|
396
|
-
const scheduled = storageOperation("get alarm", () => ctx.storage.getAlarm()).pipe(Effect.map((deadline) => deadline === null ? Option.none() : Option.some(deadline)));
|
|
397
|
-
const scheduleAt = (epochMillis) => storageOperation("set alarm", () => ctx.storage.setAlarm(epochMillis));
|
|
398
|
-
const ensureScheduledBy = (epochMillis) => storageOperation("ensure alarm", () => ctx.storage.transaction(async (transaction) => {
|
|
399
|
-
const existing = await transaction.getAlarm();
|
|
400
|
-
if (existing === null || existing > epochMillis) await transaction.setAlarm(epochMillis);
|
|
401
|
-
}));
|
|
402
|
-
const armNow = Clock.currentTimeMillis.pipe(Effect.flatMap((now) => storageOperation("wake maintenance lanes", () => ctx.storage.transaction(async (transaction) => {
|
|
403
|
-
const next$7 = next(make(ctx.storage).read());
|
|
404
|
-
if (Number.isFinite(next$7)) await ensureTransactionAlarmBy(transaction, Math.max(now, next$7));
|
|
405
|
-
}))));
|
|
406
|
-
const scheduleNow = Ref.get(runningPasses).pipe(Effect.flatMap((passes) => passes > 0 ? Effect.void : armNow));
|
|
407
|
-
const withWakesDeferred = (body) => Ref.update(runningPasses, (passes) => passes + 1).pipe(Effect.andThen(body), Effect.ensuring(Ref.update(runningPasses, (passes) => passes - 1)));
|
|
408
|
-
const cancel = storageOperation("delete alarm", () => ctx.storage.deleteAlarm());
|
|
378
|
+
const run = yield* makeStorageEffect;
|
|
379
|
+
const inlineWork = yield* Ref.make(0);
|
|
380
|
+
const wrap = (operation, effect) => run(operation, effect.pipe(Effect.mapError(alarmFailure(operation))));
|
|
409
381
|
return DurableAlarmService.of({
|
|
410
|
-
scheduled,
|
|
411
|
-
scheduleAt,
|
|
412
|
-
ensureScheduledBy,
|
|
413
|
-
scheduleNow,
|
|
414
|
-
|
|
415
|
-
|
|
382
|
+
scheduled: wrap("read maintenance checkpoint", alarms.getAlarmStatus(maintenanceAlarmRef)).pipe(Effect.map((at) => at === void 0 ? Option.none() : Option.some(DateTime.toEpochMillis(at.runAt)))),
|
|
383
|
+
scheduleAt: (at) => wrap("replace maintenance checkpoint", alarms.scheduleAlarm(maintenanceAlarmInput(at))),
|
|
384
|
+
ensureScheduledBy: (at) => wrap("advance maintenance checkpoint", alarms.scheduleAlarmEarlier(maintenanceAlarmInput(at))),
|
|
385
|
+
scheduleNow: Ref.get(inlineWork).pipe(Effect.flatMap((active) => active > 0 ? Effect.void : wrap("hint maintenance checkpoint", alarms.transaction((tx) => Effect.gen(function* () {
|
|
386
|
+
const now = yield* Clock.currentTimeMillis;
|
|
387
|
+
const next$4 = next(make(ctx.storage).read());
|
|
388
|
+
if (Number.isFinite(next$4)) yield* tx.scheduleAlarmEarlier(maintenanceAlarmInput(Math.max(now, next$4)));
|
|
389
|
+
}))))),
|
|
390
|
+
withHintsDeferred: (body) => Effect.acquireUseRelease(Ref.update(inlineWork, (active) => active + 1), () => body, () => Ref.update(inlineWork, (active) => active - 1)),
|
|
391
|
+
cancel: wrap("cancel maintenance checkpoint", alarms.cancelAlarm(maintenanceAlarmRef))
|
|
416
392
|
});
|
|
417
393
|
}));
|
|
418
394
|
};
|
|
419
|
-
|
|
395
|
+
const CurrentCheckpoint = Context.Reference("@yielded/agent/cloudflare/CurrentCheckpoint", { defaultValue: () => void 0 });
|
|
396
|
+
/** One source transaction owns queue intent and the library checkpoint through commit. */
|
|
397
|
+
const makeAlarmOperation = Effect.gen(function* () {
|
|
398
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
399
|
+
const sql = yield* SqlClient;
|
|
400
|
+
const { ctx } = yield* DurableObjectContext;
|
|
401
|
+
const queue = make(ctx.storage);
|
|
402
|
+
const transaction = (body) => alarms.transaction((mutations) => Effect.withFiber((owner) => {
|
|
403
|
+
let active = true;
|
|
404
|
+
const requireActive = Effect.withFiber((fiber) => active && fiber === owner ? Effect.void : Effect.fail(DurableAlarmError.make({
|
|
405
|
+
operation: "enroll maintenance source",
|
|
406
|
+
message: "Source intent requires its active transaction callback's fiber"
|
|
407
|
+
})));
|
|
408
|
+
return body.pipe(Effect.provideService(CurrentCheckpoint, {
|
|
409
|
+
mutations,
|
|
410
|
+
requireActive
|
|
411
|
+
}), Effect.ensuring(Effect.sync(() => {
|
|
412
|
+
active = false;
|
|
413
|
+
})));
|
|
414
|
+
}), { sqlClient: sql }).pipe(Effect.mapError((cause) => cause instanceof DurableObjectStorage.StorageOperationError ? SqlError.make({ reason: UnknownError.make({
|
|
415
|
+
operation: "commit maintenance checkpoint",
|
|
416
|
+
message: "Source facts and maintenance checkpoint could not commit",
|
|
417
|
+
cause
|
|
418
|
+
}) }) : cause));
|
|
419
|
+
const withTransaction = (body) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (ambient) => {
|
|
420
|
+
if (ambient._tag === "Some" && !ownsTransaction(ctx.storage, ambient.value[1])) return Effect.fail(SqlError.make({ reason: UnknownError.make({
|
|
421
|
+
operation: "begin maintenance source transaction",
|
|
422
|
+
message: "Use ThreadMutationGate.withTransaction at the outermost source transaction",
|
|
423
|
+
cause: /* @__PURE__ */ new Error("Unwrapped SQL transaction cannot own buffered maintenance intent")
|
|
424
|
+
}) }));
|
|
425
|
+
return Effect.uninterruptibleMask((restore) => Effect.suspend(() => {
|
|
426
|
+
let frame;
|
|
427
|
+
return transaction(Effect.flatMap(Effect.serviceOption(sql.transactionService), (transaction) => Effect.sync(() => {
|
|
428
|
+
if (transaction._tag === "None") throw new Error("SQL transaction unavailable");
|
|
429
|
+
frame = queue.begin(transaction.value[1]);
|
|
430
|
+
})).pipe(Effect.andThen(restore(body)), Effect.tap(() => Effect.try({
|
|
431
|
+
try: () => {
|
|
432
|
+
if (frame !== void 0) queue.flush(frame);
|
|
433
|
+
},
|
|
434
|
+
catch: (cause) => SqlError.make({ reason: UnknownError.make({
|
|
435
|
+
cause,
|
|
436
|
+
operation: "flush maintenance due queue",
|
|
437
|
+
message: "Maintenance intent could not commit"
|
|
438
|
+
}) })
|
|
439
|
+
})))).pipe(Effect.onExit((exit) => Effect.sync(() => {
|
|
440
|
+
if (frame !== void 0) queue.finish(frame, Exit.isSuccess(exit));
|
|
441
|
+
})), Effect.provideService(Scheduler.PreventSchedulerYield, true));
|
|
442
|
+
}));
|
|
443
|
+
}).pipe(Effect.provideService(SqlClient, sql));
|
|
444
|
+
const run = (operation, execute) => {
|
|
445
|
+
const commit = Effect.flatMap(CurrentCheckpoint, (tx) => {
|
|
446
|
+
if (tx === void 0) return Effect.die(/* @__PURE__ */ new Error("Maintenance checkpoint transaction unavailable"));
|
|
447
|
+
return tx.requireActive.pipe(Effect.andThen(Effect.suspend(() => {
|
|
448
|
+
let checkpoint = Effect.void;
|
|
449
|
+
const intent = {
|
|
450
|
+
at: (at) => {
|
|
451
|
+
checkpoint = tx.mutations.scheduleAlarm(maintenanceAlarmInput(at));
|
|
452
|
+
},
|
|
453
|
+
earlier: (at) => {
|
|
454
|
+
checkpoint = tx.mutations.scheduleAlarmEarlier(maintenanceAlarmInput(at));
|
|
455
|
+
},
|
|
456
|
+
cancel: () => {
|
|
457
|
+
checkpoint = tx.mutations.cancelAlarm(maintenanceAlarmRef);
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
return Effect.tryPromise({
|
|
461
|
+
try: () => execute(ctx.storage, intent),
|
|
462
|
+
catch: alarmFailure(operation)
|
|
463
|
+
}).pipe(Effect.tap(() => checkpoint));
|
|
464
|
+
})));
|
|
465
|
+
});
|
|
466
|
+
return Effect.gen(function* () {
|
|
467
|
+
if ((yield* CurrentCheckpoint) !== void 0) return yield* commit;
|
|
468
|
+
return yield* (yield* Effect.serviceOption(sql.transactionService))._tag === "Some" && !buffering(ctx.storage) ? transaction(commit).pipe(Effect.ensuring(Effect.sync(() => invalidate(ctx.storage)))) : withTransaction(commit);
|
|
469
|
+
}).pipe(Effect.uninterruptible, Effect.mapError(alarmFailure(operation)));
|
|
470
|
+
};
|
|
471
|
+
return {
|
|
472
|
+
run,
|
|
473
|
+
withTransaction
|
|
474
|
+
};
|
|
475
|
+
});
|
|
420
476
|
var MaintenancePassReport = class extends Schema.Class("@effect-agent/platform-cloudflare/MaintenancePassReport")({
|
|
421
477
|
/** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
|
|
422
478
|
phase: Schema.Literals(["caught-up", "actionable"]),
|
|
@@ -549,9 +605,13 @@ const recoveryEvent = (fault, submissionId, transition, occurredAt) => ({
|
|
|
549
605
|
firstFailedAt: fault.firstFailedAt,
|
|
550
606
|
failure: fault.failure
|
|
551
607
|
});
|
|
552
|
-
const nativeDispatchConcurrency = 2;
|
|
553
608
|
/** Post-native delivery shares a finite connection budget across independent host lanes. */
|
|
554
609
|
const afterNativeDispatchConcurrency = 2;
|
|
610
|
+
const NativeWorkCursor = Schema.Struct({
|
|
611
|
+
threadId: ThreadId,
|
|
612
|
+
cursor: Schema.NonEmptyString.check(Schema.isMaxLength(8192))
|
|
613
|
+
});
|
|
614
|
+
const workCursorKey = (threadId) => `effect-agent:work-recovery-cursor:v1:${threadId}`;
|
|
555
615
|
var MaintenanceRetry = class extends Schema.Class("MaintenanceRetry")({
|
|
556
616
|
generation: MaintenanceGeneration,
|
|
557
617
|
notBefore: Schema.Finite,
|
|
@@ -597,10 +657,6 @@ const readMaintenanceState = async (transaction) => {
|
|
|
597
657
|
initialized: true
|
|
598
658
|
};
|
|
599
659
|
};
|
|
600
|
-
const ensureTransactionAlarmBy = async (transaction, deadline) => {
|
|
601
|
-
const scheduled = await transaction.getAlarm();
|
|
602
|
-
if (scheduled === null || scheduled > deadline) await transaction.setAlarm(deadline);
|
|
603
|
-
};
|
|
604
660
|
const stableExternalWait = (snapshot, reports) => {
|
|
605
661
|
const report = reports.get(snapshot.submissionId);
|
|
606
662
|
if (report?.decision._tag === "SettleAborted") return false;
|
|
@@ -612,9 +668,7 @@ const stableExternalWait = (snapshot, reports) => {
|
|
|
612
668
|
case "input-applied":
|
|
613
669
|
case "joining":
|
|
614
670
|
case "ready":
|
|
615
|
-
case "
|
|
616
|
-
case "settled":
|
|
617
|
-
case "terminalizing": return false;
|
|
671
|
+
case "settled": return false;
|
|
618
672
|
}
|
|
619
673
|
};
|
|
620
674
|
/**
|
|
@@ -623,7 +677,6 @@ const stableExternalWait = (snapshot, reports) => {
|
|
|
623
677
|
* Layers must reuse that instance; a second gate cannot observe the native producers' activity.
|
|
624
678
|
*/
|
|
625
679
|
const CurrentMutationLanes = Context.Reference("@effect-agent/platform-cloudflare/CurrentMutationLanes", { defaultValue: () => [] });
|
|
626
|
-
const CurrentNativeSource = Context.Reference("@effect-agent/platform-cloudflare/CurrentNativeSource", { defaultValue: () => false });
|
|
627
680
|
var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@effect-agent/platform-cloudflare/internal/ThreadMutationGate") {
|
|
628
681
|
static layer = Layer.effect(this)(Effect.gen(function* () {
|
|
629
682
|
const { ctx } = yield* DurableObjectContext;
|
|
@@ -644,6 +697,7 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
|
|
|
644
697
|
});
|
|
645
698
|
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
646
699
|
const runTransaction = yield* makeStorageOperation;
|
|
700
|
+
const { run: runAlarmTransaction, withTransaction } = yield* makeAlarmOperation;
|
|
647
701
|
const validateSourceBoundary = Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => current._tag === "Some" && buffering(ctx.storage) && !ownsTransaction(ctx.storage, current.value[1]) ? Effect.fail(DurableAlarmError.make({
|
|
648
702
|
operation: "enroll maintenance source",
|
|
649
703
|
message: "Nested source scheduling requires ThreadMutationGate.withTransaction"
|
|
@@ -652,11 +706,10 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
|
|
|
652
706
|
dueQueue.initialize();
|
|
653
707
|
dueQueue.register(Native);
|
|
654
708
|
});
|
|
655
|
-
const schedule = Effect.
|
|
709
|
+
const schedule = Effect.fnUntraced(function* (id, dueAt, progressCursor) {
|
|
656
710
|
yield* validateSourceBoundary;
|
|
657
711
|
const now = yield* Clock.currentTimeMillis;
|
|
658
|
-
const
|
|
659
|
-
const enroll = async (transaction) => {
|
|
712
|
+
const enroll = async (transaction, checkpoint) => {
|
|
660
713
|
if (progressCursor === void 0) dueQueue.dirty(id, dueAt);
|
|
661
714
|
else if (dueQueue.progress(id, dueAt, progressCursor) && (id === "effect-agent:native" || id === "effect-agent:publication")) {
|
|
662
715
|
if (id === "effect-agent:publication") dueQueue.dirty(Native, dueAt, true);
|
|
@@ -666,32 +719,31 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
|
|
|
666
719
|
dirty: state.dirty + 1n
|
|
667
720
|
})));
|
|
668
721
|
}
|
|
669
|
-
const next$
|
|
670
|
-
if (Number.isFinite(next$
|
|
722
|
+
const next$6 = next(dueQueue.read());
|
|
723
|
+
if (Number.isFinite(next$6)) checkpoint.earlier(Math.max(next$6, now + minimumAlarmDelay));
|
|
671
724
|
};
|
|
672
|
-
yield*
|
|
725
|
+
yield* runAlarmTransaction("schedule maintenance lane", enroll);
|
|
673
726
|
yield* notify;
|
|
674
727
|
});
|
|
675
|
-
const beginMutation = Effect.
|
|
728
|
+
const beginMutation = Effect.fnUntraced(function* (invalidatesRecovery, lanes) {
|
|
676
729
|
yield* failpoint.hit("maintenance:dirty:before");
|
|
677
730
|
const now = yield* Clock.currentTimeMillis;
|
|
678
731
|
const nativeSource = yield* CurrentNativeSource;
|
|
679
732
|
const publicationRequired = yield* requiresPublication;
|
|
680
|
-
if (invalidatesRecovery || lanes.length > 0) yield*
|
|
733
|
+
if (invalidatesRecovery || lanes.length > 0) yield* runAlarmTransaction("advance maintenance generation", async (transaction, checkpoint) => {
|
|
681
734
|
const { state, initialized } = await readMaintenanceState(transaction);
|
|
682
735
|
if (invalidatesRecovery) {
|
|
683
736
|
dueQueue.dirty(Native, now);
|
|
684
737
|
if (publicationRequired) dueQueue.dirty(Publication, now);
|
|
685
738
|
}
|
|
686
739
|
for (const id of lanes) dueQueue.dirty(id, now, false);
|
|
687
|
-
|
|
740
|
+
if (!initialized || invalidatesRecovery && !nativeSource) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
688
741
|
...state,
|
|
689
742
|
dirty: state.dirty + (invalidatesRecovery && !nativeSource ? 1n : 0n)
|
|
690
|
-
});
|
|
691
|
-
if (invalidatesRecovery || !initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next$1));
|
|
743
|
+
})));
|
|
692
744
|
const deadline = next(dueQueue.read());
|
|
693
|
-
if (Number.isFinite(deadline))
|
|
694
|
-
})
|
|
745
|
+
if (Number.isFinite(deadline)) checkpoint.earlier(Math.max(deadline, now + minimumAlarmDelay));
|
|
746
|
+
});
|
|
695
747
|
yield* failpoint.hit("maintenance:dirty:after");
|
|
696
748
|
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
697
749
|
const enrolled = /* @__PURE__ */ new Set([...lanes, ...invalidatesRecovery ? [Native, ...publicationRequired ? [Publication] : []] : []]);
|
|
@@ -712,7 +764,7 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
|
|
|
712
764
|
operation: "prearm maintenance mutation",
|
|
713
765
|
message: "Run withMutation outside the source SQL transaction so prearming commits first"
|
|
714
766
|
})) : Effect.acquireUseRelease(generationGate.withPermit(beginMutation(options?.invalidatesRecovery ?? true, options?.lanes ?? [])), (enrolled) => failpoint.hit("maintenance:mutation:armed").pipe(Effect.andThen(Effect.flatMap(CurrentMutationLanes, (current) => body.pipe(Effect.provideService(CurrentMutationLanes, [.../* @__PURE__ */ new Set([...current, ...enrolled])])))), Effect.tap(() => releaseLanes(enrolled, true).pipe(Effect.andThen(notify))), Effect.tap(() => failpoint.hit("maintenance:mutation:finished"))), (enrolled) => endMutation(enrolled).pipe(Effect.andThen(notify))));
|
|
715
|
-
const recordProgress = Effect.
|
|
767
|
+
const recordProgress = Effect.fnUntraced(function* (lanes) {
|
|
716
768
|
yield* validateSourceBoundary;
|
|
717
769
|
if (Option.isNone(yield* Effect.serviceOption(sql.transactionService))) return yield* DurableAlarmError.make({
|
|
718
770
|
operation: "record source progress",
|
|
@@ -724,21 +776,21 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
|
|
|
724
776
|
if (affected.size === 0) return;
|
|
725
777
|
const now = yield* Clock.currentTimeMillis;
|
|
726
778
|
const nativeSource = yield* CurrentNativeSource;
|
|
727
|
-
yield*
|
|
779
|
+
yield* runAlarmTransaction("record maintenance producer progress", async (transaction, checkpoint) => {
|
|
728
780
|
for (const id of affected) dueQueue.dirty(id, now, true);
|
|
729
781
|
if (affected.has("effect-agent:native")) {
|
|
730
|
-
const { state } = await readMaintenanceState(
|
|
731
|
-
await
|
|
782
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
783
|
+
if (!initialized || !nativeSource || state.retry !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
732
784
|
...Struct.omit(state, ["retry"]),
|
|
733
785
|
dirty: state.dirty + (nativeSource ? 0n : 1n)
|
|
734
786
|
})));
|
|
735
787
|
}
|
|
736
|
-
|
|
788
|
+
checkpoint.earlier(now + minimumAlarmDelay);
|
|
737
789
|
});
|
|
738
790
|
yield* notify;
|
|
739
791
|
});
|
|
740
792
|
return ThreadMutationGate.of({
|
|
741
|
-
withTransaction
|
|
793
|
+
withTransaction,
|
|
742
794
|
withMutation,
|
|
743
795
|
schedule,
|
|
744
796
|
recordProgress,
|
|
@@ -759,7 +811,8 @@ var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@ef
|
|
|
759
811
|
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
760
812
|
* producer keeps its generation dirty and immediately eligible. Quiescent native retries
|
|
761
813
|
* retain their durable backoff.
|
|
762
|
-
* 3. Admit
|
|
814
|
+
* 3. Admit one Attempt for a single-Thread runtime, or at most two independent Thread
|
|
815
|
+
* Attempts for a shared host, with one active head per Thread.
|
|
763
816
|
* Keep native and delivery admission open together while finite waves remain active, so
|
|
764
817
|
* fresh replies and abort controls can progress during unrelated cleanup. Close atomically
|
|
765
818
|
* at quiescence, or at the original ten-minute yield deadline, then join admitted waves.
|
|
@@ -775,7 +828,8 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
775
828
|
const recoveryConfig = yield* DurableRuntimeConfig;
|
|
776
829
|
const ledger = yield* SubmissionLedger;
|
|
777
830
|
const wakes = yield* WakeScheduler;
|
|
778
|
-
const
|
|
831
|
+
const hints = yield* DurableAlarmService;
|
|
832
|
+
const alarms = yield* ThreadMaintenanceAlarm;
|
|
779
833
|
const config = yield* CloudflareDurableRuntimeConfig;
|
|
780
834
|
const { ctx } = yield* DurableObjectContext;
|
|
781
835
|
const dueQueue = make(ctx.storage);
|
|
@@ -789,12 +843,14 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
789
843
|
const messages = yield* ThreadMessageDelivery;
|
|
790
844
|
const host = yield* ThreadHostMaintenance;
|
|
791
845
|
const framework = yield* ThreadNativeMaintenance;
|
|
846
|
+
const nativeDispatchConcurrency = () => framework.singleThreadRuntime === runtime ? 1 : 2;
|
|
792
847
|
for (const lane of host.lanes) yield* Schema.decodeEffect(HostLaneId)(lane.id).pipe(Effect.mapError(alarmFailure("maintenance lane ID")));
|
|
793
848
|
const recoveryEvents = Option.getOrElse(yield* Effect.serviceOption(ThreadRecoveryEvents), () => ThreadRecoveryEvents.of({ publish: () => Effect.void }));
|
|
794
849
|
let recoveryEventsPending = false;
|
|
795
850
|
const maintenancePassGate = yield* Semaphore.make(1);
|
|
796
851
|
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
797
852
|
const runTransaction = yield* makeStorageOperation;
|
|
853
|
+
const { run: runAlarmTransaction } = yield* makeAlarmOperation;
|
|
798
854
|
const queueSnapshot = runTransaction("read maintenance due queue", async () => dueQueue.read());
|
|
799
855
|
const reportParked = Effect.gen(function* () {
|
|
800
856
|
if (!(yield* queueSnapshot).some((row) => row.state === "parked" && row.reported === 0)) return;
|
|
@@ -834,13 +890,13 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
834
890
|
let charged = false;
|
|
835
891
|
return Effect.gen(function* () {
|
|
836
892
|
const now = yield* Clock.currentTimeMillis;
|
|
837
|
-
const claimed = yield*
|
|
893
|
+
const claimed = yield* runAlarmTransaction("charge maintenance attempt", async (_transaction, checkpoint) => {
|
|
838
894
|
const claimed = dueQueue.claim(selected, now);
|
|
839
|
-
const next$
|
|
840
|
-
if (Number.isFinite(next$
|
|
841
|
-
else
|
|
895
|
+
const next$1 = next(dueQueue.read());
|
|
896
|
+
if (Number.isFinite(next$1)) checkpoint.at(Math.max(now + minimumAlarmDelay, next$1));
|
|
897
|
+
else checkpoint.cancel();
|
|
842
898
|
return claimed;
|
|
843
|
-
})
|
|
899
|
+
});
|
|
844
900
|
if (claimed === void 0) return Option.none();
|
|
845
901
|
row = claimed;
|
|
846
902
|
charged = true;
|
|
@@ -922,9 +978,11 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
922
978
|
return recoveryEventsPending ? Option.some(0) : Option.none();
|
|
923
979
|
})
|
|
924
980
|
};
|
|
925
|
-
const recordRecoveryFaults = Effect.
|
|
981
|
+
const recordRecoveryFaults = Effect.fnUntraced(function* (result, recovery, current, selectedThread) {
|
|
926
982
|
const threads = /* @__PURE__ */ new Map();
|
|
983
|
+
if (selectedThread !== void 0) threads.set(selectedThread, void 0);
|
|
927
984
|
for (const report of result.reports) threads.set(report.threadId, void 0);
|
|
985
|
+
for (const report of result.workReports ?? []) threads.set(report.threadId, void 0);
|
|
928
986
|
for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
|
|
929
987
|
if (threads.size === 0) return /* @__PURE__ */ new Map();
|
|
930
988
|
const now = yield* Clock.currentTimeMillis;
|
|
@@ -985,14 +1043,35 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
985
1043
|
for (const fault of retained.newlyBlocked) yield* Effect.logError("Native Thread recovery blocked; accepted work remains pending", fault.failure.reason === "defect" ? Cause.die(fault.failure) : Cause.fail(fault.failure)).pipe(Effect.annotateLogs({ threadId: fault.threadId }));
|
|
986
1044
|
return retained.faults;
|
|
987
1045
|
});
|
|
988
|
-
const recoverThread = Effect.
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
1046
|
+
const recoverThread = Effect.fnUntraced(function* (threadId, recovery) {
|
|
1047
|
+
let cursor = yield* runTransaction("read work recovery cursor", async () => {
|
|
1048
|
+
const retained = await ctx.storage.get(workCursorKey(threadId));
|
|
1049
|
+
if (retained === void 0) return void 0;
|
|
1050
|
+
const decoded = Schema.decodeUnknownSync(NativeWorkCursor)(retained);
|
|
1051
|
+
if (decoded.threadId !== threadId) throw new Error("Work cursor belongs to another Thread");
|
|
1052
|
+
return decoded.cursor;
|
|
1053
|
+
});
|
|
1054
|
+
do {
|
|
1055
|
+
const result = yield* runtime.runRecovery({
|
|
1056
|
+
threadId,
|
|
1057
|
+
...cursor === void 0 ? {} : { cursor }
|
|
1058
|
+
}).pipe(Effect.provideService(CurrentNativeSource, true));
|
|
1059
|
+
const faults = yield* recordRecoveryFaults(result, recovery, void 0, threadId);
|
|
1060
|
+
yield* runTransaction("checkpoint work recovery cursor", async () => {
|
|
1061
|
+
if (result.cursor === void 0) await ctx.storage.delete(workCursorKey(threadId));
|
|
1062
|
+
else await ctx.storage.put(workCursorKey(threadId), Schema.encodeSync(NativeWorkCursor)({
|
|
1063
|
+
threadId,
|
|
1064
|
+
cursor: result.cursor
|
|
1065
|
+
}));
|
|
1066
|
+
});
|
|
1067
|
+
for (const report of result.reports) recovery.reports.set(report.submissionId, report);
|
|
1068
|
+
recovery.faults.delete(threadId);
|
|
1069
|
+
for (const [id, fault] of faults) recovery.faults.set(id, fault);
|
|
1070
|
+
recovery.recovered += (result.workReports?.length ?? result.reports.length) + result.blocked.length;
|
|
1071
|
+
recovery.repaired ||= (result.workReports ?? result.reports).some((report) => report.disposition === "repaired");
|
|
1072
|
+
for (const report of result.workReports ?? []) if (report.retryAtMillis !== void 0) recovery.retryAtMillis = Math.min(recovery.retryAtMillis ?? Infinity, report.retryAtMillis);
|
|
1073
|
+
cursor = result.blocked.length > 0 ? void 0 : result.cursor;
|
|
1074
|
+
} while (cursor !== void 0);
|
|
996
1075
|
});
|
|
997
1076
|
const readCurrentMaintenanceState = async (transaction) => {
|
|
998
1077
|
const current = await readMaintenanceState(transaction);
|
|
@@ -1010,10 +1089,10 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1010
1089
|
initialized: true
|
|
1011
1090
|
};
|
|
1012
1091
|
};
|
|
1013
|
-
const ensureAlarm = Effect.
|
|
1092
|
+
const ensureAlarm = Effect.fnUntraced(function* () {
|
|
1014
1093
|
yield* failpoint.hit("maintenance:ensure:before");
|
|
1015
1094
|
const now = yield* Clock.currentTimeMillis;
|
|
1016
|
-
yield*
|
|
1095
|
+
yield* runAlarmTransaction("ensure maintenance alarm", async (transaction, checkpoint) => {
|
|
1017
1096
|
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1018
1097
|
recoveryEventsPending = hasRecoveryEvents(state);
|
|
1019
1098
|
const rows = dueQueue.read();
|
|
@@ -1022,14 +1101,14 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1022
1101
|
const native = state.dirty > state.processed ? state.retry?.generation === state.dirty ? state.retry.notBefore : rows.find((row) => row.id === "effect-agent:native")?.dueAt ?? now + minimumAlarmDelay : null;
|
|
1023
1102
|
dueQueue.checkpointNative(native);
|
|
1024
1103
|
const next$3 = next(dueQueue.read());
|
|
1025
|
-
if (Number.isFinite(next$3))
|
|
1026
|
-
})
|
|
1104
|
+
if (Number.isFinite(next$3)) checkpoint.earlier(Math.max(now + minimumAlarmDelay, next$3));
|
|
1105
|
+
});
|
|
1027
1106
|
yield* failpoint.hit("maintenance:ensure:after");
|
|
1028
1107
|
});
|
|
1029
|
-
const beginPass = Effect.
|
|
1108
|
+
const beginPass = Effect.fnUntraced(function* (observed) {
|
|
1030
1109
|
yield* failpoint.hit("maintenance:begin:before");
|
|
1031
1110
|
const now = yield* Clock.currentTimeMillis;
|
|
1032
|
-
const result = yield*
|
|
1111
|
+
const result = yield* runAlarmTransaction("begin maintenance pass", async (transaction, checkpoint) => {
|
|
1033
1112
|
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1034
1113
|
recoveryEventsPending = hasRecoveryEvents(state);
|
|
1035
1114
|
observed.generation = state.dirty;
|
|
@@ -1050,8 +1129,8 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1050
1129
|
observed.queue.set(Native, charged);
|
|
1051
1130
|
observed.charged.set(Native, charged.revision);
|
|
1052
1131
|
const deadline = next(dueQueue.read());
|
|
1053
|
-
if (Number.isFinite(deadline))
|
|
1054
|
-
else
|
|
1132
|
+
if (Number.isFinite(deadline)) checkpoint.at(Math.max(now + minimumAlarmDelay, deadline));
|
|
1133
|
+
else checkpoint.cancel();
|
|
1055
1134
|
return {
|
|
1056
1135
|
_tag: "Actionable",
|
|
1057
1136
|
generation: state.dirty,
|
|
@@ -1059,7 +1138,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1059
1138
|
nonterminal: state.nonterminal,
|
|
1060
1139
|
stalls: charged.stalls
|
|
1061
1140
|
};
|
|
1062
|
-
})
|
|
1141
|
+
});
|
|
1063
1142
|
yield* failpoint.hit("maintenance:begin:after");
|
|
1064
1143
|
return result;
|
|
1065
1144
|
});
|
|
@@ -1067,17 +1146,17 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1067
1146
|
const backoff = Math.min(config.alarmBackoffCap, config.alarmBackoffBase * 2 ** Math.min(priorStalls, 30));
|
|
1068
1147
|
return Math.ceil(backoff / 2 + backoff / 2 * jitter);
|
|
1069
1148
|
};
|
|
1070
|
-
const rearmDelay = Effect.
|
|
1149
|
+
const rearmDelay = Effect.fnUntraced(function* (progressed, priorStalls) {
|
|
1071
1150
|
return progressed ? config.alarmBackoffBase : backoffDelay(priorStalls, yield* Random.next);
|
|
1072
1151
|
});
|
|
1073
|
-
const rearmFailure = Effect.
|
|
1152
|
+
const rearmFailure = Effect.fnUntraced(function* (observed) {
|
|
1074
1153
|
const { generation, nativeOnly } = observed;
|
|
1075
1154
|
if (generation === void 0) return;
|
|
1076
1155
|
yield* failpoint.hit("maintenance:retry:before");
|
|
1077
1156
|
yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
1078
1157
|
const now = yield* Clock.currentTimeMillis;
|
|
1079
1158
|
const jitter = yield* Random.next;
|
|
1080
|
-
yield*
|
|
1159
|
+
yield* runAlarmTransaction("back off failed maintenance", async (transaction, checkpoint) => {
|
|
1081
1160
|
const { state } = await readMaintenanceState(transaction);
|
|
1082
1161
|
const previous = state.retry?.generation === generation && state.retry.nativeOnly === nativeOnly ? state.retry : void 0;
|
|
1083
1162
|
const retry = state.retry?.generation === generation && state.retry.nativeOnly ? state.retry : MaintenanceRetry.make({
|
|
@@ -1096,27 +1175,25 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1096
1175
|
const charged = observed.charged.get(row.id) === row.revision ? row : dueQueue.claim(row, now);
|
|
1097
1176
|
if (charged !== void 0) dueQueue.complete(charged, now + backoffDelay(charged.stalls, jitter));
|
|
1098
1177
|
}
|
|
1099
|
-
const next$
|
|
1100
|
-
if (Number.isFinite(next$
|
|
1101
|
-
else
|
|
1102
|
-
})
|
|
1178
|
+
const next$5 = next(dueQueue.read());
|
|
1179
|
+
if (Number.isFinite(next$5)) checkpoint.at(Math.max(now + minimumAlarmDelay, next$5));
|
|
1180
|
+
else checkpoint.cancel();
|
|
1181
|
+
});
|
|
1103
1182
|
}));
|
|
1104
1183
|
yield* failpoint.hit("maintenance:retry:after");
|
|
1105
1184
|
});
|
|
1106
|
-
const beginNative =
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
});
|
|
1119
|
-
const attemptSelected = Effect.fn("ThreadMaintenance.attemptSelected")(function* (selected, previousWaits, yieldAfter, recovery) {
|
|
1185
|
+
const beginNative = (observed) => mutations.withSnapshot((activeAtStart) => Effect.gen(function* () {
|
|
1186
|
+
const generation = yield* beginPass(observed);
|
|
1187
|
+
if (activeAtStart === 0) {
|
|
1188
|
+
const native = (yield* queueSnapshot).find((row) => row.id === Native);
|
|
1189
|
+
if (native !== void 0) observed.available.set(native.id, native.revision);
|
|
1190
|
+
}
|
|
1191
|
+
return {
|
|
1192
|
+
...generation,
|
|
1193
|
+
activeAtStart
|
|
1194
|
+
};
|
|
1195
|
+
}));
|
|
1196
|
+
const attemptSelected = Effect.fnUntraced(function* (selected, previousWaits, yieldAfter, recovery) {
|
|
1120
1197
|
yield* recoverThread(selected.threadId, recovery);
|
|
1121
1198
|
let bindingFailure;
|
|
1122
1199
|
const settlement = recovery.faults.has(selected.threadId) ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.provideService(CurrentNativeSource, true), Effect.catchTag("BindingUnavailable", (failure) => {
|
|
@@ -1148,7 +1225,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1148
1225
|
}
|
|
1149
1226
|
return Option.isSome(settlement) ? 1 : 0;
|
|
1150
1227
|
});
|
|
1151
|
-
const advance = Effect.
|
|
1228
|
+
const advance = Effect.fnUntraced(function* (started, yieldAfter, observed, recovery, native, dispatch = true, reserved = new Set(native.active.keys())) {
|
|
1152
1229
|
let publicationRow = (yield* queueSnapshot).find((row) => row.id === Publication);
|
|
1153
1230
|
if (publicationRow !== void 0 && publicationRow.dueAt !== null && publicationRow.dueAt <= (yield* Clock.currentTimeMillis)) {
|
|
1154
1231
|
yield* runQueued(publicationRow, mutations.withSnapshot((active) => started._tag === "Actionable" && active === 0 ? publication.prepareGeneration(started.generation) : Effect.void).pipe(Effect.andThen(publication.drain)), observed);
|
|
@@ -1186,7 +1263,20 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1186
1263
|
generation: started.generation,
|
|
1187
1264
|
activeAtStart: started.activeAtStart
|
|
1188
1265
|
};
|
|
1266
|
+
const { scannedGeneration, afterWorkThread } = yield* runTransaction("read work discovery position", async () => {
|
|
1267
|
+
const { state } = await readMaintenanceState(ctx.storage);
|
|
1268
|
+
return {
|
|
1269
|
+
scannedGeneration: state.dirty,
|
|
1270
|
+
afterWorkThread: state.lastRecoveredThreadId
|
|
1271
|
+
};
|
|
1272
|
+
});
|
|
1189
1273
|
const current = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
1274
|
+
let workOwners = yield* runtime.discoverWorkThreads({
|
|
1275
|
+
limit: 32,
|
|
1276
|
+
...afterWorkThread === void 0 ? {} : { afterThreadId: afterWorkThread }
|
|
1277
|
+
}).pipe(Effect.provideService(CurrentNativeSource, true));
|
|
1278
|
+
if (workOwners.threadIds.length === 0 && afterWorkThread !== void 0) workOwners = yield* runtime.discoverWorkThreads({ limit: 32 }).pipe(Effect.provideService(CurrentNativeSource, true));
|
|
1279
|
+
recovery.discoveryPending ||= workOwners.afterThreadId !== void 0;
|
|
1190
1280
|
const selectionTime = yield* Clock.currentTimeMillis;
|
|
1191
1281
|
const submissionsByThread = /* @__PURE__ */ new Map();
|
|
1192
1282
|
for (const row of current) {
|
|
@@ -1194,6 +1284,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1194
1284
|
ids.push(row.submissionId);
|
|
1195
1285
|
submissionsByThread.set(row.threadId, ids);
|
|
1196
1286
|
}
|
|
1287
|
+
for (const threadId of workOwners.threadIds) if (!submissionsByThread.has(threadId)) submissionsByThread.set(threadId, []);
|
|
1197
1288
|
yield* runTransaction("read Thread recovery deadlines", async () => {
|
|
1198
1289
|
for (const [threadId, ids] of submissionsByThread) {
|
|
1199
1290
|
if (!recovery.loaded.has(threadId)) {
|
|
@@ -1265,13 +1356,13 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1265
1356
|
const waits = (state.bindingWaits ?? []).filter((retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId);
|
|
1266
1357
|
const runnable = eligible.filter((threadId) => !reserved.has(threadId) && !native.active.has(threadId) && !native.deferred.has(threadId) && !waits.some((wait) => wait.threadId === threadId));
|
|
1267
1358
|
const pivot = Math.max(0, runnable.findIndex((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId));
|
|
1268
|
-
const selected = dispatch ? [...runnable.slice(pivot), ...runnable.slice(0, pivot)].slice(0, nativeDispatchConcurrency - reserved.size) : [];
|
|
1359
|
+
const selected = dispatch ? [...runnable.slice(pivot), ...runnable.slice(0, pivot)].slice(0, nativeDispatchConcurrency() - reserved.size) : [];
|
|
1269
1360
|
const next = selected.at(-1);
|
|
1270
1361
|
if (next !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
1271
1362
|
...state,
|
|
1272
1363
|
lastServedThreadId: next
|
|
1273
1364
|
})));
|
|
1274
|
-
const backlog = recovery.started ? [] : [...heads.keys()].filter((threadId) => {
|
|
1365
|
+
const backlog = recovery.started ? [] : [.../* @__PURE__ */ new Set([...heads.keys(), ...workOwners.threadIds])].filter((threadId) => {
|
|
1275
1366
|
const fault = recoveryFaults.get(threadId);
|
|
1276
1367
|
return !eligible.includes(threadId) && (fault === void 0 || fault.retryAt <= selectionTime);
|
|
1277
1368
|
});
|
|
@@ -1280,13 +1371,15 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1280
1371
|
return {
|
|
1281
1372
|
selected,
|
|
1282
1373
|
waits,
|
|
1283
|
-
backlog: [...after, ...before]
|
|
1374
|
+
backlog: [...after, ...before].slice(0, 32),
|
|
1375
|
+
discoveryPending: backlog.length > 32
|
|
1284
1376
|
};
|
|
1285
1377
|
}));
|
|
1286
1378
|
yield* failpoint.hit("maintenance:select:after");
|
|
1287
1379
|
if (!recovery.started) {
|
|
1288
1380
|
recovery.started = true;
|
|
1289
1381
|
recovery.needsCheckpoint = selection.backlog.length > 0;
|
|
1382
|
+
recovery.discoveryPending ||= selection.discoveryPending;
|
|
1290
1383
|
for (const threadId of selection.backlog) recovery.pending.add(threadId);
|
|
1291
1384
|
yield* Deferred.succeed(recovery.queue, selection.backlog);
|
|
1292
1385
|
}
|
|
@@ -1302,10 +1395,9 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1302
1395
|
nextAttemptAt: void 0,
|
|
1303
1396
|
dispatched: selection.selected.length > 0
|
|
1304
1397
|
};
|
|
1305
|
-
const scannedGeneration = yield* runTransaction("observe native source generation", async () => (await readMaintenanceState(ctx.storage)).state.dirty);
|
|
1306
1398
|
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
1307
1399
|
const waitingHeads = /* @__PURE__ */ new Map();
|
|
1308
|
-
const autonomous = remaining.some((snapshot) => {
|
|
1400
|
+
const autonomous = recovery.pending.size > 0 || recovery.discoveryPending || recoveryFaults.size > 0 || recovery.retryAtMillis !== void 0 || remaining.some((snapshot) => {
|
|
1309
1401
|
if (snapshot.state === "unknown" && waiting(snapshot)) return false;
|
|
1310
1402
|
const headWaiting = waitingHeads.get(snapshot.threadId);
|
|
1311
1403
|
if (headWaiting === void 0) waitingHeads.set(snapshot.threadId, waiting(snapshot));
|
|
@@ -1316,13 +1408,14 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1316
1408
|
const now = yield* Clock.currentTimeMillis;
|
|
1317
1409
|
const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
|
|
1318
1410
|
const nextEligible = [...recoveryFaults.values()].map((fault) => fault.retryAt).concat(eligible.filter((threadId) => !selection.waits.some((wait) => wait.threadId === threadId)).map(() => now));
|
|
1411
|
+
if (recovery.retryAtMillis !== void 0) nextEligible.push(recovery.retryAtMillis);
|
|
1319
1412
|
const retryDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
|
|
1320
1413
|
const delay = Math.max(ordinaryDelay, retryDelay);
|
|
1321
1414
|
yield* failpoint.hit("maintenance:checkpoint:before");
|
|
1322
1415
|
const nextAttemptAt = yield* mutations.withSnapshot((active) => runTransaction("checkpoint native maintenance", () => dueQueue.transaction(async (transaction) => {
|
|
1323
1416
|
const { state } = await readMaintenanceState(transaction);
|
|
1324
1417
|
const mutationOverlap = observation.activeAtStart > 0 || started.activeAtStart > 0 || active > 0;
|
|
1325
|
-
const generation = remaining.length === 0 && !mutationOverlap && state.dirty === scannedGeneration ? scannedGeneration : observation.generation;
|
|
1418
|
+
const generation = remaining.length === 0 && workOwners.threadIds.length === 0 && !mutationOverlap && state.dirty === scannedGeneration ? scannedGeneration : observation.generation;
|
|
1326
1419
|
const processed = autonomous || mutationOverlap ? state.processed : state.processed > generation ? state.processed : generation;
|
|
1327
1420
|
const canBackoff = !mutationOverlap && state.dirty === started.generation;
|
|
1328
1421
|
const next = ThreadMaintenanceState.make({
|
|
@@ -1343,7 +1436,10 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1343
1436
|
return dueAt ?? void 0;
|
|
1344
1437
|
})));
|
|
1345
1438
|
yield* failpoint.hit("maintenance:checkpoint:after");
|
|
1346
|
-
if (remaining.length === 0 && recovery.pending.size === 0)
|
|
1439
|
+
if (remaining.length === 0 && recovery.pending.size === 0) recovery.observation = {
|
|
1440
|
+
generation: observation.generation,
|
|
1441
|
+
activeAtStart: 0
|
|
1442
|
+
};
|
|
1347
1443
|
native.deferred.clear();
|
|
1348
1444
|
native.progressed = false;
|
|
1349
1445
|
native.needsCheckpoint = false;
|
|
@@ -1353,7 +1449,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1353
1449
|
nextAttemptAt
|
|
1354
1450
|
};
|
|
1355
1451
|
});
|
|
1356
|
-
const validateLanes = Effect.
|
|
1452
|
+
const validateLanes = Effect.fnUntraced(function* (lanes, queued) {
|
|
1357
1453
|
const ids = /* @__PURE__ */ new Set([
|
|
1358
1454
|
Native,
|
|
1359
1455
|
Publication,
|
|
@@ -1373,7 +1469,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1373
1469
|
message: `No handler registered for scheduled lane: ${row.id}`
|
|
1374
1470
|
});
|
|
1375
1471
|
});
|
|
1376
|
-
const dispatch = Effect.
|
|
1472
|
+
const dispatch = Effect.fnUntraced(function* (yieldAfter, dispatchUntil, observed) {
|
|
1377
1473
|
const notified = (yield* Stream.toPull(wakes.wakes)).pipe(Effect.asVoid, Effect.catch(() => Effect.never));
|
|
1378
1474
|
const started = yield* beginNative(observed);
|
|
1379
1475
|
yield* mutations.withSnapshot(() => Effect.gen(function* () {
|
|
@@ -1408,7 +1504,8 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1408
1504
|
started: false,
|
|
1409
1505
|
needsCheckpoint: false,
|
|
1410
1506
|
recovered: 0,
|
|
1411
|
-
repaired: false
|
|
1507
|
+
repaired: false,
|
|
1508
|
+
discoveryPending: false
|
|
1412
1509
|
};
|
|
1413
1510
|
const recoveryFiber = yield* fork(Effect.gen(function* () {
|
|
1414
1511
|
for (const threadId of yield* Deferred.await(recovery.queue)) {
|
|
@@ -1459,7 +1556,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1459
1556
|
native = void 0;
|
|
1460
1557
|
if (result.phase === "actionable") phase = "actionable";
|
|
1461
1558
|
observed.nativeOnly = dispatch.active.size > 0;
|
|
1462
|
-
if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency) {
|
|
1559
|
+
if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency()) {
|
|
1463
1560
|
dispatch.scanGeneration = void 0;
|
|
1464
1561
|
nativeCheck = true;
|
|
1465
1562
|
}
|
|
@@ -1548,7 +1645,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1548
1645
|
}), observed).pipe(Effect.asVoid));
|
|
1549
1646
|
}
|
|
1550
1647
|
if (native === void 0 && dispatch.active.size === 0 && recoveryObserved && !recovery.needsCheckpoint && backfillObserved && delivery === void 0 && lanes.every((lane) => lane.fiber === void 0)) break;
|
|
1551
|
-
if (native === void 0 && nativeCheck && dispatch.active.size < nativeDispatchConcurrency) {
|
|
1648
|
+
if (native === void 0 && nativeCheck && dispatch.active.size < nativeDispatchConcurrency()) {
|
|
1552
1649
|
nativeCheck = false;
|
|
1553
1650
|
const nativeCheckpoint = dispatch.needsCheckpoint && dispatch.active.size === 0;
|
|
1554
1651
|
const checkpoint = recoveryObserved && recovery.needsCheckpoint || nativeCheckpoint;
|
|
@@ -1610,7 +1707,7 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1610
1707
|
settled: dispatch.settled
|
|
1611
1708
|
};
|
|
1612
1709
|
});
|
|
1613
|
-
const pass = Effect.
|
|
1710
|
+
const pass = Effect.fnUntraced(function* (yieldAfter, dispatchUntil, observed) {
|
|
1614
1711
|
const concurrent = yield* Effect.exit(Effect.scoped(dispatch(yieldAfter, dispatchUntil, observed)));
|
|
1615
1712
|
const afterNative = yield* Effect.gen(function* () {
|
|
1616
1713
|
const lanes = [
|
|
@@ -1678,18 +1775,18 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1678
1775
|
yield* failpoint.hit("maintenance:finish:before");
|
|
1679
1776
|
const disposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
1680
1777
|
const now = yield* Clock.currentTimeMillis;
|
|
1681
|
-
return yield*
|
|
1778
|
+
return yield* runAlarmTransaction("finish maintenance event", async (transaction, checkpoint) => {
|
|
1682
1779
|
const { state } = await readMaintenanceState(transaction);
|
|
1683
1780
|
const native = active > 0 || state.dirty > state.processed ? state.retry?.generation === state.dirty && active === 0 ? Math.max(now + minimumAlarmDelay, state.retry.notBefore) : state.dirty === observed.generation && active === 0 ? result.nextAttemptAt ?? now + minimumAlarmDelay : now + minimumAlarmDelay : Infinity;
|
|
1684
1781
|
dueQueue.checkpointNative(Number.isFinite(native) ? native : null);
|
|
1685
|
-
const next$
|
|
1686
|
-
if (Number.isFinite(next$
|
|
1687
|
-
|
|
1782
|
+
const next$2 = next(dueQueue.read());
|
|
1783
|
+
if (Number.isFinite(next$2)) {
|
|
1784
|
+
checkpoint.at(Math.max(now + minimumAlarmDelay, next$2));
|
|
1688
1785
|
return "rearmed";
|
|
1689
1786
|
}
|
|
1690
|
-
|
|
1787
|
+
checkpoint.cancel();
|
|
1691
1788
|
return "cleared";
|
|
1692
|
-
})
|
|
1789
|
+
});
|
|
1693
1790
|
}));
|
|
1694
1791
|
yield* failpoint.hit("maintenance:finish:after");
|
|
1695
1792
|
const report = MaintenancePassReport.make({
|
|
@@ -1719,7 +1816,10 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1719
1816
|
charged: /* @__PURE__ */ new Map(),
|
|
1720
1817
|
available: /* @__PURE__ */ new Map()
|
|
1721
1818
|
};
|
|
1722
|
-
return yield*
|
|
1819
|
+
return yield* alarms.deferWakes(maintenancePassGate.withPermit(dueQueue.withView(Effect.scoped(pass(yieldAfter, dispatchUntil, observed))).pipe(Effect.onErrorIf(() => true, () => rearmFailure(observed))))).pipe(hints.withHintsDeferred, Effect.catchTags({
|
|
1820
|
+
StorageOperationError: (error) => alarmFailure("defer maintenance wakes")(error),
|
|
1821
|
+
InvalidScheduleConfigurationError: (error) => alarmFailure("defer maintenance wakes")(error)
|
|
1822
|
+
}));
|
|
1723
1823
|
}).pipe(Effect.timeoutOrElse({
|
|
1724
1824
|
duration: "14 minutes",
|
|
1725
1825
|
orElse: () => DurableAlarmError.make({
|
|
@@ -1733,6 +1833,6 @@ var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effe
|
|
|
1733
1833
|
}));
|
|
1734
1834
|
};
|
|
1735
1835
|
//#endregion
|
|
1736
|
-
export {
|
|
1836
|
+
export { Lifecycle as _, ThreadHostMaintenance as a, Projection as b, ThreadMaintenanceFailpoint as c, ThreadNativeMaintenance as d, ThreadPublication as f, HostLaneId as g, publishCommitted as h, MaintenancePassReport as i, ThreadMessageDelivery as l, ThreadRecoveryFaultEvent as m, DurableAlarmError as n, ThreadMaintenance as o, ThreadRecoveryEvents as p, DurableAlarmService as r, ThreadMaintenanceAlarm as s, Alarm_exports as t, ThreadMutationGate as u, Messages as v, Publication as x, Native as y };
|
|
1737
1837
|
|
|
1738
|
-
//# sourceMappingURL=Alarm-
|
|
1838
|
+
//# sourceMappingURL=Alarm-DxODgAZZ.mjs.map
|