@effect-agent/platform-cloudflare 0.1.0-beta.10 → 0.1.0-beta.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Alarm.d.mts +207 -0
- package/dist/Alarm.mjs +578 -0
- package/dist/Alarm.mjs.map +1 -0
- package/dist/BrowserRestCapture.d.mts +34 -0
- package/dist/BrowserRestCapture.mjs +238 -0
- package/dist/BrowserRestCapture.mjs.map +1 -0
- package/dist/BrowserRestCrawl.d.mts +16 -0
- package/dist/BrowserRestCrawl.mjs +244 -0
- package/dist/BrowserRestCrawl.mjs.map +1 -0
- package/dist/CloudflareAiGateway.d.mts +64 -0
- package/dist/CloudflareAiGateway.mjs +70 -0
- package/dist/CloudflareAiGateway.mjs.map +1 -0
- package/dist/CloudflareBindings.d.mts +115 -0
- package/dist/CloudflareBindings.mjs +113 -0
- package/dist/CloudflareBindings.mjs.map +1 -0
- package/dist/CloudflareBrowser-DV6kya1H.mjs +495 -0
- package/dist/CloudflareBrowser-DV6kya1H.mjs.map +1 -0
- package/dist/CloudflareBrowser-Doe1M7R5.d.mts +80 -0
- package/dist/CloudflareBrowser.d.mts +2 -0
- package/dist/CloudflareBrowser.mjs +2 -0
- package/dist/CloudflareCodeMode.d.mts +44 -0
- package/dist/CloudflareCodeMode.mjs +616 -0
- package/dist/CloudflareCodeMode.mjs.map +1 -0
- package/dist/CloudflareConfig-DKXGo8L3.d.mts +106 -0
- package/dist/CloudflareConfig.d.mts +2 -0
- package/dist/CloudflareConfig.mjs +128 -0
- package/dist/CloudflareConfig.mjs.map +1 -0
- package/dist/CloudflareMemory.d.mts +142 -0
- package/dist/CloudflareMemory.mjs +202 -0
- package/dist/CloudflareMemory.mjs.map +1 -0
- package/dist/CloudflareScheduling.d.mts +52 -0
- package/dist/CloudflareScheduling.mjs +396 -0
- package/dist/CloudflareScheduling.mjs.map +1 -0
- package/dist/CloudflareSubscriptions.d.mts +87 -0
- package/dist/CloudflareSubscriptions.mjs +530 -0
- package/dist/CloudflareSubscriptions.mjs.map +1 -0
- package/dist/CloudflareThreadClient.d.mts +1600 -0
- package/dist/CloudflareThreadClient.mjs +382 -0
- package/dist/CloudflareThreadClient.mjs.map +1 -0
- package/dist/InteractiveBrowser-BCS9g0tn.mjs +1180 -0
- package/dist/InteractiveBrowser-BCS9g0tn.mjs.map +1 -0
- package/dist/InteractiveBrowser-BR16UPbF.d.mts +169 -0
- package/dist/InteractiveBrowser.d.mts +2 -0
- package/dist/InteractiveBrowser.mjs +2 -0
- package/dist/ProtectedBrowser.d.mts +123 -0
- package/dist/ProtectedBrowser.mjs +1135 -0
- package/dist/ProtectedBrowser.mjs.map +1 -0
- package/dist/ThreadObject-CMQiW2uX.d.mts +827 -0
- package/dist/ThreadObject-DXavwaU-.mjs +853 -0
- package/dist/ThreadObject-DXavwaU-.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +2 -0
- package/dist/ThreadObject.mjs +3 -0
- package/dist/WakeScheduler.d.mts +23 -0
- package/dist/WakeScheduler.mjs +57 -0
- package/dist/WakeScheduler.mjs.map +1 -0
- package/dist/boundary-BguazVkh.mjs +42 -0
- package/dist/boundary-BguazVkh.mjs.map +1 -0
- package/dist/index.d.mts +13 -1548
- package/dist/index.mjs +13 -1636
- package/dist/prepared-admission-DU0_T-ev.mjs +73 -0
- package/dist/prepared-admission-DU0_T-ev.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -53
- package/src/Alarm.ts +1270 -0
- package/src/BrowserRestCapture.ts +477 -0
- package/src/BrowserRestCrawl.ts +540 -0
- package/src/CloudflareAiGateway.ts +170 -0
- package/src/CloudflareBindings.ts +199 -0
- package/src/CloudflareBrowser.ts +15 -0
- package/src/{code-mode-executor.ts → CloudflareCodeMode.ts} +372 -204
- package/src/{config.ts → CloudflareConfig.ts} +19 -12
- package/src/CloudflareMemory.ts +420 -0
- package/src/CloudflareScheduling.ts +747 -0
- package/src/CloudflareSubscriptions.ts +1041 -0
- package/src/CloudflareThreadClient.ts +809 -0
- package/src/InteractiveBrowser.ts +2418 -0
- package/src/ProtectedBrowser.ts +9 -0
- package/src/ThreadObject.ts +1179 -0
- package/src/{wake-scheduler.ts → WakeScheduler.ts} +32 -26
- package/src/index.ts +12 -23
- package/src/internal/boundary.ts +55 -0
- package/src/internal/browser-quick-action.ts +857 -0
- package/src/internal/browser-session-lifecycle.ts +160 -0
- package/src/internal/layers.ts +756 -0
- package/src/internal/message-delivery.ts +181 -0
- package/src/internal/prepared-admission.ts +116 -0
- package/src/internal/progress-wait.ts +121 -0
- package/src/internal/transport.ts +44 -0
- package/src/protected-browser/binding.ts +313 -0
- package/src/protected-browser/host.ts +344 -0
- package/src/protected-browser/inspect-frame.ts +125 -0
- package/src/protected-browser/native.ts +445 -0
- package/src/protected-browser/policy.ts +821 -0
- package/dist/index.mjs.map +0 -1
- package/src/alarm.ts +0 -334
- package/src/bindings.ts +0 -145
- package/src/client.ts +0 -646
- package/src/conversation-object.ts +0 -768
- package/src/layers.ts +0 -376
- package/src/transport.ts +0 -38
package/dist/Alarm.mjs
ADDED
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
2
|
+
import { DurableObjectContext } from "./CloudflareBindings.mjs";
|
|
3
|
+
import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.mjs";
|
|
4
|
+
import { r as safeCauseMessage } from "./boundary-BguazVkh.mjs";
|
|
5
|
+
import { Cause, Clock, Context, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Random, Ref, Schema, Scope, Semaphore, Stream, Struct } from "effect";
|
|
6
|
+
import "effect-agent/agent-registration";
|
|
7
|
+
import { DurableAgentRuntime } from "effect-agent/durable-agent-runtime";
|
|
8
|
+
import { SubmissionId, ThreadId } from "effect-agent/identifiers";
|
|
9
|
+
import { SubmissionLedger } from "effect-agent/submission-ledger";
|
|
10
|
+
import { ThreadProjectionMaintenance, drainDue } from "effect-agent/thread-projection-maintenance";
|
|
11
|
+
import { SqlClient } from "effect/unstable/sql/SqlClient";
|
|
12
|
+
//#region src/Alarm.ts
|
|
13
|
+
var Alarm_exports = /* @__PURE__ */ __exportAll({
|
|
14
|
+
DurableAlarmError: () => DurableAlarmError,
|
|
15
|
+
DurableAlarmService: () => DurableAlarmService,
|
|
16
|
+
MaintenancePassReport: () => MaintenancePassReport,
|
|
17
|
+
ThreadHostMaintenance: () => ThreadHostMaintenance,
|
|
18
|
+
ThreadMaintenance: () => ThreadMaintenance,
|
|
19
|
+
ThreadMaintenanceFailpoint: () => ThreadMaintenanceFailpoint,
|
|
20
|
+
ThreadMessageDelivery: () => ThreadMessageDelivery,
|
|
21
|
+
ThreadMutationGate: () => ThreadMutationGate,
|
|
22
|
+
ThreadPublication: () => ThreadPublication,
|
|
23
|
+
publishCommitted: () => publishCommitted
|
|
24
|
+
});
|
|
25
|
+
/**
|
|
26
|
+
* The single multiplexed Durable Object alarm (decision D-P6-2). A Durable Object has ONE
|
|
27
|
+
* alarm slot; every cadence the Node host ran on fibers (wake scan, lease expiry, settlement
|
|
28
|
+
* and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
|
|
29
|
+
* the slot always holds the EARLIEST deadline any caller asked for.
|
|
30
|
+
*
|
|
31
|
+
* The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
|
|
32
|
+
* maintenance generation and a committed alarm. Stable externally-driven waits may be
|
|
33
|
+
* nonterminal without retaining an alarm; their resolving mutation advances the generation and
|
|
34
|
+
* restores the alarm atomically.
|
|
35
|
+
*/
|
|
36
|
+
/** The Durable Object alarm API failed; surfaces on host entry points as a typed refusal. */
|
|
37
|
+
var DurableAlarmError = class extends Schema.TaggedError()("DurableAlarmError", {
|
|
38
|
+
operation: Schema.String,
|
|
39
|
+
message: Schema.String,
|
|
40
|
+
cause: Schema.optionalKey(Schema.Defect())
|
|
41
|
+
}) {};
|
|
42
|
+
const alarmFailure = (operation) => (cause) => DurableAlarmError.make({
|
|
43
|
+
operation,
|
|
44
|
+
message: safeCauseMessage(cause, "The Cloudflare alarm API failed without a diagnostic"),
|
|
45
|
+
cause
|
|
46
|
+
});
|
|
47
|
+
const makeStorageOperation = Effect.map(SqlClient, (sql) => (operation, execute) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => {
|
|
48
|
+
const body = Effect.uninterruptible(Effect.tryPromise({
|
|
49
|
+
try: execute,
|
|
50
|
+
catch: alarmFailure(operation)
|
|
51
|
+
}));
|
|
52
|
+
return current._tag === "Some" ? body : Effect.scoped(Effect.andThen(sql.reserve.pipe(Effect.mapError(alarmFailure(operation))), body));
|
|
53
|
+
}));
|
|
54
|
+
/** `ctx.storage` alarm slot as an Effect service; storage is truth, never a memory field. */
|
|
55
|
+
var DurableAlarmService = class DurableAlarmService extends Context.Service()("@effect-agent/platform-cloudflare/DurableAlarmService") {
|
|
56
|
+
static layer = Layer.effect(DurableAlarmService)(Effect.gen(function* () {
|
|
57
|
+
const { ctx } = yield* DurableObjectContext;
|
|
58
|
+
/**
|
|
59
|
+
* In-memory pass bookkeeping — a pure CACHE, never state: a fresh incarnation has no
|
|
60
|
+
* running pass, and a deferred wake lost to eviction was only ever a promptness hint
|
|
61
|
+
* on top of the already-committed pre-armed alarm.
|
|
62
|
+
*/
|
|
63
|
+
const runningPasses = yield* Ref.make(0);
|
|
64
|
+
const storageOperation = yield* makeStorageOperation;
|
|
65
|
+
const scheduled = storageOperation("get alarm", () => ctx.storage.getAlarm()).pipe(Effect.map((deadline) => deadline === null ? Option.none() : Option.some(deadline)));
|
|
66
|
+
const scheduleAt = (epochMillis) => storageOperation("set alarm", () => ctx.storage.setAlarm(epochMillis));
|
|
67
|
+
const ensureScheduledBy = (epochMillis) => storageOperation("ensure alarm", () => ctx.storage.transaction(async (transaction) => {
|
|
68
|
+
const existing = await transaction.getAlarm();
|
|
69
|
+
if (existing === null || existing > epochMillis) await transaction.setAlarm(epochMillis);
|
|
70
|
+
}));
|
|
71
|
+
const armNow = Clock.currentTimeMillis.pipe(Effect.flatMap((now) => ensureScheduledBy(now)));
|
|
72
|
+
const scheduleNow = Ref.get(runningPasses).pipe(Effect.flatMap((passes) => passes > 0 ? Effect.void : armNow));
|
|
73
|
+
const withWakesDeferred = (body) => Ref.update(runningPasses, (passes) => passes + 1).pipe(Effect.andThen(body), Effect.ensuring(Ref.update(runningPasses, (passes) => passes - 1)));
|
|
74
|
+
const cancel = storageOperation("delete alarm", () => ctx.storage.deleteAlarm());
|
|
75
|
+
return DurableAlarmService.of({
|
|
76
|
+
scheduled,
|
|
77
|
+
scheduleAt,
|
|
78
|
+
ensureScheduledBy,
|
|
79
|
+
scheduleNow,
|
|
80
|
+
withWakesDeferred,
|
|
81
|
+
cancel
|
|
82
|
+
});
|
|
83
|
+
}));
|
|
84
|
+
};
|
|
85
|
+
/** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
|
|
86
|
+
var MaintenancePassReport = class extends Schema.Class("@effect-agent/platform-cloudflare/MaintenancePassReport")({
|
|
87
|
+
/** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
|
|
88
|
+
phase: Schema.Literals(["caught-up", "actionable"]),
|
|
89
|
+
/** Recovery decisions executed (or deferred) BEFORE any new claim in this pass. */
|
|
90
|
+
recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
91
|
+
/** Whether the head Attempt settled. Joined input may settle with that head. */
|
|
92
|
+
settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
93
|
+
/** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
|
|
94
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
95
|
+
/** `rearmed` for dirty/autonomous work, `cleared` for stable waits or settlement. */
|
|
96
|
+
alarm: Schema.Literals(["rearmed", "cleared"])
|
|
97
|
+
}) {};
|
|
98
|
+
/** Test-only fault authority; production uses the inert layer. */
|
|
99
|
+
var ThreadMaintenanceFailpoint = class extends Context.Service()("@effect-agent/platform-cloudflare/ThreadMaintenanceFailpoint") {
|
|
100
|
+
static layer = Layer.succeed(this)({ hit: () => Effect.void });
|
|
101
|
+
};
|
|
102
|
+
/** Opt in with `ThreadObject.layer(registrations, { publication: Layer.effect(ThreadPublication)(...) })`. */
|
|
103
|
+
var ThreadPublication = class extends Context.Service()("@effect-agent/platform-cloudflare/ThreadPublication") {
|
|
104
|
+
static layer = Layer.succeed(this)({
|
|
105
|
+
invalidate: Effect.void,
|
|
106
|
+
prepareGeneration: () => Effect.void,
|
|
107
|
+
drain: Effect.void,
|
|
108
|
+
pendingDeadline: Effect.succeed(Option.none())
|
|
109
|
+
});
|
|
110
|
+
};
|
|
111
|
+
/**
|
|
112
|
+
* Host-assembled native message recovery. The driver bounds each actual Claim and persists its
|
|
113
|
+
* timeout/retry before this pump returns. Do not add a second timer starting at batch selection:
|
|
114
|
+
* local Claim setup may take time, and expiration still owes the driver's local retry commit.
|
|
115
|
+
*/
|
|
116
|
+
const ThreadMessageDelivery = Context.Reference("@effect-agent/platform-cloudflare/ThreadMessageDelivery", { defaultValue: () => ({
|
|
117
|
+
drainUntil: () => Effect.void,
|
|
118
|
+
pendingDeadline: Effect.succeed(Option.none())
|
|
119
|
+
}) });
|
|
120
|
+
/**
|
|
121
|
+
* Application obligations sharing this Object's alarm. Admit one initial external wave even on
|
|
122
|
+
* a caught-up pass, then respond to wakes while native work runs. sourceFinished closes only
|
|
123
|
+
* admission of NEW external waves. Keep local admission/hub subscriptions in the supplied event
|
|
124
|
+
* Scope until maintenance tears it down; do not scope them to sourceFinished or to this Effect.
|
|
125
|
+
* No deadline sleeps or automatic retry loops. Return after already-admitted waves finish.
|
|
126
|
+
*
|
|
127
|
+
* Declare a finite whole-wave allowance (1..300000ms): maximum for parallel lanes, sum for
|
|
128
|
+
* sequential operations. Admit a wave only if its allowance fits before dispatchUntil. Later
|
|
129
|
+
* arrivals cannot renew the retirement window. Maintenance bounds the join after sourceFinished,
|
|
130
|
+
* interrupts and joins event Scope, then reads local deadlines under the mutation gate.
|
|
131
|
+
*
|
|
132
|
+
* Setup and pendingDeadline are bounded local operations. Persist claims/envelopes before
|
|
133
|
+
* dispatch; interrupted waits leave exact retries/receipts recoverable. Cancellation is local,
|
|
134
|
+
* not remote rollback. Retry accounting is unchanged. Network waits/finalizers must be
|
|
135
|
+
* interruptible; short local atomic commits may be uninterruptible. Hooks never write the raw
|
|
136
|
+
* alarm slot. Required native publication gates belong to ThreadPublication.
|
|
137
|
+
*/
|
|
138
|
+
const ThreadHostMaintenance = Context.Reference("@effect-agent/platform-cloudflare/ThreadHostMaintenance", { defaultValue: () => ({
|
|
139
|
+
dispatchTimeoutMillis: 1,
|
|
140
|
+
drainUntil: () => Effect.void,
|
|
141
|
+
pendingDeadline: Effect.succeed(Option.none())
|
|
142
|
+
}) });
|
|
143
|
+
const earliestDeadline = (left, right) => Option.isSome(left) ? Option.isSome(right) ? Option.some(Math.min(left.value, right.value)) : left : right;
|
|
144
|
+
/** @internal A committed source operation must not become a failed operation because delivery failed. */
|
|
145
|
+
const publishCommitted = Effect.gen(function* () {
|
|
146
|
+
const publication = yield* ThreadPublication;
|
|
147
|
+
yield* publication.invalidate.pipe(Effect.andThen(publication.drain));
|
|
148
|
+
}).pipe(Effect.catchCause((cause) => Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.logError("Thread publication deferred after source commit", cause)));
|
|
149
|
+
const MaintenanceGeneration = Schema.BigIntFromString.check(Schema.isGreaterThanOrEqualToBigInt(0n));
|
|
150
|
+
var BindingRetry = class extends Schema.Class("BindingRetry")({
|
|
151
|
+
threadId: ThreadId,
|
|
152
|
+
submissionId: SubmissionId,
|
|
153
|
+
attempts: Schema.Natural,
|
|
154
|
+
notBefore: Schema.Finite,
|
|
155
|
+
reportedAt: Schema.Finite
|
|
156
|
+
}) {};
|
|
157
|
+
var MaintenanceRetry = class extends Schema.Class("MaintenanceRetry")({
|
|
158
|
+
generation: MaintenanceGeneration,
|
|
159
|
+
notBefore: Schema.Finite,
|
|
160
|
+
nativeOnly: Schema.Boolean,
|
|
161
|
+
stalls: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(30))
|
|
162
|
+
}) {};
|
|
163
|
+
/** Versioned, platform-private maintenance state stored through Durable Object KV. */
|
|
164
|
+
var ThreadMaintenanceState = class extends Schema.Class("@effect-agent/platform-cloudflare/ThreadMaintenanceState")({
|
|
165
|
+
schemaVersion: Schema.Literal(1),
|
|
166
|
+
dirty: MaintenanceGeneration,
|
|
167
|
+
processed: MaintenanceGeneration,
|
|
168
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
169
|
+
/** One physical-owner cursor; old single-lane records need no conversion. */
|
|
170
|
+
lastServedThreadId: Schema.optionalKey(ThreadId),
|
|
171
|
+
bindingRetries: Schema.optionalKey(Schema.Array(BindingRetry)),
|
|
172
|
+
/** Absent on older records. A newer mutation makes this retry obsolete. */
|
|
173
|
+
retry: Schema.optionalKey(MaintenanceRetry)
|
|
174
|
+
}) {};
|
|
175
|
+
const MAINTENANCE_STATE_KEY = "effect-agent:thread-maintenance:v1";
|
|
176
|
+
const decodeMaintenanceState = Schema.decodeUnknownSync(ThreadMaintenanceState);
|
|
177
|
+
const encodeMaintenanceState = Schema.encodeSync(ThreadMaintenanceState);
|
|
178
|
+
const initialMaintenanceState = () => ThreadMaintenanceState.make({
|
|
179
|
+
schemaVersion: 1,
|
|
180
|
+
dirty: 1n,
|
|
181
|
+
processed: 0n,
|
|
182
|
+
nonterminal: 0
|
|
183
|
+
});
|
|
184
|
+
const readMaintenanceState = async (transaction) => {
|
|
185
|
+
const encoded = await transaction.get(MAINTENANCE_STATE_KEY);
|
|
186
|
+
return encoded === void 0 ? {
|
|
187
|
+
state: initialMaintenanceState(),
|
|
188
|
+
initialized: false
|
|
189
|
+
} : {
|
|
190
|
+
state: decodeMaintenanceState(encoded),
|
|
191
|
+
initialized: true
|
|
192
|
+
};
|
|
193
|
+
};
|
|
194
|
+
const ensureTransactionAlarmBy = async (transaction, deadline) => {
|
|
195
|
+
const scheduled = await transaction.getAlarm();
|
|
196
|
+
if (scheduled === null || scheduled > deadline) await transaction.setAlarm(deadline);
|
|
197
|
+
};
|
|
198
|
+
const stableExternalWait = (snapshot, reports) => {
|
|
199
|
+
const report = reports.get(snapshot.submissionId);
|
|
200
|
+
if (report?.decision._tag === "SettleAborted") return false;
|
|
201
|
+
switch (snapshot.state) {
|
|
202
|
+
case "suspended":
|
|
203
|
+
case "joined": return true;
|
|
204
|
+
case "unknown": return report?.disposition === "unknown";
|
|
205
|
+
case "admitted": return reports.get(snapshot.submissionId)?.decision._tag === "AwaitParentEstablishment";
|
|
206
|
+
case "input-applied":
|
|
207
|
+
case "joining":
|
|
208
|
+
case "ready":
|
|
209
|
+
case "running":
|
|
210
|
+
case "settled":
|
|
211
|
+
case "terminalizing": return false;
|
|
212
|
+
}
|
|
213
|
+
};
|
|
214
|
+
/**
|
|
215
|
+
* Shared prearm/acknowledgement boundary for ingress and runtime-owned producers.
|
|
216
|
+
* `ThreadObject.layer` provides this same instance in its Services. Rebuilt runtime/maintenance
|
|
217
|
+
* Layers must reuse that instance; a second gate cannot observe the native producers' activity.
|
|
218
|
+
*/
|
|
219
|
+
var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@effect-agent/platform-cloudflare/internal/ThreadMutationGate") {
|
|
220
|
+
static layer = Layer.effect(this)(Effect.gen(function* () {
|
|
221
|
+
const { ctx } = yield* DurableObjectContext;
|
|
222
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
223
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
224
|
+
const activeMutations = yield* Ref.make(0);
|
|
225
|
+
const generationGate = yield* Semaphore.make(1);
|
|
226
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
227
|
+
const runTransaction = yield* makeStorageOperation;
|
|
228
|
+
const beginMutation = Effect.fn("ThreadMaintenance.beginMutation")(function* (invalidatesRecovery) {
|
|
229
|
+
yield* failpoint.hit("maintenance:dirty:before");
|
|
230
|
+
const now = yield* Clock.currentTimeMillis;
|
|
231
|
+
yield* runTransaction("advance maintenance generation", () => ctx.storage.transaction(async (transaction) => {
|
|
232
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
233
|
+
const next = ThreadMaintenanceState.make({
|
|
234
|
+
...state,
|
|
235
|
+
dirty: state.dirty + (invalidatesRecovery ? 1n : 0n)
|
|
236
|
+
});
|
|
237
|
+
if (invalidatesRecovery || !initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
238
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
239
|
+
}));
|
|
240
|
+
yield* failpoint.hit("maintenance:dirty:after");
|
|
241
|
+
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
242
|
+
});
|
|
243
|
+
const endMutation = generationGate.withPermit(Ref.update(activeMutations, (active) => Math.max(0, active - 1)));
|
|
244
|
+
const withMutation = (body, options) => Effect.acquireUseRelease(generationGate.withPermit(beginMutation(options?.invalidatesRecovery ?? true)), () => failpoint.hit("maintenance:mutation:armed").pipe(Effect.andThen(body), Effect.tap(() => failpoint.hit("maintenance:mutation:finished"))), () => endMutation);
|
|
245
|
+
return ThreadMutationGate.of({
|
|
246
|
+
withMutation,
|
|
247
|
+
withSnapshot: (body) => generationGate.withPermit(Effect.flatMap(Ref.get(activeMutations), body))
|
|
248
|
+
});
|
|
249
|
+
}));
|
|
250
|
+
};
|
|
251
|
+
/**
|
|
252
|
+
* Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
|
|
253
|
+
*
|
|
254
|
+
* `pass` = generation snapshot/pre-arm → recovery → one head Attempt → generation acknowledgement:
|
|
255
|
+
*
|
|
256
|
+
* 1. One storage transaction reads dirty/processed and re-arms before work. A caught-up forced
|
|
257
|
+
* alarm takes an O(1) path without recovery, ledger scans, or canonical-history reads.
|
|
258
|
+
* 2. Recovery strictly precedes a new claim. One head Attempt advances the lane and requests
|
|
259
|
+
* a safe yield after ten minutes. The whole event has a fourteen-minute cooperative timeout.
|
|
260
|
+
* 3. Auxiliary dispatch and listeners belong to the event Scope. After native completion,
|
|
261
|
+
* stop new external waves, join native delivery through its driver-owned Claim deadline,
|
|
262
|
+
* and bound host/backfill work by explicit allowances before closing and joining Scope.
|
|
263
|
+
* Ordinary auxiliary setup failures are reported after the one native opportunity.
|
|
264
|
+
* 4. The final transaction acknowledges only the generation observed at pass start. A racing
|
|
265
|
+
* mutation therefore remains `dirty > processed` and retains its atomically-established alarm.
|
|
266
|
+
* 5. Stable external waits acknowledge and clear. Autonomous retry, indeterminate, and lease
|
|
267
|
+
* recovery states leave their generation dirty and retain bounded backoff rearming.
|
|
268
|
+
*/
|
|
269
|
+
var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
|
|
270
|
+
static layer = Layer.effect(ThreadMaintenance)(Effect.gen(function* () {
|
|
271
|
+
const runtime = yield* DurableAgentRuntime;
|
|
272
|
+
const ledger = yield* SubmissionLedger;
|
|
273
|
+
const alarm = yield* DurableAlarmService;
|
|
274
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
275
|
+
const { ctx } = yield* DurableObjectContext;
|
|
276
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
277
|
+
const mutations = yield* ThreadMutationGate;
|
|
278
|
+
const publication = yield* ThreadPublication;
|
|
279
|
+
const projection = yield* ThreadProjectionMaintenance;
|
|
280
|
+
const messages = yield* ThreadMessageDelivery;
|
|
281
|
+
const host = yield* ThreadHostMaintenance;
|
|
282
|
+
const projectionDeadline = projection.pendingDeadline.pipe(Effect.catchCauseIf((cause) => !Cause.hasInterrupts(cause), (cause) => Effect.logError("Thread projection deadline unavailable", cause).pipe(Effect.as(Option.some(0)))));
|
|
283
|
+
const pendingDeadline = Effect.gen(function* () {
|
|
284
|
+
return earliestDeadline(earliestDeadline(yield* publication.pendingDeadline, yield* messages.pendingDeadline), earliestDeadline(yield* projectionDeadline, yield* host.pendingDeadline));
|
|
285
|
+
});
|
|
286
|
+
const maintenancePassGate = yield* Semaphore.make(1);
|
|
287
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
288
|
+
const runTransaction = yield* makeStorageOperation;
|
|
289
|
+
const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
|
|
290
|
+
yield* failpoint.hit("maintenance:ensure:before");
|
|
291
|
+
const now = yield* Clock.currentTimeMillis;
|
|
292
|
+
const retry = yield* runTransaction("ensure maintenance alarm", () => ctx.storage.transaction(async (transaction) => {
|
|
293
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
294
|
+
if (!initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
295
|
+
if (state.dirty > state.processed) await ensureTransactionAlarmBy(transaction, state.retry?.generation === state.dirty ? Math.max(now + minimumAlarmDelay, state.retry.notBefore) : now + config.wakeScanInterval);
|
|
296
|
+
return state.retry?.generation === state.dirty ? state.retry : void 0;
|
|
297
|
+
}));
|
|
298
|
+
const deadline = yield* pendingDeadline;
|
|
299
|
+
if (Option.isSome(deadline)) yield* runTransaction("ensure publication alarm", () => ctx.storage.transaction((transaction) => ensureTransactionAlarmBy(transaction, Math.max(now + minimumAlarmDelay, deadline.value <= now && retry !== void 0 && !retry.nativeOnly ? Math.max(deadline.value, retry.notBefore) : deadline.value))));
|
|
300
|
+
yield* failpoint.hit("maintenance:ensure:after");
|
|
301
|
+
});
|
|
302
|
+
const beginPass = Effect.fn("ThreadMaintenance.beginPass")(function* (observed) {
|
|
303
|
+
yield* failpoint.hit("maintenance:begin:before");
|
|
304
|
+
const now = yield* Clock.currentTimeMillis;
|
|
305
|
+
const result = yield* runTransaction("begin maintenance pass", () => ctx.storage.transaction(async (transaction) => {
|
|
306
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
307
|
+
observed.generation = state.dirty;
|
|
308
|
+
if (!initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
309
|
+
const retryAt = state.retry?.generation === state.dirty ? state.retry.notBefore : 0;
|
|
310
|
+
await ensureTransactionAlarmBy(transaction, now + minimumAlarmDelay);
|
|
311
|
+
if (state.processed >= state.dirty || retryAt > now) return {
|
|
312
|
+
_tag: "CaughtUp",
|
|
313
|
+
nonterminal: state.nonterminal
|
|
314
|
+
};
|
|
315
|
+
return {
|
|
316
|
+
_tag: "Actionable",
|
|
317
|
+
generation: state.dirty,
|
|
318
|
+
nonterminal: state.nonterminal,
|
|
319
|
+
stalls: state.retry?.generation === state.dirty ? state.retry.stalls : 0
|
|
320
|
+
};
|
|
321
|
+
}));
|
|
322
|
+
yield* failpoint.hit("maintenance:begin:after");
|
|
323
|
+
return result;
|
|
324
|
+
});
|
|
325
|
+
const backoffDelay = (priorStalls, jitter) => {
|
|
326
|
+
const backoff = Math.min(config.alarmBackoffCap, config.alarmBackoffBase * 2 ** Math.min(priorStalls, 30));
|
|
327
|
+
return Math.ceil(backoff / 2 + backoff / 2 * jitter);
|
|
328
|
+
};
|
|
329
|
+
const rearmDelay = Effect.fn("ThreadMaintenance.rearmDelay")(function* (progressed, priorStalls) {
|
|
330
|
+
return progressed ? config.alarmBackoffBase : backoffDelay(priorStalls, yield* Random.next);
|
|
331
|
+
});
|
|
332
|
+
const rearmFailure = Effect.fn("ThreadMaintenance.rearmFailure")(function* (observed) {
|
|
333
|
+
const { generation, nativeOnly } = observed;
|
|
334
|
+
if (generation === void 0) return;
|
|
335
|
+
yield* failpoint.hit("maintenance:retry:before");
|
|
336
|
+
yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
337
|
+
const deadline = yield* pendingDeadline.pipe(Effect.catchCause(() => Effect.succeed(Option.none())));
|
|
338
|
+
const now = yield* Clock.currentTimeMillis;
|
|
339
|
+
const jitter = yield* Random.next;
|
|
340
|
+
yield* runTransaction("back off failed maintenance", () => ctx.storage.transaction(async (transaction) => {
|
|
341
|
+
const { state } = await readMaintenanceState(transaction);
|
|
342
|
+
const previous = state.retry?.generation === generation ? state.retry : void 0;
|
|
343
|
+
const retry = MaintenanceRetry.make({
|
|
344
|
+
generation,
|
|
345
|
+
notBefore: Math.max(previous?.notBefore ?? 0, now + backoffDelay(previous?.stalls ?? 0, jitter)),
|
|
346
|
+
nativeOnly,
|
|
347
|
+
stalls: Math.min(30, (previous?.stalls ?? 0) + 1)
|
|
348
|
+
});
|
|
349
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
350
|
+
...state,
|
|
351
|
+
retry
|
|
352
|
+
})));
|
|
353
|
+
const nativeDeadline = active > 0 || state.dirty !== generation ? now + minimumAlarmDelay : retry.notBefore;
|
|
354
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, Option.isSome(deadline) && (nativeOnly || deadline.value > now) ? Math.min(nativeDeadline, deadline.value) : nativeDeadline));
|
|
355
|
+
}));
|
|
356
|
+
}));
|
|
357
|
+
yield* failpoint.hit("maintenance:retry:after");
|
|
358
|
+
});
|
|
359
|
+
const pass = Effect.fn("ThreadMaintenance.pass")(function* (yieldAfter, dispatchUntil, observed) {
|
|
360
|
+
const annotate = (report) => Effect.annotateCurrentSpan({
|
|
361
|
+
phase: report.phase,
|
|
362
|
+
recovered: report.recovered,
|
|
363
|
+
settled: report.settled,
|
|
364
|
+
nonterminal: report.nonterminal,
|
|
365
|
+
alarm: report.alarm
|
|
366
|
+
}).pipe(Effect.as(report));
|
|
367
|
+
const started = yield* mutations.withSnapshot((activeAtStart) => Effect.gen(function* () {
|
|
368
|
+
const generation = yield* beginPass(observed);
|
|
369
|
+
if (generation._tag === "Actionable" && activeAtStart === 0) yield* publication.prepareGeneration(generation.generation);
|
|
370
|
+
return {
|
|
371
|
+
...generation,
|
|
372
|
+
activeAtStart
|
|
373
|
+
};
|
|
374
|
+
}));
|
|
375
|
+
const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) => Scope.close(scope, exit));
|
|
376
|
+
const sourceFinished = yield* Deferred.make();
|
|
377
|
+
const finished = Deferred.await(sourceFinished);
|
|
378
|
+
const deliveryFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(messages.drainUntil(finished, dispatchUntil)), auxiliaryScope);
|
|
379
|
+
const hostFiber = yield* Effect.forkIn(Scope.provide(auxiliaryScope)(Effect.gen(function* () {
|
|
380
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(host.dispatchTimeoutMillis).pipe(Effect.mapError((cause) => DurableAlarmError.make({
|
|
381
|
+
operation: "host dispatch allowance",
|
|
382
|
+
message: "Declare an integer whole-wave allowance between 1 and 300000 milliseconds",
|
|
383
|
+
cause
|
|
384
|
+
})));
|
|
385
|
+
yield* host.drainUntil(finished, dispatchUntil);
|
|
386
|
+
})), auxiliaryScope);
|
|
387
|
+
const backfill = yield* Effect.forkIn(drainDue.pipe(Effect.provideService(ThreadProjectionMaintenance, projection), Effect.timeoutOption(config.projectionDispatchTimeoutMillis)), auxiliaryScope);
|
|
388
|
+
const finishAuxiliary = Effect.gen(function* () {
|
|
389
|
+
yield* Deferred.succeed(sourceFinished, void 0);
|
|
390
|
+
const remaining = Math.max(1, DateTime.toEpochMillis(dispatchUntil) - (yield* Clock.currentTimeMillis));
|
|
391
|
+
const hostJoin = yield* Effect.forkIn(Fiber.join(hostFiber).pipe(Effect.timeoutOption(Math.min(host.dispatchTimeoutMillis, remaining)), Effect.tap((result) => Effect.annotateCurrentSpan({ "host.timedOut": Option.isNone(result) }))), auxiliaryScope);
|
|
392
|
+
yield* Fiber.joinAll([
|
|
393
|
+
deliveryFiber,
|
|
394
|
+
hostJoin,
|
|
395
|
+
backfill
|
|
396
|
+
]);
|
|
397
|
+
yield* Scope.close(auxiliaryScope, Exit.void);
|
|
398
|
+
});
|
|
399
|
+
const deadline = yield* publication.pendingDeadline;
|
|
400
|
+
if (started._tag === "Actionable" || Option.isSome(deadline) && deadline.value <= (yield* Clock.currentTimeMillis)) yield* publication.drain;
|
|
401
|
+
const pending = yield* publication.pendingDeadline;
|
|
402
|
+
if (started._tag === "CaughtUp" || Option.isSome(pending)) {
|
|
403
|
+
yield* finishAuxiliary;
|
|
404
|
+
yield* failpoint.hit("maintenance:finish:before");
|
|
405
|
+
const disposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
406
|
+
const latest = yield* pendingDeadline;
|
|
407
|
+
const now = yield* Clock.currentTimeMillis;
|
|
408
|
+
return yield* runTransaction("finish publication pass", () => ctx.storage.transaction(async (transaction) => {
|
|
409
|
+
const { state } = await readMaintenanceState(transaction);
|
|
410
|
+
const nativeDeadline = active > 0 || state.dirty > state.processed ? state.retry?.generation === state.dirty && active === 0 ? Math.max(now + minimumAlarmDelay, state.retry.notBefore) : now + config.wakeScanInterval : Infinity;
|
|
411
|
+
const next = Option.isSome(latest) ? Math.min(nativeDeadline, latest.value) : nativeDeadline;
|
|
412
|
+
if (Number.isFinite(next)) {
|
|
413
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next));
|
|
414
|
+
return "rearmed";
|
|
415
|
+
}
|
|
416
|
+
await transaction.deleteAlarm();
|
|
417
|
+
return "cleared";
|
|
418
|
+
}));
|
|
419
|
+
}));
|
|
420
|
+
yield* failpoint.hit("maintenance:finish:after");
|
|
421
|
+
return yield* annotate(MaintenancePassReport.make({
|
|
422
|
+
phase: "caught-up",
|
|
423
|
+
recovered: 0,
|
|
424
|
+
settled: 0,
|
|
425
|
+
nonterminal: started.nonterminal,
|
|
426
|
+
alarm: disposition
|
|
427
|
+
}));
|
|
428
|
+
}
|
|
429
|
+
observed.nativeOnly = true;
|
|
430
|
+
const recovered = yield* runtime.runRecovery;
|
|
431
|
+
const reports = new Map(recovered.map((report) => [report.submissionId, report]));
|
|
432
|
+
const current = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
433
|
+
const heads = /* @__PURE__ */ new Map();
|
|
434
|
+
for (const row of current) {
|
|
435
|
+
if (row.state === "unknown" && stableExternalWait(row, reports)) continue;
|
|
436
|
+
if (!heads.has(row.threadId)) heads.set(row.threadId, row);
|
|
437
|
+
}
|
|
438
|
+
const eligible = [...heads.values()].filter((head) => !stableExternalWait(head, reports)).map((head) => head.threadId).sort();
|
|
439
|
+
const selectionTime = yield* Clock.currentTimeMillis;
|
|
440
|
+
yield* failpoint.hit("maintenance:select:before");
|
|
441
|
+
const selection = yield* runTransaction("select maintenance lane", () => ctx.storage.transaction(async (transaction) => {
|
|
442
|
+
const { state } = await readMaintenanceState(transaction);
|
|
443
|
+
const retries = (state.bindingRetries ?? []).filter((retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId);
|
|
444
|
+
const runnable = eligible.filter((threadId) => !retries.some((retry) => retry.threadId === threadId && retry.notBefore > selectionTime));
|
|
445
|
+
const next = runnable.find((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId) ?? runnable[0];
|
|
446
|
+
if (next !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
447
|
+
...state,
|
|
448
|
+
lastServedThreadId: next
|
|
449
|
+
})));
|
|
450
|
+
return {
|
|
451
|
+
selected: next,
|
|
452
|
+
retries
|
|
453
|
+
};
|
|
454
|
+
}));
|
|
455
|
+
yield* failpoint.hit("maintenance:select:after");
|
|
456
|
+
const selected = selection.selected === void 0 ? void 0 : heads.get(selection.selected);
|
|
457
|
+
let retries = selection.retries;
|
|
458
|
+
let bindingFailure;
|
|
459
|
+
const settlement = selected === void 0 ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.catchTag("BindingUnavailable", (failure) => {
|
|
460
|
+
bindingFailure = failure;
|
|
461
|
+
return Effect.succeed(Option.none());
|
|
462
|
+
}));
|
|
463
|
+
if (selected !== void 0) {
|
|
464
|
+
const previous = retries.find((retry) => retry.submissionId === selected.submissionId);
|
|
465
|
+
let retry;
|
|
466
|
+
let reportBindingFailure = false;
|
|
467
|
+
if (bindingFailure !== void 0) {
|
|
468
|
+
const now = yield* Clock.currentTimeMillis;
|
|
469
|
+
const attempts = Math.min(30, (previous?.attempts ?? 0) + 1);
|
|
470
|
+
reportBindingFailure = previous === void 0 || now - previous.reportedAt >= 9e5;
|
|
471
|
+
retry = BindingRetry.make({
|
|
472
|
+
threadId: selected.threadId,
|
|
473
|
+
submissionId: selected.submissionId,
|
|
474
|
+
attempts,
|
|
475
|
+
notBefore: now + Math.min(6e4, 5e3 * 2 ** (attempts - 1)),
|
|
476
|
+
reportedAt: reportBindingFailure ? now : previous?.reportedAt ?? now
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
if (retry !== void 0 || previous !== void 0) {
|
|
480
|
+
yield* failpoint.hit("maintenance:binding-retry:before");
|
|
481
|
+
retries = yield* runTransaction("record submission binding retry", () => ctx.storage.transaction(async (transaction) => {
|
|
482
|
+
const { state } = await readMaintenanceState(transaction);
|
|
483
|
+
const bindingRetries = [...(state.bindingRetries ?? []).filter((entry) => entry.submissionId !== selected.submissionId), ...retry === void 0 ? [] : [retry]];
|
|
484
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
485
|
+
...state,
|
|
486
|
+
bindingRetries
|
|
487
|
+
})));
|
|
488
|
+
return bindingRetries;
|
|
489
|
+
}));
|
|
490
|
+
yield* failpoint.hit("maintenance:binding-retry:after");
|
|
491
|
+
}
|
|
492
|
+
if (bindingFailure !== void 0) yield* reportBindingFailure ? Effect.logError("Thread awaits a current agent binding; original work remains pending", Cause.fail(bindingFailure)) : Effect.logDebug("Thread binding retry remains pending", Cause.fail(bindingFailure));
|
|
493
|
+
}
|
|
494
|
+
observed.nativeOnly = false;
|
|
495
|
+
yield* finishAuxiliary;
|
|
496
|
+
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
497
|
+
const waitingHeads = /* @__PURE__ */ new Map();
|
|
498
|
+
const autonomous = remaining.some((snapshot) => {
|
|
499
|
+
if (snapshot.state === "unknown" && stableExternalWait(snapshot, reports)) return false;
|
|
500
|
+
const headWaiting = waitingHeads.get(snapshot.threadId);
|
|
501
|
+
if (headWaiting === void 0) waitingHeads.set(snapshot.threadId, stableExternalWait(snapshot, reports));
|
|
502
|
+
if (headWaiting === true && snapshot.state === "ready" && reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput") return false;
|
|
503
|
+
return !stableExternalWait(snapshot, reports);
|
|
504
|
+
});
|
|
505
|
+
const progressed = Option.isSome(settlement) || recovered.some((report) => report.disposition === "repaired");
|
|
506
|
+
const now = yield* Clock.currentTimeMillis;
|
|
507
|
+
const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
|
|
508
|
+
const nextEligible = eligible.map((threadId) => retries.find((retry) => retry.submissionId === heads.get(threadId)?.submissionId)?.notBefore ?? now);
|
|
509
|
+
const bindingDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
|
|
510
|
+
const delay = Math.max(ordinaryDelay, bindingDelay);
|
|
511
|
+
yield* failpoint.hit("maintenance:finish:before");
|
|
512
|
+
const alarmDisposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
513
|
+
const publicationDeadline = yield* pendingDeadline;
|
|
514
|
+
return yield* runTransaction("finish maintenance pass", () => ctx.storage.transaction(async (transaction) => {
|
|
515
|
+
const { state } = await readMaintenanceState(transaction);
|
|
516
|
+
const processed = autonomous || started.activeAtStart > 0 || active > 0 ? state.processed : state.processed > started.generation ? state.processed : started.generation;
|
|
517
|
+
const next = ThreadMaintenanceState.make({
|
|
518
|
+
...Struct.omit(state, ["retry"]),
|
|
519
|
+
processed,
|
|
520
|
+
nonterminal: remaining.length,
|
|
521
|
+
bindingRetries: (state.bindingRetries ?? []).filter((retry) => remaining.some((row) => row.submissionId === retry.submissionId)),
|
|
522
|
+
...autonomous && !progressed ? { retry: MaintenanceRetry.make({
|
|
523
|
+
generation: started.generation,
|
|
524
|
+
notBefore: now + delay,
|
|
525
|
+
nativeOnly: true,
|
|
526
|
+
stalls: Math.min(30, started.stalls + 1)
|
|
527
|
+
}) } : {}
|
|
528
|
+
});
|
|
529
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
530
|
+
if (autonomous) {
|
|
531
|
+
const nativeDeadline = started.activeAtStart > 0 || active > 0 || state.dirty !== started.generation ? now + minimumAlarmDelay : now + delay;
|
|
532
|
+
await transaction.setAlarm(Option.isSome(publicationDeadline) ? Math.max(now + minimumAlarmDelay, Math.min(nativeDeadline, publicationDeadline.value)) : nativeDeadline);
|
|
533
|
+
return "rearmed";
|
|
534
|
+
}
|
|
535
|
+
if (started.activeAtStart > 0 || active > 0 || next.dirty > next.processed) {
|
|
536
|
+
await ensureTransactionAlarmBy(transaction, Option.isSome(publicationDeadline) ? Math.max(now + minimumAlarmDelay, Math.min(now + config.wakeScanInterval, publicationDeadline.value)) : now + config.wakeScanInterval);
|
|
537
|
+
return "rearmed";
|
|
538
|
+
}
|
|
539
|
+
if (Option.isSome(publicationDeadline)) {
|
|
540
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, publicationDeadline.value));
|
|
541
|
+
return "rearmed";
|
|
542
|
+
}
|
|
543
|
+
await transaction.deleteAlarm();
|
|
544
|
+
return "cleared";
|
|
545
|
+
}));
|
|
546
|
+
}));
|
|
547
|
+
yield* failpoint.hit("maintenance:finish:after");
|
|
548
|
+
return yield* annotate(MaintenancePassReport.make({
|
|
549
|
+
phase: "actionable",
|
|
550
|
+
recovered: recovered.length,
|
|
551
|
+
settled: Option.isSome(settlement) ? 1 : 0,
|
|
552
|
+
nonterminal: remaining.length,
|
|
553
|
+
alarm: alarmDisposition
|
|
554
|
+
}));
|
|
555
|
+
});
|
|
556
|
+
return ThreadMaintenance.of({
|
|
557
|
+
pass: Effect.gen(function* () {
|
|
558
|
+
const now = yield* Clock.currentTimeMillis;
|
|
559
|
+
const yieldAfter = DateTime.makeUnsafe(now + 6e5);
|
|
560
|
+
const dispatchUntil = DateTime.makeUnsafe(now + 84e4);
|
|
561
|
+
const observed = { nativeOnly: false };
|
|
562
|
+
return yield* alarm.withWakesDeferred(maintenancePassGate.withPermit(Effect.scoped(pass(yieldAfter, dispatchUntil, observed)).pipe(Effect.onErrorIf(() => true, () => rearmFailure(observed)))));
|
|
563
|
+
}).pipe(Effect.timeoutOrElse({
|
|
564
|
+
duration: "14 minutes",
|
|
565
|
+
orElse: () => DurableAlarmError.make({
|
|
566
|
+
operation: "maintenance pass deadline",
|
|
567
|
+
message: "The maintenance event exceeded its 14 minute deadline; durable recovery remains pending"
|
|
568
|
+
})
|
|
569
|
+
})),
|
|
570
|
+
ensureAlarm: mutations.withSnapshot(() => ensureAlarm()),
|
|
571
|
+
withMutation: (body) => mutations.withMutation(body.pipe(Effect.tap(() => publishCommitted.pipe(Effect.provideService(ThreadPublication, publication)))))
|
|
572
|
+
});
|
|
573
|
+
}));
|
|
574
|
+
};
|
|
575
|
+
//#endregion
|
|
576
|
+
export { DurableAlarmError, DurableAlarmService, MaintenancePassReport, ThreadHostMaintenance, ThreadMaintenance, ThreadMaintenanceFailpoint, ThreadMessageDelivery, ThreadMutationGate, ThreadPublication, publishCommitted, Alarm_exports as t };
|
|
577
|
+
|
|
578
|
+
//# sourceMappingURL=Alarm.mjs.map
|