@yielded/agent-platform-cloudflare 0.0.0-stage → 0.1.0-beta.167
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/Alarm-B0ODr1NE.mjs +1738 -0
- package/dist/Alarm-B0ODr1NE.mjs.map +1 -0
- package/dist/Alarm.d.mts +267 -0
- package/dist/Alarm.mjs +2 -0
- package/dist/BrowserCredentials.d.mts +91 -0
- package/dist/BrowserCredentials.mjs +120 -0
- package/dist/BrowserCredentials.mjs.map +1 -0
- package/dist/BrowserRestCapture.d.mts +34 -0
- package/dist/BrowserRestCapture.mjs +238 -0
- package/dist/BrowserRestCapture.mjs.map +1 -0
- package/dist/BrowserRestCrawl.d.mts +16 -0
- package/dist/BrowserRestCrawl.mjs +244 -0
- package/dist/BrowserRestCrawl.mjs.map +1 -0
- package/dist/BrowserSession-B969Xg-2.d.mts +132 -0
- package/dist/BrowserSession-RfIzxGCH.mjs +515 -0
- package/dist/BrowserSession-RfIzxGCH.mjs.map +1 -0
- package/dist/BrowserSession.d.mts +3 -0
- package/dist/BrowserSession.mjs +4 -0
- package/dist/CloudflareAiGateway.d.mts +64 -0
- package/dist/CloudflareAiGateway.mjs +70 -0
- package/dist/CloudflareAiGateway.mjs.map +1 -0
- package/dist/CloudflareBindings.d.mts +115 -0
- package/dist/CloudflareBindings.mjs +113 -0
- package/dist/CloudflareBindings.mjs.map +1 -0
- package/dist/CloudflareBrowser-5an6m-Qb.mjs +495 -0
- package/dist/CloudflareBrowser-5an6m-Qb.mjs.map +1 -0
- package/dist/CloudflareBrowser-gu7gma0Y.d.mts +80 -0
- package/dist/CloudflareBrowser.d.mts +2 -0
- package/dist/CloudflareBrowser.mjs +2 -0
- package/dist/CloudflareCodeMode.d.mts +44 -0
- package/dist/CloudflareCodeMode.mjs +616 -0
- package/dist/CloudflareCodeMode.mjs.map +1 -0
- package/dist/CloudflareConfig-qDrRLofL.d.mts +100 -0
- package/dist/CloudflareConfig.d.mts +2 -0
- package/dist/CloudflareConfig.mjs +122 -0
- package/dist/CloudflareConfig.mjs.map +1 -0
- package/dist/CloudflareMemory.d.mts +142 -0
- package/dist/CloudflareMemory.mjs +202 -0
- package/dist/CloudflareMemory.mjs.map +1 -0
- package/dist/CloudflareScheduling.d.mts +52 -0
- package/dist/CloudflareScheduling.mjs +396 -0
- package/dist/CloudflareScheduling.mjs.map +1 -0
- package/dist/CloudflareSubscriptions.d.mts +87 -0
- package/dist/CloudflareSubscriptions.mjs +529 -0
- package/dist/CloudflareSubscriptions.mjs.map +1 -0
- package/dist/CloudflareThreadClient.d.mts +1796 -0
- package/dist/CloudflareThreadClient.mjs +382 -0
- package/dist/CloudflareThreadClient.mjs.map +1 -0
- package/dist/InteractiveBrowser-BbG3kGA9.mjs +1934 -0
- package/dist/InteractiveBrowser-BbG3kGA9.mjs.map +1 -0
- package/dist/InteractiveBrowser-wV2Bti5e.d.mts +228 -0
- package/dist/InteractiveBrowser.d.mts +2 -0
- package/dist/InteractiveBrowser.mjs +2 -0
- package/dist/ThreadObject-BcfXbzSO.d.mts +867 -0
- package/dist/ThreadObject-DHjrnlX4.mjs +937 -0
- package/dist/ThreadObject-DHjrnlX4.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +2 -0
- package/dist/ThreadObject.mjs +3 -0
- package/dist/WakeScheduler.d.mts +23 -0
- package/dist/WakeScheduler.mjs +57 -0
- package/dist/WakeScheduler.mjs.map +1 -0
- package/dist/boundary-BguazVkh.mjs +42 -0
- package/dist/boundary-BguazVkh.mjs.map +1 -0
- package/dist/index.d.mts +15 -0
- package/dist/index.mjs +15 -0
- package/dist/prepared-admission-DMX5p9dC.mjs +73 -0
- package/dist/prepared-admission-DMX5p9dC.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -6
- package/src/Alarm.ts +2961 -0
- package/src/BrowserCredentials.ts +146 -0
- package/src/BrowserRestCapture.ts +477 -0
- package/src/BrowserRestCrawl.ts +540 -0
- package/src/BrowserSession.ts +667 -0
- package/src/CloudflareAiGateway.ts +170 -0
- package/src/CloudflareBindings.ts +199 -0
- package/src/CloudflareBrowser.ts +15 -0
- package/src/CloudflareCodeMode.ts +878 -0
- package/src/CloudflareConfig.ts +138 -0
- package/src/CloudflareMemory.ts +420 -0
- package/src/CloudflareScheduling.ts +745 -0
- package/src/CloudflareSubscriptions.ts +1044 -0
- package/src/CloudflareThreadClient.ts +809 -0
- package/src/InteractiveBrowser.ts +3275 -0
- package/src/ThreadObject.ts +1198 -0
- package/src/WakeScheduler.ts +91 -0
- package/src/index.ts +14 -0
- package/src/internal/boundary.ts +55 -0
- package/src/internal/browser-binding.ts +322 -0
- package/src/internal/browser-credentials.ts +362 -0
- package/src/internal/browser-failure.ts +92 -0
- package/src/internal/browser-file-selection.ts +126 -0
- package/src/internal/browser-quick-action.ts +857 -0
- package/src/internal/browser-readonly-live-view.ts +111 -0
- package/src/internal/browser-session-lifecycle.ts +193 -0
- package/src/internal/due-queue.ts +493 -0
- package/src/internal/layers.ts +1068 -0
- package/src/internal/message-delivery.ts +120 -0
- package/src/internal/prepared-admission.ts +116 -0
- package/src/internal/progress-wait.ts +121 -0
- package/src/internal/transport.ts +56 -0
- package/README.md +0 -3
|
@@ -0,0 +1,1738 @@
|
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
2
|
+
import { DurableObjectContext } from "./CloudflareBindings.mjs";
|
|
3
|
+
import { AuxiliaryDispatchMillis, CloudflareDurableRuntimeConfig } from "./CloudflareConfig.mjs";
|
|
4
|
+
import { r as safeCauseMessage } from "./boundary-BguazVkh.mjs";
|
|
5
|
+
import "@yielded/agent/agent-registration";
|
|
6
|
+
import { DurableAgentRuntime, DurableRuntimeConfig, RecoveryBlocked, RecoveryFailure, RecoverySweepResult, isolateRecovery } from "@yielded/agent/durable-agent-runtime";
|
|
7
|
+
import { SubmissionId, ThreadId } from "@yielded/agent/identifiers";
|
|
8
|
+
import { AbortIntentRequest, SubmissionLedger } from "@yielded/agent/submission-ledger";
|
|
9
|
+
import { ThreadProjectionMaintenance } from "@yielded/agent/thread-projection-maintenance";
|
|
10
|
+
import { WakeScheduler } from "@yielded/agent/wake-scheduler";
|
|
11
|
+
import { Cause, Clock, Context, DateTime, Deferred, Effect, ErrorReporter, Exit, Fiber, Layer, Option, Random, Ref, Result, Scheduler, Schema, Scope, Semaphore, Stream, Struct } from "effect";
|
|
12
|
+
import { DurableObjectStorage } from "effect-cf";
|
|
13
|
+
import { SqlClient } from "effect/sql/SqlClient";
|
|
14
|
+
import { SqlError, UnknownError } from "effect/sql/SqlError";
|
|
15
|
+
//#region src/internal/due-queue.ts
|
|
16
|
+
const LaneId = Schema.NonEmptyString.check(Schema.isMaxLength(256));
|
|
17
|
+
const HostLaneId = LaneId.check(Schema.isPattern(/^(?!effect-agent:)/, { message: "The effect-agent: namespace is reserved for framework maintenance lanes" }));
|
|
18
|
+
const Deadline = Schema.NullOr(Schema.Finite);
|
|
19
|
+
const MinimumRetryMillis = 1e3;
|
|
20
|
+
const ParkedRetryMillis = 36e5;
|
|
21
|
+
/** Scheduling metadata only. Domain outboxes, claims and receipts remain authoritative. */
|
|
22
|
+
var DueLane = class extends Schema.Class("CloudflareDueLane")({
|
|
23
|
+
id: LaneId,
|
|
24
|
+
revision: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
25
|
+
dueAt: Deadline,
|
|
26
|
+
stalls: Schema.Int.check(Schema.isBetween({
|
|
27
|
+
minimum: 0,
|
|
28
|
+
maximum: 30
|
|
29
|
+
})),
|
|
30
|
+
progressKey: Schema.NullOr(Schema.NonEmptyString.check(Schema.isMaxLength(512))),
|
|
31
|
+
notBefore: Schema.Finite,
|
|
32
|
+
state: Schema.Literals([
|
|
33
|
+
"idle",
|
|
34
|
+
"pending",
|
|
35
|
+
"parked"
|
|
36
|
+
]),
|
|
37
|
+
reported: Schema.Literals([0, 1])
|
|
38
|
+
}) {};
|
|
39
|
+
const Native = "effect-agent:native";
|
|
40
|
+
const Publication = "effect-agent:publication";
|
|
41
|
+
const Projection = "effect-agent:projection";
|
|
42
|
+
const Messages = "effect-agent:messages";
|
|
43
|
+
const RecoveryEvents = "effect-agent:recovery-events";
|
|
44
|
+
const Lifecycle = "effect-agent:lifecycle";
|
|
45
|
+
const decode = Schema.decodeUnknownSync(Schema.Array(DueLane));
|
|
46
|
+
const equivalent = Schema.toEquivalence(DueLane);
|
|
47
|
+
const views = /* @__PURE__ */ new WeakMap();
|
|
48
|
+
/** Only the explicit source boundary may retain speculative rows across operations. */
|
|
49
|
+
const buffering = (storage) => views.get(storage)?.frame !== void 0;
|
|
50
|
+
const ownsTransaction = (storage, depth) => views.get(storage)?.frame?.sqlDepth === depth;
|
|
51
|
+
const invalidate = (storage) => {
|
|
52
|
+
const view = views.get(storage);
|
|
53
|
+
if (view !== void 0) view.rows = void 0;
|
|
54
|
+
};
|
|
55
|
+
/** Callers reserve the shared SQL connection through commit. Keep the complete view
|
|
56
|
+
* during a transaction, then retain at most 128 lanes between transactions. Intent
|
|
57
|
+
* flushes before source commit; claims and prearming commit before fallible work. */
|
|
58
|
+
const make = (storage) => {
|
|
59
|
+
const sql = storage.sql;
|
|
60
|
+
let view = views.get(storage);
|
|
61
|
+
if (view === void 0) {
|
|
62
|
+
view = {
|
|
63
|
+
rows: void 0,
|
|
64
|
+
frame: void 0,
|
|
65
|
+
retained: 0,
|
|
66
|
+
changes: /* @__PURE__ */ new Map(),
|
|
67
|
+
deadlines: /* @__PURE__ */ new Map()
|
|
68
|
+
};
|
|
69
|
+
views.set(storage, view);
|
|
70
|
+
}
|
|
71
|
+
const current = view;
|
|
72
|
+
const trim = () => {
|
|
73
|
+
if (current.frame === void 0 && current.retained === 0 && (current.rows?.length ?? 0) > 128) current.rows = void 0;
|
|
74
|
+
};
|
|
75
|
+
/** Retain the pass's write-through view without deferring claims or source commits.
|
|
76
|
+
* Failed source transactions still invalidate it; large views are released at exit. */
|
|
77
|
+
const withView = (body) => Effect.acquireUseRelease(Effect.sync(() => {
|
|
78
|
+
current.retained++;
|
|
79
|
+
}), () => body, () => Effect.sync(() => {
|
|
80
|
+
current.retained--;
|
|
81
|
+
trim();
|
|
82
|
+
}));
|
|
83
|
+
const begin = (sqlDepth) => {
|
|
84
|
+
const frame = {
|
|
85
|
+
parent: current.frame,
|
|
86
|
+
sqlDepth,
|
|
87
|
+
rows: current.rows,
|
|
88
|
+
changes: current.changes,
|
|
89
|
+
deadlines: current.deadlines
|
|
90
|
+
};
|
|
91
|
+
current.frame = frame;
|
|
92
|
+
current.changes = new Map(current.changes);
|
|
93
|
+
current.deadlines = new Map(current.deadlines);
|
|
94
|
+
return frame;
|
|
95
|
+
};
|
|
96
|
+
const finish = (frame, success) => {
|
|
97
|
+
current.frame = frame.parent;
|
|
98
|
+
if (!success) {
|
|
99
|
+
current.rows = frame.parent === void 0 ? void 0 : frame.rows;
|
|
100
|
+
current.changes = frame.changes;
|
|
101
|
+
current.deadlines = frame.deadlines;
|
|
102
|
+
}
|
|
103
|
+
if (frame.parent === void 0) {
|
|
104
|
+
current.changes.clear();
|
|
105
|
+
current.deadlines.clear();
|
|
106
|
+
trim();
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
const write = (changes) => {
|
|
110
|
+
if (decode(sql.exec(`WITH changes (id, revision, dueAt, stalls, progressKey, notBefore, state, reported, expected_revision) AS (
|
|
111
|
+
VALUES ${changes.map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?)").join(", ")}
|
|
112
|
+
) INSERT INTO platform_cloudflare_due_queue
|
|
113
|
+
(id, revision, dueAt, stalls, progressKey, notBefore, state, reported)
|
|
114
|
+
SELECT id, revision, dueAt, stalls, progressKey, notBefore, state, reported FROM changes WHERE 1
|
|
115
|
+
ON CONFLICT(id) DO UPDATE SET revision = excluded.revision, dueAt = excluded.dueAt,
|
|
116
|
+
stalls = excluded.stalls, progressKey = excluded.progressKey,
|
|
117
|
+
notBefore = excluded.notBefore, state = excluded.state, reported = excluded.reported
|
|
118
|
+
WHERE platform_cloudflare_due_queue.revision = (
|
|
119
|
+
SELECT expected_revision FROM changes WHERE changes.id = excluded.id
|
|
120
|
+
) RETURNING id, revision, dueAt, stalls, progressKey, notBefore, state, reported`, ...changes.flatMap(({ before, after }) => [
|
|
121
|
+
after.id,
|
|
122
|
+
after.revision,
|
|
123
|
+
after.dueAt,
|
|
124
|
+
after.stalls,
|
|
125
|
+
after.progressKey,
|
|
126
|
+
after.notBefore,
|
|
127
|
+
after.state,
|
|
128
|
+
after.reported,
|
|
129
|
+
before?.revision ?? -1
|
|
130
|
+
])).toArray()).length !== changes.length) throw new Error("Maintenance due revision changed during transaction");
|
|
131
|
+
};
|
|
132
|
+
const flush = (frame) => {
|
|
133
|
+
if (frame.parent !== void 0) return;
|
|
134
|
+
const changes = [...current.changes.values()].filter(({ before, after }) => before === void 0 || !equivalent(before, after));
|
|
135
|
+
for (let offset = 0; offset < changes.length; offset += 11) write(changes.slice(offset, offset + 11));
|
|
136
|
+
};
|
|
137
|
+
/** Own the flush inside the source transaction without changing the SQL service.
|
|
138
|
+
* Acquire its shared connection permit through Effect requirements. Cleanup follows
|
|
139
|
+
* native settlement, including commit rejection or interruption. */
|
|
140
|
+
const withTransaction = (body) => Effect.flatMap(SqlClient, (client) => Effect.flatMap(Effect.serviceOption(client.transactionService), (ambient) => {
|
|
141
|
+
if (ambient._tag === "Some" && !ownsTransaction(storage, ambient.value[1])) return Effect.fail(SqlError.make({ reason: UnknownError.make({
|
|
142
|
+
operation: "begin maintenance source transaction",
|
|
143
|
+
message: "Use ThreadMutationGate.withTransaction at the outermost source transaction",
|
|
144
|
+
cause: /* @__PURE__ */ new Error("Unwrapped SQL transaction cannot own buffered maintenance intent")
|
|
145
|
+
}) }));
|
|
146
|
+
return Effect.uninterruptibleMask((restore) => Effect.suspend(() => {
|
|
147
|
+
let frame;
|
|
148
|
+
return client.withTransaction(Effect.flatMap(Effect.serviceOption(client.transactionService), (transaction) => Effect.sync(() => {
|
|
149
|
+
if (transaction._tag === "None") throw new Error("SQL transaction unavailable");
|
|
150
|
+
frame = begin(transaction.value[1]);
|
|
151
|
+
})).pipe(Effect.andThen(restore(body)), Effect.tap(() => Effect.try({
|
|
152
|
+
try: () => {
|
|
153
|
+
if (frame !== void 0) flush(frame);
|
|
154
|
+
},
|
|
155
|
+
catch: (cause) => SqlError.make({ reason: UnknownError.make({
|
|
156
|
+
cause,
|
|
157
|
+
operation: "flush maintenance due queue",
|
|
158
|
+
message: "Maintenance intent could not commit"
|
|
159
|
+
}) })
|
|
160
|
+
})))).pipe(Effect.onExit((exit) => Effect.sync(() => {
|
|
161
|
+
if (frame !== void 0) finish(frame, Exit.isSuccess(exit));
|
|
162
|
+
})), Effect.provideService(Scheduler.PreventSchedulerYield, true));
|
|
163
|
+
}));
|
|
164
|
+
}));
|
|
165
|
+
const transaction = async (body) => {
|
|
166
|
+
let frame;
|
|
167
|
+
try {
|
|
168
|
+
const result = await storage.transaction(async (transaction) => {
|
|
169
|
+
frame = begin();
|
|
170
|
+
const result = await body(transaction);
|
|
171
|
+
flush(frame);
|
|
172
|
+
return result;
|
|
173
|
+
});
|
|
174
|
+
if (frame !== void 0) finish(frame, true);
|
|
175
|
+
return result;
|
|
176
|
+
} catch (cause) {
|
|
177
|
+
if (frame !== void 0) finish(frame, false);
|
|
178
|
+
throw cause;
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
const initialize = () => {
|
|
182
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS platform_cloudflare_due_queue (
|
|
183
|
+
id TEXT PRIMARY KEY, revision INTEGER NOT NULL, dueAt REAL, stalls INTEGER NOT NULL,
|
|
184
|
+
progressKey TEXT, notBefore REAL NOT NULL DEFAULT 0, state TEXT NOT NULL DEFAULT 'pending',
|
|
185
|
+
reported INTEGER NOT NULL DEFAULT 0
|
|
186
|
+
)`);
|
|
187
|
+
const columns = new Set(sql.exec("PRAGMA table_info(platform_cloudflare_due_queue)").toArray().map((row) => row.name));
|
|
188
|
+
if (!columns.has("reported")) sql.exec("ALTER TABLE platform_cloudflare_due_queue ADD COLUMN reported INTEGER NOT NULL DEFAULT 0");
|
|
189
|
+
if (!columns.has("progressKey")) sql.exec("ALTER TABLE platform_cloudflare_due_queue ADD COLUMN progressKey TEXT");
|
|
190
|
+
if (!columns.has("notBefore")) sql.exec("ALTER TABLE platform_cloudflare_due_queue ADD COLUMN notBefore REAL NOT NULL DEFAULT 0");
|
|
191
|
+
if (!columns.has("state")) {
|
|
192
|
+
sql.exec("ALTER TABLE platform_cloudflare_due_queue ADD COLUMN state TEXT NOT NULL DEFAULT 'pending'");
|
|
193
|
+
sql.exec("UPDATE platform_cloudflare_due_queue SET state = 'idle' WHERE dueAt IS NULL");
|
|
194
|
+
}
|
|
195
|
+
};
|
|
196
|
+
/** Keep dormant rows so delete/reinsert cannot reuse a revision held by an older wave. */
|
|
197
|
+
const read = () => {
|
|
198
|
+
if (current.rows !== void 0) return current.rows;
|
|
199
|
+
const rows = decode(sql.exec("SELECT id, revision, dueAt, stalls, progressKey, notBefore, state, reported FROM platform_cloudflare_due_queue").toArray());
|
|
200
|
+
if (current.frame !== void 0 || current.retained > 0 || rows.length <= 128) current.rows = rows;
|
|
201
|
+
return rows;
|
|
202
|
+
};
|
|
203
|
+
const set = (after) => {
|
|
204
|
+
const rows = read();
|
|
205
|
+
const before = rows.find((row) => row.id === after.id);
|
|
206
|
+
if (before !== void 0 && equivalent(before, after)) return before;
|
|
207
|
+
const previous = current.changes.get(after.id);
|
|
208
|
+
const change = {
|
|
209
|
+
before: previous === void 0 ? before : previous.before,
|
|
210
|
+
after
|
|
211
|
+
};
|
|
212
|
+
if (current.frame === void 0) write([change]);
|
|
213
|
+
else current.changes.set(after.id, change);
|
|
214
|
+
current.rows = [...rows.filter((row) => row.id !== after.id), after];
|
|
215
|
+
trim();
|
|
216
|
+
return after;
|
|
217
|
+
};
|
|
218
|
+
const dirty = (id, dueAt, progress = false, progressKey) => {
|
|
219
|
+
Schema.decodeSync(LaneId)(id);
|
|
220
|
+
Schema.decodeSync(Schema.Finite)(dueAt);
|
|
221
|
+
const row = read().find((row) => row.id === id);
|
|
222
|
+
const earliest = Math.min(current.deadlines.get(id) ?? dueAt, dueAt);
|
|
223
|
+
if (current.frame !== void 0) current.deadlines.set(id, earliest);
|
|
224
|
+
return set(DueLane.make({
|
|
225
|
+
id,
|
|
226
|
+
revision: row === void 0 ? 1 : row.revision + Number(progress),
|
|
227
|
+
dueAt: row === void 0 || progress ? earliest : Math.max(row.notBefore, Math.min(row.dueAt ?? Infinity, earliest)),
|
|
228
|
+
state: progress || (row?.stalls ?? 0) < 8 ? "pending" : row?.state ?? "pending",
|
|
229
|
+
stalls: progress ? 0 : row?.stalls ?? 0,
|
|
230
|
+
notBefore: progress ? 0 : row?.notBefore ?? 0,
|
|
231
|
+
reported: progress ? 0 : row?.reported ?? 0,
|
|
232
|
+
progressKey: progressKey ?? row?.progressKey ?? null
|
|
233
|
+
}));
|
|
234
|
+
};
|
|
235
|
+
/** A strictly increasing source cursor; old or repeated notices cannot renew a budget. */
|
|
236
|
+
const progress = (id, dueAt, cursor) => {
|
|
237
|
+
Schema.decodeSync(Schema.BigInt.check(Schema.isGreaterThanOrEqualToBigInt(0n)))(cursor);
|
|
238
|
+
const previous = read().find((row) => row.id === id)?.progressKey;
|
|
239
|
+
if (previous !== void 0 && previous !== null && cursor <= BigInt(previous)) return false;
|
|
240
|
+
dirty(id, dueAt, true, String(cursor));
|
|
241
|
+
return true;
|
|
242
|
+
};
|
|
243
|
+
/** One initial native discovery wave; host lanes always require explicit enrollment. */
|
|
244
|
+
const register = (id) => {
|
|
245
|
+
if (read().some((row) => row.id === id)) return;
|
|
246
|
+
set(DueLane.make({
|
|
247
|
+
id,
|
|
248
|
+
revision: 0,
|
|
249
|
+
dueAt: 0,
|
|
250
|
+
stalls: 0,
|
|
251
|
+
progressKey: null,
|
|
252
|
+
notBefore: 0,
|
|
253
|
+
state: "pending",
|
|
254
|
+
reported: 0
|
|
255
|
+
}));
|
|
256
|
+
};
|
|
257
|
+
/** This transaction must commit before fallible work: never defer a debit across a wave. */
|
|
258
|
+
const claim = (lane, nowMillis, native = false) => {
|
|
259
|
+
const row = read().find((row) => row.id === lane.id);
|
|
260
|
+
if (row === void 0 || row.revision !== lane.revision || row.dueAt === null || !(row.dueAt <= nowMillis || native && row.notBefore <= nowMillis)) return;
|
|
261
|
+
const stalls = Math.min(row.stalls + 1, 8);
|
|
262
|
+
const notBefore = nowMillis + (stalls >= 8 ? ParkedRetryMillis : Math.min(6e4, MinimumRetryMillis * 2 ** row.stalls));
|
|
263
|
+
return set(DueLane.make({
|
|
264
|
+
...row,
|
|
265
|
+
stalls,
|
|
266
|
+
notBefore,
|
|
267
|
+
dueAt: notBefore,
|
|
268
|
+
state: stalls >= 8 ? "parked" : "pending"
|
|
269
|
+
}));
|
|
270
|
+
};
|
|
271
|
+
const complete = (lane, dueAt) => {
|
|
272
|
+
Schema.decodeSync(Deadline)(dueAt);
|
|
273
|
+
const row = read().find((row) => row.id === lane.id);
|
|
274
|
+
if (row === void 0 || row.revision !== lane.revision) return;
|
|
275
|
+
const after = DueLane.make({
|
|
276
|
+
...row,
|
|
277
|
+
dueAt: dueAt === null ? null : Math.max(dueAt, row.notBefore),
|
|
278
|
+
state: dueAt === null ? "idle" : row.stalls >= 8 ? "parked" : "pending",
|
|
279
|
+
stalls: dueAt === null ? 0 : row.stalls,
|
|
280
|
+
notBefore: dueAt === null ? 0 : row.notBefore,
|
|
281
|
+
reported: dueAt === null ? 0 : row.reported
|
|
282
|
+
});
|
|
283
|
+
return set(DueLane.make({
|
|
284
|
+
...after,
|
|
285
|
+
revision: row.revision + 1
|
|
286
|
+
}));
|
|
287
|
+
};
|
|
288
|
+
/** Native checkpoints already fence their own dirty/processed generation. */
|
|
289
|
+
const checkpointNative = (dueAt) => {
|
|
290
|
+
Schema.decodeSync(Deadline)(dueAt);
|
|
291
|
+
const rows = read();
|
|
292
|
+
const row = rows.find((row) => row.id === Native);
|
|
293
|
+
if (row === void 0) return;
|
|
294
|
+
const publication = rows.find((row) => row.id === Publication);
|
|
295
|
+
set(DueLane.make({
|
|
296
|
+
...row,
|
|
297
|
+
dueAt: dueAt === null || publication?.state === "parked" ? null : Math.max(dueAt, row.notBefore),
|
|
298
|
+
state: dueAt === null ? "idle" : row.stalls >= 8 ? "parked" : "pending",
|
|
299
|
+
stalls: dueAt === null ? 0 : row.stalls,
|
|
300
|
+
notBefore: dueAt === null ? 0 : row.notBefore,
|
|
301
|
+
reported: dueAt === null ? 0 : row.reported
|
|
302
|
+
}));
|
|
303
|
+
};
|
|
304
|
+
const takeParkedReports = () => read().filter((row) => row.state === "parked" && row.reported === 0).map((row) => set(DueLane.make({
|
|
305
|
+
...row,
|
|
306
|
+
reported: 1
|
|
307
|
+
})));
|
|
308
|
+
return {
|
|
309
|
+
initialize,
|
|
310
|
+
withView,
|
|
311
|
+
withTransaction,
|
|
312
|
+
transaction,
|
|
313
|
+
read,
|
|
314
|
+
dirty,
|
|
315
|
+
progress,
|
|
316
|
+
register,
|
|
317
|
+
claim,
|
|
318
|
+
complete,
|
|
319
|
+
checkpointNative,
|
|
320
|
+
takeParkedReports
|
|
321
|
+
};
|
|
322
|
+
};
|
|
323
|
+
const next = (rows) => rows.reduce((next, row) => Math.min(next, row.dueAt ?? Infinity), Infinity);
|
|
324
|
+
//#endregion
|
|
325
|
+
//#region src/Alarm.ts
|
|
326
|
+
var Alarm_exports = /* @__PURE__ */ __exportAll({
|
|
327
|
+
DurableAlarmError: () => DurableAlarmError,
|
|
328
|
+
DurableAlarmService: () => DurableAlarmService,
|
|
329
|
+
MaintenancePassReport: () => MaintenancePassReport,
|
|
330
|
+
ThreadHostMaintenance: () => ThreadHostMaintenance,
|
|
331
|
+
ThreadMaintenance: () => ThreadMaintenance,
|
|
332
|
+
ThreadMaintenanceFailpoint: () => ThreadMaintenanceFailpoint,
|
|
333
|
+
ThreadMessageDelivery: () => ThreadMessageDelivery,
|
|
334
|
+
ThreadMutationGate: () => ThreadMutationGate,
|
|
335
|
+
ThreadNativeMaintenance: () => ThreadNativeMaintenance,
|
|
336
|
+
ThreadPublication: () => ThreadPublication,
|
|
337
|
+
ThreadRecoveryEvents: () => ThreadRecoveryEvents,
|
|
338
|
+
ThreadRecoveryFaultEvent: () => ThreadRecoveryFaultEvent,
|
|
339
|
+
publishCommitted: () => publishCommitted
|
|
340
|
+
});
|
|
341
|
+
/**
|
|
342
|
+
* The single multiplexed Durable Object alarm (decision D-P6-2). A Durable Object has ONE
|
|
343
|
+
* alarm slot; due work (lease expiry, settlement
|
|
344
|
+
* and abort re-checks, retry backoff) multiplexes into one idempotent maintenance pass, and
|
|
345
|
+
* the slot always holds the EARLIEST deadline any caller asked for.
|
|
346
|
+
*
|
|
347
|
+
* The alarm invariant (plan §1.4): every committed actionable mutation carries a newer durable
|
|
348
|
+
* maintenance generation and a committed alarm. Stable externally-driven waits may be
|
|
349
|
+
* nonterminal without retaining an alarm; their resolving mutation advances the generation and
|
|
350
|
+
* restores the alarm atomically.
|
|
351
|
+
*/
|
|
352
|
+
/** The Durable Object alarm API failed; surfaces on host entry points as a typed refusal. */
|
|
353
|
+
var DurableAlarmError = class extends Schema.TaggedError()("DurableAlarmError", {
|
|
354
|
+
operation: Schema.String,
|
|
355
|
+
message: Schema.String,
|
|
356
|
+
cause: Schema.optionalKey(Schema.Defect())
|
|
357
|
+
}) {};
|
|
358
|
+
/** Content-free operational notice: pending work now retries at the hourly ceiling. */
|
|
359
|
+
var MaintenanceRetryParked = class extends Schema.TaggedError()("MaintenanceRetryParked", {
|
|
360
|
+
lane: Schema.String,
|
|
361
|
+
retryAfterMillis: Schema.Number
|
|
362
|
+
}) {};
|
|
363
|
+
const alarmFailure = (operation) => (cause) => DurableAlarmError.make({
|
|
364
|
+
operation,
|
|
365
|
+
message: safeCauseMessage(cause, "The Cloudflare alarm API failed without a diagnostic"),
|
|
366
|
+
cause
|
|
367
|
+
});
|
|
368
|
+
const makeStorageEffect = Effect.gen(function* () {
|
|
369
|
+
const sql = yield* SqlClient;
|
|
370
|
+
const { ctx } = yield* DurableObjectContext;
|
|
371
|
+
const invalidate$1 = Effect.sync(() => invalidate(ctx.storage));
|
|
372
|
+
return (operation, execute) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => {
|
|
373
|
+
const body = Effect.uninterruptible(execute);
|
|
374
|
+
if (current._tag === "Some") {
|
|
375
|
+
if (buffering(ctx.storage)) return body;
|
|
376
|
+
return body.pipe(Effect.ensuring(invalidate$1));
|
|
377
|
+
}
|
|
378
|
+
return Effect.scoped(Effect.andThen(sql.reserve.pipe(Effect.mapError(alarmFailure(operation))), body.pipe(Effect.onExit((exit) => Exit.isFailure(exit) ? invalidate$1 : Effect.void))));
|
|
379
|
+
});
|
|
380
|
+
});
|
|
381
|
+
const makeStorageOperation = Effect.map(makeStorageEffect, (run) => (operation, execute) => run(operation, Effect.tryPromise({
|
|
382
|
+
try: execute,
|
|
383
|
+
catch: alarmFailure(operation)
|
|
384
|
+
})));
|
|
385
|
+
/** Native `ctx.storage` alarm slot owned by ThreadMaintenance; storage is truth. */
|
|
386
|
+
var DurableAlarmService = class DurableAlarmService extends Context.Service()("@effect-agent/platform-cloudflare/DurableAlarmService") {
|
|
387
|
+
static layer = Layer.effect(DurableAlarmService)(Effect.gen(function* () {
|
|
388
|
+
const { ctx } = yield* DurableObjectContext;
|
|
389
|
+
/**
|
|
390
|
+
* In-memory pass bookkeeping — a pure CACHE, never state: a fresh incarnation has no
|
|
391
|
+
* running pass, and a deferred wake lost to eviction was only ever a promptness hint
|
|
392
|
+
* on top of the already-committed pre-armed alarm.
|
|
393
|
+
*/
|
|
394
|
+
const runningPasses = yield* Ref.make(0);
|
|
395
|
+
const storageOperation = yield* makeStorageOperation;
|
|
396
|
+
const scheduled = storageOperation("get alarm", () => ctx.storage.getAlarm()).pipe(Effect.map((deadline) => deadline === null ? Option.none() : Option.some(deadline)));
|
|
397
|
+
const scheduleAt = (epochMillis) => storageOperation("set alarm", () => ctx.storage.setAlarm(epochMillis));
|
|
398
|
+
const ensureScheduledBy = (epochMillis) => storageOperation("ensure alarm", () => ctx.storage.transaction(async (transaction) => {
|
|
399
|
+
const existing = await transaction.getAlarm();
|
|
400
|
+
if (existing === null || existing > epochMillis) await transaction.setAlarm(epochMillis);
|
|
401
|
+
}));
|
|
402
|
+
const armNow = Clock.currentTimeMillis.pipe(Effect.flatMap((now) => storageOperation("wake maintenance lanes", () => ctx.storage.transaction(async (transaction) => {
|
|
403
|
+
const next$7 = next(make(ctx.storage).read());
|
|
404
|
+
if (Number.isFinite(next$7)) await ensureTransactionAlarmBy(transaction, Math.max(now, next$7));
|
|
405
|
+
}))));
|
|
406
|
+
const scheduleNow = Ref.get(runningPasses).pipe(Effect.flatMap((passes) => passes > 0 ? Effect.void : armNow));
|
|
407
|
+
const withWakesDeferred = (body) => Ref.update(runningPasses, (passes) => passes + 1).pipe(Effect.andThen(body), Effect.ensuring(Ref.update(runningPasses, (passes) => passes - 1)));
|
|
408
|
+
const cancel = storageOperation("delete alarm", () => ctx.storage.deleteAlarm());
|
|
409
|
+
return DurableAlarmService.of({
|
|
410
|
+
scheduled,
|
|
411
|
+
scheduleAt,
|
|
412
|
+
ensureScheduledBy,
|
|
413
|
+
scheduleNow,
|
|
414
|
+
withWakesDeferred,
|
|
415
|
+
cancel
|
|
416
|
+
});
|
|
417
|
+
}));
|
|
418
|
+
};
|
|
419
|
+
/** What one maintenance pass did — auditable evidence mirroring `NodeDurableHost`'s report. */
|
|
420
|
+
var MaintenancePassReport = class extends Schema.Class("@effect-agent/platform-cloudflare/MaintenancePassReport")({
|
|
421
|
+
/** `caught-up` ran no runtime work (publication may be pending); `actionable` ran recovery. */
|
|
422
|
+
phase: Schema.Literals(["caught-up", "actionable"]),
|
|
423
|
+
/** Recovery decisions and Thread faults observed during this event. */
|
|
424
|
+
recovered: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
425
|
+
/** Head Attempts settled during the event. Joined input may settle with each head. */
|
|
426
|
+
settled: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
427
|
+
/** Submissions still nonterminal after the pass (suspended/unknown lanes stay honest). */
|
|
428
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
429
|
+
/** `rearmed` for dirty/autonomous work, `cleared` for stable waits or settlement. */
|
|
430
|
+
alarm: Schema.Literals(["rearmed", "cleared"])
|
|
431
|
+
}) {};
|
|
432
|
+
/** Test-only fault authority; production uses the inert layer. */
|
|
433
|
+
var ThreadMaintenanceFailpoint = class extends Context.Service()("@effect-agent/platform-cloudflare/ThreadMaintenanceFailpoint") {
|
|
434
|
+
static layer = Layer.succeed(this)({ hit: () => Effect.void });
|
|
435
|
+
};
|
|
436
|
+
const emptyPublication = {
|
|
437
|
+
invalidate: Effect.void,
|
|
438
|
+
prepareGeneration: () => Effect.void,
|
|
439
|
+
drain: Effect.succeed(Option.none())
|
|
440
|
+
};
|
|
441
|
+
/** Opt in with `ThreadObject.layer(registrations, { publication: Layer.effect(ThreadPublication)(...) })`. */
|
|
442
|
+
var ThreadPublication = class extends Context.Service()("@effect-agent/platform-cloudflare/ThreadPublication") {
|
|
443
|
+
static layer = Layer.succeed(this)(emptyPublication);
|
|
444
|
+
};
|
|
445
|
+
const requiresPublication = Effect.map(Effect.serviceOption(ThreadPublication), (service) => Option.isNone(service) || service.value !== emptyPublication);
|
|
446
|
+
/**
|
|
447
|
+
* Host-assembled native message recovery. Preparation is a bounded local selection; the
|
|
448
|
+
* driver bounds each actual Claim and persists its timeout/retry before `run` returns.
|
|
449
|
+
* The alarm must not add a timer starting at selection: Claim setup and retry commits belong
|
|
450
|
+
* to the driver. The prepared allowance decides only whether a wave fits this event.
|
|
451
|
+
*/
|
|
452
|
+
const ThreadMessageDelivery = Context.Reference("@effect-agent/platform-cloudflare/ThreadMessageDelivery", { defaultValue: () => ({ prepare: Effect.succeed({
|
|
453
|
+
timeoutMillis: 1,
|
|
454
|
+
run: Effect.succeed(Option.none())
|
|
455
|
+
}) }) });
|
|
456
|
+
/** Independent lanes share the existing alarm; compose hosts by concatenating their lanes. */
|
|
457
|
+
const ThreadHostMaintenance = Context.Reference("@effect-agent/platform-cloudflare/ThreadHostMaintenance", { defaultValue: () => ({ lanes: [] }) });
|
|
458
|
+
/** @internal Framework handlers are separate from host registrations and their namespace. */
|
|
459
|
+
const ThreadNativeMaintenance = Context.Reference("@effect-agent/platform-cloudflare/ThreadNativeMaintenance", { defaultValue: () => ({ lanes: [] }) });
|
|
460
|
+
/** @internal Required host publication only; native lifecycle facts use a maintenance lane. */
|
|
461
|
+
const publishCommitted = Effect.gen(function* () {
|
|
462
|
+
const publication = yield* ThreadPublication;
|
|
463
|
+
yield* publication.invalidate;
|
|
464
|
+
yield* publication.drain;
|
|
465
|
+
}).pipe(Effect.catchCause((cause) => Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.logError("Thread publication deferred after source commit", cause)));
|
|
466
|
+
const MaintenanceGeneration = Schema.BigIntFromString.check(Schema.isGreaterThanOrEqualToBigInt(0n));
|
|
467
|
+
/** A committed recovery transition for one affected Submission; never a Settlement. */
|
|
468
|
+
var ThreadRecoveryFaultEvent = class extends Schema.Class("@effect-agent/platform-cloudflare/ThreadRecoveryFaultEvent")({
|
|
469
|
+
schemaVersion: Schema.Literal(1),
|
|
470
|
+
/** Monotonic within the physical Object; retained unchanged on delivery retries. */
|
|
471
|
+
sequence: MaintenanceGeneration,
|
|
472
|
+
transition: Schema.Literals([
|
|
473
|
+
"created",
|
|
474
|
+
"changed",
|
|
475
|
+
"cleared"
|
|
476
|
+
]),
|
|
477
|
+
threadId: ThreadId,
|
|
478
|
+
submissionId: SubmissionId,
|
|
479
|
+
occurredAt: Schema.Finite,
|
|
480
|
+
firstFailedAt: Schema.Finite,
|
|
481
|
+
/** The current failure, or the last failure for a cleared event. */
|
|
482
|
+
failure: RecoveryFailure
|
|
483
|
+
}) {};
|
|
484
|
+
const recoveryEventKey = (sequence) => `effect-agent:thread-recovery-event:v1:${sequence}`;
|
|
485
|
+
const encodeRecoveryEvent = Schema.encodeSync(ThreadRecoveryFaultEvent);
|
|
486
|
+
const decodeRecoveryEvent = Schema.decodeUnknownSync(ThreadRecoveryFaultEvent);
|
|
487
|
+
/**
|
|
488
|
+
* Ordered, at-least-once delivery through the existing host maintenance lane. Return only
|
|
489
|
+
* after durably applying the event or retaining it in a host outbox. Deduplicate by physical
|
|
490
|
+
* Object and sequence; callbacks may repeat after interruption. This is a trusted host hook,
|
|
491
|
+
* not pre-authorized UI data: the host owns recipient authorization and safe presentation.
|
|
492
|
+
* Capture services in the Layer. Do not call producers or write the raw alarm slot.
|
|
493
|
+
*/
|
|
494
|
+
var ThreadRecoveryEvents = class extends Context.Service()("@effect-agent/platform-cloudflare/ThreadRecoveryEvents") {};
|
|
495
|
+
var BindingWait = class extends Schema.Class("BindingWait")({
|
|
496
|
+
threadId: ThreadId,
|
|
497
|
+
submissionId: SubmissionId,
|
|
498
|
+
reportedAt: Schema.Finite
|
|
499
|
+
}) {};
|
|
500
|
+
/**
|
|
501
|
+
* A durable observation of blocked recovery, independent of the execution journal. This is
|
|
502
|
+
* neither a Settlement nor proof that external effects did not happen. A successful recovery
|
|
503
|
+
* sweep clears it; repair must preserve canonical history and the original admission identity.
|
|
504
|
+
*/
|
|
505
|
+
var ThreadRecoveryFault = class extends Schema.Class("@effect-agent/platform-cloudflare/ThreadRecoveryFault")({
|
|
506
|
+
schemaVersion: Schema.Literal(1),
|
|
507
|
+
threadId: ThreadId,
|
|
508
|
+
firstFailedAt: Schema.Finite,
|
|
509
|
+
lastFailedAt: Schema.Finite,
|
|
510
|
+
/** Earliest automatic recovery retry; new admissions do not erase this deadline. */
|
|
511
|
+
retryAt: Schema.Finite,
|
|
512
|
+
/** Saturates at 2^31 - 1; one observation per Thread per recovery sweep. */
|
|
513
|
+
attempts: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(2147483647)),
|
|
514
|
+
failure: RecoveryFailure,
|
|
515
|
+
/** Individually retained recipients survive settlement until their clear commits. */
|
|
516
|
+
recipientCount: Schema.optionalKey(Schema.Natural)
|
|
517
|
+
}) {};
|
|
518
|
+
const recoveryFaultKey = (threadId) => `effect-agent:thread-recovery-fault:v1:${threadId}`;
|
|
519
|
+
const decodeRecoveryFaultValue = Schema.decodeUnknownSync(ThreadRecoveryFault);
|
|
520
|
+
const decodeRecoveryFault = (threadId, encoded) => {
|
|
521
|
+
const fault = decodeRecoveryFaultValue(encoded);
|
|
522
|
+
if (fault.threadId !== threadId) throw new Error("Recovery status does not match its Thread key");
|
|
523
|
+
return fault;
|
|
524
|
+
};
|
|
525
|
+
const encodeRecoveryFault = Schema.encodeSync(ThreadRecoveryFault);
|
|
526
|
+
const sameRecoveryFailure = Schema.toEquivalence(RecoveryFailure);
|
|
527
|
+
const decodeRecoveryRecipient = Schema.decodeUnknownSync(SubmissionId);
|
|
528
|
+
const encodeRecoveryRecipient = Schema.encodeSync(SubmissionId);
|
|
529
|
+
const recoveryRecipientKey = (threadId, index) => `effect-agent:recovery-recipient:v1:${threadId.length}:${threadId}:${index}`;
|
|
530
|
+
const recoveryRecipientKeys = (fault, start) => Array.from({ length: Math.min(128, (fault.recipientCount ?? 0) - start) }, (_, offset) => recoveryRecipientKey(fault.threadId, start + offset));
|
|
531
|
+
const readRecoveryRecipients = async (storage, fault) => {
|
|
532
|
+
const ids = [];
|
|
533
|
+
for (let start = 0; start < (fault.recipientCount ?? 0); start += 128) {
|
|
534
|
+
const keys = recoveryRecipientKeys(fault, start);
|
|
535
|
+
const values = await storage.get(keys);
|
|
536
|
+
for (const key of keys) ids.push(decodeRecoveryRecipient(values.get(key)));
|
|
537
|
+
}
|
|
538
|
+
return ids;
|
|
539
|
+
};
|
|
540
|
+
const appendRecoveryRecipients = async (transaction, threadId, count, added) => {
|
|
541
|
+
for (let start = 0; start < added.length; start += 128) await transaction.put(Object.fromEntries(added.slice(start, start + 128).map((id, offset) => [recoveryRecipientKey(threadId, count + start + offset), encodeRecoveryRecipient(id)])));
|
|
542
|
+
};
|
|
543
|
+
const recoveryEvent = (fault, submissionId, transition, occurredAt) => ({
|
|
544
|
+
schemaVersion: 1,
|
|
545
|
+
transition,
|
|
546
|
+
threadId: fault.threadId,
|
|
547
|
+
submissionId,
|
|
548
|
+
occurredAt,
|
|
549
|
+
firstFailedAt: fault.firstFailedAt,
|
|
550
|
+
failure: fault.failure
|
|
551
|
+
});
|
|
552
|
+
const nativeDispatchConcurrency = 2;
|
|
553
|
+
/** Post-native delivery shares a finite connection budget across independent host lanes. */
|
|
554
|
+
const afterNativeDispatchConcurrency = 2;
|
|
555
|
+
var MaintenanceRetry = class extends Schema.Class("MaintenanceRetry")({
|
|
556
|
+
generation: MaintenanceGeneration,
|
|
557
|
+
notBefore: Schema.Finite,
|
|
558
|
+
nativeOnly: Schema.Boolean,
|
|
559
|
+
stalls: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(30))
|
|
560
|
+
}) {};
|
|
561
|
+
/** Versioned, platform-private maintenance state stored through Durable Object KV. */
|
|
562
|
+
var ThreadMaintenanceState = class extends Schema.Class("@effect-agent/platform-cloudflare/ThreadMaintenanceState")({
|
|
563
|
+
schemaVersion: Schema.Literal(1),
|
|
564
|
+
dirty: MaintenanceGeneration,
|
|
565
|
+
processed: MaintenanceGeneration,
|
|
566
|
+
nonterminal: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
567
|
+
/** One physical-owner cursor; old single-lane records need no conversion. */
|
|
568
|
+
lastServedThreadId: Schema.optionalKey(ThreadId),
|
|
569
|
+
/** Rotate old recovery independently of dispatch, including after eviction or timeout. */
|
|
570
|
+
lastRecoveredThreadId: Schema.optionalKey(ThreadId),
|
|
571
|
+
/** Rotate bounded post-native admission across full events and Object eviction. */
|
|
572
|
+
lastAfterNativeLaneId: Schema.optionalKey(LaneId),
|
|
573
|
+
bindingRegistryKey: Schema.optionalKey(Schema.String),
|
|
574
|
+
bindingWaits: Schema.optionalKey(Schema.Array(BindingWait)),
|
|
575
|
+
recoveryEventSequence: Schema.optionalKey(MaintenanceGeneration),
|
|
576
|
+
recoveryEventAcknowledged: Schema.optionalKey(MaintenanceGeneration),
|
|
577
|
+
/** Absent on older records. A newer mutation makes this retry obsolete. */
|
|
578
|
+
retry: Schema.optionalKey(MaintenanceRetry)
|
|
579
|
+
}) {};
|
|
580
|
+
const MAINTENANCE_STATE_KEY = "effect-agent:thread-maintenance:v1";
|
|
581
|
+
const decodeMaintenanceState = Schema.decodeUnknownSync(ThreadMaintenanceState);
|
|
582
|
+
const encodeMaintenanceState = Schema.encodeSync(ThreadMaintenanceState);
|
|
583
|
+
const hasRecoveryEvents = (state) => (state.recoveryEventSequence ?? 0n) > (state.recoveryEventAcknowledged ?? 0n);
|
|
584
|
+
const initialMaintenanceState = () => ThreadMaintenanceState.make({
|
|
585
|
+
schemaVersion: 1,
|
|
586
|
+
dirty: 1n,
|
|
587
|
+
processed: 0n,
|
|
588
|
+
nonterminal: 0
|
|
589
|
+
});
|
|
590
|
+
const readMaintenanceState = async (transaction) => {
|
|
591
|
+
const encoded = await transaction.get(MAINTENANCE_STATE_KEY);
|
|
592
|
+
return encoded === void 0 ? {
|
|
593
|
+
state: initialMaintenanceState(),
|
|
594
|
+
initialized: false
|
|
595
|
+
} : {
|
|
596
|
+
state: decodeMaintenanceState(encoded),
|
|
597
|
+
initialized: true
|
|
598
|
+
};
|
|
599
|
+
};
|
|
600
|
+
const ensureTransactionAlarmBy = async (transaction, deadline) => {
|
|
601
|
+
const scheduled = await transaction.getAlarm();
|
|
602
|
+
if (scheduled === null || scheduled > deadline) await transaction.setAlarm(deadline);
|
|
603
|
+
};
|
|
604
|
+
const stableExternalWait = (snapshot, reports) => {
|
|
605
|
+
const report = reports.get(snapshot.submissionId);
|
|
606
|
+
if (report?.decision._tag === "SettleAborted") return false;
|
|
607
|
+
switch (snapshot.state) {
|
|
608
|
+
case "suspended":
|
|
609
|
+
case "joined": return true;
|
|
610
|
+
case "unknown": return report?.disposition === "unknown";
|
|
611
|
+
case "admitted": return reports.get(snapshot.submissionId)?.decision._tag === "AwaitParentEstablishment";
|
|
612
|
+
case "input-applied":
|
|
613
|
+
case "joining":
|
|
614
|
+
case "ready":
|
|
615
|
+
case "running":
|
|
616
|
+
case "settled":
|
|
617
|
+
case "terminalizing": return false;
|
|
618
|
+
}
|
|
619
|
+
};
|
|
620
|
+
/**
|
|
621
|
+
* Shared prearm/acknowledgement boundary for ingress and runtime-owned producers.
|
|
622
|
+
* `ThreadObject.layer` provides this same instance in its Services. Rebuilt runtime/maintenance
|
|
623
|
+
* Layers must reuse that instance; a second gate cannot observe the native producers' activity.
|
|
624
|
+
*/
|
|
625
|
+
const CurrentMutationLanes = Context.Reference("@effect-agent/platform-cloudflare/CurrentMutationLanes", { defaultValue: () => [] });
|
|
626
|
+
const CurrentNativeSource = Context.Reference("@effect-agent/platform-cloudflare/CurrentNativeSource", { defaultValue: () => false });
|
|
627
|
+
var ThreadMutationGate = class ThreadMutationGate extends Context.Service()("@effect-agent/platform-cloudflare/internal/ThreadMutationGate") {
|
|
628
|
+
static layer = Layer.effect(this)(Effect.gen(function* () {
|
|
629
|
+
const { ctx } = yield* DurableObjectContext;
|
|
630
|
+
const sql = yield* SqlClient;
|
|
631
|
+
const dueQueue = make(ctx.storage);
|
|
632
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
633
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
634
|
+
const activeMutations = yield* Ref.make(0);
|
|
635
|
+
const activeLanes = /* @__PURE__ */ new Map();
|
|
636
|
+
const generationGate = yield* Semaphore.make(1);
|
|
637
|
+
let revision = 0;
|
|
638
|
+
let changed = yield* Deferred.make();
|
|
639
|
+
const notify = Effect.gen(function* () {
|
|
640
|
+
const previous = changed;
|
|
641
|
+
changed = yield* Deferred.make();
|
|
642
|
+
revision++;
|
|
643
|
+
yield* Deferred.succeed(previous, void 0);
|
|
644
|
+
});
|
|
645
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
646
|
+
const runTransaction = yield* makeStorageOperation;
|
|
647
|
+
const validateSourceBoundary = Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => current._tag === "Some" && buffering(ctx.storage) && !ownsTransaction(ctx.storage, current.value[1]) ? Effect.fail(DurableAlarmError.make({
|
|
648
|
+
operation: "enroll maintenance source",
|
|
649
|
+
message: "Nested source scheduling requires ThreadMutationGate.withTransaction"
|
|
650
|
+
})) : Effect.void);
|
|
651
|
+
yield* runTransaction("initialize maintenance due queue", async () => {
|
|
652
|
+
dueQueue.initialize();
|
|
653
|
+
dueQueue.register(Native);
|
|
654
|
+
});
|
|
655
|
+
const schedule = Effect.fn("ThreadMutationGate.schedule")(function* (id, dueAt, progressCursor) {
|
|
656
|
+
yield* validateSourceBoundary;
|
|
657
|
+
const now = yield* Clock.currentTimeMillis;
|
|
658
|
+
const current = yield* Effect.serviceOption(sql.transactionService);
|
|
659
|
+
const enroll = async (transaction) => {
|
|
660
|
+
if (progressCursor === void 0) dueQueue.dirty(id, dueAt);
|
|
661
|
+
else if (dueQueue.progress(id, dueAt, progressCursor) && (id === "effect-agent:native" || id === "effect-agent:publication")) {
|
|
662
|
+
if (id === "effect-agent:publication") dueQueue.dirty(Native, dueAt, true);
|
|
663
|
+
const { state } = await readMaintenanceState(transaction);
|
|
664
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
665
|
+
...Struct.omit(state, ["retry"]),
|
|
666
|
+
dirty: state.dirty + 1n
|
|
667
|
+
})));
|
|
668
|
+
}
|
|
669
|
+
const next$2 = next(dueQueue.read());
|
|
670
|
+
if (Number.isFinite(next$2)) await ensureTransactionAlarmBy(transaction, Math.max(next$2, now + minimumAlarmDelay));
|
|
671
|
+
};
|
|
672
|
+
yield* runTransaction("schedule maintenance lane", () => Option.isSome(current) ? enroll(ctx.storage) : dueQueue.transaction(enroll));
|
|
673
|
+
yield* notify;
|
|
674
|
+
});
|
|
675
|
+
const beginMutation = Effect.fn("ThreadMaintenance.beginMutation")(function* (invalidatesRecovery, lanes) {
|
|
676
|
+
yield* failpoint.hit("maintenance:dirty:before");
|
|
677
|
+
const now = yield* Clock.currentTimeMillis;
|
|
678
|
+
const nativeSource = yield* CurrentNativeSource;
|
|
679
|
+
const publicationRequired = yield* requiresPublication;
|
|
680
|
+
if (invalidatesRecovery || lanes.length > 0) yield* runTransaction("advance maintenance generation", () => dueQueue.transaction(async (transaction) => {
|
|
681
|
+
const { state, initialized } = await readMaintenanceState(transaction);
|
|
682
|
+
if (invalidatesRecovery) {
|
|
683
|
+
dueQueue.dirty(Native, now);
|
|
684
|
+
if (publicationRequired) dueQueue.dirty(Publication, now);
|
|
685
|
+
}
|
|
686
|
+
for (const id of lanes) dueQueue.dirty(id, now, false);
|
|
687
|
+
const next$1 = ThreadMaintenanceState.make({
|
|
688
|
+
...state,
|
|
689
|
+
dirty: state.dirty + (invalidatesRecovery && !nativeSource ? 1n : 0n)
|
|
690
|
+
});
|
|
691
|
+
if (invalidatesRecovery || !initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next$1));
|
|
692
|
+
const deadline = next(dueQueue.read());
|
|
693
|
+
if (Number.isFinite(deadline)) await ensureTransactionAlarmBy(transaction, Math.max(deadline, now + minimumAlarmDelay));
|
|
694
|
+
}));
|
|
695
|
+
yield* failpoint.hit("maintenance:dirty:after");
|
|
696
|
+
yield* Ref.update(activeMutations, (active) => active + 1);
|
|
697
|
+
const enrolled = /* @__PURE__ */ new Set([...lanes, ...invalidatesRecovery ? [Native, ...publicationRequired ? [Publication] : []] : []]);
|
|
698
|
+
for (const id of enrolled) activeLanes.set(id, (activeLanes.get(id) ?? 0) + 1);
|
|
699
|
+
return enrolled;
|
|
700
|
+
});
|
|
701
|
+
const releaseLanes = (enrolled, committed = false) => Effect.sync(() => {
|
|
702
|
+
for (const id of enrolled) {
|
|
703
|
+
if (committed && (id === "effect-agent:native" || id === "effect-agent:publication")) continue;
|
|
704
|
+
const remaining = (activeLanes.get(id) ?? 1) - 1;
|
|
705
|
+
if (remaining === 0) activeLanes.delete(id);
|
|
706
|
+
else activeLanes.set(id, remaining);
|
|
707
|
+
enrolled.delete(id);
|
|
708
|
+
}
|
|
709
|
+
});
|
|
710
|
+
const endMutation = (enrolled) => generationGate.withPermit(releaseLanes(enrolled).pipe(Effect.andThen(Ref.update(activeMutations, (active) => Math.max(0, active - 1)))));
|
|
711
|
+
const withMutation = (body, options) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (current) => Option.isSome(current) ? Effect.fail(DurableAlarmError.make({
|
|
712
|
+
operation: "prearm maintenance mutation",
|
|
713
|
+
message: "Run withMutation outside the source SQL transaction so prearming commits first"
|
|
714
|
+
})) : Effect.acquireUseRelease(generationGate.withPermit(beginMutation(options?.invalidatesRecovery ?? true, options?.lanes ?? [])), (enrolled) => failpoint.hit("maintenance:mutation:armed").pipe(Effect.andThen(Effect.flatMap(CurrentMutationLanes, (current) => body.pipe(Effect.provideService(CurrentMutationLanes, [.../* @__PURE__ */ new Set([...current, ...enrolled])])))), Effect.tap(() => releaseLanes(enrolled, true).pipe(Effect.andThen(notify))), Effect.tap(() => failpoint.hit("maintenance:mutation:finished"))), (enrolled) => endMutation(enrolled).pipe(Effect.andThen(notify))));
|
|
715
|
+
const recordProgress = Effect.fn("ThreadMutationGate.recordProgress")(function* (lanes) {
|
|
716
|
+
yield* validateSourceBoundary;
|
|
717
|
+
if (Option.isNone(yield* Effect.serviceOption(sql.transactionService))) return yield* DurableAlarmError.make({
|
|
718
|
+
operation: "record source progress",
|
|
719
|
+
message: "Source progress requires its authoritative SQL transaction"
|
|
720
|
+
});
|
|
721
|
+
const affected = /* @__PURE__ */ new Set([...lanes, ...yield* CurrentMutationLanes]);
|
|
722
|
+
if (!(yield* requiresPublication)) affected.delete(Publication);
|
|
723
|
+
if (affected.has("effect-agent:publication")) affected.add(Native);
|
|
724
|
+
if (affected.size === 0) return;
|
|
725
|
+
const now = yield* Clock.currentTimeMillis;
|
|
726
|
+
const nativeSource = yield* CurrentNativeSource;
|
|
727
|
+
yield* runTransaction("record maintenance producer progress", async () => {
|
|
728
|
+
for (const id of affected) dueQueue.dirty(id, now, true);
|
|
729
|
+
if (affected.has("effect-agent:native")) {
|
|
730
|
+
const { state } = await readMaintenanceState(ctx.storage);
|
|
731
|
+
await ctx.storage.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
732
|
+
...Struct.omit(state, ["retry"]),
|
|
733
|
+
dirty: state.dirty + (nativeSource ? 0n : 1n)
|
|
734
|
+
})));
|
|
735
|
+
}
|
|
736
|
+
await ensureTransactionAlarmBy(ctx.storage, now + minimumAlarmDelay);
|
|
737
|
+
});
|
|
738
|
+
yield* notify;
|
|
739
|
+
});
|
|
740
|
+
return ThreadMutationGate.of({
|
|
741
|
+
withTransaction: (body) => dueQueue.withTransaction(body).pipe(Effect.provideService(SqlClient, sql)),
|
|
742
|
+
withMutation,
|
|
743
|
+
schedule,
|
|
744
|
+
recordProgress,
|
|
745
|
+
activeLanes: Effect.sync(() => new Set(activeLanes.keys())),
|
|
746
|
+
revision: Effect.sync(() => revision),
|
|
747
|
+
awaitChange: (observed) => Effect.suspend(() => observed === revision ? Deferred.await(changed) : Effect.void),
|
|
748
|
+
withSnapshot: (body) => generationGate.withPermit(Effect.flatMap(Ref.get(activeMutations), body))
|
|
749
|
+
});
|
|
750
|
+
}));
|
|
751
|
+
};
|
|
752
|
+
/**
|
|
753
|
+
* Incremental, quiescent maintenance over a durable dirty/processed generation (issue #93).
|
|
754
|
+
*
|
|
755
|
+
* One physical event owns native scheduling, auxiliary delivery and final alarm rearming.
|
|
756
|
+
*
|
|
757
|
+
* 1. Prearm before any work. A caught-up native step uses the O(1) generation record without
|
|
758
|
+
* recovery, ledger scans or canonical-history reads.
|
|
759
|
+
* 2. Reconcile before each head Attempt, then checkpoint only the observed generation. A racing
|
|
760
|
+
* producer keeps its generation dirty and immediately eligible. Quiescent native retries
|
|
761
|
+
* retain their durable backoff.
|
|
762
|
+
* 3. Admit at most two independent Thread Attempts, one active head per Thread.
|
|
763
|
+
* Keep native and delivery admission open together while finite waves remain active, so
|
|
764
|
+
* fresh replies and abort controls can progress during unrelated cleanup. Close atomically
|
|
765
|
+
* at quiescence, or at the original ten-minute yield deadline, then join admitted waves.
|
|
766
|
+
* 4. Native message delivery retains its driver-owned Claim deadline. Host/backfill waves are
|
|
767
|
+
* bounded independently; incoming native work never restarts or cancels their attempts.
|
|
768
|
+
* Auxiliary failures are reported after the current native opportunity.
|
|
769
|
+
* 5. Close every event resource before the final gated deadline snapshot and alarm decision.
|
|
770
|
+
* The whole event retains one fourteen-minute cooperative timeout.
|
|
771
|
+
*/
|
|
772
|
+
var ThreadMaintenance = class ThreadMaintenance extends Context.Service()("@effect-agent/platform-cloudflare/ThreadMaintenance") {
|
|
773
|
+
static layer = Layer.effect(ThreadMaintenance)(Effect.gen(function* () {
|
|
774
|
+
const runtime = yield* DurableAgentRuntime;
|
|
775
|
+
const recoveryConfig = yield* DurableRuntimeConfig;
|
|
776
|
+
const ledger = yield* SubmissionLedger;
|
|
777
|
+
const wakes = yield* WakeScheduler;
|
|
778
|
+
const alarm = yield* DurableAlarmService;
|
|
779
|
+
const config = yield* CloudflareDurableRuntimeConfig;
|
|
780
|
+
const { ctx } = yield* DurableObjectContext;
|
|
781
|
+
const dueQueue = make(ctx.storage);
|
|
782
|
+
const storage = DurableObjectStorage.fromDurableObjectStorage(ctx.storage);
|
|
783
|
+
const runStorage = yield* makeStorageEffect;
|
|
784
|
+
const failpoint = yield* ThreadMaintenanceFailpoint;
|
|
785
|
+
const mutations = yield* ThreadMutationGate;
|
|
786
|
+
const publication = yield* ThreadPublication;
|
|
787
|
+
const publicationContext = yield* Effect.context();
|
|
788
|
+
const projection = yield* ThreadProjectionMaintenance;
|
|
789
|
+
const messages = yield* ThreadMessageDelivery;
|
|
790
|
+
const host = yield* ThreadHostMaintenance;
|
|
791
|
+
const framework = yield* ThreadNativeMaintenance;
|
|
792
|
+
for (const lane of host.lanes) yield* Schema.decodeEffect(HostLaneId)(lane.id).pipe(Effect.mapError(alarmFailure("maintenance lane ID")));
|
|
793
|
+
const recoveryEvents = Option.getOrElse(yield* Effect.serviceOption(ThreadRecoveryEvents), () => ThreadRecoveryEvents.of({ publish: () => Effect.void }));
|
|
794
|
+
let recoveryEventsPending = false;
|
|
795
|
+
const maintenancePassGate = yield* Semaphore.make(1);
|
|
796
|
+
const minimumAlarmDelay = Math.max(1, Math.ceil(config.alarmBackoffBase / 2));
|
|
797
|
+
const runTransaction = yield* makeStorageOperation;
|
|
798
|
+
const queueSnapshot = runTransaction("read maintenance due queue", async () => dueQueue.read());
|
|
799
|
+
const reportParked = Effect.gen(function* () {
|
|
800
|
+
if (!(yield* queueSnapshot).some((row) => row.state === "parked" && row.reported === 0)) return;
|
|
801
|
+
const rows = yield* runTransaction("mark parked maintenance reports", () => dueQueue.transaction(async () => dueQueue.takeParkedReports()));
|
|
802
|
+
for (const row of rows) {
|
|
803
|
+
const lane = [
|
|
804
|
+
"effect-agent:native",
|
|
805
|
+
"effect-agent:publication",
|
|
806
|
+
"effect-agent:projection",
|
|
807
|
+
"effect-agent:messages",
|
|
808
|
+
"effect-agent:recovery-events",
|
|
809
|
+
"effect-agent:lifecycle"
|
|
810
|
+
].includes(row.id) ? row.id.slice(13) : "host";
|
|
811
|
+
yield* ErrorReporter.report(Cause.fail(MaintenanceRetryParked.make({
|
|
812
|
+
lane,
|
|
813
|
+
retryAfterMillis: ParkedRetryMillis
|
|
814
|
+
})));
|
|
815
|
+
}
|
|
816
|
+
}).pipe(Effect.catchCause((cause) => ErrorReporter.report(cause)));
|
|
817
|
+
yield* runTransaction("register native maintenance lanes", () => dueQueue.transaction(async () => {
|
|
818
|
+
for (const id of [
|
|
819
|
+
Publication,
|
|
820
|
+
Projection,
|
|
821
|
+
Messages,
|
|
822
|
+
...framework.lanes.map((lane) => lane.id)
|
|
823
|
+
]) dueQueue.register(id);
|
|
824
|
+
if (framework.lanes.some((lane) => lane.id === "effect-agent:lifecycle")) {
|
|
825
|
+
const start = dueQueue.read().find((row) => row.id === "effect-agent:lifecycle-start");
|
|
826
|
+
if (start !== void 0 && start.dueAt !== null) {
|
|
827
|
+
dueQueue.dirty(Lifecycle, start.dueAt);
|
|
828
|
+
dueQueue.complete(start, null);
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
}));
|
|
832
|
+
const runQueued = (selected, work, observed, yielded) => {
|
|
833
|
+
let row = selected;
|
|
834
|
+
let charged = false;
|
|
835
|
+
return Effect.gen(function* () {
|
|
836
|
+
const now = yield* Clock.currentTimeMillis;
|
|
837
|
+
const claimed = yield* runTransaction("charge maintenance attempt", () => dueQueue.transaction(async (transaction) => {
|
|
838
|
+
const claimed = dueQueue.claim(selected, now);
|
|
839
|
+
const next$5 = next(dueQueue.read());
|
|
840
|
+
if (Number.isFinite(next$5)) await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next$5));
|
|
841
|
+
else await transaction.deleteAlarm();
|
|
842
|
+
return claimed;
|
|
843
|
+
}));
|
|
844
|
+
if (claimed === void 0) return Option.none();
|
|
845
|
+
row = claimed;
|
|
846
|
+
charged = true;
|
|
847
|
+
observed.charged.set(row.id, row.revision);
|
|
848
|
+
observed.queue.set(row.id, row);
|
|
849
|
+
observed.available.set(row.id, row.revision);
|
|
850
|
+
return yield* work;
|
|
851
|
+
}).pipe(Effect.onExit((exit) => {
|
|
852
|
+
if (!charged) return Effect.void;
|
|
853
|
+
if (yielded?.() === true && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
|
|
854
|
+
observed.queue.delete(row.id);
|
|
855
|
+
return Effect.void;
|
|
856
|
+
}
|
|
857
|
+
return mutations.withSnapshot(() => Effect.gen(function* () {
|
|
858
|
+
const active = (yield* mutations.activeLanes).has(row.id);
|
|
859
|
+
const now = yield* Clock.currentTimeMillis;
|
|
860
|
+
const next = Exit.isFailure(exit) ? now + backoffDelay(row.stalls, yield* Random.next) : Option.getOrNull(exit.value);
|
|
861
|
+
const checkpointed = yield* runTransaction("checkpoint maintenance lane", () => dueQueue.transaction(async () => {
|
|
862
|
+
if (active) return;
|
|
863
|
+
if (dueQueue.read().find((lane) => lane.id === row.id)?.revision !== row.revision) return;
|
|
864
|
+
return dueQueue.complete(row, next)?.revision;
|
|
865
|
+
}));
|
|
866
|
+
if (checkpointed !== void 0) observed.available.set(row.id, checkpointed);
|
|
867
|
+
})).pipe(Effect.interruptible);
|
|
868
|
+
}));
|
|
869
|
+
};
|
|
870
|
+
const appendRecoveryEvents = async (transaction, events) => {
|
|
871
|
+
if (events.length === 0) return;
|
|
872
|
+
const { state } = await readMaintenanceState(transaction);
|
|
873
|
+
let sequence = state.recoveryEventSequence ?? 0n;
|
|
874
|
+
for (const event of events) {
|
|
875
|
+
sequence++;
|
|
876
|
+
await transaction.put(recoveryEventKey(sequence), encodeRecoveryEvent(ThreadRecoveryFaultEvent.make({
|
|
877
|
+
...event,
|
|
878
|
+
sequence
|
|
879
|
+
})));
|
|
880
|
+
}
|
|
881
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
882
|
+
...state,
|
|
883
|
+
recoveryEventSequence: sequence
|
|
884
|
+
})));
|
|
885
|
+
dueQueue.dirty(RecoveryEvents, 0, true);
|
|
886
|
+
recoveryEventsPending = true;
|
|
887
|
+
};
|
|
888
|
+
const recoveryEventLane = {
|
|
889
|
+
id: RecoveryEvents,
|
|
890
|
+
dispatchTimeoutMillis: 3e4,
|
|
891
|
+
run: Effect.gen(function* () {
|
|
892
|
+
if (!recoveryEventsPending) return Option.none();
|
|
893
|
+
const range = yield* runTransaction("select recovery events", () => dueQueue.transaction(async (transaction) => {
|
|
894
|
+
const { state } = await readMaintenanceState(transaction);
|
|
895
|
+
return {
|
|
896
|
+
next: (state.recoveryEventAcknowledged ?? 0n) + 1n,
|
|
897
|
+
through: state.recoveryEventSequence ?? 0n
|
|
898
|
+
};
|
|
899
|
+
}));
|
|
900
|
+
for (let sequence = range.next; sequence <= range.through; sequence++) {
|
|
901
|
+
const event = yield* runTransaction("read recovery event", async () => {
|
|
902
|
+
const event = decodeRecoveryEvent(await ctx.storage.get(recoveryEventKey(sequence)));
|
|
903
|
+
if (event.sequence !== sequence) throw new Error("Recovery event does not match its sequence key");
|
|
904
|
+
return event;
|
|
905
|
+
});
|
|
906
|
+
yield* recoveryEvents.publish(event);
|
|
907
|
+
const acknowledgedAt = yield* Clock.currentTimeMillis;
|
|
908
|
+
yield* runTransaction("acknowledge recovery event", () => dueQueue.transaction(async (transaction) => {
|
|
909
|
+
const { state } = await readMaintenanceState(transaction);
|
|
910
|
+
const next = ThreadMaintenanceState.make({
|
|
911
|
+
...state,
|
|
912
|
+
recoveryEventAcknowledged: sequence
|
|
913
|
+
});
|
|
914
|
+
await transaction.delete(recoveryEventKey(sequence));
|
|
915
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
916
|
+
recoveryEventsPending = hasRecoveryEvents(next);
|
|
917
|
+
dueQueue.progress(RecoveryEvents, acknowledgedAt, sequence);
|
|
918
|
+
const lane = dueQueue.read().find((row) => row.id === RecoveryEvents);
|
|
919
|
+
if (lane !== void 0) dueQueue.complete(lane, recoveryEventsPending ? acknowledgedAt : null);
|
|
920
|
+
}));
|
|
921
|
+
}
|
|
922
|
+
return recoveryEventsPending ? Option.some(0) : Option.none();
|
|
923
|
+
})
|
|
924
|
+
};
|
|
925
|
+
const recordRecoveryFaults = Effect.fn("ThreadMaintenance.recordRecoveryFaults")(function* (result, recovery, current) {
|
|
926
|
+
const threads = /* @__PURE__ */ new Map();
|
|
927
|
+
for (const report of result.reports) threads.set(report.threadId, void 0);
|
|
928
|
+
for (const blocked of result.blocked) threads.set(blocked.threadId, blocked.failure);
|
|
929
|
+
if (threads.size === 0) return /* @__PURE__ */ new Map();
|
|
930
|
+
const now = yield* Clock.currentTimeMillis;
|
|
931
|
+
const submissions = current ?? (result.blocked.length === 0 ? [] : yield* Stream.runCollect(ledger.scanNonterminal));
|
|
932
|
+
yield* failpoint.hit("maintenance:recovery-status:before");
|
|
933
|
+
const retained = yield* runTransaction("record Thread recovery faults", () => dueQueue.transaction(async (transaction) => {
|
|
934
|
+
const events = [];
|
|
935
|
+
const newlyBlocked = [];
|
|
936
|
+
const faults = /* @__PURE__ */ new Map();
|
|
937
|
+
const recipients = /* @__PURE__ */ new Map();
|
|
938
|
+
for (const [threadId, failure] of threads) {
|
|
939
|
+
const key = recoveryFaultKey(threadId);
|
|
940
|
+
const encoded = await transaction.get(key);
|
|
941
|
+
const previous = encoded === void 0 ? void 0 : decodeRecoveryFault(threadId, encoded);
|
|
942
|
+
const previousIds = previous === void 0 ? [] : await readRecoveryRecipients(transaction, previous);
|
|
943
|
+
const known = new Set(previousIds);
|
|
944
|
+
recipients.set(threadId, known);
|
|
945
|
+
if (failure === void 0) {
|
|
946
|
+
if (previous !== void 0) {
|
|
947
|
+
await transaction.delete(key);
|
|
948
|
+
for (let start = 0; start < (previous.recipientCount ?? 0); start += 128) await transaction.delete(recoveryRecipientKeys(previous, start));
|
|
949
|
+
for (const id of previous.recipientCount === void 0 ? result.reports.filter((report) => report.threadId === threadId).map((report) => report.submissionId) : previousIds) events.push(recoveryEvent(previous, id, "cleared", now));
|
|
950
|
+
}
|
|
951
|
+
continue;
|
|
952
|
+
}
|
|
953
|
+
const added = submissions.filter((row) => row.threadId === threadId && !known.has(row.submissionId)).map((row) => row.submissionId);
|
|
954
|
+
await appendRecoveryRecipients(transaction, threadId, previousIds.length, added);
|
|
955
|
+
const fault = ThreadRecoveryFault.make({
|
|
956
|
+
schemaVersion: 1,
|
|
957
|
+
threadId,
|
|
958
|
+
firstFailedAt: previous?.firstFailedAt ?? now,
|
|
959
|
+
lastFailedAt: now,
|
|
960
|
+
attempts: Math.min(2147483647, (previous?.attempts ?? 0) + 1),
|
|
961
|
+
retryAt: now + Math.min(6e4, 5e3 * 2 ** Math.min(30, previous?.attempts ?? 0)),
|
|
962
|
+
failure,
|
|
963
|
+
recipientCount: previousIds.length + added.length
|
|
964
|
+
});
|
|
965
|
+
await transaction.put(key, encodeRecoveryFault(fault));
|
|
966
|
+
if (previous !== void 0 && !sameRecoveryFailure(previous.failure, failure)) for (const id of previousIds) events.push(recoveryEvent(fault, id, "changed", now));
|
|
967
|
+
for (const id of added) {
|
|
968
|
+
known.add(id);
|
|
969
|
+
events.push(recoveryEvent(fault, id, "created", now));
|
|
970
|
+
}
|
|
971
|
+
faults.set(threadId, fault);
|
|
972
|
+
if (previous === void 0) newlyBlocked.push(fault);
|
|
973
|
+
}
|
|
974
|
+
await appendRecoveryEvents(transaction, events);
|
|
975
|
+
return {
|
|
976
|
+
changed: new Set(events.map((event) => event.threadId)),
|
|
977
|
+
newlyBlocked,
|
|
978
|
+
faults,
|
|
979
|
+
recipients
|
|
980
|
+
};
|
|
981
|
+
}));
|
|
982
|
+
yield* failpoint.hit("maintenance:recovery-status:after");
|
|
983
|
+
for (const [threadId, ids] of retained.recipients) recovery.recipients.set(threadId, ids);
|
|
984
|
+
for (const threadId of retained.changed) yield* wakes.notify(threadId);
|
|
985
|
+
for (const fault of retained.newlyBlocked) yield* Effect.logError("Native Thread recovery blocked; accepted work remains pending", fault.failure.reason === "defect" ? Cause.die(fault.failure) : Cause.fail(fault.failure)).pipe(Effect.annotateLogs({ threadId: fault.threadId }));
|
|
986
|
+
return retained.faults;
|
|
987
|
+
});
|
|
988
|
+
const recoverThread = Effect.fn("ThreadMaintenance.recoverThread")(function* (threadId, recovery) {
|
|
989
|
+
const result = yield* runtime.runRecovery({ threadId }).pipe(Effect.provideService(CurrentNativeSource, true));
|
|
990
|
+
const faults = yield* recordRecoveryFaults(result, recovery);
|
|
991
|
+
for (const report of result.reports) recovery.reports.set(report.submissionId, report);
|
|
992
|
+
recovery.faults.delete(threadId);
|
|
993
|
+
for (const [id, fault] of faults) recovery.faults.set(id, fault);
|
|
994
|
+
recovery.recovered += result.reports.length + result.blocked.length;
|
|
995
|
+
recovery.repaired ||= result.reports.some((report) => report.disposition === "repaired");
|
|
996
|
+
});
|
|
997
|
+
const readCurrentMaintenanceState = async (transaction) => {
|
|
998
|
+
const current = await readMaintenanceState(transaction);
|
|
999
|
+
if (current.state.bindingRegistryKey === runtime.bindingRegistryKey) return current;
|
|
1000
|
+
const state = ThreadMaintenanceState.make({
|
|
1001
|
+
...Struct.omit(current.state, ["retry"]),
|
|
1002
|
+
bindingRegistryKey: runtime.bindingRegistryKey,
|
|
1003
|
+
bindingWaits: [],
|
|
1004
|
+
dirty: current.state.dirty + 1n
|
|
1005
|
+
});
|
|
1006
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
1007
|
+
dueQueue.dirty(Native, 0, true);
|
|
1008
|
+
return {
|
|
1009
|
+
state,
|
|
1010
|
+
initialized: true
|
|
1011
|
+
};
|
|
1012
|
+
};
|
|
1013
|
+
const ensureAlarm = Effect.fn("ThreadMaintenance.ensureAlarm")(function* () {
|
|
1014
|
+
yield* failpoint.hit("maintenance:ensure:before");
|
|
1015
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1016
|
+
yield* runTransaction("ensure maintenance alarm", () => dueQueue.transaction(async (transaction) => {
|
|
1017
|
+
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1018
|
+
recoveryEventsPending = hasRecoveryEvents(state);
|
|
1019
|
+
const rows = dueQueue.read();
|
|
1020
|
+
if (recoveryEventsPending && !rows.some((row) => row.id === "effect-agent:recovery-events")) dueQueue.dirty(RecoveryEvents, now);
|
|
1021
|
+
if (!initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
1022
|
+
const native = state.dirty > state.processed ? state.retry?.generation === state.dirty ? state.retry.notBefore : rows.find((row) => row.id === "effect-agent:native")?.dueAt ?? now + minimumAlarmDelay : null;
|
|
1023
|
+
dueQueue.checkpointNative(native);
|
|
1024
|
+
const next$3 = next(dueQueue.read());
|
|
1025
|
+
if (Number.isFinite(next$3)) await ensureTransactionAlarmBy(transaction, Math.max(now + minimumAlarmDelay, next$3));
|
|
1026
|
+
}));
|
|
1027
|
+
yield* failpoint.hit("maintenance:ensure:after");
|
|
1028
|
+
});
|
|
1029
|
+
const beginPass = Effect.fn("ThreadMaintenance.beginPass")(function* (observed) {
|
|
1030
|
+
yield* failpoint.hit("maintenance:begin:before");
|
|
1031
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1032
|
+
const result = yield* runTransaction("begin maintenance pass", () => dueQueue.transaction(async (transaction) => {
|
|
1033
|
+
const { state, initialized } = await readCurrentMaintenanceState(transaction);
|
|
1034
|
+
recoveryEventsPending = hasRecoveryEvents(state);
|
|
1035
|
+
observed.generation = state.dirty;
|
|
1036
|
+
if (!initialized) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(state));
|
|
1037
|
+
const retryAt = state.retry?.generation === state.dirty ? state.retry.notBefore : 0;
|
|
1038
|
+
const rows = dueQueue.read();
|
|
1039
|
+
const native = rows.find((row) => row.id === Native);
|
|
1040
|
+
const publication = rows.find((row) => row.id === Publication);
|
|
1041
|
+
if (state.processed >= state.dirty || retryAt > now || native === void 0 || native.dueAt === null || native.notBefore > now || publication?.state === "parked") return {
|
|
1042
|
+
_tag: "CaughtUp",
|
|
1043
|
+
nonterminal: state.nonterminal
|
|
1044
|
+
};
|
|
1045
|
+
const charged = observed.charged.get("effect-agent:native") === native.revision ? native : dueQueue.claim(native, now, true);
|
|
1046
|
+
if (charged === void 0) return {
|
|
1047
|
+
_tag: "CaughtUp",
|
|
1048
|
+
nonterminal: state.nonterminal
|
|
1049
|
+
};
|
|
1050
|
+
observed.queue.set(Native, charged);
|
|
1051
|
+
observed.charged.set(Native, charged.revision);
|
|
1052
|
+
const deadline = next(dueQueue.read());
|
|
1053
|
+
if (Number.isFinite(deadline)) await transaction.setAlarm(Math.max(now + minimumAlarmDelay, deadline));
|
|
1054
|
+
else await transaction.deleteAlarm();
|
|
1055
|
+
return {
|
|
1056
|
+
_tag: "Actionable",
|
|
1057
|
+
generation: state.dirty,
|
|
1058
|
+
sourceRevision: charged.revision,
|
|
1059
|
+
nonterminal: state.nonterminal,
|
|
1060
|
+
stalls: charged.stalls
|
|
1061
|
+
};
|
|
1062
|
+
}));
|
|
1063
|
+
yield* failpoint.hit("maintenance:begin:after");
|
|
1064
|
+
return result;
|
|
1065
|
+
});
|
|
1066
|
+
const backoffDelay = (priorStalls, jitter) => {
|
|
1067
|
+
const backoff = Math.min(config.alarmBackoffCap, config.alarmBackoffBase * 2 ** Math.min(priorStalls, 30));
|
|
1068
|
+
return Math.ceil(backoff / 2 + backoff / 2 * jitter);
|
|
1069
|
+
};
|
|
1070
|
+
const rearmDelay = Effect.fn("ThreadMaintenance.rearmDelay")(function* (progressed, priorStalls) {
|
|
1071
|
+
return progressed ? config.alarmBackoffBase : backoffDelay(priorStalls, yield* Random.next);
|
|
1072
|
+
});
|
|
1073
|
+
const rearmFailure = Effect.fn("ThreadMaintenance.rearmFailure")(function* (observed) {
|
|
1074
|
+
const { generation, nativeOnly } = observed;
|
|
1075
|
+
if (generation === void 0) return;
|
|
1076
|
+
yield* failpoint.hit("maintenance:retry:before");
|
|
1077
|
+
yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
1078
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1079
|
+
const jitter = yield* Random.next;
|
|
1080
|
+
yield* runTransaction("back off failed maintenance", () => dueQueue.transaction(async (transaction) => {
|
|
1081
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1082
|
+
const previous = state.retry?.generation === generation && state.retry.nativeOnly === nativeOnly ? state.retry : void 0;
|
|
1083
|
+
const retry = state.retry?.generation === generation && state.retry.nativeOnly ? state.retry : MaintenanceRetry.make({
|
|
1084
|
+
generation,
|
|
1085
|
+
notBefore: Math.max(previous?.notBefore ?? 0, now + backoffDelay(previous?.stalls ?? 0, jitter)),
|
|
1086
|
+
nativeOnly,
|
|
1087
|
+
stalls: Math.min(30, (previous?.stalls ?? 0) + 1)
|
|
1088
|
+
});
|
|
1089
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
1090
|
+
...state,
|
|
1091
|
+
retry
|
|
1092
|
+
})));
|
|
1093
|
+
const nativeDeadline = active > 0 || state.dirty !== generation ? now + minimumAlarmDelay : retry.notBefore;
|
|
1094
|
+
dueQueue.checkpointNative(state.dirty > state.processed || active > 0 ? nativeDeadline : null);
|
|
1095
|
+
for (const row of observed.queue.values()) if (row.id !== "effect-agent:native" && row.dueAt !== null && row.dueAt <= now) {
|
|
1096
|
+
const charged = observed.charged.get(row.id) === row.revision ? row : dueQueue.claim(row, now);
|
|
1097
|
+
if (charged !== void 0) dueQueue.complete(charged, now + backoffDelay(charged.stalls, jitter));
|
|
1098
|
+
}
|
|
1099
|
+
const next$4 = next(dueQueue.read());
|
|
1100
|
+
if (Number.isFinite(next$4)) await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next$4));
|
|
1101
|
+
else await transaction.deleteAlarm();
|
|
1102
|
+
}));
|
|
1103
|
+
}));
|
|
1104
|
+
yield* failpoint.hit("maintenance:retry:after");
|
|
1105
|
+
});
|
|
1106
|
+
const beginNative = Effect.fn("ThreadMaintenance.beginNative")(function* (observed) {
|
|
1107
|
+
return yield* mutations.withSnapshot((activeAtStart) => Effect.gen(function* () {
|
|
1108
|
+
const generation = yield* beginPass(observed);
|
|
1109
|
+
if (activeAtStart === 0) {
|
|
1110
|
+
const native = (yield* queueSnapshot).find((row) => row.id === Native);
|
|
1111
|
+
if (native !== void 0) observed.available.set(native.id, native.revision);
|
|
1112
|
+
}
|
|
1113
|
+
return {
|
|
1114
|
+
...generation,
|
|
1115
|
+
activeAtStart
|
|
1116
|
+
};
|
|
1117
|
+
}));
|
|
1118
|
+
});
|
|
1119
|
+
const attemptSelected = Effect.fn("ThreadMaintenance.attemptSelected")(function* (selected, previousWaits, yieldAfter, recovery) {
|
|
1120
|
+
yield* recoverThread(selected.threadId, recovery);
|
|
1121
|
+
let bindingFailure;
|
|
1122
|
+
const settlement = recovery.faults.has(selected.threadId) ? Option.none() : yield* runtime.processThreadHead(selected.threadId, { yieldAfter }).pipe(Effect.provideService(CurrentNativeSource, true), Effect.catchTag("BindingUnavailable", (failure) => {
|
|
1123
|
+
bindingFailure = failure;
|
|
1124
|
+
return Effect.succeed(Option.none());
|
|
1125
|
+
}));
|
|
1126
|
+
const previous = previousWaits.find((wait) => wait.submissionId === selected.submissionId);
|
|
1127
|
+
const wait = bindingFailure === void 0 ? void 0 : BindingWait.make({
|
|
1128
|
+
threadId: selected.threadId,
|
|
1129
|
+
submissionId: selected.submissionId,
|
|
1130
|
+
reportedAt: previous?.reportedAt ?? (yield* Clock.currentTimeMillis)
|
|
1131
|
+
});
|
|
1132
|
+
if (bindingFailure !== void 0 && previous === void 0) yield* Effect.logError("Thread parked until its agent binding registry changes; original work remains pending", Cause.fail(bindingFailure)).pipe(Effect.annotateLogs({
|
|
1133
|
+
threadId: selected.threadId,
|
|
1134
|
+
submissionId: selected.submissionId
|
|
1135
|
+
}));
|
|
1136
|
+
if (wait !== void 0 || previous !== void 0) {
|
|
1137
|
+
yield* failpoint.hit("maintenance:binding-retry:before");
|
|
1138
|
+
yield* runStorage("record submission binding wait", storage.transaction((transaction) => Effect.gen(function* () {
|
|
1139
|
+
const encoded = yield* transaction.get(MAINTENANCE_STATE_KEY);
|
|
1140
|
+
const state = encoded === void 0 ? initialMaintenanceState() : yield* Schema.decodeUnknownEffect(ThreadMaintenanceState)(encoded);
|
|
1141
|
+
const bindingWaits = [...(state.bindingWaits ?? []).filter((entry) => entry.submissionId !== selected.submissionId), ...wait === void 0 ? [] : [wait]];
|
|
1142
|
+
yield* transaction.put(MAINTENANCE_STATE_KEY, yield* Schema.encodeEffect(ThreadMaintenanceState)(ThreadMaintenanceState.make({
|
|
1143
|
+
...state,
|
|
1144
|
+
bindingWaits
|
|
1145
|
+
})));
|
|
1146
|
+
})).pipe(Effect.mapError(alarmFailure("record submission binding wait"))));
|
|
1147
|
+
yield* failpoint.hit("maintenance:binding-retry:after");
|
|
1148
|
+
}
|
|
1149
|
+
return Option.isSome(settlement) ? 1 : 0;
|
|
1150
|
+
});
|
|
1151
|
+
const advance = Effect.fn("ThreadMaintenance.advance")(function* (started, yieldAfter, observed, recovery, native, dispatch = true, reserved = new Set(native.active.keys())) {
|
|
1152
|
+
let publicationRow = (yield* queueSnapshot).find((row) => row.id === Publication);
|
|
1153
|
+
if (publicationRow !== void 0 && publicationRow.dueAt !== null && publicationRow.dueAt <= (yield* Clock.currentTimeMillis)) {
|
|
1154
|
+
yield* runQueued(publicationRow, mutations.withSnapshot((active) => started._tag === "Actionable" && active === 0 ? publication.prepareGeneration(started.generation) : Effect.void).pipe(Effect.andThen(publication.drain)), observed);
|
|
1155
|
+
publicationRow = (yield* queueSnapshot).find((row) => row.id === Publication);
|
|
1156
|
+
}
|
|
1157
|
+
const pending = Option.fromNullishOr(publicationRow?.dueAt);
|
|
1158
|
+
if (started._tag === "CaughtUp" || Option.isSome(pending) || publicationRow?.state === "parked") {
|
|
1159
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
1160
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1161
|
+
const { state } = yield* runTransaction("read native maintenance deadline", () => dueQueue.transaction((transaction) => readMaintenanceState(transaction)));
|
|
1162
|
+
const native = active > 0 || state.dirty > state.processed ? state.retry?.generation === state.dirty && active === 0 ? state.retry.notBefore : now + minimumAlarmDelay : Infinity;
|
|
1163
|
+
const next = Option.isSome(pending) ? pending.value : native;
|
|
1164
|
+
return Number.isFinite(next) ? Math.max(now + minimumAlarmDelay, next) : void 0;
|
|
1165
|
+
}));
|
|
1166
|
+
return {
|
|
1167
|
+
phase: "caught-up",
|
|
1168
|
+
nonterminal: started.nonterminal,
|
|
1169
|
+
nextAttemptAt
|
|
1170
|
+
};
|
|
1171
|
+
}
|
|
1172
|
+
if (native.generation !== started.generation || native.sourceRevision !== started.sourceRevision) {
|
|
1173
|
+
native.deferred.clear();
|
|
1174
|
+
native.generation = started.generation;
|
|
1175
|
+
native.sourceRevision = started.sourceRevision;
|
|
1176
|
+
}
|
|
1177
|
+
if (native.active.size > 0 && native.scanGeneration === started.generation && native.scanRevision === started.sourceRevision) return {
|
|
1178
|
+
phase: "actionable",
|
|
1179
|
+
nonterminal: started.nonterminal,
|
|
1180
|
+
nextAttemptAt: void 0
|
|
1181
|
+
};
|
|
1182
|
+
native.scanGeneration = started.activeAtStart === 0 ? started.generation : void 0;
|
|
1183
|
+
native.scanRevision = started.activeAtStart === 0 ? started.sourceRevision : void 0;
|
|
1184
|
+
observed.nativeOnly = true;
|
|
1185
|
+
const observation = recovery.observation ??= {
|
|
1186
|
+
generation: started.generation,
|
|
1187
|
+
activeAtStart: started.activeAtStart
|
|
1188
|
+
};
|
|
1189
|
+
const current = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
1190
|
+
const selectionTime = yield* Clock.currentTimeMillis;
|
|
1191
|
+
const submissionsByThread = /* @__PURE__ */ new Map();
|
|
1192
|
+
for (const row of current) {
|
|
1193
|
+
const ids = submissionsByThread.get(row.threadId) ?? [];
|
|
1194
|
+
ids.push(row.submissionId);
|
|
1195
|
+
submissionsByThread.set(row.threadId, ids);
|
|
1196
|
+
}
|
|
1197
|
+
yield* runTransaction("read Thread recovery deadlines", async () => {
|
|
1198
|
+
for (const [threadId, ids] of submissionsByThread) {
|
|
1199
|
+
if (!recovery.loaded.has(threadId)) {
|
|
1200
|
+
const encoded = await ctx.storage.get(recoveryFaultKey(threadId));
|
|
1201
|
+
if (encoded !== void 0) {
|
|
1202
|
+
const fault = decodeRecoveryFault(threadId, encoded);
|
|
1203
|
+
recovery.faults.set(threadId, fault);
|
|
1204
|
+
recovery.recipients.set(threadId, new Set(await readRecoveryRecipients(ctx.storage, fault)));
|
|
1205
|
+
}
|
|
1206
|
+
recovery.loaded.add(threadId);
|
|
1207
|
+
}
|
|
1208
|
+
if (recovery.faults.get(threadId) === void 0 || ids.every((id) => recovery.recipients.get(threadId)?.has(id))) continue;
|
|
1209
|
+
await dueQueue.transaction(async (transaction) => {
|
|
1210
|
+
const encoded = await transaction.get(recoveryFaultKey(threadId));
|
|
1211
|
+
if (encoded === void 0) return;
|
|
1212
|
+
const previous = decodeRecoveryFault(threadId, encoded);
|
|
1213
|
+
const previousIds = await readRecoveryRecipients(transaction, previous);
|
|
1214
|
+
const known = new Set(previousIds);
|
|
1215
|
+
const added = ids.filter((id) => !known.has(id));
|
|
1216
|
+
if (added.length === 0) return;
|
|
1217
|
+
const updated = ThreadRecoveryFault.make({
|
|
1218
|
+
...previous,
|
|
1219
|
+
recipientCount: previousIds.length + added.length
|
|
1220
|
+
});
|
|
1221
|
+
await appendRecoveryRecipients(transaction, threadId, previousIds.length, added);
|
|
1222
|
+
await transaction.put(recoveryFaultKey(threadId), encodeRecoveryFault(updated));
|
|
1223
|
+
await appendRecoveryEvents(transaction, added.map((id) => recoveryEvent(updated, id, "created", selectionTime)));
|
|
1224
|
+
recovery.faults.set(threadId, updated);
|
|
1225
|
+
recovery.recipients.set(threadId, /* @__PURE__ */ new Set([...previousIds, ...added]));
|
|
1226
|
+
});
|
|
1227
|
+
}
|
|
1228
|
+
});
|
|
1229
|
+
const reports = recovery.reports;
|
|
1230
|
+
const recoveryFaults = recovery.faults;
|
|
1231
|
+
const bindingWaits = yield* runTransaction("read binding waits", async () => (await readMaintenanceState(ctx.storage)).state.bindingWaits ?? []);
|
|
1232
|
+
const parked = new Set(bindingWaits.map((wait) => wait.submissionId));
|
|
1233
|
+
const waiting = (row) => !recovery.pending.has(row.threadId) && !recoveryFaults.has(row.threadId) && (parked.has(row.submissionId) || stableExternalWait(row, reports));
|
|
1234
|
+
const heads = /* @__PURE__ */ new Map();
|
|
1235
|
+
for (const row of current) {
|
|
1236
|
+
if (row.state === "unknown" && waiting(row)) continue;
|
|
1237
|
+
if (!heads.has(row.threadId)) heads.set(row.threadId, row);
|
|
1238
|
+
}
|
|
1239
|
+
const stopping = /* @__PURE__ */ new Set();
|
|
1240
|
+
for (const head of heads.values()) {
|
|
1241
|
+
if (head.state !== "ready" || native.active.has(head.threadId) || recovery.pending.has(head.threadId) || recoveryFaults.has(head.threadId)) continue;
|
|
1242
|
+
const intent = yield* isolateRecovery(ledger.readAbortIntent(AbortIntentRequest.make({ submissionId: head.submissionId })), {
|
|
1243
|
+
timeout: recoveryConfig.recoveryTimeout,
|
|
1244
|
+
phase: () => "recovery",
|
|
1245
|
+
operation: "read abort intent"
|
|
1246
|
+
});
|
|
1247
|
+
if (Result.isSuccess(intent)) {
|
|
1248
|
+
if (intent.success !== void 0) stopping.add(head.threadId);
|
|
1249
|
+
} else {
|
|
1250
|
+
const faults = yield* recordRecoveryFaults(RecoverySweepResult.make({
|
|
1251
|
+
reports: [],
|
|
1252
|
+
blocked: [RecoveryBlocked.make({
|
|
1253
|
+
threadId: head.threadId,
|
|
1254
|
+
failure: intent.failure
|
|
1255
|
+
})]
|
|
1256
|
+
}), recovery, current);
|
|
1257
|
+
for (const [threadId, fault] of faults) recoveryFaults.set(threadId, fault);
|
|
1258
|
+
recovery.recovered++;
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
const eligible = [...heads.values()].filter((head) => !recovery.pending.has(head.threadId) && !stopping.has(head.threadId) && !recoveryFaults.has(head.threadId) && !waiting(head) && (head.state === "ready" || reports.has(head.submissionId))).map((head) => head.threadId).sort();
|
|
1262
|
+
yield* failpoint.hit("maintenance:select:before");
|
|
1263
|
+
const selection = yield* runTransaction("select maintenance lane", () => dueQueue.transaction(async (transaction) => {
|
|
1264
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1265
|
+
const waits = (state.bindingWaits ?? []).filter((retry) => heads.get(retry.threadId)?.submissionId === retry.submissionId);
|
|
1266
|
+
const runnable = eligible.filter((threadId) => !reserved.has(threadId) && !native.active.has(threadId) && !native.deferred.has(threadId) && !waits.some((wait) => wait.threadId === threadId));
|
|
1267
|
+
const pivot = Math.max(0, runnable.findIndex((threadId) => state.lastServedThreadId === void 0 || threadId > state.lastServedThreadId));
|
|
1268
|
+
const selected = dispatch ? [...runnable.slice(pivot), ...runnable.slice(0, pivot)].slice(0, nativeDispatchConcurrency - reserved.size) : [];
|
|
1269
|
+
const next = selected.at(-1);
|
|
1270
|
+
if (next !== void 0) await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
1271
|
+
...state,
|
|
1272
|
+
lastServedThreadId: next
|
|
1273
|
+
})));
|
|
1274
|
+
const backlog = recovery.started ? [] : [...heads.keys()].filter((threadId) => {
|
|
1275
|
+
const fault = recoveryFaults.get(threadId);
|
|
1276
|
+
return !eligible.includes(threadId) && (fault === void 0 || fault.retryAt <= selectionTime);
|
|
1277
|
+
});
|
|
1278
|
+
const after = backlog.filter((threadId) => state.lastRecoveredThreadId === void 0 || threadId > state.lastRecoveredThreadId);
|
|
1279
|
+
const before = backlog.filter((threadId) => state.lastRecoveredThreadId !== void 0 && threadId <= state.lastRecoveredThreadId);
|
|
1280
|
+
return {
|
|
1281
|
+
selected,
|
|
1282
|
+
waits,
|
|
1283
|
+
backlog: [...after, ...before]
|
|
1284
|
+
};
|
|
1285
|
+
}));
|
|
1286
|
+
yield* failpoint.hit("maintenance:select:after");
|
|
1287
|
+
if (!recovery.started) {
|
|
1288
|
+
recovery.started = true;
|
|
1289
|
+
recovery.needsCheckpoint = selection.backlog.length > 0;
|
|
1290
|
+
for (const threadId of selection.backlog) recovery.pending.add(threadId);
|
|
1291
|
+
yield* Deferred.succeed(recovery.queue, selection.backlog);
|
|
1292
|
+
}
|
|
1293
|
+
for (const threadId of selection.selected) {
|
|
1294
|
+
const selected = heads.get(threadId);
|
|
1295
|
+
if (selected === void 0) continue;
|
|
1296
|
+
native.dispatched++;
|
|
1297
|
+
native.active.set(threadId, yield* Effect.forkIn(attemptSelected(selected, selection.waits, yieldAfter, recovery), native.scope));
|
|
1298
|
+
}
|
|
1299
|
+
if (native.active.size > 0) return {
|
|
1300
|
+
phase: "actionable",
|
|
1301
|
+
nonterminal: current.length,
|
|
1302
|
+
nextAttemptAt: void 0,
|
|
1303
|
+
dispatched: selection.selected.length > 0
|
|
1304
|
+
};
|
|
1305
|
+
const scannedGeneration = yield* runTransaction("observe native source generation", async () => (await readMaintenanceState(ctx.storage)).state.dirty);
|
|
1306
|
+
const remaining = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
1307
|
+
const waitingHeads = /* @__PURE__ */ new Map();
|
|
1308
|
+
const autonomous = remaining.some((snapshot) => {
|
|
1309
|
+
if (snapshot.state === "unknown" && waiting(snapshot)) return false;
|
|
1310
|
+
const headWaiting = waitingHeads.get(snapshot.threadId);
|
|
1311
|
+
if (headWaiting === void 0) waitingHeads.set(snapshot.threadId, waiting(snapshot));
|
|
1312
|
+
if (headWaiting === true && snapshot.state === "ready" && reports.get(snapshot.submissionId)?.decision._tag === "ApplyInput") return false;
|
|
1313
|
+
return !waiting(snapshot);
|
|
1314
|
+
});
|
|
1315
|
+
const progressed = native.progressed || recovery.repaired;
|
|
1316
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1317
|
+
const ordinaryDelay = autonomous ? yield* rearmDelay(progressed, started.stalls) : 0;
|
|
1318
|
+
const nextEligible = [...recoveryFaults.values()].map((fault) => fault.retryAt).concat(eligible.filter((threadId) => !selection.waits.some((wait) => wait.threadId === threadId)).map(() => now));
|
|
1319
|
+
const retryDelay = nextEligible.length === 0 ? 0 : Math.max(0, Math.min(...nextEligible) - now);
|
|
1320
|
+
const delay = Math.max(ordinaryDelay, retryDelay);
|
|
1321
|
+
yield* failpoint.hit("maintenance:checkpoint:before");
|
|
1322
|
+
const nextAttemptAt = yield* mutations.withSnapshot((active) => runTransaction("checkpoint native maintenance", () => dueQueue.transaction(async (transaction) => {
|
|
1323
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1324
|
+
const mutationOverlap = observation.activeAtStart > 0 || started.activeAtStart > 0 || active > 0;
|
|
1325
|
+
const generation = remaining.length === 0 && !mutationOverlap && state.dirty === scannedGeneration ? scannedGeneration : observation.generation;
|
|
1326
|
+
const processed = autonomous || mutationOverlap ? state.processed : state.processed > generation ? state.processed : generation;
|
|
1327
|
+
const canBackoff = !mutationOverlap && state.dirty === started.generation;
|
|
1328
|
+
const next = ThreadMaintenanceState.make({
|
|
1329
|
+
...Struct.omit(state, ["retry"]),
|
|
1330
|
+
processed,
|
|
1331
|
+
nonterminal: remaining.length,
|
|
1332
|
+
bindingWaits: (state.bindingWaits ?? []).filter((retry) => remaining.some((row) => row.submissionId === retry.submissionId)),
|
|
1333
|
+
...autonomous && !progressed && canBackoff ? { retry: MaintenanceRetry.make({
|
|
1334
|
+
generation: started.generation,
|
|
1335
|
+
notBefore: now + delay,
|
|
1336
|
+
nativeOnly: true,
|
|
1337
|
+
stalls: Math.min(30, started.stalls + 1)
|
|
1338
|
+
}) } : {}
|
|
1339
|
+
});
|
|
1340
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(next));
|
|
1341
|
+
const dueAt = autonomous ? now + (canBackoff ? delay : minimumAlarmDelay) : mutationOverlap || next.dirty > next.processed ? now + minimumAlarmDelay : null;
|
|
1342
|
+
dueQueue.checkpointNative(dueAt);
|
|
1343
|
+
return dueAt ?? void 0;
|
|
1344
|
+
})));
|
|
1345
|
+
yield* failpoint.hit("maintenance:checkpoint:after");
|
|
1346
|
+
if (remaining.length === 0 && recovery.pending.size === 0) delete recovery.observation;
|
|
1347
|
+
native.deferred.clear();
|
|
1348
|
+
native.progressed = false;
|
|
1349
|
+
native.needsCheckpoint = false;
|
|
1350
|
+
return {
|
|
1351
|
+
phase: "actionable",
|
|
1352
|
+
nonterminal: remaining.length,
|
|
1353
|
+
nextAttemptAt
|
|
1354
|
+
};
|
|
1355
|
+
});
|
|
1356
|
+
const validateLanes = Effect.fn("ThreadMaintenance.validateLanes")(function* (lanes, queued) {
|
|
1357
|
+
const ids = /* @__PURE__ */ new Set([
|
|
1358
|
+
Native,
|
|
1359
|
+
Publication,
|
|
1360
|
+
Projection,
|
|
1361
|
+
Messages
|
|
1362
|
+
]);
|
|
1363
|
+
for (const lane of lanes) {
|
|
1364
|
+
yield* Schema.decodeEffect(LaneId)(lane.id).pipe(Effect.mapError(alarmFailure("maintenance lane ID")));
|
|
1365
|
+
if (ids.has(lane.id)) return yield* DurableAlarmError.make({
|
|
1366
|
+
operation: "maintenance lane ID",
|
|
1367
|
+
message: `Duplicate maintenance lane: ${lane.id}`
|
|
1368
|
+
});
|
|
1369
|
+
ids.add(lane.id);
|
|
1370
|
+
}
|
|
1371
|
+
for (const row of queued) if (row.dueAt !== null && !ids.has(row.id)) return yield* DurableAlarmError.make({
|
|
1372
|
+
operation: "maintenance lane ID",
|
|
1373
|
+
message: `No handler registered for scheduled lane: ${row.id}`
|
|
1374
|
+
});
|
|
1375
|
+
});
|
|
1376
|
+
const dispatch = Effect.fn("ThreadMaintenance.dispatch")(function* (yieldAfter, dispatchUntil, observed) {
|
|
1377
|
+
const notified = (yield* Stream.toPull(wakes.wakes)).pipe(Effect.asVoid, Effect.catch(() => Effect.never));
|
|
1378
|
+
const started = yield* beginNative(observed);
|
|
1379
|
+
yield* mutations.withSnapshot(() => Effect.gen(function* () {
|
|
1380
|
+
const queued = yield* queueSnapshot;
|
|
1381
|
+
const producing = yield* mutations.activeLanes;
|
|
1382
|
+
for (const row of queued) {
|
|
1383
|
+
observed.queue.set(row.id, row);
|
|
1384
|
+
if (!producing.has(row.id)) observed.available.set(row.id, row.revision);
|
|
1385
|
+
}
|
|
1386
|
+
}));
|
|
1387
|
+
const auxiliaryScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope, exit) => Scope.close(scope, exit));
|
|
1388
|
+
const fork = (work) => Effect.forkIn(work, auxiliaryScope);
|
|
1389
|
+
const lanes = [
|
|
1390
|
+
recoveryEventLane,
|
|
1391
|
+
...framework.lanes,
|
|
1392
|
+
...host.lanes
|
|
1393
|
+
].map((lane) => ({
|
|
1394
|
+
...lane,
|
|
1395
|
+
exhausted: false,
|
|
1396
|
+
fiber: void 0
|
|
1397
|
+
}));
|
|
1398
|
+
let messageExhausted = false;
|
|
1399
|
+
let delivery;
|
|
1400
|
+
let failure;
|
|
1401
|
+
const recovery = {
|
|
1402
|
+
queue: yield* Deferred.make(),
|
|
1403
|
+
pending: /* @__PURE__ */ new Set(),
|
|
1404
|
+
loaded: /* @__PURE__ */ new Set(),
|
|
1405
|
+
reports: /* @__PURE__ */ new Map(),
|
|
1406
|
+
faults: /* @__PURE__ */ new Map(),
|
|
1407
|
+
recipients: /* @__PURE__ */ new Map(),
|
|
1408
|
+
started: false,
|
|
1409
|
+
needsCheckpoint: false,
|
|
1410
|
+
recovered: 0,
|
|
1411
|
+
repaired: false
|
|
1412
|
+
};
|
|
1413
|
+
const recoveryFiber = yield* fork(Effect.gen(function* () {
|
|
1414
|
+
for (const threadId of yield* Deferred.await(recovery.queue)) {
|
|
1415
|
+
yield* failpoint.hit("maintenance:select:before");
|
|
1416
|
+
yield* runTransaction("select old recovery lane", () => dueQueue.transaction(async (transaction) => {
|
|
1417
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1418
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
1419
|
+
...state,
|
|
1420
|
+
lastRecoveredThreadId: threadId
|
|
1421
|
+
})));
|
|
1422
|
+
}));
|
|
1423
|
+
yield* failpoint.hit("maintenance:select:after");
|
|
1424
|
+
yield* recoverThread(threadId, recovery);
|
|
1425
|
+
recovery.pending.delete(threadId);
|
|
1426
|
+
if (!recovery.faults.has(threadId)) yield* wakes.notify(threadId);
|
|
1427
|
+
}
|
|
1428
|
+
}));
|
|
1429
|
+
const backfill = yield* fork(Effect.gen(function* () {
|
|
1430
|
+
if ((yield* mutations.activeLanes).has("effect-agent:projection")) return;
|
|
1431
|
+
const row = (yield* queueSnapshot).find((row) => row.id === Projection);
|
|
1432
|
+
if (row?.dueAt === null || row === void 0 || row.dueAt > (yield* Clock.currentTimeMillis)) return;
|
|
1433
|
+
return yield* runQueued(row, projection.drain.pipe(Effect.andThen(projection.pendingDeadline)), observed).pipe(Effect.timeoutOption(config.projectionDispatchTimeoutMillis));
|
|
1434
|
+
}));
|
|
1435
|
+
let backfillObserved = false;
|
|
1436
|
+
let recoveryObserved = false;
|
|
1437
|
+
const dispatch = {
|
|
1438
|
+
scope: auxiliaryScope,
|
|
1439
|
+
active: /* @__PURE__ */ new Map(),
|
|
1440
|
+
deferred: /* @__PURE__ */ new Set(),
|
|
1441
|
+
dispatched: 0,
|
|
1442
|
+
settled: 0,
|
|
1443
|
+
progressed: false,
|
|
1444
|
+
needsCheckpoint: false
|
|
1445
|
+
};
|
|
1446
|
+
let native = yield* fork(advance(started, yieldAfter, observed, recovery, dispatch));
|
|
1447
|
+
let result = {
|
|
1448
|
+
phase: "caught-up",
|
|
1449
|
+
nonterminal: started.nonterminal,
|
|
1450
|
+
nextAttemptAt: void 0
|
|
1451
|
+
};
|
|
1452
|
+
let phase = result.phase;
|
|
1453
|
+
let nativeCheck = false;
|
|
1454
|
+
const until = DateTime.toEpochMillis(yieldAfter);
|
|
1455
|
+
const dispatchEnd = DateTime.toEpochMillis(dispatchUntil);
|
|
1456
|
+
while (true) {
|
|
1457
|
+
if (native?.pollUnsafe() !== void 0) {
|
|
1458
|
+
result = yield* Fiber.join(native);
|
|
1459
|
+
native = void 0;
|
|
1460
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
1461
|
+
observed.nativeOnly = dispatch.active.size > 0;
|
|
1462
|
+
if (result.dispatched === true && dispatch.active.size < nativeDispatchConcurrency) {
|
|
1463
|
+
dispatch.scanGeneration = void 0;
|
|
1464
|
+
nativeCheck = true;
|
|
1465
|
+
}
|
|
1466
|
+
recovery.started = true;
|
|
1467
|
+
yield* Deferred.succeed(recovery.queue, []);
|
|
1468
|
+
}
|
|
1469
|
+
for (const [threadId, attempt] of dispatch.active) {
|
|
1470
|
+
const exit = attempt.pollUnsafe();
|
|
1471
|
+
if (exit === void 0) continue;
|
|
1472
|
+
dispatch.active.delete(threadId);
|
|
1473
|
+
dispatch.scanGeneration = void 0;
|
|
1474
|
+
if (Exit.isFailure(exit)) return yield* Effect.failCause(exit.cause);
|
|
1475
|
+
dispatch.settled += exit.value;
|
|
1476
|
+
dispatch.progressed ||= exit.value > 0;
|
|
1477
|
+
dispatch.needsCheckpoint = true;
|
|
1478
|
+
if (exit.value === 0) dispatch.deferred.add(threadId);
|
|
1479
|
+
nativeCheck = true;
|
|
1480
|
+
}
|
|
1481
|
+
if (!recoveryObserved && recoveryFiber.pollUnsafe() !== void 0) {
|
|
1482
|
+
yield* Fiber.join(recoveryFiber);
|
|
1483
|
+
recoveryObserved = true;
|
|
1484
|
+
dispatch.scanGeneration = void 0;
|
|
1485
|
+
nativeCheck ||= recovery.needsCheckpoint;
|
|
1486
|
+
}
|
|
1487
|
+
if (!backfillObserved) {
|
|
1488
|
+
const exit = backfill.pollUnsafe();
|
|
1489
|
+
if (exit !== void 0) {
|
|
1490
|
+
backfillObserved = true;
|
|
1491
|
+
if (Exit.isFailure(exit)) failure ??= exit.cause;
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
for (const lane of lanes) {
|
|
1495
|
+
const exit = lane.fiber?.pollUnsafe();
|
|
1496
|
+
if (exit === void 0) continue;
|
|
1497
|
+
lane.fiber = void 0;
|
|
1498
|
+
if (Exit.isFailure(exit)) {
|
|
1499
|
+
failure ??= exit.cause;
|
|
1500
|
+
lane.exhausted = true;
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
const deliveryExit = delivery?.pollUnsafe();
|
|
1504
|
+
if (deliveryExit !== void 0) {
|
|
1505
|
+
delivery = void 0;
|
|
1506
|
+
if (Exit.isFailure(deliveryExit)) {
|
|
1507
|
+
failure ??= deliveryExit.cause;
|
|
1508
|
+
messageExhausted = true;
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1512
|
+
if (now >= until) break;
|
|
1513
|
+
const queueRevision = yield* mutations.revision;
|
|
1514
|
+
const queued = yield* queueSnapshot;
|
|
1515
|
+
const producing = yield* mutations.activeLanes;
|
|
1516
|
+
yield* validateLanes(lanes, queued);
|
|
1517
|
+
for (const lane of lanes) {
|
|
1518
|
+
if (lane.phase === "after-native" || lane.fiber !== void 0 || lane.exhausted || producing.has(lane.id)) continue;
|
|
1519
|
+
const row = queued.find((row) => row.id === lane.id);
|
|
1520
|
+
if (row === void 0 || row.dueAt === null || row.dueAt > now) continue;
|
|
1521
|
+
const dueAt = row.dueAt;
|
|
1522
|
+
lane.fiber = yield* fork(Effect.gen(function* () {
|
|
1523
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(lane.dispatchTimeoutMillis).pipe(Effect.mapError(alarmFailure("host dispatch allowance")));
|
|
1524
|
+
const selectedAt = yield* Clock.currentTimeMillis;
|
|
1525
|
+
if (selectedAt >= until || selectedAt + lane.dispatchTimeoutMillis > dispatchEnd) {
|
|
1526
|
+
lane.exhausted = true;
|
|
1527
|
+
return Option.some(dueAt);
|
|
1528
|
+
}
|
|
1529
|
+
return yield* runQueued(row, lane.run.pipe(Effect.scoped), observed).pipe(Effect.timeoutOrElse({
|
|
1530
|
+
duration: lane.dispatchTimeoutMillis,
|
|
1531
|
+
orElse: () => DurableAlarmError.make({
|
|
1532
|
+
operation: "host dispatch allowance",
|
|
1533
|
+
message: "The admitted host wave exceeded its allowance; durable work remains pending"
|
|
1534
|
+
})
|
|
1535
|
+
}));
|
|
1536
|
+
}));
|
|
1537
|
+
}
|
|
1538
|
+
if (delivery === void 0 && !messageExhausted && !producing.has("effect-agent:messages")) {
|
|
1539
|
+
const row = queued.find((row) => row.id === Messages);
|
|
1540
|
+
if (row !== void 0 && row.dueAt !== null && row.dueAt <= now) delivery = yield* fork(runQueued(row, Effect.gen(function* () {
|
|
1541
|
+
const wave = yield* messages.prepare;
|
|
1542
|
+
const selectedAt = yield* Clock.currentTimeMillis;
|
|
1543
|
+
if (selectedAt >= until || selectedAt + wave.timeoutMillis > dispatchEnd) {
|
|
1544
|
+
messageExhausted = true;
|
|
1545
|
+
return Option.fromNullishOr(row.dueAt);
|
|
1546
|
+
}
|
|
1547
|
+
return yield* wave.run;
|
|
1548
|
+
}), observed).pipe(Effect.asVoid));
|
|
1549
|
+
}
|
|
1550
|
+
if (native === void 0 && dispatch.active.size === 0 && recoveryObserved && !recovery.needsCheckpoint && backfillObserved && delivery === void 0 && lanes.every((lane) => lane.fiber === void 0)) break;
|
|
1551
|
+
if (native === void 0 && nativeCheck && dispatch.active.size < nativeDispatchConcurrency) {
|
|
1552
|
+
nativeCheck = false;
|
|
1553
|
+
const nativeCheckpoint = dispatch.needsCheckpoint && dispatch.active.size === 0;
|
|
1554
|
+
const checkpoint = recoveryObserved && recovery.needsCheckpoint || nativeCheckpoint;
|
|
1555
|
+
if (checkpoint) recovery.needsCheckpoint = false;
|
|
1556
|
+
const reserved = new Set(dispatch.active.keys());
|
|
1557
|
+
native = yield* fork(Effect.gen(function* () {
|
|
1558
|
+
const awakened = checkpoint ? started : yield* beginNative(observed);
|
|
1559
|
+
return yield* advance(awakened, yieldAfter, observed, recovery, dispatch, !nativeCheckpoint && (!checkpoint || dispatch.settled === 0), reserved);
|
|
1560
|
+
}));
|
|
1561
|
+
}
|
|
1562
|
+
const next = Math.min(native === void 0 ? result.nextAttemptAt ?? Infinity : Infinity, ...queued.flatMap((row) => row.id !== "effect-agent:native" && row.dueAt !== null && row.dueAt > now ? [row.dueAt] : []), until);
|
|
1563
|
+
const completing = [
|
|
1564
|
+
...native === void 0 ? [] : [native],
|
|
1565
|
+
...dispatch.active.values(),
|
|
1566
|
+
...recoveryObserved ? [] : [recoveryFiber],
|
|
1567
|
+
...backfillObserved ? [] : [backfill],
|
|
1568
|
+
...delivery === void 0 ? [] : [delivery],
|
|
1569
|
+
...lanes.flatMap((lane) => lane.fiber === void 0 ? [] : [lane.fiber])
|
|
1570
|
+
];
|
|
1571
|
+
if ((yield* Effect.raceAllFirst([...completing.map((fiber) => Fiber.await(fiber).pipe(Effect.as("completed"))), Effect.raceAllFirst([
|
|
1572
|
+
notified,
|
|
1573
|
+
mutations.awaitChange(queueRevision),
|
|
1574
|
+
Effect.sleep(Math.max(0, next - now))
|
|
1575
|
+
]).pipe(Effect.as("scan"))])) === "scan") nativeCheck = true;
|
|
1576
|
+
}
|
|
1577
|
+
if (native !== void 0) {
|
|
1578
|
+
result = yield* Fiber.join(native);
|
|
1579
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
1580
|
+
observed.nativeOnly = dispatch.active.size > 0;
|
|
1581
|
+
recovery.started = true;
|
|
1582
|
+
yield* Deferred.succeed(recovery.queue, []);
|
|
1583
|
+
}
|
|
1584
|
+
for (const attempt of dispatch.active.values()) {
|
|
1585
|
+
const settled = yield* Fiber.join(attempt);
|
|
1586
|
+
dispatch.settled += settled;
|
|
1587
|
+
dispatch.progressed ||= settled > 0;
|
|
1588
|
+
}
|
|
1589
|
+
dispatch.active.clear();
|
|
1590
|
+
observed.nativeOnly = false;
|
|
1591
|
+
for (const fiber of [
|
|
1592
|
+
recoveryFiber,
|
|
1593
|
+
backfill,
|
|
1594
|
+
...delivery === void 0 ? [] : [delivery],
|
|
1595
|
+
...lanes.flatMap((lane) => lane.fiber === void 0 ? [] : [lane.fiber])
|
|
1596
|
+
]) {
|
|
1597
|
+
const exit = yield* Fiber.await(fiber);
|
|
1598
|
+
if (Exit.isFailure(exit)) failure ??= exit.cause;
|
|
1599
|
+
}
|
|
1600
|
+
if (failure !== void 0) return yield* Effect.failCause(failure);
|
|
1601
|
+
if (recovery.needsCheckpoint || dispatch.dispatched > 0) {
|
|
1602
|
+
result = yield* advance(started, yieldAfter, observed, recovery, dispatch, false);
|
|
1603
|
+
if (result.phase === "actionable") phase = "actionable";
|
|
1604
|
+
}
|
|
1605
|
+
yield* Scope.close(auxiliaryScope, Exit.void);
|
|
1606
|
+
return {
|
|
1607
|
+
result,
|
|
1608
|
+
phase,
|
|
1609
|
+
recovered: recovery.recovered,
|
|
1610
|
+
settled: dispatch.settled
|
|
1611
|
+
};
|
|
1612
|
+
});
|
|
1613
|
+
const pass = Effect.fn("ThreadMaintenance.pass")(function* (yieldAfter, dispatchUntil, observed) {
|
|
1614
|
+
const concurrent = yield* Effect.exit(Effect.scoped(dispatch(yieldAfter, dispatchUntil, observed)));
|
|
1615
|
+
const afterNative = yield* Effect.gen(function* () {
|
|
1616
|
+
const lanes = [
|
|
1617
|
+
recoveryEventLane,
|
|
1618
|
+
...framework.lanes,
|
|
1619
|
+
...host.lanes
|
|
1620
|
+
];
|
|
1621
|
+
const selected = lanes.filter((lane) => lane.phase === "after-native");
|
|
1622
|
+
if (selected.length === 0) return [];
|
|
1623
|
+
const queued = yield* queueSnapshot;
|
|
1624
|
+
const producing = yield* mutations.activeLanes;
|
|
1625
|
+
for (const row of queued) if (!producing.has(row.id)) observed.available.set(row.id, row.revision);
|
|
1626
|
+
const dispatchEnd = DateTime.toEpochMillis(dispatchUntil);
|
|
1627
|
+
const afterNativeIds = new Set(selected.map((lane) => lane.id));
|
|
1628
|
+
const completed = [];
|
|
1629
|
+
let yielded = false;
|
|
1630
|
+
const yieldToConcurrent = Effect.gen(function* () {
|
|
1631
|
+
while (true) {
|
|
1632
|
+
const revision = yield* mutations.revision;
|
|
1633
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1634
|
+
const current = yield* queueSnapshot;
|
|
1635
|
+
const active = yield* mutations.activeLanes;
|
|
1636
|
+
if (current.some((row) => !afterNativeIds.has(row.id) && !active.has(row.id) && row.dueAt !== null && row.dueAt <= now && row.revision !== observed.available.get(row.id))) {
|
|
1637
|
+
yielded = true;
|
|
1638
|
+
return completed;
|
|
1639
|
+
}
|
|
1640
|
+
yield* mutations.awaitChange(revision);
|
|
1641
|
+
}
|
|
1642
|
+
});
|
|
1643
|
+
yield* validateLanes(lanes, queued);
|
|
1644
|
+
const cursor = yield* mutations.withSnapshot(() => runTransaction("read post-native dispatch cursor", () => dueQueue.transaction(async (transaction) => (await readMaintenanceState(transaction)).state.lastAfterNativeLaneId)));
|
|
1645
|
+
const ordered = selected.toSorted((left, right) => left.id < right.id ? -1 : left.id > right.id ? 1 : 0);
|
|
1646
|
+
const after = cursor === void 0 ? 0 : ordered.findIndex((lane) => lane.id > cursor);
|
|
1647
|
+
const rotated = after <= 0 ? ordered : [...ordered.slice(after), ...ordered.slice(0, after)];
|
|
1648
|
+
const waves = Effect.forEach(rotated, (lane) => Effect.gen(function* () {
|
|
1649
|
+
const row = queued.find((row) => row.id === lane.id);
|
|
1650
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1651
|
+
if (row === void 0 || row.dueAt === null || row.dueAt > now || producing.has(lane.id)) return;
|
|
1652
|
+
yield* Schema.decodeEffect(AuxiliaryDispatchMillis)(lane.dispatchTimeoutMillis).pipe(Effect.mapError(alarmFailure("host dispatch allowance")));
|
|
1653
|
+
if (now + lane.dispatchTimeoutMillis > dispatchEnd) return;
|
|
1654
|
+
const admit = mutations.withSnapshot(() => runTransaction("advance post-native dispatch cursor", () => dueQueue.transaction(async (transaction) => {
|
|
1655
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1656
|
+
await transaction.put(MAINTENANCE_STATE_KEY, encodeMaintenanceState(ThreadMaintenanceState.make({
|
|
1657
|
+
...state,
|
|
1658
|
+
lastAfterNativeLaneId: lane.id
|
|
1659
|
+
})));
|
|
1660
|
+
})));
|
|
1661
|
+
yield* runQueued(row, admit.pipe(Effect.andThen(lane.run.pipe(Effect.scoped))), observed, () => yielded).pipe(Effect.timeoutOrElse({
|
|
1662
|
+
duration: lane.dispatchTimeoutMillis,
|
|
1663
|
+
orElse: () => DurableAlarmError.make({
|
|
1664
|
+
operation: "host dispatch allowance",
|
|
1665
|
+
message: "The admitted host wave exceeded its allowance; durable work remains pending"
|
|
1666
|
+
})
|
|
1667
|
+
}));
|
|
1668
|
+
}).pipe(Effect.exit, Effect.tap((exit) => Effect.sync(() => {
|
|
1669
|
+
if (!yielded || Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) completed.push(exit);
|
|
1670
|
+
}))), { concurrency: afterNativeDispatchConcurrency }).pipe(Effect.as(completed));
|
|
1671
|
+
return yield* Effect.raceFirst(waves, yieldToConcurrent);
|
|
1672
|
+
}).pipe(Effect.exit);
|
|
1673
|
+
let failure = Exit.isFailure(concurrent) ? concurrent.cause : Cause.empty;
|
|
1674
|
+
if (Exit.isFailure(afterNative)) failure = Cause.combine(failure, afterNative.cause);
|
|
1675
|
+
else for (const exit of afterNative.value) if (Exit.isFailure(exit)) failure = Cause.combine(failure, exit.cause);
|
|
1676
|
+
if (Exit.isFailure(concurrent) || failure.reasons.length > 0) return yield* Effect.failCause(failure);
|
|
1677
|
+
const { result, phase, recovered, settled } = concurrent.value;
|
|
1678
|
+
yield* failpoint.hit("maintenance:finish:before");
|
|
1679
|
+
const disposition = yield* mutations.withSnapshot((active) => Effect.gen(function* () {
|
|
1680
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1681
|
+
return yield* runTransaction("finish maintenance event", () => dueQueue.transaction(async (transaction) => {
|
|
1682
|
+
const { state } = await readMaintenanceState(transaction);
|
|
1683
|
+
const native = active > 0 || state.dirty > state.processed ? state.retry?.generation === state.dirty && active === 0 ? Math.max(now + minimumAlarmDelay, state.retry.notBefore) : state.dirty === observed.generation && active === 0 ? result.nextAttemptAt ?? now + minimumAlarmDelay : now + minimumAlarmDelay : Infinity;
|
|
1684
|
+
dueQueue.checkpointNative(Number.isFinite(native) ? native : null);
|
|
1685
|
+
const next$6 = next(dueQueue.read());
|
|
1686
|
+
if (Number.isFinite(next$6)) {
|
|
1687
|
+
await transaction.setAlarm(Math.max(now + minimumAlarmDelay, next$6));
|
|
1688
|
+
return "rearmed";
|
|
1689
|
+
}
|
|
1690
|
+
await transaction.deleteAlarm();
|
|
1691
|
+
return "cleared";
|
|
1692
|
+
}));
|
|
1693
|
+
}));
|
|
1694
|
+
yield* failpoint.hit("maintenance:finish:after");
|
|
1695
|
+
const report = MaintenancePassReport.make({
|
|
1696
|
+
phase,
|
|
1697
|
+
recovered,
|
|
1698
|
+
settled,
|
|
1699
|
+
nonterminal: result.nonterminal,
|
|
1700
|
+
alarm: disposition
|
|
1701
|
+
});
|
|
1702
|
+
yield* Effect.annotateCurrentSpan({
|
|
1703
|
+
phase: report.phase,
|
|
1704
|
+
recovered: report.recovered,
|
|
1705
|
+
settled: report.settled,
|
|
1706
|
+
nonterminal: report.nonterminal,
|
|
1707
|
+
alarm: report.alarm
|
|
1708
|
+
});
|
|
1709
|
+
return report;
|
|
1710
|
+
});
|
|
1711
|
+
return ThreadMaintenance.of({
|
|
1712
|
+
pass: Effect.gen(function* () {
|
|
1713
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1714
|
+
const yieldAfter = DateTime.makeUnsafe(now + 6e5);
|
|
1715
|
+
const dispatchUntil = DateTime.makeUnsafe(now + 84e4);
|
|
1716
|
+
const observed = {
|
|
1717
|
+
nativeOnly: false,
|
|
1718
|
+
queue: /* @__PURE__ */ new Map(),
|
|
1719
|
+
charged: /* @__PURE__ */ new Map(),
|
|
1720
|
+
available: /* @__PURE__ */ new Map()
|
|
1721
|
+
};
|
|
1722
|
+
return yield* alarm.withWakesDeferred(maintenancePassGate.withPermit(dueQueue.withView(Effect.scoped(pass(yieldAfter, dispatchUntil, observed))).pipe(Effect.onErrorIf(() => true, () => rearmFailure(observed)))));
|
|
1723
|
+
}).pipe(Effect.timeoutOrElse({
|
|
1724
|
+
duration: "14 minutes",
|
|
1725
|
+
orElse: () => DurableAlarmError.make({
|
|
1726
|
+
operation: "maintenance pass deadline",
|
|
1727
|
+
message: "The maintenance event exceeded its 14 minute deadline; durable recovery remains pending"
|
|
1728
|
+
})
|
|
1729
|
+
}), Effect.ensuring(reportParked)),
|
|
1730
|
+
ensureAlarm: mutations.withSnapshot(() => ensureAlarm()).pipe(Effect.tap(() => reportParked)),
|
|
1731
|
+
withMutation: (body) => mutations.withMutation(body.pipe(Effect.tap(() => publishCommitted.pipe(Effect.provide(publicationContext)))))
|
|
1732
|
+
});
|
|
1733
|
+
}));
|
|
1734
|
+
};
|
|
1735
|
+
//#endregion
|
|
1736
|
+
export { Messages as _, ThreadHostMaintenance as a, Publication as b, ThreadMessageDelivery as c, ThreadPublication as d, ThreadRecoveryEvents as f, Lifecycle as g, HostLaneId as h, MaintenancePassReport as i, ThreadMutationGate as l, publishCommitted as m, DurableAlarmError as n, ThreadMaintenance as o, ThreadRecoveryFaultEvent as p, DurableAlarmService as r, ThreadMaintenanceFailpoint as s, Alarm_exports as t, ThreadNativeMaintenance as u, Native as v, Projection as y };
|
|
1737
|
+
|
|
1738
|
+
//# sourceMappingURL=Alarm-B0ODr1NE.mjs.map
|