@nimbus-sh/fabric 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +208 -293
- package/dist/bindings.js +5 -5
- package/dist/budgets.d.ts +132 -0
- package/dist/budgets.d.ts.map +1 -0
- package/dist/budgets.js +248 -0
- package/dist/composition.d.ts +3 -0
- package/dist/composition.d.ts.map +1 -0
- package/dist/composition.js +2 -0
- package/dist/connections.d.ts +81 -0
- package/dist/connections.d.ts.map +1 -0
- package/dist/connections.js +114 -0
- package/dist/derived.d.ts +65 -0
- package/dist/derived.d.ts.map +1 -0
- package/dist/derived.js +95 -0
- package/dist/do-calls.d.ts +94 -0
- package/dist/do-calls.d.ts.map +1 -0
- package/dist/do-calls.js +111 -0
- package/dist/facet-pool.d.ts +90 -0
- package/dist/facet-pool.d.ts.map +1 -0
- package/dist/facet-pool.js +113 -0
- package/dist/{fanout-pool.d.ts → fanout.d.ts} +20 -20
- package/dist/fanout.d.ts.map +1 -0
- package/dist/{fanout-pool.js → fanout.js} +20 -20
- package/dist/{launch-journal.d.ts → fenced-work.d.ts} +58 -17
- package/dist/fenced-work.d.ts.map +1 -0
- package/dist/fenced-work.js +241 -0
- package/dist/generation.d.ts +69 -0
- package/dist/generation.d.ts.map +1 -0
- package/dist/generation.js +118 -0
- package/dist/{facet-image-store.d.ts → image-store.d.ts} +8 -8
- package/dist/image-store.d.ts.map +1 -0
- package/dist/{facet-image-store.js → image-store.js} +4 -4
- package/dist/index.d.ts +16 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +16 -8
- package/dist/{loader-pool.d.ts → isolate-pool.d.ts} +19 -19
- package/dist/isolate-pool.d.ts.map +1 -0
- package/dist/{loader-pool.js → isolate-pool.js} +20 -20
- package/dist/journal.d.ts +111 -0
- package/dist/journal.d.ts.map +1 -0
- package/dist/journal.js +177 -0
- package/dist/outbox.d.ts +249 -0
- package/dist/outbox.d.ts.map +1 -0
- package/dist/outbox.js +355 -0
- package/dist/process-fabric.d.ts +33 -15
- package/dist/process-fabric.d.ts.map +1 -1
- package/dist/process-fabric.js +25 -15
- package/dist/process-host.d.ts +1 -1
- package/dist/process-host.d.ts.map +1 -1
- package/dist/process-host.js +19 -11
- package/dist/sealed.d.ts +78 -0
- package/dist/sealed.d.ts.map +1 -0
- package/dist/sealed.js +145 -0
- package/dist/timers.d.ts +138 -0
- package/dist/timers.d.ts.map +1 -0
- package/dist/timers.js +231 -0
- package/dist/{launch-pacer.d.ts → turn-budget.d.ts} +24 -21
- package/dist/turn-budget.d.ts.map +1 -0
- package/dist/{launch-pacer.js → turn-budget.js} +24 -12
- package/dist/workerd-facet-host.d.ts +67 -70
- package/dist/workerd-facet-host.d.ts.map +1 -1
- package/dist/workerd-facet-host.js +129 -181
- package/examples/agent-core-adapter.ts +191 -0
- package/package.json +4 -2
- package/src/bindings.ts +6 -6
- package/src/budgets.ts +308 -0
- package/src/composition.ts +16 -0
- package/src/connections.ts +140 -0
- package/src/derived.ts +135 -0
- package/src/do-calls.ts +156 -0
- package/src/facet-pool.ts +157 -0
- package/src/{fanout-pool.ts → fanout.ts} +35 -35
- package/src/{launch-journal.ts → fenced-work.ts} +129 -42
- package/src/generation.ts +144 -0
- package/src/{facet-image-store.ts → image-store.ts} +9 -9
- package/src/index.ts +16 -8
- package/src/{loader-pool.ts → isolate-pool.ts} +34 -34
- package/src/journal.ts +242 -0
- package/src/node-async-hooks.d.ts +14 -0
- package/src/outbox.ts +520 -0
- package/src/process-fabric.ts +43 -34
- package/src/process-host.ts +22 -20
- package/src/sealed.ts +150 -0
- package/src/timers.ts +294 -0
- package/src/{launch-pacer.ts → turn-budget.ts} +34 -27
- package/src/workerd-facet-host.ts +159 -208
- package/dist/alarms.d.ts +0 -134
- package/dist/alarms.d.ts.map +0 -1
- package/dist/alarms.js +0 -214
- package/dist/ctx-exports.d.ts +0 -47
- package/dist/ctx-exports.d.ts.map +0 -1
- package/dist/ctx-exports.js +0 -54
- package/dist/facet-image-store.d.ts.map +0 -1
- package/dist/fanout-pool.d.ts.map +0 -1
- package/dist/launch-journal.d.ts.map +0 -1
- package/dist/launch-journal.js +0 -154
- package/dist/launch-pacer.d.ts.map +0 -1
- package/dist/loader-ledger.d.ts +0 -57
- package/dist/loader-ledger.d.ts.map +0 -1
- package/dist/loader-ledger.js +0 -91
- package/dist/loader-pool.d.ts.map +0 -1
- package/src/alarms.ts +0 -275
- package/src/ctx-exports.ts +0 -77
- package/src/loader-ledger.ts +0 -112
package/src/outbox.ts
ADDED
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* outbox.ts — a durable retry outbox over a Durable Object's own SQLite.
|
|
3
|
+
*
|
|
4
|
+
* Proteus built this discipline twice by hand and says so: its email outbox
|
|
5
|
+
* (`cf-backend/src/email/outbox.ts:7` — "The discipline mirrors the peer
|
|
6
|
+
* outbox", 8 attempts from a 30s base) and its peer transport
|
|
7
|
+
* (`core/src/events/ingress/peer.ts` — 8 attempts from a 5s base, per-receiver
|
|
8
|
+
* ordering, an ask/reply waiter over the same rows). Both carry the same
|
|
9
|
+
* mechanism: a write-ahead intent row in `state='pending'`, an
|
|
10
|
+
* `attempt_count`, a `next_attempt_at` with a partial index over pending rows,
|
|
11
|
+
* backoff `next = now + base * 2**(attempts-1)`, and a `nextRetryAt()` folded
|
|
12
|
+
* into the object's single alarm. This module is that mechanism once.
|
|
13
|
+
*
|
|
14
|
+
* What the consumers proved and this keeps:
|
|
15
|
+
* - WRITE-AHEAD. The row commits before send runs. On a Durable Object the
|
|
16
|
+
* output gate orders it: an outbound message cannot leave before the
|
|
17
|
+
* writes of its turn are durable, so no explicit sync is needed here.
|
|
18
|
+
* - DISPOSITION, not a boolean. peer.ts:476-526 separates three failures: a
|
|
19
|
+
* RESOLVED refusal is permanent (dlq now), a THROWN send is transport
|
|
20
|
+
* trouble (backoff), a malformed row is poison (dlq now, no send). A
|
|
21
|
+
* boolean cannot carry that, and retrying a refusal re-offends.
|
|
22
|
+
* - ORDERING per key. Rows drain in id order. A transient failure blocks
|
|
23
|
+
* later rows with the same order key until the head clears (peer.ts:465
|
|
24
|
+
* head-of-line set); a dead-lettered head does not block, and keyless rows
|
|
25
|
+
* never block each other (the email outbox has no ordering at all).
|
|
26
|
+
* - IDEMPOTENCY. A dedupe key already queued or sent is refused with the
|
|
27
|
+
* existing id, and a sent key never reaches send again.
|
|
28
|
+
*
|
|
29
|
+
* What the consumers lacked and this adds:
|
|
30
|
+
* - The drain registers with `timers` (one reason in the shared map) instead
|
|
31
|
+
* of owning an alarm, and the dispatch-side handler re-arms through its
|
|
32
|
+
* RETURN value — calling schedule() from inside the dispatcher's chain
|
|
33
|
+
* would deadlock on the chain that serializes the reason map. A host
|
|
34
|
+
* whose alarm belongs to another framework (the Agents SDK, agent-core's
|
|
35
|
+
* reconciler) uses the scheduler-seam form instead: fabric hands every
|
|
36
|
+
* next due time to the policy's `schedule` and touches no timer storage.
|
|
37
|
+
* - The drain is turn-bounded through a {@link TurnBudget}: both consumer
|
|
38
|
+
* drains iterate every due row in one turn, which is the same
|
|
39
|
+
* thread-holding shape the pacer exists to end.
|
|
40
|
+
*
|
|
41
|
+
* ADOPTION IS A DATA MIGRATION for a live consumer. The rows live in
|
|
42
|
+
* fabric's own table (`outbox_<name>`), and nothing moves itself: a consumer
|
|
43
|
+
* with pending intents or a sent-key dedupe history in its old table keeps
|
|
44
|
+
* them there. The ported EmailOutbox went from `email_outbox` to
|
|
45
|
+
* `outbox_email` — deployed live without a migration, its pending mail would
|
|
46
|
+
* never send and every sent key would deliver a second time.
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
import { z } from 'zod/v4';
|
|
50
|
+
import { timers, type TimerContext, type TimerHandlerResult, type TimerHost, type TimerStorage } from './timers.js';
|
|
51
|
+
import type { TurnBudget } from './turn-budget.js';
|
|
52
|
+
|
|
53
|
+
/** Synchronous DO SQLite, as the outbox uses it. `exec` returns row objects. */
|
|
54
|
+
export interface OutboxSqlExec {
|
|
55
|
+
exec(query: string, ...bindings: Array<string | number | null>): Iterable<unknown>;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The hosting actor's storage: its SQLite plus the shared timer map's keys. */
|
|
59
|
+
export interface OutboxStorage extends TimerStorage {
|
|
60
|
+
sql: OutboxSqlExec;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface OutboxContext extends TimerContext {
|
|
64
|
+
storage: OutboxStorage;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* All a scheduler-seam outbox needs from its host: the SQLite the rows live
|
|
69
|
+
* in. The timer map's keys belong to the {@link TimerHost} path only.
|
|
70
|
+
*/
|
|
71
|
+
export interface OutboxSqlContext {
|
|
72
|
+
storage: { sql: OutboxSqlExec };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* What one send attempt concluded. A resolved `retry` and a thrown send are
|
|
77
|
+
* the same transient class; `poison` is the resolved refusal that must never
|
|
78
|
+
* be retried — the split the peer transport proved necessary.
|
|
79
|
+
*/
|
|
80
|
+
export type OutboxDisposition =
|
|
81
|
+
| { status: 'sent' }
|
|
82
|
+
| { status: 'retry'; reason: string }
|
|
83
|
+
| { status: 'poison'; reason: string };
|
|
84
|
+
|
|
85
|
+
export interface OutboxPolicy<M, C = void> {
|
|
86
|
+
/** Attempts before a row dead-letters. Email uses 8; peer uses 8. */
|
|
87
|
+
maxAttempts: number;
|
|
88
|
+
/** Backoff base: `next = now + baseMs * 2**(attempts-1)`. */
|
|
89
|
+
baseMs: number;
|
|
90
|
+
/**
|
|
91
|
+
* Per-key delivery order (the peer transport's per-receiver key). Rows
|
|
92
|
+
* without a key deliver independently, as the email outbox's do.
|
|
93
|
+
*/
|
|
94
|
+
orderBy?(message: M): string;
|
|
95
|
+
/**
|
|
96
|
+
* Deliver one message. Throwing is transient, same as `retry`. `context`
|
|
97
|
+
* is the caller's per-drain state — Proteus's email outbox resolves its
|
|
98
|
+
* transport binding per call, and a policy closed at construction cannot
|
|
99
|
+
* hold it (the ported consumer smuggled it through an instance field).
|
|
100
|
+
*/
|
|
101
|
+
send(message: M, info: { id: string; attempt: number }, context: C): Promise<OutboxDisposition>;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* `context` carries the caller's per-drain state into `send`. Required when
|
|
106
|
+
* the policy declares one; absent (and unpayable) when it does not.
|
|
107
|
+
*/
|
|
108
|
+
export type OutboxDrainOptions<C> = { budget?: TurnBudget } & ([C] extends [void]
|
|
109
|
+
? { context?: C }
|
|
110
|
+
: { context: C });
|
|
111
|
+
|
|
112
|
+
type OutboxDrainArgs<C> = [C] extends [void]
|
|
113
|
+
? [opts?: OutboxDrainOptions<C>]
|
|
114
|
+
: [opts: OutboxDrainOptions<C>];
|
|
115
|
+
|
|
116
|
+
type OutboxHandlerArgs<C> = [C] extends [void] ? [] : [context: C];
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The policy of an outbox whose host owns the alarm. Proteus's Durable
|
|
120
|
+
* Object alarm belongs to the Agents SDK and agent-core's to its own
|
|
121
|
+
* reconciler; neither can give fabric the alarm slot. `schedule` receives
|
|
122
|
+
* every next due time — queue's admission instant, and the earliest pending
|
|
123
|
+
* deadline after each drain — and the consumer folds it into its own alarm.
|
|
124
|
+
*/
|
|
125
|
+
export interface ScheduledOutboxPolicy<M, C = void> extends OutboxPolicy<M, C> {
|
|
126
|
+
schedule(at: number): Promise<void>;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
type OutboxScheduling =
|
|
130
|
+
| { kind: 'timers'; host: TimerHost; ctx: OutboxContext }
|
|
131
|
+
| { kind: 'schedule'; schedule: (at: number) => Promise<void> };
|
|
132
|
+
|
|
133
|
+
export interface OutboxDrainResult {
|
|
134
|
+
sent: number;
|
|
135
|
+
retried: number;
|
|
136
|
+
deadLettered: number;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* One row, as {@link Outbox.status} and {@link Outbox.find} report it. Both
|
|
141
|
+
* ported consumers read fabric's table with raw SQL for exactly this: the
|
|
142
|
+
* email outbox to answer sent/deduped/failed per key, the peer transport to
|
|
143
|
+
* correlate a reply with its stored ask — which is why the record carries
|
|
144
|
+
* the stored message, not just the state.
|
|
145
|
+
*/
|
|
146
|
+
export interface OutboxRecord<M> {
|
|
147
|
+
id: string;
|
|
148
|
+
state: 'pending' | 'sent' | 'dlq';
|
|
149
|
+
/** Null when the stored payload no longer parses (a poison-parse row). */
|
|
150
|
+
message: M | null;
|
|
151
|
+
dedupeKey: string | null;
|
|
152
|
+
attemptCount: number;
|
|
153
|
+
lastError: string | null;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export interface OutboxDeadLetter<M> {
|
|
157
|
+
id: string;
|
|
158
|
+
/** Null when the stored payload no longer parses (a poison-parse row). */
|
|
159
|
+
message: M | null;
|
|
160
|
+
dedupeKey: string | null;
|
|
161
|
+
attemptCount: number;
|
|
162
|
+
lastError: string;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const PendingRowSchema = z.object({
|
|
166
|
+
id: z.string(),
|
|
167
|
+
message: z.string(),
|
|
168
|
+
order_key: z.string().nullable(),
|
|
169
|
+
attempt_count: z.number(),
|
|
170
|
+
next_attempt_at: z.number(),
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
const DlqRowSchema = z.object({
|
|
174
|
+
id: z.string(),
|
|
175
|
+
message: z.string(),
|
|
176
|
+
dedupe_key: z.string().nullable(),
|
|
177
|
+
attempt_count: z.number(),
|
|
178
|
+
last_error: z.string().nullable(),
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
const RecordRowSchema = z.object({
|
|
182
|
+
id: z.string(),
|
|
183
|
+
state: z.enum(['pending', 'sent', 'dlq']),
|
|
184
|
+
message: z.string(),
|
|
185
|
+
dedupe_key: z.string().nullable(),
|
|
186
|
+
attempt_count: z.number(),
|
|
187
|
+
last_error: z.string().nullable(),
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
const NAME_PATTERN = /^[a-z][a-z0-9_]{0,40}$/;
|
|
191
|
+
|
|
192
|
+
/** One named outbox on one hosting actor. Cheap accessor, like `timers()`. */
|
|
193
|
+
export function outbox<M, C = void>(
|
|
194
|
+
host: TimerHost,
|
|
195
|
+
ctx: OutboxContext,
|
|
196
|
+
name: string,
|
|
197
|
+
policy: OutboxPolicy<M, C>,
|
|
198
|
+
): Outbox<M, C>;
|
|
199
|
+
/** The scheduler-seam form: the host owns the alarm, fabric owns the rows. */
|
|
200
|
+
export function outbox<M, C = void>(
|
|
201
|
+
ctx: OutboxSqlContext,
|
|
202
|
+
name: string,
|
|
203
|
+
policy: ScheduledOutboxPolicy<M, C>,
|
|
204
|
+
): Outbox<M, C>;
|
|
205
|
+
export function outbox<M, C = void>(
|
|
206
|
+
...args:
|
|
207
|
+
| [host: TimerHost, ctx: OutboxContext, name: string, policy: OutboxPolicy<M, C>]
|
|
208
|
+
| [ctx: OutboxSqlContext, name: string, policy: ScheduledOutboxPolicy<M, C>]
|
|
209
|
+
): Outbox<M, C> {
|
|
210
|
+
if (args.length === 4) {
|
|
211
|
+
const [host, ctx, name, policy] = args;
|
|
212
|
+
return new Outbox(ctx, name, policy, { kind: 'timers', host, ctx });
|
|
213
|
+
}
|
|
214
|
+
const [ctx, name, policy] = args;
|
|
215
|
+
return new Outbox(ctx, name, policy, { kind: 'schedule', schedule: (at) => policy.schedule(at) });
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export class Outbox<M, C = void> {
|
|
219
|
+
/** The timer reason this outbox arms in the shared map. */
|
|
220
|
+
readonly reason: string;
|
|
221
|
+
|
|
222
|
+
private readonly table: string;
|
|
223
|
+
private schemaReady = false;
|
|
224
|
+
private draining = false;
|
|
225
|
+
/** Largest id ever seen, so a replacement instance mints above it. */
|
|
226
|
+
private lastId = '';
|
|
227
|
+
private seq = 0;
|
|
228
|
+
|
|
229
|
+
constructor(
|
|
230
|
+
private readonly ctx: OutboxSqlContext,
|
|
231
|
+
name: string,
|
|
232
|
+
private readonly policy: OutboxPolicy<M, C>,
|
|
233
|
+
private readonly scheduling: OutboxScheduling,
|
|
234
|
+
) {
|
|
235
|
+
if (!NAME_PATTERN.test(name)) {
|
|
236
|
+
throw new Error(`fabric: outbox name '${name}' must match ${NAME_PATTERN}`);
|
|
237
|
+
}
|
|
238
|
+
this.table = `outbox_${name}`;
|
|
239
|
+
this.reason = `outbox:${name}`;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
private ensureSchema(): void {
|
|
243
|
+
if (this.schemaReady) return;
|
|
244
|
+
const sql = this.ctx.storage.sql;
|
|
245
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
246
|
+
id TEXT PRIMARY KEY,
|
|
247
|
+
dedupe_key TEXT,
|
|
248
|
+
order_key TEXT,
|
|
249
|
+
message TEXT NOT NULL,
|
|
250
|
+
state TEXT NOT NULL DEFAULT 'pending'
|
|
251
|
+
CHECK (state IN ('pending', 'sent', 'dlq')),
|
|
252
|
+
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
253
|
+
next_attempt_at INTEGER NOT NULL,
|
|
254
|
+
created_at INTEGER NOT NULL,
|
|
255
|
+
sent_at INTEGER,
|
|
256
|
+
last_error TEXT
|
|
257
|
+
)`);
|
|
258
|
+
sql.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_${this.table}_dedupe
|
|
259
|
+
ON ${this.table} (dedupe_key) WHERE dedupe_key IS NOT NULL`);
|
|
260
|
+
// The recovery read (`nextRetryAt`, the drain) must not table-scan sent
|
|
261
|
+
// history, which only grows — the same reason both consumer schemas carry
|
|
262
|
+
// exactly this partial index.
|
|
263
|
+
sql.exec(`CREATE INDEX IF NOT EXISTS idx_${this.table}_pending
|
|
264
|
+
ON ${this.table} (next_attempt_at) WHERE state = 'pending'`);
|
|
265
|
+
const rows = [...sql.exec(`SELECT MAX(id) AS id FROM ${this.table}`)] as Array<{ id: string | null }>;
|
|
266
|
+
this.lastId = rows[0]?.id ?? '';
|
|
267
|
+
this.schemaReady = true;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Ids order the drain, so they must grow: time-prefixed, tie-broken by a
|
|
272
|
+
* per-instance counter, and forced above the largest stored id so a
|
|
273
|
+
* replacement instance with a lagging clock cannot mint into the past.
|
|
274
|
+
*/
|
|
275
|
+
private mintId(now: number): string {
|
|
276
|
+
let id = `${now.toString(36).padStart(9, '0')}-${(this.seq++).toString(36).padStart(6, '0')}`;
|
|
277
|
+
if (this.lastId !== '' && id <= this.lastId) id = `${this.lastId}0`;
|
|
278
|
+
this.lastId = id;
|
|
279
|
+
return id;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Write the intent ahead of any send. Returns `admitted: false` with the
|
|
284
|
+
* existing id when the dedupe key is already queued, sent, or dead-lettered
|
|
285
|
+
* — a sent key never reaches send again (the email outbox's short-circuit).
|
|
286
|
+
*
|
|
287
|
+
* `onDuplicate: 'retry-now'` treats a re-ask for an UNSENT key as new
|
|
288
|
+
* intent: the row's retry instant is clamped to `now` and a dead letter
|
|
289
|
+
* returns to 'pending', its attempt count kept — so every re-ask buys one
|
|
290
|
+
* more delivery attempt, past the attempt budget. The monitor's alert-once
|
|
291
|
+
* contract needs exactly this: a failed alert is retried by each sweep
|
|
292
|
+
* until it lands. A sent key stays final either way.
|
|
293
|
+
*
|
|
294
|
+
* Arms the scheduler at `now`, so delivery is owed by the alarm even
|
|
295
|
+
* when the caller never drains inline.
|
|
296
|
+
*/
|
|
297
|
+
async queue(
|
|
298
|
+
message: M,
|
|
299
|
+
opts: { dedupeKey?: string; now?: number; onDuplicate?: 'refuse' | 'retry-now' } = {},
|
|
300
|
+
): Promise<{ id: string; admitted: boolean }> {
|
|
301
|
+
this.ensureSchema();
|
|
302
|
+
const now = opts.now ?? Date.now();
|
|
303
|
+
const sql = this.ctx.storage.sql;
|
|
304
|
+
if (opts.dedupeKey !== undefined) {
|
|
305
|
+
const existing = [...sql.exec(
|
|
306
|
+
`SELECT id, state FROM ${this.table} WHERE dedupe_key = ?`, opts.dedupeKey,
|
|
307
|
+
)] as Array<{ id: string; state: string }>;
|
|
308
|
+
if (existing.length > 0) {
|
|
309
|
+
if (opts.onDuplicate === 'retry-now' && existing[0].state !== 'sent') {
|
|
310
|
+
sql.exec(
|
|
311
|
+
`UPDATE ${this.table} SET state = 'pending', next_attempt_at = ? WHERE id = ?`,
|
|
312
|
+
now, existing[0].id,
|
|
313
|
+
);
|
|
314
|
+
await this.arm(now);
|
|
315
|
+
}
|
|
316
|
+
return { id: existing[0].id, admitted: false };
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
const id = this.mintId(now);
|
|
320
|
+
sql.exec(
|
|
321
|
+
`INSERT INTO ${this.table} (id, dedupe_key, order_key, message, next_attempt_at, created_at)
|
|
322
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
323
|
+
id,
|
|
324
|
+
opts.dedupeKey ?? null,
|
|
325
|
+
this.policy.orderBy?.(message) ?? null,
|
|
326
|
+
JSON.stringify(message),
|
|
327
|
+
now,
|
|
328
|
+
now,
|
|
329
|
+
);
|
|
330
|
+
await this.arm(now);
|
|
331
|
+
return { id, admitted: true };
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Hand a due time to whichever scheduler this outbox was built on. */
|
|
335
|
+
private async arm(at: number): Promise<void> {
|
|
336
|
+
if (this.scheduling.kind === 'timers') {
|
|
337
|
+
await timers(this.scheduling.host, this.scheduling.ctx).schedule(this.reason, at);
|
|
338
|
+
} else {
|
|
339
|
+
await this.scheduling.schedule(at);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** The single-alarm fold: the earliest pending deadline, or null. */
|
|
344
|
+
nextRetryAt(): number | null {
|
|
345
|
+
this.ensureSchema();
|
|
346
|
+
const rows = [...this.ctx.storage.sql.exec(
|
|
347
|
+
`SELECT MIN(next_attempt_at) AS next FROM ${this.table} WHERE state = 'pending'`,
|
|
348
|
+
)] as Array<{ next: number | null }>;
|
|
349
|
+
return rows[0]?.next ?? null;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** The row queue() named, by its id. Null when no such row exists. */
|
|
353
|
+
status(id: string): OutboxRecord<M> | null {
|
|
354
|
+
return this.record('id', id);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/** The row a dedupe key admitted, whatever its state. Null when none. */
|
|
358
|
+
find(dedupeKey: string): OutboxRecord<M> | null {
|
|
359
|
+
return this.record('dedupe_key', dedupeKey);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
private record(column: 'id' | 'dedupe_key', value: string): OutboxRecord<M> | null {
|
|
363
|
+
this.ensureSchema();
|
|
364
|
+
const rows = [...this.ctx.storage.sql.exec(
|
|
365
|
+
`SELECT id, state, message, dedupe_key, attempt_count, last_error
|
|
366
|
+
FROM ${this.table} WHERE ${column} = ?`, value,
|
|
367
|
+
)];
|
|
368
|
+
if (rows.length === 0) return null;
|
|
369
|
+
const row = RecordRowSchema.parse(rows[0]);
|
|
370
|
+
let message: M | null = null;
|
|
371
|
+
try { message = JSON.parse(row.message) as M; } catch { /* poison-parse row */ }
|
|
372
|
+
return {
|
|
373
|
+
id: row.id,
|
|
374
|
+
state: row.state,
|
|
375
|
+
message,
|
|
376
|
+
dedupeKey: row.dedupe_key,
|
|
377
|
+
attemptCount: row.attempt_count,
|
|
378
|
+
lastError: row.last_error,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/** Dead-lettered rows, for inspection. Terminal: nothing retries out. */
|
|
383
|
+
dlq(): Array<OutboxDeadLetter<M>> {
|
|
384
|
+
this.ensureSchema();
|
|
385
|
+
const rows = [...this.ctx.storage.sql.exec(
|
|
386
|
+
`SELECT id, message, dedupe_key, attempt_count, last_error FROM ${this.table} WHERE state = 'dlq' ORDER BY id`,
|
|
387
|
+
)].map((row) => DlqRowSchema.parse(row));
|
|
388
|
+
return rows.map((row) => {
|
|
389
|
+
let message: M | null = null;
|
|
390
|
+
try { message = JSON.parse(row.message) as M; } catch { /* poison-parse row */ }
|
|
391
|
+
return {
|
|
392
|
+
id: row.id,
|
|
393
|
+
message,
|
|
394
|
+
dedupeKey: row.dedupe_key,
|
|
395
|
+
attemptCount: row.attempt_count,
|
|
396
|
+
lastError: row.last_error ?? '',
|
|
397
|
+
};
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* Deliver every due pending row, in id order, honouring per-key blocking.
|
|
403
|
+
* Reentrancy-guarded: the alarm and an inline post-queue drain overlap on
|
|
404
|
+
* the same activation (peer.ts:452 carries the same guard).
|
|
405
|
+
*
|
|
406
|
+
* `budget` bounds the turn: the drain spends each processed row's payload
|
|
407
|
+
* size and suspends when a chunk is full, so a large backlog crosses turns
|
|
408
|
+
* instead of holding the actor's only thread.
|
|
409
|
+
*
|
|
410
|
+
* `context` is handed to every `send` this drain makes — the seam for
|
|
411
|
+
* per-call state such as a transport binding resolved by the caller.
|
|
412
|
+
*/
|
|
413
|
+
async drain(now = Date.now(), ...rest: OutboxDrainArgs<C>): Promise<OutboxDrainResult> {
|
|
414
|
+
const opts = rest[0] ?? {};
|
|
415
|
+
const result: OutboxDrainResult = { sent: 0, retried: 0, deadLettered: 0 };
|
|
416
|
+
if (this.draining) return result;
|
|
417
|
+
this.draining = true;
|
|
418
|
+
try {
|
|
419
|
+
this.ensureSchema();
|
|
420
|
+
const sql = this.ctx.storage.sql;
|
|
421
|
+
const rows = [...sql.exec(
|
|
422
|
+
`SELECT id, message, order_key, attempt_count, next_attempt_at
|
|
423
|
+
FROM ${this.table} WHERE state = 'pending' ORDER BY id`,
|
|
424
|
+
)].map((row) => PendingRowSchema.parse(row));
|
|
425
|
+
const blocked = new Set<string>();
|
|
426
|
+
for (const row of rows) {
|
|
427
|
+
if (row.order_key !== null && blocked.has(row.order_key)) continue;
|
|
428
|
+
if (row.next_attempt_at > now) {
|
|
429
|
+
// A backed-off head still blocks the rows queued behind it.
|
|
430
|
+
if (row.order_key !== null) blocked.add(row.order_key);
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
433
|
+
let message: M;
|
|
434
|
+
try {
|
|
435
|
+
message = JSON.parse(row.message) as M;
|
|
436
|
+
} catch (e) {
|
|
437
|
+
// Unparseable is poison: it can never succeed, so it never blocks.
|
|
438
|
+
this.deadLetter(row.id, row.attempt_count, `outbox row does not parse: ${errorText(e)}`);
|
|
439
|
+
result.deadLettered++;
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
442
|
+
const attempt = row.attempt_count + 1;
|
|
443
|
+
let disposition: OutboxDisposition;
|
|
444
|
+
try {
|
|
445
|
+
disposition = await this.policy.send(message, { id: row.id, attempt }, opts.context as C);
|
|
446
|
+
} catch (e) {
|
|
447
|
+
disposition = { status: 'retry', reason: errorText(e) };
|
|
448
|
+
}
|
|
449
|
+
if (disposition.status === 'sent') {
|
|
450
|
+
sql.exec(
|
|
451
|
+
`UPDATE ${this.table} SET state = 'sent', attempt_count = ?, sent_at = ?, last_error = NULL WHERE id = ?`,
|
|
452
|
+
attempt, now, row.id,
|
|
453
|
+
);
|
|
454
|
+
result.sent++;
|
|
455
|
+
} else if (disposition.status === 'poison') {
|
|
456
|
+
this.deadLetter(row.id, attempt, disposition.reason);
|
|
457
|
+
result.deadLettered++;
|
|
458
|
+
} else if (attempt >= this.policy.maxAttempts) {
|
|
459
|
+
this.deadLetter(row.id, attempt, `undeliverable after ${attempt} attempts: ${disposition.reason}`);
|
|
460
|
+
result.deadLettered++;
|
|
461
|
+
} else {
|
|
462
|
+
const next = now + this.policy.baseMs * 2 ** (attempt - 1);
|
|
463
|
+
sql.exec(
|
|
464
|
+
`UPDATE ${this.table} SET attempt_count = ?, next_attempt_at = ?, last_error = ? WHERE id = ?`,
|
|
465
|
+
attempt, next, disposition.reason, row.id,
|
|
466
|
+
);
|
|
467
|
+
result.retried++;
|
|
468
|
+
if (row.order_key !== null) blocked.add(row.order_key);
|
|
469
|
+
}
|
|
470
|
+
// Code-unit length: equal to UTF-8 bytes for the ASCII JSON the rows
|
|
471
|
+
// hold, an undercount otherwise — the bound names where a turn ends,
|
|
472
|
+
// it is not an admission ceiling (same reading as budgets.ts).
|
|
473
|
+
await opts.budget?.spend(row.message.length);
|
|
474
|
+
}
|
|
475
|
+
// A seam-scheduled outbox re-arms itself: there is no dispatcher whose
|
|
476
|
+
// return value could, and the ported consumer re-armed by hand after
|
|
477
|
+
// every drain — delivery stays owed either way.
|
|
478
|
+
if (this.scheduling.kind === 'schedule') {
|
|
479
|
+
const next = this.nextRetryAt();
|
|
480
|
+
if (next !== null) await this.scheduling.schedule(next);
|
|
481
|
+
}
|
|
482
|
+
return result;
|
|
483
|
+
} finally {
|
|
484
|
+
this.draining = false;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
private deadLetter(id: string, attempts: number, error: string): void {
|
|
489
|
+
this.ctx.storage.sql.exec(
|
|
490
|
+
`UPDATE ${this.table} SET state = 'dlq', attempt_count = ?, last_error = ? WHERE id = ?`,
|
|
491
|
+
attempts, error, id,
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* The dispatch-side entry for the embedder's timer handler map. Re-arms
|
|
497
|
+
* through the RETURN value: the dispatcher runs handlers inside the chain
|
|
498
|
+
* that serializes the reason map, so a schedule() call from here would
|
|
499
|
+
* deadlock on its own chain.
|
|
500
|
+
*
|
|
501
|
+
* An alarm fires with no caller, so a policy that declares a drain context
|
|
502
|
+
* receives it here once, closed over every alarm-driven drain.
|
|
503
|
+
*/
|
|
504
|
+
handler(...context: OutboxHandlerArgs<C>): (now: number) => Promise<TimerHandlerResult> {
|
|
505
|
+
if (this.scheduling.kind === 'schedule') {
|
|
506
|
+
throw new Error(
|
|
507
|
+
`fabric: outbox '${this.reason}' uses a scheduler seam — there is no timer dispatcher to hand this handler to`,
|
|
508
|
+
);
|
|
509
|
+
}
|
|
510
|
+
return async (now: number) => {
|
|
511
|
+
await this.drain(now, ...([{ context: context[0] }] as OutboxDrainArgs<C>));
|
|
512
|
+
const next = this.nextRetryAt();
|
|
513
|
+
return next === null ? undefined : { rearmAt: next };
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
function errorText(error: unknown): string {
|
|
519
|
+
return error instanceof Error ? error.message : String(error);
|
|
520
|
+
}
|
package/src/process-fabric.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Every long-lived process Nimbus runs — node servers, python/ruby socket
|
|
6
6
|
* servers, an agent TUI and its headless server — runs as a **DO Facet**:
|
|
7
7
|
* a named child actor whose class comes from a dynamic worker, opened by
|
|
8
|
-
* `
|
|
8
|
+
* `processes(ctx, env).spawn` in `workerd-facet-host.ts`.
|
|
9
9
|
*
|
|
10
10
|
* ctx.facets.get(`proc-${pid}`, () => ({
|
|
11
11
|
* class: env.LOADER.get(workerKey, buildConfig)
|
|
@@ -76,6 +76,7 @@
|
|
|
76
76
|
|
|
77
77
|
import { z } from 'zod/v4';
|
|
78
78
|
import type { RouteableFacetTarget } from '@nimbus-sh/core/runtime/os-contracts.js';
|
|
79
|
+
import type { ServiceStub } from './vendor/types.js';
|
|
79
80
|
|
|
80
81
|
/**
|
|
81
82
|
* The class every generated resident runner exports. One name for every
|
|
@@ -131,6 +132,19 @@ export const ResidentCodeSpecSchema = z.object({
|
|
|
131
132
|
* image store below and the spec names it.
|
|
132
133
|
*/
|
|
133
134
|
vfsTextModules: z.record(z.string(), z.string()).optional(),
|
|
135
|
+
/**
|
|
136
|
+
* The isolate's exact `env`: one entry per binding the embedder minted,
|
|
137
|
+
* carried by reference so loopback stubs survive untouched. Defined —
|
|
138
|
+
* even as `{}` — means the embedder takes the whole env and no
|
|
139
|
+
* SUPERVISOR binding is injected; absent keeps the default (a SUPERVISOR
|
|
140
|
+
* minted from the composed supervisor entrypoint).
|
|
141
|
+
*/
|
|
142
|
+
env: z.record(z.string(), z.unknown()).optional(),
|
|
143
|
+
/** Absent inherits outbound, null denies it, a binding mediates it by reference. */
|
|
144
|
+
globalOutbound: z.custom<ServiceStub>((value) =>
|
|
145
|
+
value !== null && (typeof value === 'object' || typeof value === 'function')
|
|
146
|
+
&& 'fetch' in value && typeof value.fetch === 'function',
|
|
147
|
+
).nullable().optional(),
|
|
134
148
|
});
|
|
135
149
|
|
|
136
150
|
export type ResidentCodeSpec = z.infer<typeof ResidentCodeSpecSchema>;
|
|
@@ -156,37 +170,10 @@ export type ResidentBootSpec =
|
|
|
156
170
|
| { kind: 'code'; code: ResidentCodeSpec };
|
|
157
171
|
|
|
158
172
|
// ── Staged boots ────────────────────────────────────────────────────────────
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
* knows how to fetch (Nimbus's largest staged artifact is a ~23 MB module map from
|
|
164
|
-
* ASSETS), and the assembler runs inside the loader's cache-miss callback so
|
|
165
|
-
* those sources are materialized only while the facet actually loads.
|
|
166
|
-
* `env` is whichever hosting actor's env the facet is opened with.
|
|
167
|
-
*/
|
|
168
|
-
export type StagedBootAssembler = (
|
|
169
|
-
env: unknown,
|
|
170
|
-
stage: unknown,
|
|
171
|
-
) => Promise<object>;
|
|
172
|
-
|
|
173
|
-
let _stagedBootAssembler: StagedBootAssembler | null = null;
|
|
174
|
-
|
|
175
|
-
/** Registered once at composition time, first-write-wins. */
|
|
176
|
-
export function setStagedBootAssembler(assembler: StagedBootAssembler): void {
|
|
177
|
-
if (_stagedBootAssembler) return;
|
|
178
|
-
_stagedBootAssembler = assembler;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
export function requireStagedBootAssembler(): StagedBootAssembler {
|
|
182
|
-
if (!_stagedBootAssembler) {
|
|
183
|
-
throw new Error(
|
|
184
|
-
'fabric: no staged-boot assembler registered; a \'staged\' boot spec '
|
|
185
|
-
+ 'cannot be assembled without one (setStagedBootAssembler)',
|
|
186
|
-
);
|
|
187
|
-
}
|
|
188
|
-
return _stagedBootAssembler;
|
|
189
|
-
}
|
|
173
|
+
//
|
|
174
|
+
// A 'staged' boot spec assembles through the embedder's composed
|
|
175
|
+
// StagedBootAssembler — see composition.ts. What a stage IS belongs to
|
|
176
|
+
// whoever composed the assembler.
|
|
190
177
|
|
|
191
178
|
// ── Boot-image store ────────────────────────────────────────────────────────
|
|
192
179
|
|
|
@@ -265,6 +252,11 @@ export interface ResidentDiskReader {
|
|
|
265
252
|
* path, verifying each generated image against the digest its own path claims.
|
|
266
253
|
* Runs inside the loader's cache-miss callback, so the bytes exist only for
|
|
267
254
|
* the duration of the load.
|
|
255
|
+
*
|
|
256
|
+
* The spec's isolation posture rides along verbatim: an explicit `env` is
|
|
257
|
+
* the isolate's whole env (loopback stubs by reference, never cloned or
|
|
258
|
+
* re-minted). An absent env stays absent
|
|
259
|
+
* so the worker config can tell "embedder takes the env" from the default.
|
|
268
260
|
*/
|
|
269
261
|
export async function residentLoaderConfig(
|
|
270
262
|
spec: ResidentCodeSpec,
|
|
@@ -285,6 +277,8 @@ export async function residentLoaderConfig(
|
|
|
285
277
|
compatibilityFlags: spec.compatibilityFlags,
|
|
286
278
|
mainModule: spec.mainModule,
|
|
287
279
|
modules: { ...spec.modules, ...resolved },
|
|
280
|
+
...(spec.env !== undefined ? { env: spec.env } : {}),
|
|
281
|
+
...(spec.globalOutbound !== undefined ? { globalOutbound: spec.globalOutbound } : {}),
|
|
288
282
|
};
|
|
289
283
|
}
|
|
290
284
|
|
|
@@ -336,6 +330,14 @@ export interface ProcessHostParams {
|
|
|
336
330
|
writerId: string;
|
|
337
331
|
/** Forwarded verbatim to the runner's startProcess. */
|
|
338
332
|
startArgs: unknown;
|
|
333
|
+
/**
|
|
334
|
+
* Set only by the coordinator's durable-application path: an explicit facet
|
|
335
|
+
* name (`app-slot-<n>`) allocated from DO storage, plus the release split
|
|
336
|
+
* that keeps its SQLite across aborts. Absent, the host allocates an
|
|
337
|
+
* ephemeral `proc-slot-<n>` name from its in-memory free list and deletes
|
|
338
|
+
* the store on release.
|
|
339
|
+
*/
|
|
340
|
+
facet?: { name: string; durable: boolean };
|
|
339
341
|
}
|
|
340
342
|
|
|
341
343
|
/**
|
|
@@ -425,7 +427,7 @@ export interface ProcessImageDelivery {
|
|
|
425
427
|
* ends: the source exists and is populated before, the destination is
|
|
426
428
|
* non-empty after. A blocklist of bad names would pass a typo straight
|
|
427
429
|
* through and wipe a process's filesystem while returning ok. Enforced by
|
|
428
|
-
* `
|
|
430
|
+
* `cloneStorage` in the workerd host, which is the one way the fabric
|
|
429
431
|
* calls clone.
|
|
430
432
|
*/
|
|
431
433
|
readonly reflink: 'same-object' | 'impossible';
|
|
@@ -638,10 +640,16 @@ export interface ResidentProcessSpawn {
|
|
|
638
640
|
pid: number;
|
|
639
641
|
/** Keyed dynamic-worker identity (`nimbus-process:${doId}:${pid}`). */
|
|
640
642
|
workerKey: string;
|
|
641
|
-
/** What the
|
|
643
|
+
/** What the process boots from. */
|
|
642
644
|
boot: ResidentBootSpec;
|
|
643
645
|
/** Forwarded verbatim to the runner's startProcess. */
|
|
644
646
|
startArgs?: unknown;
|
|
647
|
+
/**
|
|
648
|
+
* A durable application's explicit facet name (`app-slot-<n>`) and the
|
|
649
|
+
* release split that keeps its SQLite. Coordinator-allocated; absent for an
|
|
650
|
+
* ephemeral process, which takes a `proc-slot-<n>` name from the book.
|
|
651
|
+
*/
|
|
652
|
+
facet?: { name: string; durable: boolean };
|
|
645
653
|
/**
|
|
646
654
|
* Called before any concrete host capability can expose this writer.
|
|
647
655
|
* A spawn must not proceed unless the supervisor accepts the authority.
|
|
@@ -685,6 +693,7 @@ export class ProcessFabric {
|
|
|
685
693
|
boot: spawn.boot,
|
|
686
694
|
writerId,
|
|
687
695
|
startArgs: spawn.startArgs,
|
|
696
|
+
...(spawn.facet !== undefined ? { facet: spawn.facet } : {}),
|
|
688
697
|
});
|
|
689
698
|
} catch (error) {
|
|
690
699
|
spawn.onWriterRetired(writerId);
|