@nimbus-sh/fabric 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -55
- package/dist/bindings.js +5 -5
- package/dist/budgets.d.ts +132 -0
- package/dist/budgets.d.ts.map +1 -0
- package/dist/budgets.js +248 -0
- package/dist/composition.d.ts +87 -0
- package/dist/composition.d.ts.map +1 -0
- package/dist/composition.js +76 -0
- package/dist/connections.d.ts +81 -0
- package/dist/connections.d.ts.map +1 -0
- package/dist/connections.js +114 -0
- package/dist/derived.d.ts +65 -0
- package/dist/derived.d.ts.map +1 -0
- package/dist/derived.js +95 -0
- package/dist/do-calls.d.ts +94 -0
- package/dist/do-calls.d.ts.map +1 -0
- package/dist/do-calls.js +111 -0
- package/dist/facet-pool.d.ts +90 -0
- package/dist/facet-pool.d.ts.map +1 -0
- package/dist/facet-pool.js +113 -0
- package/dist/{fanout-pool.d.ts → fanout.d.ts} +20 -20
- package/dist/fanout.d.ts.map +1 -0
- package/dist/{fanout-pool.js → fanout.js} +20 -20
- package/dist/{launch-journal.d.ts → fenced-work.d.ts} +25 -13
- package/dist/fenced-work.d.ts.map +1 -0
- package/dist/{launch-journal.js → fenced-work.js} +47 -13
- package/dist/generation.d.ts +69 -0
- package/dist/generation.d.ts.map +1 -0
- package/dist/generation.js +118 -0
- package/dist/{facet-image-store.d.ts → image-store.d.ts} +8 -8
- package/dist/image-store.d.ts.map +1 -0
- package/dist/{facet-image-store.js → image-store.js} +4 -4
- package/dist/index.d.ts +16 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +16 -8
- package/dist/{loader-pool.d.ts → isolate-pool.d.ts} +19 -19
- package/dist/isolate-pool.d.ts.map +1 -0
- package/dist/{loader-pool.js → isolate-pool.js} +20 -20
- package/dist/journal.d.ts +111 -0
- package/dist/journal.d.ts.map +1 -0
- package/dist/journal.js +177 -0
- package/dist/outbox.d.ts +249 -0
- package/dist/outbox.d.ts.map +1 -0
- package/dist/outbox.js +355 -0
- package/dist/process-fabric.d.ts +2 -14
- package/dist/process-fabric.d.ts.map +1 -1
- package/dist/process-fabric.js +6 -15
- package/dist/process-host.d.ts +1 -1
- package/dist/process-host.d.ts.map +1 -1
- package/dist/process-host.js +10 -9
- package/dist/sealed.d.ts +78 -0
- package/dist/sealed.d.ts.map +1 -0
- package/dist/sealed.js +145 -0
- package/dist/timers.d.ts +138 -0
- package/dist/timers.d.ts.map +1 -0
- package/dist/timers.js +231 -0
- package/dist/{launch-pacer.d.ts → turn-budget.d.ts} +19 -21
- package/dist/turn-budget.d.ts.map +1 -0
- package/dist/{launch-pacer.js → turn-budget.js} +22 -11
- package/dist/workerd-facet-host.d.ts +28 -67
- package/dist/workerd-facet-host.d.ts.map +1 -1
- package/dist/workerd-facet-host.js +49 -171
- package/examples/agent-core-adapter.ts +191 -0
- package/package.json +4 -2
- package/src/bindings.ts +6 -6
- package/src/budgets.ts +308 -0
- package/src/composition.ts +127 -0
- package/src/connections.ts +140 -0
- package/src/derived.ts +135 -0
- package/src/do-calls.ts +156 -0
- package/src/facet-pool.ts +157 -0
- package/src/{fanout-pool.ts → fanout.ts} +35 -35
- package/src/{launch-journal.ts → fenced-work.ts} +58 -22
- package/src/generation.ts +144 -0
- package/src/{facet-image-store.ts → image-store.ts} +9 -9
- package/src/index.ts +16 -8
- package/src/{loader-pool.ts → isolate-pool.ts} +34 -34
- package/src/journal.ts +242 -0
- package/src/node-async-hooks.d.ts +14 -0
- package/src/outbox.ts +520 -0
- package/src/process-fabric.ts +6 -33
- package/src/process-host.ts +10 -15
- package/src/sealed.ts +150 -0
- package/src/timers.ts +294 -0
- package/src/{launch-pacer.ts → turn-budget.ts} +30 -24
- package/src/workerd-facet-host.ts +67 -193
- package/dist/alarms.d.ts +0 -134
- package/dist/alarms.d.ts.map +0 -1
- package/dist/alarms.js +0 -214
- package/dist/ctx-exports.d.ts +0 -47
- package/dist/ctx-exports.d.ts.map +0 -1
- package/dist/ctx-exports.js +0 -54
- package/dist/facet-image-store.d.ts.map +0 -1
- package/dist/fanout-pool.d.ts.map +0 -1
- package/dist/launch-journal.d.ts.map +0 -1
- package/dist/launch-pacer.d.ts.map +0 -1
- package/dist/loader-ledger.d.ts +0 -57
- package/dist/loader-ledger.d.ts.map +0 -1
- package/dist/loader-ledger.js +0 -91
- package/dist/loader-pool.d.ts.map +0 -1
- package/src/alarms.ts +0 -275
- package/src/ctx-exports.ts +0 -77
- package/src/loader-ledger.ts +0 -112
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* A single Durable Object method can drive at most four concurrent
|
|
6
6
|
* Worker Loader fetches before extra dispatches serialize or fail. Small
|
|
7
|
-
* batches therefore run in the coordinator DO through
|
|
7
|
+
* batches therefore run in the coordinator DO through IsolatePool.
|
|
8
8
|
* Wider batches are sharded across sibling NimbusSession DOs, each of
|
|
9
9
|
* which owns its own four-loader budget.
|
|
10
10
|
*
|
|
@@ -16,9 +16,9 @@
|
|
|
16
16
|
|
|
17
17
|
import { serializeFunction } from './vendor/serialize.js';
|
|
18
18
|
import { BindingError } from './vendor/errors.js';
|
|
19
|
-
import {
|
|
20
|
-
import { disposeRpcResource } from '@nimbus-sh/
|
|
21
|
-
import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/
|
|
19
|
+
import { IsolatePool, type FacetTaskFn } from './isolate-pool.js';
|
|
20
|
+
import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
|
|
21
|
+
import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/platform/oom-classify.js';
|
|
22
22
|
import type { WorkerLoader } from './vendor/types.js';
|
|
23
23
|
|
|
24
24
|
/**
|
|
@@ -31,7 +31,7 @@ interface PeerSessionNamespace {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
/** The bindings a fan-out needs off the coordinator DO's env. */
|
|
34
|
-
export interface
|
|
34
|
+
export interface FanoutEnv {
|
|
35
35
|
LOADER?: WorkerLoader;
|
|
36
36
|
NIMBUS_SESSION?: PeerSessionNamespace;
|
|
37
37
|
}
|
|
@@ -104,45 +104,45 @@ export interface FanoutTask<A> {
|
|
|
104
104
|
args: A;
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
/** Options handed to
|
|
108
|
-
export interface
|
|
107
|
+
/** Options handed to Fanout's constructor. */
|
|
108
|
+
export interface FanoutOptions {
|
|
109
109
|
/**
|
|
110
110
|
* Tag prepended to peer-DO ids and in-DO loader ids for debugging
|
|
111
111
|
* (e.g. "npm-install-batch"). Affects neither isolate identity (in-DO
|
|
112
|
-
* path uses the existing
|
|
112
|
+
* path uses the existing IsolatePool's tag-fold) nor peer-DO
|
|
113
113
|
* deterministic placement (peer ids fold tag + key).
|
|
114
114
|
*/
|
|
115
115
|
tag: string;
|
|
116
116
|
/**
|
|
117
117
|
* Per-task timeout in ms. Default 60_000. Forwarded to the in-DO
|
|
118
|
-
*
|
|
119
|
-
*
|
|
118
|
+
* IsolatePool's submit calls and to the peer-DO RPC's own
|
|
119
|
+
* IsolatePool.
|
|
120
120
|
*/
|
|
121
121
|
timeoutMs?: number;
|
|
122
122
|
/**
|
|
123
123
|
* Preamble bundled into every facet (in-DO and inside each peer
|
|
124
|
-
* DO). Same semantics as
|
|
124
|
+
* DO). Same semantics as IsolatePool's preamble option.
|
|
125
125
|
*/
|
|
126
126
|
preamble?: string;
|
|
127
127
|
/**
|
|
128
128
|
* Wasm modules forwarded to every facet. Same semantics as
|
|
129
|
-
*
|
|
129
|
+
* IsolatePool's wasmModules option.
|
|
130
130
|
*/
|
|
131
131
|
wasmModules?: Record<string, ArrayBuffer>;
|
|
132
132
|
/**
|
|
133
133
|
* Extra bindings forwarded to every facet. Same semantics as
|
|
134
|
-
*
|
|
134
|
+
* IsolatePool's extraBindings option.
|
|
135
135
|
*/
|
|
136
136
|
extraBindings?: Record<string, unknown>;
|
|
137
137
|
/**
|
|
138
138
|
* If set, skip the supervisor-RPC binding injection (mirrors
|
|
139
|
-
*
|
|
139
|
+
* IsolatePool's omitSupervisor flag).
|
|
140
140
|
*/
|
|
141
141
|
omitSupervisor?: boolean;
|
|
142
142
|
/**
|
|
143
143
|
* Invoking process pid, baked into each facet's SUPERVISOR binding so
|
|
144
144
|
* filesystem RPCs (writeBatchStream) are authorized under the caller's
|
|
145
|
-
* credential (mirrors
|
|
145
|
+
* credential (mirrors IsolatePool's supervisorPid). Threaded to both
|
|
146
146
|
* the in-DO loader pool and, via `_rpcFanoutExecute`, the peer-DO pools.
|
|
147
147
|
* npm install passes the shell command's `ctx.pid`; resolve leaves it 0.
|
|
148
148
|
*/
|
|
@@ -186,10 +186,10 @@ function isFanoutPeerStub(value: unknown): value is FanoutPeerStub {
|
|
|
186
186
|
|
|
187
187
|
function fanoutPeerStub(value: unknown): FanoutPeerStub {
|
|
188
188
|
if ((typeof value !== 'object' && typeof value !== 'function') || value === null) {
|
|
189
|
-
throw new BindingError('
|
|
189
|
+
throw new BindingError('Fanout: NIMBUS_SESSION.get() did not return a peer stub.');
|
|
190
190
|
}
|
|
191
191
|
if (!isFanoutPeerStub(value)) {
|
|
192
|
-
throw new BindingError('
|
|
192
|
+
throw new BindingError('Fanout: peer stub does not expose _rpcFanoutExecute().');
|
|
193
193
|
}
|
|
194
194
|
return value;
|
|
195
195
|
}
|
|
@@ -203,23 +203,23 @@ function fanoutPeerStub(value: unknown): FanoutPeerStub {
|
|
|
203
203
|
* exists primarily as a clean API surface; per-call dispatch state
|
|
204
204
|
* lives only inside submitMany's promise.
|
|
205
205
|
*/
|
|
206
|
-
export class
|
|
207
|
-
private readonly env:
|
|
206
|
+
export class Fanout {
|
|
207
|
+
private readonly env: FanoutEnv;
|
|
208
208
|
private readonly ctx: DurableObjectState;
|
|
209
|
-
private readonly opts:
|
|
209
|
+
private readonly opts: FanoutOptions;
|
|
210
210
|
private readonly coordDoId: string;
|
|
211
211
|
private readonly coordDoIdShort: string;
|
|
212
212
|
|
|
213
|
-
constructor(rawEnv: unknown, ctx: DurableObjectState, opts:
|
|
213
|
+
constructor(rawEnv: unknown, ctx: DurableObjectState, opts: FanoutOptions) {
|
|
214
214
|
// A host hands its whole env over; the bindings are claimed here and the
|
|
215
215
|
// LOADER claim is checked immediately. Hard-fail on a missing LOADER —
|
|
216
|
-
//
|
|
217
|
-
// diagnostic at the fanout
|
|
218
|
-
// deferred
|
|
219
|
-
const env = (rawEnv as
|
|
216
|
+
// IsolatePool also enforces this, but checking up front points the
|
|
217
|
+
// diagnostic at the fanout construction site rather than the
|
|
218
|
+
// deferred isolate-pool one.
|
|
219
|
+
const env = (rawEnv as FanoutEnv | null | undefined) ?? {};
|
|
220
220
|
if (!env.LOADER || typeof env.LOADER.get !== 'function') {
|
|
221
221
|
throw new BindingError(
|
|
222
|
-
'
|
|
222
|
+
'Fanout: env.LOADER binding missing or invalid. ' +
|
|
223
223
|
'Add a [[worker_loaders]] entry to wrangler.jsonc.',
|
|
224
224
|
);
|
|
225
225
|
}
|
|
@@ -235,21 +235,21 @@ export class FanoutPool {
|
|
|
235
235
|
* results in input order.
|
|
236
236
|
*
|
|
237
237
|
* Routing:
|
|
238
|
-
* tasks.length < 5 -> coordinator-local
|
|
238
|
+
* tasks.length < 5 -> coordinator-local IsolatePool
|
|
239
239
|
* tasks.length >= 5 -> sibling NimbusSession DOs
|
|
240
240
|
*
|
|
241
241
|
* Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
|
|
242
242
|
* are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
|
|
243
243
|
* runs serially inside its assigned peer DO via the in-peer
|
|
244
|
-
*
|
|
244
|
+
* IsolatePool's concurrency (capped at 4 there too). A
|
|
245
245
|
* single submitMany call returns when ALL tasks complete (or any
|
|
246
246
|
* throws).
|
|
247
247
|
*
|
|
248
248
|
* `fn` is the user function executed per task. It runs INSIDE a
|
|
249
249
|
* Worker Loader isolate (in the in-DO path) or inside a peer DO's
|
|
250
250
|
* Worker Loader isolate (in the peer-DO path); same trust posture
|
|
251
|
-
* as
|
|
252
|
-
* the vendored serializeFunction (same as
|
|
251
|
+
* as IsolatePool.submit. The function is serialized via
|
|
252
|
+
* the vendored serializeFunction (same as IsolatePool#prepare).
|
|
253
253
|
*/
|
|
254
254
|
async submitMany<A, R>(
|
|
255
255
|
tasks: FanoutTask<A>[],
|
|
@@ -287,12 +287,12 @@ export class FanoutPool {
|
|
|
287
287
|
tasks: FanoutTask<A>[],
|
|
288
288
|
fn: FacetTaskFn<A, R>,
|
|
289
289
|
): Promise<R[]> {
|
|
290
|
-
// Use the existing
|
|
290
|
+
// Use the existing IsolatePool. Concurrency = task count
|
|
291
291
|
// (capped at 4 by constructor — tasks.length is already < 5
|
|
292
292
|
// here, so the cap won't bite). Each task = one pool.submit;
|
|
293
293
|
// pool.map runs them with stable-slot reuse.
|
|
294
294
|
const concurrency = Math.min(tasks.length, IN_DO_THRESHOLD - 1);
|
|
295
|
-
const pool = new
|
|
295
|
+
const pool = new IsolatePool(this.env, this.ctx, {
|
|
296
296
|
concurrency,
|
|
297
297
|
timeoutMs: this.opts.timeoutMs,
|
|
298
298
|
tag: this.opts.tag,
|
|
@@ -326,7 +326,7 @@ export class FanoutPool {
|
|
|
326
326
|
const ns = this.env?.NIMBUS_SESSION;
|
|
327
327
|
if (!ns || typeof ns.idFromName !== 'function' || typeof ns.get !== 'function') {
|
|
328
328
|
throw new BindingError(
|
|
329
|
-
'
|
|
329
|
+
'Fanout: env.NIMBUS_SESSION binding missing or invalid. ' +
|
|
330
330
|
'The peer-DO topology requires it. ' +
|
|
331
331
|
'Add the binding via durable_objects.bindings in wrangler.jsonc.',
|
|
332
332
|
);
|
|
@@ -340,7 +340,7 @@ export class FanoutPool {
|
|
|
340
340
|
|
|
341
341
|
// Cap peer count at MAX_PEER_FANOUT. Tasks beyond N=32 are
|
|
342
342
|
// bucketed into existing shards — each shard's peer DO then
|
|
343
|
-
// runs its bucket through its in-DO
|
|
343
|
+
// runs its bucket through its in-DO IsolatePool.map
|
|
344
344
|
// (concurrency capped at 4 there).
|
|
345
345
|
const peerCount = Math.min(tasks.length, this.opts.maxPeers ?? MAX_PEER_FANOUT);
|
|
346
346
|
// Group tasks by deterministic shard. Same key → same shard, so
|
|
@@ -397,7 +397,7 @@ export class FanoutPool {
|
|
|
397
397
|
extraBindings: this.opts.extraBindings,
|
|
398
398
|
omitSupervisor: this.opts.omitSupervisor,
|
|
399
399
|
// INSTALL-HONESTY: forward the COORDINATOR's full doId so
|
|
400
|
-
// the peer's
|
|
400
|
+
// the peer's IsolatePool can mint a SUPERVISOR
|
|
401
401
|
// binding that routes back HERE (the user's session DO),
|
|
402
402
|
// not to the peer DO itself. Without this, peer DOs'
|
|
403
403
|
// env.SUPERVISOR.writeBatch / writeBatchStream / stdout /
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* fenced-work.ts — durable record of the resident launches a Durable Object
|
|
3
3
|
* owes, and their recovery after an instance reset.
|
|
4
4
|
*
|
|
5
5
|
* The platform resets a session Durable Object over what one turn has
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* What a launch IS stays the embedder's: the journal stores the record it is
|
|
16
16
|
* given and hands it back on recovery. The mechanism reads only the fields in
|
|
17
|
-
* {@link
|
|
17
|
+
* {@link FencedWorkRecord}; everything else in the record rides through
|
|
18
18
|
* opaquely.
|
|
19
19
|
*/
|
|
20
20
|
|
|
@@ -37,10 +37,10 @@
|
|
|
37
37
|
* storage key is a migration, and orphaned rows are the least of what it
|
|
38
38
|
* breaks.
|
|
39
39
|
*/
|
|
40
|
-
export const
|
|
40
|
+
export const FENCED_WORK_KEY_PREFIX = 'resident-launch:';
|
|
41
41
|
|
|
42
42
|
/** A launch is re-driven once. A reset that recurs is not the transient one. */
|
|
43
|
-
export const
|
|
43
|
+
export const FENCED_WORK_MAX_ATTEMPT = 1;
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
46
|
* A resident process this session owes the user, as a later instance would
|
|
@@ -60,7 +60,7 @@ export const RESIDENT_LAUNCH_MAX_ATTEMPT = 1;
|
|
|
60
60
|
* process host's held-open leg dies with it), so a row from a previous
|
|
61
61
|
* generation always names a process that is genuinely gone.
|
|
62
62
|
*/
|
|
63
|
-
export interface
|
|
63
|
+
export interface FencedWorkRecord {
|
|
64
64
|
pid: number;
|
|
65
65
|
command: string;
|
|
66
66
|
/** 0 for a launch the user asked for; 1 for the one re-drive it may get. */
|
|
@@ -74,9 +74,9 @@ export interface ResidentLaunchRecord {
|
|
|
74
74
|
/**
|
|
75
75
|
* The slice of Durable Object storage the journal writes through. Exactly a
|
|
76
76
|
* `DurableObjectStorage`, narrowed to what the mechanism performs — `sync()`
|
|
77
|
-
* is load-bearing, see {@link
|
|
77
|
+
* is load-bearing, see {@link FencedWork.journal}.
|
|
78
78
|
*/
|
|
79
|
-
export interface
|
|
79
|
+
export interface FencedWorkStorage {
|
|
80
80
|
put(key: string, value: unknown): Promise<void>;
|
|
81
81
|
delete(key: string): Promise<boolean>;
|
|
82
82
|
list<T = unknown>(options: { prefix: string }): Promise<Map<string, T>>;
|
|
@@ -84,7 +84,7 @@ export interface LaunchJournalStorage {
|
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
/** What the journal's recovery needs from its embedder. */
|
|
87
|
-
export interface
|
|
87
|
+
export interface FencedWorkHost<R extends FencedWorkRecord> {
|
|
88
88
|
/**
|
|
89
89
|
* The current instance generation's pid floor. A pid at or below it was
|
|
90
90
|
* allocated by a PREVIOUS instance (core's process-table, PID_GEN_STRIDE),
|
|
@@ -94,8 +94,13 @@ export interface LaunchJournalHost<R extends ResidentLaunchRecord> {
|
|
|
94
94
|
*/
|
|
95
95
|
generationBase(): number;
|
|
96
96
|
/**
|
|
97
|
-
*
|
|
98
|
-
*
|
|
97
|
+
* Where an un-awaited re-drive goes so it is not a floating rejection —
|
|
98
|
+
* `ctx.waitUntil` in practice. Hygiene, not retention: a Durable Object
|
|
99
|
+
* cancels an in-flight promise on reset with no signal, and workerd's
|
|
100
|
+
* `waitUntil` is a no-op there (PLATFORM.md, probe 2026-08-17 — awaiting
|
|
101
|
+
* inside the invocation is the only retention an actor has). The recovery
|
|
102
|
+
* guarantee comes from the journal row and the platform's alarm
|
|
103
|
+
* re-delivery, never from this hook.
|
|
99
104
|
*/
|
|
100
105
|
waitUntil(promise: Promise<unknown>): void;
|
|
101
106
|
/**
|
|
@@ -121,7 +126,7 @@ export interface LaunchJournalHost<R extends ResidentLaunchRecord> {
|
|
|
121
126
|
* read the journal a reset leaves behind. Rows from a previous instance are
|
|
122
127
|
* recovery's to consume, never the release path's.
|
|
123
128
|
*/
|
|
124
|
-
export class
|
|
129
|
+
export class FencedWork<R extends FencedWorkRecord> {
|
|
125
130
|
/**
|
|
126
131
|
* Pids THIS instance holds journal rows for. What keeps the terminal hook —
|
|
127
132
|
* which fires for every process, shells and one-shots included — from
|
|
@@ -132,8 +137,8 @@ export class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
|
|
|
132
137
|
private recovered = false;
|
|
133
138
|
|
|
134
139
|
constructor(
|
|
135
|
-
private readonly storage:
|
|
136
|
-
private readonly host:
|
|
140
|
+
private readonly storage: FencedWorkStorage,
|
|
141
|
+
private readonly host: FencedWorkHost<R>,
|
|
137
142
|
) {}
|
|
138
143
|
|
|
139
144
|
/**
|
|
@@ -154,7 +159,7 @@ export class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
|
|
|
154
159
|
async journal(record: R): Promise<void> {
|
|
155
160
|
try {
|
|
156
161
|
this.journalledPids.add(record.pid);
|
|
157
|
-
await this.storage.put(`${
|
|
162
|
+
await this.storage.put(`${FENCED_WORK_KEY_PREFIX}${record.pid}`, record);
|
|
158
163
|
await this.storage.sync();
|
|
159
164
|
} catch (e: unknown) {
|
|
160
165
|
console.warn('[nimbus] resident launch journal write failed:', errorMessage(e));
|
|
@@ -174,7 +179,7 @@ export class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
|
|
|
174
179
|
async release(pid: number): Promise<void> {
|
|
175
180
|
if (!this.journalledPids.delete(pid)) return;
|
|
176
181
|
try {
|
|
177
|
-
await this.storage.delete(`${
|
|
182
|
+
await this.storage.delete(`${FENCED_WORK_KEY_PREFIX}${pid}`);
|
|
178
183
|
await this.storage.sync();
|
|
179
184
|
} catch (e: unknown) {
|
|
180
185
|
console.warn('[nimbus] resident launch journal delete failed:', errorMessage(e));
|
|
@@ -196,19 +201,34 @@ export class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
|
|
|
196
201
|
async recoverInterrupted(): Promise<void> {
|
|
197
202
|
if (this.recovered) return;
|
|
198
203
|
this.recovered = true;
|
|
199
|
-
const journal = await this.storage.list<R>({ prefix:
|
|
204
|
+
const journal = await this.storage.list<R>({ prefix: FENCED_WORK_KEY_PREFIX });
|
|
200
205
|
const base = this.host.generationBase();
|
|
206
|
+
const abandoned: Array<[string, R]> = [];
|
|
207
|
+
const redriven: Array<[string, R]> = [];
|
|
201
208
|
for (const [key, record] of journal) {
|
|
202
209
|
// A pid at or below this instance's base was allocated by a PREVIOUS one
|
|
203
210
|
// (process-table.ts, PID_GEN_STRIDE), so its launch never finished; above
|
|
204
211
|
// the base is this instance's own, still running. Same predicate as
|
|
205
212
|
// `session/rpc.ts` uses to attribute a prior generation's pid.
|
|
206
213
|
if (!(record.pid > 0 && record.pid <= base)) continue;
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
214
|
+
(record.attempt >= FENCED_WORK_MAX_ATTEMPT ? abandoned : redriven).push([key, record]);
|
|
215
|
+
}
|
|
216
|
+
// The attempt is SPENT in storage, synced, before any re-drive starts.
|
|
217
|
+
// Deleting the row here instead would open a loss window: writes flush in
|
|
218
|
+
// order, so a reset between the delete's flush and the re-driven launch's
|
|
219
|
+
// own journal write leaves a durable state with no row for an owed
|
|
220
|
+
// launch — and the un-awaited re-drive dies with the instance (waitUntil
|
|
221
|
+
// retains nothing). With the rewrite, every durable cut is either the
|
|
222
|
+
// untouched row (recovery re-runs) or a spent attempt (the recurrence
|
|
223
|
+
// abandons loudly). The superseded row is deleted only when its re-drive
|
|
224
|
+
// settles, after the launch's own row exists.
|
|
225
|
+
for (const [key] of abandoned) await this.storage.delete(key);
|
|
226
|
+
for (const [key, record] of redriven) {
|
|
227
|
+
await this.storage.put(key, { ...record, attempt: record.attempt + 1 });
|
|
228
|
+
}
|
|
229
|
+
if (abandoned.length > 0 || redriven.length > 0) await this.storage.sync();
|
|
230
|
+
for (const [, record] of abandoned) this.host.onAbandoned?.(record);
|
|
231
|
+
for (const [key, record] of redriven) {
|
|
212
232
|
this.host.onRedrive?.(record);
|
|
213
233
|
// Not awaited: this call is running inside the alarm that granted the
|
|
214
234
|
// turn, and the launch it starts asks for turns of its own through that
|
|
@@ -218,10 +238,26 @@ export class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
|
|
|
218
238
|
this.host.redrive(record, record.attempt + 1)
|
|
219
239
|
.catch((e: unknown) => {
|
|
220
240
|
this.host.onRedriveFailed?.(record, e);
|
|
221
|
-
})
|
|
241
|
+
})
|
|
242
|
+
.then(() => this.supersede(key)),
|
|
222
243
|
);
|
|
223
244
|
}
|
|
224
245
|
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Delete a row whose re-drive has settled — succeeded, failed and been
|
|
249
|
+
* reported, or handed off to its own journal row. Not `release()`: that
|
|
250
|
+
* path is for pids THIS instance journalled, and this row's pid belongs to
|
|
251
|
+
* a previous generation the terminal hook will never fire for.
|
|
252
|
+
*/
|
|
253
|
+
private async supersede(key: string): Promise<void> {
|
|
254
|
+
try {
|
|
255
|
+
await this.storage.delete(key);
|
|
256
|
+
await this.storage.sync();
|
|
257
|
+
} catch (e: unknown) {
|
|
258
|
+
console.warn('[nimbus] resident launch journal supersede failed:', errorMessage(e));
|
|
259
|
+
}
|
|
260
|
+
}
|
|
225
261
|
}
|
|
226
262
|
|
|
227
263
|
function errorMessage(error: unknown): string {
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* generation.ts — the isolate-generation clock, and the deferred
|
|
3
|
+
* reconciliation that runs once per fresh incarnation.
|
|
4
|
+
*
|
|
5
|
+
* Workerd hibernates Durable Objects between requests to free memory. On
|
|
6
|
+
* wake, the new isolate must rebuild its in-memory state from SQL — but it
|
|
7
|
+
* also needs to know "is this the same lifecycle as before, or did workerd
|
|
8
|
+
* recycle me?" That distinction matters for recovery (warmJoin vs cold init)
|
|
9
|
+
* and is captured by the isolate generation, a counter persisted across
|
|
10
|
+
* hibernations.
|
|
11
|
+
*
|
|
12
|
+
* State is keyed weakly off the actor's `ctx`, which lives exactly as long as
|
|
13
|
+
* the incarnation: a fresh isolate gets a fresh `ctx`, adopts the persisted
|
|
14
|
+
* counter, and bumps it once. The embedder reads `generation(ctx)` wherever it
|
|
15
|
+
* needs the incarnation number — one source of truth instead of mirrored host
|
|
16
|
+
* fields.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { errorText } from '@nimbus-sh/core/_shared/error-text.js';
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Storage key for the isolate-generation counter (cold-start +
|
|
23
|
+
* post-hibernation wake; one increment per fresh isolate).
|
|
24
|
+
*
|
|
25
|
+
* The VALUE is live production DO storage ('w9_isolate_gen') and must never
|
|
26
|
+
* change — renaming a storage key is a migration, and orphaned rows are the
|
|
27
|
+
* least of what it breaks.
|
|
28
|
+
*/
|
|
29
|
+
export const GENERATION_KEY = 'w9_isolate_gen';
|
|
30
|
+
|
|
31
|
+
/** The storage the generation counter persists through. */
|
|
32
|
+
export interface GenerationStorage {
|
|
33
|
+
get(key: string): Promise<unknown>;
|
|
34
|
+
put(key: string, value: unknown): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The hosting actor's context, as the generation clock reads it. */
|
|
38
|
+
export interface GenerationContext {
|
|
39
|
+
storage: GenerationStorage;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
interface GenerationState {
|
|
43
|
+
value: number;
|
|
44
|
+
adopted: boolean;
|
|
45
|
+
/** Deferred reconciliation tasks, drained by {@link runColdStart}. */
|
|
46
|
+
coldStart: Array<() => Promise<unknown>>;
|
|
47
|
+
/** Serializes drains so two callers never run one task twice. */
|
|
48
|
+
coldStartChain: Promise<void>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const states = new WeakMap<object, GenerationState>();
|
|
52
|
+
|
|
53
|
+
function stateOf(ctx: object): GenerationState {
|
|
54
|
+
let state = states.get(ctx);
|
|
55
|
+
if (!state) {
|
|
56
|
+
state = { value: 0, adopted: false, coldStart: [], coldStartChain: Promise.resolve() };
|
|
57
|
+
states.set(ctx, state);
|
|
58
|
+
}
|
|
59
|
+
return state;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** This incarnation's generation. Zero until {@link adoptGeneration} ran. */
|
|
63
|
+
export function generation(ctx: object): number {
|
|
64
|
+
return states.get(ctx)?.value ?? 0;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Increment + persist the generation counter once per fresh isolate. */
|
|
68
|
+
export async function adoptGeneration(ctx: GenerationContext): Promise<void> {
|
|
69
|
+
const state = stateOf(ctx);
|
|
70
|
+
if (state.adopted) return;
|
|
71
|
+
state.adopted = true;
|
|
72
|
+
try {
|
|
73
|
+
const prev = (await ctx.storage.get(GENERATION_KEY)) as number | undefined;
|
|
74
|
+
// Adopt the persisted truth first, and adopt the bump only after the
|
|
75
|
+
// put resolves. An unpersisted `next` would be re-read as `prev` by the
|
|
76
|
+
// NEXT boot and re-issued — two instances sharing one generation is
|
|
77
|
+
// exactly the pid-aliasing this counter exists to prevent. Running on
|
|
78
|
+
// the previous persisted generation is the lesser lapse, and the
|
|
79
|
+
// put-failure case is replica-only in practice (replicas never spawn).
|
|
80
|
+
//
|
|
81
|
+
// What holds the guarantee is the output gate, not this await: measured,
|
|
82
|
+
// the block body resolves in 0 ms even with a confirmed put, because
|
|
83
|
+
// `await storage.put()` returns before durability. The gate is what
|
|
84
|
+
// keeps a pid from generation N from escaping before N is durable, which
|
|
85
|
+
// is why marking this put `allowUnconfirmed` is not a free speedup — see
|
|
86
|
+
// scratchpad/coldstart-s1.md.
|
|
87
|
+
state.value = typeof prev === 'number' ? prev : 0;
|
|
88
|
+
const next = state.value + 1;
|
|
89
|
+
await ctx.storage.put(GENERATION_KEY, next);
|
|
90
|
+
state.value = next;
|
|
91
|
+
} catch (e) {
|
|
92
|
+
console.warn('[nimbus/W9] generation bump failed:', errorText(e));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Take on a generation without persisting it, and clear the adopted guard so
|
|
98
|
+
* a later {@link adoptGeneration} re-derives from storage. The
|
|
99
|
+
* destroy-and-recreate path uses this: it wipes storage, re-persists the
|
|
100
|
+
* pre-destroy counter, and runs the rest of this incarnation on the successor
|
|
101
|
+
* generation the next boot will derive.
|
|
102
|
+
*/
|
|
103
|
+
export function assumeGeneration(ctx: object, value: number): void {
|
|
104
|
+
const state = stateOf(ctx);
|
|
105
|
+
state.value = value;
|
|
106
|
+
state.adopted = false;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Queue deferred async reconciliation for this incarnation.
|
|
111
|
+
*
|
|
112
|
+
* The task runs on the first {@link runColdStart} after registration — a turn
|
|
113
|
+
* the embedder already owns, NEVER the constructor's init gate. Awaiting
|
|
114
|
+
* recovery on the gate path is the trap this helper exists to avoid: the gate
|
|
115
|
+
* blocks every request to the object, and reconciliation wants a filesystem
|
|
116
|
+
* and a terminal that only a later turn has. The gate is also a wall, not
|
|
117
|
+
* just a stall: a `blockConcurrencyWhile` callback still pending at ~30 s
|
|
118
|
+
* (BLOCK_CONCURRENCY_CANCEL_MS, proven by probe) is cancelled and RESETS the
|
|
119
|
+
* object with every queued event — so never call {@link runColdStart} from
|
|
120
|
+
* inside one; the pump and the embedder's own turns are the places it runs.
|
|
121
|
+
*/
|
|
122
|
+
export function onColdStart(ctx: object, task: () => Promise<unknown>): void {
|
|
123
|
+
stateOf(ctx).coldStart.push(task);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Drain the queued cold-start tasks, serialized, each awaited so the turn
|
|
128
|
+
* that runs them pays for them. Idempotent between registrations: a drained
|
|
129
|
+
* queue is a cheap no-op, and a task registered after a drain runs on the
|
|
130
|
+
* next call.
|
|
131
|
+
*/
|
|
132
|
+
export function runColdStart(ctx: object): Promise<void> {
|
|
133
|
+
const state = stateOf(ctx);
|
|
134
|
+
if (state.coldStart.length === 0) return state.coldStartChain;
|
|
135
|
+
const tasks = state.coldStart;
|
|
136
|
+
state.coldStart = [];
|
|
137
|
+
const chained = state.coldStartChain.then(async () => {
|
|
138
|
+
for (const task of tasks) {
|
|
139
|
+
await task();
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
state.coldStartChain = chained.catch(() => {});
|
|
143
|
+
return chained;
|
|
144
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* image-store.ts — materializing resident-process boot images into the
|
|
3
3
|
* content-addressed image store, and sweeping the ones nothing boots from.
|
|
4
4
|
*
|
|
5
5
|
* A resident process's module map is sized by the user's disk, so it does not
|
|
@@ -12,14 +12,14 @@
|
|
|
12
12
|
* table.
|
|
13
13
|
*
|
|
14
14
|
* The filesystem itself stays the embedder's, reached through the
|
|
15
|
-
* {@link
|
|
15
|
+
* {@link ImageBlobStore} port — the store decides what is written where
|
|
16
16
|
* and when; the port decides how bytes land on a disk and with what modes and
|
|
17
17
|
* credentials.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
-
import { MAX_TX_BLOB_BYTES, CHUNK_SIZE } from '@nimbus-sh/
|
|
20
|
+
import { MAX_TX_BLOB_BYTES, CHUNK_SIZE } from '@nimbus-sh/platform/limits.js';
|
|
21
21
|
import { FACET_IMAGE_DIR, facetImageDigest, facetImagePath } from './process-fabric.js';
|
|
22
|
-
import type {
|
|
22
|
+
import type { TurnBudget } from './turn-budget.js';
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
25
|
* Bytes of an image written in one storage transaction.
|
|
@@ -39,7 +39,7 @@ export const FACET_IMAGE_WRITE_SLICE_BYTES = Math.floor(MAX_TX_BLOB_BYTES / CHUN
|
|
|
39
39
|
* normalization are the implementation's: the store passes the same
|
|
40
40
|
* store-relative paths it later roots and sweeps by.
|
|
41
41
|
*/
|
|
42
|
-
export interface
|
|
42
|
+
export interface ImageBlobStore {
|
|
43
43
|
/** Create a directory (and its parents) if it does not exist. */
|
|
44
44
|
mkdirp(dir: string): void;
|
|
45
45
|
/** The file's current size in bytes, or null when it does not exist. */
|
|
@@ -62,7 +62,7 @@ export interface FacetImageBlobStore {
|
|
|
62
62
|
* hash's problem; everything else is idempotent — an image already present
|
|
63
63
|
* at its own digest is already the bytes we were about to write.
|
|
64
64
|
*/
|
|
65
|
-
export class
|
|
65
|
+
export class ImageStore {
|
|
66
66
|
/** pid → the boot images its facet loads from; the image sweep's root set. */
|
|
67
67
|
private residentImages = new Map<number, string[]>();
|
|
68
68
|
private dirReady = false;
|
|
@@ -75,7 +75,7 @@ export class FacetImageStore {
|
|
|
75
75
|
* is the process table, reached through this one predicate.
|
|
76
76
|
*/
|
|
77
77
|
constructor(
|
|
78
|
-
private readonly blobs: () =>
|
|
78
|
+
private readonly blobs: () => ImageBlobStore,
|
|
79
79
|
private readonly isLive: (pid: number) => boolean,
|
|
80
80
|
) {}
|
|
81
81
|
|
|
@@ -113,7 +113,7 @@ export class FacetImageStore {
|
|
|
113
113
|
async materialize(
|
|
114
114
|
pid: number,
|
|
115
115
|
modules: Record<string, string>,
|
|
116
|
-
pacer:
|
|
116
|
+
pacer: TurnBudget,
|
|
117
117
|
): Promise<Record<string, string>> {
|
|
118
118
|
const fs = this.blobs();
|
|
119
119
|
const images: Record<string, string> = {};
|
|
@@ -177,7 +177,7 @@ export class FacetImageStore {
|
|
|
177
177
|
* Nothing is left for a TTL or an eviction heuristic to guess at, and after
|
|
178
178
|
* a DO reset the table is empty so every orphan goes.
|
|
179
179
|
*/
|
|
180
|
-
private sweep(fs:
|
|
180
|
+
private sweep(fs: ImageBlobStore): void {
|
|
181
181
|
const live = new Set<string>();
|
|
182
182
|
for (const [pid, paths] of this.residentImages) {
|
|
183
183
|
if (this.isLive(pid)) {
|
package/src/index.ts
CHANGED
|
@@ -7,16 +7,24 @@
|
|
|
7
7
|
* modules they need instead.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
export * from './
|
|
10
|
+
export * from './generation.js';
|
|
11
|
+
export * from './timers.js';
|
|
12
|
+
export * from './outbox.js';
|
|
13
|
+
export * from './journal.js';
|
|
14
|
+
export * from './do-calls.js';
|
|
15
|
+
export * from './facet-pool.js';
|
|
16
|
+
export * from './derived.js';
|
|
17
|
+
export * from './connections.js';
|
|
18
|
+
export * from './sealed.js';
|
|
11
19
|
export * from './bindings.js';
|
|
12
|
-
export * from './
|
|
13
|
-
export * from './
|
|
14
|
-
export * from './fanout
|
|
20
|
+
export * from './composition.js';
|
|
21
|
+
export * from './image-store.js';
|
|
22
|
+
export * from './fanout.js';
|
|
15
23
|
export * from './inner-do-registry.js';
|
|
16
|
-
export * from './
|
|
17
|
-
export * from './
|
|
18
|
-
export * from './
|
|
19
|
-
export * from './
|
|
24
|
+
export * from './fenced-work.js';
|
|
25
|
+
export * from './turn-budget.js';
|
|
26
|
+
export * from './budgets.js';
|
|
27
|
+
export * from './isolate-pool.js';
|
|
20
28
|
export * from './process-fabric.js';
|
|
21
29
|
export * from './process-host.js';
|
|
22
30
|
export * from './workerd-facet-host.js';
|