@nimbus-sh/fabric 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +208 -293
- package/dist/bindings.js +5 -5
- package/dist/budgets.d.ts +132 -0
- package/dist/budgets.d.ts.map +1 -0
- package/dist/budgets.js +248 -0
- package/dist/composition.d.ts +3 -0
- package/dist/composition.d.ts.map +1 -0
- package/dist/composition.js +2 -0
- package/dist/connections.d.ts +81 -0
- package/dist/connections.d.ts.map +1 -0
- package/dist/connections.js +114 -0
- package/dist/derived.d.ts +65 -0
- package/dist/derived.d.ts.map +1 -0
- package/dist/derived.js +95 -0
- package/dist/do-calls.d.ts +94 -0
- package/dist/do-calls.d.ts.map +1 -0
- package/dist/do-calls.js +111 -0
- package/dist/facet-pool.d.ts +90 -0
- package/dist/facet-pool.d.ts.map +1 -0
- package/dist/facet-pool.js +113 -0
- package/dist/{fanout-pool.d.ts → fanout.d.ts} +20 -20
- package/dist/fanout.d.ts.map +1 -0
- package/dist/{fanout-pool.js → fanout.js} +20 -20
- package/dist/{launch-journal.d.ts → fenced-work.d.ts} +58 -17
- package/dist/fenced-work.d.ts.map +1 -0
- package/dist/fenced-work.js +241 -0
- package/dist/generation.d.ts +69 -0
- package/dist/generation.d.ts.map +1 -0
- package/dist/generation.js +118 -0
- package/dist/{facet-image-store.d.ts → image-store.d.ts} +8 -8
- package/dist/image-store.d.ts.map +1 -0
- package/dist/{facet-image-store.js → image-store.js} +4 -4
- package/dist/index.d.ts +16 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +16 -8
- package/dist/{loader-pool.d.ts → isolate-pool.d.ts} +19 -19
- package/dist/isolate-pool.d.ts.map +1 -0
- package/dist/{loader-pool.js → isolate-pool.js} +20 -20
- package/dist/journal.d.ts +111 -0
- package/dist/journal.d.ts.map +1 -0
- package/dist/journal.js +177 -0
- package/dist/outbox.d.ts +249 -0
- package/dist/outbox.d.ts.map +1 -0
- package/dist/outbox.js +355 -0
- package/dist/process-fabric.d.ts +33 -15
- package/dist/process-fabric.d.ts.map +1 -1
- package/dist/process-fabric.js +25 -15
- package/dist/process-host.d.ts +1 -1
- package/dist/process-host.d.ts.map +1 -1
- package/dist/process-host.js +19 -11
- package/dist/sealed.d.ts +78 -0
- package/dist/sealed.d.ts.map +1 -0
- package/dist/sealed.js +145 -0
- package/dist/timers.d.ts +138 -0
- package/dist/timers.d.ts.map +1 -0
- package/dist/timers.js +231 -0
- package/dist/{launch-pacer.d.ts → turn-budget.d.ts} +24 -21
- package/dist/turn-budget.d.ts.map +1 -0
- package/dist/{launch-pacer.js → turn-budget.js} +24 -12
- package/dist/workerd-facet-host.d.ts +67 -70
- package/dist/workerd-facet-host.d.ts.map +1 -1
- package/dist/workerd-facet-host.js +129 -181
- package/examples/agent-core-adapter.ts +191 -0
- package/package.json +4 -2
- package/src/bindings.ts +6 -6
- package/src/budgets.ts +308 -0
- package/src/composition.ts +16 -0
- package/src/connections.ts +140 -0
- package/src/derived.ts +135 -0
- package/src/do-calls.ts +156 -0
- package/src/facet-pool.ts +157 -0
- package/src/{fanout-pool.ts → fanout.ts} +35 -35
- package/src/{launch-journal.ts → fenced-work.ts} +129 -42
- package/src/generation.ts +144 -0
- package/src/{facet-image-store.ts → image-store.ts} +9 -9
- package/src/index.ts +16 -8
- package/src/{loader-pool.ts → isolate-pool.ts} +34 -34
- package/src/journal.ts +242 -0
- package/src/node-async-hooks.d.ts +14 -0
- package/src/outbox.ts +520 -0
- package/src/process-fabric.ts +43 -34
- package/src/process-host.ts +22 -20
- package/src/sealed.ts +150 -0
- package/src/timers.ts +294 -0
- package/src/{launch-pacer.ts → turn-budget.ts} +34 -27
- package/src/workerd-facet-host.ts +159 -208
- package/dist/alarms.d.ts +0 -134
- package/dist/alarms.d.ts.map +0 -1
- package/dist/alarms.js +0 -214
- package/dist/ctx-exports.d.ts +0 -47
- package/dist/ctx-exports.d.ts.map +0 -1
- package/dist/ctx-exports.js +0 -54
- package/dist/facet-image-store.d.ts.map +0 -1
- package/dist/fanout-pool.d.ts.map +0 -1
- package/dist/launch-journal.d.ts.map +0 -1
- package/dist/launch-journal.js +0 -154
- package/dist/launch-pacer.d.ts.map +0 -1
- package/dist/loader-ledger.d.ts +0 -57
- package/dist/loader-ledger.d.ts.map +0 -1
- package/dist/loader-ledger.js +0 -91
- package/dist/loader-pool.d.ts.map +0 -1
- package/src/alarms.ts +0 -275
- package/src/ctx-exports.ts +0 -77
- package/src/loader-ledger.ts +0 -112
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* A single Durable Object method can drive at most four concurrent
|
|
6
6
|
* Worker Loader fetches before extra dispatches serialize or fail. Small
|
|
7
|
-
* batches therefore run in the coordinator DO through
|
|
7
|
+
* batches therefore run in the coordinator DO through IsolatePool.
|
|
8
8
|
* Wider batches are sharded across sibling NimbusSession DOs, each of
|
|
9
9
|
* which owns its own four-loader budget.
|
|
10
10
|
*
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* missing LOADER or NIMBUS_SESSION bindings fail loudly so install and
|
|
14
14
|
* runtime operations do not appear successful after partial dispatch.
|
|
15
15
|
*/
|
|
16
|
-
import { type FacetTaskFn } from './
|
|
16
|
+
import { type FacetTaskFn } from './isolate-pool.js';
|
|
17
17
|
import type { WorkerLoader } from './vendor/types.js';
|
|
18
18
|
/**
|
|
19
19
|
* The sibling-session namespace the peer-DO topology routes through. Ids are
|
|
@@ -24,7 +24,7 @@ interface PeerSessionNamespace {
|
|
|
24
24
|
get(id: DurableObjectId): unknown;
|
|
25
25
|
}
|
|
26
26
|
/** The bindings a fan-out needs off the coordinator DO's env. */
|
|
27
|
-
export interface
|
|
27
|
+
export interface FanoutEnv {
|
|
28
28
|
LOADER?: WorkerLoader;
|
|
29
29
|
NIMBUS_SESSION?: PeerSessionNamespace;
|
|
30
30
|
}
|
|
@@ -90,45 +90,45 @@ export interface FanoutTask<A> {
|
|
|
90
90
|
/** Argument passed to the user fn. */
|
|
91
91
|
args: A;
|
|
92
92
|
}
|
|
93
|
-
/** Options handed to
|
|
94
|
-
export interface
|
|
93
|
+
/** Options handed to Fanout's constructor. */
|
|
94
|
+
export interface FanoutOptions {
|
|
95
95
|
/**
|
|
96
96
|
* Tag prepended to peer-DO ids and in-DO loader ids for debugging
|
|
97
97
|
* (e.g. "npm-install-batch"). Affects neither isolate identity (in-DO
|
|
98
|
-
* path uses the existing
|
|
98
|
+
* path uses the existing IsolatePool's tag-fold) nor peer-DO
|
|
99
99
|
* deterministic placement (peer ids fold tag + key).
|
|
100
100
|
*/
|
|
101
101
|
tag: string;
|
|
102
102
|
/**
|
|
103
103
|
* Per-task timeout in ms. Default 60_000. Forwarded to the in-DO
|
|
104
|
-
*
|
|
105
|
-
*
|
|
104
|
+
* IsolatePool's submit calls and to the peer-DO RPC's own
|
|
105
|
+
* IsolatePool.
|
|
106
106
|
*/
|
|
107
107
|
timeoutMs?: number;
|
|
108
108
|
/**
|
|
109
109
|
* Preamble bundled into every facet (in-DO and inside each peer
|
|
110
|
-
* DO). Same semantics as
|
|
110
|
+
* DO). Same semantics as IsolatePool's preamble option.
|
|
111
111
|
*/
|
|
112
112
|
preamble?: string;
|
|
113
113
|
/**
|
|
114
114
|
* Wasm modules forwarded to every facet. Same semantics as
|
|
115
|
-
*
|
|
115
|
+
* IsolatePool's wasmModules option.
|
|
116
116
|
*/
|
|
117
117
|
wasmModules?: Record<string, ArrayBuffer>;
|
|
118
118
|
/**
|
|
119
119
|
* Extra bindings forwarded to every facet. Same semantics as
|
|
120
|
-
*
|
|
120
|
+
* IsolatePool's extraBindings option.
|
|
121
121
|
*/
|
|
122
122
|
extraBindings?: Record<string, unknown>;
|
|
123
123
|
/**
|
|
124
124
|
* If set, skip the supervisor-RPC binding injection (mirrors
|
|
125
|
-
*
|
|
125
|
+
* IsolatePool's omitSupervisor flag).
|
|
126
126
|
*/
|
|
127
127
|
omitSupervisor?: boolean;
|
|
128
128
|
/**
|
|
129
129
|
* Invoking process pid, baked into each facet's SUPERVISOR binding so
|
|
130
130
|
* filesystem RPCs (writeBatchStream) are authorized under the caller's
|
|
131
|
-
* credential (mirrors
|
|
131
|
+
* credential (mirrors IsolatePool's supervisorPid). Threaded to both
|
|
132
132
|
* the in-DO loader pool and, via `_rpcFanoutExecute`, the peer-DO pools.
|
|
133
133
|
* npm install passes the shell command's `ctx.pid`; resolve leaves it 0.
|
|
134
134
|
*/
|
|
@@ -162,33 +162,33 @@ export interface FanoutPoolOptions {
|
|
|
162
162
|
* exists primarily as a clean API surface; per-call dispatch state
|
|
163
163
|
* lives only inside submitMany's promise.
|
|
164
164
|
*/
|
|
165
|
-
export declare class
|
|
165
|
+
export declare class Fanout {
|
|
166
166
|
private readonly env;
|
|
167
167
|
private readonly ctx;
|
|
168
168
|
private readonly opts;
|
|
169
169
|
private readonly coordDoId;
|
|
170
170
|
private readonly coordDoIdShort;
|
|
171
|
-
constructor(rawEnv: unknown, ctx: DurableObjectState, opts:
|
|
171
|
+
constructor(rawEnv: unknown, ctx: DurableObjectState, opts: FanoutOptions);
|
|
172
172
|
/**
|
|
173
173
|
* Dispatch `tasks` across the appropriate topology and return
|
|
174
174
|
* results in input order.
|
|
175
175
|
*
|
|
176
176
|
* Routing:
|
|
177
|
-
* tasks.length < 5 -> coordinator-local
|
|
177
|
+
* tasks.length < 5 -> coordinator-local IsolatePool
|
|
178
178
|
* tasks.length >= 5 -> sibling NimbusSession DOs
|
|
179
179
|
*
|
|
180
180
|
* Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
|
|
181
181
|
* are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
|
|
182
182
|
* runs serially inside its assigned peer DO via the in-peer
|
|
183
|
-
*
|
|
183
|
+
* IsolatePool's concurrency (capped at 4 there too). A
|
|
184
184
|
* single submitMany call returns when ALL tasks complete (or any
|
|
185
185
|
* throws).
|
|
186
186
|
*
|
|
187
187
|
* `fn` is the user function executed per task. It runs INSIDE a
|
|
188
188
|
* Worker Loader isolate (in the in-DO path) or inside a peer DO's
|
|
189
189
|
* Worker Loader isolate (in the peer-DO path); same trust posture
|
|
190
|
-
* as
|
|
191
|
-
* the vendored serializeFunction (same as
|
|
190
|
+
* as IsolatePool.submit. The function is serialized via
|
|
191
|
+
* the vendored serializeFunction (same as IsolatePool#prepare).
|
|
192
192
|
*/
|
|
193
193
|
submitMany<A, R>(tasks: FanoutTask<A>[], fn: FacetTaskFn<A, R>): Promise<R[]>;
|
|
194
194
|
/** Report which topology a task count uses without dispatching. */
|
|
@@ -220,4 +220,4 @@ export declare class FanoutPool {
|
|
|
220
220
|
*/
|
|
221
221
|
export declare function hashKeyToShard(key: string, peerCount: number): number;
|
|
222
222
|
export {};
|
|
223
|
-
//# sourceMappingURL=fanout
|
|
223
|
+
//# sourceMappingURL=fanout.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fanout.d.ts","sourceRoot":"","sources":["../src/fanout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGlE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD;;;GAGG;AACH,UAAU,oBAAoB;IAC5B,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC;IAC1C,GAAG,CAAC,EAAE,EAAE,eAAe,GAAG,OAAO,CAAC;CACnC;AAED,iEAAiE;AACjE,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC;;;GAGG;AACH,eAAO,MAAM,eAAe,KAAK,CAAC;AAElC;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAC9C,eAAO,MAAM,qBAAqB,UAAmB,CAAC;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,UAAqB,CAAC;AAE3D;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC,uCAAuC;AACvC,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B;;;OAGG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ,sCAAsC;IACtC,IAAI,EAAE,CAAC,CAAC;CACT;AAED,8CAA8C;AAC9C,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC1C;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AA4BD;;;;;;;;GAQG;AACH,qBAAa,MAAM;IACjB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAY;IAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAqB;IACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,aAAa;IAoBzE;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,CAAC,EAAE,CAAC,EACnB,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,EACtB,EAAE,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,EAAE,CAAC;IASf,mEAAmE;IACnE,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,OAAO;IAK7D;;;;;;OAMG;IACH,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;YAOvC,aAAa;YAoCb,eAAe;CA2J9B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAUrE"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* A single Durable Object method can drive at most four concurrent
|
|
6
6
|
* Worker Loader fetches before extra dispatches serialize or fail. Small
|
|
7
|
-
* batches therefore run in the coordinator DO through
|
|
7
|
+
* batches therefore run in the coordinator DO through IsolatePool.
|
|
8
8
|
* Wider batches are sharded across sibling NimbusSession DOs, each of
|
|
9
9
|
* which owns its own four-loader budget.
|
|
10
10
|
*
|
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { serializeFunction } from './vendor/serialize.js';
|
|
17
17
|
import { BindingError } from './vendor/errors.js';
|
|
18
|
-
import {
|
|
19
|
-
import { disposeRpcResource } from '@nimbus-sh/
|
|
20
|
-
import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/
|
|
18
|
+
import { IsolatePool } from './isolate-pool.js';
|
|
19
|
+
import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
|
|
20
|
+
import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/platform/oom-classify.js';
|
|
21
21
|
/**
|
|
22
22
|
* Threshold at which routing switches from coordinator-local loaders to
|
|
23
23
|
* sibling Durable Objects.
|
|
@@ -79,10 +79,10 @@ function isFanoutPeerStub(value) {
|
|
|
79
79
|
}
|
|
80
80
|
function fanoutPeerStub(value) {
|
|
81
81
|
if ((typeof value !== 'object' && typeof value !== 'function') || value === null) {
|
|
82
|
-
throw new BindingError('
|
|
82
|
+
throw new BindingError('Fanout: NIMBUS_SESSION.get() did not return a peer stub.');
|
|
83
83
|
}
|
|
84
84
|
if (!isFanoutPeerStub(value)) {
|
|
85
|
-
throw new BindingError('
|
|
85
|
+
throw new BindingError('Fanout: peer stub does not expose _rpcFanoutExecute().');
|
|
86
86
|
}
|
|
87
87
|
return value;
|
|
88
88
|
}
|
|
@@ -95,7 +95,7 @@ function fanoutPeerStub(value) {
|
|
|
95
95
|
* exists primarily as a clean API surface; per-call dispatch state
|
|
96
96
|
* lives only inside submitMany's promise.
|
|
97
97
|
*/
|
|
98
|
-
export class
|
|
98
|
+
export class Fanout {
|
|
99
99
|
env;
|
|
100
100
|
ctx;
|
|
101
101
|
opts;
|
|
@@ -104,12 +104,12 @@ export class FanoutPool {
|
|
|
104
104
|
constructor(rawEnv, ctx, opts) {
|
|
105
105
|
// A host hands its whole env over; the bindings are claimed here and the
|
|
106
106
|
// LOADER claim is checked immediately. Hard-fail on a missing LOADER —
|
|
107
|
-
//
|
|
108
|
-
// diagnostic at the fanout
|
|
109
|
-
// deferred
|
|
107
|
+
// IsolatePool also enforces this, but checking up front points the
|
|
108
|
+
// diagnostic at the fanout construction site rather than the
|
|
109
|
+
// deferred isolate-pool one.
|
|
110
110
|
const env = rawEnv ?? {};
|
|
111
111
|
if (!env.LOADER || typeof env.LOADER.get !== 'function') {
|
|
112
|
-
throw new BindingError('
|
|
112
|
+
throw new BindingError('Fanout: env.LOADER binding missing or invalid. ' +
|
|
113
113
|
'Add a [[worker_loaders]] entry to wrangler.jsonc.');
|
|
114
114
|
}
|
|
115
115
|
this.env = env;
|
|
@@ -123,21 +123,21 @@ export class FanoutPool {
|
|
|
123
123
|
* results in input order.
|
|
124
124
|
*
|
|
125
125
|
* Routing:
|
|
126
|
-
* tasks.length < 5 -> coordinator-local
|
|
126
|
+
* tasks.length < 5 -> coordinator-local IsolatePool
|
|
127
127
|
* tasks.length >= 5 -> sibling NimbusSession DOs
|
|
128
128
|
*
|
|
129
129
|
* Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
|
|
130
130
|
* are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
|
|
131
131
|
* runs serially inside its assigned peer DO via the in-peer
|
|
132
|
-
*
|
|
132
|
+
* IsolatePool's concurrency (capped at 4 there too). A
|
|
133
133
|
* single submitMany call returns when ALL tasks complete (or any
|
|
134
134
|
* throws).
|
|
135
135
|
*
|
|
136
136
|
* `fn` is the user function executed per task. It runs INSIDE a
|
|
137
137
|
* Worker Loader isolate (in the in-DO path) or inside a peer DO's
|
|
138
138
|
* Worker Loader isolate (in the peer-DO path); same trust posture
|
|
139
|
-
* as
|
|
140
|
-
* the vendored serializeFunction (same as
|
|
139
|
+
* as IsolatePool.submit. The function is serialized via
|
|
140
|
+
* the vendored serializeFunction (same as IsolatePool#prepare).
|
|
141
141
|
*/
|
|
142
142
|
async submitMany(tasks, fn) {
|
|
143
143
|
if (tasks.length === 0)
|
|
@@ -166,12 +166,12 @@ export class FanoutPool {
|
|
|
166
166
|
}
|
|
167
167
|
// ── Private: in-DO dispatch (in-DO fanout) ──────────────────────────────
|
|
168
168
|
async _dispatchInDo(tasks, fn) {
|
|
169
|
-
// Use the existing
|
|
169
|
+
// Use the existing IsolatePool. Concurrency = task count
|
|
170
170
|
// (capped at 4 by constructor — tasks.length is already < 5
|
|
171
171
|
// here, so the cap won't bite). Each task = one pool.submit;
|
|
172
172
|
// pool.map runs them with stable-slot reuse.
|
|
173
173
|
const concurrency = Math.min(tasks.length, IN_DO_THRESHOLD - 1);
|
|
174
|
-
const pool = new
|
|
174
|
+
const pool = new IsolatePool(this.env, this.ctx, {
|
|
175
175
|
concurrency,
|
|
176
176
|
timeoutMs: this.opts.timeoutMs,
|
|
177
177
|
tag: this.opts.tag,
|
|
@@ -203,7 +203,7 @@ export class FanoutPool {
|
|
|
203
203
|
async _dispatchPeerDo(tasks, fn) {
|
|
204
204
|
const ns = this.env?.NIMBUS_SESSION;
|
|
205
205
|
if (!ns || typeof ns.idFromName !== 'function' || typeof ns.get !== 'function') {
|
|
206
|
-
throw new BindingError('
|
|
206
|
+
throw new BindingError('Fanout: env.NIMBUS_SESSION binding missing or invalid. ' +
|
|
207
207
|
'The peer-DO topology requires it. ' +
|
|
208
208
|
'Add the binding via durable_objects.bindings in wrangler.jsonc.');
|
|
209
209
|
}
|
|
@@ -214,7 +214,7 @@ export class FanoutPool {
|
|
|
214
214
|
const fnSource = serializeFunction(fn);
|
|
215
215
|
// Cap peer count at MAX_PEER_FANOUT. Tasks beyond N=32 are
|
|
216
216
|
// bucketed into existing shards — each shard's peer DO then
|
|
217
|
-
// runs its bucket through its in-DO
|
|
217
|
+
// runs its bucket through its in-DO IsolatePool.map
|
|
218
218
|
// (concurrency capped at 4 there).
|
|
219
219
|
const peerCount = Math.min(tasks.length, this.opts.maxPeers ?? MAX_PEER_FANOUT);
|
|
220
220
|
// Group tasks by deterministic shard. Same key → same shard, so
|
|
@@ -266,7 +266,7 @@ export class FanoutPool {
|
|
|
266
266
|
extraBindings: this.opts.extraBindings,
|
|
267
267
|
omitSupervisor: this.opts.omitSupervisor,
|
|
268
268
|
// INSTALL-HONESTY: forward the COORDINATOR's full doId so
|
|
269
|
-
// the peer's
|
|
269
|
+
// the peer's IsolatePool can mint a SUPERVISOR
|
|
270
270
|
// binding that routes back HERE (the user's session DO),
|
|
271
271
|
// not to the peer DO itself. Without this, peer DOs'
|
|
272
272
|
// env.SUPERVISOR.writeBatch / writeBatchStream / stdout /
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* fenced-work.ts — durable record of the resident launches a Durable Object
|
|
3
3
|
* owes, and their recovery after an instance reset.
|
|
4
4
|
*
|
|
5
5
|
* The platform resets a session Durable Object over what one turn has
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* What a launch IS stays the embedder's: the journal stores the record it is
|
|
16
16
|
* given and hands it back on recovery. The mechanism reads only the fields in
|
|
17
|
-
* {@link
|
|
17
|
+
* {@link FencedWorkRecord}; everything else in the record rides through
|
|
18
18
|
* opaquely.
|
|
19
19
|
*/
|
|
20
20
|
/**
|
|
@@ -36,9 +36,9 @@
|
|
|
36
36
|
* storage key is a migration, and orphaned rows are the least of what it
|
|
37
37
|
* breaks.
|
|
38
38
|
*/
|
|
39
|
-
export declare const
|
|
39
|
+
export declare const FENCED_WORK_KEY_PREFIX = "resident-launch:";
|
|
40
40
|
/** A launch is re-driven once. A reset that recurs is not the transient one. */
|
|
41
|
-
export declare const
|
|
41
|
+
export declare const FENCED_WORK_MAX_ATTEMPT = 1;
|
|
42
42
|
/**
|
|
43
43
|
* A resident process this session owes the user, as a later instance would
|
|
44
44
|
* have to re-drive it.
|
|
@@ -57,7 +57,7 @@ export declare const RESIDENT_LAUNCH_MAX_ATTEMPT = 1;
|
|
|
57
57
|
* process host's held-open leg dies with it), so a row from a previous
|
|
58
58
|
* generation always names a process that is genuinely gone.
|
|
59
59
|
*/
|
|
60
|
-
export interface
|
|
60
|
+
export interface FencedWorkRecord {
|
|
61
61
|
pid: number;
|
|
62
62
|
command: string;
|
|
63
63
|
/** 0 for a launch the user asked for; 1 for the one re-drive it may get. */
|
|
@@ -70,9 +70,9 @@ export interface ResidentLaunchRecord {
|
|
|
70
70
|
/**
|
|
71
71
|
* The slice of Durable Object storage the journal writes through. Exactly a
|
|
72
72
|
* `DurableObjectStorage`, narrowed to what the mechanism performs — `sync()`
|
|
73
|
-
* is load-bearing, see {@link
|
|
73
|
+
* is load-bearing, see {@link FencedWork.journal}.
|
|
74
74
|
*/
|
|
75
|
-
export interface
|
|
75
|
+
export interface FencedWorkStorage {
|
|
76
76
|
put(key: string, value: unknown): Promise<void>;
|
|
77
77
|
delete(key: string): Promise<boolean>;
|
|
78
78
|
list<T = unknown>(options: {
|
|
@@ -81,7 +81,7 @@ export interface LaunchJournalStorage {
|
|
|
81
81
|
sync(): Promise<void>;
|
|
82
82
|
}
|
|
83
83
|
/** What the journal's recovery needs from its embedder. */
|
|
84
|
-
export interface
|
|
84
|
+
export interface FencedWorkHost<R extends FencedWorkRecord> {
|
|
85
85
|
/**
|
|
86
86
|
* The current instance generation's pid floor. A pid at or below it was
|
|
87
87
|
* allocated by a PREVIOUS instance (core's process-table, PID_GEN_STRIDE),
|
|
@@ -91,8 +91,13 @@ export interface LaunchJournalHost<R extends ResidentLaunchRecord> {
|
|
|
91
91
|
*/
|
|
92
92
|
generationBase(): number;
|
|
93
93
|
/**
|
|
94
|
-
*
|
|
95
|
-
*
|
|
94
|
+
* Where an un-awaited re-drive goes so it is not a floating rejection —
|
|
95
|
+
* `ctx.waitUntil` in practice. Hygiene, not retention: a Durable Object
|
|
96
|
+
* cancels an in-flight promise on reset with no signal, and workerd's
|
|
97
|
+
* `waitUntil` is a no-op there (PLATFORM.md, probe 2026-08-17 — awaiting
|
|
98
|
+
* inside the invocation is the only retention an actor has). The recovery
|
|
99
|
+
* guarantee comes from the journal row and the platform's alarm
|
|
100
|
+
* re-delivery, never from this hook.
|
|
96
101
|
*/
|
|
97
102
|
waitUntil(promise: Promise<unknown>): void;
|
|
98
103
|
/**
|
|
@@ -117,7 +122,7 @@ export interface LaunchJournalHost<R extends ResidentLaunchRecord> {
|
|
|
117
122
|
* read the journal a reset leaves behind. Rows from a previous instance are
|
|
118
123
|
* recovery's to consume, never the release path's.
|
|
119
124
|
*/
|
|
120
|
-
export declare class
|
|
125
|
+
export declare class FencedWork<R extends FencedWorkRecord> {
|
|
121
126
|
private readonly storage;
|
|
122
127
|
private readonly host;
|
|
123
128
|
/**
|
|
@@ -126,13 +131,16 @@ export declare class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
|
|
|
126
131
|
* paying a storage delete for pids that never had a row.
|
|
127
132
|
*/
|
|
128
133
|
private journalledPids;
|
|
134
|
+
/** Re-drives in flight, by journal key — recovery's un-awaited ones and a
|
|
135
|
+
* caller-driven one share the same drive for the same row. */
|
|
136
|
+
private drives;
|
|
129
137
|
/** Whether this instance has already read the journal a reset leaves behind. */
|
|
130
138
|
private recovered;
|
|
131
|
-
constructor(storage:
|
|
139
|
+
constructor(storage: FencedWorkStorage, host: FencedWorkHost<R>);
|
|
132
140
|
/**
|
|
133
141
|
* Record a launch as in flight, so an instance that replaces this one knows
|
|
134
|
-
* it never finished.
|
|
135
|
-
*
|
|
142
|
+
* it never finished. A launch that cannot be journalled does not start: the
|
|
143
|
+
* rejection reaches the caller, which reports it like any launch failure.
|
|
136
144
|
*
|
|
137
145
|
* Synced, not merely put: `await put()` resolves before durability, and the
|
|
138
146
|
* reset this journal exists for destroys every write its turn still had
|
|
@@ -153,6 +161,28 @@ export declare class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
|
|
|
153
161
|
* resurrect a process the user watched end.
|
|
154
162
|
*/
|
|
155
163
|
release(pid: number): Promise<void>;
|
|
164
|
+
/**
|
|
165
|
+
* Drop every row `predicate` claims, live-pid bookkeeping included, synced
|
|
166
|
+
* like {@link release}. The one bulk delete the journal admits: an owner
|
|
167
|
+
* that removes its durable application is owed no recovery, however many
|
|
168
|
+
* generations back its rows were written.
|
|
169
|
+
*/
|
|
170
|
+
purgeWhere(predicate: (record: R) => boolean): Promise<number>;
|
|
171
|
+
/**
|
|
172
|
+
* Every journal row, storage-true — the rows this instance wrote and the
|
|
173
|
+
* ones a previous instance left behind. The one read surface a request-
|
|
174
|
+
* driven recovery needs to find the durable launch a port belongs to.
|
|
175
|
+
*/
|
|
176
|
+
rows(): Promise<Map<string, R>>;
|
|
177
|
+
/**
|
|
178
|
+
* Re-drive one journal row — the awaited sibling of recovery's un-awaited
|
|
179
|
+
* re-drives, for a caller that must know whether the launch actually came
|
|
180
|
+
* back. Single-flight per row: a request-driven drive and recovery's own
|
|
181
|
+
* never boot the same launch twice. Resolves true only when the re-drive
|
|
182
|
+
* itself FAILED and the failure was reported; a settled drive supersedes
|
|
183
|
+
* the row the same way recovery's does.
|
|
184
|
+
*/
|
|
185
|
+
drive(key: string, record: R): Promise<boolean>;
|
|
156
186
|
/**
|
|
157
187
|
* Re-drive the launches a previous instance was building when it was reset.
|
|
158
188
|
*
|
|
@@ -162,9 +192,20 @@ export declare class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
|
|
|
162
192
|
* that replaces this one. So the first turn after a reset is already this
|
|
163
193
|
* one.
|
|
164
194
|
*
|
|
165
|
-
* Runs once per instance
|
|
166
|
-
*
|
|
195
|
+
* Runs once per instance — re-calls in the same instance are no-ops — and
|
|
196
|
+
* re-drives every row whose pid is `> 0` and at or below `generationBase()`
|
|
197
|
+
* with `attempt < FENCED_WORK_MAX_ATTEMPT`; the rest are abandoned. What the
|
|
198
|
+
* re-drive resolver receives is the journalled recipe and nothing else:
|
|
199
|
+
* env and credentials are never written to storage, so the resolver's
|
|
200
|
+
* embedder re-resolves them rather than reading them back.
|
|
167
201
|
*/
|
|
168
202
|
recoverInterrupted(): Promise<void>;
|
|
203
|
+
/**
|
|
204
|
+
* Delete a row whose re-drive has settled — succeeded, failed and been
|
|
205
|
+
* reported, or handed off to its own journal row. Not `release()`: that
|
|
206
|
+
* path is for pids THIS instance journalled, and this row's pid belongs to
|
|
207
|
+
* a previous generation the terminal hook will never fire for.
|
|
208
|
+
*/
|
|
209
|
+
private supersede;
|
|
169
210
|
}
|
|
170
|
-
//# sourceMappingURL=
|
|
211
|
+
//# sourceMappingURL=fenced-work.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fenced-work.d.ts","sourceRoot":"","sources":["../src/fenced-work.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,sBAAsB,qBAAqB,CAAC;AAEzD,gFAAgF;AAChF,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,4EAA4E;IAC5E,OAAO,EAAE,MAAM,CAAC;IAChB;;4DAEwD;IACxD,KAAK,EAAE,UAAU,GAAG,SAAS,CAAC;CAC/B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACxE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc,CAAC,CAAC,SAAS,gBAAgB;IACxD;;;;;;OAMG;IACH,cAAc,IAAI,MAAM,CAAC;IACzB;;;;;;;;OAQG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;IAC3C;;;;;OAKG;IACH,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtD,mDAAmD;IACnD,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC;IAC5B,yEAAyE;IACzE,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC;IAC9B,kCAAkC;IAClC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;CACnD;AAED;;;;;;;GAOG;AACH,qBAAa,UAAU,CAAC,CAAC,SAAS,gBAAgB;IAc9C,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAdvB;;;;OAIG;IACH,OAAO,CAAC,cAAc,CAAqB;IAC3C;mEAC+D;IAC/D,OAAO,CAAC,MAAM,CAAuC;IACrD,gFAAgF;IAChF,OAAO,CAAC,SAAS,CAAS;gBAGP,OAAO,EAAE,iBAAiB,EAC1B,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;IAG1C;;;;;;;;;;;;;;OAcG;IACG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAUvC,8DAA8D;IAC9D,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIzB;;;;OAIG;IACG,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMzC;;;;;OAKG;IACG,UAAU,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAapE;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAIrC;;;;;;;OAOG;IACH,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;IAwB/C;;;;;;;;;;;;;;;OAeG;IACG,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IA0CzC;;;;;OAKG;YACW,SAAS;CAIxB"}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* fenced-work.ts — durable record of the resident launches a Durable Object
|
|
3
|
+
* owes, and their recovery after an instance reset.
|
|
4
|
+
*
|
|
5
|
+
* The platform resets a session Durable Object over what one turn has
|
|
6
|
+
* outstanding in storage ("Internal error in Durable Object storage caused
|
|
7
|
+
* object to be reset"), and a resident launch is the largest writer a session
|
|
8
|
+
* has. Everything a launch holds is in memory, so the process it is building
|
|
9
|
+
* and the terminal watching it both go with the instance — the journal is what
|
|
10
|
+
* a LATER instance reads to know that happened, and this module is the whole
|
|
11
|
+
* of that mechanism: the put→sync durability barrier on the way in, the
|
|
12
|
+
* delete→sync release on the way out, and the once-per-instance recovery pump
|
|
13
|
+
* that re-drives what a previous generation left behind.
|
|
14
|
+
*
|
|
15
|
+
* What a launch IS stays the embedder's: the journal stores the record it is
|
|
16
|
+
* given and hands it back on recovery. The mechanism reads only the fields in
|
|
17
|
+
* {@link FencedWorkRecord}; everything else in the record rides through
|
|
18
|
+
* opaquely.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Prefix for the resident-process journal: one row per resident this session
|
|
22
|
+
* owes the user, keyed by the pid it was built for.
|
|
23
|
+
*
|
|
24
|
+
* A resident holds its state in memory — the process table entry, the facet
|
|
25
|
+
* handle, the terminal — so an instance reset destroys it silently. The row
|
|
26
|
+
* is what a LATER instance reads to know a resident ended that way rather
|
|
27
|
+
* than on purpose: a pid at or below the reader's own pid base was allocated
|
|
28
|
+
* by a previous generation (PID_GEN_STRIDE, core's process-table). Written
|
|
29
|
+
* (and synced) before the launch's first byte of work, rewritten as `running`
|
|
30
|
+
* when the launch settles, and released only when the PROCESS ends — because
|
|
31
|
+
* the resets this row survives strike after the launch as often as during it
|
|
32
|
+
* (measured live, staging 2026-08-13: every observed reset landed seconds
|
|
33
|
+
* AFTER settle).
|
|
34
|
+
*
|
|
35
|
+
* The VALUE is live production DO storage and must never change — renaming a
|
|
36
|
+
* storage key is a migration, and orphaned rows are the least of what it
|
|
37
|
+
* breaks.
|
|
38
|
+
*/
|
|
39
|
+
export const FENCED_WORK_KEY_PREFIX = 'resident-launch:';
|
|
40
|
+
/** A launch is re-driven once. A reset that recurs is not the transient one. */
|
|
41
|
+
export const FENCED_WORK_MAX_ATTEMPT = 1;
|
|
42
|
+
/**
|
|
43
|
+
* The resident-launch journal of one Durable Object instance.
|
|
44
|
+
*
|
|
45
|
+
* In-memory state here is per-instance on purpose: `journalledPids` tracks the
|
|
46
|
+
* rows THIS instance wrote, and `recovered` whether this instance has already
|
|
47
|
+
* read the journal a reset leaves behind. Rows from a previous instance are
|
|
48
|
+
* recovery's to consume, never the release path's.
|
|
49
|
+
*/
|
|
50
|
+
export class FencedWork {
|
|
51
|
+
storage;
|
|
52
|
+
host;
|
|
53
|
+
/**
|
|
54
|
+
* Pids THIS instance holds journal rows for. What keeps the terminal hook —
|
|
55
|
+
* which fires for every process, shells and one-shots included — from
|
|
56
|
+
* paying a storage delete for pids that never had a row.
|
|
57
|
+
*/
|
|
58
|
+
journalledPids = new Set();
|
|
59
|
+
/** Re-drives in flight, by journal key — recovery's un-awaited ones and a
|
|
60
|
+
* caller-driven one share the same drive for the same row. */
|
|
61
|
+
drives = new Map();
|
|
62
|
+
/** Whether this instance has already read the journal a reset leaves behind. */
|
|
63
|
+
recovered = false;
|
|
64
|
+
constructor(storage, host) {
|
|
65
|
+
this.storage = storage;
|
|
66
|
+
this.host = host;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Record a launch as in flight, so an instance that replaces this one knows
|
|
70
|
+
* it never finished. A launch that cannot be journalled does not start: the
|
|
71
|
+
* rejection reaches the caller, which reports it like any launch failure.
|
|
72
|
+
*
|
|
73
|
+
* Synced, not merely put: `await put()` resolves before durability, and the
|
|
74
|
+
* reset this journal exists for destroys every write its turn still had
|
|
75
|
+
* outstanding — measured live, a launch killed in its first chunks left NO
|
|
76
|
+
* row for the replacement instance to find, which is how the recovery this
|
|
77
|
+
* feeds sat inert while its own test stayed green. `sync()` is the storage
|
|
78
|
+
* layer's durability barrier: the row is on disk before the launch performs
|
|
79
|
+
* its first byte of real work. What remains is a reset between the put and
|
|
80
|
+
* the sync's completion — and a launch that dies there has not started, so
|
|
81
|
+
* losing its row costs a retype, not a recovery.
|
|
82
|
+
*/
|
|
83
|
+
async journal(record) {
|
|
84
|
+
this.journalledPids.add(record.pid);
|
|
85
|
+
try {
|
|
86
|
+
await this.storage.put(`${FENCED_WORK_KEY_PREFIX}${record.pid}`, record);
|
|
87
|
+
await this.storage.sync();
|
|
88
|
+
}
|
|
89
|
+
catch (cause) {
|
|
90
|
+
throw new Error('resident launch journal write failed', { cause });
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** True while this instance holds a journal row for `pid`. */
|
|
94
|
+
has(pid) {
|
|
95
|
+
return this.journalledPids.has(pid);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* The journal row's one release: the process is over, nothing is owed.
|
|
99
|
+
* Synced so an instance reset moments later cannot roll the delete back and
|
|
100
|
+
* resurrect a process the user watched end.
|
|
101
|
+
*/
|
|
102
|
+
async release(pid) {
|
|
103
|
+
if (!this.journalledPids.delete(pid))
|
|
104
|
+
return;
|
|
105
|
+
await this.storage.delete(`${FENCED_WORK_KEY_PREFIX}${pid}`);
|
|
106
|
+
await this.storage.sync();
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Drop every row `predicate` claims, live-pid bookkeeping included, synced
|
|
110
|
+
* like {@link release}. The one bulk delete the journal admits: an owner
|
|
111
|
+
* that removes its durable application is owed no recovery, however many
|
|
112
|
+
* generations back its rows were written.
|
|
113
|
+
*/
|
|
114
|
+
async purgeWhere(predicate) {
|
|
115
|
+
const rows = await this.storage.list({ prefix: FENCED_WORK_KEY_PREFIX });
|
|
116
|
+
let purged = 0;
|
|
117
|
+
for (const [key, record] of rows) {
|
|
118
|
+
if (!predicate(record))
|
|
119
|
+
continue;
|
|
120
|
+
this.journalledPids.delete(record.pid);
|
|
121
|
+
await this.storage.delete(key);
|
|
122
|
+
purged += 1;
|
|
123
|
+
}
|
|
124
|
+
if (purged > 0)
|
|
125
|
+
await this.storage.sync();
|
|
126
|
+
return purged;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Every journal row, storage-true — the rows this instance wrote and the
|
|
130
|
+
* ones a previous instance left behind. The one read surface a request-
|
|
131
|
+
* driven recovery needs to find the durable launch a port belongs to.
|
|
132
|
+
*/
|
|
133
|
+
async rows() {
|
|
134
|
+
return this.storage.list({ prefix: FENCED_WORK_KEY_PREFIX });
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Re-drive one journal row — the awaited sibling of recovery's un-awaited
|
|
138
|
+
* re-drives, for a caller that must know whether the launch actually came
|
|
139
|
+
* back. Single-flight per row: a request-driven drive and recovery's own
|
|
140
|
+
* never boot the same launch twice. Resolves true only when the re-drive
|
|
141
|
+
* itself FAILED and the failure was reported; a settled drive supersedes
|
|
142
|
+
* the row the same way recovery's does.
|
|
143
|
+
*/
|
|
144
|
+
drive(key, record) {
|
|
145
|
+
let inflight = this.drives.get(key);
|
|
146
|
+
if (inflight === undefined) {
|
|
147
|
+
inflight = (async () => {
|
|
148
|
+
let failed = false;
|
|
149
|
+
try {
|
|
150
|
+
await this.host.redrive(record, record.attempt + 1);
|
|
151
|
+
}
|
|
152
|
+
catch (e) {
|
|
153
|
+
this.host.onRedriveFailed?.(record, e);
|
|
154
|
+
failed = true;
|
|
155
|
+
}
|
|
156
|
+
// A reported failure is settled business — supersede either way, so
|
|
157
|
+
// the row never re-surfaces on the next recovery.
|
|
158
|
+
await this.supersede(key);
|
|
159
|
+
return failed;
|
|
160
|
+
})();
|
|
161
|
+
this.drives.set(key, inflight);
|
|
162
|
+
inflight.finally(() => {
|
|
163
|
+
if (this.drives.get(key) === inflight)
|
|
164
|
+
this.drives.delete(key);
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
return inflight;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Re-drive the launches a previous instance was building when it was reset.
|
|
171
|
+
*
|
|
172
|
+
* Sited on the launch-turn pump because the pump is what an alarm calls, and
|
|
173
|
+
* a launch that was suspended has an alarm armed for it — a reset during a
|
|
174
|
+
* chunk fails that alarm, and the platform re-delivers it to the instance
|
|
175
|
+
* that replaces this one. So the first turn after a reset is already this
|
|
176
|
+
* one.
|
|
177
|
+
*
|
|
178
|
+
* Runs once per instance — re-calls in the same instance are no-ops — and
|
|
179
|
+
* re-drives every row whose pid is `> 0` and at or below `generationBase()`
|
|
180
|
+
* with `attempt < FENCED_WORK_MAX_ATTEMPT`; the rest are abandoned. What the
|
|
181
|
+
* re-drive resolver receives is the journalled recipe and nothing else:
|
|
182
|
+
* env and credentials are never written to storage, so the resolver's
|
|
183
|
+
* embedder re-resolves them rather than reading them back.
|
|
184
|
+
*/
|
|
185
|
+
async recoverInterrupted() {
|
|
186
|
+
if (this.recovered)
|
|
187
|
+
return;
|
|
188
|
+
this.recovered = true;
|
|
189
|
+
const journal = await this.storage.list({ prefix: FENCED_WORK_KEY_PREFIX });
|
|
190
|
+
const base = this.host.generationBase();
|
|
191
|
+
const abandoned = [];
|
|
192
|
+
const redriven = [];
|
|
193
|
+
for (const [key, record] of journal) {
|
|
194
|
+
// A pid at or below this instance's base was allocated by a PREVIOUS one
|
|
195
|
+
// (process-table.ts, PID_GEN_STRIDE), so its launch never finished; above
|
|
196
|
+
// the base is this instance's own, still running. Same predicate as
|
|
197
|
+
// `session/rpc.ts` uses to attribute a prior generation's pid.
|
|
198
|
+
if (!(record.pid > 0 && record.pid <= base))
|
|
199
|
+
continue;
|
|
200
|
+
(record.attempt >= FENCED_WORK_MAX_ATTEMPT ? abandoned : redriven).push([key, record]);
|
|
201
|
+
}
|
|
202
|
+
// The attempt is SPENT in storage, synced, before any re-drive starts.
|
|
203
|
+
// Deleting the row here instead would open a loss window: writes flush in
|
|
204
|
+
// order, so a reset between the delete's flush and the re-driven launch's
|
|
205
|
+
// own journal write leaves a durable state with no row for an owed
|
|
206
|
+
// launch — and the un-awaited re-drive dies with the instance (waitUntil
|
|
207
|
+
// retains nothing). With the rewrite, every durable cut is either the
|
|
208
|
+
// untouched row (recovery re-runs) or a spent attempt (the recurrence
|
|
209
|
+
// abandons loudly). The superseded row is deleted only when its re-drive
|
|
210
|
+
// settles, after the launch's own row exists.
|
|
211
|
+
for (const [key] of abandoned)
|
|
212
|
+
await this.storage.delete(key);
|
|
213
|
+
for (const [key, record] of redriven) {
|
|
214
|
+
await this.storage.put(key, { ...record, attempt: record.attempt + 1 });
|
|
215
|
+
}
|
|
216
|
+
if (abandoned.length > 0 || redriven.length > 0)
|
|
217
|
+
await this.storage.sync();
|
|
218
|
+
for (const [, record] of abandoned)
|
|
219
|
+
this.host.onAbandoned?.(record);
|
|
220
|
+
for (const [key, record] of redriven) {
|
|
221
|
+
this.host.onRedrive?.(record);
|
|
222
|
+
// Not awaited: this call is running inside the alarm that granted the
|
|
223
|
+
// turn, and the launch it starts asks for turns of its own through that
|
|
224
|
+
// same alarm — awaiting it here would be waiting on an alarm that cannot
|
|
225
|
+
// be scheduled until this one returns. `drive` single-flights it: a
|
|
226
|
+
// request that arrives mid-launch waits on this same drive rather than
|
|
227
|
+
// booting a second process.
|
|
228
|
+
this.host.waitUntil(this.drive(key, record));
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Delete a row whose re-drive has settled — succeeded, failed and been
|
|
233
|
+
* reported, or handed off to its own journal row. Not `release()`: that
|
|
234
|
+
* path is for pids THIS instance journalled, and this row's pid belongs to
|
|
235
|
+
* a previous generation the terminal hook will never fire for.
|
|
236
|
+
*/
|
|
237
|
+
async supersede(key) {
|
|
238
|
+
await this.storage.delete(key);
|
|
239
|
+
await this.storage.sync();
|
|
240
|
+
}
|
|
241
|
+
}
|