@cortexkit/subc-client 0.16.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -0
- package/dist/client.d.ts +36 -8
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +269 -46
- package/dist/client.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/launch-nonce.d.ts +173 -0
- package/dist/launch-nonce.d.ts.map +1 -0
- package/dist/launch-nonce.js +334 -0
- package/dist/launch-nonce.js.map +1 -0
- package/dist/provider.d.ts +41 -4
- package/dist/provider.d.ts.map +1 -1
- package/dist/provider.js +69 -4
- package/dist/provider.js.map +1 -1
- package/package.json +1 -1
- package/src/client.ts +303 -52
- package/src/index.ts +13 -0
- package/src/launch-nonce.ts +447 -0
- package/src/provider.ts +109 -8
package/README.md
CHANGED
|
@@ -78,6 +78,40 @@ fails loud with an `AuthError` rather than connecting insecurely.
|
|
|
78
78
|
- **Connection-file security.** On unix the file must be owner-only (`0600`); a
|
|
79
79
|
group/world-readable file is rejected, because the key has effectively leaked.
|
|
80
80
|
|
|
81
|
+
### Launch nonce
|
|
82
|
+
|
|
83
|
+
A module the daemon spawns proves who it is with a launch nonce. On macOS and
|
|
84
|
+
Linux the daemon hands it over as a pipe at descriptor 3, named by
|
|
85
|
+
`SUBC_LAUNCH_NONCE_FD=3:<inode>`, and during the rollout also as
|
|
86
|
+
`SUBC_LAUNCH_NONCE` (which any same-user process can read with `ps eww`).
|
|
87
|
+
`launchNonce()` is the one reader: it takes the descriptor only when it is a
|
|
88
|
+
pipe with the named inode that holds bytes, reads it to end of file, closes it,
|
|
89
|
+
and caches the value and its source (`fd` or `env`) for the life of the
|
|
90
|
+
process. A named descriptor that is closed, not a pipe, a different pipe, empty
|
|
91
|
+
or malformed throws a `LaunchNonceError` (`NotOpen`, `NotAPipe`, `WrongPipe`,
|
|
92
|
+
`Empty`, `Malformed`), leaves the descriptor as it was, and never falls back to
|
|
93
|
+
the environment copy. The environment copy is read only when
|
|
94
|
+
`SUBC_LAUNCH_NONCE_FD` is absent. `process.env` is never modified.
|
|
95
|
+
|
|
96
|
+
- `SubcProvider.connect()` sends the nonce in HELLO and rejects with a
|
|
97
|
+
`SubcProviderError` coded `launch_nonce_unavailable` on a refusal. A declared
|
|
98
|
+
`manifest.provenance` gets `launch_nonce_source` filled in when unset.
|
|
99
|
+
- `route.open` presents the nonce as consumer identity; a refusal means no
|
|
100
|
+
identity.
|
|
101
|
+
- Call `launchNonce()` (or connect the provider) before spawning anything: until
|
|
102
|
+
the first read, descriptor 3 is inheritable. A process spawned afterwards
|
|
103
|
+
inherits the variable but not the pipe, and its accessor refuses by name.
|
|
104
|
+
- Every copy of this package in a JavaScript realm shares the cache. A worker
|
|
105
|
+
thread is a separate realm, so read the nonce on the main thread.
|
|
106
|
+
- It relies on the write end of the pipe being closed before the module
|
|
107
|
+
starts, as the daemon's handoff does. Node and Bun cannot count the bytes
|
|
108
|
+
waiting in a pipe without reading them, so emptiness is checked with a first
|
|
109
|
+
read, which returns nothing and consumes nothing at end of file; if a writer
|
|
110
|
+
still held the pipe open, that read would block instead of reporting
|
|
111
|
+
`Empty`. (fstat's size is no substitute: macOS reports the waiting bytes for
|
|
112
|
+
an anonymous pipe, but 0 for a mkfifo pipe, and Linux reports 0 for every
|
|
113
|
+
pipe.)
|
|
114
|
+
|
|
81
115
|
## Testing
|
|
82
116
|
|
|
83
117
|
```sh
|
|
@@ -100,4 +134,5 @@ package builds the `ck-subc` executable).
|
|
|
100
134
|
| `src/auth.ts` | HMAC-SHA256 handshake (`computeProof`, constant-time verify) |
|
|
101
135
|
| `src/route-handle.ts` | immutable connection-bound `(channel, epoch)` route identity |
|
|
102
136
|
| `src/client.ts` | `SubcClient`: route handles, channel-0 RPCs, epoch-aware corr-mux |
|
|
137
|
+
| `src/launch-nonce.ts` | the one launch-nonce reader: descriptor 3 or the environment copy, cached |
|
|
103
138
|
| `src/provider.ts` | provider bind publication, epoch validation, and routed serving |
|
package/dist/client.d.ts
CHANGED
|
@@ -4,9 +4,10 @@ import { ReverseRequestRegistry, RouteHandle } from "./route-handle.js";
|
|
|
4
4
|
export declare const DEFAULT_REQUEST_TIMEOUT_MS = 30000;
|
|
5
5
|
export declare const TIMEOUT_ARBITRATION_GRACE_MS = 50;
|
|
6
6
|
export declare const LIVENESS_PROBE_WINDOW_MS = 2000;
|
|
7
|
-
export declare const ROUTE_OPEN_RETRY_DEADLINE_MS =
|
|
7
|
+
export declare const ROUTE_OPEN_RETRY_DEADLINE_MS = 90000;
|
|
8
|
+
export declare const MAX_ROUTE_OPENS_IN_FLIGHT = 8;
|
|
8
9
|
export declare const SUBC_MODULE_ID_ENV = "SUBC_MODULE_ID";
|
|
9
|
-
export
|
|
10
|
+
export { SUBC_LAUNCH_NONCE_ENV } from "./launch-nonce.js";
|
|
10
11
|
export interface BindIdentity {
|
|
11
12
|
project_root: string;
|
|
12
13
|
harness: string;
|
|
@@ -29,7 +30,7 @@ export interface ConsumerIdentity {
|
|
|
29
30
|
launch_nonce: string;
|
|
30
31
|
}
|
|
31
32
|
export interface RouteOpenOptions {
|
|
32
|
-
/** Optional override for the consumer identity; by default the SUBC_MODULE_ID and
|
|
33
|
+
/** Optional override for the consumer identity; by default the SUBC_MODULE_ID environment variable and the process's launch nonce (see `launchNonce`) are used when both are non-empty; a refused launch-nonce descriptor counts as no nonce. Set null to send route.open without consumer_identity. */
|
|
33
34
|
consumerIdentity?: ConsumerIdentity | null;
|
|
34
35
|
/**
|
|
35
36
|
* Handlers prepared before route.open. The client derives consumer_capabilities
|
|
@@ -116,7 +117,7 @@ export interface ManagedCallOptions extends RequestOptions {
|
|
|
116
117
|
identity?: BindIdentity;
|
|
117
118
|
/** Defaults to management_surface, matching the store/host management APIs. */
|
|
118
119
|
targetKind?: ManagedRouteKind;
|
|
119
|
-
/** Optional override for the consumer identity; by default the SUBC_MODULE_ID and
|
|
120
|
+
/** Optional override for the consumer identity; by default the SUBC_MODULE_ID environment variable and the process's launch nonce (see `launchNonce`) are used when both are non-empty; a refused launch-nonce descriptor counts as no nonce. Set null to send route.open without consumer_identity. */
|
|
120
121
|
consumerIdentity?: ConsumerIdentity | null;
|
|
121
122
|
/** Reverse-request handlers used by the managed route and all of its reconnect reopens. */
|
|
122
123
|
reverseRequests?: ReverseRequestRegistry;
|
|
@@ -229,12 +230,23 @@ export interface ConnectOptions {
|
|
|
229
230
|
reconnectBackoff?: ReconnectBackoff;
|
|
230
231
|
/** Injectable sleep for timer-free reconnect tests. */
|
|
231
232
|
sleep?: (ms: number) => Promise<void>;
|
|
233
|
+
/**
|
|
234
|
+
* Injectable clock (milliseconds) for the route.open retry deadline. Tests pair
|
|
235
|
+
* it with `sleep` so a minute-long module restart runs in fake time.
|
|
236
|
+
*/
|
|
237
|
+
now?: () => number;
|
|
238
|
+
/**
|
|
239
|
+
* Injectable random source in [0, 1) for route.open retry jitter, so tests can
|
|
240
|
+
* seed it. Defaults to Math.random.
|
|
241
|
+
*/
|
|
242
|
+
random?: () => number;
|
|
232
243
|
/**
|
|
233
244
|
* How long managed calls keep retrying route.open in-place on retryable
|
|
234
245
|
* refusals (module_reloading, module_warming, target_unavailable, …) before
|
|
235
|
-
* settling not_sent. This deadline
|
|
236
|
-
*
|
|
237
|
-
*
|
|
246
|
+
* settling not_sent. This deadline and the call's own timeoutMs (when set) are
|
|
247
|
+
* the only binders on those retries: a module restart legitimately keeps its
|
|
248
|
+
* routes refused for about a minute (drain, stop, boot), so the default is
|
|
249
|
+
* ROUTE_OPEN_RETRY_DEADLINE_MS.
|
|
238
250
|
*/
|
|
239
251
|
routeOpenRetryDeadlineMs?: number;
|
|
240
252
|
/**
|
|
@@ -305,9 +317,11 @@ export declare class SubcClient {
|
|
|
305
317
|
private closeStarted;
|
|
306
318
|
private reconnecting;
|
|
307
319
|
private generation;
|
|
320
|
+
private routeOpenGateInstance;
|
|
308
321
|
private readerActive;
|
|
309
322
|
private constructor();
|
|
310
323
|
get conn(): ConnectionInfo;
|
|
324
|
+
private get routeOpenGate();
|
|
311
325
|
/** Read the connection file, connect, authenticate, and start the read loop. */
|
|
312
326
|
static connect(opts: ConnectOptions): Promise<SubcClient>;
|
|
313
327
|
/** List modules subc knows about (channel-0 catalog.list). */
|
|
@@ -321,8 +335,14 @@ export declare class SubcClient {
|
|
|
321
335
|
resolveProvider(capability: string): Promise<string>;
|
|
322
336
|
/** Resolve every catalog claimant in deterministic module-id order. */
|
|
323
337
|
resolveProviders(capability: string): Promise<string[]>;
|
|
324
|
-
/**
|
|
338
|
+
/**
|
|
339
|
+
* Open a route and return its connection-bound immutable handle. At most
|
|
340
|
+
* MAX_ROUTE_OPENS_IN_FLIGHT opens are outstanding at once; further calls wait
|
|
341
|
+
* their turn.
|
|
342
|
+
*/
|
|
325
343
|
routeOpen(target: RouteTarget, identity: BindIdentity, opts?: RouteOpenOptions): Promise<RouteHandle>;
|
|
344
|
+
/** One route.open exchange. The caller holds a route-open gate slot. */
|
|
345
|
+
private sendRouteOpen;
|
|
326
346
|
/** Send a data-plane request on exactly the supplied route generation. */
|
|
327
347
|
request(handle: RouteHandle, body: unknown, opts?: RequestOptions): Promise<unknown>;
|
|
328
348
|
/**
|
|
@@ -382,7 +402,15 @@ export declare class SubcClient {
|
|
|
382
402
|
private managedRequest;
|
|
383
403
|
private sendManaged;
|
|
384
404
|
private cachedRouteHandle;
|
|
405
|
+
private awaitOpeningUntil;
|
|
385
406
|
private openCachedRoute;
|
|
407
|
+
/**
|
|
408
|
+
* The not_sent error for a route.open whose retry deadline ran out. Its code
|
|
409
|
+
* and cause are the most recent refusal, as they always were; its message
|
|
410
|
+
* describes the most informative refusal seen, so a module that was reloading
|
|
411
|
+
* is not reported as admission pressure on the client's own connection.
|
|
412
|
+
*/
|
|
413
|
+
private routeOpenRetryExhausted;
|
|
386
414
|
private lastInboundAtMs;
|
|
387
415
|
private livenessProbe;
|
|
388
416
|
/**
|
package/dist/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAeA,OAAO,EAA2C,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACpG,OAAO,EACL,cAAc,EAMd,QAAQ,EAGT,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAeA,OAAO,EAA2C,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACpG,OAAO,EACL,cAAc,EAMd,QAAQ,EAGT,MAAM,eAAe,CAAC;AAEvB,OAAO,EAKL,sBAAsB,EACtB,WAAW,EAGZ,MAAM,mBAAmB,CAAC;AAc3B,eAAO,MAAM,0BAA0B,QAAS,CAAC;AAWjD,eAAO,MAAM,4BAA4B,KAAK,CAAC;AAC/C,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAuB7C,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAOnD,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAM3C,eAAO,MAAM,kBAAkB,mBAAmB,CAAC;AACnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,WAAW,GACnB;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACjD;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAExE,MAAM,MAAM,gBAAgB,GAAG,oBAAoB,GAAG,eAAe,CAAC;AAEtE,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,wSAAwS;IACxS,gBAAgB,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC3C;;;OAGG;IACH,eAAe,CAAC,EAAE,sBAAsB,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,GAAG,UAAU,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,4BAA4B,EAAE,CAAC;IACzC,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,yEAAyE;AACzE,MAAM,MAAM,mBAAmB,GAC3B,gBAAgB,GAChB;IAAE,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEjC,uDAAuD;AACvD,MAAM,MAAM,oBAAoB,GAC5B;IAAE,OAAO,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GACpC;IAAE,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEpC,4EAA4E;AAC5E,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC7B,WAAW,EAAE,mBAAmB,CAAC;IACjC,OAAO,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC1C,+EAA+E;IAC/E,YAAY,CAAC,EAAE,4BAA4B,EAAE,GAAG,IAAI,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iFAAiF;IACjF,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,IAAI,CAAC;IACxC;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,kBAAmB,SAAQ,cAAc;IACxD,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,+EAA+E;IAC/E,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,wSAAwS;IACxS,gBAAgB,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC3C,2FAA2F;IAC3F,eAAe,CAAC,EAAE,sBAAsB,CAAC;CAC1C;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAED,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,UAAU,CAAC;AAElD,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,EAAE,OAAO,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,wBAAyB,SAAQ,iBAAiB;IACjE,0DAA0D;IAC1D,gBAAgB,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC3C,6EAA6E;IAC7E,eAAe,CAAC,EAAE,sBAAsB,CAAC;CAC1C;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yBAAyB;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,yBAAyB,EAAE,gBAIvC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,UAAU,GAAG,iBAAiB,GAAG,UAAU,CAAC;AAE5E;;;;;;;;;;;;;GAaG;AACH,qBAAa,aAAc,SAAQ,KAAK;IAEpC,QAAQ,CAAC,IAAI,EAAE,iBAAiB;IAEhC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO;gBAHf,IAAI,EAAE,iBAAiB,EAChC,OAAO,EAAE,MAAM,EACN,IAAI,CAAC,EAAE,MAAM,YAAA,EACb,KAAK,CAAC,EAAE,OAAO,YAAA;IAM1B;;;;;OAKG;IACH,IAAI,MAAM,IAAI,OAAO,CAEpB;CACF;AAED;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,WAAW,IAAI,IAAI,CAAC;IACpB,+FAA+F;IAC/F,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAChC;AAED,qBAAa,SAAU,SAAQ,KAAK;IAGhC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM;IACtB;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO;gBARzB,OAAO,EAAE,MAAM,EACN,IAAI,CAAC,EAAE,MAAM,YAAA;IACtB;;;;;OAKG;IACM,MAAM,CAAC,EAAE,OAAO,YAAA;CAI5B;AA6BD,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,iFAAiF;IACjF,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,wFAAwF;IACxF,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,uDAAuD;IACvD,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;;OAMG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC;;;;;;;;;;OAUG;IACH,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,WAAW,KAAK,IAAI,CAAC;CAC7C;AAED,yDAAyD;AACzD,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,EAAE,EAAE,MAAM,CAAC;IACX,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,+EAA+E;AAC/E,MAAM,MAAM,qBAAqB,GAC7B,QAAQ,GACR,SAAS,GACT,SAAS,GACT,OAAO,GACP,mBAAmB,CAAC;AAExB,+DAA+D;AAC/D,MAAM,MAAM,gBAAgB,GAAG,qBAAqB,GAAG,SAAS,CAAC;AAEjE,8EAA8E;AAC9E,MAAM,MAAM,qBAAqB,GAAG,YAAY,GAAG,iBAAiB,CAAC;AAErE;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAWvE;AAED,qFAAqF;AACrF,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,OAAO,GAAG,qBAAqB,CAW/E;AAED,6EAA6E;AAC7E,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,WAAW,GAAG,gBAAgB,GAAG,SAAS,CAGhF;AAsHD,qBAAa,UAAU;IAoBnB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI;IArBvB,OAAO,CAAC,QAAQ,CAAM;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8B;IACtD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA6C;IAC3E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;IACzD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAkC;IAC7D,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,SAAS,CAAsB;IACvC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAA4B;IAKzD,OAAO,CAAC,YAAY,CAAS;IAE7B,OAAO;IAQP,IAAI,IAAI,IAAI,cAAc,CAEzB;IAID,OAAO,KAAK,aAAa,GAExB;IAED,gFAAgF;WACnE,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC;IAM/D,8DAA8D;IACxD,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAS7D;;;;;OAKG;IACG,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAc1D,uEAAuE;IACjE,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAc7D;;;;OAIG;IACG,SAAS,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,GAAE,gBAAqB,GAAG,OAAO,CAAC,WAAW,CAAC;IAU/G,wEAAwE;YAC1D,aAAa;IA8C3B,0EAA0E;IACpE,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,GAAE,cAAmB,GAAG,OAAO,CAAC,OAAO,CAAC;IAqB9F;;;OAGG;IACG,IAAI,CAAC,QAAQ,GAAG,OAAO,EAC3B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,OAAO,EAChB,IAAI,GAAE,kBAAuB,GAC5B,OAAO,CAAC,QAAQ,CAAC;IAWpB;;;;;OAKG;IACG,UAAU,CACd,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,UAAU,EAChB,IAAI,GAAE,kBAAuB,GAC5B,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC;YAKlB,WAAW;IA6DvB,2DAA2D;IAC7D,SAAS,CACP,MAAM,EAAE,WAAW,EACnB,IAAI,EAAE,OAAO,EACb,OAAO,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,EACpC,IAAI,GAAE,gBAAqB,GAC1B,YAAY;IA+Df;;;;;;;;OAQG;IACH,OAAO,CAAC,kBAAkB;IAmC1B,gEAAgE;IAChE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAE,QAA+B,GAAG,IAAI;IAa1F,yEAAyE;IACnE,SAAS,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAgBjF,uDAAuD;IACnD,UAAU,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,GAAE,iBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAelF,qEAAqE;IAC/D,iBAAiB,CACrB,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,IAAI,EAAE,gBAAgB,CAAA;KAAE,CAAC,EACxD,QAAQ,EAAE,YAAY,EACtB,IAAI,GAAE,wBAA6B,GAClC,OAAO,CAAC,IAAI,CAAC;IAgBd,+GAA+G;IAC3G,iBAAiB,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,GAAE,iBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzF,KAAK,IAAI,IAAI;IAMb,OAAO,CAAC,kBAAkB;IAkB1B,OAAO,CAAC,gBAAgB;mBAqBH,cAAc;YAcrB,UAAU;IAQxB,OAAO,CAAC,IAAI;IAqDZ;;;;;;;;;OASG;IACH,OAAO,CAAC,gBAAgB;YA2BV,cAAc;IAuB5B,OAAO,CAAC,WAAW;YAyEL,iBAAiB;IAmD/B,OAAO,CAAC,iBAAiB;YAmBX,eAAe;IA4E7B;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB;IAiB/B,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,aAAa,CAA8B;IAEnD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,+EAA+E;IAC/E,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,0BAA0B;YA2CpB,yBAAyB;IAMvC,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,kBAAkB;YAWZ,kBAAkB;IAsChC,OAAO,CAAC,iBAAiB;YAgBX,kBAAkB;IAkChC,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,qBAAqB;YAIf,QAAQ;IAwBtB,OAAO,CAAC,QAAQ;IAsIhB;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM;IASd,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,sBAAsB;YAgEhB,gBAAgB;YAWhB,gBAAgB;IA4B9B,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,IAAI;IAOZ,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,uBAAuB;IAI/B,OAAO,CAAC,iBAAiB;IAKzB,OAAO,CAAC,oBAAoB;IAM5B,qFAAqF;IACrF,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,oBAAoB;IAI5B,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,kCAAkC;IAO1C,OAAO,CAAC,MAAM;IAId,8EAA8E;IAC9E,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,SAAS;CAIlB;AAED,wBAAgB,4BAA4B,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAkBlE;AAWD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAS1E;AAqDD,eAAO,MAAM,eAAe,oBAAoB,CAAC;AACjD,eAAO,MAAM,iBAAiB,sBAAsB,CAAC;AAErD;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAExF;AAED,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAOzE;AAmBD,2EAA2E;AAC3E,wBAAgB,2BAA2B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAYvE"}
|
package/dist/client.js
CHANGED
|
@@ -13,6 +13,7 @@ import { debuglog } from "node:util";
|
|
|
13
13
|
import { AuthError, authenticateClient } from "./auth.js";
|
|
14
14
|
import { ConnectionFileError, readConnectionFile } from "./connection-file.js";
|
|
15
15
|
import { AdmissionClass, buildFrame, buildFlags, encodeFrame, FrameType, hasBinary, Priority, } from "./envelope.js";
|
|
16
|
+
import { launchNonceOrUndefined } from "./launch-nonce.js";
|
|
16
17
|
import { belongsToConnection, createRouteHandle, newConnectionToken, reverseRequestHandler, ReverseRequestRegistry, sameRouteHandle, StaleRouteHandleError, } from "./route-handle.js";
|
|
17
18
|
import { SocketClosedError, SocketTimeoutError, SocketWriteNotQueuedError, SocketWriteQueuedError, SubcSocket, writeBorrowed, writeTrackedBorrowed, } from "./socket.js";
|
|
18
19
|
const debug = debuglog("subc-client");
|
|
@@ -44,21 +45,29 @@ const DEADLINE_NO_DROP_CODE = "deadline_exceeded_no_drop_observed";
|
|
|
44
45
|
// A retryable route.open rejection (target booting / reloading / momentarily
|
|
45
46
|
// absent) is retried in-place against the same connection up to this deadline
|
|
46
47
|
// before it is surfaced as not_sent. Mirrors subc-client-rs
|
|
47
|
-
// DEFAULT_ROUTE_RETRY_DEADLINE so a target that is
|
|
48
|
-
//
|
|
49
|
-
// Sized against
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
|
|
48
|
+
// DEFAULT_ROUTE_RETRY_DEADLINE so a target that is unavailable while it restarts
|
|
49
|
+
// recovers without a misleading terminal error.
|
|
50
|
+
// Sized against a real module restart, not a single bind: the daemon drains the
|
|
51
|
+
// module's routes (up to 30s), waits for the old process to stop (up to its stop
|
|
52
|
+
// budget, 25s by default), then boots the new one. A measured prefrontal-core
|
|
53
|
+
// restart kept route.open refused for 62.5s end to end, which a 30s budget gave
|
|
54
|
+
// up on halfway through. 90s covers that window with room for a slow boot. A
|
|
55
|
+
// call that sets its own timeoutMs still stops retrying at that timeout.
|
|
56
|
+
export const ROUTE_OPEN_RETRY_DEADLINE_MS = 90_000;
|
|
57
|
+
// The most route.open requests this client keeps outstanding on one connection.
|
|
58
|
+
// Mirrors the daemon's MAX_PENDING_ROUTE_OPENS_PER_CONNECTION
|
|
59
|
+
// (crates/subc-daemon/src/server.rs): the daemon refuses every open beyond that
|
|
60
|
+
// many with a retryable target_unavailable, so sending more at once only
|
|
61
|
+
// manufactures refusals. Extra opens wait here, first come first served, and the
|
|
62
|
+
// wait counts against the same retry deadline as the refusals do.
|
|
63
|
+
export const MAX_ROUTE_OPENS_IN_FLIGHT = 8;
|
|
55
64
|
// Once a header arrives, its body must follow promptly; bound it so a truncated
|
|
56
65
|
// frame cannot wedge the read loop forever.
|
|
57
66
|
const BODY_READ_TIMEOUT_MS = 30_000;
|
|
58
67
|
const EMPTY_BODY = new Uint8Array(0);
|
|
59
68
|
const DEFAULT_MANAGED_TARGET_KIND = "management_surface";
|
|
60
69
|
export const SUBC_MODULE_ID_ENV = "SUBC_MODULE_ID";
|
|
61
|
-
export
|
|
70
|
+
export { SUBC_LAUNCH_NONCE_ENV } from "./launch-nonce.js";
|
|
62
71
|
export const DEFAULT_RECONNECT_BACKOFF = {
|
|
63
72
|
baseMs: 100,
|
|
64
73
|
capMs: 2_000,
|
|
@@ -161,6 +170,85 @@ export function routeCloseReason(push) {
|
|
|
161
170
|
return undefined;
|
|
162
171
|
return parseRouteCloseReason(push.body.reason);
|
|
163
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* First-come-first-served limit on the route.open requests one client has
|
|
175
|
+
* outstanding. A slot is held from just before the open is written until its
|
|
176
|
+
* answer, or its failure, settles. The client holds one connection at a time,
|
|
177
|
+
* and replacing it fails every open still in flight on the old one, so one gate
|
|
178
|
+
* per client is a per-connection limit.
|
|
179
|
+
*/
|
|
180
|
+
class RouteOpenGate {
|
|
181
|
+
limit;
|
|
182
|
+
inFlight = 0;
|
|
183
|
+
waiters = [];
|
|
184
|
+
constructor(limit) {
|
|
185
|
+
this.limit = limit;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Resolve with a release function once a slot is free, or with null when
|
|
189
|
+
* `timeoutMs` passes first. Without a timeout the caller waits until a slot
|
|
190
|
+
* frees, which happens as each open in flight settles.
|
|
191
|
+
*/
|
|
192
|
+
acquire(timeoutMs) {
|
|
193
|
+
if (this.inFlight < this.limit && this.waiters.length === 0) {
|
|
194
|
+
this.inFlight += 1;
|
|
195
|
+
return Promise.resolve(this.releaser());
|
|
196
|
+
}
|
|
197
|
+
if (timeoutMs !== undefined && timeoutMs <= 0)
|
|
198
|
+
return Promise.resolve(null);
|
|
199
|
+
return new Promise((resolve) => {
|
|
200
|
+
let timer;
|
|
201
|
+
const waiter = (release) => {
|
|
202
|
+
if (timer !== undefined)
|
|
203
|
+
clearTimeout(timer);
|
|
204
|
+
resolve(release);
|
|
205
|
+
};
|
|
206
|
+
this.waiters.push(waiter);
|
|
207
|
+
if (timeoutMs !== undefined) {
|
|
208
|
+
timer = setTimeout(() => {
|
|
209
|
+
const index = this.waiters.indexOf(waiter);
|
|
210
|
+
if (index === -1)
|
|
211
|
+
return;
|
|
212
|
+
this.waiters.splice(index, 1);
|
|
213
|
+
resolve(null);
|
|
214
|
+
}, timeoutMs);
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
releaser() {
|
|
219
|
+
let released = false;
|
|
220
|
+
return () => {
|
|
221
|
+
if (released)
|
|
222
|
+
return;
|
|
223
|
+
released = true;
|
|
224
|
+
// Hand the slot straight to the oldest waiter, so a newcomer cannot take
|
|
225
|
+
// it between this release and that waiter resuming.
|
|
226
|
+
const next = this.waiters.shift();
|
|
227
|
+
if (next)
|
|
228
|
+
next(this.releaser());
|
|
229
|
+
else
|
|
230
|
+
this.inFlight -= 1;
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* The route.open refusals one retry loop has seen: the most recent one, and the
|
|
236
|
+
* one that best explains why the target was unreachable. When the deadline runs
|
|
237
|
+
* out, the error describes the best one. After a module restart the most recent
|
|
238
|
+
* refusal is often admission pressure ("8 binds in flight"), which points a
|
|
239
|
+
* reader at the client's concurrency when the module was simply down.
|
|
240
|
+
*/
|
|
241
|
+
class RouteOpenRefusals {
|
|
242
|
+
best;
|
|
243
|
+
last;
|
|
244
|
+
record(refusal) {
|
|
245
|
+
this.last = refusal;
|
|
246
|
+
// At equal rank the newer refusal wins: it describes the target's latest state.
|
|
247
|
+
if (!this.best || routeOpenRefusalRank(refusal) >= routeOpenRefusalRank(this.best)) {
|
|
248
|
+
this.best = refusal;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
164
252
|
export class SubcClient {
|
|
165
253
|
sock;
|
|
166
254
|
currentConn;
|
|
@@ -176,6 +264,7 @@ export class SubcClient {
|
|
|
176
264
|
closeStarted = false;
|
|
177
265
|
reconnecting = null;
|
|
178
266
|
generation = 1;
|
|
267
|
+
routeOpenGateInstance;
|
|
179
268
|
// True while the read loop is actively reading/dispatching a frame off the
|
|
180
269
|
// current socket (between reading a header and finishing its dispatch). The
|
|
181
270
|
// timeout arbitration reads it to decide whether a just-fired timeout should
|
|
@@ -190,6 +279,11 @@ export class SubcClient {
|
|
|
190
279
|
get conn() {
|
|
191
280
|
return this.currentConn;
|
|
192
281
|
}
|
|
282
|
+
// Created on first use, so a client assembled without running its constructor
|
|
283
|
+
// (as the unit tests do) still paces its route.opens.
|
|
284
|
+
get routeOpenGate() {
|
|
285
|
+
return (this.routeOpenGateInstance ??= new RouteOpenGate(MAX_ROUTE_OPENS_IN_FLIGHT));
|
|
286
|
+
}
|
|
193
287
|
/** Read the connection file, connect, authenticate, and start the read loop. */
|
|
194
288
|
static async connect(opts) {
|
|
195
289
|
const normalized = normalizeConnectOptions(opts);
|
|
@@ -230,8 +324,23 @@ export class SubcClient {
|
|
|
230
324
|
.map((module) => module.module_id)
|
|
231
325
|
.sort();
|
|
232
326
|
}
|
|
233
|
-
/**
|
|
327
|
+
/**
|
|
328
|
+
* Open a route and return its connection-bound immutable handle. At most
|
|
329
|
+
* MAX_ROUTE_OPENS_IN_FLIGHT opens are outstanding at once; further calls wait
|
|
330
|
+
* their turn.
|
|
331
|
+
*/
|
|
234
332
|
async routeOpen(target, identity, opts = {}) {
|
|
333
|
+
// Without a timeout the gate always grants a slot eventually.
|
|
334
|
+
const release = (await this.routeOpenGate.acquire());
|
|
335
|
+
try {
|
|
336
|
+
return await this.sendRouteOpen(target, identity, opts);
|
|
337
|
+
}
|
|
338
|
+
finally {
|
|
339
|
+
release();
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
/** One route.open exchange. The caller holds a route-open gate slot. */
|
|
343
|
+
async sendRouteOpen(target, identity, opts = {}) {
|
|
235
344
|
const consumerIdentity = routeOpenConsumerIdentity(opts);
|
|
236
345
|
const reverseRequests = opts.reverseRequests ?? new ReverseRequestRegistry();
|
|
237
346
|
reverseRequests.seal();
|
|
@@ -741,17 +850,41 @@ export class SubcClient {
|
|
|
741
850
|
}
|
|
742
851
|
if (cached.handle && this.isLiveHandle(cached.handle))
|
|
743
852
|
return cached.handle;
|
|
853
|
+
// The retries end at the retry deadline or at the call's own timeoutMs,
|
|
854
|
+
// whichever comes first. Without timeoutMs the call has no overall deadline
|
|
855
|
+
// here: the default request timeout bounds only the response wait.
|
|
856
|
+
const startedAt = this.opts.now();
|
|
857
|
+
const callDeadline = opts.timeoutMs === undefined ? undefined : startedAt + opts.timeoutMs;
|
|
744
858
|
if (!cached.opening) {
|
|
745
|
-
|
|
859
|
+
const retryDeadline = startedAt + this.opts.routeOpenRetryDeadlineMs;
|
|
860
|
+
const deadline = callDeadline === undefined ? retryDeadline : Math.min(retryDeadline, callDeadline);
|
|
861
|
+
cached.opening = this.openCachedRoute(cached, deadline).finally(() => {
|
|
746
862
|
cached.opening = null;
|
|
747
863
|
});
|
|
864
|
+
return cached.opening;
|
|
748
865
|
}
|
|
749
|
-
|
|
866
|
+
// Another call is already opening this route, on its own deadline. Share its
|
|
867
|
+
// result, but never wait past this call's own timeout for it.
|
|
868
|
+
if (callDeadline === undefined)
|
|
869
|
+
return cached.opening;
|
|
870
|
+
return this.awaitOpeningUntil(cached.opening, callDeadline);
|
|
871
|
+
}
|
|
872
|
+
awaitOpeningUntil(opening, deadline) {
|
|
873
|
+
return new Promise((resolve, reject) => {
|
|
874
|
+
const timer = setTimeout(() => reject(this.notSentCallError("call deadline elapsed waiting for another call's route.open")), Math.max(0, deadline - this.opts.now()));
|
|
875
|
+
opening.then((handle) => {
|
|
876
|
+
clearTimeout(timer);
|
|
877
|
+
resolve(handle);
|
|
878
|
+
}, (error) => {
|
|
879
|
+
clearTimeout(timer);
|
|
880
|
+
reject(error);
|
|
881
|
+
});
|
|
882
|
+
});
|
|
750
883
|
}
|
|
751
|
-
async openCachedRoute(cached) {
|
|
752
|
-
const routeRetryDeadline = Date.now() + this.opts.routeOpenRetryDeadlineMs;
|
|
884
|
+
async openCachedRoute(cached, deadline) {
|
|
753
885
|
let routeRetryDelay = this.opts.reconnectBackoff.baseMs;
|
|
754
886
|
let routeRetryAttempt = 0;
|
|
887
|
+
const refusals = new RouteOpenRefusals();
|
|
755
888
|
for (;;) {
|
|
756
889
|
if (cached.closed)
|
|
757
890
|
throw this.routeClosedDuringOpen();
|
|
@@ -763,11 +896,27 @@ export class SubcClient {
|
|
|
763
896
|
}
|
|
764
897
|
if (cached.handle && this.isLiveHandle(cached.handle))
|
|
765
898
|
return cached.handle;
|
|
899
|
+
// Waiting for a free slot spends the same deadline as retrying refusals.
|
|
900
|
+
const release = await this.routeOpenGate.acquire(Math.max(0, deadline - this.opts.now()));
|
|
901
|
+
if (!release)
|
|
902
|
+
throw this.routeOpenRetryExhausted(cached.moduleId, routeRetryAttempt, refusals);
|
|
903
|
+
let outcome;
|
|
766
904
|
try {
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
905
|
+
outcome = {
|
|
906
|
+
handle: await this.sendRouteOpen(cached.target, cached.identity, {
|
|
907
|
+
consumerIdentity: cached.consumerIdentity ?? null,
|
|
908
|
+
reverseRequests: cached.reverseRequests,
|
|
909
|
+
}),
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
catch (error) {
|
|
913
|
+
outcome = { error };
|
|
914
|
+
}
|
|
915
|
+
finally {
|
|
916
|
+
release();
|
|
917
|
+
}
|
|
918
|
+
if ("handle" in outcome) {
|
|
919
|
+
const { handle } = outcome;
|
|
771
920
|
if (cached.closed) {
|
|
772
921
|
this.liveRoutes.delete(handle.channel);
|
|
773
922
|
this.sendRouteGoodbye(handle);
|
|
@@ -776,37 +925,55 @@ export class SubcClient {
|
|
|
776
925
|
cached.handle = handle;
|
|
777
926
|
return handle;
|
|
778
927
|
}
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
}
|
|
786
|
-
catch (reconnectError) {
|
|
787
|
-
throw this.notSentRecoveryError("route.open was not sent and reconnect failed", reconnectError);
|
|
788
|
-
}
|
|
789
|
-
continue;
|
|
928
|
+
const { error } = outcome;
|
|
929
|
+
if (error instanceof SubcCallError && error.code === "route_closed")
|
|
930
|
+
throw error;
|
|
931
|
+
if (!this.closeStarted && isConsumerReconnectTransient(error)) {
|
|
932
|
+
try {
|
|
933
|
+
await this.reconnectAfterDrop(error);
|
|
790
934
|
}
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
// this condition, and because the capped backoff sums to ~3.1s it
|
|
794
|
-
// strictly dominated the 30s deadline — the advertised reload
|
|
795
|
-
// patience was never delivered, so every module restart whose reload
|
|
796
|
-
// exceeded ~3s failed managed callers with module_reloading. Module
|
|
797
|
-
// reloads legitimately take tens of seconds (drain alone defaults to
|
|
798
|
-
// 30s); the backoff cap below bounds retry pressure instead.
|
|
799
|
-
routeRetryAttempt += 1;
|
|
800
|
-
if (Date.now() < routeRetryDeadline) {
|
|
801
|
-
await this.opts.sleep(routeRetryDelay);
|
|
802
|
-
routeRetryDelay = Math.min(routeRetryDelay * 2, this.opts.reconnectBackoff.capMs);
|
|
803
|
-
continue;
|
|
804
|
-
}
|
|
805
|
-
throw this.notSentCallError(`route.open failed for module ${cached.moduleId}: ${error.code} (retry deadline exhausted after ${routeRetryAttempt} attempts)`, error);
|
|
935
|
+
catch (reconnectError) {
|
|
936
|
+
throw this.notSentRecoveryError("route.open was not sent and reconnect failed", reconnectError);
|
|
806
937
|
}
|
|
807
|
-
|
|
938
|
+
continue;
|
|
808
939
|
}
|
|
940
|
+
if (!this.closeStarted && error instanceof SubcError && isRetryableRouteOpenCode(error.code)) {
|
|
941
|
+
// The deadline is the ONLY binder here. An attempt cap used to share
|
|
942
|
+
// this condition, and because the capped backoff sums to ~3.1s it
|
|
943
|
+
// strictly dominated the deadline — the advertised reload patience was
|
|
944
|
+
// never delivered, so every module restart whose reload exceeded ~3s
|
|
945
|
+
// failed managed callers with module_reloading. A restart legitimately
|
|
946
|
+
// keeps routes refused for about a minute; the backoff cap below bounds
|
|
947
|
+
// retry pressure instead.
|
|
948
|
+
routeRetryAttempt += 1;
|
|
949
|
+
refusals.record(error);
|
|
950
|
+
const remaining = deadline - this.opts.now();
|
|
951
|
+
if (remaining > 0) {
|
|
952
|
+
// Every route on a connection is closed together when its module
|
|
953
|
+
// drains, so without jitter they all retry in lock step and arrive at
|
|
954
|
+
// the daemon as one burst each round.
|
|
955
|
+
await this.opts.sleep(Math.min(equalJitter(routeRetryDelay, this.opts.random), remaining));
|
|
956
|
+
routeRetryDelay = Math.min(routeRetryDelay * 2, this.opts.reconnectBackoff.capMs);
|
|
957
|
+
continue;
|
|
958
|
+
}
|
|
959
|
+
throw this.routeOpenRetryExhausted(cached.moduleId, routeRetryAttempt, refusals);
|
|
960
|
+
}
|
|
961
|
+
throw this.terminalCallError(`route.open failed for module ${cached.moduleId}`, error);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
/**
|
|
965
|
+
* The not_sent error for a route.open whose retry deadline ran out. Its code
|
|
966
|
+
* and cause are the most recent refusal, as they always were; its message
|
|
967
|
+
* describes the most informative refusal seen, so a module that was reloading
|
|
968
|
+
* is not reported as admission pressure on the client's own connection.
|
|
969
|
+
*/
|
|
970
|
+
routeOpenRetryExhausted(moduleId, attempts, refusals) {
|
|
971
|
+
const { best, last } = refusals;
|
|
972
|
+
if (!best || !last) {
|
|
973
|
+
return this.notSentCallError(`route.open for module ${moduleId} waited until its deadline for one of ${MAX_ROUTE_OPENS_IN_FLIGHT} route.open slots on this connection`);
|
|
809
974
|
}
|
|
975
|
+
const recent = last === best ? "" : `; the most recent refusal was ${describeRouteOpenRefusal(last)}`;
|
|
976
|
+
return new SubcCallError("not_sent", `route.open failed for module ${moduleId}: ${describeRouteOpenRefusal(best)} (retry deadline exhausted after ${attempts} attempts${recent})`, last.code, last);
|
|
810
977
|
}
|
|
811
978
|
// Stamped by dispatch() on every inbound frame; read by the liveness probe.
|
|
812
979
|
lastInboundAtMs = 0;
|
|
@@ -1403,6 +1570,55 @@ export function isRetryableRouteOpenCode(code) {
|
|
|
1403
1570
|
code === "target_unavailable" ||
|
|
1404
1571
|
code === "module_timeout");
|
|
1405
1572
|
}
|
|
1573
|
+
/**
|
|
1574
|
+
* "Equal jitter": half the delay is kept and the other half is random, so
|
|
1575
|
+
* retries spread out while still backing off. `random` returns a value in [0, 1).
|
|
1576
|
+
*/
|
|
1577
|
+
function equalJitter(delayMs, random) {
|
|
1578
|
+
const half = delayMs / 2;
|
|
1579
|
+
return half + random() * half;
|
|
1580
|
+
}
|
|
1581
|
+
// The daemon's reasons for refusing a route.open because too many binds are
|
|
1582
|
+
// already in flight: on the client's connection, or towards the target module
|
|
1583
|
+
// (crates/subc-daemon/src/control.rs, route_open_capacity_refusal and
|
|
1584
|
+
// route_open_target_capacity_refusal). Both are sent as target_unavailable.
|
|
1585
|
+
const ROUTE_OPEN_ADMISSION_PRESSURE_REASONS = new Set(["open_admission_full", "target_binds_full"]);
|
|
1586
|
+
// Matches the messages of those same admission-pressure refusals. The daemon
|
|
1587
|
+
// currently logs the reason but leaves it out of the refusal's detail, so the
|
|
1588
|
+
// message is the only way to recognise them.
|
|
1589
|
+
const ROUTE_OPEN_ADMISSION_PRESSURE_MESSAGE = /route\.open binds in flight|route\.bind relays in flight/;
|
|
1590
|
+
function refusalReason(refusal) {
|
|
1591
|
+
const detail = refusal.detail;
|
|
1592
|
+
if (typeof detail !== "object" || detail === null || !("reason" in detail))
|
|
1593
|
+
return undefined;
|
|
1594
|
+
const reason = detail.reason;
|
|
1595
|
+
return typeof reason === "string" ? reason : undefined;
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* How much a route.open refusal says about why the target was unreachable,
|
|
1599
|
+
* highest first: the module was changing state (reloading or warming); a
|
|
1600
|
+
* delegation was not registered; the daemon was refusing for admission pressure
|
|
1601
|
+
* (too many binds in flight); anything else.
|
|
1602
|
+
*/
|
|
1603
|
+
function routeOpenRefusalRank(refusal) {
|
|
1604
|
+
if (refusal.code === "module_reloading" || refusal.code === "module_warming")
|
|
1605
|
+
return 3;
|
|
1606
|
+
const reason = refusalReason(refusal);
|
|
1607
|
+
if (refusal.code === "delegation_not_registered" || reason === "delegation_not_registered")
|
|
1608
|
+
return 2;
|
|
1609
|
+
if (refusal.code === "target_unavailable") {
|
|
1610
|
+
const admissionPressure = reason === undefined
|
|
1611
|
+
? ROUTE_OPEN_ADMISSION_PRESSURE_MESSAGE.test(refusal.message)
|
|
1612
|
+
: ROUTE_OPEN_ADMISSION_PRESSURE_REASONS.has(reason);
|
|
1613
|
+
if (admissionPressure)
|
|
1614
|
+
return 1;
|
|
1615
|
+
}
|
|
1616
|
+
return 0;
|
|
1617
|
+
}
|
|
1618
|
+
function describeRouteOpenRefusal(refusal) {
|
|
1619
|
+
const reason = refusalReason(refusal);
|
|
1620
|
+
return `${refusal.code ?? "refused"}${reason === undefined ? "" : ` (reason ${reason})`}: ${refusal.message}`;
|
|
1621
|
+
}
|
|
1406
1622
|
export const UNKNOWN_CHANNEL = "unknown_channel";
|
|
1407
1623
|
export const STALE_ROUTE_EPOCH = "stale_route_epoch";
|
|
1408
1624
|
/** Whether to evict this route, reopen it, and resend once. A consumer deriving
|
|
@@ -1432,6 +1648,8 @@ function normalizeConnectOptions(opts) {
|
|
|
1432
1648
|
targetKind: opts.targetKind ?? DEFAULT_MANAGED_TARGET_KIND,
|
|
1433
1649
|
reconnectBackoff: opts.reconnectBackoff ?? DEFAULT_RECONNECT_BACKOFF,
|
|
1434
1650
|
sleep: opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
|
|
1651
|
+
now: opts.now ?? Date.now,
|
|
1652
|
+
random: opts.random ?? Math.random,
|
|
1435
1653
|
routeOpenRetryDeadlineMs: opts.routeOpenRetryDeadlineMs ?? ROUTE_OPEN_RETRY_DEADLINE_MS,
|
|
1436
1654
|
timeoutArbitrationGraceMs: opts.timeoutArbitrationGraceMs ?? TIMEOUT_ARBITRATION_GRACE_MS,
|
|
1437
1655
|
livenessProbeWindowMs: opts.livenessProbeWindowMs ?? LIVENESS_PROBE_WINDOW_MS,
|
|
@@ -1469,7 +1687,12 @@ function routeOpenConsumerIdentity(opts = {}) {
|
|
|
1469
1687
|
if (opts.consumerIdentity !== undefined)
|
|
1470
1688
|
return opts.consumerIdentity ?? undefined;
|
|
1471
1689
|
const moduleId = process.env[SUBC_MODULE_ID_ENV];
|
|
1472
|
-
|
|
1690
|
+
// The nonce comes from the one process-wide accessor, never straight from the
|
|
1691
|
+
// environment: the module's HELLO may already have read and closed the
|
|
1692
|
+
// descriptor, and a second reader would find some other file at its number.
|
|
1693
|
+
// A refused descriptor means this process has no identity to present, so the
|
|
1694
|
+
// route opens without one, as the Rust consumer does.
|
|
1695
|
+
const launchNonce = launchNonceOrUndefined()?.value;
|
|
1473
1696
|
if (!moduleId || !launchNonce)
|
|
1474
1697
|
return undefined;
|
|
1475
1698
|
return { module_id: moduleId, launch_nonce: launchNonce };
|