@cortexkit/subc-client 0.16.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -0
- package/dist/client.d.ts +36 -8
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +269 -46
- package/dist/client.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/launch-nonce.d.ts +173 -0
- package/dist/launch-nonce.d.ts.map +1 -0
- package/dist/launch-nonce.js +334 -0
- package/dist/launch-nonce.js.map +1 -0
- package/dist/provider.d.ts +41 -4
- package/dist/provider.d.ts.map +1 -1
- package/dist/provider.js +69 -4
- package/dist/provider.js.map +1 -1
- package/package.json +1 -1
- package/src/client.ts +303 -52
- package/src/index.ts +13 -0
- package/src/launch-nonce.ts +447 -0
- package/src/provider.ts +109 -8
package/src/client.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
SUBSCRIPTION_FLAG,
|
|
26
26
|
type Frame,
|
|
27
27
|
} from "./envelope.js";
|
|
28
|
+
import { launchNonceOrUndefined } from "./launch-nonce.js";
|
|
28
29
|
import {
|
|
29
30
|
belongsToConnection,
|
|
30
31
|
createRouteHandle,
|
|
@@ -75,21 +76,29 @@ const DEADLINE_NO_DROP_CODE = "deadline_exceeded_no_drop_observed";
|
|
|
75
76
|
// A retryable route.open rejection (target booting / reloading / momentarily
|
|
76
77
|
// absent) is retried in-place against the same connection up to this deadline
|
|
77
78
|
// before it is surfaced as not_sent. Mirrors subc-client-rs
|
|
78
|
-
// DEFAULT_ROUTE_RETRY_DEADLINE so a target that is
|
|
79
|
-
//
|
|
80
|
-
// Sized against
|
|
81
|
-
//
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
|
|
79
|
+
// DEFAULT_ROUTE_RETRY_DEADLINE so a target that is unavailable while it restarts
|
|
80
|
+
// recovers without a misleading terminal error.
|
|
81
|
+
// Sized against a real module restart, not a single bind: the daemon drains the
|
|
82
|
+
// module's routes (up to 30s), waits for the old process to stop (up to its stop
|
|
83
|
+
// budget, 25s by default), then boots the new one. A measured prefrontal-core
|
|
84
|
+
// restart kept route.open refused for 62.5s end to end, which a 30s budget gave
|
|
85
|
+
// up on halfway through. 90s covers that window with room for a slow boot. A
|
|
86
|
+
// call that sets its own timeoutMs still stops retrying at that timeout.
|
|
87
|
+
export const ROUTE_OPEN_RETRY_DEADLINE_MS = 90_000;
|
|
88
|
+
// The most route.open requests this client keeps outstanding on one connection.
|
|
89
|
+
// Mirrors the daemon's MAX_PENDING_ROUTE_OPENS_PER_CONNECTION
|
|
90
|
+
// (crates/subc-daemon/src/server.rs): the daemon refuses every open beyond that
|
|
91
|
+
// many with a retryable target_unavailable, so sending more at once only
|
|
92
|
+
// manufactures refusals. Extra opens wait here, first come first served, and the
|
|
93
|
+
// wait counts against the same retry deadline as the refusals do.
|
|
94
|
+
export const MAX_ROUTE_OPENS_IN_FLIGHT = 8;
|
|
86
95
|
// Once a header arrives, its body must follow promptly; bound it so a truncated
|
|
87
96
|
// frame cannot wedge the read loop forever.
|
|
88
97
|
const BODY_READ_TIMEOUT_MS = 30_000;
|
|
89
98
|
const EMPTY_BODY = new Uint8Array(0);
|
|
90
99
|
const DEFAULT_MANAGED_TARGET_KIND: ManagedRouteKind = "management_surface";
|
|
91
100
|
export const SUBC_MODULE_ID_ENV = "SUBC_MODULE_ID";
|
|
92
|
-
export
|
|
101
|
+
export { SUBC_LAUNCH_NONCE_ENV } from "./launch-nonce.js";
|
|
93
102
|
|
|
94
103
|
export interface BindIdentity {
|
|
95
104
|
project_root: string;
|
|
@@ -110,7 +119,7 @@ export interface ConsumerIdentity {
|
|
|
110
119
|
}
|
|
111
120
|
|
|
112
121
|
export interface RouteOpenOptions {
|
|
113
|
-
/** Optional override for the consumer identity; by default the SUBC_MODULE_ID and
|
|
122
|
+
/** Optional override for the consumer identity; by default the SUBC_MODULE_ID environment variable and the process's launch nonce (see `launchNonce`) are used when both are non-empty; a refused launch-nonce descriptor counts as no nonce. Set null to send route.open without consumer_identity. */
|
|
114
123
|
consumerIdentity?: ConsumerIdentity | null;
|
|
115
124
|
/**
|
|
116
125
|
* Handlers prepared before route.open. The client derives consumer_capabilities
|
|
@@ -197,7 +206,7 @@ export interface ManagedCallOptions extends RequestOptions {
|
|
|
197
206
|
identity?: BindIdentity;
|
|
198
207
|
/** Defaults to management_surface, matching the store/host management APIs. */
|
|
199
208
|
targetKind?: ManagedRouteKind;
|
|
200
|
-
/** Optional override for the consumer identity; by default the SUBC_MODULE_ID and
|
|
209
|
+
/** Optional override for the consumer identity; by default the SUBC_MODULE_ID environment variable and the process's launch nonce (see `launchNonce`) are used when both are non-empty; a refused launch-nonce descriptor counts as no nonce. Set null to send route.open without consumer_identity. */
|
|
201
210
|
consumerIdentity?: ConsumerIdentity | null;
|
|
202
211
|
/** Reverse-request handlers used by the managed route and all of its reconnect reopens. */
|
|
203
212
|
reverseRequests?: ReverseRequestRegistry;
|
|
@@ -358,12 +367,23 @@ export interface ConnectOptions {
|
|
|
358
367
|
reconnectBackoff?: ReconnectBackoff;
|
|
359
368
|
/** Injectable sleep for timer-free reconnect tests. */
|
|
360
369
|
sleep?: (ms: number) => Promise<void>;
|
|
370
|
+
/**
|
|
371
|
+
* Injectable clock (milliseconds) for the route.open retry deadline. Tests pair
|
|
372
|
+
* it with `sleep` so a minute-long module restart runs in fake time.
|
|
373
|
+
*/
|
|
374
|
+
now?: () => number;
|
|
375
|
+
/**
|
|
376
|
+
* Injectable random source in [0, 1) for route.open retry jitter, so tests can
|
|
377
|
+
* seed it. Defaults to Math.random.
|
|
378
|
+
*/
|
|
379
|
+
random?: () => number;
|
|
361
380
|
/**
|
|
362
381
|
* How long managed calls keep retrying route.open in-place on retryable
|
|
363
382
|
* refusals (module_reloading, module_warming, target_unavailable, …) before
|
|
364
|
-
* settling not_sent. This deadline
|
|
365
|
-
*
|
|
366
|
-
*
|
|
383
|
+
* settling not_sent. This deadline and the call's own timeoutMs (when set) are
|
|
384
|
+
* the only binders on those retries: a module restart legitimately keeps its
|
|
385
|
+
* routes refused for about a minute (drain, stop, boot), so the default is
|
|
386
|
+
* ROUTE_OPEN_RETRY_DEADLINE_MS.
|
|
367
387
|
*/
|
|
368
388
|
routeOpenRetryDeadlineMs?: number;
|
|
369
389
|
/**
|
|
@@ -465,6 +485,8 @@ interface NormalizedConnectOptions {
|
|
|
465
485
|
targetKind: ManagedRouteKind;
|
|
466
486
|
reconnectBackoff: ReconnectBackoff;
|
|
467
487
|
sleep: (ms: number) => Promise<void>;
|
|
488
|
+
now: () => number;
|
|
489
|
+
random: () => number;
|
|
468
490
|
routeOpenRetryDeadlineMs: number;
|
|
469
491
|
timeoutArbitrationGraceMs: number;
|
|
470
492
|
livenessProbeWindowMs: number;
|
|
@@ -476,6 +498,82 @@ interface OpenedConnection {
|
|
|
476
498
|
conn: ConnectionInfo;
|
|
477
499
|
}
|
|
478
500
|
|
|
501
|
+
/**
|
|
502
|
+
* First-come-first-served limit on the route.open requests one client has
|
|
503
|
+
* outstanding. A slot is held from just before the open is written until its
|
|
504
|
+
* answer, or its failure, settles. The client holds one connection at a time,
|
|
505
|
+
* and replacing it fails every open still in flight on the old one, so one gate
|
|
506
|
+
* per client is a per-connection limit.
|
|
507
|
+
*/
|
|
508
|
+
class RouteOpenGate {
|
|
509
|
+
private inFlight = 0;
|
|
510
|
+
private readonly waiters: Array<(release: () => void) => void> = [];
|
|
511
|
+
|
|
512
|
+
constructor(private readonly limit: number) {}
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Resolve with a release function once a slot is free, or with null when
|
|
516
|
+
* `timeoutMs` passes first. Without a timeout the caller waits until a slot
|
|
517
|
+
* frees, which happens as each open in flight settles.
|
|
518
|
+
*/
|
|
519
|
+
acquire(timeoutMs?: number): Promise<(() => void) | null> {
|
|
520
|
+
if (this.inFlight < this.limit && this.waiters.length === 0) {
|
|
521
|
+
this.inFlight += 1;
|
|
522
|
+
return Promise.resolve(this.releaser());
|
|
523
|
+
}
|
|
524
|
+
if (timeoutMs !== undefined && timeoutMs <= 0) return Promise.resolve(null);
|
|
525
|
+
return new Promise((resolve) => {
|
|
526
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
527
|
+
const waiter = (release: () => void): void => {
|
|
528
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
529
|
+
resolve(release);
|
|
530
|
+
};
|
|
531
|
+
this.waiters.push(waiter);
|
|
532
|
+
if (timeoutMs !== undefined) {
|
|
533
|
+
timer = setTimeout(() => {
|
|
534
|
+
const index = this.waiters.indexOf(waiter);
|
|
535
|
+
if (index === -1) return;
|
|
536
|
+
this.waiters.splice(index, 1);
|
|
537
|
+
resolve(null);
|
|
538
|
+
}, timeoutMs);
|
|
539
|
+
}
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
private releaser(): () => void {
|
|
544
|
+
let released = false;
|
|
545
|
+
return () => {
|
|
546
|
+
if (released) return;
|
|
547
|
+
released = true;
|
|
548
|
+
// Hand the slot straight to the oldest waiter, so a newcomer cannot take
|
|
549
|
+
// it between this release and that waiter resuming.
|
|
550
|
+
const next = this.waiters.shift();
|
|
551
|
+
if (next) next(this.releaser());
|
|
552
|
+
else this.inFlight -= 1;
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* The route.open refusals one retry loop has seen: the most recent one, and the
|
|
559
|
+
* one that best explains why the target was unreachable. When the deadline runs
|
|
560
|
+
* out, the error describes the best one. After a module restart the most recent
|
|
561
|
+
* refusal is often admission pressure ("8 binds in flight"), which points a
|
|
562
|
+
* reader at the client's concurrency when the module was simply down.
|
|
563
|
+
*/
|
|
564
|
+
class RouteOpenRefusals {
|
|
565
|
+
best: SubcError | undefined;
|
|
566
|
+
last: SubcError | undefined;
|
|
567
|
+
|
|
568
|
+
record(refusal: SubcError): void {
|
|
569
|
+
this.last = refusal;
|
|
570
|
+
// At equal rank the newer refusal wins: it describes the target's latest state.
|
|
571
|
+
if (!this.best || routeOpenRefusalRank(refusal) >= routeOpenRefusalRank(this.best)) {
|
|
572
|
+
this.best = refusal;
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
479
577
|
interface CachedRoute {
|
|
480
578
|
key: string;
|
|
481
579
|
moduleId: string;
|
|
@@ -508,6 +606,7 @@ export class SubcClient {
|
|
|
508
606
|
private closeStarted = false;
|
|
509
607
|
private reconnecting: Promise<void> | null = null;
|
|
510
608
|
private generation = 1;
|
|
609
|
+
private routeOpenGateInstance: RouteOpenGate | undefined;
|
|
511
610
|
// True while the read loop is actively reading/dispatching a frame off the
|
|
512
611
|
// current socket (between reading a header and finishing its dispatch). The
|
|
513
612
|
// timeout arbitration reads it to decide whether a just-fired timeout should
|
|
@@ -526,6 +625,12 @@ export class SubcClient {
|
|
|
526
625
|
return this.currentConn;
|
|
527
626
|
}
|
|
528
627
|
|
|
628
|
+
// Created on first use, so a client assembled without running its constructor
|
|
629
|
+
// (as the unit tests do) still paces its route.opens.
|
|
630
|
+
private get routeOpenGate(): RouteOpenGate {
|
|
631
|
+
return (this.routeOpenGateInstance ??= new RouteOpenGate(MAX_ROUTE_OPENS_IN_FLIGHT));
|
|
632
|
+
}
|
|
633
|
+
|
|
529
634
|
/** Read the connection file, connect, authenticate, and start the read loop. */
|
|
530
635
|
static async connect(opts: ConnectOptions): Promise<SubcClient> {
|
|
531
636
|
const normalized = normalizeConnectOptions(opts);
|
|
@@ -578,8 +683,27 @@ export class SubcClient {
|
|
|
578
683
|
.sort();
|
|
579
684
|
}
|
|
580
685
|
|
|
581
|
-
/**
|
|
686
|
+
/**
|
|
687
|
+
* Open a route and return its connection-bound immutable handle. At most
|
|
688
|
+
* MAX_ROUTE_OPENS_IN_FLIGHT opens are outstanding at once; further calls wait
|
|
689
|
+
* their turn.
|
|
690
|
+
*/
|
|
582
691
|
async routeOpen(target: RouteTarget, identity: BindIdentity, opts: RouteOpenOptions = {}): Promise<RouteHandle> {
|
|
692
|
+
// Without a timeout the gate always grants a slot eventually.
|
|
693
|
+
const release = (await this.routeOpenGate.acquire())!;
|
|
694
|
+
try {
|
|
695
|
+
return await this.sendRouteOpen(target, identity, opts);
|
|
696
|
+
} finally {
|
|
697
|
+
release();
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
/** One route.open exchange. The caller holds a route-open gate slot. */
|
|
702
|
+
private async sendRouteOpen(
|
|
703
|
+
target: RouteTarget,
|
|
704
|
+
identity: BindIdentity,
|
|
705
|
+
opts: RouteOpenOptions = {},
|
|
706
|
+
): Promise<RouteHandle> {
|
|
583
707
|
const consumerIdentity = routeOpenConsumerIdentity(opts);
|
|
584
708
|
const reverseRequests = opts.reverseRequests ?? new ReverseRequestRegistry();
|
|
585
709
|
reverseRequests.seal();
|
|
@@ -1210,18 +1334,48 @@ export class SubcClient {
|
|
|
1210
1334
|
this.routes.set(key, cached);
|
|
1211
1335
|
}
|
|
1212
1336
|
if (cached.handle && this.isLiveHandle(cached.handle)) return cached.handle;
|
|
1337
|
+
// The retries end at the retry deadline or at the call's own timeoutMs,
|
|
1338
|
+
// whichever comes first. Without timeoutMs the call has no overall deadline
|
|
1339
|
+
// here: the default request timeout bounds only the response wait.
|
|
1340
|
+
const startedAt = this.opts.now();
|
|
1341
|
+
const callDeadline = opts.timeoutMs === undefined ? undefined : startedAt + opts.timeoutMs;
|
|
1213
1342
|
if (!cached.opening) {
|
|
1214
|
-
|
|
1343
|
+
const retryDeadline = startedAt + this.opts.routeOpenRetryDeadlineMs;
|
|
1344
|
+
const deadline = callDeadline === undefined ? retryDeadline : Math.min(retryDeadline, callDeadline);
|
|
1345
|
+
cached.opening = this.openCachedRoute(cached, deadline).finally(() => {
|
|
1215
1346
|
cached.opening = null;
|
|
1216
1347
|
});
|
|
1348
|
+
return cached.opening;
|
|
1217
1349
|
}
|
|
1218
|
-
|
|
1350
|
+
// Another call is already opening this route, on its own deadline. Share its
|
|
1351
|
+
// result, but never wait past this call's own timeout for it.
|
|
1352
|
+
if (callDeadline === undefined) return cached.opening;
|
|
1353
|
+
return this.awaitOpeningUntil(cached.opening, callDeadline);
|
|
1219
1354
|
}
|
|
1220
1355
|
|
|
1221
|
-
private
|
|
1222
|
-
|
|
1356
|
+
private awaitOpeningUntil(opening: Promise<RouteHandle>, deadline: number): Promise<RouteHandle> {
|
|
1357
|
+
return new Promise<RouteHandle>((resolve, reject) => {
|
|
1358
|
+
const timer = setTimeout(
|
|
1359
|
+
() => reject(this.notSentCallError("call deadline elapsed waiting for another call's route.open")),
|
|
1360
|
+
Math.max(0, deadline - this.opts.now()),
|
|
1361
|
+
);
|
|
1362
|
+
opening.then(
|
|
1363
|
+
(handle) => {
|
|
1364
|
+
clearTimeout(timer);
|
|
1365
|
+
resolve(handle);
|
|
1366
|
+
},
|
|
1367
|
+
(error: unknown) => {
|
|
1368
|
+
clearTimeout(timer);
|
|
1369
|
+
reject(error);
|
|
1370
|
+
},
|
|
1371
|
+
);
|
|
1372
|
+
});
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
private async openCachedRoute(cached: CachedRoute, deadline: number): Promise<RouteHandle> {
|
|
1223
1376
|
let routeRetryDelay = this.opts.reconnectBackoff.baseMs;
|
|
1224
1377
|
let routeRetryAttempt = 0;
|
|
1378
|
+
const refusals = new RouteOpenRefusals();
|
|
1225
1379
|
for (;;) {
|
|
1226
1380
|
if (cached.closed) throw this.routeClosedDuringOpen();
|
|
1227
1381
|
try {
|
|
@@ -1231,11 +1385,25 @@ export class SubcClient {
|
|
|
1231
1385
|
}
|
|
1232
1386
|
if (cached.handle && this.isLiveHandle(cached.handle)) return cached.handle;
|
|
1233
1387
|
|
|
1388
|
+
// Waiting for a free slot spends the same deadline as retrying refusals.
|
|
1389
|
+
const release = await this.routeOpenGate.acquire(Math.max(0, deadline - this.opts.now()));
|
|
1390
|
+
if (!release) throw this.routeOpenRetryExhausted(cached.moduleId, routeRetryAttempt, refusals);
|
|
1391
|
+
let outcome: { handle: RouteHandle } | { error: unknown };
|
|
1234
1392
|
try {
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1393
|
+
outcome = {
|
|
1394
|
+
handle: await this.sendRouteOpen(cached.target, cached.identity, {
|
|
1395
|
+
consumerIdentity: cached.consumerIdentity ?? null,
|
|
1396
|
+
reverseRequests: cached.reverseRequests,
|
|
1397
|
+
}),
|
|
1398
|
+
};
|
|
1399
|
+
} catch (error) {
|
|
1400
|
+
outcome = { error };
|
|
1401
|
+
} finally {
|
|
1402
|
+
release();
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
if ("handle" in outcome) {
|
|
1406
|
+
const { handle } = outcome;
|
|
1239
1407
|
if (cached.closed) {
|
|
1240
1408
|
this.liveRoutes.delete(handle.channel);
|
|
1241
1409
|
this.sendRouteGoodbye(handle);
|
|
@@ -1243,40 +1411,65 @@ export class SubcClient {
|
|
|
1243
1411
|
}
|
|
1244
1412
|
cached.handle = handle;
|
|
1245
1413
|
return handle;
|
|
1246
|
-
}
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
const { error } = outcome;
|
|
1417
|
+
if (error instanceof SubcCallError && error.code === "route_closed") throw error;
|
|
1418
|
+
if (!this.closeStarted && isConsumerReconnectTransient(error)) {
|
|
1419
|
+
try {
|
|
1420
|
+
await this.reconnectAfterDrop(error);
|
|
1421
|
+
} catch (reconnectError) {
|
|
1422
|
+
throw this.notSentRecoveryError("route.open was not sent and reconnect failed", reconnectError);
|
|
1255
1423
|
}
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
);
|
|
1424
|
+
continue;
|
|
1425
|
+
}
|
|
1426
|
+
if (!this.closeStarted && error instanceof SubcError && isRetryableRouteOpenCode(error.code)) {
|
|
1427
|
+
// The deadline is the ONLY binder here. An attempt cap used to share
|
|
1428
|
+
// this condition, and because the capped backoff sums to ~3.1s it
|
|
1429
|
+
// strictly dominated the deadline — the advertised reload patience was
|
|
1430
|
+
// never delivered, so every module restart whose reload exceeded ~3s
|
|
1431
|
+
// failed managed callers with module_reloading. A restart legitimately
|
|
1432
|
+
// keeps routes refused for about a minute; the backoff cap below bounds
|
|
1433
|
+
// retry pressure instead.
|
|
1434
|
+
routeRetryAttempt += 1;
|
|
1435
|
+
refusals.record(error);
|
|
1436
|
+
const remaining = deadline - this.opts.now();
|
|
1437
|
+
if (remaining > 0) {
|
|
1438
|
+
// Every route on a connection is closed together when its module
|
|
1439
|
+
// drains, so without jitter they all retry in lock step and arrive at
|
|
1440
|
+
// the daemon as one burst each round.
|
|
1441
|
+
await this.opts.sleep(Math.min(equalJitter(routeRetryDelay, this.opts.random), remaining));
|
|
1442
|
+
routeRetryDelay = Math.min(routeRetryDelay * 2, this.opts.reconnectBackoff.capMs);
|
|
1443
|
+
continue;
|
|
1274
1444
|
}
|
|
1275
|
-
throw this.
|
|
1445
|
+
throw this.routeOpenRetryExhausted(cached.moduleId, routeRetryAttempt, refusals);
|
|
1276
1446
|
}
|
|
1447
|
+
throw this.terminalCallError(`route.open failed for module ${cached.moduleId}`, error);
|
|
1277
1448
|
}
|
|
1278
1449
|
}
|
|
1279
1450
|
|
|
1451
|
+
/**
|
|
1452
|
+
* The not_sent error for a route.open whose retry deadline ran out. Its code
|
|
1453
|
+
* and cause are the most recent refusal, as they always were; its message
|
|
1454
|
+
* describes the most informative refusal seen, so a module that was reloading
|
|
1455
|
+
* is not reported as admission pressure on the client's own connection.
|
|
1456
|
+
*/
|
|
1457
|
+
private routeOpenRetryExhausted(moduleId: string, attempts: number, refusals: RouteOpenRefusals): SubcCallError {
|
|
1458
|
+
const { best, last } = refusals;
|
|
1459
|
+
if (!best || !last) {
|
|
1460
|
+
return this.notSentCallError(
|
|
1461
|
+
`route.open for module ${moduleId} waited until its deadline for one of ${MAX_ROUTE_OPENS_IN_FLIGHT} route.open slots on this connection`,
|
|
1462
|
+
);
|
|
1463
|
+
}
|
|
1464
|
+
const recent = last === best ? "" : `; the most recent refusal was ${describeRouteOpenRefusal(last)}`;
|
|
1465
|
+
return new SubcCallError(
|
|
1466
|
+
"not_sent",
|
|
1467
|
+
`route.open failed for module ${moduleId}: ${describeRouteOpenRefusal(best)} (retry deadline exhausted after ${attempts} attempts${recent})`,
|
|
1468
|
+
last.code,
|
|
1469
|
+
last,
|
|
1470
|
+
);
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1280
1473
|
// Stamped by dispatch() on every inbound frame; read by the liveness probe.
|
|
1281
1474
|
private lastInboundAtMs = 0;
|
|
1282
1475
|
// Single-flight guard: overlapping deadline-no-drop settles share one probe.
|
|
@@ -1949,6 +2142,57 @@ export function isRetryableRouteOpenCode(code: string | undefined): boolean {
|
|
|
1949
2142
|
);
|
|
1950
2143
|
}
|
|
1951
2144
|
|
|
2145
|
+
/**
|
|
2146
|
+
* "Equal jitter": half the delay is kept and the other half is random, so
|
|
2147
|
+
* retries spread out while still backing off. `random` returns a value in [0, 1).
|
|
2148
|
+
*/
|
|
2149
|
+
function equalJitter(delayMs: number, random: () => number): number {
|
|
2150
|
+
const half = delayMs / 2;
|
|
2151
|
+
return half + random() * half;
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
// The daemon's reasons for refusing a route.open because too many binds are
|
|
2155
|
+
// already in flight: on the client's connection, or towards the target module
|
|
2156
|
+
// (crates/subc-daemon/src/control.rs, route_open_capacity_refusal and
|
|
2157
|
+
// route_open_target_capacity_refusal). Both are sent as target_unavailable.
|
|
2158
|
+
const ROUTE_OPEN_ADMISSION_PRESSURE_REASONS = new Set(["open_admission_full", "target_binds_full"]);
|
|
2159
|
+
// Matches the messages of those same admission-pressure refusals. The daemon
|
|
2160
|
+
// currently logs the reason but leaves it out of the refusal's detail, so the
|
|
2161
|
+
// message is the only way to recognise them.
|
|
2162
|
+
const ROUTE_OPEN_ADMISSION_PRESSURE_MESSAGE = /route\.open binds in flight|route\.bind relays in flight/;
|
|
2163
|
+
|
|
2164
|
+
function refusalReason(refusal: SubcError): string | undefined {
|
|
2165
|
+
const detail = refusal.detail;
|
|
2166
|
+
if (typeof detail !== "object" || detail === null || !("reason" in detail)) return undefined;
|
|
2167
|
+
const reason = (detail as { reason?: unknown }).reason;
|
|
2168
|
+
return typeof reason === "string" ? reason : undefined;
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
/**
|
|
2172
|
+
* How much a route.open refusal says about why the target was unreachable,
|
|
2173
|
+
* highest first: the module was changing state (reloading or warming); a
|
|
2174
|
+
* delegation was not registered; the daemon was refusing for admission pressure
|
|
2175
|
+
* (too many binds in flight); anything else.
|
|
2176
|
+
*/
|
|
2177
|
+
function routeOpenRefusalRank(refusal: SubcError): number {
|
|
2178
|
+
if (refusal.code === "module_reloading" || refusal.code === "module_warming") return 3;
|
|
2179
|
+
const reason = refusalReason(refusal);
|
|
2180
|
+
if (refusal.code === "delegation_not_registered" || reason === "delegation_not_registered") return 2;
|
|
2181
|
+
if (refusal.code === "target_unavailable") {
|
|
2182
|
+
const admissionPressure =
|
|
2183
|
+
reason === undefined
|
|
2184
|
+
? ROUTE_OPEN_ADMISSION_PRESSURE_MESSAGE.test(refusal.message)
|
|
2185
|
+
: ROUTE_OPEN_ADMISSION_PRESSURE_REASONS.has(reason);
|
|
2186
|
+
if (admissionPressure) return 1;
|
|
2187
|
+
}
|
|
2188
|
+
return 0;
|
|
2189
|
+
}
|
|
2190
|
+
|
|
2191
|
+
function describeRouteOpenRefusal(refusal: SubcError): string {
|
|
2192
|
+
const reason = refusalReason(refusal);
|
|
2193
|
+
return `${refusal.code ?? "refused"}${reason === undefined ? "" : ` (reason ${reason})`}: ${refusal.message}`;
|
|
2194
|
+
}
|
|
2195
|
+
|
|
1952
2196
|
export const UNKNOWN_CHANNEL = "unknown_channel";
|
|
1953
2197
|
export const STALE_ROUTE_EPOCH = "stale_route_epoch";
|
|
1954
2198
|
|
|
@@ -1980,6 +2224,8 @@ function normalizeConnectOptions(opts: ConnectOptions): NormalizedConnectOptions
|
|
|
1980
2224
|
targetKind: opts.targetKind ?? DEFAULT_MANAGED_TARGET_KIND,
|
|
1981
2225
|
reconnectBackoff: opts.reconnectBackoff ?? DEFAULT_RECONNECT_BACKOFF,
|
|
1982
2226
|
sleep: opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
|
|
2227
|
+
now: opts.now ?? Date.now,
|
|
2228
|
+
random: opts.random ?? Math.random,
|
|
1983
2229
|
routeOpenRetryDeadlineMs: opts.routeOpenRetryDeadlineMs ?? ROUTE_OPEN_RETRY_DEADLINE_MS,
|
|
1984
2230
|
timeoutArbitrationGraceMs: opts.timeoutArbitrationGraceMs ?? TIMEOUT_ARBITRATION_GRACE_MS,
|
|
1985
2231
|
livenessProbeWindowMs: opts.livenessProbeWindowMs ?? LIVENESS_PROBE_WINDOW_MS,
|
|
@@ -2018,7 +2264,12 @@ function routeCacheKey(
|
|
|
2018
2264
|
function routeOpenConsumerIdentity(opts: RouteOpenOptions = {}): ConsumerIdentity | undefined {
|
|
2019
2265
|
if (opts.consumerIdentity !== undefined) return opts.consumerIdentity ?? undefined;
|
|
2020
2266
|
const moduleId = process.env[SUBC_MODULE_ID_ENV];
|
|
2021
|
-
|
|
2267
|
+
// The nonce comes from the one process-wide accessor, never straight from the
|
|
2268
|
+
// environment: the module's HELLO may already have read and closed the
|
|
2269
|
+
// descriptor, and a second reader would find some other file at its number.
|
|
2270
|
+
// A refused descriptor means this process has no identity to present, so the
|
|
2271
|
+
// route opens without one, as the Rust consumer does.
|
|
2272
|
+
const launchNonce = launchNonceOrUndefined()?.value;
|
|
2022
2273
|
if (!moduleId || !launchNonce) return undefined;
|
|
2023
2274
|
return { module_id: moduleId, launch_nonce: launchNonce };
|
|
2024
2275
|
}
|
package/src/index.ts
CHANGED
|
@@ -5,6 +5,7 @@ export {
|
|
|
5
5
|
UNKNOWN_CHANNEL,
|
|
6
6
|
STALE_ROUTE_EPOCH,
|
|
7
7
|
isEstablishedRouteDead,
|
|
8
|
+
isRetryableRouteOpenCode,
|
|
8
9
|
DEFAULT_RECONNECT_BACKOFF,
|
|
9
10
|
SUBC_MODULE_ID_ENV,
|
|
10
11
|
SUBC_LAUNCH_NONCE_ENV,
|
|
@@ -83,6 +84,17 @@ export {
|
|
|
83
84
|
CLIENT_AUTH_DOMAIN,
|
|
84
85
|
} from "./auth.js";
|
|
85
86
|
export { SubcSocket, SocketClosedError, SocketTimeoutError } from "./socket.js";
|
|
87
|
+
export {
|
|
88
|
+
launchNonce,
|
|
89
|
+
launchNonceOrUndefined,
|
|
90
|
+
isLaunchNonceError,
|
|
91
|
+
LaunchNonceError,
|
|
92
|
+
LAUNCH_NONCE_FD,
|
|
93
|
+
SUBC_LAUNCH_NONCE_FD_ENV,
|
|
94
|
+
type LaunchNonce,
|
|
95
|
+
type LaunchNonceErrorKind,
|
|
96
|
+
type LaunchNonceSource,
|
|
97
|
+
} from "./launch-nonce.js";
|
|
86
98
|
export {
|
|
87
99
|
SubcProvider,
|
|
88
100
|
SubcProviderError,
|
|
@@ -104,6 +116,7 @@ export {
|
|
|
104
116
|
type ManagementOperationKind,
|
|
105
117
|
type ManagementSurfaceManifestOptions,
|
|
106
118
|
type ManifestInput,
|
|
119
|
+
type ManifestProvenance,
|
|
107
120
|
machineIdFromHelloAck,
|
|
108
121
|
type ModuleHelloAckBody,
|
|
109
122
|
type ObservabilityKind,
|