@cortexkit/aft-bridge 0.58.2 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bash-format.d.ts.map +1 -1
- package/dist/bash-format.js +5 -0
- package/dist/bash-format.js.map +1 -1
- package/dist/bash-hints.d.ts +76 -30
- package/dist/bash-hints.d.ts.map +1 -1
- package/dist/bash-hints.js +225 -47
- package/dist/bash-hints.js.map +1 -1
- package/dist/bash-host-fallback.d.ts +27 -0
- package/dist/bash-host-fallback.d.ts.map +1 -1
- package/dist/bash-host-fallback.js +55 -16
- package/dist/bash-host-fallback.js.map +1 -1
- package/dist/binary-identity.d.ts +13 -0
- package/dist/binary-identity.d.ts.map +1 -1
- package/dist/binary-identity.js +91 -2
- package/dist/binary-identity.js.map +1 -1
- package/dist/bridge.d.ts +45 -3
- package/dist/bridge.d.ts.map +1 -1
- package/dist/bridge.js +161 -42
- package/dist/bridge.js.map +1 -1
- package/dist/callgraph-format.d.ts.map +1 -1
- package/dist/callgraph-format.js +7 -7
- package/dist/callgraph-format.js.map +1 -1
- package/dist/child-process.d.ts.map +1 -1
- package/dist/child-process.js +10 -1
- package/dist/child-process.js.map +1 -1
- package/dist/config-error.d.ts +2 -2
- package/dist/config-error.js +2 -2
- package/dist/config-watch.d.ts +10 -6
- package/dist/config-watch.d.ts.map +1 -1
- package/dist/config-watch.js +46 -12
- package/dist/config-watch.js.map +1 -1
- package/dist/durable-log.d.ts +11 -0
- package/dist/durable-log.d.ts.map +1 -1
- package/dist/durable-log.js +67 -24
- package/dist/durable-log.js.map +1 -1
- package/dist/error-contract.d.ts +29 -5
- package/dist/error-contract.d.ts.map +1 -1
- package/dist/error-contract.js +68 -9
- package/dist/error-contract.js.map +1 -1
- package/dist/feature-config.d.ts +35 -18
- package/dist/feature-config.d.ts.map +1 -1
- package/dist/feature-config.js +236 -94
- package/dist/feature-config.js.map +1 -1
- package/dist/index.d.ts +11 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -9
- package/dist/index.js.map +1 -1
- package/dist/location-lifecycle.d.ts.map +1 -1
- package/dist/location-lifecycle.js +4 -2
- package/dist/location-lifecycle.js.map +1 -1
- package/dist/message-detach.d.ts +8 -0
- package/dist/message-detach.d.ts.map +1 -1
- package/dist/message-detach.js +32 -0
- package/dist/message-detach.js.map +1 -1
- package/dist/migration-notices.d.ts.map +1 -1
- package/dist/migration-notices.js +12 -5
- package/dist/migration-notices.js.map +1 -1
- package/dist/migration.d.ts +1 -0
- package/dist/migration.d.ts.map +1 -1
- package/dist/migration.js +39 -12
- package/dist/migration.js.map +1 -1
- package/dist/onnx-runtime.d.ts.map +1 -1
- package/dist/onnx-runtime.js +37 -32
- package/dist/onnx-runtime.js.map +1 -1
- package/dist/path-display.d.ts +5 -0
- package/dist/path-display.d.ts.map +1 -0
- package/dist/path-display.js +21 -0
- package/dist/path-display.js.map +1 -0
- package/dist/paths.d.ts +3 -0
- package/dist/paths.d.ts.map +1 -1
- package/dist/paths.js +47 -11
- package/dist/paths.js.map +1 -1
- package/dist/pool.d.ts +2 -0
- package/dist/pool.d.ts.map +1 -1
- package/dist/pool.js +4 -1
- package/dist/pool.js.map +1 -1
- package/dist/private-storage.d.ts +9 -0
- package/dist/private-storage.d.ts.map +1 -0
- package/dist/private-storage.js +75 -0
- package/dist/private-storage.js.map +1 -0
- package/dist/protocol.d.ts +4 -0
- package/dist/protocol.d.ts.map +1 -1
- package/dist/revivable-transport.d.ts +8 -2
- package/dist/revivable-transport.d.ts.map +1 -1
- package/dist/revivable-transport.js +63 -23
- package/dist/revivable-transport.js.map +1 -1
- package/dist/storage-paths.d.ts +3 -4
- package/dist/storage-paths.d.ts.map +1 -1
- package/dist/storage-paths.js +4 -5
- package/dist/storage-paths.js.map +1 -1
- package/dist/subc-transport.d.ts +104 -7
- package/dist/subc-transport.d.ts.map +1 -1
- package/dist/subc-transport.js +955 -61
- package/dist/subc-transport.js.map +1 -1
- package/dist/tar-executable.d.ts +1 -1
- package/dist/tar-executable.d.ts.map +1 -1
- package/dist/tar-executable.js +16 -2
- package/dist/tar-executable.js.map +1 -1
- package/dist/test-child-environment.d.ts +8 -0
- package/dist/test-child-environment.d.ts.map +1 -0
- package/dist/test-child-environment.js +105 -0
- package/dist/test-child-environment.js.map +1 -0
- package/dist/transport-factory.d.ts.map +1 -1
- package/dist/transport-factory.js +5 -4
- package/dist/transport-factory.js.map +1 -1
- package/dist/transport.d.ts +39 -0
- package/dist/transport.d.ts.map +1 -1
- package/dist/transport.js +26 -0
- package/dist/transport.js.map +1 -1
- package/dist/zoom-format.d.ts +1 -0
- package/dist/zoom-format.d.ts.map +1 -1
- package/dist/zoom-format.js +5 -1
- package/dist/zoom-format.js.map +1 -1
- package/package.json +2 -2
package/dist/subc-transport.js
CHANGED
|
@@ -16,13 +16,13 @@
|
|
|
16
16
|
* top of it. subc-client is a build-time path dependency bundled into the
|
|
17
17
|
* published plugin dist; it is never a published runtime dependency.
|
|
18
18
|
*/
|
|
19
|
-
import {
|
|
20
|
-
import { AuthError, connectionFileExists, isConsumerReconnectTransient, StaleRouteHandleError, SubcCallError, SubcClient, SubcError, } from "@cortexkit/subc-client";
|
|
19
|
+
import { statSync } from "node:fs";
|
|
20
|
+
import { AuthError, connectionFileExists, isConsumerReconnectTransient, routeCloseReason, SocketTimeoutError, StaleRouteHandleError, SubcCallError, SubcClient, SubcError, } from "@cortexkit/subc-client";
|
|
21
21
|
import { log, warn } from "./active-logger.js";
|
|
22
22
|
import { isRouteOpenReloadWindowError, isRouteRequestReloadRefusal } from "./error-contract.js";
|
|
23
23
|
import { asCanonicalRootPath, asRootGeneration, LifecycleRegistry, } from "./lifecycle-registry.js";
|
|
24
24
|
import { canonicalizeProjectRoot } from "./project-identity.js";
|
|
25
|
-
import { WORKER_SESSION_FIELD } from "./transport.js";
|
|
25
|
+
import { callPresetFor, HOST_TOOL_NAMES_FIELD, PRESET_FIELD, WORKER_SESSION_FIELD, } from "./transport.js";
|
|
26
26
|
/** The subc pool is closing and cannot carry another request. */
|
|
27
27
|
export class SubcTransportShuttingDownError extends SubcCallError {
|
|
28
28
|
constructor() {
|
|
@@ -30,6 +30,106 @@ export class SubcTransportShuttingDownError extends SubcCallError {
|
|
|
30
30
|
this.name = "SubcTransportShuttingDownError";
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* The plugin closed one of its own routes (pool shutdown, session close,
|
|
35
|
+
* project-root close, or a sibling call on the same session discarding the
|
|
36
|
+
* shared route) while this call's request was already in flight on it.
|
|
37
|
+
*
|
|
38
|
+
* subc-client records a request as pending and starts writing its frame in the
|
|
39
|
+
* same synchronous step, and its `closeRoute` rejects exactly those pending
|
|
40
|
+
* requests. So a request rejected that way was written, or queued to be
|
|
41
|
+
* written, and the daemon may have run it: the outcome is unknown. The call
|
|
42
|
+
* must not be blind-retried, and bash must not re-run it on the host.
|
|
43
|
+
*/
|
|
44
|
+
export class SubcRouteClosedMidCallError extends SubcCallError {
|
|
45
|
+
reason;
|
|
46
|
+
constructor(reason, cause) {
|
|
47
|
+
super("outcome_unknown", `the command may have run; the route closed mid-call: ${reason}`, "route_closed", cause);
|
|
48
|
+
this.name = "SubcRouteClosedMidCallError";
|
|
49
|
+
this.reason = reason;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* A data-plane request was queued and sent, but reading the daemon's response
|
|
54
|
+
* frame timed out waiting for body bytes after the header was received.
|
|
55
|
+
*
|
|
56
|
+
* subc-client raises SocketTimeoutError from readExact when the body does not
|
|
57
|
+
* arrive within BODY_READ_TIMEOUT_MS after the header. Because the request was
|
|
58
|
+
* already sent to the daemon, it may have executed: the outcome is unknown.
|
|
59
|
+
* The call must not be blind-retried, and bash must not re-run it on the host.
|
|
60
|
+
*/
|
|
61
|
+
export class SubcFrameReadTimeoutError extends SubcCallError {
|
|
62
|
+
constructor(cause) {
|
|
63
|
+
super("outcome_unknown", "the command may have run; timed out waiting for frame body bytes from the AFT daemon", "frame_read_timeout", cause);
|
|
64
|
+
this.name = "SubcFrameReadTimeoutError";
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Matches the frame-body read timeout raised by subc-client (socket.ts:185/190):
|
|
69
|
+
* "timed out waiting for <N> bytes".
|
|
70
|
+
*
|
|
71
|
+
* Excludes connect timeouts ("timed out connecting to ...") and write timeouts
|
|
72
|
+
* ("timed out writing to subc"): connect timeouts are provably pre-send, while
|
|
73
|
+
* write timeouts keep their existing transport classification.
|
|
74
|
+
*/
|
|
75
|
+
function isFrameBodyReadTimeoutMessage(message) {
|
|
76
|
+
return (typeof message === "string" &&
|
|
77
|
+
(/timed out waiting for \d+ bytes/i.test(message) ||
|
|
78
|
+
message.includes("timed out waiting for frame body bytes")));
|
|
79
|
+
}
|
|
80
|
+
export function isSubcFrameReadTimeoutError(error) {
|
|
81
|
+
if (error instanceof SubcFrameReadTimeoutError)
|
|
82
|
+
return true;
|
|
83
|
+
if (error instanceof SocketTimeoutError && isFrameBodyReadTimeoutMessage(error.message)) {
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
if (error instanceof SubcError && isFrameBodyReadTimeoutMessage(error.message)) {
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
if (error instanceof SubcCallError) {
|
|
90
|
+
if (error.code === "frame_read_timeout")
|
|
91
|
+
return true;
|
|
92
|
+
if (error.cause !== undefined &&
|
|
93
|
+
error.cause !== error &&
|
|
94
|
+
isSubcFrameReadTimeoutError(error.cause)) {
|
|
95
|
+
return true;
|
|
96
|
+
}
|
|
97
|
+
if (isFrameBodyReadTimeoutMessage(error.message))
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
export function frameReadTimeoutError(error) {
|
|
103
|
+
if (!isSubcFrameReadTimeoutError(error))
|
|
104
|
+
return error;
|
|
105
|
+
return error instanceof SubcFrameReadTimeoutError ? error : new SubcFrameReadTimeoutError(error);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* subc-client's rejection of a request that was pending when the client itself
|
|
109
|
+
* closed the route: `closeRoute` tearing that route down, or the client closing
|
|
110
|
+
* its connection. The client types both with `closeReason: "closed_by_caller"`;
|
|
111
|
+
* a daemon GOODBYE carries the daemon's reason and a lost connection carries
|
|
112
|
+
* `"connection_lost"`. A route closed with no request pending produces no error
|
|
113
|
+
* at all, and a request started on an already-closed route fails with
|
|
114
|
+
* `StaleRouteHandleError`. A route closed while still opening is reported as a
|
|
115
|
+
* not-sent `SubcCallError`, which is not a `SubcError` and so never matches.
|
|
116
|
+
*/
|
|
117
|
+
export function isLocalRouteCloseRejection(error) {
|
|
118
|
+
return error instanceof SubcError && error.closeReason === "closed_by_caller";
|
|
119
|
+
}
|
|
120
|
+
/** The route-close reason that matches a session's teardown, if it was torn down. */
|
|
121
|
+
function teardownCloseReason(teardown) {
|
|
122
|
+
switch (teardown) {
|
|
123
|
+
case "shutdown":
|
|
124
|
+
return "transport shutting down";
|
|
125
|
+
case "session_closed":
|
|
126
|
+
return "session closed";
|
|
127
|
+
case "root_reaped":
|
|
128
|
+
return "project root closed";
|
|
129
|
+
default:
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
33
133
|
class ReconnectKeyChangedError extends Error {
|
|
34
134
|
constructor() {
|
|
35
135
|
super("the subc daemon restarted and its connection key changed; the reconnect did not complete within the deadline, retry");
|
|
@@ -56,6 +156,12 @@ export function isSubcClientClosedError(error) {
|
|
|
56
156
|
* the connection has already been torn down locally.
|
|
57
157
|
*/
|
|
58
158
|
export function isSocketDeathError(error, depth = 0) {
|
|
159
|
+
if (isSubcFrameReadTimeoutError(error))
|
|
160
|
+
return true;
|
|
161
|
+
// Raw data-plane requests wrap socket death in a SubcError; its typed
|
|
162
|
+
// connection-loss reason survives even when the underlying errno does not.
|
|
163
|
+
if (error instanceof SubcError && error.closeReason === "connection_lost")
|
|
164
|
+
return true;
|
|
59
165
|
if (error instanceof SubcCallError) {
|
|
60
166
|
const cause = error.cause;
|
|
61
167
|
return depth < 4 && cause !== undefined && cause !== error
|
|
@@ -115,6 +221,14 @@ export class SubcLivenessProbeTimeoutError extends Error {
|
|
|
115
221
|
}
|
|
116
222
|
/** The subc module id AFT registers under (matches the daemon manifest). */
|
|
117
223
|
const AFT_MODULE_ID = "aft";
|
|
224
|
+
const HOST_SCOPE_LOOKUP_TIMEOUT_MS = 2_000;
|
|
225
|
+
const HOST_SCOPE_RETRY_FLOOR_MS = 5_000;
|
|
226
|
+
const HOST_SCOPE_RETRY_CAP_MS = 300_000;
|
|
227
|
+
function isHostScopeSyncRefusal(code) {
|
|
228
|
+
// crates/aft/src/gh_shim_relay/mod.rs lists scope_not_synced and scope_changed as retryable.
|
|
229
|
+
// A live scope freshly reported by core can also be scope_not_live until the daemon applies it.
|
|
230
|
+
return code === "scope_not_live" || code === "scope_not_synced" || code === "scope_changed";
|
|
231
|
+
}
|
|
118
232
|
/**
|
|
119
233
|
* A run of consecutive per-call failures that got no answer (timeout / route
|
|
120
234
|
* GOODBYE / refusal) on the SAME client raises the suspicion of a dead half-open
|
|
@@ -256,7 +370,7 @@ class BgSubscription {
|
|
|
256
370
|
identity;
|
|
257
371
|
acquireClient;
|
|
258
372
|
dropClient;
|
|
259
|
-
|
|
373
|
+
openRoute;
|
|
260
374
|
onNudge;
|
|
261
375
|
sleep;
|
|
262
376
|
canAttach;
|
|
@@ -267,23 +381,25 @@ class BgSubscription {
|
|
|
267
381
|
isCurrent;
|
|
268
382
|
noteClientAlive;
|
|
269
383
|
noteUnansweredCall;
|
|
384
|
+
onRouteAttached;
|
|
270
385
|
stopped = false;
|
|
271
386
|
/** The live subscription handle, read by stop() to wake the loop's `await closed`. */
|
|
272
387
|
current = null;
|
|
273
388
|
/** Client carrying the live subscription route, counted when that client is dropped. */
|
|
274
389
|
attached = null;
|
|
390
|
+
route = null;
|
|
275
391
|
loop;
|
|
276
392
|
lifecycleLogState = new Map();
|
|
277
393
|
nudgeReceiptLogState = null;
|
|
278
|
-
constructor(identity, acquireClient, dropClient,
|
|
394
|
+
constructor(identity, acquireClient, dropClient, openRoute, onNudge, sleep, canAttach, onRootAttachFailure, onDormant, dispatchProbeIntervalMs, nudgeRef, isCurrent = () => true,
|
|
279
395
|
/** Reports a reply seen on the client, which proves its socket still delivers. */
|
|
280
396
|
noteClientAlive = () => undefined,
|
|
281
397
|
/** Reports a call on the client that got no answer; enough in a row start a liveness probe. */
|
|
282
|
-
noteUnansweredCall = () => undefined) {
|
|
398
|
+
noteUnansweredCall = () => undefined, onRouteAttached = () => undefined) {
|
|
283
399
|
this.identity = identity;
|
|
284
400
|
this.acquireClient = acquireClient;
|
|
285
401
|
this.dropClient = dropClient;
|
|
286
|
-
this.
|
|
402
|
+
this.openRoute = openRoute;
|
|
287
403
|
this.onNudge = onNudge;
|
|
288
404
|
this.sleep = sleep;
|
|
289
405
|
this.canAttach = canAttach;
|
|
@@ -294,12 +410,16 @@ class BgSubscription {
|
|
|
294
410
|
this.isCurrent = isCurrent;
|
|
295
411
|
this.noteClientAlive = noteClientAlive;
|
|
296
412
|
this.noteUnansweredCall = noteUnansweredCall;
|
|
413
|
+
this.onRouteAttached = onRouteAttached;
|
|
297
414
|
this.loop = this.run();
|
|
298
415
|
}
|
|
299
416
|
/** The client whose route currently carries this subscription, if any. */
|
|
300
417
|
get attachedClient() {
|
|
301
418
|
return this.attached;
|
|
302
419
|
}
|
|
420
|
+
get attachedRoute() {
|
|
421
|
+
return this.route;
|
|
422
|
+
}
|
|
303
423
|
async stop() {
|
|
304
424
|
this.stopped = true;
|
|
305
425
|
// Wake a live `await sub.closed`; the loop's `finally` remains the sole
|
|
@@ -350,6 +470,11 @@ class BgSubscription {
|
|
|
350
470
|
const initial = client.droppedIngressFrames;
|
|
351
471
|
if (typeof initial !== "number")
|
|
352
472
|
return () => undefined;
|
|
473
|
+
const existing = dispatchProbes.get(client);
|
|
474
|
+
if (existing) {
|
|
475
|
+
existing.users += 1;
|
|
476
|
+
return () => releaseDispatchProbe(client);
|
|
477
|
+
}
|
|
353
478
|
let previous = initial;
|
|
354
479
|
const timer = setInterval(() => {
|
|
355
480
|
const total = client.droppedIngressFrames;
|
|
@@ -359,8 +484,10 @@ class BgSubscription {
|
|
|
359
484
|
previous = total;
|
|
360
485
|
this.info("dispatch-epoch-drop", `client ingress epoch drops scope=client observed_while_channel=${routeId} delta=${delta} total=${total}`);
|
|
361
486
|
}, this.dispatchProbeIntervalMs);
|
|
487
|
+
dispatchProbeTimers += 1;
|
|
362
488
|
timer.unref?.();
|
|
363
|
-
|
|
489
|
+
dispatchProbes.set(client, { users: 1, timer });
|
|
490
|
+
return () => releaseDispatchProbe(client);
|
|
364
491
|
}
|
|
365
492
|
async run() {
|
|
366
493
|
let backoffAttempt = 0;
|
|
@@ -421,7 +548,7 @@ class BgSubscription {
|
|
|
421
548
|
try {
|
|
422
549
|
// A dedicated route keeps bg_events independent of the tool route's
|
|
423
550
|
// credit window while preserving the client's connection-bound handle.
|
|
424
|
-
route = await
|
|
551
|
+
route = await this.openRoute(client);
|
|
425
552
|
}
|
|
426
553
|
catch (err) {
|
|
427
554
|
if (this.isCurrent() && this.onRootAttachFailure(err)) {
|
|
@@ -476,10 +603,12 @@ class BgSubscription {
|
|
|
476
603
|
if (!this.isCurrent()) {
|
|
477
604
|
this.info("nudge-stale-carrier", `nudge carried by stale subscription; checking current session channel=${routeId}`);
|
|
478
605
|
}
|
|
479
|
-
this.onNudge();
|
|
606
|
+
this.onNudge(true);
|
|
480
607
|
});
|
|
481
608
|
this.current = sub;
|
|
482
609
|
this.attached = client;
|
|
610
|
+
this.route = route;
|
|
611
|
+
this.onRouteAttached();
|
|
483
612
|
stopDispatchProbe = this.startDispatchProbe(client, routeId);
|
|
484
613
|
this.info("subscription-open", `subscription open channel=${routeId}`);
|
|
485
614
|
if (reconnecting) {
|
|
@@ -494,7 +623,7 @@ class BgSubscription {
|
|
|
494
623
|
// A reconnect may have missed completions, so force a drain after every
|
|
495
624
|
// successful subscribe to replay anything queued while the stream was down.
|
|
496
625
|
if (!this.stopped && this.isCurrent())
|
|
497
|
-
this.onNudge();
|
|
626
|
+
this.onNudge(false);
|
|
498
627
|
await sub.closed;
|
|
499
628
|
this.info("stream-end", `stream ended channel=${routeId}`);
|
|
500
629
|
if (this.stopped) {
|
|
@@ -530,6 +659,7 @@ class BgSubscription {
|
|
|
530
659
|
stopDispatchProbe();
|
|
531
660
|
this.current = null;
|
|
532
661
|
this.attached = null;
|
|
662
|
+
this.route = null;
|
|
533
663
|
safeCloseRoute(client, route);
|
|
534
664
|
}
|
|
535
665
|
await this.backoff(backoffAttempt++);
|
|
@@ -545,7 +675,8 @@ function isRecord(value) {
|
|
|
545
675
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
546
676
|
}
|
|
547
677
|
function isRouteProvenAbsentError(err) {
|
|
548
|
-
return ((err instanceof SubcError &&
|
|
678
|
+
return ((err instanceof SubcError &&
|
|
679
|
+
(err.code === "unknown_channel" || err.code === "stale_route_epoch")) ||
|
|
549
680
|
err instanceof StaleRouteHandleError);
|
|
550
681
|
}
|
|
551
682
|
/**
|
|
@@ -610,6 +741,21 @@ function abortedRequestError() {
|
|
|
610
741
|
error.name = "AbortError";
|
|
611
742
|
return error;
|
|
612
743
|
}
|
|
744
|
+
// The metric is connection-wide, so observing it needs one timer, not one per
|
|
745
|
+
// session. Last unsubscribe releases the timer, including on a dead socket.
|
|
746
|
+
const dispatchProbes = new Map();
|
|
747
|
+
let dispatchProbeTimers = 0;
|
|
748
|
+
function releaseDispatchProbe(client) {
|
|
749
|
+
const probe = dispatchProbes.get(client);
|
|
750
|
+
if (!probe || --probe.users > 0)
|
|
751
|
+
return;
|
|
752
|
+
clearInterval(probe.timer);
|
|
753
|
+
dispatchProbeTimers -= 1;
|
|
754
|
+
dispatchProbes.delete(client);
|
|
755
|
+
}
|
|
756
|
+
export function __dispatchProbeCountForTests() {
|
|
757
|
+
return dispatchProbeTimers;
|
|
758
|
+
}
|
|
613
759
|
/**
|
|
614
760
|
* A route open that resolved AFTER its session was torn down (closeSession /
|
|
615
761
|
* shutdown / client swap). The caller's request can't proceed, but this is an
|
|
@@ -618,6 +764,25 @@ function abortedRequestError() {
|
|
|
618
764
|
*/
|
|
619
765
|
class RouteTornDownError extends Error {
|
|
620
766
|
}
|
|
767
|
+
/**
|
|
768
|
+
* True when a call stopped at a teardown before its request was handed to the
|
|
769
|
+
* client (a closed session, or a route that opened after the teardown), so the
|
|
770
|
+
* request provably never reached AFT.
|
|
771
|
+
*/
|
|
772
|
+
export function isSubcRouteTornDownError(error) {
|
|
773
|
+
return error instanceof RouteTornDownError;
|
|
774
|
+
}
|
|
775
|
+
/**
|
|
776
|
+
* Turn subc-client's bare rejection of a request that was in flight on a route
|
|
777
|
+
* the plugin closed into an outcome-unknown error that names the close reason.
|
|
778
|
+
* Any other error passes through unchanged.
|
|
779
|
+
*/
|
|
780
|
+
function routeClosedMidCallError(error, entry, record) {
|
|
781
|
+
if (!isLocalRouteCloseRejection(error))
|
|
782
|
+
return error;
|
|
783
|
+
const reason = teardownCloseReason(record.teardownReason) ?? entry.closedBy ?? "route closed by the plugin";
|
|
784
|
+
return new SubcRouteClosedMidCallError(reason, error);
|
|
785
|
+
}
|
|
621
786
|
/**
|
|
622
787
|
* Re-lift the route reply into the flat {@link ToolCallResult} shape the standalone
|
|
623
788
|
* `BinaryBridge.toolCall` returns. The Rust module wraps the full flat response
|
|
@@ -709,6 +874,9 @@ class SubcTransport {
|
|
|
709
874
|
body.preview = true;
|
|
710
875
|
if (options?.workerSession === true)
|
|
711
876
|
body[WORKER_SESSION_FIELD] = true;
|
|
877
|
+
if (options?.hostToolNames)
|
|
878
|
+
body[HOST_TOOL_NAMES_FIELD] = options.hostToolNames;
|
|
879
|
+
body[PRESET_FIELD] = callPresetFor(options?.workerSession === true);
|
|
712
880
|
const reply = await this.pool.routeRequest(this.identityFor(sessionId), body, timeoutMs, onProgress, this.generation, abortSignal);
|
|
713
881
|
return reliftReply(reply);
|
|
714
882
|
}
|
|
@@ -732,10 +900,15 @@ class SubcTransport {
|
|
|
732
900
|
// The caller's role travels beside the call, like the session (which the
|
|
733
901
|
// route bind carries), not inside the command's arguments: the module
|
|
734
902
|
// reads it from the call body.
|
|
735
|
-
|
|
903
|
+
// The caller's tool names travel the same way: RouteBind has no field for
|
|
904
|
+
// them, and one route's host must not decide another route's wording.
|
|
905
|
+
const { [WORKER_SESSION_FIELD]: workerSession, [HOST_TOOL_NAMES_FIELD]: hostToolNames, ...args } = params;
|
|
736
906
|
const body = { name: command, arguments: args };
|
|
737
907
|
if (workerSession === true)
|
|
738
908
|
body[WORKER_SESSION_FIELD] = true;
|
|
909
|
+
if (hostToolNames !== undefined)
|
|
910
|
+
body[HOST_TOOL_NAMES_FIELD] = hostToolNames;
|
|
911
|
+
body[PRESET_FIELD] = callPresetFor(workerSession === true);
|
|
739
912
|
const editSlotSurvives = this.pool.getEditSlotSurvives();
|
|
740
913
|
if (editSlotSurvives !== undefined)
|
|
741
914
|
body.edit_slot_survives = editSlotSurvives;
|
|
@@ -765,6 +938,11 @@ export class SubcTransportPool {
|
|
|
765
938
|
connectionFile;
|
|
766
939
|
handshakeTimeoutMs;
|
|
767
940
|
consumerIdentity;
|
|
941
|
+
hostScopes;
|
|
942
|
+
/** Scope identity is host-session-wide, even when the session uses multiple project roots. */
|
|
943
|
+
hostScopeCache = new Map();
|
|
944
|
+
hostScopeRefusals = new Set();
|
|
945
|
+
hostScopeOutcomes = new Map();
|
|
768
946
|
connectFn;
|
|
769
947
|
onBgEventsNudge;
|
|
770
948
|
onBgEventsNudgeRef;
|
|
@@ -791,6 +969,12 @@ export class SubcTransportPool {
|
|
|
791
969
|
routeReopenRetryMs = null;
|
|
792
970
|
/** Per-session records keyed by the opaque identity key. */
|
|
793
971
|
sessions = new Map();
|
|
972
|
+
/** Keep global insertion order so a session using several roots selects the same first live route. */
|
|
973
|
+
sessionIndex = new Map();
|
|
974
|
+
// Pending host creation events are bounded. Forgetting one only makes that
|
|
975
|
+
// session unknown (retained), never falsely eligible for eviction.
|
|
976
|
+
observedStarts = new Set();
|
|
977
|
+
sessionSweep = null;
|
|
794
978
|
/**
|
|
795
979
|
* The sole root-scoped session enumeration authority. Keys are opaque and are
|
|
796
980
|
* removed by the same detacher that removes the corresponding session record.
|
|
@@ -820,6 +1004,7 @@ export class SubcTransportPool {
|
|
|
820
1004
|
this.harness = options.harness;
|
|
821
1005
|
this.handshakeTimeoutMs = options.handshakeTimeoutMs;
|
|
822
1006
|
this.consumerIdentity = options.consumerIdentity;
|
|
1007
|
+
this.hostScopes = options.hostScopes ?? false;
|
|
823
1008
|
this.connectFn = options.connect ?? ((opts) => SubcClient.connect(opts));
|
|
824
1009
|
this.onBgEventsNudge = options.onBgEventsNudge;
|
|
825
1010
|
this.onBgEventsNudgeRef = options.onBgEventsNudgeRef;
|
|
@@ -912,9 +1097,10 @@ export class SubcTransportPool {
|
|
|
912
1097
|
return asCanonicalRootPath(canonicalizeProjectRoot(projectRoot));
|
|
913
1098
|
}
|
|
914
1099
|
/**
|
|
915
|
-
* Check a root immediately before opening a route.
|
|
916
|
-
*
|
|
917
|
-
*
|
|
1100
|
+
* Check a root immediately before opening a route. Only the directory's
|
|
1101
|
+
* existence decides: a reclaim marker beside it (`<root>.reclaimed`) needs
|
|
1102
|
+
* no check, since an absent directory suspends the bind either way and a
|
|
1103
|
+
* directory that has returned wins over a stale marker.
|
|
918
1104
|
*/
|
|
919
1105
|
rootCanAttach(root) {
|
|
920
1106
|
let directoryExists = false;
|
|
@@ -928,13 +1114,6 @@ export class SubcTransportPool {
|
|
|
928
1114
|
this.dormantRoots.delete(root);
|
|
929
1115
|
return true;
|
|
930
1116
|
}
|
|
931
|
-
// Read the marker only as a latency hint. Directory absence is sufficient
|
|
932
|
-
// to suspend the bind, and the marker's JSON is intentionally irrelevant.
|
|
933
|
-
const reclaimedMarkerPresent = existsSync(`${root}.reclaimed`);
|
|
934
|
-
if (reclaimedMarkerPresent) {
|
|
935
|
-
this.markRootDormant(root);
|
|
936
|
-
return false;
|
|
937
|
-
}
|
|
938
1117
|
this.markRootDormant(root);
|
|
939
1118
|
return false;
|
|
940
1119
|
}
|
|
@@ -1084,7 +1263,9 @@ export class SubcTransportPool {
|
|
|
1084
1263
|
* always has a registered root and captured generation.
|
|
1085
1264
|
*/
|
|
1086
1265
|
getBridge(projectRoot) {
|
|
1087
|
-
|
|
1266
|
+
return this.getBridgeForCanonicalRoot(this.canonicalRoot(projectRoot));
|
|
1267
|
+
}
|
|
1268
|
+
getBridgeForCanonicalRoot(root) {
|
|
1088
1269
|
if (this.shuttingDown && this.lifecycleEnabled()) {
|
|
1089
1270
|
throw new SubcTransportShuttingDownError();
|
|
1090
1271
|
}
|
|
@@ -1166,6 +1347,7 @@ export class SubcTransportPool {
|
|
|
1166
1347
|
if (this.lifecycleEnabled() && record.generation !== generation) {
|
|
1167
1348
|
throw this.generationExpiredError(root, generation ?? asRootGeneration(1));
|
|
1168
1349
|
}
|
|
1350
|
+
record.lastUsedAt = Date.now();
|
|
1169
1351
|
return record;
|
|
1170
1352
|
}
|
|
1171
1353
|
record = {
|
|
@@ -1174,20 +1356,199 @@ export class SubcTransportPool {
|
|
|
1174
1356
|
canonicalRoot: root,
|
|
1175
1357
|
generation,
|
|
1176
1358
|
routeEntry: null,
|
|
1359
|
+
retiredRoutes: new Set(),
|
|
1177
1360
|
bgSub: null,
|
|
1178
1361
|
closed: false,
|
|
1179
1362
|
teardownReason: null,
|
|
1180
1363
|
inflight: 0,
|
|
1364
|
+
lastUsedAt: Date.now(),
|
|
1365
|
+
observedFromStart: this.observedStarts.delete(key),
|
|
1366
|
+
backgroundTasks: new Map(),
|
|
1367
|
+
backgroundWatches: new Map(),
|
|
1368
|
+
unknownBackgroundWork: false,
|
|
1369
|
+
pendingBgNudge: false,
|
|
1370
|
+
pendingDetach: false,
|
|
1181
1371
|
};
|
|
1182
1372
|
this.sessions.set(key, record);
|
|
1373
|
+
let records = this.sessionIndex.get(identity.session);
|
|
1374
|
+
if (!records) {
|
|
1375
|
+
records = new Set();
|
|
1376
|
+
this.sessionIndex.set(identity.session, records);
|
|
1377
|
+
}
|
|
1378
|
+
records.add(record);
|
|
1183
1379
|
let keys = this.rootIndex.get(root);
|
|
1184
1380
|
if (!keys) {
|
|
1185
1381
|
keys = new Set();
|
|
1186
1382
|
this.rootIndex.set(root, keys);
|
|
1187
1383
|
}
|
|
1188
1384
|
keys.add(key);
|
|
1385
|
+
if (!this.sessionSweep) {
|
|
1386
|
+
this.sessionSweep = setInterval(() => {
|
|
1387
|
+
void this.reapIdleSessions();
|
|
1388
|
+
}, 60_000);
|
|
1389
|
+
this.sessionSweep.unref?.();
|
|
1390
|
+
}
|
|
1189
1391
|
return record;
|
|
1190
1392
|
}
|
|
1393
|
+
/** Release only idle carriers, not the daemon's durable session or completion queue.
|
|
1394
|
+
* The next request opens a new route and subscribes again; subscription startup
|
|
1395
|
+
* always nudges a drain, replaying completions queued while no carrier existed.
|
|
1396
|
+
*/
|
|
1397
|
+
async reapIdleSessions(now = Date.now()) {
|
|
1398
|
+
for (const [key, state] of this.nudgeDeliveryLogState) {
|
|
1399
|
+
if (now - state.lastEmittedAt >= 15 * 60_000)
|
|
1400
|
+
this.nudgeDeliveryLogState.delete(key);
|
|
1401
|
+
}
|
|
1402
|
+
const detached = [];
|
|
1403
|
+
for (const [key, record] of this.sessions) {
|
|
1404
|
+
if (
|
|
1405
|
+
// The wire has no session-wide background census. A resumed/rebound
|
|
1406
|
+
// session may have tasks or watches this pool never saw, so only a
|
|
1407
|
+
// host-observed creation can establish a complete lifetime history.
|
|
1408
|
+
!record.observedFromStart ||
|
|
1409
|
+
record.backgroundTasks.size > 0 ||
|
|
1410
|
+
record.backgroundWatches.size > 0 ||
|
|
1411
|
+
record.unknownBackgroundWork ||
|
|
1412
|
+
record.pendingBgNudge ||
|
|
1413
|
+
record.pendingDetach ||
|
|
1414
|
+
record.inflight > 0 ||
|
|
1415
|
+
record.retiredRoutes.size > 0 ||
|
|
1416
|
+
record.routeEntry?.opening ||
|
|
1417
|
+
now - record.lastUsedAt < 15 * 60_000)
|
|
1418
|
+
continue;
|
|
1419
|
+
const owner = this.detachSession(key, "session_closed");
|
|
1420
|
+
if (owner)
|
|
1421
|
+
detached.push(owner);
|
|
1422
|
+
}
|
|
1423
|
+
if (this.sessions.size === 0 && this.sessionSweep) {
|
|
1424
|
+
clearInterval(this.sessionSweep);
|
|
1425
|
+
this.sessionSweep = null;
|
|
1426
|
+
}
|
|
1427
|
+
const cleanup = Promise.allSettled(detached.map((owner) => this.cleanupDetached(owner)));
|
|
1428
|
+
this.pendingRootCleanups.add(cleanup);
|
|
1429
|
+
try {
|
|
1430
|
+
await cleanup;
|
|
1431
|
+
}
|
|
1432
|
+
finally {
|
|
1433
|
+
this.pendingRootCleanups.delete(cleanup);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
observeSessionStart(projectRoot, session) {
|
|
1437
|
+
if (this.shuttingDown || !session)
|
|
1438
|
+
return;
|
|
1439
|
+
const key = identityKey({
|
|
1440
|
+
project_root: canonicalizeProjectRoot(projectRoot),
|
|
1441
|
+
harness: this.harness,
|
|
1442
|
+
session,
|
|
1443
|
+
});
|
|
1444
|
+
const record = this.sessions.get(key);
|
|
1445
|
+
if (record) {
|
|
1446
|
+
record.observedFromStart = true;
|
|
1447
|
+
return;
|
|
1448
|
+
}
|
|
1449
|
+
if (this.observedStarts.size >= 256)
|
|
1450
|
+
this.observedStarts.delete(this.observedStarts.values().next().value);
|
|
1451
|
+
this.observedStarts.add(key);
|
|
1452
|
+
}
|
|
1453
|
+
__pendingSessionStartsForTests() {
|
|
1454
|
+
return this.observedStarts.size;
|
|
1455
|
+
}
|
|
1456
|
+
observeBackgroundReply(record, body, reply) {
|
|
1457
|
+
const name = String(body.name);
|
|
1458
|
+
const args = isRecord(body.arguments) ? body.arguments : {};
|
|
1459
|
+
const data = isRecord(reply) && isRecord(reply.structuredContent) ? reply.structuredContent : null;
|
|
1460
|
+
const affectsBackground = name === "bash" ||
|
|
1461
|
+
name === "powershell" ||
|
|
1462
|
+
[
|
|
1463
|
+
"bash_status",
|
|
1464
|
+
"bash_kill",
|
|
1465
|
+
"bash_write",
|
|
1466
|
+
"bash_promote",
|
|
1467
|
+
"bash_watch",
|
|
1468
|
+
"bash_notify",
|
|
1469
|
+
"bash_unnotify",
|
|
1470
|
+
"bash_wait_detach",
|
|
1471
|
+
"bash_drain_completions",
|
|
1472
|
+
"bash_ack_completions",
|
|
1473
|
+
].includes(name);
|
|
1474
|
+
if (!data || data.success !== true) {
|
|
1475
|
+
if (affectsBackground) {
|
|
1476
|
+
if (typeof args.task_id === "string") {
|
|
1477
|
+
if (!record.backgroundTasks.has(args.task_id))
|
|
1478
|
+
record.backgroundTasks.set(args.task_id, "unknown");
|
|
1479
|
+
}
|
|
1480
|
+
else
|
|
1481
|
+
record.unknownBackgroundWork = true;
|
|
1482
|
+
}
|
|
1483
|
+
return;
|
|
1484
|
+
}
|
|
1485
|
+
if (typeof data.task_id === "string") {
|
|
1486
|
+
// Terminal tasks remain pinned until the daemon explicitly ACKs them.
|
|
1487
|
+
const terminal = ["completed", "failed", "killed", "timed_out"].includes(String(data.status));
|
|
1488
|
+
const running = ["starting", "running", "killing"].includes(String(data.status));
|
|
1489
|
+
record.backgroundTasks.set(data.task_id, terminal ? "terminal" : running ? "running" : "unknown");
|
|
1490
|
+
if (name === "bash" || name === "powershell")
|
|
1491
|
+
record.pendingDetach = false;
|
|
1492
|
+
}
|
|
1493
|
+
if (Array.isArray(data.bg_completions)) {
|
|
1494
|
+
for (const completion of data.bg_completions) {
|
|
1495
|
+
if (isRecord(completion) && typeof completion.task_id === "string")
|
|
1496
|
+
record.backgroundTasks.set(completion.task_id, "terminal");
|
|
1497
|
+
else
|
|
1498
|
+
record.unknownBackgroundWork = true;
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
if (Array.isArray(data.pending_matches)) {
|
|
1502
|
+
for (const match of data.pending_matches) {
|
|
1503
|
+
if (isRecord(match) &&
|
|
1504
|
+
typeof match.watch_id === "string" &&
|
|
1505
|
+
typeof match.task_id === "string")
|
|
1506
|
+
record.backgroundWatches.set(match.watch_id, match.task_id);
|
|
1507
|
+
else
|
|
1508
|
+
record.unknownBackgroundWork = true;
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
if (name === "bash_drain_completions" &&
|
|
1512
|
+
Array.isArray(data.bg_completions) &&
|
|
1513
|
+
Array.isArray(data.pending_matches))
|
|
1514
|
+
record.pendingBgNudge = false;
|
|
1515
|
+
if (name === "bash_notify") {
|
|
1516
|
+
if (typeof data.watch_id === "string" && typeof args.task_id === "string")
|
|
1517
|
+
record.backgroundWatches.set(data.watch_id, args.task_id);
|
|
1518
|
+
else
|
|
1519
|
+
record.unknownBackgroundWork = true;
|
|
1520
|
+
}
|
|
1521
|
+
if (name === "bash_unnotify" && data.unregistered === true && typeof args.watch_id === "string")
|
|
1522
|
+
record.backgroundWatches.delete(args.watch_id);
|
|
1523
|
+
if (name === "bash_ack_completions" && Array.isArray(data.acked_task_ids)) {
|
|
1524
|
+
// ACK also acknowledges mid-run pattern matches. It does not prove a
|
|
1525
|
+
// running/unknown task is terminal, so those must keep their carrier.
|
|
1526
|
+
for (const task of data.acked_task_ids)
|
|
1527
|
+
if (typeof task === "string" && record.backgroundTasks.get(task) === "terminal") {
|
|
1528
|
+
record.backgroundTasks.delete(task);
|
|
1529
|
+
// The daemon's terminal ACK also removes every watch on that task.
|
|
1530
|
+
for (const [watch, owner] of record.backgroundWatches) {
|
|
1531
|
+
if (owner === task)
|
|
1532
|
+
record.backgroundWatches.delete(watch);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
if (name === "bash_wait_detach") {
|
|
1537
|
+
if (typeof data.detached === "boolean")
|
|
1538
|
+
record.pendingDetach = data.detached;
|
|
1539
|
+
else
|
|
1540
|
+
record.unknownBackgroundWork = true;
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
__retainedSessionCountsForTests() {
|
|
1544
|
+
return {
|
|
1545
|
+
sessions: this.sessions.size,
|
|
1546
|
+
roots: this.rootIndex.size,
|
|
1547
|
+
routes: [...this.sessions.values()].filter((r) => r.routeEntry !== null).length,
|
|
1548
|
+
subscriptions: [...this.sessions.values()].filter((r) => r.bgSub !== null).length,
|
|
1549
|
+
sweepTimers: this.sessionSweep ? 1 : 0,
|
|
1550
|
+
};
|
|
1551
|
+
}
|
|
1191
1552
|
isCurrentSession(key, record) {
|
|
1192
1553
|
return this.sessions.get(key) === record && !record.closed;
|
|
1193
1554
|
}
|
|
@@ -1221,6 +1582,10 @@ export class SubcTransportPool {
|
|
|
1221
1582
|
log(`subc bg_events: ${message}${suffix}`, { sessionId: record.identity.session });
|
|
1222
1583
|
}
|
|
1223
1584
|
removeIndexMembership(record) {
|
|
1585
|
+
const records = this.sessionIndex.get(record.identity.session);
|
|
1586
|
+
records?.delete(record);
|
|
1587
|
+
if (records?.size === 0)
|
|
1588
|
+
this.sessionIndex.delete(record.identity.session);
|
|
1224
1589
|
const keys = this.rootIndex.get(record.canonicalRoot);
|
|
1225
1590
|
if (!keys)
|
|
1226
1591
|
return;
|
|
@@ -1232,11 +1597,18 @@ export class SubcTransportPool {
|
|
|
1232
1597
|
if (this.sessions.get(record.identityKey) === record &&
|
|
1233
1598
|
!record.closed &&
|
|
1234
1599
|
record.inflight === 0 &&
|
|
1600
|
+
record.retiredRoutes.size === 0 &&
|
|
1601
|
+
record.backgroundTasks.size === 0 &&
|
|
1602
|
+
record.backgroundWatches.size === 0 &&
|
|
1603
|
+
!record.unknownBackgroundWork &&
|
|
1604
|
+
!record.pendingBgNudge &&
|
|
1605
|
+
!record.pendingDetach &&
|
|
1235
1606
|
record.routeEntry === null &&
|
|
1236
1607
|
record.bgSub === null) {
|
|
1237
1608
|
this.sessions.delete(record.identityKey);
|
|
1238
1609
|
this.removeIndexMembership(record);
|
|
1239
1610
|
}
|
|
1611
|
+
this.stopUnusedHostScopeRetry(record.identity.session);
|
|
1240
1612
|
}
|
|
1241
1613
|
/**
|
|
1242
1614
|
* Atomically detaches one opaque identity. Nothing in this method awaits: it
|
|
@@ -1250,26 +1622,36 @@ export class SubcTransportPool {
|
|
|
1250
1622
|
record.teardownReason = reason;
|
|
1251
1623
|
this.sessions.delete(key);
|
|
1252
1624
|
this.removeIndexMembership(record);
|
|
1625
|
+
for (const logKey of this.nudgeDeliveryLogState.keys()) {
|
|
1626
|
+
if (logKey.endsWith(`\u0000${key}`))
|
|
1627
|
+
this.nudgeDeliveryLogState.delete(logKey);
|
|
1628
|
+
}
|
|
1253
1629
|
const bgSub = record.bgSub;
|
|
1254
1630
|
record.bgSub = null;
|
|
1255
1631
|
const routeEntry = record.routeEntry;
|
|
1256
1632
|
record.routeEntry = null;
|
|
1257
1633
|
if (routeEntry)
|
|
1258
1634
|
routeEntry.closed = true;
|
|
1635
|
+
this.stopUnusedHostScopeRetry(record.identity.session);
|
|
1259
1636
|
return { record, bgSub, routeEntry };
|
|
1260
1637
|
}
|
|
1261
1638
|
async cleanupDetached(detached) {
|
|
1262
1639
|
const cleanup = [];
|
|
1263
1640
|
if (detached.bgSub)
|
|
1264
1641
|
cleanup.push(detached.bgSub.stop());
|
|
1265
|
-
const
|
|
1266
|
-
|
|
1267
|
-
if (routeEntry
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1642
|
+
const entries = new Set(detached.record.retiredRoutes);
|
|
1643
|
+
detached.record.retiredRoutes.clear();
|
|
1644
|
+
if (detached.routeEntry)
|
|
1645
|
+
entries.add(detached.routeEntry);
|
|
1646
|
+
for (const entry of entries) {
|
|
1647
|
+
entry.closed = true;
|
|
1648
|
+
if (entry.handle != null) {
|
|
1649
|
+
try {
|
|
1650
|
+
cleanup.push(Promise.resolve(entry.client.closeRouteChannel(entry.handle)).catch(() => undefined));
|
|
1651
|
+
}
|
|
1652
|
+
catch {
|
|
1653
|
+
// A client may throw synchronously when its socket is already closed.
|
|
1654
|
+
}
|
|
1273
1655
|
}
|
|
1274
1656
|
}
|
|
1275
1657
|
await Promise.allSettled(cleanup);
|
|
@@ -1299,6 +1681,28 @@ export class SubcTransportPool {
|
|
|
1299
1681
|
}
|
|
1300
1682
|
return this.rootReapedError(record);
|
|
1301
1683
|
}
|
|
1684
|
+
closeRetiredRoute(record, entry) {
|
|
1685
|
+
if (entry.inflight > 0 || !record.retiredRoutes.delete(entry))
|
|
1686
|
+
return;
|
|
1687
|
+
if (entry.handle != null)
|
|
1688
|
+
safeCloseRoute(entry.client, entry.handle);
|
|
1689
|
+
this.deleteSessionIfEmpty(record.identityKey, record);
|
|
1690
|
+
}
|
|
1691
|
+
releaseRouteRequest(record, entry) {
|
|
1692
|
+
entry.inflight -= 1;
|
|
1693
|
+
this.closeRetiredRoute(record, entry);
|
|
1694
|
+
}
|
|
1695
|
+
retireHostToolRoute(record) {
|
|
1696
|
+
const entry = record.routeEntry;
|
|
1697
|
+
if (!entry)
|
|
1698
|
+
return;
|
|
1699
|
+
entry.closed = true;
|
|
1700
|
+
entry.closedBy ??= "route replaced";
|
|
1701
|
+
record.routeEntry = null;
|
|
1702
|
+
record.retiredRoutes.add(entry);
|
|
1703
|
+
this.closeRetiredRoute(record, entry);
|
|
1704
|
+
this.stopUnusedHostScopeRetry(record.identity.session);
|
|
1705
|
+
}
|
|
1302
1706
|
/** Open or reuse a route while guarding every lifecycle boundary. */
|
|
1303
1707
|
async routeRequest(identity, body, timeoutMs, onProgress, expectedGeneration, abortSignal) {
|
|
1304
1708
|
const callStartedAt = performance.now();
|
|
@@ -1470,16 +1874,16 @@ export class SubcTransportPool {
|
|
|
1470
1874
|
const clearRouteEntry = (entry) => {
|
|
1471
1875
|
if (record.routeEntry !== entry)
|
|
1472
1876
|
return;
|
|
1473
|
-
|
|
1474
|
-
record.routeEntry = null;
|
|
1475
|
-
if (entry.handle != null)
|
|
1476
|
-
safeCloseRoute(entry.client, entry.handle);
|
|
1877
|
+
this.retireHostToolRoute(record);
|
|
1477
1878
|
};
|
|
1478
1879
|
const handleRequestFailure = (error, entry) => {
|
|
1479
1880
|
const aborted = error instanceof Error && error.name === "AbortError";
|
|
1480
1881
|
const ownsClient = this.isCurrentSession(key, record) &&
|
|
1481
1882
|
this.client === client &&
|
|
1482
1883
|
!this.isReapInduced(record);
|
|
1884
|
+
if (ownsClient && error instanceof SubcError && error.closeReason?.startsWith("scope_")) {
|
|
1885
|
+
this.invalidateHostScope(identity.session);
|
|
1886
|
+
}
|
|
1483
1887
|
if (!aborted && ownsClient && isSocketDeathError(error)) {
|
|
1484
1888
|
// Dropped before the route is cleared so the drop log counts it.
|
|
1485
1889
|
this.dropClient(client, {
|
|
@@ -1489,20 +1893,31 @@ export class SubcTransportPool {
|
|
|
1489
1893
|
route: routeLabel(entry.handle),
|
|
1490
1894
|
});
|
|
1491
1895
|
}
|
|
1492
|
-
|
|
1896
|
+
// A call's timeout or refusal says nothing about the route's health.
|
|
1897
|
+
// Retire only a cancelled carrier or one known to be gone; a slow live
|
|
1898
|
+
// route remains reusable, and no replacement may cut its other calls.
|
|
1899
|
+
if (aborted ||
|
|
1900
|
+
isRouteProvenAbsentError(error) ||
|
|
1901
|
+
(error instanceof SubcError && error.closeReason !== undefined) ||
|
|
1902
|
+
isSubcFrameReadTimeoutError(error)) {
|
|
1903
|
+
clearRouteEntry(entry);
|
|
1904
|
+
}
|
|
1493
1905
|
if (aborted || !ownsClient || this.client !== client)
|
|
1494
1906
|
return;
|
|
1495
1907
|
if (isRouteProvenAbsentError(error))
|
|
1496
1908
|
return;
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1909
|
+
if (isDaemonAnswerError(error)) {
|
|
1910
|
+
this.transportFailures = 0;
|
|
1911
|
+
this.noteClientAlive(client);
|
|
1912
|
+
}
|
|
1913
|
+
else if (isUnansweredCallError(error)) {
|
|
1914
|
+
this.noteUnansweredCall(client, {
|
|
1915
|
+
error,
|
|
1916
|
+
trigger: "request",
|
|
1917
|
+
session: identity.session,
|
|
1918
|
+
route: routeLabel(entry.handle),
|
|
1919
|
+
});
|
|
1920
|
+
}
|
|
1506
1921
|
};
|
|
1507
1922
|
const requestOnRoute = async (route, entry, requestTimeoutMs) => {
|
|
1508
1923
|
this.assertRecordLive(record);
|
|
@@ -1520,10 +1935,22 @@ export class SubcTransportPool {
|
|
|
1520
1935
|
};
|
|
1521
1936
|
abortSignal?.addEventListener("abort", onAbort, { once: true });
|
|
1522
1937
|
try {
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1938
|
+
// Track the actual client promise, not the abort race. A cancelled
|
|
1939
|
+
// caller has no terminal from the daemon yet, so closing its carrier
|
|
1940
|
+
// here would still cut sibling calls and invalidate request credits.
|
|
1941
|
+
entry.inflight += 1;
|
|
1942
|
+
let request;
|
|
1943
|
+
try {
|
|
1944
|
+
request = client.request(route, body, {
|
|
1945
|
+
timeoutMs: Math.ceil(requestTimeoutMs),
|
|
1946
|
+
onProgress,
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
1949
|
+
catch (error) {
|
|
1950
|
+
this.releaseRouteRequest(record, entry);
|
|
1951
|
+
throw error;
|
|
1952
|
+
}
|
|
1953
|
+
request = request.finally(() => this.releaseRouteRequest(record, entry));
|
|
1527
1954
|
const reply = abortSignal ? await Promise.race([request, aborted]) : await request;
|
|
1528
1955
|
// A legacy closeSession may intentionally let an already-delivered reply
|
|
1529
1956
|
// settle. It must not mutate shared state or recreate a subscription.
|
|
@@ -1536,6 +1963,7 @@ export class SubcTransportPool {
|
|
|
1536
1963
|
if (this.client === client)
|
|
1537
1964
|
this.transportFailures = 0;
|
|
1538
1965
|
this.noteClientAlive(client);
|
|
1966
|
+
this.observeBackgroundReply(record, body, reply);
|
|
1539
1967
|
this.ensureBgSubscription(identity, record);
|
|
1540
1968
|
return reply;
|
|
1541
1969
|
}
|
|
@@ -1547,6 +1975,10 @@ export class SubcTransportPool {
|
|
|
1547
1975
|
let reopened = false;
|
|
1548
1976
|
let retriedAbsentRoute = false;
|
|
1549
1977
|
while (true) {
|
|
1978
|
+
// Another call can retire a just-acquired carrier before this caller
|
|
1979
|
+
// dispatches. Only already-pending requests belong on retired routes.
|
|
1980
|
+
if (routeAndEntry.entry.closed)
|
|
1981
|
+
routeAndEntry = await openRouteAfterReloadWindow();
|
|
1550
1982
|
// Checked outside the try below so the retry branches never mistake
|
|
1551
1983
|
// this call's own expiry for a daemon refusal worth retrying.
|
|
1552
1984
|
const requestTimeoutMs = remainingMs();
|
|
@@ -1558,10 +1990,16 @@ export class SubcTransportPool {
|
|
|
1558
1990
|
this.resetRouteReopenBackoff();
|
|
1559
1991
|
return reply;
|
|
1560
1992
|
}
|
|
1561
|
-
catch (
|
|
1993
|
+
catch (requestError) {
|
|
1994
|
+
const error = frameReadTimeoutError(routeClosedMidCallError(requestError, routeAndEntry.entry, record));
|
|
1562
1995
|
if (this.isReapInduced(record))
|
|
1563
1996
|
throw this.annotateReapError(error, record);
|
|
1564
1997
|
const ownsRoute = this.isCurrentSession(key, record) && this.client === client;
|
|
1998
|
+
if (ownsRoute &&
|
|
1999
|
+
((error instanceof SubcError && error.code === "stale_route_epoch") ||
|
|
2000
|
+
error instanceof StaleRouteHandleError)) {
|
|
2001
|
+
this.invalidateHostScope(identity.session);
|
|
2002
|
+
}
|
|
1565
2003
|
// A route that was bound before a module reload started gets its
|
|
1566
2004
|
// requests refused by the daemon for as long as the drain lasts. The
|
|
1567
2005
|
// refusal happens before the request is forwarded, so it is handled
|
|
@@ -1607,10 +2045,14 @@ export class SubcTransportPool {
|
|
|
1607
2045
|
catch (error) {
|
|
1608
2046
|
if (this.isReapInduced(record))
|
|
1609
2047
|
throw this.annotateReapError(error, record);
|
|
2048
|
+
this.observeBackgroundReply(record, body, null);
|
|
2049
|
+
if (record.backgroundTasks.size > 0 || record.unknownBackgroundWork)
|
|
2050
|
+
this.ensureBgSubscription(identity, record);
|
|
1610
2051
|
throw error;
|
|
1611
2052
|
}
|
|
1612
2053
|
finally {
|
|
1613
2054
|
record.inflight -= 1;
|
|
2055
|
+
record.lastUsedAt = Date.now();
|
|
1614
2056
|
this.deleteSessionIfEmpty(key, record);
|
|
1615
2057
|
}
|
|
1616
2058
|
}
|
|
@@ -1655,9 +2097,14 @@ export class SubcTransportPool {
|
|
|
1655
2097
|
return this.client;
|
|
1656
2098
|
if (this.connecting)
|
|
1657
2099
|
return this.connecting;
|
|
2100
|
+
let owner;
|
|
1658
2101
|
this.connecting = this.connectFn({
|
|
1659
2102
|
connectionFile: this.connectionFile,
|
|
1660
2103
|
handshakeTimeoutMs: this.handshakeTimeoutMs,
|
|
2104
|
+
onControlPush: (push) => {
|
|
2105
|
+
if (owner && this.client === owner)
|
|
2106
|
+
this.onScopeRouteClose(push);
|
|
2107
|
+
},
|
|
1661
2108
|
})
|
|
1662
2109
|
.then((client) => {
|
|
1663
2110
|
this.connecting = null;
|
|
@@ -1671,6 +2118,7 @@ export class SubcTransportPool {
|
|
|
1671
2118
|
throw new SubcTransportShuttingDownError();
|
|
1672
2119
|
}
|
|
1673
2120
|
this.client = client;
|
|
2121
|
+
owner = client;
|
|
1674
2122
|
this.hadWorkingClient = true;
|
|
1675
2123
|
this.transportFailures = 0;
|
|
1676
2124
|
return client;
|
|
@@ -1700,12 +2148,11 @@ export class SubcTransportPool {
|
|
|
1700
2148
|
opening: null,
|
|
1701
2149
|
handle: null,
|
|
1702
2150
|
closed: false,
|
|
2151
|
+
inflight: 0,
|
|
2152
|
+
closedBy: null,
|
|
1703
2153
|
};
|
|
1704
2154
|
this.assertRootCanAttach(record.canonicalRoot);
|
|
1705
|
-
const opening = client
|
|
1706
|
-
.routeOpen({ kind: "tool_provider", module_id: AFT_MODULE_ID }, identity, {
|
|
1707
|
-
consumerIdentity: this.consumerIdentity,
|
|
1708
|
-
})
|
|
2155
|
+
const opening = this.openHostToolRoute(client, identity, record, entry)
|
|
1709
2156
|
.then((route) => {
|
|
1710
2157
|
if (!this.isCurrentSession(record.identityKey, record) ||
|
|
1711
2158
|
record.routeEntry !== entry ||
|
|
@@ -1721,6 +2168,15 @@ export class SubcTransportPool {
|
|
|
1721
2168
|
entry.handle = route;
|
|
1722
2169
|
entry.opening = null;
|
|
1723
2170
|
this.noteClientAlive(client);
|
|
2171
|
+
const state = this.hostScopeCache.get(identity.session);
|
|
2172
|
+
if (state)
|
|
2173
|
+
this.scheduleHostScopeRetry(identity.session, state);
|
|
2174
|
+
if (state?.answer?.result === "scope" &&
|
|
2175
|
+
!state.scopeBindPending &&
|
|
2176
|
+
!entry.hostScope &&
|
|
2177
|
+
!this.hostScopeRefusals.has(identity.session)) {
|
|
2178
|
+
this.promoteHostScope(identity.session);
|
|
2179
|
+
}
|
|
1724
2180
|
return route;
|
|
1725
2181
|
})
|
|
1726
2182
|
.catch((error) => {
|
|
@@ -1729,6 +2185,7 @@ export class SubcTransportPool {
|
|
|
1729
2185
|
entry.closed = true;
|
|
1730
2186
|
record.routeEntry = null;
|
|
1731
2187
|
}
|
|
2188
|
+
this.stopUnusedHostScopeRetry(identity.session);
|
|
1732
2189
|
if (!this.isCurrentSession(record.identityKey, record) &&
|
|
1733
2190
|
!(error instanceof RouteTornDownError)) {
|
|
1734
2191
|
throw new RouteTornDownError("subc route opened after session closed");
|
|
@@ -1742,17 +2199,404 @@ export class SubcTransportPool {
|
|
|
1742
2199
|
record.routeEntry = entry;
|
|
1743
2200
|
return { route: await opening, entry };
|
|
1744
2201
|
}
|
|
2202
|
+
hostScope(client, identity) {
|
|
2203
|
+
if (this.hostScopeRefusals.has(identity.session))
|
|
2204
|
+
return Promise.resolve(undefined);
|
|
2205
|
+
let state = this.hostScopeCache.get(identity.session);
|
|
2206
|
+
if (!state) {
|
|
2207
|
+
state = { retryAt: 0, retryMs: HOST_SCOPE_RETRY_FLOOR_MS };
|
|
2208
|
+
this.hostScopeCache.set(identity.session, state);
|
|
2209
|
+
}
|
|
2210
|
+
if (state.answer) {
|
|
2211
|
+
return Promise.resolve(state.answer.result === "scope" && !state.scopeBindPending ? state.answer.scope : undefined);
|
|
2212
|
+
}
|
|
2213
|
+
if (state.pending)
|
|
2214
|
+
return state.pending;
|
|
2215
|
+
// Backoff skips the lookup, not the bind, so an unavailable core cannot delay every root.
|
|
2216
|
+
if (Date.now() < state.retryAt)
|
|
2217
|
+
return Promise.resolve(undefined);
|
|
2218
|
+
const current = state;
|
|
2219
|
+
clearTimeout(current.retryTimer);
|
|
2220
|
+
current.retryTimer = undefined;
|
|
2221
|
+
const abort = new AbortController();
|
|
2222
|
+
current.lookupAbort = abort;
|
|
2223
|
+
const lookup = this.readHostScope(client, identity, abort.signal).then((outcome) => {
|
|
2224
|
+
// A scope-close push may invalidate the session while the management request is in flight.
|
|
2225
|
+
if (this.hostScopeCache.get(identity.session) !== current)
|
|
2226
|
+
return undefined;
|
|
2227
|
+
current.pending = undefined;
|
|
2228
|
+
current.lookupAbort = undefined;
|
|
2229
|
+
if (abort.signal.aborted)
|
|
2230
|
+
return undefined;
|
|
2231
|
+
this.logHostScopeOutcome(identity.session, outcome);
|
|
2232
|
+
if (outcome.result === "scope" || outcome.result === "no_scope") {
|
|
2233
|
+
current.answer = outcome;
|
|
2234
|
+
current.retryAt = 0;
|
|
2235
|
+
current.retryMs = HOST_SCOPE_RETRY_FLOOR_MS;
|
|
2236
|
+
if (outcome.result === "scope")
|
|
2237
|
+
this.promoteHostScope(identity.session);
|
|
2238
|
+
}
|
|
2239
|
+
else {
|
|
2240
|
+
current.retryAt = Date.now() + current.retryMs;
|
|
2241
|
+
current.retryMs = Math.min(current.retryMs * 2, HOST_SCOPE_RETRY_CAP_MS);
|
|
2242
|
+
this.scheduleHostScopeRetry(identity.session, current);
|
|
2243
|
+
}
|
|
2244
|
+
return outcome.result === "scope" ? outcome.scope : undefined;
|
|
2245
|
+
});
|
|
2246
|
+
current.pending = lookup;
|
|
2247
|
+
return lookup;
|
|
2248
|
+
}
|
|
2249
|
+
unscopedHostRecord(session) {
|
|
2250
|
+
for (const record of this.sessionIndex.get(session) ?? []) {
|
|
2251
|
+
const entry = record.routeEntry;
|
|
2252
|
+
if (record.identity.session === session &&
|
|
2253
|
+
!record.closed &&
|
|
2254
|
+
((entry &&
|
|
2255
|
+
!entry.closed &&
|
|
2256
|
+
!entry.hostScope &&
|
|
2257
|
+
entry.client === this.client &&
|
|
2258
|
+
(entry.handle || entry.opening)) ||
|
|
2259
|
+
(record.bgSub?.attachedClient === this.client && record.bgSub?.attachedRoute)))
|
|
2260
|
+
return record;
|
|
2261
|
+
}
|
|
2262
|
+
return undefined;
|
|
2263
|
+
}
|
|
2264
|
+
cancelHostScopeRetry(state) {
|
|
2265
|
+
clearTimeout(state.retryTimer);
|
|
2266
|
+
state.retryTimer = undefined;
|
|
2267
|
+
state.lookupAbort?.abort();
|
|
2268
|
+
}
|
|
2269
|
+
stopUnusedHostScopeRetry(session) {
|
|
2270
|
+
const state = this.hostScopeCache.get(session);
|
|
2271
|
+
if (state && !this.unscopedHostRecord(session)) {
|
|
2272
|
+
if (state.retryTimer || (state.scopeBindPending && state.bindPending))
|
|
2273
|
+
log(`subc head scope: session=${session} retry stopped reason=route_closed`);
|
|
2274
|
+
this.cancelHostScopeRetry(state);
|
|
2275
|
+
state.scopeBindPending = false;
|
|
2276
|
+
}
|
|
2277
|
+
}
|
|
2278
|
+
scheduleHostScopeRetry(session, state) {
|
|
2279
|
+
if (this.shuttingDown ||
|
|
2280
|
+
this.hostScopeCache.get(session) !== state ||
|
|
2281
|
+
(state.answer && !state.scopeBindPending) ||
|
|
2282
|
+
state.pending ||
|
|
2283
|
+
state.bindPending ||
|
|
2284
|
+
state.retryTimer ||
|
|
2285
|
+
this.hostScopeRefusals.has(session) ||
|
|
2286
|
+
!this.unscopedHostRecord(session))
|
|
2287
|
+
return;
|
|
2288
|
+
// A head can keep its tool route for its whole life, so recovery cannot rely on another bind.
|
|
2289
|
+
state.retryTimer = setTimeout(() => {
|
|
2290
|
+
state.retryTimer = undefined;
|
|
2291
|
+
const record = this.unscopedHostRecord(session);
|
|
2292
|
+
const client = record?.routeEntry?.client ?? record?.bgSub?.attachedClient;
|
|
2293
|
+
if (this.shuttingDown || this.hostScopeCache.get(session) !== state || !record || !client)
|
|
2294
|
+
return;
|
|
2295
|
+
let retry;
|
|
2296
|
+
if (state.scopeBindPending && state.answer?.result === "scope") {
|
|
2297
|
+
state.bindPending = this.retryHostScopeBind(client, record.identity, state, state.answer.scope).finally(() => {
|
|
2298
|
+
state.bindPending = undefined;
|
|
2299
|
+
this.scheduleHostScopeRetry(session, state);
|
|
2300
|
+
});
|
|
2301
|
+
retry = state.bindPending;
|
|
2302
|
+
}
|
|
2303
|
+
else {
|
|
2304
|
+
retry = this.hostScope(client, record.identity);
|
|
2305
|
+
}
|
|
2306
|
+
void retry.catch((error) => {
|
|
2307
|
+
warn(`subc head scope: background retry failed code=${error instanceof Error ? error.name : "Error"}`);
|
|
2308
|
+
});
|
|
2309
|
+
}, Math.max(0, state.retryAt - Date.now()));
|
|
2310
|
+
state.retryTimer.unref();
|
|
2311
|
+
}
|
|
2312
|
+
deferHostScopeBind(session, state, code) {
|
|
2313
|
+
state.scopeBindPending = true;
|
|
2314
|
+
if (Date.now() >= state.retryAt) {
|
|
2315
|
+
state.retryAt = Date.now() + state.retryMs;
|
|
2316
|
+
state.retryMs = Math.min(state.retryMs * 2, HOST_SCOPE_RETRY_CAP_MS);
|
|
2317
|
+
}
|
|
2318
|
+
warn(`subc head scope: session=${session} scoped bind refused code=${code}; binding unscoped; retry in=${Math.max(0, state.retryAt - Date.now())}ms`);
|
|
2319
|
+
this.scheduleHostScopeRetry(session, state);
|
|
2320
|
+
}
|
|
2321
|
+
refuseHostScopeBind(session, state, code) {
|
|
2322
|
+
if (!this.hostScopeRefusals.has(session)) {
|
|
2323
|
+
this.hostScopeRefusals.add(session);
|
|
2324
|
+
warn(`subc head scope: session=${session} scoped bind refused code=${code}; binding unscoped; retry stopped`);
|
|
2325
|
+
}
|
|
2326
|
+
if (state) {
|
|
2327
|
+
state.scopeBindPending = false;
|
|
2328
|
+
this.cancelHostScopeRetry(state);
|
|
2329
|
+
}
|
|
2330
|
+
// Remember the downgrade without misreporting it as core's no_scope answer.
|
|
2331
|
+
}
|
|
2332
|
+
async retryHostScopeBind(client, identity, state, scope) {
|
|
2333
|
+
try {
|
|
2334
|
+
// Validate daemon acceptance before retiring a working fallback route or its held requests.
|
|
2335
|
+
const route = await client.routeOpen({ kind: "tool_provider", module_id: AFT_MODULE_ID }, identity, { consumerIdentity: this.consumerIdentity, scope });
|
|
2336
|
+
safeCloseRoute(client, route);
|
|
2337
|
+
if (this.hostScopeCache.get(identity.session) !== state ||
|
|
2338
|
+
!state.scopeBindPending ||
|
|
2339
|
+
this.shuttingDown ||
|
|
2340
|
+
client !== this.client ||
|
|
2341
|
+
!this.unscopedHostRecord(identity.session))
|
|
2342
|
+
return;
|
|
2343
|
+
state.scopeBindPending = false;
|
|
2344
|
+
state.retryAt = 0;
|
|
2345
|
+
state.retryMs = HOST_SCOPE_RETRY_FLOOR_MS;
|
|
2346
|
+
log(`subc head scope: session=${identity.session} scoped bind recovered ref=${scope.ref.slice(0, 8)} epoch=${scope.scopeEpoch}`);
|
|
2347
|
+
this.promoteHostScope(identity.session);
|
|
2348
|
+
}
|
|
2349
|
+
catch (error) {
|
|
2350
|
+
if (this.hostScopeCache.get(identity.session) !== state ||
|
|
2351
|
+
!state.scopeBindPending ||
|
|
2352
|
+
this.shuttingDown ||
|
|
2353
|
+
!this.unscopedHostRecord(identity.session))
|
|
2354
|
+
return;
|
|
2355
|
+
if (error instanceof SubcError &&
|
|
2356
|
+
error.code &&
|
|
2357
|
+
!isSocketDeathError(error) &&
|
|
2358
|
+
!isRouteOpenReloadWindowError(error) &&
|
|
2359
|
+
!isHostScopeSyncRefusal(error.code)) {
|
|
2360
|
+
this.refuseHostScopeBind(identity.session, state, error.code);
|
|
2361
|
+
}
|
|
2362
|
+
else {
|
|
2363
|
+
this.deferHostScopeBind(identity.session, state, error instanceof SubcError ? (error.code ?? error.name) : "Error");
|
|
2364
|
+
}
|
|
2365
|
+
}
|
|
2366
|
+
}
|
|
2367
|
+
promoteHostScope(session) {
|
|
2368
|
+
for (const record of this.sessions.values()) {
|
|
2369
|
+
const entry = record.routeEntry;
|
|
2370
|
+
if (record.identity.session !== session)
|
|
2371
|
+
continue;
|
|
2372
|
+
// Retirement stops new dispatches; the existing request counter defers closure of held waits.
|
|
2373
|
+
if (entry?.handle && !entry.hostScope)
|
|
2374
|
+
this.retireHostToolRoute(record);
|
|
2375
|
+
const sub = record.bgSub;
|
|
2376
|
+
if (!sub?.attachedRoute)
|
|
2377
|
+
continue;
|
|
2378
|
+
// The old subscription must finish closing before its normal attach path opens a scoped one.
|
|
2379
|
+
void sub.stop().then(() => {
|
|
2380
|
+
if (record.bgSub !== sub)
|
|
2381
|
+
return;
|
|
2382
|
+
record.bgSub = null;
|
|
2383
|
+
this.ensureBgSubscription(record.identity, record);
|
|
2384
|
+
});
|
|
2385
|
+
}
|
|
2386
|
+
}
|
|
2387
|
+
logHostScopeOutcome(session, outcome) {
|
|
2388
|
+
let key;
|
|
2389
|
+
let detail;
|
|
2390
|
+
switch (outcome.result) {
|
|
2391
|
+
case "scope":
|
|
2392
|
+
key = JSON.stringify([outcome.result, outcome.scope.ref, outcome.scope.scopeEpoch]);
|
|
2393
|
+
detail = `scope ref=${outcome.scope.ref.slice(0, 8)} epoch=${outcome.scope.scopeEpoch}`;
|
|
2394
|
+
break;
|
|
2395
|
+
case "error":
|
|
2396
|
+
key = JSON.stringify([outcome.result, outcome.code]);
|
|
2397
|
+
detail = `error code=${outcome.code} after=${outcome.afterMs}`;
|
|
2398
|
+
break;
|
|
2399
|
+
case "timeout":
|
|
2400
|
+
key = outcome.result;
|
|
2401
|
+
detail = `timeout after=${outcome.afterMs}`;
|
|
2402
|
+
break;
|
|
2403
|
+
default:
|
|
2404
|
+
key = outcome.result;
|
|
2405
|
+
detail = outcome.result;
|
|
2406
|
+
}
|
|
2407
|
+
// Elapsed time varies on every retry; only a changed answer or error code merits another line.
|
|
2408
|
+
if (this.hostScopeOutcomes.get(session) === key)
|
|
2409
|
+
return;
|
|
2410
|
+
this.hostScopeOutcomes.set(session, key);
|
|
2411
|
+
log(`subc head scope: session=${session} result=${detail}`);
|
|
2412
|
+
}
|
|
2413
|
+
async readHostScope(client, identity, signal) {
|
|
2414
|
+
const startedAt = Date.now();
|
|
2415
|
+
const timeout = () => ({ result: "timeout", afterMs: Date.now() - startedAt });
|
|
2416
|
+
let expired = false;
|
|
2417
|
+
let timer;
|
|
2418
|
+
const abort = new AbortController();
|
|
2419
|
+
let cancel;
|
|
2420
|
+
const cancelled = new Promise((resolve) => {
|
|
2421
|
+
cancel = () => {
|
|
2422
|
+
expired = true;
|
|
2423
|
+
abort.abort();
|
|
2424
|
+
resolve({ result: "error", code: "AbortError", afterMs: Date.now() - startedAt });
|
|
2425
|
+
};
|
|
2426
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
2427
|
+
});
|
|
2428
|
+
const read = async () => {
|
|
2429
|
+
let route;
|
|
2430
|
+
try {
|
|
2431
|
+
route = await client.routeOpen({ kind: "management_surface", module_id: "prefrontal-core" }, identity, { consumerIdentity: this.consumerIdentity });
|
|
2432
|
+
if (expired)
|
|
2433
|
+
return timeout();
|
|
2434
|
+
const envelope = await client.request(route, {
|
|
2435
|
+
// Core decodes every management request as {method, params}.
|
|
2436
|
+
method: "session.scope_identity",
|
|
2437
|
+
params: { harness: identity.harness, session: identity.session },
|
|
2438
|
+
}, { timeoutMs: HOST_SCOPE_LOOKUP_TIMEOUT_MS, signal: abort.signal });
|
|
2439
|
+
// Core answers a management request as {result: <answer>}.
|
|
2440
|
+
const reply = isRecord(envelope) ? envelope.result : undefined;
|
|
2441
|
+
if (isRecord(reply) && reply.answer === "no_scope")
|
|
2442
|
+
return { result: "no_scope" };
|
|
2443
|
+
if (!isRecord(reply) ||
|
|
2444
|
+
!isRecord(reply.owner) ||
|
|
2445
|
+
reply.owner.kind !== "reserved" ||
|
|
2446
|
+
reply.owner.module_id !== "prefrontal-core" ||
|
|
2447
|
+
typeof reply.ref !== "string" ||
|
|
2448
|
+
reply.ref.length === 0 ||
|
|
2449
|
+
!Number.isSafeInteger(reply.scope_epoch) ||
|
|
2450
|
+
reply.scope_epoch < 0)
|
|
2451
|
+
return { result: "malformed" };
|
|
2452
|
+
return {
|
|
2453
|
+
result: "scope",
|
|
2454
|
+
scope: {
|
|
2455
|
+
owner: { kind: "reserved", module_id: "prefrontal-core" },
|
|
2456
|
+
ref: reply.ref,
|
|
2457
|
+
scopeEpoch: reply.scope_epoch,
|
|
2458
|
+
},
|
|
2459
|
+
};
|
|
2460
|
+
}
|
|
2461
|
+
catch (error) {
|
|
2462
|
+
// A missing or unavailable scope service must not disable the host's tools.
|
|
2463
|
+
if (expired)
|
|
2464
|
+
return timeout();
|
|
2465
|
+
return {
|
|
2466
|
+
result: "error",
|
|
2467
|
+
code: error instanceof SubcError && error.code
|
|
2468
|
+
? error.code
|
|
2469
|
+
: error instanceof Error
|
|
2470
|
+
? error.name
|
|
2471
|
+
: "Error",
|
|
2472
|
+
afterMs: Date.now() - startedAt,
|
|
2473
|
+
};
|
|
2474
|
+
}
|
|
2475
|
+
finally {
|
|
2476
|
+
if (route)
|
|
2477
|
+
safeCloseRoute(client, route);
|
|
2478
|
+
}
|
|
2479
|
+
};
|
|
2480
|
+
try {
|
|
2481
|
+
// Keep the two-second bind budget during core startup; fallback routes retry in the background.
|
|
2482
|
+
return await Promise.race([
|
|
2483
|
+
read(),
|
|
2484
|
+
cancelled,
|
|
2485
|
+
new Promise((resolve) => {
|
|
2486
|
+
timer = setTimeout(() => {
|
|
2487
|
+
expired = true;
|
|
2488
|
+
abort.abort();
|
|
2489
|
+
resolve(timeout());
|
|
2490
|
+
}, HOST_SCOPE_LOOKUP_TIMEOUT_MS);
|
|
2491
|
+
}),
|
|
2492
|
+
]);
|
|
2493
|
+
}
|
|
2494
|
+
finally {
|
|
2495
|
+
clearTimeout(timer);
|
|
2496
|
+
signal.removeEventListener("abort", cancel);
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
openHostToolRoute(client, identity, record, entry) {
|
|
2500
|
+
if (!this.hostScopes ||
|
|
2501
|
+
!["opencode", "pi"].includes(identity.harness) ||
|
|
2502
|
+
identity.session === DEFAULT_SESSION_ID) {
|
|
2503
|
+
return client.routeOpen({ kind: "tool_provider", module_id: AFT_MODULE_ID }, identity, {
|
|
2504
|
+
consumerIdentity: this.consumerIdentity,
|
|
2505
|
+
});
|
|
2506
|
+
}
|
|
2507
|
+
return this.openScopedHostToolRoute(client, identity, record, entry);
|
|
2508
|
+
}
|
|
2509
|
+
async openScopedHostToolRoute(client, identity, record, entry) {
|
|
2510
|
+
const scope = await this.hostScope(client, identity);
|
|
2511
|
+
this.assertRecordLive(record);
|
|
2512
|
+
if (scope) {
|
|
2513
|
+
try {
|
|
2514
|
+
const route = await client.routeOpen({ kind: "tool_provider", module_id: AFT_MODULE_ID }, identity, { consumerIdentity: this.consumerIdentity, scope });
|
|
2515
|
+
if (entry)
|
|
2516
|
+
entry.hostScope = scope;
|
|
2517
|
+
return route;
|
|
2518
|
+
}
|
|
2519
|
+
catch (error) {
|
|
2520
|
+
if (!(error instanceof SubcError) || !error.code || isSocketDeathError(error))
|
|
2521
|
+
throw error;
|
|
2522
|
+
// A restarting provider has not rejected the scope. Keep it for the normal reopen retry.
|
|
2523
|
+
if (isRouteOpenReloadWindowError(error))
|
|
2524
|
+
throw error;
|
|
2525
|
+
this.assertRecordLive(record);
|
|
2526
|
+
const state = this.hostScopeCache.get(identity.session);
|
|
2527
|
+
if (state && isHostScopeSyncRefusal(error.code)) {
|
|
2528
|
+
this.deferHostScopeBind(identity.session, state, error.code);
|
|
2529
|
+
}
|
|
2530
|
+
else {
|
|
2531
|
+
this.refuseHostScopeBind(identity.session, state, error.code);
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
this.assertRecordLive(record);
|
|
2536
|
+
const route = await client.routeOpen({ kind: "tool_provider", module_id: AFT_MODULE_ID }, identity, {
|
|
2537
|
+
consumerIdentity: this.consumerIdentity,
|
|
2538
|
+
});
|
|
2539
|
+
// Scope recovery can finish while a fallback bind waits for the provider, including bg_events.
|
|
2540
|
+
if (this.hostScopeCache.get(identity.session)?.answer?.result === "scope" &&
|
|
2541
|
+
!this.hostScopeCache.get(identity.session)?.scopeBindPending &&
|
|
2542
|
+
!this.hostScopeRefusals.has(identity.session)) {
|
|
2543
|
+
safeCloseRoute(client, route);
|
|
2544
|
+
return this.openScopedHostToolRoute(client, identity, record, entry);
|
|
2545
|
+
}
|
|
2546
|
+
return route;
|
|
2547
|
+
}
|
|
2548
|
+
invalidateHostScope(session) {
|
|
2549
|
+
const state = this.hostScopeCache.get(session);
|
|
2550
|
+
if (state) {
|
|
2551
|
+
if (state.scopeBindPending)
|
|
2552
|
+
log(`subc head scope: session=${session} retry stopped reason=scope_route_closed`);
|
|
2553
|
+
this.cancelHostScopeRetry(state);
|
|
2554
|
+
}
|
|
2555
|
+
this.hostScopeCache.delete(session);
|
|
2556
|
+
this.hostScopeRefusals.delete(session);
|
|
2557
|
+
for (const record of this.sessions.values()) {
|
|
2558
|
+
if (record.identity.session !== session || !record.routeEntry)
|
|
2559
|
+
continue;
|
|
2560
|
+
this.retireHostToolRoute(record);
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
onScopeRouteClose(push) {
|
|
2564
|
+
if (push.op !== "route.closed")
|
|
2565
|
+
return;
|
|
2566
|
+
const reason = routeCloseReason(push);
|
|
2567
|
+
if (!reason?.startsWith("scope_")) {
|
|
2568
|
+
for (const record of this.sessions.values()) {
|
|
2569
|
+
const route = record.routeEntry?.handle;
|
|
2570
|
+
if (!route)
|
|
2571
|
+
continue;
|
|
2572
|
+
if ((Array.isArray(push.body.channels) && push.body.channels.includes(route.channel)) ||
|
|
2573
|
+
(!Array.isArray(push.body.channels) && push.body.module_id === AFT_MODULE_ID))
|
|
2574
|
+
this.retireHostToolRoute(record);
|
|
2575
|
+
}
|
|
2576
|
+
return;
|
|
2577
|
+
}
|
|
2578
|
+
const sessions = new Set();
|
|
2579
|
+
for (const record of this.sessions.values()) {
|
|
2580
|
+
const routes = [record.routeEntry?.handle, record.bgSub?.attachedRoute].filter((route) => route != null);
|
|
2581
|
+
if ((Array.isArray(push.body.channels) &&
|
|
2582
|
+
routes.some((route) => push.body.channels.includes(route.channel))) ||
|
|
2583
|
+
(!Array.isArray(push.body.channels) && push.body.module_id === AFT_MODULE_ID))
|
|
2584
|
+
sessions.add(record.identity.session);
|
|
2585
|
+
}
|
|
2586
|
+
for (const session of sessions)
|
|
2587
|
+
this.invalidateHostScope(session);
|
|
2588
|
+
}
|
|
1745
2589
|
ensureBgSubscription(identity, record) {
|
|
1746
2590
|
if (this.shuttingDown || (!this.onBgEventsNudge && !this.onBgEventsNudgeRef))
|
|
1747
2591
|
return;
|
|
1748
2592
|
if (!this.isCurrentSession(record.identityKey, record))
|
|
1749
2593
|
return;
|
|
1750
|
-
if (!this.rootCanAttach(record.canonicalRoot))
|
|
1751
|
-
return;
|
|
1752
2594
|
if (record.bgSub)
|
|
1753
2595
|
return;
|
|
2596
|
+
if (!this.rootCanAttach(record.canonicalRoot))
|
|
2597
|
+
return;
|
|
1754
2598
|
const nudgeRef = this.nudgeRefFor(record);
|
|
1755
|
-
const onNudge = () => {
|
|
2599
|
+
const onNudge = (observedEvent) => {
|
|
1756
2600
|
const currentRecord = this.currentSessionForNudge(identity);
|
|
1757
2601
|
if (!currentRecord) {
|
|
1758
2602
|
this.logNudgeDelivery("drop-no-current-session", record, `nudge dropped cause=no-current-session root=${record.canonicalRoot}`);
|
|
@@ -1761,6 +2605,9 @@ export class SubcTransportPool {
|
|
|
1761
2605
|
if (currentRecord !== record) {
|
|
1762
2606
|
this.logNudgeDelivery("forward-superseded-carrier", currentRecord, `nudge forwarding cause=superseded-carrying-record root=${currentRecord.canonicalRoot}`);
|
|
1763
2607
|
}
|
|
2608
|
+
currentRecord.lastUsedAt = Date.now();
|
|
2609
|
+
if (observedEvent)
|
|
2610
|
+
currentRecord.pendingBgNudge = true;
|
|
1764
2611
|
const currentRef = this.nudgeRefFor(currentRecord);
|
|
1765
2612
|
let delivered = false;
|
|
1766
2613
|
if (currentRef && this.onBgEventsNudgeRef) {
|
|
@@ -1778,18 +2625,32 @@ export class SubcTransportPool {
|
|
|
1778
2625
|
return;
|
|
1779
2626
|
this.logNudgeDelivery("drop-no-compatible-handler", currentRecord, `nudge dropped cause=generation-provenance-unavailable-and-root-session-handler-unwired root=${currentRecord.canonicalRoot}`);
|
|
1780
2627
|
};
|
|
2628
|
+
const binding = {};
|
|
1781
2629
|
let sub = null;
|
|
1782
2630
|
const clearDormantSubscription = () => {
|
|
1783
2631
|
if (sub && record.bgSub === sub)
|
|
1784
2632
|
record.bgSub = null;
|
|
1785
2633
|
};
|
|
1786
|
-
sub = new BgSubscription(identity, () => this.ensureClient(), (client, cause) => this.dropClient(client, cause),
|
|
2634
|
+
sub = new BgSubscription(identity, () => this.ensureClient(), (client, cause) => this.dropClient(client, cause), (client) => {
|
|
2635
|
+
binding.hostScope = undefined;
|
|
2636
|
+
return this.openHostToolRoute(client, identity, record, binding);
|
|
2637
|
+
}, onNudge, this.bgBackoffSleep, () => this.rootCanAttach(record.canonicalRoot), (error) => {
|
|
1787
2638
|
if (!isAbsentRootRouteError(error))
|
|
1788
2639
|
return false;
|
|
1789
2640
|
this.markRootDormant(record.canonicalRoot, error);
|
|
1790
2641
|
return true;
|
|
1791
2642
|
}, clearDormantSubscription, this.bgDispatchProbeIntervalMs, nudgeRef, () => this.isCurrentSession(record.identityKey, record) &&
|
|
1792
|
-
this.isCurrentLiveGeneration(record.canonicalRoot, record.generation), (client) => this.noteClientAlive(client), (client, cause) => this.noteUnansweredCall(client, cause))
|
|
2643
|
+
this.isCurrentLiveGeneration(record.canonicalRoot, record.generation), (client) => this.noteClientAlive(client), (client, cause) => this.noteUnansweredCall(client, cause), () => {
|
|
2644
|
+
const state = this.hostScopeCache.get(identity.session);
|
|
2645
|
+
if (state)
|
|
2646
|
+
this.scheduleHostScopeRetry(identity.session, state);
|
|
2647
|
+
if (state?.answer?.result === "scope" &&
|
|
2648
|
+
!state.scopeBindPending &&
|
|
2649
|
+
!binding.hostScope &&
|
|
2650
|
+
!this.hostScopeRefusals.has(identity.session)) {
|
|
2651
|
+
this.promoteHostScope(identity.session);
|
|
2652
|
+
}
|
|
2653
|
+
});
|
|
1793
2654
|
record.bgSub = sub;
|
|
1794
2655
|
if (!this.rootCanAttach(record.canonicalRoot)) {
|
|
1795
2656
|
record.bgSub = null;
|
|
@@ -1825,6 +2686,12 @@ export class SubcTransportPool {
|
|
|
1825
2686
|
` message=${JSON.stringify(message)}`);
|
|
1826
2687
|
this.client = null;
|
|
1827
2688
|
for (const record of this.sessions.values()) {
|
|
2689
|
+
for (const retired of record.retiredRoutes) {
|
|
2690
|
+
if (retired.client !== client)
|
|
2691
|
+
continue;
|
|
2692
|
+
retired.closed = true;
|
|
2693
|
+
record.retiredRoutes.delete(retired);
|
|
2694
|
+
}
|
|
1828
2695
|
const entry = record.routeEntry;
|
|
1829
2696
|
if (entry?.client !== client)
|
|
1830
2697
|
continue;
|
|
@@ -1832,6 +2699,8 @@ export class SubcTransportPool {
|
|
|
1832
2699
|
record.routeEntry = null;
|
|
1833
2700
|
this.deleteSessionIfEmpty(record.identityKey, record);
|
|
1834
2701
|
}
|
|
2702
|
+
for (const session of this.hostScopeCache.keys())
|
|
2703
|
+
this.stopUnusedHostScopeRetry(session);
|
|
1835
2704
|
this.transportFailures = 0;
|
|
1836
2705
|
try {
|
|
1837
2706
|
client.close();
|
|
@@ -1996,7 +2865,20 @@ export class SubcTransportPool {
|
|
|
1996
2865
|
if (this.shuttingDown)
|
|
1997
2866
|
return;
|
|
1998
2867
|
this.shuttingDown = true;
|
|
2868
|
+
if (this.sessionSweep)
|
|
2869
|
+
clearInterval(this.sessionSweep);
|
|
2870
|
+
this.sessionSweep = null;
|
|
2871
|
+
this.nudgeDeliveryLogState.clear();
|
|
2872
|
+
this.generationRejections.clear();
|
|
2873
|
+
this.observedStarts.clear();
|
|
1999
2874
|
this.dormantRoots.clear();
|
|
2875
|
+
const scopeLookups = [...this.hostScopeCache.values()].flatMap((state) => {
|
|
2876
|
+
this.cancelHostScopeRetry(state);
|
|
2877
|
+
return state.pending ? [state.pending] : [];
|
|
2878
|
+
});
|
|
2879
|
+
this.hostScopeCache.clear();
|
|
2880
|
+
this.hostScopeRefusals.clear();
|
|
2881
|
+
this.hostScopeOutcomes.clear();
|
|
2000
2882
|
// Deregistration synchronously tombstones all registered roots and invokes
|
|
2001
2883
|
// closeProjectRoot before this method reaches its first await.
|
|
2002
2884
|
const registration = this.lifecycleRegistration;
|
|
@@ -2009,11 +2891,16 @@ export class SubcTransportPool {
|
|
|
2009
2891
|
detached.push(owner);
|
|
2010
2892
|
}
|
|
2011
2893
|
this.rootIndex.clear();
|
|
2894
|
+
this.sessionIndex.clear();
|
|
2012
2895
|
this.transports.clear();
|
|
2013
2896
|
const client = this.client;
|
|
2014
2897
|
this.client = null;
|
|
2015
2898
|
const pending = [...this.pendingRootCleanups];
|
|
2016
|
-
await Promise.allSettled([
|
|
2899
|
+
await Promise.allSettled([
|
|
2900
|
+
...detached.map((owner) => this.cleanupDetached(owner)),
|
|
2901
|
+
...pending,
|
|
2902
|
+
...scopeLookups,
|
|
2903
|
+
]);
|
|
2017
2904
|
if (client) {
|
|
2018
2905
|
try {
|
|
2019
2906
|
client.close();
|
|
@@ -2030,6 +2917,13 @@ export class SubcTransportPool {
|
|
|
2030
2917
|
session: session && session.length > 0 ? session : DEFAULT_SESSION_ID,
|
|
2031
2918
|
};
|
|
2032
2919
|
const owner = this.detachSession(identityKey(identity), "session_closed");
|
|
2920
|
+
if (!this.sessionIndex.has(identity.session)) {
|
|
2921
|
+
const state = this.hostScopeCache.get(identity.session);
|
|
2922
|
+
if (state)
|
|
2923
|
+
this.cancelHostScopeRetry(state);
|
|
2924
|
+
this.hostScopeCache.delete(identity.session);
|
|
2925
|
+
this.hostScopeRefusals.delete(identity.session);
|
|
2926
|
+
}
|
|
2033
2927
|
if (owner)
|
|
2034
2928
|
await this.cleanupDetached(owner);
|
|
2035
2929
|
}
|