@intx/hub-sessions 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/package.json +23 -0
- package/src/agent-repo.test.ts +310 -0
- package/src/agent-repo.ts +165 -0
- package/src/agent-state-kind.test.ts +247 -0
- package/src/agent-state-kind.ts +204 -0
- package/src/asset-service.test.ts +540 -0
- package/src/asset-service.ts +378 -0
- package/src/available-skills-stanza.test.ts +87 -0
- package/src/available-skills-stanza.ts +47 -0
- package/src/credential-push.ts +65 -0
- package/src/event-collector-registry.test.ts +73 -0
- package/src/event-collector-registry.ts +171 -0
- package/src/event-collector.test.ts +1387 -0
- package/src/event-collector.ts +424 -0
- package/src/hub-session-lookups.ts +206 -0
- package/src/hub-session-orchestrator.test.ts +510 -0
- package/src/hub-session-orchestrator.ts +213 -0
- package/src/index.ts +78 -0
- package/src/repo-store/index.ts +15 -0
- package/src/repo-store/store.test.ts +1169 -0
- package/src/repo-store/store.ts +428 -0
- package/src/repo-store/types.ts +253 -0
- package/src/session-service.test.ts +895 -0
- package/src/session-service.ts +464 -0
- package/src/skill-kind.test.ts +599 -0
- package/src/skill-kind.ts +350 -0
- package/src/ws/index.ts +18 -0
- package/src/ws/sidecar-events.test.ts +96 -0
- package/src/ws/sidecar-events.ts +231 -0
- package/src/ws/sidecar-handler.test.ts +2217 -0
- package/src/ws/sidecar-handler.ts +1574 -0
- package/tsconfig.json +4 -0
- package/tsconfig.tsbuildinfo +1 -0
|
@@ -0,0 +1,1574 @@
|
|
|
1
|
+
// Hub-side websocket handler for sidecar connections.
|
|
2
|
+
//
|
|
3
|
+
// Accepts websocket upgrades, processes register frames, maintains a routing
|
|
4
|
+
// table of agentAddress → sidecar connection, and dispatches frames between
|
|
5
|
+
// sidecars and the hub's internal systems.
|
|
6
|
+
|
|
7
|
+
import { randomBytes } from "node:crypto";
|
|
8
|
+
import { getLogger } from "@intx/log";
|
|
9
|
+
import { verifyEd25519 } from "@intx/crypto-node";
|
|
10
|
+
import { chunkPack, createPackReceiver } from "@intx/pack-transport";
|
|
11
|
+
import { hexDecode, hexEncode } from "@intx/types";
|
|
12
|
+
import { type } from "arktype";
|
|
13
|
+
import {
|
|
14
|
+
SidecarFrame,
|
|
15
|
+
type HubFrame,
|
|
16
|
+
type PackPushFrame,
|
|
17
|
+
type PackDoneFrame,
|
|
18
|
+
type RepoId,
|
|
19
|
+
} from "@intx/types/sidecar";
|
|
20
|
+
import type {
|
|
21
|
+
AbortReason,
|
|
22
|
+
ConnectorThreadState,
|
|
23
|
+
HarnessConfig,
|
|
24
|
+
InferenceSource,
|
|
25
|
+
} from "@intx/types/runtime";
|
|
26
|
+
import type { GrantRule } from "@intx/types/authz";
|
|
27
|
+
import {
|
|
28
|
+
createSidecarEmitter,
|
|
29
|
+
type SidecarEventEmitter,
|
|
30
|
+
type SidecarLookups,
|
|
31
|
+
type SidecarMailPersistedRow,
|
|
32
|
+
} from "./sidecar-events";
|
|
33
|
+
|
|
34
|
+
const logger = getLogger(["hub", "ws", "sidecar"]);
|
|
35
|
+
|
|
36
|
+
export type SidecarConnection = {
|
|
37
|
+
sidecarId: string;
|
|
38
|
+
agentAddresses: Set<string>;
|
|
39
|
+
send(frame: HubFrame): void;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
type PendingRequest = {
|
|
43
|
+
requestId: string;
|
|
44
|
+
ws: WsHandle;
|
|
45
|
+
resolve(): void;
|
|
46
|
+
reject(error: string): void;
|
|
47
|
+
timer: ReturnType<typeof setTimeout>;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
export type SendPackOptions = {
|
|
51
|
+
/**
|
|
52
|
+
* Repo-relative mount path under the sidecar's per-agent workspace.
|
|
53
|
+
* When set, the receiving sidecar materializes the pack as plain
|
|
54
|
+
* files at `<workspaceRoot>/<mountPath>/` and does NOT apply it to
|
|
55
|
+
* the agent's deploy git tree. Absent for agent-state deploy/state
|
|
56
|
+
* packs, which continue to apply to the deploy tree.
|
|
57
|
+
*/
|
|
58
|
+
mountPath?: string;
|
|
59
|
+
/**
|
|
60
|
+
* Override the `repoId` emitted on the wire. The agent-state flow
|
|
61
|
+
* defaults to `{ kind: "agent-state", id: agentAddress }`; asset
|
|
62
|
+
* packs must pass the SOURCE asset's id so audit can correlate the
|
|
63
|
+
* pack back to its hub-side origin.
|
|
64
|
+
*/
|
|
65
|
+
repoId?: RepoId;
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
export type SidecarRouter = {
|
|
69
|
+
handleOpen(ws: WsHandle): void;
|
|
70
|
+
handleMessage(ws: WsHandle, data: string): void;
|
|
71
|
+
handleClose(ws: WsHandle): void;
|
|
72
|
+
|
|
73
|
+
routeMail(agentAddress: string, rawMessage: string): boolean;
|
|
74
|
+
/**
|
|
75
|
+
* Returns the current connector-thread state for the named agent, or
|
|
76
|
+
* `null` if the agent has no active connector thread (or if the
|
|
77
|
+
* sidecar has not yet reported any state — e.g. mid-reconnect, before
|
|
78
|
+
* the harness has loaded its context store). The state is cached
|
|
79
|
+
* from `connector.state.changed` frames; callers should treat `null`
|
|
80
|
+
* as "no threading info available" and fall through to whatever
|
|
81
|
+
* default the calling path uses.
|
|
82
|
+
*/
|
|
83
|
+
getConnectorState(agentAddress: string): ConnectorThreadState | null;
|
|
84
|
+
sendAgentDeploy(agentAddress: string, config: HarnessConfig): Promise<void>;
|
|
85
|
+
sendAgentUndeploy(agentAddress: string, reason: string): Promise<void>;
|
|
86
|
+
sendSessionStart(agentAddress: string): Promise<void>;
|
|
87
|
+
sendSessionAbort(agentAddress: string, reason: AbortReason): Promise<void>;
|
|
88
|
+
sendGrantsUpdate(agentAddress: string, grants: GrantRule[]): Promise<void>;
|
|
89
|
+
sendSourcesUpdate(
|
|
90
|
+
agentAddress: string,
|
|
91
|
+
sources: InferenceSource[],
|
|
92
|
+
defaultSource: string,
|
|
93
|
+
): Promise<void>;
|
|
94
|
+
sendPack(
|
|
95
|
+
agentAddress: string,
|
|
96
|
+
pack: Uint8Array,
|
|
97
|
+
ref: string,
|
|
98
|
+
commitSha: string,
|
|
99
|
+
options?: SendPackOptions,
|
|
100
|
+
): Promise<void>;
|
|
101
|
+
sendSyncRequest(agentAddress: string): void;
|
|
102
|
+
|
|
103
|
+
subscribeAgent(
|
|
104
|
+
agentAddress: string,
|
|
105
|
+
callback: (event: unknown) => void,
|
|
106
|
+
): () => void;
|
|
107
|
+
dispatchAgentEvent(agentAddress: string, event: unknown): void;
|
|
108
|
+
|
|
109
|
+
getConnectedSidecars(): string[];
|
|
110
|
+
getRoutableAddresses(): string[];
|
|
111
|
+
|
|
112
|
+
/** Typed event emitter for the receiver-dispatch surface. See
|
|
113
|
+
* `sidecar-events.ts` for the event map and emission semantics. */
|
|
114
|
+
events: SidecarEventEmitter;
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
export type SidecarRouterConfig = {
|
|
118
|
+
requestTimeoutMs?: number;
|
|
119
|
+
/** Hex-encoded 32-byte Ed25519 public key for signing deploy commits.
|
|
120
|
+
* Included in agent.deploy frames so sidecars can verify pack signatures. */
|
|
121
|
+
hubPublicKey?: string;
|
|
122
|
+
validateToken?: (sidecarId: string, token: string) => boolean;
|
|
123
|
+
challengeTimeoutMs?: number;
|
|
124
|
+
disconnectQueueMaxSize?: number;
|
|
125
|
+
disconnectQueueTTLMs?: number;
|
|
126
|
+
pingTimeoutMs?: number;
|
|
127
|
+
/** Query handlers the wire layer issues during frame processing.
|
|
128
|
+
* Each lookup is one-handler-returns-a-value; for multi-subscriber
|
|
129
|
+
* notifications use `router.events.on(...)` instead.
|
|
130
|
+
*
|
|
131
|
+
* `lookupDeployRef` and the `deploy.ref.stale` event are paired by
|
|
132
|
+
* convention: the wire layer only issues the staleness comparison
|
|
133
|
+
* when the lookup is set, and only emits the event on a confirmed
|
|
134
|
+
* mismatch. The host is responsible for subscribing a listener
|
|
135
|
+
* whenever the lookup is provided; the router does not enforce
|
|
136
|
+
* the pairing. */
|
|
137
|
+
lookups?: SidecarLookups;
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
// Minimal handle so the router doesn't depend on a specific WebSocket impl.
|
|
141
|
+
export type WsHandle = {
|
|
142
|
+
send(data: string): void;
|
|
143
|
+
close(): void;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
147
|
+
const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
|
|
148
|
+
const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100;
|
|
149
|
+
const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
|
|
150
|
+
const DEFAULT_PING_TIMEOUT_MS = 60_000;
|
|
151
|
+
|
|
152
|
+
export function createSidecarRouter(
|
|
153
|
+
config: SidecarRouterConfig = {},
|
|
154
|
+
): SidecarRouter {
|
|
155
|
+
const {
|
|
156
|
+
requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,
|
|
157
|
+
challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS,
|
|
158
|
+
hubPublicKey: hubPublicKeyHex,
|
|
159
|
+
validateToken,
|
|
160
|
+
disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE,
|
|
161
|
+
disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS,
|
|
162
|
+
pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS,
|
|
163
|
+
lookups = {},
|
|
164
|
+
} = config;
|
|
165
|
+
|
|
166
|
+
// Receiver-dispatch surface. Wire-layer callsites emit events here;
|
|
167
|
+
// host code subscribes via `router.events`.
|
|
168
|
+
const events = createSidecarEmitter();
|
|
169
|
+
|
|
170
|
+
// ws handle → registered connection
|
|
171
|
+
const connections = new Map<WsHandle, SidecarConnection>();
|
|
172
|
+
// agentAddress → ws handle (routing table)
|
|
173
|
+
const addressIndex = new Map<string, WsHandle>();
|
|
174
|
+
// requestId → pending promise
|
|
175
|
+
const pending = new Map<string, PendingRequest>();
|
|
176
|
+
// agentAddress → pending deploy promise (matched by agent.deploy.ack/agent.error)
|
|
177
|
+
type PendingDeploy = {
|
|
178
|
+
agentAddress: string;
|
|
179
|
+
ws: WsHandle;
|
|
180
|
+
resolve(): void;
|
|
181
|
+
reject(error: string): void;
|
|
182
|
+
timer: ReturnType<typeof setTimeout>;
|
|
183
|
+
};
|
|
184
|
+
const pendingDeploys = new Map<string, PendingDeploy>();
|
|
185
|
+
// ws handle → pending challenge (awaiting challenge.response)
|
|
186
|
+
type PendingChallenge = {
|
|
187
|
+
sidecarId: string;
|
|
188
|
+
challenges: Map<string, { nonce: Uint8Array; publicKey: Uint8Array }>;
|
|
189
|
+
deployRefs: Record<string, string>;
|
|
190
|
+
timer: ReturnType<typeof setTimeout>;
|
|
191
|
+
};
|
|
192
|
+
const pendingChallenges = new Map<WsHandle, PendingChallenge>();
|
|
193
|
+
// agentAddress → queued frames for disconnected agents awaiting reconnect
|
|
194
|
+
type DisconnectedAgent = {
|
|
195
|
+
queue: HubFrame[];
|
|
196
|
+
timer: ReturnType<typeof setTimeout>;
|
|
197
|
+
};
|
|
198
|
+
const disconnectedAgents = new Map<string, DisconnectedAgent>();
|
|
199
|
+
// agentAddress → set of subscriber callbacks for agent events
|
|
200
|
+
const agentSubscribers = new Map<string, Set<(event: unknown) => void>>();
|
|
201
|
+
// agentAddress → cached connector-thread state, populated by
|
|
202
|
+
// connector.state.changed frames. Hub-side mail composition reads this
|
|
203
|
+
// to set threading headers on user-originated mail. Absent entries mean
|
|
204
|
+
// "no state reported yet" (e.g. mid-reconnect); callers must treat that
|
|
205
|
+
// identically to a null entry (no active thread).
|
|
206
|
+
const connectorStates = new Map<string, ConnectorThreadState | null>();
|
|
207
|
+
// ws handle → liveness timer (reset on each ping from the sidecar)
|
|
208
|
+
const livenessTimers = new Map<WsHandle, ReturnType<typeof setTimeout>>();
|
|
209
|
+
|
|
210
|
+
// transferId → pending pack transfer (resolved by repo.pack.ack, rejected by repo.pack.reject)
|
|
211
|
+
type PendingPack = {
|
|
212
|
+
transferId: string;
|
|
213
|
+
ws: WsHandle;
|
|
214
|
+
resolve(): void;
|
|
215
|
+
reject(error: string): void;
|
|
216
|
+
timer: ReturnType<typeof setTimeout>;
|
|
217
|
+
};
|
|
218
|
+
const pendingPacks = new Map<string, PendingPack>();
|
|
219
|
+
let packCounter = 0;
|
|
220
|
+
|
|
221
|
+
// agentAddress → pending session start (resolved by session.start.ack)
|
|
222
|
+
type PendingSessionStart = {
|
|
223
|
+
agentAddress: string;
|
|
224
|
+
ws: WsHandle;
|
|
225
|
+
resolve(): void;
|
|
226
|
+
reject(error: string): void;
|
|
227
|
+
timer: ReturnType<typeof setTimeout>;
|
|
228
|
+
};
|
|
229
|
+
const pendingSessionStarts = new Map<string, PendingSessionStart>();
|
|
230
|
+
|
|
231
|
+
// agentAddress → pending undeploy (resolved by agent.undeploy.ack)
|
|
232
|
+
type PendingUndeploy = {
|
|
233
|
+
agentAddress: string;
|
|
234
|
+
ws: WsHandle;
|
|
235
|
+
resolve(): void;
|
|
236
|
+
reject(error: string): void;
|
|
237
|
+
timer: ReturnType<typeof setTimeout>;
|
|
238
|
+
};
|
|
239
|
+
const pendingUndeploys = new Map<string, PendingUndeploy>();
|
|
240
|
+
|
|
241
|
+
// Receives state packs pushed from sidecars.
|
|
242
|
+
const statePackReceiver = createPackReceiver();
|
|
243
|
+
|
|
244
|
+
let requestCounter = 0;
|
|
245
|
+
|
|
246
|
+
function enqueueForDisconnected(
|
|
247
|
+
agentAddress: string,
|
|
248
|
+
frame: HubFrame,
|
|
249
|
+
): boolean {
|
|
250
|
+
const entry = disconnectedAgents.get(agentAddress);
|
|
251
|
+
if (entry === undefined) return false;
|
|
252
|
+
|
|
253
|
+
if (entry.queue.length >= disconnectQueueMaxSize) {
|
|
254
|
+
entry.queue.shift();
|
|
255
|
+
}
|
|
256
|
+
entry.queue.push(frame);
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function flushDisconnectedQueue(
|
|
261
|
+
agentAddress: string,
|
|
262
|
+
conn: SidecarConnection,
|
|
263
|
+
): void {
|
|
264
|
+
const entry = disconnectedAgents.get(agentAddress);
|
|
265
|
+
if (entry === undefined) return;
|
|
266
|
+
|
|
267
|
+
clearTimeout(entry.timer);
|
|
268
|
+
disconnectedAgents.delete(agentAddress);
|
|
269
|
+
|
|
270
|
+
for (const frame of entry.queue) {
|
|
271
|
+
conn.send(frame);
|
|
272
|
+
}
|
|
273
|
+
if (entry.queue.length > 0) {
|
|
274
|
+
logger.info`Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function resetLivenessTimer(ws: WsHandle): void {
|
|
279
|
+
const existing = livenessTimers.get(ws);
|
|
280
|
+
if (existing !== undefined) clearTimeout(existing);
|
|
281
|
+
|
|
282
|
+
const timer = setTimeout(() => {
|
|
283
|
+
livenessTimers.delete(ws);
|
|
284
|
+
logger.warn`Sidecar ping timeout, closing connection`;
|
|
285
|
+
ws.close();
|
|
286
|
+
}, pingTimeoutMs);
|
|
287
|
+
livenessTimers.set(ws, timer);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function handlePing(ws: WsHandle): void {
|
|
291
|
+
resetLivenessTimer(ws);
|
|
292
|
+
// Always respond with pong, even before register/reconnect completes.
|
|
293
|
+
// The sidecar's ping timer starts on open, which may fire before the
|
|
294
|
+
// async registration handshake finishes.
|
|
295
|
+
ws.send(JSON.stringify({ type: "pong" }));
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function handleOpen(ws: WsHandle): void {
|
|
299
|
+
// Connection is not usable until a register frame arrives.
|
|
300
|
+
// Start the liveness timer immediately — a sidecar that connects
|
|
301
|
+
// but never sends a ping will be reaped.
|
|
302
|
+
resetLivenessTimer(ws);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function handleMessage(ws: WsHandle, data: string): void {
|
|
306
|
+
let raw: unknown;
|
|
307
|
+
try {
|
|
308
|
+
raw = JSON.parse(data) as unknown;
|
|
309
|
+
} catch {
|
|
310
|
+
logger.warn`Unparseable frame from sidecar connection`;
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
const validated = SidecarFrame(raw);
|
|
314
|
+
if (validated instanceof type.errors) {
|
|
315
|
+
logger.warn`Invalid sidecar frame: ${validated.summary}`;
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
const frame = validated;
|
|
319
|
+
|
|
320
|
+
switch (frame.type) {
|
|
321
|
+
case "register":
|
|
322
|
+
handleRegister(ws, frame.sidecarId, frame.token, frame.agentAddresses);
|
|
323
|
+
break;
|
|
324
|
+
case "reconnect":
|
|
325
|
+
void handleReconnect(
|
|
326
|
+
ws,
|
|
327
|
+
frame.sidecarId,
|
|
328
|
+
frame.token,
|
|
329
|
+
frame.agentAddresses,
|
|
330
|
+
frame.deployRefs ?? {},
|
|
331
|
+
);
|
|
332
|
+
break;
|
|
333
|
+
case "challenge.response":
|
|
334
|
+
void handleChallengeResponse(ws, frame.responses);
|
|
335
|
+
break;
|
|
336
|
+
case "agent.deploy.ack":
|
|
337
|
+
void handleDeployAck(frame.agentAddress, frame.publicKey);
|
|
338
|
+
break;
|
|
339
|
+
case "agent.error":
|
|
340
|
+
rejectDeployPending(frame.agentAddress, frame.error);
|
|
341
|
+
rejectSessionStartPending(frame.agentAddress, frame.error);
|
|
342
|
+
rejectUndeployPending(frame.agentAddress, frame.error);
|
|
343
|
+
break;
|
|
344
|
+
case "session.start.ack":
|
|
345
|
+
resolveSessionStartPending(frame.agentAddress);
|
|
346
|
+
break;
|
|
347
|
+
case "agent.undeploy.ack":
|
|
348
|
+
resolveUndeployPending(frame.agentAddress);
|
|
349
|
+
break;
|
|
350
|
+
case "ping":
|
|
351
|
+
handlePing(ws);
|
|
352
|
+
break;
|
|
353
|
+
case "mail.outbound":
|
|
354
|
+
if (frame.delivered !== true) {
|
|
355
|
+
handleMailOutbound(frame.rawMessage, frame.recipients);
|
|
356
|
+
} else if (lookups.persistMail && frame.senderAddress) {
|
|
357
|
+
void handleMailPersist(
|
|
358
|
+
lookups.persistMail,
|
|
359
|
+
frame.rawMessage,
|
|
360
|
+
frame.senderAddress,
|
|
361
|
+
frame.recipients,
|
|
362
|
+
);
|
|
363
|
+
} else if (frame.delivered === true) {
|
|
364
|
+
if (!frame.senderAddress) {
|
|
365
|
+
logger.warn`Dropping delivered mail.outbound frame with no senderAddress`;
|
|
366
|
+
} else {
|
|
367
|
+
logger.warn`Dropping delivered mail.outbound frame: no persistMail lookup configured`;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
break;
|
|
371
|
+
case "agent.event":
|
|
372
|
+
events.emit("agent.event", {
|
|
373
|
+
agentAddress: frame.agentAddress,
|
|
374
|
+
sessionId: frame.sessionId,
|
|
375
|
+
event: frame.event,
|
|
376
|
+
});
|
|
377
|
+
dispatchToSubscribers(frame.agentAddress, frame.event);
|
|
378
|
+
break;
|
|
379
|
+
case "connector.state.changed":
|
|
380
|
+
// Gate the cache write on the sending sidecar actually owning
|
|
381
|
+
// the named agent. A misbehaving sidecar that knows another
|
|
382
|
+
// agent's address could otherwise poison the cached state.
|
|
383
|
+
if (addressIndex.get(frame.agentAddress) !== ws) {
|
|
384
|
+
logger.warn`Dropping connector.state.changed for ${frame.agentAddress}: not registered to this sidecar`;
|
|
385
|
+
break;
|
|
386
|
+
}
|
|
387
|
+
connectorStates.set(frame.agentAddress, frame.connectorState);
|
|
388
|
+
events.emit("connector.state.changed", {
|
|
389
|
+
agentAddress: frame.agentAddress,
|
|
390
|
+
connectorState: frame.connectorState,
|
|
391
|
+
});
|
|
392
|
+
break;
|
|
393
|
+
case "session.ack":
|
|
394
|
+
resolvePending(frame.requestId);
|
|
395
|
+
break;
|
|
396
|
+
case "session.error":
|
|
397
|
+
rejectPending(frame.requestId, frame.error);
|
|
398
|
+
break;
|
|
399
|
+
case "repo.pack.ack":
|
|
400
|
+
resolvePackPending(frame.transferId);
|
|
401
|
+
break;
|
|
402
|
+
case "repo.pack.reject":
|
|
403
|
+
rejectPackPending(frame.transferId, frame.reason);
|
|
404
|
+
break;
|
|
405
|
+
case "repo.pack.push":
|
|
406
|
+
handleStatePackPush(ws, frame);
|
|
407
|
+
break;
|
|
408
|
+
case "repo.pack.done":
|
|
409
|
+
void handleStatePackDone(ws, frame);
|
|
410
|
+
break;
|
|
411
|
+
default:
|
|
412
|
+
logger.warn`Unknown frame type from sidecar: ${(frame as { type: string }).type}`;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function handleRegister(
|
|
417
|
+
ws: WsHandle,
|
|
418
|
+
sidecarId: string,
|
|
419
|
+
token: string,
|
|
420
|
+
agentAddresses: string[],
|
|
421
|
+
): void {
|
|
422
|
+
if (validateToken !== undefined && !validateToken(sidecarId, token)) {
|
|
423
|
+
logger.warn`Rejected registration from sidecar ${sidecarId}: invalid token`;
|
|
424
|
+
ws.close();
|
|
425
|
+
return;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
// If this sidecar was already connected, clean up old state.
|
|
429
|
+
const existing = connections.get(ws);
|
|
430
|
+
if (existing !== undefined) {
|
|
431
|
+
for (const addr of existing.agentAddresses) {
|
|
432
|
+
addressIndex.delete(addr);
|
|
433
|
+
// The harness on this ws is restarting; the cached connector
|
|
434
|
+
// state from its previous incarnation is now stale. The new
|
|
435
|
+
// harness will bootstrap via restore-fires-callback.
|
|
436
|
+
connectorStates.delete(addr);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
const addrSet = new Set(agentAddresses);
|
|
441
|
+
|
|
442
|
+
// Clean up ghost entries from other connections that previously
|
|
443
|
+
// owned addresses this sidecar is now claiming.
|
|
444
|
+
for (const addr of addrSet) {
|
|
445
|
+
const prevWs = addressIndex.get(addr);
|
|
446
|
+
if (prevWs !== undefined && prevWs !== ws) {
|
|
447
|
+
const prevConn = connections.get(prevWs);
|
|
448
|
+
if (prevConn !== undefined) {
|
|
449
|
+
prevConn.agentAddresses.delete(addr);
|
|
450
|
+
}
|
|
451
|
+
// The new owner is about to take over; the prior owner's
|
|
452
|
+
// cached state must not survive into the new owner's window
|
|
453
|
+
// before its bootstrap frame arrives.
|
|
454
|
+
connectorStates.delete(addr);
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Cancel any in-flight deploy for this address since the
|
|
458
|
+
// reconnecting sidecar is taking ownership.
|
|
459
|
+
const staleDeployReq = pendingDeploys.get(addr);
|
|
460
|
+
if (staleDeployReq !== undefined) {
|
|
461
|
+
clearTimeout(staleDeployReq.timer);
|
|
462
|
+
pendingDeploys.delete(addr);
|
|
463
|
+
staleDeployReq.reject(
|
|
464
|
+
`Sidecar ${sidecarId} reconnected and claimed address "${addr}"`,
|
|
465
|
+
);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const conn: SidecarConnection = {
|
|
470
|
+
sidecarId,
|
|
471
|
+
agentAddresses: addrSet,
|
|
472
|
+
send(frame: HubFrame) {
|
|
473
|
+
ws.send(JSON.stringify(frame));
|
|
474
|
+
},
|
|
475
|
+
};
|
|
476
|
+
|
|
477
|
+
connections.set(ws, conn);
|
|
478
|
+
for (const addr of addrSet) {
|
|
479
|
+
addressIndex.set(addr, ws);
|
|
480
|
+
// Discard any disconnect queue — register has no identity
|
|
481
|
+
// verification, so flushing to an unverified connection is unsafe.
|
|
482
|
+
// Use reconnect with challenge/response to preserve queued messages.
|
|
483
|
+
const staleQueue = disconnectedAgents.get(addr);
|
|
484
|
+
if (staleQueue !== undefined) {
|
|
485
|
+
clearTimeout(staleQueue.timer);
|
|
486
|
+
if (staleQueue.queue.length > 0) {
|
|
487
|
+
logger.warn`Discarding ${String(staleQueue.queue.length)} queued message(s) for ${addr} on unverified register`;
|
|
488
|
+
}
|
|
489
|
+
disconnectedAgents.delete(addr);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
logger.info`Sidecar ${sidecarId} registered with ${String(agentAddresses.length)} agents`;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
async function handleReconnect(
|
|
497
|
+
ws: WsHandle,
|
|
498
|
+
sidecarId: string,
|
|
499
|
+
token: string,
|
|
500
|
+
agentAddresses: string[],
|
|
501
|
+
deployRefs: Record<string, string> = {},
|
|
502
|
+
): Promise<void> {
|
|
503
|
+
if (validateToken !== undefined && !validateToken(sidecarId, token)) {
|
|
504
|
+
logger.warn`Rejected reconnect from sidecar ${sidecarId}: invalid token`;
|
|
505
|
+
ws.close();
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
const lookupKey = lookups.lookupPublicKey;
|
|
510
|
+
if (lookupKey === undefined) {
|
|
511
|
+
logger.error`Received reconnect frame but no lookupPublicKey is configured`;
|
|
512
|
+
ws.close();
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// Cancel any existing pending challenge for this ws before doing
|
|
517
|
+
// async work, so concurrent reconnect frames don't race.
|
|
518
|
+
const existingChallenge = pendingChallenges.get(ws);
|
|
519
|
+
if (existingChallenge !== undefined) {
|
|
520
|
+
clearTimeout(existingChallenge.timer);
|
|
521
|
+
pendingChallenges.delete(ws);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// Register the sidecar connection immediately (with no addresses)
|
|
525
|
+
// so it can receive frames while the challenge is pending.
|
|
526
|
+
handleRegister(ws, sidecarId, token, []);
|
|
527
|
+
|
|
528
|
+
const conn = connections.get(ws);
|
|
529
|
+
if (conn === undefined) return;
|
|
530
|
+
|
|
531
|
+
// Look up stored public keys for all claimed addresses.
|
|
532
|
+
const keyLookups = await Promise.all(
|
|
533
|
+
agentAddresses.map(async (addr) => ({
|
|
534
|
+
address: addr,
|
|
535
|
+
publicKeyHex: await lookupKey(addr),
|
|
536
|
+
})),
|
|
537
|
+
);
|
|
538
|
+
|
|
539
|
+
// If the connection was closed or superseded while we were awaiting
|
|
540
|
+
// key lookups, bail out.
|
|
541
|
+
if (!connections.has(ws)) return;
|
|
542
|
+
|
|
543
|
+
const challenges = new Map<
|
|
544
|
+
string,
|
|
545
|
+
{ nonce: Uint8Array; publicKey: Uint8Array }
|
|
546
|
+
>();
|
|
547
|
+
const challengeEntries: { address: string; nonce: string }[] = [];
|
|
548
|
+
|
|
549
|
+
for (const { address, publicKeyHex } of keyLookups) {
|
|
550
|
+
if (publicKeyHex === null) {
|
|
551
|
+
conn.send({
|
|
552
|
+
type: "challenge.failed",
|
|
553
|
+
address,
|
|
554
|
+
reason: "Unknown agent address",
|
|
555
|
+
});
|
|
556
|
+
continue;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
let publicKey: Uint8Array;
|
|
560
|
+
try {
|
|
561
|
+
publicKey = hexDecode(publicKeyHex);
|
|
562
|
+
} catch {
|
|
563
|
+
conn.send({
|
|
564
|
+
type: "challenge.failed",
|
|
565
|
+
address,
|
|
566
|
+
reason: "Stored public key is corrupt",
|
|
567
|
+
});
|
|
568
|
+
logger.error`Corrupt stored public key for ${address}`;
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
const nonce = randomBytes(32);
|
|
573
|
+
challenges.set(address, { nonce, publicKey });
|
|
574
|
+
challengeEntries.push({ address, nonce: hexEncode(nonce) });
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
if (challenges.size === 0) return;
|
|
578
|
+
|
|
579
|
+
// If another reconnect completed while we were building the
|
|
580
|
+
// challenge, it will have written its own entry. Don't overwrite.
|
|
581
|
+
if (pendingChallenges.has(ws)) return;
|
|
582
|
+
|
|
583
|
+
const timer = setTimeout(() => {
|
|
584
|
+
pendingChallenges.delete(ws);
|
|
585
|
+
logger.warn`Challenge timed out for sidecar ${sidecarId}`;
|
|
586
|
+
}, challengeTimeoutMs);
|
|
587
|
+
|
|
588
|
+
pendingChallenges.set(ws, {
|
|
589
|
+
sidecarId,
|
|
590
|
+
challenges,
|
|
591
|
+
deployRefs,
|
|
592
|
+
timer,
|
|
593
|
+
});
|
|
594
|
+
|
|
595
|
+
conn.send({ type: "challenge", challenges: challengeEntries });
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
async function handleChallengeResponse(
|
|
599
|
+
ws: WsHandle,
|
|
600
|
+
responses: { address: string; signature: string }[],
|
|
601
|
+
): Promise<void> {
|
|
602
|
+
const challenge = pendingChallenges.get(ws);
|
|
603
|
+
if (challenge === undefined) {
|
|
604
|
+
logger.warn`Received challenge.response with no pending challenge`;
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
clearTimeout(challenge.timer);
|
|
609
|
+
pendingChallenges.delete(ws);
|
|
610
|
+
|
|
611
|
+
const conn = connections.get(ws);
|
|
612
|
+
if (conn === undefined) return;
|
|
613
|
+
|
|
614
|
+
const verified: string[] = [];
|
|
615
|
+
const responded = new Set<string>();
|
|
616
|
+
|
|
617
|
+
for (const { address, signature } of responses) {
|
|
618
|
+
responded.add(address);
|
|
619
|
+
|
|
620
|
+
const entry = challenge.challenges.get(address);
|
|
621
|
+
if (entry === undefined) {
|
|
622
|
+
conn.send({
|
|
623
|
+
type: "challenge.failed",
|
|
624
|
+
address,
|
|
625
|
+
reason: "Address was not challenged",
|
|
626
|
+
});
|
|
627
|
+
continue;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
let valid = false;
|
|
631
|
+
try {
|
|
632
|
+
const nonceBytes = entry.nonce;
|
|
633
|
+
const addressBytes = new TextEncoder().encode(address);
|
|
634
|
+
const payload = new Uint8Array(nonceBytes.length + addressBytes.length);
|
|
635
|
+
payload.set(nonceBytes);
|
|
636
|
+
payload.set(addressBytes, nonceBytes.length);
|
|
637
|
+
|
|
638
|
+
const sigBytes = hexDecode(signature);
|
|
639
|
+
valid = verifyEd25519(payload, sigBytes, entry.publicKey);
|
|
640
|
+
} catch (err) {
|
|
641
|
+
logger.warn`Challenge failed for ${address}: ${err instanceof Error ? err.message : String(err)}`;
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
if (valid) {
|
|
645
|
+
verified.push(address);
|
|
646
|
+
} else {
|
|
647
|
+
conn.send({
|
|
648
|
+
type: "challenge.failed",
|
|
649
|
+
address,
|
|
650
|
+
reason: "Signature verification failed",
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
// Notify about challenged addresses that were omitted from the response.
|
|
656
|
+
for (const address of challenge.challenges.keys()) {
|
|
657
|
+
if (!responded.has(address)) {
|
|
658
|
+
conn.send({
|
|
659
|
+
type: "challenge.failed",
|
|
660
|
+
address,
|
|
661
|
+
reason: "No response provided for challenged address",
|
|
662
|
+
});
|
|
663
|
+
logger.warn`Challenge failed for ${address}: no response provided`;
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
// Add verified addresses to the routing table immediately so that
|
|
668
|
+
// `agent.reconnected` subscribers can use sendRequest-based methods
|
|
669
|
+
// (e.g. sendGrantsUpdate). Addresses that fail governance are
|
|
670
|
+
// rolled back from the routing table afterward.
|
|
671
|
+
for (const addr of verified) {
|
|
672
|
+
// If a different ws still owns this address (live takeover via
|
|
673
|
+
// verified reconnect), evict its cached connector state before
|
|
674
|
+
// routing flips. The new owner's harness will bootstrap via
|
|
675
|
+
// restore-fires-callback.
|
|
676
|
+
const prevWs = addressIndex.get(addr);
|
|
677
|
+
if (prevWs !== undefined && prevWs !== ws) {
|
|
678
|
+
connectorStates.delete(addr);
|
|
679
|
+
}
|
|
680
|
+
conn.agentAddresses.add(addr);
|
|
681
|
+
addressIndex.set(addr, ws);
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
const ready: string[] = [];
|
|
685
|
+
const failed: string[] = [];
|
|
686
|
+
|
|
687
|
+
for (const addr of verified) {
|
|
688
|
+
if (events.listenerCount("agent.reconnected") === 0) {
|
|
689
|
+
ready.push(addr);
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
try {
|
|
693
|
+
await events.emitAndAwait("agent.reconnected", { agentAddress: addr });
|
|
694
|
+
ready.push(addr);
|
|
695
|
+
} catch (err) {
|
|
696
|
+
logger.error`Failed to handle reconnection for ${addr}: ${err instanceof Error ? err.message : String(err)}`;
|
|
697
|
+
failed.push(addr);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// If a second reconnect arrived during the callback loop, our conn
|
|
702
|
+
// is orphaned — handleRegister already rebuilt the connection and
|
|
703
|
+
// cleared addressIndex. Bail out; the new reconnect flow will
|
|
704
|
+
// re-verify these addresses from scratch.
|
|
705
|
+
if (connections.get(ws) !== conn) {
|
|
706
|
+
logger.warn`Challenge response processing aborted: connection superseded by new reconnect`;
|
|
707
|
+
return;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
// Roll back failed addresses from the routing table.
|
|
711
|
+
for (const addr of failed) {
|
|
712
|
+
conn.agentAddresses.delete(addr);
|
|
713
|
+
addressIndex.delete(addr);
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
// Flush queued messages only for ready addresses.
|
|
717
|
+
for (const addr of ready) {
|
|
718
|
+
flushDisconnectedQueue(addr, conn);
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
|
|
722
|
+
// so reconnect completion is not blocked on pack transfer. The
|
|
723
|
+
// wire layer owns the staleness comparison; the event fires only
|
|
724
|
+
// when staleness is confirmed.
|
|
725
|
+
const checkDeployRef = lookups.lookupDeployRef;
|
|
726
|
+
if (checkDeployRef !== undefined) {
|
|
727
|
+
for (const addr of ready) {
|
|
728
|
+
void (async () => {
|
|
729
|
+
try {
|
|
730
|
+
const hubRef = await checkDeployRef(addr);
|
|
731
|
+
if (hubRef === null) return;
|
|
732
|
+
const sidecarRef = challenge.deployRefs[addr];
|
|
733
|
+
if (sidecarRef === hubRef) return;
|
|
734
|
+
|
|
735
|
+
logger.info`Re-deploying ${addr}: sidecar ref ${sidecarRef ?? "(none)"} != hub ref ${hubRef.slice(0, 8)}`;
|
|
736
|
+
await events.emitAndAwait("deploy.ref.stale", {
|
|
737
|
+
agentAddress: addr,
|
|
738
|
+
});
|
|
739
|
+
} catch (err) {
|
|
740
|
+
logger.error`Failed to re-deploy ${addr} after reconnect: ${err instanceof Error ? err.message : String(err)}`;
|
|
741
|
+
}
|
|
742
|
+
})();
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
// Failed addresses: reset their queue TTL so messages survive until
|
|
747
|
+
// the next reconnect attempt, and notify the sidecar.
|
|
748
|
+
for (const addr of failed) {
|
|
749
|
+
const entry = disconnectedAgents.get(addr);
|
|
750
|
+
if (entry !== undefined) {
|
|
751
|
+
clearTimeout(entry.timer);
|
|
752
|
+
entry.timer = setTimeout(() => {
|
|
753
|
+
disconnectedAgents.delete(addr);
|
|
754
|
+
}, disconnectQueueTTLMs);
|
|
755
|
+
}
|
|
756
|
+
conn.send({
|
|
757
|
+
type: "challenge.failed",
|
|
758
|
+
address: addr,
|
|
759
|
+
reason: "Reconnection rejected by governance",
|
|
760
|
+
});
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
logger.info`Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
function handleMailOutbound(rawMessage: string, recipients: string[]): void {
|
|
767
|
+
// Route to locally connected sidecars first, then try disconnect queues.
|
|
768
|
+
const unrouted: string[] = [];
|
|
769
|
+
for (const recipient of recipients) {
|
|
770
|
+
const targetWs = addressIndex.get(recipient);
|
|
771
|
+
if (targetWs !== undefined) {
|
|
772
|
+
const conn = connections.get(targetWs);
|
|
773
|
+
if (conn !== undefined) {
|
|
774
|
+
conn.send({
|
|
775
|
+
type: "mail.inbound",
|
|
776
|
+
agentAddress: recipient,
|
|
777
|
+
rawMessage,
|
|
778
|
+
});
|
|
779
|
+
continue;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
const frame: HubFrame = {
|
|
784
|
+
type: "mail.inbound",
|
|
785
|
+
agentAddress: recipient,
|
|
786
|
+
rawMessage,
|
|
787
|
+
};
|
|
788
|
+
if (enqueueForDisconnected(recipient, frame)) continue;
|
|
789
|
+
|
|
790
|
+
unrouted.push(recipient);
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
// Anything not routed locally is emitted as a notification. The
|
|
794
|
+
// host decides whether to relay onto an external transport, log,
|
|
795
|
+
// or drop. The wire layer takes no stance.
|
|
796
|
+
if (unrouted.length > 0) {
|
|
797
|
+
events.emit("mail.outbound.undelivered", {
|
|
798
|
+
rawMessage,
|
|
799
|
+
recipients: unrouted,
|
|
800
|
+
});
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
async function handleMailPersist(
|
|
805
|
+
persist: NonNullable<SidecarLookups["persistMail"]>,
|
|
806
|
+
rawMessage: string,
|
|
807
|
+
senderAddress: string,
|
|
808
|
+
recipients: string[],
|
|
809
|
+
): Promise<void> {
|
|
810
|
+
let results: SidecarMailPersistedRow[];
|
|
811
|
+
let raw: Uint8Array;
|
|
812
|
+
try {
|
|
813
|
+
raw = Uint8Array.from(atob(rawMessage), (c) => c.charCodeAt(0));
|
|
814
|
+
results = await persist({
|
|
815
|
+
senderAddress,
|
|
816
|
+
recipients,
|
|
817
|
+
raw,
|
|
818
|
+
});
|
|
819
|
+
} catch (err) {
|
|
820
|
+
logger.error`Failed to persist mail from ${senderAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
for (const result of results) {
|
|
825
|
+
events.emit("mail.persisted", {
|
|
826
|
+
id: result.id,
|
|
827
|
+
raw,
|
|
828
|
+
createdAt: result.createdAt,
|
|
829
|
+
direction: result.direction,
|
|
830
|
+
instanceId: result.instanceId,
|
|
831
|
+
address: result.address,
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
function handleClose(ws: WsHandle): void {
|
|
837
|
+
const conn = connections.get(ws);
|
|
838
|
+
if (conn === undefined) return;
|
|
839
|
+
|
|
840
|
+
for (const addr of conn.agentAddresses) {
|
|
841
|
+
// Only remove routing and pending state if this connection still
|
|
842
|
+
// owns the address. A reconnected sidecar may have already claimed it.
|
|
843
|
+
if (addressIndex.get(addr) === ws) {
|
|
844
|
+
addressIndex.delete(addr);
|
|
845
|
+
// Drop cached connector state for the same reason: a takeover
|
|
846
|
+
// sidecar's state lives in connectorStates under the same key,
|
|
847
|
+
// and only this owner's close should evict it. The next
|
|
848
|
+
// reconnect re-bootstraps via the router's
|
|
849
|
+
// restore-fires-callback path.
|
|
850
|
+
connectorStates.delete(addr);
|
|
851
|
+
const deployReq = pendingDeploys.get(addr);
|
|
852
|
+
if (deployReq !== undefined && deployReq.ws === ws) {
|
|
853
|
+
clearTimeout(deployReq.timer);
|
|
854
|
+
pendingDeploys.delete(addr);
|
|
855
|
+
deployReq.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
// Create a queue entry so messages can accumulate while the
|
|
859
|
+
// sidecar is disconnected. Skip if the agent is being undeployed
|
|
860
|
+
// or has a pending session start — there is no point queuing
|
|
861
|
+
// messages for an agent being torn down or one that never started.
|
|
862
|
+
if (!pendingUndeploys.has(addr) && !pendingSessionStarts.has(addr)) {
|
|
863
|
+
const timer = setTimeout(() => {
|
|
864
|
+
disconnectedAgents.delete(addr);
|
|
865
|
+
}, disconnectQueueTTLMs);
|
|
866
|
+
disconnectedAgents.set(addr, { queue: [], timer });
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
connections.delete(ws);
|
|
871
|
+
|
|
872
|
+
// Cancel the liveness timer for this connection.
|
|
873
|
+
const livenessTimer = livenessTimers.get(ws);
|
|
874
|
+
if (livenessTimer !== undefined) {
|
|
875
|
+
clearTimeout(livenessTimer);
|
|
876
|
+
livenessTimers.delete(ws);
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
// Cancel any pending challenge for this connection.
|
|
880
|
+
const challengeReq = pendingChallenges.get(ws);
|
|
881
|
+
if (challengeReq !== undefined) {
|
|
882
|
+
clearTimeout(challengeReq.timer);
|
|
883
|
+
pendingChallenges.delete(ws);
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
// Reject any in-flight requests that were sent to this sidecar.
|
|
887
|
+
for (const [requestId, req] of pending) {
|
|
888
|
+
if (req.ws !== ws) continue;
|
|
889
|
+
clearTimeout(req.timer);
|
|
890
|
+
pending.delete(requestId);
|
|
891
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
// Reject any in-flight pack transfers for this sidecar.
|
|
895
|
+
for (const [transferId, pack] of pendingPacks) {
|
|
896
|
+
if (pack.ws !== ws) continue;
|
|
897
|
+
clearTimeout(pack.timer);
|
|
898
|
+
pendingPacks.delete(transferId);
|
|
899
|
+
pack.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
// Reject any in-flight session starts for this sidecar.
|
|
903
|
+
for (const [addr, req] of pendingSessionStarts) {
|
|
904
|
+
if (req.ws !== ws) continue;
|
|
905
|
+
clearTimeout(req.timer);
|
|
906
|
+
pendingSessionStarts.delete(addr);
|
|
907
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
// Reject any in-flight undeploys for this sidecar.
|
|
911
|
+
for (const [addr, req] of pendingUndeploys) {
|
|
912
|
+
if (req.ws !== ws) continue;
|
|
913
|
+
clearTimeout(req.timer);
|
|
914
|
+
pendingUndeploys.delete(addr);
|
|
915
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
// Cancel any in-flight inbound state transfers from this sidecar.
|
|
919
|
+
for (const addr of conn.agentAddresses) {
|
|
920
|
+
statePackReceiver.cancelByAgent(addr);
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
events.emit("sidecar.disconnect", {
|
|
924
|
+
agentAddresses: [...conn.agentAddresses],
|
|
925
|
+
});
|
|
926
|
+
|
|
927
|
+
logger.info`Sidecar ${conn.sidecarId} disconnected`;
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
function nextRequestId(): string {
|
|
931
|
+
return `req-${++requestCounter}`;
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
function sendRequest(
|
|
935
|
+
agentAddress: string,
|
|
936
|
+
buildFrame: (requestId: string) => HubFrame,
|
|
937
|
+
): Promise<void> {
|
|
938
|
+
const ws = addressIndex.get(agentAddress);
|
|
939
|
+
if (ws === undefined) {
|
|
940
|
+
return Promise.reject(
|
|
941
|
+
new Error(`No sidecar connected for agent "${agentAddress}"`),
|
|
942
|
+
);
|
|
943
|
+
}
|
|
944
|
+
const conn = connections.get(ws);
|
|
945
|
+
if (conn === undefined) {
|
|
946
|
+
return Promise.reject(
|
|
947
|
+
new Error(`No sidecar connected for agent "${agentAddress}"`),
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
const requestId = nextRequestId();
|
|
952
|
+
const frame = buildFrame(requestId);
|
|
953
|
+
|
|
954
|
+
return new Promise<void>((resolve, reject) => {
|
|
955
|
+
const timer = setTimeout(() => {
|
|
956
|
+
pending.delete(requestId);
|
|
957
|
+
reject(
|
|
958
|
+
new Error(
|
|
959
|
+
`Request ${requestId} timed out after ${requestTimeoutMs}ms`,
|
|
960
|
+
),
|
|
961
|
+
);
|
|
962
|
+
}, requestTimeoutMs);
|
|
963
|
+
|
|
964
|
+
pending.set(requestId, {
|
|
965
|
+
requestId,
|
|
966
|
+
ws,
|
|
967
|
+
resolve,
|
|
968
|
+
reject(error: string) {
|
|
969
|
+
reject(new Error(error));
|
|
970
|
+
},
|
|
971
|
+
timer,
|
|
972
|
+
});
|
|
973
|
+
|
|
974
|
+
conn.send(frame);
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
function resolvePending(requestId: string): void {
|
|
979
|
+
const req = pending.get(requestId);
|
|
980
|
+
if (req === undefined) return;
|
|
981
|
+
clearTimeout(req.timer);
|
|
982
|
+
pending.delete(requestId);
|
|
983
|
+
req.resolve();
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
function rejectPending(requestId: string, error: string): void {
|
|
987
|
+
const req = pending.get(requestId);
|
|
988
|
+
if (req === undefined) return;
|
|
989
|
+
clearTimeout(req.timer);
|
|
990
|
+
pending.delete(requestId);
|
|
991
|
+
req.reject(error);
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
function resolvePackPending(transferId: string): void {
|
|
995
|
+
const entry = pendingPacks.get(transferId);
|
|
996
|
+
if (entry === undefined) return;
|
|
997
|
+
clearTimeout(entry.timer);
|
|
998
|
+
pendingPacks.delete(transferId);
|
|
999
|
+
entry.resolve();
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
function rejectPackPending(transferId: string, reason: string): void {
|
|
1003
|
+
const entry = pendingPacks.get(transferId);
|
|
1004
|
+
if (entry === undefined) return;
|
|
1005
|
+
clearTimeout(entry.timer);
|
|
1006
|
+
pendingPacks.delete(transferId);
|
|
1007
|
+
entry.reject(reason);
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
function resolveSessionStartPending(agentAddress: string): void {
|
|
1011
|
+
const req = pendingSessionStarts.get(agentAddress);
|
|
1012
|
+
if (req === undefined) {
|
|
1013
|
+
logger.warn`Received session.start.ack for "${agentAddress}" with no pending start`;
|
|
1014
|
+
return;
|
|
1015
|
+
}
|
|
1016
|
+
clearTimeout(req.timer);
|
|
1017
|
+
pendingSessionStarts.delete(agentAddress);
|
|
1018
|
+
req.resolve();
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
function rejectSessionStartPending(
|
|
1022
|
+
agentAddress: string,
|
|
1023
|
+
error: string,
|
|
1024
|
+
): void {
|
|
1025
|
+
const req = pendingSessionStarts.get(agentAddress);
|
|
1026
|
+
if (req === undefined) return;
|
|
1027
|
+
clearTimeout(req.timer);
|
|
1028
|
+
pendingSessionStarts.delete(agentAddress);
|
|
1029
|
+
req.reject(error);
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
function resolveUndeployPending(agentAddress: string): void {
|
|
1033
|
+
const req = pendingUndeploys.get(agentAddress);
|
|
1034
|
+
if (req === undefined) {
|
|
1035
|
+
logger.warn`Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`;
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
clearTimeout(req.timer);
|
|
1039
|
+
pendingUndeploys.delete(agentAddress);
|
|
1040
|
+
req.resolve();
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
function rejectUndeployPending(agentAddress: string, error: string): void {
|
|
1044
|
+
const req = pendingUndeploys.get(agentAddress);
|
|
1045
|
+
if (req === undefined) return;
|
|
1046
|
+
clearTimeout(req.timer);
|
|
1047
|
+
pendingUndeploys.delete(agentAddress);
|
|
1048
|
+
req.reject(error);
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
function handleStatePackPush(ws: WsHandle, frame: PackPushFrame): void {
|
|
1052
|
+
const conn = connections.get(ws);
|
|
1053
|
+
if (conn === undefined) return;
|
|
1054
|
+
if (!conn.agentAddresses.has(frame.agentAddress)) {
|
|
1055
|
+
logger.warn`Received repo.pack.push for unrouted agent ${frame.agentAddress}`;
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
const reason = statePackReceiver.handlePush(frame);
|
|
1060
|
+
if (reason !== null) {
|
|
1061
|
+
conn.send({
|
|
1062
|
+
type: "repo.pack.reject",
|
|
1063
|
+
agentAddress: frame.agentAddress,
|
|
1064
|
+
repoId: frame.repoId,
|
|
1065
|
+
transferId: frame.transferId,
|
|
1066
|
+
reason,
|
|
1067
|
+
});
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
async function handleStatePackDone(
|
|
1072
|
+
ws: WsHandle,
|
|
1073
|
+
frame: PackDoneFrame,
|
|
1074
|
+
): Promise<void> {
|
|
1075
|
+
const conn = connections.get(ws);
|
|
1076
|
+
if (conn === undefined) return;
|
|
1077
|
+
if (!conn.agentAddresses.has(frame.agentAddress)) {
|
|
1078
|
+
logger.warn`Received repo.pack.done for unrouted agent ${frame.agentAddress}`;
|
|
1079
|
+
return;
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
const result = statePackReceiver.handleDone(frame);
|
|
1083
|
+
if (result === null) {
|
|
1084
|
+
conn.send({
|
|
1085
|
+
type: "repo.pack.reject",
|
|
1086
|
+
agentAddress: frame.agentAddress,
|
|
1087
|
+
repoId: frame.repoId,
|
|
1088
|
+
transferId: frame.transferId,
|
|
1089
|
+
reason: "corrupt",
|
|
1090
|
+
});
|
|
1091
|
+
return;
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
const receiveStatePack = lookups.receiveStatePack;
|
|
1095
|
+
if (receiveStatePack === undefined) {
|
|
1096
|
+
conn.send({
|
|
1097
|
+
type: "repo.pack.ack",
|
|
1098
|
+
agentAddress: frame.agentAddress,
|
|
1099
|
+
repoId: frame.repoId,
|
|
1100
|
+
transferId: frame.transferId,
|
|
1101
|
+
});
|
|
1102
|
+
return;
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
const verdict = await receiveStatePack(
|
|
1106
|
+
frame.repoId,
|
|
1107
|
+
result.pack,
|
|
1108
|
+
result.ref,
|
|
1109
|
+
result.commitSha,
|
|
1110
|
+
);
|
|
1111
|
+
|
|
1112
|
+
// Connection may have closed during async verification.
|
|
1113
|
+
const currentConn = connections.get(ws);
|
|
1114
|
+
if (currentConn === undefined) return;
|
|
1115
|
+
|
|
1116
|
+
if (verdict.accepted) {
|
|
1117
|
+
currentConn.send({
|
|
1118
|
+
type: "repo.pack.ack",
|
|
1119
|
+
agentAddress: frame.agentAddress,
|
|
1120
|
+
repoId: frame.repoId,
|
|
1121
|
+
transferId: frame.transferId,
|
|
1122
|
+
});
|
|
1123
|
+
} else {
|
|
1124
|
+
currentConn.send({
|
|
1125
|
+
type: "repo.pack.reject",
|
|
1126
|
+
agentAddress: frame.agentAddress,
|
|
1127
|
+
repoId: frame.repoId,
|
|
1128
|
+
transferId: frame.transferId,
|
|
1129
|
+
reason: verdict.reason,
|
|
1130
|
+
});
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
// Pack transfers may take longer than session requests due to data volume.
|
|
1135
|
+
const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
|
|
1136
|
+
|
|
1137
|
+
function sendPack(
|
|
1138
|
+
agentAddress: string,
|
|
1139
|
+
pack: Uint8Array,
|
|
1140
|
+
ref: string,
|
|
1141
|
+
commitSha: string,
|
|
1142
|
+
options?: SendPackOptions,
|
|
1143
|
+
): Promise<void> {
|
|
1144
|
+
const ws = addressIndex.get(agentAddress);
|
|
1145
|
+
if (ws === undefined) {
|
|
1146
|
+
return Promise.reject(
|
|
1147
|
+
new Error(`No sidecar connected for agent "${agentAddress}"`),
|
|
1148
|
+
);
|
|
1149
|
+
}
|
|
1150
|
+
const conn = connections.get(ws);
|
|
1151
|
+
if (conn === undefined) {
|
|
1152
|
+
return Promise.reject(
|
|
1153
|
+
new Error(`No sidecar connected for agent "${agentAddress}"`),
|
|
1154
|
+
);
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
const transferId = `pack-${++packCounter}`;
|
|
1158
|
+
// For the agent-state flow the destination agent and the source repo
|
|
1159
|
+
// are the same entity, so `repoId.id === agentAddress`. Asset packs
|
|
1160
|
+
// override this with the SOURCE asset's id so audit can correlate
|
|
1161
|
+
// the pack back to its hub-side origin.
|
|
1162
|
+
const repoId: RepoId = options?.repoId ?? {
|
|
1163
|
+
kind: "agent-state",
|
|
1164
|
+
id: agentAddress,
|
|
1165
|
+
};
|
|
1166
|
+
const mountPath = options?.mountPath;
|
|
1167
|
+
|
|
1168
|
+
// Register pending entry before sending frames so that a synchronous
|
|
1169
|
+
// repo.pack.ack (e.g. in tests or loopback transports) resolves correctly.
|
|
1170
|
+
return new Promise<void>((resolve, reject) => {
|
|
1171
|
+
const timer = setTimeout(() => {
|
|
1172
|
+
pendingPacks.delete(transferId);
|
|
1173
|
+
reject(
|
|
1174
|
+
new Error(
|
|
1175
|
+
`Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`,
|
|
1176
|
+
),
|
|
1177
|
+
);
|
|
1178
|
+
}, PACK_TIMEOUT_MS);
|
|
1179
|
+
|
|
1180
|
+
pendingPacks.set(transferId, {
|
|
1181
|
+
transferId,
|
|
1182
|
+
ws,
|
|
1183
|
+
resolve,
|
|
1184
|
+
reject(error: string) {
|
|
1185
|
+
reject(new Error(`Pack rejected: ${error}`));
|
|
1186
|
+
},
|
|
1187
|
+
timer,
|
|
1188
|
+
});
|
|
1189
|
+
|
|
1190
|
+
// Send chunks
|
|
1191
|
+
for (const chunk of chunkPack(pack)) {
|
|
1192
|
+
conn.send({
|
|
1193
|
+
type: "repo.pack.push",
|
|
1194
|
+
agentAddress,
|
|
1195
|
+
repoId,
|
|
1196
|
+
transferId,
|
|
1197
|
+
seq: chunk.seq,
|
|
1198
|
+
data: chunk.data,
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
// Send done
|
|
1203
|
+
conn.send({
|
|
1204
|
+
type: "repo.pack.done",
|
|
1205
|
+
agentAddress,
|
|
1206
|
+
repoId,
|
|
1207
|
+
transferId,
|
|
1208
|
+
ref,
|
|
1209
|
+
commitSha,
|
|
1210
|
+
...(mountPath !== undefined ? { mountPath } : {}),
|
|
1211
|
+
});
|
|
1212
|
+
});
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
function routeMail(agentAddress: string, rawMessage: string): boolean {
|
|
1216
|
+
const ws = addressIndex.get(agentAddress);
|
|
1217
|
+
if (ws !== undefined) {
|
|
1218
|
+
const conn = connections.get(ws);
|
|
1219
|
+
if (conn !== undefined) {
|
|
1220
|
+
conn.send({
|
|
1221
|
+
type: "mail.inbound",
|
|
1222
|
+
agentAddress,
|
|
1223
|
+
rawMessage,
|
|
1224
|
+
});
|
|
1225
|
+
return true;
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
// If the agent recently disconnected, queue for delivery on reconnect.
|
|
1230
|
+
const frame: HubFrame = { type: "mail.inbound", agentAddress, rawMessage };
|
|
1231
|
+
return enqueueForDisconnected(agentAddress, frame);
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
async function handleDeployAck(
|
|
1235
|
+
agentAddress: string,
|
|
1236
|
+
publicKey: string,
|
|
1237
|
+
): Promise<void> {
|
|
1238
|
+
if (!pendingDeploys.has(agentAddress)) {
|
|
1239
|
+
logger.warn`Received agent.deploy.ack for "${agentAddress}" with no pending deploy`;
|
|
1240
|
+
return;
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
if (events.listenerCount("agent.deploy.ack") > 0) {
|
|
1244
|
+
try {
|
|
1245
|
+
await events.emitAndAwait("agent.deploy.ack", {
|
|
1246
|
+
agentAddress,
|
|
1247
|
+
publicKey,
|
|
1248
|
+
});
|
|
1249
|
+
} catch (err) {
|
|
1250
|
+
rejectDeployPending(
|
|
1251
|
+
agentAddress,
|
|
1252
|
+
`Failed to store public key: ${err instanceof Error ? err.message : String(err)}`,
|
|
1253
|
+
);
|
|
1254
|
+
return;
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
resolveDeployPending(agentAddress);
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
function resolveDeployPending(agentAddress: string): void {
|
|
1261
|
+
const req = pendingDeploys.get(agentAddress);
|
|
1262
|
+
if (req === undefined) return;
|
|
1263
|
+
clearTimeout(req.timer);
|
|
1264
|
+
pendingDeploys.delete(agentAddress);
|
|
1265
|
+
req.resolve();
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
function rejectDeployPending(agentAddress: string, error: string): void {
|
|
1269
|
+
const req = pendingDeploys.get(agentAddress);
|
|
1270
|
+
if (req === undefined) return;
|
|
1271
|
+
clearTimeout(req.timer);
|
|
1272
|
+
pendingDeploys.delete(agentAddress);
|
|
1273
|
+
req.reject(error);
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
async function sendAgentDeploy(
|
|
1277
|
+
agentAddress: string,
|
|
1278
|
+
harnessConfig: HarnessConfig,
|
|
1279
|
+
): Promise<void> {
|
|
1280
|
+
if (hubPublicKeyHex === undefined) {
|
|
1281
|
+
throw new Error("Hub signing key is required for agent deployment");
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
if (pendingDeploys.has(agentAddress)) {
|
|
1285
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
const ws =
|
|
1289
|
+
addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
|
|
1290
|
+
|
|
1291
|
+
if (ws === undefined) {
|
|
1292
|
+
throw new Error(`No sidecar available for agent "${agentAddress}"`);
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
const conn = connections.get(ws);
|
|
1296
|
+
if (conn === undefined) {
|
|
1297
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
conn.agentAddresses.add(agentAddress);
|
|
1301
|
+
addressIndex.set(agentAddress, ws);
|
|
1302
|
+
|
|
1303
|
+
return new Promise<void>((resolve, reject) => {
|
|
1304
|
+
const timer = setTimeout(() => {
|
|
1305
|
+
pendingDeploys.delete(agentAddress);
|
|
1306
|
+
if (addressIndex.get(agentAddress) === ws) {
|
|
1307
|
+
conn.agentAddresses.delete(agentAddress);
|
|
1308
|
+
addressIndex.delete(agentAddress);
|
|
1309
|
+
}
|
|
1310
|
+
reject(
|
|
1311
|
+
new Error(
|
|
1312
|
+
`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
1313
|
+
),
|
|
1314
|
+
);
|
|
1315
|
+
}, requestTimeoutMs);
|
|
1316
|
+
|
|
1317
|
+
pendingDeploys.set(agentAddress, {
|
|
1318
|
+
agentAddress,
|
|
1319
|
+
ws,
|
|
1320
|
+
resolve() {
|
|
1321
|
+
resolve();
|
|
1322
|
+
},
|
|
1323
|
+
reject(error: string) {
|
|
1324
|
+
if (addressIndex.get(agentAddress) === ws) {
|
|
1325
|
+
conn.agentAddresses.delete(agentAddress);
|
|
1326
|
+
addressIndex.delete(agentAddress);
|
|
1327
|
+
}
|
|
1328
|
+
reject(new Error(error));
|
|
1329
|
+
},
|
|
1330
|
+
timer,
|
|
1331
|
+
});
|
|
1332
|
+
|
|
1333
|
+
conn.send({
|
|
1334
|
+
type: "agent.deploy",
|
|
1335
|
+
agentAddress,
|
|
1336
|
+
agentId: harnessConfig.agentId,
|
|
1337
|
+
config: harnessConfig,
|
|
1338
|
+
hubPublicKey: hubPublicKeyHex,
|
|
1339
|
+
});
|
|
1340
|
+
});
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
function findSidecarForNewAgent(_agentAddress: string): WsHandle | undefined {
|
|
1344
|
+
const first = connections.entries().next();
|
|
1345
|
+
if (first.done) return undefined;
|
|
1346
|
+
return first.value[0];
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
function sendAgentUndeploy(
|
|
1350
|
+
agentAddress: string,
|
|
1351
|
+
reason: string,
|
|
1352
|
+
): Promise<void> {
|
|
1353
|
+
const ws = addressIndex.get(agentAddress);
|
|
1354
|
+
if (ws === undefined) {
|
|
1355
|
+
return Promise.reject(
|
|
1356
|
+
new Error(`No sidecar connected for agent "${agentAddress}"`),
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
const conn = connections.get(ws);
|
|
1360
|
+
if (conn === undefined) {
|
|
1361
|
+
return Promise.reject(
|
|
1362
|
+
new Error(`No sidecar connected for agent "${agentAddress}"`),
|
|
1363
|
+
);
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
return new Promise<void>((resolve, reject) => {
|
|
1367
|
+
const timer = setTimeout(() => {
|
|
1368
|
+
pendingUndeploys.delete(agentAddress);
|
|
1369
|
+
removeAgentAddress(ws, agentAddress);
|
|
1370
|
+
reject(
|
|
1371
|
+
new Error(
|
|
1372
|
+
`Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
1373
|
+
),
|
|
1374
|
+
);
|
|
1375
|
+
}, requestTimeoutMs);
|
|
1376
|
+
|
|
1377
|
+
pendingUndeploys.set(agentAddress, {
|
|
1378
|
+
agentAddress,
|
|
1379
|
+
ws,
|
|
1380
|
+
resolve() {
|
|
1381
|
+
removeAgentAddress(ws, agentAddress);
|
|
1382
|
+
resolve();
|
|
1383
|
+
},
|
|
1384
|
+
reject(error: string) {
|
|
1385
|
+
removeAgentAddress(ws, agentAddress);
|
|
1386
|
+
reject(new Error(error));
|
|
1387
|
+
},
|
|
1388
|
+
timer,
|
|
1389
|
+
});
|
|
1390
|
+
|
|
1391
|
+
conn.send({
|
|
1392
|
+
type: "agent.undeploy",
|
|
1393
|
+
agentAddress,
|
|
1394
|
+
reason,
|
|
1395
|
+
});
|
|
1396
|
+
});
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
function sendSessionStart(agentAddress: string): Promise<void> {
|
|
1400
|
+
const ws = addressIndex.get(agentAddress);
|
|
1401
|
+
if (ws === undefined) {
|
|
1402
|
+
return Promise.reject(
|
|
1403
|
+
new Error(`No sidecar connected for agent "${agentAddress}"`),
|
|
1404
|
+
);
|
|
1405
|
+
}
|
|
1406
|
+
const conn = connections.get(ws);
|
|
1407
|
+
if (conn === undefined) {
|
|
1408
|
+
return Promise.reject(
|
|
1409
|
+
new Error(`No sidecar connected for agent "${agentAddress}"`),
|
|
1410
|
+
);
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
return new Promise<void>((resolve, reject) => {
|
|
1414
|
+
const timer = setTimeout(() => {
|
|
1415
|
+
pendingSessionStarts.delete(agentAddress);
|
|
1416
|
+
removeAgentAddress(ws, agentAddress);
|
|
1417
|
+
reject(
|
|
1418
|
+
new Error(
|
|
1419
|
+
`Session start for "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
1420
|
+
),
|
|
1421
|
+
);
|
|
1422
|
+
}, requestTimeoutMs);
|
|
1423
|
+
|
|
1424
|
+
pendingSessionStarts.set(agentAddress, {
|
|
1425
|
+
agentAddress,
|
|
1426
|
+
ws,
|
|
1427
|
+
resolve() {
|
|
1428
|
+
resolve();
|
|
1429
|
+
},
|
|
1430
|
+
reject(error: string) {
|
|
1431
|
+
removeAgentAddress(ws, agentAddress);
|
|
1432
|
+
reject(new Error(error));
|
|
1433
|
+
},
|
|
1434
|
+
timer,
|
|
1435
|
+
});
|
|
1436
|
+
|
|
1437
|
+
conn.send({
|
|
1438
|
+
type: "session.start",
|
|
1439
|
+
agentAddress,
|
|
1440
|
+
});
|
|
1441
|
+
});
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
async function sendSessionAbort(
|
|
1445
|
+
agentAddress: string,
|
|
1446
|
+
reason: AbortReason,
|
|
1447
|
+
): Promise<void> {
|
|
1448
|
+
await sendRequest(agentAddress, (requestId) => ({
|
|
1449
|
+
type: "session.abort",
|
|
1450
|
+
requestId,
|
|
1451
|
+
agentAddress,
|
|
1452
|
+
reason,
|
|
1453
|
+
}));
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
function removeAgentAddress(ws: WsHandle, agentAddress: string): void {
|
|
1457
|
+
addressIndex.delete(agentAddress);
|
|
1458
|
+
const conn = connections.get(ws);
|
|
1459
|
+
if (conn !== undefined) {
|
|
1460
|
+
conn.agentAddresses.delete(agentAddress);
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
function dispatchToSubscribers(agentAddress: string, event: unknown): void {
|
|
1465
|
+
const subs = agentSubscribers.get(agentAddress);
|
|
1466
|
+
if (subs === undefined) return;
|
|
1467
|
+
for (const cb of [...subs]) {
|
|
1468
|
+
try {
|
|
1469
|
+
cb(event);
|
|
1470
|
+
} catch (err) {
|
|
1471
|
+
logger.warn`Agent subscriber threw: ${err instanceof Error ? err.message : String(err)}`;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
function subscribeAgent(
|
|
1477
|
+
agentAddress: string,
|
|
1478
|
+
callback: (event: unknown) => void,
|
|
1479
|
+
): () => void {
|
|
1480
|
+
let subs = agentSubscribers.get(agentAddress);
|
|
1481
|
+
if (subs === undefined) {
|
|
1482
|
+
subs = new Set();
|
|
1483
|
+
agentSubscribers.set(agentAddress, subs);
|
|
1484
|
+
}
|
|
1485
|
+
subs.add(callback);
|
|
1486
|
+
return () => {
|
|
1487
|
+
const current = agentSubscribers.get(agentAddress);
|
|
1488
|
+
if (current === undefined) return;
|
|
1489
|
+
current.delete(callback);
|
|
1490
|
+
if (current.size === 0) {
|
|
1491
|
+
agentSubscribers.delete(agentAddress);
|
|
1492
|
+
}
|
|
1493
|
+
};
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
function getConnectedSidecars(): string[] {
|
|
1497
|
+
return Array.from(connections.values()).map((c) => c.sidecarId);
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
function getRoutableAddresses(): string[] {
|
|
1501
|
+
return Array.from(addressIndex.keys());
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
function getConnectorState(
|
|
1505
|
+
agentAddress: string,
|
|
1506
|
+
): ConnectorThreadState | null {
|
|
1507
|
+
return connectorStates.get(agentAddress) ?? null;
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
async function sendGrantsUpdate(
|
|
1511
|
+
agentAddress: string,
|
|
1512
|
+
grants: GrantRule[],
|
|
1513
|
+
): Promise<void> {
|
|
1514
|
+
await sendRequest(agentAddress, (requestId) => ({
|
|
1515
|
+
type: "grants.update",
|
|
1516
|
+
requestId,
|
|
1517
|
+
agentAddress,
|
|
1518
|
+
grants,
|
|
1519
|
+
}));
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
async function sendSourcesUpdate(
|
|
1523
|
+
agentAddress: string,
|
|
1524
|
+
sources: InferenceSource[],
|
|
1525
|
+
defaultSource: string,
|
|
1526
|
+
): Promise<void> {
|
|
1527
|
+
await sendRequest(agentAddress, (requestId) => ({
|
|
1528
|
+
type: "sources.update",
|
|
1529
|
+
requestId,
|
|
1530
|
+
agentAddress,
|
|
1531
|
+
sources,
|
|
1532
|
+
defaultSource,
|
|
1533
|
+
}));
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
function sendSyncRequest(agentAddress: string): void {
|
|
1537
|
+
const ws = addressIndex.get(agentAddress);
|
|
1538
|
+
if (ws === undefined) {
|
|
1539
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1540
|
+
}
|
|
1541
|
+
const conn = connections.get(ws);
|
|
1542
|
+
if (conn === undefined) {
|
|
1543
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
const transferId = `sync-${++packCounter}`;
|
|
1547
|
+
conn.send({
|
|
1548
|
+
type: "sync.request",
|
|
1549
|
+
agentAddress,
|
|
1550
|
+
transferId,
|
|
1551
|
+
});
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
return {
|
|
1555
|
+
handleOpen,
|
|
1556
|
+
handleMessage,
|
|
1557
|
+
handleClose,
|
|
1558
|
+
routeMail,
|
|
1559
|
+
sendAgentDeploy,
|
|
1560
|
+
sendAgentUndeploy,
|
|
1561
|
+
sendSessionStart,
|
|
1562
|
+
sendSessionAbort,
|
|
1563
|
+
sendGrantsUpdate,
|
|
1564
|
+
sendSourcesUpdate,
|
|
1565
|
+
sendPack,
|
|
1566
|
+
sendSyncRequest,
|
|
1567
|
+
subscribeAgent,
|
|
1568
|
+
dispatchAgentEvent: dispatchToSubscribers,
|
|
1569
|
+
getConnectedSidecars,
|
|
1570
|
+
getRoutableAddresses,
|
|
1571
|
+
getConnectorState,
|
|
1572
|
+
events,
|
|
1573
|
+
};
|
|
1574
|
+
}
|