@intx/hub-sessions 0.1.2 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +176 -0
- package/README.md +84 -1
- package/dist/agent-repo.d.ts +89 -0
- package/dist/agent-repo.js +109 -0
- package/dist/agent-state-kind.d.ts +12 -0
- package/dist/agent-state-kind.js +185 -0
- package/dist/asset-service.d.ts +123 -0
- package/dist/asset-service.js +349 -0
- package/dist/available-skills-stanza.d.ts +21 -0
- package/dist/available-skills-stanza.js +32 -0
- package/dist/credential-push.d.ts +32 -0
- package/dist/credential-push.js +85 -0
- package/dist/event-collector-registry.d.ts +20 -0
- package/dist/event-collector-registry.js +115 -0
- package/dist/event-collector.d.ts +39 -0
- package/dist/event-collector.js +357 -0
- package/dist/hub-session-lookups.d.ts +17 -0
- package/dist/hub-session-lookups.js +204 -0
- package/dist/hub-session-orchestrator.d.ts +25 -0
- package/dist/hub-session-orchestrator.js +122 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +16 -0
- package/dist/package-registry-kind.d.ts +70 -0
- package/dist/package-registry-kind.js +260 -0
- package/dist/repo-store/index.d.ts +4 -0
- package/dist/repo-store/index.js +3 -0
- package/dist/repo-store/store.d.ts +41 -0
- package/dist/repo-store/store.js +1692 -0
- package/dist/repo-store/subscribe-kind.d.ts +53 -0
- package/dist/repo-store/subscribe-kind.js +179 -0
- package/dist/repo-store/types.d.ts +483 -0
- package/dist/repo-store/types.js +42 -0
- package/dist/session-service.d.ts +235 -0
- package/dist/session-service.js +997 -0
- package/dist/skill-kind.d.ts +41 -0
- package/dist/skill-kind.js +288 -0
- package/dist/substrate.d.ts +8 -0
- package/dist/substrate.js +21 -0
- package/dist/workflow-kind.d.ts +21 -0
- package/dist/workflow-kind.js +263 -0
- package/dist/workflow-run-event-log.d.ts +21 -0
- package/dist/workflow-run-event-log.js +51 -0
- package/dist/workflow-run-kind.d.ts +326 -0
- package/dist/workflow-run-kind.js +2646 -0
- package/dist/workflow-run-reader.d.ts +47 -0
- package/dist/workflow-run-reader.js +157 -0
- package/dist/ws/index.d.ts +3 -0
- package/dist/ws/index.js +3 -0
- package/dist/ws/sidecar-events.d.ts +134 -0
- package/dist/ws/sidecar-events.js +70 -0
- package/dist/ws/sidecar-handler.d.ts +184 -0
- package/dist/ws/sidecar-handler.js +1603 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +15 -0
- package/dist/ws/sidecar-token-authenticator.js +24 -0
- package/package.json +34 -12
- package/src/agent-repo.test.ts +0 -310
- package/src/agent-repo.ts +0 -165
- package/src/agent-state-kind.test.ts +0 -247
- package/src/agent-state-kind.ts +0 -204
- package/src/asset-service.test.ts +0 -540
- package/src/asset-service.ts +0 -378
- package/src/available-skills-stanza.test.ts +0 -87
- package/src/available-skills-stanza.ts +0 -47
- package/src/credential-push.ts +0 -65
- package/src/event-collector-registry.test.ts +0 -73
- package/src/event-collector-registry.ts +0 -171
- package/src/event-collector.test.ts +0 -1387
- package/src/event-collector.ts +0 -424
- package/src/hub-session-lookups.ts +0 -206
- package/src/hub-session-orchestrator.test.ts +0 -510
- package/src/hub-session-orchestrator.ts +0 -213
- package/src/index.ts +0 -78
- package/src/repo-store/index.ts +0 -15
- package/src/repo-store/store.test.ts +0 -1169
- package/src/repo-store/store.ts +0 -428
- package/src/repo-store/types.ts +0 -253
- package/src/session-service.test.ts +0 -895
- package/src/session-service.ts +0 -464
- package/src/skill-kind.test.ts +0 -599
- package/src/skill-kind.ts +0 -350
- package/src/ws/index.ts +0 -18
- package/src/ws/sidecar-events.test.ts +0 -96
- package/src/ws/sidecar-events.ts +0 -231
- package/src/ws/sidecar-handler.test.ts +0 -2217
- package/src/ws/sidecar-handler.ts +0 -1574
- package/tsconfig.json +0 -4
- package/tsconfig.tsbuildinfo +0 -1
|
@@ -0,0 +1,1603 @@
|
|
|
1
|
+
// Hub-side websocket handler for sidecar connections.
|
|
2
|
+
//
|
|
3
|
+
// Accepts websocket upgrades, processes register frames, maintains a routing
|
|
4
|
+
// table of agentAddress → sidecar connection, and dispatches frames between
|
|
5
|
+
// sidecars and the hub's internal systems.
|
|
6
|
+
import { getLogger } from "@intx/log";
|
|
7
|
+
import { verifyEd25519 } from "@intx/crypto";
|
|
8
|
+
import { chunkPack, createPackReceiver } from "@intx/pack-transport";
|
|
9
|
+
import { hexDecode, hexEncode } from "@intx/types";
|
|
10
|
+
import { isWorkflowDerivedAddress } from "@intx/workflow-deploy";
|
|
11
|
+
import { type } from "arktype";
|
|
12
|
+
import { SidecarFrame, } from "@intx/types/sidecar";
|
|
13
|
+
import { createSidecarEmitter, } from "./sidecar-events.js";
|
|
14
|
+
const logger = getLogger(["hub", "ws", "sidecar"]);
|
|
15
|
+
/**
|
|
16
|
+
* Whether this connection owns `address` for routing/lifecycle purposes --
|
|
17
|
+
* as a challenged session address OR a hub-minted workflow-substrate address.
|
|
18
|
+
* The two sets are kept physically distinct (they differ on the challenge /
|
|
19
|
+
* re-add path), but ownership readers -- pack-transfer authorization,
|
|
20
|
+
* in-flight cancellation, disconnect teardown -- must see the union, or a
|
|
21
|
+
* reconnected workflow deployment (which lives only in `workflowAddresses`)
|
|
22
|
+
* is silently treated as unowned even though its mail routes.
|
|
23
|
+
*/
|
|
24
|
+
function connOwnsAddress(conn, address) {
|
|
25
|
+
return (conn.agentAddresses.has(address) || conn.workflowAddresses.has(address));
|
|
26
|
+
}
|
|
27
|
+
/** The deduped set of every address this connection owns (session + workflow). */
|
|
28
|
+
function ownedAddresses(conn) {
|
|
29
|
+
return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
|
|
30
|
+
}
|
|
31
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
32
|
+
const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
|
|
33
|
+
const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100;
|
|
34
|
+
const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
|
|
35
|
+
const DEFAULT_PING_TIMEOUT_MS = 60_000;
|
|
36
|
+
export function createSidecarRouter(config) {
|
|
37
|
+
const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, lookups = {}, } = config;
|
|
38
|
+
// Receiver-dispatch surface. Wire-layer callsites emit events here;
|
|
39
|
+
// host code subscribes via `router.events`.
|
|
40
|
+
const events = createSidecarEmitter();
|
|
41
|
+
// ws handle → registered connection
|
|
42
|
+
const connections = new Map();
|
|
43
|
+
// agentAddress → ws handle (routing table)
|
|
44
|
+
const addressIndex = new Map();
|
|
45
|
+
// requestId → pending promise
|
|
46
|
+
const pending = new Map();
|
|
47
|
+
const pendingDeploys = new Map();
|
|
48
|
+
const pendingChallenges = new Map();
|
|
49
|
+
const disconnectedAgents = new Map();
|
|
50
|
+
// agentAddress → set of subscriber callbacks for agent events
|
|
51
|
+
const agentSubscribers = new Map();
|
|
52
|
+
// agentAddress → cached connector-thread state, populated by
|
|
53
|
+
// connector.state.changed frames. Hub-side mail composition reads this
|
|
54
|
+
// to set threading headers on user-originated mail. Absent entries mean
|
|
55
|
+
// "no state reported yet" (e.g. mid-reconnect); callers must treat that
|
|
56
|
+
// identically to a null entry (no active thread).
|
|
57
|
+
const connectorStates = new Map();
|
|
58
|
+
// ws handle → liveness timer (reset on each ping from the sidecar)
|
|
59
|
+
const livenessTimers = new Map();
|
|
60
|
+
// Per-ws serialization chain for QUEUE-class frames (see `frameBypassesQueue`
|
|
61
|
+
// for the split and the invariant behind it). A frame that establishes or
|
|
62
|
+
// reads routing waits for earlier queued frames on the same ws to complete,
|
|
63
|
+
// so it observes their finished effects -- most importantly an async
|
|
64
|
+
// register's routing write, which would otherwise land after a following
|
|
65
|
+
// connector.state.changed / mail / pack frame and silently drop it. It holds
|
|
66
|
+
// a single in-flight promise per ws (replaced each queued frame), cleared on
|
|
67
|
+
// close.
|
|
68
|
+
const messageChains = new Map();
|
|
69
|
+
const pendingPacks = new Map();
|
|
70
|
+
let packCounter = 0;
|
|
71
|
+
const pendingUndeploys = new Map();
|
|
72
|
+
// Receives agent-state packs pushed from sidecars. The wire frames
|
|
73
|
+
// (`repo.pack.push` / `repo.pack.done`) are shared with the
|
|
74
|
+
// workflow-run flow; dispatch on `repoId.kind` picks which receiver
|
|
75
|
+
// observes the chunks. The two receivers maintain independent
|
|
76
|
+
// in-flight pack state and independent cancel-by-agent semantics so a
|
|
77
|
+
// pending workflow-run transfer cannot disturb a concurrent agent-
|
|
78
|
+
// state transfer for the same agent and vice versa.
|
|
79
|
+
const agentStatePackReceiver = createPackReceiver();
|
|
80
|
+
const workflowRunPackReceiver = createPackReceiver();
|
|
81
|
+
let requestCounter = 0;
|
|
82
|
+
// Surface disconnect-queue mail that is being dropped rather than delivered.
|
|
83
|
+
// Every dropped frame reaches the same channel routing failures already use
|
|
84
|
+
// (`mail.outbound.undelivered`, logged by the orchestrator), plus a warn that
|
|
85
|
+
// names the recipient and the drop count so a size-cap eviction or a TTL
|
|
86
|
+
// expiry of a still-full queue is visible instead of silent. A queued frame
|
|
87
|
+
// is always a `mail.inbound` carrying the sender's rawMessage; a frame of any
|
|
88
|
+
// other shape has no rawMessage to relay and is surfaced by the warn alone.
|
|
89
|
+
function surfaceDroppedFrames(agentAddress, frames, reason) {
|
|
90
|
+
if (frames.length === 0)
|
|
91
|
+
return;
|
|
92
|
+
logger.warn `Dropping ${String(frames.length)} queued message(s) for ${agentAddress}: ${reason}`;
|
|
93
|
+
for (const frame of frames) {
|
|
94
|
+
if (frame.type !== "mail.inbound")
|
|
95
|
+
continue;
|
|
96
|
+
events.emit("mail.outbound.undelivered", {
|
|
97
|
+
rawMessage: frame.rawMessage,
|
|
98
|
+
recipients: [agentAddress],
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
function enqueueForDisconnected(agentAddress, frame) {
|
|
103
|
+
const entry = disconnectedAgents.get(agentAddress);
|
|
104
|
+
if (entry === undefined)
|
|
105
|
+
return false;
|
|
106
|
+
if (entry.queue.length >= disconnectQueueMaxSize) {
|
|
107
|
+
const evicted = entry.queue.shift();
|
|
108
|
+
if (evicted !== undefined) {
|
|
109
|
+
surfaceDroppedFrames(agentAddress, [evicted], "disconnect queue full");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
entry.queue.push(frame);
|
|
113
|
+
return true;
|
|
114
|
+
}
|
|
115
|
+
function flushDisconnectedQueue(agentAddress, conn) {
|
|
116
|
+
const entry = disconnectedAgents.get(agentAddress);
|
|
117
|
+
if (entry === undefined)
|
|
118
|
+
return;
|
|
119
|
+
clearTimeout(entry.timer);
|
|
120
|
+
disconnectedAgents.delete(agentAddress);
|
|
121
|
+
for (const frame of entry.queue) {
|
|
122
|
+
conn.send(frame);
|
|
123
|
+
}
|
|
124
|
+
if (entry.queue.length > 0) {
|
|
125
|
+
logger.info `Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
function resetLivenessTimer(ws) {
|
|
129
|
+
const existing = livenessTimers.get(ws);
|
|
130
|
+
if (existing !== undefined)
|
|
131
|
+
clearTimeout(existing);
|
|
132
|
+
const timer = setTimeout(() => {
|
|
133
|
+
livenessTimers.delete(ws);
|
|
134
|
+
logger.warn `Sidecar ping timeout, closing connection`;
|
|
135
|
+
ws.close();
|
|
136
|
+
}, pingTimeoutMs);
|
|
137
|
+
livenessTimers.set(ws, timer);
|
|
138
|
+
}
|
|
139
|
+
function handlePing(ws) {
|
|
140
|
+
resetLivenessTimer(ws);
|
|
141
|
+
// Always respond with pong, even before register/reconnect completes.
|
|
142
|
+
// The sidecar's ping timer starts on open, which may fire before the
|
|
143
|
+
// async registration handshake finishes.
|
|
144
|
+
ws.send(JSON.stringify({ type: "pong" }));
|
|
145
|
+
}
|
|
146
|
+
function handleOpen(ws) {
|
|
147
|
+
// Connection is not usable until a register frame arrives.
|
|
148
|
+
// Start the liveness timer immediately — a sidecar that connects
|
|
149
|
+
// but never sends a ping will be reaped.
|
|
150
|
+
resetLivenessTimer(ws);
|
|
151
|
+
}
|
|
152
|
+
function handleMessage(ws, data) {
|
|
153
|
+
let raw;
|
|
154
|
+
try {
|
|
155
|
+
raw = JSON.parse(data);
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
logger.warn `Unparseable frame from sidecar connection`;
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
const validated = SidecarFrame(raw);
|
|
162
|
+
if (validated instanceof type.errors) {
|
|
163
|
+
logger.warn `Invalid sidecar frame: ${validated.summary}`;
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
const frame = validated;
|
|
167
|
+
// Bypass frames (liveness + terminal responses to outbound requests)
|
|
168
|
+
// dispatch immediately: they resolve the very promises a queued handler
|
|
169
|
+
// may be blocked on, so queuing them would deadlock the round-trip.
|
|
170
|
+
if (frameBypassesQueue(frame)) {
|
|
171
|
+
// Guard so a bypass handler's failure -- a synchronous throw or an async
|
|
172
|
+
// ack handler's rejection -- is logged rather than floating out of the
|
|
173
|
+
// immediate dispatch. The async wrapper turns a synchronous throw into a
|
|
174
|
+
// rejection too, matching the queue path's .then/.catch coverage.
|
|
175
|
+
void (async () => dispatchFrame(ws, frame))().catch((err) => {
|
|
176
|
+
logger.warn `Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`;
|
|
177
|
+
});
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
// Everything else serializes per ws so a frame that establishes or reads
|
|
181
|
+
// routing observes earlier queued frames' completed effects.
|
|
182
|
+
const prev = messageChains.get(ws) ?? Promise.resolve();
|
|
183
|
+
const next = prev
|
|
184
|
+
.then(() => dispatchFrame(ws, frame))
|
|
185
|
+
.catch((err) => {
|
|
186
|
+
logger.warn `Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`;
|
|
187
|
+
});
|
|
188
|
+
messageChains.set(ws, next);
|
|
189
|
+
}
|
|
190
|
+
function assertNever(x) {
|
|
191
|
+
throw new Error(`Unclassified sidecar frame type: ${JSON.stringify(x)}`);
|
|
192
|
+
}
|
|
193
|
+
// Whether `frame` bypasses the per-ws serialization chain. Invariant: a frame
|
|
194
|
+
// bypasses IFF it is liveness (ping) OR a terminal response to an
|
|
195
|
+
// already-issued outbound request -- correlated purely by
|
|
196
|
+
// requestId/transferId/agentAddress in the pending maps, touching no routing
|
|
197
|
+
// state. Such a frame has no ordering obligation against new inbound frames
|
|
198
|
+
// (a response cannot resolve "too early" for a request that already went
|
|
199
|
+
// out), and it is exactly what in-flight queued handlers block on, so it MUST
|
|
200
|
+
// run out of band or the challenge round-trip deadlocks (challenge.response
|
|
201
|
+
// -> agent.reconnected reaction -> sendSourcesUpdate awaits a later
|
|
202
|
+
// session.ack). Every other frame establishes or reads routing, or carries an
|
|
203
|
+
// inbound payload whose order matters, so it queues. The exhaustive switch +
|
|
204
|
+
// assertNever makes adding a SidecarFrame variant without classifying it a
|
|
205
|
+
// compile error, not a latent deadlock or a silent bypass hole.
|
|
206
|
+
function frameBypassesQueue(frame) {
|
|
207
|
+
switch (frame.type) {
|
|
208
|
+
case "ping":
|
|
209
|
+
case "session.ack":
|
|
210
|
+
case "session.error":
|
|
211
|
+
case "agent.deploy.ack":
|
|
212
|
+
case "agent.error":
|
|
213
|
+
case "agent.undeploy.ack":
|
|
214
|
+
case "repo.pack.ack":
|
|
215
|
+
case "repo.pack.reject":
|
|
216
|
+
return true;
|
|
217
|
+
case "register":
|
|
218
|
+
case "reconnect":
|
|
219
|
+
case "challenge.response":
|
|
220
|
+
case "mail.outbound":
|
|
221
|
+
case "agent.event":
|
|
222
|
+
case "connector.state.changed":
|
|
223
|
+
case "repo.pack.push":
|
|
224
|
+
case "repo.pack.done":
|
|
225
|
+
return false;
|
|
226
|
+
default:
|
|
227
|
+
return assertNever(frame);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
// Runs one frame's handler. Returns the handler's promise for async handlers
|
|
231
|
+
// so the per-ws chain can await bounded completion; sync handlers return
|
|
232
|
+
// void. Never awaits a promise that resolves on a LATER same-ws frame -- the
|
|
233
|
+
// only such await (the challenge round-trip's session.ack) is reached via a
|
|
234
|
+
// bypass frame, which does not queue.
|
|
235
|
+
function dispatchFrame(ws, frame) {
|
|
236
|
+
switch (frame.type) {
|
|
237
|
+
case "register": {
|
|
238
|
+
const agentAddresses = frame.agentAddresses;
|
|
239
|
+
return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses));
|
|
240
|
+
}
|
|
241
|
+
case "reconnect": {
|
|
242
|
+
const agentAddresses = frame.agentAddresses;
|
|
243
|
+
const deployRefs = frame.deployRefs ?? {};
|
|
244
|
+
return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, deployRefs));
|
|
245
|
+
}
|
|
246
|
+
case "challenge.response":
|
|
247
|
+
return handleChallengeResponse(ws, frame.responses);
|
|
248
|
+
case "agent.deploy.ack":
|
|
249
|
+
return handleDeployAck(frame.agentAddress, frame.publicKey);
|
|
250
|
+
case "agent.error":
|
|
251
|
+
rejectDeployPending(frame.agentAddress, frame.error);
|
|
252
|
+
rejectUndeployPending(frame.agentAddress, frame.error);
|
|
253
|
+
return;
|
|
254
|
+
case "agent.undeploy.ack":
|
|
255
|
+
resolveUndeployPending(frame.agentAddress);
|
|
256
|
+
return;
|
|
257
|
+
case "ping":
|
|
258
|
+
handlePing(ws);
|
|
259
|
+
return;
|
|
260
|
+
case "mail.outbound":
|
|
261
|
+
if (frame.delivered !== true) {
|
|
262
|
+
handleMailOutbound(frame.rawMessage, frame.recipients);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
if (lookups.persistMail && frame.senderAddress) {
|
|
266
|
+
return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
|
|
267
|
+
}
|
|
268
|
+
if (!frame.senderAddress) {
|
|
269
|
+
logger.warn `Dropping delivered mail.outbound frame with no senderAddress`;
|
|
270
|
+
}
|
|
271
|
+
else {
|
|
272
|
+
logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
|
|
273
|
+
}
|
|
274
|
+
return;
|
|
275
|
+
case "agent.event":
|
|
276
|
+
events.emit("agent.event", {
|
|
277
|
+
agentAddress: frame.agentAddress,
|
|
278
|
+
sessionId: frame.sessionId,
|
|
279
|
+
event: frame.event,
|
|
280
|
+
});
|
|
281
|
+
dispatchToSubscribers(frame.agentAddress, frame.event);
|
|
282
|
+
return;
|
|
283
|
+
case "connector.state.changed":
|
|
284
|
+
// Gate the cache write on the sending sidecar actually owning
|
|
285
|
+
// the named agent. A misbehaving sidecar that knows another
|
|
286
|
+
// agent's address could otherwise poison the cached state.
|
|
287
|
+
if (addressIndex.get(frame.agentAddress) !== ws) {
|
|
288
|
+
logger.warn `Dropping connector.state.changed for ${frame.agentAddress}: not registered to this sidecar`;
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
connectorStates.set(frame.agentAddress, frame.connectorState);
|
|
292
|
+
events.emit("connector.state.changed", {
|
|
293
|
+
agentAddress: frame.agentAddress,
|
|
294
|
+
connectorState: frame.connectorState,
|
|
295
|
+
});
|
|
296
|
+
return;
|
|
297
|
+
case "session.ack":
|
|
298
|
+
resolvePending(frame.requestId);
|
|
299
|
+
return;
|
|
300
|
+
case "session.error":
|
|
301
|
+
rejectPending(frame.requestId, frame.error);
|
|
302
|
+
return;
|
|
303
|
+
case "repo.pack.ack":
|
|
304
|
+
resolvePackPending(frame.transferId);
|
|
305
|
+
return;
|
|
306
|
+
case "repo.pack.reject":
|
|
307
|
+
rejectPackPending(frame.transferId, frame.reason);
|
|
308
|
+
return;
|
|
309
|
+
case "repo.pack.push":
|
|
310
|
+
handlePackPush(ws, frame);
|
|
311
|
+
return;
|
|
312
|
+
case "repo.pack.done":
|
|
313
|
+
return handlePackDone(ws, frame);
|
|
314
|
+
default:
|
|
315
|
+
return assertNever(frame);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
// Authenticate a register/reconnect handshake exactly once, then run the
|
|
319
|
+
// frame's handler with the verified identity. The claimed `sidecarId` on
|
|
320
|
+
// the frame is an unauthenticated hint: it is logged if it disagrees with
|
|
321
|
+
// the verified id but never trusted -- routing keys off the verified id.
|
|
322
|
+
// Fails closed by closing the connection when the authenticator rejects
|
|
323
|
+
// (returns null) or throws (e.g. a database failure), so a handshake never
|
|
324
|
+
// proceeds on unverified credentials.
|
|
325
|
+
async function authenticateHandshake(ws, frame, run) {
|
|
326
|
+
let identity;
|
|
327
|
+
try {
|
|
328
|
+
identity = await authenticateSidecar({
|
|
329
|
+
sidecarId: frame.sidecarId,
|
|
330
|
+
token: frame.token,
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
catch (err) {
|
|
334
|
+
logger.error `Rejected ${frame.type} from claimed sidecar ${frame.sidecarId}: authenticator failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
335
|
+
ws.close();
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
if (identity === null) {
|
|
339
|
+
logger.warn `Rejected ${frame.type} from claimed sidecar ${frame.sidecarId}: invalid token`;
|
|
340
|
+
ws.close();
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
if (identity.sidecarId !== frame.sidecarId) {
|
|
344
|
+
logger.warn `Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${identity.sidecarId}; keying off the verified id`;
|
|
345
|
+
}
|
|
346
|
+
await run(identity);
|
|
347
|
+
}
|
|
348
|
+
async function handleRegister(ws, identity, agentAddresses) {
|
|
349
|
+
const sidecarId = identity.sidecarId;
|
|
350
|
+
// Key-existence gate. A register frame is token-authenticated but carries
|
|
351
|
+
// no per-address ownership proof, so it may route an address ONLY if that
|
|
352
|
+
// address has no stored key yet -- a genuine keyless first-deploy (the
|
|
353
|
+
// token-bounded first-deploy trust model). An address that already has a
|
|
354
|
+
// key must prove ownership through the challenged reconnect path; routing
|
|
355
|
+
// it here on token auth alone is the register-frame sibling of the
|
|
356
|
+
// reconnect hijack. The keyless-only set is computed up front, BEFORE the
|
|
357
|
+
// ghost-cleanup and every routing mutation below, so a rejected address
|
|
358
|
+
// touches nothing: no eviction of a live owner, hence no downgrade from
|
|
359
|
+
// hijack to denial-of-service on the victim.
|
|
360
|
+
const lookupKey = lookups.lookupPublicKey;
|
|
361
|
+
const routableAddresses = [];
|
|
362
|
+
for (const addr of agentAddresses) {
|
|
363
|
+
if (lookupKey === undefined) {
|
|
364
|
+
// Fail closed: without the ownership lookup a keyed address cannot be
|
|
365
|
+
// told apart from a first-deploy, so route nothing and surface the
|
|
366
|
+
// misconfiguration. Empty first-connect registers never reach here.
|
|
367
|
+
logger.error `Cannot gate register routing for ${addr}: lookupPublicKey is not configured; refusing to route (challenged reconnect required)`;
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
let existingKey;
|
|
371
|
+
try {
|
|
372
|
+
existingKey = await lookupKey(addr);
|
|
373
|
+
}
|
|
374
|
+
catch (err) {
|
|
375
|
+
// Fail closed on a lookup error (e.g. a transient DB failure): route
|
|
376
|
+
// nothing for this address and surface the failure, rather than let
|
|
377
|
+
// the rejection float out of this void-dispatched handler and take
|
|
378
|
+
// down the hub.
|
|
379
|
+
logger.error `Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`;
|
|
380
|
+
continue;
|
|
381
|
+
}
|
|
382
|
+
if (existingKey !== null) {
|
|
383
|
+
logger.warn `Refusing to route ${addr} via register: address already has a stored key; ownership must be proven via challenged reconnect`;
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
routableAddresses.push(addr);
|
|
387
|
+
}
|
|
388
|
+
// Additive re-register: inherit every address this ws already owns and ADD
|
|
389
|
+
// the frame's keyless first-deploys. Register never drops an owned route --
|
|
390
|
+
// an address proved via challenged reconnect stays routed, and removal
|
|
391
|
+
// happens via undeploy/disconnect, not register-omission.
|
|
392
|
+
//
|
|
393
|
+
// (Under the retired full-set model a register frame carried the sidecar's
|
|
394
|
+
// complete live set, so an omitted address meant "removed". Now the frame
|
|
395
|
+
// carries only keyless first-deploys the sidecar is adding -- keyed
|
|
396
|
+
// addresses arrive via reconnect -- so omission is meaningless, and a
|
|
397
|
+
// replace-on-register would wrongly drop an earlier first-deploy when a
|
|
398
|
+
// later one is registered, as well as any reconnect-verified keyed route.)
|
|
399
|
+
const existing = connections.get(ws);
|
|
400
|
+
const inheritedAgent = new Set(existing?.agentAddresses);
|
|
401
|
+
const inheritedWorkflow = new Set(existing?.workflowAddresses);
|
|
402
|
+
// Clean up ghost entries from OTHER connections for the newly-claimed
|
|
403
|
+
// addresses only. An inherited address is already owned by this ws
|
|
404
|
+
// (prevWs === ws), so it needs no eviction and its in-flight deploy must
|
|
405
|
+
// not be cancelled.
|
|
406
|
+
for (const addr of routableAddresses) {
|
|
407
|
+
const prevWs = addressIndex.get(addr);
|
|
408
|
+
if (prevWs !== undefined && prevWs !== ws) {
|
|
409
|
+
const prevConn = connections.get(prevWs);
|
|
410
|
+
if (prevConn !== undefined) {
|
|
411
|
+
prevConn.agentAddresses.delete(addr);
|
|
412
|
+
}
|
|
413
|
+
// The new owner is about to take over; the prior owner's
|
|
414
|
+
// cached state must not survive into the new owner's window
|
|
415
|
+
// before its bootstrap frame arrives.
|
|
416
|
+
connectorStates.delete(addr);
|
|
417
|
+
}
|
|
418
|
+
// Cancel any in-flight deploy for this address since the
|
|
419
|
+
// reconnecting sidecar is taking ownership.
|
|
420
|
+
const staleDeployReq = pendingDeploys.get(addr);
|
|
421
|
+
if (staleDeployReq !== undefined) {
|
|
422
|
+
clearTimeout(staleDeployReq.timer);
|
|
423
|
+
pendingDeploys.delete(addr);
|
|
424
|
+
staleDeployReq.reject(`Sidecar ${sidecarId} reconnected and claimed address "${addr}"`);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
// New keyless first-deploys join the inherited session set; the workflow
|
|
428
|
+
// set is inherited unchanged (register never adds to it -- a
|
|
429
|
+
// workflow-derived address routes only through the challenged reconnect).
|
|
430
|
+
for (const addr of routableAddresses) {
|
|
431
|
+
inheritedAgent.add(addr);
|
|
432
|
+
}
|
|
433
|
+
const addrSet = inheritedAgent;
|
|
434
|
+
const workflowSet = inheritedWorkflow;
|
|
435
|
+
const conn = {
|
|
436
|
+
sidecarId,
|
|
437
|
+
agentAddresses: addrSet,
|
|
438
|
+
workflowAddresses: workflowSet,
|
|
439
|
+
send(frame) {
|
|
440
|
+
ws.send(JSON.stringify(frame));
|
|
441
|
+
},
|
|
442
|
+
};
|
|
443
|
+
connections.set(ws, conn);
|
|
444
|
+
// Only the newly-claimed addresses need a routing write + queue discard;
|
|
445
|
+
// inherited addresses already point at this ws in addressIndex.
|
|
446
|
+
for (const addr of routableAddresses) {
|
|
447
|
+
addressIndex.set(addr, ws);
|
|
448
|
+
// Discard any disconnect queue — register has no identity
|
|
449
|
+
// verification, so flushing to an unverified connection is unsafe.
|
|
450
|
+
// Use reconnect with challenge/response to preserve queued messages.
|
|
451
|
+
const staleQueue = disconnectedAgents.get(addr);
|
|
452
|
+
if (staleQueue !== undefined) {
|
|
453
|
+
clearTimeout(staleQueue.timer);
|
|
454
|
+
if (staleQueue.queue.length > 0) {
|
|
455
|
+
logger.warn `Discarding ${String(staleQueue.queue.length)} queued message(s) for ${addr} on unverified register`;
|
|
456
|
+
}
|
|
457
|
+
disconnectedAgents.delete(addr);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
// Only keyless first-deploy addresses reach `addressIndex` here; an
|
|
461
|
+
// address that already has a stored key was filtered out by the gate
|
|
462
|
+
// above and must re-enter routing through the challenged reconnect path.
|
|
463
|
+
// Because the gate runs before the ghost-cleanup, a rejected (keyed)
|
|
464
|
+
// address never evicts its prior owner -- register cannot reclaim or
|
|
465
|
+
// disrupt a victim's route on token auth alone.
|
|
466
|
+
logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
|
|
467
|
+
}
|
|
468
|
+
async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
|
|
469
|
+
const sidecarId = identity.sidecarId;
|
|
470
|
+
const lookupKey = lookups.lookupPublicKey;
|
|
471
|
+
if (lookupKey === undefined) {
|
|
472
|
+
logger.error `Received reconnect frame but no lookupPublicKey is configured`;
|
|
473
|
+
ws.close();
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
// Cancel any existing pending challenge for this ws before doing
|
|
477
|
+
// async work, so concurrent reconnect frames don't race.
|
|
478
|
+
const existingChallenge = pendingChallenges.get(ws);
|
|
479
|
+
if (existingChallenge !== undefined) {
|
|
480
|
+
clearTimeout(existingChallenge.timer);
|
|
481
|
+
pendingChallenges.delete(ws);
|
|
482
|
+
}
|
|
483
|
+
// Capture the addresses this live connection already owns. The
|
|
484
|
+
// internal register below clears them, but ones the reconnect is not
|
|
485
|
+
// re-challenging are still valid — an agent provisioned during the
|
|
486
|
+
// sidecar's restore window is routed into addressIndex by its deploy
|
|
487
|
+
// and is not part of the disk-restored set this reconnect carries.
|
|
488
|
+
// Without preserving them, the reconnect silently drops a
|
|
489
|
+
// freshly-deployed agent from routing.
|
|
490
|
+
const previouslyOwned = new Set(connections.get(ws)?.agentAddresses);
|
|
491
|
+
// Register the sidecar connection immediately (with no addresses) so it
|
|
492
|
+
// can receive frames while the ownership challenge is pending. Every
|
|
493
|
+
// reconnect address -- session and workflow-derived alike -- enters
|
|
494
|
+
// routing only through the verified path below, never unchallenged here.
|
|
495
|
+
// Empty address list, so the key-existence gate has nothing to await.
|
|
496
|
+
// The identity was already verified at the dispatch boundary, so the
|
|
497
|
+
// internal register does not re-authenticate.
|
|
498
|
+
await handleRegister(ws, identity, []);
|
|
499
|
+
const conn = connections.get(ws);
|
|
500
|
+
if (conn === undefined)
|
|
501
|
+
return;
|
|
502
|
+
// Re-add the still-owned addresses the register cleared but this
|
|
503
|
+
// reconnect is not re-challenging. The challenged addresses below
|
|
504
|
+
// re-enter addressIndex through the verified path instead.
|
|
505
|
+
const claimedAddresses = new Set(agentAddresses);
|
|
506
|
+
for (const addr of previouslyOwned) {
|
|
507
|
+
if (claimedAddresses.has(addr))
|
|
508
|
+
continue;
|
|
509
|
+
conn.agentAddresses.add(addr);
|
|
510
|
+
addressIndex.set(addr, ws);
|
|
511
|
+
}
|
|
512
|
+
// Look up stored public keys for all claimed addresses. Fail closed on a
|
|
513
|
+
// lookup error (e.g. a transient DB failure): treat the address as
|
|
514
|
+
// unverifiable so it fails its challenge and stays unrouted, rather than
|
|
515
|
+
// letting the rejection float out of this void-dispatched handler as an
|
|
516
|
+
// unhandled rejection that could take down the hub.
|
|
517
|
+
const keyLookups = await Promise.all(agentAddresses.map(async (addr) => {
|
|
518
|
+
try {
|
|
519
|
+
return { address: addr, publicKeyHex: await lookupKey(addr) };
|
|
520
|
+
}
|
|
521
|
+
catch (err) {
|
|
522
|
+
logger.error `Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`;
|
|
523
|
+
return { address: addr, publicKeyHex: null };
|
|
524
|
+
}
|
|
525
|
+
}));
|
|
526
|
+
// If the connection was closed or superseded while we were awaiting
|
|
527
|
+
// key lookups, bail out.
|
|
528
|
+
if (!connections.has(ws))
|
|
529
|
+
return;
|
|
530
|
+
const challenges = new Map();
|
|
531
|
+
const challengeEntries = [];
|
|
532
|
+
for (const { address, publicKeyHex } of keyLookups) {
|
|
533
|
+
if (publicKeyHex === null) {
|
|
534
|
+
conn.send({
|
|
535
|
+
type: "challenge.failed",
|
|
536
|
+
address,
|
|
537
|
+
reason: "Unknown agent address",
|
|
538
|
+
});
|
|
539
|
+
continue;
|
|
540
|
+
}
|
|
541
|
+
let publicKey;
|
|
542
|
+
try {
|
|
543
|
+
publicKey = hexDecode(publicKeyHex);
|
|
544
|
+
}
|
|
545
|
+
catch {
|
|
546
|
+
conn.send({
|
|
547
|
+
type: "challenge.failed",
|
|
548
|
+
address,
|
|
549
|
+
reason: "Stored public key is corrupt",
|
|
550
|
+
});
|
|
551
|
+
logger.error `Corrupt stored public key for ${address}`;
|
|
552
|
+
continue;
|
|
553
|
+
}
|
|
554
|
+
const nonce = crypto.getRandomValues(new Uint8Array(32));
|
|
555
|
+
challenges.set(address, { nonce, publicKey });
|
|
556
|
+
challengeEntries.push({ address, nonce: hexEncode(nonce) });
|
|
557
|
+
}
|
|
558
|
+
if (challenges.size === 0)
|
|
559
|
+
return;
|
|
560
|
+
// If another reconnect completed while we were building the
|
|
561
|
+
// challenge, it will have written its own entry. Don't overwrite.
|
|
562
|
+
if (pendingChallenges.has(ws))
|
|
563
|
+
return;
|
|
564
|
+
const timer = setTimeout(() => {
|
|
565
|
+
pendingChallenges.delete(ws);
|
|
566
|
+
logger.warn `Challenge timed out for sidecar ${sidecarId}`;
|
|
567
|
+
}, challengeTimeoutMs);
|
|
568
|
+
pendingChallenges.set(ws, {
|
|
569
|
+
sidecarId,
|
|
570
|
+
challenges,
|
|
571
|
+
deployRefs,
|
|
572
|
+
timer,
|
|
573
|
+
});
|
|
574
|
+
conn.send({ type: "challenge", challenges: challengeEntries });
|
|
575
|
+
}
|
|
576
|
+
async function handleChallengeResponse(ws, responses) {
|
|
577
|
+
const challenge = pendingChallenges.get(ws);
|
|
578
|
+
if (challenge === undefined) {
|
|
579
|
+
logger.warn `Received challenge.response with no pending challenge`;
|
|
580
|
+
return;
|
|
581
|
+
}
|
|
582
|
+
clearTimeout(challenge.timer);
|
|
583
|
+
pendingChallenges.delete(ws);
|
|
584
|
+
const conn = connections.get(ws);
|
|
585
|
+
if (conn === undefined)
|
|
586
|
+
return;
|
|
587
|
+
const verified = [];
|
|
588
|
+
const responded = new Set();
|
|
589
|
+
for (const { address, signature } of responses) {
|
|
590
|
+
responded.add(address);
|
|
591
|
+
const entry = challenge.challenges.get(address);
|
|
592
|
+
if (entry === undefined) {
|
|
593
|
+
conn.send({
|
|
594
|
+
type: "challenge.failed",
|
|
595
|
+
address,
|
|
596
|
+
reason: "Address was not challenged",
|
|
597
|
+
});
|
|
598
|
+
continue;
|
|
599
|
+
}
|
|
600
|
+
let valid = false;
|
|
601
|
+
try {
|
|
602
|
+
const nonceBytes = entry.nonce;
|
|
603
|
+
const addressBytes = new TextEncoder().encode(address);
|
|
604
|
+
const payload = new Uint8Array(nonceBytes.length + addressBytes.length);
|
|
605
|
+
payload.set(nonceBytes);
|
|
606
|
+
payload.set(addressBytes, nonceBytes.length);
|
|
607
|
+
const sigBytes = hexDecode(signature);
|
|
608
|
+
valid = await verifyEd25519(payload, sigBytes, entry.publicKey);
|
|
609
|
+
}
|
|
610
|
+
catch (err) {
|
|
611
|
+
logger.warn `Challenge failed for ${address}: ${err instanceof Error ? err.message : String(err)}`;
|
|
612
|
+
}
|
|
613
|
+
if (valid) {
|
|
614
|
+
verified.push(address);
|
|
615
|
+
}
|
|
616
|
+
else {
|
|
617
|
+
conn.send({
|
|
618
|
+
type: "challenge.failed",
|
|
619
|
+
address,
|
|
620
|
+
reason: "Signature verification failed",
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
// Notify about challenged addresses that were omitted from the response.
|
|
625
|
+
for (const address of challenge.challenges.keys()) {
|
|
626
|
+
if (!responded.has(address)) {
|
|
627
|
+
conn.send({
|
|
628
|
+
type: "challenge.failed",
|
|
629
|
+
address,
|
|
630
|
+
reason: "No response provided for challenged address",
|
|
631
|
+
});
|
|
632
|
+
logger.warn `Challenge failed for ${address}: no response provided`;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
// Add verified addresses to the routing table immediately so that
|
|
636
|
+
// `agent.reconnected` subscribers can use sendRequest-based methods
|
|
637
|
+
// (e.g. sendSourcesUpdate). Addresses that fail governance are
|
|
638
|
+
// rolled back from the routing table afterward.
|
|
639
|
+
for (const addr of verified) {
|
|
640
|
+
// If a different ws still owns this address (live takeover via
|
|
641
|
+
// verified reconnect), evict its cached connector state before
|
|
642
|
+
// routing flips. The new owner's harness will bootstrap via
|
|
643
|
+
// restore-fires-callback.
|
|
644
|
+
const prevWs = addressIndex.get(addr);
|
|
645
|
+
if (prevWs !== undefined && prevWs !== ws) {
|
|
646
|
+
connectorStates.delete(addr);
|
|
647
|
+
// Evict the reclaimed address from the superseded connection's owned
|
|
648
|
+
// set. handleClose's cancelByAgent sweep iterates a connection's owned
|
|
649
|
+
// union WITHOUT an ownership guard, so if the stale connection still
|
|
650
|
+
// listed this address it would cancel THIS connection's in-flight pack
|
|
651
|
+
// transfer for it when it finally closes. Delete from both sets: a
|
|
652
|
+
// workflow-derived address lives on the workflow set, a launched agent
|
|
653
|
+
// on the session set, and delete is a no-op for the absent one.
|
|
654
|
+
const prevConn = connections.get(prevWs);
|
|
655
|
+
if (prevConn !== undefined) {
|
|
656
|
+
prevConn.workflowAddresses.delete(addr);
|
|
657
|
+
prevConn.agentAddresses.delete(addr);
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
// Track the address on the set that matches its lifecycle so
|
|
661
|
+
// handleClose reclaims it correctly: a workflow-derived deployment
|
|
662
|
+
// address goes on the workflow set (no disconnect queue -- its
|
|
663
|
+
// in-flight state is reconstructed sidecar-locally on the next
|
|
664
|
+
// reconnect), a launched agent on the session set (queued for
|
|
665
|
+
// reconnect). The routing pointer is the same either way; only now
|
|
666
|
+
// it is written behind a passed challenge.
|
|
667
|
+
if (isWorkflowDerivedAddress(addr)) {
|
|
668
|
+
conn.workflowAddresses.add(addr);
|
|
669
|
+
}
|
|
670
|
+
else {
|
|
671
|
+
conn.agentAddresses.add(addr);
|
|
672
|
+
}
|
|
673
|
+
addressIndex.set(addr, ws);
|
|
674
|
+
}
|
|
675
|
+
const ready = [];
|
|
676
|
+
const failed = [];
|
|
677
|
+
for (const addr of verified) {
|
|
678
|
+
// The `agent.reconnected` reaction is session lifecycle -- instance
|
|
679
|
+
// status flip, event-collector restore -- owned by the agent_instance
|
|
680
|
+
// concept. A workflow-derived deployment address has no agent_instance
|
|
681
|
+
// row, so the reaction's `requireInstance` would throw and roll the
|
|
682
|
+
// just-verified address back out of routing. It needs routing + queue
|
|
683
|
+
// flush only, which the passed challenge has now made safe; skip the
|
|
684
|
+
// session reaction for it.
|
|
685
|
+
if (isWorkflowDerivedAddress(addr)) {
|
|
686
|
+
ready.push(addr);
|
|
687
|
+
continue;
|
|
688
|
+
}
|
|
689
|
+
if (events.listenerCount("agent.reconnected") === 0) {
|
|
690
|
+
ready.push(addr);
|
|
691
|
+
continue;
|
|
692
|
+
}
|
|
693
|
+
try {
|
|
694
|
+
await events.emitAndAwait("agent.reconnected", { agentAddress: addr });
|
|
695
|
+
ready.push(addr);
|
|
696
|
+
}
|
|
697
|
+
catch (err) {
|
|
698
|
+
logger.error `Failed to handle reconnection for ${addr}: ${err instanceof Error ? err.message : String(err)}`;
|
|
699
|
+
failed.push(addr);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
// If a second reconnect arrived during the callback loop, our conn
|
|
703
|
+
// is orphaned — handleRegister already rebuilt the connection and
|
|
704
|
+
// cleared addressIndex. Bail out; the new reconnect flow will
|
|
705
|
+
// re-verify these addresses from scratch.
|
|
706
|
+
if (connections.get(ws) !== conn) {
|
|
707
|
+
logger.warn `Challenge response processing aborted: connection superseded by new reconnect`;
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
710
|
+
// Roll back failed addresses from the routing table. Only the session set
|
|
711
|
+
// is touched: a workflow-derived address can never be in `failed` -- it
|
|
712
|
+
// early-`continue`s to `ready` above, before the reaction that populates
|
|
713
|
+
// `failed` -- so it is never on the workflow set at this point.
|
|
714
|
+
for (const addr of failed) {
|
|
715
|
+
conn.agentAddresses.delete(addr);
|
|
716
|
+
addressIndex.delete(addr);
|
|
717
|
+
}
|
|
718
|
+
// Flush queued messages only for ready addresses.
|
|
719
|
+
for (const addr of ready) {
|
|
720
|
+
flushDisconnectedQueue(addr, conn);
|
|
721
|
+
}
|
|
722
|
+
// Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
|
|
723
|
+
// so reconnect completion is not blocked on pack transfer. The
|
|
724
|
+
// wire layer owns the staleness comparison; the event fires only
|
|
725
|
+
// when staleness is confirmed.
|
|
726
|
+
const checkDeployRef = lookups.lookupDeployRef;
|
|
727
|
+
if (checkDeployRef !== undefined) {
|
|
728
|
+
for (const addr of ready) {
|
|
729
|
+
// Workflow deployments are pinned-forever: a deployment keeps its
|
|
730
|
+
// deploy-time definition until an explicit undeploy/redeploy, so the
|
|
731
|
+
// deploy-ref freshness catch-up is deliberately NOT run for a
|
|
732
|
+
// workflow-derived address. A definition edited on the hub while the
|
|
733
|
+
// sidecar was disconnected does not reconcile on reconnect; it affects
|
|
734
|
+
// only newly created deployments. The deployment's in-flight run state
|
|
735
|
+
// is reconstructed sidecar-locally at restore, not re-fetched here. Do
|
|
736
|
+
// NOT add a reconcile path for these addresses -- see the "Workflow
|
|
737
|
+
// Definition Versioning: Pinned-Forever" note under "Reconnect
|
|
738
|
+
// Sequencing" in docs/IMPLEMENTATION.md.
|
|
739
|
+
if (isWorkflowDerivedAddress(addr))
|
|
740
|
+
continue;
|
|
741
|
+
void (async () => {
|
|
742
|
+
try {
|
|
743
|
+
const hubRef = await checkDeployRef(addr);
|
|
744
|
+
if (hubRef === null)
|
|
745
|
+
return;
|
|
746
|
+
const sidecarRef = challenge.deployRefs[addr];
|
|
747
|
+
if (sidecarRef === hubRef)
|
|
748
|
+
return;
|
|
749
|
+
logger.info `Re-deploying ${addr}: sidecar ref ${sidecarRef ?? "(none)"} != hub ref ${hubRef.slice(0, 8)}`;
|
|
750
|
+
await events.emitAndAwait("deploy.ref.stale", {
|
|
751
|
+
agentAddress: addr,
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
catch (err) {
|
|
755
|
+
logger.error `Failed to re-deploy ${addr} after reconnect: ${err instanceof Error ? err.message : String(err)}`;
|
|
756
|
+
}
|
|
757
|
+
})();
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
// Failed addresses: reset their queue TTL so messages survive until
|
|
761
|
+
// the next reconnect attempt, and notify the sidecar.
|
|
762
|
+
for (const addr of failed) {
|
|
763
|
+
const entry = disconnectedAgents.get(addr);
|
|
764
|
+
if (entry !== undefined) {
|
|
765
|
+
clearTimeout(entry.timer);
|
|
766
|
+
entry.timer = setTimeout(() => {
|
|
767
|
+
const expired = disconnectedAgents.get(addr);
|
|
768
|
+
disconnectedAgents.delete(addr);
|
|
769
|
+
if (expired !== undefined) {
|
|
770
|
+
surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
|
|
771
|
+
}
|
|
772
|
+
}, disconnectQueueTTLMs);
|
|
773
|
+
}
|
|
774
|
+
conn.send({
|
|
775
|
+
type: "challenge.failed",
|
|
776
|
+
address: addr,
|
|
777
|
+
reason: "Reconnection rejected by governance",
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
|
|
781
|
+
}
|
|
782
|
+
function handleMailOutbound(rawMessage, recipients) {
|
|
783
|
+
// Route to locally connected sidecars first, then try disconnect queues.
|
|
784
|
+
const unrouted = [];
|
|
785
|
+
for (const recipient of recipients) {
|
|
786
|
+
const targetWs = addressIndex.get(recipient);
|
|
787
|
+
if (targetWs !== undefined) {
|
|
788
|
+
const conn = connections.get(targetWs);
|
|
789
|
+
if (conn !== undefined) {
|
|
790
|
+
conn.send({
|
|
791
|
+
type: "mail.inbound",
|
|
792
|
+
agentAddress: recipient,
|
|
793
|
+
rawMessage,
|
|
794
|
+
});
|
|
795
|
+
continue;
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
const frame = {
|
|
799
|
+
type: "mail.inbound",
|
|
800
|
+
agentAddress: recipient,
|
|
801
|
+
rawMessage,
|
|
802
|
+
};
|
|
803
|
+
if (enqueueForDisconnected(recipient, frame))
|
|
804
|
+
continue;
|
|
805
|
+
unrouted.push(recipient);
|
|
806
|
+
}
|
|
807
|
+
// Anything not routed locally is emitted as a notification. The
|
|
808
|
+
// host decides whether to relay onto an external transport, log,
|
|
809
|
+
// or drop. The wire layer takes no stance.
|
|
810
|
+
if (unrouted.length > 0) {
|
|
811
|
+
events.emit("mail.outbound.undelivered", {
|
|
812
|
+
rawMessage,
|
|
813
|
+
recipients: unrouted,
|
|
814
|
+
});
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
|
|
818
|
+
let results;
|
|
819
|
+
let raw;
|
|
820
|
+
try {
|
|
821
|
+
raw = Uint8Array.from(atob(rawMessage), (c) => c.charCodeAt(0));
|
|
822
|
+
results = await persist({
|
|
823
|
+
senderAddress,
|
|
824
|
+
recipients,
|
|
825
|
+
raw,
|
|
826
|
+
});
|
|
827
|
+
}
|
|
828
|
+
catch (err) {
|
|
829
|
+
logger.error `Failed to persist mail from ${senderAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
for (const result of results) {
|
|
833
|
+
events.emit("mail.persisted", {
|
|
834
|
+
id: result.id,
|
|
835
|
+
raw,
|
|
836
|
+
createdAt: result.createdAt,
|
|
837
|
+
direction: result.direction,
|
|
838
|
+
instanceId: result.instanceId,
|
|
839
|
+
address: result.address,
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
function handleClose(ws) {
|
|
844
|
+
const conn = connections.get(ws);
|
|
845
|
+
if (conn === undefined)
|
|
846
|
+
return;
|
|
847
|
+
for (const addr of conn.agentAddresses) {
|
|
848
|
+
// Only remove routing and pending state if this connection still
|
|
849
|
+
// owns the address. A reconnected sidecar may have already claimed it.
|
|
850
|
+
if (addressIndex.get(addr) === ws) {
|
|
851
|
+
addressIndex.delete(addr);
|
|
852
|
+
// Drop cached connector state for the same reason: a takeover
|
|
853
|
+
// sidecar's state lives in connectorStates under the same key,
|
|
854
|
+
// and only this owner's close should evict it. The next
|
|
855
|
+
// reconnect re-bootstraps via the router's
|
|
856
|
+
// restore-fires-callback path.
|
|
857
|
+
connectorStates.delete(addr);
|
|
858
|
+
const deployReq = pendingDeploys.get(addr);
|
|
859
|
+
if (deployReq !== undefined && deployReq.ws === ws) {
|
|
860
|
+
clearTimeout(deployReq.timer);
|
|
861
|
+
pendingDeploys.delete(addr);
|
|
862
|
+
deployReq.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
863
|
+
}
|
|
864
|
+
// Create a queue entry so messages can accumulate while the
|
|
865
|
+
// sidecar is disconnected. Skip if the agent is being undeployed --
|
|
866
|
+
// there is no point queuing messages for an agent being torn down.
|
|
867
|
+
if (!pendingUndeploys.has(addr)) {
|
|
868
|
+
const timer = setTimeout(() => {
|
|
869
|
+
const expired = disconnectedAgents.get(addr);
|
|
870
|
+
disconnectedAgents.delete(addr);
|
|
871
|
+
if (expired !== undefined) {
|
|
872
|
+
surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
|
|
873
|
+
}
|
|
874
|
+
}, disconnectQueueTTLMs);
|
|
875
|
+
disconnectedAgents.set(addr, { queue: [], timer });
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
// Remove this connection's workflow-substrate routes. No disconnect queue
|
|
880
|
+
// is created: these addresses re-register (with the complete live set)
|
|
881
|
+
// when the sidecar reconnects, and their in-flight run state is
|
|
882
|
+
// reconstructed sidecar-locally, not from a hub-side queue. The ownership
|
|
883
|
+
// guard mirrors the session loop above so a takeover by a newer ws is not
|
|
884
|
+
// clobbered by the prior owner's close.
|
|
885
|
+
for (const addr of conn.workflowAddresses) {
|
|
886
|
+
if (addressIndex.get(addr) === ws) {
|
|
887
|
+
addressIndex.delete(addr);
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
connections.delete(ws);
|
|
891
|
+
// Cancel the liveness timer for this connection.
|
|
892
|
+
const livenessTimer = livenessTimers.get(ws);
|
|
893
|
+
if (livenessTimer !== undefined) {
|
|
894
|
+
clearTimeout(livenessTimer);
|
|
895
|
+
livenessTimers.delete(ws);
|
|
896
|
+
}
|
|
897
|
+
// Drop the per-ws serialization chain; no more frames will queue on it.
|
|
898
|
+
messageChains.delete(ws);
|
|
899
|
+
// Cancel any pending challenge for this connection.
|
|
900
|
+
const challengeReq = pendingChallenges.get(ws);
|
|
901
|
+
if (challengeReq !== undefined) {
|
|
902
|
+
clearTimeout(challengeReq.timer);
|
|
903
|
+
pendingChallenges.delete(ws);
|
|
904
|
+
}
|
|
905
|
+
// Reject any in-flight requests that were sent to this sidecar.
|
|
906
|
+
for (const [requestId, req] of pending) {
|
|
907
|
+
if (req.ws !== ws)
|
|
908
|
+
continue;
|
|
909
|
+
clearTimeout(req.timer);
|
|
910
|
+
pending.delete(requestId);
|
|
911
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
912
|
+
}
|
|
913
|
+
// Reject any in-flight pack transfers for this sidecar.
|
|
914
|
+
for (const [transferId, pack] of pendingPacks) {
|
|
915
|
+
if (pack.ws !== ws)
|
|
916
|
+
continue;
|
|
917
|
+
clearTimeout(pack.timer);
|
|
918
|
+
pendingPacks.delete(transferId);
|
|
919
|
+
pack.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
920
|
+
}
|
|
921
|
+
// Reject any in-flight undeploys for this sidecar.
|
|
922
|
+
for (const [addr, req] of pendingUndeploys) {
|
|
923
|
+
if (req.ws !== ws)
|
|
924
|
+
continue;
|
|
925
|
+
clearTimeout(req.timer);
|
|
926
|
+
pendingUndeploys.delete(addr);
|
|
927
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
928
|
+
}
|
|
929
|
+
// Cancel any in-flight inbound pack transfers from this sidecar
|
|
930
|
+
// across both receivers. The two receivers track their own in-
|
|
931
|
+
// flight transferIds, so a pending workflow-run transfer for an
|
|
932
|
+
// agent that just disconnected won't outlive the connection just
|
|
933
|
+
// because the agent-state receiver has nothing to cancel. Iterate the
|
|
934
|
+
// owned union so a reconnected workflow deployment's transfer is
|
|
935
|
+
// cancelled too; the deduped set avoids a double-cancel for an address
|
|
936
|
+
// that is in both sets. A reclaimed address is not present here -- the
|
|
937
|
+
// verified reconnect path that took it over evicts it from this
|
|
938
|
+
// (superseded) connection's owned set -- so a stale close does not cancel
|
|
939
|
+
// the new owner's work.
|
|
940
|
+
const owned = ownedAddresses(conn);
|
|
941
|
+
for (const addr of owned) {
|
|
942
|
+
agentStatePackReceiver.cancelByAgent(addr);
|
|
943
|
+
workflowRunPackReceiver.cancelByAgent(addr);
|
|
944
|
+
}
|
|
945
|
+
events.emit("sidecar.disconnect", {
|
|
946
|
+
ownedAddresses: [...owned],
|
|
947
|
+
});
|
|
948
|
+
logger.info `Sidecar ${conn.sidecarId} disconnected`;
|
|
949
|
+
}
|
|
950
|
+
function nextRequestId() {
|
|
951
|
+
return `req-${++requestCounter}`;
|
|
952
|
+
}
|
|
953
|
+
function sendRequest(agentAddress, buildFrame) {
|
|
954
|
+
const ws = addressIndex.get(agentAddress);
|
|
955
|
+
if (ws === undefined) {
|
|
956
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
957
|
+
}
|
|
958
|
+
const conn = connections.get(ws);
|
|
959
|
+
if (conn === undefined) {
|
|
960
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
961
|
+
}
|
|
962
|
+
const requestId = nextRequestId();
|
|
963
|
+
const frame = buildFrame(requestId);
|
|
964
|
+
return new Promise((resolve, reject) => {
|
|
965
|
+
const timer = setTimeout(() => {
|
|
966
|
+
pending.delete(requestId);
|
|
967
|
+
reject(new Error(`Request ${requestId} timed out after ${requestTimeoutMs}ms`));
|
|
968
|
+
}, requestTimeoutMs);
|
|
969
|
+
pending.set(requestId, {
|
|
970
|
+
requestId,
|
|
971
|
+
ws,
|
|
972
|
+
resolve,
|
|
973
|
+
reject(error) {
|
|
974
|
+
reject(new Error(error));
|
|
975
|
+
},
|
|
976
|
+
timer,
|
|
977
|
+
});
|
|
978
|
+
conn.send(frame);
|
|
979
|
+
});
|
|
980
|
+
}
|
|
981
|
+
function resolvePending(requestId) {
|
|
982
|
+
const req = pending.get(requestId);
|
|
983
|
+
if (req === undefined)
|
|
984
|
+
return;
|
|
985
|
+
clearTimeout(req.timer);
|
|
986
|
+
pending.delete(requestId);
|
|
987
|
+
req.resolve();
|
|
988
|
+
}
|
|
989
|
+
function rejectPending(requestId, error) {
|
|
990
|
+
const req = pending.get(requestId);
|
|
991
|
+
if (req === undefined)
|
|
992
|
+
return;
|
|
993
|
+
clearTimeout(req.timer);
|
|
994
|
+
pending.delete(requestId);
|
|
995
|
+
req.reject(error);
|
|
996
|
+
}
|
|
997
|
+
function resolvePackPending(transferId) {
|
|
998
|
+
const entry = pendingPacks.get(transferId);
|
|
999
|
+
if (entry === undefined)
|
|
1000
|
+
return;
|
|
1001
|
+
clearTimeout(entry.timer);
|
|
1002
|
+
pendingPacks.delete(transferId);
|
|
1003
|
+
entry.resolve();
|
|
1004
|
+
}
|
|
1005
|
+
function rejectPackPending(transferId, reason) {
|
|
1006
|
+
const entry = pendingPacks.get(transferId);
|
|
1007
|
+
if (entry === undefined)
|
|
1008
|
+
return;
|
|
1009
|
+
clearTimeout(entry.timer);
|
|
1010
|
+
pendingPacks.delete(transferId);
|
|
1011
|
+
entry.reject(reason);
|
|
1012
|
+
}
|
|
1013
|
+
function resolveUndeployPending(agentAddress) {
|
|
1014
|
+
const req = pendingUndeploys.get(agentAddress);
|
|
1015
|
+
if (req === undefined) {
|
|
1016
|
+
logger.warn `Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`;
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
clearTimeout(req.timer);
|
|
1020
|
+
pendingUndeploys.delete(agentAddress);
|
|
1021
|
+
req.resolve();
|
|
1022
|
+
}
|
|
1023
|
+
function rejectUndeployPending(agentAddress, error) {
|
|
1024
|
+
const req = pendingUndeploys.get(agentAddress);
|
|
1025
|
+
if (req === undefined)
|
|
1026
|
+
return;
|
|
1027
|
+
clearTimeout(req.timer);
|
|
1028
|
+
pendingUndeploys.delete(agentAddress);
|
|
1029
|
+
req.reject(error);
|
|
1030
|
+
}
|
|
1031
|
+
// Routing rule: pick the receiver dedicated to the repoId.kind the
|
|
1032
|
+
// frame carries. The receivers' in-flight state is independent, so a
|
|
1033
|
+
// workflow-run transferId can never collide with or evict an
|
|
1034
|
+
// agent-state transferId for the same agentAddress.
|
|
1035
|
+
function pickPackReceiver(repoId) {
|
|
1036
|
+
switch (repoId.kind) {
|
|
1037
|
+
case "agent-state":
|
|
1038
|
+
return { receiver: agentStatePackReceiver };
|
|
1039
|
+
case "workflow-run":
|
|
1040
|
+
return { receiver: workflowRunPackReceiver };
|
|
1041
|
+
// The remaining kinds in `RepoKind` (`skill`, `package-registry`,
|
|
1042
|
+
// `workflow`) have no sidecar->hub pack flow today. A frame
|
|
1043
|
+
// arriving with those kinds is malformed at this layer.
|
|
1044
|
+
default:
|
|
1045
|
+
return null;
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
function pickReceivePackLookup(repoId) {
|
|
1049
|
+
switch (repoId.kind) {
|
|
1050
|
+
case "agent-state":
|
|
1051
|
+
return lookups.receiveAgentStatePack;
|
|
1052
|
+
case "workflow-run":
|
|
1053
|
+
return lookups.receiveWorkflowRunPack;
|
|
1054
|
+
default:
|
|
1055
|
+
return undefined;
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
function handlePackPush(ws, frame) {
|
|
1059
|
+
const conn = connections.get(ws);
|
|
1060
|
+
if (conn === undefined)
|
|
1061
|
+
return;
|
|
1062
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
1063
|
+
logger.warn `Received repo.pack.push for unrouted agent ${frame.agentAddress}`;
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
const picked = pickPackReceiver(frame.repoId);
|
|
1067
|
+
if (picked === null) {
|
|
1068
|
+
logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
1069
|
+
conn.send({
|
|
1070
|
+
type: "repo.pack.reject",
|
|
1071
|
+
agentAddress: frame.agentAddress,
|
|
1072
|
+
repoId: frame.repoId,
|
|
1073
|
+
transferId: frame.transferId,
|
|
1074
|
+
reason: "corrupt",
|
|
1075
|
+
});
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
1078
|
+
const reason = picked.receiver.handlePush(frame);
|
|
1079
|
+
if (reason !== null) {
|
|
1080
|
+
conn.send({
|
|
1081
|
+
type: "repo.pack.reject",
|
|
1082
|
+
agentAddress: frame.agentAddress,
|
|
1083
|
+
repoId: frame.repoId,
|
|
1084
|
+
transferId: frame.transferId,
|
|
1085
|
+
reason,
|
|
1086
|
+
});
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
async function handlePackDone(ws, frame) {
|
|
1090
|
+
const conn = connections.get(ws);
|
|
1091
|
+
if (conn === undefined)
|
|
1092
|
+
return;
|
|
1093
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
1094
|
+
logger.warn `Received repo.pack.done for unrouted agent ${frame.agentAddress}`;
|
|
1095
|
+
return;
|
|
1096
|
+
}
|
|
1097
|
+
const picked = pickPackReceiver(frame.repoId);
|
|
1098
|
+
if (picked === null) {
|
|
1099
|
+
logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
1100
|
+
conn.send({
|
|
1101
|
+
type: "repo.pack.reject",
|
|
1102
|
+
agentAddress: frame.agentAddress,
|
|
1103
|
+
repoId: frame.repoId,
|
|
1104
|
+
transferId: frame.transferId,
|
|
1105
|
+
reason: "corrupt",
|
|
1106
|
+
});
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1109
|
+
const result = picked.receiver.handleDone(frame);
|
|
1110
|
+
if (result === null) {
|
|
1111
|
+
conn.send({
|
|
1112
|
+
type: "repo.pack.reject",
|
|
1113
|
+
agentAddress: frame.agentAddress,
|
|
1114
|
+
repoId: frame.repoId,
|
|
1115
|
+
transferId: frame.transferId,
|
|
1116
|
+
reason: "corrupt",
|
|
1117
|
+
});
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
const receivePackLookup = pickReceivePackLookup(frame.repoId);
|
|
1121
|
+
if (receivePackLookup === undefined) {
|
|
1122
|
+
conn.send({
|
|
1123
|
+
type: "repo.pack.ack",
|
|
1124
|
+
agentAddress: frame.agentAddress,
|
|
1125
|
+
repoId: frame.repoId,
|
|
1126
|
+
transferId: frame.transferId,
|
|
1127
|
+
});
|
|
1128
|
+
return;
|
|
1129
|
+
}
|
|
1130
|
+
const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha);
|
|
1131
|
+
// Connection may have closed during async verification.
|
|
1132
|
+
const currentConn = connections.get(ws);
|
|
1133
|
+
if (currentConn === undefined)
|
|
1134
|
+
return;
|
|
1135
|
+
if (verdict.accepted) {
|
|
1136
|
+
currentConn.send({
|
|
1137
|
+
type: "repo.pack.ack",
|
|
1138
|
+
agentAddress: frame.agentAddress,
|
|
1139
|
+
repoId: frame.repoId,
|
|
1140
|
+
transferId: frame.transferId,
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
else {
|
|
1144
|
+
currentConn.send({
|
|
1145
|
+
type: "repo.pack.reject",
|
|
1146
|
+
agentAddress: frame.agentAddress,
|
|
1147
|
+
repoId: frame.repoId,
|
|
1148
|
+
transferId: frame.transferId,
|
|
1149
|
+
reason: verdict.reason,
|
|
1150
|
+
});
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
/**
|
|
1154
|
+
* Bind a per-step workflow-substrate address to a sidecar for the staging
|
|
1155
|
+
* window of a multi-step deploy, so `sendPack` can route the step's deploy
|
|
1156
|
+
* and asset packs before the deployment-level frame spawns the child.
|
|
1157
|
+
*
|
|
1158
|
+
* The address is hub-minted and workflow-derived (no per-address key), so
|
|
1159
|
+
* it enters the keyless `workflowAddresses` set -- never the challenged
|
|
1160
|
+
* `agentAddresses` set -- and is torn down by `unbindStepRoute` once the
|
|
1161
|
+
* step's packs land. `handleClose` reclaims it if the sidecar drops
|
|
1162
|
+
* mid-stage. Per-step addresses are not runtime-routed (mail, signals, and
|
|
1163
|
+
* drains use the deployment address), so the binding is transient: it is
|
|
1164
|
+
* never persisted into the reconnect set and never resurrected on
|
|
1165
|
+
* reconnect.
|
|
1166
|
+
*/
|
|
1167
|
+
function bindStepRoute(stepAddress) {
|
|
1168
|
+
const ws = addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress);
|
|
1169
|
+
if (ws === undefined) {
|
|
1170
|
+
throw new Error(`No sidecar available to stage workflow step "${stepAddress}"`);
|
|
1171
|
+
}
|
|
1172
|
+
const conn = connections.get(ws);
|
|
1173
|
+
if (conn === undefined) {
|
|
1174
|
+
throw new Error(`No sidecar connected to stage workflow step "${stepAddress}"`);
|
|
1175
|
+
}
|
|
1176
|
+
conn.workflowAddresses.add(stepAddress);
|
|
1177
|
+
addressIndex.set(stepAddress, ws);
|
|
1178
|
+
}
|
|
1179
|
+
/**
|
|
1180
|
+
* Remove a per-step route bound by `bindStepRoute` once the step's packs
|
|
1181
|
+
* have landed. Idempotent: an address that was never bound (or already
|
|
1182
|
+
* unbound, e.g. by a mid-stage `handleClose`) is a no-op.
|
|
1183
|
+
*/
|
|
1184
|
+
function unbindStepRoute(stepAddress) {
|
|
1185
|
+
const ws = addressIndex.get(stepAddress);
|
|
1186
|
+
if (ws === undefined)
|
|
1187
|
+
return;
|
|
1188
|
+
const conn = connections.get(ws);
|
|
1189
|
+
if (conn !== undefined) {
|
|
1190
|
+
conn.workflowAddresses.delete(stepAddress);
|
|
1191
|
+
}
|
|
1192
|
+
addressIndex.delete(stepAddress);
|
|
1193
|
+
}
|
|
1194
|
+
// Pack transfers may take longer than session requests due to data volume.
|
|
1195
|
+
const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
|
|
1196
|
+
function sendPack(agentAddress, pack, ref, commitSha, options) {
|
|
1197
|
+
const ws = addressIndex.get(agentAddress);
|
|
1198
|
+
if (ws === undefined) {
|
|
1199
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1200
|
+
}
|
|
1201
|
+
const conn = connections.get(ws);
|
|
1202
|
+
if (conn === undefined) {
|
|
1203
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1204
|
+
}
|
|
1205
|
+
const transferId = `pack-${++packCounter}`;
|
|
1206
|
+
// For the agent-state flow the destination agent and the source repo
|
|
1207
|
+
// are the same entity, so `repoId.id === agentAddress`. Asset packs
|
|
1208
|
+
// override this with the SOURCE asset's id so audit can correlate
|
|
1209
|
+
// the pack back to its hub-side origin.
|
|
1210
|
+
const repoId = options?.repoId ?? {
|
|
1211
|
+
kind: "agent-state",
|
|
1212
|
+
id: agentAddress,
|
|
1213
|
+
};
|
|
1214
|
+
const mountPath = options?.mountPath;
|
|
1215
|
+
// Register pending entry before sending frames so that a synchronous
|
|
1216
|
+
// repo.pack.ack (e.g. in tests or loopback transports) resolves correctly.
|
|
1217
|
+
return new Promise((resolve, reject) => {
|
|
1218
|
+
const timer = setTimeout(() => {
|
|
1219
|
+
pendingPacks.delete(transferId);
|
|
1220
|
+
reject(new Error(`Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`));
|
|
1221
|
+
}, PACK_TIMEOUT_MS);
|
|
1222
|
+
pendingPacks.set(transferId, {
|
|
1223
|
+
transferId,
|
|
1224
|
+
ws,
|
|
1225
|
+
resolve,
|
|
1226
|
+
reject(error) {
|
|
1227
|
+
reject(new Error(`Pack rejected: ${error}`));
|
|
1228
|
+
},
|
|
1229
|
+
timer,
|
|
1230
|
+
});
|
|
1231
|
+
// Send chunks
|
|
1232
|
+
for (const chunk of chunkPack(pack)) {
|
|
1233
|
+
conn.send({
|
|
1234
|
+
type: "repo.pack.push",
|
|
1235
|
+
agentAddress,
|
|
1236
|
+
repoId,
|
|
1237
|
+
transferId,
|
|
1238
|
+
seq: chunk.seq,
|
|
1239
|
+
data: chunk.data,
|
|
1240
|
+
});
|
|
1241
|
+
}
|
|
1242
|
+
// Send done
|
|
1243
|
+
conn.send({
|
|
1244
|
+
type: "repo.pack.done",
|
|
1245
|
+
agentAddress,
|
|
1246
|
+
repoId,
|
|
1247
|
+
transferId,
|
|
1248
|
+
ref,
|
|
1249
|
+
commitSha,
|
|
1250
|
+
...(mountPath !== undefined ? { mountPath } : {}),
|
|
1251
|
+
});
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
function routeMail(agentAddress, rawMessage) {
|
|
1255
|
+
const ws = addressIndex.get(agentAddress);
|
|
1256
|
+
if (ws !== undefined) {
|
|
1257
|
+
const conn = connections.get(ws);
|
|
1258
|
+
if (conn !== undefined) {
|
|
1259
|
+
conn.send({
|
|
1260
|
+
type: "mail.inbound",
|
|
1261
|
+
agentAddress,
|
|
1262
|
+
rawMessage,
|
|
1263
|
+
});
|
|
1264
|
+
return true;
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
// If the agent recently disconnected, queue for delivery on reconnect.
|
|
1268
|
+
const frame = { type: "mail.inbound", agentAddress, rawMessage };
|
|
1269
|
+
return enqueueForDisconnected(agentAddress, frame);
|
|
1270
|
+
}
|
|
1271
|
+
async function handleDeployAck(agentAddress, publicKey) {
|
|
1272
|
+
if (!pendingDeploys.has(agentAddress)) {
|
|
1273
|
+
logger.warn `Received agent.deploy.ack for "${agentAddress}" with no pending deploy`;
|
|
1274
|
+
return;
|
|
1275
|
+
}
|
|
1276
|
+
if (events.listenerCount("agent.deploy.ack") > 0) {
|
|
1277
|
+
try {
|
|
1278
|
+
await events.emitAndAwait("agent.deploy.ack", {
|
|
1279
|
+
agentAddress,
|
|
1280
|
+
publicKey,
|
|
1281
|
+
});
|
|
1282
|
+
}
|
|
1283
|
+
catch (err) {
|
|
1284
|
+
rejectDeployPending(agentAddress, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
|
|
1285
|
+
return;
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
resolveDeployPending(agentAddress);
|
|
1289
|
+
}
|
|
1290
|
+
function resolveDeployPending(agentAddress) {
|
|
1291
|
+
const req = pendingDeploys.get(agentAddress);
|
|
1292
|
+
if (req === undefined)
|
|
1293
|
+
return;
|
|
1294
|
+
clearTimeout(req.timer);
|
|
1295
|
+
pendingDeploys.delete(agentAddress);
|
|
1296
|
+
req.resolve();
|
|
1297
|
+
}
|
|
1298
|
+
function rejectDeployPending(agentAddress, error) {
|
|
1299
|
+
const req = pendingDeploys.get(agentAddress);
|
|
1300
|
+
if (req === undefined)
|
|
1301
|
+
return;
|
|
1302
|
+
clearTimeout(req.timer);
|
|
1303
|
+
pendingDeploys.delete(agentAddress);
|
|
1304
|
+
req.reject(error);
|
|
1305
|
+
}
|
|
1306
|
+
async function sendAgentDeploy(agentAddress, harnessConfig, workflow) {
|
|
1307
|
+
if (hubPublicKeyHex === undefined) {
|
|
1308
|
+
throw new Error("Hub signing key is required for agent deployment");
|
|
1309
|
+
}
|
|
1310
|
+
if (pendingDeploys.has(agentAddress)) {
|
|
1311
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
1312
|
+
}
|
|
1313
|
+
const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
|
|
1314
|
+
if (ws === undefined) {
|
|
1315
|
+
throw new Error(`No sidecar available for agent "${agentAddress}"`);
|
|
1316
|
+
}
|
|
1317
|
+
const conn = connections.get(ws);
|
|
1318
|
+
if (conn === undefined) {
|
|
1319
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1320
|
+
}
|
|
1321
|
+
conn.agentAddresses.add(agentAddress);
|
|
1322
|
+
addressIndex.set(agentAddress, ws);
|
|
1323
|
+
return new Promise((resolve, reject) => {
|
|
1324
|
+
// The deploy-ack handler does not currently thread the
|
|
1325
|
+
// sidecar-reported public key back through the pending-deploy
|
|
1326
|
+
// resolver; it only fires `agent.deploy.ack` listeners and then
|
|
1327
|
+
// resolves the pending deploy. Capture the key via a one-shot
|
|
1328
|
+
// listener so the return value carries it without a wire-shape
|
|
1329
|
+
// change to `agent.deploy.ack`.
|
|
1330
|
+
let capturedPublicKey;
|
|
1331
|
+
const detachListener = events.on("agent.deploy.ack", (payload) => {
|
|
1332
|
+
if (payload.agentAddress === agentAddress) {
|
|
1333
|
+
capturedPublicKey = payload.publicKey;
|
|
1334
|
+
}
|
|
1335
|
+
});
|
|
1336
|
+
const timer = setTimeout(() => {
|
|
1337
|
+
detachListener();
|
|
1338
|
+
pendingDeploys.delete(agentAddress);
|
|
1339
|
+
if (addressIndex.get(agentAddress) === ws) {
|
|
1340
|
+
conn.agentAddresses.delete(agentAddress);
|
|
1341
|
+
addressIndex.delete(agentAddress);
|
|
1342
|
+
}
|
|
1343
|
+
reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
1344
|
+
}, requestTimeoutMs);
|
|
1345
|
+
pendingDeploys.set(agentAddress, {
|
|
1346
|
+
agentAddress,
|
|
1347
|
+
ws,
|
|
1348
|
+
resolve() {
|
|
1349
|
+
detachListener();
|
|
1350
|
+
if (capturedPublicKey === undefined) {
|
|
1351
|
+
reject(new Error(`Deploy of "${agentAddress}" resolved without an agent.deploy.ack publicKey payload`));
|
|
1352
|
+
return;
|
|
1353
|
+
}
|
|
1354
|
+
resolve({ publicKey: capturedPublicKey });
|
|
1355
|
+
},
|
|
1356
|
+
reject(error) {
|
|
1357
|
+
detachListener();
|
|
1358
|
+
if (addressIndex.get(agentAddress) === ws) {
|
|
1359
|
+
conn.agentAddresses.delete(agentAddress);
|
|
1360
|
+
addressIndex.delete(agentAddress);
|
|
1361
|
+
}
|
|
1362
|
+
reject(new Error(error));
|
|
1363
|
+
},
|
|
1364
|
+
timer,
|
|
1365
|
+
});
|
|
1366
|
+
conn.send({
|
|
1367
|
+
type: "agent.deploy",
|
|
1368
|
+
agentAddress,
|
|
1369
|
+
agentId: harnessConfig.agentId,
|
|
1370
|
+
config: harnessConfig,
|
|
1371
|
+
hubPublicKey: hubPublicKeyHex,
|
|
1372
|
+
...(workflow !== undefined ? { workflow } : {}),
|
|
1373
|
+
});
|
|
1374
|
+
});
|
|
1375
|
+
}
|
|
1376
|
+
/**
|
|
1377
|
+
* Provision one step of a multi-step deploy on the sidecar WITHOUT
|
|
1378
|
+
* spawning: the sidecar initializes the step's agent-state repo and
|
|
1379
|
+
* records the hub key, so the follow-up deploy pack applies into a repo
|
|
1380
|
+
* and verifies against the recorded key -- but no supervisor or child is
|
|
1381
|
+
* constructed. The deployment-level workflow frame, sent once after every
|
|
1382
|
+
* step is provisioned, spawns the child.
|
|
1383
|
+
*
|
|
1384
|
+
* The step address must already be bound via `bindStepRoute`, which
|
|
1385
|
+
* resolves and records the sidecar; this reuses that route rather than
|
|
1386
|
+
* touching `agentAddresses`. Waits for the sidecar's `agent.deploy.ack`
|
|
1387
|
+
* so the caller can safely deliver the deploy pack afterward. On failure
|
|
1388
|
+
* the caller owns tearing the route down via `unbindStepRoute`.
|
|
1389
|
+
*/
|
|
1390
|
+
function sendProvisionStep(agentAddress, harnessConfig) {
|
|
1391
|
+
if (hubPublicKeyHex === undefined) {
|
|
1392
|
+
throw new Error("Hub signing key is required for step provisioning");
|
|
1393
|
+
}
|
|
1394
|
+
if (pendingDeploys.has(agentAddress)) {
|
|
1395
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
1396
|
+
}
|
|
1397
|
+
const ws = addressIndex.get(agentAddress);
|
|
1398
|
+
if (ws === undefined) {
|
|
1399
|
+
throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
|
|
1400
|
+
}
|
|
1401
|
+
const conn = connections.get(ws);
|
|
1402
|
+
if (conn === undefined) {
|
|
1403
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1404
|
+
}
|
|
1405
|
+
const hubKey = hubPublicKeyHex;
|
|
1406
|
+
return new Promise((resolve, reject) => {
|
|
1407
|
+
const timer = setTimeout(() => {
|
|
1408
|
+
pendingDeploys.delete(agentAddress);
|
|
1409
|
+
reject(new Error(`Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
1410
|
+
}, requestTimeoutMs);
|
|
1411
|
+
// The sidecar's `agent.deploy.ack` resolves this through
|
|
1412
|
+
// `resolveDeployPending` -> `req.resolve()`. The per-step address is
|
|
1413
|
+
// workflow-derived and records no hub-side key, so the ack's public
|
|
1414
|
+
// key is not needed and this resolves void.
|
|
1415
|
+
pendingDeploys.set(agentAddress, {
|
|
1416
|
+
agentAddress,
|
|
1417
|
+
ws,
|
|
1418
|
+
resolve() {
|
|
1419
|
+
resolve();
|
|
1420
|
+
},
|
|
1421
|
+
reject(error) {
|
|
1422
|
+
reject(new Error(error));
|
|
1423
|
+
},
|
|
1424
|
+
timer,
|
|
1425
|
+
});
|
|
1426
|
+
conn.send({
|
|
1427
|
+
type: "agent.deploy",
|
|
1428
|
+
agentAddress,
|
|
1429
|
+
agentId: harnessConfig.agentId,
|
|
1430
|
+
config: harnessConfig,
|
|
1431
|
+
hubPublicKey: hubKey,
|
|
1432
|
+
provisionStep: true,
|
|
1433
|
+
});
|
|
1434
|
+
});
|
|
1435
|
+
}
|
|
1436
|
+
function findSidecarForNewAgent(_agentAddress) {
|
|
1437
|
+
const first = connections.entries().next();
|
|
1438
|
+
if (first.done)
|
|
1439
|
+
return undefined;
|
|
1440
|
+
return first.value[0];
|
|
1441
|
+
}
|
|
1442
|
+
function sendAgentUndeploy(agentAddress, reason) {
|
|
1443
|
+
const ws = addressIndex.get(agentAddress);
|
|
1444
|
+
if (ws === undefined) {
|
|
1445
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1446
|
+
}
|
|
1447
|
+
const conn = connections.get(ws);
|
|
1448
|
+
if (conn === undefined) {
|
|
1449
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1450
|
+
}
|
|
1451
|
+
return new Promise((resolve, reject) => {
|
|
1452
|
+
const timer = setTimeout(() => {
|
|
1453
|
+
pendingUndeploys.delete(agentAddress);
|
|
1454
|
+
removeAgentAddress(ws, agentAddress);
|
|
1455
|
+
reject(new Error(`Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
1456
|
+
}, requestTimeoutMs);
|
|
1457
|
+
pendingUndeploys.set(agentAddress, {
|
|
1458
|
+
agentAddress,
|
|
1459
|
+
ws,
|
|
1460
|
+
resolve() {
|
|
1461
|
+
removeAgentAddress(ws, agentAddress);
|
|
1462
|
+
resolve();
|
|
1463
|
+
},
|
|
1464
|
+
reject(error) {
|
|
1465
|
+
removeAgentAddress(ws, agentAddress);
|
|
1466
|
+
reject(new Error(error));
|
|
1467
|
+
},
|
|
1468
|
+
timer,
|
|
1469
|
+
});
|
|
1470
|
+
conn.send({
|
|
1471
|
+
type: "agent.undeploy",
|
|
1472
|
+
agentAddress,
|
|
1473
|
+
reason,
|
|
1474
|
+
});
|
|
1475
|
+
});
|
|
1476
|
+
}
|
|
1477
|
+
function removeAgentAddress(ws, agentAddress) {
|
|
1478
|
+
addressIndex.delete(agentAddress);
|
|
1479
|
+
const conn = connections.get(ws);
|
|
1480
|
+
if (conn !== undefined) {
|
|
1481
|
+
conn.agentAddresses.delete(agentAddress);
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
function dispatchToSubscribers(agentAddress, event) {
|
|
1485
|
+
const subs = agentSubscribers.get(agentAddress);
|
|
1486
|
+
if (subs === undefined)
|
|
1487
|
+
return;
|
|
1488
|
+
for (const cb of [...subs]) {
|
|
1489
|
+
try {
|
|
1490
|
+
cb(event);
|
|
1491
|
+
}
|
|
1492
|
+
catch (err) {
|
|
1493
|
+
logger.warn `Agent subscriber threw: ${err instanceof Error ? err.message : String(err)}`;
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
function subscribeAgent(agentAddress, callback) {
|
|
1498
|
+
let subs = agentSubscribers.get(agentAddress);
|
|
1499
|
+
if (subs === undefined) {
|
|
1500
|
+
subs = new Set();
|
|
1501
|
+
agentSubscribers.set(agentAddress, subs);
|
|
1502
|
+
}
|
|
1503
|
+
subs.add(callback);
|
|
1504
|
+
return () => {
|
|
1505
|
+
const current = agentSubscribers.get(agentAddress);
|
|
1506
|
+
if (current === undefined)
|
|
1507
|
+
return;
|
|
1508
|
+
current.delete(callback);
|
|
1509
|
+
if (current.size === 0) {
|
|
1510
|
+
agentSubscribers.delete(agentAddress);
|
|
1511
|
+
}
|
|
1512
|
+
};
|
|
1513
|
+
}
|
|
1514
|
+
function getConnectedSidecars() {
|
|
1515
|
+
return Array.from(connections.values()).map((c) => c.sidecarId);
|
|
1516
|
+
}
|
|
1517
|
+
function getRoutableAddresses() {
|
|
1518
|
+
return Array.from(addressIndex.keys());
|
|
1519
|
+
}
|
|
1520
|
+
function getConnectorState(agentAddress) {
|
|
1521
|
+
return connectorStates.get(agentAddress) ?? null;
|
|
1522
|
+
}
|
|
1523
|
+
async function sendSourcesUpdate(agentAddress, sources, defaultSource) {
|
|
1524
|
+
await sendRequest(agentAddress, (requestId) => ({
|
|
1525
|
+
type: "sources.update",
|
|
1526
|
+
requestId,
|
|
1527
|
+
agentAddress,
|
|
1528
|
+
sources,
|
|
1529
|
+
defaultSource,
|
|
1530
|
+
}));
|
|
1531
|
+
}
|
|
1532
|
+
function sendSyncRequest(agentAddress) {
|
|
1533
|
+
const ws = addressIndex.get(agentAddress);
|
|
1534
|
+
if (ws === undefined) {
|
|
1535
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1536
|
+
}
|
|
1537
|
+
const conn = connections.get(ws);
|
|
1538
|
+
if (conn === undefined) {
|
|
1539
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1540
|
+
}
|
|
1541
|
+
const transferId = `sync-${++packCounter}`;
|
|
1542
|
+
conn.send({
|
|
1543
|
+
type: "sync.request",
|
|
1544
|
+
agentAddress,
|
|
1545
|
+
transferId,
|
|
1546
|
+
});
|
|
1547
|
+
}
|
|
1548
|
+
function sendSignalDeliver(opts) {
|
|
1549
|
+
const ws = addressIndex.get(opts.agentAddress);
|
|
1550
|
+
if (ws === undefined) {
|
|
1551
|
+
throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
|
|
1552
|
+
}
|
|
1553
|
+
const conn = connections.get(ws);
|
|
1554
|
+
if (conn === undefined) {
|
|
1555
|
+
throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
|
|
1556
|
+
}
|
|
1557
|
+
conn.send({
|
|
1558
|
+
type: "signal.deliver",
|
|
1559
|
+
agentAddress: opts.agentAddress,
|
|
1560
|
+
runId: opts.runId,
|
|
1561
|
+
signalName: opts.signalName,
|
|
1562
|
+
signalId: opts.signalId,
|
|
1563
|
+
payload: opts.payload,
|
|
1564
|
+
});
|
|
1565
|
+
}
|
|
1566
|
+
function sendDrain(opts) {
|
|
1567
|
+
const ws = addressIndex.get(opts.agentAddress);
|
|
1568
|
+
if (ws === undefined) {
|
|
1569
|
+
throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
|
|
1570
|
+
}
|
|
1571
|
+
const conn = connections.get(ws);
|
|
1572
|
+
if (conn === undefined) {
|
|
1573
|
+
throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
|
|
1574
|
+
}
|
|
1575
|
+
conn.send({
|
|
1576
|
+
type: "drain.deliver",
|
|
1577
|
+
agentAddress: opts.agentAddress,
|
|
1578
|
+
deadlineMs: opts.deadlineMs,
|
|
1579
|
+
});
|
|
1580
|
+
}
|
|
1581
|
+
return {
|
|
1582
|
+
handleOpen,
|
|
1583
|
+
handleMessage,
|
|
1584
|
+
handleClose,
|
|
1585
|
+
routeMail,
|
|
1586
|
+
sendAgentDeploy,
|
|
1587
|
+
sendAgentUndeploy,
|
|
1588
|
+
sendSourcesUpdate,
|
|
1589
|
+
sendPack,
|
|
1590
|
+
bindStepRoute,
|
|
1591
|
+
unbindStepRoute,
|
|
1592
|
+
sendProvisionStep,
|
|
1593
|
+
sendSyncRequest,
|
|
1594
|
+
sendSignalDeliver,
|
|
1595
|
+
sendDrain,
|
|
1596
|
+
subscribeAgent,
|
|
1597
|
+
dispatchAgentEvent: dispatchToSubscribers,
|
|
1598
|
+
getConnectedSidecars,
|
|
1599
|
+
getRoutableAddresses,
|
|
1600
|
+
getConnectorState,
|
|
1601
|
+
events,
|
|
1602
|
+
};
|
|
1603
|
+
}
|