@intx/hub-sessions 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +176 -0
- package/README.md +82 -1
- package/dist/agent-repo.d.ts +93 -0
- package/dist/agent-repo.js +109 -0
- package/dist/agent-state-kind.d.ts +12 -0
- package/dist/agent-state-kind.js +189 -0
- package/dist/asset-service.d.ts +104 -0
- package/dist/asset-service.js +267 -0
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +33 -0
- package/dist/credential-push.js +109 -0
- package/dist/event-collector-registry.d.ts +20 -0
- package/dist/event-collector-registry.js +115 -0
- package/dist/event-collector.d.ts +39 -0
- package/dist/event-collector.js +365 -0
- package/dist/hub-session-lookups.d.ts +135 -0
- package/dist/hub-session-lookups.js +663 -0
- package/dist/hub-session-orchestrator.d.ts +25 -0
- package/dist/hub-session-orchestrator.js +87 -0
- package/dist/index.d.ts +27 -0
- package/dist/index.js +24 -0
- package/dist/package-registry-kind.d.ts +70 -0
- package/dist/package-registry-kind.js +260 -0
- package/dist/repo-store/index.d.ts +4 -0
- package/dist/repo-store/index.js +3 -0
- package/dist/repo-store/store.d.ts +41 -0
- package/dist/repo-store/store.js +1829 -0
- package/dist/repo-store/subscribe-kind.d.ts +56 -0
- package/dist/repo-store/subscribe-kind.js +144 -0
- package/dist/repo-store/types.d.ts +571 -0
- package/dist/repo-store/types.js +42 -0
- package/dist/session-service.d.ts +416 -0
- package/dist/session-service.js +1191 -0
- package/dist/sidecar-allocation/contracts.d.ts +78 -0
- package/dist/sidecar-allocation/contracts.js +21 -0
- package/dist/sidecar-allocation/index.d.ts +4 -0
- package/dist/sidecar-allocation/index.js +3 -0
- package/dist/sidecar-allocation/placement-policy.d.ts +11 -0
- package/dist/sidecar-allocation/placement-policy.js +21 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +11 -0
- package/dist/sidecar-allocation/plugin-registry.js +37 -0
- package/dist/sidecar-allocation/reconciler.d.ts +42 -0
- package/dist/sidecar-allocation/reconciler.js +431 -0
- package/dist/skill-kind.d.ts +41 -0
- package/dist/skill-kind.js +292 -0
- package/dist/substrate.d.ts +8 -0
- package/dist/substrate.js +21 -0
- package/dist/workflow-allocation-service.d.ts +58 -0
- package/dist/workflow-allocation-service.js +239 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +40 -0
- package/dist/workflow-dispatch-service.js +146 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +37 -0
- package/dist/workflow-kind.js +310 -0
- package/dist/workflow-probe-gate.d.ts +214 -0
- package/dist/workflow-probe-gate.js +207 -0
- package/dist/workflow-run-event-log.d.ts +21 -0
- package/dist/workflow-run-event-log.js +51 -0
- package/dist/workflow-run-kind.d.ts +440 -0
- package/dist/workflow-run-kind.js +2916 -0
- package/dist/workflow-run-reader.d.ts +47 -0
- package/dist/workflow-run-reader.js +153 -0
- package/dist/workflow-run-restore.d.ts +15 -0
- package/dist/workflow-run-restore.js +26 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/ws/index.d.ts +3 -0
- package/dist/ws/index.js +3 -0
- package/dist/ws/sidecar-events.d.ts +222 -0
- package/dist/ws/sidecar-events.js +72 -0
- package/dist/ws/sidecar-handler.d.ts +305 -0
- package/dist/ws/sidecar-handler.js +2537 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +17 -0
- package/dist/ws/sidecar-token-authenticator.js +81 -0
- package/package.json +35 -12
- package/src/agent-repo.test.ts +0 -310
- package/src/agent-repo.ts +0 -165
- package/src/agent-state-kind.test.ts +0 -247
- package/src/agent-state-kind.ts +0 -204
- package/src/asset-service.test.ts +0 -540
- package/src/asset-service.ts +0 -378
- package/src/available-skills-stanza.test.ts +0 -87
- package/src/available-skills-stanza.ts +0 -47
- package/src/credential-push.ts +0 -65
- package/src/event-collector-registry.test.ts +0 -73
- package/src/event-collector-registry.ts +0 -171
- package/src/event-collector.test.ts +0 -1387
- package/src/event-collector.ts +0 -424
- package/src/hub-session-lookups.ts +0 -206
- package/src/hub-session-orchestrator.test.ts +0 -510
- package/src/hub-session-orchestrator.ts +0 -213
- package/src/index.ts +0 -78
- package/src/repo-store/index.ts +0 -15
- package/src/repo-store/store.test.ts +0 -1169
- package/src/repo-store/store.ts +0 -428
- package/src/repo-store/types.ts +0 -253
- package/src/session-service.test.ts +0 -895
- package/src/session-service.ts +0 -464
- package/src/skill-kind.test.ts +0 -599
- package/src/skill-kind.ts +0 -350
- package/src/ws/index.ts +0 -18
- package/src/ws/sidecar-events.test.ts +0 -96
- package/src/ws/sidecar-events.ts +0 -231
- package/src/ws/sidecar-handler.test.ts +0 -2217
- package/src/ws/sidecar-handler.ts +0 -1574
- package/tsconfig.json +0 -4
- package/tsconfig.tsbuildinfo +0 -1
|
@@ -0,0 +1,2537 @@
|
|
|
1
|
+
// Hub-side websocket handler for sidecar connections.
|
|
2
|
+
//
|
|
3
|
+
// Accepts websocket upgrades, processes register frames, maintains a routing
|
|
4
|
+
// table of agentAddress → sidecar connection, and dispatches frames between
|
|
5
|
+
// sidecars and the hub's internal systems.
|
|
6
|
+
import { getLogger } from "@intx/log";
|
|
7
|
+
import { verifyEd25519 } from "@intx/crypto";
|
|
8
|
+
import { chunkPack, createPackReceiver } from "@intx/pack-transport";
|
|
9
|
+
import { base64Decode, deriveMessageId, deriveWorkflowRunId, hexDecode, hexEncode, isRunAddress, } from "@intx/types";
|
|
10
|
+
import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
|
|
11
|
+
import { type } from "arktype";
|
|
12
|
+
import { SidecarFrame, } from "@intx/types/sidecar";
|
|
13
|
+
import { createSidecarEmitter, } from "./sidecar-events.js";
|
|
14
|
+
const logger = getLogger(["hub", "ws", "sidecar"]);
|
|
15
|
+
/**
|
|
16
|
+
* Whether this connection owns `address` for routing/lifecycle purposes --
|
|
17
|
+
* as a challenged session address OR a hub-minted workflow-substrate address.
|
|
18
|
+
* The two sets are kept physically distinct (they differ on the challenge /
|
|
19
|
+
* re-add path), but ownership readers -- pack-transfer authorization,
|
|
20
|
+
* in-flight cancellation, disconnect teardown -- must see the union, or a
|
|
21
|
+
* reconnected workflow deployment (which lives only in `workflowAddresses`)
|
|
22
|
+
* is silently treated as unowned even though its mail routes.
|
|
23
|
+
*/
|
|
24
|
+
function connOwnsAddress(conn, address) {
|
|
25
|
+
return (conn.agentAddresses.has(address) || conn.workflowAddresses.has(address));
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Bind pack writes to the repository implied by the authenticated address.
|
|
29
|
+
* An allocated credential is narrower still: it may only write its one
|
|
30
|
+
* deployment's workflow-run repository and never a standalone agent-state
|
|
31
|
+
* repository.
|
|
32
|
+
*/
|
|
33
|
+
function connCanPushRepo(conn, agentAddress, repoId) {
|
|
34
|
+
if (conn.identity.kind === "allocated" &&
|
|
35
|
+
agentAddress !== conn.identity.workflowRunAddress) {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
if (repoId.kind === "agent-state") {
|
|
39
|
+
return conn.identity.kind === "shared" && repoId.id === agentAddress;
|
|
40
|
+
}
|
|
41
|
+
return (repoId.kind === "workflow-run" &&
|
|
42
|
+
repoId.id === deriveWorkflowRunRepoId(agentAddress));
|
|
43
|
+
}
|
|
44
|
+
/** The deduped set of every address this connection owns (session + workflow). */
|
|
45
|
+
function ownedAddresses(conn) {
|
|
46
|
+
return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
|
|
47
|
+
}
|
|
48
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
49
|
+
const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
|
|
50
|
+
// A probe fetches a workflow's dependency closure from a registry and
|
|
51
|
+
// evaluates it on the sidecar, so it runs longer than a routine request; its
|
|
52
|
+
// default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS.
|
|
53
|
+
export const DEFAULT_PROBE_TIMEOUT_MS = 60_000;
|
|
54
|
+
const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100;
|
|
55
|
+
const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
|
|
56
|
+
const DEFAULT_PING_TIMEOUT_MS = 60_000;
|
|
57
|
+
const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000;
|
|
58
|
+
const DEFAULT_MAIL_ACK_MAX_RETRIES = 5;
|
|
59
|
+
export function createSidecarRouter(config) {
|
|
60
|
+
const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity = async () => true, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
|
|
61
|
+
// Receiver-dispatch surface. Wire-layer callsites emit events here;
|
|
62
|
+
// host code subscribes via `router.events`.
|
|
63
|
+
const events = createSidecarEmitter();
|
|
64
|
+
// ws handle → registered connection
|
|
65
|
+
const connections = new Map();
|
|
66
|
+
const allocatedConnections = new Map();
|
|
67
|
+
const allocationFences = new Map();
|
|
68
|
+
const allocationWaiters = new Map();
|
|
69
|
+
// agentAddress → ws handle (routing table)
|
|
70
|
+
const addressIndex = new Map();
|
|
71
|
+
// requestId → pending promise
|
|
72
|
+
const pending = new Map();
|
|
73
|
+
const pendingDeploys = new Map();
|
|
74
|
+
const pendingChallenges = new Map();
|
|
75
|
+
const disconnectedAgents = new Map();
|
|
76
|
+
const pendingMail = new Map();
|
|
77
|
+
// agentAddress → retention TTL timer for un-acked pending mail held across a
|
|
78
|
+
// disconnect. On close the per-entry retry timers are cleared (the socket is
|
|
79
|
+
// gone) but the entries are RETAINED so a verified reconnect can redeliver
|
|
80
|
+
// them; this timer bounds that retention so a sidecar that never reconnects
|
|
81
|
+
// does not leak entries. Cleared when the address reconnects (redelivery) or
|
|
82
|
+
// its last pending entry is acked.
|
|
83
|
+
const pendingMailRetention = new Map();
|
|
84
|
+
// agentAddress → set of subscriber callbacks for agent events
|
|
85
|
+
const agentSubscribers = new Map();
|
|
86
|
+
// agentAddress → cached connector-thread state, populated by
|
|
87
|
+
// connector.state.changed frames. Hub-side mail composition reads this
|
|
88
|
+
// to set threading headers on user-originated mail. Absent entries mean
|
|
89
|
+
// "no state reported yet" (e.g. mid-reconnect); callers must treat that
|
|
90
|
+
// identically to a null entry (no active thread).
|
|
91
|
+
const connectorStates = new Map();
|
|
92
|
+
// ws handle → liveness timer (reset on each ping from the sidecar)
|
|
93
|
+
const livenessTimers = new Map();
|
|
94
|
+
// Per-ws serialization chain for QUEUE-class frames (see `frameBypassesQueue`
|
|
95
|
+
// for the split and the invariant behind it). A frame that establishes or
|
|
96
|
+
// reads routing waits for earlier queued frames on the same ws to complete,
|
|
97
|
+
// so it observes their finished effects -- most importantly an async
|
|
98
|
+
// register's routing write, which would otherwise land after a following
|
|
99
|
+
// connector.state.changed / mail / pack frame and silently drop it. It holds
|
|
100
|
+
// a single in-flight promise per ws (replaced each queued frame), cleared on
|
|
101
|
+
// close.
|
|
102
|
+
const messageChains = new Map();
|
|
103
|
+
const pendingPacks = new Map();
|
|
104
|
+
let packCounter = 0;
|
|
105
|
+
const pendingUndeploys = new Map();
|
|
106
|
+
const pendingProbes = new Map();
|
|
107
|
+
// Receives agent-state packs pushed from sidecars. The wire frames
|
|
108
|
+
// (`repo.pack.push` / `repo.pack.done`) are shared with the
|
|
109
|
+
// workflow-run flow; dispatch on `repoId.kind` picks which receiver
|
|
110
|
+
// observes the chunks. The two receivers maintain independent
|
|
111
|
+
// in-flight pack state and independent cancel-by-agent semantics so a
|
|
112
|
+
// pending workflow-run transfer cannot disturb a concurrent agent-
|
|
113
|
+
// state transfer for the same agent and vice versa.
|
|
114
|
+
const agentStatePackReceiver = createPackReceiver();
|
|
115
|
+
const workflowRunPackReceiver = createPackReceiver();
|
|
116
|
+
let requestCounter = 0;
|
|
117
|
+
// Surface disconnect-queue mail that is being dropped rather than delivered.
|
|
118
|
+
// Every dropped frame reaches the same channel routing failures already use
|
|
119
|
+
// (`mail.outbound.undelivered`, logged by the orchestrator), plus a warn that
|
|
120
|
+
// names the recipient and the drop count so a size-cap eviction or a TTL
|
|
121
|
+
// expiry of a still-full queue is visible instead of silent. A queued frame
|
|
122
|
+
// is always a `mail.inbound` carrying the sender's rawMessage; a frame of any
|
|
123
|
+
// other shape has no rawMessage to relay and is surfaced by the warn alone.
|
|
124
|
+
function surfaceDroppedFrames(agentAddress, frames, reason) {
|
|
125
|
+
if (frames.length === 0)
|
|
126
|
+
return;
|
|
127
|
+
logger.warn `Dropping ${String(frames.length)} queued message(s) for ${agentAddress}: ${reason}`;
|
|
128
|
+
for (const frame of frames) {
|
|
129
|
+
if (frame.type !== "mail.inbound")
|
|
130
|
+
continue;
|
|
131
|
+
events.emit("mail.outbound.undelivered", {
|
|
132
|
+
rawMessage: frame.rawMessage,
|
|
133
|
+
recipients: [agentAddress],
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
function enqueueForDisconnected(agentAddress, frame) {
|
|
138
|
+
const entry = disconnectedAgents.get(agentAddress);
|
|
139
|
+
if (entry === undefined)
|
|
140
|
+
return false;
|
|
141
|
+
if (entry.queue.length >= disconnectQueueMaxSize) {
|
|
142
|
+
const evicted = entry.queue.shift();
|
|
143
|
+
if (evicted !== undefined) {
|
|
144
|
+
surfaceDroppedFrames(agentAddress, [evicted], "disconnect queue full");
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
entry.queue.push(frame);
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
150
|
+
function flushDisconnectedQueue(agentAddress, conn) {
|
|
151
|
+
const entry = disconnectedAgents.get(agentAddress);
|
|
152
|
+
if (entry === undefined)
|
|
153
|
+
return;
|
|
154
|
+
clearTimeout(entry.timer);
|
|
155
|
+
disconnectedAgents.delete(agentAddress);
|
|
156
|
+
for (const frame of entry.queue) {
|
|
157
|
+
conn.send(frame);
|
|
158
|
+
}
|
|
159
|
+
if (entry.queue.length > 0) {
|
|
160
|
+
logger.info `Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
// Track a connected-window `mail.inbound` for redelivery until the sidecar
|
|
164
|
+
// acks its durable inbox write. Replaces any prior entry for the same
|
|
165
|
+
// (agentAddress, messageId) -- clearing its timer first so no timer leaks --
|
|
166
|
+
// which keeps a re-sent delivery from arming a second concurrent retry loop.
|
|
167
|
+
function trackPendingMail(agentAddress, messageId, frame, runGrants, allocatedTarget) {
|
|
168
|
+
let byId = pendingMail.get(agentAddress);
|
|
169
|
+
if (byId === undefined) {
|
|
170
|
+
byId = new Map();
|
|
171
|
+
pendingMail.set(agentAddress, byId);
|
|
172
|
+
}
|
|
173
|
+
const existing = byId.get(messageId);
|
|
174
|
+
if (existing !== undefined)
|
|
175
|
+
clearTimeout(existing.timer);
|
|
176
|
+
byId.set(messageId, {
|
|
177
|
+
agentAddress,
|
|
178
|
+
messageId,
|
|
179
|
+
frame,
|
|
180
|
+
attempts: 0,
|
|
181
|
+
timer: setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs),
|
|
182
|
+
...(runGrants !== undefined ? { runGrants } : {}),
|
|
183
|
+
...(allocatedTarget !== undefined ? { allocatedTarget } : {}),
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
// Replay a mail-triggered run's grants ahead of a redelivery of its trigger
|
|
187
|
+
// mail. Same-connection FIFO lands the `run.grants` frame before the mail, so
|
|
188
|
+
// the redelivered run resolves its onRunStart barrier instead of failing
|
|
189
|
+
// closed on missing grants.
|
|
190
|
+
function replayRunGrantsAhead(conn, entry) {
|
|
191
|
+
if (entry.runGrants === undefined)
|
|
192
|
+
return;
|
|
193
|
+
conn.send({
|
|
194
|
+
type: "run.grants",
|
|
195
|
+
agentAddress: entry.agentAddress,
|
|
196
|
+
runId: entry.runGrants.runId,
|
|
197
|
+
stepGrants: entry.runGrants.stepGrants,
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
function deletePendingMail(byId, agentAddress, messageId) {
|
|
201
|
+
byId.delete(messageId);
|
|
202
|
+
if (byId.size === 0) {
|
|
203
|
+
pendingMail.delete(agentAddress);
|
|
204
|
+
// The retention TTL guards a non-empty pending set; drop it once the set
|
|
205
|
+
// is empty so it never outlives the entries it was bounding.
|
|
206
|
+
const retention = pendingMailRetention.get(agentAddress);
|
|
207
|
+
if (retention !== undefined) {
|
|
208
|
+
clearTimeout(retention);
|
|
209
|
+
pendingMailRetention.delete(agentAddress);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
function retryPendingMail(agentAddress, messageId) {
|
|
214
|
+
const byId = pendingMail.get(agentAddress);
|
|
215
|
+
if (byId === undefined)
|
|
216
|
+
return;
|
|
217
|
+
const entry = byId.get(messageId);
|
|
218
|
+
if (entry === undefined)
|
|
219
|
+
return;
|
|
220
|
+
if (entry.attempts >= mailAckMaxRetries) {
|
|
221
|
+
// The sidecar never acked within the retry budget. The ack is withheld
|
|
222
|
+
// precisely because the sidecar's durable inbox write failed, so the
|
|
223
|
+
// mail was NOT delivered: surface it as undelivered so the host can relay
|
|
224
|
+
// it onto an external transport, then drop the pending entry so its timer
|
|
225
|
+
// does not leak.
|
|
226
|
+
if (entry.frame.type === "mail.inbound") {
|
|
227
|
+
events.emit("mail.outbound.undelivered", {
|
|
228
|
+
rawMessage: entry.frame.rawMessage,
|
|
229
|
+
recipients: [agentAddress],
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
233
|
+
logger.warn `Gave up redelivering mail ${messageId} to ${agentAddress} after ${String(entry.attempts)} un-acked attempt(s)`;
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
// Redeliver over the address's CURRENT owner: a verified reconnect may have
|
|
237
|
+
// moved the address to a new connection since the original delivery.
|
|
238
|
+
const ws = addressIndex.get(agentAddress);
|
|
239
|
+
const conn = ws !== undefined ? connections.get(ws) : undefined;
|
|
240
|
+
const allocated = entry.allocatedTarget === undefined
|
|
241
|
+
? undefined
|
|
242
|
+
: allocatedConnections.get(entry.allocatedTarget.allocationId);
|
|
243
|
+
const targetStillOwnsAddress = entry.allocatedTarget === undefined ||
|
|
244
|
+
(allocated !== undefined &&
|
|
245
|
+
allocated.identity.generation === entry.allocatedTarget.generation &&
|
|
246
|
+
allocated.ws === ws);
|
|
247
|
+
if (conn === undefined || !targetStillOwnsAddress) {
|
|
248
|
+
// No live connection to recover into. Connected-window redelivery only
|
|
249
|
+
// applies while the address is routable; a disconnected address is not
|
|
250
|
+
// retried here.
|
|
251
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
252
|
+
logger.warn `Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`;
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
entry.attempts += 1;
|
|
256
|
+
replayRunGrantsAhead(conn, entry);
|
|
257
|
+
conn.send(entry.frame);
|
|
258
|
+
entry.timer = setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs);
|
|
259
|
+
}
|
|
260
|
+
function resolvePendingMail(agentAddress, messageId) {
|
|
261
|
+
const byId = pendingMail.get(agentAddress);
|
|
262
|
+
if (byId === undefined)
|
|
263
|
+
return;
|
|
264
|
+
const entry = byId.get(messageId);
|
|
265
|
+
if (entry === undefined)
|
|
266
|
+
return;
|
|
267
|
+
clearTimeout(entry.timer);
|
|
268
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
269
|
+
}
|
|
270
|
+
// Hold an address's un-acked pending mail across a disconnect. The per-entry
|
|
271
|
+
// retry timers are cleared -- retrying over the dead socket is pointless --
|
|
272
|
+
// but the entries are KEPT so a verified reconnect can redeliver them. A
|
|
273
|
+
// retention TTL (the disconnect-queue horizon) bounds the hold so a sidecar
|
|
274
|
+
// that never reconnects does not leak; on expiry the still-un-acked entries
|
|
275
|
+
// are surfaced as `mail.outbound.undelivered` so the host can relay them,
|
|
276
|
+
// since a withheld ack means the sidecar's durable write never landed.
|
|
277
|
+
function retainPendingMailForAddress(agentAddress) {
|
|
278
|
+
const byId = pendingMail.get(agentAddress);
|
|
279
|
+
if (byId === undefined)
|
|
280
|
+
return;
|
|
281
|
+
for (const entry of byId.values())
|
|
282
|
+
clearTimeout(entry.timer);
|
|
283
|
+
const existing = pendingMailRetention.get(agentAddress);
|
|
284
|
+
if (existing !== undefined)
|
|
285
|
+
clearTimeout(existing);
|
|
286
|
+
const timer = setTimeout(() => {
|
|
287
|
+
pendingMailRetention.delete(agentAddress);
|
|
288
|
+
const expired = pendingMail.get(agentAddress);
|
|
289
|
+
pendingMail.delete(agentAddress);
|
|
290
|
+
if (expired !== undefined && expired.size > 0) {
|
|
291
|
+
for (const entry of expired.values()) {
|
|
292
|
+
if (entry.frame.type !== "mail.inbound")
|
|
293
|
+
continue;
|
|
294
|
+
events.emit("mail.outbound.undelivered", {
|
|
295
|
+
rawMessage: entry.frame.rawMessage,
|
|
296
|
+
recipients: [agentAddress],
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
logger.warn `Dropping ${String(expired.size)} un-acked message(s) for ${agentAddress}: pending-mail retention TTL expired`;
|
|
300
|
+
}
|
|
301
|
+
}, disconnectQueueTTLMs);
|
|
302
|
+
pendingMailRetention.set(agentAddress, timer);
|
|
303
|
+
}
|
|
304
|
+
// Redeliver an address's retained un-acked pending mail on a verified
|
|
305
|
+
// reconnect. Replays identical bytes (same messageId) over the new
|
|
306
|
+
// connection, so the sidecar's inbox dedups a message it already wrote
|
|
307
|
+
// (effectively-once) and processes one it had dropped (no loss). Re-arms the
|
|
308
|
+
// connected-window retry over the new connection with a fresh per-generation
|
|
309
|
+
// budget, so a redelivery that is itself dropped before its ack is retried.
|
|
310
|
+
function redeliverPendingMail(agentAddress, conn) {
|
|
311
|
+
const retention = pendingMailRetention.get(agentAddress);
|
|
312
|
+
if (retention !== undefined) {
|
|
313
|
+
clearTimeout(retention);
|
|
314
|
+
pendingMailRetention.delete(agentAddress);
|
|
315
|
+
}
|
|
316
|
+
const byId = pendingMail.get(agentAddress);
|
|
317
|
+
if (byId === undefined)
|
|
318
|
+
return;
|
|
319
|
+
for (const entry of [...byId.values()]) {
|
|
320
|
+
if (entry.allocatedTarget !== undefined &&
|
|
321
|
+
(conn.identity.kind !== "allocated" ||
|
|
322
|
+
conn.identity.allocationId !== entry.allocatedTarget.allocationId ||
|
|
323
|
+
conn.identity.generation !== entry.allocatedTarget.generation)) {
|
|
324
|
+
// The Hub-owned dispatch row survives generation replacement and will
|
|
325
|
+
// be requeued by the allocation-ready callback. Do not leak or replay
|
|
326
|
+
// this generation-local retry entry onto a different worker.
|
|
327
|
+
clearTimeout(entry.timer);
|
|
328
|
+
deletePendingMail(byId, agentAddress, entry.messageId);
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
331
|
+
replayRunGrantsAhead(conn, entry);
|
|
332
|
+
conn.send(entry.frame);
|
|
333
|
+
entry.attempts = 0;
|
|
334
|
+
entry.timer = setTimeout(() => retryPendingMail(agentAddress, entry.messageId), mailAckRetryIntervalMs);
|
|
335
|
+
}
|
|
336
|
+
if (byId.size > 0) {
|
|
337
|
+
logger.info `Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
function resetLivenessTimer(ws) {
|
|
341
|
+
const existing = livenessTimers.get(ws);
|
|
342
|
+
if (existing !== undefined)
|
|
343
|
+
clearTimeout(existing);
|
|
344
|
+
const timer = setTimeout(() => {
|
|
345
|
+
livenessTimers.delete(ws);
|
|
346
|
+
logger.warn `Sidecar ping timeout, closing connection`;
|
|
347
|
+
ws.close();
|
|
348
|
+
}, pingTimeoutMs);
|
|
349
|
+
livenessTimers.set(ws, timer);
|
|
350
|
+
}
|
|
351
|
+
function handlePing(ws) {
|
|
352
|
+
resetLivenessTimer(ws);
|
|
353
|
+
// Always respond with pong, even before register/reconnect completes.
|
|
354
|
+
// The sidecar's ping timer starts on open, which may fire before the
|
|
355
|
+
// async registration handshake finishes.
|
|
356
|
+
ws.send(JSON.stringify({ type: "pong" }));
|
|
357
|
+
}
|
|
358
|
+
function handleOpen(ws) {
|
|
359
|
+
// Connection is not usable until a register frame arrives.
|
|
360
|
+
// Start the liveness timer immediately — a sidecar that connects
|
|
361
|
+
// but never sends a ping will be reaped.
|
|
362
|
+
resetLivenessTimer(ws);
|
|
363
|
+
}
|
|
364
|
+
function handleMessage(ws, data) {
|
|
365
|
+
let raw;
|
|
366
|
+
try {
|
|
367
|
+
raw = JSON.parse(data);
|
|
368
|
+
}
|
|
369
|
+
catch {
|
|
370
|
+
logger.warn `Unparseable frame from sidecar connection`;
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const validated = SidecarFrame(raw);
|
|
374
|
+
if (validated instanceof type.errors) {
|
|
375
|
+
logger.warn `Invalid sidecar frame: ${validated.summary}`;
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
const frame = validated;
|
|
379
|
+
// Bypass frames (liveness + terminal responses to outbound requests)
|
|
380
|
+
// dispatch immediately: they resolve the very promises a queued handler
|
|
381
|
+
// may be blocked on, so queuing them would deadlock the round-trip.
|
|
382
|
+
if (frameBypassesQueue(frame)) {
|
|
383
|
+
// Guard so a bypass handler's failure -- a synchronous throw or an async
|
|
384
|
+
// ack handler's rejection -- is logged rather than floating out of the
|
|
385
|
+
// immediate dispatch. The async wrapper turns a synchronous throw into a
|
|
386
|
+
// rejection too, matching the queue path's .then/.catch coverage.
|
|
387
|
+
void (async () => dispatchFrame(ws, frame))().catch((err) => {
|
|
388
|
+
logger.warn `Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`;
|
|
389
|
+
});
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
// Everything else serializes per ws so a frame that establishes or reads
|
|
393
|
+
// routing observes earlier queued frames' completed effects.
|
|
394
|
+
const prev = messageChains.get(ws) ?? Promise.resolve();
|
|
395
|
+
const next = prev
|
|
396
|
+
.then(() => dispatchFrame(ws, frame))
|
|
397
|
+
.catch((err) => {
|
|
398
|
+
logger.warn `Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`;
|
|
399
|
+
});
|
|
400
|
+
messageChains.set(ws, next);
|
|
401
|
+
}
|
|
402
|
+
function assertNever(x) {
|
|
403
|
+
throw new Error(`Unclassified sidecar frame type: ${JSON.stringify(x)}`);
|
|
404
|
+
}
|
|
405
|
+
// Whether `frame` bypasses the per-ws serialization chain. Invariant: a frame
|
|
406
|
+
// bypasses IFF it is liveness (ping) OR a terminal response to an
|
|
407
|
+
// already-issued outbound request -- correlated purely by
|
|
408
|
+
// requestId/transferId/agentAddress in the pending maps, touching no routing
|
|
409
|
+
// state. Such a frame has no ordering obligation against new inbound frames
|
|
410
|
+
// (a response cannot resolve "too early" for a request that already went
|
|
411
|
+
// out), and it is exactly what in-flight queued handlers block on, so it MUST
|
|
412
|
+
// run out of band or the challenge round-trip deadlocks (challenge.response
|
|
413
|
+
// -> agent.reconnected reaction -> sendSourcesUpdate awaits a later
|
|
414
|
+
// session.ack). Every other frame establishes or reads routing, or carries an
|
|
415
|
+
// inbound payload whose order matters, so it queues. The exhaustive switch +
|
|
416
|
+
// assertNever makes adding a SidecarFrame variant without classifying it a
|
|
417
|
+
// compile error, not a latent deadlock or a silent bypass hole.
|
|
418
|
+
function frameBypassesQueue(frame) {
|
|
419
|
+
switch (frame.type) {
|
|
420
|
+
case "ping":
|
|
421
|
+
case "session.ack":
|
|
422
|
+
case "session.error":
|
|
423
|
+
case "agent.deploy.ack":
|
|
424
|
+
case "agent.error":
|
|
425
|
+
case "agent.undeploy.ack":
|
|
426
|
+
case "repo.pack.ack":
|
|
427
|
+
case "repo.pack.reject":
|
|
428
|
+
case "workflow.probe.result":
|
|
429
|
+
case "workflow.probe.error":
|
|
430
|
+
return true;
|
|
431
|
+
case "register":
|
|
432
|
+
case "reconnect":
|
|
433
|
+
case "challenge.response":
|
|
434
|
+
case "mail.outbound":
|
|
435
|
+
case "agent.event":
|
|
436
|
+
case "connector.state.changed":
|
|
437
|
+
case "mail.inbound.ack":
|
|
438
|
+
case "signal.correlation.register":
|
|
439
|
+
case "repo.pack.push":
|
|
440
|
+
case "repo.pack.done":
|
|
441
|
+
return false;
|
|
442
|
+
default:
|
|
443
|
+
return assertNever(frame);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
// Runs one frame's handler. Returns the handler's promise for async handlers
|
|
447
|
+
// so the per-ws chain can await bounded completion; sync handlers return
|
|
448
|
+
// void. Never awaits a promise that resolves on a LATER same-ws frame -- the
|
|
449
|
+
// only such await (the challenge round-trip's session.ack) is reached via a
|
|
450
|
+
// bypass frame, which does not queue.
|
|
451
|
+
function dispatchFrame(ws, frame) {
|
|
452
|
+
switch (frame.type) {
|
|
453
|
+
case "register": {
|
|
454
|
+
const agentAddresses = frame.agentAddresses;
|
|
455
|
+
return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses));
|
|
456
|
+
}
|
|
457
|
+
case "reconnect": {
|
|
458
|
+
const agentAddresses = frame.agentAddresses;
|
|
459
|
+
const deployRefs = frame.deployRefs ?? {};
|
|
460
|
+
return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, deployRefs));
|
|
461
|
+
}
|
|
462
|
+
case "challenge.response":
|
|
463
|
+
return handleChallengeResponse(ws, frame.responses);
|
|
464
|
+
case "agent.deploy.ack":
|
|
465
|
+
return handleDeployAck(ws, frame);
|
|
466
|
+
case "agent.error":
|
|
467
|
+
rejectDeployPendingFromFrame(ws, frame.agentAddress, frame.error);
|
|
468
|
+
rejectUndeployPending(ws, frame.agentAddress, frame.error);
|
|
469
|
+
return;
|
|
470
|
+
case "agent.undeploy.ack":
|
|
471
|
+
resolveUndeployPending(ws, frame.agentAddress);
|
|
472
|
+
return;
|
|
473
|
+
case "ping":
|
|
474
|
+
handlePing(ws);
|
|
475
|
+
return;
|
|
476
|
+
case "mail.outbound":
|
|
477
|
+
if (frame.delivered !== true) {
|
|
478
|
+
return handleMailOutbound(frame.rawMessage, frame.recipients);
|
|
479
|
+
}
|
|
480
|
+
if (lookups.persistMail && frame.senderAddress) {
|
|
481
|
+
return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
|
|
482
|
+
}
|
|
483
|
+
if (!frame.senderAddress) {
|
|
484
|
+
logger.warn `Dropping delivered mail.outbound frame with no senderAddress`;
|
|
485
|
+
}
|
|
486
|
+
else {
|
|
487
|
+
logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
|
|
488
|
+
}
|
|
489
|
+
return;
|
|
490
|
+
case "agent.event":
|
|
491
|
+
events.emit("agent.event", {
|
|
492
|
+
agentAddress: frame.agentAddress,
|
|
493
|
+
sessionId: frame.sessionId,
|
|
494
|
+
event: frame.event,
|
|
495
|
+
});
|
|
496
|
+
dispatchToSubscribers(frame.agentAddress, frame.event);
|
|
497
|
+
return;
|
|
498
|
+
case "connector.state.changed":
|
|
499
|
+
// Gate the cache write on the sending sidecar actually owning
|
|
500
|
+
// the named agent. A misbehaving sidecar that knows another
|
|
501
|
+
// agent's address could otherwise poison the cached state.
|
|
502
|
+
if (addressIndex.get(frame.agentAddress) !== ws) {
|
|
503
|
+
logger.warn `Dropping connector.state.changed for ${frame.agentAddress}: not registered to this sidecar`;
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
connectorStates.set(frame.agentAddress, frame.connectorState);
|
|
507
|
+
events.emit("connector.state.changed", {
|
|
508
|
+
agentAddress: frame.agentAddress,
|
|
509
|
+
connectorState: frame.connectorState,
|
|
510
|
+
});
|
|
511
|
+
return;
|
|
512
|
+
case "mail.inbound.ack": {
|
|
513
|
+
// Terminal receipt for a connected-window `mail.inbound`: the sidecar
|
|
514
|
+
// has durably written the message to its inbox. Gate on ownership --
|
|
515
|
+
// like connector.state.changed and signal.correlation.register -- so a
|
|
516
|
+
// sidecar cannot clear another sidecar's pending mail. The messageId is
|
|
517
|
+
// a hub-minted id only the owning sidecar ever received on the frame,
|
|
518
|
+
// so the ownership check is defense-in-depth, not the sole guard.
|
|
519
|
+
const conn = connections.get(ws);
|
|
520
|
+
if (conn === undefined)
|
|
521
|
+
return;
|
|
522
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
523
|
+
logger.warn `Dropping mail.inbound.ack for ${frame.agentAddress}: not registered to this sidecar`;
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
resolvePendingMail(frame.agentAddress, frame.messageId);
|
|
527
|
+
events.emit("mail.inbound.acknowledged", {
|
|
528
|
+
agentAddress: frame.agentAddress,
|
|
529
|
+
messageId: frame.messageId,
|
|
530
|
+
...(conn.identity.kind === "allocated"
|
|
531
|
+
? {
|
|
532
|
+
allocated: {
|
|
533
|
+
allocationId: conn.identity.allocationId,
|
|
534
|
+
anchorRunId: conn.identity.anchorRunId,
|
|
535
|
+
generation: conn.identity.generation,
|
|
536
|
+
},
|
|
537
|
+
}
|
|
538
|
+
: {}),
|
|
539
|
+
});
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
542
|
+
case "signal.correlation.register":
|
|
543
|
+
return handleSignalCorrelationRegister(ws, frame);
|
|
544
|
+
case "session.ack":
|
|
545
|
+
resolvePending(frame.requestId);
|
|
546
|
+
return;
|
|
547
|
+
case "session.error":
|
|
548
|
+
rejectPending(frame.requestId, frame.error);
|
|
549
|
+
return;
|
|
550
|
+
case "repo.pack.ack":
|
|
551
|
+
resolvePackPending(ws, frame);
|
|
552
|
+
return;
|
|
553
|
+
case "repo.pack.reject":
|
|
554
|
+
rejectPackPending(ws, frame);
|
|
555
|
+
return;
|
|
556
|
+
case "repo.pack.push":
|
|
557
|
+
handlePackPush(ws, frame);
|
|
558
|
+
return;
|
|
559
|
+
case "repo.pack.done":
|
|
560
|
+
return handlePackDone(ws, frame);
|
|
561
|
+
case "workflow.probe.result":
|
|
562
|
+
resolveProbe(frame.requestId, {
|
|
563
|
+
projection: frame.projection,
|
|
564
|
+
grants: frame.grants,
|
|
565
|
+
grantWalkSnapshot: frame.grantWalkSnapshot,
|
|
566
|
+
wireHash: frame.wireHash,
|
|
567
|
+
});
|
|
568
|
+
return;
|
|
569
|
+
case "workflow.probe.error":
|
|
570
|
+
rejectProbe(frame.requestId, frame.error);
|
|
571
|
+
return;
|
|
572
|
+
default:
|
|
573
|
+
return assertNever(frame);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
// Authenticate a register/reconnect handshake exactly once, then run the
|
|
577
|
+
// frame's handler with the verified identity. The claimed `sidecarId` on
|
|
578
|
+
// the frame is an unauthenticated hint: it is logged if it disagrees with
|
|
579
|
+
// the verified id but never trusted -- routing keys off the verified id.
|
|
580
|
+
// Fails closed by closing the connection when the authenticator rejects
|
|
581
|
+
// (returns null) or throws (e.g. a database failure), so a handshake never
|
|
582
|
+
// proceeds on unverified credentials.
|
|
583
|
+
async function authenticateHandshake(ws, frame, run) {
|
|
584
|
+
let identity;
|
|
585
|
+
try {
|
|
586
|
+
identity = await authenticateSidecar({
|
|
587
|
+
sidecarId: frame.sidecarId,
|
|
588
|
+
token: frame.token,
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
catch (err) {
|
|
592
|
+
logger.error `Rejected ${frame.type} from claimed sidecar ${frame.sidecarId}: authenticator failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
593
|
+
ws.close();
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
if (identity === null) {
|
|
597
|
+
logger.warn `Rejected ${frame.type} from claimed sidecar ${frame.sidecarId}: invalid token`;
|
|
598
|
+
ws.close();
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
if (identity.sidecarId !== frame.sidecarId) {
|
|
602
|
+
logger.warn `Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${identity.sidecarId}; keying off the verified id`;
|
|
603
|
+
}
|
|
604
|
+
if (!(await validateSidecarIdentity(identity, "registration"))) {
|
|
605
|
+
logger.warn `Rejected ${frame.type} from sidecar ${identity.sidecarId}: credential identity is no longer current`;
|
|
606
|
+
ws.close();
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
await run(identity);
|
|
610
|
+
}
|
|
611
|
+
async function notifyAllocationWaiters(allocationId) {
|
|
612
|
+
const waiters = allocationWaiters.get(allocationId);
|
|
613
|
+
const current = allocatedConnections.get(allocationId);
|
|
614
|
+
if (waiters === undefined || current === undefined)
|
|
615
|
+
return;
|
|
616
|
+
if (!(await validateSidecarIdentity(current.identity, "readiness")))
|
|
617
|
+
return;
|
|
618
|
+
for (const waiter of [...waiters]) {
|
|
619
|
+
if (waiter.generation !== current.identity.generation)
|
|
620
|
+
continue;
|
|
621
|
+
clearTimeout(waiter.timer);
|
|
622
|
+
waiters.delete(waiter);
|
|
623
|
+
waiter.resolve();
|
|
624
|
+
}
|
|
625
|
+
if (waiters.size === 0)
|
|
626
|
+
allocationWaiters.delete(allocationId);
|
|
627
|
+
}
|
|
628
|
+
async function handleAllocatedRegister(ws, identity, agentAddresses) {
|
|
629
|
+
if (allocationFences.get(identity.allocationId) !== identity.generation) {
|
|
630
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`;
|
|
631
|
+
ws.close();
|
|
632
|
+
return;
|
|
633
|
+
}
|
|
634
|
+
if (agentAddresses.length > 0 &&
|
|
635
|
+
!(await validateSidecarIdentity(identity, "routing"))) {
|
|
636
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`;
|
|
637
|
+
ws.close();
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
const existingOnSocket = connections.get(ws);
|
|
641
|
+
const newlyRoutedAddresses = new Set();
|
|
642
|
+
for (const address of agentAddresses) {
|
|
643
|
+
const alreadyOwned = existingOnSocket?.identity.kind === "allocated" &&
|
|
644
|
+
existingOnSocket.identity.allocationId === identity.allocationId &&
|
|
645
|
+
addressIndex.get(address) === ws &&
|
|
646
|
+
connOwnsAddress(existingOnSocket, address);
|
|
647
|
+
if (!alreadyOwned && address !== identity.workflowRunAddress) {
|
|
648
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`;
|
|
649
|
+
ws.close();
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
if (!alreadyOwned)
|
|
653
|
+
newlyRoutedAddresses.add(address);
|
|
654
|
+
}
|
|
655
|
+
if (allocationFences.get(identity.allocationId) !== identity.generation) {
|
|
656
|
+
ws.close();
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
const current = allocatedConnections.get(identity.allocationId);
|
|
660
|
+
if (current !== undefined && current.ws !== ws) {
|
|
661
|
+
// A current-generation takeover is a reconnect, not a capacity loss.
|
|
662
|
+
// Remove the old socket from the allocation index before closing it so
|
|
663
|
+
// handleClose does not emit a false allocated-disconnect event.
|
|
664
|
+
allocatedConnections.delete(identity.allocationId);
|
|
665
|
+
handleClose(current.ws);
|
|
666
|
+
current.ws.close();
|
|
667
|
+
}
|
|
668
|
+
const conn = existingOnSocket ?? {
|
|
669
|
+
sidecarId: identity.sidecarId,
|
|
670
|
+
identity,
|
|
671
|
+
agentAddresses: new Set(),
|
|
672
|
+
workflowAddresses: new Set(),
|
|
673
|
+
send(frame) {
|
|
674
|
+
ws.send(JSON.stringify(frame));
|
|
675
|
+
},
|
|
676
|
+
};
|
|
677
|
+
if (conn.identity.kind !== "allocated" ||
|
|
678
|
+
conn.identity.allocationId !== identity.allocationId ||
|
|
679
|
+
conn.identity.generation !== identity.generation) {
|
|
680
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`;
|
|
681
|
+
ws.close();
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
684
|
+
connections.set(ws, conn);
|
|
685
|
+
for (const address of agentAddresses) {
|
|
686
|
+
conn.workflowAddresses.add(address);
|
|
687
|
+
addressIndex.set(address, ws);
|
|
688
|
+
}
|
|
689
|
+
allocatedConnections.set(identity.allocationId, { ws, identity });
|
|
690
|
+
for (const address of newlyRoutedAddresses) {
|
|
691
|
+
redeliverPendingMail(address, conn);
|
|
692
|
+
}
|
|
693
|
+
logger.info `Allocated sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
|
|
694
|
+
await notifyAllocationWaiters(identity.allocationId);
|
|
695
|
+
events.emit("sidecar.allocated.connected", {
|
|
696
|
+
allocationId: identity.allocationId,
|
|
697
|
+
generation: identity.generation,
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
async function handleRegister(ws, identity, agentAddresses) {
|
|
701
|
+
if (identity.kind === "allocated") {
|
|
702
|
+
await handleAllocatedRegister(ws, identity, agentAddresses);
|
|
703
|
+
return;
|
|
704
|
+
}
|
|
705
|
+
const sidecarId = identity.sidecarId;
|
|
706
|
+
// Key-existence gate. A register frame is token-authenticated but carries
|
|
707
|
+
// no per-address ownership proof, so it may route an address ONLY if that
|
|
708
|
+
// address has no stored key yet -- a genuine keyless first-deploy (the
|
|
709
|
+
// token-bounded first-deploy trust model). An address that already has a
|
|
710
|
+
// key must prove ownership through the challenged reconnect path; routing
|
|
711
|
+
// it here on token auth alone is the register-frame sibling of the
|
|
712
|
+
// reconnect hijack. The keyless-only set is computed up front, BEFORE the
|
|
713
|
+
// ghost-cleanup and every routing mutation below, so a rejected address
|
|
714
|
+
// touches nothing: no eviction of a live owner, hence no downgrade from
|
|
715
|
+
// hijack to denial-of-service on the victim.
|
|
716
|
+
const lookupKey = lookups.lookupPublicKey;
|
|
717
|
+
const routableAddresses = [];
|
|
718
|
+
for (const addr of agentAddresses) {
|
|
719
|
+
if (lookupKey === undefined) {
|
|
720
|
+
// Fail closed: without the ownership lookup a keyed address cannot be
|
|
721
|
+
// told apart from a first-deploy, so route nothing and surface the
|
|
722
|
+
// misconfiguration. Empty first-connect registers never reach here.
|
|
723
|
+
logger.error `Cannot gate register routing for ${addr}: lookupPublicKey is not configured; refusing to route (challenged reconnect required)`;
|
|
724
|
+
continue;
|
|
725
|
+
}
|
|
726
|
+
let existingKey;
|
|
727
|
+
try {
|
|
728
|
+
existingKey = await lookupKey(addr);
|
|
729
|
+
}
|
|
730
|
+
catch (err) {
|
|
731
|
+
// Fail closed on a lookup error (e.g. a transient DB failure): route
|
|
732
|
+
// nothing for this address and surface the failure, rather than let
|
|
733
|
+
// the rejection float out of this void-dispatched handler and take
|
|
734
|
+
// down the hub.
|
|
735
|
+
logger.error `Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`;
|
|
736
|
+
continue;
|
|
737
|
+
}
|
|
738
|
+
if (existingKey !== null) {
|
|
739
|
+
logger.warn `Refusing to route ${addr} via register: address already has a stored key; ownership must be proven via challenged reconnect`;
|
|
740
|
+
continue;
|
|
741
|
+
}
|
|
742
|
+
routableAddresses.push(addr);
|
|
743
|
+
}
|
|
744
|
+
// Additive re-register: inherit every address this ws already owns and ADD
|
|
745
|
+
// the frame's keyless first-deploys. Register never drops an owned route --
|
|
746
|
+
// an address proved via challenged reconnect stays routed, and removal
|
|
747
|
+
// happens via undeploy/disconnect, not register-omission.
|
|
748
|
+
//
|
|
749
|
+
// (Under the retired full-set model a register frame carried the sidecar's
|
|
750
|
+
// complete live set, so an omitted address meant "removed". Now the frame
|
|
751
|
+
// carries only keyless first-deploys the sidecar is adding -- keyed
|
|
752
|
+
// addresses arrive via reconnect -- so omission is meaningless, and a
|
|
753
|
+
// replace-on-register would wrongly drop an earlier first-deploy when a
|
|
754
|
+
// later one is registered, as well as any reconnect-verified keyed route.)
|
|
755
|
+
const existing = connections.get(ws);
|
|
756
|
+
const inheritedAgent = new Set(existing?.agentAddresses);
|
|
757
|
+
const inheritedWorkflow = new Set(existing?.workflowAddresses);
|
|
758
|
+
// Clean up ghost entries from OTHER connections for the newly-claimed
|
|
759
|
+
// addresses only. An inherited address is already owned by this ws
|
|
760
|
+
// (prevWs === ws), so it needs no eviction and its in-flight deploy must
|
|
761
|
+
// not be cancelled.
|
|
762
|
+
for (const addr of routableAddresses) {
|
|
763
|
+
const prevWs = addressIndex.get(addr);
|
|
764
|
+
if (prevWs !== undefined && prevWs !== ws) {
|
|
765
|
+
const prevConn = connections.get(prevWs);
|
|
766
|
+
if (prevConn !== undefined) {
|
|
767
|
+
prevConn.agentAddresses.delete(addr);
|
|
768
|
+
}
|
|
769
|
+
// The new owner is about to take over; the prior owner's
|
|
770
|
+
// cached state must not survive into the new owner's window
|
|
771
|
+
// before its bootstrap frame arrives.
|
|
772
|
+
connectorStates.delete(addr);
|
|
773
|
+
}
|
|
774
|
+
// Cancel any in-flight deploy for this address since the
|
|
775
|
+
// reconnecting sidecar is taking ownership.
|
|
776
|
+
const staleDeployReq = pendingDeploys.get(addr);
|
|
777
|
+
if (staleDeployReq !== undefined) {
|
|
778
|
+
clearTimeout(staleDeployReq.timer);
|
|
779
|
+
pendingDeploys.delete(addr);
|
|
780
|
+
staleDeployReq.reject(`Sidecar ${sidecarId} reconnected and claimed address "${addr}"`);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
// New keyless first-deploys join the inherited session set; the workflow
|
|
784
|
+
// set is inherited unchanged (register never adds to it -- a
|
|
785
|
+
// workflow-derived address routes only through the challenged reconnect).
|
|
786
|
+
for (const addr of routableAddresses) {
|
|
787
|
+
inheritedAgent.add(addr);
|
|
788
|
+
}
|
|
789
|
+
const addrSet = inheritedAgent;
|
|
790
|
+
const workflowSet = inheritedWorkflow;
|
|
791
|
+
const conn = {
|
|
792
|
+
sidecarId,
|
|
793
|
+
identity,
|
|
794
|
+
agentAddresses: addrSet,
|
|
795
|
+
workflowAddresses: workflowSet,
|
|
796
|
+
send(frame) {
|
|
797
|
+
ws.send(JSON.stringify(frame));
|
|
798
|
+
},
|
|
799
|
+
};
|
|
800
|
+
connections.set(ws, conn);
|
|
801
|
+
// Only the newly-claimed addresses need a routing write + queue discard;
|
|
802
|
+
// inherited addresses already point at this ws in addressIndex.
|
|
803
|
+
for (const addr of routableAddresses) {
|
|
804
|
+
addressIndex.set(addr, ws);
|
|
805
|
+
// Discard any disconnect queue — register has no identity
|
|
806
|
+
// verification, so flushing to an unverified connection is unsafe.
|
|
807
|
+
// Use reconnect with challenge/response to preserve queued messages.
|
|
808
|
+
const staleQueue = disconnectedAgents.get(addr);
|
|
809
|
+
if (staleQueue !== undefined) {
|
|
810
|
+
clearTimeout(staleQueue.timer);
|
|
811
|
+
if (staleQueue.queue.length > 0) {
|
|
812
|
+
logger.warn `Discarding ${String(staleQueue.queue.length)} queued message(s) for ${addr} on unverified register`;
|
|
813
|
+
}
|
|
814
|
+
disconnectedAgents.delete(addr);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
// Only keyless first-deploy addresses reach `addressIndex` here; an
|
|
818
|
+
// address that already has a stored key was filtered out by the gate
|
|
819
|
+
// above and must re-enter routing through the challenged reconnect path.
|
|
820
|
+
// Because the gate runs before the ghost-cleanup, a rejected (keyed)
|
|
821
|
+
// address never evicts its prior owner -- register cannot reclaim or
|
|
822
|
+
// disrupt a victim's route on token auth alone.
|
|
823
|
+
logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
|
|
824
|
+
}
|
|
825
|
+
async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
|
|
826
|
+
if (identity.kind === "allocated") {
|
|
827
|
+
await handleAllocatedRegister(ws, identity, agentAddresses);
|
|
828
|
+
return;
|
|
829
|
+
}
|
|
830
|
+
const sidecarId = identity.sidecarId;
|
|
831
|
+
const lookupKey = lookups.lookupPublicKey;
|
|
832
|
+
if (lookupKey === undefined) {
|
|
833
|
+
logger.error `Received reconnect frame but no lookupPublicKey is configured`;
|
|
834
|
+
ws.close();
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
// Cancel any existing pending challenge for this ws before doing
|
|
838
|
+
// async work, so concurrent reconnect frames don't race.
|
|
839
|
+
const existingChallenge = pendingChallenges.get(ws);
|
|
840
|
+
if (existingChallenge !== undefined) {
|
|
841
|
+
clearTimeout(existingChallenge.timer);
|
|
842
|
+
pendingChallenges.delete(ws);
|
|
843
|
+
}
|
|
844
|
+
// Capture the addresses this live connection already owns. The
|
|
845
|
+
// internal register below clears them, but ones the reconnect is not
|
|
846
|
+
// re-challenging are still valid — an agent provisioned during the
|
|
847
|
+
// sidecar's restore window is routed into addressIndex by its deploy
|
|
848
|
+
// and is not part of the disk-restored set this reconnect carries.
|
|
849
|
+
// Without preserving them, the reconnect silently drops a
|
|
850
|
+
// freshly-deployed agent from routing.
|
|
851
|
+
const previouslyOwned = new Set(connections.get(ws)?.agentAddresses);
|
|
852
|
+
// Register the sidecar connection immediately (with no addresses) so it
|
|
853
|
+
// can receive frames while the ownership challenge is pending. Every
|
|
854
|
+
// reconnect address -- session and workflow-derived alike -- enters
|
|
855
|
+
// routing only through the verified path below, never unchallenged here.
|
|
856
|
+
// Empty address list, so the key-existence gate has nothing to await.
|
|
857
|
+
// The identity was already verified at the dispatch boundary, so the
|
|
858
|
+
// internal register does not re-authenticate.
|
|
859
|
+
await handleRegister(ws, identity, []);
|
|
860
|
+
const conn = connections.get(ws);
|
|
861
|
+
if (conn === undefined)
|
|
862
|
+
return;
|
|
863
|
+
// Re-add the still-owned addresses the register cleared but this
|
|
864
|
+
// reconnect is not re-challenging. The challenged addresses below
|
|
865
|
+
// re-enter addressIndex through the verified path instead.
|
|
866
|
+
const claimedAddresses = new Set(agentAddresses);
|
|
867
|
+
for (const addr of previouslyOwned) {
|
|
868
|
+
if (claimedAddresses.has(addr))
|
|
869
|
+
continue;
|
|
870
|
+
conn.agentAddresses.add(addr);
|
|
871
|
+
addressIndex.set(addr, ws);
|
|
872
|
+
}
|
|
873
|
+
// Look up stored public keys for all claimed addresses. Fail closed on a
|
|
874
|
+
// lookup error (e.g. a transient DB failure): treat the address as
|
|
875
|
+
// unverifiable so it fails its challenge and stays unrouted, rather than
|
|
876
|
+
// letting the rejection float out of this void-dispatched handler as an
|
|
877
|
+
// unhandled rejection that could take down the hub.
|
|
878
|
+
const keyLookups = await Promise.all(agentAddresses.map(async (addr) => {
|
|
879
|
+
try {
|
|
880
|
+
return { address: addr, publicKeyHex: await lookupKey(addr) };
|
|
881
|
+
}
|
|
882
|
+
catch (err) {
|
|
883
|
+
logger.error `Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`;
|
|
884
|
+
return { address: addr, publicKeyHex: null };
|
|
885
|
+
}
|
|
886
|
+
}));
|
|
887
|
+
// If the connection was closed or superseded while we were awaiting
|
|
888
|
+
// key lookups, bail out.
|
|
889
|
+
if (!connections.has(ws))
|
|
890
|
+
return;
|
|
891
|
+
const challenges = new Map();
|
|
892
|
+
const challengeEntries = [];
|
|
893
|
+
for (const { address, publicKeyHex } of keyLookups) {
|
|
894
|
+
if (publicKeyHex === null) {
|
|
895
|
+
conn.send({
|
|
896
|
+
type: "challenge.failed",
|
|
897
|
+
address,
|
|
898
|
+
reason: "Unknown run address",
|
|
899
|
+
});
|
|
900
|
+
continue;
|
|
901
|
+
}
|
|
902
|
+
let publicKey;
|
|
903
|
+
try {
|
|
904
|
+
publicKey = hexDecode(publicKeyHex);
|
|
905
|
+
}
|
|
906
|
+
catch {
|
|
907
|
+
conn.send({
|
|
908
|
+
type: "challenge.failed",
|
|
909
|
+
address,
|
|
910
|
+
reason: "Stored public key is corrupt",
|
|
911
|
+
});
|
|
912
|
+
logger.error `Corrupt stored public key for ${address}`;
|
|
913
|
+
continue;
|
|
914
|
+
}
|
|
915
|
+
const nonce = crypto.getRandomValues(new Uint8Array(32));
|
|
916
|
+
challenges.set(address, { nonce, publicKey });
|
|
917
|
+
challengeEntries.push({ address, nonce: hexEncode(nonce) });
|
|
918
|
+
}
|
|
919
|
+
if (challenges.size === 0)
|
|
920
|
+
return;
|
|
921
|
+
// If another reconnect completed while we were building the
|
|
922
|
+
// challenge, it will have written its own entry. Don't overwrite.
|
|
923
|
+
if (pendingChallenges.has(ws))
|
|
924
|
+
return;
|
|
925
|
+
const timer = setTimeout(() => {
|
|
926
|
+
pendingChallenges.delete(ws);
|
|
927
|
+
logger.warn `Challenge timed out for sidecar ${sidecarId}`;
|
|
928
|
+
}, challengeTimeoutMs);
|
|
929
|
+
pendingChallenges.set(ws, {
|
|
930
|
+
sidecarId,
|
|
931
|
+
challenges,
|
|
932
|
+
deployRefs,
|
|
933
|
+
timer,
|
|
934
|
+
});
|
|
935
|
+
conn.send({ type: "challenge", challenges: challengeEntries });
|
|
936
|
+
}
|
|
937
|
+
async function handleChallengeResponse(ws, responses) {
|
|
938
|
+
const challenge = pendingChallenges.get(ws);
|
|
939
|
+
if (challenge === undefined) {
|
|
940
|
+
logger.warn `Received challenge.response with no pending challenge`;
|
|
941
|
+
return;
|
|
942
|
+
}
|
|
943
|
+
clearTimeout(challenge.timer);
|
|
944
|
+
pendingChallenges.delete(ws);
|
|
945
|
+
const conn = connections.get(ws);
|
|
946
|
+
if (conn === undefined)
|
|
947
|
+
return;
|
|
948
|
+
const verified = [];
|
|
949
|
+
const responded = new Set();
|
|
950
|
+
for (const { address, signature } of responses) {
|
|
951
|
+
responded.add(address);
|
|
952
|
+
const entry = challenge.challenges.get(address);
|
|
953
|
+
if (entry === undefined) {
|
|
954
|
+
conn.send({
|
|
955
|
+
type: "challenge.failed",
|
|
956
|
+
address,
|
|
957
|
+
reason: "Address was not challenged",
|
|
958
|
+
});
|
|
959
|
+
continue;
|
|
960
|
+
}
|
|
961
|
+
let valid = false;
|
|
962
|
+
try {
|
|
963
|
+
const nonceBytes = entry.nonce;
|
|
964
|
+
const addressBytes = new TextEncoder().encode(address);
|
|
965
|
+
const payload = new Uint8Array(nonceBytes.length + addressBytes.length);
|
|
966
|
+
payload.set(nonceBytes);
|
|
967
|
+
payload.set(addressBytes, nonceBytes.length);
|
|
968
|
+
const sigBytes = hexDecode(signature);
|
|
969
|
+
valid = await verifyEd25519(payload, sigBytes, entry.publicKey);
|
|
970
|
+
}
|
|
971
|
+
catch (err) {
|
|
972
|
+
logger.warn `Challenge failed for ${address}: ${err instanceof Error ? err.message : String(err)}`;
|
|
973
|
+
}
|
|
974
|
+
if (valid) {
|
|
975
|
+
verified.push(address);
|
|
976
|
+
}
|
|
977
|
+
else {
|
|
978
|
+
conn.send({
|
|
979
|
+
type: "challenge.failed",
|
|
980
|
+
address,
|
|
981
|
+
reason: "Signature verification failed",
|
|
982
|
+
});
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
// Notify about challenged addresses that were omitted from the response.
|
|
986
|
+
for (const address of challenge.challenges.keys()) {
|
|
987
|
+
if (!responded.has(address)) {
|
|
988
|
+
conn.send({
|
|
989
|
+
type: "challenge.failed",
|
|
990
|
+
address,
|
|
991
|
+
reason: "No response provided for challenged address",
|
|
992
|
+
});
|
|
993
|
+
logger.warn `Challenge failed for ${address}: no response provided`;
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
// Add verified addresses to the routing table immediately so that
|
|
997
|
+
// `agent.reconnected` subscribers can use sendRequest-based methods
|
|
998
|
+
// (e.g. sendSourcesUpdate). Addresses that fail governance are
|
|
999
|
+
// rolled back from the routing table afterward.
|
|
1000
|
+
for (const addr of verified) {
|
|
1001
|
+
// If a different ws still owns this address (live takeover via
|
|
1002
|
+
// verified reconnect), evict its cached connector state before
|
|
1003
|
+
// routing flips. The new owner's harness will bootstrap via
|
|
1004
|
+
// restore-fires-callback.
|
|
1005
|
+
const prevWs = addressIndex.get(addr);
|
|
1006
|
+
if (prevWs !== undefined && prevWs !== ws) {
|
|
1007
|
+
connectorStates.delete(addr);
|
|
1008
|
+
// Evict the reclaimed address from the superseded connection's owned
|
|
1009
|
+
// set. handleClose's cancelByAgent sweep iterates a connection's owned
|
|
1010
|
+
// union WITHOUT an ownership guard, so if the stale connection still
|
|
1011
|
+
// listed this address it would cancel THIS connection's in-flight pack
|
|
1012
|
+
// transfer for it when it finally closes. Delete from both sets: a
|
|
1013
|
+
// workflow-derived address lives on the workflow set, a launched agent
|
|
1014
|
+
// on the session set, and delete is a no-op for the absent one.
|
|
1015
|
+
const prevConn = connections.get(prevWs);
|
|
1016
|
+
if (prevConn !== undefined) {
|
|
1017
|
+
prevConn.workflowAddresses.delete(addr);
|
|
1018
|
+
prevConn.agentAddresses.delete(addr);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
// A run address goes on the workflow set (no disconnect queue -- its
|
|
1022
|
+
// in-flight state is reconstructed sidecar-locally on the next reconnect),
|
|
1023
|
+
// so handleClose reclaims it correctly. The routing pointer is the same
|
|
1024
|
+
// either way; only now it is written behind a passed challenge.
|
|
1025
|
+
//
|
|
1026
|
+
// The `else` (session set, queued for reconnect) is the retired
|
|
1027
|
+
// launched-agent path: launched agents no longer exist (the folded-launch
|
|
1028
|
+
// route was removed), so no current producer reaches it. It is left in
|
|
1029
|
+
// place for the reconnect-subsystem teardown to remove as its own
|
|
1030
|
+
// reviewable change, not folded into this collapse.
|
|
1031
|
+
if (isRunAddress(addr)) {
|
|
1032
|
+
conn.workflowAddresses.add(addr);
|
|
1033
|
+
}
|
|
1034
|
+
else {
|
|
1035
|
+
conn.agentAddresses.add(addr);
|
|
1036
|
+
}
|
|
1037
|
+
addressIndex.set(addr, ws);
|
|
1038
|
+
}
|
|
1039
|
+
const ready = [];
|
|
1040
|
+
const failed = [];
|
|
1041
|
+
for (const addr of verified) {
|
|
1042
|
+
// A workflow run needs routing + queue flush only, which the passed
|
|
1043
|
+
// challenge has now made safe: its in-flight state is reconstructed
|
|
1044
|
+
// sidecar-locally, so it does not enrol in the `agent.reconnected` session
|
|
1045
|
+
// reaction (event-collector restore). Skip the reaction for it.
|
|
1046
|
+
//
|
|
1047
|
+
// The reaction below was the retired launched-agent path (restore a
|
|
1048
|
+
// launched agent's collector). Launched agents no longer exist, so no
|
|
1049
|
+
// current producer reaches it; the `agent.reconnected` listener is gone,
|
|
1050
|
+
// so `listenerCount` is 0 and the emit is inert. Left for the
|
|
1051
|
+
// reconnect-subsystem teardown to remove.
|
|
1052
|
+
if (isRunAddress(addr)) {
|
|
1053
|
+
ready.push(addr);
|
|
1054
|
+
continue;
|
|
1055
|
+
}
|
|
1056
|
+
if (events.listenerCount("agent.reconnected") === 0) {
|
|
1057
|
+
ready.push(addr);
|
|
1058
|
+
continue;
|
|
1059
|
+
}
|
|
1060
|
+
try {
|
|
1061
|
+
await events.emitAndAwait("agent.reconnected", { agentAddress: addr });
|
|
1062
|
+
ready.push(addr);
|
|
1063
|
+
}
|
|
1064
|
+
catch (err) {
|
|
1065
|
+
logger.error `Failed to handle reconnection for ${addr}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1066
|
+
failed.push(addr);
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
// If a second reconnect arrived during the callback loop, our conn
|
|
1070
|
+
// is orphaned — handleRegister already rebuilt the connection and
|
|
1071
|
+
// cleared addressIndex. Bail out; the new reconnect flow will
|
|
1072
|
+
// re-verify these addresses from scratch.
|
|
1073
|
+
if (connections.get(ws) !== conn) {
|
|
1074
|
+
logger.warn `Challenge response processing aborted: connection superseded by new reconnect`;
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
// Roll back failed addresses from the routing table. Only the session set
|
|
1078
|
+
// is touched: a workflow-derived address can never be in `failed` -- it
|
|
1079
|
+
// early-`continue`s to `ready` above, before the reaction that populates
|
|
1080
|
+
// `failed` -- so it is never on the workflow set at this point.
|
|
1081
|
+
for (const addr of failed) {
|
|
1082
|
+
conn.agentAddresses.delete(addr);
|
|
1083
|
+
addressIndex.delete(addr);
|
|
1084
|
+
}
|
|
1085
|
+
// Flush queued messages and redeliver retained un-acked mail, only for
|
|
1086
|
+
// ready addresses. The disconnect queue carries mail that arrived WHILE
|
|
1087
|
+
// disconnected; pending-mail redelivery carries mail that was delivered
|
|
1088
|
+
// over the prior live connection but never acked (the connected-window
|
|
1089
|
+
// drop). Both replay over the verified new connection.
|
|
1090
|
+
for (const addr of ready) {
|
|
1091
|
+
flushDisconnectedQueue(addr, conn);
|
|
1092
|
+
redeliverPendingMail(addr, conn);
|
|
1093
|
+
}
|
|
1094
|
+
// Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
|
|
1095
|
+
// so reconnect completion is not blocked on pack transfer. The
|
|
1096
|
+
// wire layer owns the staleness comparison; the event fires only
|
|
1097
|
+
// when staleness is confirmed.
|
|
1098
|
+
const checkDeployRef = lookups.lookupDeployRef;
|
|
1099
|
+
if (checkDeployRef !== undefined) {
|
|
1100
|
+
for (const addr of ready) {
|
|
1101
|
+
// Workflow deployments are pinned-forever: a deployment keeps its
|
|
1102
|
+
// deploy-time definition until an explicit undeploy/redeploy, so the
|
|
1103
|
+
// deploy-ref freshness catch-up is deliberately NOT run for a run
|
|
1104
|
+
// address. A definition edited on the hub while the sidecar was
|
|
1105
|
+
// disconnected does not reconcile on reconnect; it affects only newly
|
|
1106
|
+
// created deployments. The deployment's in-flight run state is
|
|
1107
|
+
// reconstructed sidecar-locally at restore, not re-fetched here. Do NOT
|
|
1108
|
+
// add a reconcile path for these addresses -- see the "Workflow
|
|
1109
|
+
// Definition Versioning: Pinned-Forever" note under "Reconnect
|
|
1110
|
+
// Sequencing" in docs/IMPLEMENTATION.md. (Every routable address is a
|
|
1111
|
+
// run address now; the fall-through was the retired launched-agent
|
|
1112
|
+
// catch-up path.)
|
|
1113
|
+
if (isRunAddress(addr))
|
|
1114
|
+
continue;
|
|
1115
|
+
void (async () => {
|
|
1116
|
+
try {
|
|
1117
|
+
const hubRef = await checkDeployRef(addr);
|
|
1118
|
+
if (hubRef === null)
|
|
1119
|
+
return;
|
|
1120
|
+
const sidecarRef = challenge.deployRefs[addr];
|
|
1121
|
+
if (sidecarRef === hubRef)
|
|
1122
|
+
return;
|
|
1123
|
+
logger.info `Re-deploying ${addr}: sidecar ref ${sidecarRef ?? "(none)"} != hub ref ${hubRef.slice(0, 8)}`;
|
|
1124
|
+
await events.emitAndAwait("deploy.ref.stale", {
|
|
1125
|
+
agentAddress: addr,
|
|
1126
|
+
});
|
|
1127
|
+
}
|
|
1128
|
+
catch (err) {
|
|
1129
|
+
logger.error `Failed to re-deploy ${addr} after reconnect: ${err instanceof Error ? err.message : String(err)}`;
|
|
1130
|
+
}
|
|
1131
|
+
})();
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
// Failed addresses: reset their queue TTL so messages survive until
|
|
1135
|
+
// the next reconnect attempt, and notify the sidecar.
|
|
1136
|
+
for (const addr of failed) {
|
|
1137
|
+
const entry = disconnectedAgents.get(addr);
|
|
1138
|
+
if (entry !== undefined) {
|
|
1139
|
+
clearTimeout(entry.timer);
|
|
1140
|
+
entry.timer = setTimeout(() => {
|
|
1141
|
+
const expired = disconnectedAgents.get(addr);
|
|
1142
|
+
disconnectedAgents.delete(addr);
|
|
1143
|
+
if (expired !== undefined) {
|
|
1144
|
+
surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
|
|
1145
|
+
}
|
|
1146
|
+
}, disconnectQueueTTLMs);
|
|
1147
|
+
}
|
|
1148
|
+
conn.send({
|
|
1149
|
+
type: "challenge.failed",
|
|
1150
|
+
address: addr,
|
|
1151
|
+
reason: "Reconnection rejected by governance",
|
|
1152
|
+
});
|
|
1153
|
+
}
|
|
1154
|
+
logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
|
|
1155
|
+
}
|
|
1156
|
+
async function handleMailOutbound(rawMessage, recipients) {
|
|
1157
|
+
// A mail addressed to more than one workflow deployment would birth a
|
|
1158
|
+
// run per recipient from a single inbound mail. The stable runId
|
|
1159
|
+
// removed the Message-ID collision that originally forced this guard --
|
|
1160
|
+
// each recipient now derives its own per-deployment runId (its mail
|
|
1161
|
+
// address), so it is no longer a runId-collision guard. It stays a
|
|
1162
|
+
// deliberate one-workflow-recipient-per-mail restriction because the
|
|
1163
|
+
// fan-out is not verified end-to-end: per-recipient grants
|
|
1164
|
+
// materialization, consumed-tracking, and reply-addressing all assume a
|
|
1165
|
+
// single workflow recipient today. Lifting it means proving those three
|
|
1166
|
+
// hold per recipient, not just relaxing this check -- so fail loudly
|
|
1167
|
+
// rather than materialize a partial set. The guard only applies when a
|
|
1168
|
+
// materializer is wired -- absent one, no run is born from the mail, so
|
|
1169
|
+
// there is nothing to restrict.
|
|
1170
|
+
if (lookups.materializeMailTriggeredRunGrants !== undefined) {
|
|
1171
|
+
// A workflow recipient is one this hub owns: its address parses as a run
|
|
1172
|
+
// address. An external/federated address does not, and is not ours to
|
|
1173
|
+
// materialize a run for.
|
|
1174
|
+
const workflowRecipients = recipients.filter(isRunAddress);
|
|
1175
|
+
if (workflowRecipients.length > 1) {
|
|
1176
|
+
throw new Error(`mail addressed to multiple workflow-derived recipients (${workflowRecipients.join(", ")}); materializing a run for more than one workflow deployment from a single mail is unsupported`);
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
// Route to locally connected sidecars first, then try disconnect queues.
|
|
1180
|
+
const unrouted = [];
|
|
1181
|
+
for (const recipient of recipients) {
|
|
1182
|
+
// Each recipient is isolated: a materialization failure or a
|
|
1183
|
+
// fail-closed rejection for one must not drop the mail for its
|
|
1184
|
+
// co-recipients. The catch fails THIS recipient closed (its run never
|
|
1185
|
+
// starts under-authorized) and continues to the rest.
|
|
1186
|
+
try {
|
|
1187
|
+
const outcome = await deliverMailToRecipient(recipient, rawMessage);
|
|
1188
|
+
if (outcome === "unrouted")
|
|
1189
|
+
unrouted.push(recipient);
|
|
1190
|
+
}
|
|
1191
|
+
catch (err) {
|
|
1192
|
+
logger.error `Failed to deliver mail to ${recipient}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
// Anything not routed locally is emitted as a notification. The
|
|
1196
|
+
// host decides whether to relay onto an external transport, log,
|
|
1197
|
+
// or drop. The wire layer takes no stance.
|
|
1198
|
+
if (unrouted.length > 0) {
|
|
1199
|
+
events.emit("mail.outbound.undelivered", {
|
|
1200
|
+
rawMessage,
|
|
1201
|
+
recipients: unrouted,
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
// Deliver an inbound mail to one recipient, materializing a
|
|
1206
|
+
// mail-triggered run's grants first when the recipient is a workflow
|
|
1207
|
+
// deployment. Returns:
|
|
1208
|
+
// - `routed`: the mail reached a live connection or disconnect queue.
|
|
1209
|
+
// - `unrouted`: the mail was locally undeliverable and should be
|
|
1210
|
+
// relayed externally by the host.
|
|
1211
|
+
// - `failed-closed`: the run's grants could not be materialized safely,
|
|
1212
|
+
// so the mail is deliberately DROPPED for this recipient (not relayed)
|
|
1213
|
+
// to keep its run from starting under-authorized.
|
|
1214
|
+
//
|
|
1215
|
+
// A workflow deployment is the only recipient whose inbound mail can first
|
|
1216
|
+
// fire its stable run. Its grants are reserved, and the `run.grants` frame is
|
|
1217
|
+
// sent BEFORE the mail. Same-address FIFO guarantees it lands ahead of the
|
|
1218
|
+
// mail that dispatches the run, so the run's `onRunStart` barrier resolves its
|
|
1219
|
+
// grants rather than failing closed. Reservation happens before routing so
|
|
1220
|
+
// concurrent first deliveries cannot send different snapshots; a routing
|
|
1221
|
+
// failure leaves a grants-only, still-unfired run.
|
|
1222
|
+
async function deliverMailToRecipient(recipient, rawMessage) {
|
|
1223
|
+
if (lookups.materializeMailTriggeredRunGrants !== undefined &&
|
|
1224
|
+
isRunAddress(recipient)) {
|
|
1225
|
+
const runId = deriveWorkflowRunId(recipient);
|
|
1226
|
+
const result = await lookups.materializeMailTriggeredRunGrants({
|
|
1227
|
+
agentAddress: recipient,
|
|
1228
|
+
runId,
|
|
1229
|
+
});
|
|
1230
|
+
if (result.outcome === "rejected") {
|
|
1231
|
+
// The run's grants could not be materialized with sufficient
|
|
1232
|
+
// authority or it is already terminal. Fail the mail closed for this
|
|
1233
|
+
// recipient: routing or external relay would bypass that decision.
|
|
1234
|
+
logger.error `Refusing mail-triggered run ${runId} for ${recipient}: grant materialization rejected (${result.code}): ${result.message}`;
|
|
1235
|
+
return "failed-closed";
|
|
1236
|
+
}
|
|
1237
|
+
if (result.outcome === "materialized") {
|
|
1238
|
+
// Send the run's grants ahead of the mail. A `false` here means the
|
|
1239
|
+
// deployment is unroutable. Do not route the mail that would dispatch
|
|
1240
|
+
// it; the grants-only reservation remains the canonical snapshot for a
|
|
1241
|
+
// later first-delivery attempt.
|
|
1242
|
+
if (!sendRunGrants(recipient, runId, result.stepGrants)) {
|
|
1243
|
+
logger.error `Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`;
|
|
1244
|
+
return "unrouted";
|
|
1245
|
+
}
|
|
1246
|
+
// Route through the messageId handshake `routeMail` -- NOT a
|
|
1247
|
+
// fire-and-forget send. This branch COMMITS a run, so a mail dropped in
|
|
1248
|
+
// the connected window (a socket that half-dies before the sidecar's
|
|
1249
|
+
// durable-write ack) would otherwise leave the run row "running"
|
|
1250
|
+
// forever with no body and no error. `routeMail` tracks the delivery
|
|
1251
|
+
// and redelivers identical bytes on reconnect, bringing the mail-relay
|
|
1252
|
+
// run-trigger to parity with the HTTP-trigger path. The messageId is the
|
|
1253
|
+
// mail's own id (derived over the same bytes the sidecar derives), so a
|
|
1254
|
+
// redelivery replays identically and the downstream RunStarted /
|
|
1255
|
+
// stable-runId dedup makes it effectively-once.
|
|
1256
|
+
const messageId = await deriveMessageId(base64Decode(rawMessage));
|
|
1257
|
+
const outcome = routeMail(recipient, rawMessage, messageId, { runId, stepGrants: result.stepGrants })
|
|
1258
|
+
? "routed"
|
|
1259
|
+
: "unrouted";
|
|
1260
|
+
return outcome;
|
|
1261
|
+
}
|
|
1262
|
+
// `skip`: the address named no deployed workflow deployment. Forward
|
|
1263
|
+
// the mail without grants -- the run, if any, is not ours to
|
|
1264
|
+
// authorize. No run is committed here, so no ack handshake is needed.
|
|
1265
|
+
}
|
|
1266
|
+
return routeMail(recipient, rawMessage) ? "routed" : "unrouted";
|
|
1267
|
+
}
|
|
1268
|
+
async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
|
|
1269
|
+
let results;
|
|
1270
|
+
let raw;
|
|
1271
|
+
try {
|
|
1272
|
+
raw = Uint8Array.from(atob(rawMessage), (c) => c.charCodeAt(0));
|
|
1273
|
+
results = await persist({
|
|
1274
|
+
senderAddress,
|
|
1275
|
+
recipients,
|
|
1276
|
+
raw,
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
catch (err) {
|
|
1280
|
+
logger.error `Failed to persist mail from ${senderAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1281
|
+
return;
|
|
1282
|
+
}
|
|
1283
|
+
for (const result of results) {
|
|
1284
|
+
events.emit("mail.persisted", {
|
|
1285
|
+
id: result.id,
|
|
1286
|
+
raw,
|
|
1287
|
+
createdAt: result.createdAt,
|
|
1288
|
+
direction: result.direction,
|
|
1289
|
+
runId: result.runId,
|
|
1290
|
+
address: result.address,
|
|
1291
|
+
});
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
async function handleSignalCorrelationRegister(ws, frame) {
|
|
1295
|
+
// Gate the co-write on the sending sidecar actually owning the named
|
|
1296
|
+
// deployment address, mirroring the connector.state.changed and pack
|
|
1297
|
+
// handlers. A workflow deployment routes on the keyless workflow set, so
|
|
1298
|
+
// ownership is the union check, not addressIndex identity alone. Without
|
|
1299
|
+
// it a misbehaving sidecar that knows another deployment's address could
|
|
1300
|
+
// register a spurious correlation against it.
|
|
1301
|
+
const conn = connections.get(ws);
|
|
1302
|
+
if (conn === undefined)
|
|
1303
|
+
return;
|
|
1304
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
1305
|
+
logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: not registered to this sidecar`;
|
|
1306
|
+
return;
|
|
1307
|
+
}
|
|
1308
|
+
const register = lookups.registerSignalCorrelation;
|
|
1309
|
+
if (register === undefined) {
|
|
1310
|
+
logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: no registerSignalCorrelation lookup configured`;
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
try {
|
|
1314
|
+
await register({
|
|
1315
|
+
correlationId: frame.correlationId,
|
|
1316
|
+
runId: frame.runId,
|
|
1317
|
+
anchorRunId: frame.anchorRunId,
|
|
1318
|
+
agentAddress: frame.agentAddress,
|
|
1319
|
+
kind: frame.kind,
|
|
1320
|
+
approvalSnapshot: frame.snapshot,
|
|
1321
|
+
});
|
|
1322
|
+
// The co-write resolves only when a row exists -- freshly inserted or
|
|
1323
|
+
// already present (both stores are idempotent on the correlationId). Ack
|
|
1324
|
+
// so the sidecar's link stops retrying a register whose frame may have
|
|
1325
|
+
// been lost on an open socket. A thrown co-write (undeployed deployment,
|
|
1326
|
+
// id mismatch) means no row, so no ack: the sidecar keeps retrying and
|
|
1327
|
+
// the reconnect re-emit remains the ultimate backstop.
|
|
1328
|
+
conn.send({
|
|
1329
|
+
type: "signal.correlation.register.ack",
|
|
1330
|
+
agentAddress: frame.agentAddress,
|
|
1331
|
+
correlationId: frame.correlationId,
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
catch (err) {
|
|
1335
|
+
logger.error `Failed to register signal correlation ${frame.correlationId} for ${frame.agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
function handleClose(ws) {
|
|
1339
|
+
const conn = connections.get(ws);
|
|
1340
|
+
if (conn === undefined)
|
|
1341
|
+
return;
|
|
1342
|
+
let allocated;
|
|
1343
|
+
for (const addr of conn.agentAddresses) {
|
|
1344
|
+
// Only remove routing and pending state if this connection still
|
|
1345
|
+
// owns the address. A reconnected sidecar may have already claimed it.
|
|
1346
|
+
if (addressIndex.get(addr) === ws) {
|
|
1347
|
+
addressIndex.delete(addr);
|
|
1348
|
+
// Drop cached connector state for the same reason: a takeover
|
|
1349
|
+
// sidecar's state lives in connectorStates under the same key,
|
|
1350
|
+
// and only this owner's close should evict it. The next
|
|
1351
|
+
// reconnect re-bootstraps via the router's
|
|
1352
|
+
// restore-fires-callback path.
|
|
1353
|
+
connectorStates.delete(addr);
|
|
1354
|
+
// Retain this address's un-acked pending mail across the disconnect:
|
|
1355
|
+
// its in-flight retry timers target a dead socket (cleared), but the
|
|
1356
|
+
// entries are held so a verified reconnect redelivers them, closing the
|
|
1357
|
+
// connected-window drop rather than losing the mail. Bounded by a
|
|
1358
|
+
// retention TTL.
|
|
1359
|
+
retainPendingMailForAddress(addr);
|
|
1360
|
+
// Create a queue entry so messages can accumulate while the
|
|
1361
|
+
// sidecar is disconnected. Skip if the agent is being undeployed --
|
|
1362
|
+
// there is no point queuing messages for an agent being torn down.
|
|
1363
|
+
if (!pendingUndeploys.has(addr)) {
|
|
1364
|
+
const timer = setTimeout(() => {
|
|
1365
|
+
const expired = disconnectedAgents.get(addr);
|
|
1366
|
+
disconnectedAgents.delete(addr);
|
|
1367
|
+
if (expired !== undefined) {
|
|
1368
|
+
surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
|
|
1369
|
+
}
|
|
1370
|
+
}, disconnectQueueTTLMs);
|
|
1371
|
+
disconnectedAgents.set(addr, { queue: [], timer });
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
// Remove this connection's workflow-substrate routes. No disconnect queue
|
|
1376
|
+
// is created: these addresses re-register (with the complete live set)
|
|
1377
|
+
// when the sidecar reconnects, and their in-flight run state is
|
|
1378
|
+
// reconstructed sidecar-locally, not from a hub-side queue. The ownership
|
|
1379
|
+
// guard mirrors the session loop above so a takeover by a newer ws is not
|
|
1380
|
+
// clobbered by the prior owner's close.
|
|
1381
|
+
for (const addr of conn.workflowAddresses) {
|
|
1382
|
+
if (addressIndex.get(addr) === ws) {
|
|
1383
|
+
addressIndex.delete(addr);
|
|
1384
|
+
// Retain un-acked workflow trigger mail across the disconnect for the
|
|
1385
|
+
// same reason as the session loop above -- a challenged reconnect
|
|
1386
|
+
// redelivers it. This is un-acked TRIGGER mail, distinct from the
|
|
1387
|
+
// deployment's in-flight run state (reconstructed sidecar-locally); the
|
|
1388
|
+
// "no disconnect queue" note above is about that run state, not this.
|
|
1389
|
+
retainPendingMailForAddress(addr);
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
if (conn.identity.kind === "allocated") {
|
|
1393
|
+
const current = allocatedConnections.get(conn.identity.allocationId);
|
|
1394
|
+
if (current?.ws === ws) {
|
|
1395
|
+
allocatedConnections.delete(conn.identity.allocationId);
|
|
1396
|
+
allocated = {
|
|
1397
|
+
allocationId: conn.identity.allocationId,
|
|
1398
|
+
generation: conn.identity.generation,
|
|
1399
|
+
};
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
connections.delete(ws);
|
|
1403
|
+
// Cancel the liveness timer for this connection.
|
|
1404
|
+
const livenessTimer = livenessTimers.get(ws);
|
|
1405
|
+
if (livenessTimer !== undefined) {
|
|
1406
|
+
clearTimeout(livenessTimer);
|
|
1407
|
+
livenessTimers.delete(ws);
|
|
1408
|
+
}
|
|
1409
|
+
// Drop the per-ws serialization chain; no more frames will queue on it.
|
|
1410
|
+
messageChains.delete(ws);
|
|
1411
|
+
// Cancel any pending challenge for this connection.
|
|
1412
|
+
const challengeReq = pendingChallenges.get(ws);
|
|
1413
|
+
if (challengeReq !== undefined) {
|
|
1414
|
+
clearTimeout(challengeReq.timer);
|
|
1415
|
+
pendingChallenges.delete(ws);
|
|
1416
|
+
}
|
|
1417
|
+
// Reject any in-flight requests that were sent to this sidecar.
|
|
1418
|
+
for (const [requestId, req] of pending) {
|
|
1419
|
+
if (req.ws !== ws)
|
|
1420
|
+
continue;
|
|
1421
|
+
clearTimeout(req.timer);
|
|
1422
|
+
pending.delete(requestId);
|
|
1423
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1424
|
+
}
|
|
1425
|
+
// Reject every deploy issued on this socket, including allocated
|
|
1426
|
+
// workflow deployments stored in `workflowAddresses` rather than
|
|
1427
|
+
// `agentAddresses`.
|
|
1428
|
+
for (const [agentAddress, req] of pendingDeploys) {
|
|
1429
|
+
if (req.ws !== ws)
|
|
1430
|
+
continue;
|
|
1431
|
+
clearTimeout(req.timer);
|
|
1432
|
+
pendingDeploys.delete(agentAddress);
|
|
1433
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1434
|
+
}
|
|
1435
|
+
// Reject any in-flight pack transfers for this sidecar.
|
|
1436
|
+
for (const [transferId, pack] of pendingPacks) {
|
|
1437
|
+
if (pack.ws !== ws)
|
|
1438
|
+
continue;
|
|
1439
|
+
clearTimeout(pack.timer);
|
|
1440
|
+
pendingPacks.delete(transferId);
|
|
1441
|
+
pack.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1442
|
+
}
|
|
1443
|
+
// Reject any in-flight undeploys for this sidecar.
|
|
1444
|
+
for (const [addr, req] of pendingUndeploys) {
|
|
1445
|
+
if (req.ws !== ws)
|
|
1446
|
+
continue;
|
|
1447
|
+
clearTimeout(req.timer);
|
|
1448
|
+
pendingUndeploys.delete(addr);
|
|
1449
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1450
|
+
}
|
|
1451
|
+
// Reject any in-flight probes sent to this sidecar. A probe never enters
|
|
1452
|
+
// the address maps, so this ws-keyed sweep is its ONLY disconnect cleanup:
|
|
1453
|
+
// without it a probe whose sidecar drops mid-flight would hang until its
|
|
1454
|
+
// own timeout instead of failing fast on the disconnect.
|
|
1455
|
+
for (const [requestId, probe] of pendingProbes) {
|
|
1456
|
+
if (probe.ws !== ws)
|
|
1457
|
+
continue;
|
|
1458
|
+
clearTimeout(probe.timer);
|
|
1459
|
+
pendingProbes.delete(requestId);
|
|
1460
|
+
probe.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1461
|
+
}
|
|
1462
|
+
// Cancel any in-flight inbound pack transfers from this sidecar
|
|
1463
|
+
// across both receivers. The two receivers track their own in-
|
|
1464
|
+
// flight transferIds, so a pending workflow-run transfer for an
|
|
1465
|
+
// agent that just disconnected won't outlive the connection just
|
|
1466
|
+
// because the agent-state receiver has nothing to cancel. Iterate the
|
|
1467
|
+
// owned union so a reconnected workflow deployment's transfer is
|
|
1468
|
+
// cancelled too; the deduped set avoids a double-cancel for an address
|
|
1469
|
+
// that is in both sets. A reclaimed address is not present here -- the
|
|
1470
|
+
// verified reconnect path that took it over evicts it from this
|
|
1471
|
+
// (superseded) connection's owned set -- so a stale close does not cancel
|
|
1472
|
+
// the new owner's work.
|
|
1473
|
+
const owned = ownedAddresses(conn);
|
|
1474
|
+
for (const addr of owned) {
|
|
1475
|
+
agentStatePackReceiver.cancelByAgent(addr);
|
|
1476
|
+
workflowRunPackReceiver.cancelByAgent(addr);
|
|
1477
|
+
}
|
|
1478
|
+
events.emit("sidecar.disconnect", {
|
|
1479
|
+
ownedAddresses: [...owned],
|
|
1480
|
+
...(allocated !== undefined ? { allocated } : {}),
|
|
1481
|
+
});
|
|
1482
|
+
logger.info `Sidecar ${conn.sidecarId} disconnected`;
|
|
1483
|
+
}
|
|
1484
|
+
function nextRequestId() {
|
|
1485
|
+
return `req-${++requestCounter}`;
|
|
1486
|
+
}
|
|
1487
|
+
function sendRequest(agentAddress, buildFrame) {
|
|
1488
|
+
const ws = addressIndex.get(agentAddress);
|
|
1489
|
+
if (ws === undefined) {
|
|
1490
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1491
|
+
}
|
|
1492
|
+
const conn = connections.get(ws);
|
|
1493
|
+
if (conn === undefined) {
|
|
1494
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1495
|
+
}
|
|
1496
|
+
const requestId = nextRequestId();
|
|
1497
|
+
const frame = buildFrame(requestId);
|
|
1498
|
+
return new Promise((resolve, reject) => {
|
|
1499
|
+
const timer = setTimeout(() => {
|
|
1500
|
+
pending.delete(requestId);
|
|
1501
|
+
reject(new Error(`Request ${requestId} timed out after ${requestTimeoutMs}ms`));
|
|
1502
|
+
}, requestTimeoutMs);
|
|
1503
|
+
pending.set(requestId, {
|
|
1504
|
+
requestId,
|
|
1505
|
+
ws,
|
|
1506
|
+
resolve,
|
|
1507
|
+
reject(error) {
|
|
1508
|
+
reject(new Error(error));
|
|
1509
|
+
},
|
|
1510
|
+
timer,
|
|
1511
|
+
});
|
|
1512
|
+
conn.send(frame);
|
|
1513
|
+
});
|
|
1514
|
+
}
|
|
1515
|
+
function resolvePending(requestId) {
|
|
1516
|
+
const req = pending.get(requestId);
|
|
1517
|
+
if (req === undefined)
|
|
1518
|
+
return;
|
|
1519
|
+
clearTimeout(req.timer);
|
|
1520
|
+
pending.delete(requestId);
|
|
1521
|
+
req.resolve();
|
|
1522
|
+
}
|
|
1523
|
+
function rejectPending(requestId, error) {
|
|
1524
|
+
const req = pending.get(requestId);
|
|
1525
|
+
if (req === undefined)
|
|
1526
|
+
return;
|
|
1527
|
+
clearTimeout(req.timer);
|
|
1528
|
+
pending.delete(requestId);
|
|
1529
|
+
req.reject(error);
|
|
1530
|
+
}
|
|
1531
|
+
function packResponseMatches(entry, ws, frame) {
|
|
1532
|
+
return (entry.ws === ws &&
|
|
1533
|
+
entry.agentAddress === frame.agentAddress &&
|
|
1534
|
+
entry.repoId.kind === frame.repoId.kind &&
|
|
1535
|
+
entry.repoId.id === frame.repoId.id);
|
|
1536
|
+
}
|
|
1537
|
+
function resolvePackPending(ws, frame) {
|
|
1538
|
+
const entry = pendingPacks.get(frame.transferId);
|
|
1539
|
+
if (entry === undefined)
|
|
1540
|
+
return;
|
|
1541
|
+
if (!packResponseMatches(entry, ws, frame)) {
|
|
1542
|
+
logger.warn `Ignoring repo.pack.ack for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
|
|
1543
|
+
return;
|
|
1544
|
+
}
|
|
1545
|
+
clearTimeout(entry.timer);
|
|
1546
|
+
pendingPacks.delete(frame.transferId);
|
|
1547
|
+
entry.resolve();
|
|
1548
|
+
}
|
|
1549
|
+
function rejectPackPending(ws, frame) {
|
|
1550
|
+
const entry = pendingPacks.get(frame.transferId);
|
|
1551
|
+
if (entry === undefined)
|
|
1552
|
+
return;
|
|
1553
|
+
if (!packResponseMatches(entry, ws, frame)) {
|
|
1554
|
+
logger.warn `Ignoring repo.pack.reject for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
|
|
1555
|
+
return;
|
|
1556
|
+
}
|
|
1557
|
+
clearTimeout(entry.timer);
|
|
1558
|
+
pendingPacks.delete(frame.transferId);
|
|
1559
|
+
// Surface the receiver's specific cause when it carried one, so the awaiting
|
|
1560
|
+
// push sees "corrupt: <detail>" rather than only the coarse reason.
|
|
1561
|
+
entry.reject(frame.detail !== undefined
|
|
1562
|
+
? `${frame.reason}: ${frame.detail}`
|
|
1563
|
+
: frame.reason);
|
|
1564
|
+
}
|
|
1565
|
+
function resolveUndeployPending(ws, agentAddress) {
|
|
1566
|
+
const req = pendingUndeploys.get(agentAddress);
|
|
1567
|
+
if (req === undefined) {
|
|
1568
|
+
logger.warn `Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`;
|
|
1569
|
+
return;
|
|
1570
|
+
}
|
|
1571
|
+
if (req.ws !== ws)
|
|
1572
|
+
return;
|
|
1573
|
+
clearTimeout(req.timer);
|
|
1574
|
+
pendingUndeploys.delete(agentAddress);
|
|
1575
|
+
req.resolve();
|
|
1576
|
+
}
|
|
1577
|
+
function rejectUndeployPending(ws, agentAddress, error) {
|
|
1578
|
+
const req = pendingUndeploys.get(agentAddress);
|
|
1579
|
+
if (req === undefined)
|
|
1580
|
+
return;
|
|
1581
|
+
if (req.ws !== ws)
|
|
1582
|
+
return;
|
|
1583
|
+
clearTimeout(req.timer);
|
|
1584
|
+
pendingUndeploys.delete(agentAddress);
|
|
1585
|
+
req.reject(error);
|
|
1586
|
+
}
|
|
1587
|
+
function resolveProbe(requestId, result) {
|
|
1588
|
+
const req = pendingProbes.get(requestId);
|
|
1589
|
+
if (req === undefined)
|
|
1590
|
+
return;
|
|
1591
|
+
clearTimeout(req.timer);
|
|
1592
|
+
pendingProbes.delete(requestId);
|
|
1593
|
+
req.resolve(result);
|
|
1594
|
+
}
|
|
1595
|
+
function rejectProbe(requestId, error) {
|
|
1596
|
+
const req = pendingProbes.get(requestId);
|
|
1597
|
+
if (req === undefined)
|
|
1598
|
+
return;
|
|
1599
|
+
clearTimeout(req.timer);
|
|
1600
|
+
pendingProbes.delete(requestId);
|
|
1601
|
+
req.reject(error);
|
|
1602
|
+
}
|
|
1603
|
+
// Routing rule: pick the receiver dedicated to the repoId.kind the
|
|
1604
|
+
// frame carries. The receivers' in-flight state is independent, so a
|
|
1605
|
+
// workflow-run transferId can never collide with or evict an
|
|
1606
|
+
// agent-state transferId for the same agentAddress.
|
|
1607
|
+
function pickPackReceiver(repoId) {
|
|
1608
|
+
switch (repoId.kind) {
|
|
1609
|
+
case "agent-state":
|
|
1610
|
+
return { receiver: agentStatePackReceiver };
|
|
1611
|
+
case "workflow-run":
|
|
1612
|
+
return { receiver: workflowRunPackReceiver };
|
|
1613
|
+
// The remaining kinds in `RepoKind` (`skill`, `package-registry`,
|
|
1614
|
+
// `workflow`) have no sidecar->hub pack flow today. A frame
|
|
1615
|
+
// arriving with those kinds is malformed at this layer.
|
|
1616
|
+
default:
|
|
1617
|
+
return null;
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
function pickReceivePackLookup(repoId) {
|
|
1621
|
+
switch (repoId.kind) {
|
|
1622
|
+
case "agent-state":
|
|
1623
|
+
// The agent-state lookup ignores the `source` argument the workflow-run
|
|
1624
|
+
// lookup takes; the two are otherwise the same contract.
|
|
1625
|
+
return lookups.receiveAgentStatePack;
|
|
1626
|
+
case "workflow-run":
|
|
1627
|
+
return lookups.receiveWorkflowRunPack;
|
|
1628
|
+
default:
|
|
1629
|
+
return undefined;
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
function handlePackPush(ws, frame) {
|
|
1633
|
+
const conn = connections.get(ws);
|
|
1634
|
+
if (conn === undefined)
|
|
1635
|
+
return;
|
|
1636
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
1637
|
+
logger.warn `Received repo.pack.push for unrouted agent ${frame.agentAddress}`;
|
|
1638
|
+
return;
|
|
1639
|
+
}
|
|
1640
|
+
if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
|
|
1641
|
+
logger.warn `Rejected repo.pack.push outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
|
|
1642
|
+
conn.send({
|
|
1643
|
+
type: "repo.pack.reject",
|
|
1644
|
+
agentAddress: frame.agentAddress,
|
|
1645
|
+
repoId: frame.repoId,
|
|
1646
|
+
transferId: frame.transferId,
|
|
1647
|
+
reason: "path_violation",
|
|
1648
|
+
});
|
|
1649
|
+
return;
|
|
1650
|
+
}
|
|
1651
|
+
const picked = pickPackReceiver(frame.repoId);
|
|
1652
|
+
if (picked === null) {
|
|
1653
|
+
logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
1654
|
+
conn.send({
|
|
1655
|
+
type: "repo.pack.reject",
|
|
1656
|
+
agentAddress: frame.agentAddress,
|
|
1657
|
+
repoId: frame.repoId,
|
|
1658
|
+
transferId: frame.transferId,
|
|
1659
|
+
reason: "corrupt",
|
|
1660
|
+
});
|
|
1661
|
+
return;
|
|
1662
|
+
}
|
|
1663
|
+
const reason = picked.receiver.handlePush(frame);
|
|
1664
|
+
if (reason !== null) {
|
|
1665
|
+
conn.send({
|
|
1666
|
+
type: "repo.pack.reject",
|
|
1667
|
+
agentAddress: frame.agentAddress,
|
|
1668
|
+
repoId: frame.repoId,
|
|
1669
|
+
transferId: frame.transferId,
|
|
1670
|
+
reason,
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
async function handlePackDone(ws, frame) {
|
|
1675
|
+
const conn = connections.get(ws);
|
|
1676
|
+
if (conn === undefined)
|
|
1677
|
+
return;
|
|
1678
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
1679
|
+
logger.warn `Received repo.pack.done for unrouted agent ${frame.agentAddress}`;
|
|
1680
|
+
return;
|
|
1681
|
+
}
|
|
1682
|
+
if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
|
|
1683
|
+
logger.warn `Rejected repo.pack.done outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
|
|
1684
|
+
conn.send({
|
|
1685
|
+
type: "repo.pack.reject",
|
|
1686
|
+
agentAddress: frame.agentAddress,
|
|
1687
|
+
repoId: frame.repoId,
|
|
1688
|
+
transferId: frame.transferId,
|
|
1689
|
+
reason: "path_violation",
|
|
1690
|
+
});
|
|
1691
|
+
return;
|
|
1692
|
+
}
|
|
1693
|
+
const picked = pickPackReceiver(frame.repoId);
|
|
1694
|
+
if (picked === null) {
|
|
1695
|
+
logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
1696
|
+
conn.send({
|
|
1697
|
+
type: "repo.pack.reject",
|
|
1698
|
+
agentAddress: frame.agentAddress,
|
|
1699
|
+
repoId: frame.repoId,
|
|
1700
|
+
transferId: frame.transferId,
|
|
1701
|
+
reason: "corrupt",
|
|
1702
|
+
});
|
|
1703
|
+
return;
|
|
1704
|
+
}
|
|
1705
|
+
const result = picked.receiver.handleDone(frame);
|
|
1706
|
+
if (result === null) {
|
|
1707
|
+
conn.send({
|
|
1708
|
+
type: "repo.pack.reject",
|
|
1709
|
+
agentAddress: frame.agentAddress,
|
|
1710
|
+
repoId: frame.repoId,
|
|
1711
|
+
transferId: frame.transferId,
|
|
1712
|
+
reason: "corrupt",
|
|
1713
|
+
});
|
|
1714
|
+
return;
|
|
1715
|
+
}
|
|
1716
|
+
const receivePackLookup = pickReceivePackLookup(frame.repoId);
|
|
1717
|
+
if (receivePackLookup === undefined) {
|
|
1718
|
+
conn.send({
|
|
1719
|
+
type: "repo.pack.ack",
|
|
1720
|
+
agentAddress: frame.agentAddress,
|
|
1721
|
+
repoId: frame.repoId,
|
|
1722
|
+
transferId: frame.transferId,
|
|
1723
|
+
});
|
|
1724
|
+
return;
|
|
1725
|
+
}
|
|
1726
|
+
const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, conn.identity.kind === "allocated"
|
|
1727
|
+
? {
|
|
1728
|
+
kind: "allocated",
|
|
1729
|
+
agentAddress: frame.agentAddress,
|
|
1730
|
+
allocationId: conn.identity.allocationId,
|
|
1731
|
+
anchorRunId: conn.identity.anchorRunId,
|
|
1732
|
+
generation: conn.identity.generation,
|
|
1733
|
+
}
|
|
1734
|
+
: { kind: "shared", agentAddress: frame.agentAddress });
|
|
1735
|
+
// Connection may have closed during async verification.
|
|
1736
|
+
const currentConn = connections.get(ws);
|
|
1737
|
+
if (currentConn === undefined)
|
|
1738
|
+
return;
|
|
1739
|
+
if (verdict.accepted) {
|
|
1740
|
+
currentConn.send({
|
|
1741
|
+
type: "repo.pack.ack",
|
|
1742
|
+
agentAddress: frame.agentAddress,
|
|
1743
|
+
repoId: frame.repoId,
|
|
1744
|
+
transferId: frame.transferId,
|
|
1745
|
+
});
|
|
1746
|
+
}
|
|
1747
|
+
else {
|
|
1748
|
+
currentConn.send({
|
|
1749
|
+
type: "repo.pack.reject",
|
|
1750
|
+
agentAddress: frame.agentAddress,
|
|
1751
|
+
repoId: frame.repoId,
|
|
1752
|
+
transferId: frame.transferId,
|
|
1753
|
+
reason: verdict.reason,
|
|
1754
|
+
});
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
/**
|
|
1758
|
+
* Bind a per-step workflow-substrate address to a sidecar for the staging
|
|
1759
|
+
* window of a multi-step deploy, so `sendPack` can route the step's deploy
|
|
1760
|
+
* and asset packs before the deployment-level frame spawns the child.
|
|
1761
|
+
*
|
|
1762
|
+
* The address is hub-minted and workflow-derived (no per-address key), so
|
|
1763
|
+
* it enters the keyless `workflowAddresses` set -- never the challenged
|
|
1764
|
+
* `agentAddresses` set -- and is torn down by `unbindStepRoute` once the
|
|
1765
|
+
* step's packs land. `handleClose` reclaims it if the sidecar drops
|
|
1766
|
+
* mid-stage. Per-step addresses are not runtime-routed (mail, signals, and
|
|
1767
|
+
* drains use the deployment address), so the binding is transient: it is
|
|
1768
|
+
* never persisted into the reconnect set and never resurrected on
|
|
1769
|
+
* reconnect.
|
|
1770
|
+
*/
|
|
1771
|
+
function fenceAllocation(allocationId, generation) {
|
|
1772
|
+
const existing = allocationFences.get(allocationId);
|
|
1773
|
+
if (existing !== undefined && generation < existing) {
|
|
1774
|
+
throw new Error(`Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`);
|
|
1775
|
+
}
|
|
1776
|
+
allocationFences.set(allocationId, generation);
|
|
1777
|
+
const current = allocatedConnections.get(allocationId);
|
|
1778
|
+
if (current !== undefined && current.identity.generation !== generation) {
|
|
1779
|
+
handleClose(current.ws);
|
|
1780
|
+
current.ws.close();
|
|
1781
|
+
}
|
|
1782
|
+
const waiters = allocationWaiters.get(allocationId);
|
|
1783
|
+
if (waiters === undefined)
|
|
1784
|
+
return;
|
|
1785
|
+
for (const waiter of [...waiters]) {
|
|
1786
|
+
if (waiter.generation === generation)
|
|
1787
|
+
continue;
|
|
1788
|
+
clearTimeout(waiter.timer);
|
|
1789
|
+
waiters.delete(waiter);
|
|
1790
|
+
waiter.reject(new Error(`Allocation ${allocationId} advanced to generation ${String(generation)}`));
|
|
1791
|
+
}
|
|
1792
|
+
if (waiters.size === 0)
|
|
1793
|
+
allocationWaiters.delete(allocationId);
|
|
1794
|
+
}
|
|
1795
|
+
async function getAllocatedConnection(target, use) {
|
|
1796
|
+
if (allocationFences.get(target.allocationId) !== target.generation) {
|
|
1797
|
+
throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
|
|
1798
|
+
}
|
|
1799
|
+
const current = allocatedConnections.get(target.allocationId);
|
|
1800
|
+
if (current === undefined ||
|
|
1801
|
+
current.identity.generation !== target.generation) {
|
|
1802
|
+
throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`);
|
|
1803
|
+
}
|
|
1804
|
+
if (!(await validateSidecarIdentity(current.identity, use))) {
|
|
1805
|
+
if (allocatedConnections.get(target.allocationId) === current) {
|
|
1806
|
+
handleClose(current.ws);
|
|
1807
|
+
current.ws.close();
|
|
1808
|
+
}
|
|
1809
|
+
throw new Error(`Allocated sidecar identity is no longer current for allocation ${target.allocationId}`);
|
|
1810
|
+
}
|
|
1811
|
+
if (allocatedConnections.get(target.allocationId) !== current) {
|
|
1812
|
+
throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
|
|
1813
|
+
}
|
|
1814
|
+
const conn = connections.get(current.ws);
|
|
1815
|
+
if (conn === undefined ||
|
|
1816
|
+
conn.identity.kind !== "allocated" ||
|
|
1817
|
+
conn.identity.allocationId !== target.allocationId ||
|
|
1818
|
+
conn.identity.generation !== target.generation) {
|
|
1819
|
+
throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId}`);
|
|
1820
|
+
}
|
|
1821
|
+
return { ws: current.ws, conn };
|
|
1822
|
+
}
|
|
1823
|
+
async function isAllocatedSidecarReady(target) {
|
|
1824
|
+
try {
|
|
1825
|
+
await getAllocatedConnection(target, "readiness");
|
|
1826
|
+
return true;
|
|
1827
|
+
}
|
|
1828
|
+
catch {
|
|
1829
|
+
return false;
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
async function isAllocatedWorkflowActive(target) {
|
|
1833
|
+
try {
|
|
1834
|
+
const { conn } = await getAllocatedConnection(target, "readiness");
|
|
1835
|
+
if (conn.identity.kind !== "allocated")
|
|
1836
|
+
return false;
|
|
1837
|
+
return conn.workflowAddresses.has(conn.identity.workflowRunAddress);
|
|
1838
|
+
}
|
|
1839
|
+
catch {
|
|
1840
|
+
return false;
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
async function waitForAllocatedSidecar(target, timeoutMs) {
|
|
1844
|
+
if (await isAllocatedSidecarReady(target))
|
|
1845
|
+
return;
|
|
1846
|
+
if (allocationFences.get(target.allocationId) !== target.generation) {
|
|
1847
|
+
throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
|
|
1848
|
+
}
|
|
1849
|
+
if (timeoutMs <= 0) {
|
|
1850
|
+
throw new Error(`Timed out waiting for allocated sidecar ${target.allocationId}`);
|
|
1851
|
+
}
|
|
1852
|
+
await new Promise((resolve, reject) => {
|
|
1853
|
+
const waiter = {
|
|
1854
|
+
generation: target.generation,
|
|
1855
|
+
resolve,
|
|
1856
|
+
reject,
|
|
1857
|
+
timer: setTimeout(() => {
|
|
1858
|
+
const current = allocationWaiters.get(target.allocationId);
|
|
1859
|
+
current?.delete(waiter);
|
|
1860
|
+
if (current?.size === 0) {
|
|
1861
|
+
allocationWaiters.delete(target.allocationId);
|
|
1862
|
+
}
|
|
1863
|
+
reject(new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`));
|
|
1864
|
+
}, timeoutMs),
|
|
1865
|
+
};
|
|
1866
|
+
let waiters = allocationWaiters.get(target.allocationId);
|
|
1867
|
+
if (waiters === undefined) {
|
|
1868
|
+
waiters = new Set();
|
|
1869
|
+
allocationWaiters.set(target.allocationId, waiters);
|
|
1870
|
+
}
|
|
1871
|
+
waiters.add(waiter);
|
|
1872
|
+
void notifyAllocationWaiters(target.allocationId);
|
|
1873
|
+
});
|
|
1874
|
+
}
|
|
1875
|
+
function bindStepRoute(stepAddress) {
|
|
1876
|
+
const ws = addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress);
|
|
1877
|
+
if (ws === undefined) {
|
|
1878
|
+
throw new Error(`No sidecar available to stage workflow step "${stepAddress}"`);
|
|
1879
|
+
}
|
|
1880
|
+
const conn = connections.get(ws);
|
|
1881
|
+
if (conn === undefined) {
|
|
1882
|
+
throw new Error(`No sidecar connected to stage workflow step "${stepAddress}"`);
|
|
1883
|
+
}
|
|
1884
|
+
if (conn.identity.kind !== "shared") {
|
|
1885
|
+
throw new Error(`Allocated sidecar ${conn.sidecarId} cannot receive ordinary workflow step ${stepAddress}`);
|
|
1886
|
+
}
|
|
1887
|
+
conn.workflowAddresses.add(stepAddress);
|
|
1888
|
+
addressIndex.set(stepAddress, ws);
|
|
1889
|
+
}
|
|
1890
|
+
async function bindAllocatedStepRoute(target, stepAddress) {
|
|
1891
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
1892
|
+
const existing = addressIndex.get(stepAddress);
|
|
1893
|
+
if (existing !== undefined && existing !== ws) {
|
|
1894
|
+
throw new Error(`Workflow step ${stepAddress} is already routed to another sidecar`);
|
|
1895
|
+
}
|
|
1896
|
+
conn.workflowAddresses.add(stepAddress);
|
|
1897
|
+
addressIndex.set(stepAddress, ws);
|
|
1898
|
+
}
|
|
1899
|
+
/**
|
|
1900
|
+
* Remove a per-step route bound by `bindStepRoute` once the step's packs
|
|
1901
|
+
* have landed. Idempotent: an address that was never bound (or already
|
|
1902
|
+
* unbound, e.g. by a mid-stage `handleClose`) is a no-op.
|
|
1903
|
+
*/
|
|
1904
|
+
function unbindStepRoute(stepAddress) {
|
|
1905
|
+
const ws = addressIndex.get(stepAddress);
|
|
1906
|
+
if (ws === undefined)
|
|
1907
|
+
return;
|
|
1908
|
+
const conn = connections.get(ws);
|
|
1909
|
+
if (conn !== undefined) {
|
|
1910
|
+
conn.workflowAddresses.delete(stepAddress);
|
|
1911
|
+
}
|
|
1912
|
+
addressIndex.delete(stepAddress);
|
|
1913
|
+
}
|
|
1914
|
+
function unbindAllocatedStepRoute(target, stepAddress) {
|
|
1915
|
+
const current = allocatedConnections.get(target.allocationId);
|
|
1916
|
+
if (current === undefined ||
|
|
1917
|
+
current.identity.generation !== target.generation) {
|
|
1918
|
+
return;
|
|
1919
|
+
}
|
|
1920
|
+
if (addressIndex.get(stepAddress) !== current.ws)
|
|
1921
|
+
return;
|
|
1922
|
+
connections.get(current.ws)?.workflowAddresses.delete(stepAddress);
|
|
1923
|
+
addressIndex.delete(stepAddress);
|
|
1924
|
+
}
|
|
1925
|
+
// Pack transfers may take longer than session requests due to data volume.
|
|
1926
|
+
const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
|
|
1927
|
+
function sendPack(agentAddress, pack, ref, commitSha, options) {
|
|
1928
|
+
const ws = addressIndex.get(agentAddress);
|
|
1929
|
+
if (ws === undefined) {
|
|
1930
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1931
|
+
}
|
|
1932
|
+
const conn = connections.get(ws);
|
|
1933
|
+
if (conn === undefined) {
|
|
1934
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1935
|
+
}
|
|
1936
|
+
if (conn.identity.kind !== "shared") {
|
|
1937
|
+
return Promise.reject(new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound pack routing for "${agentAddress}"`));
|
|
1938
|
+
}
|
|
1939
|
+
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
|
|
1940
|
+
}
|
|
1941
|
+
function sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options) {
|
|
1942
|
+
const transferId = `pack-${++packCounter}`;
|
|
1943
|
+
// For the agent-state flow the destination agent and the source repo
|
|
1944
|
+
// are the same entity, so `repoId.id === agentAddress`. Asset packs
|
|
1945
|
+
// override this with the SOURCE asset's id so audit can correlate
|
|
1946
|
+
// the pack back to its hub-side origin.
|
|
1947
|
+
const repoId = options?.repoId ?? {
|
|
1948
|
+
kind: "agent-state",
|
|
1949
|
+
id: agentAddress,
|
|
1950
|
+
};
|
|
1951
|
+
const mountPath = options?.mountPath;
|
|
1952
|
+
// Register pending entry before sending frames so that a synchronous
|
|
1953
|
+
// repo.pack.ack (e.g. in tests or loopback transports) resolves correctly.
|
|
1954
|
+
return new Promise((resolve, reject) => {
|
|
1955
|
+
const timer = setTimeout(() => {
|
|
1956
|
+
pendingPacks.delete(transferId);
|
|
1957
|
+
reject(new Error(`Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`));
|
|
1958
|
+
}, PACK_TIMEOUT_MS);
|
|
1959
|
+
pendingPacks.set(transferId, {
|
|
1960
|
+
transferId,
|
|
1961
|
+
ws,
|
|
1962
|
+
agentAddress,
|
|
1963
|
+
repoId,
|
|
1964
|
+
resolve,
|
|
1965
|
+
reject(error) {
|
|
1966
|
+
reject(new Error(`Pack rejected: ${error}`));
|
|
1967
|
+
},
|
|
1968
|
+
timer,
|
|
1969
|
+
});
|
|
1970
|
+
// Send chunks
|
|
1971
|
+
for (const chunk of chunkPack(pack)) {
|
|
1972
|
+
conn.send({
|
|
1973
|
+
type: "repo.pack.push",
|
|
1974
|
+
agentAddress,
|
|
1975
|
+
repoId,
|
|
1976
|
+
transferId,
|
|
1977
|
+
seq: chunk.seq,
|
|
1978
|
+
data: chunk.data,
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
// Send done
|
|
1982
|
+
conn.send({
|
|
1983
|
+
type: "repo.pack.done",
|
|
1984
|
+
agentAddress,
|
|
1985
|
+
repoId,
|
|
1986
|
+
transferId,
|
|
1987
|
+
ref,
|
|
1988
|
+
commitSha,
|
|
1989
|
+
...(mountPath !== undefined ? { mountPath } : {}),
|
|
1990
|
+
});
|
|
1991
|
+
});
|
|
1992
|
+
}
|
|
1993
|
+
async function sendPackToAllocation(target, agentAddress, pack, ref, commitSha, options) {
|
|
1994
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
1995
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
1996
|
+
throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
1997
|
+
}
|
|
1998
|
+
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
|
|
1999
|
+
}
|
|
2000
|
+
async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha) {
|
|
2001
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2002
|
+
if (conn.identity.kind !== "allocated") {
|
|
2003
|
+
throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
|
|
2004
|
+
}
|
|
2005
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
2006
|
+
throw new Error(`Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`);
|
|
2007
|
+
}
|
|
2008
|
+
if (conn.workflowAddresses.has(agentAddress)) {
|
|
2009
|
+
throw new Error(`Allocation ${target.allocationId} already hosts active workflow ${agentAddress}; refusing to overwrite its run history`);
|
|
2010
|
+
}
|
|
2011
|
+
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, {
|
|
2012
|
+
repoId: {
|
|
2013
|
+
kind: "workflow-run",
|
|
2014
|
+
id: deriveWorkflowRunRepoId(agentAddress),
|
|
2015
|
+
},
|
|
2016
|
+
});
|
|
2017
|
+
}
|
|
2018
|
+
function routeMail(agentAddress, rawMessage, messageId, runGrants) {
|
|
2019
|
+
// Carry the hub-minted messageId on the frame so the sidecar's durable-
|
|
2020
|
+
// receipt ack (`mail.inbound.ack`) keys on the same id the hub tracks, and
|
|
2021
|
+
// a redelivery replays identical bytes for the downstream RunStarted dedup.
|
|
2022
|
+
// Optional: the workflow-trigger and session-conversation callers supply
|
|
2023
|
+
// it (they participate in the ack/retry handshake); a caller without a
|
|
2024
|
+
// hub-minted id omits it and the delivery is not tracked for redelivery.
|
|
2025
|
+
const frame = {
|
|
2026
|
+
type: "mail.inbound",
|
|
2027
|
+
agentAddress,
|
|
2028
|
+
rawMessage,
|
|
2029
|
+
...(messageId !== undefined ? { messageId } : {}),
|
|
2030
|
+
};
|
|
2031
|
+
const ws = addressIndex.get(agentAddress);
|
|
2032
|
+
if (ws !== undefined) {
|
|
2033
|
+
const conn = connections.get(ws);
|
|
2034
|
+
if (conn !== undefined) {
|
|
2035
|
+
conn.send(frame);
|
|
2036
|
+
// Track the delivery for redelivery until the sidecar acks its durable
|
|
2037
|
+
// inbox write. Only mail carrying a hub-minted messageId participates
|
|
2038
|
+
// in the ack handshake; relayed agent-to-agent mail omits it and is
|
|
2039
|
+
// delivered fire-and-forget as before. A mail that triggered a workflow
|
|
2040
|
+
// run carries the run's grants so redelivery can replay them ahead of
|
|
2041
|
+
// the mail.
|
|
2042
|
+
if (messageId !== undefined) {
|
|
2043
|
+
trackPendingMail(agentAddress, messageId, frame, runGrants);
|
|
2044
|
+
}
|
|
2045
|
+
return true;
|
|
2046
|
+
}
|
|
2047
|
+
}
|
|
2048
|
+
// If the agent recently disconnected, queue for delivery on reconnect.
|
|
2049
|
+
return enqueueForDisconnected(agentAddress, frame);
|
|
2050
|
+
}
|
|
2051
|
+
function sendRunGrants(agentAddress, runId, stepGrants) {
|
|
2052
|
+
const frame = {
|
|
2053
|
+
type: "run.grants",
|
|
2054
|
+
agentAddress,
|
|
2055
|
+
runId,
|
|
2056
|
+
stepGrants,
|
|
2057
|
+
};
|
|
2058
|
+
const ws = addressIndex.get(agentAddress);
|
|
2059
|
+
if (ws !== undefined) {
|
|
2060
|
+
const conn = connections.get(ws);
|
|
2061
|
+
if (conn !== undefined) {
|
|
2062
|
+
conn.send(frame);
|
|
2063
|
+
return true;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
// Mirror routeMail: if the address has a live disconnect queue, ride it
|
|
2067
|
+
// so a run.grants issued in the window between deploy and the first
|
|
2068
|
+
// reconnect survives the same way the dispatching trigger mail does.
|
|
2069
|
+
// A queue exists only while the deployment address is still on
|
|
2070
|
+
// agentAddresses (pre-first-reconnect); after a challenged reconnect it
|
|
2071
|
+
// moves to workflowAddresses, which handleClose leaves unqueued because
|
|
2072
|
+
// that generation's in-flight run state is reconstructed sidecar-locally.
|
|
2073
|
+
// Returning without enqueueing there is correct; enqueueing is what keeps
|
|
2074
|
+
// grants and mail from diverging in the pre-reconnect window.
|
|
2075
|
+
return enqueueForDisconnected(agentAddress, frame);
|
|
2076
|
+
}
|
|
2077
|
+
async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, messageId) {
|
|
2078
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2079
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
2080
|
+
throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2081
|
+
}
|
|
2082
|
+
const runGrants = { runId, stepGrants };
|
|
2083
|
+
conn.send({
|
|
2084
|
+
type: "run.grants",
|
|
2085
|
+
agentAddress,
|
|
2086
|
+
runId,
|
|
2087
|
+
stepGrants,
|
|
2088
|
+
});
|
|
2089
|
+
const frame = {
|
|
2090
|
+
type: "mail.inbound",
|
|
2091
|
+
agentAddress,
|
|
2092
|
+
rawMessage,
|
|
2093
|
+
messageId,
|
|
2094
|
+
};
|
|
2095
|
+
conn.send(frame);
|
|
2096
|
+
trackPendingMail(agentAddress, messageId, frame, runGrants, target);
|
|
2097
|
+
}
|
|
2098
|
+
async function handleDeployAck(ws, frame) {
|
|
2099
|
+
const req = pendingDeploys.get(frame.agentAddress);
|
|
2100
|
+
if (req === undefined) {
|
|
2101
|
+
logger.warn `Received agent.deploy.ack for "${frame.agentAddress}" with no pending deploy`;
|
|
2102
|
+
return;
|
|
2103
|
+
}
|
|
2104
|
+
if (req.ws !== ws)
|
|
2105
|
+
return;
|
|
2106
|
+
if (events.listenerCount("agent.deploy.ack") > 0) {
|
|
2107
|
+
try {
|
|
2108
|
+
const identity = connections.get(ws)?.identity;
|
|
2109
|
+
await events.emitAndAwait("agent.deploy.ack", {
|
|
2110
|
+
agentAddress: frame.agentAddress,
|
|
2111
|
+
publicKey: frame.publicKey,
|
|
2112
|
+
...(identity?.kind === "allocated"
|
|
2113
|
+
? {
|
|
2114
|
+
allocated: {
|
|
2115
|
+
allocationId: identity.allocationId,
|
|
2116
|
+
anchorRunId: identity.anchorRunId,
|
|
2117
|
+
generation: identity.generation,
|
|
2118
|
+
},
|
|
2119
|
+
}
|
|
2120
|
+
: {}),
|
|
2121
|
+
});
|
|
2122
|
+
}
|
|
2123
|
+
catch (err) {
|
|
2124
|
+
rejectDeployPending(req, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
|
|
2125
|
+
return;
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
resolveDeployPending(req, frame.publicKey);
|
|
2129
|
+
}
|
|
2130
|
+
function resolveDeployPending(req, publicKey) {
|
|
2131
|
+
if (pendingDeploys.get(req.agentAddress) !== req)
|
|
2132
|
+
return;
|
|
2133
|
+
clearTimeout(req.timer);
|
|
2134
|
+
pendingDeploys.delete(req.agentAddress);
|
|
2135
|
+
req.resolve(publicKey);
|
|
2136
|
+
}
|
|
2137
|
+
function rejectDeployPending(req, error) {
|
|
2138
|
+
if (pendingDeploys.get(req.agentAddress) !== req)
|
|
2139
|
+
return;
|
|
2140
|
+
clearTimeout(req.timer);
|
|
2141
|
+
pendingDeploys.delete(req.agentAddress);
|
|
2142
|
+
req.reject(error);
|
|
2143
|
+
}
|
|
2144
|
+
function rejectDeployPendingFromFrame(ws, agentAddress, error) {
|
|
2145
|
+
const req = pendingDeploys.get(agentAddress);
|
|
2146
|
+
if (req === undefined || req.ws !== ws)
|
|
2147
|
+
return;
|
|
2148
|
+
rejectDeployPending(req, error);
|
|
2149
|
+
}
|
|
2150
|
+
async function sendAgentDeploy(agentAddress, harnessConfig, workflow) {
|
|
2151
|
+
if (hubPublicKeyHex === undefined) {
|
|
2152
|
+
throw new Error("Hub signing key is required for agent deployment");
|
|
2153
|
+
}
|
|
2154
|
+
const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
|
|
2155
|
+
if (ws === undefined) {
|
|
2156
|
+
throw new Error(`No sidecar available for agent "${agentAddress}"`);
|
|
2157
|
+
}
|
|
2158
|
+
const conn = connections.get(ws);
|
|
2159
|
+
if (conn === undefined) {
|
|
2160
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
2161
|
+
}
|
|
2162
|
+
if (conn.identity.kind !== "shared") {
|
|
2163
|
+
throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound deploy routing`);
|
|
2164
|
+
}
|
|
2165
|
+
return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
|
|
2166
|
+
}
|
|
2167
|
+
function sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow) {
|
|
2168
|
+
if (hubPublicKeyHex === undefined) {
|
|
2169
|
+
throw new Error("Hub signing key is required for agent deployment");
|
|
2170
|
+
}
|
|
2171
|
+
if (pendingDeploys.has(agentAddress)) {
|
|
2172
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
2173
|
+
}
|
|
2174
|
+
const addressSet = conn.identity.kind === "allocated"
|
|
2175
|
+
? conn.workflowAddresses
|
|
2176
|
+
: conn.agentAddresses;
|
|
2177
|
+
addressSet.add(agentAddress);
|
|
2178
|
+
addressIndex.set(agentAddress, ws);
|
|
2179
|
+
return new Promise((resolve, reject) => {
|
|
2180
|
+
const timer = setTimeout(() => {
|
|
2181
|
+
pendingDeploys.delete(agentAddress);
|
|
2182
|
+
if (addressIndex.get(agentAddress) === ws) {
|
|
2183
|
+
addressSet.delete(agentAddress);
|
|
2184
|
+
addressIndex.delete(agentAddress);
|
|
2185
|
+
}
|
|
2186
|
+
reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
2187
|
+
}, requestTimeoutMs);
|
|
2188
|
+
pendingDeploys.set(agentAddress, {
|
|
2189
|
+
agentAddress,
|
|
2190
|
+
ws,
|
|
2191
|
+
resolve(publicKey) {
|
|
2192
|
+
resolve({ publicKey });
|
|
2193
|
+
},
|
|
2194
|
+
reject(error) {
|
|
2195
|
+
if (addressIndex.get(agentAddress) === ws) {
|
|
2196
|
+
addressSet.delete(agentAddress);
|
|
2197
|
+
addressIndex.delete(agentAddress);
|
|
2198
|
+
}
|
|
2199
|
+
reject(new Error(error));
|
|
2200
|
+
},
|
|
2201
|
+
timer,
|
|
2202
|
+
});
|
|
2203
|
+
conn.send({
|
|
2204
|
+
type: "agent.deploy",
|
|
2205
|
+
agentAddress,
|
|
2206
|
+
agentId: harnessConfig.agentId,
|
|
2207
|
+
config: harnessConfig,
|
|
2208
|
+
hubPublicKey: hubPublicKeyHex,
|
|
2209
|
+
...(workflow !== undefined ? { workflow } : {}),
|
|
2210
|
+
});
|
|
2211
|
+
});
|
|
2212
|
+
}
|
|
2213
|
+
async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow) {
|
|
2214
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2215
|
+
if (conn.identity.kind !== "allocated") {
|
|
2216
|
+
throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
|
|
2217
|
+
}
|
|
2218
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
2219
|
+
throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
|
|
2220
|
+
}
|
|
2221
|
+
const existing = addressIndex.get(agentAddress);
|
|
2222
|
+
if (existing !== undefined && existing !== ws) {
|
|
2223
|
+
throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
|
|
2224
|
+
}
|
|
2225
|
+
return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
|
|
2226
|
+
}
|
|
2227
|
+
/**
|
|
2228
|
+
* Provision one step of a multi-step deploy on the sidecar WITHOUT
|
|
2229
|
+
* spawning: the sidecar initializes the step's agent-state repo and
|
|
2230
|
+
* records the hub key, so the follow-up deploy pack applies into a repo
|
|
2231
|
+
* and verifies against the recorded key -- but no supervisor or child is
|
|
2232
|
+
* constructed. The deployment-level workflow frame, sent once after every
|
|
2233
|
+
* step is provisioned, spawns the child.
|
|
2234
|
+
*
|
|
2235
|
+
* The step address must already be bound via `bindStepRoute`, which
|
|
2236
|
+
* resolves and records the sidecar; this reuses that route rather than
|
|
2237
|
+
* touching `agentAddresses`. Waits for the sidecar's `agent.deploy.ack`
|
|
2238
|
+
* so the caller can safely deliver the deploy pack afterward. On failure
|
|
2239
|
+
* the caller owns tearing the route down via `unbindStepRoute`.
|
|
2240
|
+
*/
|
|
2241
|
+
function sendProvisionStep(agentAddress, harnessConfig) {
|
|
2242
|
+
const ws = addressIndex.get(agentAddress);
|
|
2243
|
+
if (ws === undefined) {
|
|
2244
|
+
throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
|
|
2245
|
+
}
|
|
2246
|
+
const conn = connections.get(ws);
|
|
2247
|
+
if (conn === undefined) {
|
|
2248
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
2249
|
+
}
|
|
2250
|
+
if (conn.identity.kind !== "shared") {
|
|
2251
|
+
throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound step provisioning`);
|
|
2252
|
+
}
|
|
2253
|
+
return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
|
|
2254
|
+
}
|
|
2255
|
+
function sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig) {
|
|
2256
|
+
if (hubPublicKeyHex === undefined) {
|
|
2257
|
+
throw new Error("Hub signing key is required for step provisioning");
|
|
2258
|
+
}
|
|
2259
|
+
if (pendingDeploys.has(agentAddress)) {
|
|
2260
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
2261
|
+
}
|
|
2262
|
+
const hubKey = hubPublicKeyHex;
|
|
2263
|
+
return new Promise((resolve, reject) => {
|
|
2264
|
+
const timer = setTimeout(() => {
|
|
2265
|
+
pendingDeploys.delete(agentAddress);
|
|
2266
|
+
reject(new Error(`Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
2267
|
+
}, requestTimeoutMs);
|
|
2268
|
+
// The sidecar's `agent.deploy.ack` resolves this through
|
|
2269
|
+
// `resolveDeployPending` -> `req.resolve()`. The per-step address is
|
|
2270
|
+
// workflow-derived and records no hub-side key, so the ack's public
|
|
2271
|
+
// key is not needed and this resolves void.
|
|
2272
|
+
pendingDeploys.set(agentAddress, {
|
|
2273
|
+
agentAddress,
|
|
2274
|
+
ws,
|
|
2275
|
+
resolve(_publicKey) {
|
|
2276
|
+
resolve();
|
|
2277
|
+
},
|
|
2278
|
+
reject(error) {
|
|
2279
|
+
reject(new Error(error));
|
|
2280
|
+
},
|
|
2281
|
+
timer,
|
|
2282
|
+
});
|
|
2283
|
+
conn.send({
|
|
2284
|
+
type: "agent.deploy",
|
|
2285
|
+
agentAddress,
|
|
2286
|
+
agentId: harnessConfig.agentId,
|
|
2287
|
+
config: harnessConfig,
|
|
2288
|
+
hubPublicKey: hubKey,
|
|
2289
|
+
provisionStep: true,
|
|
2290
|
+
});
|
|
2291
|
+
});
|
|
2292
|
+
}
|
|
2293
|
+
async function sendProvisionStepToAllocation(target, agentAddress, harnessConfig) {
|
|
2294
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2295
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
2296
|
+
throw new Error(`Step route ${agentAddress} is not bound to allocation ${target.allocationId}`);
|
|
2297
|
+
}
|
|
2298
|
+
return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
|
|
2299
|
+
}
|
|
2300
|
+
function findSidecarForNewAgent(_agentAddress) {
|
|
2301
|
+
for (const [ws, conn] of connections) {
|
|
2302
|
+
if (conn.identity.kind === "shared")
|
|
2303
|
+
return ws;
|
|
2304
|
+
}
|
|
2305
|
+
return undefined;
|
|
2306
|
+
}
|
|
2307
|
+
function sendProbe(args) {
|
|
2308
|
+
// Select any connected sidecar, exactly as the pre-deploy window does.
|
|
2309
|
+
// A probe runs in the sidecar's pre-deploy state, so it targets a
|
|
2310
|
+
// connection, not an address -- `findSidecarForNewAgent` ignores its
|
|
2311
|
+
// argument and returns the first connection. Throw immediately on an empty
|
|
2312
|
+
// registry rather than register a probe that could only time out.
|
|
2313
|
+
const ws = findSidecarForNewAgent("");
|
|
2314
|
+
if (ws === undefined) {
|
|
2315
|
+
return Promise.reject(new Error("No sidecar available to probe workflow"));
|
|
2316
|
+
}
|
|
2317
|
+
const conn = connections.get(ws);
|
|
2318
|
+
if (conn === undefined) {
|
|
2319
|
+
return Promise.reject(new Error("No sidecar connected to probe workflow"));
|
|
2320
|
+
}
|
|
2321
|
+
const requestId = nextRequestId();
|
|
2322
|
+
return new Promise((resolve, reject) => {
|
|
2323
|
+
const timer = setTimeout(() => {
|
|
2324
|
+
pendingProbes.delete(requestId);
|
|
2325
|
+
reject(new Error(`Probe ${requestId} timed out after ${probeTimeoutMs}ms`));
|
|
2326
|
+
}, probeTimeoutMs);
|
|
2327
|
+
pendingProbes.set(requestId, {
|
|
2328
|
+
requestId,
|
|
2329
|
+
ws,
|
|
2330
|
+
resolve,
|
|
2331
|
+
reject(error) {
|
|
2332
|
+
reject(new Error(error));
|
|
2333
|
+
},
|
|
2334
|
+
timer,
|
|
2335
|
+
});
|
|
2336
|
+
conn.send({
|
|
2337
|
+
type: "workflow.probe.request",
|
|
2338
|
+
requestId,
|
|
2339
|
+
source: args.source,
|
|
2340
|
+
closure: args.closure,
|
|
2341
|
+
entry: args.entry,
|
|
2342
|
+
...(args.assets !== undefined ? { assets: args.assets } : {}),
|
|
2343
|
+
});
|
|
2344
|
+
});
|
|
2345
|
+
}
|
|
2346
|
+
function sendAgentUndeploy(agentAddress, reason) {
|
|
2347
|
+
const ws = addressIndex.get(agentAddress);
|
|
2348
|
+
if (ws === undefined) {
|
|
2349
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
2350
|
+
}
|
|
2351
|
+
const conn = connections.get(ws);
|
|
2352
|
+
if (conn === undefined) {
|
|
2353
|
+
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
2354
|
+
}
|
|
2355
|
+
return new Promise((resolve, reject) => {
|
|
2356
|
+
const timer = setTimeout(() => {
|
|
2357
|
+
pendingUndeploys.delete(agentAddress);
|
|
2358
|
+
removeAgentAddress(ws, agentAddress);
|
|
2359
|
+
reject(new Error(`Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
2360
|
+
}, requestTimeoutMs);
|
|
2361
|
+
pendingUndeploys.set(agentAddress, {
|
|
2362
|
+
agentAddress,
|
|
2363
|
+
ws,
|
|
2364
|
+
resolve() {
|
|
2365
|
+
removeAgentAddress(ws, agentAddress);
|
|
2366
|
+
resolve();
|
|
2367
|
+
},
|
|
2368
|
+
reject(error) {
|
|
2369
|
+
removeAgentAddress(ws, agentAddress);
|
|
2370
|
+
reject(new Error(error));
|
|
2371
|
+
},
|
|
2372
|
+
timer,
|
|
2373
|
+
});
|
|
2374
|
+
conn.send({
|
|
2375
|
+
type: "agent.undeploy",
|
|
2376
|
+
agentAddress,
|
|
2377
|
+
reason,
|
|
2378
|
+
});
|
|
2379
|
+
});
|
|
2380
|
+
}
|
|
2381
|
+
function removeAgentAddress(ws, agentAddress) {
|
|
2382
|
+
addressIndex.delete(agentAddress);
|
|
2383
|
+
const conn = connections.get(ws);
|
|
2384
|
+
if (conn !== undefined) {
|
|
2385
|
+
conn.agentAddresses.delete(agentAddress);
|
|
2386
|
+
}
|
|
2387
|
+
}
|
|
2388
|
+
function dispatchToSubscribers(agentAddress, event) {
|
|
2389
|
+
const subs = agentSubscribers.get(agentAddress);
|
|
2390
|
+
if (subs === undefined)
|
|
2391
|
+
return;
|
|
2392
|
+
for (const cb of [...subs]) {
|
|
2393
|
+
try {
|
|
2394
|
+
cb(event);
|
|
2395
|
+
}
|
|
2396
|
+
catch (err) {
|
|
2397
|
+
logger.warn `Agent subscriber threw: ${err instanceof Error ? err.message : String(err)}`;
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
function subscribeAgent(agentAddress, callback) {
|
|
2402
|
+
let subs = agentSubscribers.get(agentAddress);
|
|
2403
|
+
if (subs === undefined) {
|
|
2404
|
+
subs = new Set();
|
|
2405
|
+
agentSubscribers.set(agentAddress, subs);
|
|
2406
|
+
}
|
|
2407
|
+
subs.add(callback);
|
|
2408
|
+
return () => {
|
|
2409
|
+
const current = agentSubscribers.get(agentAddress);
|
|
2410
|
+
if (current === undefined)
|
|
2411
|
+
return;
|
|
2412
|
+
current.delete(callback);
|
|
2413
|
+
if (current.size === 0) {
|
|
2414
|
+
agentSubscribers.delete(agentAddress);
|
|
2415
|
+
}
|
|
2416
|
+
};
|
|
2417
|
+
}
|
|
2418
|
+
function getConnectedSidecars() {
|
|
2419
|
+
return Array.from(connections.values()).map((c) => c.sidecarId);
|
|
2420
|
+
}
|
|
2421
|
+
function getRoutableAddresses() {
|
|
2422
|
+
return Array.from(addressIndex.keys());
|
|
2423
|
+
}
|
|
2424
|
+
function getConnectorState(agentAddress) {
|
|
2425
|
+
return connectorStates.get(agentAddress) ?? null;
|
|
2426
|
+
}
|
|
2427
|
+
async function sendSourcesUpdate(agentAddress, sources, defaultSource) {
|
|
2428
|
+
await sendRequest(agentAddress, (requestId) => ({
|
|
2429
|
+
type: "sources.update",
|
|
2430
|
+
requestId,
|
|
2431
|
+
agentAddress,
|
|
2432
|
+
sources,
|
|
2433
|
+
defaultSource,
|
|
2434
|
+
}));
|
|
2435
|
+
}
|
|
2436
|
+
async function sendCredentialsUpdate(agentAddress, delivery) {
|
|
2437
|
+
await sendRequest(agentAddress, (requestId) => ({
|
|
2438
|
+
type: "credentials.update",
|
|
2439
|
+
requestId,
|
|
2440
|
+
agentAddress,
|
|
2441
|
+
delivery,
|
|
2442
|
+
}));
|
|
2443
|
+
}
|
|
2444
|
+
function sendSyncRequest(agentAddress) {
|
|
2445
|
+
const ws = addressIndex.get(agentAddress);
|
|
2446
|
+
if (ws === undefined) {
|
|
2447
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
2448
|
+
}
|
|
2449
|
+
const conn = connections.get(ws);
|
|
2450
|
+
if (conn === undefined) {
|
|
2451
|
+
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
2452
|
+
}
|
|
2453
|
+
const transferId = `sync-${++packCounter}`;
|
|
2454
|
+
conn.send({
|
|
2455
|
+
type: "sync.request",
|
|
2456
|
+
agentAddress,
|
|
2457
|
+
transferId,
|
|
2458
|
+
});
|
|
2459
|
+
}
|
|
2460
|
+
function sendSignalDeliver(opts) {
|
|
2461
|
+
const ws = addressIndex.get(opts.agentAddress);
|
|
2462
|
+
if (ws === undefined) {
|
|
2463
|
+
throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
|
|
2464
|
+
}
|
|
2465
|
+
const conn = connections.get(ws);
|
|
2466
|
+
if (conn === undefined) {
|
|
2467
|
+
throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
|
|
2468
|
+
}
|
|
2469
|
+
conn.send({
|
|
2470
|
+
type: "signal.deliver",
|
|
2471
|
+
agentAddress: opts.agentAddress,
|
|
2472
|
+
runId: opts.runId,
|
|
2473
|
+
signalName: opts.signalName,
|
|
2474
|
+
signalId: opts.signalId,
|
|
2475
|
+
payload: opts.payload,
|
|
2476
|
+
});
|
|
2477
|
+
}
|
|
2478
|
+
async function sendSignalDeliverToAllocation(target, opts) {
|
|
2479
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2480
|
+
if (addressIndex.get(opts.agentAddress) !== ws) {
|
|
2481
|
+
throw new Error(`Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2482
|
+
}
|
|
2483
|
+
conn.send({ type: "signal.deliver", ...opts });
|
|
2484
|
+
}
|
|
2485
|
+
function sendDrain(opts) {
|
|
2486
|
+
const ws = addressIndex.get(opts.agentAddress);
|
|
2487
|
+
if (ws === undefined) {
|
|
2488
|
+
throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
|
|
2489
|
+
}
|
|
2490
|
+
const conn = connections.get(ws);
|
|
2491
|
+
if (conn === undefined) {
|
|
2492
|
+
throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
|
|
2493
|
+
}
|
|
2494
|
+
conn.send({
|
|
2495
|
+
type: "drain.deliver",
|
|
2496
|
+
agentAddress: opts.agentAddress,
|
|
2497
|
+
deadlineMs: opts.deadlineMs,
|
|
2498
|
+
});
|
|
2499
|
+
}
|
|
2500
|
+
return {
|
|
2501
|
+
handleOpen,
|
|
2502
|
+
handleMessage,
|
|
2503
|
+
handleClose,
|
|
2504
|
+
routeMail,
|
|
2505
|
+
sendRunGrants,
|
|
2506
|
+
sendAgentDeploy,
|
|
2507
|
+
sendProbe,
|
|
2508
|
+
sendAgentUndeploy,
|
|
2509
|
+
sendSourcesUpdate,
|
|
2510
|
+
sendCredentialsUpdate,
|
|
2511
|
+
sendPack,
|
|
2512
|
+
sendPackToAllocation,
|
|
2513
|
+
sendWorkflowRunPackToAllocation,
|
|
2514
|
+
fenceAllocation,
|
|
2515
|
+
waitForAllocatedSidecar,
|
|
2516
|
+
isAllocatedSidecarReady,
|
|
2517
|
+
isAllocatedWorkflowActive,
|
|
2518
|
+
sendAgentDeployToAllocation,
|
|
2519
|
+
bindStepRoute,
|
|
2520
|
+
bindAllocatedStepRoute,
|
|
2521
|
+
unbindStepRoute,
|
|
2522
|
+
unbindAllocatedStepRoute,
|
|
2523
|
+
sendProvisionStep,
|
|
2524
|
+
sendProvisionStepToAllocation,
|
|
2525
|
+
sendWorkflowRunDispatchToAllocation,
|
|
2526
|
+
sendSyncRequest,
|
|
2527
|
+
sendSignalDeliver,
|
|
2528
|
+
sendSignalDeliverToAllocation,
|
|
2529
|
+
sendDrain,
|
|
2530
|
+
subscribeAgent,
|
|
2531
|
+
dispatchAgentEvent: dispatchToSubscribers,
|
|
2532
|
+
getConnectedSidecars,
|
|
2533
|
+
getRoutableAddresses,
|
|
2534
|
+
getConnectorState,
|
|
2535
|
+
events,
|
|
2536
|
+
};
|
|
2537
|
+
}
|