@intx/hub-sessions 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-repo.d.ts +14 -2
- package/dist/agent-repo.js +17 -4
- package/dist/agent-state-kind.js +14 -63
- package/dist/asset-service.js +14 -10
- package/dist/credential-push.d.ts +48 -4
- package/dist/credential-push.js +138 -6
- package/dist/event-collector-registry.d.ts +2 -1
- package/dist/event-collector-registry.js +38 -9
- package/dist/event-collector.d.ts +11 -1
- package/dist/event-collector.js +36 -3
- package/dist/hub-session-lookups.d.ts +1 -1
- package/dist/hub-session-lookups.js +68 -72
- package/dist/hub-session-orchestrator.d.ts +2 -3
- package/dist/hub-session-orchestrator.js +13 -12
- package/dist/index.d.ts +7 -6
- package/dist/index.js +7 -6
- package/dist/reconciliation-scheduler.d.ts +14 -0
- package/dist/reconciliation-scheduler.js +55 -0
- package/dist/repo-store/index.d.ts +1 -0
- package/dist/repo-store/index.js +1 -0
- package/dist/repo-store/user-principal-gate.d.ts +26 -0
- package/dist/repo-store/user-principal-gate.js +78 -0
- package/dist/session-service.d.ts +66 -121
- package/dist/session-service.js +444 -411
- package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
- package/dist/sidecar-allocation/capability-policy.js +124 -0
- package/dist/sidecar-allocation/contracts.d.ts +29 -6
- package/dist/sidecar-allocation/contracts.js +7 -2
- package/dist/sidecar-allocation/index.d.ts +4 -3
- package/dist/sidecar-allocation/index.js +3 -2
- package/dist/sidecar-allocation/operation.d.ts +10 -0
- package/dist/sidecar-allocation/operation.js +54 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +16 -3
- package/dist/sidecar-allocation/plugin-registry.js +36 -12
- package/dist/sidecar-allocation/reconciler.d.ts +16 -4
- package/dist/sidecar-allocation/reconciler.js +486 -92
- package/dist/skill-kind.js +8 -62
- package/dist/substrate.d.ts +1 -1
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +21 -15
- package/dist/workflow-allocation-service.js +440 -125
- package/dist/workflow-dispatch-service.d.ts +4 -2
- package/dist/workflow-dispatch-service.js +89 -26
- package/dist/workflow-kind.d.ts +12 -0
- package/dist/workflow-kind.js +17 -60
- package/dist/workflow-probe-gate.d.ts +99 -27
- package/dist/workflow-probe-gate.js +196 -21
- package/dist/workflow-run-kind.d.ts +112 -19
- package/dist/workflow-run-kind.js +626 -210
- package/dist/workflow-run-restore.d.ts +1 -0
- package/dist/workflow-run-restore.js +5 -1
- package/dist/workflow-source-pins.d.ts +8 -0
- package/dist/workflow-source-pins.js +14 -0
- package/dist/ws/index.d.ts +1 -1
- package/dist/ws/index.js +1 -1
- package/dist/ws/pending-tracker.d.ts +93 -0
- package/dist/ws/pending-tracker.js +132 -0
- package/dist/ws/sidecar-events.d.ts +43 -29
- package/dist/ws/sidecar-events.js +0 -2
- package/dist/ws/sidecar-handler.d.ts +122 -85
- package/dist/ws/sidecar-handler.js +925 -878
- package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
- package/dist/ws/sidecar-handler.test-helpers.js +95 -0
- package/dist/ws/sidecar-token-authenticator.js +37 -23
- package/package.json +13 -13
- package/dist/sidecar-allocation/placement-policy.d.ts +0 -11
- package/dist/sidecar-allocation/placement-policy.js +0 -21
|
@@ -4,19 +4,37 @@
|
|
|
4
4
|
// table of agentAddress → sidecar connection, and dispatches frames between
|
|
5
5
|
// sidecars and the hub's internal systems.
|
|
6
6
|
import { getLogger } from "@intx/log";
|
|
7
|
-
import { verifyEd25519 } from "@intx/crypto";
|
|
8
7
|
import { chunkPack, createPackReceiver } from "@intx/pack-transport";
|
|
9
|
-
import { base64Decode, deriveMessageId, deriveWorkflowRunId,
|
|
8
|
+
import { base64Decode, deriveMessageId, deriveWorkflowRunId, isRunAddress, } from "@intx/types";
|
|
10
9
|
import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
|
|
11
10
|
import { type } from "arktype";
|
|
12
|
-
import { SidecarFrame, } from "@intx/types/sidecar";
|
|
11
|
+
import { MAX_MAIL_OUTBOUND_BODY_BYTES, SidecarFrame, } from "@intx/types/sidecar";
|
|
13
12
|
import { createSidecarEmitter, } from "./sidecar-events.js";
|
|
13
|
+
import { PendingTracker, } from "./pending-tracker.js";
|
|
14
14
|
const logger = getLogger(["hub", "ws", "sidecar"]);
|
|
15
|
+
function deployFrameFailure(message, frameSent, cause) {
|
|
16
|
+
return Object.assign(new Error(message, { cause }), { frameSent });
|
|
17
|
+
}
|
|
18
|
+
export function isDeployFrameFailure(err) {
|
|
19
|
+
return (err instanceof Error &&
|
|
20
|
+
"frameSent" in err &&
|
|
21
|
+
typeof err.frameSent === "boolean");
|
|
22
|
+
}
|
|
15
23
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
|
|
24
|
+
* Identity validation failed or remained pending at the connection deadline.
|
|
25
|
+
* Readiness is unknown: the worker may be healthy behind the lookup, so
|
|
26
|
+
* callers must retry rather than treat this as a missed connection deadline.
|
|
27
|
+
*/
|
|
28
|
+
export class SidecarIdentityValidationError extends Error {
|
|
29
|
+
constructor(allocationId, generation, cause) {
|
|
30
|
+
super(`Cannot validate sidecar identity for allocation ${allocationId} generation ${String(generation)}`, { cause });
|
|
31
|
+
this.name = "SidecarIdentityValidationError";
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Whether this connection owns `address` for routing/lifecycle purposes.
|
|
36
|
+
* The legacy and workflow sets remain physically distinct, but ownership
|
|
37
|
+
* readers -- pack-transfer authorization,
|
|
20
38
|
* in-flight cancellation, disconnect teardown -- must see the union, or a
|
|
21
39
|
* reconnected workflow deployment (which lives only in `workflowAddresses`)
|
|
22
40
|
* is silently treated as unowned even though its mail routes.
|
|
@@ -31,12 +49,10 @@ function connOwnsAddress(conn, address) {
|
|
|
31
49
|
* repository.
|
|
32
50
|
*/
|
|
33
51
|
function connCanPushRepo(conn, agentAddress, repoId) {
|
|
34
|
-
if (conn.identity.kind
|
|
35
|
-
|
|
52
|
+
if (conn.identity.kind !== "allocated")
|
|
53
|
+
return false;
|
|
54
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
36
55
|
return false;
|
|
37
|
-
}
|
|
38
|
-
if (repoId.kind === "agent-state") {
|
|
39
|
-
return conn.identity.kind === "shared" && repoId.id === agentAddress;
|
|
40
56
|
}
|
|
41
57
|
return (repoId.kind === "workflow-run" &&
|
|
42
58
|
repoId.id === deriveWorkflowRunRepoId(agentAddress));
|
|
@@ -46,7 +62,6 @@ function ownedAddresses(conn) {
|
|
|
46
62
|
return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
|
|
47
63
|
}
|
|
48
64
|
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
49
|
-
const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
|
|
50
65
|
// A probe fetches a workflow's dependency closure from a registry and
|
|
51
66
|
// evaluates it on the sidecar, so it runs longer than a routine request; its
|
|
52
67
|
// default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS.
|
|
@@ -56,8 +71,25 @@ const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
|
|
|
56
71
|
const DEFAULT_PING_TIMEOUT_MS = 60_000;
|
|
57
72
|
const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000;
|
|
58
73
|
const DEFAULT_MAIL_ACK_MAX_RETRIES = 5;
|
|
74
|
+
// The hub re-resolves and re-pushes a key for each rotatable sender a sidecar
|
|
75
|
+
// reports on (re)connect. A legitimate sidecar caches keys for tens, maybe low
|
|
76
|
+
// hundreds of distinct user senders, so this cap sits well above ten times that
|
|
77
|
+
// ceiling: it NEVER truncates a real report -- dropping a genuine sender would
|
|
78
|
+
// leave its key stale, the exact failure this refresh exists to prevent. It
|
|
79
|
+
// bounds only a hostile or buggy sidecar, since a compromised authenticated
|
|
80
|
+
// sidecar could otherwise report an unbounded set and drive that many sequential
|
|
81
|
+
// DB resolves on every reconnect. The cap lives in the handler, not on the
|
|
82
|
+
// arktype frame schema, on purpose: rejecting an over-cap frame at parse would
|
|
83
|
+
// fail the whole reconnect (a hard outage) rather than degrade gracefully to a
|
|
84
|
+
// bounded refresh.
|
|
85
|
+
export const MAX_RESYNC_SENDER_ADDRESSES = 2048;
|
|
59
86
|
export function createSidecarRouter(config) {
|
|
60
|
-
const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,
|
|
87
|
+
const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, scheduleTimeout = (handler, ms) => {
|
|
88
|
+
const handle = setTimeout(handler, ms);
|
|
89
|
+
return () => {
|
|
90
|
+
clearTimeout(handle);
|
|
91
|
+
};
|
|
92
|
+
}, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
|
|
61
93
|
// Receiver-dispatch surface. Wire-layer callsites emit events here;
|
|
62
94
|
// host code subscribes via `router.events`.
|
|
63
95
|
const events = createSidecarEmitter();
|
|
@@ -68,10 +100,20 @@ export function createSidecarRouter(config) {
|
|
|
68
100
|
const allocationWaiters = new Map();
|
|
69
101
|
// agentAddress → ws handle (routing table)
|
|
70
102
|
const addressIndex = new Map();
|
|
71
|
-
// requestId → pending promise
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
103
|
+
// requestId → pending promise (resolved by session.ack, rejected by
|
|
104
|
+
// session.error). `PendingTracker` owns the register/timeout/settle/sweep
|
|
105
|
+
// lifecycle shared by all five pending round-trips below; each entry's
|
|
106
|
+
// resolve/reject closures carry the per-round-trip cleanup.
|
|
107
|
+
const pendingRequests = new PendingTracker();
|
|
108
|
+
// agentAddress → pending deploy promise (matched by agent.deploy.ack/agent.error)
|
|
109
|
+
const pendingDeploys = new PendingTracker();
|
|
110
|
+
// Run addresses whose ALLOCATED deploy is mid-flight -- key-record has been
|
|
111
|
+
// started but not yet committed. pendingDeploys clears at the deploy ack, but an
|
|
112
|
+
// allocated run's key is recorded LATER by session-service's anchor-key update,
|
|
113
|
+
// so pendingDeploys alone under-covers the allocated pre-ack window. session-
|
|
114
|
+
// service brackets this marker across its deploy try/catch: set before the
|
|
115
|
+
// deploy emit, cleared by noteSenderDeploySettled on record or failure.
|
|
116
|
+
const allocatedKeyRecordInFlight = new Map();
|
|
75
117
|
const disconnectedAgents = new Map();
|
|
76
118
|
const pendingMail = new Map();
|
|
77
119
|
// agentAddress → retention TTL timer for un-acked pending mail held across a
|
|
@@ -81,6 +123,7 @@ export function createSidecarRouter(config) {
|
|
|
81
123
|
// does not leak entries. Cleared when the address reconnects (redelivery) or
|
|
82
124
|
// its last pending entry is acked.
|
|
83
125
|
const pendingMailRetention = new Map();
|
|
126
|
+
const deferredSenderMail = new Map();
|
|
84
127
|
// agentAddress → set of subscriber callbacks for agent events
|
|
85
128
|
const agentSubscribers = new Map();
|
|
86
129
|
// agentAddress → cached connector-thread state, populated by
|
|
@@ -100,10 +143,17 @@ export function createSidecarRouter(config) {
|
|
|
100
143
|
// a single in-flight promise per ws (replaced each queued frame), cleared on
|
|
101
144
|
// close.
|
|
102
145
|
const messageChains = new Map();
|
|
103
|
-
const pendingPacks = new
|
|
146
|
+
const pendingPacks = new PendingTracker();
|
|
104
147
|
let packCounter = 0;
|
|
105
|
-
|
|
106
|
-
const
|
|
148
|
+
// agentAddress → pending undeploy (resolved by agent.undeploy.ack)
|
|
149
|
+
const pendingUndeploys = new PendingTracker();
|
|
150
|
+
// requestId → pending workflow probe (resolved by workflow.probe.result,
|
|
151
|
+
// rejected by workflow.probe.error). Result-carrying, unlike the other
|
|
152
|
+
// trackers (which resolve void): a probe returns the sidecar's inert
|
|
153
|
+
// projection + grant set + wire hash. Keyed on requestId alone -- the
|
|
154
|
+
// probe runs in the sidecar's pre-deploy state and enters no address map,
|
|
155
|
+
// so `handleClose`'s ws-keyed sweep is its ONLY disconnect cleanup.
|
|
156
|
+
const pendingProbes = new PendingTracker();
|
|
107
157
|
// Receives agent-state packs pushed from sidecars. The wire frames
|
|
108
158
|
// (`repo.pack.push` / `repo.pack.done`) are shared with the
|
|
109
159
|
// workflow-run flow; dispatch on `repoId.kind` picks which receiver
|
|
@@ -147,18 +197,16 @@ export function createSidecarRouter(config) {
|
|
|
147
197
|
entry.queue.push(frame);
|
|
148
198
|
return true;
|
|
149
199
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
logger.info `Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
|
|
161
|
-
}
|
|
200
|
+
// Arm a redelivery-retry timer for a tracked pending mail. Wraps the async
|
|
201
|
+
// `retryPendingMail` so a rejection -- a socket write that throws once the
|
|
202
|
+
// sidecar is gone -- is logged rather than floating out of the timer as an
|
|
203
|
+
// unhandled rejection.
|
|
204
|
+
function scheduleMailRetry(agentAddress, messageId) {
|
|
205
|
+
return scheduleTimeout(() => {
|
|
206
|
+
void retryPendingMail(agentAddress, messageId).catch((err) => {
|
|
207
|
+
logger.warn `Redelivery retry for mail ${messageId} to ${agentAddress} failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
208
|
+
});
|
|
209
|
+
}, mailAckRetryIntervalMs);
|
|
162
210
|
}
|
|
163
211
|
// Track a connected-window `mail.inbound` for redelivery until the sidecar
|
|
164
212
|
// acks its durable inbox write. Replaces any prior entry for the same
|
|
@@ -172,30 +220,129 @@ export function createSidecarRouter(config) {
|
|
|
172
220
|
}
|
|
173
221
|
const existing = byId.get(messageId);
|
|
174
222
|
if (existing !== undefined)
|
|
175
|
-
|
|
223
|
+
existing.cancelRetry();
|
|
176
224
|
byId.set(messageId, {
|
|
177
225
|
agentAddress,
|
|
178
226
|
messageId,
|
|
179
227
|
frame,
|
|
180
228
|
attempts: 0,
|
|
181
|
-
|
|
229
|
+
cancelRetry: scheduleMailRetry(agentAddress, messageId),
|
|
182
230
|
...(runGrants !== undefined ? { runGrants } : {}),
|
|
183
231
|
...(allocatedTarget !== undefined ? { allocatedTarget } : {}),
|
|
184
232
|
});
|
|
185
233
|
}
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
//
|
|
189
|
-
//
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
234
|
+
// Resolve the frame that must precede a redelivery of a trigger mail on the
|
|
235
|
+
// FIFO socket, re-resolving a keyless run sender's key so it still
|
|
236
|
+
// co-delivers. Returns:
|
|
237
|
+
// - a `run.grants` frame when the entry carries run grants (the redelivered
|
|
238
|
+
// run resolves its onRunStart barrier instead of failing closed on
|
|
239
|
+
// missing grants);
|
|
240
|
+
// - a bare `sender.key.refresh` frame when the entry carries NO run grants
|
|
241
|
+
// but its run sender's key was never co-delivered, so the recipient still
|
|
242
|
+
// caches the key ahead of the mail;
|
|
243
|
+
// - `undefined` when nothing must precede the mail.
|
|
244
|
+
//
|
|
245
|
+
// The re-resolve is KIND-GATED to run-address senders only. A run's
|
|
246
|
+
// deployment key is immutable once acked, so the re-resolved key equals the
|
|
247
|
+
// signing-time key -- safe. A user (non-run) sender's key may have rotated
|
|
248
|
+
// since it signed, so re-resolving would check the fixed signed bytes against
|
|
249
|
+
// a newer key and turn a valid message into a false `invalid`; such a sender
|
|
250
|
+
// lacking a captured key stays keyless (an honest `unknown`). An entry that
|
|
251
|
+
// captured `senderIdentities` at track time replays that snapshot as-is: it
|
|
252
|
+
// holds the signing-time key and is never re-resolved.
|
|
253
|
+
//
|
|
254
|
+
// Awaits any key resolve so the caller sends the returned frame and the mail
|
|
255
|
+
// back-to-back with no await between them, keeping the co-delivered key ahead
|
|
256
|
+
// of the mail on the FIFO socket.
|
|
257
|
+
async function resolveReplayLeadFrame(entry) {
|
|
258
|
+
const authenticatedSender = entry.frame.type === "mail.inbound"
|
|
259
|
+
? entry.frame.authenticatedSender
|
|
260
|
+
: undefined;
|
|
261
|
+
const senderIsRun = authenticatedSender !== undefined && isRunAddress(authenticatedSender);
|
|
262
|
+
if (entry.runGrants === undefined) {
|
|
263
|
+
if (authenticatedSender === undefined || !senderIsRun)
|
|
264
|
+
return undefined;
|
|
265
|
+
const key = await reresolveRunSenderKey(authenticatedSender);
|
|
266
|
+
if (key === null)
|
|
267
|
+
return undefined;
|
|
268
|
+
return {
|
|
269
|
+
type: "sender.key.refresh",
|
|
270
|
+
address: authenticatedSender,
|
|
271
|
+
publicKey: key,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
let senderIdentities = entry.runGrants.senderIdentities;
|
|
275
|
+
if (senderIdentities === undefined &&
|
|
276
|
+
authenticatedSender !== undefined &&
|
|
277
|
+
senderIsRun) {
|
|
278
|
+
const key = await reresolveRunSenderKey(authenticatedSender);
|
|
279
|
+
senderIdentities = senderIdentitiesFromKey(authenticatedSender, key);
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
194
282
|
type: "run.grants",
|
|
195
283
|
agentAddress: entry.agentAddress,
|
|
196
284
|
runId: entry.runGrants.runId,
|
|
197
285
|
stepGrants: entry.runGrants.stepGrants,
|
|
198
|
-
|
|
286
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
// Best-effort re-resolve of a run sender's hub-held key at replay time. Only
|
|
290
|
+
// called for a run-address sender, whose deployment key is immutable once
|
|
291
|
+
// acked, so the current key equals the signing-time key. Returns null when no
|
|
292
|
+
// resolver is wired or the sender has no durable key.
|
|
293
|
+
//
|
|
294
|
+
// This relies on `lookups.resolveSenderKey` being the BEST-EFFORT,
|
|
295
|
+
// NEVER-THROWS resolver (the contract at sidecar-events.ts:274-279, wired to
|
|
296
|
+
// resolveFrameSenderKey, which swallows faults to null). That contract is
|
|
297
|
+
// load-bearing here: `redeliverPendingMail` clears the retention TTL up-front
|
|
298
|
+
// and re-arms each entry's per-entry timer only on a successful send, so a
|
|
299
|
+
// resolver that THREW would abort the redeliver loop and strand the
|
|
300
|
+
// not-yet-processed entries with no timer and no TTL until a process restart.
|
|
301
|
+
// A strict/throwing resolver must NOT be wired here. Do not add a try/catch:
|
|
302
|
+
// the boundary owns the never-throws contract; duplicating it here would
|
|
303
|
+
// violate that ownership. The dispatch-time resolveSenderKey call
|
|
304
|
+
// (sendWorkflowRunDispatchToAllocation path) carries the same dependency
|
|
305
|
+
// note.
|
|
306
|
+
async function reresolveRunSenderKey(authenticatedSender) {
|
|
307
|
+
const resolveSenderKey = lookups.resolveSenderKey;
|
|
308
|
+
if (resolveSenderKey === undefined)
|
|
309
|
+
return null;
|
|
310
|
+
return resolveSenderKey(authenticatedSender);
|
|
311
|
+
}
|
|
312
|
+
// Replay a pending mail's lead frame (its run grants or a re-resolved sender
|
|
313
|
+
// key) and then the mail itself over `conn`. Awaits the resolve FIRST, then
|
|
314
|
+
// sends the lead frame and the mail back-to-back with NO await between them,
|
|
315
|
+
// so the co-delivered key always precedes the mail on the FIFO socket.
|
|
316
|
+
// Returns whether the mail was (re)sent, so the caller re-arms the retry timer
|
|
317
|
+
// only for an entry it actually redelivered.
|
|
318
|
+
async function replaySendPendingMail(conn, entry) {
|
|
319
|
+
const lead = await resolveReplayLeadFrame(entry);
|
|
320
|
+
// The resolve above may have awaited real I/O; during that gap a queued
|
|
321
|
+
// `mail.inbound.ack` can advance and run `resolvePendingMail` (delete +
|
|
322
|
+
// clearTimeout) on this entry. The window is opened by the timer-macrotask
|
|
323
|
+
// retry path, NOT by any bypass: `mail.inbound.ack` is a QUEUED frame
|
|
324
|
+
// (frameBypassesQueue returns false for it). It can interleave because the
|
|
325
|
+
// retry runs as an independent setTimeout macrotask (retryPendingMail), so
|
|
326
|
+
// the owning ws's message chain is free to advance the ack during the
|
|
327
|
+
// resolve await. On the reconnect/redeliver path the ack cannot interleave
|
|
328
|
+
// at all -- it queues behind the still-running reconnect handler on the
|
|
329
|
+
// same ws -- so here this guard is pure defense-in-depth. Re-confirm it is
|
|
330
|
+
// still the tracked entry before sending, or a post-ack redelivery would
|
|
331
|
+
// arm a retry timer on a detached entry.
|
|
332
|
+
if (pendingMail.get(entry.agentAddress)?.get(entry.messageId) !== entry) {
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
// The same gap can span a disconnect or a takeover that moves the address
|
|
336
|
+
// off `conn`. Sending on the stale conn would write to a dead socket and
|
|
337
|
+
// re-arm a retry that later drops a still-retained entry. Skip so the entry
|
|
338
|
+
// survives for the reconnect redelivery.
|
|
339
|
+
const ws = addressIndex.get(entry.agentAddress);
|
|
340
|
+
if (ws === undefined || connections.get(ws) !== conn)
|
|
341
|
+
return false;
|
|
342
|
+
if (lead !== undefined)
|
|
343
|
+
conn.send(lead);
|
|
344
|
+
conn.send(entry.frame);
|
|
345
|
+
return true;
|
|
199
346
|
}
|
|
200
347
|
function deletePendingMail(byId, agentAddress, messageId) {
|
|
201
348
|
byId.delete(messageId);
|
|
@@ -210,7 +357,7 @@ export function createSidecarRouter(config) {
|
|
|
210
357
|
}
|
|
211
358
|
}
|
|
212
359
|
}
|
|
213
|
-
function retryPendingMail(agentAddress, messageId) {
|
|
360
|
+
async function retryPendingMail(agentAddress, messageId) {
|
|
214
361
|
const byId = pendingMail.get(agentAddress);
|
|
215
362
|
if (byId === undefined)
|
|
216
363
|
return;
|
|
@@ -252,10 +399,10 @@ export function createSidecarRouter(config) {
|
|
|
252
399
|
logger.warn `Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`;
|
|
253
400
|
return;
|
|
254
401
|
}
|
|
402
|
+
if (!(await replaySendPendingMail(conn, entry)))
|
|
403
|
+
return;
|
|
255
404
|
entry.attempts += 1;
|
|
256
|
-
|
|
257
|
-
conn.send(entry.frame);
|
|
258
|
-
entry.timer = setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs);
|
|
405
|
+
entry.cancelRetry = scheduleMailRetry(agentAddress, messageId);
|
|
259
406
|
}
|
|
260
407
|
function resolvePendingMail(agentAddress, messageId) {
|
|
261
408
|
const byId = pendingMail.get(agentAddress);
|
|
@@ -264,7 +411,7 @@ export function createSidecarRouter(config) {
|
|
|
264
411
|
const entry = byId.get(messageId);
|
|
265
412
|
if (entry === undefined)
|
|
266
413
|
return;
|
|
267
|
-
|
|
414
|
+
entry.cancelRetry();
|
|
268
415
|
deletePendingMail(byId, agentAddress, messageId);
|
|
269
416
|
}
|
|
270
417
|
// Hold an address's un-acked pending mail across a disconnect. The per-entry
|
|
@@ -279,7 +426,7 @@ export function createSidecarRouter(config) {
|
|
|
279
426
|
if (byId === undefined)
|
|
280
427
|
return;
|
|
281
428
|
for (const entry of byId.values())
|
|
282
|
-
|
|
429
|
+
entry.cancelRetry();
|
|
283
430
|
const existing = pendingMailRetention.get(agentAddress);
|
|
284
431
|
if (existing !== undefined)
|
|
285
432
|
clearTimeout(existing);
|
|
@@ -307,7 +454,7 @@ export function createSidecarRouter(config) {
|
|
|
307
454
|
// (effectively-once) and processes one it had dropped (no loss). Re-arms the
|
|
308
455
|
// connected-window retry over the new connection with a fresh per-generation
|
|
309
456
|
// budget, so a redelivery that is itself dropped before its ack is retried.
|
|
310
|
-
function redeliverPendingMail(agentAddress, conn) {
|
|
457
|
+
async function redeliverPendingMail(agentAddress, conn) {
|
|
311
458
|
const retention = pendingMailRetention.get(agentAddress);
|
|
312
459
|
if (retention !== undefined) {
|
|
313
460
|
clearTimeout(retention);
|
|
@@ -324,14 +471,14 @@ export function createSidecarRouter(config) {
|
|
|
324
471
|
// The Hub-owned dispatch row survives generation replacement and will
|
|
325
472
|
// be requeued by the allocation-ready callback. Do not leak or replay
|
|
326
473
|
// this generation-local retry entry onto a different worker.
|
|
327
|
-
|
|
474
|
+
entry.cancelRetry();
|
|
328
475
|
deletePendingMail(byId, agentAddress, entry.messageId);
|
|
329
476
|
continue;
|
|
330
477
|
}
|
|
331
|
-
|
|
332
|
-
|
|
478
|
+
if (!(await replaySendPendingMail(conn, entry)))
|
|
479
|
+
continue;
|
|
333
480
|
entry.attempts = 0;
|
|
334
|
-
entry.
|
|
481
|
+
entry.cancelRetry = scheduleMailRetry(agentAddress, entry.messageId);
|
|
335
482
|
}
|
|
336
483
|
if (byId.size > 0) {
|
|
337
484
|
logger.info `Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`;
|
|
@@ -340,13 +487,13 @@ export function createSidecarRouter(config) {
|
|
|
340
487
|
function resetLivenessTimer(ws) {
|
|
341
488
|
const existing = livenessTimers.get(ws);
|
|
342
489
|
if (existing !== undefined)
|
|
343
|
-
|
|
344
|
-
const
|
|
490
|
+
existing();
|
|
491
|
+
const cancel = scheduleTimeout(() => {
|
|
345
492
|
livenessTimers.delete(ws);
|
|
346
493
|
logger.warn `Sidecar ping timeout, closing connection`;
|
|
347
494
|
ws.close();
|
|
348
495
|
}, pingTimeoutMs);
|
|
349
|
-
livenessTimers.set(ws,
|
|
496
|
+
livenessTimers.set(ws, cancel);
|
|
350
497
|
}
|
|
351
498
|
function handlePing(ws) {
|
|
352
499
|
resetLivenessTimer(ws);
|
|
@@ -409,11 +556,9 @@ export function createSidecarRouter(config) {
|
|
|
409
556
|
// state. Such a frame has no ordering obligation against new inbound frames
|
|
410
557
|
// (a response cannot resolve "too early" for a request that already went
|
|
411
558
|
// out), and it is exactly what in-flight queued handlers block on, so it MUST
|
|
412
|
-
// run out of band or
|
|
413
|
-
//
|
|
414
|
-
//
|
|
415
|
-
// inbound payload whose order matters, so it queues. The exhaustive switch +
|
|
416
|
-
// assertNever makes adding a SidecarFrame variant without classifying it a
|
|
559
|
+
// run out of band. Every other frame establishes or reads routing, or carries
|
|
560
|
+
// an inbound payload whose order matters, so it queues. The exhaustive switch
|
|
561
|
+
// + assertNever makes adding a SidecarFrame variant without classifying it a
|
|
417
562
|
// compile error, not a latent deadlock or a silent bypass hole.
|
|
418
563
|
function frameBypassesQueue(frame) {
|
|
419
564
|
switch (frame.type) {
|
|
@@ -430,7 +575,6 @@ export function createSidecarRouter(config) {
|
|
|
430
575
|
return true;
|
|
431
576
|
case "register":
|
|
432
577
|
case "reconnect":
|
|
433
|
-
case "challenge.response":
|
|
434
578
|
case "mail.outbound":
|
|
435
579
|
case "agent.event":
|
|
436
580
|
case "connector.state.changed":
|
|
@@ -445,22 +589,31 @@ export function createSidecarRouter(config) {
|
|
|
445
589
|
}
|
|
446
590
|
// Runs one frame's handler. Returns the handler's promise for async handlers
|
|
447
591
|
// so the per-ws chain can await bounded completion; sync handlers return
|
|
448
|
-
// void. Never awaits a promise that resolves on a
|
|
449
|
-
// only such await (the challenge round-trip's session.ack) is reached via a
|
|
450
|
-
// bypass frame, which does not queue.
|
|
592
|
+
// void. Never awaits a promise that resolves on a later same-ws frame.
|
|
451
593
|
function dispatchFrame(ws, frame) {
|
|
594
|
+
const registeredIdentity = connections.get(ws)?.identity;
|
|
595
|
+
if (registeredIdentity?.kind === "probe" &&
|
|
596
|
+
frame.type !== "register" &&
|
|
597
|
+
frame.type !== "reconnect" &&
|
|
598
|
+
frame.type !== "ping" &&
|
|
599
|
+
frame.type !== "workflow.probe.result" &&
|
|
600
|
+
frame.type !== "workflow.probe.error") {
|
|
601
|
+
logger.warn `Rejected ${frame.type} from probe sidecar ${registeredIdentity.sidecarId}`;
|
|
602
|
+
handleClose(ws);
|
|
603
|
+
ws.close();
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
452
606
|
switch (frame.type) {
|
|
453
607
|
case "register": {
|
|
454
608
|
const agentAddresses = frame.agentAddresses;
|
|
455
|
-
|
|
609
|
+
const cachedSenderAddresses = frame.cachedSenderAddresses ?? [];
|
|
610
|
+
return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses, cachedSenderAddresses));
|
|
456
611
|
}
|
|
457
612
|
case "reconnect": {
|
|
458
613
|
const agentAddresses = frame.agentAddresses;
|
|
459
|
-
const
|
|
460
|
-
return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses,
|
|
614
|
+
const cachedSenderAddresses = frame.cachedSenderAddresses ?? [];
|
|
615
|
+
return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses));
|
|
461
616
|
}
|
|
462
|
-
case "challenge.response":
|
|
463
|
-
return handleChallengeResponse(ws, frame.responses);
|
|
464
617
|
case "agent.deploy.ack":
|
|
465
618
|
return handleDeployAck(ws, frame);
|
|
466
619
|
case "agent.error":
|
|
@@ -473,21 +626,45 @@ export function createSidecarRouter(config) {
|
|
|
473
626
|
case "ping":
|
|
474
627
|
handlePing(ws);
|
|
475
628
|
return;
|
|
476
|
-
case "mail.outbound":
|
|
477
|
-
|
|
478
|
-
|
|
629
|
+
case "mail.outbound": {
|
|
630
|
+
const conn = connections.get(ws);
|
|
631
|
+
if (conn === undefined)
|
|
632
|
+
return;
|
|
633
|
+
if (!connOwnsAddress(conn, frame.senderAddress)) {
|
|
634
|
+
logger.warn `Dropping mail.outbound from ${frame.senderAddress}: not registered to this sidecar`;
|
|
635
|
+
return;
|
|
479
636
|
}
|
|
480
|
-
|
|
481
|
-
|
|
637
|
+
// The DoS backstop and the trust boundary for an untrusted sidecar's
|
|
638
|
+
// mail body: measure the true byte cost (a hostile sidecar can send
|
|
639
|
+
// multi-byte UTF-8, so `.length` would undercount) and drop an over-cap
|
|
640
|
+
// frame here before either delivery path allocates on it. The socket's
|
|
641
|
+
// maxPayloadLength has already closed the connection for a truly huge
|
|
642
|
+
// frame; this catches one between the mail cap and that ceiling.
|
|
643
|
+
const bodyBytes = Buffer.byteLength(frame.rawMessage, "utf8");
|
|
644
|
+
if (bodyBytes > MAX_MAIL_OUTBOUND_BODY_BYTES) {
|
|
645
|
+
logger.warn `Dropping mail.outbound from ${frame.senderAddress}: rawMessage of ${String(bodyBytes)} bytes exceeds the ${String(MAX_MAIL_OUTBOUND_BODY_BYTES)}-byte cap`;
|
|
646
|
+
return;
|
|
482
647
|
}
|
|
483
|
-
if (
|
|
484
|
-
|
|
648
|
+
if (frame.delivered !== true) {
|
|
649
|
+
// frame.senderAddress is the sender this connection was just gated
|
|
650
|
+
// on by connOwnsAddress above -- a hub-verified value. Thread it so
|
|
651
|
+
// the relayed inbound frame is stamped with it, not the MIME From.
|
|
652
|
+
return handleMailOutbound(frame.rawMessage, frame.senderAddress, frame.recipients);
|
|
485
653
|
}
|
|
486
|
-
|
|
487
|
-
|
|
654
|
+
if (lookups.persistMail) {
|
|
655
|
+
return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
|
|
488
656
|
}
|
|
657
|
+
logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
|
|
489
658
|
return;
|
|
490
|
-
|
|
659
|
+
}
|
|
660
|
+
case "agent.event": {
|
|
661
|
+
const conn = connections.get(ws);
|
|
662
|
+
if (conn === undefined)
|
|
663
|
+
return;
|
|
664
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
665
|
+
logger.warn `Dropping agent.event for ${frame.agentAddress}: not registered to this sidecar`;
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
491
668
|
events.emit("agent.event", {
|
|
492
669
|
agentAddress: frame.agentAddress,
|
|
493
670
|
sessionId: frame.sessionId,
|
|
@@ -495,6 +672,7 @@ export function createSidecarRouter(config) {
|
|
|
495
672
|
});
|
|
496
673
|
dispatchToSubscribers(frame.agentAddress, frame.event);
|
|
497
674
|
return;
|
|
675
|
+
}
|
|
498
676
|
case "connector.state.changed":
|
|
499
677
|
// Gate the cache write on the sending sidecar actually owning
|
|
500
678
|
// the named agent. A misbehaving sidecar that knows another
|
|
@@ -542,10 +720,10 @@ export function createSidecarRouter(config) {
|
|
|
542
720
|
case "signal.correlation.register":
|
|
543
721
|
return handleSignalCorrelationRegister(ws, frame);
|
|
544
722
|
case "session.ack":
|
|
545
|
-
|
|
723
|
+
pendingRequests.resolve(frame.requestId);
|
|
546
724
|
return;
|
|
547
725
|
case "session.error":
|
|
548
|
-
|
|
726
|
+
pendingRequests.reject(frame.requestId, frame.error);
|
|
549
727
|
return;
|
|
550
728
|
case "repo.pack.ack":
|
|
551
729
|
resolvePackPending(ws, frame);
|
|
@@ -559,7 +737,7 @@ export function createSidecarRouter(config) {
|
|
|
559
737
|
case "repo.pack.done":
|
|
560
738
|
return handlePackDone(ws, frame);
|
|
561
739
|
case "workflow.probe.result":
|
|
562
|
-
resolveProbe(frame.requestId, {
|
|
740
|
+
resolveProbe(ws, frame.requestId, {
|
|
563
741
|
projection: frame.projection,
|
|
564
742
|
grants: frame.grants,
|
|
565
743
|
grantWalkSnapshot: frame.grantWalkSnapshot,
|
|
@@ -567,7 +745,7 @@ export function createSidecarRouter(config) {
|
|
|
567
745
|
});
|
|
568
746
|
return;
|
|
569
747
|
case "workflow.probe.error":
|
|
570
|
-
rejectProbe(frame.requestId, frame.error);
|
|
748
|
+
rejectProbe(ws, frame.requestId, frame.error);
|
|
571
749
|
return;
|
|
572
750
|
default:
|
|
573
751
|
return assertNever(frame);
|
|
@@ -613,24 +791,61 @@ export function createSidecarRouter(config) {
|
|
|
613
791
|
const current = allocatedConnections.get(allocationId);
|
|
614
792
|
if (waiters === undefined || current === undefined)
|
|
615
793
|
return;
|
|
616
|
-
|
|
794
|
+
const matchingWaiters = [...waiters].filter((waiter) => waiter.generation === current.identity.generation);
|
|
795
|
+
if (matchingWaiters.length === 0)
|
|
617
796
|
return;
|
|
618
|
-
|
|
619
|
-
|
|
797
|
+
const validation = Promise.resolve().then(() => validateSidecarIdentity(current.identity, "readiness"));
|
|
798
|
+
for (const waiter of matchingWaiters) {
|
|
799
|
+
waiter.validations.add(validation);
|
|
800
|
+
waiter.onValidation?.(validation);
|
|
801
|
+
}
|
|
802
|
+
let identityCurrent;
|
|
803
|
+
try {
|
|
804
|
+
identityCurrent = await validation;
|
|
805
|
+
}
|
|
806
|
+
catch (cause) {
|
|
807
|
+
// A failed revalidation leaves the waiters parked: a later register
|
|
808
|
+
// revalidates, and at expiry the wait reports the failure rather than a
|
|
809
|
+
// missed deadline. Registration itself was already gated, so this must
|
|
810
|
+
// not fail the connection that just registered.
|
|
811
|
+
const validationFailure = new SidecarIdentityValidationError(allocationId, current.identity.generation, cause);
|
|
812
|
+
for (const waiter of matchingWaiters) {
|
|
813
|
+
waiter.validationFailure = validationFailure;
|
|
814
|
+
}
|
|
815
|
+
return;
|
|
816
|
+
}
|
|
817
|
+
finally {
|
|
818
|
+
for (const waiter of matchingWaiters) {
|
|
819
|
+
waiter.validations.delete(validation);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
// A clean validation supersedes earlier failures: expiry must report the
|
|
823
|
+
// current reading, not a stale transient.
|
|
824
|
+
for (const waiter of matchingWaiters) {
|
|
825
|
+
delete waiter.validationFailure;
|
|
826
|
+
}
|
|
827
|
+
if (!identityCurrent || allocatedConnections.get(allocationId) !== current)
|
|
828
|
+
return;
|
|
829
|
+
for (const waiter of matchingWaiters) {
|
|
830
|
+
if (!waiters.delete(waiter))
|
|
620
831
|
continue;
|
|
621
832
|
clearTimeout(waiter.timer);
|
|
622
|
-
waiters.delete(waiter);
|
|
623
833
|
waiter.resolve();
|
|
624
834
|
}
|
|
625
|
-
if (waiters.size === 0)
|
|
835
|
+
if (waiters.size === 0 && allocationWaiters.get(allocationId) === waiters)
|
|
626
836
|
allocationWaiters.delete(allocationId);
|
|
627
837
|
}
|
|
628
|
-
async function handleAllocatedRegister(ws, identity, agentAddresses) {
|
|
838
|
+
async function handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses) {
|
|
629
839
|
if (allocationFences.get(identity.allocationId) !== identity.generation) {
|
|
630
840
|
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`;
|
|
631
841
|
ws.close();
|
|
632
842
|
return;
|
|
633
843
|
}
|
|
844
|
+
if (identity.kind === "probe" && agentAddresses.length > 0) {
|
|
845
|
+
logger.warn `Rejected probe sidecar ${identity.sidecarId}: probe ${identity.allocationId} claimed workflow addresses`;
|
|
846
|
+
ws.close();
|
|
847
|
+
return;
|
|
848
|
+
}
|
|
634
849
|
if (agentAddresses.length > 0 &&
|
|
635
850
|
!(await validateSidecarIdentity(identity, "routing"))) {
|
|
636
851
|
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`;
|
|
@@ -644,7 +859,8 @@ export function createSidecarRouter(config) {
|
|
|
644
859
|
existingOnSocket.identity.allocationId === identity.allocationId &&
|
|
645
860
|
addressIndex.get(address) === ws &&
|
|
646
861
|
connOwnsAddress(existingOnSocket, address);
|
|
647
|
-
if (
|
|
862
|
+
if (identity.kind !== "allocated" ||
|
|
863
|
+
(!alreadyOwned && address !== identity.workflowRunAddress)) {
|
|
648
864
|
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`;
|
|
649
865
|
ws.close();
|
|
650
866
|
return;
|
|
@@ -674,7 +890,7 @@ export function createSidecarRouter(config) {
|
|
|
674
890
|
ws.send(JSON.stringify(frame));
|
|
675
891
|
},
|
|
676
892
|
};
|
|
677
|
-
if (conn.identity.kind !==
|
|
893
|
+
if (conn.identity.kind !== identity.kind ||
|
|
678
894
|
conn.identity.allocationId !== identity.allocationId ||
|
|
679
895
|
conn.identity.generation !== identity.generation) {
|
|
680
896
|
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`;
|
|
@@ -688,472 +904,275 @@ export function createSidecarRouter(config) {
|
|
|
688
904
|
}
|
|
689
905
|
allocatedConnections.set(identity.allocationId, { ws, identity });
|
|
690
906
|
for (const address of newlyRoutedAddresses) {
|
|
691
|
-
redeliverPendingMail(address, conn);
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
}
|
|
705
|
-
const sidecarId = identity.sidecarId;
|
|
706
|
-
// Key-existence gate. A register frame is token-authenticated but carries
|
|
707
|
-
// no per-address ownership proof, so it may route an address ONLY if that
|
|
708
|
-
// address has no stored key yet -- a genuine keyless first-deploy (the
|
|
709
|
-
// token-bounded first-deploy trust model). An address that already has a
|
|
710
|
-
// key must prove ownership through the challenged reconnect path; routing
|
|
711
|
-
// it here on token auth alone is the register-frame sibling of the
|
|
712
|
-
// reconnect hijack. The keyless-only set is computed up front, BEFORE the
|
|
713
|
-
// ghost-cleanup and every routing mutation below, so a rejected address
|
|
714
|
-
// touches nothing: no eviction of a live owner, hence no downgrade from
|
|
715
|
-
// hijack to denial-of-service on the victim.
|
|
716
|
-
const lookupKey = lookups.lookupPublicKey;
|
|
717
|
-
const routableAddresses = [];
|
|
718
|
-
for (const addr of agentAddresses) {
|
|
719
|
-
if (lookupKey === undefined) {
|
|
720
|
-
// Fail closed: without the ownership lookup a keyed address cannot be
|
|
721
|
-
// told apart from a first-deploy, so route nothing and surface the
|
|
722
|
-
// misconfiguration. Empty first-connect registers never reach here.
|
|
723
|
-
logger.error `Cannot gate register routing for ${addr}: lookupPublicKey is not configured; refusing to route (challenged reconnect required)`;
|
|
724
|
-
continue;
|
|
725
|
-
}
|
|
726
|
-
let existingKey;
|
|
727
|
-
try {
|
|
728
|
-
existingKey = await lookupKey(addr);
|
|
729
|
-
}
|
|
730
|
-
catch (err) {
|
|
731
|
-
// Fail closed on a lookup error (e.g. a transient DB failure): route
|
|
732
|
-
// nothing for this address and surface the failure, rather than let
|
|
733
|
-
// the rejection float out of this void-dispatched handler and take
|
|
734
|
-
// down the hub.
|
|
735
|
-
logger.error `Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`;
|
|
736
|
-
continue;
|
|
737
|
-
}
|
|
738
|
-
if (existingKey !== null) {
|
|
739
|
-
logger.warn `Refusing to route ${addr} via register: address already has a stored key; ownership must be proven via challenged reconnect`;
|
|
740
|
-
continue;
|
|
907
|
+
await redeliverPendingMail(address, conn);
|
|
908
|
+
}
|
|
909
|
+
// Reconcile a reconnecting deployment's credentials, closing the offline
|
|
910
|
+
// window: a credential revoked, deleted, or rotated while the sidecar was
|
|
911
|
+
// disconnected is applied to the child now. Fire-and-forget so
|
|
912
|
+
// registration is not blocked; the lookup no-ops for a run that persisted
|
|
913
|
+
// no credential refs.
|
|
914
|
+
const resyncCredentials = lookups.resyncCredentials;
|
|
915
|
+
if (resyncCredentials !== undefined) {
|
|
916
|
+
for (const address of newlyRoutedAddresses) {
|
|
917
|
+
if (!isRunAddress(address))
|
|
918
|
+
continue;
|
|
919
|
+
resyncCredentials(address);
|
|
741
920
|
}
|
|
742
|
-
routableAddresses.push(addr);
|
|
743
921
|
}
|
|
744
|
-
//
|
|
745
|
-
//
|
|
746
|
-
//
|
|
747
|
-
//
|
|
748
|
-
//
|
|
749
|
-
//
|
|
750
|
-
//
|
|
751
|
-
//
|
|
752
|
-
//
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
if (prevConn !== undefined) {
|
|
767
|
-
prevConn.agentAddresses.delete(addr);
|
|
768
|
-
}
|
|
769
|
-
// The new owner is about to take over; the prior owner's
|
|
770
|
-
// cached state must not survive into the new owner's window
|
|
771
|
-
// before its bootstrap frame arrives.
|
|
772
|
-
connectorStates.delete(addr);
|
|
922
|
+
// Reconcile the sidecar's cached sender keys, closing the offline window: a
|
|
923
|
+
// user-principal key that rotated while the sidecar was disconnected is
|
|
924
|
+
// re-resolved and re-pushed, and a sender whose principal was DELETED while
|
|
925
|
+
// the sidecar was disconnected is evicted, so the recipient stops verifying
|
|
926
|
+
// either against a key the hub no longer vouches for. Only allocated
|
|
927
|
+
// sidecars host a sender cache worth reconciling. Resolve and push
|
|
928
|
+
// SEQUENTIALLY in one detached task: registration is never blocked, and a
|
|
929
|
+
// large cache cannot fan out into one concurrent DB query per reported
|
|
930
|
+
// sender on every reconnect.
|
|
931
|
+
const resolveSenderKeyStrict = lookups.resolveSenderKeyStrict;
|
|
932
|
+
if (identity.kind === "allocated" && resolveSenderKeyStrict !== undefined) {
|
|
933
|
+
const rotatableSenders = new Set(cachedSenderAddresses);
|
|
934
|
+
// Resolve-don't-trust applied to input SIZE: bound the reported set before
|
|
935
|
+
// acting on it. Run addresses count toward the cap by design -- the
|
|
936
|
+
// isRunAddress skip below is inside the loop, so the iteration, and thus
|
|
937
|
+
// the DB resolves, can never exceed the cap regardless of the run/non-run
|
|
938
|
+
// mix. Over the cap, reconcile the first MAX_RESYNC_SENDER_ADDRESSES and
|
|
939
|
+
// log the overflow so a misbehaving sidecar is detectable.
|
|
940
|
+
let sendersToResync = [...rotatableSenders];
|
|
941
|
+
if (sendersToResync.length > MAX_RESYNC_SENDER_ADDRESSES) {
|
|
942
|
+
logger.warn `Sidecar ${identity.sidecarId} reported ${String(sendersToResync.length)} cached sender addresses on allocation ${identity.allocationId} generation ${String(identity.generation)}, over the ${String(MAX_RESYNC_SENDER_ADDRESSES)} resync cap; reconciling the first ${String(MAX_RESYNC_SENDER_ADDRESSES)} and ignoring the rest`;
|
|
943
|
+
sendersToResync = sendersToResync.slice(0, MAX_RESYNC_SENDER_ADDRESSES);
|
|
773
944
|
}
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
// Use reconnect with challenge/response to preserve queued messages.
|
|
808
|
-
const staleQueue = disconnectedAgents.get(addr);
|
|
809
|
-
if (staleQueue !== undefined) {
|
|
810
|
-
clearTimeout(staleQueue.timer);
|
|
811
|
-
if (staleQueue.queue.length > 0) {
|
|
812
|
-
logger.warn `Discarding ${String(staleQueue.queue.length)} queued message(s) for ${addr} on unverified register`;
|
|
945
|
+
void (async () => {
|
|
946
|
+
for (const address of sendersToResync) {
|
|
947
|
+
// The sidecar already reports only non-run senders, but do not trust
|
|
948
|
+
// the report: a run sender's key is the immutable
|
|
949
|
+
// workflow_run.public_key and is never refreshed or evicted, so skip
|
|
950
|
+
// it here too rather than couple correctness to the sidecar's filter.
|
|
951
|
+
if (isRunAddress(address))
|
|
952
|
+
continue;
|
|
953
|
+
// Tri-state, deleted-vs-fault distinguished by the STRICT resolver:
|
|
954
|
+
// - resolves to a key -> refresh the sidecar's cached key;
|
|
955
|
+
// - CONFIRMED null (no matching principal = a deleted sender) ->
|
|
956
|
+
// evict it;
|
|
957
|
+
// - THROWS (fault: ambiguous address, keyless-principal invariant
|
|
958
|
+
// break, DB error) -> keep the stale key, evict nothing.
|
|
959
|
+
// Never evicting on a fault is the load-bearing property: dropping a
|
|
960
|
+
// live key on a transient DB fault would be worse than doing nothing.
|
|
961
|
+
// Only the resolve is guarded here; conn.send stays outside so a
|
|
962
|
+
// socket-gone throw propagates to the outer catch and stops the loop.
|
|
963
|
+
let publicKey;
|
|
964
|
+
try {
|
|
965
|
+
publicKey = await resolveSenderKeyStrict(address);
|
|
966
|
+
}
|
|
967
|
+
catch (cause) {
|
|
968
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
969
|
+
logger.error `Keeping the stale cached key for ${address}: resolving it faulted (a fault, not a deleted sender): ${message}`;
|
|
970
|
+
continue;
|
|
971
|
+
}
|
|
972
|
+
if (publicKey !== null) {
|
|
973
|
+
conn.send({ type: "sender.key.refresh", address, publicKey });
|
|
974
|
+
}
|
|
975
|
+
else {
|
|
976
|
+
conn.send({ type: "sender.key.evict", address });
|
|
977
|
+
}
|
|
813
978
|
}
|
|
814
|
-
|
|
815
|
-
|
|
979
|
+
})().catch((cause) => {
|
|
980
|
+
// The per-address resolve is guarded above, so the only throw reaching
|
|
981
|
+
// here is conn.send (JSON.stringify + the socket write) once the sidecar
|
|
982
|
+
// is gone. That means the connection left, so stop -- the remaining
|
|
983
|
+
// sends would fail the same way.
|
|
984
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
985
|
+
logger.warn `Sender-key resync for sidecar ${identity.sidecarId} stopped: ${message}`;
|
|
986
|
+
});
|
|
816
987
|
}
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
// above and must re-enter routing through the challenged reconnect path.
|
|
820
|
-
// Because the gate runs before the ghost-cleanup, a rejected (keyed)
|
|
821
|
-
// address never evicts its prior owner -- register cannot reclaim or
|
|
822
|
-
// disrupt a victim's route on token auth alone.
|
|
823
|
-
logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
|
|
824
|
-
}
|
|
825
|
-
async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
|
|
988
|
+
logger.info `Provisioned sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
|
|
989
|
+
await notifyAllocationWaiters(identity.allocationId);
|
|
826
990
|
if (identity.kind === "allocated") {
|
|
827
|
-
|
|
828
|
-
|
|
991
|
+
events.emit("sidecar.allocated.connected", {
|
|
992
|
+
allocationId: identity.allocationId,
|
|
993
|
+
generation: identity.generation,
|
|
994
|
+
});
|
|
829
995
|
}
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
996
|
+
}
|
|
997
|
+
async function handleRegister(ws, identity, agentAddresses, cachedSenderAddresses) {
|
|
998
|
+
await handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses);
|
|
999
|
+
}
|
|
1000
|
+
async function handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses) {
|
|
1001
|
+
await handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses);
|
|
1002
|
+
}
|
|
1003
|
+
// Park a pre-ack sender's mail synchronously and return its entry. Registering
|
|
1004
|
+
// the entry BEFORE the caller awaits `resolveSenderKey` is the interlock that
|
|
1005
|
+
// guarantees a settle landing during the resolve has an entry to find: the
|
|
1006
|
+
// event loop is single-threaded, so no settle can interleave between this
|
|
1007
|
+
// synchronous registration and the caller's first await.
|
|
1008
|
+
function parkDeferredSenderMail(authenticatedSender, rawMessage, recipients) {
|
|
1009
|
+
let parked = deferredSenderMail.get(authenticatedSender);
|
|
1010
|
+
if (parked === undefined) {
|
|
1011
|
+
parked = new Set();
|
|
1012
|
+
deferredSenderMail.set(authenticatedSender, parked);
|
|
836
1013
|
}
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
// freshly-deployed agent from routing.
|
|
851
|
-
const previouslyOwned = new Set(connections.get(ws)?.agentAddresses);
|
|
852
|
-
// Register the sidecar connection immediately (with no addresses) so it
|
|
853
|
-
// can receive frames while the ownership challenge is pending. Every
|
|
854
|
-
// reconnect address -- session and workflow-derived alike -- enters
|
|
855
|
-
// routing only through the verified path below, never unchallenged here.
|
|
856
|
-
// Empty address list, so the key-existence gate has nothing to await.
|
|
857
|
-
// The identity was already verified at the dispatch boundary, so the
|
|
858
|
-
// internal register does not re-authenticate.
|
|
859
|
-
await handleRegister(ws, identity, []);
|
|
860
|
-
const conn = connections.get(ws);
|
|
861
|
-
if (conn === undefined)
|
|
862
|
-
return;
|
|
863
|
-
// Re-add the still-owned addresses the register cleared but this
|
|
864
|
-
// reconnect is not re-challenging. The challenged addresses below
|
|
865
|
-
// re-enter addressIndex through the verified path instead.
|
|
866
|
-
const claimedAddresses = new Set(agentAddresses);
|
|
867
|
-
for (const addr of previouslyOwned) {
|
|
868
|
-
if (claimedAddresses.has(addr))
|
|
869
|
-
continue;
|
|
870
|
-
conn.agentAddresses.add(addr);
|
|
871
|
-
addressIndex.set(addr, ws);
|
|
872
|
-
}
|
|
873
|
-
// Look up stored public keys for all claimed addresses. Fail closed on a
|
|
874
|
-
// lookup error (e.g. a transient DB failure): treat the address as
|
|
875
|
-
// unverifiable so it fails its challenge and stays unrouted, rather than
|
|
876
|
-
// letting the rejection float out of this void-dispatched handler as an
|
|
877
|
-
// unhandled rejection that could take down the hub.
|
|
878
|
-
const keyLookups = await Promise.all(agentAddresses.map(async (addr) => {
|
|
879
|
-
try {
|
|
880
|
-
return { address: addr, publicKeyHex: await lookupKey(addr) };
|
|
881
|
-
}
|
|
882
|
-
catch (err) {
|
|
883
|
-
logger.error `Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`;
|
|
884
|
-
return { address: addr, publicKeyHex: null };
|
|
885
|
-
}
|
|
886
|
-
}));
|
|
887
|
-
// If the connection was closed or superseded while we were awaiting
|
|
888
|
-
// key lookups, bail out.
|
|
889
|
-
if (!connections.has(ws))
|
|
890
|
-
return;
|
|
891
|
-
const challenges = new Map();
|
|
892
|
-
const challengeEntries = [];
|
|
893
|
-
for (const { address, publicKeyHex } of keyLookups) {
|
|
894
|
-
if (publicKeyHex === null) {
|
|
895
|
-
conn.send({
|
|
896
|
-
type: "challenge.failed",
|
|
897
|
-
address,
|
|
898
|
-
reason: "Unknown run address",
|
|
899
|
-
});
|
|
900
|
-
continue;
|
|
901
|
-
}
|
|
902
|
-
let publicKey;
|
|
903
|
-
try {
|
|
904
|
-
publicKey = hexDecode(publicKeyHex);
|
|
905
|
-
}
|
|
906
|
-
catch {
|
|
907
|
-
conn.send({
|
|
908
|
-
type: "challenge.failed",
|
|
909
|
-
address,
|
|
910
|
-
reason: "Stored public key is corrupt",
|
|
1014
|
+
const entry = {
|
|
1015
|
+
authenticatedSender,
|
|
1016
|
+
rawMessage,
|
|
1017
|
+
recipients,
|
|
1018
|
+
timer: setTimeout(() => {
|
|
1019
|
+
// TTL backstop for the case where a settle never arrives (the sender's
|
|
1020
|
+
// deploy never acked and never failed loudly). Claim the entry and
|
|
1021
|
+
// surface it as undelivered so the mail is not held forever.
|
|
1022
|
+
if (!claimDeferredSenderEntry(entry))
|
|
1023
|
+
return;
|
|
1024
|
+
events.emit("mail.outbound.undelivered", {
|
|
1025
|
+
rawMessage: entry.rawMessage,
|
|
1026
|
+
recipients: entry.recipients,
|
|
911
1027
|
});
|
|
912
|
-
logger.
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
const
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
conn.send({ type: "challenge", challenges: challengeEntries });
|
|
1028
|
+
logger.warn `Dropping mail from ${entry.authenticatedSender}: its sender key was not recorded before the deferred-mail TTL expired`;
|
|
1029
|
+
}, disconnectQueueTTLMs),
|
|
1030
|
+
};
|
|
1031
|
+
parked.add(entry);
|
|
1032
|
+
return entry;
|
|
1033
|
+
}
|
|
1034
|
+
// Remove one parked entry by identity, clearing its TTL timer. Returns whether
|
|
1035
|
+
// THIS call removed it. The inline-deliver path, a settle, and the TTL all
|
|
1036
|
+
// race to claim the same entry; only the claimer acts on it, so a claim that
|
|
1037
|
+
// finds nothing (already claimed) is a no-op. This is the idempotent
|
|
1038
|
+
// remove-by-key that keeps a settle and the inline non-null branch from both
|
|
1039
|
+
// delivering the same message.
|
|
1040
|
+
function claimDeferredSenderEntry(entry) {
|
|
1041
|
+
const parked = deferredSenderMail.get(entry.authenticatedSender);
|
|
1042
|
+
if (parked === undefined)
|
|
1043
|
+
return false;
|
|
1044
|
+
const claimed = parked.delete(entry);
|
|
1045
|
+
if (!claimed)
|
|
1046
|
+
return false;
|
|
1047
|
+
clearTimeout(entry.timer);
|
|
1048
|
+
if (parked.size === 0)
|
|
1049
|
+
deferredSenderMail.delete(entry.authenticatedSender);
|
|
1050
|
+
return true;
|
|
936
1051
|
}
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
const
|
|
946
|
-
|
|
1052
|
+
// Claim every entry parked for a sender, clearing their TTL timers. A later
|
|
1053
|
+
// settle or TTL for the same sender then finds nothing.
|
|
1054
|
+
function claimAllDeferredSenderMail(authenticatedSender) {
|
|
1055
|
+
const parked = deferredSenderMail.get(authenticatedSender);
|
|
1056
|
+
if (parked === undefined)
|
|
1057
|
+
return [];
|
|
1058
|
+
deferredSenderMail.delete(authenticatedSender);
|
|
1059
|
+
const entries = [...parked];
|
|
1060
|
+
for (const entry of entries)
|
|
1061
|
+
clearTimeout(entry.timer);
|
|
1062
|
+
return entries;
|
|
1063
|
+
}
|
|
1064
|
+
function drainDeferredSenderMail(authenticatedSender, reason) {
|
|
1065
|
+
const entries = claimAllDeferredSenderMail(authenticatedSender);
|
|
1066
|
+
if (entries.length === 0)
|
|
947
1067
|
return;
|
|
948
|
-
const
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
if (entry === undefined) {
|
|
954
|
-
conn.send({
|
|
955
|
-
type: "challenge.failed",
|
|
956
|
-
address,
|
|
957
|
-
reason: "Address was not challenged",
|
|
958
|
-
});
|
|
959
|
-
continue;
|
|
960
|
-
}
|
|
961
|
-
let valid = false;
|
|
962
|
-
try {
|
|
963
|
-
const nonceBytes = entry.nonce;
|
|
964
|
-
const addressBytes = new TextEncoder().encode(address);
|
|
965
|
-
const payload = new Uint8Array(nonceBytes.length + addressBytes.length);
|
|
966
|
-
payload.set(nonceBytes);
|
|
967
|
-
payload.set(addressBytes, nonceBytes.length);
|
|
968
|
-
const sigBytes = hexDecode(signature);
|
|
969
|
-
valid = await verifyEd25519(payload, sigBytes, entry.publicKey);
|
|
970
|
-
}
|
|
971
|
-
catch (err) {
|
|
972
|
-
logger.warn `Challenge failed for ${address}: ${err instanceof Error ? err.message : String(err)}`;
|
|
973
|
-
}
|
|
974
|
-
if (valid) {
|
|
975
|
-
verified.push(address);
|
|
976
|
-
}
|
|
977
|
-
else {
|
|
978
|
-
conn.send({
|
|
979
|
-
type: "challenge.failed",
|
|
980
|
-
address,
|
|
981
|
-
reason: "Signature verification failed",
|
|
982
|
-
});
|
|
983
|
-
}
|
|
1068
|
+
for (const entry of entries) {
|
|
1069
|
+
events.emit("mail.outbound.undelivered", {
|
|
1070
|
+
rawMessage: entry.rawMessage,
|
|
1071
|
+
recipients: entry.recipients,
|
|
1072
|
+
});
|
|
984
1073
|
}
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
address,
|
|
991
|
-
reason: "No response provided for challenged address",
|
|
992
|
-
});
|
|
993
|
-
logger.warn `Challenge failed for ${address}: no response provided`;
|
|
994
|
-
}
|
|
1074
|
+
logger.warn `Dropping ${String(entries.length)} deferred message(s) from ${authenticatedSender}: ${reason}`;
|
|
1075
|
+
}
|
|
1076
|
+
function noteSenderDeployStarted(address, attempt) {
|
|
1077
|
+
if (allocatedKeyRecordInFlight.has(address)) {
|
|
1078
|
+
throw new Error(`Sender deployment ${address} has an unresolved attempt`);
|
|
995
1079
|
}
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
const prevWs = addressIndex.get(addr);
|
|
1006
|
-
if (prevWs !== undefined && prevWs !== ws) {
|
|
1007
|
-
connectorStates.delete(addr);
|
|
1008
|
-
// Evict the reclaimed address from the superseded connection's owned
|
|
1009
|
-
// set. handleClose's cancelByAgent sweep iterates a connection's owned
|
|
1010
|
-
// union WITHOUT an ownership guard, so if the stale connection still
|
|
1011
|
-
// listed this address it would cancel THIS connection's in-flight pack
|
|
1012
|
-
// transfer for it when it finally closes. Delete from both sets: a
|
|
1013
|
-
// workflow-derived address lives on the workflow set, a launched agent
|
|
1014
|
-
// on the session set, and delete is a no-op for the absent one.
|
|
1015
|
-
const prevConn = connections.get(prevWs);
|
|
1016
|
-
if (prevConn !== undefined) {
|
|
1017
|
-
prevConn.workflowAddresses.delete(addr);
|
|
1018
|
-
prevConn.agentAddresses.delete(addr);
|
|
1080
|
+
allocatedKeyRecordInFlight.set(address, attempt);
|
|
1081
|
+
}
|
|
1082
|
+
function noteSenderDeploySettled(sender, outcome) {
|
|
1083
|
+
if (typeof sender !== "string") {
|
|
1084
|
+
for (const [address, attempt] of [...allocatedKeyRecordInFlight]) {
|
|
1085
|
+
if (attempt.allocationId !== sender.allocationId ||
|
|
1086
|
+
attempt.generation !== sender.generation ||
|
|
1087
|
+
("leaseId" in sender && attempt.leaseId !== sender.leaseId)) {
|
|
1088
|
+
continue;
|
|
1019
1089
|
}
|
|
1090
|
+
allocatedKeyRecordInFlight.delete(address);
|
|
1091
|
+
settleSenderMail(address, outcome);
|
|
1020
1092
|
}
|
|
1021
|
-
|
|
1022
|
-
// in-flight state is reconstructed sidecar-locally on the next reconnect),
|
|
1023
|
-
// so handleClose reclaims it correctly. The routing pointer is the same
|
|
1024
|
-
// either way; only now it is written behind a passed challenge.
|
|
1025
|
-
//
|
|
1026
|
-
// The `else` (session set, queued for reconnect) is the retired
|
|
1027
|
-
// launched-agent path: launched agents no longer exist (the folded-launch
|
|
1028
|
-
// route was removed), so no current producer reaches it. It is left in
|
|
1029
|
-
// place for the reconnect-subsystem teardown to remove as its own
|
|
1030
|
-
// reviewable change, not folded into this collapse.
|
|
1031
|
-
if (isRunAddress(addr)) {
|
|
1032
|
-
conn.workflowAddresses.add(addr);
|
|
1033
|
-
}
|
|
1034
|
-
else {
|
|
1035
|
-
conn.agentAddresses.add(addr);
|
|
1036
|
-
}
|
|
1037
|
-
addressIndex.set(addr, ws);
|
|
1038
|
-
}
|
|
1039
|
-
const ready = [];
|
|
1040
|
-
const failed = [];
|
|
1041
|
-
for (const addr of verified) {
|
|
1042
|
-
// A workflow run needs routing + queue flush only, which the passed
|
|
1043
|
-
// challenge has now made safe: its in-flight state is reconstructed
|
|
1044
|
-
// sidecar-locally, so it does not enrol in the `agent.reconnected` session
|
|
1045
|
-
// reaction (event-collector restore). Skip the reaction for it.
|
|
1046
|
-
//
|
|
1047
|
-
// The reaction below was the retired launched-agent path (restore a
|
|
1048
|
-
// launched agent's collector). Launched agents no longer exist, so no
|
|
1049
|
-
// current producer reaches it; the `agent.reconnected` listener is gone,
|
|
1050
|
-
// so `listenerCount` is 0 and the emit is inert. Left for the
|
|
1051
|
-
// reconnect-subsystem teardown to remove.
|
|
1052
|
-
if (isRunAddress(addr)) {
|
|
1053
|
-
ready.push(addr);
|
|
1054
|
-
continue;
|
|
1055
|
-
}
|
|
1056
|
-
if (events.listenerCount("agent.reconnected") === 0) {
|
|
1057
|
-
ready.push(addr);
|
|
1058
|
-
continue;
|
|
1059
|
-
}
|
|
1060
|
-
try {
|
|
1061
|
-
await events.emitAndAwait("agent.reconnected", { agentAddress: addr });
|
|
1062
|
-
ready.push(addr);
|
|
1063
|
-
}
|
|
1064
|
-
catch (err) {
|
|
1065
|
-
logger.error `Failed to handle reconnection for ${addr}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1066
|
-
failed.push(addr);
|
|
1067
|
-
}
|
|
1093
|
+
return;
|
|
1068
1094
|
}
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
// Roll back failed addresses from the routing table. Only the session set
|
|
1078
|
-
// is touched: a workflow-derived address can never be in `failed` -- it
|
|
1079
|
-
// early-`continue`s to `ready` above, before the reaction that populates
|
|
1080
|
-
// `failed` -- so it is never on the workflow set at this point.
|
|
1081
|
-
for (const addr of failed) {
|
|
1082
|
-
conn.agentAddresses.delete(addr);
|
|
1083
|
-
addressIndex.delete(addr);
|
|
1084
|
-
}
|
|
1085
|
-
// Flush queued messages and redeliver retained un-acked mail, only for
|
|
1086
|
-
// ready addresses. The disconnect queue carries mail that arrived WHILE
|
|
1087
|
-
// disconnected; pending-mail redelivery carries mail that was delivered
|
|
1088
|
-
// over the prior live connection but never acked (the connected-window
|
|
1089
|
-
// drop). Both replay over the verified new connection.
|
|
1090
|
-
for (const addr of ready) {
|
|
1091
|
-
flushDisconnectedQueue(addr, conn);
|
|
1092
|
-
redeliverPendingMail(addr, conn);
|
|
1093
|
-
}
|
|
1094
|
-
// Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
|
|
1095
|
-
// so reconnect completion is not blocked on pack transfer. The
|
|
1096
|
-
// wire layer owns the staleness comparison; the event fires only
|
|
1097
|
-
// when staleness is confirmed.
|
|
1098
|
-
const checkDeployRef = lookups.lookupDeployRef;
|
|
1099
|
-
if (checkDeployRef !== undefined) {
|
|
1100
|
-
for (const addr of ready) {
|
|
1101
|
-
// Workflow deployments are pinned-forever: a deployment keeps its
|
|
1102
|
-
// deploy-time definition until an explicit undeploy/redeploy, so the
|
|
1103
|
-
// deploy-ref freshness catch-up is deliberately NOT run for a run
|
|
1104
|
-
// address. A definition edited on the hub while the sidecar was
|
|
1105
|
-
// disconnected does not reconcile on reconnect; it affects only newly
|
|
1106
|
-
// created deployments. The deployment's in-flight run state is
|
|
1107
|
-
// reconstructed sidecar-locally at restore, not re-fetched here. Do NOT
|
|
1108
|
-
// add a reconcile path for these addresses -- see the "Workflow
|
|
1109
|
-
// Definition Versioning: Pinned-Forever" note under "Reconnect
|
|
1110
|
-
// Sequencing" in docs/IMPLEMENTATION.md. (Every routable address is a
|
|
1111
|
-
// run address now; the fall-through was the retired launched-agent
|
|
1112
|
-
// catch-up path.)
|
|
1113
|
-
if (isRunAddress(addr))
|
|
1114
|
-
continue;
|
|
1115
|
-
void (async () => {
|
|
1116
|
-
try {
|
|
1117
|
-
const hubRef = await checkDeployRef(addr);
|
|
1118
|
-
if (hubRef === null)
|
|
1119
|
-
return;
|
|
1120
|
-
const sidecarRef = challenge.deployRefs[addr];
|
|
1121
|
-
if (sidecarRef === hubRef)
|
|
1122
|
-
return;
|
|
1123
|
-
logger.info `Re-deploying ${addr}: sidecar ref ${sidecarRef ?? "(none)"} != hub ref ${hubRef.slice(0, 8)}`;
|
|
1124
|
-
await events.emitAndAwait("deploy.ref.stale", {
|
|
1125
|
-
agentAddress: addr,
|
|
1126
|
-
});
|
|
1127
|
-
}
|
|
1128
|
-
catch (err) {
|
|
1129
|
-
logger.error `Failed to re-deploy ${addr} after reconnect: ${err instanceof Error ? err.message : String(err)}`;
|
|
1130
|
-
}
|
|
1131
|
-
})();
|
|
1132
|
-
}
|
|
1095
|
+
if (allocatedKeyRecordInFlight.has(sender))
|
|
1096
|
+
return;
|
|
1097
|
+
settleSenderMail(sender, outcome);
|
|
1098
|
+
}
|
|
1099
|
+
function settleSenderMail(address, outcome) {
|
|
1100
|
+
if ("failed" in outcome) {
|
|
1101
|
+
drainDeferredSenderMail(address, `sender deploy failed: ${outcome.failed}`);
|
|
1102
|
+
return;
|
|
1133
1103
|
}
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
if (expired !== undefined) {
|
|
1144
|
-
surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
|
|
1145
|
-
}
|
|
1146
|
-
}, disconnectQueueTTLMs);
|
|
1147
|
-
}
|
|
1148
|
-
conn.send({
|
|
1149
|
-
type: "challenge.failed",
|
|
1150
|
-
address: addr,
|
|
1151
|
-
reason: "Reconnection rejected by governance",
|
|
1104
|
+
for (const entry of claimAllDeferredSenderMail(address)) {
|
|
1105
|
+
// Re-drive delivery as its OWN task, off the settle's stack, so delivery
|
|
1106
|
+
// work never runs on the deploy-ack handler's stack. Carry the confirmed
|
|
1107
|
+
// key: another attempt may start before this task runs, and must not
|
|
1108
|
+
// capture this mail or change the key that authenticates it.
|
|
1109
|
+
void Promise.resolve()
|
|
1110
|
+
.then(() => handleMailOutbound(entry.rawMessage, entry.authenticatedSender, entry.recipients, outcome.recorded))
|
|
1111
|
+
.catch((err) => {
|
|
1112
|
+
logger.error `Re-driving deferred mail from ${entry.authenticatedSender} failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
1152
1113
|
});
|
|
1153
1114
|
}
|
|
1154
|
-
logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
|
|
1155
1115
|
}
|
|
1156
|
-
|
|
1116
|
+
function senderIdentitiesFromKey(address, publicKey) {
|
|
1117
|
+
return publicKey !== null ? [{ address, publicKey }] : undefined;
|
|
1118
|
+
}
|
|
1119
|
+
// Resolve the co-delivered sender identities for a message, applying the
|
|
1120
|
+
// register-before-read interlock for a pre-ack run sender. Returns either
|
|
1121
|
+
// `deliver: true` with the resolved identities (undefined when there is no
|
|
1122
|
+
// resolvable key), or `deliver: false` when the message is parked and will be
|
|
1123
|
+
// driven later by a settle (`noteSenderDeploySettled`) or the TTL.
|
|
1124
|
+
async function resolveSenderIdentitiesOrPark(rawMessage, authenticatedSender, recipients) {
|
|
1125
|
+
const resolveSenderKey = lookups.resolveSenderKey;
|
|
1126
|
+
if (resolveSenderKey === undefined)
|
|
1127
|
+
return { deliver: true, senderIdentities: undefined };
|
|
1128
|
+
// The co-delivered key is consumed only by a run recipient caching it from the
|
|
1129
|
+
// run.grants frame. Purely external/federated mail never uses it and is never
|
|
1130
|
+
// locally verified, so resolve nothing and never park it.
|
|
1131
|
+
if (!recipients.some(isRunAddress))
|
|
1132
|
+
return { deliver: true, senderIdentities: undefined };
|
|
1133
|
+
// A stable-key (non-run) sender has no pre-ack window; resolve inline.
|
|
1134
|
+
if (!isRunAddress(authenticatedSender)) {
|
|
1135
|
+
const key = await resolveSenderKey(authenticatedSender);
|
|
1136
|
+
return {
|
|
1137
|
+
deliver: true,
|
|
1138
|
+
senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
// Park a run sender ONLY while a key-record settle is guaranteed to arrive -- a
|
|
1142
|
+
// deploy is in flight. Without one, a null resolve is a transient fault or a
|
|
1143
|
+
// genuine absence on an already-settled run: no settle is coming, so parking
|
|
1144
|
+
// would strand the mail to the TTL. Deliver on the normal path instead.
|
|
1145
|
+
const settleGuaranteed = pendingDeploys.has(authenticatedSender) ||
|
|
1146
|
+
allocatedKeyRecordInFlight.has(authenticatedSender);
|
|
1147
|
+
if (!settleGuaranteed) {
|
|
1148
|
+
const key = await resolveSenderKey(authenticatedSender);
|
|
1149
|
+
return {
|
|
1150
|
+
deliver: true,
|
|
1151
|
+
senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
// Register-before-read: park a waiter entry synchronously (NO await) so a
|
|
1155
|
+
// settle that lands while we resolve below has an entry to find, THEN
|
|
1156
|
+
// resolve. The single-threaded event loop cannot interleave a settle between
|
|
1157
|
+
// this registration and the await.
|
|
1158
|
+
const entry = parkDeferredSenderMail(authenticatedSender, rawMessage, recipients);
|
|
1159
|
+
const key = await resolveSenderKey(authenticatedSender);
|
|
1160
|
+
if (key === null) {
|
|
1161
|
+
// Not recorded yet. Leave the entry parked; a settle or the TTL drives it.
|
|
1162
|
+
return { deliver: false };
|
|
1163
|
+
}
|
|
1164
|
+
// The key was already recorded before we parked. Claim our entry and deliver
|
|
1165
|
+
// inline -- unless a concurrent settle already claimed it and is re-driving
|
|
1166
|
+
// this message, in which case claiming fails and we must NOT deliver again.
|
|
1167
|
+
if (!claimDeferredSenderEntry(entry)) {
|
|
1168
|
+
return { deliver: false };
|
|
1169
|
+
}
|
|
1170
|
+
return {
|
|
1171
|
+
deliver: true,
|
|
1172
|
+
senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
|
|
1173
|
+
};
|
|
1174
|
+
}
|
|
1175
|
+
async function handleMailOutbound(rawMessage, authenticatedSender, recipients, recordedSenderKey) {
|
|
1157
1176
|
// A mail addressed to more than one workflow deployment would birth a
|
|
1158
1177
|
// run per recipient from a single inbound mail. The stable runId
|
|
1159
1178
|
// removed the Message-ID collision that originally forced this guard --
|
|
@@ -1176,6 +1195,23 @@ export function createSidecarRouter(config) {
|
|
|
1176
1195
|
throw new Error(`mail addressed to multiple workflow-derived recipients (${workflowRecipients.join(", ")}); materializing a run for more than one workflow deployment from a single mail is unsupported`);
|
|
1177
1196
|
}
|
|
1178
1197
|
}
|
|
1198
|
+
// Resolve the sender's hub-held key ONCE for the whole message, ahead of the
|
|
1199
|
+
// recipient fan-out and any grant materialization, so every recipient in
|
|
1200
|
+
// this fan-out binds the same key snapshot. A run-address sender may be
|
|
1201
|
+
// pre-ack -- it minted its keypair locally and can send before the hub
|
|
1202
|
+
// records its public key. The register-before-read interlock holds such mail
|
|
1203
|
+
// until the key lands rather than delivering it keyless, which a strict
|
|
1204
|
+
// recipient drops as an unknown sender. A parked message returns here and is
|
|
1205
|
+
// re-driven later by a settle or the TTL.
|
|
1206
|
+
const resolution = recordedSenderKey === undefined
|
|
1207
|
+
? await resolveSenderIdentitiesOrPark(rawMessage, authenticatedSender, recipients)
|
|
1208
|
+
: {
|
|
1209
|
+
deliver: true,
|
|
1210
|
+
senderIdentities: senderIdentitiesFromKey(authenticatedSender, recordedSenderKey),
|
|
1211
|
+
};
|
|
1212
|
+
if (!resolution.deliver)
|
|
1213
|
+
return;
|
|
1214
|
+
const senderIdentities = resolution.senderIdentities;
|
|
1179
1215
|
// Route to locally connected sidecars first, then try disconnect queues.
|
|
1180
1216
|
const unrouted = [];
|
|
1181
1217
|
for (const recipient of recipients) {
|
|
@@ -1184,7 +1220,7 @@ export function createSidecarRouter(config) {
|
|
|
1184
1220
|
// co-recipients. The catch fails THIS recipient closed (its run never
|
|
1185
1221
|
// starts under-authorized) and continues to the rest.
|
|
1186
1222
|
try {
|
|
1187
|
-
const outcome = await deliverMailToRecipient(recipient, rawMessage);
|
|
1223
|
+
const outcome = await deliverMailToRecipient(recipient, rawMessage, authenticatedSender, senderIdentities);
|
|
1188
1224
|
if (outcome === "unrouted")
|
|
1189
1225
|
unrouted.push(recipient);
|
|
1190
1226
|
}
|
|
@@ -1219,10 +1255,19 @@ export function createSidecarRouter(config) {
|
|
|
1219
1255
|
// grants rather than failing closed. Reservation happens before routing so
|
|
1220
1256
|
// concurrent first deliveries cannot send different snapshots; a routing
|
|
1221
1257
|
// failure leaves a grants-only, still-unfired run.
|
|
1222
|
-
async function deliverMailToRecipient(recipient, rawMessage) {
|
|
1258
|
+
async function deliverMailToRecipient(recipient, rawMessage, authenticatedSender, senderIdentities) {
|
|
1223
1259
|
if (lookups.materializeMailTriggeredRunGrants !== undefined &&
|
|
1224
1260
|
isRunAddress(recipient)) {
|
|
1225
1261
|
const runId = deriveWorkflowRunId(recipient);
|
|
1262
|
+
// This does NOT let mail mutate a run's authorization. First delivery
|
|
1263
|
+
// reserves and commits the run's grants (the mail IS the trigger);
|
|
1264
|
+
// every later delivery only RE-READS the current committed grants
|
|
1265
|
+
// (`loadCommittedRunGrants`) and re-asserts them ahead of the dispatch.
|
|
1266
|
+
// The committed rows already carry any standing-approval change (an
|
|
1267
|
+
// approve/reject-with-`always` resolution mutates them through its own
|
|
1268
|
+
// path), so this re-send is idempotent -- it re-establishes the run's
|
|
1269
|
+
// current floor on the sidecar, self-healing a `grants.json` a sidecar
|
|
1270
|
+
// may have lost, and never overwrites it with anything staler.
|
|
1226
1271
|
const result = await lookups.materializeMailTriggeredRunGrants({
|
|
1227
1272
|
agentAddress: recipient,
|
|
1228
1273
|
runId,
|
|
@@ -1235,11 +1280,23 @@ export function createSidecarRouter(config) {
|
|
|
1235
1280
|
return "failed-closed";
|
|
1236
1281
|
}
|
|
1237
1282
|
if (result.outcome === "materialized") {
|
|
1283
|
+
// The sender's hub-held key was resolved ONCE in handleMailOutbound,
|
|
1284
|
+
// ahead of this fan-out, and threaded in as `senderIdentities`. Co-
|
|
1285
|
+
// deliver it on the run's grants barrier so a recipient that caches from
|
|
1286
|
+
// the `run.grants` frame binds the sender address to the key and can
|
|
1287
|
+
// verify the sender's mail locally. A null key is never carried (the
|
|
1288
|
+
// list is undefined then), so the "authorized-with-a-key implies key
|
|
1289
|
+
// cached" invariant holds; a recipient with no cached key resolves such
|
|
1290
|
+
// mail as `unknown`, which its admission policy rejects by default (a
|
|
1291
|
+
// workflow may relax `unknown` to admit).
|
|
1292
|
+
// Finish asynchronous preparation before sending the grants and mail
|
|
1293
|
+
// together, keeping another delivery's key out of the gap between them.
|
|
1294
|
+
const messageId = await deriveMessageId(base64Decode(rawMessage));
|
|
1238
1295
|
// Send the run's grants ahead of the mail. A `false` here means the
|
|
1239
1296
|
// deployment is unroutable. Do not route the mail that would dispatch
|
|
1240
1297
|
// it; the grants-only reservation remains the canonical snapshot for a
|
|
1241
1298
|
// later first-delivery attempt.
|
|
1242
|
-
if (!sendRunGrants(recipient, runId, result.stepGrants)) {
|
|
1299
|
+
if (!sendRunGrants(recipient, runId, result.stepGrants, senderIdentities)) {
|
|
1243
1300
|
logger.error `Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`;
|
|
1244
1301
|
return "unrouted";
|
|
1245
1302
|
}
|
|
@@ -1253,8 +1310,11 @@ export function createSidecarRouter(config) {
|
|
|
1253
1310
|
// mail's own id (derived over the same bytes the sidecar derives), so a
|
|
1254
1311
|
// redelivery replays identically and the downstream RunStarted /
|
|
1255
1312
|
// stable-runId dedup makes it effectively-once.
|
|
1256
|
-
const
|
|
1257
|
-
|
|
1313
|
+
const outcome = routeMail(recipient, rawMessage, authenticatedSender, messageId, {
|
|
1314
|
+
runId,
|
|
1315
|
+
stepGrants: result.stepGrants,
|
|
1316
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
1317
|
+
})
|
|
1258
1318
|
? "routed"
|
|
1259
1319
|
: "unrouted";
|
|
1260
1320
|
return outcome;
|
|
@@ -1263,7 +1323,9 @@ export function createSidecarRouter(config) {
|
|
|
1263
1323
|
// the mail without grants -- the run, if any, is not ours to
|
|
1264
1324
|
// authorize. No run is committed here, so no ack handshake is needed.
|
|
1265
1325
|
}
|
|
1266
|
-
return routeMail(recipient, rawMessage)
|
|
1326
|
+
return routeMail(recipient, rawMessage, authenticatedSender)
|
|
1327
|
+
? "routed"
|
|
1328
|
+
: "unrouted";
|
|
1267
1329
|
}
|
|
1268
1330
|
async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
|
|
1269
1331
|
let results;
|
|
@@ -1381,18 +1443,19 @@ export function createSidecarRouter(config) {
|
|
|
1381
1443
|
for (const addr of conn.workflowAddresses) {
|
|
1382
1444
|
if (addressIndex.get(addr) === ws) {
|
|
1383
1445
|
addressIndex.delete(addr);
|
|
1446
|
+
connectorStates.delete(addr);
|
|
1384
1447
|
// Retain un-acked workflow trigger mail across the disconnect for the
|
|
1385
|
-
// same reason as the session loop above --
|
|
1448
|
+
// same reason as the session loop above -- an authenticated reconnect
|
|
1386
1449
|
// redelivers it. This is un-acked TRIGGER mail, distinct from the
|
|
1387
1450
|
// deployment's in-flight run state (reconstructed sidecar-locally); the
|
|
1388
1451
|
// "no disconnect queue" note above is about that run state, not this.
|
|
1389
1452
|
retainPendingMailForAddress(addr);
|
|
1390
1453
|
}
|
|
1391
1454
|
}
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1455
|
+
const current = allocatedConnections.get(conn.identity.allocationId);
|
|
1456
|
+
if (current?.ws === ws) {
|
|
1457
|
+
allocatedConnections.delete(conn.identity.allocationId);
|
|
1458
|
+
if (conn.identity.kind === "allocated") {
|
|
1396
1459
|
allocated = {
|
|
1397
1460
|
allocationId: conn.identity.allocationId,
|
|
1398
1461
|
generation: conn.identity.generation,
|
|
@@ -1401,64 +1464,31 @@ export function createSidecarRouter(config) {
|
|
|
1401
1464
|
}
|
|
1402
1465
|
connections.delete(ws);
|
|
1403
1466
|
// Cancel the liveness timer for this connection.
|
|
1404
|
-
const
|
|
1405
|
-
if (
|
|
1406
|
-
|
|
1467
|
+
const cancelLiveness = livenessTimers.get(ws);
|
|
1468
|
+
if (cancelLiveness !== undefined) {
|
|
1469
|
+
cancelLiveness();
|
|
1407
1470
|
livenessTimers.delete(ws);
|
|
1408
1471
|
}
|
|
1409
1472
|
// Drop the per-ws serialization chain; no more frames will queue on it.
|
|
1410
1473
|
messageChains.delete(ws);
|
|
1411
|
-
//
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
}
|
|
1417
|
-
// Reject any in-flight requests that were sent to this sidecar.
|
|
1418
|
-
for (const [requestId, req] of pending) {
|
|
1419
|
-
if (req.ws !== ws)
|
|
1420
|
-
continue;
|
|
1421
|
-
clearTimeout(req.timer);
|
|
1422
|
-
pending.delete(requestId);
|
|
1423
|
-
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1424
|
-
}
|
|
1474
|
+
// Reject any in-flight requests that were sent to this sidecar. Each
|
|
1475
|
+
// entry's reject closure runs its own per-site cleanup (the deploy and
|
|
1476
|
+
// undeploy closures roll routing back), exactly as a frame-error
|
|
1477
|
+
// rejection would.
|
|
1478
|
+
pendingRequests.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
1425
1479
|
// Reject every deploy issued on this socket, including allocated
|
|
1426
1480
|
// workflow deployments stored in `workflowAddresses` rather than
|
|
1427
1481
|
// `agentAddresses`.
|
|
1428
|
-
|
|
1429
|
-
if (req.ws !== ws)
|
|
1430
|
-
continue;
|
|
1431
|
-
clearTimeout(req.timer);
|
|
1432
|
-
pendingDeploys.delete(agentAddress);
|
|
1433
|
-
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1434
|
-
}
|
|
1482
|
+
pendingDeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
1435
1483
|
// Reject any in-flight pack transfers for this sidecar.
|
|
1436
|
-
|
|
1437
|
-
if (pack.ws !== ws)
|
|
1438
|
-
continue;
|
|
1439
|
-
clearTimeout(pack.timer);
|
|
1440
|
-
pendingPacks.delete(transferId);
|
|
1441
|
-
pack.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1442
|
-
}
|
|
1484
|
+
pendingPacks.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
1443
1485
|
// Reject any in-flight undeploys for this sidecar.
|
|
1444
|
-
|
|
1445
|
-
if (req.ws !== ws)
|
|
1446
|
-
continue;
|
|
1447
|
-
clearTimeout(req.timer);
|
|
1448
|
-
pendingUndeploys.delete(addr);
|
|
1449
|
-
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1450
|
-
}
|
|
1486
|
+
pendingUndeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
1451
1487
|
// Reject any in-flight probes sent to this sidecar. A probe never enters
|
|
1452
1488
|
// the address maps, so this ws-keyed sweep is its ONLY disconnect cleanup:
|
|
1453
1489
|
// without it a probe whose sidecar drops mid-flight would hang until its
|
|
1454
1490
|
// own timeout instead of failing fast on the disconnect.
|
|
1455
|
-
|
|
1456
|
-
if (probe.ws !== ws)
|
|
1457
|
-
continue;
|
|
1458
|
-
clearTimeout(probe.timer);
|
|
1459
|
-
pendingProbes.delete(requestId);
|
|
1460
|
-
probe.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1461
|
-
}
|
|
1491
|
+
pendingProbes.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
1462
1492
|
// Cancel any in-flight inbound pack transfers from this sidecar
|
|
1463
1493
|
// across both receivers. The two receivers track their own in-
|
|
1464
1494
|
// flight transferIds, so a pending workflow-run transfer for an
|
|
@@ -1496,43 +1526,22 @@ export function createSidecarRouter(config) {
|
|
|
1496
1526
|
const requestId = nextRequestId();
|
|
1497
1527
|
const frame = buildFrame(requestId);
|
|
1498
1528
|
return new Promise((resolve, reject) => {
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
}, requestTimeoutMs);
|
|
1503
|
-
pending.set(requestId, {
|
|
1504
|
-
requestId,
|
|
1505
|
-
ws,
|
|
1529
|
+
pendingRequests.register(requestId, ws, {
|
|
1530
|
+
timeoutMs: requestTimeoutMs,
|
|
1531
|
+
timeoutMessage: `Request ${requestId} timed out after ${requestTimeoutMs}ms`,
|
|
1506
1532
|
resolve,
|
|
1507
1533
|
reject(error) {
|
|
1508
1534
|
reject(new Error(error));
|
|
1509
1535
|
},
|
|
1510
|
-
|
|
1511
|
-
});
|
|
1536
|
+
}, undefined);
|
|
1512
1537
|
conn.send(frame);
|
|
1513
1538
|
});
|
|
1514
1539
|
}
|
|
1515
|
-
function resolvePending(requestId) {
|
|
1516
|
-
const req = pending.get(requestId);
|
|
1517
|
-
if (req === undefined)
|
|
1518
|
-
return;
|
|
1519
|
-
clearTimeout(req.timer);
|
|
1520
|
-
pending.delete(requestId);
|
|
1521
|
-
req.resolve();
|
|
1522
|
-
}
|
|
1523
|
-
function rejectPending(requestId, error) {
|
|
1524
|
-
const req = pending.get(requestId);
|
|
1525
|
-
if (req === undefined)
|
|
1526
|
-
return;
|
|
1527
|
-
clearTimeout(req.timer);
|
|
1528
|
-
pending.delete(requestId);
|
|
1529
|
-
req.reject(error);
|
|
1530
|
-
}
|
|
1531
1540
|
function packResponseMatches(entry, ws, frame) {
|
|
1532
1541
|
return (entry.ws === ws &&
|
|
1533
|
-
entry.agentAddress === frame.agentAddress &&
|
|
1534
|
-
entry.repoId.kind === frame.repoId.kind &&
|
|
1535
|
-
entry.repoId.id === frame.repoId.id);
|
|
1542
|
+
entry.meta.agentAddress === frame.agentAddress &&
|
|
1543
|
+
entry.meta.repoId.kind === frame.repoId.kind &&
|
|
1544
|
+
entry.meta.repoId.id === frame.repoId.id);
|
|
1536
1545
|
}
|
|
1537
1546
|
function resolvePackPending(ws, frame) {
|
|
1538
1547
|
const entry = pendingPacks.get(frame.transferId);
|
|
@@ -1542,9 +1551,7 @@ export function createSidecarRouter(config) {
|
|
|
1542
1551
|
logger.warn `Ignoring repo.pack.ack for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
|
|
1543
1552
|
return;
|
|
1544
1553
|
}
|
|
1545
|
-
|
|
1546
|
-
pendingPacks.delete(frame.transferId);
|
|
1547
|
-
entry.resolve();
|
|
1554
|
+
pendingPacks.resolve(frame.transferId);
|
|
1548
1555
|
}
|
|
1549
1556
|
function rejectPackPending(ws, frame) {
|
|
1550
1557
|
const entry = pendingPacks.get(frame.transferId);
|
|
@@ -1554,13 +1561,13 @@ export function createSidecarRouter(config) {
|
|
|
1554
1561
|
logger.warn `Ignoring repo.pack.reject for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
|
|
1555
1562
|
return;
|
|
1556
1563
|
}
|
|
1557
|
-
clearTimeout(entry.timer);
|
|
1558
|
-
pendingPacks.delete(frame.transferId);
|
|
1559
1564
|
// Surface the receiver's specific cause when it carried one, so the awaiting
|
|
1560
|
-
// push sees "corrupt: <detail>" rather than only the coarse reason.
|
|
1561
|
-
|
|
1565
|
+
// push sees "corrupt: <detail>" rather than only the coarse reason. The
|
|
1566
|
+
// "Pack rejected:" prefix is applied here rather than in the entry's
|
|
1567
|
+
// reject closure because a TIMEOUT rejection must not carry it.
|
|
1568
|
+
pendingPacks.reject(frame.transferId, `Pack rejected: ${frame.detail !== undefined
|
|
1562
1569
|
? `${frame.reason}: ${frame.detail}`
|
|
1563
|
-
: frame.reason);
|
|
1570
|
+
: frame.reason}`);
|
|
1564
1571
|
}
|
|
1565
1572
|
function resolveUndeployPending(ws, agentAddress) {
|
|
1566
1573
|
const req = pendingUndeploys.get(agentAddress);
|
|
@@ -1570,9 +1577,7 @@ export function createSidecarRouter(config) {
|
|
|
1570
1577
|
}
|
|
1571
1578
|
if (req.ws !== ws)
|
|
1572
1579
|
return;
|
|
1573
|
-
|
|
1574
|
-
pendingUndeploys.delete(agentAddress);
|
|
1575
|
-
req.resolve();
|
|
1580
|
+
pendingUndeploys.resolve(agentAddress);
|
|
1576
1581
|
}
|
|
1577
1582
|
function rejectUndeployPending(ws, agentAddress, error) {
|
|
1578
1583
|
const req = pendingUndeploys.get(agentAddress);
|
|
@@ -1580,25 +1585,19 @@ export function createSidecarRouter(config) {
|
|
|
1580
1585
|
return;
|
|
1581
1586
|
if (req.ws !== ws)
|
|
1582
1587
|
return;
|
|
1583
|
-
|
|
1584
|
-
pendingUndeploys.delete(agentAddress);
|
|
1585
|
-
req.reject(error);
|
|
1588
|
+
pendingUndeploys.reject(agentAddress, error);
|
|
1586
1589
|
}
|
|
1587
|
-
function resolveProbe(requestId, result) {
|
|
1590
|
+
function resolveProbe(ws, requestId, result) {
|
|
1588
1591
|
const req = pendingProbes.get(requestId);
|
|
1589
|
-
if (req === undefined)
|
|
1592
|
+
if (req === undefined || req.ws !== ws)
|
|
1590
1593
|
return;
|
|
1591
|
-
|
|
1592
|
-
pendingProbes.delete(requestId);
|
|
1593
|
-
req.resolve(result);
|
|
1594
|
+
pendingProbes.resolve(requestId, result);
|
|
1594
1595
|
}
|
|
1595
|
-
function rejectProbe(requestId, error) {
|
|
1596
|
+
function rejectProbe(ws, requestId, error) {
|
|
1596
1597
|
const req = pendingProbes.get(requestId);
|
|
1597
|
-
if (req === undefined)
|
|
1598
|
+
if (req === undefined || req.ws !== ws)
|
|
1598
1599
|
return;
|
|
1599
|
-
|
|
1600
|
-
pendingProbes.delete(requestId);
|
|
1601
|
-
req.reject(error);
|
|
1600
|
+
pendingProbes.reject(requestId, error);
|
|
1602
1601
|
}
|
|
1603
1602
|
// Routing rule: pick the receiver dedicated to the repoId.kind the
|
|
1604
1603
|
// frame carries. The receivers' in-flight state is independent, so a
|
|
@@ -1648,6 +1647,8 @@ export function createSidecarRouter(config) {
|
|
|
1648
1647
|
});
|
|
1649
1648
|
return;
|
|
1650
1649
|
}
|
|
1650
|
+
if (conn.identity.kind !== "allocated")
|
|
1651
|
+
return;
|
|
1651
1652
|
const picked = pickPackReceiver(frame.repoId);
|
|
1652
1653
|
if (picked === null) {
|
|
1653
1654
|
logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
@@ -1690,6 +1691,9 @@ export function createSidecarRouter(config) {
|
|
|
1690
1691
|
});
|
|
1691
1692
|
return;
|
|
1692
1693
|
}
|
|
1694
|
+
if (conn.identity.kind !== "allocated")
|
|
1695
|
+
return;
|
|
1696
|
+
const identity = conn.identity;
|
|
1693
1697
|
const picked = pickPackReceiver(frame.repoId);
|
|
1694
1698
|
if (picked === null) {
|
|
1695
1699
|
logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
@@ -1723,15 +1727,13 @@ export function createSidecarRouter(config) {
|
|
|
1723
1727
|
});
|
|
1724
1728
|
return;
|
|
1725
1729
|
}
|
|
1726
|
-
const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha,
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
}
|
|
1734
|
-
: { kind: "shared", agentAddress: frame.agentAddress });
|
|
1730
|
+
const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, {
|
|
1731
|
+
kind: "allocated",
|
|
1732
|
+
agentAddress: frame.agentAddress,
|
|
1733
|
+
allocationId: identity.allocationId,
|
|
1734
|
+
anchorRunId: identity.anchorRunId,
|
|
1735
|
+
generation: identity.generation,
|
|
1736
|
+
});
|
|
1735
1737
|
// Connection may have closed during async verification.
|
|
1736
1738
|
const currentConn = connections.get(ws);
|
|
1737
1739
|
if (currentConn === undefined)
|
|
@@ -1759,9 +1761,9 @@ export function createSidecarRouter(config) {
|
|
|
1759
1761
|
* window of a multi-step deploy, so `sendPack` can route the step's deploy
|
|
1760
1762
|
* and asset packs before the deployment-level frame spawns the child.
|
|
1761
1763
|
*
|
|
1762
|
-
* The address is
|
|
1763
|
-
*
|
|
1764
|
-
*
|
|
1764
|
+
* The address is Hub-minted and workflow-derived, so it enters the
|
|
1765
|
+
* `workflowAddresses` set rather than the legacy `agentAddresses` set and is
|
|
1766
|
+
* torn down by `unbindStepRoute` once the
|
|
1765
1767
|
* step's packs land. `handleClose` reclaims it if the sidecar drops
|
|
1766
1768
|
* mid-stage. Per-step addresses are not runtime-routed (mail, signals, and
|
|
1767
1769
|
* drains use the deployment address), so the binding is transient: it is
|
|
@@ -1774,6 +1776,17 @@ export function createSidecarRouter(config) {
|
|
|
1774
1776
|
throw new Error(`Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`);
|
|
1775
1777
|
}
|
|
1776
1778
|
allocationFences.set(allocationId, generation);
|
|
1779
|
+
// A durable generation advance resolves unfinished initialization as failed.
|
|
1780
|
+
// This also covers a cleanup transaction whose response was lost: the next
|
|
1781
|
+
// reconciliation rebuilds this fence before it can start a replacement.
|
|
1782
|
+
for (const attempt of [...allocatedKeyRecordInFlight.values()]) {
|
|
1783
|
+
if (attempt.allocationId === allocationId &&
|
|
1784
|
+
attempt.generation < generation) {
|
|
1785
|
+
noteSenderDeploySettled(attempt, {
|
|
1786
|
+
failed: `Allocation ${allocationId} advanced beyond the deployment attempt`,
|
|
1787
|
+
});
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1777
1790
|
const current = allocatedConnections.get(allocationId);
|
|
1778
1791
|
if (current !== undefined && current.identity.generation !== generation) {
|
|
1779
1792
|
handleClose(current.ws);
|
|
@@ -1792,7 +1805,31 @@ export function createSidecarRouter(config) {
|
|
|
1792
1805
|
if (waiters.size === 0)
|
|
1793
1806
|
allocationWaiters.delete(allocationId);
|
|
1794
1807
|
}
|
|
1795
|
-
|
|
1808
|
+
function retireAllocation(target) {
|
|
1809
|
+
if (allocationFences.get(target.allocationId) !== target.generation)
|
|
1810
|
+
return;
|
|
1811
|
+
disconnectAllocation(target);
|
|
1812
|
+
allocationFences.delete(target.allocationId);
|
|
1813
|
+
// The fence is gone, so a lingering attempt can never settle normally.
|
|
1814
|
+
// Fail it here rather than leaving a marker that blocks the address.
|
|
1815
|
+
for (const attempt of [...allocatedKeyRecordInFlight.values()]) {
|
|
1816
|
+
if (attempt.allocationId === target.allocationId &&
|
|
1817
|
+
attempt.generation <= target.generation) {
|
|
1818
|
+
noteSenderDeploySettled(attempt, {
|
|
1819
|
+
failed: `Allocation ${target.allocationId} generation ${String(target.generation)} retired`,
|
|
1820
|
+
});
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
const waiters = allocationWaiters.get(target.allocationId);
|
|
1824
|
+
if (waiters === undefined)
|
|
1825
|
+
return;
|
|
1826
|
+
allocationWaiters.delete(target.allocationId);
|
|
1827
|
+
for (const waiter of waiters) {
|
|
1828
|
+
clearTimeout(waiter.timer);
|
|
1829
|
+
waiter.reject(new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} retired`));
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
async function getProvisionedConnection(target, use) {
|
|
1796
1833
|
if (allocationFences.get(target.allocationId) !== target.generation) {
|
|
1797
1834
|
throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
|
|
1798
1835
|
}
|
|
@@ -1801,7 +1838,14 @@ export function createSidecarRouter(config) {
|
|
|
1801
1838
|
current.identity.generation !== target.generation) {
|
|
1802
1839
|
throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`);
|
|
1803
1840
|
}
|
|
1804
|
-
|
|
1841
|
+
let identityCurrent;
|
|
1842
|
+
try {
|
|
1843
|
+
identityCurrent = await validateSidecarIdentity(current.identity, use);
|
|
1844
|
+
}
|
|
1845
|
+
catch (cause) {
|
|
1846
|
+
throw new SidecarIdentityValidationError(target.allocationId, target.generation, cause);
|
|
1847
|
+
}
|
|
1848
|
+
if (!identityCurrent) {
|
|
1805
1849
|
if (allocatedConnections.get(target.allocationId) === current) {
|
|
1806
1850
|
handleClose(current.ws);
|
|
1807
1851
|
current.ws.close();
|
|
@@ -1813,19 +1857,33 @@ export function createSidecarRouter(config) {
|
|
|
1813
1857
|
}
|
|
1814
1858
|
const conn = connections.get(current.ws);
|
|
1815
1859
|
if (conn === undefined ||
|
|
1816
|
-
conn.identity.kind !== "allocated" ||
|
|
1817
1860
|
conn.identity.allocationId !== target.allocationId ||
|
|
1818
1861
|
conn.identity.generation !== target.generation) {
|
|
1819
1862
|
throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId}`);
|
|
1820
1863
|
}
|
|
1821
1864
|
return { ws: current.ws, conn };
|
|
1822
1865
|
}
|
|
1866
|
+
async function getAllocatedConnection(target, use) {
|
|
1867
|
+
const current = await getProvisionedConnection(target, use);
|
|
1868
|
+
if (current.conn.identity.kind !== "allocated") {
|
|
1869
|
+
throw new Error(`Allocation ${target.allocationId} is connected as probe capacity`);
|
|
1870
|
+
}
|
|
1871
|
+
return {
|
|
1872
|
+
ws: current.ws,
|
|
1873
|
+
conn: { ...current.conn, identity: current.conn.identity },
|
|
1874
|
+
};
|
|
1875
|
+
}
|
|
1823
1876
|
async function isAllocatedSidecarReady(target) {
|
|
1824
1877
|
try {
|
|
1825
|
-
await
|
|
1878
|
+
await getProvisionedConnection(target, "readiness");
|
|
1826
1879
|
return true;
|
|
1827
1880
|
}
|
|
1828
|
-
catch {
|
|
1881
|
+
catch (error) {
|
|
1882
|
+
// A failed validation is unknown, not absent: the worker may be healthy
|
|
1883
|
+
// behind a failed lookup, so report it distinctly instead of answering
|
|
1884
|
+
// `false` and letting the caller release a live worker.
|
|
1885
|
+
if (error instanceof SidecarIdentityValidationError)
|
|
1886
|
+
throw error;
|
|
1829
1887
|
return false;
|
|
1830
1888
|
}
|
|
1831
1889
|
}
|
|
@@ -1836,22 +1894,40 @@ export function createSidecarRouter(config) {
|
|
|
1836
1894
|
return false;
|
|
1837
1895
|
return conn.workflowAddresses.has(conn.identity.workflowRunAddress);
|
|
1838
1896
|
}
|
|
1839
|
-
catch {
|
|
1897
|
+
catch (error) {
|
|
1898
|
+
if (error instanceof SidecarIdentityValidationError)
|
|
1899
|
+
throw error;
|
|
1840
1900
|
return false;
|
|
1841
1901
|
}
|
|
1842
1902
|
}
|
|
1843
|
-
async function waitForAllocatedSidecar(target, timeoutMs) {
|
|
1844
|
-
|
|
1845
|
-
|
|
1903
|
+
async function waitForAllocatedSidecar(target, timeoutMs, onValidation) {
|
|
1904
|
+
// An indeterminable worker waits out the unknown while time remains: only
|
|
1905
|
+
// confirmed absence may surface as a connection timeout. At expiry the
|
|
1906
|
+
// wait reports the validation failure rather than a missed deadline, so
|
|
1907
|
+
// the caller retries instead of releasing a worker that may be healthy.
|
|
1908
|
+
let validationFailure;
|
|
1909
|
+
try {
|
|
1910
|
+
if (await isAllocatedSidecarReady(target))
|
|
1911
|
+
return;
|
|
1912
|
+
}
|
|
1913
|
+
catch (error) {
|
|
1914
|
+
if (!(error instanceof SidecarIdentityValidationError))
|
|
1915
|
+
throw error;
|
|
1916
|
+
validationFailure = error;
|
|
1917
|
+
}
|
|
1846
1918
|
if (allocationFences.get(target.allocationId) !== target.generation) {
|
|
1847
1919
|
throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
|
|
1848
1920
|
}
|
|
1849
1921
|
if (timeoutMs <= 0) {
|
|
1922
|
+
if (validationFailure !== undefined)
|
|
1923
|
+
throw validationFailure;
|
|
1850
1924
|
throw new Error(`Timed out waiting for allocated sidecar ${target.allocationId}`);
|
|
1851
1925
|
}
|
|
1852
1926
|
await new Promise((resolve, reject) => {
|
|
1853
1927
|
const waiter = {
|
|
1854
1928
|
generation: target.generation,
|
|
1929
|
+
validations: new Set(),
|
|
1930
|
+
...(onValidation !== undefined ? { onValidation } : {}),
|
|
1855
1931
|
resolve,
|
|
1856
1932
|
reject,
|
|
1857
1933
|
timer: setTimeout(() => {
|
|
@@ -1860,8 +1936,12 @@ export function createSidecarRouter(config) {
|
|
|
1860
1936
|
if (current?.size === 0) {
|
|
1861
1937
|
allocationWaiters.delete(target.allocationId);
|
|
1862
1938
|
}
|
|
1863
|
-
reject(
|
|
1939
|
+
reject(waiter.validationFailure ??
|
|
1940
|
+
(waiter.validations.size > 0
|
|
1941
|
+
? new SidecarIdentityValidationError(target.allocationId, target.generation)
|
|
1942
|
+
: new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`)));
|
|
1864
1943
|
}, timeoutMs),
|
|
1944
|
+
...(validationFailure !== undefined ? { validationFailure } : {}),
|
|
1865
1945
|
};
|
|
1866
1946
|
let waiters = allocationWaiters.get(target.allocationId);
|
|
1867
1947
|
if (waiters === undefined) {
|
|
@@ -1872,21 +1952,6 @@ export function createSidecarRouter(config) {
|
|
|
1872
1952
|
void notifyAllocationWaiters(target.allocationId);
|
|
1873
1953
|
});
|
|
1874
1954
|
}
|
|
1875
|
-
function bindStepRoute(stepAddress) {
|
|
1876
|
-
const ws = addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress);
|
|
1877
|
-
if (ws === undefined) {
|
|
1878
|
-
throw new Error(`No sidecar available to stage workflow step "${stepAddress}"`);
|
|
1879
|
-
}
|
|
1880
|
-
const conn = connections.get(ws);
|
|
1881
|
-
if (conn === undefined) {
|
|
1882
|
-
throw new Error(`No sidecar connected to stage workflow step "${stepAddress}"`);
|
|
1883
|
-
}
|
|
1884
|
-
if (conn.identity.kind !== "shared") {
|
|
1885
|
-
throw new Error(`Allocated sidecar ${conn.sidecarId} cannot receive ordinary workflow step ${stepAddress}`);
|
|
1886
|
-
}
|
|
1887
|
-
conn.workflowAddresses.add(stepAddress);
|
|
1888
|
-
addressIndex.set(stepAddress, ws);
|
|
1889
|
-
}
|
|
1890
1955
|
async function bindAllocatedStepRoute(target, stepAddress) {
|
|
1891
1956
|
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
1892
1957
|
const existing = addressIndex.get(stepAddress);
|
|
@@ -1896,21 +1961,6 @@ export function createSidecarRouter(config) {
|
|
|
1896
1961
|
conn.workflowAddresses.add(stepAddress);
|
|
1897
1962
|
addressIndex.set(stepAddress, ws);
|
|
1898
1963
|
}
|
|
1899
|
-
/**
|
|
1900
|
-
* Remove a per-step route bound by `bindStepRoute` once the step's packs
|
|
1901
|
-
* have landed. Idempotent: an address that was never bound (or already
|
|
1902
|
-
* unbound, e.g. by a mid-stage `handleClose`) is a no-op.
|
|
1903
|
-
*/
|
|
1904
|
-
function unbindStepRoute(stepAddress) {
|
|
1905
|
-
const ws = addressIndex.get(stepAddress);
|
|
1906
|
-
if (ws === undefined)
|
|
1907
|
-
return;
|
|
1908
|
-
const conn = connections.get(ws);
|
|
1909
|
-
if (conn !== undefined) {
|
|
1910
|
-
conn.workflowAddresses.delete(stepAddress);
|
|
1911
|
-
}
|
|
1912
|
-
addressIndex.delete(stepAddress);
|
|
1913
|
-
}
|
|
1914
1964
|
function unbindAllocatedStepRoute(target, stepAddress) {
|
|
1915
1965
|
const current = allocatedConnections.get(target.allocationId);
|
|
1916
1966
|
if (current === undefined ||
|
|
@@ -1924,20 +1974,6 @@ export function createSidecarRouter(config) {
|
|
|
1924
1974
|
}
|
|
1925
1975
|
// Pack transfers may take longer than session requests due to data volume.
|
|
1926
1976
|
const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
|
|
1927
|
-
function sendPack(agentAddress, pack, ref, commitSha, options) {
|
|
1928
|
-
const ws = addressIndex.get(agentAddress);
|
|
1929
|
-
if (ws === undefined) {
|
|
1930
|
-
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1931
|
-
}
|
|
1932
|
-
const conn = connections.get(ws);
|
|
1933
|
-
if (conn === undefined) {
|
|
1934
|
-
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1935
|
-
}
|
|
1936
|
-
if (conn.identity.kind !== "shared") {
|
|
1937
|
-
return Promise.reject(new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound pack routing for "${agentAddress}"`));
|
|
1938
|
-
}
|
|
1939
|
-
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
|
|
1940
|
-
}
|
|
1941
1977
|
function sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options) {
|
|
1942
1978
|
const transferId = `pack-${++packCounter}`;
|
|
1943
1979
|
// For the agent-state flow the destination agent and the source repo
|
|
@@ -1952,21 +1988,14 @@ export function createSidecarRouter(config) {
|
|
|
1952
1988
|
// Register pending entry before sending frames so that a synchronous
|
|
1953
1989
|
// repo.pack.ack (e.g. in tests or loopback transports) resolves correctly.
|
|
1954
1990
|
return new Promise((resolve, reject) => {
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
}, PACK_TIMEOUT_MS);
|
|
1959
|
-
pendingPacks.set(transferId, {
|
|
1960
|
-
transferId,
|
|
1961
|
-
ws,
|
|
1962
|
-
agentAddress,
|
|
1963
|
-
repoId,
|
|
1991
|
+
pendingPacks.register(transferId, ws, {
|
|
1992
|
+
timeoutMs: PACK_TIMEOUT_MS,
|
|
1993
|
+
timeoutMessage: `Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`,
|
|
1964
1994
|
resolve,
|
|
1965
1995
|
reject(error) {
|
|
1966
|
-
reject(new Error(
|
|
1996
|
+
reject(new Error(error));
|
|
1967
1997
|
},
|
|
1968
|
-
|
|
1969
|
-
});
|
|
1998
|
+
}, { agentAddress, repoId });
|
|
1970
1999
|
// Send chunks
|
|
1971
2000
|
for (const chunk of chunkPack(pack)) {
|
|
1972
2001
|
conn.send({
|
|
@@ -1997,11 +2026,10 @@ export function createSidecarRouter(config) {
|
|
|
1997
2026
|
}
|
|
1998
2027
|
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
|
|
1999
2028
|
}
|
|
2000
|
-
async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha) {
|
|
2029
|
+
async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha, signal) {
|
|
2030
|
+
signal?.throwIfAborted();
|
|
2001
2031
|
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2002
|
-
|
|
2003
|
-
throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
|
|
2004
|
-
}
|
|
2032
|
+
signal?.throwIfAborted();
|
|
2005
2033
|
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
2006
2034
|
throw new Error(`Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`);
|
|
2007
2035
|
}
|
|
@@ -2015,7 +2043,16 @@ export function createSidecarRouter(config) {
|
|
|
2015
2043
|
},
|
|
2016
2044
|
});
|
|
2017
2045
|
}
|
|
2018
|
-
function routeMail(agentAddress, rawMessage, messageId, runGrants) {
|
|
2046
|
+
function routeMail(agentAddress, rawMessage, authenticatedSender, messageId, runGrants) {
|
|
2047
|
+
// `authenticatedSender` is hub-assigned by the caller from a hub-verified
|
|
2048
|
+
// value (the ownership-gated sender of a relayed mail, or the triggering
|
|
2049
|
+
// principal's address) -- never the message's own MIME `From`. It rides
|
|
2050
|
+
// the frame as the hub-verified sender of record, so a recipient can take
|
|
2051
|
+
// the sender from it rather than the forgeable `From`. The recipient's
|
|
2052
|
+
// signature check reads it as the sender of record -- resolving the
|
|
2053
|
+
// sender's key from its local cache to verify the signature -- and its
|
|
2054
|
+
// admission policy gates delivery on the verdict.
|
|
2055
|
+
//
|
|
2019
2056
|
// Carry the hub-minted messageId on the frame so the sidecar's durable-
|
|
2020
2057
|
// receipt ack (`mail.inbound.ack`) keys on the same id the hub tracks, and
|
|
2021
2058
|
// a redelivery replays identical bytes for the downstream RunStarted dedup.
|
|
@@ -2026,6 +2063,7 @@ export function createSidecarRouter(config) {
|
|
|
2026
2063
|
type: "mail.inbound",
|
|
2027
2064
|
agentAddress,
|
|
2028
2065
|
rawMessage,
|
|
2066
|
+
authenticatedSender,
|
|
2029
2067
|
...(messageId !== undefined ? { messageId } : {}),
|
|
2030
2068
|
};
|
|
2031
2069
|
const ws = addressIndex.get(agentAddress);
|
|
@@ -2048,12 +2086,13 @@ export function createSidecarRouter(config) {
|
|
|
2048
2086
|
// If the agent recently disconnected, queue for delivery on reconnect.
|
|
2049
2087
|
return enqueueForDisconnected(agentAddress, frame);
|
|
2050
2088
|
}
|
|
2051
|
-
function sendRunGrants(agentAddress, runId, stepGrants) {
|
|
2089
|
+
function sendRunGrants(agentAddress, runId, stepGrants, senderIdentities) {
|
|
2052
2090
|
const frame = {
|
|
2053
2091
|
type: "run.grants",
|
|
2054
2092
|
agentAddress,
|
|
2055
2093
|
runId,
|
|
2056
2094
|
stepGrants,
|
|
2095
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
2057
2096
|
};
|
|
2058
2097
|
const ws = addressIndex.get(agentAddress);
|
|
2059
2098
|
if (ws !== undefined) {
|
|
@@ -2067,29 +2106,65 @@ export function createSidecarRouter(config) {
|
|
|
2067
2106
|
// so a run.grants issued in the window between deploy and the first
|
|
2068
2107
|
// reconnect survives the same way the dispatching trigger mail does.
|
|
2069
2108
|
// A queue exists only while the deployment address is still on
|
|
2070
|
-
// agentAddresses (pre-first-reconnect); after
|
|
2109
|
+
// agentAddresses (pre-first-reconnect); after an authenticated reconnect it
|
|
2071
2110
|
// moves to workflowAddresses, which handleClose leaves unqueued because
|
|
2072
2111
|
// that generation's in-flight run state is reconstructed sidecar-locally.
|
|
2073
2112
|
// Returning without enqueueing there is correct; enqueueing is what keeps
|
|
2074
2113
|
// grants and mail from diverging in the pre-reconnect window.
|
|
2075
2114
|
return enqueueForDisconnected(agentAddress, frame);
|
|
2076
2115
|
}
|
|
2077
|
-
async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, messageId) {
|
|
2116
|
+
async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, authenticatedSender, messageId, signal) {
|
|
2117
|
+
signal?.throwIfAborted();
|
|
2078
2118
|
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2119
|
+
signal?.throwIfAborted();
|
|
2079
2120
|
if (addressIndex.get(agentAddress) !== ws) {
|
|
2080
2121
|
throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2081
2122
|
}
|
|
2082
|
-
|
|
2123
|
+
// authenticatedSender is the sender persisted at enqueue on the dispatch
|
|
2124
|
+
// row (the triggering principal's hub-verified address); the caller reads
|
|
2125
|
+
// it from that row. It is never the message's MIME From.
|
|
2126
|
+
//
|
|
2127
|
+
// Resolve its key here, at dispatch (redelivery) time, from that persisted
|
|
2128
|
+
// address, to co-deliver on the run's grants barrier so the recipient
|
|
2129
|
+
// caches the sender's current hub-held key. A run sender's deployment key
|
|
2130
|
+
// is immutable once acked; a user sender's key rotating mid-flight would
|
|
2131
|
+
// leave the fixed signed bytes checked against the new key, which the
|
|
2132
|
+
// recipient logs as unverifiable. Null when unresolvable (no resolver
|
|
2133
|
+
// wired, or the sender has no durable key). The lookup contract (see
|
|
2134
|
+
// SidecarLookups.resolveSenderKey) is best-effort and never throws, so
|
|
2135
|
+
// resolving ahead of the run.grants send cannot block it.
|
|
2136
|
+
const authenticatedSenderPublicKey = lookups.resolveSenderKey !== undefined
|
|
2137
|
+
? await lookups.resolveSenderKey(authenticatedSender)
|
|
2138
|
+
: null;
|
|
2139
|
+
signal?.throwIfAborted();
|
|
2140
|
+
// Co-deliver the resolved key on the run's grants barrier, omitting a null
|
|
2141
|
+
// key so it is never cached (see deliverMailToRecipient). The same list
|
|
2142
|
+
// rides the pending-mail entry so the reconnect replay carries it too.
|
|
2143
|
+
const senderIdentities = authenticatedSenderPublicKey !== null
|
|
2144
|
+
? [
|
|
2145
|
+
{
|
|
2146
|
+
address: authenticatedSender,
|
|
2147
|
+
publicKey: authenticatedSenderPublicKey,
|
|
2148
|
+
},
|
|
2149
|
+
]
|
|
2150
|
+
: undefined;
|
|
2151
|
+
const runGrants = {
|
|
2152
|
+
runId,
|
|
2153
|
+
stepGrants,
|
|
2154
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
2155
|
+
};
|
|
2083
2156
|
conn.send({
|
|
2084
2157
|
type: "run.grants",
|
|
2085
2158
|
agentAddress,
|
|
2086
2159
|
runId,
|
|
2087
2160
|
stepGrants,
|
|
2161
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
2088
2162
|
});
|
|
2089
2163
|
const frame = {
|
|
2090
2164
|
type: "mail.inbound",
|
|
2091
2165
|
agentAddress,
|
|
2092
2166
|
rawMessage,
|
|
2167
|
+
authenticatedSender,
|
|
2093
2168
|
messageId,
|
|
2094
2169
|
};
|
|
2095
2170
|
conn.send(frame);
|
|
@@ -2121,55 +2196,26 @@ export function createSidecarRouter(config) {
|
|
|
2121
2196
|
});
|
|
2122
2197
|
}
|
|
2123
2198
|
catch (err) {
|
|
2124
|
-
|
|
2199
|
+
pendingDeploys.reject(frame.agentAddress, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
|
|
2125
2200
|
return;
|
|
2126
2201
|
}
|
|
2127
2202
|
}
|
|
2128
|
-
|
|
2129
|
-
}
|
|
2130
|
-
function resolveDeployPending(req, publicKey) {
|
|
2131
|
-
if (pendingDeploys.get(req.agentAddress) !== req)
|
|
2132
|
-
return;
|
|
2133
|
-
clearTimeout(req.timer);
|
|
2134
|
-
pendingDeploys.delete(req.agentAddress);
|
|
2135
|
-
req.resolve(publicKey);
|
|
2136
|
-
}
|
|
2137
|
-
function rejectDeployPending(req, error) {
|
|
2138
|
-
if (pendingDeploys.get(req.agentAddress) !== req)
|
|
2139
|
-
return;
|
|
2140
|
-
clearTimeout(req.timer);
|
|
2141
|
-
pendingDeploys.delete(req.agentAddress);
|
|
2142
|
-
req.reject(error);
|
|
2203
|
+
pendingDeploys.resolve(frame.agentAddress, frame.publicKey);
|
|
2143
2204
|
}
|
|
2144
2205
|
function rejectDeployPendingFromFrame(ws, agentAddress, error) {
|
|
2145
2206
|
const req = pendingDeploys.get(agentAddress);
|
|
2146
2207
|
if (req === undefined || req.ws !== ws)
|
|
2147
2208
|
return;
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
if (hubPublicKeyHex === undefined) {
|
|
2152
|
-
throw new Error("Hub signing key is required for agent deployment");
|
|
2153
|
-
}
|
|
2154
|
-
const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
|
|
2155
|
-
if (ws === undefined) {
|
|
2156
|
-
throw new Error(`No sidecar available for agent "${agentAddress}"`);
|
|
2157
|
-
}
|
|
2158
|
-
const conn = connections.get(ws);
|
|
2159
|
-
if (conn === undefined) {
|
|
2160
|
-
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
2161
|
-
}
|
|
2162
|
-
if (conn.identity.kind !== "shared") {
|
|
2163
|
-
throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound deploy routing`);
|
|
2164
|
-
}
|
|
2165
|
-
return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
|
|
2209
|
+
// Settle by key, not by the `req` object: a key lookup observes the
|
|
2210
|
+
// CURRENT entry, so a stale handle cannot settle a replaced round-trip.
|
|
2211
|
+
pendingDeploys.reject(agentAddress, error);
|
|
2166
2212
|
}
|
|
2167
2213
|
function sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow) {
|
|
2168
2214
|
if (hubPublicKeyHex === undefined) {
|
|
2169
|
-
throw
|
|
2215
|
+
throw deployFrameFailure("Hub signing key is required for agent deployment", false);
|
|
2170
2216
|
}
|
|
2171
2217
|
if (pendingDeploys.has(agentAddress)) {
|
|
2172
|
-
throw
|
|
2218
|
+
throw deployFrameFailure(`Deploy already in progress for agent "${agentAddress}"`, false);
|
|
2173
2219
|
}
|
|
2174
2220
|
const addressSet = conn.identity.kind === "allocated"
|
|
2175
2221
|
? conn.workflowAddresses
|
|
@@ -2177,17 +2223,11 @@ export function createSidecarRouter(config) {
|
|
|
2177
2223
|
addressSet.add(agentAddress);
|
|
2178
2224
|
addressIndex.set(agentAddress, ws);
|
|
2179
2225
|
return new Promise((resolve, reject) => {
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
}
|
|
2186
|
-
reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
2187
|
-
}, requestTimeoutMs);
|
|
2188
|
-
pendingDeploys.set(agentAddress, {
|
|
2189
|
-
agentAddress,
|
|
2190
|
-
ws,
|
|
2226
|
+
// Timeout and frame-error rejections share this closure, so the routing
|
|
2227
|
+
// rollback and the `frameSent: true` tag live in one place.
|
|
2228
|
+
pendingDeploys.register(agentAddress, ws, {
|
|
2229
|
+
timeoutMs: requestTimeoutMs,
|
|
2230
|
+
timeoutMessage: `Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
2191
2231
|
resolve(publicKey) {
|
|
2192
2232
|
resolve({ publicKey });
|
|
2193
2233
|
},
|
|
@@ -2196,33 +2236,73 @@ export function createSidecarRouter(config) {
|
|
|
2196
2236
|
addressSet.delete(agentAddress);
|
|
2197
2237
|
addressIndex.delete(agentAddress);
|
|
2198
2238
|
}
|
|
2199
|
-
|
|
2239
|
+
// A non-allocated deployment's key is recorded by the deploy-ack
|
|
2240
|
+
// projection, whose failure (reject/timeout/agent.error/disconnect)
|
|
2241
|
+
// is observed only here. Drain any pre-ack sender mail parked on
|
|
2242
|
+
// this address so it surfaces as undelivered rather than waiting out
|
|
2243
|
+
// the TTL. An allocated deployment's failure is drained by its
|
|
2244
|
+
// session-service owner instead, so skip it here to keep one owner
|
|
2245
|
+
// per case.
|
|
2246
|
+
if (conn.identity.kind !== "allocated") {
|
|
2247
|
+
drainDeferredSenderMail(agentAddress, `deploy failed: ${error}`);
|
|
2248
|
+
}
|
|
2249
|
+
reject(deployFrameFailure(error, true));
|
|
2200
2250
|
},
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2251
|
+
}, undefined);
|
|
2252
|
+
try {
|
|
2253
|
+
conn.send({
|
|
2254
|
+
type: "agent.deploy",
|
|
2255
|
+
agentAddress,
|
|
2256
|
+
agentId: harnessConfig.agentId,
|
|
2257
|
+
config: harnessConfig,
|
|
2258
|
+
hubPublicKey: hubPublicKeyHex,
|
|
2259
|
+
...(workflow !== undefined ? { workflow } : {}),
|
|
2260
|
+
});
|
|
2261
|
+
}
|
|
2262
|
+
catch (err) {
|
|
2263
|
+
// A synchronous send failure means the frame never reached the wire.
|
|
2264
|
+
// Drop the pending entry (and its armed timer) and reject as not-sent
|
|
2265
|
+
// so a caller may safely roll back what it staged. The drop bypasses
|
|
2266
|
+
// the entry's reject closure: this failure must report
|
|
2267
|
+
// `frameSent: false`, and the timer must not fire later and
|
|
2268
|
+
// double-reject.
|
|
2269
|
+
pendingDeploys.delete(agentAddress);
|
|
2270
|
+
if (addressIndex.get(agentAddress) === ws) {
|
|
2271
|
+
addressSet.delete(agentAddress);
|
|
2272
|
+
addressIndex.delete(agentAddress);
|
|
2273
|
+
}
|
|
2274
|
+
reject(deployFrameFailure(`Deploy of "${agentAddress}" failed to send: ${err instanceof Error ? err.message : String(err)}`, false));
|
|
2275
|
+
}
|
|
2211
2276
|
});
|
|
2212
2277
|
}
|
|
2213
|
-
async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow) {
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2278
|
+
async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow, signal, beforeSend) {
|
|
2279
|
+
try {
|
|
2280
|
+
signal?.throwIfAborted();
|
|
2281
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2282
|
+
signal?.throwIfAborted();
|
|
2283
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
2284
|
+
throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
|
|
2285
|
+
}
|
|
2286
|
+
const existing = addressIndex.get(agentAddress);
|
|
2287
|
+
if (existing !== undefined && existing !== ws) {
|
|
2288
|
+
throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
|
|
2289
|
+
}
|
|
2290
|
+
if (hubPublicKeyHex === undefined)
|
|
2291
|
+
throw new Error("Hub signing key is required for agent deployment");
|
|
2292
|
+
if (pendingDeploys.has(agentAddress))
|
|
2293
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
2294
|
+
await beforeSend?.();
|
|
2295
|
+
signal?.throwIfAborted();
|
|
2296
|
+
if (allocatedConnections.get(target.allocationId)?.ws !== ws ||
|
|
2297
|
+
allocationFences.get(target.allocationId) !== target.generation) {
|
|
2298
|
+
throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
|
|
2299
|
+
}
|
|
2300
|
+
// Return without awaiting: only pre-send failures belong to this catch.
|
|
2301
|
+
return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
|
|
2220
2302
|
}
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
|
|
2303
|
+
catch (cause) {
|
|
2304
|
+
throw deployFrameFailure(cause instanceof Error ? cause.message : String(cause), false, cause);
|
|
2224
2305
|
}
|
|
2225
|
-
return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
|
|
2226
2306
|
}
|
|
2227
2307
|
/**
|
|
2228
2308
|
* Provision one step of a multi-step deploy on the sidecar WITHOUT
|
|
@@ -2238,20 +2318,6 @@ export function createSidecarRouter(config) {
|
|
|
2238
2318
|
* so the caller can safely deliver the deploy pack afterward. On failure
|
|
2239
2319
|
* the caller owns tearing the route down via `unbindStepRoute`.
|
|
2240
2320
|
*/
|
|
2241
|
-
function sendProvisionStep(agentAddress, harnessConfig) {
|
|
2242
|
-
const ws = addressIndex.get(agentAddress);
|
|
2243
|
-
if (ws === undefined) {
|
|
2244
|
-
throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
|
|
2245
|
-
}
|
|
2246
|
-
const conn = connections.get(ws);
|
|
2247
|
-
if (conn === undefined) {
|
|
2248
|
-
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
2249
|
-
}
|
|
2250
|
-
if (conn.identity.kind !== "shared") {
|
|
2251
|
-
throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound step provisioning`);
|
|
2252
|
-
}
|
|
2253
|
-
return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
|
|
2254
|
-
}
|
|
2255
2321
|
function sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig) {
|
|
2256
2322
|
if (hubPublicKeyHex === undefined) {
|
|
2257
2323
|
throw new Error("Hub signing key is required for step provisioning");
|
|
@@ -2261,25 +2327,20 @@ export function createSidecarRouter(config) {
|
|
|
2261
2327
|
}
|
|
2262
2328
|
const hubKey = hubPublicKeyHex;
|
|
2263
2329
|
return new Promise((resolve, reject) => {
|
|
2264
|
-
const timer = setTimeout(() => {
|
|
2265
|
-
pendingDeploys.delete(agentAddress);
|
|
2266
|
-
reject(new Error(`Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
2267
|
-
}, requestTimeoutMs);
|
|
2268
2330
|
// The sidecar's `agent.deploy.ack` resolves this through
|
|
2269
|
-
// `
|
|
2270
|
-
//
|
|
2271
|
-
//
|
|
2272
|
-
pendingDeploys.
|
|
2273
|
-
|
|
2274
|
-
|
|
2331
|
+
// `pendingDeploys.resolve`. The per-step address is workflow-derived
|
|
2332
|
+
// and records no hub-side key, so the ack's public key is not needed
|
|
2333
|
+
// and this resolves void.
|
|
2334
|
+
pendingDeploys.register(agentAddress, ws, {
|
|
2335
|
+
timeoutMs: requestTimeoutMs,
|
|
2336
|
+
timeoutMessage: `Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
2275
2337
|
resolve(_publicKey) {
|
|
2276
2338
|
resolve();
|
|
2277
2339
|
},
|
|
2278
2340
|
reject(error) {
|
|
2279
2341
|
reject(new Error(error));
|
|
2280
2342
|
},
|
|
2281
|
-
|
|
2282
|
-
});
|
|
2343
|
+
}, undefined);
|
|
2283
2344
|
conn.send({
|
|
2284
2345
|
type: "agent.deploy",
|
|
2285
2346
|
agentAddress,
|
|
@@ -2297,42 +2358,17 @@ export function createSidecarRouter(config) {
|
|
|
2297
2358
|
}
|
|
2298
2359
|
return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
|
|
2299
2360
|
}
|
|
2300
|
-
function
|
|
2301
|
-
for (const [ws, conn] of connections) {
|
|
2302
|
-
if (conn.identity.kind === "shared")
|
|
2303
|
-
return ws;
|
|
2304
|
-
}
|
|
2305
|
-
return undefined;
|
|
2306
|
-
}
|
|
2307
|
-
function sendProbe(args) {
|
|
2308
|
-
// Select any connected sidecar, exactly as the pre-deploy window does.
|
|
2309
|
-
// A probe runs in the sidecar's pre-deploy state, so it targets a
|
|
2310
|
-
// connection, not an address -- `findSidecarForNewAgent` ignores its
|
|
2311
|
-
// argument and returns the first connection. Throw immediately on an empty
|
|
2312
|
-
// registry rather than register a probe that could only time out.
|
|
2313
|
-
const ws = findSidecarForNewAgent("");
|
|
2314
|
-
if (ws === undefined) {
|
|
2315
|
-
return Promise.reject(new Error("No sidecar available to probe workflow"));
|
|
2316
|
-
}
|
|
2317
|
-
const conn = connections.get(ws);
|
|
2318
|
-
if (conn === undefined) {
|
|
2319
|
-
return Promise.reject(new Error("No sidecar connected to probe workflow"));
|
|
2320
|
-
}
|
|
2361
|
+
function sendProbeOnConnection(ws, conn, args) {
|
|
2321
2362
|
const requestId = nextRequestId();
|
|
2322
2363
|
return new Promise((resolve, reject) => {
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
}, probeTimeoutMs);
|
|
2327
|
-
pendingProbes.set(requestId, {
|
|
2328
|
-
requestId,
|
|
2329
|
-
ws,
|
|
2364
|
+
pendingProbes.register(requestId, ws, {
|
|
2365
|
+
timeoutMs: probeTimeoutMs,
|
|
2366
|
+
timeoutMessage: `Probe ${requestId} timed out after ${probeTimeoutMs}ms`,
|
|
2330
2367
|
resolve,
|
|
2331
2368
|
reject(error) {
|
|
2332
2369
|
reject(new Error(error));
|
|
2333
2370
|
},
|
|
2334
|
-
|
|
2335
|
-
});
|
|
2371
|
+
}, undefined);
|
|
2336
2372
|
conn.send({
|
|
2337
2373
|
type: "workflow.probe.request",
|
|
2338
2374
|
requestId,
|
|
@@ -2343,6 +2379,19 @@ export function createSidecarRouter(config) {
|
|
|
2343
2379
|
});
|
|
2344
2380
|
});
|
|
2345
2381
|
}
|
|
2382
|
+
async function sendProbeToAllocation(target, args) {
|
|
2383
|
+
const { ws, conn } = await getProvisionedConnection(target, "routing");
|
|
2384
|
+
return sendProbeOnConnection(ws, conn, args);
|
|
2385
|
+
}
|
|
2386
|
+
function disconnectAllocation(target) {
|
|
2387
|
+
const current = allocatedConnections.get(target.allocationId);
|
|
2388
|
+
if (current === undefined ||
|
|
2389
|
+
current.identity.generation !== target.generation) {
|
|
2390
|
+
return;
|
|
2391
|
+
}
|
|
2392
|
+
handleClose(current.ws);
|
|
2393
|
+
current.ws.close();
|
|
2394
|
+
}
|
|
2346
2395
|
function sendAgentUndeploy(agentAddress, reason) {
|
|
2347
2396
|
const ws = addressIndex.get(agentAddress);
|
|
2348
2397
|
if (ws === undefined) {
|
|
@@ -2353,14 +2402,11 @@ export function createSidecarRouter(config) {
|
|
|
2353
2402
|
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
2354
2403
|
}
|
|
2355
2404
|
return new Promise((resolve, reject) => {
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
pendingUndeploys.set(agentAddress, {
|
|
2362
|
-
agentAddress,
|
|
2363
|
-
ws,
|
|
2405
|
+
// Timeout, ack, and error rejection share one closure so the routing
|
|
2406
|
+
// teardown runs exactly once no matter how the round-trip settles.
|
|
2407
|
+
pendingUndeploys.register(agentAddress, ws, {
|
|
2408
|
+
timeoutMs: requestTimeoutMs,
|
|
2409
|
+
timeoutMessage: `Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
2364
2410
|
resolve() {
|
|
2365
2411
|
removeAgentAddress(ws, agentAddress);
|
|
2366
2412
|
resolve();
|
|
@@ -2369,8 +2415,7 @@ export function createSidecarRouter(config) {
|
|
|
2369
2415
|
removeAgentAddress(ws, agentAddress);
|
|
2370
2416
|
reject(new Error(error));
|
|
2371
2417
|
},
|
|
2372
|
-
|
|
2373
|
-
});
|
|
2418
|
+
}, undefined);
|
|
2374
2419
|
conn.send({
|
|
2375
2420
|
type: "agent.undeploy",
|
|
2376
2421
|
agentAddress,
|
|
@@ -2433,12 +2478,13 @@ export function createSidecarRouter(config) {
|
|
|
2433
2478
|
defaultSource,
|
|
2434
2479
|
}));
|
|
2435
2480
|
}
|
|
2436
|
-
async function sendCredentialsUpdate(agentAddress, delivery) {
|
|
2481
|
+
async function sendCredentialsUpdate(agentAddress, delivery, revoke) {
|
|
2437
2482
|
await sendRequest(agentAddress, (requestId) => ({
|
|
2438
2483
|
type: "credentials.update",
|
|
2439
2484
|
requestId,
|
|
2440
2485
|
agentAddress,
|
|
2441
2486
|
delivery,
|
|
2487
|
+
...(revoke !== undefined ? { revoke } : {}),
|
|
2442
2488
|
}));
|
|
2443
2489
|
}
|
|
2444
2490
|
function sendSyncRequest(agentAddress) {
|
|
@@ -2475,8 +2521,10 @@ export function createSidecarRouter(config) {
|
|
|
2475
2521
|
payload: opts.payload,
|
|
2476
2522
|
});
|
|
2477
2523
|
}
|
|
2478
|
-
async function sendSignalDeliverToAllocation(target, opts) {
|
|
2524
|
+
async function sendSignalDeliverToAllocation(target, opts, signal) {
|
|
2525
|
+
signal?.throwIfAborted();
|
|
2479
2526
|
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2527
|
+
signal?.throwIfAborted();
|
|
2480
2528
|
if (addressIndex.get(opts.agentAddress) !== ws) {
|
|
2481
2529
|
throw new Error(`Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2482
2530
|
}
|
|
@@ -2503,24 +2551,23 @@ export function createSidecarRouter(config) {
|
|
|
2503
2551
|
handleClose,
|
|
2504
2552
|
routeMail,
|
|
2505
2553
|
sendRunGrants,
|
|
2506
|
-
|
|
2507
|
-
|
|
2554
|
+
noteSenderDeployStarted,
|
|
2555
|
+
noteSenderDeploySettled,
|
|
2556
|
+
sendProbeToAllocation,
|
|
2557
|
+
disconnectAllocation,
|
|
2508
2558
|
sendAgentUndeploy,
|
|
2509
2559
|
sendSourcesUpdate,
|
|
2510
2560
|
sendCredentialsUpdate,
|
|
2511
|
-
sendPack,
|
|
2512
2561
|
sendPackToAllocation,
|
|
2513
2562
|
sendWorkflowRunPackToAllocation,
|
|
2514
2563
|
fenceAllocation,
|
|
2564
|
+
retireAllocation,
|
|
2515
2565
|
waitForAllocatedSidecar,
|
|
2516
2566
|
isAllocatedSidecarReady,
|
|
2517
2567
|
isAllocatedWorkflowActive,
|
|
2518
2568
|
sendAgentDeployToAllocation,
|
|
2519
|
-
bindStepRoute,
|
|
2520
2569
|
bindAllocatedStepRoute,
|
|
2521
|
-
unbindStepRoute,
|
|
2522
2570
|
unbindAllocatedStepRoute,
|
|
2523
|
-
sendProvisionStep,
|
|
2524
2571
|
sendProvisionStepToAllocation,
|
|
2525
2572
|
sendWorkflowRunDispatchToAllocation,
|
|
2526
2573
|
sendSyncRequest,
|