@intx/hub-sessions 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/agent-repo.d.ts +23 -7
- package/dist/agent-repo.js +19 -6
- package/dist/agent-state-kind.js +18 -63
- package/dist/asset-service.d.ts +1 -20
- package/dist/asset-service.js +19 -97
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +52 -7
- package/dist/credential-push.js +178 -22
- package/dist/event-collector-registry.d.ts +3 -2
- package/dist/event-collector-registry.js +42 -13
- package/dist/event-collector.d.ts +12 -2
- package/dist/event-collector.js +45 -4
- package/dist/hub-session-lookups.d.ts +125 -7
- package/dist/hub-session-lookups.js +541 -86
- package/dist/hub-session-orchestrator.d.ts +2 -3
- package/dist/hub-session-orchestrator.js +22 -56
- package/dist/index.d.ts +19 -9
- package/dist/index.js +16 -7
- package/dist/reconciliation-scheduler.d.ts +14 -0
- package/dist/reconciliation-scheduler.js +55 -0
- package/dist/repo-store/index.d.ts +2 -1
- package/dist/repo-store/index.js +1 -0
- package/dist/repo-store/store.d.ts +1 -1
- package/dist/repo-store/store.js +138 -1
- package/dist/repo-store/subscribe-kind.d.ts +6 -3
- package/dist/repo-store/subscribe-kind.js +42 -77
- package/dist/repo-store/types.d.ts +94 -6
- package/dist/repo-store/user-principal-gate.d.ts +26 -0
- package/dist/repo-store/user-principal-gate.js +78 -0
- package/dist/session-service.d.ts +252 -126
- package/dist/session-service.js +851 -624
- package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
- package/dist/sidecar-allocation/capability-policy.js +124 -0
- package/dist/sidecar-allocation/contracts.d.ts +101 -0
- package/dist/sidecar-allocation/contracts.js +26 -0
- package/dist/sidecar-allocation/index.d.ts +5 -0
- package/dist/sidecar-allocation/index.js +4 -0
- package/dist/sidecar-allocation/operation.d.ts +10 -0
- package/dist/sidecar-allocation/operation.js +54 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +24 -0
- package/dist/sidecar-allocation/plugin-registry.js +61 -0
- package/dist/sidecar-allocation/reconciler.d.ts +54 -0
- package/dist/sidecar-allocation/reconciler.js +825 -0
- package/dist/skill-kind.js +12 -62
- package/dist/substrate.d.ts +3 -3
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +64 -0
- package/dist/workflow-allocation-service.js +554 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +42 -0
- package/dist/workflow-dispatch-service.js +209 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +29 -1
- package/dist/workflow-kind.js +143 -139
- package/dist/workflow-probe-gate.d.ts +286 -0
- package/dist/workflow-probe-gate.js +382 -0
- package/dist/workflow-run-kind.d.ts +239 -32
- package/dist/workflow-run-kind.js +874 -188
- package/dist/workflow-run-reader.d.ts +1 -1
- package/dist/workflow-run-reader.js +3 -7
- package/dist/workflow-run-restore.d.ts +16 -0
- package/dist/workflow-run-restore.js +30 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/workflow-source-pins.d.ts +8 -0
- package/dist/workflow-source-pins.js +14 -0
- package/dist/ws/index.d.ts +3 -3
- package/dist/ws/index.js +2 -2
- package/dist/ws/pending-tracker.d.ts +93 -0
- package/dist/ws/pending-tracker.js +132 -0
- package/dist/ws/sidecar-events.d.ts +139 -37
- package/dist/ws/sidecar-events.js +2 -2
- package/dist/ws/sidecar-handler.d.ts +218 -60
- package/dist/ws/sidecar-handler.js +1713 -732
- package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
- package/dist/ws/sidecar-handler.test-helpers.js +95 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +3 -1
- package/dist/ws/sidecar-token-authenticator.js +78 -7
- package/package.json +14 -13
- package/dist/available-skills-stanza.d.ts +0 -21
- package/dist/available-skills-stanza.js +0 -32
|
@@ -4,19 +4,37 @@
|
|
|
4
4
|
// table of agentAddress → sidecar connection, and dispatches frames between
|
|
5
5
|
// sidecars and the hub's internal systems.
|
|
6
6
|
import { getLogger } from "@intx/log";
|
|
7
|
-
import { verifyEd25519 } from "@intx/crypto";
|
|
8
7
|
import { chunkPack, createPackReceiver } from "@intx/pack-transport";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
8
|
+
import { base64Decode, deriveMessageId, deriveWorkflowRunId, isRunAddress, } from "@intx/types";
|
|
9
|
+
import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
|
|
11
10
|
import { type } from "arktype";
|
|
12
|
-
import { SidecarFrame, } from "@intx/types/sidecar";
|
|
11
|
+
import { MAX_MAIL_OUTBOUND_BODY_BYTES, SidecarFrame, } from "@intx/types/sidecar";
|
|
13
12
|
import { createSidecarEmitter, } from "./sidecar-events.js";
|
|
13
|
+
import { PendingTracker, } from "./pending-tracker.js";
|
|
14
14
|
const logger = getLogger(["hub", "ws", "sidecar"]);
|
|
15
|
+
function deployFrameFailure(message, frameSent, cause) {
|
|
16
|
+
return Object.assign(new Error(message, { cause }), { frameSent });
|
|
17
|
+
}
|
|
18
|
+
export function isDeployFrameFailure(err) {
|
|
19
|
+
return (err instanceof Error &&
|
|
20
|
+
"frameSent" in err &&
|
|
21
|
+
typeof err.frameSent === "boolean");
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Identity validation failed or remained pending at the connection deadline.
|
|
25
|
+
* Readiness is unknown: the worker may be healthy behind the lookup, so
|
|
26
|
+
* callers must retry rather than treat this as a missed connection deadline.
|
|
27
|
+
*/
|
|
28
|
+
export class SidecarIdentityValidationError extends Error {
|
|
29
|
+
constructor(allocationId, generation, cause) {
|
|
30
|
+
super(`Cannot validate sidecar identity for allocation ${allocationId} generation ${String(generation)}`, { cause });
|
|
31
|
+
this.name = "SidecarIdentityValidationError";
|
|
32
|
+
}
|
|
33
|
+
}
|
|
15
34
|
/**
|
|
16
|
-
* Whether this connection owns `address` for routing/lifecycle purposes
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* re-add path), but ownership readers -- pack-transfer authorization,
|
|
35
|
+
* Whether this connection owns `address` for routing/lifecycle purposes.
|
|
36
|
+
* The legacy and workflow sets remain physically distinct, but ownership
|
|
37
|
+
* readers -- pack-transfer authorization,
|
|
20
38
|
* in-flight cancellation, disconnect teardown -- must see the union, or a
|
|
21
39
|
* reconnected workflow deployment (which lives only in `workflowAddresses`)
|
|
22
40
|
* is silently treated as unowned even though its mail routes.
|
|
@@ -24,29 +42,88 @@ const logger = getLogger(["hub", "ws", "sidecar"]);
|
|
|
24
42
|
function connOwnsAddress(conn, address) {
|
|
25
43
|
return (conn.agentAddresses.has(address) || conn.workflowAddresses.has(address));
|
|
26
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Bind pack writes to the repository implied by the authenticated address.
|
|
47
|
+
* An allocated credential is narrower still: it may only write its one
|
|
48
|
+
* deployment's workflow-run repository and never a standalone agent-state
|
|
49
|
+
* repository.
|
|
50
|
+
*/
|
|
51
|
+
function connCanPushRepo(conn, agentAddress, repoId) {
|
|
52
|
+
if (conn.identity.kind !== "allocated")
|
|
53
|
+
return false;
|
|
54
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
return (repoId.kind === "workflow-run" &&
|
|
58
|
+
repoId.id === deriveWorkflowRunRepoId(agentAddress));
|
|
59
|
+
}
|
|
27
60
|
/** The deduped set of every address this connection owns (session + workflow). */
|
|
28
61
|
function ownedAddresses(conn) {
|
|
29
62
|
return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
|
|
30
63
|
}
|
|
31
64
|
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
32
|
-
|
|
65
|
+
// A probe fetches a workflow's dependency closure from a registry and
|
|
66
|
+
// evaluates it on the sidecar, so it runs longer than a routine request; its
|
|
67
|
+
// default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS.
|
|
68
|
+
export const DEFAULT_PROBE_TIMEOUT_MS = 60_000;
|
|
33
69
|
const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100;
|
|
34
70
|
const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
|
|
35
71
|
const DEFAULT_PING_TIMEOUT_MS = 60_000;
|
|
72
|
+
const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000;
|
|
73
|
+
const DEFAULT_MAIL_ACK_MAX_RETRIES = 5;
|
|
74
|
+
// The hub re-resolves and re-pushes a key for each rotatable sender a sidecar
|
|
75
|
+
// reports on (re)connect. A legitimate sidecar caches keys for tens, maybe low
|
|
76
|
+
// hundreds of distinct user senders, so this cap sits well above ten times that
|
|
77
|
+
// ceiling: it NEVER truncates a real report -- dropping a genuine sender would
|
|
78
|
+
// leave its key stale, the exact failure this refresh exists to prevent. It
|
|
79
|
+
// bounds only a hostile or buggy sidecar, since a compromised authenticated
|
|
80
|
+
// sidecar could otherwise report an unbounded set and drive that many sequential
|
|
81
|
+
// DB resolves on every reconnect. The cap lives in the handler, not on the
|
|
82
|
+
// arktype frame schema, on purpose: rejecting an over-cap frame at parse would
|
|
83
|
+
// fail the whole reconnect (a hard outage) rather than degrade gracefully to a
|
|
84
|
+
// bounded refresh.
|
|
85
|
+
export const MAX_RESYNC_SENDER_ADDRESSES = 2048;
|
|
36
86
|
export function createSidecarRouter(config) {
|
|
37
|
-
const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,
|
|
87
|
+
const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, scheduleTimeout = (handler, ms) => {
|
|
88
|
+
const handle = setTimeout(handler, ms);
|
|
89
|
+
return () => {
|
|
90
|
+
clearTimeout(handle);
|
|
91
|
+
};
|
|
92
|
+
}, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
|
|
38
93
|
// Receiver-dispatch surface. Wire-layer callsites emit events here;
|
|
39
94
|
// host code subscribes via `router.events`.
|
|
40
95
|
const events = createSidecarEmitter();
|
|
41
96
|
// ws handle → registered connection
|
|
42
97
|
const connections = new Map();
|
|
98
|
+
const allocatedConnections = new Map();
|
|
99
|
+
const allocationFences = new Map();
|
|
100
|
+
const allocationWaiters = new Map();
|
|
43
101
|
// agentAddress → ws handle (routing table)
|
|
44
102
|
const addressIndex = new Map();
|
|
45
|
-
// requestId → pending promise
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
103
|
+
// requestId → pending promise (resolved by session.ack, rejected by
|
|
104
|
+
// session.error). `PendingTracker` owns the register/timeout/settle/sweep
|
|
105
|
+
// lifecycle shared by all five pending round-trips below; each entry's
|
|
106
|
+
// resolve/reject closures carry the per-round-trip cleanup.
|
|
107
|
+
const pendingRequests = new PendingTracker();
|
|
108
|
+
// agentAddress → pending deploy promise (matched by agent.deploy.ack/agent.error)
|
|
109
|
+
const pendingDeploys = new PendingTracker();
|
|
110
|
+
// Run addresses whose ALLOCATED deploy is mid-flight -- key-record has been
|
|
111
|
+
// started but not yet committed. pendingDeploys clears at the deploy ack, but an
|
|
112
|
+
// allocated run's key is recorded LATER by session-service's anchor-key update,
|
|
113
|
+
// so pendingDeploys alone under-covers the allocated pre-ack window. session-
|
|
114
|
+
// service brackets this marker across its deploy try/catch: set before the
|
|
115
|
+
// deploy emit, cleared by noteSenderDeploySettled on record or failure.
|
|
116
|
+
const allocatedKeyRecordInFlight = new Map();
|
|
49
117
|
const disconnectedAgents = new Map();
|
|
118
|
+
const pendingMail = new Map();
|
|
119
|
+
// agentAddress → retention TTL timer for un-acked pending mail held across a
|
|
120
|
+
// disconnect. On close the per-entry retry timers are cleared (the socket is
|
|
121
|
+
// gone) but the entries are RETAINED so a verified reconnect can redeliver
|
|
122
|
+
// them; this timer bounds that retention so a sidecar that never reconnects
|
|
123
|
+
// does not leak entries. Cleared when the address reconnects (redelivery) or
|
|
124
|
+
// its last pending entry is acked.
|
|
125
|
+
const pendingMailRetention = new Map();
|
|
126
|
+
const deferredSenderMail = new Map();
|
|
50
127
|
// agentAddress → set of subscriber callbacks for agent events
|
|
51
128
|
const agentSubscribers = new Map();
|
|
52
129
|
// agentAddress → cached connector-thread state, populated by
|
|
@@ -66,9 +143,17 @@ export function createSidecarRouter(config) {
|
|
|
66
143
|
// a single in-flight promise per ws (replaced each queued frame), cleared on
|
|
67
144
|
// close.
|
|
68
145
|
const messageChains = new Map();
|
|
69
|
-
const pendingPacks = new
|
|
146
|
+
const pendingPacks = new PendingTracker();
|
|
70
147
|
let packCounter = 0;
|
|
71
|
-
|
|
148
|
+
// agentAddress → pending undeploy (resolved by agent.undeploy.ack)
|
|
149
|
+
const pendingUndeploys = new PendingTracker();
|
|
150
|
+
// requestId → pending workflow probe (resolved by workflow.probe.result,
|
|
151
|
+
// rejected by workflow.probe.error). Result-carrying, unlike the other
|
|
152
|
+
// trackers (which resolve void): a probe returns the sidecar's inert
|
|
153
|
+
// projection + grant set + wire hash. Keyed on requestId alone -- the
|
|
154
|
+
// probe runs in the sidecar's pre-deploy state and enters no address map,
|
|
155
|
+
// so `handleClose`'s ws-keyed sweep is its ONLY disconnect cleanup.
|
|
156
|
+
const pendingProbes = new PendingTracker();
|
|
72
157
|
// Receives agent-state packs pushed from sidecars. The wire frames
|
|
73
158
|
// (`repo.pack.push` / `repo.pack.done`) are shared with the
|
|
74
159
|
// workflow-run flow; dispatch on `repoId.kind` picks which receiver
|
|
@@ -112,29 +197,303 @@ export function createSidecarRouter(config) {
|
|
|
112
197
|
entry.queue.push(frame);
|
|
113
198
|
return true;
|
|
114
199
|
}
|
|
115
|
-
|
|
116
|
-
|
|
200
|
+
// Arm a redelivery-retry timer for a tracked pending mail. Wraps the async
|
|
201
|
+
// `retryPendingMail` so a rejection -- a socket write that throws once the
|
|
202
|
+
// sidecar is gone -- is logged rather than floating out of the timer as an
|
|
203
|
+
// unhandled rejection.
|
|
204
|
+
function scheduleMailRetry(agentAddress, messageId) {
|
|
205
|
+
return scheduleTimeout(() => {
|
|
206
|
+
void retryPendingMail(agentAddress, messageId).catch((err) => {
|
|
207
|
+
logger.warn `Redelivery retry for mail ${messageId} to ${agentAddress} failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
208
|
+
});
|
|
209
|
+
}, mailAckRetryIntervalMs);
|
|
210
|
+
}
|
|
211
|
+
// Track a connected-window `mail.inbound` for redelivery until the sidecar
|
|
212
|
+
// acks its durable inbox write. Replaces any prior entry for the same
|
|
213
|
+
// (agentAddress, messageId) -- clearing its timer first so no timer leaks --
|
|
214
|
+
// which keeps a re-sent delivery from arming a second concurrent retry loop.
|
|
215
|
+
function trackPendingMail(agentAddress, messageId, frame, runGrants, allocatedTarget) {
|
|
216
|
+
let byId = pendingMail.get(agentAddress);
|
|
217
|
+
if (byId === undefined) {
|
|
218
|
+
byId = new Map();
|
|
219
|
+
pendingMail.set(agentAddress, byId);
|
|
220
|
+
}
|
|
221
|
+
const existing = byId.get(messageId);
|
|
222
|
+
if (existing !== undefined)
|
|
223
|
+
existing.cancelRetry();
|
|
224
|
+
byId.set(messageId, {
|
|
225
|
+
agentAddress,
|
|
226
|
+
messageId,
|
|
227
|
+
frame,
|
|
228
|
+
attempts: 0,
|
|
229
|
+
cancelRetry: scheduleMailRetry(agentAddress, messageId),
|
|
230
|
+
...(runGrants !== undefined ? { runGrants } : {}),
|
|
231
|
+
...(allocatedTarget !== undefined ? { allocatedTarget } : {}),
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
// Resolve the frame that must precede a redelivery of a trigger mail on the
|
|
235
|
+
// FIFO socket, re-resolving a keyless run sender's key so it still
|
|
236
|
+
// co-delivers. Returns:
|
|
237
|
+
// - a `run.grants` frame when the entry carries run grants (the redelivered
|
|
238
|
+
// run resolves its onRunStart barrier instead of failing closed on
|
|
239
|
+
// missing grants);
|
|
240
|
+
// - a bare `sender.key.refresh` frame when the entry carries NO run grants
|
|
241
|
+
// but its run sender's key was never co-delivered, so the recipient still
|
|
242
|
+
// caches the key ahead of the mail;
|
|
243
|
+
// - `undefined` when nothing must precede the mail.
|
|
244
|
+
//
|
|
245
|
+
// The re-resolve is KIND-GATED to run-address senders only. A run's
|
|
246
|
+
// deployment key is immutable once acked, so the re-resolved key equals the
|
|
247
|
+
// signing-time key -- safe. A user (non-run) sender's key may have rotated
|
|
248
|
+
// since it signed, so re-resolving would check the fixed signed bytes against
|
|
249
|
+
// a newer key and turn a valid message into a false `invalid`; such a sender
|
|
250
|
+
// lacking a captured key stays keyless (an honest `unknown`). An entry that
|
|
251
|
+
// captured `senderIdentities` at track time replays that snapshot as-is: it
|
|
252
|
+
// holds the signing-time key and is never re-resolved.
|
|
253
|
+
//
|
|
254
|
+
// Awaits any key resolve so the caller sends the returned frame and the mail
|
|
255
|
+
// back-to-back with no await between them, keeping the co-delivered key ahead
|
|
256
|
+
// of the mail on the FIFO socket.
|
|
257
|
+
async function resolveReplayLeadFrame(entry) {
|
|
258
|
+
const authenticatedSender = entry.frame.type === "mail.inbound"
|
|
259
|
+
? entry.frame.authenticatedSender
|
|
260
|
+
: undefined;
|
|
261
|
+
const senderIsRun = authenticatedSender !== undefined && isRunAddress(authenticatedSender);
|
|
262
|
+
if (entry.runGrants === undefined) {
|
|
263
|
+
if (authenticatedSender === undefined || !senderIsRun)
|
|
264
|
+
return undefined;
|
|
265
|
+
const key = await reresolveRunSenderKey(authenticatedSender);
|
|
266
|
+
if (key === null)
|
|
267
|
+
return undefined;
|
|
268
|
+
return {
|
|
269
|
+
type: "sender.key.refresh",
|
|
270
|
+
address: authenticatedSender,
|
|
271
|
+
publicKey: key,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
let senderIdentities = entry.runGrants.senderIdentities;
|
|
275
|
+
if (senderIdentities === undefined &&
|
|
276
|
+
authenticatedSender !== undefined &&
|
|
277
|
+
senderIsRun) {
|
|
278
|
+
const key = await reresolveRunSenderKey(authenticatedSender);
|
|
279
|
+
senderIdentities = senderIdentitiesFromKey(authenticatedSender, key);
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
type: "run.grants",
|
|
283
|
+
agentAddress: entry.agentAddress,
|
|
284
|
+
runId: entry.runGrants.runId,
|
|
285
|
+
stepGrants: entry.runGrants.stepGrants,
|
|
286
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
// Best-effort re-resolve of a run sender's hub-held key at replay time. Only
|
|
290
|
+
// called for a run-address sender, whose deployment key is immutable once
|
|
291
|
+
// acked, so the current key equals the signing-time key. Returns null when no
|
|
292
|
+
// resolver is wired or the sender has no durable key.
|
|
293
|
+
//
|
|
294
|
+
// This relies on `lookups.resolveSenderKey` being the BEST-EFFORT,
|
|
295
|
+
// NEVER-THROWS resolver (the contract at sidecar-events.ts:274-279, wired to
|
|
296
|
+
// resolveFrameSenderKey, which swallows faults to null). That contract is
|
|
297
|
+
// load-bearing here: `redeliverPendingMail` clears the retention TTL up-front
|
|
298
|
+
// and re-arms each entry's per-entry timer only on a successful send, so a
|
|
299
|
+
// resolver that THREW would abort the redeliver loop and strand the
|
|
300
|
+
// not-yet-processed entries with no timer and no TTL until a process restart.
|
|
301
|
+
// A strict/throwing resolver must NOT be wired here. Do not add a try/catch:
|
|
302
|
+
// the boundary owns the never-throws contract; duplicating it here would
|
|
303
|
+
// violate that ownership. The dispatch-time resolveSenderKey call
|
|
304
|
+
// (sendWorkflowRunDispatchToAllocation path) carries the same dependency
|
|
305
|
+
// note.
|
|
306
|
+
async function reresolveRunSenderKey(authenticatedSender) {
|
|
307
|
+
const resolveSenderKey = lookups.resolveSenderKey;
|
|
308
|
+
if (resolveSenderKey === undefined)
|
|
309
|
+
return null;
|
|
310
|
+
return resolveSenderKey(authenticatedSender);
|
|
311
|
+
}
|
|
312
|
+
// Replay a pending mail's lead frame (its run grants or a re-resolved sender
|
|
313
|
+
// key) and then the mail itself over `conn`. Awaits the resolve FIRST, then
|
|
314
|
+
// sends the lead frame and the mail back-to-back with NO await between them,
|
|
315
|
+
// so the co-delivered key always precedes the mail on the FIFO socket.
|
|
316
|
+
// Returns whether the mail was (re)sent, so the caller re-arms the retry timer
|
|
317
|
+
// only for an entry it actually redelivered.
|
|
318
|
+
async function replaySendPendingMail(conn, entry) {
|
|
319
|
+
const lead = await resolveReplayLeadFrame(entry);
|
|
320
|
+
// The resolve above may have awaited real I/O; during that gap a queued
|
|
321
|
+
// `mail.inbound.ack` can advance and run `resolvePendingMail` (delete +
|
|
322
|
+
// clearTimeout) on this entry. The window is opened by the timer-macrotask
|
|
323
|
+
// retry path, NOT by any bypass: `mail.inbound.ack` is a QUEUED frame
|
|
324
|
+
// (frameBypassesQueue returns false for it). It can interleave because the
|
|
325
|
+
// retry runs as an independent setTimeout macrotask (retryPendingMail), so
|
|
326
|
+
// the owning ws's message chain is free to advance the ack during the
|
|
327
|
+
// resolve await. On the reconnect/redeliver path the ack cannot interleave
|
|
328
|
+
// at all -- it queues behind the still-running reconnect handler on the
|
|
329
|
+
// same ws -- so here this guard is pure defense-in-depth. Re-confirm it is
|
|
330
|
+
// still the tracked entry before sending, or a post-ack redelivery would
|
|
331
|
+
// arm a retry timer on a detached entry.
|
|
332
|
+
if (pendingMail.get(entry.agentAddress)?.get(entry.messageId) !== entry) {
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
// The same gap can span a disconnect or a takeover that moves the address
|
|
336
|
+
// off `conn`. Sending on the stale conn would write to a dead socket and
|
|
337
|
+
// re-arm a retry that later drops a still-retained entry. Skip so the entry
|
|
338
|
+
// survives for the reconnect redelivery.
|
|
339
|
+
const ws = addressIndex.get(entry.agentAddress);
|
|
340
|
+
if (ws === undefined || connections.get(ws) !== conn)
|
|
341
|
+
return false;
|
|
342
|
+
if (lead !== undefined)
|
|
343
|
+
conn.send(lead);
|
|
344
|
+
conn.send(entry.frame);
|
|
345
|
+
return true;
|
|
346
|
+
}
|
|
347
|
+
function deletePendingMail(byId, agentAddress, messageId) {
|
|
348
|
+
byId.delete(messageId);
|
|
349
|
+
if (byId.size === 0) {
|
|
350
|
+
pendingMail.delete(agentAddress);
|
|
351
|
+
// The retention TTL guards a non-empty pending set; drop it once the set
|
|
352
|
+
// is empty so it never outlives the entries it was bounding.
|
|
353
|
+
const retention = pendingMailRetention.get(agentAddress);
|
|
354
|
+
if (retention !== undefined) {
|
|
355
|
+
clearTimeout(retention);
|
|
356
|
+
pendingMailRetention.delete(agentAddress);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
async function retryPendingMail(agentAddress, messageId) {
|
|
361
|
+
const byId = pendingMail.get(agentAddress);
|
|
362
|
+
if (byId === undefined)
|
|
363
|
+
return;
|
|
364
|
+
const entry = byId.get(messageId);
|
|
117
365
|
if (entry === undefined)
|
|
118
366
|
return;
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
367
|
+
if (entry.attempts >= mailAckMaxRetries) {
|
|
368
|
+
// The sidecar never acked within the retry budget. The ack is withheld
|
|
369
|
+
// precisely because the sidecar's durable inbox write failed, so the
|
|
370
|
+
// mail was NOT delivered: surface it as undelivered so the host can relay
|
|
371
|
+
// it onto an external transport, then drop the pending entry so its timer
|
|
372
|
+
// does not leak.
|
|
373
|
+
if (entry.frame.type === "mail.inbound") {
|
|
374
|
+
events.emit("mail.outbound.undelivered", {
|
|
375
|
+
rawMessage: entry.frame.rawMessage,
|
|
376
|
+
recipients: [agentAddress],
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
380
|
+
logger.warn `Gave up redelivering mail ${messageId} to ${agentAddress} after ${String(entry.attempts)} un-acked attempt(s)`;
|
|
381
|
+
return;
|
|
123
382
|
}
|
|
124
|
-
|
|
125
|
-
|
|
383
|
+
// Redeliver over the address's CURRENT owner: a verified reconnect may have
|
|
384
|
+
// moved the address to a new connection since the original delivery.
|
|
385
|
+
const ws = addressIndex.get(agentAddress);
|
|
386
|
+
const conn = ws !== undefined ? connections.get(ws) : undefined;
|
|
387
|
+
const allocated = entry.allocatedTarget === undefined
|
|
388
|
+
? undefined
|
|
389
|
+
: allocatedConnections.get(entry.allocatedTarget.allocationId);
|
|
390
|
+
const targetStillOwnsAddress = entry.allocatedTarget === undefined ||
|
|
391
|
+
(allocated !== undefined &&
|
|
392
|
+
allocated.identity.generation === entry.allocatedTarget.generation &&
|
|
393
|
+
allocated.ws === ws);
|
|
394
|
+
if (conn === undefined || !targetStillOwnsAddress) {
|
|
395
|
+
// No live connection to recover into. Connected-window redelivery only
|
|
396
|
+
// applies while the address is routable; a disconnected address is not
|
|
397
|
+
// retried here.
|
|
398
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
399
|
+
logger.warn `Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`;
|
|
400
|
+
return;
|
|
126
401
|
}
|
|
402
|
+
if (!(await replaySendPendingMail(conn, entry)))
|
|
403
|
+
return;
|
|
404
|
+
entry.attempts += 1;
|
|
405
|
+
entry.cancelRetry = scheduleMailRetry(agentAddress, messageId);
|
|
127
406
|
}
|
|
128
|
-
function
|
|
129
|
-
const
|
|
407
|
+
function resolvePendingMail(agentAddress, messageId) {
|
|
408
|
+
const byId = pendingMail.get(agentAddress);
|
|
409
|
+
if (byId === undefined)
|
|
410
|
+
return;
|
|
411
|
+
const entry = byId.get(messageId);
|
|
412
|
+
if (entry === undefined)
|
|
413
|
+
return;
|
|
414
|
+
entry.cancelRetry();
|
|
415
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
416
|
+
}
|
|
417
|
+
// Hold an address's un-acked pending mail across a disconnect. The per-entry
|
|
418
|
+
// retry timers are cleared -- retrying over the dead socket is pointless --
|
|
419
|
+
// but the entries are KEPT so a verified reconnect can redeliver them. A
|
|
420
|
+
// retention TTL (the disconnect-queue horizon) bounds the hold so a sidecar
|
|
421
|
+
// that never reconnects does not leak; on expiry the still-un-acked entries
|
|
422
|
+
// are surfaced as `mail.outbound.undelivered` so the host can relay them,
|
|
423
|
+
// since a withheld ack means the sidecar's durable write never landed.
|
|
424
|
+
function retainPendingMailForAddress(agentAddress) {
|
|
425
|
+
const byId = pendingMail.get(agentAddress);
|
|
426
|
+
if (byId === undefined)
|
|
427
|
+
return;
|
|
428
|
+
for (const entry of byId.values())
|
|
429
|
+
entry.cancelRetry();
|
|
430
|
+
const existing = pendingMailRetention.get(agentAddress);
|
|
130
431
|
if (existing !== undefined)
|
|
131
432
|
clearTimeout(existing);
|
|
132
433
|
const timer = setTimeout(() => {
|
|
434
|
+
pendingMailRetention.delete(agentAddress);
|
|
435
|
+
const expired = pendingMail.get(agentAddress);
|
|
436
|
+
pendingMail.delete(agentAddress);
|
|
437
|
+
if (expired !== undefined && expired.size > 0) {
|
|
438
|
+
for (const entry of expired.values()) {
|
|
439
|
+
if (entry.frame.type !== "mail.inbound")
|
|
440
|
+
continue;
|
|
441
|
+
events.emit("mail.outbound.undelivered", {
|
|
442
|
+
rawMessage: entry.frame.rawMessage,
|
|
443
|
+
recipients: [agentAddress],
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
logger.warn `Dropping ${String(expired.size)} un-acked message(s) for ${agentAddress}: pending-mail retention TTL expired`;
|
|
447
|
+
}
|
|
448
|
+
}, disconnectQueueTTLMs);
|
|
449
|
+
pendingMailRetention.set(agentAddress, timer);
|
|
450
|
+
}
|
|
451
|
+
// Redeliver an address's retained un-acked pending mail on a verified
|
|
452
|
+
// reconnect. Replays identical bytes (same messageId) over the new
|
|
453
|
+
// connection, so the sidecar's inbox dedups a message it already wrote
|
|
454
|
+
// (effectively-once) and processes one it had dropped (no loss). Re-arms the
|
|
455
|
+
// connected-window retry over the new connection with a fresh per-generation
|
|
456
|
+
// budget, so a redelivery that is itself dropped before its ack is retried.
|
|
457
|
+
async function redeliverPendingMail(agentAddress, conn) {
|
|
458
|
+
const retention = pendingMailRetention.get(agentAddress);
|
|
459
|
+
if (retention !== undefined) {
|
|
460
|
+
clearTimeout(retention);
|
|
461
|
+
pendingMailRetention.delete(agentAddress);
|
|
462
|
+
}
|
|
463
|
+
const byId = pendingMail.get(agentAddress);
|
|
464
|
+
if (byId === undefined)
|
|
465
|
+
return;
|
|
466
|
+
for (const entry of [...byId.values()]) {
|
|
467
|
+
if (entry.allocatedTarget !== undefined &&
|
|
468
|
+
(conn.identity.kind !== "allocated" ||
|
|
469
|
+
conn.identity.allocationId !== entry.allocatedTarget.allocationId ||
|
|
470
|
+
conn.identity.generation !== entry.allocatedTarget.generation)) {
|
|
471
|
+
// The Hub-owned dispatch row survives generation replacement and will
|
|
472
|
+
// be requeued by the allocation-ready callback. Do not leak or replay
|
|
473
|
+
// this generation-local retry entry onto a different worker.
|
|
474
|
+
entry.cancelRetry();
|
|
475
|
+
deletePendingMail(byId, agentAddress, entry.messageId);
|
|
476
|
+
continue;
|
|
477
|
+
}
|
|
478
|
+
if (!(await replaySendPendingMail(conn, entry)))
|
|
479
|
+
continue;
|
|
480
|
+
entry.attempts = 0;
|
|
481
|
+
entry.cancelRetry = scheduleMailRetry(agentAddress, entry.messageId);
|
|
482
|
+
}
|
|
483
|
+
if (byId.size > 0) {
|
|
484
|
+
logger.info `Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
function resetLivenessTimer(ws) {
|
|
488
|
+
const existing = livenessTimers.get(ws);
|
|
489
|
+
if (existing !== undefined)
|
|
490
|
+
existing();
|
|
491
|
+
const cancel = scheduleTimeout(() => {
|
|
133
492
|
livenessTimers.delete(ws);
|
|
134
493
|
logger.warn `Sidecar ping timeout, closing connection`;
|
|
135
494
|
ws.close();
|
|
136
495
|
}, pingTimeoutMs);
|
|
137
|
-
livenessTimers.set(ws,
|
|
496
|
+
livenessTimers.set(ws, cancel);
|
|
138
497
|
}
|
|
139
498
|
function handlePing(ws) {
|
|
140
499
|
resetLivenessTimer(ws);
|
|
@@ -197,11 +556,9 @@ export function createSidecarRouter(config) {
|
|
|
197
556
|
// state. Such a frame has no ordering obligation against new inbound frames
|
|
198
557
|
// (a response cannot resolve "too early" for a request that already went
|
|
199
558
|
// out), and it is exactly what in-flight queued handlers block on, so it MUST
|
|
200
|
-
// run out of band or
|
|
201
|
-
//
|
|
202
|
-
//
|
|
203
|
-
// inbound payload whose order matters, so it queues. The exhaustive switch +
|
|
204
|
-
// assertNever makes adding a SidecarFrame variant without classifying it a
|
|
559
|
+
// run out of band. Every other frame establishes or reads routing, or carries
|
|
560
|
+
// an inbound payload whose order matters, so it queues. The exhaustive switch
|
|
561
|
+
// + assertNever makes adding a SidecarFrame variant without classifying it a
|
|
205
562
|
// compile error, not a latent deadlock or a silent bypass hole.
|
|
206
563
|
function frameBypassesQueue(frame) {
|
|
207
564
|
switch (frame.type) {
|
|
@@ -213,13 +570,16 @@ export function createSidecarRouter(config) {
|
|
|
213
570
|
case "agent.undeploy.ack":
|
|
214
571
|
case "repo.pack.ack":
|
|
215
572
|
case "repo.pack.reject":
|
|
573
|
+
case "workflow.probe.result":
|
|
574
|
+
case "workflow.probe.error":
|
|
216
575
|
return true;
|
|
217
576
|
case "register":
|
|
218
577
|
case "reconnect":
|
|
219
|
-
case "challenge.response":
|
|
220
578
|
case "mail.outbound":
|
|
221
579
|
case "agent.event":
|
|
222
580
|
case "connector.state.changed":
|
|
581
|
+
case "mail.inbound.ack":
|
|
582
|
+
case "signal.correlation.register":
|
|
223
583
|
case "repo.pack.push":
|
|
224
584
|
case "repo.pack.done":
|
|
225
585
|
return false;
|
|
@@ -229,50 +589,82 @@ export function createSidecarRouter(config) {
|
|
|
229
589
|
}
|
|
230
590
|
// Runs one frame's handler. Returns the handler's promise for async handlers
|
|
231
591
|
// so the per-ws chain can await bounded completion; sync handlers return
|
|
232
|
-
// void. Never awaits a promise that resolves on a
|
|
233
|
-
// only such await (the challenge round-trip's session.ack) is reached via a
|
|
234
|
-
// bypass frame, which does not queue.
|
|
592
|
+
// void. Never awaits a promise that resolves on a later same-ws frame.
|
|
235
593
|
function dispatchFrame(ws, frame) {
|
|
594
|
+
const registeredIdentity = connections.get(ws)?.identity;
|
|
595
|
+
if (registeredIdentity?.kind === "probe" &&
|
|
596
|
+
frame.type !== "register" &&
|
|
597
|
+
frame.type !== "reconnect" &&
|
|
598
|
+
frame.type !== "ping" &&
|
|
599
|
+
frame.type !== "workflow.probe.result" &&
|
|
600
|
+
frame.type !== "workflow.probe.error") {
|
|
601
|
+
logger.warn `Rejected ${frame.type} from probe sidecar ${registeredIdentity.sidecarId}`;
|
|
602
|
+
handleClose(ws);
|
|
603
|
+
ws.close();
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
236
606
|
switch (frame.type) {
|
|
237
607
|
case "register": {
|
|
238
608
|
const agentAddresses = frame.agentAddresses;
|
|
239
|
-
|
|
609
|
+
const cachedSenderAddresses = frame.cachedSenderAddresses ?? [];
|
|
610
|
+
return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses, cachedSenderAddresses));
|
|
240
611
|
}
|
|
241
612
|
case "reconnect": {
|
|
242
613
|
const agentAddresses = frame.agentAddresses;
|
|
243
|
-
const
|
|
244
|
-
return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses,
|
|
614
|
+
const cachedSenderAddresses = frame.cachedSenderAddresses ?? [];
|
|
615
|
+
return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses));
|
|
245
616
|
}
|
|
246
|
-
case "challenge.response":
|
|
247
|
-
return handleChallengeResponse(ws, frame.responses);
|
|
248
617
|
case "agent.deploy.ack":
|
|
249
|
-
return handleDeployAck(
|
|
618
|
+
return handleDeployAck(ws, frame);
|
|
250
619
|
case "agent.error":
|
|
251
|
-
|
|
252
|
-
rejectUndeployPending(frame.agentAddress, frame.error);
|
|
620
|
+
rejectDeployPendingFromFrame(ws, frame.agentAddress, frame.error);
|
|
621
|
+
rejectUndeployPending(ws, frame.agentAddress, frame.error);
|
|
253
622
|
return;
|
|
254
623
|
case "agent.undeploy.ack":
|
|
255
|
-
resolveUndeployPending(frame.agentAddress);
|
|
624
|
+
resolveUndeployPending(ws, frame.agentAddress);
|
|
256
625
|
return;
|
|
257
626
|
case "ping":
|
|
258
627
|
handlePing(ws);
|
|
259
628
|
return;
|
|
260
|
-
case "mail.outbound":
|
|
261
|
-
|
|
262
|
-
|
|
629
|
+
case "mail.outbound": {
|
|
630
|
+
const conn = connections.get(ws);
|
|
631
|
+
if (conn === undefined)
|
|
632
|
+
return;
|
|
633
|
+
if (!connOwnsAddress(conn, frame.senderAddress)) {
|
|
634
|
+
logger.warn `Dropping mail.outbound from ${frame.senderAddress}: not registered to this sidecar`;
|
|
263
635
|
return;
|
|
264
636
|
}
|
|
265
|
-
|
|
266
|
-
|
|
637
|
+
// The DoS backstop and the trust boundary for an untrusted sidecar's
|
|
638
|
+
// mail body: measure the true byte cost (a hostile sidecar can send
|
|
639
|
+
// multi-byte UTF-8, so `.length` would undercount) and drop an over-cap
|
|
640
|
+
// frame here before either delivery path allocates on it. The socket's
|
|
641
|
+
// maxPayloadLength has already closed the connection for a truly huge
|
|
642
|
+
// frame; this catches one between the mail cap and that ceiling.
|
|
643
|
+
const bodyBytes = Buffer.byteLength(frame.rawMessage, "utf8");
|
|
644
|
+
if (bodyBytes > MAX_MAIL_OUTBOUND_BODY_BYTES) {
|
|
645
|
+
logger.warn `Dropping mail.outbound from ${frame.senderAddress}: rawMessage of ${String(bodyBytes)} bytes exceeds the ${String(MAX_MAIL_OUTBOUND_BODY_BYTES)}-byte cap`;
|
|
646
|
+
return;
|
|
267
647
|
}
|
|
268
|
-
if (
|
|
269
|
-
|
|
648
|
+
if (frame.delivered !== true) {
|
|
649
|
+
// frame.senderAddress is the sender this connection was just gated
|
|
650
|
+
// on by connOwnsAddress above -- a hub-verified value. Thread it so
|
|
651
|
+
// the relayed inbound frame is stamped with it, not the MIME From.
|
|
652
|
+
return handleMailOutbound(frame.rawMessage, frame.senderAddress, frame.recipients);
|
|
270
653
|
}
|
|
271
|
-
|
|
272
|
-
|
|
654
|
+
if (lookups.persistMail) {
|
|
655
|
+
return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
|
|
273
656
|
}
|
|
657
|
+
logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
|
|
274
658
|
return;
|
|
275
|
-
|
|
659
|
+
}
|
|
660
|
+
case "agent.event": {
|
|
661
|
+
const conn = connections.get(ws);
|
|
662
|
+
if (conn === undefined)
|
|
663
|
+
return;
|
|
664
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
665
|
+
logger.warn `Dropping agent.event for ${frame.agentAddress}: not registered to this sidecar`;
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
276
668
|
events.emit("agent.event", {
|
|
277
669
|
agentAddress: frame.agentAddress,
|
|
278
670
|
sessionId: frame.sessionId,
|
|
@@ -280,6 +672,7 @@ export function createSidecarRouter(config) {
|
|
|
280
672
|
});
|
|
281
673
|
dispatchToSubscribers(frame.agentAddress, frame.event);
|
|
282
674
|
return;
|
|
675
|
+
}
|
|
283
676
|
case "connector.state.changed":
|
|
284
677
|
// Gate the cache write on the sending sidecar actually owning
|
|
285
678
|
// the named agent. A misbehaving sidecar that knows another
|
|
@@ -294,23 +687,66 @@ export function createSidecarRouter(config) {
|
|
|
294
687
|
connectorState: frame.connectorState,
|
|
295
688
|
});
|
|
296
689
|
return;
|
|
690
|
+
case "mail.inbound.ack": {
|
|
691
|
+
// Terminal receipt for a connected-window `mail.inbound`: the sidecar
|
|
692
|
+
// has durably written the message to its inbox. Gate on ownership --
|
|
693
|
+
// like connector.state.changed and signal.correlation.register -- so a
|
|
694
|
+
// sidecar cannot clear another sidecar's pending mail. The messageId is
|
|
695
|
+
// a hub-minted id only the owning sidecar ever received on the frame,
|
|
696
|
+
// so the ownership check is defense-in-depth, not the sole guard.
|
|
697
|
+
const conn = connections.get(ws);
|
|
698
|
+
if (conn === undefined)
|
|
699
|
+
return;
|
|
700
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
701
|
+
logger.warn `Dropping mail.inbound.ack for ${frame.agentAddress}: not registered to this sidecar`;
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
resolvePendingMail(frame.agentAddress, frame.messageId);
|
|
705
|
+
events.emit("mail.inbound.acknowledged", {
|
|
706
|
+
agentAddress: frame.agentAddress,
|
|
707
|
+
messageId: frame.messageId,
|
|
708
|
+
...(conn.identity.kind === "allocated"
|
|
709
|
+
? {
|
|
710
|
+
allocated: {
|
|
711
|
+
allocationId: conn.identity.allocationId,
|
|
712
|
+
anchorRunId: conn.identity.anchorRunId,
|
|
713
|
+
generation: conn.identity.generation,
|
|
714
|
+
},
|
|
715
|
+
}
|
|
716
|
+
: {}),
|
|
717
|
+
});
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
case "signal.correlation.register":
|
|
721
|
+
return handleSignalCorrelationRegister(ws, frame);
|
|
297
722
|
case "session.ack":
|
|
298
|
-
|
|
723
|
+
pendingRequests.resolve(frame.requestId);
|
|
299
724
|
return;
|
|
300
725
|
case "session.error":
|
|
301
|
-
|
|
726
|
+
pendingRequests.reject(frame.requestId, frame.error);
|
|
302
727
|
return;
|
|
303
728
|
case "repo.pack.ack":
|
|
304
|
-
resolvePackPending(frame
|
|
729
|
+
resolvePackPending(ws, frame);
|
|
305
730
|
return;
|
|
306
731
|
case "repo.pack.reject":
|
|
307
|
-
rejectPackPending(
|
|
732
|
+
rejectPackPending(ws, frame);
|
|
308
733
|
return;
|
|
309
734
|
case "repo.pack.push":
|
|
310
735
|
handlePackPush(ws, frame);
|
|
311
736
|
return;
|
|
312
737
|
case "repo.pack.done":
|
|
313
738
|
return handlePackDone(ws, frame);
|
|
739
|
+
case "workflow.probe.result":
|
|
740
|
+
resolveProbe(ws, frame.requestId, {
|
|
741
|
+
projection: frame.projection,
|
|
742
|
+
grants: frame.grants,
|
|
743
|
+
grantWalkSnapshot: frame.grantWalkSnapshot,
|
|
744
|
+
wireHash: frame.wireHash,
|
|
745
|
+
});
|
|
746
|
+
return;
|
|
747
|
+
case "workflow.probe.error":
|
|
748
|
+
rejectProbe(ws, frame.requestId, frame.error);
|
|
749
|
+
return;
|
|
314
750
|
default:
|
|
315
751
|
return assertNever(frame);
|
|
316
752
|
}
|
|
@@ -343,466 +779,454 @@ export function createSidecarRouter(config) {
|
|
|
343
779
|
if (identity.sidecarId !== frame.sidecarId) {
|
|
344
780
|
logger.warn `Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${identity.sidecarId}; keying off the verified id`;
|
|
345
781
|
}
|
|
782
|
+
if (!(await validateSidecarIdentity(identity, "registration"))) {
|
|
783
|
+
logger.warn `Rejected ${frame.type} from sidecar ${identity.sidecarId}: credential identity is no longer current`;
|
|
784
|
+
ws.close();
|
|
785
|
+
return;
|
|
786
|
+
}
|
|
346
787
|
await run(identity);
|
|
347
788
|
}
|
|
348
|
-
async function
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
789
|
+
async function notifyAllocationWaiters(allocationId) {
|
|
790
|
+
const waiters = allocationWaiters.get(allocationId);
|
|
791
|
+
const current = allocatedConnections.get(allocationId);
|
|
792
|
+
if (waiters === undefined || current === undefined)
|
|
793
|
+
return;
|
|
794
|
+
const matchingWaiters = [...waiters].filter((waiter) => waiter.generation === current.identity.generation);
|
|
795
|
+
if (matchingWaiters.length === 0)
|
|
796
|
+
return;
|
|
797
|
+
const validation = Promise.resolve().then(() => validateSidecarIdentity(current.identity, "readiness"));
|
|
798
|
+
for (const waiter of matchingWaiters) {
|
|
799
|
+
waiter.validations.add(validation);
|
|
800
|
+
waiter.onValidation?.(validation);
|
|
801
|
+
}
|
|
802
|
+
let identityCurrent;
|
|
803
|
+
try {
|
|
804
|
+
identityCurrent = await validation;
|
|
805
|
+
}
|
|
806
|
+
catch (cause) {
|
|
807
|
+
// A failed revalidation leaves the waiters parked: a later register
|
|
808
|
+
// revalidates, and at expiry the wait reports the failure rather than a
|
|
809
|
+
// missed deadline. Registration itself was already gated, so this must
|
|
810
|
+
// not fail the connection that just registered.
|
|
811
|
+
const validationFailure = new SidecarIdentityValidationError(allocationId, current.identity.generation, cause);
|
|
812
|
+
for (const waiter of matchingWaiters) {
|
|
813
|
+
waiter.validationFailure = validationFailure;
|
|
373
814
|
}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
logger.error `Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`;
|
|
380
|
-
continue;
|
|
815
|
+
return;
|
|
816
|
+
}
|
|
817
|
+
finally {
|
|
818
|
+
for (const waiter of matchingWaiters) {
|
|
819
|
+
waiter.validations.delete(validation);
|
|
381
820
|
}
|
|
382
|
-
|
|
383
|
-
|
|
821
|
+
}
|
|
822
|
+
// A clean validation supersedes earlier failures: expiry must report the
|
|
823
|
+
// current reading, not a stale transient.
|
|
824
|
+
for (const waiter of matchingWaiters) {
|
|
825
|
+
delete waiter.validationFailure;
|
|
826
|
+
}
|
|
827
|
+
if (!identityCurrent || allocatedConnections.get(allocationId) !== current)
|
|
828
|
+
return;
|
|
829
|
+
for (const waiter of matchingWaiters) {
|
|
830
|
+
if (!waiters.delete(waiter))
|
|
384
831
|
continue;
|
|
385
|
-
|
|
386
|
-
|
|
832
|
+
clearTimeout(waiter.timer);
|
|
833
|
+
waiter.resolve();
|
|
387
834
|
}
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
const staleDeployReq = pendingDeploys.get(addr);
|
|
421
|
-
if (staleDeployReq !== undefined) {
|
|
422
|
-
clearTimeout(staleDeployReq.timer);
|
|
423
|
-
pendingDeploys.delete(addr);
|
|
424
|
-
staleDeployReq.reject(`Sidecar ${sidecarId} reconnected and claimed address "${addr}"`);
|
|
835
|
+
if (waiters.size === 0 && allocationWaiters.get(allocationId) === waiters)
|
|
836
|
+
allocationWaiters.delete(allocationId);
|
|
837
|
+
}
|
|
838
|
+
async function handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses) {
|
|
839
|
+
if (allocationFences.get(identity.allocationId) !== identity.generation) {
|
|
840
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`;
|
|
841
|
+
ws.close();
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
844
|
+
if (identity.kind === "probe" && agentAddresses.length > 0) {
|
|
845
|
+
logger.warn `Rejected probe sidecar ${identity.sidecarId}: probe ${identity.allocationId} claimed workflow addresses`;
|
|
846
|
+
ws.close();
|
|
847
|
+
return;
|
|
848
|
+
}
|
|
849
|
+
if (agentAddresses.length > 0 &&
|
|
850
|
+
!(await validateSidecarIdentity(identity, "routing"))) {
|
|
851
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`;
|
|
852
|
+
ws.close();
|
|
853
|
+
return;
|
|
854
|
+
}
|
|
855
|
+
const existingOnSocket = connections.get(ws);
|
|
856
|
+
const newlyRoutedAddresses = new Set();
|
|
857
|
+
for (const address of agentAddresses) {
|
|
858
|
+
const alreadyOwned = existingOnSocket?.identity.kind === "allocated" &&
|
|
859
|
+
existingOnSocket.identity.allocationId === identity.allocationId &&
|
|
860
|
+
addressIndex.get(address) === ws &&
|
|
861
|
+
connOwnsAddress(existingOnSocket, address);
|
|
862
|
+
if (identity.kind !== "allocated" ||
|
|
863
|
+
(!alreadyOwned && address !== identity.workflowRunAddress)) {
|
|
864
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`;
|
|
865
|
+
ws.close();
|
|
866
|
+
return;
|
|
425
867
|
}
|
|
868
|
+
if (!alreadyOwned)
|
|
869
|
+
newlyRoutedAddresses.add(address);
|
|
870
|
+
}
|
|
871
|
+
if (allocationFences.get(identity.allocationId) !== identity.generation) {
|
|
872
|
+
ws.close();
|
|
873
|
+
return;
|
|
426
874
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
875
|
+
const current = allocatedConnections.get(identity.allocationId);
|
|
876
|
+
if (current !== undefined && current.ws !== ws) {
|
|
877
|
+
// A current-generation takeover is a reconnect, not a capacity loss.
|
|
878
|
+
// Remove the old socket from the allocation index before closing it so
|
|
879
|
+
// handleClose does not emit a false allocated-disconnect event.
|
|
880
|
+
allocatedConnections.delete(identity.allocationId);
|
|
881
|
+
handleClose(current.ws);
|
|
882
|
+
current.ws.close();
|
|
883
|
+
}
|
|
884
|
+
const conn = existingOnSocket ?? {
|
|
885
|
+
sidecarId: identity.sidecarId,
|
|
886
|
+
identity,
|
|
887
|
+
agentAddresses: new Set(),
|
|
888
|
+
workflowAddresses: new Set(),
|
|
439
889
|
send(frame) {
|
|
440
890
|
ws.send(JSON.stringify(frame));
|
|
441
891
|
},
|
|
442
892
|
};
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
addressIndex.set(addr, ws);
|
|
448
|
-
// Discard any disconnect queue — register has no identity
|
|
449
|
-
// verification, so flushing to an unverified connection is unsafe.
|
|
450
|
-
// Use reconnect with challenge/response to preserve queued messages.
|
|
451
|
-
const staleQueue = disconnectedAgents.get(addr);
|
|
452
|
-
if (staleQueue !== undefined) {
|
|
453
|
-
clearTimeout(staleQueue.timer);
|
|
454
|
-
if (staleQueue.queue.length > 0) {
|
|
455
|
-
logger.warn `Discarding ${String(staleQueue.queue.length)} queued message(s) for ${addr} on unverified register`;
|
|
456
|
-
}
|
|
457
|
-
disconnectedAgents.delete(addr);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
// Only keyless first-deploy addresses reach `addressIndex` here; an
|
|
461
|
-
// address that already has a stored key was filtered out by the gate
|
|
462
|
-
// above and must re-enter routing through the challenged reconnect path.
|
|
463
|
-
// Because the gate runs before the ghost-cleanup, a rejected (keyed)
|
|
464
|
-
// address never evicts its prior owner -- register cannot reclaim or
|
|
465
|
-
// disrupt a victim's route on token auth alone.
|
|
466
|
-
logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
|
|
467
|
-
}
|
|
468
|
-
async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
|
|
469
|
-
const sidecarId = identity.sidecarId;
|
|
470
|
-
const lookupKey = lookups.lookupPublicKey;
|
|
471
|
-
if (lookupKey === undefined) {
|
|
472
|
-
logger.error `Received reconnect frame but no lookupPublicKey is configured`;
|
|
893
|
+
if (conn.identity.kind !== identity.kind ||
|
|
894
|
+
conn.identity.allocationId !== identity.allocationId ||
|
|
895
|
+
conn.identity.generation !== identity.generation) {
|
|
896
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`;
|
|
473
897
|
ws.close();
|
|
474
898
|
return;
|
|
475
899
|
}
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
//
|
|
486
|
-
//
|
|
487
|
-
//
|
|
488
|
-
//
|
|
489
|
-
//
|
|
490
|
-
const
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
// The identity was already verified at the dispatch boundary, so the
|
|
497
|
-
// internal register does not re-authenticate.
|
|
498
|
-
await handleRegister(ws, identity, []);
|
|
499
|
-
const conn = connections.get(ws);
|
|
500
|
-
if (conn === undefined)
|
|
501
|
-
return;
|
|
502
|
-
// Re-add the still-owned addresses the register cleared but this
|
|
503
|
-
// reconnect is not re-challenging. The challenged addresses below
|
|
504
|
-
// re-enter addressIndex through the verified path instead.
|
|
505
|
-
const claimedAddresses = new Set(agentAddresses);
|
|
506
|
-
for (const addr of previouslyOwned) {
|
|
507
|
-
if (claimedAddresses.has(addr))
|
|
508
|
-
continue;
|
|
509
|
-
conn.agentAddresses.add(addr);
|
|
510
|
-
addressIndex.set(addr, ws);
|
|
511
|
-
}
|
|
512
|
-
// Look up stored public keys for all claimed addresses. Fail closed on a
|
|
513
|
-
// lookup error (e.g. a transient DB failure): treat the address as
|
|
514
|
-
// unverifiable so it fails its challenge and stays unrouted, rather than
|
|
515
|
-
// letting the rejection float out of this void-dispatched handler as an
|
|
516
|
-
// unhandled rejection that could take down the hub.
|
|
517
|
-
const keyLookups = await Promise.all(agentAddresses.map(async (addr) => {
|
|
518
|
-
try {
|
|
519
|
-
return { address: addr, publicKeyHex: await lookupKey(addr) };
|
|
520
|
-
}
|
|
521
|
-
catch (err) {
|
|
522
|
-
logger.error `Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`;
|
|
523
|
-
return { address: addr, publicKeyHex: null };
|
|
524
|
-
}
|
|
525
|
-
}));
|
|
526
|
-
// If the connection was closed or superseded while we were awaiting
|
|
527
|
-
// key lookups, bail out.
|
|
528
|
-
if (!connections.has(ws))
|
|
529
|
-
return;
|
|
530
|
-
const challenges = new Map();
|
|
531
|
-
const challengeEntries = [];
|
|
532
|
-
for (const { address, publicKeyHex } of keyLookups) {
|
|
533
|
-
if (publicKeyHex === null) {
|
|
534
|
-
conn.send({
|
|
535
|
-
type: "challenge.failed",
|
|
536
|
-
address,
|
|
537
|
-
reason: "Unknown agent address",
|
|
538
|
-
});
|
|
539
|
-
continue;
|
|
540
|
-
}
|
|
541
|
-
let publicKey;
|
|
542
|
-
try {
|
|
543
|
-
publicKey = hexDecode(publicKeyHex);
|
|
900
|
+
connections.set(ws, conn);
|
|
901
|
+
for (const address of agentAddresses) {
|
|
902
|
+
conn.workflowAddresses.add(address);
|
|
903
|
+
addressIndex.set(address, ws);
|
|
904
|
+
}
|
|
905
|
+
allocatedConnections.set(identity.allocationId, { ws, identity });
|
|
906
|
+
for (const address of newlyRoutedAddresses) {
|
|
907
|
+
await redeliverPendingMail(address, conn);
|
|
908
|
+
}
|
|
909
|
+
// Reconcile a reconnecting deployment's credentials, closing the offline
|
|
910
|
+
// window: a credential revoked, deleted, or rotated while the sidecar was
|
|
911
|
+
// disconnected is applied to the child now. Fire-and-forget so
|
|
912
|
+
// registration is not blocked; the lookup no-ops for a run that persisted
|
|
913
|
+
// no credential refs.
|
|
914
|
+
const resyncCredentials = lookups.resyncCredentials;
|
|
915
|
+
if (resyncCredentials !== undefined) {
|
|
916
|
+
for (const address of newlyRoutedAddresses) {
|
|
917
|
+
if (!isRunAddress(address))
|
|
918
|
+
continue;
|
|
919
|
+
resyncCredentials(address);
|
|
544
920
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
921
|
+
}
|
|
922
|
+
// Reconcile the sidecar's cached sender keys, closing the offline window: a
|
|
923
|
+
// user-principal key that rotated while the sidecar was disconnected is
|
|
924
|
+
// re-resolved and re-pushed, and a sender whose principal was DELETED while
|
|
925
|
+
// the sidecar was disconnected is evicted, so the recipient stops verifying
|
|
926
|
+
// either against a key the hub no longer vouches for. Only allocated
|
|
927
|
+
// sidecars host a sender cache worth reconciling. Resolve and push
|
|
928
|
+
// SEQUENTIALLY in one detached task: registration is never blocked, and a
|
|
929
|
+
// large cache cannot fan out into one concurrent DB query per reported
|
|
930
|
+
// sender on every reconnect.
|
|
931
|
+
const resolveSenderKeyStrict = lookups.resolveSenderKeyStrict;
|
|
932
|
+
if (identity.kind === "allocated" && resolveSenderKeyStrict !== undefined) {
|
|
933
|
+
const rotatableSenders = new Set(cachedSenderAddresses);
|
|
934
|
+
// Resolve-don't-trust applied to input SIZE: bound the reported set before
|
|
935
|
+
// acting on it. Run addresses count toward the cap by design -- the
|
|
936
|
+
// isRunAddress skip below is inside the loop, so the iteration, and thus
|
|
937
|
+
// the DB resolves, can never exceed the cap regardless of the run/non-run
|
|
938
|
+
// mix. Over the cap, reconcile the first MAX_RESYNC_SENDER_ADDRESSES and
|
|
939
|
+
// log the overflow so a misbehaving sidecar is detectable.
|
|
940
|
+
let sendersToResync = [...rotatableSenders];
|
|
941
|
+
if (sendersToResync.length > MAX_RESYNC_SENDER_ADDRESSES) {
|
|
942
|
+
logger.warn `Sidecar ${identity.sidecarId} reported ${String(sendersToResync.length)} cached sender addresses on allocation ${identity.allocationId} generation ${String(identity.generation)}, over the ${String(MAX_RESYNC_SENDER_ADDRESSES)} resync cap; reconciling the first ${String(MAX_RESYNC_SENDER_ADDRESSES)} and ignoring the rest`;
|
|
943
|
+
sendersToResync = sendersToResync.slice(0, MAX_RESYNC_SENDER_ADDRESSES);
|
|
553
944
|
}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
945
|
+
void (async () => {
|
|
946
|
+
for (const address of sendersToResync) {
|
|
947
|
+
// The sidecar already reports only non-run senders, but do not trust
|
|
948
|
+
// the report: a run sender's key is the immutable
|
|
949
|
+
// workflow_run.public_key and is never refreshed or evicted, so skip
|
|
950
|
+
// it here too rather than couple correctness to the sidecar's filter.
|
|
951
|
+
if (isRunAddress(address))
|
|
952
|
+
continue;
|
|
953
|
+
// Tri-state, deleted-vs-fault distinguished by the STRICT resolver:
|
|
954
|
+
// - resolves to a key -> refresh the sidecar's cached key;
|
|
955
|
+
// - CONFIRMED null (no matching principal = a deleted sender) ->
|
|
956
|
+
// evict it;
|
|
957
|
+
// - THROWS (fault: ambiguous address, keyless-principal invariant
|
|
958
|
+
// break, DB error) -> keep the stale key, evict nothing.
|
|
959
|
+
// Never evicting on a fault is the load-bearing property: dropping a
|
|
960
|
+
// live key on a transient DB fault would be worse than doing nothing.
|
|
961
|
+
// Only the resolve is guarded here; conn.send stays outside so a
|
|
962
|
+
// socket-gone throw propagates to the outer catch and stops the loop.
|
|
963
|
+
let publicKey;
|
|
964
|
+
try {
|
|
965
|
+
publicKey = await resolveSenderKeyStrict(address);
|
|
966
|
+
}
|
|
967
|
+
catch (cause) {
|
|
968
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
969
|
+
logger.error `Keeping the stale cached key for ${address}: resolving it faulted (a fault, not a deleted sender): ${message}`;
|
|
970
|
+
continue;
|
|
971
|
+
}
|
|
972
|
+
if (publicKey !== null) {
|
|
973
|
+
conn.send({ type: "sender.key.refresh", address, publicKey });
|
|
974
|
+
}
|
|
975
|
+
else {
|
|
976
|
+
conn.send({ type: "sender.key.evict", address });
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
})().catch((cause) => {
|
|
980
|
+
// The per-address resolve is guarded above, so the only throw reaching
|
|
981
|
+
// here is conn.send (JSON.stringify + the socket write) once the sidecar
|
|
982
|
+
// is gone. That means the connection left, so stop -- the remaining
|
|
983
|
+
// sends would fail the same way.
|
|
984
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
985
|
+
logger.warn `Sender-key resync for sidecar ${identity.sidecarId} stopped: ${message}`;
|
|
986
|
+
});
|
|
557
987
|
}
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
pendingChallenges.delete(ws);
|
|
566
|
-
logger.warn `Challenge timed out for sidecar ${sidecarId}`;
|
|
567
|
-
}, challengeTimeoutMs);
|
|
568
|
-
pendingChallenges.set(ws, {
|
|
569
|
-
sidecarId,
|
|
570
|
-
challenges,
|
|
571
|
-
deployRefs,
|
|
572
|
-
timer,
|
|
573
|
-
});
|
|
574
|
-
conn.send({ type: "challenge", challenges: challengeEntries });
|
|
575
|
-
}
|
|
576
|
-
async function handleChallengeResponse(ws, responses) {
|
|
577
|
-
const challenge = pendingChallenges.get(ws);
|
|
578
|
-
if (challenge === undefined) {
|
|
579
|
-
logger.warn `Received challenge.response with no pending challenge`;
|
|
580
|
-
return;
|
|
988
|
+
logger.info `Provisioned sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
|
|
989
|
+
await notifyAllocationWaiters(identity.allocationId);
|
|
990
|
+
if (identity.kind === "allocated") {
|
|
991
|
+
events.emit("sidecar.allocated.connected", {
|
|
992
|
+
allocationId: identity.allocationId,
|
|
993
|
+
generation: identity.generation,
|
|
994
|
+
});
|
|
581
995
|
}
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
if (valid) {
|
|
614
|
-
verified.push(address);
|
|
615
|
-
}
|
|
616
|
-
else {
|
|
617
|
-
conn.send({
|
|
618
|
-
type: "challenge.failed",
|
|
619
|
-
address,
|
|
620
|
-
reason: "Signature verification failed",
|
|
996
|
+
}
|
|
997
|
+
async function handleRegister(ws, identity, agentAddresses, cachedSenderAddresses) {
|
|
998
|
+
await handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses);
|
|
999
|
+
}
|
|
1000
|
+
async function handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses) {
|
|
1001
|
+
await handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses);
|
|
1002
|
+
}
|
|
1003
|
+
// Park a pre-ack sender's mail synchronously and return its entry. Registering
|
|
1004
|
+
// the entry BEFORE the caller awaits `resolveSenderKey` is the interlock that
|
|
1005
|
+
// guarantees a settle landing during the resolve has an entry to find: the
|
|
1006
|
+
// event loop is single-threaded, so no settle can interleave between this
|
|
1007
|
+
// synchronous registration and the caller's first await.
|
|
1008
|
+
function parkDeferredSenderMail(authenticatedSender, rawMessage, recipients) {
|
|
1009
|
+
let parked = deferredSenderMail.get(authenticatedSender);
|
|
1010
|
+
if (parked === undefined) {
|
|
1011
|
+
parked = new Set();
|
|
1012
|
+
deferredSenderMail.set(authenticatedSender, parked);
|
|
1013
|
+
}
|
|
1014
|
+
const entry = {
|
|
1015
|
+
authenticatedSender,
|
|
1016
|
+
rawMessage,
|
|
1017
|
+
recipients,
|
|
1018
|
+
timer: setTimeout(() => {
|
|
1019
|
+
// TTL backstop for the case where a settle never arrives (the sender's
|
|
1020
|
+
// deploy never acked and never failed loudly). Claim the entry and
|
|
1021
|
+
// surface it as undelivered so the mail is not held forever.
|
|
1022
|
+
if (!claimDeferredSenderEntry(entry))
|
|
1023
|
+
return;
|
|
1024
|
+
events.emit("mail.outbound.undelivered", {
|
|
1025
|
+
rawMessage: entry.rawMessage,
|
|
1026
|
+
recipients: entry.recipients,
|
|
621
1027
|
});
|
|
622
|
-
|
|
1028
|
+
logger.warn `Dropping mail from ${entry.authenticatedSender}: its sender key was not recorded before the deferred-mail TTL expired`;
|
|
1029
|
+
}, disconnectQueueTTLMs),
|
|
1030
|
+
};
|
|
1031
|
+
parked.add(entry);
|
|
1032
|
+
return entry;
|
|
1033
|
+
}
|
|
1034
|
+
// Remove one parked entry by identity, clearing its TTL timer. Returns whether
|
|
1035
|
+
// THIS call removed it. The inline-deliver path, a settle, and the TTL all
|
|
1036
|
+
// race to claim the same entry; only the claimer acts on it, so a claim that
|
|
1037
|
+
// finds nothing (already claimed) is a no-op. This is the idempotent
|
|
1038
|
+
// remove-by-key that keeps a settle and the inline non-null branch from both
|
|
1039
|
+
// delivering the same message.
|
|
1040
|
+
function claimDeferredSenderEntry(entry) {
|
|
1041
|
+
const parked = deferredSenderMail.get(entry.authenticatedSender);
|
|
1042
|
+
if (parked === undefined)
|
|
1043
|
+
return false;
|
|
1044
|
+
const claimed = parked.delete(entry);
|
|
1045
|
+
if (!claimed)
|
|
1046
|
+
return false;
|
|
1047
|
+
clearTimeout(entry.timer);
|
|
1048
|
+
if (parked.size === 0)
|
|
1049
|
+
deferredSenderMail.delete(entry.authenticatedSender);
|
|
1050
|
+
return true;
|
|
1051
|
+
}
|
|
1052
|
+
// Claim every entry parked for a sender, clearing their TTL timers. A later
|
|
1053
|
+
// settle or TTL for the same sender then finds nothing.
|
|
1054
|
+
function claimAllDeferredSenderMail(authenticatedSender) {
|
|
1055
|
+
const parked = deferredSenderMail.get(authenticatedSender);
|
|
1056
|
+
if (parked === undefined)
|
|
1057
|
+
return [];
|
|
1058
|
+
deferredSenderMail.delete(authenticatedSender);
|
|
1059
|
+
const entries = [...parked];
|
|
1060
|
+
for (const entry of entries)
|
|
1061
|
+
clearTimeout(entry.timer);
|
|
1062
|
+
return entries;
|
|
1063
|
+
}
|
|
1064
|
+
function drainDeferredSenderMail(authenticatedSender, reason) {
|
|
1065
|
+
const entries = claimAllDeferredSenderMail(authenticatedSender);
|
|
1066
|
+
if (entries.length === 0)
|
|
1067
|
+
return;
|
|
1068
|
+
for (const entry of entries) {
|
|
1069
|
+
events.emit("mail.outbound.undelivered", {
|
|
1070
|
+
rawMessage: entry.rawMessage,
|
|
1071
|
+
recipients: entry.recipients,
|
|
1072
|
+
});
|
|
623
1073
|
}
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
address,
|
|
630
|
-
reason: "No response provided for challenged address",
|
|
631
|
-
});
|
|
632
|
-
logger.warn `Challenge failed for ${address}: no response provided`;
|
|
633
|
-
}
|
|
1074
|
+
logger.warn `Dropping ${String(entries.length)} deferred message(s) from ${authenticatedSender}: ${reason}`;
|
|
1075
|
+
}
|
|
1076
|
+
function noteSenderDeployStarted(address, attempt) {
|
|
1077
|
+
if (allocatedKeyRecordInFlight.has(address)) {
|
|
1078
|
+
throw new Error(`Sender deployment ${address} has an unresolved attempt`);
|
|
634
1079
|
}
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
const prevWs = addressIndex.get(addr);
|
|
645
|
-
if (prevWs !== undefined && prevWs !== ws) {
|
|
646
|
-
connectorStates.delete(addr);
|
|
647
|
-
// Evict the reclaimed address from the superseded connection's owned
|
|
648
|
-
// set. handleClose's cancelByAgent sweep iterates a connection's owned
|
|
649
|
-
// union WITHOUT an ownership guard, so if the stale connection still
|
|
650
|
-
// listed this address it would cancel THIS connection's in-flight pack
|
|
651
|
-
// transfer for it when it finally closes. Delete from both sets: a
|
|
652
|
-
// workflow-derived address lives on the workflow set, a launched agent
|
|
653
|
-
// on the session set, and delete is a no-op for the absent one.
|
|
654
|
-
const prevConn = connections.get(prevWs);
|
|
655
|
-
if (prevConn !== undefined) {
|
|
656
|
-
prevConn.workflowAddresses.delete(addr);
|
|
657
|
-
prevConn.agentAddresses.delete(addr);
|
|
1080
|
+
allocatedKeyRecordInFlight.set(address, attempt);
|
|
1081
|
+
}
|
|
1082
|
+
function noteSenderDeploySettled(sender, outcome) {
|
|
1083
|
+
if (typeof sender !== "string") {
|
|
1084
|
+
for (const [address, attempt] of [...allocatedKeyRecordInFlight]) {
|
|
1085
|
+
if (attempt.allocationId !== sender.allocationId ||
|
|
1086
|
+
attempt.generation !== sender.generation ||
|
|
1087
|
+
("leaseId" in sender && attempt.leaseId !== sender.leaseId)) {
|
|
1088
|
+
continue;
|
|
658
1089
|
}
|
|
1090
|
+
allocatedKeyRecordInFlight.delete(address);
|
|
1091
|
+
settleSenderMail(address, outcome);
|
|
659
1092
|
}
|
|
660
|
-
|
|
661
|
-
// handleClose reclaims it correctly: a workflow-derived deployment
|
|
662
|
-
// address goes on the workflow set (no disconnect queue -- its
|
|
663
|
-
// in-flight state is reconstructed sidecar-locally on the next
|
|
664
|
-
// reconnect), a launched agent on the session set (queued for
|
|
665
|
-
// reconnect). The routing pointer is the same either way; only now
|
|
666
|
-
// it is written behind a passed challenge.
|
|
667
|
-
if (isWorkflowDerivedAddress(addr)) {
|
|
668
|
-
conn.workflowAddresses.add(addr);
|
|
669
|
-
}
|
|
670
|
-
else {
|
|
671
|
-
conn.agentAddresses.add(addr);
|
|
672
|
-
}
|
|
673
|
-
addressIndex.set(addr, ws);
|
|
674
|
-
}
|
|
675
|
-
const ready = [];
|
|
676
|
-
const failed = [];
|
|
677
|
-
for (const addr of verified) {
|
|
678
|
-
// The `agent.reconnected` reaction is session lifecycle -- instance
|
|
679
|
-
// status flip, event-collector restore -- owned by the agent_instance
|
|
680
|
-
// concept. A workflow-derived deployment address has no agent_instance
|
|
681
|
-
// row, so the reaction's `requireInstance` would throw and roll the
|
|
682
|
-
// just-verified address back out of routing. It needs routing + queue
|
|
683
|
-
// flush only, which the passed challenge has now made safe; skip the
|
|
684
|
-
// session reaction for it.
|
|
685
|
-
if (isWorkflowDerivedAddress(addr)) {
|
|
686
|
-
ready.push(addr);
|
|
687
|
-
continue;
|
|
688
|
-
}
|
|
689
|
-
if (events.listenerCount("agent.reconnected") === 0) {
|
|
690
|
-
ready.push(addr);
|
|
691
|
-
continue;
|
|
692
|
-
}
|
|
693
|
-
try {
|
|
694
|
-
await events.emitAndAwait("agent.reconnected", { agentAddress: addr });
|
|
695
|
-
ready.push(addr);
|
|
696
|
-
}
|
|
697
|
-
catch (err) {
|
|
698
|
-
logger.error `Failed to handle reconnection for ${addr}: ${err instanceof Error ? err.message : String(err)}`;
|
|
699
|
-
failed.push(addr);
|
|
700
|
-
}
|
|
1093
|
+
return;
|
|
701
1094
|
}
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
// Roll back failed addresses from the routing table. Only the session set
|
|
711
|
-
// is touched: a workflow-derived address can never be in `failed` -- it
|
|
712
|
-
// early-`continue`s to `ready` above, before the reaction that populates
|
|
713
|
-
// `failed` -- so it is never on the workflow set at this point.
|
|
714
|
-
for (const addr of failed) {
|
|
715
|
-
conn.agentAddresses.delete(addr);
|
|
716
|
-
addressIndex.delete(addr);
|
|
717
|
-
}
|
|
718
|
-
// Flush queued messages only for ready addresses.
|
|
719
|
-
for (const addr of ready) {
|
|
720
|
-
flushDisconnectedQueue(addr, conn);
|
|
721
|
-
}
|
|
722
|
-
// Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
|
|
723
|
-
// so reconnect completion is not blocked on pack transfer. The
|
|
724
|
-
// wire layer owns the staleness comparison; the event fires only
|
|
725
|
-
// when staleness is confirmed.
|
|
726
|
-
const checkDeployRef = lookups.lookupDeployRef;
|
|
727
|
-
if (checkDeployRef !== undefined) {
|
|
728
|
-
for (const addr of ready) {
|
|
729
|
-
// Workflow deployments are pinned-forever: a deployment keeps its
|
|
730
|
-
// deploy-time definition until an explicit undeploy/redeploy, so the
|
|
731
|
-
// deploy-ref freshness catch-up is deliberately NOT run for a
|
|
732
|
-
// workflow-derived address. A definition edited on the hub while the
|
|
733
|
-
// sidecar was disconnected does not reconcile on reconnect; it affects
|
|
734
|
-
// only newly created deployments. The deployment's in-flight run state
|
|
735
|
-
// is reconstructed sidecar-locally at restore, not re-fetched here. Do
|
|
736
|
-
// NOT add a reconcile path for these addresses -- see the "Workflow
|
|
737
|
-
// Definition Versioning: Pinned-Forever" note under "Reconnect
|
|
738
|
-
// Sequencing" in docs/IMPLEMENTATION.md.
|
|
739
|
-
if (isWorkflowDerivedAddress(addr))
|
|
740
|
-
continue;
|
|
741
|
-
void (async () => {
|
|
742
|
-
try {
|
|
743
|
-
const hubRef = await checkDeployRef(addr);
|
|
744
|
-
if (hubRef === null)
|
|
745
|
-
return;
|
|
746
|
-
const sidecarRef = challenge.deployRefs[addr];
|
|
747
|
-
if (sidecarRef === hubRef)
|
|
748
|
-
return;
|
|
749
|
-
logger.info `Re-deploying ${addr}: sidecar ref ${sidecarRef ?? "(none)"} != hub ref ${hubRef.slice(0, 8)}`;
|
|
750
|
-
await events.emitAndAwait("deploy.ref.stale", {
|
|
751
|
-
agentAddress: addr,
|
|
752
|
-
});
|
|
753
|
-
}
|
|
754
|
-
catch (err) {
|
|
755
|
-
logger.error `Failed to re-deploy ${addr} after reconnect: ${err instanceof Error ? err.message : String(err)}`;
|
|
756
|
-
}
|
|
757
|
-
})();
|
|
758
|
-
}
|
|
1095
|
+
if (allocatedKeyRecordInFlight.has(sender))
|
|
1096
|
+
return;
|
|
1097
|
+
settleSenderMail(sender, outcome);
|
|
1098
|
+
}
|
|
1099
|
+
function settleSenderMail(address, outcome) {
|
|
1100
|
+
if ("failed" in outcome) {
|
|
1101
|
+
drainDeferredSenderMail(address, `sender deploy failed: ${outcome.failed}`);
|
|
1102
|
+
return;
|
|
759
1103
|
}
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
if (expired !== undefined) {
|
|
770
|
-
surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
|
|
771
|
-
}
|
|
772
|
-
}, disconnectQueueTTLMs);
|
|
773
|
-
}
|
|
774
|
-
conn.send({
|
|
775
|
-
type: "challenge.failed",
|
|
776
|
-
address: addr,
|
|
777
|
-
reason: "Reconnection rejected by governance",
|
|
1104
|
+
for (const entry of claimAllDeferredSenderMail(address)) {
|
|
1105
|
+
// Re-drive delivery as its OWN task, off the settle's stack, so delivery
|
|
1106
|
+
// work never runs on the deploy-ack handler's stack. Carry the confirmed
|
|
1107
|
+
// key: another attempt may start before this task runs, and must not
|
|
1108
|
+
// capture this mail or change the key that authenticates it.
|
|
1109
|
+
void Promise.resolve()
|
|
1110
|
+
.then(() => handleMailOutbound(entry.rawMessage, entry.authenticatedSender, entry.recipients, outcome.recorded))
|
|
1111
|
+
.catch((err) => {
|
|
1112
|
+
logger.error `Re-driving deferred mail from ${entry.authenticatedSender} failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
778
1113
|
});
|
|
779
1114
|
}
|
|
780
|
-
logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
|
|
781
1115
|
}
|
|
782
|
-
function
|
|
1116
|
+
function senderIdentitiesFromKey(address, publicKey) {
|
|
1117
|
+
return publicKey !== null ? [{ address, publicKey }] : undefined;
|
|
1118
|
+
}
|
|
1119
|
+
// Resolve the co-delivered sender identities for a message, applying the
|
|
1120
|
+
// register-before-read interlock for a pre-ack run sender. Returns either
|
|
1121
|
+
// `deliver: true` with the resolved identities (undefined when there is no
|
|
1122
|
+
// resolvable key), or `deliver: false` when the message is parked and will be
|
|
1123
|
+
// driven later by a settle (`noteSenderDeploySettled`) or the TTL.
|
|
1124
|
+
async function resolveSenderIdentitiesOrPark(rawMessage, authenticatedSender, recipients) {
|
|
1125
|
+
const resolveSenderKey = lookups.resolveSenderKey;
|
|
1126
|
+
if (resolveSenderKey === undefined)
|
|
1127
|
+
return { deliver: true, senderIdentities: undefined };
|
|
1128
|
+
// The co-delivered key is consumed only by a run recipient caching it from the
|
|
1129
|
+
// run.grants frame. Purely external/federated mail never uses it and is never
|
|
1130
|
+
// locally verified, so resolve nothing and never park it.
|
|
1131
|
+
if (!recipients.some(isRunAddress))
|
|
1132
|
+
return { deliver: true, senderIdentities: undefined };
|
|
1133
|
+
// A stable-key (non-run) sender has no pre-ack window; resolve inline.
|
|
1134
|
+
if (!isRunAddress(authenticatedSender)) {
|
|
1135
|
+
const key = await resolveSenderKey(authenticatedSender);
|
|
1136
|
+
return {
|
|
1137
|
+
deliver: true,
|
|
1138
|
+
senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
// Park a run sender ONLY while a key-record settle is guaranteed to arrive -- a
|
|
1142
|
+
// deploy is in flight. Without one, a null resolve is a transient fault or a
|
|
1143
|
+
// genuine absence on an already-settled run: no settle is coming, so parking
|
|
1144
|
+
// would strand the mail to the TTL. Deliver on the normal path instead.
|
|
1145
|
+
const settleGuaranteed = pendingDeploys.has(authenticatedSender) ||
|
|
1146
|
+
allocatedKeyRecordInFlight.has(authenticatedSender);
|
|
1147
|
+
if (!settleGuaranteed) {
|
|
1148
|
+
const key = await resolveSenderKey(authenticatedSender);
|
|
1149
|
+
return {
|
|
1150
|
+
deliver: true,
|
|
1151
|
+
senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
// Register-before-read: park a waiter entry synchronously (NO await) so a
|
|
1155
|
+
// settle that lands while we resolve below has an entry to find, THEN
|
|
1156
|
+
// resolve. The single-threaded event loop cannot interleave a settle between
|
|
1157
|
+
// this registration and the await.
|
|
1158
|
+
const entry = parkDeferredSenderMail(authenticatedSender, rawMessage, recipients);
|
|
1159
|
+
const key = await resolveSenderKey(authenticatedSender);
|
|
1160
|
+
if (key === null) {
|
|
1161
|
+
// Not recorded yet. Leave the entry parked; a settle or the TTL drives it.
|
|
1162
|
+
return { deliver: false };
|
|
1163
|
+
}
|
|
1164
|
+
// The key was already recorded before we parked. Claim our entry and deliver
|
|
1165
|
+
// inline -- unless a concurrent settle already claimed it and is re-driving
|
|
1166
|
+
// this message, in which case claiming fails and we must NOT deliver again.
|
|
1167
|
+
if (!claimDeferredSenderEntry(entry)) {
|
|
1168
|
+
return { deliver: false };
|
|
1169
|
+
}
|
|
1170
|
+
return {
|
|
1171
|
+
deliver: true,
|
|
1172
|
+
senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
|
|
1173
|
+
};
|
|
1174
|
+
}
|
|
1175
|
+
async function handleMailOutbound(rawMessage, authenticatedSender, recipients, recordedSenderKey) {
|
|
1176
|
+
// A mail addressed to more than one workflow deployment would birth a
|
|
1177
|
+
// run per recipient from a single inbound mail. The stable runId
|
|
1178
|
+
// removed the Message-ID collision that originally forced this guard --
|
|
1179
|
+
// each recipient now derives its own per-deployment runId (its mail
|
|
1180
|
+
// address), so it is no longer a runId-collision guard. It stays a
|
|
1181
|
+
// deliberate one-workflow-recipient-per-mail restriction because the
|
|
1182
|
+
// fan-out is not verified end-to-end: per-recipient grants
|
|
1183
|
+
// materialization, consumed-tracking, and reply-addressing all assume a
|
|
1184
|
+
// single workflow recipient today. Lifting it means proving those three
|
|
1185
|
+
// hold per recipient, not just relaxing this check -- so fail loudly
|
|
1186
|
+
// rather than materialize a partial set. The guard only applies when a
|
|
1187
|
+
// materializer is wired -- absent one, no run is born from the mail, so
|
|
1188
|
+
// there is nothing to restrict.
|
|
1189
|
+
if (lookups.materializeMailTriggeredRunGrants !== undefined) {
|
|
1190
|
+
// A workflow recipient is one this hub owns: its address parses as a run
|
|
1191
|
+
// address. An external/federated address does not, and is not ours to
|
|
1192
|
+
// materialize a run for.
|
|
1193
|
+
const workflowRecipients = recipients.filter(isRunAddress);
|
|
1194
|
+
if (workflowRecipients.length > 1) {
|
|
1195
|
+
throw new Error(`mail addressed to multiple workflow-derived recipients (${workflowRecipients.join(", ")}); materializing a run for more than one workflow deployment from a single mail is unsupported`);
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
// Resolve the sender's hub-held key ONCE for the whole message, ahead of the
|
|
1199
|
+
// recipient fan-out and any grant materialization, so every recipient in
|
|
1200
|
+
// this fan-out binds the same key snapshot. A run-address sender may be
|
|
1201
|
+
// pre-ack -- it minted its keypair locally and can send before the hub
|
|
1202
|
+
// records its public key. The register-before-read interlock holds such mail
|
|
1203
|
+
// until the key lands rather than delivering it keyless, which a strict
|
|
1204
|
+
// recipient drops as an unknown sender. A parked message returns here and is
|
|
1205
|
+
// re-driven later by a settle or the TTL.
|
|
1206
|
+
const resolution = recordedSenderKey === undefined
|
|
1207
|
+
? await resolveSenderIdentitiesOrPark(rawMessage, authenticatedSender, recipients)
|
|
1208
|
+
: {
|
|
1209
|
+
deliver: true,
|
|
1210
|
+
senderIdentities: senderIdentitiesFromKey(authenticatedSender, recordedSenderKey),
|
|
1211
|
+
};
|
|
1212
|
+
if (!resolution.deliver)
|
|
1213
|
+
return;
|
|
1214
|
+
const senderIdentities = resolution.senderIdentities;
|
|
783
1215
|
// Route to locally connected sidecars first, then try disconnect queues.
|
|
784
1216
|
const unrouted = [];
|
|
785
1217
|
for (const recipient of recipients) {
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
}
|
|
1218
|
+
// Each recipient is isolated: a materialization failure or a
|
|
1219
|
+
// fail-closed rejection for one must not drop the mail for its
|
|
1220
|
+
// co-recipients. The catch fails THIS recipient closed (its run never
|
|
1221
|
+
// starts under-authorized) and continues to the rest.
|
|
1222
|
+
try {
|
|
1223
|
+
const outcome = await deliverMailToRecipient(recipient, rawMessage, authenticatedSender, senderIdentities);
|
|
1224
|
+
if (outcome === "unrouted")
|
|
1225
|
+
unrouted.push(recipient);
|
|
1226
|
+
}
|
|
1227
|
+
catch (err) {
|
|
1228
|
+
logger.error `Failed to deliver mail to ${recipient}: ${err instanceof Error ? err.message : String(err)}`;
|
|
797
1229
|
}
|
|
798
|
-
const frame = {
|
|
799
|
-
type: "mail.inbound",
|
|
800
|
-
agentAddress: recipient,
|
|
801
|
-
rawMessage,
|
|
802
|
-
};
|
|
803
|
-
if (enqueueForDisconnected(recipient, frame))
|
|
804
|
-
continue;
|
|
805
|
-
unrouted.push(recipient);
|
|
806
1230
|
}
|
|
807
1231
|
// Anything not routed locally is emitted as a notification. The
|
|
808
1232
|
// host decides whether to relay onto an external transport, log,
|
|
@@ -814,6 +1238,95 @@ export function createSidecarRouter(config) {
|
|
|
814
1238
|
});
|
|
815
1239
|
}
|
|
816
1240
|
}
|
|
1241
|
+
// Deliver an inbound mail to one recipient, materializing a
|
|
1242
|
+
// mail-triggered run's grants first when the recipient is a workflow
|
|
1243
|
+
// deployment. Returns:
|
|
1244
|
+
// - `routed`: the mail reached a live connection or disconnect queue.
|
|
1245
|
+
// - `unrouted`: the mail was locally undeliverable and should be
|
|
1246
|
+
// relayed externally by the host.
|
|
1247
|
+
// - `failed-closed`: the run's grants could not be materialized safely,
|
|
1248
|
+
// so the mail is deliberately DROPPED for this recipient (not relayed)
|
|
1249
|
+
// to keep its run from starting under-authorized.
|
|
1250
|
+
//
|
|
1251
|
+
// A workflow deployment is the only recipient whose inbound mail can first
|
|
1252
|
+
// fire its stable run. Its grants are reserved, and the `run.grants` frame is
|
|
1253
|
+
// sent BEFORE the mail. Same-address FIFO guarantees it lands ahead of the
|
|
1254
|
+
// mail that dispatches the run, so the run's `onRunStart` barrier resolves its
|
|
1255
|
+
// grants rather than failing closed. Reservation happens before routing so
|
|
1256
|
+
// concurrent first deliveries cannot send different snapshots; a routing
|
|
1257
|
+
// failure leaves a grants-only, still-unfired run.
|
|
1258
|
+
async function deliverMailToRecipient(recipient, rawMessage, authenticatedSender, senderIdentities) {
|
|
1259
|
+
if (lookups.materializeMailTriggeredRunGrants !== undefined &&
|
|
1260
|
+
isRunAddress(recipient)) {
|
|
1261
|
+
const runId = deriveWorkflowRunId(recipient);
|
|
1262
|
+
// This does NOT let mail mutate a run's authorization. First delivery
|
|
1263
|
+
// reserves and commits the run's grants (the mail IS the trigger);
|
|
1264
|
+
// every later delivery only RE-READS the current committed grants
|
|
1265
|
+
// (`loadCommittedRunGrants`) and re-asserts them ahead of the dispatch.
|
|
1266
|
+
// The committed rows already carry any standing-approval change (an
|
|
1267
|
+
// approve/reject-with-`always` resolution mutates them through its own
|
|
1268
|
+
// path), so this re-send is idempotent -- it re-establishes the run's
|
|
1269
|
+
// current floor on the sidecar, self-healing a `grants.json` a sidecar
|
|
1270
|
+
// may have lost, and never overwrites it with anything staler.
|
|
1271
|
+
const result = await lookups.materializeMailTriggeredRunGrants({
|
|
1272
|
+
agentAddress: recipient,
|
|
1273
|
+
runId,
|
|
1274
|
+
});
|
|
1275
|
+
if (result.outcome === "rejected") {
|
|
1276
|
+
// The run's grants could not be materialized with sufficient
|
|
1277
|
+
// authority or it is already terminal. Fail the mail closed for this
|
|
1278
|
+
// recipient: routing or external relay would bypass that decision.
|
|
1279
|
+
logger.error `Refusing mail-triggered run ${runId} for ${recipient}: grant materialization rejected (${result.code}): ${result.message}`;
|
|
1280
|
+
return "failed-closed";
|
|
1281
|
+
}
|
|
1282
|
+
if (result.outcome === "materialized") {
|
|
1283
|
+
// The sender's hub-held key was resolved ONCE in handleMailOutbound,
|
|
1284
|
+
// ahead of this fan-out, and threaded in as `senderIdentities`. Co-
|
|
1285
|
+
// deliver it on the run's grants barrier so a recipient that caches from
|
|
1286
|
+
// the `run.grants` frame binds the sender address to the key and can
|
|
1287
|
+
// verify the sender's mail locally. A null key is never carried (the
|
|
1288
|
+
// list is undefined then), so the "authorized-with-a-key implies key
|
|
1289
|
+
// cached" invariant holds; a recipient with no cached key resolves such
|
|
1290
|
+
// mail as `unknown`, which its admission policy rejects by default (a
|
|
1291
|
+
// workflow may relax `unknown` to admit).
|
|
1292
|
+
// Finish asynchronous preparation before sending the grants and mail
|
|
1293
|
+
// together, keeping another delivery's key out of the gap between them.
|
|
1294
|
+
const messageId = await deriveMessageId(base64Decode(rawMessage));
|
|
1295
|
+
// Send the run's grants ahead of the mail. A `false` here means the
|
|
1296
|
+
// deployment is unroutable. Do not route the mail that would dispatch
|
|
1297
|
+
// it; the grants-only reservation remains the canonical snapshot for a
|
|
1298
|
+
// later first-delivery attempt.
|
|
1299
|
+
if (!sendRunGrants(recipient, runId, result.stepGrants, senderIdentities)) {
|
|
1300
|
+
logger.error `Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`;
|
|
1301
|
+
return "unrouted";
|
|
1302
|
+
}
|
|
1303
|
+
// Route through the messageId handshake `routeMail` -- NOT a
|
|
1304
|
+
// fire-and-forget send. This branch COMMITS a run, so a mail dropped in
|
|
1305
|
+
// the connected window (a socket that half-dies before the sidecar's
|
|
1306
|
+
// durable-write ack) would otherwise leave the run row "running"
|
|
1307
|
+
// forever with no body and no error. `routeMail` tracks the delivery
|
|
1308
|
+
// and redelivers identical bytes on reconnect, bringing the mail-relay
|
|
1309
|
+
// run-trigger to parity with the HTTP-trigger path. The messageId is the
|
|
1310
|
+
// mail's own id (derived over the same bytes the sidecar derives), so a
|
|
1311
|
+
// redelivery replays identically and the downstream RunStarted /
|
|
1312
|
+
// stable-runId dedup makes it effectively-once.
|
|
1313
|
+
const outcome = routeMail(recipient, rawMessage, authenticatedSender, messageId, {
|
|
1314
|
+
runId,
|
|
1315
|
+
stepGrants: result.stepGrants,
|
|
1316
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
1317
|
+
})
|
|
1318
|
+
? "routed"
|
|
1319
|
+
: "unrouted";
|
|
1320
|
+
return outcome;
|
|
1321
|
+
}
|
|
1322
|
+
// `skip`: the address named no deployed workflow deployment. Forward
|
|
1323
|
+
// the mail without grants -- the run, if any, is not ours to
|
|
1324
|
+
// authorize. No run is committed here, so no ack handshake is needed.
|
|
1325
|
+
}
|
|
1326
|
+
return routeMail(recipient, rawMessage, authenticatedSender)
|
|
1327
|
+
? "routed"
|
|
1328
|
+
: "unrouted";
|
|
1329
|
+
}
|
|
817
1330
|
async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
|
|
818
1331
|
let results;
|
|
819
1332
|
let raw;
|
|
@@ -835,15 +1348,60 @@ export function createSidecarRouter(config) {
|
|
|
835
1348
|
raw,
|
|
836
1349
|
createdAt: result.createdAt,
|
|
837
1350
|
direction: result.direction,
|
|
838
|
-
|
|
1351
|
+
runId: result.runId,
|
|
839
1352
|
address: result.address,
|
|
840
1353
|
});
|
|
841
1354
|
}
|
|
842
1355
|
}
|
|
1356
|
+
async function handleSignalCorrelationRegister(ws, frame) {
|
|
1357
|
+
// Gate the co-write on the sending sidecar actually owning the named
|
|
1358
|
+
// deployment address, mirroring the connector.state.changed and pack
|
|
1359
|
+
// handlers. A workflow deployment routes on the keyless workflow set, so
|
|
1360
|
+
// ownership is the union check, not addressIndex identity alone. Without
|
|
1361
|
+
// it a misbehaving sidecar that knows another deployment's address could
|
|
1362
|
+
// register a spurious correlation against it.
|
|
1363
|
+
const conn = connections.get(ws);
|
|
1364
|
+
if (conn === undefined)
|
|
1365
|
+
return;
|
|
1366
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
1367
|
+
logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: not registered to this sidecar`;
|
|
1368
|
+
return;
|
|
1369
|
+
}
|
|
1370
|
+
const register = lookups.registerSignalCorrelation;
|
|
1371
|
+
if (register === undefined) {
|
|
1372
|
+
logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: no registerSignalCorrelation lookup configured`;
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
try {
|
|
1376
|
+
await register({
|
|
1377
|
+
correlationId: frame.correlationId,
|
|
1378
|
+
runId: frame.runId,
|
|
1379
|
+
anchorRunId: frame.anchorRunId,
|
|
1380
|
+
agentAddress: frame.agentAddress,
|
|
1381
|
+
kind: frame.kind,
|
|
1382
|
+
approvalSnapshot: frame.snapshot,
|
|
1383
|
+
});
|
|
1384
|
+
// The co-write resolves only when a row exists -- freshly inserted or
|
|
1385
|
+
// already present (both stores are idempotent on the correlationId). Ack
|
|
1386
|
+
// so the sidecar's link stops retrying a register whose frame may have
|
|
1387
|
+
// been lost on an open socket. A thrown co-write (undeployed deployment,
|
|
1388
|
+
// id mismatch) means no row, so no ack: the sidecar keeps retrying and
|
|
1389
|
+
// the reconnect re-emit remains the ultimate backstop.
|
|
1390
|
+
conn.send({
|
|
1391
|
+
type: "signal.correlation.register.ack",
|
|
1392
|
+
agentAddress: frame.agentAddress,
|
|
1393
|
+
correlationId: frame.correlationId,
|
|
1394
|
+
});
|
|
1395
|
+
}
|
|
1396
|
+
catch (err) {
|
|
1397
|
+
logger.error `Failed to register signal correlation ${frame.correlationId} for ${frame.agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
843
1400
|
function handleClose(ws) {
|
|
844
1401
|
const conn = connections.get(ws);
|
|
845
1402
|
if (conn === undefined)
|
|
846
1403
|
return;
|
|
1404
|
+
let allocated;
|
|
847
1405
|
for (const addr of conn.agentAddresses) {
|
|
848
1406
|
// Only remove routing and pending state if this connection still
|
|
849
1407
|
// owns the address. A reconnected sidecar may have already claimed it.
|
|
@@ -855,12 +1413,12 @@ export function createSidecarRouter(config) {
|
|
|
855
1413
|
// reconnect re-bootstraps via the router's
|
|
856
1414
|
// restore-fires-callback path.
|
|
857
1415
|
connectorStates.delete(addr);
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
1416
|
+
// Retain this address's un-acked pending mail across the disconnect:
|
|
1417
|
+
// its in-flight retry timers target a dead socket (cleared), but the
|
|
1418
|
+
// entries are held so a verified reconnect redelivers them, closing the
|
|
1419
|
+
// connected-window drop rather than losing the mail. Bounded by a
|
|
1420
|
+
// retention TTL.
|
|
1421
|
+
retainPendingMailForAddress(addr);
|
|
864
1422
|
// Create a queue entry so messages can accumulate while the
|
|
865
1423
|
// sidecar is disconnected. Skip if the agent is being undeployed --
|
|
866
1424
|
// there is no point queuing messages for an agent being torn down.
|
|
@@ -885,47 +1443,52 @@ export function createSidecarRouter(config) {
|
|
|
885
1443
|
for (const addr of conn.workflowAddresses) {
|
|
886
1444
|
if (addressIndex.get(addr) === ws) {
|
|
887
1445
|
addressIndex.delete(addr);
|
|
1446
|
+
connectorStates.delete(addr);
|
|
1447
|
+
// Retain un-acked workflow trigger mail across the disconnect for the
|
|
1448
|
+
// same reason as the session loop above -- an authenticated reconnect
|
|
1449
|
+
// redelivers it. This is un-acked TRIGGER mail, distinct from the
|
|
1450
|
+
// deployment's in-flight run state (reconstructed sidecar-locally); the
|
|
1451
|
+
// "no disconnect queue" note above is about that run state, not this.
|
|
1452
|
+
retainPendingMailForAddress(addr);
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
const current = allocatedConnections.get(conn.identity.allocationId);
|
|
1456
|
+
if (current?.ws === ws) {
|
|
1457
|
+
allocatedConnections.delete(conn.identity.allocationId);
|
|
1458
|
+
if (conn.identity.kind === "allocated") {
|
|
1459
|
+
allocated = {
|
|
1460
|
+
allocationId: conn.identity.allocationId,
|
|
1461
|
+
generation: conn.identity.generation,
|
|
1462
|
+
};
|
|
888
1463
|
}
|
|
889
1464
|
}
|
|
890
1465
|
connections.delete(ws);
|
|
891
1466
|
// Cancel the liveness timer for this connection.
|
|
892
|
-
const
|
|
893
|
-
if (
|
|
894
|
-
|
|
1467
|
+
const cancelLiveness = livenessTimers.get(ws);
|
|
1468
|
+
if (cancelLiveness !== undefined) {
|
|
1469
|
+
cancelLiveness();
|
|
895
1470
|
livenessTimers.delete(ws);
|
|
896
1471
|
}
|
|
897
1472
|
// Drop the per-ws serialization chain; no more frames will queue on it.
|
|
898
1473
|
messageChains.delete(ws);
|
|
899
|
-
//
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
//
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
continue;
|
|
909
|
-
clearTimeout(req.timer);
|
|
910
|
-
pending.delete(requestId);
|
|
911
|
-
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
912
|
-
}
|
|
1474
|
+
// Reject any in-flight requests that were sent to this sidecar. Each
|
|
1475
|
+
// entry's reject closure runs its own per-site cleanup (the deploy and
|
|
1476
|
+
// undeploy closures roll routing back), exactly as a frame-error
|
|
1477
|
+
// rejection would.
|
|
1478
|
+
pendingRequests.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
1479
|
+
// Reject every deploy issued on this socket, including allocated
|
|
1480
|
+
// workflow deployments stored in `workflowAddresses` rather than
|
|
1481
|
+
// `agentAddresses`.
|
|
1482
|
+
pendingDeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
913
1483
|
// Reject any in-flight pack transfers for this sidecar.
|
|
914
|
-
|
|
915
|
-
if (pack.ws !== ws)
|
|
916
|
-
continue;
|
|
917
|
-
clearTimeout(pack.timer);
|
|
918
|
-
pendingPacks.delete(transferId);
|
|
919
|
-
pack.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
920
|
-
}
|
|
1484
|
+
pendingPacks.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
921
1485
|
// Reject any in-flight undeploys for this sidecar.
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
}
|
|
1486
|
+
pendingUndeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
1487
|
+
// Reject any in-flight probes sent to this sidecar. A probe never enters
|
|
1488
|
+
// the address maps, so this ws-keyed sweep is its ONLY disconnect cleanup:
|
|
1489
|
+
// without it a probe whose sidecar drops mid-flight would hang until its
|
|
1490
|
+
// own timeout instead of failing fast on the disconnect.
|
|
1491
|
+
pendingProbes.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
|
|
929
1492
|
// Cancel any in-flight inbound pack transfers from this sidecar
|
|
930
1493
|
// across both receivers. The two receivers track their own in-
|
|
931
1494
|
// flight transferIds, so a pending workflow-run transfer for an
|
|
@@ -944,6 +1507,7 @@ export function createSidecarRouter(config) {
|
|
|
944
1507
|
}
|
|
945
1508
|
events.emit("sidecar.disconnect", {
|
|
946
1509
|
ownedAddresses: [...owned],
|
|
1510
|
+
...(allocated !== undefined ? { allocated } : {}),
|
|
947
1511
|
});
|
|
948
1512
|
logger.info `Sidecar ${conn.sidecarId} disconnected`;
|
|
949
1513
|
}
|
|
@@ -962,71 +1526,78 @@ export function createSidecarRouter(config) {
|
|
|
962
1526
|
const requestId = nextRequestId();
|
|
963
1527
|
const frame = buildFrame(requestId);
|
|
964
1528
|
return new Promise((resolve, reject) => {
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
}, requestTimeoutMs);
|
|
969
|
-
pending.set(requestId, {
|
|
970
|
-
requestId,
|
|
971
|
-
ws,
|
|
1529
|
+
pendingRequests.register(requestId, ws, {
|
|
1530
|
+
timeoutMs: requestTimeoutMs,
|
|
1531
|
+
timeoutMessage: `Request ${requestId} timed out after ${requestTimeoutMs}ms`,
|
|
972
1532
|
resolve,
|
|
973
1533
|
reject(error) {
|
|
974
1534
|
reject(new Error(error));
|
|
975
1535
|
},
|
|
976
|
-
|
|
977
|
-
});
|
|
1536
|
+
}, undefined);
|
|
978
1537
|
conn.send(frame);
|
|
979
1538
|
});
|
|
980
1539
|
}
|
|
981
|
-
function
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
pending.delete(requestId);
|
|
987
|
-
req.resolve();
|
|
1540
|
+
function packResponseMatches(entry, ws, frame) {
|
|
1541
|
+
return (entry.ws === ws &&
|
|
1542
|
+
entry.meta.agentAddress === frame.agentAddress &&
|
|
1543
|
+
entry.meta.repoId.kind === frame.repoId.kind &&
|
|
1544
|
+
entry.meta.repoId.id === frame.repoId.id);
|
|
988
1545
|
}
|
|
989
|
-
function
|
|
990
|
-
const
|
|
991
|
-
if (req === undefined)
|
|
992
|
-
return;
|
|
993
|
-
clearTimeout(req.timer);
|
|
994
|
-
pending.delete(requestId);
|
|
995
|
-
req.reject(error);
|
|
996
|
-
}
|
|
997
|
-
function resolvePackPending(transferId) {
|
|
998
|
-
const entry = pendingPacks.get(transferId);
|
|
1546
|
+
function resolvePackPending(ws, frame) {
|
|
1547
|
+
const entry = pendingPacks.get(frame.transferId);
|
|
999
1548
|
if (entry === undefined)
|
|
1000
1549
|
return;
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1550
|
+
if (!packResponseMatches(entry, ws, frame)) {
|
|
1551
|
+
logger.warn `Ignoring repo.pack.ack for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
|
|
1552
|
+
return;
|
|
1553
|
+
}
|
|
1554
|
+
pendingPacks.resolve(frame.transferId);
|
|
1004
1555
|
}
|
|
1005
|
-
function rejectPackPending(
|
|
1006
|
-
const entry = pendingPacks.get(transferId);
|
|
1556
|
+
function rejectPackPending(ws, frame) {
|
|
1557
|
+
const entry = pendingPacks.get(frame.transferId);
|
|
1007
1558
|
if (entry === undefined)
|
|
1008
1559
|
return;
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1560
|
+
if (!packResponseMatches(entry, ws, frame)) {
|
|
1561
|
+
logger.warn `Ignoring repo.pack.reject for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
|
|
1562
|
+
return;
|
|
1563
|
+
}
|
|
1564
|
+
// Surface the receiver's specific cause when it carried one, so the awaiting
|
|
1565
|
+
// push sees "corrupt: <detail>" rather than only the coarse reason. The
|
|
1566
|
+
// "Pack rejected:" prefix is applied here rather than in the entry's
|
|
1567
|
+
// reject closure because a TIMEOUT rejection must not carry it.
|
|
1568
|
+
pendingPacks.reject(frame.transferId, `Pack rejected: ${frame.detail !== undefined
|
|
1569
|
+
? `${frame.reason}: ${frame.detail}`
|
|
1570
|
+
: frame.reason}`);
|
|
1012
1571
|
}
|
|
1013
|
-
function resolveUndeployPending(agentAddress) {
|
|
1572
|
+
function resolveUndeployPending(ws, agentAddress) {
|
|
1014
1573
|
const req = pendingUndeploys.get(agentAddress);
|
|
1015
1574
|
if (req === undefined) {
|
|
1016
1575
|
logger.warn `Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`;
|
|
1017
1576
|
return;
|
|
1018
1577
|
}
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1578
|
+
if (req.ws !== ws)
|
|
1579
|
+
return;
|
|
1580
|
+
pendingUndeploys.resolve(agentAddress);
|
|
1022
1581
|
}
|
|
1023
|
-
function rejectUndeployPending(agentAddress, error) {
|
|
1582
|
+
function rejectUndeployPending(ws, agentAddress, error) {
|
|
1024
1583
|
const req = pendingUndeploys.get(agentAddress);
|
|
1025
1584
|
if (req === undefined)
|
|
1026
1585
|
return;
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1586
|
+
if (req.ws !== ws)
|
|
1587
|
+
return;
|
|
1588
|
+
pendingUndeploys.reject(agentAddress, error);
|
|
1589
|
+
}
|
|
1590
|
+
function resolveProbe(ws, requestId, result) {
|
|
1591
|
+
const req = pendingProbes.get(requestId);
|
|
1592
|
+
if (req === undefined || req.ws !== ws)
|
|
1593
|
+
return;
|
|
1594
|
+
pendingProbes.resolve(requestId, result);
|
|
1595
|
+
}
|
|
1596
|
+
function rejectProbe(ws, requestId, error) {
|
|
1597
|
+
const req = pendingProbes.get(requestId);
|
|
1598
|
+
if (req === undefined || req.ws !== ws)
|
|
1599
|
+
return;
|
|
1600
|
+
pendingProbes.reject(requestId, error);
|
|
1030
1601
|
}
|
|
1031
1602
|
// Routing rule: pick the receiver dedicated to the repoId.kind the
|
|
1032
1603
|
// frame carries. The receivers' in-flight state is independent, so a
|
|
@@ -1048,6 +1619,8 @@ export function createSidecarRouter(config) {
|
|
|
1048
1619
|
function pickReceivePackLookup(repoId) {
|
|
1049
1620
|
switch (repoId.kind) {
|
|
1050
1621
|
case "agent-state":
|
|
1622
|
+
// The agent-state lookup ignores the `source` argument the workflow-run
|
|
1623
|
+
// lookup takes; the two are otherwise the same contract.
|
|
1051
1624
|
return lookups.receiveAgentStatePack;
|
|
1052
1625
|
case "workflow-run":
|
|
1053
1626
|
return lookups.receiveWorkflowRunPack;
|
|
@@ -1063,6 +1636,19 @@ export function createSidecarRouter(config) {
|
|
|
1063
1636
|
logger.warn `Received repo.pack.push for unrouted agent ${frame.agentAddress}`;
|
|
1064
1637
|
return;
|
|
1065
1638
|
}
|
|
1639
|
+
if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
|
|
1640
|
+
logger.warn `Rejected repo.pack.push outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
|
|
1641
|
+
conn.send({
|
|
1642
|
+
type: "repo.pack.reject",
|
|
1643
|
+
agentAddress: frame.agentAddress,
|
|
1644
|
+
repoId: frame.repoId,
|
|
1645
|
+
transferId: frame.transferId,
|
|
1646
|
+
reason: "path_violation",
|
|
1647
|
+
});
|
|
1648
|
+
return;
|
|
1649
|
+
}
|
|
1650
|
+
if (conn.identity.kind !== "allocated")
|
|
1651
|
+
return;
|
|
1066
1652
|
const picked = pickPackReceiver(frame.repoId);
|
|
1067
1653
|
if (picked === null) {
|
|
1068
1654
|
logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
@@ -1094,6 +1680,20 @@ export function createSidecarRouter(config) {
|
|
|
1094
1680
|
logger.warn `Received repo.pack.done for unrouted agent ${frame.agentAddress}`;
|
|
1095
1681
|
return;
|
|
1096
1682
|
}
|
|
1683
|
+
if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
|
|
1684
|
+
logger.warn `Rejected repo.pack.done outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
|
|
1685
|
+
conn.send({
|
|
1686
|
+
type: "repo.pack.reject",
|
|
1687
|
+
agentAddress: frame.agentAddress,
|
|
1688
|
+
repoId: frame.repoId,
|
|
1689
|
+
transferId: frame.transferId,
|
|
1690
|
+
reason: "path_violation",
|
|
1691
|
+
});
|
|
1692
|
+
return;
|
|
1693
|
+
}
|
|
1694
|
+
if (conn.identity.kind !== "allocated")
|
|
1695
|
+
return;
|
|
1696
|
+
const identity = conn.identity;
|
|
1097
1697
|
const picked = pickPackReceiver(frame.repoId);
|
|
1098
1698
|
if (picked === null) {
|
|
1099
1699
|
logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
@@ -1127,7 +1727,13 @@ export function createSidecarRouter(config) {
|
|
|
1127
1727
|
});
|
|
1128
1728
|
return;
|
|
1129
1729
|
}
|
|
1130
|
-
const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha
|
|
1730
|
+
const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, {
|
|
1731
|
+
kind: "allocated",
|
|
1732
|
+
agentAddress: frame.agentAddress,
|
|
1733
|
+
allocationId: identity.allocationId,
|
|
1734
|
+
anchorRunId: identity.anchorRunId,
|
|
1735
|
+
generation: identity.generation,
|
|
1736
|
+
});
|
|
1131
1737
|
// Connection may have closed during async verification.
|
|
1132
1738
|
const currentConn = connections.get(ws);
|
|
1133
1739
|
if (currentConn === undefined)
|
|
@@ -1155,53 +1761,220 @@ export function createSidecarRouter(config) {
|
|
|
1155
1761
|
* window of a multi-step deploy, so `sendPack` can route the step's deploy
|
|
1156
1762
|
* and asset packs before the deployment-level frame spawns the child.
|
|
1157
1763
|
*
|
|
1158
|
-
* The address is
|
|
1159
|
-
*
|
|
1160
|
-
*
|
|
1764
|
+
* The address is Hub-minted and workflow-derived, so it enters the
|
|
1765
|
+
* `workflowAddresses` set rather than the legacy `agentAddresses` set and is
|
|
1766
|
+
* torn down by `unbindStepRoute` once the
|
|
1161
1767
|
* step's packs land. `handleClose` reclaims it if the sidecar drops
|
|
1162
1768
|
* mid-stage. Per-step addresses are not runtime-routed (mail, signals, and
|
|
1163
1769
|
* drains use the deployment address), so the binding is transient: it is
|
|
1164
1770
|
* never persisted into the reconnect set and never resurrected on
|
|
1165
1771
|
* reconnect.
|
|
1166
1772
|
*/
|
|
1167
|
-
function
|
|
1168
|
-
const
|
|
1169
|
-
if (
|
|
1170
|
-
throw new Error(`
|
|
1773
|
+
function fenceAllocation(allocationId, generation) {
|
|
1774
|
+
const existing = allocationFences.get(allocationId);
|
|
1775
|
+
if (existing !== undefined && generation < existing) {
|
|
1776
|
+
throw new Error(`Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`);
|
|
1777
|
+
}
|
|
1778
|
+
allocationFences.set(allocationId, generation);
|
|
1779
|
+
// A durable generation advance resolves unfinished initialization as failed.
|
|
1780
|
+
// This also covers a cleanup transaction whose response was lost: the next
|
|
1781
|
+
// reconciliation rebuilds this fence before it can start a replacement.
|
|
1782
|
+
for (const attempt of [...allocatedKeyRecordInFlight.values()]) {
|
|
1783
|
+
if (attempt.allocationId === allocationId &&
|
|
1784
|
+
attempt.generation < generation) {
|
|
1785
|
+
noteSenderDeploySettled(attempt, {
|
|
1786
|
+
failed: `Allocation ${allocationId} advanced beyond the deployment attempt`,
|
|
1787
|
+
});
|
|
1788
|
+
}
|
|
1171
1789
|
}
|
|
1172
|
-
const
|
|
1173
|
-
if (
|
|
1174
|
-
|
|
1790
|
+
const current = allocatedConnections.get(allocationId);
|
|
1791
|
+
if (current !== undefined && current.identity.generation !== generation) {
|
|
1792
|
+
handleClose(current.ws);
|
|
1793
|
+
current.ws.close();
|
|
1794
|
+
}
|
|
1795
|
+
const waiters = allocationWaiters.get(allocationId);
|
|
1796
|
+
if (waiters === undefined)
|
|
1797
|
+
return;
|
|
1798
|
+
for (const waiter of [...waiters]) {
|
|
1799
|
+
if (waiter.generation === generation)
|
|
1800
|
+
continue;
|
|
1801
|
+
clearTimeout(waiter.timer);
|
|
1802
|
+
waiters.delete(waiter);
|
|
1803
|
+
waiter.reject(new Error(`Allocation ${allocationId} advanced to generation ${String(generation)}`));
|
|
1804
|
+
}
|
|
1805
|
+
if (waiters.size === 0)
|
|
1806
|
+
allocationWaiters.delete(allocationId);
|
|
1807
|
+
}
|
|
1808
|
+
function retireAllocation(target) {
|
|
1809
|
+
if (allocationFences.get(target.allocationId) !== target.generation)
|
|
1810
|
+
return;
|
|
1811
|
+
disconnectAllocation(target);
|
|
1812
|
+
allocationFences.delete(target.allocationId);
|
|
1813
|
+
// The fence is gone, so a lingering attempt can never settle normally.
|
|
1814
|
+
// Fail it here rather than leaving a marker that blocks the address.
|
|
1815
|
+
for (const attempt of [...allocatedKeyRecordInFlight.values()]) {
|
|
1816
|
+
if (attempt.allocationId === target.allocationId &&
|
|
1817
|
+
attempt.generation <= target.generation) {
|
|
1818
|
+
noteSenderDeploySettled(attempt, {
|
|
1819
|
+
failed: `Allocation ${target.allocationId} generation ${String(target.generation)} retired`,
|
|
1820
|
+
});
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
const waiters = allocationWaiters.get(target.allocationId);
|
|
1824
|
+
if (waiters === undefined)
|
|
1825
|
+
return;
|
|
1826
|
+
allocationWaiters.delete(target.allocationId);
|
|
1827
|
+
for (const waiter of waiters) {
|
|
1828
|
+
clearTimeout(waiter.timer);
|
|
1829
|
+
waiter.reject(new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} retired`));
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
async function getProvisionedConnection(target, use) {
|
|
1833
|
+
if (allocationFences.get(target.allocationId) !== target.generation) {
|
|
1834
|
+
throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
|
|
1835
|
+
}
|
|
1836
|
+
const current = allocatedConnections.get(target.allocationId);
|
|
1837
|
+
if (current === undefined ||
|
|
1838
|
+
current.identity.generation !== target.generation) {
|
|
1839
|
+
throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`);
|
|
1840
|
+
}
|
|
1841
|
+
let identityCurrent;
|
|
1842
|
+
try {
|
|
1843
|
+
identityCurrent = await validateSidecarIdentity(current.identity, use);
|
|
1844
|
+
}
|
|
1845
|
+
catch (cause) {
|
|
1846
|
+
throw new SidecarIdentityValidationError(target.allocationId, target.generation, cause);
|
|
1847
|
+
}
|
|
1848
|
+
if (!identityCurrent) {
|
|
1849
|
+
if (allocatedConnections.get(target.allocationId) === current) {
|
|
1850
|
+
handleClose(current.ws);
|
|
1851
|
+
current.ws.close();
|
|
1852
|
+
}
|
|
1853
|
+
throw new Error(`Allocated sidecar identity is no longer current for allocation ${target.allocationId}`);
|
|
1854
|
+
}
|
|
1855
|
+
if (allocatedConnections.get(target.allocationId) !== current) {
|
|
1856
|
+
throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
|
|
1857
|
+
}
|
|
1858
|
+
const conn = connections.get(current.ws);
|
|
1859
|
+
if (conn === undefined ||
|
|
1860
|
+
conn.identity.allocationId !== target.allocationId ||
|
|
1861
|
+
conn.identity.generation !== target.generation) {
|
|
1862
|
+
throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId}`);
|
|
1863
|
+
}
|
|
1864
|
+
return { ws: current.ws, conn };
|
|
1865
|
+
}
|
|
1866
|
+
async function getAllocatedConnection(target, use) {
|
|
1867
|
+
const current = await getProvisionedConnection(target, use);
|
|
1868
|
+
if (current.conn.identity.kind !== "allocated") {
|
|
1869
|
+
throw new Error(`Allocation ${target.allocationId} is connected as probe capacity`);
|
|
1870
|
+
}
|
|
1871
|
+
return {
|
|
1872
|
+
ws: current.ws,
|
|
1873
|
+
conn: { ...current.conn, identity: current.conn.identity },
|
|
1874
|
+
};
|
|
1875
|
+
}
|
|
1876
|
+
async function isAllocatedSidecarReady(target) {
|
|
1877
|
+
try {
|
|
1878
|
+
await getProvisionedConnection(target, "readiness");
|
|
1879
|
+
return true;
|
|
1880
|
+
}
|
|
1881
|
+
catch (error) {
|
|
1882
|
+
// A failed validation is unknown, not absent: the worker may be healthy
|
|
1883
|
+
// behind a failed lookup, so report it distinctly instead of answering
|
|
1884
|
+
// `false` and letting the caller release a live worker.
|
|
1885
|
+
if (error instanceof SidecarIdentityValidationError)
|
|
1886
|
+
throw error;
|
|
1887
|
+
return false;
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
async function isAllocatedWorkflowActive(target) {
|
|
1891
|
+
try {
|
|
1892
|
+
const { conn } = await getAllocatedConnection(target, "readiness");
|
|
1893
|
+
if (conn.identity.kind !== "allocated")
|
|
1894
|
+
return false;
|
|
1895
|
+
return conn.workflowAddresses.has(conn.identity.workflowRunAddress);
|
|
1896
|
+
}
|
|
1897
|
+
catch (error) {
|
|
1898
|
+
if (error instanceof SidecarIdentityValidationError)
|
|
1899
|
+
throw error;
|
|
1900
|
+
return false;
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
async function waitForAllocatedSidecar(target, timeoutMs, onValidation) {
|
|
1904
|
+
// An indeterminable worker waits out the unknown while time remains: only
|
|
1905
|
+
// confirmed absence may surface as a connection timeout. At expiry the
|
|
1906
|
+
// wait reports the validation failure rather than a missed deadline, so
|
|
1907
|
+
// the caller retries instead of releasing a worker that may be healthy.
|
|
1908
|
+
let validationFailure;
|
|
1909
|
+
try {
|
|
1910
|
+
if (await isAllocatedSidecarReady(target))
|
|
1911
|
+
return;
|
|
1912
|
+
}
|
|
1913
|
+
catch (error) {
|
|
1914
|
+
if (!(error instanceof SidecarIdentityValidationError))
|
|
1915
|
+
throw error;
|
|
1916
|
+
validationFailure = error;
|
|
1917
|
+
}
|
|
1918
|
+
if (allocationFences.get(target.allocationId) !== target.generation) {
|
|
1919
|
+
throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
|
|
1920
|
+
}
|
|
1921
|
+
if (timeoutMs <= 0) {
|
|
1922
|
+
if (validationFailure !== undefined)
|
|
1923
|
+
throw validationFailure;
|
|
1924
|
+
throw new Error(`Timed out waiting for allocated sidecar ${target.allocationId}`);
|
|
1925
|
+
}
|
|
1926
|
+
await new Promise((resolve, reject) => {
|
|
1927
|
+
const waiter = {
|
|
1928
|
+
generation: target.generation,
|
|
1929
|
+
validations: new Set(),
|
|
1930
|
+
...(onValidation !== undefined ? { onValidation } : {}),
|
|
1931
|
+
resolve,
|
|
1932
|
+
reject,
|
|
1933
|
+
timer: setTimeout(() => {
|
|
1934
|
+
const current = allocationWaiters.get(target.allocationId);
|
|
1935
|
+
current?.delete(waiter);
|
|
1936
|
+
if (current?.size === 0) {
|
|
1937
|
+
allocationWaiters.delete(target.allocationId);
|
|
1938
|
+
}
|
|
1939
|
+
reject(waiter.validationFailure ??
|
|
1940
|
+
(waiter.validations.size > 0
|
|
1941
|
+
? new SidecarIdentityValidationError(target.allocationId, target.generation)
|
|
1942
|
+
: new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`)));
|
|
1943
|
+
}, timeoutMs),
|
|
1944
|
+
...(validationFailure !== undefined ? { validationFailure } : {}),
|
|
1945
|
+
};
|
|
1946
|
+
let waiters = allocationWaiters.get(target.allocationId);
|
|
1947
|
+
if (waiters === undefined) {
|
|
1948
|
+
waiters = new Set();
|
|
1949
|
+
allocationWaiters.set(target.allocationId, waiters);
|
|
1950
|
+
}
|
|
1951
|
+
waiters.add(waiter);
|
|
1952
|
+
void notifyAllocationWaiters(target.allocationId);
|
|
1953
|
+
});
|
|
1954
|
+
}
|
|
1955
|
+
async function bindAllocatedStepRoute(target, stepAddress) {
|
|
1956
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
1957
|
+
const existing = addressIndex.get(stepAddress);
|
|
1958
|
+
if (existing !== undefined && existing !== ws) {
|
|
1959
|
+
throw new Error(`Workflow step ${stepAddress} is already routed to another sidecar`);
|
|
1175
1960
|
}
|
|
1176
1961
|
conn.workflowAddresses.add(stepAddress);
|
|
1177
1962
|
addressIndex.set(stepAddress, ws);
|
|
1178
1963
|
}
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
*/
|
|
1184
|
-
function unbindStepRoute(stepAddress) {
|
|
1185
|
-
const ws = addressIndex.get(stepAddress);
|
|
1186
|
-
if (ws === undefined)
|
|
1964
|
+
function unbindAllocatedStepRoute(target, stepAddress) {
|
|
1965
|
+
const current = allocatedConnections.get(target.allocationId);
|
|
1966
|
+
if (current === undefined ||
|
|
1967
|
+
current.identity.generation !== target.generation) {
|
|
1187
1968
|
return;
|
|
1188
|
-
const conn = connections.get(ws);
|
|
1189
|
-
if (conn !== undefined) {
|
|
1190
|
-
conn.workflowAddresses.delete(stepAddress);
|
|
1191
1969
|
}
|
|
1970
|
+
if (addressIndex.get(stepAddress) !== current.ws)
|
|
1971
|
+
return;
|
|
1972
|
+
connections.get(current.ws)?.workflowAddresses.delete(stepAddress);
|
|
1192
1973
|
addressIndex.delete(stepAddress);
|
|
1193
1974
|
}
|
|
1194
1975
|
// Pack transfers may take longer than session requests due to data volume.
|
|
1195
1976
|
const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
|
|
1196
|
-
function
|
|
1197
|
-
const ws = addressIndex.get(agentAddress);
|
|
1198
|
-
if (ws === undefined) {
|
|
1199
|
-
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1200
|
-
}
|
|
1201
|
-
const conn = connections.get(ws);
|
|
1202
|
-
if (conn === undefined) {
|
|
1203
|
-
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1204
|
-
}
|
|
1977
|
+
function sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options) {
|
|
1205
1978
|
const transferId = `pack-${++packCounter}`;
|
|
1206
1979
|
// For the agent-state flow the destination agent and the source repo
|
|
1207
1980
|
// are the same entity, so `repoId.id === agentAddress`. Asset packs
|
|
@@ -1215,19 +1988,14 @@ export function createSidecarRouter(config) {
|
|
|
1215
1988
|
// Register pending entry before sending frames so that a synchronous
|
|
1216
1989
|
// repo.pack.ack (e.g. in tests or loopback transports) resolves correctly.
|
|
1217
1990
|
return new Promise((resolve, reject) => {
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
}, PACK_TIMEOUT_MS);
|
|
1222
|
-
pendingPacks.set(transferId, {
|
|
1223
|
-
transferId,
|
|
1224
|
-
ws,
|
|
1991
|
+
pendingPacks.register(transferId, ws, {
|
|
1992
|
+
timeoutMs: PACK_TIMEOUT_MS,
|
|
1993
|
+
timeoutMessage: `Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`,
|
|
1225
1994
|
resolve,
|
|
1226
1995
|
reject(error) {
|
|
1227
|
-
reject(new Error(
|
|
1996
|
+
reject(new Error(error));
|
|
1228
1997
|
},
|
|
1229
|
-
|
|
1230
|
-
});
|
|
1998
|
+
}, { agentAddress, repoId });
|
|
1231
1999
|
// Send chunks
|
|
1232
2000
|
for (const chunk of chunkPack(pack)) {
|
|
1233
2001
|
conn.send({
|
|
@@ -1251,128 +2019,291 @@ export function createSidecarRouter(config) {
|
|
|
1251
2019
|
});
|
|
1252
2020
|
});
|
|
1253
2021
|
}
|
|
1254
|
-
function
|
|
2022
|
+
async function sendPackToAllocation(target, agentAddress, pack, ref, commitSha, options) {
|
|
2023
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2024
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
2025
|
+
throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2026
|
+
}
|
|
2027
|
+
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
|
|
2028
|
+
}
|
|
2029
|
+
async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha, signal) {
|
|
2030
|
+
signal?.throwIfAborted();
|
|
2031
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2032
|
+
signal?.throwIfAborted();
|
|
2033
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
2034
|
+
throw new Error(`Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`);
|
|
2035
|
+
}
|
|
2036
|
+
if (conn.workflowAddresses.has(agentAddress)) {
|
|
2037
|
+
throw new Error(`Allocation ${target.allocationId} already hosts active workflow ${agentAddress}; refusing to overwrite its run history`);
|
|
2038
|
+
}
|
|
2039
|
+
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, {
|
|
2040
|
+
repoId: {
|
|
2041
|
+
kind: "workflow-run",
|
|
2042
|
+
id: deriveWorkflowRunRepoId(agentAddress),
|
|
2043
|
+
},
|
|
2044
|
+
});
|
|
2045
|
+
}
|
|
2046
|
+
function routeMail(agentAddress, rawMessage, authenticatedSender, messageId, runGrants) {
|
|
2047
|
+
// `authenticatedSender` is hub-assigned by the caller from a hub-verified
|
|
2048
|
+
// value (the ownership-gated sender of a relayed mail, or the triggering
|
|
2049
|
+
// principal's address) -- never the message's own MIME `From`. It rides
|
|
2050
|
+
// the frame as the hub-verified sender of record, so a recipient can take
|
|
2051
|
+
// the sender from it rather than the forgeable `From`. The recipient's
|
|
2052
|
+
// signature check reads it as the sender of record -- resolving the
|
|
2053
|
+
// sender's key from its local cache to verify the signature -- and its
|
|
2054
|
+
// admission policy gates delivery on the verdict.
|
|
2055
|
+
//
|
|
2056
|
+
// Carry the hub-minted messageId on the frame so the sidecar's durable-
|
|
2057
|
+
// receipt ack (`mail.inbound.ack`) keys on the same id the hub tracks, and
|
|
2058
|
+
// a redelivery replays identical bytes for the downstream RunStarted dedup.
|
|
2059
|
+
// Optional: the workflow-trigger and session-conversation callers supply
|
|
2060
|
+
// it (they participate in the ack/retry handshake); a caller without a
|
|
2061
|
+
// hub-minted id omits it and the delivery is not tracked for redelivery.
|
|
2062
|
+
const frame = {
|
|
2063
|
+
type: "mail.inbound",
|
|
2064
|
+
agentAddress,
|
|
2065
|
+
rawMessage,
|
|
2066
|
+
authenticatedSender,
|
|
2067
|
+
...(messageId !== undefined ? { messageId } : {}),
|
|
2068
|
+
};
|
|
1255
2069
|
const ws = addressIndex.get(agentAddress);
|
|
1256
2070
|
if (ws !== undefined) {
|
|
1257
2071
|
const conn = connections.get(ws);
|
|
1258
2072
|
if (conn !== undefined) {
|
|
1259
|
-
conn.send(
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
2073
|
+
conn.send(frame);
|
|
2074
|
+
// Track the delivery for redelivery until the sidecar acks its durable
|
|
2075
|
+
// inbox write. Only mail carrying a hub-minted messageId participates
|
|
2076
|
+
// in the ack handshake; relayed agent-to-agent mail omits it and is
|
|
2077
|
+
// delivered fire-and-forget as before. A mail that triggered a workflow
|
|
2078
|
+
// run carries the run's grants so redelivery can replay them ahead of
|
|
2079
|
+
// the mail.
|
|
2080
|
+
if (messageId !== undefined) {
|
|
2081
|
+
trackPendingMail(agentAddress, messageId, frame, runGrants);
|
|
2082
|
+
}
|
|
1264
2083
|
return true;
|
|
1265
2084
|
}
|
|
1266
2085
|
}
|
|
1267
2086
|
// If the agent recently disconnected, queue for delivery on reconnect.
|
|
1268
|
-
const frame = { type: "mail.inbound", agentAddress, rawMessage };
|
|
1269
2087
|
return enqueueForDisconnected(agentAddress, frame);
|
|
1270
2088
|
}
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
2089
|
+
function sendRunGrants(agentAddress, runId, stepGrants, senderIdentities) {
|
|
2090
|
+
const frame = {
|
|
2091
|
+
type: "run.grants",
|
|
2092
|
+
agentAddress,
|
|
2093
|
+
runId,
|
|
2094
|
+
stepGrants,
|
|
2095
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
2096
|
+
};
|
|
2097
|
+
const ws = addressIndex.get(agentAddress);
|
|
2098
|
+
if (ws !== undefined) {
|
|
2099
|
+
const conn = connections.get(ws);
|
|
2100
|
+
if (conn !== undefined) {
|
|
2101
|
+
conn.send(frame);
|
|
2102
|
+
return true;
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
// Mirror routeMail: if the address has a live disconnect queue, ride it
|
|
2106
|
+
// so a run.grants issued in the window between deploy and the first
|
|
2107
|
+
// reconnect survives the same way the dispatching trigger mail does.
|
|
2108
|
+
// A queue exists only while the deployment address is still on
|
|
2109
|
+
// agentAddresses (pre-first-reconnect); after an authenticated reconnect it
|
|
2110
|
+
// moves to workflowAddresses, which handleClose leaves unqueued because
|
|
2111
|
+
// that generation's in-flight run state is reconstructed sidecar-locally.
|
|
2112
|
+
// Returning without enqueueing there is correct; enqueueing is what keeps
|
|
2113
|
+
// grants and mail from diverging in the pre-reconnect window.
|
|
2114
|
+
return enqueueForDisconnected(agentAddress, frame);
|
|
2115
|
+
}
|
|
2116
|
+
async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, authenticatedSender, messageId, signal) {
|
|
2117
|
+
signal?.throwIfAborted();
|
|
2118
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2119
|
+
signal?.throwIfAborted();
|
|
2120
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
2121
|
+
throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2122
|
+
}
|
|
2123
|
+
// authenticatedSender is the sender persisted at enqueue on the dispatch
|
|
2124
|
+
// row (the triggering principal's hub-verified address); the caller reads
|
|
2125
|
+
// it from that row. It is never the message's MIME From.
|
|
2126
|
+
//
|
|
2127
|
+
// Resolve its key here, at dispatch (redelivery) time, from that persisted
|
|
2128
|
+
// address, to co-deliver on the run's grants barrier so the recipient
|
|
2129
|
+
// caches the sender's current hub-held key. A run sender's deployment key
|
|
2130
|
+
// is immutable once acked; a user sender's key rotating mid-flight would
|
|
2131
|
+
// leave the fixed signed bytes checked against the new key, which the
|
|
2132
|
+
// recipient logs as unverifiable. Null when unresolvable (no resolver
|
|
2133
|
+
// wired, or the sender has no durable key). The lookup contract (see
|
|
2134
|
+
// SidecarLookups.resolveSenderKey) is best-effort and never throws, so
|
|
2135
|
+
// resolving ahead of the run.grants send cannot block it.
|
|
2136
|
+
const authenticatedSenderPublicKey = lookups.resolveSenderKey !== undefined
|
|
2137
|
+
? await lookups.resolveSenderKey(authenticatedSender)
|
|
2138
|
+
: null;
|
|
2139
|
+
signal?.throwIfAborted();
|
|
2140
|
+
// Co-deliver the resolved key on the run's grants barrier, omitting a null
|
|
2141
|
+
// key so it is never cached (see deliverMailToRecipient). The same list
|
|
2142
|
+
// rides the pending-mail entry so the reconnect replay carries it too.
|
|
2143
|
+
const senderIdentities = authenticatedSenderPublicKey !== null
|
|
2144
|
+
? [
|
|
2145
|
+
{
|
|
2146
|
+
address: authenticatedSender,
|
|
2147
|
+
publicKey: authenticatedSenderPublicKey,
|
|
2148
|
+
},
|
|
2149
|
+
]
|
|
2150
|
+
: undefined;
|
|
2151
|
+
const runGrants = {
|
|
2152
|
+
runId,
|
|
2153
|
+
stepGrants,
|
|
2154
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
2155
|
+
};
|
|
2156
|
+
conn.send({
|
|
2157
|
+
type: "run.grants",
|
|
2158
|
+
agentAddress,
|
|
2159
|
+
runId,
|
|
2160
|
+
stepGrants,
|
|
2161
|
+
...(senderIdentities !== undefined ? { senderIdentities } : {}),
|
|
2162
|
+
});
|
|
2163
|
+
const frame = {
|
|
2164
|
+
type: "mail.inbound",
|
|
2165
|
+
agentAddress,
|
|
2166
|
+
rawMessage,
|
|
2167
|
+
authenticatedSender,
|
|
2168
|
+
messageId,
|
|
2169
|
+
};
|
|
2170
|
+
conn.send(frame);
|
|
2171
|
+
trackPendingMail(agentAddress, messageId, frame, runGrants, target);
|
|
2172
|
+
}
|
|
2173
|
+
async function handleDeployAck(ws, frame) {
|
|
2174
|
+
const req = pendingDeploys.get(frame.agentAddress);
|
|
2175
|
+
if (req === undefined) {
|
|
2176
|
+
logger.warn `Received agent.deploy.ack for "${frame.agentAddress}" with no pending deploy`;
|
|
1274
2177
|
return;
|
|
1275
2178
|
}
|
|
2179
|
+
if (req.ws !== ws)
|
|
2180
|
+
return;
|
|
1276
2181
|
if (events.listenerCount("agent.deploy.ack") > 0) {
|
|
1277
2182
|
try {
|
|
2183
|
+
const identity = connections.get(ws)?.identity;
|
|
1278
2184
|
await events.emitAndAwait("agent.deploy.ack", {
|
|
1279
|
-
agentAddress,
|
|
1280
|
-
publicKey,
|
|
2185
|
+
agentAddress: frame.agentAddress,
|
|
2186
|
+
publicKey: frame.publicKey,
|
|
2187
|
+
...(identity?.kind === "allocated"
|
|
2188
|
+
? {
|
|
2189
|
+
allocated: {
|
|
2190
|
+
allocationId: identity.allocationId,
|
|
2191
|
+
anchorRunId: identity.anchorRunId,
|
|
2192
|
+
generation: identity.generation,
|
|
2193
|
+
},
|
|
2194
|
+
}
|
|
2195
|
+
: {}),
|
|
1281
2196
|
});
|
|
1282
2197
|
}
|
|
1283
2198
|
catch (err) {
|
|
1284
|
-
|
|
2199
|
+
pendingDeploys.reject(frame.agentAddress, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
|
|
1285
2200
|
return;
|
|
1286
2201
|
}
|
|
1287
2202
|
}
|
|
1288
|
-
|
|
1289
|
-
}
|
|
1290
|
-
function resolveDeployPending(agentAddress) {
|
|
1291
|
-
const req = pendingDeploys.get(agentAddress);
|
|
1292
|
-
if (req === undefined)
|
|
1293
|
-
return;
|
|
1294
|
-
clearTimeout(req.timer);
|
|
1295
|
-
pendingDeploys.delete(agentAddress);
|
|
1296
|
-
req.resolve();
|
|
2203
|
+
pendingDeploys.resolve(frame.agentAddress, frame.publicKey);
|
|
1297
2204
|
}
|
|
1298
|
-
function
|
|
2205
|
+
function rejectDeployPendingFromFrame(ws, agentAddress, error) {
|
|
1299
2206
|
const req = pendingDeploys.get(agentAddress);
|
|
1300
|
-
if (req === undefined)
|
|
2207
|
+
if (req === undefined || req.ws !== ws)
|
|
1301
2208
|
return;
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
2209
|
+
// Settle by key, not by the `req` object: a key lookup observes the
|
|
2210
|
+
// CURRENT entry, so a stale handle cannot settle a replaced round-trip.
|
|
2211
|
+
pendingDeploys.reject(agentAddress, error);
|
|
1305
2212
|
}
|
|
1306
|
-
|
|
2213
|
+
function sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow) {
|
|
1307
2214
|
if (hubPublicKeyHex === undefined) {
|
|
1308
|
-
throw
|
|
2215
|
+
throw deployFrameFailure("Hub signing key is required for agent deployment", false);
|
|
1309
2216
|
}
|
|
1310
2217
|
if (pendingDeploys.has(agentAddress)) {
|
|
1311
|
-
throw
|
|
1312
|
-
}
|
|
1313
|
-
const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
|
|
1314
|
-
if (ws === undefined) {
|
|
1315
|
-
throw new Error(`No sidecar available for agent "${agentAddress}"`);
|
|
1316
|
-
}
|
|
1317
|
-
const conn = connections.get(ws);
|
|
1318
|
-
if (conn === undefined) {
|
|
1319
|
-
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
2218
|
+
throw deployFrameFailure(`Deploy already in progress for agent "${agentAddress}"`, false);
|
|
1320
2219
|
}
|
|
1321
|
-
conn.
|
|
2220
|
+
const addressSet = conn.identity.kind === "allocated"
|
|
2221
|
+
? conn.workflowAddresses
|
|
2222
|
+
: conn.agentAddresses;
|
|
2223
|
+
addressSet.add(agentAddress);
|
|
1322
2224
|
addressIndex.set(agentAddress, ws);
|
|
1323
2225
|
return new Promise((resolve, reject) => {
|
|
1324
|
-
//
|
|
1325
|
-
//
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
const detachListener = events.on("agent.deploy.ack", (payload) => {
|
|
1332
|
-
if (payload.agentAddress === agentAddress) {
|
|
1333
|
-
capturedPublicKey = payload.publicKey;
|
|
1334
|
-
}
|
|
1335
|
-
});
|
|
1336
|
-
const timer = setTimeout(() => {
|
|
1337
|
-
detachListener();
|
|
1338
|
-
pendingDeploys.delete(agentAddress);
|
|
1339
|
-
if (addressIndex.get(agentAddress) === ws) {
|
|
1340
|
-
conn.agentAddresses.delete(agentAddress);
|
|
1341
|
-
addressIndex.delete(agentAddress);
|
|
1342
|
-
}
|
|
1343
|
-
reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
1344
|
-
}, requestTimeoutMs);
|
|
1345
|
-
pendingDeploys.set(agentAddress, {
|
|
1346
|
-
agentAddress,
|
|
1347
|
-
ws,
|
|
1348
|
-
resolve() {
|
|
1349
|
-
detachListener();
|
|
1350
|
-
if (capturedPublicKey === undefined) {
|
|
1351
|
-
reject(new Error(`Deploy of "${agentAddress}" resolved without an agent.deploy.ack publicKey payload`));
|
|
1352
|
-
return;
|
|
1353
|
-
}
|
|
1354
|
-
resolve({ publicKey: capturedPublicKey });
|
|
2226
|
+
// Timeout and frame-error rejections share this closure, so the routing
|
|
2227
|
+
// rollback and the `frameSent: true` tag live in one place.
|
|
2228
|
+
pendingDeploys.register(agentAddress, ws, {
|
|
2229
|
+
timeoutMs: requestTimeoutMs,
|
|
2230
|
+
timeoutMessage: `Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
2231
|
+
resolve(publicKey) {
|
|
2232
|
+
resolve({ publicKey });
|
|
1355
2233
|
},
|
|
1356
2234
|
reject(error) {
|
|
1357
|
-
detachListener();
|
|
1358
2235
|
if (addressIndex.get(agentAddress) === ws) {
|
|
1359
|
-
|
|
2236
|
+
addressSet.delete(agentAddress);
|
|
1360
2237
|
addressIndex.delete(agentAddress);
|
|
1361
2238
|
}
|
|
1362
|
-
|
|
2239
|
+
// A non-allocated deployment's key is recorded by the deploy-ack
|
|
2240
|
+
// projection, whose failure (reject/timeout/agent.error/disconnect)
|
|
2241
|
+
// is observed only here. Drain any pre-ack sender mail parked on
|
|
2242
|
+
// this address so it surfaces as undelivered rather than waiting out
|
|
2243
|
+
// the TTL. An allocated deployment's failure is drained by its
|
|
2244
|
+
// session-service owner instead, so skip it here to keep one owner
|
|
2245
|
+
// per case.
|
|
2246
|
+
if (conn.identity.kind !== "allocated") {
|
|
2247
|
+
drainDeferredSenderMail(agentAddress, `deploy failed: ${error}`);
|
|
2248
|
+
}
|
|
2249
|
+
reject(deployFrameFailure(error, true));
|
|
1363
2250
|
},
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
2251
|
+
}, undefined);
|
|
2252
|
+
try {
|
|
2253
|
+
conn.send({
|
|
2254
|
+
type: "agent.deploy",
|
|
2255
|
+
agentAddress,
|
|
2256
|
+
agentId: harnessConfig.agentId,
|
|
2257
|
+
config: harnessConfig,
|
|
2258
|
+
hubPublicKey: hubPublicKeyHex,
|
|
2259
|
+
...(workflow !== undefined ? { workflow } : {}),
|
|
2260
|
+
});
|
|
2261
|
+
}
|
|
2262
|
+
catch (err) {
|
|
2263
|
+
// A synchronous send failure means the frame never reached the wire.
|
|
2264
|
+
// Drop the pending entry (and its armed timer) and reject as not-sent
|
|
2265
|
+
// so a caller may safely roll back what it staged. The drop bypasses
|
|
2266
|
+
// the entry's reject closure: this failure must report
|
|
2267
|
+
// `frameSent: false`, and the timer must not fire later and
|
|
2268
|
+
// double-reject.
|
|
2269
|
+
pendingDeploys.delete(agentAddress);
|
|
2270
|
+
if (addressIndex.get(agentAddress) === ws) {
|
|
2271
|
+
addressSet.delete(agentAddress);
|
|
2272
|
+
addressIndex.delete(agentAddress);
|
|
2273
|
+
}
|
|
2274
|
+
reject(deployFrameFailure(`Deploy of "${agentAddress}" failed to send: ${err instanceof Error ? err.message : String(err)}`, false));
|
|
2275
|
+
}
|
|
1374
2276
|
});
|
|
1375
2277
|
}
|
|
2278
|
+
async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow, signal, beforeSend) {
|
|
2279
|
+
try {
|
|
2280
|
+
signal?.throwIfAborted();
|
|
2281
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2282
|
+
signal?.throwIfAborted();
|
|
2283
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
2284
|
+
throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
|
|
2285
|
+
}
|
|
2286
|
+
const existing = addressIndex.get(agentAddress);
|
|
2287
|
+
if (existing !== undefined && existing !== ws) {
|
|
2288
|
+
throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
|
|
2289
|
+
}
|
|
2290
|
+
if (hubPublicKeyHex === undefined)
|
|
2291
|
+
throw new Error("Hub signing key is required for agent deployment");
|
|
2292
|
+
if (pendingDeploys.has(agentAddress))
|
|
2293
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
2294
|
+
await beforeSend?.();
|
|
2295
|
+
signal?.throwIfAborted();
|
|
2296
|
+
if (allocatedConnections.get(target.allocationId)?.ws !== ws ||
|
|
2297
|
+
allocationFences.get(target.allocationId) !== target.generation) {
|
|
2298
|
+
throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
|
|
2299
|
+
}
|
|
2300
|
+
// Return without awaiting: only pre-send failures belong to this catch.
|
|
2301
|
+
return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
|
|
2302
|
+
}
|
|
2303
|
+
catch (cause) {
|
|
2304
|
+
throw deployFrameFailure(cause instanceof Error ? cause.message : String(cause), false, cause);
|
|
2305
|
+
}
|
|
2306
|
+
}
|
|
1376
2307
|
/**
|
|
1377
2308
|
* Provision one step of a multi-step deploy on the sidecar WITHOUT
|
|
1378
2309
|
* spawning: the sidecar initializes the step's agent-state repo and
|
|
@@ -1387,42 +2318,29 @@ export function createSidecarRouter(config) {
|
|
|
1387
2318
|
* so the caller can safely deliver the deploy pack afterward. On failure
|
|
1388
2319
|
* the caller owns tearing the route down via `unbindStepRoute`.
|
|
1389
2320
|
*/
|
|
1390
|
-
function
|
|
2321
|
+
function sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig) {
|
|
1391
2322
|
if (hubPublicKeyHex === undefined) {
|
|
1392
2323
|
throw new Error("Hub signing key is required for step provisioning");
|
|
1393
2324
|
}
|
|
1394
2325
|
if (pendingDeploys.has(agentAddress)) {
|
|
1395
2326
|
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
1396
2327
|
}
|
|
1397
|
-
const ws = addressIndex.get(agentAddress);
|
|
1398
|
-
if (ws === undefined) {
|
|
1399
|
-
throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
|
|
1400
|
-
}
|
|
1401
|
-
const conn = connections.get(ws);
|
|
1402
|
-
if (conn === undefined) {
|
|
1403
|
-
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1404
|
-
}
|
|
1405
2328
|
const hubKey = hubPublicKeyHex;
|
|
1406
2329
|
return new Promise((resolve, reject) => {
|
|
1407
|
-
const timer = setTimeout(() => {
|
|
1408
|
-
pendingDeploys.delete(agentAddress);
|
|
1409
|
-
reject(new Error(`Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
1410
|
-
}, requestTimeoutMs);
|
|
1411
2330
|
// The sidecar's `agent.deploy.ack` resolves this through
|
|
1412
|
-
// `
|
|
1413
|
-
//
|
|
1414
|
-
//
|
|
1415
|
-
pendingDeploys.
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
resolve() {
|
|
2331
|
+
// `pendingDeploys.resolve`. The per-step address is workflow-derived
|
|
2332
|
+
// and records no hub-side key, so the ack's public key is not needed
|
|
2333
|
+
// and this resolves void.
|
|
2334
|
+
pendingDeploys.register(agentAddress, ws, {
|
|
2335
|
+
timeoutMs: requestTimeoutMs,
|
|
2336
|
+
timeoutMessage: `Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
2337
|
+
resolve(_publicKey) {
|
|
1419
2338
|
resolve();
|
|
1420
2339
|
},
|
|
1421
2340
|
reject(error) {
|
|
1422
2341
|
reject(new Error(error));
|
|
1423
2342
|
},
|
|
1424
|
-
|
|
1425
|
-
});
|
|
2343
|
+
}, undefined);
|
|
1426
2344
|
conn.send({
|
|
1427
2345
|
type: "agent.deploy",
|
|
1428
2346
|
agentAddress,
|
|
@@ -1433,11 +2351,46 @@ export function createSidecarRouter(config) {
|
|
|
1433
2351
|
});
|
|
1434
2352
|
});
|
|
1435
2353
|
}
|
|
1436
|
-
function
|
|
1437
|
-
const
|
|
1438
|
-
if (
|
|
1439
|
-
|
|
1440
|
-
|
|
2354
|
+
async function sendProvisionStepToAllocation(target, agentAddress, harnessConfig) {
|
|
2355
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2356
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
2357
|
+
throw new Error(`Step route ${agentAddress} is not bound to allocation ${target.allocationId}`);
|
|
2358
|
+
}
|
|
2359
|
+
return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
|
|
2360
|
+
}
|
|
2361
|
+
function sendProbeOnConnection(ws, conn, args) {
|
|
2362
|
+
const requestId = nextRequestId();
|
|
2363
|
+
return new Promise((resolve, reject) => {
|
|
2364
|
+
pendingProbes.register(requestId, ws, {
|
|
2365
|
+
timeoutMs: probeTimeoutMs,
|
|
2366
|
+
timeoutMessage: `Probe ${requestId} timed out after ${probeTimeoutMs}ms`,
|
|
2367
|
+
resolve,
|
|
2368
|
+
reject(error) {
|
|
2369
|
+
reject(new Error(error));
|
|
2370
|
+
},
|
|
2371
|
+
}, undefined);
|
|
2372
|
+
conn.send({
|
|
2373
|
+
type: "workflow.probe.request",
|
|
2374
|
+
requestId,
|
|
2375
|
+
source: args.source,
|
|
2376
|
+
closure: args.closure,
|
|
2377
|
+
entry: args.entry,
|
|
2378
|
+
...(args.assets !== undefined ? { assets: args.assets } : {}),
|
|
2379
|
+
});
|
|
2380
|
+
});
|
|
2381
|
+
}
|
|
2382
|
+
async function sendProbeToAllocation(target, args) {
|
|
2383
|
+
const { ws, conn } = await getProvisionedConnection(target, "routing");
|
|
2384
|
+
return sendProbeOnConnection(ws, conn, args);
|
|
2385
|
+
}
|
|
2386
|
+
function disconnectAllocation(target) {
|
|
2387
|
+
const current = allocatedConnections.get(target.allocationId);
|
|
2388
|
+
if (current === undefined ||
|
|
2389
|
+
current.identity.generation !== target.generation) {
|
|
2390
|
+
return;
|
|
2391
|
+
}
|
|
2392
|
+
handleClose(current.ws);
|
|
2393
|
+
current.ws.close();
|
|
1441
2394
|
}
|
|
1442
2395
|
function sendAgentUndeploy(agentAddress, reason) {
|
|
1443
2396
|
const ws = addressIndex.get(agentAddress);
|
|
@@ -1449,14 +2402,11 @@ export function createSidecarRouter(config) {
|
|
|
1449
2402
|
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1450
2403
|
}
|
|
1451
2404
|
return new Promise((resolve, reject) => {
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
pendingUndeploys.set(agentAddress, {
|
|
1458
|
-
agentAddress,
|
|
1459
|
-
ws,
|
|
2405
|
+
// Timeout, ack, and error rejection share one closure so the routing
|
|
2406
|
+
// teardown runs exactly once no matter how the round-trip settles.
|
|
2407
|
+
pendingUndeploys.register(agentAddress, ws, {
|
|
2408
|
+
timeoutMs: requestTimeoutMs,
|
|
2409
|
+
timeoutMessage: `Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
|
|
1460
2410
|
resolve() {
|
|
1461
2411
|
removeAgentAddress(ws, agentAddress);
|
|
1462
2412
|
resolve();
|
|
@@ -1465,8 +2415,7 @@ export function createSidecarRouter(config) {
|
|
|
1465
2415
|
removeAgentAddress(ws, agentAddress);
|
|
1466
2416
|
reject(new Error(error));
|
|
1467
2417
|
},
|
|
1468
|
-
|
|
1469
|
-
});
|
|
2418
|
+
}, undefined);
|
|
1470
2419
|
conn.send({
|
|
1471
2420
|
type: "agent.undeploy",
|
|
1472
2421
|
agentAddress,
|
|
@@ -1529,6 +2478,15 @@ export function createSidecarRouter(config) {
|
|
|
1529
2478
|
defaultSource,
|
|
1530
2479
|
}));
|
|
1531
2480
|
}
|
|
2481
|
+
async function sendCredentialsUpdate(agentAddress, delivery, revoke) {
|
|
2482
|
+
await sendRequest(agentAddress, (requestId) => ({
|
|
2483
|
+
type: "credentials.update",
|
|
2484
|
+
requestId,
|
|
2485
|
+
agentAddress,
|
|
2486
|
+
delivery,
|
|
2487
|
+
...(revoke !== undefined ? { revoke } : {}),
|
|
2488
|
+
}));
|
|
2489
|
+
}
|
|
1532
2490
|
function sendSyncRequest(agentAddress) {
|
|
1533
2491
|
const ws = addressIndex.get(agentAddress);
|
|
1534
2492
|
if (ws === undefined) {
|
|
@@ -1563,6 +2521,15 @@ export function createSidecarRouter(config) {
|
|
|
1563
2521
|
payload: opts.payload,
|
|
1564
2522
|
});
|
|
1565
2523
|
}
|
|
2524
|
+
async function sendSignalDeliverToAllocation(target, opts, signal) {
|
|
2525
|
+
signal?.throwIfAborted();
|
|
2526
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2527
|
+
signal?.throwIfAborted();
|
|
2528
|
+
if (addressIndex.get(opts.agentAddress) !== ws) {
|
|
2529
|
+
throw new Error(`Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2530
|
+
}
|
|
2531
|
+
conn.send({ type: "signal.deliver", ...opts });
|
|
2532
|
+
}
|
|
1566
2533
|
function sendDrain(opts) {
|
|
1567
2534
|
const ws = addressIndex.get(opts.agentAddress);
|
|
1568
2535
|
if (ws === undefined) {
|
|
@@ -1583,15 +2550,29 @@ export function createSidecarRouter(config) {
|
|
|
1583
2550
|
handleMessage,
|
|
1584
2551
|
handleClose,
|
|
1585
2552
|
routeMail,
|
|
1586
|
-
|
|
2553
|
+
sendRunGrants,
|
|
2554
|
+
noteSenderDeployStarted,
|
|
2555
|
+
noteSenderDeploySettled,
|
|
2556
|
+
sendProbeToAllocation,
|
|
2557
|
+
disconnectAllocation,
|
|
1587
2558
|
sendAgentUndeploy,
|
|
1588
2559
|
sendSourcesUpdate,
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
2560
|
+
sendCredentialsUpdate,
|
|
2561
|
+
sendPackToAllocation,
|
|
2562
|
+
sendWorkflowRunPackToAllocation,
|
|
2563
|
+
fenceAllocation,
|
|
2564
|
+
retireAllocation,
|
|
2565
|
+
waitForAllocatedSidecar,
|
|
2566
|
+
isAllocatedSidecarReady,
|
|
2567
|
+
isAllocatedWorkflowActive,
|
|
2568
|
+
sendAgentDeployToAllocation,
|
|
2569
|
+
bindAllocatedStepRoute,
|
|
2570
|
+
unbindAllocatedStepRoute,
|
|
2571
|
+
sendProvisionStepToAllocation,
|
|
2572
|
+
sendWorkflowRunDispatchToAllocation,
|
|
1593
2573
|
sendSyncRequest,
|
|
1594
2574
|
sendSignalDeliver,
|
|
2575
|
+
sendSignalDeliverToAllocation,
|
|
1595
2576
|
sendDrain,
|
|
1596
2577
|
subscribeAgent,
|
|
1597
2578
|
dispatchAgentEvent: dispatchToSubscribers,
|