@intx/hub-sessions 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/agent-repo.d.ts +9 -5
- package/dist/agent-repo.js +2 -2
- package/dist/agent-state-kind.js +4 -0
- package/dist/asset-service.d.ts +1 -20
- package/dist/asset-service.js +9 -91
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +7 -6
- package/dist/credential-push.js +42 -18
- package/dist/event-collector-registry.d.ts +1 -1
- package/dist/event-collector-registry.js +4 -4
- package/dist/event-collector.d.ts +1 -1
- package/dist/event-collector.js +10 -2
- package/dist/hub-session-lookups.d.ts +125 -7
- package/dist/hub-session-lookups.js +539 -80
- package/dist/hub-session-orchestrator.js +14 -49
- package/dist/index.d.ts +17 -8
- package/dist/index.js +14 -6
- package/dist/repo-store/index.d.ts +1 -1
- package/dist/repo-store/store.d.ts +1 -1
- package/dist/repo-store/store.js +138 -1
- package/dist/repo-store/subscribe-kind.d.ts +6 -3
- package/dist/repo-store/subscribe-kind.js +42 -77
- package/dist/repo-store/types.d.ts +94 -6
- package/dist/session-service.d.ts +277 -96
- package/dist/session-service.js +741 -547
- package/dist/sidecar-allocation/contracts.d.ts +78 -0
- package/dist/sidecar-allocation/contracts.js +21 -0
- package/dist/sidecar-allocation/index.d.ts +4 -0
- package/dist/sidecar-allocation/index.js +3 -0
- package/dist/sidecar-allocation/placement-policy.d.ts +11 -0
- package/dist/sidecar-allocation/placement-policy.js +21 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +11 -0
- package/dist/sidecar-allocation/plugin-registry.js +37 -0
- package/dist/sidecar-allocation/reconciler.d.ts +42 -0
- package/dist/sidecar-allocation/reconciler.js +431 -0
- package/dist/skill-kind.js +4 -0
- package/dist/substrate.d.ts +3 -3
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +58 -0
- package/dist/workflow-allocation-service.js +239 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +40 -0
- package/dist/workflow-dispatch-service.js +146 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +17 -1
- package/dist/workflow-kind.js +127 -80
- package/dist/workflow-probe-gate.d.ts +214 -0
- package/dist/workflow-probe-gate.js +207 -0
- package/dist/workflow-run-kind.d.ts +128 -14
- package/dist/workflow-run-kind.js +353 -83
- package/dist/workflow-run-reader.d.ts +1 -1
- package/dist/workflow-run-reader.js +3 -7
- package/dist/workflow-run-restore.d.ts +15 -0
- package/dist/workflow-run-restore.js +26 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/ws/index.d.ts +3 -3
- package/dist/ws/index.js +1 -1
- package/dist/ws/sidecar-events.d.ts +100 -12
- package/dist/ws/sidecar-events.js +2 -0
- package/dist/ws/sidecar-handler.d.ts +128 -7
- package/dist/ws/sidecar-handler.js +1069 -135
- package/dist/ws/sidecar-token-authenticator.d.ts +3 -1
- package/dist/ws/sidecar-token-authenticator.js +64 -7
- package/package.json +14 -13
- package/dist/available-skills-stanza.d.ts +0 -21
- package/dist/available-skills-stanza.js +0 -32
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
import { getLogger } from "@intx/log";
|
|
7
7
|
import { verifyEd25519 } from "@intx/crypto";
|
|
8
8
|
import { chunkPack, createPackReceiver } from "@intx/pack-transport";
|
|
9
|
-
import { hexDecode, hexEncode } from "@intx/types";
|
|
10
|
-
import {
|
|
9
|
+
import { base64Decode, deriveMessageId, deriveWorkflowRunId, hexDecode, hexEncode, isRunAddress, } from "@intx/types";
|
|
10
|
+
import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
|
|
11
11
|
import { type } from "arktype";
|
|
12
12
|
import { SidecarFrame, } from "@intx/types/sidecar";
|
|
13
13
|
import { createSidecarEmitter, } from "./sidecar-events.js";
|
|
@@ -24,22 +24,48 @@ const logger = getLogger(["hub", "ws", "sidecar"]);
|
|
|
24
24
|
function connOwnsAddress(conn, address) {
|
|
25
25
|
return (conn.agentAddresses.has(address) || conn.workflowAddresses.has(address));
|
|
26
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Bind pack writes to the repository implied by the authenticated address.
|
|
29
|
+
* An allocated credential is narrower still: it may only write its one
|
|
30
|
+
* deployment's workflow-run repository and never a standalone agent-state
|
|
31
|
+
* repository.
|
|
32
|
+
*/
|
|
33
|
+
function connCanPushRepo(conn, agentAddress, repoId) {
|
|
34
|
+
if (conn.identity.kind === "allocated" &&
|
|
35
|
+
agentAddress !== conn.identity.workflowRunAddress) {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
if (repoId.kind === "agent-state") {
|
|
39
|
+
return conn.identity.kind === "shared" && repoId.id === agentAddress;
|
|
40
|
+
}
|
|
41
|
+
return (repoId.kind === "workflow-run" &&
|
|
42
|
+
repoId.id === deriveWorkflowRunRepoId(agentAddress));
|
|
43
|
+
}
|
|
27
44
|
/** The deduped set of every address this connection owns (session + workflow). */
|
|
28
45
|
function ownedAddresses(conn) {
|
|
29
46
|
return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
|
|
30
47
|
}
|
|
31
48
|
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
32
49
|
const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
|
|
50
|
+
// A probe fetches a workflow's dependency closure from a registry and
|
|
51
|
+
// evaluates it on the sidecar, so it runs longer than a routine request; its
|
|
52
|
+
// default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS.
|
|
53
|
+
export const DEFAULT_PROBE_TIMEOUT_MS = 60_000;
|
|
33
54
|
const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100;
|
|
34
55
|
const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
|
|
35
56
|
const DEFAULT_PING_TIMEOUT_MS = 60_000;
|
|
57
|
+
const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000;
|
|
58
|
+
const DEFAULT_MAIL_ACK_MAX_RETRIES = 5;
|
|
36
59
|
export function createSidecarRouter(config) {
|
|
37
|
-
const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, lookups = {}, } = config;
|
|
60
|
+
const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity = async () => true, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
|
|
38
61
|
// Receiver-dispatch surface. Wire-layer callsites emit events here;
|
|
39
62
|
// host code subscribes via `router.events`.
|
|
40
63
|
const events = createSidecarEmitter();
|
|
41
64
|
// ws handle → registered connection
|
|
42
65
|
const connections = new Map();
|
|
66
|
+
const allocatedConnections = new Map();
|
|
67
|
+
const allocationFences = new Map();
|
|
68
|
+
const allocationWaiters = new Map();
|
|
43
69
|
// agentAddress → ws handle (routing table)
|
|
44
70
|
const addressIndex = new Map();
|
|
45
71
|
// requestId → pending promise
|
|
@@ -47,6 +73,14 @@ export function createSidecarRouter(config) {
|
|
|
47
73
|
const pendingDeploys = new Map();
|
|
48
74
|
const pendingChallenges = new Map();
|
|
49
75
|
const disconnectedAgents = new Map();
|
|
76
|
+
const pendingMail = new Map();
|
|
77
|
+
// agentAddress → retention TTL timer for un-acked pending mail held across a
|
|
78
|
+
// disconnect. On close the per-entry retry timers are cleared (the socket is
|
|
79
|
+
// gone) but the entries are RETAINED so a verified reconnect can redeliver
|
|
80
|
+
// them; this timer bounds that retention so a sidecar that never reconnects
|
|
81
|
+
// does not leak entries. Cleared when the address reconnects (redelivery) or
|
|
82
|
+
// its last pending entry is acked.
|
|
83
|
+
const pendingMailRetention = new Map();
|
|
50
84
|
// agentAddress → set of subscriber callbacks for agent events
|
|
51
85
|
const agentSubscribers = new Map();
|
|
52
86
|
// agentAddress → cached connector-thread state, populated by
|
|
@@ -69,6 +103,7 @@ export function createSidecarRouter(config) {
|
|
|
69
103
|
const pendingPacks = new Map();
|
|
70
104
|
let packCounter = 0;
|
|
71
105
|
const pendingUndeploys = new Map();
|
|
106
|
+
const pendingProbes = new Map();
|
|
72
107
|
// Receives agent-state packs pushed from sidecars. The wire frames
|
|
73
108
|
// (`repo.pack.push` / `repo.pack.done`) are shared with the
|
|
74
109
|
// workflow-run flow; dispatch on `repoId.kind` picks which receiver
|
|
@@ -125,6 +160,183 @@ export function createSidecarRouter(config) {
|
|
|
125
160
|
logger.info `Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
|
|
126
161
|
}
|
|
127
162
|
}
|
|
163
|
+
// Track a connected-window `mail.inbound` for redelivery until the sidecar
|
|
164
|
+
// acks its durable inbox write. Replaces any prior entry for the same
|
|
165
|
+
// (agentAddress, messageId) -- clearing its timer first so no timer leaks --
|
|
166
|
+
// which keeps a re-sent delivery from arming a second concurrent retry loop.
|
|
167
|
+
function trackPendingMail(agentAddress, messageId, frame, runGrants, allocatedTarget) {
|
|
168
|
+
let byId = pendingMail.get(agentAddress);
|
|
169
|
+
if (byId === undefined) {
|
|
170
|
+
byId = new Map();
|
|
171
|
+
pendingMail.set(agentAddress, byId);
|
|
172
|
+
}
|
|
173
|
+
const existing = byId.get(messageId);
|
|
174
|
+
if (existing !== undefined)
|
|
175
|
+
clearTimeout(existing.timer);
|
|
176
|
+
byId.set(messageId, {
|
|
177
|
+
agentAddress,
|
|
178
|
+
messageId,
|
|
179
|
+
frame,
|
|
180
|
+
attempts: 0,
|
|
181
|
+
timer: setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs),
|
|
182
|
+
...(runGrants !== undefined ? { runGrants } : {}),
|
|
183
|
+
...(allocatedTarget !== undefined ? { allocatedTarget } : {}),
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
// Replay a mail-triggered run's grants ahead of a redelivery of its trigger
|
|
187
|
+
// mail. Same-connection FIFO lands the `run.grants` frame before the mail, so
|
|
188
|
+
// the redelivered run resolves its onRunStart barrier instead of failing
|
|
189
|
+
// closed on missing grants.
|
|
190
|
+
function replayRunGrantsAhead(conn, entry) {
|
|
191
|
+
if (entry.runGrants === undefined)
|
|
192
|
+
return;
|
|
193
|
+
conn.send({
|
|
194
|
+
type: "run.grants",
|
|
195
|
+
agentAddress: entry.agentAddress,
|
|
196
|
+
runId: entry.runGrants.runId,
|
|
197
|
+
stepGrants: entry.runGrants.stepGrants,
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
function deletePendingMail(byId, agentAddress, messageId) {
|
|
201
|
+
byId.delete(messageId);
|
|
202
|
+
if (byId.size === 0) {
|
|
203
|
+
pendingMail.delete(agentAddress);
|
|
204
|
+
// The retention TTL guards a non-empty pending set; drop it once the set
|
|
205
|
+
// is empty so it never outlives the entries it was bounding.
|
|
206
|
+
const retention = pendingMailRetention.get(agentAddress);
|
|
207
|
+
if (retention !== undefined) {
|
|
208
|
+
clearTimeout(retention);
|
|
209
|
+
pendingMailRetention.delete(agentAddress);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
function retryPendingMail(agentAddress, messageId) {
|
|
214
|
+
const byId = pendingMail.get(agentAddress);
|
|
215
|
+
if (byId === undefined)
|
|
216
|
+
return;
|
|
217
|
+
const entry = byId.get(messageId);
|
|
218
|
+
if (entry === undefined)
|
|
219
|
+
return;
|
|
220
|
+
if (entry.attempts >= mailAckMaxRetries) {
|
|
221
|
+
// The sidecar never acked within the retry budget. The ack is withheld
|
|
222
|
+
// precisely because the sidecar's durable inbox write failed, so the
|
|
223
|
+
// mail was NOT delivered: surface it as undelivered so the host can relay
|
|
224
|
+
// it onto an external transport, then drop the pending entry so its timer
|
|
225
|
+
// does not leak.
|
|
226
|
+
if (entry.frame.type === "mail.inbound") {
|
|
227
|
+
events.emit("mail.outbound.undelivered", {
|
|
228
|
+
rawMessage: entry.frame.rawMessage,
|
|
229
|
+
recipients: [agentAddress],
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
233
|
+
logger.warn `Gave up redelivering mail ${messageId} to ${agentAddress} after ${String(entry.attempts)} un-acked attempt(s)`;
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
// Redeliver over the address's CURRENT owner: a verified reconnect may have
|
|
237
|
+
// moved the address to a new connection since the original delivery.
|
|
238
|
+
const ws = addressIndex.get(agentAddress);
|
|
239
|
+
const conn = ws !== undefined ? connections.get(ws) : undefined;
|
|
240
|
+
const allocated = entry.allocatedTarget === undefined
|
|
241
|
+
? undefined
|
|
242
|
+
: allocatedConnections.get(entry.allocatedTarget.allocationId);
|
|
243
|
+
const targetStillOwnsAddress = entry.allocatedTarget === undefined ||
|
|
244
|
+
(allocated !== undefined &&
|
|
245
|
+
allocated.identity.generation === entry.allocatedTarget.generation &&
|
|
246
|
+
allocated.ws === ws);
|
|
247
|
+
if (conn === undefined || !targetStillOwnsAddress) {
|
|
248
|
+
// No live connection to recover into. Connected-window redelivery only
|
|
249
|
+
// applies while the address is routable; a disconnected address is not
|
|
250
|
+
// retried here.
|
|
251
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
252
|
+
logger.warn `Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`;
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
entry.attempts += 1;
|
|
256
|
+
replayRunGrantsAhead(conn, entry);
|
|
257
|
+
conn.send(entry.frame);
|
|
258
|
+
entry.timer = setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs);
|
|
259
|
+
}
|
|
260
|
+
function resolvePendingMail(agentAddress, messageId) {
|
|
261
|
+
const byId = pendingMail.get(agentAddress);
|
|
262
|
+
if (byId === undefined)
|
|
263
|
+
return;
|
|
264
|
+
const entry = byId.get(messageId);
|
|
265
|
+
if (entry === undefined)
|
|
266
|
+
return;
|
|
267
|
+
clearTimeout(entry.timer);
|
|
268
|
+
deletePendingMail(byId, agentAddress, messageId);
|
|
269
|
+
}
|
|
270
|
+
// Hold an address's un-acked pending mail across a disconnect. The per-entry
|
|
271
|
+
// retry timers are cleared -- retrying over the dead socket is pointless --
|
|
272
|
+
// but the entries are KEPT so a verified reconnect can redeliver them. A
|
|
273
|
+
// retention TTL (the disconnect-queue horizon) bounds the hold so a sidecar
|
|
274
|
+
// that never reconnects does not leak; on expiry the still-un-acked entries
|
|
275
|
+
// are surfaced as `mail.outbound.undelivered` so the host can relay them,
|
|
276
|
+
// since a withheld ack means the sidecar's durable write never landed.
|
|
277
|
+
function retainPendingMailForAddress(agentAddress) {
|
|
278
|
+
const byId = pendingMail.get(agentAddress);
|
|
279
|
+
if (byId === undefined)
|
|
280
|
+
return;
|
|
281
|
+
for (const entry of byId.values())
|
|
282
|
+
clearTimeout(entry.timer);
|
|
283
|
+
const existing = pendingMailRetention.get(agentAddress);
|
|
284
|
+
if (existing !== undefined)
|
|
285
|
+
clearTimeout(existing);
|
|
286
|
+
const timer = setTimeout(() => {
|
|
287
|
+
pendingMailRetention.delete(agentAddress);
|
|
288
|
+
const expired = pendingMail.get(agentAddress);
|
|
289
|
+
pendingMail.delete(agentAddress);
|
|
290
|
+
if (expired !== undefined && expired.size > 0) {
|
|
291
|
+
for (const entry of expired.values()) {
|
|
292
|
+
if (entry.frame.type !== "mail.inbound")
|
|
293
|
+
continue;
|
|
294
|
+
events.emit("mail.outbound.undelivered", {
|
|
295
|
+
rawMessage: entry.frame.rawMessage,
|
|
296
|
+
recipients: [agentAddress],
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
logger.warn `Dropping ${String(expired.size)} un-acked message(s) for ${agentAddress}: pending-mail retention TTL expired`;
|
|
300
|
+
}
|
|
301
|
+
}, disconnectQueueTTLMs);
|
|
302
|
+
pendingMailRetention.set(agentAddress, timer);
|
|
303
|
+
}
|
|
304
|
+
// Redeliver an address's retained un-acked pending mail on a verified
|
|
305
|
+
// reconnect. Replays identical bytes (same messageId) over the new
|
|
306
|
+
// connection, so the sidecar's inbox dedups a message it already wrote
|
|
307
|
+
// (effectively-once) and processes one it had dropped (no loss). Re-arms the
|
|
308
|
+
// connected-window retry over the new connection with a fresh per-generation
|
|
309
|
+
// budget, so a redelivery that is itself dropped before its ack is retried.
|
|
310
|
+
function redeliverPendingMail(agentAddress, conn) {
|
|
311
|
+
const retention = pendingMailRetention.get(agentAddress);
|
|
312
|
+
if (retention !== undefined) {
|
|
313
|
+
clearTimeout(retention);
|
|
314
|
+
pendingMailRetention.delete(agentAddress);
|
|
315
|
+
}
|
|
316
|
+
const byId = pendingMail.get(agentAddress);
|
|
317
|
+
if (byId === undefined)
|
|
318
|
+
return;
|
|
319
|
+
for (const entry of [...byId.values()]) {
|
|
320
|
+
if (entry.allocatedTarget !== undefined &&
|
|
321
|
+
(conn.identity.kind !== "allocated" ||
|
|
322
|
+
conn.identity.allocationId !== entry.allocatedTarget.allocationId ||
|
|
323
|
+
conn.identity.generation !== entry.allocatedTarget.generation)) {
|
|
324
|
+
// The Hub-owned dispatch row survives generation replacement and will
|
|
325
|
+
// be requeued by the allocation-ready callback. Do not leak or replay
|
|
326
|
+
// this generation-local retry entry onto a different worker.
|
|
327
|
+
clearTimeout(entry.timer);
|
|
328
|
+
deletePendingMail(byId, agentAddress, entry.messageId);
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
331
|
+
replayRunGrantsAhead(conn, entry);
|
|
332
|
+
conn.send(entry.frame);
|
|
333
|
+
entry.attempts = 0;
|
|
334
|
+
entry.timer = setTimeout(() => retryPendingMail(agentAddress, entry.messageId), mailAckRetryIntervalMs);
|
|
335
|
+
}
|
|
336
|
+
if (byId.size > 0) {
|
|
337
|
+
logger.info `Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
128
340
|
function resetLivenessTimer(ws) {
|
|
129
341
|
const existing = livenessTimers.get(ws);
|
|
130
342
|
if (existing !== undefined)
|
|
@@ -213,6 +425,8 @@ export function createSidecarRouter(config) {
|
|
|
213
425
|
case "agent.undeploy.ack":
|
|
214
426
|
case "repo.pack.ack":
|
|
215
427
|
case "repo.pack.reject":
|
|
428
|
+
case "workflow.probe.result":
|
|
429
|
+
case "workflow.probe.error":
|
|
216
430
|
return true;
|
|
217
431
|
case "register":
|
|
218
432
|
case "reconnect":
|
|
@@ -220,6 +434,8 @@ export function createSidecarRouter(config) {
|
|
|
220
434
|
case "mail.outbound":
|
|
221
435
|
case "agent.event":
|
|
222
436
|
case "connector.state.changed":
|
|
437
|
+
case "mail.inbound.ack":
|
|
438
|
+
case "signal.correlation.register":
|
|
223
439
|
case "repo.pack.push":
|
|
224
440
|
case "repo.pack.done":
|
|
225
441
|
return false;
|
|
@@ -246,21 +462,20 @@ export function createSidecarRouter(config) {
|
|
|
246
462
|
case "challenge.response":
|
|
247
463
|
return handleChallengeResponse(ws, frame.responses);
|
|
248
464
|
case "agent.deploy.ack":
|
|
249
|
-
return handleDeployAck(
|
|
465
|
+
return handleDeployAck(ws, frame);
|
|
250
466
|
case "agent.error":
|
|
251
|
-
|
|
252
|
-
rejectUndeployPending(frame.agentAddress, frame.error);
|
|
467
|
+
rejectDeployPendingFromFrame(ws, frame.agentAddress, frame.error);
|
|
468
|
+
rejectUndeployPending(ws, frame.agentAddress, frame.error);
|
|
253
469
|
return;
|
|
254
470
|
case "agent.undeploy.ack":
|
|
255
|
-
resolveUndeployPending(frame.agentAddress);
|
|
471
|
+
resolveUndeployPending(ws, frame.agentAddress);
|
|
256
472
|
return;
|
|
257
473
|
case "ping":
|
|
258
474
|
handlePing(ws);
|
|
259
475
|
return;
|
|
260
476
|
case "mail.outbound":
|
|
261
477
|
if (frame.delivered !== true) {
|
|
262
|
-
handleMailOutbound(frame.rawMessage, frame.recipients);
|
|
263
|
-
return;
|
|
478
|
+
return handleMailOutbound(frame.rawMessage, frame.recipients);
|
|
264
479
|
}
|
|
265
480
|
if (lookups.persistMail && frame.senderAddress) {
|
|
266
481
|
return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
|
|
@@ -294,6 +509,38 @@ export function createSidecarRouter(config) {
|
|
|
294
509
|
connectorState: frame.connectorState,
|
|
295
510
|
});
|
|
296
511
|
return;
|
|
512
|
+
case "mail.inbound.ack": {
|
|
513
|
+
// Terminal receipt for a connected-window `mail.inbound`: the sidecar
|
|
514
|
+
// has durably written the message to its inbox. Gate on ownership --
|
|
515
|
+
// like connector.state.changed and signal.correlation.register -- so a
|
|
516
|
+
// sidecar cannot clear another sidecar's pending mail. The messageId is
|
|
517
|
+
// a hub-minted id only the owning sidecar ever received on the frame,
|
|
518
|
+
// so the ownership check is defense-in-depth, not the sole guard.
|
|
519
|
+
const conn = connections.get(ws);
|
|
520
|
+
if (conn === undefined)
|
|
521
|
+
return;
|
|
522
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
523
|
+
logger.warn `Dropping mail.inbound.ack for ${frame.agentAddress}: not registered to this sidecar`;
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
resolvePendingMail(frame.agentAddress, frame.messageId);
|
|
527
|
+
events.emit("mail.inbound.acknowledged", {
|
|
528
|
+
agentAddress: frame.agentAddress,
|
|
529
|
+
messageId: frame.messageId,
|
|
530
|
+
...(conn.identity.kind === "allocated"
|
|
531
|
+
? {
|
|
532
|
+
allocated: {
|
|
533
|
+
allocationId: conn.identity.allocationId,
|
|
534
|
+
anchorRunId: conn.identity.anchorRunId,
|
|
535
|
+
generation: conn.identity.generation,
|
|
536
|
+
},
|
|
537
|
+
}
|
|
538
|
+
: {}),
|
|
539
|
+
});
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
542
|
+
case "signal.correlation.register":
|
|
543
|
+
return handleSignalCorrelationRegister(ws, frame);
|
|
297
544
|
case "session.ack":
|
|
298
545
|
resolvePending(frame.requestId);
|
|
299
546
|
return;
|
|
@@ -301,16 +548,27 @@ export function createSidecarRouter(config) {
|
|
|
301
548
|
rejectPending(frame.requestId, frame.error);
|
|
302
549
|
return;
|
|
303
550
|
case "repo.pack.ack":
|
|
304
|
-
resolvePackPending(frame
|
|
551
|
+
resolvePackPending(ws, frame);
|
|
305
552
|
return;
|
|
306
553
|
case "repo.pack.reject":
|
|
307
|
-
rejectPackPending(
|
|
554
|
+
rejectPackPending(ws, frame);
|
|
308
555
|
return;
|
|
309
556
|
case "repo.pack.push":
|
|
310
557
|
handlePackPush(ws, frame);
|
|
311
558
|
return;
|
|
312
559
|
case "repo.pack.done":
|
|
313
560
|
return handlePackDone(ws, frame);
|
|
561
|
+
case "workflow.probe.result":
|
|
562
|
+
resolveProbe(frame.requestId, {
|
|
563
|
+
projection: frame.projection,
|
|
564
|
+
grants: frame.grants,
|
|
565
|
+
grantWalkSnapshot: frame.grantWalkSnapshot,
|
|
566
|
+
wireHash: frame.wireHash,
|
|
567
|
+
});
|
|
568
|
+
return;
|
|
569
|
+
case "workflow.probe.error":
|
|
570
|
+
rejectProbe(frame.requestId, frame.error);
|
|
571
|
+
return;
|
|
314
572
|
default:
|
|
315
573
|
return assertNever(frame);
|
|
316
574
|
}
|
|
@@ -343,9 +601,107 @@ export function createSidecarRouter(config) {
|
|
|
343
601
|
if (identity.sidecarId !== frame.sidecarId) {
|
|
344
602
|
logger.warn `Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${identity.sidecarId}; keying off the verified id`;
|
|
345
603
|
}
|
|
604
|
+
if (!(await validateSidecarIdentity(identity, "registration"))) {
|
|
605
|
+
logger.warn `Rejected ${frame.type} from sidecar ${identity.sidecarId}: credential identity is no longer current`;
|
|
606
|
+
ws.close();
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
346
609
|
await run(identity);
|
|
347
610
|
}
|
|
611
|
+
async function notifyAllocationWaiters(allocationId) {
|
|
612
|
+
const waiters = allocationWaiters.get(allocationId);
|
|
613
|
+
const current = allocatedConnections.get(allocationId);
|
|
614
|
+
if (waiters === undefined || current === undefined)
|
|
615
|
+
return;
|
|
616
|
+
if (!(await validateSidecarIdentity(current.identity, "readiness")))
|
|
617
|
+
return;
|
|
618
|
+
for (const waiter of [...waiters]) {
|
|
619
|
+
if (waiter.generation !== current.identity.generation)
|
|
620
|
+
continue;
|
|
621
|
+
clearTimeout(waiter.timer);
|
|
622
|
+
waiters.delete(waiter);
|
|
623
|
+
waiter.resolve();
|
|
624
|
+
}
|
|
625
|
+
if (waiters.size === 0)
|
|
626
|
+
allocationWaiters.delete(allocationId);
|
|
627
|
+
}
|
|
628
|
+
async function handleAllocatedRegister(ws, identity, agentAddresses) {
|
|
629
|
+
if (allocationFences.get(identity.allocationId) !== identity.generation) {
|
|
630
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`;
|
|
631
|
+
ws.close();
|
|
632
|
+
return;
|
|
633
|
+
}
|
|
634
|
+
if (agentAddresses.length > 0 &&
|
|
635
|
+
!(await validateSidecarIdentity(identity, "routing"))) {
|
|
636
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`;
|
|
637
|
+
ws.close();
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
const existingOnSocket = connections.get(ws);
|
|
641
|
+
const newlyRoutedAddresses = new Set();
|
|
642
|
+
for (const address of agentAddresses) {
|
|
643
|
+
const alreadyOwned = existingOnSocket?.identity.kind === "allocated" &&
|
|
644
|
+
existingOnSocket.identity.allocationId === identity.allocationId &&
|
|
645
|
+
addressIndex.get(address) === ws &&
|
|
646
|
+
connOwnsAddress(existingOnSocket, address);
|
|
647
|
+
if (!alreadyOwned && address !== identity.workflowRunAddress) {
|
|
648
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`;
|
|
649
|
+
ws.close();
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
if (!alreadyOwned)
|
|
653
|
+
newlyRoutedAddresses.add(address);
|
|
654
|
+
}
|
|
655
|
+
if (allocationFences.get(identity.allocationId) !== identity.generation) {
|
|
656
|
+
ws.close();
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
const current = allocatedConnections.get(identity.allocationId);
|
|
660
|
+
if (current !== undefined && current.ws !== ws) {
|
|
661
|
+
// A current-generation takeover is a reconnect, not a capacity loss.
|
|
662
|
+
// Remove the old socket from the allocation index before closing it so
|
|
663
|
+
// handleClose does not emit a false allocated-disconnect event.
|
|
664
|
+
allocatedConnections.delete(identity.allocationId);
|
|
665
|
+
handleClose(current.ws);
|
|
666
|
+
current.ws.close();
|
|
667
|
+
}
|
|
668
|
+
const conn = existingOnSocket ?? {
|
|
669
|
+
sidecarId: identity.sidecarId,
|
|
670
|
+
identity,
|
|
671
|
+
agentAddresses: new Set(),
|
|
672
|
+
workflowAddresses: new Set(),
|
|
673
|
+
send(frame) {
|
|
674
|
+
ws.send(JSON.stringify(frame));
|
|
675
|
+
},
|
|
676
|
+
};
|
|
677
|
+
if (conn.identity.kind !== "allocated" ||
|
|
678
|
+
conn.identity.allocationId !== identity.allocationId ||
|
|
679
|
+
conn.identity.generation !== identity.generation) {
|
|
680
|
+
logger.warn `Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`;
|
|
681
|
+
ws.close();
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
684
|
+
connections.set(ws, conn);
|
|
685
|
+
for (const address of agentAddresses) {
|
|
686
|
+
conn.workflowAddresses.add(address);
|
|
687
|
+
addressIndex.set(address, ws);
|
|
688
|
+
}
|
|
689
|
+
allocatedConnections.set(identity.allocationId, { ws, identity });
|
|
690
|
+
for (const address of newlyRoutedAddresses) {
|
|
691
|
+
redeliverPendingMail(address, conn);
|
|
692
|
+
}
|
|
693
|
+
logger.info `Allocated sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
|
|
694
|
+
await notifyAllocationWaiters(identity.allocationId);
|
|
695
|
+
events.emit("sidecar.allocated.connected", {
|
|
696
|
+
allocationId: identity.allocationId,
|
|
697
|
+
generation: identity.generation,
|
|
698
|
+
});
|
|
699
|
+
}
|
|
348
700
|
async function handleRegister(ws, identity, agentAddresses) {
|
|
701
|
+
if (identity.kind === "allocated") {
|
|
702
|
+
await handleAllocatedRegister(ws, identity, agentAddresses);
|
|
703
|
+
return;
|
|
704
|
+
}
|
|
349
705
|
const sidecarId = identity.sidecarId;
|
|
350
706
|
// Key-existence gate. A register frame is token-authenticated but carries
|
|
351
707
|
// no per-address ownership proof, so it may route an address ONLY if that
|
|
@@ -434,6 +790,7 @@ export function createSidecarRouter(config) {
|
|
|
434
790
|
const workflowSet = inheritedWorkflow;
|
|
435
791
|
const conn = {
|
|
436
792
|
sidecarId,
|
|
793
|
+
identity,
|
|
437
794
|
agentAddresses: addrSet,
|
|
438
795
|
workflowAddresses: workflowSet,
|
|
439
796
|
send(frame) {
|
|
@@ -466,6 +823,10 @@ export function createSidecarRouter(config) {
|
|
|
466
823
|
logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
|
|
467
824
|
}
|
|
468
825
|
async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
|
|
826
|
+
if (identity.kind === "allocated") {
|
|
827
|
+
await handleAllocatedRegister(ws, identity, agentAddresses);
|
|
828
|
+
return;
|
|
829
|
+
}
|
|
469
830
|
const sidecarId = identity.sidecarId;
|
|
470
831
|
const lookupKey = lookups.lookupPublicKey;
|
|
471
832
|
if (lookupKey === undefined) {
|
|
@@ -534,7 +895,7 @@ export function createSidecarRouter(config) {
|
|
|
534
895
|
conn.send({
|
|
535
896
|
type: "challenge.failed",
|
|
536
897
|
address,
|
|
537
|
-
reason: "Unknown
|
|
898
|
+
reason: "Unknown run address",
|
|
538
899
|
});
|
|
539
900
|
continue;
|
|
540
901
|
}
|
|
@@ -657,14 +1018,17 @@ export function createSidecarRouter(config) {
|
|
|
657
1018
|
prevConn.agentAddresses.delete(addr);
|
|
658
1019
|
}
|
|
659
1020
|
}
|
|
660
|
-
//
|
|
661
|
-
//
|
|
662
|
-
//
|
|
663
|
-
//
|
|
664
|
-
//
|
|
665
|
-
//
|
|
666
|
-
//
|
|
667
|
-
|
|
1021
|
+
// A run address goes on the workflow set (no disconnect queue -- its
|
|
1022
|
+
// in-flight state is reconstructed sidecar-locally on the next reconnect),
|
|
1023
|
+
// so handleClose reclaims it correctly. The routing pointer is the same
|
|
1024
|
+
// either way; only now it is written behind a passed challenge.
|
|
1025
|
+
//
|
|
1026
|
+
// The `else` (session set, queued for reconnect) is the retired
|
|
1027
|
+
// launched-agent path: launched agents no longer exist (the folded-launch
|
|
1028
|
+
// route was removed), so no current producer reaches it. It is left in
|
|
1029
|
+
// place for the reconnect-subsystem teardown to remove as its own
|
|
1030
|
+
// reviewable change, not folded into this collapse.
|
|
1031
|
+
if (isRunAddress(addr)) {
|
|
668
1032
|
conn.workflowAddresses.add(addr);
|
|
669
1033
|
}
|
|
670
1034
|
else {
|
|
@@ -675,14 +1039,17 @@ export function createSidecarRouter(config) {
|
|
|
675
1039
|
const ready = [];
|
|
676
1040
|
const failed = [];
|
|
677
1041
|
for (const addr of verified) {
|
|
678
|
-
//
|
|
679
|
-
//
|
|
680
|
-
//
|
|
681
|
-
//
|
|
682
|
-
//
|
|
683
|
-
//
|
|
684
|
-
//
|
|
685
|
-
|
|
1042
|
+
// A workflow run needs routing + queue flush only, which the passed
|
|
1043
|
+
// challenge has now made safe: its in-flight state is reconstructed
|
|
1044
|
+
// sidecar-locally, so it does not enrol in the `agent.reconnected` session
|
|
1045
|
+
// reaction (event-collector restore). Skip the reaction for it.
|
|
1046
|
+
//
|
|
1047
|
+
// The reaction below was the retired launched-agent path (restore a
|
|
1048
|
+
// launched agent's collector). Launched agents no longer exist, so no
|
|
1049
|
+
// current producer reaches it; the `agent.reconnected` listener is gone,
|
|
1050
|
+
// so `listenerCount` is 0 and the emit is inert. Left for the
|
|
1051
|
+
// reconnect-subsystem teardown to remove.
|
|
1052
|
+
if (isRunAddress(addr)) {
|
|
686
1053
|
ready.push(addr);
|
|
687
1054
|
continue;
|
|
688
1055
|
}
|
|
@@ -715,9 +1082,14 @@ export function createSidecarRouter(config) {
|
|
|
715
1082
|
conn.agentAddresses.delete(addr);
|
|
716
1083
|
addressIndex.delete(addr);
|
|
717
1084
|
}
|
|
718
|
-
// Flush queued messages only for
|
|
1085
|
+
// Flush queued messages and redeliver retained un-acked mail, only for
|
|
1086
|
+
// ready addresses. The disconnect queue carries mail that arrived WHILE
|
|
1087
|
+
// disconnected; pending-mail redelivery carries mail that was delivered
|
|
1088
|
+
// over the prior live connection but never acked (the connected-window
|
|
1089
|
+
// drop). Both replay over the verified new connection.
|
|
719
1090
|
for (const addr of ready) {
|
|
720
1091
|
flushDisconnectedQueue(addr, conn);
|
|
1092
|
+
redeliverPendingMail(addr, conn);
|
|
721
1093
|
}
|
|
722
1094
|
// Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
|
|
723
1095
|
// so reconnect completion is not blocked on pack transfer. The
|
|
@@ -728,15 +1100,17 @@ export function createSidecarRouter(config) {
|
|
|
728
1100
|
for (const addr of ready) {
|
|
729
1101
|
// Workflow deployments are pinned-forever: a deployment keeps its
|
|
730
1102
|
// deploy-time definition until an explicit undeploy/redeploy, so the
|
|
731
|
-
// deploy-ref freshness catch-up is deliberately NOT run for a
|
|
732
|
-
//
|
|
733
|
-
//
|
|
734
|
-
//
|
|
735
|
-
//
|
|
736
|
-
//
|
|
1103
|
+
// deploy-ref freshness catch-up is deliberately NOT run for a run
|
|
1104
|
+
// address. A definition edited on the hub while the sidecar was
|
|
1105
|
+
// disconnected does not reconcile on reconnect; it affects only newly
|
|
1106
|
+
// created deployments. The deployment's in-flight run state is
|
|
1107
|
+
// reconstructed sidecar-locally at restore, not re-fetched here. Do NOT
|
|
1108
|
+
// add a reconcile path for these addresses -- see the "Workflow
|
|
737
1109
|
// Definition Versioning: Pinned-Forever" note under "Reconnect
|
|
738
|
-
// Sequencing" in docs/IMPLEMENTATION.md.
|
|
739
|
-
|
|
1110
|
+
// Sequencing" in docs/IMPLEMENTATION.md. (Every routable address is a
|
|
1111
|
+
// run address now; the fall-through was the retired launched-agent
|
|
1112
|
+
// catch-up path.)
|
|
1113
|
+
if (isRunAddress(addr))
|
|
740
1114
|
continue;
|
|
741
1115
|
void (async () => {
|
|
742
1116
|
try {
|
|
@@ -779,30 +1153,44 @@ export function createSidecarRouter(config) {
|
|
|
779
1153
|
}
|
|
780
1154
|
logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
|
|
781
1155
|
}
|
|
782
|
-
function handleMailOutbound(rawMessage, recipients) {
|
|
1156
|
+
async function handleMailOutbound(rawMessage, recipients) {
|
|
1157
|
+
// A mail addressed to more than one workflow deployment would birth a
|
|
1158
|
+
// run per recipient from a single inbound mail. The stable runId
|
|
1159
|
+
// removed the Message-ID collision that originally forced this guard --
|
|
1160
|
+
// each recipient now derives its own per-deployment runId (its mail
|
|
1161
|
+
// address), so it is no longer a runId-collision guard. It stays a
|
|
1162
|
+
// deliberate one-workflow-recipient-per-mail restriction because the
|
|
1163
|
+
// fan-out is not verified end-to-end: per-recipient grants
|
|
1164
|
+
// materialization, consumed-tracking, and reply-addressing all assume a
|
|
1165
|
+
// single workflow recipient today. Lifting it means proving those three
|
|
1166
|
+
// hold per recipient, not just relaxing this check -- so fail loudly
|
|
1167
|
+
// rather than materialize a partial set. The guard only applies when a
|
|
1168
|
+
// materializer is wired -- absent one, no run is born from the mail, so
|
|
1169
|
+
// there is nothing to restrict.
|
|
1170
|
+
if (lookups.materializeMailTriggeredRunGrants !== undefined) {
|
|
1171
|
+
// A workflow recipient is one this hub owns: its address parses as a run
|
|
1172
|
+
// address. An external/federated address does not, and is not ours to
|
|
1173
|
+
// materialize a run for.
|
|
1174
|
+
const workflowRecipients = recipients.filter(isRunAddress);
|
|
1175
|
+
if (workflowRecipients.length > 1) {
|
|
1176
|
+
throw new Error(`mail addressed to multiple workflow-derived recipients (${workflowRecipients.join(", ")}); materializing a run for more than one workflow deployment from a single mail is unsupported`);
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
783
1179
|
// Route to locally connected sidecars first, then try disconnect queues.
|
|
784
1180
|
const unrouted = [];
|
|
785
1181
|
for (const recipient of recipients) {
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
}
|
|
1182
|
+
// Each recipient is isolated: a materialization failure or a
|
|
1183
|
+
// fail-closed rejection for one must not drop the mail for its
|
|
1184
|
+
// co-recipients. The catch fails THIS recipient closed (its run never
|
|
1185
|
+
// starts under-authorized) and continues to the rest.
|
|
1186
|
+
try {
|
|
1187
|
+
const outcome = await deliverMailToRecipient(recipient, rawMessage);
|
|
1188
|
+
if (outcome === "unrouted")
|
|
1189
|
+
unrouted.push(recipient);
|
|
1190
|
+
}
|
|
1191
|
+
catch (err) {
|
|
1192
|
+
logger.error `Failed to deliver mail to ${recipient}: ${err instanceof Error ? err.message : String(err)}`;
|
|
797
1193
|
}
|
|
798
|
-
const frame = {
|
|
799
|
-
type: "mail.inbound",
|
|
800
|
-
agentAddress: recipient,
|
|
801
|
-
rawMessage,
|
|
802
|
-
};
|
|
803
|
-
if (enqueueForDisconnected(recipient, frame))
|
|
804
|
-
continue;
|
|
805
|
-
unrouted.push(recipient);
|
|
806
1194
|
}
|
|
807
1195
|
// Anything not routed locally is emitted as a notification. The
|
|
808
1196
|
// host decides whether to relay onto an external transport, log,
|
|
@@ -814,6 +1202,69 @@ export function createSidecarRouter(config) {
|
|
|
814
1202
|
});
|
|
815
1203
|
}
|
|
816
1204
|
}
|
|
1205
|
+
// Deliver an inbound mail to one recipient, materializing a
|
|
1206
|
+
// mail-triggered run's grants first when the recipient is a workflow
|
|
1207
|
+
// deployment. Returns:
|
|
1208
|
+
// - `routed`: the mail reached a live connection or disconnect queue.
|
|
1209
|
+
// - `unrouted`: the mail was locally undeliverable and should be
|
|
1210
|
+
// relayed externally by the host.
|
|
1211
|
+
// - `failed-closed`: the run's grants could not be materialized safely,
|
|
1212
|
+
// so the mail is deliberately DROPPED for this recipient (not relayed)
|
|
1213
|
+
// to keep its run from starting under-authorized.
|
|
1214
|
+
//
|
|
1215
|
+
// A workflow deployment is the only recipient whose inbound mail can first
|
|
1216
|
+
// fire its stable run. Its grants are reserved, and the `run.grants` frame is
|
|
1217
|
+
// sent BEFORE the mail. Same-address FIFO guarantees it lands ahead of the
|
|
1218
|
+
// mail that dispatches the run, so the run's `onRunStart` barrier resolves its
|
|
1219
|
+
// grants rather than failing closed. Reservation happens before routing so
|
|
1220
|
+
// concurrent first deliveries cannot send different snapshots; a routing
|
|
1221
|
+
// failure leaves a grants-only, still-unfired run.
|
|
1222
|
+
async function deliverMailToRecipient(recipient, rawMessage) {
|
|
1223
|
+
if (lookups.materializeMailTriggeredRunGrants !== undefined &&
|
|
1224
|
+
isRunAddress(recipient)) {
|
|
1225
|
+
const runId = deriveWorkflowRunId(recipient);
|
|
1226
|
+
const result = await lookups.materializeMailTriggeredRunGrants({
|
|
1227
|
+
agentAddress: recipient,
|
|
1228
|
+
runId,
|
|
1229
|
+
});
|
|
1230
|
+
if (result.outcome === "rejected") {
|
|
1231
|
+
// The run's grants could not be materialized with sufficient
|
|
1232
|
+
// authority or it is already terminal. Fail the mail closed for this
|
|
1233
|
+
// recipient: routing or external relay would bypass that decision.
|
|
1234
|
+
logger.error `Refusing mail-triggered run ${runId} for ${recipient}: grant materialization rejected (${result.code}): ${result.message}`;
|
|
1235
|
+
return "failed-closed";
|
|
1236
|
+
}
|
|
1237
|
+
if (result.outcome === "materialized") {
|
|
1238
|
+
// Send the run's grants ahead of the mail. A `false` here means the
|
|
1239
|
+
// deployment is unroutable. Do not route the mail that would dispatch
|
|
1240
|
+
// it; the grants-only reservation remains the canonical snapshot for a
|
|
1241
|
+
// later first-delivery attempt.
|
|
1242
|
+
if (!sendRunGrants(recipient, runId, result.stepGrants)) {
|
|
1243
|
+
logger.error `Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`;
|
|
1244
|
+
return "unrouted";
|
|
1245
|
+
}
|
|
1246
|
+
// Route through the messageId handshake `routeMail` -- NOT a
|
|
1247
|
+
// fire-and-forget send. This branch COMMITS a run, so a mail dropped in
|
|
1248
|
+
// the connected window (a socket that half-dies before the sidecar's
|
|
1249
|
+
// durable-write ack) would otherwise leave the run row "running"
|
|
1250
|
+
// forever with no body and no error. `routeMail` tracks the delivery
|
|
1251
|
+
// and redelivers identical bytes on reconnect, bringing the mail-relay
|
|
1252
|
+
// run-trigger to parity with the HTTP-trigger path. The messageId is the
|
|
1253
|
+
// mail's own id (derived over the same bytes the sidecar derives), so a
|
|
1254
|
+
// redelivery replays identically and the downstream RunStarted /
|
|
1255
|
+
// stable-runId dedup makes it effectively-once.
|
|
1256
|
+
const messageId = await deriveMessageId(base64Decode(rawMessage));
|
|
1257
|
+
const outcome = routeMail(recipient, rawMessage, messageId, { runId, stepGrants: result.stepGrants })
|
|
1258
|
+
? "routed"
|
|
1259
|
+
: "unrouted";
|
|
1260
|
+
return outcome;
|
|
1261
|
+
}
|
|
1262
|
+
// `skip`: the address named no deployed workflow deployment. Forward
|
|
1263
|
+
// the mail without grants -- the run, if any, is not ours to
|
|
1264
|
+
// authorize. No run is committed here, so no ack handshake is needed.
|
|
1265
|
+
}
|
|
1266
|
+
return routeMail(recipient, rawMessage) ? "routed" : "unrouted";
|
|
1267
|
+
}
|
|
817
1268
|
async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
|
|
818
1269
|
let results;
|
|
819
1270
|
let raw;
|
|
@@ -835,15 +1286,60 @@ export function createSidecarRouter(config) {
|
|
|
835
1286
|
raw,
|
|
836
1287
|
createdAt: result.createdAt,
|
|
837
1288
|
direction: result.direction,
|
|
838
|
-
|
|
1289
|
+
runId: result.runId,
|
|
839
1290
|
address: result.address,
|
|
840
1291
|
});
|
|
841
1292
|
}
|
|
842
1293
|
}
|
|
1294
|
+
async function handleSignalCorrelationRegister(ws, frame) {
|
|
1295
|
+
// Gate the co-write on the sending sidecar actually owning the named
|
|
1296
|
+
// deployment address, mirroring the connector.state.changed and pack
|
|
1297
|
+
// handlers. A workflow deployment routes on the keyless workflow set, so
|
|
1298
|
+
// ownership is the union check, not addressIndex identity alone. Without
|
|
1299
|
+
// it a misbehaving sidecar that knows another deployment's address could
|
|
1300
|
+
// register a spurious correlation against it.
|
|
1301
|
+
const conn = connections.get(ws);
|
|
1302
|
+
if (conn === undefined)
|
|
1303
|
+
return;
|
|
1304
|
+
if (!connOwnsAddress(conn, frame.agentAddress)) {
|
|
1305
|
+
logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: not registered to this sidecar`;
|
|
1306
|
+
return;
|
|
1307
|
+
}
|
|
1308
|
+
const register = lookups.registerSignalCorrelation;
|
|
1309
|
+
if (register === undefined) {
|
|
1310
|
+
logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: no registerSignalCorrelation lookup configured`;
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
try {
|
|
1314
|
+
await register({
|
|
1315
|
+
correlationId: frame.correlationId,
|
|
1316
|
+
runId: frame.runId,
|
|
1317
|
+
anchorRunId: frame.anchorRunId,
|
|
1318
|
+
agentAddress: frame.agentAddress,
|
|
1319
|
+
kind: frame.kind,
|
|
1320
|
+
approvalSnapshot: frame.snapshot,
|
|
1321
|
+
});
|
|
1322
|
+
// The co-write resolves only when a row exists -- freshly inserted or
|
|
1323
|
+
// already present (both stores are idempotent on the correlationId). Ack
|
|
1324
|
+
// so the sidecar's link stops retrying a register whose frame may have
|
|
1325
|
+
// been lost on an open socket. A thrown co-write (undeployed deployment,
|
|
1326
|
+
// id mismatch) means no row, so no ack: the sidecar keeps retrying and
|
|
1327
|
+
// the reconnect re-emit remains the ultimate backstop.
|
|
1328
|
+
conn.send({
|
|
1329
|
+
type: "signal.correlation.register.ack",
|
|
1330
|
+
agentAddress: frame.agentAddress,
|
|
1331
|
+
correlationId: frame.correlationId,
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
catch (err) {
|
|
1335
|
+
logger.error `Failed to register signal correlation ${frame.correlationId} for ${frame.agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
843
1338
|
function handleClose(ws) {
|
|
844
1339
|
const conn = connections.get(ws);
|
|
845
1340
|
if (conn === undefined)
|
|
846
1341
|
return;
|
|
1342
|
+
let allocated;
|
|
847
1343
|
for (const addr of conn.agentAddresses) {
|
|
848
1344
|
// Only remove routing and pending state if this connection still
|
|
849
1345
|
// owns the address. A reconnected sidecar may have already claimed it.
|
|
@@ -855,12 +1351,12 @@ export function createSidecarRouter(config) {
|
|
|
855
1351
|
// reconnect re-bootstraps via the router's
|
|
856
1352
|
// restore-fires-callback path.
|
|
857
1353
|
connectorStates.delete(addr);
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
1354
|
+
// Retain this address's un-acked pending mail across the disconnect:
|
|
1355
|
+
// its in-flight retry timers target a dead socket (cleared), but the
|
|
1356
|
+
// entries are held so a verified reconnect redelivers them, closing the
|
|
1357
|
+
// connected-window drop rather than losing the mail. Bounded by a
|
|
1358
|
+
// retention TTL.
|
|
1359
|
+
retainPendingMailForAddress(addr);
|
|
864
1360
|
// Create a queue entry so messages can accumulate while the
|
|
865
1361
|
// sidecar is disconnected. Skip if the agent is being undeployed --
|
|
866
1362
|
// there is no point queuing messages for an agent being torn down.
|
|
@@ -885,6 +1381,22 @@ export function createSidecarRouter(config) {
|
|
|
885
1381
|
for (const addr of conn.workflowAddresses) {
|
|
886
1382
|
if (addressIndex.get(addr) === ws) {
|
|
887
1383
|
addressIndex.delete(addr);
|
|
1384
|
+
// Retain un-acked workflow trigger mail across the disconnect for the
|
|
1385
|
+
// same reason as the session loop above -- a challenged reconnect
|
|
1386
|
+
// redelivers it. This is un-acked TRIGGER mail, distinct from the
|
|
1387
|
+
// deployment's in-flight run state (reconstructed sidecar-locally); the
|
|
1388
|
+
// "no disconnect queue" note above is about that run state, not this.
|
|
1389
|
+
retainPendingMailForAddress(addr);
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
if (conn.identity.kind === "allocated") {
|
|
1393
|
+
const current = allocatedConnections.get(conn.identity.allocationId);
|
|
1394
|
+
if (current?.ws === ws) {
|
|
1395
|
+
allocatedConnections.delete(conn.identity.allocationId);
|
|
1396
|
+
allocated = {
|
|
1397
|
+
allocationId: conn.identity.allocationId,
|
|
1398
|
+
generation: conn.identity.generation,
|
|
1399
|
+
};
|
|
888
1400
|
}
|
|
889
1401
|
}
|
|
890
1402
|
connections.delete(ws);
|
|
@@ -910,6 +1422,16 @@ export function createSidecarRouter(config) {
|
|
|
910
1422
|
pending.delete(requestId);
|
|
911
1423
|
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
912
1424
|
}
|
|
1425
|
+
// Reject every deploy issued on this socket, including allocated
|
|
1426
|
+
// workflow deployments stored in `workflowAddresses` rather than
|
|
1427
|
+
// `agentAddresses`.
|
|
1428
|
+
for (const [agentAddress, req] of pendingDeploys) {
|
|
1429
|
+
if (req.ws !== ws)
|
|
1430
|
+
continue;
|
|
1431
|
+
clearTimeout(req.timer);
|
|
1432
|
+
pendingDeploys.delete(agentAddress);
|
|
1433
|
+
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1434
|
+
}
|
|
913
1435
|
// Reject any in-flight pack transfers for this sidecar.
|
|
914
1436
|
for (const [transferId, pack] of pendingPacks) {
|
|
915
1437
|
if (pack.ws !== ws)
|
|
@@ -926,6 +1448,17 @@ export function createSidecarRouter(config) {
|
|
|
926
1448
|
pendingUndeploys.delete(addr);
|
|
927
1449
|
req.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
928
1450
|
}
|
|
1451
|
+
// Reject any in-flight probes sent to this sidecar. A probe never enters
|
|
1452
|
+
// the address maps, so this ws-keyed sweep is its ONLY disconnect cleanup:
|
|
1453
|
+
// without it a probe whose sidecar drops mid-flight would hang until its
|
|
1454
|
+
// own timeout instead of failing fast on the disconnect.
|
|
1455
|
+
for (const [requestId, probe] of pendingProbes) {
|
|
1456
|
+
if (probe.ws !== ws)
|
|
1457
|
+
continue;
|
|
1458
|
+
clearTimeout(probe.timer);
|
|
1459
|
+
pendingProbes.delete(requestId);
|
|
1460
|
+
probe.reject(`Sidecar ${conn.sidecarId} disconnected`);
|
|
1461
|
+
}
|
|
929
1462
|
// Cancel any in-flight inbound pack transfers from this sidecar
|
|
930
1463
|
// across both receivers. The two receivers track their own in-
|
|
931
1464
|
// flight transferIds, so a pending workflow-run transfer for an
|
|
@@ -944,6 +1477,7 @@ export function createSidecarRouter(config) {
|
|
|
944
1477
|
}
|
|
945
1478
|
events.emit("sidecar.disconnect", {
|
|
946
1479
|
ownedAddresses: [...owned],
|
|
1480
|
+
...(allocated !== undefined ? { allocated } : {}),
|
|
947
1481
|
});
|
|
948
1482
|
logger.info `Sidecar ${conn.sidecarId} disconnected`;
|
|
949
1483
|
}
|
|
@@ -994,40 +1528,78 @@ export function createSidecarRouter(config) {
|
|
|
994
1528
|
pending.delete(requestId);
|
|
995
1529
|
req.reject(error);
|
|
996
1530
|
}
|
|
997
|
-
function
|
|
998
|
-
|
|
1531
|
+
function packResponseMatches(entry, ws, frame) {
|
|
1532
|
+
return (entry.ws === ws &&
|
|
1533
|
+
entry.agentAddress === frame.agentAddress &&
|
|
1534
|
+
entry.repoId.kind === frame.repoId.kind &&
|
|
1535
|
+
entry.repoId.id === frame.repoId.id);
|
|
1536
|
+
}
|
|
1537
|
+
function resolvePackPending(ws, frame) {
|
|
1538
|
+
const entry = pendingPacks.get(frame.transferId);
|
|
999
1539
|
if (entry === undefined)
|
|
1000
1540
|
return;
|
|
1541
|
+
if (!packResponseMatches(entry, ws, frame)) {
|
|
1542
|
+
logger.warn `Ignoring repo.pack.ack for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
|
|
1543
|
+
return;
|
|
1544
|
+
}
|
|
1001
1545
|
clearTimeout(entry.timer);
|
|
1002
|
-
pendingPacks.delete(transferId);
|
|
1546
|
+
pendingPacks.delete(frame.transferId);
|
|
1003
1547
|
entry.resolve();
|
|
1004
1548
|
}
|
|
1005
|
-
function rejectPackPending(
|
|
1006
|
-
const entry = pendingPacks.get(transferId);
|
|
1549
|
+
function rejectPackPending(ws, frame) {
|
|
1550
|
+
const entry = pendingPacks.get(frame.transferId);
|
|
1007
1551
|
if (entry === undefined)
|
|
1008
1552
|
return;
|
|
1553
|
+
if (!packResponseMatches(entry, ws, frame)) {
|
|
1554
|
+
logger.warn `Ignoring repo.pack.reject for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
|
|
1555
|
+
return;
|
|
1556
|
+
}
|
|
1009
1557
|
clearTimeout(entry.timer);
|
|
1010
|
-
pendingPacks.delete(transferId);
|
|
1011
|
-
|
|
1558
|
+
pendingPacks.delete(frame.transferId);
|
|
1559
|
+
// Surface the receiver's specific cause when it carried one, so the awaiting
|
|
1560
|
+
// push sees "corrupt: <detail>" rather than only the coarse reason.
|
|
1561
|
+
entry.reject(frame.detail !== undefined
|
|
1562
|
+
? `${frame.reason}: ${frame.detail}`
|
|
1563
|
+
: frame.reason);
|
|
1012
1564
|
}
|
|
1013
|
-
function resolveUndeployPending(agentAddress) {
|
|
1565
|
+
function resolveUndeployPending(ws, agentAddress) {
|
|
1014
1566
|
const req = pendingUndeploys.get(agentAddress);
|
|
1015
1567
|
if (req === undefined) {
|
|
1016
1568
|
logger.warn `Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`;
|
|
1017
1569
|
return;
|
|
1018
1570
|
}
|
|
1571
|
+
if (req.ws !== ws)
|
|
1572
|
+
return;
|
|
1019
1573
|
clearTimeout(req.timer);
|
|
1020
1574
|
pendingUndeploys.delete(agentAddress);
|
|
1021
1575
|
req.resolve();
|
|
1022
1576
|
}
|
|
1023
|
-
function rejectUndeployPending(agentAddress, error) {
|
|
1577
|
+
function rejectUndeployPending(ws, agentAddress, error) {
|
|
1024
1578
|
const req = pendingUndeploys.get(agentAddress);
|
|
1025
1579
|
if (req === undefined)
|
|
1026
1580
|
return;
|
|
1581
|
+
if (req.ws !== ws)
|
|
1582
|
+
return;
|
|
1027
1583
|
clearTimeout(req.timer);
|
|
1028
1584
|
pendingUndeploys.delete(agentAddress);
|
|
1029
1585
|
req.reject(error);
|
|
1030
1586
|
}
|
|
1587
|
+
function resolveProbe(requestId, result) {
|
|
1588
|
+
const req = pendingProbes.get(requestId);
|
|
1589
|
+
if (req === undefined)
|
|
1590
|
+
return;
|
|
1591
|
+
clearTimeout(req.timer);
|
|
1592
|
+
pendingProbes.delete(requestId);
|
|
1593
|
+
req.resolve(result);
|
|
1594
|
+
}
|
|
1595
|
+
function rejectProbe(requestId, error) {
|
|
1596
|
+
const req = pendingProbes.get(requestId);
|
|
1597
|
+
if (req === undefined)
|
|
1598
|
+
return;
|
|
1599
|
+
clearTimeout(req.timer);
|
|
1600
|
+
pendingProbes.delete(requestId);
|
|
1601
|
+
req.reject(error);
|
|
1602
|
+
}
|
|
1031
1603
|
// Routing rule: pick the receiver dedicated to the repoId.kind the
|
|
1032
1604
|
// frame carries. The receivers' in-flight state is independent, so a
|
|
1033
1605
|
// workflow-run transferId can never collide with or evict an
|
|
@@ -1048,6 +1620,8 @@ export function createSidecarRouter(config) {
|
|
|
1048
1620
|
function pickReceivePackLookup(repoId) {
|
|
1049
1621
|
switch (repoId.kind) {
|
|
1050
1622
|
case "agent-state":
|
|
1623
|
+
// The agent-state lookup ignores the `source` argument the workflow-run
|
|
1624
|
+
// lookup takes; the two are otherwise the same contract.
|
|
1051
1625
|
return lookups.receiveAgentStatePack;
|
|
1052
1626
|
case "workflow-run":
|
|
1053
1627
|
return lookups.receiveWorkflowRunPack;
|
|
@@ -1063,6 +1637,17 @@ export function createSidecarRouter(config) {
|
|
|
1063
1637
|
logger.warn `Received repo.pack.push for unrouted agent ${frame.agentAddress}`;
|
|
1064
1638
|
return;
|
|
1065
1639
|
}
|
|
1640
|
+
if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
|
|
1641
|
+
logger.warn `Rejected repo.pack.push outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
|
|
1642
|
+
conn.send({
|
|
1643
|
+
type: "repo.pack.reject",
|
|
1644
|
+
agentAddress: frame.agentAddress,
|
|
1645
|
+
repoId: frame.repoId,
|
|
1646
|
+
transferId: frame.transferId,
|
|
1647
|
+
reason: "path_violation",
|
|
1648
|
+
});
|
|
1649
|
+
return;
|
|
1650
|
+
}
|
|
1066
1651
|
const picked = pickPackReceiver(frame.repoId);
|
|
1067
1652
|
if (picked === null) {
|
|
1068
1653
|
logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
@@ -1094,6 +1679,17 @@ export function createSidecarRouter(config) {
|
|
|
1094
1679
|
logger.warn `Received repo.pack.done for unrouted agent ${frame.agentAddress}`;
|
|
1095
1680
|
return;
|
|
1096
1681
|
}
|
|
1682
|
+
if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
|
|
1683
|
+
logger.warn `Rejected repo.pack.done outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
|
|
1684
|
+
conn.send({
|
|
1685
|
+
type: "repo.pack.reject",
|
|
1686
|
+
agentAddress: frame.agentAddress,
|
|
1687
|
+
repoId: frame.repoId,
|
|
1688
|
+
transferId: frame.transferId,
|
|
1689
|
+
reason: "path_violation",
|
|
1690
|
+
});
|
|
1691
|
+
return;
|
|
1692
|
+
}
|
|
1097
1693
|
const picked = pickPackReceiver(frame.repoId);
|
|
1098
1694
|
if (picked === null) {
|
|
1099
1695
|
logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
|
|
@@ -1127,7 +1723,15 @@ export function createSidecarRouter(config) {
|
|
|
1127
1723
|
});
|
|
1128
1724
|
return;
|
|
1129
1725
|
}
|
|
1130
|
-
const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha
|
|
1726
|
+
const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, conn.identity.kind === "allocated"
|
|
1727
|
+
? {
|
|
1728
|
+
kind: "allocated",
|
|
1729
|
+
agentAddress: frame.agentAddress,
|
|
1730
|
+
allocationId: conn.identity.allocationId,
|
|
1731
|
+
anchorRunId: conn.identity.anchorRunId,
|
|
1732
|
+
generation: conn.identity.generation,
|
|
1733
|
+
}
|
|
1734
|
+
: { kind: "shared", agentAddress: frame.agentAddress });
|
|
1131
1735
|
// Connection may have closed during async verification.
|
|
1132
1736
|
const currentConn = connections.get(ws);
|
|
1133
1737
|
if (currentConn === undefined)
|
|
@@ -1164,6 +1768,110 @@ export function createSidecarRouter(config) {
|
|
|
1164
1768
|
* never persisted into the reconnect set and never resurrected on
|
|
1165
1769
|
* reconnect.
|
|
1166
1770
|
*/
|
|
1771
|
+
function fenceAllocation(allocationId, generation) {
|
|
1772
|
+
const existing = allocationFences.get(allocationId);
|
|
1773
|
+
if (existing !== undefined && generation < existing) {
|
|
1774
|
+
throw new Error(`Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`);
|
|
1775
|
+
}
|
|
1776
|
+
allocationFences.set(allocationId, generation);
|
|
1777
|
+
const current = allocatedConnections.get(allocationId);
|
|
1778
|
+
if (current !== undefined && current.identity.generation !== generation) {
|
|
1779
|
+
handleClose(current.ws);
|
|
1780
|
+
current.ws.close();
|
|
1781
|
+
}
|
|
1782
|
+
const waiters = allocationWaiters.get(allocationId);
|
|
1783
|
+
if (waiters === undefined)
|
|
1784
|
+
return;
|
|
1785
|
+
for (const waiter of [...waiters]) {
|
|
1786
|
+
if (waiter.generation === generation)
|
|
1787
|
+
continue;
|
|
1788
|
+
clearTimeout(waiter.timer);
|
|
1789
|
+
waiters.delete(waiter);
|
|
1790
|
+
waiter.reject(new Error(`Allocation ${allocationId} advanced to generation ${String(generation)}`));
|
|
1791
|
+
}
|
|
1792
|
+
if (waiters.size === 0)
|
|
1793
|
+
allocationWaiters.delete(allocationId);
|
|
1794
|
+
}
|
|
1795
|
+
async function getAllocatedConnection(target, use) {
|
|
1796
|
+
if (allocationFences.get(target.allocationId) !== target.generation) {
|
|
1797
|
+
throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
|
|
1798
|
+
}
|
|
1799
|
+
const current = allocatedConnections.get(target.allocationId);
|
|
1800
|
+
if (current === undefined ||
|
|
1801
|
+
current.identity.generation !== target.generation) {
|
|
1802
|
+
throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`);
|
|
1803
|
+
}
|
|
1804
|
+
if (!(await validateSidecarIdentity(current.identity, use))) {
|
|
1805
|
+
if (allocatedConnections.get(target.allocationId) === current) {
|
|
1806
|
+
handleClose(current.ws);
|
|
1807
|
+
current.ws.close();
|
|
1808
|
+
}
|
|
1809
|
+
throw new Error(`Allocated sidecar identity is no longer current for allocation ${target.allocationId}`);
|
|
1810
|
+
}
|
|
1811
|
+
if (allocatedConnections.get(target.allocationId) !== current) {
|
|
1812
|
+
throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
|
|
1813
|
+
}
|
|
1814
|
+
const conn = connections.get(current.ws);
|
|
1815
|
+
if (conn === undefined ||
|
|
1816
|
+
conn.identity.kind !== "allocated" ||
|
|
1817
|
+
conn.identity.allocationId !== target.allocationId ||
|
|
1818
|
+
conn.identity.generation !== target.generation) {
|
|
1819
|
+
throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId}`);
|
|
1820
|
+
}
|
|
1821
|
+
return { ws: current.ws, conn };
|
|
1822
|
+
}
|
|
1823
|
+
async function isAllocatedSidecarReady(target) {
|
|
1824
|
+
try {
|
|
1825
|
+
await getAllocatedConnection(target, "readiness");
|
|
1826
|
+
return true;
|
|
1827
|
+
}
|
|
1828
|
+
catch {
|
|
1829
|
+
return false;
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
async function isAllocatedWorkflowActive(target) {
|
|
1833
|
+
try {
|
|
1834
|
+
const { conn } = await getAllocatedConnection(target, "readiness");
|
|
1835
|
+
if (conn.identity.kind !== "allocated")
|
|
1836
|
+
return false;
|
|
1837
|
+
return conn.workflowAddresses.has(conn.identity.workflowRunAddress);
|
|
1838
|
+
}
|
|
1839
|
+
catch {
|
|
1840
|
+
return false;
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
async function waitForAllocatedSidecar(target, timeoutMs) {
|
|
1844
|
+
if (await isAllocatedSidecarReady(target))
|
|
1845
|
+
return;
|
|
1846
|
+
if (allocationFences.get(target.allocationId) !== target.generation) {
|
|
1847
|
+
throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
|
|
1848
|
+
}
|
|
1849
|
+
if (timeoutMs <= 0) {
|
|
1850
|
+
throw new Error(`Timed out waiting for allocated sidecar ${target.allocationId}`);
|
|
1851
|
+
}
|
|
1852
|
+
await new Promise((resolve, reject) => {
|
|
1853
|
+
const waiter = {
|
|
1854
|
+
generation: target.generation,
|
|
1855
|
+
resolve,
|
|
1856
|
+
reject,
|
|
1857
|
+
timer: setTimeout(() => {
|
|
1858
|
+
const current = allocationWaiters.get(target.allocationId);
|
|
1859
|
+
current?.delete(waiter);
|
|
1860
|
+
if (current?.size === 0) {
|
|
1861
|
+
allocationWaiters.delete(target.allocationId);
|
|
1862
|
+
}
|
|
1863
|
+
reject(new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`));
|
|
1864
|
+
}, timeoutMs),
|
|
1865
|
+
};
|
|
1866
|
+
let waiters = allocationWaiters.get(target.allocationId);
|
|
1867
|
+
if (waiters === undefined) {
|
|
1868
|
+
waiters = new Set();
|
|
1869
|
+
allocationWaiters.set(target.allocationId, waiters);
|
|
1870
|
+
}
|
|
1871
|
+
waiters.add(waiter);
|
|
1872
|
+
void notifyAllocationWaiters(target.allocationId);
|
|
1873
|
+
});
|
|
1874
|
+
}
|
|
1167
1875
|
function bindStepRoute(stepAddress) {
|
|
1168
1876
|
const ws = addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress);
|
|
1169
1877
|
if (ws === undefined) {
|
|
@@ -1173,6 +1881,18 @@ export function createSidecarRouter(config) {
|
|
|
1173
1881
|
if (conn === undefined) {
|
|
1174
1882
|
throw new Error(`No sidecar connected to stage workflow step "${stepAddress}"`);
|
|
1175
1883
|
}
|
|
1884
|
+
if (conn.identity.kind !== "shared") {
|
|
1885
|
+
throw new Error(`Allocated sidecar ${conn.sidecarId} cannot receive ordinary workflow step ${stepAddress}`);
|
|
1886
|
+
}
|
|
1887
|
+
conn.workflowAddresses.add(stepAddress);
|
|
1888
|
+
addressIndex.set(stepAddress, ws);
|
|
1889
|
+
}
|
|
1890
|
+
async function bindAllocatedStepRoute(target, stepAddress) {
|
|
1891
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
1892
|
+
const existing = addressIndex.get(stepAddress);
|
|
1893
|
+
if (existing !== undefined && existing !== ws) {
|
|
1894
|
+
throw new Error(`Workflow step ${stepAddress} is already routed to another sidecar`);
|
|
1895
|
+
}
|
|
1176
1896
|
conn.workflowAddresses.add(stepAddress);
|
|
1177
1897
|
addressIndex.set(stepAddress, ws);
|
|
1178
1898
|
}
|
|
@@ -1191,6 +1911,17 @@ export function createSidecarRouter(config) {
|
|
|
1191
1911
|
}
|
|
1192
1912
|
addressIndex.delete(stepAddress);
|
|
1193
1913
|
}
|
|
1914
|
+
function unbindAllocatedStepRoute(target, stepAddress) {
|
|
1915
|
+
const current = allocatedConnections.get(target.allocationId);
|
|
1916
|
+
if (current === undefined ||
|
|
1917
|
+
current.identity.generation !== target.generation) {
|
|
1918
|
+
return;
|
|
1919
|
+
}
|
|
1920
|
+
if (addressIndex.get(stepAddress) !== current.ws)
|
|
1921
|
+
return;
|
|
1922
|
+
connections.get(current.ws)?.workflowAddresses.delete(stepAddress);
|
|
1923
|
+
addressIndex.delete(stepAddress);
|
|
1924
|
+
}
|
|
1194
1925
|
// Pack transfers may take longer than session requests due to data volume.
|
|
1195
1926
|
const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
|
|
1196
1927
|
function sendPack(agentAddress, pack, ref, commitSha, options) {
|
|
@@ -1202,6 +1933,12 @@ export function createSidecarRouter(config) {
|
|
|
1202
1933
|
if (conn === undefined) {
|
|
1203
1934
|
return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
|
|
1204
1935
|
}
|
|
1936
|
+
if (conn.identity.kind !== "shared") {
|
|
1937
|
+
return Promise.reject(new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound pack routing for "${agentAddress}"`));
|
|
1938
|
+
}
|
|
1939
|
+
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
|
|
1940
|
+
}
|
|
1941
|
+
function sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options) {
|
|
1205
1942
|
const transferId = `pack-${++packCounter}`;
|
|
1206
1943
|
// For the agent-state flow the destination agent and the source repo
|
|
1207
1944
|
// are the same entity, so `repoId.id === agentAddress`. Asset packs
|
|
@@ -1222,6 +1959,8 @@ export function createSidecarRouter(config) {
|
|
|
1222
1959
|
pendingPacks.set(transferId, {
|
|
1223
1960
|
transferId,
|
|
1224
1961
|
ws,
|
|
1962
|
+
agentAddress,
|
|
1963
|
+
repoId,
|
|
1225
1964
|
resolve,
|
|
1226
1965
|
reject(error) {
|
|
1227
1966
|
reject(new Error(`Pack rejected: ${error}`));
|
|
@@ -1251,65 +1990,167 @@ export function createSidecarRouter(config) {
|
|
|
1251
1990
|
});
|
|
1252
1991
|
});
|
|
1253
1992
|
}
|
|
1254
|
-
function
|
|
1993
|
+
async function sendPackToAllocation(target, agentAddress, pack, ref, commitSha, options) {
|
|
1994
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
1995
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
1996
|
+
throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
1997
|
+
}
|
|
1998
|
+
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
|
|
1999
|
+
}
|
|
2000
|
+
async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha) {
|
|
2001
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2002
|
+
if (conn.identity.kind !== "allocated") {
|
|
2003
|
+
throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
|
|
2004
|
+
}
|
|
2005
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
2006
|
+
throw new Error(`Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`);
|
|
2007
|
+
}
|
|
2008
|
+
if (conn.workflowAddresses.has(agentAddress)) {
|
|
2009
|
+
throw new Error(`Allocation ${target.allocationId} already hosts active workflow ${agentAddress}; refusing to overwrite its run history`);
|
|
2010
|
+
}
|
|
2011
|
+
return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, {
|
|
2012
|
+
repoId: {
|
|
2013
|
+
kind: "workflow-run",
|
|
2014
|
+
id: deriveWorkflowRunRepoId(agentAddress),
|
|
2015
|
+
},
|
|
2016
|
+
});
|
|
2017
|
+
}
|
|
2018
|
+
function routeMail(agentAddress, rawMessage, messageId, runGrants) {
|
|
2019
|
+
// Carry the hub-minted messageId on the frame so the sidecar's durable-
|
|
2020
|
+
// receipt ack (`mail.inbound.ack`) keys on the same id the hub tracks, and
|
|
2021
|
+
// a redelivery replays identical bytes for the downstream RunStarted dedup.
|
|
2022
|
+
// Optional: the workflow-trigger and session-conversation callers supply
|
|
2023
|
+
// it (they participate in the ack/retry handshake); a caller without a
|
|
2024
|
+
// hub-minted id omits it and the delivery is not tracked for redelivery.
|
|
2025
|
+
const frame = {
|
|
2026
|
+
type: "mail.inbound",
|
|
2027
|
+
agentAddress,
|
|
2028
|
+
rawMessage,
|
|
2029
|
+
...(messageId !== undefined ? { messageId } : {}),
|
|
2030
|
+
};
|
|
1255
2031
|
const ws = addressIndex.get(agentAddress);
|
|
1256
2032
|
if (ws !== undefined) {
|
|
1257
2033
|
const conn = connections.get(ws);
|
|
1258
2034
|
if (conn !== undefined) {
|
|
1259
|
-
conn.send(
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
2035
|
+
conn.send(frame);
|
|
2036
|
+
// Track the delivery for redelivery until the sidecar acks its durable
|
|
2037
|
+
// inbox write. Only mail carrying a hub-minted messageId participates
|
|
2038
|
+
// in the ack handshake; relayed agent-to-agent mail omits it and is
|
|
2039
|
+
// delivered fire-and-forget as before. A mail that triggered a workflow
|
|
2040
|
+
// run carries the run's grants so redelivery can replay them ahead of
|
|
2041
|
+
// the mail.
|
|
2042
|
+
if (messageId !== undefined) {
|
|
2043
|
+
trackPendingMail(agentAddress, messageId, frame, runGrants);
|
|
2044
|
+
}
|
|
1264
2045
|
return true;
|
|
1265
2046
|
}
|
|
1266
2047
|
}
|
|
1267
2048
|
// If the agent recently disconnected, queue for delivery on reconnect.
|
|
1268
|
-
const frame = { type: "mail.inbound", agentAddress, rawMessage };
|
|
1269
2049
|
return enqueueForDisconnected(agentAddress, frame);
|
|
1270
2050
|
}
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
2051
|
+
function sendRunGrants(agentAddress, runId, stepGrants) {
|
|
2052
|
+
const frame = {
|
|
2053
|
+
type: "run.grants",
|
|
2054
|
+
agentAddress,
|
|
2055
|
+
runId,
|
|
2056
|
+
stepGrants,
|
|
2057
|
+
};
|
|
2058
|
+
const ws = addressIndex.get(agentAddress);
|
|
2059
|
+
if (ws !== undefined) {
|
|
2060
|
+
const conn = connections.get(ws);
|
|
2061
|
+
if (conn !== undefined) {
|
|
2062
|
+
conn.send(frame);
|
|
2063
|
+
return true;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
// Mirror routeMail: if the address has a live disconnect queue, ride it
|
|
2067
|
+
// so a run.grants issued in the window between deploy and the first
|
|
2068
|
+
// reconnect survives the same way the dispatching trigger mail does.
|
|
2069
|
+
// A queue exists only while the deployment address is still on
|
|
2070
|
+
// agentAddresses (pre-first-reconnect); after a challenged reconnect it
|
|
2071
|
+
// moves to workflowAddresses, which handleClose leaves unqueued because
|
|
2072
|
+
// that generation's in-flight run state is reconstructed sidecar-locally.
|
|
2073
|
+
// Returning without enqueueing there is correct; enqueueing is what keeps
|
|
2074
|
+
// grants and mail from diverging in the pre-reconnect window.
|
|
2075
|
+
return enqueueForDisconnected(agentAddress, frame);
|
|
2076
|
+
}
|
|
2077
|
+
async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, messageId) {
|
|
2078
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2079
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
2080
|
+
throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2081
|
+
}
|
|
2082
|
+
const runGrants = { runId, stepGrants };
|
|
2083
|
+
conn.send({
|
|
2084
|
+
type: "run.grants",
|
|
2085
|
+
agentAddress,
|
|
2086
|
+
runId,
|
|
2087
|
+
stepGrants,
|
|
2088
|
+
});
|
|
2089
|
+
const frame = {
|
|
2090
|
+
type: "mail.inbound",
|
|
2091
|
+
agentAddress,
|
|
2092
|
+
rawMessage,
|
|
2093
|
+
messageId,
|
|
2094
|
+
};
|
|
2095
|
+
conn.send(frame);
|
|
2096
|
+
trackPendingMail(agentAddress, messageId, frame, runGrants, target);
|
|
2097
|
+
}
|
|
2098
|
+
async function handleDeployAck(ws, frame) {
|
|
2099
|
+
const req = pendingDeploys.get(frame.agentAddress);
|
|
2100
|
+
if (req === undefined) {
|
|
2101
|
+
logger.warn `Received agent.deploy.ack for "${frame.agentAddress}" with no pending deploy`;
|
|
1274
2102
|
return;
|
|
1275
2103
|
}
|
|
2104
|
+
if (req.ws !== ws)
|
|
2105
|
+
return;
|
|
1276
2106
|
if (events.listenerCount("agent.deploy.ack") > 0) {
|
|
1277
2107
|
try {
|
|
2108
|
+
const identity = connections.get(ws)?.identity;
|
|
1278
2109
|
await events.emitAndAwait("agent.deploy.ack", {
|
|
1279
|
-
agentAddress,
|
|
1280
|
-
publicKey,
|
|
2110
|
+
agentAddress: frame.agentAddress,
|
|
2111
|
+
publicKey: frame.publicKey,
|
|
2112
|
+
...(identity?.kind === "allocated"
|
|
2113
|
+
? {
|
|
2114
|
+
allocated: {
|
|
2115
|
+
allocationId: identity.allocationId,
|
|
2116
|
+
anchorRunId: identity.anchorRunId,
|
|
2117
|
+
generation: identity.generation,
|
|
2118
|
+
},
|
|
2119
|
+
}
|
|
2120
|
+
: {}),
|
|
1281
2121
|
});
|
|
1282
2122
|
}
|
|
1283
2123
|
catch (err) {
|
|
1284
|
-
rejectDeployPending(
|
|
2124
|
+
rejectDeployPending(req, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
|
|
1285
2125
|
return;
|
|
1286
2126
|
}
|
|
1287
2127
|
}
|
|
1288
|
-
resolveDeployPending(
|
|
2128
|
+
resolveDeployPending(req, frame.publicKey);
|
|
1289
2129
|
}
|
|
1290
|
-
function resolveDeployPending(
|
|
1291
|
-
|
|
1292
|
-
if (req === undefined)
|
|
2130
|
+
function resolveDeployPending(req, publicKey) {
|
|
2131
|
+
if (pendingDeploys.get(req.agentAddress) !== req)
|
|
1293
2132
|
return;
|
|
1294
2133
|
clearTimeout(req.timer);
|
|
1295
|
-
pendingDeploys.delete(agentAddress);
|
|
1296
|
-
req.resolve();
|
|
2134
|
+
pendingDeploys.delete(req.agentAddress);
|
|
2135
|
+
req.resolve(publicKey);
|
|
1297
2136
|
}
|
|
1298
|
-
function rejectDeployPending(
|
|
1299
|
-
|
|
1300
|
-
if (req === undefined)
|
|
2137
|
+
function rejectDeployPending(req, error) {
|
|
2138
|
+
if (pendingDeploys.get(req.agentAddress) !== req)
|
|
1301
2139
|
return;
|
|
1302
2140
|
clearTimeout(req.timer);
|
|
1303
|
-
pendingDeploys.delete(agentAddress);
|
|
2141
|
+
pendingDeploys.delete(req.agentAddress);
|
|
1304
2142
|
req.reject(error);
|
|
1305
2143
|
}
|
|
2144
|
+
function rejectDeployPendingFromFrame(ws, agentAddress, error) {
|
|
2145
|
+
const req = pendingDeploys.get(agentAddress);
|
|
2146
|
+
if (req === undefined || req.ws !== ws)
|
|
2147
|
+
return;
|
|
2148
|
+
rejectDeployPending(req, error);
|
|
2149
|
+
}
|
|
1306
2150
|
async function sendAgentDeploy(agentAddress, harnessConfig, workflow) {
|
|
1307
2151
|
if (hubPublicKeyHex === undefined) {
|
|
1308
2152
|
throw new Error("Hub signing key is required for agent deployment");
|
|
1309
2153
|
}
|
|
1310
|
-
if (pendingDeploys.has(agentAddress)) {
|
|
1311
|
-
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
1312
|
-
}
|
|
1313
2154
|
const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
|
|
1314
2155
|
if (ws === undefined) {
|
|
1315
2156
|
throw new Error(`No sidecar available for agent "${agentAddress}"`);
|
|
@@ -1318,26 +2159,28 @@ export function createSidecarRouter(config) {
|
|
|
1318
2159
|
if (conn === undefined) {
|
|
1319
2160
|
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1320
2161
|
}
|
|
1321
|
-
conn.
|
|
2162
|
+
if (conn.identity.kind !== "shared") {
|
|
2163
|
+
throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound deploy routing`);
|
|
2164
|
+
}
|
|
2165
|
+
return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
|
|
2166
|
+
}
|
|
2167
|
+
function sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow) {
|
|
2168
|
+
if (hubPublicKeyHex === undefined) {
|
|
2169
|
+
throw new Error("Hub signing key is required for agent deployment");
|
|
2170
|
+
}
|
|
2171
|
+
if (pendingDeploys.has(agentAddress)) {
|
|
2172
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
2173
|
+
}
|
|
2174
|
+
const addressSet = conn.identity.kind === "allocated"
|
|
2175
|
+
? conn.workflowAddresses
|
|
2176
|
+
: conn.agentAddresses;
|
|
2177
|
+
addressSet.add(agentAddress);
|
|
1322
2178
|
addressIndex.set(agentAddress, ws);
|
|
1323
2179
|
return new Promise((resolve, reject) => {
|
|
1324
|
-
// The deploy-ack handler does not currently thread the
|
|
1325
|
-
// sidecar-reported public key back through the pending-deploy
|
|
1326
|
-
// resolver; it only fires `agent.deploy.ack` listeners and then
|
|
1327
|
-
// resolves the pending deploy. Capture the key via a one-shot
|
|
1328
|
-
// listener so the return value carries it without a wire-shape
|
|
1329
|
-
// change to `agent.deploy.ack`.
|
|
1330
|
-
let capturedPublicKey;
|
|
1331
|
-
const detachListener = events.on("agent.deploy.ack", (payload) => {
|
|
1332
|
-
if (payload.agentAddress === agentAddress) {
|
|
1333
|
-
capturedPublicKey = payload.publicKey;
|
|
1334
|
-
}
|
|
1335
|
-
});
|
|
1336
2180
|
const timer = setTimeout(() => {
|
|
1337
|
-
detachListener();
|
|
1338
2181
|
pendingDeploys.delete(agentAddress);
|
|
1339
2182
|
if (addressIndex.get(agentAddress) === ws) {
|
|
1340
|
-
|
|
2183
|
+
addressSet.delete(agentAddress);
|
|
1341
2184
|
addressIndex.delete(agentAddress);
|
|
1342
2185
|
}
|
|
1343
2186
|
reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
|
|
@@ -1345,18 +2188,12 @@ export function createSidecarRouter(config) {
|
|
|
1345
2188
|
pendingDeploys.set(agentAddress, {
|
|
1346
2189
|
agentAddress,
|
|
1347
2190
|
ws,
|
|
1348
|
-
resolve() {
|
|
1349
|
-
|
|
1350
|
-
if (capturedPublicKey === undefined) {
|
|
1351
|
-
reject(new Error(`Deploy of "${agentAddress}" resolved without an agent.deploy.ack publicKey payload`));
|
|
1352
|
-
return;
|
|
1353
|
-
}
|
|
1354
|
-
resolve({ publicKey: capturedPublicKey });
|
|
2191
|
+
resolve(publicKey) {
|
|
2192
|
+
resolve({ publicKey });
|
|
1355
2193
|
},
|
|
1356
2194
|
reject(error) {
|
|
1357
|
-
detachListener();
|
|
1358
2195
|
if (addressIndex.get(agentAddress) === ws) {
|
|
1359
|
-
|
|
2196
|
+
addressSet.delete(agentAddress);
|
|
1360
2197
|
addressIndex.delete(agentAddress);
|
|
1361
2198
|
}
|
|
1362
2199
|
reject(new Error(error));
|
|
@@ -1373,6 +2210,20 @@ export function createSidecarRouter(config) {
|
|
|
1373
2210
|
});
|
|
1374
2211
|
});
|
|
1375
2212
|
}
|
|
2213
|
+
async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow) {
|
|
2214
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2215
|
+
if (conn.identity.kind !== "allocated") {
|
|
2216
|
+
throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
|
|
2217
|
+
}
|
|
2218
|
+
if (agentAddress !== conn.identity.workflowRunAddress) {
|
|
2219
|
+
throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
|
|
2220
|
+
}
|
|
2221
|
+
const existing = addressIndex.get(agentAddress);
|
|
2222
|
+
if (existing !== undefined && existing !== ws) {
|
|
2223
|
+
throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
|
|
2224
|
+
}
|
|
2225
|
+
return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
|
|
2226
|
+
}
|
|
1376
2227
|
/**
|
|
1377
2228
|
* Provision one step of a multi-step deploy on the sidecar WITHOUT
|
|
1378
2229
|
* spawning: the sidecar initializes the step's agent-state repo and
|
|
@@ -1388,12 +2239,6 @@ export function createSidecarRouter(config) {
|
|
|
1388
2239
|
* the caller owns tearing the route down via `unbindStepRoute`.
|
|
1389
2240
|
*/
|
|
1390
2241
|
function sendProvisionStep(agentAddress, harnessConfig) {
|
|
1391
|
-
if (hubPublicKeyHex === undefined) {
|
|
1392
|
-
throw new Error("Hub signing key is required for step provisioning");
|
|
1393
|
-
}
|
|
1394
|
-
if (pendingDeploys.has(agentAddress)) {
|
|
1395
|
-
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
1396
|
-
}
|
|
1397
2242
|
const ws = addressIndex.get(agentAddress);
|
|
1398
2243
|
if (ws === undefined) {
|
|
1399
2244
|
throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
|
|
@@ -1402,6 +2247,18 @@ export function createSidecarRouter(config) {
|
|
|
1402
2247
|
if (conn === undefined) {
|
|
1403
2248
|
throw new Error(`No sidecar connected for agent "${agentAddress}"`);
|
|
1404
2249
|
}
|
|
2250
|
+
if (conn.identity.kind !== "shared") {
|
|
2251
|
+
throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound step provisioning`);
|
|
2252
|
+
}
|
|
2253
|
+
return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
|
|
2254
|
+
}
|
|
2255
|
+
function sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig) {
|
|
2256
|
+
if (hubPublicKeyHex === undefined) {
|
|
2257
|
+
throw new Error("Hub signing key is required for step provisioning");
|
|
2258
|
+
}
|
|
2259
|
+
if (pendingDeploys.has(agentAddress)) {
|
|
2260
|
+
throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
|
|
2261
|
+
}
|
|
1405
2262
|
const hubKey = hubPublicKeyHex;
|
|
1406
2263
|
return new Promise((resolve, reject) => {
|
|
1407
2264
|
const timer = setTimeout(() => {
|
|
@@ -1415,7 +2272,7 @@ export function createSidecarRouter(config) {
|
|
|
1415
2272
|
pendingDeploys.set(agentAddress, {
|
|
1416
2273
|
agentAddress,
|
|
1417
2274
|
ws,
|
|
1418
|
-
resolve() {
|
|
2275
|
+
resolve(_publicKey) {
|
|
1419
2276
|
resolve();
|
|
1420
2277
|
},
|
|
1421
2278
|
reject(error) {
|
|
@@ -1433,11 +2290,58 @@ export function createSidecarRouter(config) {
|
|
|
1433
2290
|
});
|
|
1434
2291
|
});
|
|
1435
2292
|
}
|
|
2293
|
+
async function sendProvisionStepToAllocation(target, agentAddress, harnessConfig) {
|
|
2294
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2295
|
+
if (addressIndex.get(agentAddress) !== ws) {
|
|
2296
|
+
throw new Error(`Step route ${agentAddress} is not bound to allocation ${target.allocationId}`);
|
|
2297
|
+
}
|
|
2298
|
+
return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
|
|
2299
|
+
}
|
|
1436
2300
|
function findSidecarForNewAgent(_agentAddress) {
|
|
1437
|
-
const
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
2301
|
+
for (const [ws, conn] of connections) {
|
|
2302
|
+
if (conn.identity.kind === "shared")
|
|
2303
|
+
return ws;
|
|
2304
|
+
}
|
|
2305
|
+
return undefined;
|
|
2306
|
+
}
|
|
2307
|
+
function sendProbe(args) {
|
|
2308
|
+
// Select any connected sidecar, exactly as the pre-deploy window does.
|
|
2309
|
+
// A probe runs in the sidecar's pre-deploy state, so it targets a
|
|
2310
|
+
// connection, not an address -- `findSidecarForNewAgent` ignores its
|
|
2311
|
+
// argument and returns the first connection. Throw immediately on an empty
|
|
2312
|
+
// registry rather than register a probe that could only time out.
|
|
2313
|
+
const ws = findSidecarForNewAgent("");
|
|
2314
|
+
if (ws === undefined) {
|
|
2315
|
+
return Promise.reject(new Error("No sidecar available to probe workflow"));
|
|
2316
|
+
}
|
|
2317
|
+
const conn = connections.get(ws);
|
|
2318
|
+
if (conn === undefined) {
|
|
2319
|
+
return Promise.reject(new Error("No sidecar connected to probe workflow"));
|
|
2320
|
+
}
|
|
2321
|
+
const requestId = nextRequestId();
|
|
2322
|
+
return new Promise((resolve, reject) => {
|
|
2323
|
+
const timer = setTimeout(() => {
|
|
2324
|
+
pendingProbes.delete(requestId);
|
|
2325
|
+
reject(new Error(`Probe ${requestId} timed out after ${probeTimeoutMs}ms`));
|
|
2326
|
+
}, probeTimeoutMs);
|
|
2327
|
+
pendingProbes.set(requestId, {
|
|
2328
|
+
requestId,
|
|
2329
|
+
ws,
|
|
2330
|
+
resolve,
|
|
2331
|
+
reject(error) {
|
|
2332
|
+
reject(new Error(error));
|
|
2333
|
+
},
|
|
2334
|
+
timer,
|
|
2335
|
+
});
|
|
2336
|
+
conn.send({
|
|
2337
|
+
type: "workflow.probe.request",
|
|
2338
|
+
requestId,
|
|
2339
|
+
source: args.source,
|
|
2340
|
+
closure: args.closure,
|
|
2341
|
+
entry: args.entry,
|
|
2342
|
+
...(args.assets !== undefined ? { assets: args.assets } : {}),
|
|
2343
|
+
});
|
|
2344
|
+
});
|
|
1441
2345
|
}
|
|
1442
2346
|
function sendAgentUndeploy(agentAddress, reason) {
|
|
1443
2347
|
const ws = addressIndex.get(agentAddress);
|
|
@@ -1529,6 +2433,14 @@ export function createSidecarRouter(config) {
|
|
|
1529
2433
|
defaultSource,
|
|
1530
2434
|
}));
|
|
1531
2435
|
}
|
|
2436
|
+
async function sendCredentialsUpdate(agentAddress, delivery) {
|
|
2437
|
+
await sendRequest(agentAddress, (requestId) => ({
|
|
2438
|
+
type: "credentials.update",
|
|
2439
|
+
requestId,
|
|
2440
|
+
agentAddress,
|
|
2441
|
+
delivery,
|
|
2442
|
+
}));
|
|
2443
|
+
}
|
|
1532
2444
|
function sendSyncRequest(agentAddress) {
|
|
1533
2445
|
const ws = addressIndex.get(agentAddress);
|
|
1534
2446
|
if (ws === undefined) {
|
|
@@ -1563,6 +2475,13 @@ export function createSidecarRouter(config) {
|
|
|
1563
2475
|
payload: opts.payload,
|
|
1564
2476
|
});
|
|
1565
2477
|
}
|
|
2478
|
+
async function sendSignalDeliverToAllocation(target, opts) {
|
|
2479
|
+
const { ws, conn } = await getAllocatedConnection(target, "routing");
|
|
2480
|
+
if (addressIndex.get(opts.agentAddress) !== ws) {
|
|
2481
|
+
throw new Error(`Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`);
|
|
2482
|
+
}
|
|
2483
|
+
conn.send({ type: "signal.deliver", ...opts });
|
|
2484
|
+
}
|
|
1566
2485
|
function sendDrain(opts) {
|
|
1567
2486
|
const ws = addressIndex.get(opts.agentAddress);
|
|
1568
2487
|
if (ws === undefined) {
|
|
@@ -1583,15 +2502,30 @@ export function createSidecarRouter(config) {
|
|
|
1583
2502
|
handleMessage,
|
|
1584
2503
|
handleClose,
|
|
1585
2504
|
routeMail,
|
|
2505
|
+
sendRunGrants,
|
|
1586
2506
|
sendAgentDeploy,
|
|
2507
|
+
sendProbe,
|
|
1587
2508
|
sendAgentUndeploy,
|
|
1588
2509
|
sendSourcesUpdate,
|
|
2510
|
+
sendCredentialsUpdate,
|
|
1589
2511
|
sendPack,
|
|
2512
|
+
sendPackToAllocation,
|
|
2513
|
+
sendWorkflowRunPackToAllocation,
|
|
2514
|
+
fenceAllocation,
|
|
2515
|
+
waitForAllocatedSidecar,
|
|
2516
|
+
isAllocatedSidecarReady,
|
|
2517
|
+
isAllocatedWorkflowActive,
|
|
2518
|
+
sendAgentDeployToAllocation,
|
|
1590
2519
|
bindStepRoute,
|
|
2520
|
+
bindAllocatedStepRoute,
|
|
1591
2521
|
unbindStepRoute,
|
|
2522
|
+
unbindAllocatedStepRoute,
|
|
1592
2523
|
sendProvisionStep,
|
|
2524
|
+
sendProvisionStepToAllocation,
|
|
2525
|
+
sendWorkflowRunDispatchToAllocation,
|
|
1593
2526
|
sendSyncRequest,
|
|
1594
2527
|
sendSignalDeliver,
|
|
2528
|
+
sendSignalDeliverToAllocation,
|
|
1595
2529
|
sendDrain,
|
|
1596
2530
|
subscribeAgent,
|
|
1597
2531
|
dispatchAgentEvent: dispatchToSubscribers,
|