@masons/agent-network 0.5.29 → 0.5.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/broker/broker-daemon.d.ts +13 -3
- package/dist/broker/broker-daemon.d.ts.map +1 -1
- package/dist/broker/broker-daemon.js +294 -52
- package/dist/broker/closed-endpoint-lookup.d.ts +25 -0
- package/dist/broker/closed-endpoint-lookup.d.ts.map +1 -0
- package/dist/broker/closed-endpoint-lookup.js +59 -0
- package/dist/broker/control-event-dispatcher.d.ts +20 -0
- package/dist/broker/control-event-dispatcher.d.ts.map +1 -0
- package/dist/broker/control-event-dispatcher.js +45 -0
- package/dist/broker/control-event-types.d.ts +30 -0
- package/dist/broker/control-event-types.d.ts.map +1 -0
- package/dist/broker/control-event-types.js +1 -0
- package/dist/broker/endpoint-registry.d.ts +17 -1
- package/dist/broker/endpoint-registry.d.ts.map +1 -1
- package/dist/broker/endpoint-registry.js +39 -0
- package/dist/broker/endpoint-state-machine.d.ts +40 -0
- package/dist/broker/endpoint-state-machine.d.ts.map +1 -0
- package/dist/broker/endpoint-state-machine.js +92 -0
- package/dist/broker/entry.d.ts +3 -0
- package/dist/broker/entry.d.ts.map +1 -1
- package/dist/broker/entry.js +41 -3
- package/dist/broker/grace-timer.d.ts +9 -0
- package/dist/broker/grace-timer.d.ts.map +1 -0
- package/dist/broker/grace-timer.js +34 -0
- package/dist/broker/ipc-server.d.ts +8 -1
- package/dist/broker/ipc-server.d.ts.map +1 -1
- package/dist/broker/ipc-server.js +31 -0
- package/dist/broker/paths.d.ts +1 -0
- package/dist/broker/paths.d.ts.map +1 -1
- package/dist/broker/paths.js +2 -0
- package/dist/broker/plugin-liveness.d.ts +2 -0
- package/dist/broker/plugin-liveness.d.ts.map +1 -0
- package/dist/broker/plugin-liveness.js +15 -0
- package/dist/broker/reconnecting-buffer.d.ts +23 -0
- package/dist/broker/reconnecting-buffer.d.ts.map +1 -0
- package/dist/broker/reconnecting-buffer.js +107 -0
- package/dist/broker/routing-table.d.ts.map +1 -1
- package/dist/broker/routing-table.js +1 -1
- package/dist/broker/runtime-endpoint-port.d.ts +2 -1
- package/dist/broker/runtime-endpoint-port.d.ts.map +1 -1
- package/dist/broker/undispatched-inbox.d.ts +26 -0
- package/dist/broker/undispatched-inbox.d.ts.map +1 -0
- package/dist/broker/undispatched-inbox.js +53 -0
- package/dist/runtime-endpoint-client.d.ts +7 -0
- package/dist/runtime-endpoint-client.d.ts.map +1 -1
- package/dist/runtime-endpoint-client.js +11 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -1,24 +1,34 @@
|
|
|
1
1
|
import type { IncomingMessage } from "node:http";
|
|
2
|
-
import type {
|
|
2
|
+
import type { ClosedEndpointLookup } from "./closed-endpoint-lookup.js";
|
|
3
3
|
import type { ConnectorWS } from "./connector-ws.js";
|
|
4
|
+
import { type ControlEventDispatcher } from "./control-event-dispatcher.js";
|
|
5
|
+
import { type EndpointState } from "./endpoint-state-machine.js";
|
|
4
6
|
import type { BrokerLogger } from "./logger.js";
|
|
5
7
|
import type { BrokerPaths } from "./paths.js";
|
|
8
|
+
import { type BufferedMessage } from "./reconnecting-buffer.js";
|
|
6
9
|
import type { RuntimeEndpointPort } from "./runtime-endpoint-port.js";
|
|
10
|
+
import { type UndispatchedInbox } from "./undispatched-inbox.js";
|
|
7
11
|
export interface BrokerDaemonOptions {
|
|
8
12
|
paths: BrokerPaths;
|
|
9
13
|
asNodeId: string;
|
|
10
14
|
connector: ConnectorWS;
|
|
11
15
|
apiPort: RuntimeEndpointPort;
|
|
12
16
|
logger: BrokerLogger;
|
|
17
|
+
closedEndpointLookup?: ClosedEndpointLookup;
|
|
18
|
+
inboxCapacity?: number;
|
|
19
|
+
graceMs?: number;
|
|
20
|
+
livenessProbe?: (plugin_pid: number) => boolean;
|
|
13
21
|
}
|
|
14
22
|
export interface RunningBroker {
|
|
15
23
|
bearerToken: string;
|
|
16
24
|
ipcUrl: string;
|
|
17
25
|
shutdown(reason: string): Promise<void>;
|
|
18
26
|
registrySize(): number;
|
|
19
|
-
|
|
20
|
-
|
|
27
|
+
state(endpoint_id: string): EndpointState | undefined;
|
|
28
|
+
undispatchedInbox: UndispatchedInbox;
|
|
29
|
+
controlEventDispatcher: ControlEventDispatcher;
|
|
21
30
|
}
|
|
22
31
|
export declare function startBrokerDaemon(opts: BrokerDaemonOptions): Promise<RunningBroker>;
|
|
23
32
|
export declare function readPluginPidHeader(req: IncomingMessage): number | null;
|
|
33
|
+
export type { BufferedMessage };
|
|
24
34
|
//# sourceMappingURL=broker-daemon.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"broker-daemon.d.ts","sourceRoot":"","sources":["../../src/broker/broker-daemon.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"broker-daemon.d.ts","sourceRoot":"","sources":["../../src/broker/broker-daemon.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,KAAK,sBAAsB,EAE5B,MAAM,+BAA+B,CAAC;AAQvC,OAAO,EAEL,KAAK,aAAa,EAEnB,MAAM,6BAA6B,CAAC;AAarC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,OAAO,EACL,KAAK,eAAe,EAErB,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,mBAAmB,CAAC;IAC7B,MAAM,EAAE,YAAY,CAAC;IAGrB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAE5C,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,OAAO,CAAC,EAAE,MAAM,CAAC;IAGjB,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExC,YAAY,IAAI,MAAM,CAAC;IAGvB,KAAK,CAAC,WAAW,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAAC;IAEtD,iBAAiB,EAAE,iBAAiB,CAAC;IAErC,sBAAsB,EAAE,sBAAsB,CAAC;CAChD;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC,aAAa,CAAC,CAqnBxB;AAGD,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAMvE;AAED,YAAY,EAAE,eAAe,EAAE,CAAC"}
|
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { createControlEventDispatcher, } from "./control-event-dispatcher.js";
|
|
2
3
|
import { deleteDiscoveryFile, mintBearerToken, writeDiscoveryFile, } from "./discovery-file.js";
|
|
3
4
|
import { EndpointRegistry } from "./endpoint-registry.js";
|
|
5
|
+
import { DEFAULT_GRACE_MS, transition, } from "./endpoint-state-machine.js";
|
|
6
|
+
import { createGraceTimerManager } from "./grace-timer.js";
|
|
4
7
|
import { BrokerHttpError, pushToPlugin, startIPCServer, } from "./ipc-server.js";
|
|
8
|
+
import { isPluginAlive as defaultIsPluginAlive } from "./plugin-liveness.js";
|
|
9
|
+
import { createReconnectingBufferManager, } from "./reconnecting-buffer.js";
|
|
5
10
|
import { RoutingTable } from "./routing-table.js";
|
|
11
|
+
import { createUndispatchedInbox, } from "./undispatched-inbox.js";
|
|
6
12
|
export async function startBrokerDaemon(opts) {
|
|
7
|
-
const { paths, connector, apiPort, logger } = opts;
|
|
13
|
+
const { paths, asNodeId: _asNodeIdReservedForA3, connector, apiPort, logger, closedEndpointLookup, livenessProbe = defaultIsPluginAlive, } = opts;
|
|
14
|
+
const graceMs = opts.graceMs ?? DEFAULT_GRACE_MS;
|
|
15
|
+
void _asNodeIdReservedForA3;
|
|
8
16
|
logger.info("connecting_to_connector");
|
|
9
17
|
await connector.connect();
|
|
10
18
|
logger.info("connector_connected");
|
|
@@ -14,12 +22,142 @@ export async function startBrokerDaemon(opts) {
|
|
|
14
22
|
logger.warn("correlation_id_collision", { correlation_id, count });
|
|
15
23
|
},
|
|
16
24
|
});
|
|
25
|
+
const undispatchedInbox = createUndispatchedInbox({
|
|
26
|
+
capacity: opts.inboxCapacity,
|
|
27
|
+
});
|
|
28
|
+
const reconnectingBuffers = createReconnectingBufferManager(paths.buffersDir);
|
|
29
|
+
const graceTimers = createGraceTimerManager();
|
|
30
|
+
const controlEventDispatcher = createControlEventDispatcher();
|
|
17
31
|
const channels = new Map();
|
|
32
|
+
const wsEndpoints = new WeakMap();
|
|
33
|
+
const displacedWs = new WeakSet();
|
|
34
|
+
const emitTransition = (endpoint_id, state, reason) => {
|
|
35
|
+
void apiPort.transitionState(endpoint_id, {
|
|
36
|
+
state,
|
|
37
|
+
reason,
|
|
38
|
+
ts: new Date().toISOString(),
|
|
39
|
+
});
|
|
40
|
+
};
|
|
41
|
+
const handleTransition = (endpoint_id, event) => {
|
|
42
|
+
const entry = registry.get(endpoint_id);
|
|
43
|
+
if (!entry)
|
|
44
|
+
return;
|
|
45
|
+
const result = transition(entry.state, event, { graceMs });
|
|
46
|
+
if (result.ignored) {
|
|
47
|
+
logger.info("state_event_ignored", {
|
|
48
|
+
endpoint_id,
|
|
49
|
+
current: entry.state,
|
|
50
|
+
event,
|
|
51
|
+
});
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
for (const effect of result.effects) {
|
|
55
|
+
switch (effect.type) {
|
|
56
|
+
case "start_grace_timer": {
|
|
57
|
+
const deadline = new Date(Date.now() + effect.deadline_ms);
|
|
58
|
+
registry.markReconnecting(endpoint_id, deadline);
|
|
59
|
+
graceTimers.start(endpoint_id, effect.deadline_ms, () => {
|
|
60
|
+
handleTransition(endpoint_id, { type: "grace_expired" });
|
|
61
|
+
});
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
case "cancel_grace_timer":
|
|
65
|
+
graceTimers.cancel(endpoint_id);
|
|
66
|
+
break;
|
|
67
|
+
case "emit_state":
|
|
68
|
+
emitTransition(endpoint_id, effect.state, effect.reason);
|
|
69
|
+
break;
|
|
70
|
+
case "drain_buffer": {
|
|
71
|
+
const buf = reconnectingBuffers.forEndpoint(endpoint_id);
|
|
72
|
+
const drained = buf.drain();
|
|
73
|
+
const e = registry.get(endpoint_id);
|
|
74
|
+
if (e) {
|
|
75
|
+
for (const msg of drained) {
|
|
76
|
+
pushToPlugin(e.ipc_ws, {
|
|
77
|
+
event: "message_received",
|
|
78
|
+
from: msg.envelope.from,
|
|
79
|
+
content: msg.envelope.content,
|
|
80
|
+
contentType: msg.envelope.contentType,
|
|
81
|
+
metadata: msg.envelope.metadata,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
break;
|
|
86
|
+
}
|
|
87
|
+
case "delete_buffer":
|
|
88
|
+
reconnectingBuffers.cleanup(endpoint_id);
|
|
89
|
+
break;
|
|
90
|
+
case "remove_from_registry":
|
|
91
|
+
registry.unregister(endpoint_id);
|
|
92
|
+
void apiPort.unregister(endpoint_id).catch(() => {
|
|
93
|
+
});
|
|
94
|
+
break;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
controlEventDispatcher.on("dispatch_undispatched", async (event) => {
|
|
99
|
+
const taken = undispatchedInbox.take(event.undispatched_id);
|
|
100
|
+
if (!taken) {
|
|
101
|
+
return {
|
|
102
|
+
ok: false,
|
|
103
|
+
detail: `unknown undispatched_id: ${event.undispatched_id}`,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
const target = registry.get(event.target_endpoint_id);
|
|
107
|
+
if (!target) {
|
|
108
|
+
const readded = undispatchedInbox.tryAdd(taken);
|
|
109
|
+
if (!readded) {
|
|
110
|
+
logger.error("dispatch_reinsert_failed_at_capacity", {
|
|
111
|
+
undispatched_id: event.undispatched_id,
|
|
112
|
+
target_endpoint_id: event.target_endpoint_id,
|
|
113
|
+
inbox_size: undispatchedInbox.size(),
|
|
114
|
+
inbox_capacity: undispatchedInbox.capacity(),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
ok: false,
|
|
119
|
+
detail: `target endpoint not found: ${event.target_endpoint_id}`,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
const stamped = {
|
|
123
|
+
event: "message_received",
|
|
124
|
+
from: taken.sender_address,
|
|
125
|
+
content: taken.content,
|
|
126
|
+
contentType: taken.content_type,
|
|
127
|
+
metadata: {
|
|
128
|
+
...taken.metadata,
|
|
129
|
+
dispatched_from: event.undispatched_id,
|
|
130
|
+
dispatched_by: "passport",
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
if (target.state === "active") {
|
|
134
|
+
pushToPlugin(target.ipc_ws, stamped);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
const buf = reconnectingBuffers.forEndpoint(target.endpoint_id);
|
|
138
|
+
buf.append({
|
|
139
|
+
id: randomUUID(),
|
|
140
|
+
arrived_at: new Date().toISOString(),
|
|
141
|
+
envelope: {
|
|
142
|
+
from: taken.sender_address,
|
|
143
|
+
content: taken.content,
|
|
144
|
+
contentType: taken.content_type,
|
|
145
|
+
metadata: stamped.metadata,
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
logger.info("undispatched_dispatched", {
|
|
150
|
+
undispatched_id: event.undispatched_id,
|
|
151
|
+
target_endpoint_id: event.target_endpoint_id,
|
|
152
|
+
target_state: target.state,
|
|
153
|
+
});
|
|
154
|
+
return { ok: true };
|
|
155
|
+
});
|
|
18
156
|
const handlers = {
|
|
19
157
|
async registerEndpoint(body, _ipcWs) {
|
|
20
158
|
const boundWs = channels.get(body.plugin_pid);
|
|
21
159
|
if (!boundWs) {
|
|
22
|
-
throw new
|
|
160
|
+
throw new BrokerHttpError(400, "ipc_channel_missing", `no open IPC channel for plugin_pid=${body.plugin_pid}; ` +
|
|
23
161
|
"open WS /v1/stream with x-plugin-pid header first");
|
|
24
162
|
}
|
|
25
163
|
const apiResp = await apiPort.register({
|
|
@@ -46,6 +184,13 @@ export async function startBrokerDaemon(opts) {
|
|
|
46
184
|
task_hint: body.task_hint,
|
|
47
185
|
},
|
|
48
186
|
});
|
|
187
|
+
let bound = wsEndpoints.get(boundWs);
|
|
188
|
+
if (!bound) {
|
|
189
|
+
bound = new Set();
|
|
190
|
+
wsEndpoints.set(boundWs, bound);
|
|
191
|
+
}
|
|
192
|
+
bound.add(apiResp.endpoint_id);
|
|
193
|
+
emitTransition(apiResp.endpoint_id, "active", "register");
|
|
49
194
|
logger.info("endpoint_registered", {
|
|
50
195
|
endpoint_id: apiResp.endpoint_id,
|
|
51
196
|
plugin_pid: body.plugin_pid,
|
|
@@ -56,7 +201,7 @@ export async function startBrokerDaemon(opts) {
|
|
|
56
201
|
async heartbeatEndpoint(endpoint_id) {
|
|
57
202
|
const entry = registry.get(endpoint_id);
|
|
58
203
|
if (!entry) {
|
|
59
|
-
throw new
|
|
204
|
+
throw new BrokerHttpError(404, "endpoint_unknown", `unknown endpoint_id: ${endpoint_id}`);
|
|
60
205
|
}
|
|
61
206
|
await apiPort.heartbeat(endpoint_id);
|
|
62
207
|
},
|
|
@@ -64,22 +209,16 @@ export async function startBrokerDaemon(opts) {
|
|
|
64
209
|
const entry = registry.get(endpoint_id);
|
|
65
210
|
if (!entry)
|
|
66
211
|
return;
|
|
67
|
-
|
|
68
|
-
try {
|
|
69
|
-
await apiPort.unregister(endpoint_id);
|
|
70
|
-
}
|
|
71
|
-
catch (err) {
|
|
72
|
-
logger.warn("unregister_api_failed", {
|
|
73
|
-
endpoint_id,
|
|
74
|
-
err: err instanceof Error ? err.message : String(err),
|
|
75
|
-
});
|
|
76
|
-
}
|
|
212
|
+
handleTransition(endpoint_id, { type: "explicit_unregister" });
|
|
77
213
|
},
|
|
78
214
|
async send(body) {
|
|
79
215
|
const entry = registry.get(body.endpoint_id);
|
|
80
216
|
if (!entry) {
|
|
81
217
|
throw new BrokerHttpError(404, "unknown_endpoint", `unknown endpoint_id: ${body.endpoint_id}`);
|
|
82
218
|
}
|
|
219
|
+
if (entry.state !== "active") {
|
|
220
|
+
throw new BrokerHttpError(409, "endpoint_not_active", `endpoint ${body.endpoint_id} is ${entry.state}; outbound send is only allowed from Active`);
|
|
221
|
+
}
|
|
83
222
|
const incoming = body.metadata ?? {};
|
|
84
223
|
if ("correlation_id" in incoming) {
|
|
85
224
|
throw new BrokerHttpError(400, "correlation_id_forbidden", "broker is the sole issuer of correlation_id; do not pass it from the Plugin");
|
|
@@ -100,17 +239,64 @@ export async function startBrokerDaemon(opts) {
|
|
|
100
239
|
require_live: body.require_live ?? false,
|
|
101
240
|
};
|
|
102
241
|
const ack = await connector.send(body.to, body.content, body.contentType ?? "text", metadata);
|
|
103
|
-
return {
|
|
104
|
-
messageId: ack.messageId,
|
|
105
|
-
status: ack.status,
|
|
106
|
-
};
|
|
242
|
+
return { messageId: ack.messageId, status: ack.status };
|
|
107
243
|
},
|
|
108
|
-
async reattachEndpoint(endpoint_id, plugin_pid,
|
|
109
|
-
const
|
|
110
|
-
if (!
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
244
|
+
async reattachEndpoint(endpoint_id, plugin_pid, _ipcWsHint) {
|
|
245
|
+
const ipcWs = channels.get(plugin_pid);
|
|
246
|
+
if (!ipcWs) {
|
|
247
|
+
throw new BrokerHttpError(400, "ipc_channel_missing", `no open IPC channel for plugin_pid=${plugin_pid}; open WS /v1/stream first`);
|
|
248
|
+
}
|
|
249
|
+
const status = registry.getReattachStatus(endpoint_id, plugin_pid);
|
|
250
|
+
if (!status.ok) {
|
|
251
|
+
if (status.reason === "endpoint_unknown") {
|
|
252
|
+
throw new BrokerHttpError(404, "endpoint_unknown", `unknown endpoint_id: ${endpoint_id}`);
|
|
253
|
+
}
|
|
254
|
+
if (status.reason === "reattach_pid_mismatch") {
|
|
255
|
+
throw new BrokerHttpError(403, "reattach_pid_mismatch", "plugin_pid does not match the registered endpoint");
|
|
256
|
+
}
|
|
257
|
+
throw new BrokerHttpError(409, "endpoint_not_reconnecting", "endpoint is currently active; reattach is only valid from reconnecting");
|
|
258
|
+
}
|
|
259
|
+
let bound = wsEndpoints.get(ipcWs);
|
|
260
|
+
if (!bound) {
|
|
261
|
+
bound = new Set();
|
|
262
|
+
wsEndpoints.set(ipcWs, bound);
|
|
263
|
+
}
|
|
264
|
+
bound.add(endpoint_id);
|
|
265
|
+
const buf = reconnectingBuffers.forEndpoint(endpoint_id);
|
|
266
|
+
const drained = buf.drain();
|
|
267
|
+
graceTimers.cancel(endpoint_id);
|
|
268
|
+
registry.markActive(endpoint_id, ipcWs);
|
|
269
|
+
for (const msg of drained) {
|
|
270
|
+
pushToPlugin(ipcWs, {
|
|
271
|
+
event: "message_received",
|
|
272
|
+
from: msg.envelope.from,
|
|
273
|
+
content: msg.envelope.content,
|
|
274
|
+
contentType: msg.envelope.contentType,
|
|
275
|
+
metadata: msg.envelope.metadata,
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
emitTransition(endpoint_id, "active", "reattach");
|
|
279
|
+
logger.info("endpoint_reattached", {
|
|
280
|
+
endpoint_id,
|
|
281
|
+
plugin_pid,
|
|
282
|
+
reconnecting_buffer_count: drained.length,
|
|
283
|
+
});
|
|
284
|
+
return { restored: true, reconnecting_buffer_count: drained.length };
|
|
285
|
+
},
|
|
286
|
+
async listUndispatched() {
|
|
287
|
+
return undispatchedInbox.list();
|
|
288
|
+
},
|
|
289
|
+
async dispatch(undispatched_id, target_endpoint_id) {
|
|
290
|
+
const result = await controlEventDispatcher.dispatch({
|
|
291
|
+
type: "dispatch_undispatched",
|
|
292
|
+
version: 1,
|
|
293
|
+
idempotency_key: `local-${randomUUID()}`,
|
|
294
|
+
undispatched_id,
|
|
295
|
+
target_endpoint_id,
|
|
296
|
+
});
|
|
297
|
+
if (!result.ok) {
|
|
298
|
+
throw new BrokerHttpError(400, "dispatch_failed", result.detail ?? "unknown");
|
|
299
|
+
}
|
|
114
300
|
},
|
|
115
301
|
};
|
|
116
302
|
const bearerToken = mintBearerToken();
|
|
@@ -121,6 +307,7 @@ export async function startBrokerDaemon(opts) {
|
|
|
121
307
|
onChannelOpened: (plugin_pid, ws) => {
|
|
122
308
|
const prior = channels.get(plugin_pid);
|
|
123
309
|
if (prior && prior !== ws) {
|
|
310
|
+
displacedWs.add(prior);
|
|
124
311
|
try {
|
|
125
312
|
prior.close(1000, "replaced");
|
|
126
313
|
}
|
|
@@ -133,29 +320,94 @@ export async function startBrokerDaemon(opts) {
|
|
|
133
320
|
if (channels.get(plugin_pid) === ws) {
|
|
134
321
|
channels.delete(plugin_pid);
|
|
135
322
|
}
|
|
323
|
+
if (displacedWs.has(ws))
|
|
324
|
+
return;
|
|
325
|
+
const bound = wsEndpoints.get(ws);
|
|
326
|
+
if (!bound)
|
|
327
|
+
return;
|
|
328
|
+
const alive = livenessProbe(plugin_pid);
|
|
329
|
+
for (const endpoint_id of bound) {
|
|
330
|
+
handleTransition(endpoint_id, {
|
|
331
|
+
type: "ipc_close",
|
|
332
|
+
plugin_alive: alive,
|
|
333
|
+
});
|
|
334
|
+
}
|
|
136
335
|
},
|
|
137
336
|
});
|
|
138
|
-
connector.on("message_received", ({ payload }) => {
|
|
337
|
+
connector.on("message_received", async ({ payload }) => {
|
|
139
338
|
const meta = payload.metadata ?? {};
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
? meta.correlation_id
|
|
146
|
-
: undefined,
|
|
147
|
-
});
|
|
339
|
+
const target_endpoint_id = typeof meta.target_endpoint_id === "string"
|
|
340
|
+
? meta.target_endpoint_id
|
|
341
|
+
: undefined;
|
|
342
|
+
const correlation_id = typeof meta.correlation_id === "string" ? meta.correlation_id : undefined;
|
|
343
|
+
const decision = router.route({ target_endpoint_id, correlation_id });
|
|
148
344
|
if (decision.kind === "endpoint") {
|
|
149
|
-
|
|
150
|
-
|
|
345
|
+
if (decision.entry.state === "active") {
|
|
346
|
+
pushToPlugin(decision.entry.ipc_ws, {
|
|
347
|
+
event: "message_received",
|
|
348
|
+
from: payload.from,
|
|
349
|
+
content: payload.content,
|
|
350
|
+
contentType: payload.contentType,
|
|
351
|
+
metadata: meta,
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
else {
|
|
355
|
+
const buf = reconnectingBuffers.forEndpoint(decision.entry.endpoint_id);
|
|
356
|
+
buf.append({
|
|
357
|
+
id: randomUUID(),
|
|
358
|
+
arrived_at: new Date().toISOString(),
|
|
359
|
+
envelope: {
|
|
360
|
+
from: payload.from,
|
|
361
|
+
content: payload.content,
|
|
362
|
+
contentType: payload.contentType,
|
|
363
|
+
metadata: meta,
|
|
364
|
+
},
|
|
365
|
+
});
|
|
366
|
+
logger.info("buffered_for_reconnecting_endpoint", {
|
|
367
|
+
endpoint_id: decision.entry.endpoint_id,
|
|
368
|
+
buffer_size: buf.size(),
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
let reason = "unaddressed";
|
|
374
|
+
let originalTarget;
|
|
375
|
+
if (target_endpoint_id && closedEndpointLookup) {
|
|
376
|
+
try {
|
|
377
|
+
const result = await closedEndpointLookup.lookup(target_endpoint_id, opts.asNodeId);
|
|
378
|
+
if (result.exists && result.closed) {
|
|
379
|
+
reason = "closed_session_continuation";
|
|
380
|
+
originalTarget = target_endpoint_id;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
catch (err) {
|
|
384
|
+
logger.warn("closed_lookup_failed", {
|
|
385
|
+
target_endpoint_id,
|
|
386
|
+
err: err instanceof Error ? err.message : String(err),
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
const undispatched = {
|
|
391
|
+
id: randomUUID(),
|
|
392
|
+
arrived_at: new Date().toISOString(),
|
|
393
|
+
sender_address: payload.from,
|
|
394
|
+
content: payload.content,
|
|
395
|
+
content_type: payload.contentType,
|
|
396
|
+
metadata: meta,
|
|
397
|
+
reason,
|
|
398
|
+
original_target_endpoint_id: originalTarget,
|
|
399
|
+
original_correlation_id: correlation_id,
|
|
400
|
+
};
|
|
401
|
+
const added = undispatchedInbox.tryAdd(undispatched);
|
|
402
|
+
if (!added) {
|
|
403
|
+
logger.warn("undispatched_inbox_at_capacity", {
|
|
151
404
|
from: payload.from,
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
metadata: meta,
|
|
405
|
+
size: undispatchedInbox.size(),
|
|
406
|
+
capacity: undispatchedInbox.capacity(),
|
|
155
407
|
});
|
|
156
408
|
}
|
|
157
409
|
else {
|
|
158
|
-
logger.info("
|
|
410
|
+
logger.info("undispatched_added", { id: undispatched.id, reason });
|
|
159
411
|
}
|
|
160
412
|
});
|
|
161
413
|
connector.on("delivery_pending", ({ payload }) => {
|
|
@@ -175,6 +427,7 @@ export async function startBrokerDaemon(opts) {
|
|
|
175
427
|
return;
|
|
176
428
|
shutdownStarted = true;
|
|
177
429
|
logger.info("shutdown_begin", { reason });
|
|
430
|
+
graceTimers.cancelAll();
|
|
178
431
|
const endpoints = registry.list();
|
|
179
432
|
await Promise.all(endpoints.map(async (entry) => {
|
|
180
433
|
try {
|
|
@@ -203,20 +456,9 @@ export async function startBrokerDaemon(opts) {
|
|
|
203
456
|
ipcUrl: ipc.ipcUrl,
|
|
204
457
|
shutdown,
|
|
205
458
|
registrySize: () => registry.size(),
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
try {
|
|
210
|
-
prior.close(1000, "replaced");
|
|
211
|
-
}
|
|
212
|
-
catch {
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
channels.set(plugin_pid, ws);
|
|
216
|
-
},
|
|
217
|
-
_unbindPluginChannel: (plugin_pid) => {
|
|
218
|
-
channels.delete(plugin_pid);
|
|
219
|
-
},
|
|
459
|
+
state: (endpoint_id) => registry.get(endpoint_id)?.state,
|
|
460
|
+
undispatchedInbox,
|
|
461
|
+
controlEventDispatcher,
|
|
220
462
|
};
|
|
221
463
|
}
|
|
222
464
|
export function readPluginPidHeader(req) {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { PlatformClientConfig } from "../platform-client.js";
|
|
2
|
+
import { type OwnedRuntimeEndpointView } from "../runtime-endpoint-client.js";
|
|
3
|
+
export interface ClosedEndpointInfo {
|
|
4
|
+
exists: true;
|
|
5
|
+
closed: boolean;
|
|
6
|
+
view: OwnedRuntimeEndpointView;
|
|
7
|
+
}
|
|
8
|
+
export interface UnknownEndpoint {
|
|
9
|
+
exists: false;
|
|
10
|
+
}
|
|
11
|
+
export type LookupResult = ClosedEndpointInfo | UnknownEndpoint;
|
|
12
|
+
export interface ClosedEndpointLookup {
|
|
13
|
+
lookup(endpoint_id: string, asNodeId: string): Promise<LookupResult>;
|
|
14
|
+
invalidate(endpoint_id: string): void;
|
|
15
|
+
size(): number;
|
|
16
|
+
}
|
|
17
|
+
export interface ClosedEndpointLookupOptions {
|
|
18
|
+
cfg: PlatformClientConfig;
|
|
19
|
+
runtimeKey: string;
|
|
20
|
+
cacheSize?: number;
|
|
21
|
+
cacheTtlMs?: number;
|
|
22
|
+
fetcher?: (cfg: PlatformClientConfig, runtimeKey: string, endpointId: string, asNodeId: string) => Promise<OwnedRuntimeEndpointView>;
|
|
23
|
+
}
|
|
24
|
+
export declare function createClosedEndpointLookup(opts: ClosedEndpointLookupOptions): ClosedEndpointLookup;
|
|
25
|
+
//# sourceMappingURL=closed-endpoint-lookup.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"closed-endpoint-lookup.d.ts","sourceRoot":"","sources":["../../src/broker/closed-endpoint-lookup.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAEL,KAAK,wBAAwB,EAC9B,MAAM,+BAA+B,CAAC;AAEvC,MAAM,WAAW,kBAAkB;IAEjC,MAAM,EAAE,IAAI,CAAC;IAEb,MAAM,EAAE,OAAO,CAAC;IAChB,IAAI,EAAE,wBAAwB,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,KAAK,CAAC;CACf;AAED,MAAM,MAAM,YAAY,GAAG,kBAAkB,GAAG,eAAe,CAAC;AAEhE,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAIrE,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtC,IAAI,IAAI,MAAM,CAAC;CAChB;AAaD,MAAM,WAAW,2BAA2B;IAC1C,GAAG,EAAE,oBAAoB,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IAEnB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,OAAO,CAAC,EAAE,CACR,GAAG,EAAE,oBAAoB,EACzB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,KACb,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACxC;AAED,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,2BAA2B,GAChC,oBAAoB,CA6DtB"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { getOwnedRuntimeEndpoint, } from "../runtime-endpoint-client.js";
|
|
2
|
+
const DEFAULT_CACHE_SIZE = 64;
|
|
3
|
+
const DEFAULT_CACHE_TTL_MS = 5_000;
|
|
4
|
+
export function createClosedEndpointLookup(opts) {
|
|
5
|
+
const cacheSize = opts.cacheSize ?? DEFAULT_CACHE_SIZE;
|
|
6
|
+
const ttl = opts.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS;
|
|
7
|
+
const fetcher = opts.fetcher ?? getOwnedRuntimeEndpoint;
|
|
8
|
+
const cache = new Map();
|
|
9
|
+
const evict = () => {
|
|
10
|
+
if (cache.size <= cacheSize)
|
|
11
|
+
return;
|
|
12
|
+
const oldest = cache.keys().next();
|
|
13
|
+
if (!oldest.done)
|
|
14
|
+
cache.delete(oldest.value);
|
|
15
|
+
};
|
|
16
|
+
return {
|
|
17
|
+
async lookup(endpoint_id, asNodeId) {
|
|
18
|
+
const now = Date.now();
|
|
19
|
+
const hit = cache.get(endpoint_id);
|
|
20
|
+
if (hit && hit.expires_at > now) {
|
|
21
|
+
cache.delete(endpoint_id);
|
|
22
|
+
cache.set(endpoint_id, hit);
|
|
23
|
+
return hit.result;
|
|
24
|
+
}
|
|
25
|
+
let result;
|
|
26
|
+
try {
|
|
27
|
+
const view = await fetcher(opts.cfg, opts.runtimeKey, endpoint_id, asNodeId);
|
|
28
|
+
result = {
|
|
29
|
+
exists: true,
|
|
30
|
+
closed: typeof view.retired_at === "string" && view.retired_at.length > 0,
|
|
31
|
+
view,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
if (isPlatformApi404(err)) {
|
|
36
|
+
result = { exists: false };
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
return { exists: false };
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
cache.set(endpoint_id, { result, expires_at: now + ttl });
|
|
43
|
+
evict();
|
|
44
|
+
return result;
|
|
45
|
+
},
|
|
46
|
+
invalidate(endpoint_id) {
|
|
47
|
+
cache.delete(endpoint_id);
|
|
48
|
+
},
|
|
49
|
+
size() {
|
|
50
|
+
return cache.size;
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function isPlatformApi404(err) {
|
|
55
|
+
if (typeof err !== "object" || err === null)
|
|
56
|
+
return false;
|
|
57
|
+
const status = err.status;
|
|
58
|
+
return status === 404;
|
|
59
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ControlEvent } from "./control-event-types.js";
|
|
2
|
+
export declare const DEFAULT_IDEMPOTENCY_RING_SIZE = 1024;
|
|
3
|
+
export type ControlEventOutcome = {
|
|
4
|
+
ok: true;
|
|
5
|
+
} | {
|
|
6
|
+
ok: false;
|
|
7
|
+
detail: string;
|
|
8
|
+
};
|
|
9
|
+
export type ControlEventHandler<E extends ControlEvent = ControlEvent> = (event: E) => Promise<ControlEventOutcome>;
|
|
10
|
+
export interface ControlEventDispatcher {
|
|
11
|
+
on<E extends ControlEvent>(type: E["type"], handler: ControlEventHandler<E>): void;
|
|
12
|
+
dispatch(event: ControlEvent): Promise<ControlEventOutcome>;
|
|
13
|
+
ringSize(): number;
|
|
14
|
+
hasSeen(idempotency_key: string): boolean;
|
|
15
|
+
}
|
|
16
|
+
export interface ControlEventDispatcherOptions {
|
|
17
|
+
ringSize?: number;
|
|
18
|
+
}
|
|
19
|
+
export declare function createControlEventDispatcher(opts?: ControlEventDispatcherOptions): ControlEventDispatcher;
|
|
20
|
+
//# sourceMappingURL=control-event-dispatcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"control-event-dispatcher.d.ts","sourceRoot":"","sources":["../../src/broker/control-event-dispatcher.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAG7D,eAAO,MAAM,6BAA6B,OAAO,CAAC;AAElD,MAAM,MAAM,mBAAmB,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE/E,MAAM,MAAM,mBAAmB,CAAC,CAAC,SAAS,YAAY,GAAG,YAAY,IAAI,CACvE,KAAK,EAAE,CAAC,KACL,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAElC,MAAM,WAAW,sBAAsB;IAErC,EAAE,CAAC,CAAC,SAAS,YAAY,EACvB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,EACf,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC9B,IAAI,CAAC;IAKR,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAE5D,QAAQ,IAAI,MAAM,CAAC;IAEnB,OAAO,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,4BAA4B,CAC1C,IAAI,GAAE,6BAAkC,GACvC,sBAAsB,CAqDxB"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
export const DEFAULT_IDEMPOTENCY_RING_SIZE = 1024;
|
|
2
|
+
export function createControlEventDispatcher(opts = {}) {
|
|
3
|
+
const cap = opts.ringSize ?? DEFAULT_IDEMPOTENCY_RING_SIZE;
|
|
4
|
+
if (cap <= 0)
|
|
5
|
+
throw new Error(`ringSize must be positive: ${cap}`);
|
|
6
|
+
const seen = new Set();
|
|
7
|
+
const order = [];
|
|
8
|
+
const handlers = new Map();
|
|
9
|
+
return {
|
|
10
|
+
on(type, handler) {
|
|
11
|
+
handlers.set(type, handler);
|
|
12
|
+
},
|
|
13
|
+
async dispatch(event) {
|
|
14
|
+
if (seen.has(event.idempotency_key)) {
|
|
15
|
+
return { ok: true };
|
|
16
|
+
}
|
|
17
|
+
seen.add(event.idempotency_key);
|
|
18
|
+
order.push(event.idempotency_key);
|
|
19
|
+
if (order.length > cap) {
|
|
20
|
+
const oldest = order.shift();
|
|
21
|
+
if (oldest !== undefined)
|
|
22
|
+
seen.delete(oldest);
|
|
23
|
+
}
|
|
24
|
+
const handler = handlers.get(event.type);
|
|
25
|
+
if (!handler) {
|
|
26
|
+
return { ok: false, detail: `no_handler:${event.type}` };
|
|
27
|
+
}
|
|
28
|
+
try {
|
|
29
|
+
return await handler(event);
|
|
30
|
+
}
|
|
31
|
+
catch (err) {
|
|
32
|
+
return {
|
|
33
|
+
ok: false,
|
|
34
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
ringSize() {
|
|
39
|
+
return seen.size;
|
|
40
|
+
},
|
|
41
|
+
hasSeen(idempotency_key) {
|
|
42
|
+
return seen.has(idempotency_key);
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export type ControlEvent = DispatchUndispatchedEvent | SpawnRequestEvent | ForceUnregisterEvent;
|
|
2
|
+
export interface DispatchUndispatchedEvent {
|
|
3
|
+
type: "dispatch_undispatched";
|
|
4
|
+
version: 1;
|
|
5
|
+
idempotency_key: string;
|
|
6
|
+
undispatched_id: string;
|
|
7
|
+
target_endpoint_id: string;
|
|
8
|
+
}
|
|
9
|
+
export interface SpawnRequestEvent {
|
|
10
|
+
type: "spawn_request";
|
|
11
|
+
version: 1;
|
|
12
|
+
idempotency_key: string;
|
|
13
|
+
request_id: string;
|
|
14
|
+
prompt: string;
|
|
15
|
+
runtime_kind: string;
|
|
16
|
+
origin: "passport-spawn";
|
|
17
|
+
}
|
|
18
|
+
export interface ForceUnregisterEvent {
|
|
19
|
+
type: "force_unregister";
|
|
20
|
+
version: 1;
|
|
21
|
+
idempotency_key: string;
|
|
22
|
+
endpoint_id: string;
|
|
23
|
+
reason: string;
|
|
24
|
+
}
|
|
25
|
+
export interface ControlAck {
|
|
26
|
+
request_id: string;
|
|
27
|
+
status: "ok" | "error";
|
|
28
|
+
detail?: string;
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=control-event-types.d.ts.map
|