clawmatrix 0.1.15 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BOOTSTRAP.md +17 -2
- package/package.json +1 -1
- package/src/auth.ts +42 -12
- package/src/cluster-service.ts +31 -4
- package/src/compat.ts +3 -0
- package/src/connection.ts +15 -6
- package/src/handoff.ts +311 -17
- package/src/http-utils.ts +35 -0
- package/src/index.ts +52 -41
- package/src/model-proxy.ts +19 -16
- package/src/peer-manager.ts +55 -5
- package/src/router.ts +62 -28
- package/src/tool-proxy.ts +22 -7
- package/src/tools/cluster-events.ts +119 -0
- package/src/tools/cluster-exec.ts +5 -1
- package/src/tools/cluster-handoff-reply.ts +73 -0
- package/src/tools/cluster-handoff.ts +13 -9
- package/src/tools/cluster-peers.ts +15 -6
- package/src/tools/cluster-read.ts +1 -1
- package/src/tools/cluster-send.ts +1 -3
- package/src/tools/cluster-tool.ts +3 -6
- package/src/tools/cluster-write.ts +1 -1
- package/src/types.ts +93 -0
- package/src/web-ui.ts +490 -345
- package/src/web.ts +675 -53
package/src/model-proxy.ts
CHANGED
|
@@ -9,12 +9,19 @@ import type {
|
|
|
9
9
|
ModelStreamChunk,
|
|
10
10
|
} from "./types.ts";
|
|
11
11
|
import { debug } from "./debug.ts";
|
|
12
|
+
import { readBody } from "./http-utils.ts";
|
|
12
13
|
|
|
13
14
|
const MODEL_TIMEOUT = 120_000; // 2 minutes
|
|
14
15
|
const MAX_STREAM_BUFFER = 1_048_576; // 1MB — guard against upstream not sending newlines
|
|
15
16
|
|
|
16
17
|
type ResponseFormat = "chat" | "responses";
|
|
17
18
|
|
|
19
|
+
interface ProxyResponse {
|
|
20
|
+
status: number;
|
|
21
|
+
headers: Record<string, string>;
|
|
22
|
+
body: string | ReadableStream;
|
|
23
|
+
}
|
|
24
|
+
|
|
18
25
|
interface PendingModelReq {
|
|
19
26
|
resolve: (value: unknown) => void;
|
|
20
27
|
reject: (error: Error) => void;
|
|
@@ -173,12 +180,12 @@ export class ModelProxy {
|
|
|
173
180
|
debug("proxy", `${req.method} ${url.pathname} → ${p}`);
|
|
174
181
|
|
|
175
182
|
if (p === "/chat/completions" && req.method === "POST") {
|
|
176
|
-
const body = await
|
|
183
|
+
const body = await readBody(req);
|
|
177
184
|
const response = await this.handleChatCompletion(body, "openai-completions");
|
|
178
185
|
debug("proxy", `response status=${response.status}`);
|
|
179
186
|
this.sendResponse(res, response);
|
|
180
187
|
} else if (p === "/responses" && req.method === "POST") {
|
|
181
|
-
const body = await
|
|
188
|
+
const body = await readBody(req);
|
|
182
189
|
const response = await this.handleResponses(body);
|
|
183
190
|
debug("proxy", `response status=${response.status}`);
|
|
184
191
|
this.sendResponse(res, response);
|
|
@@ -215,18 +222,11 @@ export class ModelProxy {
|
|
|
215
222
|
pending.reject(new Error("Shutting down"));
|
|
216
223
|
}
|
|
217
224
|
this.pending.clear();
|
|
225
|
+
this.streamText.clear();
|
|
218
226
|
}
|
|
219
227
|
|
|
220
|
-
private readBody(req: import("node:http").IncomingMessage): Promise<string> {
|
|
221
|
-
return new Promise((resolve, reject) => {
|
|
222
|
-
const chunks: Buffer[] = [];
|
|
223
|
-
req.on("data", (chunk: Buffer) => chunks.push(chunk));
|
|
224
|
-
req.on("end", () => resolve(Buffer.concat(chunks).toString()));
|
|
225
|
-
req.on("error", reject);
|
|
226
|
-
});
|
|
227
|
-
}
|
|
228
228
|
|
|
229
|
-
private sendResponse(res: import("node:http").ServerResponse, response:
|
|
229
|
+
private sendResponse(res: import("node:http").ServerResponse, response: ProxyResponse) {
|
|
230
230
|
res.writeHead(response.status, response.headers);
|
|
231
231
|
if (typeof response.body === "string") {
|
|
232
232
|
res.end(response.body);
|
|
@@ -288,7 +288,7 @@ export class ModelProxy {
|
|
|
288
288
|
return { nodeId, modelId, proxyModel, routeNodeId: route.nodeId };
|
|
289
289
|
}
|
|
290
290
|
|
|
291
|
-
private async handleChatCompletion(rawBody: string, _api: string): Promise<
|
|
291
|
+
private async handleChatCompletion(rawBody: string, _api: string): Promise<ProxyResponse> {
|
|
292
292
|
let body: { model: string; messages: unknown[]; stream?: boolean; temperature?: number; max_tokens?: number };
|
|
293
293
|
try {
|
|
294
294
|
body = JSON.parse(rawBody);
|
|
@@ -303,6 +303,8 @@ export class ModelProxy {
|
|
|
303
303
|
|
|
304
304
|
const { modelId, proxyModel, routeNodeId } = resolved;
|
|
305
305
|
const messages = body.messages;
|
|
306
|
+
debug("proxy", `messages count=${messages?.length ?? 0} roles=${(messages ?? []).map((m: unknown) => (m as Record<string, unknown>)?.role).join(",")}`);
|
|
307
|
+
|
|
306
308
|
if (proxyModel?.description) {
|
|
307
309
|
const first = messages[0] as { role?: string; content?: string } | undefined;
|
|
308
310
|
if (first?.role === "system" && typeof first.content === "string") {
|
|
@@ -326,7 +328,7 @@ export class ModelProxy {
|
|
|
326
328
|
}
|
|
327
329
|
}
|
|
328
330
|
|
|
329
|
-
private async handleResponses(rawBody: string): Promise<
|
|
331
|
+
private async handleResponses(rawBody: string): Promise<ProxyResponse> {
|
|
330
332
|
let body: { model: string; input: unknown; stream?: boolean; temperature?: number; max_output_tokens?: number; instructions?: string };
|
|
331
333
|
try {
|
|
332
334
|
body = JSON.parse(rawBody);
|
|
@@ -382,7 +384,7 @@ export class ModelProxy {
|
|
|
382
384
|
targetNodeId: string,
|
|
383
385
|
frame: ModelRequest,
|
|
384
386
|
responseFormat: ResponseFormat,
|
|
385
|
-
):
|
|
387
|
+
): ProxyResponse & { body: ReadableStream } {
|
|
386
388
|
const encoder = new TextEncoder();
|
|
387
389
|
const model = frame.payload.model;
|
|
388
390
|
|
|
@@ -390,6 +392,7 @@ export class ModelProxy {
|
|
|
390
392
|
start: (controller) => {
|
|
391
393
|
const timer = setTimeout(() => {
|
|
392
394
|
this.pending.delete(requestId);
|
|
395
|
+
this.streamText.delete(requestId);
|
|
393
396
|
this.peerManager.router.markFailed(requestId);
|
|
394
397
|
try {
|
|
395
398
|
if (responseFormat === "responses") {
|
|
@@ -475,7 +478,7 @@ export class ModelProxy {
|
|
|
475
478
|
targetNodeId: string,
|
|
476
479
|
frame: ModelRequest,
|
|
477
480
|
responseFormat: ResponseFormat,
|
|
478
|
-
): Promise<
|
|
481
|
+
): Promise<ProxyResponse & { body: string }> {
|
|
479
482
|
try {
|
|
480
483
|
const result = await new Promise<ModelResponse["payload"]>(
|
|
481
484
|
(resolve, reject) => {
|
|
@@ -569,7 +572,7 @@ export class ModelProxy {
|
|
|
569
572
|
}
|
|
570
573
|
}
|
|
571
574
|
|
|
572
|
-
private handleListModels():
|
|
575
|
+
private handleListModels(): ProxyResponse & { body: string } {
|
|
573
576
|
// Build from proxyModels config (has full detail) and enrich with
|
|
574
577
|
// connectivity info from the router so consumers know what's reachable.
|
|
575
578
|
const reachable = new Set(
|
package/src/peer-manager.ts
CHANGED
|
@@ -27,6 +27,7 @@ export interface PeerManagerEvents {
|
|
|
27
27
|
export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
28
28
|
readonly router: Router;
|
|
29
29
|
readonly localDeviceInfo: DeviceInfo;
|
|
30
|
+
satelliteContexts: import("./types.ts").SatelliteContext[] = [];
|
|
30
31
|
private config: ClawMatrixConfig;
|
|
31
32
|
private localCapabilities: NodeCapabilities;
|
|
32
33
|
private httpServer: Server | null = null;
|
|
@@ -48,12 +49,18 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
48
49
|
models: config.models,
|
|
49
50
|
tags: config.tags,
|
|
50
51
|
deviceInfo: this.localDeviceInfo,
|
|
52
|
+
toolProxy: config.toolProxy ? {
|
|
53
|
+
enabled: config.toolProxy.enabled,
|
|
54
|
+
allow: config.toolProxy.allow,
|
|
55
|
+
deny: config.toolProxy.deny,
|
|
56
|
+
} : undefined,
|
|
51
57
|
};
|
|
52
58
|
this.router = new Router(config.nodeId, {
|
|
53
59
|
agents: config.agents,
|
|
54
60
|
models: config.models,
|
|
55
61
|
tags: config.tags,
|
|
56
62
|
deviceInfo: this.localDeviceInfo,
|
|
63
|
+
toolProxy: this.localCapabilities.toolProxy,
|
|
57
64
|
});
|
|
58
65
|
}
|
|
59
66
|
|
|
@@ -251,6 +258,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
251
258
|
agents: caps.agents,
|
|
252
259
|
models: caps.models,
|
|
253
260
|
tags: caps.tags,
|
|
261
|
+
deviceInfo: caps.deviceInfo,
|
|
262
|
+
toolProxy: caps.toolProxy,
|
|
254
263
|
},
|
|
255
264
|
} as AnyClusterFrame);
|
|
256
265
|
|
|
@@ -270,6 +279,13 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
270
279
|
|
|
271
280
|
this.router.removePeer(nodeId);
|
|
272
281
|
|
|
282
|
+
// Remove satellite contexts that were only reachable via this peer
|
|
283
|
+
this.satelliteContexts = this.satelliteContexts.filter(s => {
|
|
284
|
+
// Keep satellites that are not associated with the disconnected peer
|
|
285
|
+
// (satellite nodeIds typically differ from mesh peer nodeIds)
|
|
286
|
+
return s.nodeId !== nodeId;
|
|
287
|
+
});
|
|
288
|
+
|
|
273
289
|
this.router.broadcast({
|
|
274
290
|
type: "peer_leave",
|
|
275
291
|
from: this.config.nodeId,
|
|
@@ -285,9 +301,22 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
285
301
|
// Validate from field: must be the direct peer or a known node (relayed)
|
|
286
302
|
if (frame.from && frame.from !== from.remoteNodeId && !this.router.getRoute(frame.from)) return;
|
|
287
303
|
|
|
288
|
-
// Skip dedup for streaming frame types
|
|
289
|
-
|
|
290
|
-
|
|
304
|
+
// Skip dedup for streaming and response frame types.
|
|
305
|
+
// Stream frames share one id across many chunks.
|
|
306
|
+
// Response frames (model_res, tool_res, handoff_res, etc.) share the same id
|
|
307
|
+
// as their request — without this exemption, a relay node that forwarded
|
|
308
|
+
// the request would mark the id as seen and then drop the returning response.
|
|
309
|
+
// Skip dedup for streaming chunks (same id across many chunks) and response
|
|
310
|
+
// frames (share id with their request — relay would otherwise drop the reply).
|
|
311
|
+
// handoff_input, handoff_cancel, and handoff_status reuse the original handoff_req id,
|
|
312
|
+
// so relay nodes that already forwarded the request would drop them as duplicates.
|
|
313
|
+
const skipDedup = frame.type === "model_stream" || frame.type === "handoff_stream"
|
|
314
|
+
|| frame.type === "model_res" || frame.type === "tool_res"
|
|
315
|
+
|| frame.type === "handoff_res" || frame.type === "handoff_status_res"
|
|
316
|
+
|| frame.type === "handoff_input_required"
|
|
317
|
+
|| frame.type === "handoff_input" || frame.type === "handoff_cancel"
|
|
318
|
+
|| frame.type === "handoff_status";
|
|
319
|
+
if (frame.id && !skipDedup && this.router.isDuplicate(frame.id)) return;
|
|
291
320
|
|
|
292
321
|
if (frame.type === "peer_sync") {
|
|
293
322
|
this.handlePeerSync(frame as PeerSync, from);
|
|
@@ -321,15 +350,32 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
321
350
|
|
|
322
351
|
private sendPeerSync(conn: Connection) {
|
|
323
352
|
const peers = this.router.buildPeerSyncPayload();
|
|
353
|
+
const payload: Record<string, unknown> = { peers };
|
|
354
|
+
if (this.satelliteContexts.length > 0) {
|
|
355
|
+
payload.satellites = this.satelliteContexts;
|
|
356
|
+
}
|
|
324
357
|
conn.send({
|
|
325
358
|
type: "peer_sync",
|
|
326
359
|
from: this.config.nodeId,
|
|
327
360
|
timestamp: Date.now(),
|
|
328
|
-
payload
|
|
361
|
+
payload,
|
|
329
362
|
} as AnyClusterFrame);
|
|
330
363
|
}
|
|
331
364
|
|
|
332
365
|
private handlePeerSync(frame: PeerSync, from: Connection) {
|
|
366
|
+
// Merge satellite contexts from relay node (keep newest per nodeId)
|
|
367
|
+
if (frame.payload.satellites) {
|
|
368
|
+
const byNodeId = new Map<string, import("./types.ts").SatelliteContext>();
|
|
369
|
+
for (const s of this.satelliteContexts) byNodeId.set(s.nodeId, s);
|
|
370
|
+
for (const s of frame.payload.satellites) {
|
|
371
|
+
const existing = byNodeId.get(s.nodeId);
|
|
372
|
+
if (!existing || s.ts > existing.ts) {
|
|
373
|
+
byNodeId.set(s.nodeId, s);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
this.satelliteContexts = [...byNodeId.values()];
|
|
377
|
+
}
|
|
378
|
+
|
|
333
379
|
let changed = false;
|
|
334
380
|
for (const peer of frame.payload.peers) {
|
|
335
381
|
if (peer.nodeId === this.config.nodeId) continue;
|
|
@@ -337,9 +383,13 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
337
383
|
const prev = this.router.getRoute(peer.nodeId);
|
|
338
384
|
const hadAgents = prev?.agents.length ?? 0;
|
|
339
385
|
const hadDirectPeers = prev?.directPeers.length ?? 0;
|
|
386
|
+
const hadToolProxy = JSON.stringify(prev?.toolProxy);
|
|
387
|
+
const hadDeviceInfo = prev?.deviceInfo?.hostname;
|
|
340
388
|
this.router.updatePeerCapabilities(peer.nodeId, peer);
|
|
341
389
|
if (peer.agents.length !== hadAgents || peer.models.length !== (prev?.models.length ?? 0)
|
|
342
|
-
|| (peer.directPeers?.length ?? 0) !== hadDirectPeers
|
|
390
|
+
|| (peer.directPeers?.length ?? 0) !== hadDirectPeers
|
|
391
|
+
|| JSON.stringify(peer.toolProxy) !== hadToolProxy
|
|
392
|
+
|| peer.deviceInfo?.hostname !== hadDeviceInfo) {
|
|
343
393
|
changed = true;
|
|
344
394
|
}
|
|
345
395
|
} else {
|
package/src/router.ts
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import type { ClusterFrame, AnyClusterFrame, PeerInfo, AgentInfo, ModelInfo, DeviceInfo } from "./types.ts";
|
|
1
|
+
import type { ClusterFrame, AnyClusterFrame, PeerInfo, AgentInfo, ModelInfo, DeviceInfo, ToolProxyInfo } from "./types.ts";
|
|
2
2
|
import type { Connection } from "./connection.ts";
|
|
3
3
|
|
|
4
4
|
const DEFAULT_TTL = 3;
|
|
5
|
-
const SEEN_FRAME_TTL = 60_000; // 60s dedup window
|
|
6
5
|
const MAX_SEEN_FRAMES = 10_000;
|
|
7
|
-
const
|
|
6
|
+
const ROTATE_INTERVAL = 60_000; // rotate dedup maps every 60s
|
|
8
7
|
|
|
9
8
|
export interface RouteEntry {
|
|
10
9
|
nodeId: string;
|
|
@@ -17,6 +16,7 @@ export interface RouteEntry {
|
|
|
17
16
|
latencyMs: number;
|
|
18
17
|
directPeers: string[]; // nodeIds this node has direct connections to
|
|
19
18
|
deviceInfo?: DeviceInfo;
|
|
19
|
+
toolProxy?: ToolProxyInfo;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
export class Router {
|
|
@@ -25,38 +25,46 @@ export class Router {
|
|
|
25
25
|
private localModels: ModelInfo[];
|
|
26
26
|
private localTags: string[];
|
|
27
27
|
private localDeviceInfo?: DeviceInfo;
|
|
28
|
+
private localToolProxy?: ToolProxyInfo;
|
|
28
29
|
private routes = new Map<string, RouteEntry>();
|
|
29
30
|
private connections = new Map<string, Connection>(); // nodeId → direct connection
|
|
30
|
-
|
|
31
|
-
private
|
|
31
|
+
/** Double-map dedup: current window + previous window. Rotated periodically. */
|
|
32
|
+
private seenCurrent = new Map<string, true>();
|
|
33
|
+
private seenPrevious = new Map<string, true>();
|
|
34
|
+
private rotateTimer: ReturnType<typeof setInterval> | null = null;
|
|
35
|
+
/** Failed request IDs with expiry timestamps. Separate from dedup to support longer TTLs. */
|
|
36
|
+
private failedRequests = new Map<string, number>(); // requestId → expiresAt
|
|
32
37
|
|
|
33
38
|
constructor(
|
|
34
39
|
nodeId: string,
|
|
35
|
-
localCapabilities?: { agents: AgentInfo[]; models: ModelInfo[]; tags: string[]; deviceInfo?: DeviceInfo },
|
|
40
|
+
localCapabilities?: { agents: AgentInfo[]; models: ModelInfo[]; tags: string[]; deviceInfo?: DeviceInfo; toolProxy?: ToolProxyInfo },
|
|
36
41
|
) {
|
|
37
42
|
this.nodeId = nodeId;
|
|
38
43
|
this.localAgents = localCapabilities?.agents ?? [];
|
|
39
44
|
this.localModels = localCapabilities?.models ?? [];
|
|
40
45
|
this.localTags = localCapabilities?.tags ?? [];
|
|
41
46
|
this.localDeviceInfo = localCapabilities?.deviceInfo;
|
|
47
|
+
this.localToolProxy = localCapabilities?.toolProxy;
|
|
42
48
|
|
|
43
|
-
this.
|
|
49
|
+
this.rotateTimer = setInterval(() => this.rotateSeenFrames(), ROTATE_INTERVAL);
|
|
44
50
|
}
|
|
45
51
|
|
|
46
52
|
/** Stop periodic cleanup. Call on shutdown. */
|
|
47
53
|
destroy() {
|
|
48
|
-
if (this.
|
|
49
|
-
clearInterval(this.
|
|
50
|
-
this.
|
|
54
|
+
if (this.rotateTimer) {
|
|
55
|
+
clearInterval(this.rotateTimer);
|
|
56
|
+
this.rotateTimer = null;
|
|
51
57
|
}
|
|
52
|
-
this.
|
|
58
|
+
this.seenCurrent.clear();
|
|
59
|
+
this.seenPrevious.clear();
|
|
60
|
+
this.failedRequests.clear();
|
|
53
61
|
}
|
|
54
62
|
|
|
55
63
|
// ── Route table management ─────────────────────────────────────
|
|
56
64
|
addDirectPeer(
|
|
57
65
|
nodeId: string,
|
|
58
66
|
connection: Connection,
|
|
59
|
-
capabilities: { agents: AgentInfo[]; models: ModelInfo[]; tags: string[]; deviceInfo?: DeviceInfo },
|
|
67
|
+
capabilities: { agents: AgentInfo[]; models: ModelInfo[]; tags: string[]; deviceInfo?: DeviceInfo; toolProxy?: ToolProxyInfo },
|
|
60
68
|
) {
|
|
61
69
|
this.connections.set(nodeId, connection);
|
|
62
70
|
this.routes.set(nodeId, {
|
|
@@ -70,6 +78,7 @@ export class Router {
|
|
|
70
78
|
latencyMs: 0,
|
|
71
79
|
directPeers: [],
|
|
72
80
|
deviceInfo: capabilities.deviceInfo,
|
|
81
|
+
toolProxy: capabilities.toolProxy,
|
|
73
82
|
});
|
|
74
83
|
}
|
|
75
84
|
|
|
@@ -91,6 +100,7 @@ export class Router {
|
|
|
91
100
|
latencyMs: 0,
|
|
92
101
|
directPeers: peer.directPeers ?? [],
|
|
93
102
|
deviceInfo: peer.deviceInfo,
|
|
103
|
+
toolProxy: peer.toolProxy,
|
|
94
104
|
});
|
|
95
105
|
}
|
|
96
106
|
|
|
@@ -107,7 +117,7 @@ export class Router {
|
|
|
107
117
|
|
|
108
118
|
updatePeerCapabilities(
|
|
109
119
|
nodeId: string,
|
|
110
|
-
capabilities: { agents: AgentInfo[]; models: ModelInfo[]; tags: string[]; directPeers?: string[]; deviceInfo?: DeviceInfo },
|
|
120
|
+
capabilities: { agents: AgentInfo[]; models: ModelInfo[]; tags: string[]; directPeers?: string[]; deviceInfo?: DeviceInfo; toolProxy?: ToolProxyInfo },
|
|
111
121
|
) {
|
|
112
122
|
const entry = this.routes.get(nodeId);
|
|
113
123
|
if (entry) {
|
|
@@ -120,6 +130,7 @@ export class Router {
|
|
|
120
130
|
if (capabilities.deviceInfo) {
|
|
121
131
|
entry.deviceInfo = capabilities.deviceInfo;
|
|
122
132
|
}
|
|
133
|
+
entry.toolProxy = capabilities.toolProxy;
|
|
123
134
|
entry.lastSeen = Date.now();
|
|
124
135
|
}
|
|
125
136
|
}
|
|
@@ -236,33 +247,54 @@ export class Router {
|
|
|
236
247
|
return relayed;
|
|
237
248
|
}
|
|
238
249
|
|
|
239
|
-
// ── Deduplication
|
|
240
|
-
/**
|
|
250
|
+
// ── Deduplication (double-map rotation) ────────────────────────
|
|
251
|
+
/**
|
|
252
|
+
* Returns true if the frame has been seen before (duplicate).
|
|
253
|
+
* Uses a double-map strategy: entries live in `seenCurrent` and are
|
|
254
|
+
* promoted from `seenPrevious` on access. Every ROTATE_INTERVAL the
|
|
255
|
+
* previous map is discarded and current becomes previous — O(1) cleanup.
|
|
256
|
+
*/
|
|
241
257
|
isDuplicate(frameId: string): boolean {
|
|
242
258
|
if (!frameId) return false;
|
|
243
|
-
if (this.
|
|
259
|
+
if (this.seenCurrent.has(frameId)) return true;
|
|
260
|
+
if (this.seenPrevious.has(frameId)) {
|
|
261
|
+
// Promote to current so it survives the next rotation
|
|
262
|
+
this.seenCurrent.set(frameId, true);
|
|
263
|
+
return true;
|
|
264
|
+
}
|
|
244
265
|
|
|
245
|
-
this.
|
|
246
|
-
|
|
266
|
+
this.seenCurrent.set(frameId, true);
|
|
267
|
+
// Safety valve: if current map grows too large, force a rotation
|
|
268
|
+
if (this.seenCurrent.size > MAX_SEEN_FRAMES) {
|
|
269
|
+
this.rotateSeenFrames();
|
|
270
|
+
}
|
|
247
271
|
return false;
|
|
248
272
|
}
|
|
249
273
|
|
|
250
|
-
/** Mark a request ID as failed so late responses are ignored.
|
|
251
|
-
|
|
252
|
-
|
|
274
|
+
/** Mark a request ID as failed so late responses are ignored.
|
|
275
|
+
* TTL defaults to 15 minutes — long enough for handoff timeouts. */
|
|
276
|
+
markFailed(requestId: string, ttlMs = 900_000) {
|
|
277
|
+
this.failedRequests.set(requestId, Date.now() + ttlMs);
|
|
253
278
|
}
|
|
254
279
|
|
|
255
280
|
isFailed(requestId: string): boolean {
|
|
256
|
-
|
|
281
|
+
const expiresAt = this.failedRequests.get(requestId);
|
|
282
|
+
if (expiresAt === undefined) return false;
|
|
283
|
+
if (Date.now() > expiresAt) {
|
|
284
|
+
this.failedRequests.delete(requestId);
|
|
285
|
+
return false;
|
|
286
|
+
}
|
|
287
|
+
return true;
|
|
257
288
|
}
|
|
258
289
|
|
|
259
|
-
private
|
|
260
|
-
|
|
290
|
+
private rotateSeenFrames() {
|
|
291
|
+
this.seenPrevious = this.seenCurrent;
|
|
292
|
+
this.seenCurrent = new Map();
|
|
293
|
+
|
|
294
|
+
// Prune expired failed requests
|
|
261
295
|
const now = Date.now();
|
|
262
|
-
for (const [id,
|
|
263
|
-
if (now
|
|
264
|
-
this.seenFrames.delete(id);
|
|
265
|
-
}
|
|
296
|
+
for (const [id, expiresAt] of this.failedRequests) {
|
|
297
|
+
if (now > expiresAt) this.failedRequests.delete(id);
|
|
266
298
|
}
|
|
267
299
|
}
|
|
268
300
|
|
|
@@ -287,6 +319,7 @@ export class Router {
|
|
|
287
319
|
tags: this.localTags,
|
|
288
320
|
directPeers: myDirectPeers,
|
|
289
321
|
deviceInfo: this.localDeviceInfo,
|
|
322
|
+
toolProxy: this.localToolProxy,
|
|
290
323
|
});
|
|
291
324
|
for (const entry of this.routes.values()) {
|
|
292
325
|
peers.push({
|
|
@@ -297,6 +330,7 @@ export class Router {
|
|
|
297
330
|
reachableVia: entry.reachableVia ?? undefined,
|
|
298
331
|
directPeers: entry.directPeers.length > 0 ? entry.directPeers : undefined,
|
|
299
332
|
deviceInfo: entry.deviceInfo,
|
|
333
|
+
toolProxy: entry.toolProxy,
|
|
300
334
|
});
|
|
301
335
|
}
|
|
302
336
|
return peers;
|
package/src/tool-proxy.ts
CHANGED
|
@@ -16,11 +16,18 @@ export interface GatewayInfo {
|
|
|
16
16
|
authHeader?: string;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
/** Interface for satellite tool routing (implemented by WebHandler). */
|
|
20
|
+
export interface SatelliteToolHandler {
|
|
21
|
+
isSatelliteNode(nodeId: string): boolean;
|
|
22
|
+
queueToolForSatellite(nodeId: string, id: string, tool: string, params: Record<string, unknown>, timeout?: number): Promise<Record<string, unknown>>;
|
|
23
|
+
}
|
|
24
|
+
|
|
19
25
|
export class ToolProxy {
|
|
20
26
|
private config: ClawMatrixConfig;
|
|
21
27
|
private peerManager: PeerManager;
|
|
22
28
|
private pending = new Map<string, PendingToolReq>();
|
|
23
29
|
private gatewayInfo: GatewayInfo;
|
|
30
|
+
private satelliteHandler: SatelliteToolHandler | null = null;
|
|
24
31
|
|
|
25
32
|
constructor(config: ClawMatrixConfig, peerManager: PeerManager, gatewayInfo: GatewayInfo) {
|
|
26
33
|
this.config = config;
|
|
@@ -28,6 +35,11 @@ export class ToolProxy {
|
|
|
28
35
|
this.gatewayInfo = gatewayInfo;
|
|
29
36
|
}
|
|
30
37
|
|
|
38
|
+
/** Set the satellite tool handler (called by ClusterRuntime after WebHandler is created). */
|
|
39
|
+
setSatelliteHandler(handler: SatelliteToolHandler) {
|
|
40
|
+
this.satelliteHandler = handler;
|
|
41
|
+
}
|
|
42
|
+
|
|
31
43
|
// ── Outbound: request remote execution ─────────────────────────
|
|
32
44
|
async invoke(
|
|
33
45
|
node: string,
|
|
@@ -45,7 +57,15 @@ export class ToolProxy {
|
|
|
45
57
|
timeout?: number,
|
|
46
58
|
): Promise<Record<string, unknown>> {
|
|
47
59
|
const route = this.peerManager.router.resolveNode(node);
|
|
48
|
-
|
|
60
|
+
|
|
61
|
+
// If no WS route, try satellite fallback
|
|
62
|
+
if (!route) {
|
|
63
|
+
if (this.satelliteHandler?.isSatelliteNode(node)) {
|
|
64
|
+
const id = crypto.randomUUID();
|
|
65
|
+
return this.satelliteHandler.queueToolForSatellite(node, id, tool, params, timeout);
|
|
66
|
+
}
|
|
67
|
+
throw new Error(`Node "${node}" not reachable`);
|
|
68
|
+
}
|
|
49
69
|
|
|
50
70
|
const id = crypto.randomUUID();
|
|
51
71
|
const frame: ToolProxyRequest = {
|
|
@@ -114,17 +134,12 @@ export class ToolProxy {
|
|
|
114
134
|
}
|
|
115
135
|
|
|
116
136
|
try {
|
|
117
|
-
console.log(`Received tool request for: ${payload.tool}`);
|
|
118
|
-
console.log(`Is local tool? ${isLocalTool(payload.tool)}`);
|
|
119
|
-
|
|
120
137
|
const result = isLocalTool(payload.tool)
|
|
121
138
|
? await executeLocally(payload.tool, payload.params)
|
|
122
139
|
: await this.executeViaGateway(payload.tool, payload.params);
|
|
123
|
-
|
|
124
|
-
console.log(`Tool execution result: ${JSON.stringify(result)}`);
|
|
140
|
+
|
|
125
141
|
this.sendResponse(id, from, { success: true, result });
|
|
126
142
|
} catch (err) {
|
|
127
|
-
console.error(`Tool execution error: ${err}`);
|
|
128
143
|
this.sendResponse(id, from, {
|
|
129
144
|
success: false,
|
|
130
145
|
error: err instanceof Error ? err.message : String(err),
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import type { AnyAgentTool } from "openclaw/plugin-sdk";
|
|
2
|
+
import { getClusterRuntime } from "../cluster-service.ts";
|
|
3
|
+
|
|
4
|
+
export function createClusterEventsTool(): AnyAgentTool {
|
|
5
|
+
return {
|
|
6
|
+
name: "cluster_events",
|
|
7
|
+
label: "Cluster Events",
|
|
8
|
+
description:
|
|
9
|
+
"Query and consume events from external sources (messages, calls, location changes).",
|
|
10
|
+
parameters: {
|
|
11
|
+
type: "object",
|
|
12
|
+
properties: {
|
|
13
|
+
action: {
|
|
14
|
+
type: "string",
|
|
15
|
+
enum: ["query", "consume"],
|
|
16
|
+
description: '"query" to list events (type/source/unconsumed/since/limit filters apply), "consume" to mark as processed (requires ids)',
|
|
17
|
+
},
|
|
18
|
+
type: {
|
|
19
|
+
type: "string",
|
|
20
|
+
description: 'Filter by event type (e.g. "message_received")',
|
|
21
|
+
},
|
|
22
|
+
source: {
|
|
23
|
+
type: "string",
|
|
24
|
+
description: 'Filter by source (e.g. "shortcuts")',
|
|
25
|
+
},
|
|
26
|
+
unconsumed: {
|
|
27
|
+
type: "boolean",
|
|
28
|
+
description: "Only unconsumed events (default true)",
|
|
29
|
+
},
|
|
30
|
+
since: {
|
|
31
|
+
type: "number",
|
|
32
|
+
description: "Events after this unix timestamp (ms)",
|
|
33
|
+
},
|
|
34
|
+
limit: {
|
|
35
|
+
type: "number",
|
|
36
|
+
description: "Max events to return (default 20)",
|
|
37
|
+
},
|
|
38
|
+
ids: {
|
|
39
|
+
type: "array",
|
|
40
|
+
items: { type: "string" },
|
|
41
|
+
description: "Event IDs to consume",
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
required: ["action"],
|
|
45
|
+
},
|
|
46
|
+
async execute(_toolCallId, params) {
|
|
47
|
+
const { action, type, source, unconsumed, since, limit, ids } = params as {
|
|
48
|
+
action: "query" | "consume";
|
|
49
|
+
type?: string;
|
|
50
|
+
source?: string;
|
|
51
|
+
unconsumed?: boolean;
|
|
52
|
+
since?: number;
|
|
53
|
+
limit?: number;
|
|
54
|
+
ids?: string[];
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
try {
|
|
58
|
+
const runtime = getClusterRuntime();
|
|
59
|
+
const webHandler = runtime.webHandler;
|
|
60
|
+
if (!webHandler) {
|
|
61
|
+
return {
|
|
62
|
+
content: [{ type: "text" as const, text: "Event ingestion requires web dashboard to be enabled (web.enabled = true)" }],
|
|
63
|
+
details: { error: true },
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (action === "consume") {
|
|
68
|
+
if (!ids || ids.length === 0) {
|
|
69
|
+
return {
|
|
70
|
+
content: [{ type: "text" as const, text: "Error: ids array required for consume action" }],
|
|
71
|
+
details: { error: true },
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
const consumed = webHandler.consumeEvents(ids);
|
|
75
|
+
return {
|
|
76
|
+
content: [{ type: "text" as const, text: `Consumed ${consumed} event(s)` }],
|
|
77
|
+
details: { consumed },
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// action === "query"
|
|
82
|
+
const events = webHandler.queryEvents({
|
|
83
|
+
type,
|
|
84
|
+
source,
|
|
85
|
+
since,
|
|
86
|
+
unconsumed: unconsumed !== false, // default true
|
|
87
|
+
limit: limit ?? 20,
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
if (events.length === 0) {
|
|
91
|
+
return {
|
|
92
|
+
content: [{ type: "text" as const, text: "No matching events found." }],
|
|
93
|
+
details: { events: [] },
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const summary = events.map(e => ({
|
|
98
|
+
id: e.id,
|
|
99
|
+
type: e.type,
|
|
100
|
+
source: e.source,
|
|
101
|
+
ts: e.ts,
|
|
102
|
+
time: new Date(e.ts).toISOString(),
|
|
103
|
+
consumed: e.consumed,
|
|
104
|
+
data: e.data,
|
|
105
|
+
}));
|
|
106
|
+
|
|
107
|
+
return {
|
|
108
|
+
content: [{ type: "text" as const, text: JSON.stringify(summary) }],
|
|
109
|
+
details: { events: summary },
|
|
110
|
+
};
|
|
111
|
+
} catch (err) {
|
|
112
|
+
return {
|
|
113
|
+
content: [{ type: "text" as const, text: `Error: ${err instanceof Error ? err.message : String(err)}` }],
|
|
114
|
+
details: { error: true },
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
@@ -40,16 +40,20 @@ export function createClusterExecTool(): AnyAgentTool {
|
|
|
40
40
|
|
|
41
41
|
try {
|
|
42
42
|
const runtime = getClusterRuntime();
|
|
43
|
+
// Network timeout must cover the exec timeout plus overhead
|
|
44
|
+
const execTimeoutSec = timeout ?? 1800;
|
|
45
|
+
const networkTimeout = (execTimeoutSec + 30) * 1000;
|
|
43
46
|
const result = await runtime.toolProxy.invoke(
|
|
44
47
|
node,
|
|
45
48
|
"exec",
|
|
46
49
|
{ command, workdir, timeout },
|
|
50
|
+
networkTimeout,
|
|
47
51
|
);
|
|
48
52
|
return {
|
|
49
53
|
content: [
|
|
50
54
|
{
|
|
51
55
|
type: "text" as const,
|
|
52
|
-
text: JSON.stringify(result
|
|
56
|
+
text: JSON.stringify(result),
|
|
53
57
|
},
|
|
54
58
|
],
|
|
55
59
|
details: result,
|