clawmatrix 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BOOTSTRAP.md +55 -8
- package/package.json +4 -2
- package/src/auth.ts +42 -12
- package/src/cluster-service.ts +35 -7
- package/src/compat.ts +3 -0
- package/src/config.ts +57 -6
- package/src/connection.ts +34 -8
- package/src/device-info.ts +48 -0
- package/src/handoff.ts +330 -21
- package/src/http-utils.ts +35 -0
- package/src/index.ts +47 -19
- package/src/model-proxy.ts +546 -242
- package/src/peer-manager.ts +65 -6
- package/src/router.ts +89 -47
- package/src/tool-proxy.ts +22 -7
- package/src/tools/cluster-events.ts +119 -0
- package/src/tools/cluster-exec.ts +4 -0
- package/src/tools/cluster-handoff-reply.ts +77 -0
- package/src/tools/cluster-handoff.ts +12 -0
- package/src/tools/cluster-peers.ts +17 -1
- package/src/tools/cluster-send.ts +1 -3
- package/src/tools/cluster-tool.ts +2 -5
- package/src/types.ts +117 -0
- package/src/web-ui.ts +694 -342
- package/src/web.ts +726 -50
package/src/handoff.ts
CHANGED
|
@@ -5,30 +5,91 @@ import type {
|
|
|
5
5
|
HandoffRequest,
|
|
6
6
|
HandoffResponse,
|
|
7
7
|
HandoffStreamChunk,
|
|
8
|
+
HandoffCancel,
|
|
9
|
+
HandoffStatusQuery,
|
|
10
|
+
HandoffStatusResponse,
|
|
11
|
+
HandoffInputRequired,
|
|
12
|
+
HandoffInput,
|
|
13
|
+
HandoffStatus,
|
|
8
14
|
} from "./types.ts";
|
|
9
15
|
|
|
10
16
|
const DEFAULT_HANDOFF_TIMEOUT = 600_000; // 10 minutes (resets on each stream chunk)
|
|
11
17
|
const MAX_RETRIES = 2;
|
|
18
|
+
const INPUT_REQUIRED_TTL = 1_800_000; // 30 minutes — max time to wait for user reply
|
|
19
|
+
const STALE_CLEANUP_INTERVAL = 120_000; // check every 2 minutes
|
|
12
20
|
|
|
13
21
|
interface PendingHandoff {
|
|
14
22
|
resolve: (result: HandoffResponse["payload"]) => void;
|
|
15
23
|
reject: (error: Error) => void;
|
|
16
24
|
timer: ReturnType<typeof setTimeout>;
|
|
17
25
|
target: string;
|
|
26
|
+
targetNodeId: string;
|
|
18
27
|
retriesLeft: number;
|
|
19
28
|
task: string;
|
|
20
29
|
context?: string;
|
|
21
30
|
accumulated: string;
|
|
31
|
+
onStream?: (delta: string) => void;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface ActiveHandoff {
|
|
35
|
+
proc: { kill: () => void; exited: Promise<number> } | null;
|
|
36
|
+
status: HandoffStatus;
|
|
37
|
+
sessionId: string;
|
|
38
|
+
agent: string;
|
|
39
|
+
startedAt: number;
|
|
40
|
+
inputRequiredAt: number | null; // when status changed to input_required
|
|
41
|
+
from: string; // requesting nodeId
|
|
22
42
|
}
|
|
23
43
|
|
|
24
44
|
export class HandoffManager {
|
|
25
45
|
private config: ClawMatrixConfig;
|
|
26
46
|
private peerManager: PeerManager;
|
|
27
47
|
private pending = new Map<string, PendingHandoff>();
|
|
48
|
+
private active = new Map<string, ActiveHandoff>();
|
|
49
|
+
private inputRequiredTargets = new Map<string, string>(); // handoffId → targetNodeId
|
|
50
|
+
private staleCleanupTimer: ReturnType<typeof setInterval> | null = null;
|
|
28
51
|
|
|
29
52
|
constructor(config: ClawMatrixConfig, peerManager: PeerManager) {
|
|
30
53
|
this.config = config;
|
|
31
54
|
this.peerManager = peerManager;
|
|
55
|
+
|
|
56
|
+
// Periodically clean up stale input_required entries
|
|
57
|
+
this.staleCleanupTimer = setInterval(() => this.cleanupStale(), STALE_CLEANUP_INTERVAL);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Remove stale input_required and canceled entries that exceeded TTL. */
|
|
61
|
+
private cleanupStale() {
|
|
62
|
+
const now = Date.now();
|
|
63
|
+
for (const [id, entry] of this.active) {
|
|
64
|
+
if (entry.status === "input_required" && entry.inputRequiredAt && now - entry.inputRequiredAt > INPUT_REQUIRED_TTL) {
|
|
65
|
+
this.active.delete(id);
|
|
66
|
+
// Notify the requester that the handoff timed out
|
|
67
|
+
this.peerManager.sendTo(entry.from, {
|
|
68
|
+
type: "handoff_res",
|
|
69
|
+
id,
|
|
70
|
+
from: this.config.nodeId,
|
|
71
|
+
to: entry.from,
|
|
72
|
+
timestamp: now,
|
|
73
|
+
payload: {
|
|
74
|
+
success: false,
|
|
75
|
+
nodeId: this.config.nodeId,
|
|
76
|
+
agent: entry.agent,
|
|
77
|
+
error: "Handoff timed out waiting for input",
|
|
78
|
+
},
|
|
79
|
+
} satisfies HandoffResponse);
|
|
80
|
+
}
|
|
81
|
+
// Clean up canceled entries that were never cleaned by runAgentTurn
|
|
82
|
+
// (e.g. cancel arrived during input_required when no process was running)
|
|
83
|
+
if (entry.status === "canceled") {
|
|
84
|
+
this.active.delete(id);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Clean stale inputRequiredTargets (requester side)
|
|
88
|
+
for (const [handoffId] of this.inputRequiredTargets) {
|
|
89
|
+
if (!this.pending.has(handoffId)) {
|
|
90
|
+
this.inputRequiredTargets.delete(handoffId);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
32
93
|
}
|
|
33
94
|
|
|
34
95
|
/** Send a handoff request and wait for the response. */
|
|
@@ -36,13 +97,19 @@ export class HandoffManager {
|
|
|
36
97
|
target: string,
|
|
37
98
|
task: string,
|
|
38
99
|
context?: string,
|
|
100
|
+
options?: { nodeId?: string; onStream?: (delta: string) => void },
|
|
39
101
|
): Promise<HandoffResponse["payload"]> {
|
|
102
|
+
if (options?.nodeId) {
|
|
103
|
+
// Direct node targeting (e.g. from web UI) — skip router resolution
|
|
104
|
+
return this.sendHandoff(options.nodeId, target, task, context, 0, options.onStream);
|
|
105
|
+
}
|
|
106
|
+
|
|
40
107
|
const route = this.peerManager.router.resolveAgent(target);
|
|
41
108
|
if (!route) {
|
|
42
109
|
throw new Error(`No reachable agent for target "${target}"`);
|
|
43
110
|
}
|
|
44
111
|
|
|
45
|
-
return this.sendHandoff(route.nodeId, target, task, context, MAX_RETRIES);
|
|
112
|
+
return this.sendHandoff(route.nodeId, target, task, context, MAX_RETRIES, options?.onStream);
|
|
46
113
|
}
|
|
47
114
|
|
|
48
115
|
private sendHandoff(
|
|
@@ -51,13 +118,14 @@ export class HandoffManager {
|
|
|
51
118
|
task: string,
|
|
52
119
|
context: string | undefined,
|
|
53
120
|
retriesLeft: number,
|
|
121
|
+
onStream?: (delta: string) => void,
|
|
54
122
|
): Promise<HandoffResponse["payload"]> {
|
|
55
123
|
const id = crypto.randomUUID();
|
|
56
124
|
|
|
57
125
|
return new Promise<HandoffResponse["payload"]>((resolve, reject) => {
|
|
58
|
-
const timer = this.createTimeout(id, targetNodeId, target, task, context, retriesLeft, resolve, reject);
|
|
126
|
+
const timer = this.createTimeout(id, targetNodeId, target, task, context, retriesLeft, resolve, reject, onStream);
|
|
59
127
|
|
|
60
|
-
this.pending.set(id, { resolve, reject, timer, target, retriesLeft, task, context, accumulated: "" });
|
|
128
|
+
this.pending.set(id, { resolve, reject, timer, target, targetNodeId, retriesLeft, task, context, accumulated: "", onStream });
|
|
61
129
|
|
|
62
130
|
const frame: HandoffRequest = {
|
|
63
131
|
type: "handoff_req",
|
|
@@ -77,7 +145,7 @@ export class HandoffManager {
|
|
|
77
145
|
if (retriesLeft > 0) {
|
|
78
146
|
const nextRoute = this.peerManager.router.resolveAgent(target);
|
|
79
147
|
if (nextRoute && nextRoute.nodeId !== targetNodeId) {
|
|
80
|
-
this.sendHandoff(nextRoute.nodeId, target, task, context, retriesLeft - 1)
|
|
148
|
+
this.sendHandoff(nextRoute.nodeId, target, task, context, retriesLeft - 1, onStream)
|
|
81
149
|
.then(resolve)
|
|
82
150
|
.catch(reject);
|
|
83
151
|
return;
|
|
@@ -97,6 +165,7 @@ export class HandoffManager {
|
|
|
97
165
|
retriesLeft: number,
|
|
98
166
|
resolve: (result: HandoffResponse["payload"]) => void,
|
|
99
167
|
reject: (error: Error) => void,
|
|
168
|
+
onStream?: (delta: string) => void,
|
|
100
169
|
): ReturnType<typeof setTimeout> {
|
|
101
170
|
return setTimeout(() => {
|
|
102
171
|
this.pending.delete(id);
|
|
@@ -106,7 +175,7 @@ export class HandoffManager {
|
|
|
106
175
|
if (retriesLeft > 0) {
|
|
107
176
|
const nextRoute = this.peerManager.router.resolveAgent(target);
|
|
108
177
|
if (nextRoute && nextRoute.nodeId !== targetNodeId) {
|
|
109
|
-
this.sendHandoff(nextRoute.nodeId, target, task, context, retriesLeft - 1)
|
|
178
|
+
this.sendHandoff(nextRoute.nodeId, target, task, context, retriesLeft - 1, onStream)
|
|
110
179
|
.then(resolve)
|
|
111
180
|
.catch(reject);
|
|
112
181
|
return;
|
|
@@ -125,6 +194,11 @@ export class HandoffManager {
|
|
|
125
194
|
|
|
126
195
|
pending.accumulated += frame.payload.delta;
|
|
127
196
|
|
|
197
|
+
// Notify stream listener
|
|
198
|
+
if (pending.onStream && frame.payload.delta) {
|
|
199
|
+
pending.onStream(frame.payload.delta);
|
|
200
|
+
}
|
|
201
|
+
|
|
128
202
|
// Reset timeout — the remote agent is still working
|
|
129
203
|
clearTimeout(pending.timer);
|
|
130
204
|
pending.timer = this.createTimeout(
|
|
@@ -136,6 +210,7 @@ export class HandoffManager {
|
|
|
136
210
|
pending.retriesLeft,
|
|
137
211
|
pending.resolve,
|
|
138
212
|
pending.reject,
|
|
213
|
+
pending.onStream,
|
|
139
214
|
);
|
|
140
215
|
}
|
|
141
216
|
|
|
@@ -185,26 +260,94 @@ export class HandoffManager {
|
|
|
185
260
|
return;
|
|
186
261
|
}
|
|
187
262
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
263
|
+
const sessionId = `handoff-${id}`;
|
|
264
|
+
const message = payload.context
|
|
265
|
+
? `${payload.task}\n\nContext:\n${payload.context}`
|
|
266
|
+
: payload.task;
|
|
267
|
+
|
|
268
|
+
const activeEntry: ActiveHandoff = {
|
|
269
|
+
proc: null,
|
|
270
|
+
status: "working",
|
|
271
|
+
sessionId,
|
|
272
|
+
agent: agent.id,
|
|
273
|
+
startedAt: Date.now(),
|
|
274
|
+
inputRequiredAt: null,
|
|
275
|
+
from,
|
|
276
|
+
};
|
|
277
|
+
this.active.set(id, activeEntry);
|
|
278
|
+
|
|
279
|
+
await this.runAgentTurn(id, agent.id, message, sessionId, from, activeEntry);
|
|
280
|
+
}
|
|
193
281
|
|
|
282
|
+
/** Handle incoming handoff_input (resume agent with user reply). */
|
|
283
|
+
async handleInput(frame: HandoffInput): Promise<void> {
|
|
284
|
+
const entry = this.active.get(frame.id);
|
|
285
|
+
if (!entry || entry.status !== "input_required") return;
|
|
286
|
+
|
|
287
|
+
// Verify the input comes from the original requester
|
|
288
|
+
if (frame.from !== entry.from) return;
|
|
289
|
+
|
|
290
|
+
entry.status = "working";
|
|
291
|
+
await this.runAgentTurn(frame.id, entry.agent, frame.payload.message, entry.sessionId, entry.from, entry);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Shared logic: spawn agent subprocess, stream output, handle input_required or completion. */
|
|
295
|
+
private async runAgentTurn(
|
|
296
|
+
id: string,
|
|
297
|
+
agent: string,
|
|
298
|
+
message: string,
|
|
299
|
+
sessionId: string,
|
|
300
|
+
from: string,
|
|
301
|
+
activeEntry: ActiveHandoff,
|
|
302
|
+
): Promise<void> {
|
|
303
|
+
try {
|
|
194
304
|
const proc = spawnProcess(
|
|
195
|
-
["openclaw", "agent", "--agent", agent
|
|
305
|
+
["openclaw", "agent", "--agent", agent, "--message", message, "--session-id", sessionId],
|
|
196
306
|
{ stdout: "pipe", stderr: "pipe" },
|
|
197
307
|
);
|
|
308
|
+
activeEntry.proc = proc;
|
|
309
|
+
|
|
310
|
+
// If cancel arrived between active.set() and proc assignment, kill immediately
|
|
311
|
+
if (activeEntry.status === "canceled") {
|
|
312
|
+
proc.kill();
|
|
313
|
+
this.active.delete(id);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Consume stderr concurrently to prevent pipe buffer deadlock
|
|
318
|
+
const stderrPromise = proc.stderr ? drainStream(proc.stderr) : Promise.resolve("");
|
|
319
|
+
const { output, inputRequired } = await this.streamStdout(proc.stdout, id, from);
|
|
320
|
+
const [exitCode, stderr] = await Promise.all([proc.exited, stderrPromise]);
|
|
321
|
+
|
|
322
|
+
if (activeEntry.status === "canceled") {
|
|
323
|
+
this.active.delete(id);
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
198
326
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
327
|
+
if (inputRequired) {
|
|
328
|
+
const marker = "[INPUT_REQUIRED]";
|
|
329
|
+
const question = output.slice(output.indexOf(marker) + marker.length).trim();
|
|
330
|
+
activeEntry.status = "input_required";
|
|
331
|
+
activeEntry.inputRequiredAt = Date.now();
|
|
332
|
+
|
|
333
|
+
this.peerManager.sendTo(from, {
|
|
334
|
+
type: "handoff_input_required",
|
|
335
|
+
id,
|
|
336
|
+
from: this.config.nodeId,
|
|
337
|
+
to: from,
|
|
338
|
+
timestamp: Date.now(),
|
|
339
|
+
payload: { message: question },
|
|
340
|
+
} satisfies HandoffInputRequired);
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
202
343
|
|
|
203
344
|
if (exitCode !== 0) {
|
|
204
|
-
const stderr = proc.stderr ? await new Response(proc.stderr).text() : "";
|
|
205
345
|
throw new Error(`Agent exited with code ${exitCode}: ${stderr}`);
|
|
206
346
|
}
|
|
207
347
|
|
|
348
|
+
activeEntry.status = "completed";
|
|
349
|
+
this.active.delete(id);
|
|
350
|
+
|
|
208
351
|
this.peerManager.sendTo(from, {
|
|
209
352
|
type: "handoff_res",
|
|
210
353
|
id,
|
|
@@ -214,11 +357,18 @@ export class HandoffManager {
|
|
|
214
357
|
payload: {
|
|
215
358
|
success: true,
|
|
216
359
|
nodeId: this.config.nodeId,
|
|
217
|
-
agent
|
|
218
|
-
result:
|
|
360
|
+
agent,
|
|
361
|
+
result: output.trim(),
|
|
219
362
|
},
|
|
220
363
|
} satisfies HandoffResponse);
|
|
221
364
|
} catch (err) {
|
|
365
|
+
if (activeEntry.status === "canceled") {
|
|
366
|
+
this.active.delete(id);
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
activeEntry.status = "failed";
|
|
370
|
+
this.active.delete(id);
|
|
371
|
+
|
|
222
372
|
this.peerManager.sendTo(from, {
|
|
223
373
|
type: "handoff_res",
|
|
224
374
|
id,
|
|
@@ -228,7 +378,7 @@ export class HandoffManager {
|
|
|
228
378
|
payload: {
|
|
229
379
|
success: false,
|
|
230
380
|
nodeId: this.config.nodeId,
|
|
231
|
-
agent
|
|
381
|
+
agent,
|
|
232
382
|
error: err instanceof Error ? err.message : String(err),
|
|
233
383
|
},
|
|
234
384
|
} satisfies HandoffResponse);
|
|
@@ -240,8 +390,8 @@ export class HandoffManager {
|
|
|
240
390
|
stdout: ReadableStream | null,
|
|
241
391
|
handoffId: string,
|
|
242
392
|
to: string,
|
|
243
|
-
): Promise<string> {
|
|
244
|
-
if (!stdout) return "";
|
|
393
|
+
): Promise<{ output: string; inputRequired: boolean }> {
|
|
394
|
+
if (!stdout) return { output: "", inputRequired: false };
|
|
245
395
|
|
|
246
396
|
const reader = stdout.getReader();
|
|
247
397
|
const decoder = new TextDecoder();
|
|
@@ -268,6 +418,11 @@ export class HandoffManager {
|
|
|
268
418
|
reader.releaseLock();
|
|
269
419
|
}
|
|
270
420
|
|
|
421
|
+
const INPUT_MARKER = "[INPUT_REQUIRED]";
|
|
422
|
+
if (full.indexOf(INPUT_MARKER) !== -1) {
|
|
423
|
+
return { output: full, inputRequired: true };
|
|
424
|
+
}
|
|
425
|
+
|
|
271
426
|
// Send final done marker
|
|
272
427
|
this.peerManager.sendTo(to, {
|
|
273
428
|
type: "handoff_stream",
|
|
@@ -278,15 +433,169 @@ export class HandoffManager {
|
|
|
278
433
|
payload: { delta: "", done: true },
|
|
279
434
|
} satisfies HandoffStreamChunk);
|
|
280
435
|
|
|
281
|
-
return full;
|
|
436
|
+
return { output: full, inputRequired: false };
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/** Handle incoming input_required from remote (requester side). */
|
|
440
|
+
handleInputRequired(frame: HandoffInputRequired) {
|
|
441
|
+
const pending = this.pending.get(frame.id);
|
|
442
|
+
if (!pending) return;
|
|
443
|
+
|
|
444
|
+
clearTimeout(pending.timer);
|
|
445
|
+
this.pending.delete(frame.id);
|
|
446
|
+
|
|
447
|
+
// Cache target for sendHandoffInput
|
|
448
|
+
this.inputRequiredTargets.set(frame.id, frame.from);
|
|
449
|
+
|
|
450
|
+
pending.resolve({
|
|
451
|
+
success: true,
|
|
452
|
+
nodeId: frame.from,
|
|
453
|
+
result: frame.payload.message,
|
|
454
|
+
inputRequired: true,
|
|
455
|
+
handoffId: frame.id,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** Send a reply to a remote agent that requested more input (requester side). */
|
|
460
|
+
sendHandoffInput(handoffId: string, message: string): Promise<HandoffResponse["payload"]> {
|
|
461
|
+
const targetNodeId = this.inputRequiredTargets.get(handoffId);
|
|
462
|
+
if (!targetNodeId) throw new Error(`No pending input_required for handoff ${handoffId}`);
|
|
463
|
+
|
|
464
|
+
this.inputRequiredTargets.delete(handoffId);
|
|
465
|
+
|
|
466
|
+
return new Promise<HandoffResponse["payload"]>((resolve, reject) => {
|
|
467
|
+
const timer = setTimeout(() => {
|
|
468
|
+
this.pending.delete(handoffId);
|
|
469
|
+
reject(new Error("Handoff reply timed out"));
|
|
470
|
+
}, this.config.handoffTimeout ?? DEFAULT_HANDOFF_TIMEOUT);
|
|
471
|
+
|
|
472
|
+
this.pending.set(handoffId, {
|
|
473
|
+
resolve, reject, timer,
|
|
474
|
+
target: "", targetNodeId,
|
|
475
|
+
retriesLeft: 0, task: message, accumulated: "",
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
this.peerManager.sendTo(targetNodeId, {
|
|
479
|
+
type: "handoff_input",
|
|
480
|
+
id: handoffId,
|
|
481
|
+
from: this.config.nodeId,
|
|
482
|
+
to: targetNodeId,
|
|
483
|
+
timestamp: Date.now(),
|
|
484
|
+
payload: { message },
|
|
485
|
+
} as HandoffInput);
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
/** Handle incoming cancel request (receiver side — kill subprocess). */
|
|
490
|
+
handleCancel(frame: HandoffCancel) {
|
|
491
|
+
const entry = this.active.get(frame.id);
|
|
492
|
+
if (!entry || (entry.status !== "working" && entry.status !== "input_required")) return;
|
|
493
|
+
|
|
494
|
+
// Verify the cancel comes from the original requester
|
|
495
|
+
if (frame.from !== entry.from) return;
|
|
496
|
+
|
|
497
|
+
const wasInputRequired = entry.status === "input_required";
|
|
498
|
+
entry.status = "canceled";
|
|
499
|
+
entry.proc?.kill();
|
|
500
|
+
|
|
501
|
+
// If canceled during input_required, no runAgentTurn is running to clean up
|
|
502
|
+
if (wasInputRequired) {
|
|
503
|
+
this.active.delete(frame.id);
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
this.peerManager.sendTo(entry.from, {
|
|
507
|
+
type: "handoff_res",
|
|
508
|
+
id: frame.id,
|
|
509
|
+
from: this.config.nodeId,
|
|
510
|
+
to: entry.from,
|
|
511
|
+
timestamp: Date.now(),
|
|
512
|
+
payload: {
|
|
513
|
+
success: false,
|
|
514
|
+
nodeId: this.config.nodeId,
|
|
515
|
+
agent: entry.agent,
|
|
516
|
+
error: "canceled",
|
|
517
|
+
},
|
|
518
|
+
} satisfies HandoffResponse);
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/** Handle incoming status query (receiver side — report current state). */
|
|
522
|
+
handleStatusQuery(frame: HandoffStatusQuery) {
|
|
523
|
+
const entry = this.active.get(frame.id);
|
|
524
|
+
if (!entry) return;
|
|
525
|
+
|
|
526
|
+
this.peerManager.sendTo(frame.from, {
|
|
527
|
+
type: "handoff_status_res",
|
|
528
|
+
id: frame.id,
|
|
529
|
+
from: this.config.nodeId,
|
|
530
|
+
to: frame.from,
|
|
531
|
+
timestamp: Date.now(),
|
|
532
|
+
payload: {
|
|
533
|
+
status: entry.status,
|
|
534
|
+
nodeId: this.config.nodeId,
|
|
535
|
+
agent: entry.agent,
|
|
536
|
+
elapsedMs: Date.now() - entry.startedAt,
|
|
537
|
+
},
|
|
538
|
+
} satisfies HandoffStatusResponse);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/** Cancel an outgoing handoff (requester side — send cancel to remote). */
|
|
542
|
+
cancelHandoff(handoffId: string): boolean {
|
|
543
|
+
const pending = this.pending.get(handoffId);
|
|
544
|
+
if (!pending) return false;
|
|
545
|
+
|
|
546
|
+
clearTimeout(pending.timer);
|
|
547
|
+
this.pending.delete(handoffId);
|
|
548
|
+
pending.reject(new Error("Handoff canceled by requester"));
|
|
549
|
+
|
|
550
|
+
// Send cancel frame to the remote node
|
|
551
|
+
this.peerManager.sendTo(pending.targetNodeId, {
|
|
552
|
+
type: "handoff_cancel",
|
|
553
|
+
id: handoffId,
|
|
554
|
+
from: this.config.nodeId,
|
|
555
|
+
to: pending.targetNodeId,
|
|
556
|
+
timestamp: Date.now(),
|
|
557
|
+
} as HandoffCancel);
|
|
558
|
+
|
|
559
|
+
return true;
|
|
282
560
|
}
|
|
283
561
|
|
|
284
562
|
/** Clean up on shutdown. */
|
|
285
563
|
destroy() {
|
|
564
|
+
if (this.staleCleanupTimer) {
|
|
565
|
+
clearInterval(this.staleCleanupTimer);
|
|
566
|
+
this.staleCleanupTimer = null;
|
|
567
|
+
}
|
|
568
|
+
|
|
286
569
|
for (const [, pending] of this.pending) {
|
|
287
570
|
clearTimeout(pending.timer);
|
|
288
571
|
pending.reject(new Error("Shutting down"));
|
|
289
572
|
}
|
|
290
573
|
this.pending.clear();
|
|
574
|
+
|
|
575
|
+
// Kill all active tasks (working and input_required)
|
|
576
|
+
for (const [, entry] of this.active) {
|
|
577
|
+
if (entry.status === "working" || entry.status === "input_required") {
|
|
578
|
+
entry.proc?.kill();
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
this.active.clear();
|
|
582
|
+
this.inputRequiredTargets.clear();
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/** Drain a ReadableStream into a string (for stderr consumption). */
|
|
587
|
+
async function drainStream(stream: ReadableStream): Promise<string> {
|
|
588
|
+
const reader = stream.getReader();
|
|
589
|
+
const decoder = new TextDecoder();
|
|
590
|
+
let result = "";
|
|
591
|
+
try {
|
|
592
|
+
while (true) {
|
|
593
|
+
const { done, value } = await reader.read();
|
|
594
|
+
if (done) break;
|
|
595
|
+
result += decoder.decode(value, { stream: true });
|
|
596
|
+
}
|
|
597
|
+
} finally {
|
|
598
|
+
reader.releaseLock();
|
|
291
599
|
}
|
|
600
|
+
return result;
|
|
292
601
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { IncomingMessage } from "node:http";
|
|
2
|
+
|
|
3
|
+
const MAX_BODY_SIZE = 1024 * 1024; // 1 MB
|
|
4
|
+
|
|
5
|
+
/** Read HTTP request body with size limit (default 1 MB). */
|
|
6
|
+
export function readBody(req: IncomingMessage, maxBytes = MAX_BODY_SIZE): Promise<string> {
|
|
7
|
+
return new Promise((resolve, reject) => {
|
|
8
|
+
const chunks: Buffer[] = [];
|
|
9
|
+
let size = 0;
|
|
10
|
+
let settled = false;
|
|
11
|
+
req.on("data", (chunk: Buffer) => {
|
|
12
|
+
if (settled) return;
|
|
13
|
+
size += chunk.length;
|
|
14
|
+
if (size > maxBytes) {
|
|
15
|
+
settled = true;
|
|
16
|
+
req.destroy();
|
|
17
|
+
reject(new Error("Request body too large"));
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
chunks.push(chunk);
|
|
21
|
+
});
|
|
22
|
+
req.on("end", () => {
|
|
23
|
+
if (!settled) {
|
|
24
|
+
settled = true;
|
|
25
|
+
resolve(Buffer.concat(chunks).toString());
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
req.on("error", (err) => {
|
|
29
|
+
if (!settled) {
|
|
30
|
+
settled = true;
|
|
31
|
+
reject(err);
|
|
32
|
+
}
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -2,12 +2,14 @@ import type { OpenClawPluginApi, GatewayRequestHandlerOptions } from "openclaw/p
|
|
|
2
2
|
import { ClawMatrixConfigSchema, parseConfig } from "./config.ts";
|
|
3
3
|
import { createClusterService, getClusterRuntime } from "./cluster-service.ts";
|
|
4
4
|
import { createClusterHandoffTool } from "./tools/cluster-handoff.ts";
|
|
5
|
+
import { createClusterHandoffReplyTool } from "./tools/cluster-handoff-reply.ts";
|
|
5
6
|
import { createClusterSendTool } from "./tools/cluster-send.ts";
|
|
6
7
|
import { createClusterPeersTool } from "./tools/cluster-peers.ts";
|
|
7
8
|
import { createClusterExecTool } from "./tools/cluster-exec.ts";
|
|
8
9
|
import { createClusterReadTool } from "./tools/cluster-read.ts";
|
|
9
10
|
import { createClusterWriteTool } from "./tools/cluster-write.ts";
|
|
10
11
|
import { createClusterToolTool } from "./tools/cluster-tool.ts";
|
|
12
|
+
import { createClusterEventsTool } from "./tools/cluster-events.ts";
|
|
11
13
|
import { registerClusterCli } from "./cli.ts";
|
|
12
14
|
|
|
13
15
|
const plugin = {
|
|
@@ -39,7 +41,7 @@ const plugin = {
|
|
|
39
41
|
const config = parseConfig(api.pluginConfig);
|
|
40
42
|
|
|
41
43
|
// Background service: manages mesh connections, WS listener, heartbeat
|
|
42
|
-
api.registerService(createClusterService(config, api.config));
|
|
44
|
+
api.registerService(createClusterService(config, api.config, api.runtime.version));
|
|
43
45
|
|
|
44
46
|
// Model providers: register per-node providers so models are accessed as nodeId/modelId
|
|
45
47
|
const baseUrl = `http://127.0.0.1:${config.proxyPort}/v1`;
|
|
@@ -50,12 +52,21 @@ const plugin = {
|
|
|
50
52
|
// We must patch BOTH api.config (cfgAtStart) AND the runtimeConfigSnapshot
|
|
51
53
|
// (returned by loadConfig()) because activateSecretsRuntimeSnapshot clones the
|
|
52
54
|
// config before plugins load, so api.config and the snapshot are separate objects.
|
|
55
|
+
// Determine per-node API type from proxyModels (all models in a group share the same api)
|
|
56
|
+
const nodeApiType: Record<string, string> = {};
|
|
57
|
+
for (const m of config.proxyModels) {
|
|
58
|
+
if (m.api && !nodeApiType[m.nodeId]) {
|
|
59
|
+
nodeApiType[m.nodeId] = m.api;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
53
63
|
const patchProviders = (cfg: Record<string, unknown>) => {
|
|
54
64
|
const models = ((cfg).models ??= {}) as Record<string, unknown>;
|
|
55
65
|
const providers = (models.providers ??= {}) as Record<string, unknown>;
|
|
56
66
|
for (const [nodeId, nodeModels] of Object.entries(modelsByNode)) {
|
|
57
67
|
if (!providers[nodeId]) {
|
|
58
|
-
|
|
68
|
+
const api = nodeApiType[nodeId] ?? "openai-completions";
|
|
69
|
+
providers[nodeId] = { baseUrl, apiKey: "sk-clawmatrix-proxy", api, models: nodeModels };
|
|
59
70
|
}
|
|
60
71
|
}
|
|
61
72
|
};
|
|
@@ -63,7 +74,8 @@ const plugin = {
|
|
|
63
74
|
patchProviders(api.config as Record<string, unknown>);
|
|
64
75
|
|
|
65
76
|
// Also patch the runtime config snapshot (loadConfig returns it by reference).
|
|
66
|
-
// api.
|
|
77
|
+
// activateSecretsRuntimeSnapshot clones the config, so api.config and the
|
|
78
|
+
// snapshot returned by loadConfig() are separate objects — patch both.
|
|
67
79
|
try {
|
|
68
80
|
const snapshot = api.runtime.config.loadConfig();
|
|
69
81
|
if (snapshot && snapshot !== api.config) {
|
|
@@ -89,12 +101,14 @@ const plugin = {
|
|
|
89
101
|
|
|
90
102
|
// Agent tools
|
|
91
103
|
api.registerTool(createClusterHandoffTool(), { optional: true });
|
|
104
|
+
api.registerTool(createClusterHandoffReplyTool(), { optional: true });
|
|
92
105
|
api.registerTool(createClusterSendTool(), { optional: true });
|
|
93
106
|
api.registerTool(createClusterPeersTool(), { optional: true });
|
|
94
107
|
api.registerTool(createClusterExecTool(), { optional: true });
|
|
95
108
|
api.registerTool(createClusterReadTool(), { optional: true });
|
|
96
109
|
api.registerTool(createClusterWriteTool(), { optional: true });
|
|
97
110
|
api.registerTool(createClusterToolTool(), { optional: true });
|
|
111
|
+
api.registerTool(createClusterEventsTool(), { optional: true });
|
|
98
112
|
|
|
99
113
|
// Gateway methods (queried by CLI via `openclaw gateway call`)
|
|
100
114
|
api.registerGatewayMethod(
|
|
@@ -167,6 +181,10 @@ const plugin = {
|
|
|
167
181
|
lines.push(`Your role: ${localAgent.description}`);
|
|
168
182
|
}
|
|
169
183
|
|
|
184
|
+
// Satellite nodes (via WebHandler on relay, or gossiped via peer_sync)
|
|
185
|
+
const satellites = runtime.webHandler?.getSatelliteContexts() ?? runtime.peerManager.satelliteContexts;
|
|
186
|
+
const activeSatellites = satellites.filter(s => Date.now() - s.ts < 600_000);
|
|
187
|
+
|
|
170
188
|
lines.push("", "Remote nodes in the cluster:");
|
|
171
189
|
for (const peer of peers) {
|
|
172
190
|
const status = peer.connection?.isOpen ? "connected" : "via relay";
|
|
@@ -179,27 +197,37 @@ const plugin = {
|
|
|
179
197
|
lines.push(` models: ${peer.models.map((m) => m.id).join(", ")}`);
|
|
180
198
|
}
|
|
181
199
|
}
|
|
200
|
+
for (const sat of activeSatellites) {
|
|
201
|
+
const age = Math.floor((Date.now() - sat.ts) / 1000);
|
|
202
|
+
const country = sat.country ? `, ${sat.country}` : "";
|
|
203
|
+
lines.push(` - ${sat.nodeId} (satellite${country}, ${age}s ago)`);
|
|
204
|
+
if (sat.tools?.length) {
|
|
205
|
+
lines.push(` tools: ${sat.tools.join(", ")}`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Unconsumed events from external sources (Shortcuts automations, etc.)
|
|
210
|
+
const pendingEvents = runtime.webHandler?.getUnconsumedEvents(5) ?? [];
|
|
211
|
+
if (pendingEvents.length > 0) {
|
|
212
|
+
lines.push("", "⚡ Pending events:");
|
|
213
|
+
for (const evt of pendingEvents) {
|
|
214
|
+
const age = Math.floor((Date.now() - evt.ts) / 1000);
|
|
215
|
+
const dataStr = Object.entries(evt.data)
|
|
216
|
+
.map(([k, v]) => `${k}: ${typeof v === "string" ? v : JSON.stringify(v)}`)
|
|
217
|
+
.join(", ");
|
|
218
|
+
const truncated = dataStr.length > 120 ? dataStr.slice(0, 120) + "…" : dataStr;
|
|
219
|
+
lines.push(` - [${evt.type}] ${evt.source} (${age}s ago, id:${evt.id}): ${truncated}`);
|
|
220
|
+
}
|
|
221
|
+
lines.push("Use cluster_events for details or to mark consumed.");
|
|
222
|
+
}
|
|
182
223
|
|
|
183
224
|
lines.push(
|
|
184
225
|
"",
|
|
185
|
-
"
|
|
186
|
-
|
|
187
|
-
" - cluster_exec / cluster_read / cluster_write — run commands, read/write files on a remote node",
|
|
188
|
-
" - cluster_tool — invoke any OpenClaw tool on a remote node",
|
|
189
|
-
" - cluster_handoff — delegate a complex task to a remote agent for autonomous execution",
|
|
190
|
-
" - cluster_send — send a one-way message to a remote agent",
|
|
191
|
-
" - cluster_peers — inspect cluster topology",
|
|
192
|
-
"Use the node's description and tags to decide which node to target. " +
|
|
193
|
-
"For simple operations, prefer cluster_exec/read/write; for complex multi-step tasks, prefer cluster_handoff.",
|
|
194
|
-
"",
|
|
195
|
-
"IMPORTANT: Before calling any cluster tool or using a remote model, you MUST explicitly tell the user " +
|
|
196
|
-
"which remote node you are targeting and what you are about to do. For example:",
|
|
197
|
-
' "I\'m going to run this command on remote node «coder» ..."',
|
|
198
|
-
' "I\'m delegating this task to agent «reviewer» on node «server-b» ..."',
|
|
199
|
-
"This ensures the user always knows when operations leave the local node.",
|
|
226
|
+
"Prefer cluster_exec/read/write for simple ops; cluster_handoff for complex multi-step tasks.",
|
|
227
|
+
"IMPORTANT: Always tell user which remote node you're targeting before calling cluster tools.",
|
|
200
228
|
);
|
|
201
229
|
|
|
202
|
-
return {
|
|
230
|
+
return { prependSystemContext: lines.join("\n") };
|
|
203
231
|
} catch {
|
|
204
232
|
return;
|
|
205
233
|
}
|