clawmatrix 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BOOTSTRAP.md +17 -2
- package/package.json +1 -1
- package/src/auth.ts +42 -12
- package/src/cluster-service.ts +31 -4
- package/src/compat.ts +3 -0
- package/src/connection.ts +15 -6
- package/src/handoff.ts +311 -17
- package/src/http-utils.ts +35 -0
- package/src/index.ts +33 -15
- package/src/model-proxy.ts +19 -16
- package/src/peer-manager.ts +55 -5
- package/src/router.ts +62 -28
- package/src/tool-proxy.ts +22 -7
- package/src/tools/cluster-events.ts +119 -0
- package/src/tools/cluster-exec.ts +4 -0
- package/src/tools/cluster-handoff-reply.ts +77 -0
- package/src/tools/cluster-handoff.ts +12 -0
- package/src/tools/cluster-peers.ts +17 -1
- package/src/tools/cluster-send.ts +1 -3
- package/src/tools/cluster-tool.ts +2 -5
- package/src/types.ts +93 -0
- package/src/web-ui.ts +490 -345
- package/src/web.ts +675 -53
package/src/handoff.ts
CHANGED
|
@@ -5,16 +5,25 @@ import type {
|
|
|
5
5
|
HandoffRequest,
|
|
6
6
|
HandoffResponse,
|
|
7
7
|
HandoffStreamChunk,
|
|
8
|
+
HandoffCancel,
|
|
9
|
+
HandoffStatusQuery,
|
|
10
|
+
HandoffStatusResponse,
|
|
11
|
+
HandoffInputRequired,
|
|
12
|
+
HandoffInput,
|
|
13
|
+
HandoffStatus,
|
|
8
14
|
} from "./types.ts";
|
|
9
15
|
|
|
10
16
|
const DEFAULT_HANDOFF_TIMEOUT = 600_000; // 10 minutes (resets on each stream chunk)
|
|
11
17
|
const MAX_RETRIES = 2;
|
|
18
|
+
const INPUT_REQUIRED_TTL = 1_800_000; // 30 minutes — max time to wait for user reply
|
|
19
|
+
const STALE_CLEANUP_INTERVAL = 120_000; // check every 2 minutes
|
|
12
20
|
|
|
13
21
|
interface PendingHandoff {
|
|
14
22
|
resolve: (result: HandoffResponse["payload"]) => void;
|
|
15
23
|
reject: (error: Error) => void;
|
|
16
24
|
timer: ReturnType<typeof setTimeout>;
|
|
17
25
|
target: string;
|
|
26
|
+
targetNodeId: string;
|
|
18
27
|
retriesLeft: number;
|
|
19
28
|
task: string;
|
|
20
29
|
context?: string;
|
|
@@ -22,14 +31,65 @@ interface PendingHandoff {
|
|
|
22
31
|
onStream?: (delta: string) => void;
|
|
23
32
|
}
|
|
24
33
|
|
|
34
|
+
interface ActiveHandoff {
|
|
35
|
+
proc: { kill: () => void; exited: Promise<number> } | null;
|
|
36
|
+
status: HandoffStatus;
|
|
37
|
+
sessionId: string;
|
|
38
|
+
agent: string;
|
|
39
|
+
startedAt: number;
|
|
40
|
+
inputRequiredAt: number | null; // when status changed to input_required
|
|
41
|
+
from: string; // requesting nodeId
|
|
42
|
+
}
|
|
43
|
+
|
|
25
44
|
export class HandoffManager {
|
|
26
45
|
private config: ClawMatrixConfig;
|
|
27
46
|
private peerManager: PeerManager;
|
|
28
47
|
private pending = new Map<string, PendingHandoff>();
|
|
48
|
+
private active = new Map<string, ActiveHandoff>();
|
|
49
|
+
private inputRequiredTargets = new Map<string, string>(); // handoffId → targetNodeId
|
|
50
|
+
private staleCleanupTimer: ReturnType<typeof setInterval> | null = null;
|
|
29
51
|
|
|
30
52
|
constructor(config: ClawMatrixConfig, peerManager: PeerManager) {
|
|
31
53
|
this.config = config;
|
|
32
54
|
this.peerManager = peerManager;
|
|
55
|
+
|
|
56
|
+
// Periodically clean up stale input_required entries
|
|
57
|
+
this.staleCleanupTimer = setInterval(() => this.cleanupStale(), STALE_CLEANUP_INTERVAL);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Remove stale input_required and canceled entries that exceeded TTL. */
|
|
61
|
+
private cleanupStale() {
|
|
62
|
+
const now = Date.now();
|
|
63
|
+
for (const [id, entry] of this.active) {
|
|
64
|
+
if (entry.status === "input_required" && entry.inputRequiredAt && now - entry.inputRequiredAt > INPUT_REQUIRED_TTL) {
|
|
65
|
+
this.active.delete(id);
|
|
66
|
+
// Notify the requester that the handoff timed out
|
|
67
|
+
this.peerManager.sendTo(entry.from, {
|
|
68
|
+
type: "handoff_res",
|
|
69
|
+
id,
|
|
70
|
+
from: this.config.nodeId,
|
|
71
|
+
to: entry.from,
|
|
72
|
+
timestamp: now,
|
|
73
|
+
payload: {
|
|
74
|
+
success: false,
|
|
75
|
+
nodeId: this.config.nodeId,
|
|
76
|
+
agent: entry.agent,
|
|
77
|
+
error: "Handoff timed out waiting for input",
|
|
78
|
+
},
|
|
79
|
+
} satisfies HandoffResponse);
|
|
80
|
+
}
|
|
81
|
+
// Clean up canceled entries that were never cleaned by runAgentTurn
|
|
82
|
+
// (e.g. cancel arrived during input_required when no process was running)
|
|
83
|
+
if (entry.status === "canceled") {
|
|
84
|
+
this.active.delete(id);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Clean stale inputRequiredTargets (requester side)
|
|
88
|
+
for (const [handoffId] of this.inputRequiredTargets) {
|
|
89
|
+
if (!this.pending.has(handoffId)) {
|
|
90
|
+
this.inputRequiredTargets.delete(handoffId);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
33
93
|
}
|
|
34
94
|
|
|
35
95
|
/** Send a handoff request and wait for the response. */
|
|
@@ -65,7 +125,7 @@ export class HandoffManager {
|
|
|
65
125
|
return new Promise<HandoffResponse["payload"]>((resolve, reject) => {
|
|
66
126
|
const timer = this.createTimeout(id, targetNodeId, target, task, context, retriesLeft, resolve, reject, onStream);
|
|
67
127
|
|
|
68
|
-
this.pending.set(id, { resolve, reject, timer, target, retriesLeft, task, context, accumulated: "", onStream });
|
|
128
|
+
this.pending.set(id, { resolve, reject, timer, target, targetNodeId, retriesLeft, task, context, accumulated: "", onStream });
|
|
69
129
|
|
|
70
130
|
const frame: HandoffRequest = {
|
|
71
131
|
type: "handoff_req",
|
|
@@ -200,26 +260,94 @@ export class HandoffManager {
|
|
|
200
260
|
return;
|
|
201
261
|
}
|
|
202
262
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
263
|
+
const sessionId = `handoff-${id}`;
|
|
264
|
+
const message = payload.context
|
|
265
|
+
? `${payload.task}\n\nContext:\n${payload.context}`
|
|
266
|
+
: payload.task;
|
|
267
|
+
|
|
268
|
+
const activeEntry: ActiveHandoff = {
|
|
269
|
+
proc: null,
|
|
270
|
+
status: "working",
|
|
271
|
+
sessionId,
|
|
272
|
+
agent: agent.id,
|
|
273
|
+
startedAt: Date.now(),
|
|
274
|
+
inputRequiredAt: null,
|
|
275
|
+
from,
|
|
276
|
+
};
|
|
277
|
+
this.active.set(id, activeEntry);
|
|
278
|
+
|
|
279
|
+
await this.runAgentTurn(id, agent.id, message, sessionId, from, activeEntry);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** Handle incoming handoff_input (resume agent with user reply). */
|
|
283
|
+
async handleInput(frame: HandoffInput): Promise<void> {
|
|
284
|
+
const entry = this.active.get(frame.id);
|
|
285
|
+
if (!entry || entry.status !== "input_required") return;
|
|
286
|
+
|
|
287
|
+
// Verify the input comes from the original requester
|
|
288
|
+
if (frame.from !== entry.from) return;
|
|
289
|
+
|
|
290
|
+
entry.status = "working";
|
|
291
|
+
await this.runAgentTurn(frame.id, entry.agent, frame.payload.message, entry.sessionId, entry.from, entry);
|
|
292
|
+
}
|
|
208
293
|
|
|
294
|
+
/** Shared logic: spawn agent subprocess, stream output, handle input_required or completion. */
|
|
295
|
+
private async runAgentTurn(
|
|
296
|
+
id: string,
|
|
297
|
+
agent: string,
|
|
298
|
+
message: string,
|
|
299
|
+
sessionId: string,
|
|
300
|
+
from: string,
|
|
301
|
+
activeEntry: ActiveHandoff,
|
|
302
|
+
): Promise<void> {
|
|
303
|
+
try {
|
|
209
304
|
const proc = spawnProcess(
|
|
210
|
-
["openclaw", "agent", "--agent", agent
|
|
305
|
+
["openclaw", "agent", "--agent", agent, "--message", message, "--session-id", sessionId],
|
|
211
306
|
{ stdout: "pipe", stderr: "pipe" },
|
|
212
307
|
);
|
|
308
|
+
activeEntry.proc = proc;
|
|
309
|
+
|
|
310
|
+
// If cancel arrived between active.set() and proc assignment, kill immediately
|
|
311
|
+
if (activeEntry.status === "canceled") {
|
|
312
|
+
proc.kill();
|
|
313
|
+
this.active.delete(id);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
213
316
|
|
|
214
|
-
//
|
|
215
|
-
const
|
|
216
|
-
const
|
|
317
|
+
// Consume stderr concurrently to prevent pipe buffer deadlock
|
|
318
|
+
const stderrPromise = proc.stderr ? drainStream(proc.stderr) : Promise.resolve("");
|
|
319
|
+
const { output, inputRequired } = await this.streamStdout(proc.stdout, id, from);
|
|
320
|
+
const [exitCode, stderr] = await Promise.all([proc.exited, stderrPromise]);
|
|
321
|
+
|
|
322
|
+
if (activeEntry.status === "canceled") {
|
|
323
|
+
this.active.delete(id);
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
if (inputRequired) {
|
|
328
|
+
const marker = "[INPUT_REQUIRED]";
|
|
329
|
+
const question = output.slice(output.indexOf(marker) + marker.length).trim();
|
|
330
|
+
activeEntry.status = "input_required";
|
|
331
|
+
activeEntry.inputRequiredAt = Date.now();
|
|
332
|
+
|
|
333
|
+
this.peerManager.sendTo(from, {
|
|
334
|
+
type: "handoff_input_required",
|
|
335
|
+
id,
|
|
336
|
+
from: this.config.nodeId,
|
|
337
|
+
to: from,
|
|
338
|
+
timestamp: Date.now(),
|
|
339
|
+
payload: { message: question },
|
|
340
|
+
} satisfies HandoffInputRequired);
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
217
343
|
|
|
218
344
|
if (exitCode !== 0) {
|
|
219
|
-
const stderr = proc.stderr ? await new Response(proc.stderr).text() : "";
|
|
220
345
|
throw new Error(`Agent exited with code ${exitCode}: ${stderr}`);
|
|
221
346
|
}
|
|
222
347
|
|
|
348
|
+
activeEntry.status = "completed";
|
|
349
|
+
this.active.delete(id);
|
|
350
|
+
|
|
223
351
|
this.peerManager.sendTo(from, {
|
|
224
352
|
type: "handoff_res",
|
|
225
353
|
id,
|
|
@@ -229,11 +357,18 @@ export class HandoffManager {
|
|
|
229
357
|
payload: {
|
|
230
358
|
success: true,
|
|
231
359
|
nodeId: this.config.nodeId,
|
|
232
|
-
agent
|
|
233
|
-
result:
|
|
360
|
+
agent,
|
|
361
|
+
result: output.trim(),
|
|
234
362
|
},
|
|
235
363
|
} satisfies HandoffResponse);
|
|
236
364
|
} catch (err) {
|
|
365
|
+
if (activeEntry.status === "canceled") {
|
|
366
|
+
this.active.delete(id);
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
activeEntry.status = "failed";
|
|
370
|
+
this.active.delete(id);
|
|
371
|
+
|
|
237
372
|
this.peerManager.sendTo(from, {
|
|
238
373
|
type: "handoff_res",
|
|
239
374
|
id,
|
|
@@ -243,7 +378,7 @@ export class HandoffManager {
|
|
|
243
378
|
payload: {
|
|
244
379
|
success: false,
|
|
245
380
|
nodeId: this.config.nodeId,
|
|
246
|
-
agent
|
|
381
|
+
agent,
|
|
247
382
|
error: err instanceof Error ? err.message : String(err),
|
|
248
383
|
},
|
|
249
384
|
} satisfies HandoffResponse);
|
|
@@ -255,8 +390,8 @@ export class HandoffManager {
|
|
|
255
390
|
stdout: ReadableStream | null,
|
|
256
391
|
handoffId: string,
|
|
257
392
|
to: string,
|
|
258
|
-
): Promise<string> {
|
|
259
|
-
if (!stdout) return "";
|
|
393
|
+
): Promise<{ output: string; inputRequired: boolean }> {
|
|
394
|
+
if (!stdout) return { output: "", inputRequired: false };
|
|
260
395
|
|
|
261
396
|
const reader = stdout.getReader();
|
|
262
397
|
const decoder = new TextDecoder();
|
|
@@ -283,6 +418,11 @@ export class HandoffManager {
|
|
|
283
418
|
reader.releaseLock();
|
|
284
419
|
}
|
|
285
420
|
|
|
421
|
+
const INPUT_MARKER = "[INPUT_REQUIRED]";
|
|
422
|
+
if (full.indexOf(INPUT_MARKER) !== -1) {
|
|
423
|
+
return { output: full, inputRequired: true };
|
|
424
|
+
}
|
|
425
|
+
|
|
286
426
|
// Send final done marker
|
|
287
427
|
this.peerManager.sendTo(to, {
|
|
288
428
|
type: "handoff_stream",
|
|
@@ -293,15 +433,169 @@ export class HandoffManager {
|
|
|
293
433
|
payload: { delta: "", done: true },
|
|
294
434
|
} satisfies HandoffStreamChunk);
|
|
295
435
|
|
|
296
|
-
return full;
|
|
436
|
+
return { output: full, inputRequired: false };
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/** Handle incoming input_required from remote (requester side). */
|
|
440
|
+
handleInputRequired(frame: HandoffInputRequired) {
|
|
441
|
+
const pending = this.pending.get(frame.id);
|
|
442
|
+
if (!pending) return;
|
|
443
|
+
|
|
444
|
+
clearTimeout(pending.timer);
|
|
445
|
+
this.pending.delete(frame.id);
|
|
446
|
+
|
|
447
|
+
// Cache target for sendHandoffInput
|
|
448
|
+
this.inputRequiredTargets.set(frame.id, frame.from);
|
|
449
|
+
|
|
450
|
+
pending.resolve({
|
|
451
|
+
success: true,
|
|
452
|
+
nodeId: frame.from,
|
|
453
|
+
result: frame.payload.message,
|
|
454
|
+
inputRequired: true,
|
|
455
|
+
handoffId: frame.id,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** Send a reply to a remote agent that requested more input (requester side). */
|
|
460
|
+
sendHandoffInput(handoffId: string, message: string): Promise<HandoffResponse["payload"]> {
|
|
461
|
+
const targetNodeId = this.inputRequiredTargets.get(handoffId);
|
|
462
|
+
if (!targetNodeId) throw new Error(`No pending input_required for handoff ${handoffId}`);
|
|
463
|
+
|
|
464
|
+
this.inputRequiredTargets.delete(handoffId);
|
|
465
|
+
|
|
466
|
+
return new Promise<HandoffResponse["payload"]>((resolve, reject) => {
|
|
467
|
+
const timer = setTimeout(() => {
|
|
468
|
+
this.pending.delete(handoffId);
|
|
469
|
+
reject(new Error("Handoff reply timed out"));
|
|
470
|
+
}, this.config.handoffTimeout ?? DEFAULT_HANDOFF_TIMEOUT);
|
|
471
|
+
|
|
472
|
+
this.pending.set(handoffId, {
|
|
473
|
+
resolve, reject, timer,
|
|
474
|
+
target: "", targetNodeId,
|
|
475
|
+
retriesLeft: 0, task: message, accumulated: "",
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
this.peerManager.sendTo(targetNodeId, {
|
|
479
|
+
type: "handoff_input",
|
|
480
|
+
id: handoffId,
|
|
481
|
+
from: this.config.nodeId,
|
|
482
|
+
to: targetNodeId,
|
|
483
|
+
timestamp: Date.now(),
|
|
484
|
+
payload: { message },
|
|
485
|
+
} as HandoffInput);
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
/** Handle incoming cancel request (receiver side — kill subprocess). */
|
|
490
|
+
handleCancel(frame: HandoffCancel) {
|
|
491
|
+
const entry = this.active.get(frame.id);
|
|
492
|
+
if (!entry || (entry.status !== "working" && entry.status !== "input_required")) return;
|
|
493
|
+
|
|
494
|
+
// Verify the cancel comes from the original requester
|
|
495
|
+
if (frame.from !== entry.from) return;
|
|
496
|
+
|
|
497
|
+
const wasInputRequired = entry.status === "input_required";
|
|
498
|
+
entry.status = "canceled";
|
|
499
|
+
entry.proc?.kill();
|
|
500
|
+
|
|
501
|
+
// If canceled during input_required, no runAgentTurn is running to clean up
|
|
502
|
+
if (wasInputRequired) {
|
|
503
|
+
this.active.delete(frame.id);
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
this.peerManager.sendTo(entry.from, {
|
|
507
|
+
type: "handoff_res",
|
|
508
|
+
id: frame.id,
|
|
509
|
+
from: this.config.nodeId,
|
|
510
|
+
to: entry.from,
|
|
511
|
+
timestamp: Date.now(),
|
|
512
|
+
payload: {
|
|
513
|
+
success: false,
|
|
514
|
+
nodeId: this.config.nodeId,
|
|
515
|
+
agent: entry.agent,
|
|
516
|
+
error: "canceled",
|
|
517
|
+
},
|
|
518
|
+
} satisfies HandoffResponse);
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/** Handle incoming status query (receiver side — report current state). */
|
|
522
|
+
handleStatusQuery(frame: HandoffStatusQuery) {
|
|
523
|
+
const entry = this.active.get(frame.id);
|
|
524
|
+
if (!entry) return;
|
|
525
|
+
|
|
526
|
+
this.peerManager.sendTo(frame.from, {
|
|
527
|
+
type: "handoff_status_res",
|
|
528
|
+
id: frame.id,
|
|
529
|
+
from: this.config.nodeId,
|
|
530
|
+
to: frame.from,
|
|
531
|
+
timestamp: Date.now(),
|
|
532
|
+
payload: {
|
|
533
|
+
status: entry.status,
|
|
534
|
+
nodeId: this.config.nodeId,
|
|
535
|
+
agent: entry.agent,
|
|
536
|
+
elapsedMs: Date.now() - entry.startedAt,
|
|
537
|
+
},
|
|
538
|
+
} satisfies HandoffStatusResponse);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/** Cancel an outgoing handoff (requester side — send cancel to remote). */
|
|
542
|
+
cancelHandoff(handoffId: string): boolean {
|
|
543
|
+
const pending = this.pending.get(handoffId);
|
|
544
|
+
if (!pending) return false;
|
|
545
|
+
|
|
546
|
+
clearTimeout(pending.timer);
|
|
547
|
+
this.pending.delete(handoffId);
|
|
548
|
+
pending.reject(new Error("Handoff canceled by requester"));
|
|
549
|
+
|
|
550
|
+
// Send cancel frame to the remote node
|
|
551
|
+
this.peerManager.sendTo(pending.targetNodeId, {
|
|
552
|
+
type: "handoff_cancel",
|
|
553
|
+
id: handoffId,
|
|
554
|
+
from: this.config.nodeId,
|
|
555
|
+
to: pending.targetNodeId,
|
|
556
|
+
timestamp: Date.now(),
|
|
557
|
+
} as HandoffCancel);
|
|
558
|
+
|
|
559
|
+
return true;
|
|
297
560
|
}
|
|
298
561
|
|
|
299
562
|
/** Clean up on shutdown. */
|
|
300
563
|
destroy() {
|
|
564
|
+
if (this.staleCleanupTimer) {
|
|
565
|
+
clearInterval(this.staleCleanupTimer);
|
|
566
|
+
this.staleCleanupTimer = null;
|
|
567
|
+
}
|
|
568
|
+
|
|
301
569
|
for (const [, pending] of this.pending) {
|
|
302
570
|
clearTimeout(pending.timer);
|
|
303
571
|
pending.reject(new Error("Shutting down"));
|
|
304
572
|
}
|
|
305
573
|
this.pending.clear();
|
|
574
|
+
|
|
575
|
+
// Kill all active tasks (working and input_required)
|
|
576
|
+
for (const [, entry] of this.active) {
|
|
577
|
+
if (entry.status === "working" || entry.status === "input_required") {
|
|
578
|
+
entry.proc?.kill();
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
this.active.clear();
|
|
582
|
+
this.inputRequiredTargets.clear();
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/** Drain a ReadableStream into a string (for stderr consumption). */
|
|
587
|
+
async function drainStream(stream: ReadableStream): Promise<string> {
|
|
588
|
+
const reader = stream.getReader();
|
|
589
|
+
const decoder = new TextDecoder();
|
|
590
|
+
let result = "";
|
|
591
|
+
try {
|
|
592
|
+
while (true) {
|
|
593
|
+
const { done, value } = await reader.read();
|
|
594
|
+
if (done) break;
|
|
595
|
+
result += decoder.decode(value, { stream: true });
|
|
596
|
+
}
|
|
597
|
+
} finally {
|
|
598
|
+
reader.releaseLock();
|
|
306
599
|
}
|
|
600
|
+
return result;
|
|
307
601
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { IncomingMessage } from "node:http";
|
|
2
|
+
|
|
3
|
+
const MAX_BODY_SIZE = 1024 * 1024; // 1 MB
|
|
4
|
+
|
|
5
|
+
/** Read HTTP request body with size limit (default 1 MB). */
|
|
6
|
+
export function readBody(req: IncomingMessage, maxBytes = MAX_BODY_SIZE): Promise<string> {
|
|
7
|
+
return new Promise((resolve, reject) => {
|
|
8
|
+
const chunks: Buffer[] = [];
|
|
9
|
+
let size = 0;
|
|
10
|
+
let settled = false;
|
|
11
|
+
req.on("data", (chunk: Buffer) => {
|
|
12
|
+
if (settled) return;
|
|
13
|
+
size += chunk.length;
|
|
14
|
+
if (size > maxBytes) {
|
|
15
|
+
settled = true;
|
|
16
|
+
req.destroy();
|
|
17
|
+
reject(new Error("Request body too large"));
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
chunks.push(chunk);
|
|
21
|
+
});
|
|
22
|
+
req.on("end", () => {
|
|
23
|
+
if (!settled) {
|
|
24
|
+
settled = true;
|
|
25
|
+
resolve(Buffer.concat(chunks).toString());
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
req.on("error", (err) => {
|
|
29
|
+
if (!settled) {
|
|
30
|
+
settled = true;
|
|
31
|
+
reject(err);
|
|
32
|
+
}
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -2,12 +2,14 @@ import type { OpenClawPluginApi, GatewayRequestHandlerOptions } from "openclaw/p
|
|
|
2
2
|
import { ClawMatrixConfigSchema, parseConfig } from "./config.ts";
|
|
3
3
|
import { createClusterService, getClusterRuntime } from "./cluster-service.ts";
|
|
4
4
|
import { createClusterHandoffTool } from "./tools/cluster-handoff.ts";
|
|
5
|
+
import { createClusterHandoffReplyTool } from "./tools/cluster-handoff-reply.ts";
|
|
5
6
|
import { createClusterSendTool } from "./tools/cluster-send.ts";
|
|
6
7
|
import { createClusterPeersTool } from "./tools/cluster-peers.ts";
|
|
7
8
|
import { createClusterExecTool } from "./tools/cluster-exec.ts";
|
|
8
9
|
import { createClusterReadTool } from "./tools/cluster-read.ts";
|
|
9
10
|
import { createClusterWriteTool } from "./tools/cluster-write.ts";
|
|
10
11
|
import { createClusterToolTool } from "./tools/cluster-tool.ts";
|
|
12
|
+
import { createClusterEventsTool } from "./tools/cluster-events.ts";
|
|
11
13
|
import { registerClusterCli } from "./cli.ts";
|
|
12
14
|
|
|
13
15
|
const plugin = {
|
|
@@ -99,12 +101,14 @@ const plugin = {
|
|
|
99
101
|
|
|
100
102
|
// Agent tools
|
|
101
103
|
api.registerTool(createClusterHandoffTool(), { optional: true });
|
|
104
|
+
api.registerTool(createClusterHandoffReplyTool(), { optional: true });
|
|
102
105
|
api.registerTool(createClusterSendTool(), { optional: true });
|
|
103
106
|
api.registerTool(createClusterPeersTool(), { optional: true });
|
|
104
107
|
api.registerTool(createClusterExecTool(), { optional: true });
|
|
105
108
|
api.registerTool(createClusterReadTool(), { optional: true });
|
|
106
109
|
api.registerTool(createClusterWriteTool(), { optional: true });
|
|
107
110
|
api.registerTool(createClusterToolTool(), { optional: true });
|
|
111
|
+
api.registerTool(createClusterEventsTool(), { optional: true });
|
|
108
112
|
|
|
109
113
|
// Gateway methods (queried by CLI via `openclaw gateway call`)
|
|
110
114
|
api.registerGatewayMethod(
|
|
@@ -177,6 +181,10 @@ const plugin = {
|
|
|
177
181
|
lines.push(`Your role: ${localAgent.description}`);
|
|
178
182
|
}
|
|
179
183
|
|
|
184
|
+
// Satellite nodes (via WebHandler on relay, or gossiped via peer_sync)
|
|
185
|
+
const satellites = runtime.webHandler?.getSatelliteContexts() ?? runtime.peerManager.satelliteContexts;
|
|
186
|
+
const activeSatellites = satellites.filter(s => Date.now() - s.ts < 600_000);
|
|
187
|
+
|
|
180
188
|
lines.push("", "Remote nodes in the cluster:");
|
|
181
189
|
for (const peer of peers) {
|
|
182
190
|
const status = peer.connection?.isOpen ? "connected" : "via relay";
|
|
@@ -189,24 +197,34 @@ const plugin = {
|
|
|
189
197
|
lines.push(` models: ${peer.models.map((m) => m.id).join(", ")}`);
|
|
190
198
|
}
|
|
191
199
|
}
|
|
200
|
+
for (const sat of activeSatellites) {
|
|
201
|
+
const age = Math.floor((Date.now() - sat.ts) / 1000);
|
|
202
|
+
const country = sat.country ? `, ${sat.country}` : "";
|
|
203
|
+
lines.push(` - ${sat.nodeId} (satellite${country}, ${age}s ago)`);
|
|
204
|
+
if (sat.tools?.length) {
|
|
205
|
+
lines.push(` tools: ${sat.tools.join(", ")}`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Unconsumed events from external sources (Shortcuts automations, etc.)
|
|
210
|
+
const pendingEvents = runtime.webHandler?.getUnconsumedEvents(5) ?? [];
|
|
211
|
+
if (pendingEvents.length > 0) {
|
|
212
|
+
lines.push("", "⚡ Pending events:");
|
|
213
|
+
for (const evt of pendingEvents) {
|
|
214
|
+
const age = Math.floor((Date.now() - evt.ts) / 1000);
|
|
215
|
+
const dataStr = Object.entries(evt.data)
|
|
216
|
+
.map(([k, v]) => `${k}: ${typeof v === "string" ? v : JSON.stringify(v)}`)
|
|
217
|
+
.join(", ");
|
|
218
|
+
const truncated = dataStr.length > 120 ? dataStr.slice(0, 120) + "…" : dataStr;
|
|
219
|
+
lines.push(` - [${evt.type}] ${evt.source} (${age}s ago, id:${evt.id}): ${truncated}`);
|
|
220
|
+
}
|
|
221
|
+
lines.push("Use cluster_events for details or to mark consumed.");
|
|
222
|
+
}
|
|
192
223
|
|
|
193
224
|
lines.push(
|
|
194
225
|
"",
|
|
195
|
-
"
|
|
196
|
-
|
|
197
|
-
" - cluster_exec / cluster_read / cluster_write — run commands, read/write files on a remote node",
|
|
198
|
-
" - cluster_tool — invoke any OpenClaw tool on a remote node",
|
|
199
|
-
" - cluster_handoff — delegate a complex task to a remote agent for autonomous execution",
|
|
200
|
-
" - cluster_send — send a one-way message to a remote agent",
|
|
201
|
-
" - cluster_peers — inspect cluster topology",
|
|
202
|
-
"Use the node's description and tags to decide which node to target. " +
|
|
203
|
-
"For simple operations, prefer cluster_exec/read/write; for complex multi-step tasks, prefer cluster_handoff.",
|
|
204
|
-
"",
|
|
205
|
-
"IMPORTANT: Before calling any cluster tool or using a remote model, you MUST explicitly tell the user " +
|
|
206
|
-
"which remote node you are targeting and what you are about to do. For example:",
|
|
207
|
-
' "I\'m going to run this command on remote node «coder» ..."',
|
|
208
|
-
' "I\'m delegating this task to agent «reviewer» on node «server-b» ..."',
|
|
209
|
-
"This ensures the user always knows when operations leave the local node.",
|
|
226
|
+
"Prefer cluster_exec/read/write for simple ops; cluster_handoff for complex multi-step tasks.",
|
|
227
|
+
"IMPORTANT: Always tell user which remote node you're targeting before calling cluster tools.",
|
|
210
228
|
);
|
|
211
229
|
|
|
212
230
|
return { prependSystemContext: lines.join("\n") };
|
package/src/model-proxy.ts
CHANGED
|
@@ -9,12 +9,19 @@ import type {
|
|
|
9
9
|
ModelStreamChunk,
|
|
10
10
|
} from "./types.ts";
|
|
11
11
|
import { debug } from "./debug.ts";
|
|
12
|
+
import { readBody } from "./http-utils.ts";
|
|
12
13
|
|
|
13
14
|
const MODEL_TIMEOUT = 120_000; // 2 minutes
|
|
14
15
|
const MAX_STREAM_BUFFER = 1_048_576; // 1MB — guard against upstream not sending newlines
|
|
15
16
|
|
|
16
17
|
type ResponseFormat = "chat" | "responses";
|
|
17
18
|
|
|
19
|
+
interface ProxyResponse {
|
|
20
|
+
status: number;
|
|
21
|
+
headers: Record<string, string>;
|
|
22
|
+
body: string | ReadableStream;
|
|
23
|
+
}
|
|
24
|
+
|
|
18
25
|
interface PendingModelReq {
|
|
19
26
|
resolve: (value: unknown) => void;
|
|
20
27
|
reject: (error: Error) => void;
|
|
@@ -173,12 +180,12 @@ export class ModelProxy {
|
|
|
173
180
|
debug("proxy", `${req.method} ${url.pathname} → ${p}`);
|
|
174
181
|
|
|
175
182
|
if (p === "/chat/completions" && req.method === "POST") {
|
|
176
|
-
const body = await
|
|
183
|
+
const body = await readBody(req);
|
|
177
184
|
const response = await this.handleChatCompletion(body, "openai-completions");
|
|
178
185
|
debug("proxy", `response status=${response.status}`);
|
|
179
186
|
this.sendResponse(res, response);
|
|
180
187
|
} else if (p === "/responses" && req.method === "POST") {
|
|
181
|
-
const body = await
|
|
188
|
+
const body = await readBody(req);
|
|
182
189
|
const response = await this.handleResponses(body);
|
|
183
190
|
debug("proxy", `response status=${response.status}`);
|
|
184
191
|
this.sendResponse(res, response);
|
|
@@ -215,18 +222,11 @@ export class ModelProxy {
|
|
|
215
222
|
pending.reject(new Error("Shutting down"));
|
|
216
223
|
}
|
|
217
224
|
this.pending.clear();
|
|
225
|
+
this.streamText.clear();
|
|
218
226
|
}
|
|
219
227
|
|
|
220
|
-
private readBody(req: import("node:http").IncomingMessage): Promise<string> {
|
|
221
|
-
return new Promise((resolve, reject) => {
|
|
222
|
-
const chunks: Buffer[] = [];
|
|
223
|
-
req.on("data", (chunk: Buffer) => chunks.push(chunk));
|
|
224
|
-
req.on("end", () => resolve(Buffer.concat(chunks).toString()));
|
|
225
|
-
req.on("error", reject);
|
|
226
|
-
});
|
|
227
|
-
}
|
|
228
228
|
|
|
229
|
-
private sendResponse(res: import("node:http").ServerResponse, response:
|
|
229
|
+
private sendResponse(res: import("node:http").ServerResponse, response: ProxyResponse) {
|
|
230
230
|
res.writeHead(response.status, response.headers);
|
|
231
231
|
if (typeof response.body === "string") {
|
|
232
232
|
res.end(response.body);
|
|
@@ -288,7 +288,7 @@ export class ModelProxy {
|
|
|
288
288
|
return { nodeId, modelId, proxyModel, routeNodeId: route.nodeId };
|
|
289
289
|
}
|
|
290
290
|
|
|
291
|
-
private async handleChatCompletion(rawBody: string, _api: string): Promise<
|
|
291
|
+
private async handleChatCompletion(rawBody: string, _api: string): Promise<ProxyResponse> {
|
|
292
292
|
let body: { model: string; messages: unknown[]; stream?: boolean; temperature?: number; max_tokens?: number };
|
|
293
293
|
try {
|
|
294
294
|
body = JSON.parse(rawBody);
|
|
@@ -303,6 +303,8 @@ export class ModelProxy {
|
|
|
303
303
|
|
|
304
304
|
const { modelId, proxyModel, routeNodeId } = resolved;
|
|
305
305
|
const messages = body.messages;
|
|
306
|
+
debug("proxy", `messages count=${messages?.length ?? 0} roles=${(messages ?? []).map((m: unknown) => (m as Record<string, unknown>)?.role).join(",")}`);
|
|
307
|
+
|
|
306
308
|
if (proxyModel?.description) {
|
|
307
309
|
const first = messages[0] as { role?: string; content?: string } | undefined;
|
|
308
310
|
if (first?.role === "system" && typeof first.content === "string") {
|
|
@@ -326,7 +328,7 @@ export class ModelProxy {
|
|
|
326
328
|
}
|
|
327
329
|
}
|
|
328
330
|
|
|
329
|
-
private async handleResponses(rawBody: string): Promise<
|
|
331
|
+
private async handleResponses(rawBody: string): Promise<ProxyResponse> {
|
|
330
332
|
let body: { model: string; input: unknown; stream?: boolean; temperature?: number; max_output_tokens?: number; instructions?: string };
|
|
331
333
|
try {
|
|
332
334
|
body = JSON.parse(rawBody);
|
|
@@ -382,7 +384,7 @@ export class ModelProxy {
|
|
|
382
384
|
targetNodeId: string,
|
|
383
385
|
frame: ModelRequest,
|
|
384
386
|
responseFormat: ResponseFormat,
|
|
385
|
-
):
|
|
387
|
+
): ProxyResponse & { body: ReadableStream } {
|
|
386
388
|
const encoder = new TextEncoder();
|
|
387
389
|
const model = frame.payload.model;
|
|
388
390
|
|
|
@@ -390,6 +392,7 @@ export class ModelProxy {
|
|
|
390
392
|
start: (controller) => {
|
|
391
393
|
const timer = setTimeout(() => {
|
|
392
394
|
this.pending.delete(requestId);
|
|
395
|
+
this.streamText.delete(requestId);
|
|
393
396
|
this.peerManager.router.markFailed(requestId);
|
|
394
397
|
try {
|
|
395
398
|
if (responseFormat === "responses") {
|
|
@@ -475,7 +478,7 @@ export class ModelProxy {
|
|
|
475
478
|
targetNodeId: string,
|
|
476
479
|
frame: ModelRequest,
|
|
477
480
|
responseFormat: ResponseFormat,
|
|
478
|
-
): Promise<
|
|
481
|
+
): Promise<ProxyResponse & { body: string }> {
|
|
479
482
|
try {
|
|
480
483
|
const result = await new Promise<ModelResponse["payload"]>(
|
|
481
484
|
(resolve, reject) => {
|
|
@@ -569,7 +572,7 @@ export class ModelProxy {
|
|
|
569
572
|
}
|
|
570
573
|
}
|
|
571
574
|
|
|
572
|
-
private handleListModels():
|
|
575
|
+
private handleListModels(): ProxyResponse & { body: string } {
|
|
573
576
|
// Build from proxyModels config (has full detail) and enrich with
|
|
574
577
|
// connectivity info from the router so consumers know what's reachable.
|
|
575
578
|
const reachable = new Set(
|