@integrity-labs/agt-cli 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/agt.js +3223 -0
- package/dist/bin/agt.js.map +1 -0
- package/dist/chunk-4I4QZRBQ.js +181 -0
- package/dist/chunk-4I4QZRBQ.js.map +1 -0
- package/dist/chunk-EUF2V4N5.js +4485 -0
- package/dist/chunk-EUF2V4N5.js.map +1 -0
- package/dist/claude-scheduler-APXMZEK4.js +21 -0
- package/dist/claude-scheduler-APXMZEK4.js.map +1 -0
- package/dist/lib/manager-worker.js +3403 -0
- package/dist/lib/manager-worker.js.map +1 -0
- package/package.json +51 -0
|
@@ -0,0 +1,3403 @@
|
|
|
1
|
+
import {
|
|
2
|
+
api,
|
|
3
|
+
exchangeApiKey,
|
|
4
|
+
extractFrontmatter,
|
|
5
|
+
getApiKey,
|
|
6
|
+
getFramework,
|
|
7
|
+
getHostId,
|
|
8
|
+
provision,
|
|
9
|
+
requireHost,
|
|
10
|
+
resolveChannels
|
|
11
|
+
} from "../chunk-EUF2V4N5.js";
|
|
12
|
+
import {
|
|
13
|
+
findTaskByTemplate,
|
|
14
|
+
getProjectDir,
|
|
15
|
+
getReadyTasks,
|
|
16
|
+
loadSchedulerState,
|
|
17
|
+
markTaskFired,
|
|
18
|
+
syncTasksToScheduler
|
|
19
|
+
} from "../chunk-4I4QZRBQ.js";
|
|
20
|
+
|
|
21
|
+
// src/lib/manager-worker.ts
|
|
22
|
+
import { createHash } from "crypto";
|
|
23
|
+
import { readFileSync as readFileSync2, writeFileSync, mkdirSync, existsSync as existsSync2, rmSync, readdirSync, statSync, unlinkSync } from "fs";
|
|
24
|
+
import https from "https";
|
|
25
|
+
import { join as join2 } from "path";
|
|
26
|
+
|
|
27
|
+
// src/lib/gateway-client.ts
|
|
28
|
+
import { EventEmitter } from "events";
|
|
29
|
+
import WebSocket from "ws";
|
|
30
|
+
var DEFAULT_PORT = 18789;
|
|
31
|
+
var RECONNECT_INITIAL_MS = 1e3;
|
|
32
|
+
var RECONNECT_BASE_MS = 5e3;
|
|
33
|
+
var RECONNECT_MAX_MS = 3e4;
|
|
34
|
+
var HEARTBEAT_INTERVAL_MS = 3e4;
|
|
35
|
+
var GatewayClient = class extends EventEmitter {
|
|
36
|
+
port;
|
|
37
|
+
token;
|
|
38
|
+
ws = null;
|
|
39
|
+
_connected = false;
|
|
40
|
+
_everConnected = false;
|
|
41
|
+
reconnectAttempts = 0;
|
|
42
|
+
reconnectTimer = null;
|
|
43
|
+
heartbeatTimer = null;
|
|
44
|
+
pongReceived = true;
|
|
45
|
+
intentionalClose = false;
|
|
46
|
+
pendingRpc = /* @__PURE__ */ new Map();
|
|
47
|
+
rpcSeq = 0;
|
|
48
|
+
constructor(options = {}) {
|
|
49
|
+
super();
|
|
50
|
+
this.port = options.port ?? Number(process.env["OPENCLAW_GATEWAY_PORT"]) ?? DEFAULT_PORT;
|
|
51
|
+
this.token = options.token ?? process.env["OPENCLAW_GATEWAY_TOKEN"];
|
|
52
|
+
}
|
|
53
|
+
get connected() {
|
|
54
|
+
return this._connected;
|
|
55
|
+
}
|
|
56
|
+
connect() {
|
|
57
|
+
this.intentionalClose = false;
|
|
58
|
+
this.doConnect();
|
|
59
|
+
}
|
|
60
|
+
disconnect() {
|
|
61
|
+
this.intentionalClose = true;
|
|
62
|
+
this.clearTimers();
|
|
63
|
+
if (this.ws) {
|
|
64
|
+
this.ws.removeAllListeners();
|
|
65
|
+
this.ws.close(1e3);
|
|
66
|
+
this.ws = null;
|
|
67
|
+
}
|
|
68
|
+
if (this._connected) {
|
|
69
|
+
this._connected = false;
|
|
70
|
+
this.emit("disconnected");
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// ── Private ──────────────────────────────────────────────────────────────
|
|
74
|
+
doConnect() {
|
|
75
|
+
const url = `ws://127.0.0.1:${this.port}`;
|
|
76
|
+
try {
|
|
77
|
+
const headers = {};
|
|
78
|
+
if (this.token) {
|
|
79
|
+
headers["Authorization"] = `Bearer ${this.token}`;
|
|
80
|
+
}
|
|
81
|
+
this.ws = new WebSocket(url, { headers });
|
|
82
|
+
} catch (err) {
|
|
83
|
+
this.emit("error", err);
|
|
84
|
+
this.scheduleReconnect();
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
this.ws.on("open", () => {
|
|
88
|
+
});
|
|
89
|
+
this.ws.on("message", (data) => {
|
|
90
|
+
try {
|
|
91
|
+
const msg = JSON.parse(data.toString());
|
|
92
|
+
this.handleMessage(msg);
|
|
93
|
+
} catch {
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
this.ws.on("close", (code, reason) => {
|
|
97
|
+
this.clearHeartbeat();
|
|
98
|
+
if (this._connected) {
|
|
99
|
+
this._connected = false;
|
|
100
|
+
this.emit("disconnected");
|
|
101
|
+
} else if (!this._everConnected) {
|
|
102
|
+
this.emit("error", new Error(`WebSocket closed before auth (code=${code}, reason=${reason?.toString() || "none"})`));
|
|
103
|
+
}
|
|
104
|
+
if (!this.intentionalClose) {
|
|
105
|
+
this.scheduleReconnect();
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
this.ws.on("error", (err) => {
|
|
109
|
+
if (!this._everConnected && err.code === "ECONNREFUSED") {
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
this.emit("error", err);
|
|
113
|
+
});
|
|
114
|
+
this.ws.on("pong", () => {
|
|
115
|
+
this.pongReceived = true;
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
handleMessage(msg) {
|
|
119
|
+
switch (msg.type) {
|
|
120
|
+
case "challenge":
|
|
121
|
+
this.sendJson({
|
|
122
|
+
type: "connect",
|
|
123
|
+
nonce: msg.nonce,
|
|
124
|
+
role: "operator",
|
|
125
|
+
scopes: ["events:read"]
|
|
126
|
+
});
|
|
127
|
+
break;
|
|
128
|
+
case "hello-ok":
|
|
129
|
+
this._connected = true;
|
|
130
|
+
this._everConnected = true;
|
|
131
|
+
this.reconnectAttempts = 0;
|
|
132
|
+
this.startHeartbeat();
|
|
133
|
+
this.emit("connected");
|
|
134
|
+
break;
|
|
135
|
+
case "event":
|
|
136
|
+
this.emit("event", {
|
|
137
|
+
type: "event",
|
|
138
|
+
event: msg.event,
|
|
139
|
+
payload: msg.payload,
|
|
140
|
+
seq: msg.seq,
|
|
141
|
+
stateVersion: msg.stateVersion
|
|
142
|
+
});
|
|
143
|
+
break;
|
|
144
|
+
case "heartbeat":
|
|
145
|
+
case "tick":
|
|
146
|
+
break;
|
|
147
|
+
case "rpc-result":
|
|
148
|
+
case "rpc-error": {
|
|
149
|
+
const rpcId = msg.id;
|
|
150
|
+
const pending = this.pendingRpc.get(rpcId);
|
|
151
|
+
if (pending) {
|
|
152
|
+
this.pendingRpc.delete(rpcId);
|
|
153
|
+
clearTimeout(pending.timer);
|
|
154
|
+
if (msg.type === "rpc-error") {
|
|
155
|
+
pending.reject(new Error(msg.error ?? "RPC error"));
|
|
156
|
+
} else {
|
|
157
|
+
pending.resolve(msg.result);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
default:
|
|
163
|
+
if (msg.id && typeof msg.id === "string") {
|
|
164
|
+
const pending = this.pendingRpc.get(msg.id);
|
|
165
|
+
if (pending) {
|
|
166
|
+
this.pendingRpc.delete(msg.id);
|
|
167
|
+
clearTimeout(pending.timer);
|
|
168
|
+
if (msg.error) {
|
|
169
|
+
pending.reject(new Error(String(msg.error)));
|
|
170
|
+
} else {
|
|
171
|
+
pending.resolve(msg);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Send an RPC call to the gateway and wait for a response.
|
|
180
|
+
* Returns the response payload or throws on timeout/error.
|
|
181
|
+
*/
|
|
182
|
+
sendRpc(method, params = {}, timeoutMs = 1e4) {
|
|
183
|
+
return new Promise((resolve, reject) => {
|
|
184
|
+
if (!this._connected || this.ws?.readyState !== WebSocket.OPEN) {
|
|
185
|
+
reject(new Error("Gateway not connected"));
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
const id = `rpc-${++this.rpcSeq}-${Date.now()}`;
|
|
189
|
+
const timer = setTimeout(() => {
|
|
190
|
+
this.pendingRpc.delete(id);
|
|
191
|
+
reject(new Error(`RPC ${method} timed out after ${timeoutMs}ms`));
|
|
192
|
+
}, timeoutMs);
|
|
193
|
+
this.pendingRpc.set(id, { resolve, reject, timer });
|
|
194
|
+
this.sendJson({ type: "rpc", id, method, params });
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
sendJson(obj) {
|
|
198
|
+
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
199
|
+
this.ws.send(JSON.stringify(obj));
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
startHeartbeat() {
|
|
203
|
+
this.clearHeartbeat();
|
|
204
|
+
this.pongReceived = true;
|
|
205
|
+
this.heartbeatTimer = setInterval(() => {
|
|
206
|
+
if (!this.pongReceived) {
|
|
207
|
+
this.ws?.terminate();
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
this.pongReceived = false;
|
|
211
|
+
this.ws?.ping();
|
|
212
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
213
|
+
}
|
|
214
|
+
clearHeartbeat() {
|
|
215
|
+
if (this.heartbeatTimer) {
|
|
216
|
+
clearInterval(this.heartbeatTimer);
|
|
217
|
+
this.heartbeatTimer = null;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
scheduleReconnect() {
|
|
221
|
+
if (this.intentionalClose) return;
|
|
222
|
+
const delay = this.reconnectAttempts === 0 ? RECONNECT_INITIAL_MS : Math.min(RECONNECT_BASE_MS * Math.pow(2, this.reconnectAttempts - 1), RECONNECT_MAX_MS);
|
|
223
|
+
this.reconnectAttempts++;
|
|
224
|
+
this.reconnectTimer = setTimeout(() => {
|
|
225
|
+
this.reconnectTimer = null;
|
|
226
|
+
this.doConnect();
|
|
227
|
+
}, delay);
|
|
228
|
+
}
|
|
229
|
+
clearTimers() {
|
|
230
|
+
this.clearHeartbeat();
|
|
231
|
+
if (this.reconnectTimer) {
|
|
232
|
+
clearTimeout(this.reconnectTimer);
|
|
233
|
+
this.reconnectTimer = null;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
var GatewayClientPool = class extends EventEmitter {
|
|
238
|
+
clients = /* @__PURE__ */ new Map();
|
|
239
|
+
addAgent(codeName, port, token) {
|
|
240
|
+
if (this.clients.has(codeName)) {
|
|
241
|
+
this.removeAgent(codeName);
|
|
242
|
+
}
|
|
243
|
+
const client2 = new GatewayClient({ port, token });
|
|
244
|
+
client2.on("connected", () => {
|
|
245
|
+
this.emit("connected", codeName);
|
|
246
|
+
});
|
|
247
|
+
client2.on("disconnected", () => {
|
|
248
|
+
this.emit("disconnected", codeName);
|
|
249
|
+
});
|
|
250
|
+
client2.on("error", (err) => {
|
|
251
|
+
this.emit("error", err, codeName);
|
|
252
|
+
});
|
|
253
|
+
client2.on("event", (evt) => {
|
|
254
|
+
const pooledEvent = {
|
|
255
|
+
...evt,
|
|
256
|
+
agentCodeName: codeName
|
|
257
|
+
};
|
|
258
|
+
this.emit("event", pooledEvent);
|
|
259
|
+
});
|
|
260
|
+
this.clients.set(codeName, client2);
|
|
261
|
+
client2.connect();
|
|
262
|
+
}
|
|
263
|
+
removeAgent(codeName) {
|
|
264
|
+
const client2 = this.clients.get(codeName);
|
|
265
|
+
if (client2) {
|
|
266
|
+
client2.disconnect();
|
|
267
|
+
this.clients.delete(codeName);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
hasAgent(codeName) {
|
|
271
|
+
return this.clients.has(codeName);
|
|
272
|
+
}
|
|
273
|
+
isConnected(codeName) {
|
|
274
|
+
return this.clients.get(codeName)?.connected ?? false;
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Send an RPC call to a specific agent's gateway.
|
|
278
|
+
* Returns the response or throws on timeout/error/not connected.
|
|
279
|
+
*/
|
|
280
|
+
async sendRpc(codeName, method, params = {}, timeoutMs = 1e4) {
|
|
281
|
+
const client2 = this.clients.get(codeName);
|
|
282
|
+
if (!client2) throw new Error(`No gateway client for agent '${codeName}'`);
|
|
283
|
+
return client2.sendRpc(method, params, timeoutMs);
|
|
284
|
+
}
|
|
285
|
+
disconnectAll() {
|
|
286
|
+
for (const [, client2] of this.clients) {
|
|
287
|
+
client2.disconnect();
|
|
288
|
+
}
|
|
289
|
+
this.clients.clear();
|
|
290
|
+
}
|
|
291
|
+
get size() {
|
|
292
|
+
return this.clients.size;
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
|
|
296
|
+
// src/lib/persistent-session.ts
|
|
297
|
+
import { spawn, execSync } from "child_process";
|
|
298
|
+
import { join } from "path";
|
|
299
|
+
import { homedir } from "os";
|
|
300
|
+
import { existsSync, readFileSync } from "fs";
|
|
301
|
+
var sessions = /* @__PURE__ */ new Map();
|
|
302
|
+
function startPersistentSession(config2) {
|
|
303
|
+
const existing = sessions.get(config2.codeName);
|
|
304
|
+
if (existing && existing.status === "running" && existing.process && !existing.process.killed) {
|
|
305
|
+
return existing;
|
|
306
|
+
}
|
|
307
|
+
const session = {
|
|
308
|
+
codeName: config2.codeName,
|
|
309
|
+
process: null,
|
|
310
|
+
taskChannelPort: null,
|
|
311
|
+
startedAt: null,
|
|
312
|
+
restartCount: existing?.restartCount ?? 0,
|
|
313
|
+
status: "starting"
|
|
314
|
+
};
|
|
315
|
+
sessions.set(config2.codeName, session);
|
|
316
|
+
spawnSession(config2, session);
|
|
317
|
+
return session;
|
|
318
|
+
}
|
|
319
|
+
function spawnSession(config2, session) {
|
|
320
|
+
const { codeName, projectDir, mcpConfigPath, claudeMdPath, channels, devChannels, log: log2 } = config2;
|
|
321
|
+
const args = [];
|
|
322
|
+
if (channels.length > 0) {
|
|
323
|
+
args.push("--channels", ...channels);
|
|
324
|
+
}
|
|
325
|
+
if (devChannels.length > 0) {
|
|
326
|
+
args.push("--dangerously-load-development-channels", ...devChannels);
|
|
327
|
+
}
|
|
328
|
+
args.push("--mcp-config", mcpConfigPath);
|
|
329
|
+
const channelsConfigPath = join(projectDir, ".mcp-channels.json");
|
|
330
|
+
if (existsSync(channelsConfigPath)) {
|
|
331
|
+
args.push("--mcp-config", channelsConfigPath);
|
|
332
|
+
}
|
|
333
|
+
if (existsSync(claudeMdPath)) {
|
|
334
|
+
args.push("--system-prompt-file", claudeMdPath);
|
|
335
|
+
}
|
|
336
|
+
args.push("--allow-dangerously-skip-permissions");
|
|
337
|
+
args.push("--dangerously-skip-permissions");
|
|
338
|
+
args.push("--name", `agt-${codeName}`);
|
|
339
|
+
log2(`[persistent-session] Starting session for '${codeName}' with args: ${args.join(" ")}`);
|
|
340
|
+
const tmuxSession = `agt-${codeName}`;
|
|
341
|
+
try {
|
|
342
|
+
execSync(`tmux kill-session -t ${tmuxSession} 2>/dev/null`, { stdio: "ignore" });
|
|
343
|
+
} catch {
|
|
344
|
+
}
|
|
345
|
+
const envIntegrationsPath = join(projectDir, ".env.integrations");
|
|
346
|
+
let envPrefix = "";
|
|
347
|
+
if (existsSync(envIntegrationsPath)) {
|
|
348
|
+
try {
|
|
349
|
+
const envContent = readFileSync(envIntegrationsPath, "utf-8");
|
|
350
|
+
const envVars = envContent.split("\n").filter((line) => line && !line.startsWith("#") && line.includes("=")).map((line) => {
|
|
351
|
+
const eqIdx = line.indexOf("=");
|
|
352
|
+
const key = line.slice(0, eqIdx);
|
|
353
|
+
const value = line.slice(eqIdx + 1);
|
|
354
|
+
return `${key}=${JSON.stringify(value)}`;
|
|
355
|
+
}).join(" ");
|
|
356
|
+
if (envVars) envPrefix = `env ${envVars} `;
|
|
357
|
+
} catch {
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
const initPrompt = "You are now online. Wait for messages from your channels (Telegram, Slack) and respond to them. Use your kanban tools to track work.";
|
|
361
|
+
const claudeCmd = `${envPrefix}claude ${JSON.stringify(initPrompt)} ${args.map((a) => a.includes(" ") ? JSON.stringify(a) : a).join(" ")}`;
|
|
362
|
+
const child = spawn("tmux", [
|
|
363
|
+
"new-session",
|
|
364
|
+
"-d",
|
|
365
|
+
"-s",
|
|
366
|
+
tmuxSession,
|
|
367
|
+
"-c",
|
|
368
|
+
projectDir,
|
|
369
|
+
claudeCmd
|
|
370
|
+
], {
|
|
371
|
+
cwd: projectDir,
|
|
372
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
373
|
+
env: process.env
|
|
374
|
+
});
|
|
375
|
+
session.process = child;
|
|
376
|
+
session.startedAt = Date.now();
|
|
377
|
+
session.status = "running";
|
|
378
|
+
child.on("close", (code) => {
|
|
379
|
+
if (code !== 0) {
|
|
380
|
+
log2(`[persistent-session] Failed to create tmux session for '${codeName}' (exit ${code})`);
|
|
381
|
+
session.status = "crashed";
|
|
382
|
+
session.process = null;
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
log2(`[persistent-session] tmux session '${tmuxSession}' created for '${codeName}'`);
|
|
386
|
+
const acceptDialogs = async () => {
|
|
387
|
+
for (let i = 0; i < 15; i++) {
|
|
388
|
+
await new Promise((r) => setTimeout(r, 2e3));
|
|
389
|
+
try {
|
|
390
|
+
const { execSync: es } = await import("child_process");
|
|
391
|
+
const screen = es(`tmux capture-pane -t ${tmuxSession} -p 2>/dev/null`, { encoding: "utf-8" });
|
|
392
|
+
if (screen.includes("Yes, I trust this folder")) {
|
|
393
|
+
es(`tmux send-keys -t ${tmuxSession} Enter`, { stdio: "ignore" });
|
|
394
|
+
log2(`[persistent-session] Auto-accepted workspace trust for '${codeName}'`);
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
if (screen.includes("I am using this for local development")) {
|
|
398
|
+
es(`tmux send-keys -t ${tmuxSession} Enter`, { stdio: "ignore" });
|
|
399
|
+
log2(`[persistent-session] Auto-accepted dev channels for '${codeName}'`);
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
if (screen.includes("Enter to confirm") && screen.includes("MCP")) {
|
|
403
|
+
es(`tmux send-keys -t ${tmuxSession} Enter`, { stdio: "ignore" });
|
|
404
|
+
log2(`[persistent-session] Auto-accepted MCP servers for '${codeName}'`);
|
|
405
|
+
continue;
|
|
406
|
+
}
|
|
407
|
+
if (screen.includes("Yes, I accept") && screen.includes("Bypass Permissions")) {
|
|
408
|
+
es(`tmux send-keys -t ${tmuxSession} 2`, { stdio: "ignore" });
|
|
409
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
410
|
+
es(`tmux send-keys -t ${tmuxSession} Enter`, { stdio: "ignore" });
|
|
411
|
+
log2(`[persistent-session] Auto-accepted bypass permissions for '${codeName}'`);
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
if (screen.includes("\u276F") && !screen.includes("Enter to confirm")) {
|
|
415
|
+
log2(`[persistent-session] Session ready for '${codeName}' \u2014 no more dialogs`);
|
|
416
|
+
break;
|
|
417
|
+
}
|
|
418
|
+
} catch {
|
|
419
|
+
break;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
};
|
|
423
|
+
acceptDialogs().catch(() => {
|
|
424
|
+
});
|
|
425
|
+
});
|
|
426
|
+
child.on("error", (err) => {
|
|
427
|
+
log2(`[persistent-session] Failed to start tmux for '${codeName}': ${err.message}`);
|
|
428
|
+
session.status = "crashed";
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
function stopPersistentSession(codeName, log2) {
|
|
432
|
+
const session = sessions.get(codeName);
|
|
433
|
+
if (!session) return;
|
|
434
|
+
log2(`[persistent-session] Stopping session for '${codeName}'`);
|
|
435
|
+
session.status = "stopped";
|
|
436
|
+
try {
|
|
437
|
+
execSync(`tmux kill-session -t agt-${codeName} 2>/dev/null`, { stdio: "ignore" });
|
|
438
|
+
} catch {
|
|
439
|
+
}
|
|
440
|
+
if (session.process && !session.process.killed) {
|
|
441
|
+
session.process.kill("SIGTERM");
|
|
442
|
+
}
|
|
443
|
+
setTimeout(() => {
|
|
444
|
+
if (session.process && !session.process.killed) {
|
|
445
|
+
session.process.kill("SIGKILL");
|
|
446
|
+
}
|
|
447
|
+
}, 1e4);
|
|
448
|
+
sessions.delete(codeName);
|
|
449
|
+
}
|
|
450
|
+
async function injectMessage(codeName, type, content, meta) {
|
|
451
|
+
const session = sessions.get(codeName);
|
|
452
|
+
if (!session || session.status !== "running" || !session.process) {
|
|
453
|
+
return false;
|
|
454
|
+
}
|
|
455
|
+
if (session.taskChannelPort) {
|
|
456
|
+
try {
|
|
457
|
+
const res = await fetch(`http://127.0.0.1:${session.taskChannelPort}`, {
|
|
458
|
+
method: "POST",
|
|
459
|
+
headers: { "Content-Type": "application/json" },
|
|
460
|
+
body: JSON.stringify({ type, content, meta }),
|
|
461
|
+
signal: AbortSignal.timeout(1e4)
|
|
462
|
+
});
|
|
463
|
+
if (res.ok) return true;
|
|
464
|
+
} catch {
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
try {
|
|
468
|
+
const prefix = meta?.task_name ? `[Task: ${meta.task_name}] ` : "";
|
|
469
|
+
const text = prefix + content;
|
|
470
|
+
const escaped = text.replace(/'/g, "'\\''");
|
|
471
|
+
execSync(`tmux send-keys -t agt-${codeName} '${escaped}' Enter`, { stdio: "ignore" });
|
|
472
|
+
return true;
|
|
473
|
+
} catch {
|
|
474
|
+
return false;
|
|
475
|
+
}
|
|
476
|
+
return false;
|
|
477
|
+
}
|
|
478
|
+
function isSessionHealthy(codeName) {
|
|
479
|
+
const session = sessions.get(codeName);
|
|
480
|
+
if (!session || session.status !== "running") return false;
|
|
481
|
+
try {
|
|
482
|
+
execSync(`tmux has-session -t agt-${codeName} 2>/dev/null`, { stdio: "ignore" });
|
|
483
|
+
return true;
|
|
484
|
+
} catch {
|
|
485
|
+
session.status = "crashed";
|
|
486
|
+
return false;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
function resetRestartCount(codeName) {
|
|
490
|
+
const session = sessions.get(codeName);
|
|
491
|
+
if (session) session.restartCount = 0;
|
|
492
|
+
}
|
|
493
|
+
function getProjectDir2(codeName) {
|
|
494
|
+
return join(homedir(), ".augmented", codeName, "project");
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
// src/lib/realtime-chat.ts
|
|
498
|
+
import { createClient } from "@supabase/supabase-js";
|
|
499
|
+
var client = null;
|
|
500
|
+
var chatChannel = null;
|
|
501
|
+
var driftChannel = null;
|
|
502
|
+
var assignChannel = null;
|
|
503
|
+
var configChannel = null;
|
|
504
|
+
var kanbanChannel = null;
|
|
505
|
+
var connected = false;
|
|
506
|
+
var tearingDown = false;
|
|
507
|
+
function ensureClient(config2) {
|
|
508
|
+
if (client) return client;
|
|
509
|
+
client = createClient(config2.supabaseUrl, config2.supabaseAnonKey, {
|
|
510
|
+
global: {
|
|
511
|
+
headers: { Authorization: `Bearer ${config2.token}` }
|
|
512
|
+
},
|
|
513
|
+
realtime: {
|
|
514
|
+
params: { apikey: config2.supabaseAnonKey }
|
|
515
|
+
}
|
|
516
|
+
});
|
|
517
|
+
client.realtime.setAuth(config2.token);
|
|
518
|
+
return client;
|
|
519
|
+
}
|
|
520
|
+
function startRealtimeChat(config2) {
|
|
521
|
+
const { agentIds, onMessage, onStatusChange, log: log2 } = config2;
|
|
522
|
+
if (agentIds.length === 0) return;
|
|
523
|
+
const sb = ensureClient(config2);
|
|
524
|
+
const filterStr = agentIds.length === 1 ? `agent_id=eq.${agentIds[0]}` : `agent_id=in.(${agentIds.join(",")})`;
|
|
525
|
+
chatChannel = sb.channel("direct-chat-realtime").on("postgres_changes", {
|
|
526
|
+
event: "INSERT",
|
|
527
|
+
schema: "public",
|
|
528
|
+
table: "direct_chat_messages",
|
|
529
|
+
filter: filterStr
|
|
530
|
+
}, (payload) => {
|
|
531
|
+
const msg = payload.new;
|
|
532
|
+
if (msg.status !== "pending") return;
|
|
533
|
+
log2(`[realtime] Chat message for agent ${msg.agent_id}: id=${msg.id}`);
|
|
534
|
+
onMessage(msg);
|
|
535
|
+
}).subscribe((status) => {
|
|
536
|
+
if (status === "SUBSCRIBED") {
|
|
537
|
+
connected = true;
|
|
538
|
+
log2("[realtime] Chat channel connected");
|
|
539
|
+
onStatusChange("connected");
|
|
540
|
+
} else if (status === "CLOSED" || status === "CHANNEL_ERROR" || status === "TIMED_OUT") {
|
|
541
|
+
if (tearingDown) return;
|
|
542
|
+
connected = false;
|
|
543
|
+
log2(`[realtime] Chat channel: ${status} \u2014 will reconnect next cycle`);
|
|
544
|
+
chatChannel = null;
|
|
545
|
+
driftChannel = null;
|
|
546
|
+
assignChannel = null;
|
|
547
|
+
configChannel = null;
|
|
548
|
+
kanbanChannel = null;
|
|
549
|
+
if (client) {
|
|
550
|
+
try {
|
|
551
|
+
client.removeAllChannels();
|
|
552
|
+
} catch {
|
|
553
|
+
}
|
|
554
|
+
client = null;
|
|
555
|
+
}
|
|
556
|
+
onStatusChange(status === "TIMED_OUT" ? "error" : "disconnected");
|
|
557
|
+
}
|
|
558
|
+
});
|
|
559
|
+
log2(`[realtime] Subscribing to direct_chat_messages for ${agentIds.length} agent(s)`);
|
|
560
|
+
}
|
|
561
|
+
function startRealtimeDrift(config2) {
|
|
562
|
+
const { agentIds, onDrift, log: log2 } = config2;
|
|
563
|
+
if (agentIds.length === 0) return;
|
|
564
|
+
const sb = ensureClient(config2);
|
|
565
|
+
const filterStr = agentIds.length === 1 ? `agent_id=eq.${agentIds[0]}` : `agent_id=in.(${agentIds.join(",")})`;
|
|
566
|
+
driftChannel = sb.channel("drift-realtime").on("postgres_changes", {
|
|
567
|
+
event: "INSERT",
|
|
568
|
+
schema: "public",
|
|
569
|
+
table: "agent_doc_versions",
|
|
570
|
+
filter: filterStr
|
|
571
|
+
}, (payload) => {
|
|
572
|
+
const doc = payload.new;
|
|
573
|
+
log2(`[realtime] Doc version change for agent ${doc.agent_id}: ${doc.doc_type} v${doc.version}`);
|
|
574
|
+
onDrift(doc);
|
|
575
|
+
}).subscribe((status) => {
|
|
576
|
+
if (status === "SUBSCRIBED") {
|
|
577
|
+
log2("[realtime] Drift channel connected");
|
|
578
|
+
} else if (status === "CLOSED" || status === "CHANNEL_ERROR") {
|
|
579
|
+
log2(`[realtime] Drift channel: ${status}`);
|
|
580
|
+
}
|
|
581
|
+
});
|
|
582
|
+
log2(`[realtime] Subscribing to agent_doc_versions for ${agentIds.length} agent(s)`);
|
|
583
|
+
}
|
|
584
|
+
function startRealtimeAssignments(config2) {
|
|
585
|
+
const { hostId, onAssign, onUnassign, log: log2 } = config2;
|
|
586
|
+
const sb = ensureClient(config2);
|
|
587
|
+
assignChannel = sb.channel("assignment-realtime").on("postgres_changes", {
|
|
588
|
+
event: "INSERT",
|
|
589
|
+
schema: "public",
|
|
590
|
+
table: "host_agents",
|
|
591
|
+
filter: `host_id=eq.${hostId}`
|
|
592
|
+
}, (payload) => {
|
|
593
|
+
const row = payload.new;
|
|
594
|
+
log2(`[realtime] Agent assigned: ${row.agent_id} to host ${row.host_id}`);
|
|
595
|
+
onAssign(row);
|
|
596
|
+
}).on("postgres_changes", {
|
|
597
|
+
event: "DELETE",
|
|
598
|
+
schema: "public",
|
|
599
|
+
table: "host_agents"
|
|
600
|
+
}, (payload) => {
|
|
601
|
+
const old = payload.old;
|
|
602
|
+
if (old.host_id === hostId && old.agent_id) {
|
|
603
|
+
log2(`[realtime] Agent unassigned: ${old.agent_id} from host ${hostId}`);
|
|
604
|
+
onUnassign({ agent_id: old.agent_id });
|
|
605
|
+
}
|
|
606
|
+
}).subscribe((status) => {
|
|
607
|
+
if (status === "SUBSCRIBED") {
|
|
608
|
+
log2("[realtime] Assignment channel connected");
|
|
609
|
+
} else if (status === "CLOSED" || status === "CHANNEL_ERROR") {
|
|
610
|
+
log2(`[realtime] Assignment channel: ${status}`);
|
|
611
|
+
}
|
|
612
|
+
});
|
|
613
|
+
log2(`[realtime] Subscribing to host_agents for host ${hostId}`);
|
|
614
|
+
}
|
|
615
|
+
function startRealtimeConfig(config2) {
|
|
616
|
+
const { agentIds, onConfigChange, log: log2 } = config2;
|
|
617
|
+
if (agentIds.length === 0) return;
|
|
618
|
+
const sb = ensureClient(config2);
|
|
619
|
+
const filterStr = agentIds.length === 1 ? `agent_id=eq.${agentIds[0]}` : `agent_id=in.(${agentIds.join(",")})`;
|
|
620
|
+
const lastKnown = /* @__PURE__ */ new Map();
|
|
621
|
+
configChannel = sb.channel("config-realtime").on("postgres_changes", {
|
|
622
|
+
event: "UPDATE",
|
|
623
|
+
schema: "public",
|
|
624
|
+
table: "agents",
|
|
625
|
+
filter: filterStr
|
|
626
|
+
}, (payload) => {
|
|
627
|
+
const agent = payload.new;
|
|
628
|
+
const fingerprint = `${agent.status}|${agent.framework}|${agent.session_mode}|${agent.primary_model}`;
|
|
629
|
+
const prev = lastKnown.get(agent.agent_id);
|
|
630
|
+
if (prev === fingerprint) return;
|
|
631
|
+
lastKnown.set(agent.agent_id, fingerprint);
|
|
632
|
+
log2(`[realtime] Agent config changed: ${agent.code_name} (status=${agent.status})`);
|
|
633
|
+
onConfigChange(agent);
|
|
634
|
+
}).subscribe((status) => {
|
|
635
|
+
if (status === "SUBSCRIBED") {
|
|
636
|
+
log2("[realtime] Config channel connected");
|
|
637
|
+
} else if (status === "CLOSED" || status === "CHANNEL_ERROR") {
|
|
638
|
+
log2(`[realtime] Config channel: ${status}`);
|
|
639
|
+
}
|
|
640
|
+
});
|
|
641
|
+
log2(`[realtime] Subscribing to agents table for ${agentIds.length} agent(s)`);
|
|
642
|
+
}
|
|
643
|
+
function startRealtimeKanban(config2) {
|
|
644
|
+
const { agentIds, onTodayItem, log: log2 } = config2;
|
|
645
|
+
if (agentIds.length === 0) return;
|
|
646
|
+
const sb = ensureClient(config2);
|
|
647
|
+
const filterStr = agentIds.length === 1 ? `agent_id=eq.${agentIds[0]}` : `agent_id=in.(${agentIds.join(",")})`;
|
|
648
|
+
kanbanChannel = sb.channel("kanban-realtime").on("postgres_changes", {
|
|
649
|
+
event: "INSERT",
|
|
650
|
+
schema: "public",
|
|
651
|
+
table: "agent_kanban_items",
|
|
652
|
+
filter: filterStr
|
|
653
|
+
}, (payload) => {
|
|
654
|
+
const item = payload.new;
|
|
655
|
+
if (item.status === "today") {
|
|
656
|
+
log2(`[realtime] New kanban item in 'today': "${item.title}" for agent ${item.agent_id}`);
|
|
657
|
+
onTodayItem(item);
|
|
658
|
+
}
|
|
659
|
+
}).on("postgres_changes", {
|
|
660
|
+
event: "UPDATE",
|
|
661
|
+
schema: "public",
|
|
662
|
+
table: "agent_kanban_items",
|
|
663
|
+
filter: filterStr
|
|
664
|
+
}, (payload) => {
|
|
665
|
+
const item = payload.new;
|
|
666
|
+
const old = payload.old;
|
|
667
|
+
if (item.status === "today" && old.status !== "today") {
|
|
668
|
+
log2(`[realtime] Kanban item moved to 'today': "${item.title}" for agent ${item.agent_id}`);
|
|
669
|
+
onTodayItem(item);
|
|
670
|
+
}
|
|
671
|
+
}).subscribe((status) => {
|
|
672
|
+
if (status === "SUBSCRIBED") {
|
|
673
|
+
log2("[realtime] Kanban channel connected");
|
|
674
|
+
} else if (status === "CLOSED" || status === "CHANNEL_ERROR") {
|
|
675
|
+
log2(`[realtime] Kanban channel: ${status}`);
|
|
676
|
+
}
|
|
677
|
+
});
|
|
678
|
+
log2(`[realtime] Subscribing to agent_kanban_items for ${agentIds.length} agent(s)`);
|
|
679
|
+
}
|
|
680
|
+
function isRealtimeConnected() {
|
|
681
|
+
return connected;
|
|
682
|
+
}
|
|
683
|
+
function stopRealtimeChat() {
|
|
684
|
+
if (tearingDown) return;
|
|
685
|
+
tearingDown = true;
|
|
686
|
+
try {
|
|
687
|
+
if (chatChannel) {
|
|
688
|
+
try {
|
|
689
|
+
chatChannel.unsubscribe();
|
|
690
|
+
} catch {
|
|
691
|
+
}
|
|
692
|
+
chatChannel = null;
|
|
693
|
+
}
|
|
694
|
+
if (driftChannel) {
|
|
695
|
+
try {
|
|
696
|
+
driftChannel.unsubscribe();
|
|
697
|
+
} catch {
|
|
698
|
+
}
|
|
699
|
+
driftChannel = null;
|
|
700
|
+
}
|
|
701
|
+
if (assignChannel) {
|
|
702
|
+
try {
|
|
703
|
+
assignChannel.unsubscribe();
|
|
704
|
+
} catch {
|
|
705
|
+
}
|
|
706
|
+
assignChannel = null;
|
|
707
|
+
}
|
|
708
|
+
if (configChannel) {
|
|
709
|
+
try {
|
|
710
|
+
configChannel.unsubscribe();
|
|
711
|
+
} catch {
|
|
712
|
+
}
|
|
713
|
+
configChannel = null;
|
|
714
|
+
}
|
|
715
|
+
if (kanbanChannel) {
|
|
716
|
+
try {
|
|
717
|
+
kanbanChannel.unsubscribe();
|
|
718
|
+
} catch {
|
|
719
|
+
}
|
|
720
|
+
kanbanChannel = null;
|
|
721
|
+
}
|
|
722
|
+
if (client) {
|
|
723
|
+
try {
|
|
724
|
+
client.removeAllChannels();
|
|
725
|
+
} catch {
|
|
726
|
+
}
|
|
727
|
+
client = null;
|
|
728
|
+
}
|
|
729
|
+
connected = false;
|
|
730
|
+
} finally {
|
|
731
|
+
tearingDown = false;
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
// src/lib/manager-worker.ts
|
|
736
|
+
var GATEWAY_PORT_BASE = 18800;
|
|
737
|
+
var GATEWAY_PORT_STEP = 10;
|
|
738
|
+
var GATEWAY_PORT_MAX = 18899;
|
|
739
|
+
var AUGMENTED_DIR = join2(process.env["HOME"] ?? "/tmp", ".augmented");
|
|
740
|
+
var GATEWAY_PORTS_FILE = join2(AUGMENTED_DIR, "gateway-ports.json");
|
|
741
|
+
var config = null;
|
|
742
|
+
var running = false;
|
|
743
|
+
var pollTimer = null;
|
|
744
|
+
var knownVersions = /* @__PURE__ */ new Map();
|
|
745
|
+
var knownStatuses = /* @__PURE__ */ new Map();
|
|
746
|
+
var knownChannels = /* @__PURE__ */ new Map();
|
|
747
|
+
var writtenHashes = /* @__PURE__ */ new Map();
|
|
748
|
+
var knownSecretsHashes = /* @__PURE__ */ new Map();
|
|
749
|
+
var knownChannelConfigHashes = /* @__PURE__ */ new Map();
|
|
750
|
+
var knownModels = /* @__PURE__ */ new Map();
|
|
751
|
+
var knownTasksHashes = /* @__PURE__ */ new Map();
|
|
752
|
+
var knownIntegrationHashes = /* @__PURE__ */ new Map();
|
|
753
|
+
var losslessClawInstalled = /* @__PURE__ */ new Map();
|
|
754
|
+
var knownSkillHashes = /* @__PURE__ */ new Map();
|
|
755
|
+
var lastCronRunTs = /* @__PURE__ */ new Map();
|
|
756
|
+
var lastWorkTriggerAt = /* @__PURE__ */ new Map();
|
|
757
|
+
var alertedStaleItems = /* @__PURE__ */ new Set();
|
|
758
|
+
var apiKeyStatusCache = /* @__PURE__ */ new Map();
|
|
759
|
+
var STALE_TASK_THRESHOLD_MS = 30 * 60 * 1e3;
|
|
760
|
+
var alertSlackWebhook = null;
|
|
761
|
+
var alertedJobs = /* @__PURE__ */ new Set();
|
|
762
|
+
var taskDisplayInfo = /* @__PURE__ */ new Map();
|
|
763
|
+
var agentDisplayNames = /* @__PURE__ */ new Map();
|
|
764
|
+
var codeNameToAgentId = /* @__PURE__ */ new Map();
|
|
765
|
+
var agentChannelTokens = /* @__PURE__ */ new Map();
|
|
766
|
+
var activeChannels = /* @__PURE__ */ new Map();
|
|
767
|
+
var gatewaysStartedThisCycle = /* @__PURE__ */ new Set();
|
|
768
|
+
var state = {
|
|
769
|
+
pid: process.pid,
|
|
770
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
771
|
+
lastPollAt: null,
|
|
772
|
+
pollCount: 0,
|
|
773
|
+
errorCount: 0,
|
|
774
|
+
agents: []
|
|
775
|
+
};
|
|
776
|
+
var registeredAgentsCache = /* @__PURE__ */ new Map();
|
|
777
|
+
var agentFrameworkCache = /* @__PURE__ */ new Map();
|
|
778
|
+
function resolveAgentFramework(codeName) {
|
|
779
|
+
const frameworkId = agentFrameworkCache.get(codeName) ?? "openclaw";
|
|
780
|
+
return getFramework(frameworkId);
|
|
781
|
+
}
|
|
782
|
+
var gatewayPool = null;
|
|
783
|
+
function clearAgentCaches(agentId, codeName) {
|
|
784
|
+
knownVersions.delete(agentId);
|
|
785
|
+
knownStatuses.delete(agentId);
|
|
786
|
+
knownChannels.delete(agentId);
|
|
787
|
+
writtenHashes.delete(agentId);
|
|
788
|
+
knownSecretsHashes.delete(agentId);
|
|
789
|
+
knownModels.delete(agentId);
|
|
790
|
+
knownTasksHashes.delete(agentId);
|
|
791
|
+
knownIntegrationHashes.delete(agentId);
|
|
792
|
+
losslessClawInstalled.delete(codeName);
|
|
793
|
+
agentDisplayNames.delete(codeName);
|
|
794
|
+
codeNameToAgentId.delete(codeName);
|
|
795
|
+
agentFrameworkCache.delete(codeName);
|
|
796
|
+
kanbanBoardCache.delete(codeName);
|
|
797
|
+
lastHarvestAt.delete(codeName);
|
|
798
|
+
claudeSchedulerStates.delete(codeName);
|
|
799
|
+
claudeTaskConcurrency.delete(codeName);
|
|
800
|
+
for (const key of knownChannelConfigHashes.keys()) {
|
|
801
|
+
if (key.startsWith(`${agentId}:`)) knownChannelConfigHashes.delete(key);
|
|
802
|
+
}
|
|
803
|
+
for (const key of knownSkillHashes.keys()) {
|
|
804
|
+
if (key.startsWith(`${agentId}:`)) knownSkillHashes.delete(key);
|
|
805
|
+
}
|
|
806
|
+
for (const key of taskDisplayInfo.keys()) {
|
|
807
|
+
if (key.startsWith(`${codeName}:`)) taskDisplayInfo.delete(key);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
var cachedFrameworkVersion = null;
|
|
811
|
+
var lastVersionCheckAt = 0;
|
|
812
|
+
var VERSION_CHECK_INTERVAL_MS = 5 * 60 * 1e3;
|
|
813
|
+
function loadGatewayPorts() {
|
|
814
|
+
try {
|
|
815
|
+
return JSON.parse(readFileSync2(GATEWAY_PORTS_FILE, "utf-8"));
|
|
816
|
+
} catch {
|
|
817
|
+
return {};
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
function saveGatewayPorts(ports) {
|
|
821
|
+
mkdirSync(AUGMENTED_DIR, { recursive: true });
|
|
822
|
+
writeFileSync(GATEWAY_PORTS_FILE, JSON.stringify(ports, null, 2));
|
|
823
|
+
}
|
|
824
|
+
function allocatePort(codeName) {
|
|
825
|
+
const ports = loadGatewayPorts();
|
|
826
|
+
if (ports[codeName]) return ports[codeName];
|
|
827
|
+
const usedPorts = new Set(Object.values(ports));
|
|
828
|
+
for (let port = GATEWAY_PORT_BASE; port <= GATEWAY_PORT_MAX; port += GATEWAY_PORT_STEP) {
|
|
829
|
+
if (!usedPorts.has(port)) {
|
|
830
|
+
ports[codeName] = port;
|
|
831
|
+
saveGatewayPorts(ports);
|
|
832
|
+
return port;
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
throw new Error(`No free gateway ports in range ${GATEWAY_PORT_BASE}-${GATEWAY_PORT_MAX}`);
|
|
836
|
+
}
|
|
837
|
+
function freePort(codeName) {
|
|
838
|
+
const ports = loadGatewayPorts();
|
|
839
|
+
if (ports[codeName]) {
|
|
840
|
+
delete ports[codeName];
|
|
841
|
+
saveGatewayPorts(ports);
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
var STATE_FILE = join2(process.env["HOME"] ?? "/tmp", ".augmented", "manager-state.json");
|
|
845
|
+
function send(msg) {
|
|
846
|
+
if (msg.type === "state-update") {
|
|
847
|
+
try {
|
|
848
|
+
writeFileSync(STATE_FILE, JSON.stringify(msg.state, null, 2));
|
|
849
|
+
} catch {
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
if (msg.type === "provisioned") {
|
|
853
|
+
log(`Provisioned ${msg.codeName}`);
|
|
854
|
+
} else if (msg.type === "drift-detected") {
|
|
855
|
+
log(`Drift detected: ${msg.codeName} (${msg.files?.join(", ")})`);
|
|
856
|
+
} else if (msg.type === "error") {
|
|
857
|
+
log(`Error: ${msg.message}`);
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
function log(msg) {
|
|
861
|
+
const ts = (/* @__PURE__ */ new Date()).toISOString();
|
|
862
|
+
process.stderr.write(`[manager-worker ${ts}] ${msg}
|
|
863
|
+
`);
|
|
864
|
+
}
|
|
865
|
+
function sha256(content) {
|
|
866
|
+
return createHash("sha256").update(content, "utf8").digest("hex");
|
|
867
|
+
}
|
|
868
|
+
function hashFile(filePath) {
|
|
869
|
+
try {
|
|
870
|
+
const content = readFileSync2(filePath, "utf-8");
|
|
871
|
+
return sha256(content);
|
|
872
|
+
} catch {
|
|
873
|
+
return null;
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
async function migrateToProfiles() {
|
|
877
|
+
const homeDir = process.env["HOME"] ?? "/tmp";
|
|
878
|
+
const sharedConfigPath = join2(homeDir, ".openclaw", "openclaw.json");
|
|
879
|
+
let sharedConfig;
|
|
880
|
+
try {
|
|
881
|
+
sharedConfig = JSON.parse(readFileSync2(sharedConfigPath, "utf-8"));
|
|
882
|
+
} catch {
|
|
883
|
+
return;
|
|
884
|
+
}
|
|
885
|
+
const agents = sharedConfig["agents"];
|
|
886
|
+
const agentList = agents?.["list"] ?? [];
|
|
887
|
+
if (agentList.length === 0) return;
|
|
888
|
+
const adapter = getFramework("openclaw");
|
|
889
|
+
let migrated = 0;
|
|
890
|
+
for (const agentEntry of agentList) {
|
|
891
|
+
const codeName = agentEntry["id"];
|
|
892
|
+
if (!codeName) continue;
|
|
893
|
+
if (codeName === "main") continue;
|
|
894
|
+
const profileDir = join2(homeDir, `.openclaw-${codeName}`);
|
|
895
|
+
if (existsSync2(join2(profileDir, "openclaw.json"))) continue;
|
|
896
|
+
log(`Migrating agent '${codeName}' to per-agent profile`);
|
|
897
|
+
if (adapter.seedProfileConfig) {
|
|
898
|
+
adapter.seedProfileConfig(codeName);
|
|
899
|
+
}
|
|
900
|
+
const sharedAuthDir = join2(homeDir, ".openclaw", "agents", codeName, "agent");
|
|
901
|
+
const profileAuthDir = join2(profileDir, "agents", codeName, "agent");
|
|
902
|
+
const authFile = join2(sharedAuthDir, "auth-profiles.json");
|
|
903
|
+
if (existsSync2(authFile)) {
|
|
904
|
+
mkdirSync(profileAuthDir, { recursive: true });
|
|
905
|
+
const authContent = readFileSync2(authFile, "utf-8");
|
|
906
|
+
writeFileSync(join2(profileAuthDir, "auth-profiles.json"), authContent);
|
|
907
|
+
}
|
|
908
|
+
allocatePort(codeName);
|
|
909
|
+
migrated++;
|
|
910
|
+
}
|
|
911
|
+
if (migrated > 0) {
|
|
912
|
+
log(`Migration complete: ${migrated} agent(s) migrated to per-agent profiles`);
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
function resolveModelChain(refreshData) {
|
|
916
|
+
const agent = refreshData.agent;
|
|
917
|
+
const modelDefaults = refreshData.model_defaults;
|
|
918
|
+
const platform = modelDefaults?.platform ?? {};
|
|
919
|
+
const org = modelDefaults?.org ?? {};
|
|
920
|
+
function resolve(tier) {
|
|
921
|
+
const agentField = `${tier}_model`;
|
|
922
|
+
const platformField = `default_${tier}_model`;
|
|
923
|
+
const agentVal = agent?.[agentField];
|
|
924
|
+
if (agentVal) return agentVal;
|
|
925
|
+
const orgVal = org?.[platformField];
|
|
926
|
+
if (orgVal) return orgVal;
|
|
927
|
+
const platformVal = platform?.[platformField];
|
|
928
|
+
if (platformVal) return platformVal;
|
|
929
|
+
return void 0;
|
|
930
|
+
}
|
|
931
|
+
return {
|
|
932
|
+
primary: resolve("primary"),
|
|
933
|
+
secondary: resolve("secondary"),
|
|
934
|
+
tertiary: resolve("tertiary")
|
|
935
|
+
};
|
|
936
|
+
}
|
|
937
|
+
function readGatewayToken(codeName) {
|
|
938
|
+
const adapter = resolveAgentFramework(codeName);
|
|
939
|
+
if (adapter.readGatewayToken) {
|
|
940
|
+
return adapter.readGatewayToken(codeName);
|
|
941
|
+
}
|
|
942
|
+
const homeDir = process.env["HOME"] ?? "/tmp";
|
|
943
|
+
try {
|
|
944
|
+
const cfg = JSON.parse(readFileSync2(join2(homeDir, `.openclaw-${codeName}`, "openclaw.json"), "utf-8"));
|
|
945
|
+
return cfg?.gateway?.auth?.token;
|
|
946
|
+
} catch {
|
|
947
|
+
return void 0;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
var GATEWAY_HUNG_TIMEOUT_MS = 5 * 6e4;
|
|
951
|
+
function isGatewayHung(codeName) {
|
|
952
|
+
const homeDir = process.env["HOME"] ?? "/tmp";
|
|
953
|
+
const jobsPath = join2(homeDir, `.openclaw-${codeName}`, "cron", "jobs.json");
|
|
954
|
+
if (!existsSync2(jobsPath)) return false;
|
|
955
|
+
try {
|
|
956
|
+
const data = JSON.parse(readFileSync2(jobsPath, "utf-8"));
|
|
957
|
+
const jobs = data.jobs ?? data;
|
|
958
|
+
if (!Array.isArray(jobs)) return false;
|
|
959
|
+
const now = Date.now();
|
|
960
|
+
for (const job of jobs) {
|
|
961
|
+
const state2 = job.state;
|
|
962
|
+
if (!state2) continue;
|
|
963
|
+
const runStartedAt = state2.runStartedAtMs;
|
|
964
|
+
const isRunning = state2.status === "running" || state2.running === true;
|
|
965
|
+
if (isRunning && runStartedAt && now - runStartedAt > GATEWAY_HUNG_TIMEOUT_MS) {
|
|
966
|
+
return true;
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
} catch {
|
|
970
|
+
}
|
|
971
|
+
return false;
|
|
972
|
+
}
|
|
973
|
+
async function ensureGatewayRunning(codeName, adapter) {
|
|
974
|
+
if (!adapter.isGatewayRunning || !adapter.startGateway) {
|
|
975
|
+
return { pid: null, port: null, running: false };
|
|
976
|
+
}
|
|
977
|
+
const status = await adapter.isGatewayRunning(codeName);
|
|
978
|
+
if (status.running) {
|
|
979
|
+
if (await isGatewayHung(codeName)) {
|
|
980
|
+
log(`Gateway for '${codeName}' appears hung (cron stuck >5min) \u2014 restarting`);
|
|
981
|
+
if (adapter.stopGateway) {
|
|
982
|
+
try {
|
|
983
|
+
await adapter.stopGateway(codeName);
|
|
984
|
+
} catch {
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
await new Promise((r) => setTimeout(r, 2e3));
|
|
988
|
+
const homeDir = process.env["HOME"] ?? "/tmp";
|
|
989
|
+
const cronJobsPath = join2(homeDir, `.openclaw-${codeName}`, "cron", "jobs.json");
|
|
990
|
+
clearStaleCronRunState(cronJobsPath);
|
|
991
|
+
} else {
|
|
992
|
+
if (status.port) {
|
|
993
|
+
try {
|
|
994
|
+
const homeDir = process.env["HOME"] ?? "/tmp";
|
|
995
|
+
const configPath = join2(homeDir, `.openclaw-${codeName}`, "openclaw.json");
|
|
996
|
+
if (existsSync2(configPath)) {
|
|
997
|
+
const cfg = JSON.parse(readFileSync2(configPath, "utf-8"));
|
|
998
|
+
if (cfg.gateway?.port !== status.port) {
|
|
999
|
+
if (!cfg.gateway) cfg.gateway = {};
|
|
1000
|
+
cfg.gateway.port = status.port;
|
|
1001
|
+
writeFileSync(configPath, JSON.stringify(cfg, null, 2));
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
} catch {
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
if (gatewayPool && status.port && !gatewayPool.hasAgent(codeName)) {
|
|
1008
|
+
const token = readGatewayToken(codeName);
|
|
1009
|
+
gatewayPool.addAgent(codeName, status.port, token);
|
|
1010
|
+
}
|
|
1011
|
+
return { pid: status.pid ?? null, port: status.port ?? null, running: true };
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
const port = allocatePort(codeName);
|
|
1015
|
+
try {
|
|
1016
|
+
const result = await adapter.startGateway(codeName, port);
|
|
1017
|
+
log(`Gateway started for '${codeName}' on port ${port} (PID ${result.pid})`);
|
|
1018
|
+
gatewaysStartedThisCycle.add(codeName);
|
|
1019
|
+
try {
|
|
1020
|
+
const homeDir = process.env["HOME"] ?? "/tmp";
|
|
1021
|
+
const configPath = join2(homeDir, `.openclaw-${codeName}`, "openclaw.json");
|
|
1022
|
+
if (existsSync2(configPath)) {
|
|
1023
|
+
const cfg = JSON.parse(readFileSync2(configPath, "utf-8"));
|
|
1024
|
+
if (!cfg.gateway) cfg.gateway = {};
|
|
1025
|
+
cfg.gateway.port = port;
|
|
1026
|
+
writeFileSync(configPath, JSON.stringify(cfg, null, 2));
|
|
1027
|
+
}
|
|
1028
|
+
} catch {
|
|
1029
|
+
}
|
|
1030
|
+
if (gatewayPool) {
|
|
1031
|
+
const token = readGatewayToken(codeName);
|
|
1032
|
+
gatewayPool.addAgent(codeName, port, token);
|
|
1033
|
+
}
|
|
1034
|
+
return { pid: result.pid, port, running: true };
|
|
1035
|
+
} catch (err) {
|
|
1036
|
+
log(`Failed to start gateway for '${codeName}': ${err.message}`);
|
|
1037
|
+
return { pid: null, port, running: false };
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
async function stopGatewayIfRunning(codeName, adapter) {
|
|
1041
|
+
if (!adapter.stopGateway) return;
|
|
1042
|
+
try {
|
|
1043
|
+
const stopped = await adapter.stopGateway(codeName);
|
|
1044
|
+
if (stopped) {
|
|
1045
|
+
log(`Gateway stopped for '${codeName}'`);
|
|
1046
|
+
}
|
|
1047
|
+
} catch (err) {
|
|
1048
|
+
log(`Failed to stop gateway for '${codeName}': ${err.message}`);
|
|
1049
|
+
}
|
|
1050
|
+
if (gatewayPool) {
|
|
1051
|
+
gatewayPool.removeAgent(codeName);
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
async function stopAllGateways() {
|
|
1055
|
+
const ports = loadGatewayPorts();
|
|
1056
|
+
for (const codeName of Object.keys(ports)) {
|
|
1057
|
+
const adapter = resolveAgentFramework(codeName);
|
|
1058
|
+
await stopGatewayIfRunning(codeName, adapter);
|
|
1059
|
+
}
|
|
1060
|
+
if (gatewayPool) {
|
|
1061
|
+
gatewayPool.disconnectAll();
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
async function healthCheckGateways(agentStates) {
|
|
1065
|
+
for (const agentState of agentStates) {
|
|
1066
|
+
if (agentState.status !== "active" || !agentState.gatewayPort) continue;
|
|
1067
|
+
const adapter = resolveAgentFramework(agentState.codeName);
|
|
1068
|
+
if (!adapter.isGatewayRunning || !adapter.startGateway) continue;
|
|
1069
|
+
if (gatewaysStartedThisCycle.has(agentState.codeName)) continue;
|
|
1070
|
+
const status = await adapter.isGatewayRunning(agentState.codeName);
|
|
1071
|
+
if (!status.running && agentState.gatewayRunning) {
|
|
1072
|
+
const displayName = agentDisplayNames.get(agentState.codeName) ?? agentState.codeName;
|
|
1073
|
+
log(`Gateway for '${agentState.codeName}' crashed, restarting...`);
|
|
1074
|
+
sendSlackWebhookMessage(
|
|
1075
|
+
`:red_circle: *Host Down* \u2014 *${displayName}* (\`${agentState.codeName}\`)
|
|
1076
|
+
OpenClaw gateway crashed. Attempting automatic restart...`
|
|
1077
|
+
).catch(() => {
|
|
1078
|
+
});
|
|
1079
|
+
try {
|
|
1080
|
+
const result = await adapter.startGateway(agentState.codeName, agentState.gatewayPort);
|
|
1081
|
+
agentState.gatewayPid = result.pid;
|
|
1082
|
+
agentState.gatewayRunning = true;
|
|
1083
|
+
log(`Gateway restarted for '${agentState.codeName}' (PID ${result.pid})`);
|
|
1084
|
+
await new Promise((resolve) => setTimeout(resolve, 2e3));
|
|
1085
|
+
if (gatewayPool) {
|
|
1086
|
+
const token = readGatewayToken(agentState.codeName);
|
|
1087
|
+
gatewayPool.addAgent(agentState.codeName, agentState.gatewayPort, token);
|
|
1088
|
+
}
|
|
1089
|
+
sendSlackWebhookMessage(
|
|
1090
|
+
`:large_green_circle: *Host Recovered* \u2014 *${displayName}* (\`${agentState.codeName}\`)
|
|
1091
|
+
Gateway restarted successfully (PID ${result.pid}).`
|
|
1092
|
+
).catch(() => {
|
|
1093
|
+
});
|
|
1094
|
+
} catch (err) {
|
|
1095
|
+
agentState.gatewayRunning = false;
|
|
1096
|
+
log(`Failed to restart gateway for '${agentState.codeName}': ${err.message}`);
|
|
1097
|
+
sendSlackWebhookMessage(
|
|
1098
|
+
`:x: *Host Restart Failed* \u2014 *${displayName}* (\`${agentState.codeName}\`)
|
|
1099
|
+
Automatic restart failed: ${err.message}`
|
|
1100
|
+
).catch(() => {
|
|
1101
|
+
});
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
async function pollCycle() {
|
|
1107
|
+
if (!config) return;
|
|
1108
|
+
try {
|
|
1109
|
+
registeredAgentsCache.clear();
|
|
1110
|
+
gatewaysStartedThisCycle.clear();
|
|
1111
|
+
const hostId = await getHostId();
|
|
1112
|
+
if (!hostId) {
|
|
1113
|
+
send({ type: "error", message: "Could not resolve host ID from API key" });
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
const now = Date.now();
|
|
1117
|
+
if (now - lastVersionCheckAt > VERSION_CHECK_INTERVAL_MS) {
|
|
1118
|
+
try {
|
|
1119
|
+
const firstAgent = state.agents[0];
|
|
1120
|
+
const versionAdapter = firstAgent ? resolveAgentFramework(firstAgent.codeName) : getFramework("openclaw");
|
|
1121
|
+
if (versionAdapter.getVersion) {
|
|
1122
|
+
cachedFrameworkVersion = await versionAdapter.getVersion();
|
|
1123
|
+
}
|
|
1124
|
+
} catch {
|
|
1125
|
+
}
|
|
1126
|
+
lastVersionCheckAt = now;
|
|
1127
|
+
}
|
|
1128
|
+
try {
|
|
1129
|
+
await api.post("/host/heartbeat", {
|
|
1130
|
+
host_id: hostId,
|
|
1131
|
+
framework_version: cachedFrameworkVersion ?? void 0
|
|
1132
|
+
});
|
|
1133
|
+
} catch (err) {
|
|
1134
|
+
log(`Heartbeat failed: ${err.message}`);
|
|
1135
|
+
}
|
|
1136
|
+
const data = await api.post("/host/agents", { host_id: hostId });
|
|
1137
|
+
const agents = data.agents ?? [];
|
|
1138
|
+
activeChannels.clear();
|
|
1139
|
+
const agentStates = [];
|
|
1140
|
+
for (const agent of agents) {
|
|
1141
|
+
try {
|
|
1142
|
+
await processAgent(agent, agentStates);
|
|
1143
|
+
} catch (err) {
|
|
1144
|
+
log(`Error processing agent '${agent.code_name}': ${err.message}`);
|
|
1145
|
+
const existing = state.agents.find((a) => a.agentId === agent.agent_id);
|
|
1146
|
+
if (existing) {
|
|
1147
|
+
agentStates.push(existing);
|
|
1148
|
+
} else {
|
|
1149
|
+
agentStates.push({
|
|
1150
|
+
agentId: agent.agent_id,
|
|
1151
|
+
codeName: agent.code_name,
|
|
1152
|
+
status: agent.status,
|
|
1153
|
+
charterVersion: "",
|
|
1154
|
+
toolsVersion: "",
|
|
1155
|
+
secretsHash: null,
|
|
1156
|
+
lastRefreshAt: null,
|
|
1157
|
+
lastProvisionAt: null,
|
|
1158
|
+
lastDriftCheckAt: null,
|
|
1159
|
+
lastSecretsProvisionAt: null,
|
|
1160
|
+
gatewayPort: null,
|
|
1161
|
+
gatewayPid: null,
|
|
1162
|
+
gatewayRunning: false,
|
|
1163
|
+
acpSessions: []
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
try {
|
|
1169
|
+
for (const [channelId, codeNames] of activeChannels) {
|
|
1170
|
+
for (const codeName of codeNames) {
|
|
1171
|
+
const adapter = resolveAgentFramework(codeName);
|
|
1172
|
+
if (adapter.setChannelEnabled) {
|
|
1173
|
+
adapter.setChannelEnabled(channelId, true, codeName);
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
} catch {
|
|
1178
|
+
}
|
|
1179
|
+
const currentIds = new Set(agents.map((a) => a.agent_id));
|
|
1180
|
+
for (const prev of state.agents) {
|
|
1181
|
+
if (!currentIds.has(prev.agentId)) {
|
|
1182
|
+
log(`Agent '${prev.codeName}' unassigned from host`);
|
|
1183
|
+
const adapter = resolveAgentFramework(prev.codeName);
|
|
1184
|
+
clearAgentCaches(prev.agentId, prev.codeName);
|
|
1185
|
+
await stopGatewayIfRunning(prev.codeName, adapter);
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
await healthCheckGateways(agentStates);
|
|
1189
|
+
const lastHealthCheck = lastHarvestAt.get("__cron_health__") ?? 0;
|
|
1190
|
+
if (Date.now() - lastHealthCheck >= HARVEST_INTERVAL_MS) {
|
|
1191
|
+
lastHarvestAt.set("__cron_health__", Date.now());
|
|
1192
|
+
monitorCronHealth(agentStates).catch((err) => {
|
|
1193
|
+
log(`Cron health monitor error: ${err.message}`);
|
|
1194
|
+
});
|
|
1195
|
+
}
|
|
1196
|
+
if (!isRealtimeConnected()) {
|
|
1197
|
+
pollDirectChatMessages(agentStates).catch((err) => {
|
|
1198
|
+
log(`Direct chat poll error: ${err.message}`);
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
ensureRealtimeStarted(agentStates);
|
|
1202
|
+
ensureRealtimeDriftStarted(agentStates);
|
|
1203
|
+
ensureRealtimeAssignStarted(agentStates);
|
|
1204
|
+
ensureRealtimeConfigStarted(agentStates);
|
|
1205
|
+
ensureRealtimeKanbanStarted(agentStates);
|
|
1206
|
+
state = {
|
|
1207
|
+
...state,
|
|
1208
|
+
lastPollAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1209
|
+
pollCount: state.pollCount + 1,
|
|
1210
|
+
agents: agentStates
|
|
1211
|
+
};
|
|
1212
|
+
send({ type: "state-update", state });
|
|
1213
|
+
} catch (err) {
|
|
1214
|
+
state.errorCount++;
|
|
1215
|
+
const message = err.message;
|
|
1216
|
+
log(`Poll error: ${message}`);
|
|
1217
|
+
send({ type: "error", message });
|
|
1218
|
+
if (message.includes("exchange failed") || message.includes("401")) {
|
|
1219
|
+
log("Fatal auth error, exiting for watchdog restart");
|
|
1220
|
+
send({ type: "shutdown" });
|
|
1221
|
+
process.exit(1);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
async function getOrCacheRegisteredAgents(adapter, profile) {
|
|
1226
|
+
const cacheKey = profile ? `${adapter.id}:${profile}` : adapter.id;
|
|
1227
|
+
let cached = registeredAgentsCache.get(cacheKey);
|
|
1228
|
+
if (!cached) {
|
|
1229
|
+
cached = await adapter.getRegisteredAgents(profile);
|
|
1230
|
+
registeredAgentsCache.set(cacheKey, cached);
|
|
1231
|
+
}
|
|
1232
|
+
return cached;
|
|
1233
|
+
}
|
|
1234
|
+
async function processAgent(agent, agentStates) {
|
|
1235
|
+
if (!config) return;
|
|
1236
|
+
log(`==================== ${agent.display_name} (${agent.code_name}) ====================`);
|
|
1237
|
+
agentDisplayNames.set(agent.code_name, agent.display_name);
|
|
1238
|
+
codeNameToAgentId.set(agent.code_name, agent.agent_id);
|
|
1239
|
+
if (agent.framework) {
|
|
1240
|
+
agentFrameworkCache.set(agent.code_name, agent.framework);
|
|
1241
|
+
}
|
|
1242
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1243
|
+
const agentDir = join2(config.configDir, agent.code_name, "provision");
|
|
1244
|
+
const adapter = resolveAgentFramework(agent.code_name);
|
|
1245
|
+
if (agent.status === "draft" || agent.status === "paused") {
|
|
1246
|
+
log(`Agent '${agent.code_name}' is ${agent.status}, skipping provisioning`);
|
|
1247
|
+
await stopGatewayIfRunning(agent.code_name, adapter);
|
|
1248
|
+
stopPersistentSession(agent.code_name, log);
|
|
1249
|
+
try {
|
|
1250
|
+
const { execSync: es } = await import("child_process");
|
|
1251
|
+
es(`tmux kill-session -t agt-${agent.code_name} 2>/dev/null`, { stdio: "ignore" });
|
|
1252
|
+
log(`Killed tmux session for paused agent '${agent.code_name}'`);
|
|
1253
|
+
} catch {
|
|
1254
|
+
}
|
|
1255
|
+
agentStates.push({
|
|
1256
|
+
agentId: agent.agent_id,
|
|
1257
|
+
codeName: agent.code_name,
|
|
1258
|
+
status: agent.status,
|
|
1259
|
+
charterVersion: "",
|
|
1260
|
+
toolsVersion: "",
|
|
1261
|
+
secretsHash: null,
|
|
1262
|
+
lastRefreshAt: now,
|
|
1263
|
+
lastProvisionAt: null,
|
|
1264
|
+
lastDriftCheckAt: null,
|
|
1265
|
+
lastSecretsProvisionAt: null,
|
|
1266
|
+
gatewayPort: null,
|
|
1267
|
+
gatewayPid: null,
|
|
1268
|
+
gatewayRunning: false,
|
|
1269
|
+
acpSessions: []
|
|
1270
|
+
});
|
|
1271
|
+
return;
|
|
1272
|
+
}
|
|
1273
|
+
if (agent.status === "revoked") {
|
|
1274
|
+
log(`Agent '${agent.code_name}' is revoked, cleaning up`);
|
|
1275
|
+
await stopGatewayIfRunning(agent.code_name, adapter);
|
|
1276
|
+
stopPersistentSession(agent.code_name, log);
|
|
1277
|
+
try {
|
|
1278
|
+
const { execSync: es } = await import("child_process");
|
|
1279
|
+
es(`tmux kill-session -t agt-${agent.code_name} 2>/dev/null`, { stdio: "ignore" });
|
|
1280
|
+
} catch {
|
|
1281
|
+
}
|
|
1282
|
+
freePort(agent.code_name);
|
|
1283
|
+
await cleanupAgentFiles(agent.code_name, agentDir);
|
|
1284
|
+
clearAgentCaches(agent.agent_id, agent.code_name);
|
|
1285
|
+
knownStatuses.set(agent.agent_id, agent.status);
|
|
1286
|
+
agentStates.push({
|
|
1287
|
+
agentId: agent.agent_id,
|
|
1288
|
+
codeName: agent.code_name,
|
|
1289
|
+
status: agent.status,
|
|
1290
|
+
charterVersion: "",
|
|
1291
|
+
toolsVersion: "",
|
|
1292
|
+
secretsHash: null,
|
|
1293
|
+
lastRefreshAt: now,
|
|
1294
|
+
lastProvisionAt: null,
|
|
1295
|
+
lastDriftCheckAt: null,
|
|
1296
|
+
lastSecretsProvisionAt: null,
|
|
1297
|
+
gatewayPort: null,
|
|
1298
|
+
gatewayPid: null,
|
|
1299
|
+
gatewayRunning: false,
|
|
1300
|
+
acpSessions: []
|
|
1301
|
+
});
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1304
|
+
const previousStatus = knownStatuses.get(agent.agent_id);
|
|
1305
|
+
if (previousStatus && previousStatus !== agent.status) {
|
|
1306
|
+
log(`Agent '${agent.code_name}' status changed: ${previousStatus} \u2192 ${agent.status}`);
|
|
1307
|
+
knownVersions.delete(agent.agent_id);
|
|
1308
|
+
for (const key of knownChannelConfigHashes.keys()) {
|
|
1309
|
+
if (key.startsWith(`${agent.agent_id}:`)) knownChannelConfigHashes.delete(key);
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
knownStatuses.set(agent.agent_id, agent.status);
|
|
1313
|
+
let refreshData;
|
|
1314
|
+
try {
|
|
1315
|
+
refreshData = await api.post("/host/refresh", {
|
|
1316
|
+
agent_id: agent.agent_id
|
|
1317
|
+
});
|
|
1318
|
+
} catch (err) {
|
|
1319
|
+
log(`Refresh failed for '${agent.code_name}': ${err.message}`);
|
|
1320
|
+
const existing = state.agents.find((a) => a.agentId === agent.agent_id);
|
|
1321
|
+
agentStates.push(existing ?? {
|
|
1322
|
+
agentId: agent.agent_id,
|
|
1323
|
+
codeName: agent.code_name,
|
|
1324
|
+
status: agent.status,
|
|
1325
|
+
charterVersion: "",
|
|
1326
|
+
toolsVersion: "",
|
|
1327
|
+
secretsHash: null,
|
|
1328
|
+
lastRefreshAt: null,
|
|
1329
|
+
lastProvisionAt: null,
|
|
1330
|
+
lastDriftCheckAt: null,
|
|
1331
|
+
lastSecretsProvisionAt: null,
|
|
1332
|
+
gatewayPort: null,
|
|
1333
|
+
gatewayPid: null,
|
|
1334
|
+
gatewayRunning: false,
|
|
1335
|
+
acpSessions: []
|
|
1336
|
+
});
|
|
1337
|
+
return;
|
|
1338
|
+
}
|
|
1339
|
+
if (!alertSlackWebhook && refreshData.team?.settings) {
|
|
1340
|
+
const webhook = refreshData.team.settings["alert_slack_webhook"];
|
|
1341
|
+
if (typeof webhook === "string" && webhook.startsWith("https://")) {
|
|
1342
|
+
alertSlackWebhook = webhook;
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
if (!refreshData.charter || !refreshData.tools) {
|
|
1346
|
+
log(`No charter/tools for '${agent.code_name}', skipping`);
|
|
1347
|
+
agentStates.push({
|
|
1348
|
+
agentId: agent.agent_id,
|
|
1349
|
+
codeName: agent.code_name,
|
|
1350
|
+
status: agent.status,
|
|
1351
|
+
charterVersion: "",
|
|
1352
|
+
toolsVersion: "",
|
|
1353
|
+
secretsHash: null,
|
|
1354
|
+
lastRefreshAt: now,
|
|
1355
|
+
lastProvisionAt: null,
|
|
1356
|
+
lastDriftCheckAt: null,
|
|
1357
|
+
lastSecretsProvisionAt: null,
|
|
1358
|
+
gatewayPort: null,
|
|
1359
|
+
gatewayPid: null,
|
|
1360
|
+
gatewayRunning: false,
|
|
1361
|
+
acpSessions: []
|
|
1362
|
+
});
|
|
1363
|
+
return;
|
|
1364
|
+
}
|
|
1365
|
+
const frameworkId = refreshData.agent.framework ?? "openclaw";
|
|
1366
|
+
agentFrameworkCache.set(agent.code_name, frameworkId);
|
|
1367
|
+
const frameworkAdapter = getFramework(frameworkId);
|
|
1368
|
+
if (frameworkAdapter.seedProfileConfig) {
|
|
1369
|
+
frameworkAdapter.seedProfileConfig(agent.code_name);
|
|
1370
|
+
}
|
|
1371
|
+
const charterVersion = refreshData.charter.version;
|
|
1372
|
+
const toolsVersion = refreshData.tools.version;
|
|
1373
|
+
const known = knownVersions.get(agent.agent_id);
|
|
1374
|
+
let lastProvisionAt = state.agents.find((a) => a.agentId === agent.agent_id)?.lastProvisionAt ?? null;
|
|
1375
|
+
const currentChannelIds = new Set(Object.keys(refreshData.channel_configs ?? {}));
|
|
1376
|
+
const previousChannelIds = knownChannels.get(agent.agent_id);
|
|
1377
|
+
const channelsChanged = !previousChannelIds || currentChannelIds.size !== previousChannelIds.size || [...currentChannelIds].some((ch) => !previousChannelIds.has(ch)) || [...previousChannelIds].some((ch) => !currentChannelIds.has(ch));
|
|
1378
|
+
if (previousChannelIds && channelsChanged && frameworkAdapter.removeChannelCredentials) {
|
|
1379
|
+
for (const ch of previousChannelIds) {
|
|
1380
|
+
if (!currentChannelIds.has(ch)) {
|
|
1381
|
+
try {
|
|
1382
|
+
frameworkAdapter.removeChannelCredentials(agent.code_name, ch);
|
|
1383
|
+
log(`Removed ${ch} credentials for '${agent.code_name}'`);
|
|
1384
|
+
} catch (err) {
|
|
1385
|
+
log(`Failed to remove ${ch} credentials for '${agent.code_name}': ${err.message}`);
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
knownChannels.set(agent.agent_id, currentChannelIds);
|
|
1391
|
+
try {
|
|
1392
|
+
const artifacts = generateArtifacts(agent, refreshData, frameworkAdapter);
|
|
1393
|
+
const changedFiles = [];
|
|
1394
|
+
mkdirSync(agentDir, { recursive: true });
|
|
1395
|
+
for (const artifact of artifacts) {
|
|
1396
|
+
const filePath = join2(agentDir, artifact.relativePath);
|
|
1397
|
+
const newHash = sha256(artifact.content);
|
|
1398
|
+
const existingHash = hashFile(filePath);
|
|
1399
|
+
if (newHash !== existingHash) {
|
|
1400
|
+
changedFiles.push(artifact);
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
if (changedFiles.length > 0) {
|
|
1404
|
+
const isFirst = !existsSync2(join2(agentDir, "CHARTER.md"));
|
|
1405
|
+
const verb = isFirst ? "Provisioning" : "Updating";
|
|
1406
|
+
const fileNames = changedFiles.map((f) => f.relativePath).join(", ");
|
|
1407
|
+
log(`${verb} '${agent.code_name}': ${fileNames}`);
|
|
1408
|
+
for (const file of changedFiles) {
|
|
1409
|
+
writeFileSync(join2(agentDir, file.relativePath), file.content);
|
|
1410
|
+
}
|
|
1411
|
+
lastProvisionAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1412
|
+
knownVersions.set(agent.agent_id, { charterVersion, toolsVersion });
|
|
1413
|
+
const trackedFiles = frameworkAdapter.driftTrackedFiles();
|
|
1414
|
+
const hashes = /* @__PURE__ */ new Map();
|
|
1415
|
+
for (const file of trackedFiles) {
|
|
1416
|
+
const h = hashFile(join2(agentDir, file));
|
|
1417
|
+
if (h) hashes.set(file, h);
|
|
1418
|
+
}
|
|
1419
|
+
writtenHashes.set(agent.agent_id, hashes);
|
|
1420
|
+
const resolvedModelsForRegistration = resolveModelChain(refreshData);
|
|
1421
|
+
const primaryModel2 = resolvedModelsForRegistration.primary ?? refreshData.agent.primary_model;
|
|
1422
|
+
const registeredAgents = await getOrCacheRegisteredAgents(frameworkAdapter, agent.code_name);
|
|
1423
|
+
if (!registeredAgents.has(agent.code_name)) {
|
|
1424
|
+
const registered = await frameworkAdapter.registerAgent(agent.code_name, agentDir, primaryModel2);
|
|
1425
|
+
if (registered) {
|
|
1426
|
+
registeredAgents.add(agent.code_name);
|
|
1427
|
+
log(`Registered '${agent.code_name}' in ${frameworkAdapter.label}`);
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
send({ type: "provisioned", agentId: agent.agent_id, codeName: agent.code_name });
|
|
1431
|
+
}
|
|
1432
|
+
if (frameworkAdapter.deployArtifactsToProject) {
|
|
1433
|
+
frameworkAdapter.deployArtifactsToProject(agent.code_name, agentDir);
|
|
1434
|
+
}
|
|
1435
|
+
} catch (err) {
|
|
1436
|
+
log(`Provision failed for '${agent.code_name}': ${err.message}`);
|
|
1437
|
+
}
|
|
1438
|
+
const resolvedForModel = resolveModelChain(refreshData);
|
|
1439
|
+
const primaryModel = resolvedForModel.primary ?? refreshData.agent.primary_model;
|
|
1440
|
+
if (primaryModel && frameworkAdapter.updateAgentModel) {
|
|
1441
|
+
const previousModel = knownModels.get(agent.agent_id);
|
|
1442
|
+
if (previousModel !== primaryModel) {
|
|
1443
|
+
try {
|
|
1444
|
+
const updated = await frameworkAdapter.updateAgentModel(agent.code_name, primaryModel);
|
|
1445
|
+
if (updated) {
|
|
1446
|
+
if (previousModel) {
|
|
1447
|
+
log(`Model updated for '${agent.code_name}': ${previousModel} \u2192 ${primaryModel}`);
|
|
1448
|
+
} else {
|
|
1449
|
+
log(`Model set for '${agent.code_name}': ${primaryModel}`);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
} catch (err) {
|
|
1453
|
+
log(`Failed to update model for '${agent.code_name}': ${err.message}`);
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
knownModels.set(agent.agent_id, primaryModel);
|
|
1457
|
+
}
|
|
1458
|
+
let lastDriftCheckAt = now;
|
|
1459
|
+
const written = writtenHashes.get(agent.agent_id);
|
|
1460
|
+
if (written && existsSync2(agentDir)) {
|
|
1461
|
+
const driftedFiles = [];
|
|
1462
|
+
for (const [file, expectedHash] of written) {
|
|
1463
|
+
const localHash = hashFile(join2(agentDir, file));
|
|
1464
|
+
if (localHash && localHash !== expectedHash) {
|
|
1465
|
+
driftedFiles.push(file);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
if (driftedFiles.length > 0) {
|
|
1469
|
+
log(`Drift detected for '${agent.code_name}': ${driftedFiles.join(", ")}`);
|
|
1470
|
+
send({ type: "drift-detected", agentId: agent.agent_id, codeName: agent.code_name, files: driftedFiles });
|
|
1471
|
+
try {
|
|
1472
|
+
const localHashes = {};
|
|
1473
|
+
for (const file of driftedFiles) {
|
|
1474
|
+
localHashes[file] = hashFile(join2(agentDir, file));
|
|
1475
|
+
}
|
|
1476
|
+
await api.post("/host/drift", {
|
|
1477
|
+
agent_id: agent.agent_id,
|
|
1478
|
+
drifted_files: driftedFiles,
|
|
1479
|
+
local_hashes: localHashes
|
|
1480
|
+
});
|
|
1481
|
+
} catch (err) {
|
|
1482
|
+
log(`Failed to report drift for '${agent.code_name}': ${err.message}`);
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
if (refreshData.channel_configs) {
|
|
1487
|
+
const tokens = {};
|
|
1488
|
+
const slackCfg = refreshData.channel_configs["slack"];
|
|
1489
|
+
if (slackCfg?.config) {
|
|
1490
|
+
const bt = slackCfg.config.bot_token;
|
|
1491
|
+
if (typeof bt === "string" && bt) tokens.slack = bt;
|
|
1492
|
+
}
|
|
1493
|
+
const tgCfg = refreshData.channel_configs["telegram"];
|
|
1494
|
+
if (tgCfg?.config) {
|
|
1495
|
+
const tgConfig = tgCfg.config;
|
|
1496
|
+
const bt = tgConfig.bot_token;
|
|
1497
|
+
if (typeof bt === "string" && bt) tokens.telegram = bt;
|
|
1498
|
+
const allowedChats = tgConfig.allowed_chat_ids;
|
|
1499
|
+
if (Array.isArray(allowedChats)) tokens.telegramAllowedChats = allowedChats;
|
|
1500
|
+
}
|
|
1501
|
+
if (tokens.slack || tokens.telegram) {
|
|
1502
|
+
agentChannelTokens.set(agent.code_name, tokens);
|
|
1503
|
+
} else {
|
|
1504
|
+
agentChannelTokens.delete(agent.code_name);
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
let needsGatewayRestart = false;
|
|
1508
|
+
const hasChannelConfigs = refreshData.channel_configs && Object.keys(refreshData.channel_configs).length > 0;
|
|
1509
|
+
if (refreshData.channel_configs && frameworkAdapter.writeChannelCredentials) {
|
|
1510
|
+
if (agent.status === "active") {
|
|
1511
|
+
for (const [channelId, entry] of Object.entries(refreshData.channel_configs)) {
|
|
1512
|
+
if ((entry.status === "active" || entry.status === "pending") && entry.config) {
|
|
1513
|
+
if (!activeChannels.has(channelId)) {
|
|
1514
|
+
activeChannels.set(channelId, /* @__PURE__ */ new Set());
|
|
1515
|
+
}
|
|
1516
|
+
activeChannels.get(channelId).add(agent.code_name);
|
|
1517
|
+
const configHash = createHash("sha256").update(JSON.stringify(entry.config)).digest("hex");
|
|
1518
|
+
const cacheKey = `${agent.agent_id}:${channelId}`;
|
|
1519
|
+
if (knownChannelConfigHashes.get(cacheKey) === configHash) {
|
|
1520
|
+
continue;
|
|
1521
|
+
}
|
|
1522
|
+
try {
|
|
1523
|
+
const sessionMode2 = refreshData.agent.session_mode;
|
|
1524
|
+
frameworkAdapter.writeChannelCredentials(agent.code_name, channelId, entry.config, { sessionMode: sessionMode2 });
|
|
1525
|
+
knownChannelConfigHashes.set(cacheKey, configHash);
|
|
1526
|
+
log(`Channel credentials written for '${agent.code_name}/${channelId}'`);
|
|
1527
|
+
} catch (err) {
|
|
1528
|
+
log(`Failed to write channel credentials for '${agent.code_name}/${channelId}': ${err.message}`);
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
} else if (agent.status === "paused") {
|
|
1533
|
+
if (frameworkAdapter.setChannelEnabled) {
|
|
1534
|
+
for (const channelId of Object.keys(refreshData.channel_configs)) {
|
|
1535
|
+
frameworkAdapter.setChannelEnabled(channelId, false, agent.code_name);
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
let lastSecretsProvisionAt = state.agents.find((a) => a.agentId === agent.agent_id)?.lastSecretsProvisionAt ?? null;
|
|
1541
|
+
let secretsHash = knownSecretsHashes.get(agent.agent_id) ?? null;
|
|
1542
|
+
try {
|
|
1543
|
+
const secretsData = await api.post("/host/secrets", { agent_id: agent.agent_id });
|
|
1544
|
+
const newHash = secretsData.secrets_hash;
|
|
1545
|
+
const hashChanged = newHash !== secretsHash;
|
|
1546
|
+
if (hashChanged && secretsData.profiles.length > 0) {
|
|
1547
|
+
frameworkAdapter.writeAuthProfiles(agent.code_name, secretsData.profiles);
|
|
1548
|
+
lastSecretsProvisionAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1549
|
+
log(`Secrets updated for '${agent.code_name}'`);
|
|
1550
|
+
}
|
|
1551
|
+
if (newHash) {
|
|
1552
|
+
knownSecretsHashes.set(agent.agent_id, newHash);
|
|
1553
|
+
secretsHash = newHash;
|
|
1554
|
+
} else {
|
|
1555
|
+
knownSecretsHashes.delete(agent.agent_id);
|
|
1556
|
+
secretsHash = null;
|
|
1557
|
+
}
|
|
1558
|
+
} catch (err) {
|
|
1559
|
+
log(`Secrets fetch failed for '${agent.code_name}': ${err.message}`);
|
|
1560
|
+
}
|
|
1561
|
+
try {
|
|
1562
|
+
const integrationsData = await api.post("/host/agent-integrations", { agent_id: agent.agent_id });
|
|
1563
|
+
const integrations = integrationsData.integrations ?? [];
|
|
1564
|
+
for (const integration of integrations) {
|
|
1565
|
+
if (integration.auth_type !== "oauth2") continue;
|
|
1566
|
+
const expiresAt = integration.credentials?.token_expires_at;
|
|
1567
|
+
const refreshToken = integration.credentials?.refresh_token;
|
|
1568
|
+
if (!expiresAt || !refreshToken) continue;
|
|
1569
|
+
const msRemaining = new Date(expiresAt).getTime() - Date.now();
|
|
1570
|
+
if (msRemaining > 10 * 60 * 1e3) continue;
|
|
1571
|
+
try {
|
|
1572
|
+
const integrationId = integration.id;
|
|
1573
|
+
if (!integrationId) continue;
|
|
1574
|
+
const refreshResult = await api.post(
|
|
1575
|
+
`/integrations/oauth/${integrationId}/refresh`,
|
|
1576
|
+
{}
|
|
1577
|
+
);
|
|
1578
|
+
if (refreshResult.ok) {
|
|
1579
|
+
integration.credentials.token_expires_at = refreshResult.expires_at;
|
|
1580
|
+
if (refreshResult.access_token) {
|
|
1581
|
+
integration.credentials.access_token = refreshResult.access_token;
|
|
1582
|
+
}
|
|
1583
|
+
log(`OAuth token refreshed for '${agent.code_name}/${integration.definition_id}'`);
|
|
1584
|
+
if (frameworkAdapter.writeTokenFile) {
|
|
1585
|
+
frameworkAdapter.writeTokenFile(agent.code_name, integrations);
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
} catch (err) {
|
|
1589
|
+
log(`OAuth token refresh failed for '${agent.code_name}/${integration.definition_id}': ${err.message}`);
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
if (integrations.length > 0) {
|
|
1593
|
+
const intHash = createHash("sha256").update(JSON.stringify(integrations.map((i) => `${i.definition_id}:${JSON.stringify(i.credentials)}`))).digest("hex").slice(0, 16);
|
|
1594
|
+
const prevIntHash = knownIntegrationHashes.get(agent.agent_id);
|
|
1595
|
+
if (intHash !== prevIntHash) {
|
|
1596
|
+
if (frameworkAdapter.writeIntegrations) {
|
|
1597
|
+
frameworkAdapter.writeIntegrations(agent.code_name, integrations);
|
|
1598
|
+
}
|
|
1599
|
+
knownIntegrationHashes.set(agent.agent_id, intHash);
|
|
1600
|
+
log(`Integrations provisioned for '${agent.code_name}' (${integrations.length} integration(s))`);
|
|
1601
|
+
needsGatewayRestart = true;
|
|
1602
|
+
const hasLcm = integrations.some((i) => i.definition_id === "lossless-claw");
|
|
1603
|
+
if (hasLcm && !losslessClawInstalled.get(agent.code_name)) {
|
|
1604
|
+
try {
|
|
1605
|
+
const { execFileSync } = await import("child_process");
|
|
1606
|
+
let pluginList;
|
|
1607
|
+
try {
|
|
1608
|
+
pluginList = execFileSync(
|
|
1609
|
+
"openclaw",
|
|
1610
|
+
["--profile", agent.code_name, "plugins", "list", "--json"],
|
|
1611
|
+
{ timeout: 15e3 }
|
|
1612
|
+
).toString();
|
|
1613
|
+
} catch {
|
|
1614
|
+
pluginList = "[]";
|
|
1615
|
+
}
|
|
1616
|
+
const installed = JSON.parse(pluginList);
|
|
1617
|
+
const alreadyInstalled = installed.some(
|
|
1618
|
+
(p) => p.name === "lossless-claw" || p.name === "@martian-engineering/lossless-claw"
|
|
1619
|
+
);
|
|
1620
|
+
if (!alreadyInstalled) {
|
|
1621
|
+
log(`Installing lossless-claw plugin for '${agent.code_name}'...`);
|
|
1622
|
+
execFileSync(
|
|
1623
|
+
"openclaw",
|
|
1624
|
+
["--profile", agent.code_name, "plugins", "install", "@martian-engineering/lossless-claw"],
|
|
1625
|
+
{ stdio: "ignore", timeout: 6e4 }
|
|
1626
|
+
);
|
|
1627
|
+
log(`lossless-claw plugin installed for '${agent.code_name}'`);
|
|
1628
|
+
}
|
|
1629
|
+
losslessClawInstalled.set(agent.code_name, true);
|
|
1630
|
+
} catch (pluginErr) {
|
|
1631
|
+
log(`lossless-claw plugin install failed for '${agent.code_name}': ${pluginErr.message}`);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
const resolvedDefIds = new Set(integrations.map((i) => i.definition_id));
|
|
1636
|
+
for (const capId of resolvedDefIds) {
|
|
1637
|
+
try {
|
|
1638
|
+
const capData = await api.post("/host/capability-skill", { definition_id: capId });
|
|
1639
|
+
const allSatisfied = capData.requiredIntegrations.every((reqId) => resolvedDefIds.has(reqId));
|
|
1640
|
+
if (!allSatisfied) {
|
|
1641
|
+
const missing = capData.requiredIntegrations.filter((reqId) => !resolvedDefIds.has(reqId));
|
|
1642
|
+
log(`Capability '${capId}' skipped \u2014 missing integration(s): ${missing.join(", ")}`);
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1645
|
+
const skillContent = JSON.stringify(capData.skills.map((s) => s.files.map((f) => `${f.relativePath}:${f.content}`)));
|
|
1646
|
+
const skillHash = createHash("sha256").update(skillContent).digest("hex").slice(0, 16);
|
|
1647
|
+
const skillKey = `${agent.agent_id}:${capId}`;
|
|
1648
|
+
const prevSkillHash = knownSkillHashes.get(skillKey);
|
|
1649
|
+
if (skillHash !== prevSkillHash) {
|
|
1650
|
+
if (frameworkAdapter.installSkillFiles && capData.skills) {
|
|
1651
|
+
for (const skill of capData.skills) {
|
|
1652
|
+
if (skill.files.length > 0) {
|
|
1653
|
+
frameworkAdapter.installSkillFiles(agent.code_name, skill.id, skill.files);
|
|
1654
|
+
log(`Installed skill '${skill.id}' for '${agent.code_name}' (${skill.files.length} file(s))`);
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
knownSkillHashes.set(skillKey, skillHash);
|
|
1659
|
+
}
|
|
1660
|
+
const ALLOWED_CLI_PACKAGES = /* @__PURE__ */ new Set([
|
|
1661
|
+
"xero-cli",
|
|
1662
|
+
"@openapitools/openapi-generator-cli",
|
|
1663
|
+
"gh",
|
|
1664
|
+
"@schpet/linear-cli",
|
|
1665
|
+
"@googleworkspace/cli",
|
|
1666
|
+
"@tobilu/qmd"
|
|
1667
|
+
]);
|
|
1668
|
+
if (intHash !== prevIntHash) {
|
|
1669
|
+
const { execFileSync } = await import("child_process");
|
|
1670
|
+
for (const tool of capData.cliTools) {
|
|
1671
|
+
if (!ALLOWED_CLI_PACKAGES.has(tool.package)) {
|
|
1672
|
+
log(`Skipping CLI tool '${tool.package}' for '${agent.code_name}' \u2014 not on the allowed packages list`);
|
|
1673
|
+
continue;
|
|
1674
|
+
}
|
|
1675
|
+
try {
|
|
1676
|
+
execFileSync("which", [tool.binary], { stdio: "ignore" });
|
|
1677
|
+
} catch {
|
|
1678
|
+
log(`Installing CLI tool '${tool.package}' for '${agent.code_name}'...`);
|
|
1679
|
+
try {
|
|
1680
|
+
execFileSync("npm", ["install", "-g", tool.package], { stdio: "ignore", timeout: 6e4 });
|
|
1681
|
+
log(`CLI tool '${tool.binary}' installed successfully`);
|
|
1682
|
+
} catch (installErr) {
|
|
1683
|
+
log(`Failed to install CLI tool '${tool.package}': ${installErr.message}`);
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
if (tool.binary === "qmd") {
|
|
1687
|
+
try {
|
|
1688
|
+
const agentDir2 = join2(process.env["HOME"] ?? "/tmp", ".augmented", agent.code_name);
|
|
1689
|
+
execFileSync("qmd", ["collection", "add", agent.code_name, "project"], {
|
|
1690
|
+
stdio: "ignore",
|
|
1691
|
+
timeout: 3e4,
|
|
1692
|
+
cwd: agentDir2
|
|
1693
|
+
});
|
|
1694
|
+
log(`QMD collection '${agent.code_name}' configured`);
|
|
1695
|
+
} catch {
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
} catch (skillErr) {
|
|
1701
|
+
const msg = skillErr.message ?? "";
|
|
1702
|
+
if (!msg.includes("404") && !msg.includes("Unknown capability")) {
|
|
1703
|
+
log(`Capability fetch failed for '${capId}': ${msg}`);
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
} catch (err) {
|
|
1709
|
+
log(`Integration provisioning failed for '${agent.code_name}': ${err.message}`);
|
|
1710
|
+
}
|
|
1711
|
+
let gatewayPort = null;
|
|
1712
|
+
let gatewayPid = null;
|
|
1713
|
+
let gatewayRunning = false;
|
|
1714
|
+
if (agent.status === "active" && hasChannelConfigs) {
|
|
1715
|
+
const gwStatus = await ensureGatewayRunning(agent.code_name, frameworkAdapter);
|
|
1716
|
+
gatewayPort = gwStatus.port;
|
|
1717
|
+
gatewayPid = gwStatus.pid;
|
|
1718
|
+
gatewayRunning = gwStatus.running;
|
|
1719
|
+
} else if (agent.status === "paused") {
|
|
1720
|
+
await stopGatewayIfRunning(agent.code_name, frameworkAdapter);
|
|
1721
|
+
}
|
|
1722
|
+
let tasks = refreshData.scheduled_tasks ?? [];
|
|
1723
|
+
const existingTemplateIds = new Set(tasks.map((t) => t.template_id));
|
|
1724
|
+
try {
|
|
1725
|
+
const defaultsData = await api.post("/host/resolve-default-schedules", { agent_id: agent.agent_id });
|
|
1726
|
+
const missing = (defaultsData.schedules ?? []).filter(
|
|
1727
|
+
(s) => !existingTemplateIds.has(s.template_id)
|
|
1728
|
+
);
|
|
1729
|
+
const allSchedules = defaultsData.schedules ?? [];
|
|
1730
|
+
if (allSchedules.length > 0) {
|
|
1731
|
+
await api.post("/host/ensure-default-schedules", {
|
|
1732
|
+
agent_id: agent.agent_id,
|
|
1733
|
+
team_id: defaultsData.team_id,
|
|
1734
|
+
timezone: defaultsData.timezone,
|
|
1735
|
+
schedules: allSchedules
|
|
1736
|
+
});
|
|
1737
|
+
if (missing.length > 0) {
|
|
1738
|
+
log(`Auto-provisioned ${missing.length} default schedule(s) for '${agent.code_name}': ${missing.map((s) => s.template_id).join(", ")}`);
|
|
1739
|
+
}
|
|
1740
|
+
const freshData = await api.post("/host/refresh", { agent_id: agent.agent_id });
|
|
1741
|
+
tasks = freshData.scheduled_tasks ?? tasks;
|
|
1742
|
+
}
|
|
1743
|
+
} catch (err) {
|
|
1744
|
+
log(`Default schedule provisioning failed for '${agent.code_name}': ${err.message}`);
|
|
1745
|
+
}
|
|
1746
|
+
if (agent.status === "active") {
|
|
1747
|
+
if (frameworkAdapter.installPlugin) {
|
|
1748
|
+
try {
|
|
1749
|
+
const pluginPath = join2(process.cwd(), "packages", "openclaw-plugin-augmented", "src", "index.ts");
|
|
1750
|
+
if (existsSync2(pluginPath)) {
|
|
1751
|
+
frameworkAdapter.installPlugin(agent.code_name, "augmented", pluginPath, {
|
|
1752
|
+
agtHost: requireHost(),
|
|
1753
|
+
agtApiKey: getApiKey() ?? void 0,
|
|
1754
|
+
agentId: agent.agent_id
|
|
1755
|
+
});
|
|
1756
|
+
}
|
|
1757
|
+
} catch (err) {
|
|
1758
|
+
log(`Augmented plugin install failed for '${agent.code_name}': ${err.message}`);
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
if (frameworkAdapter.installSkillFiles) {
|
|
1762
|
+
try {
|
|
1763
|
+
const skillContent = getBuiltInSkillContent("kanban");
|
|
1764
|
+
if (skillContent) {
|
|
1765
|
+
frameworkAdapter.installSkillFiles(agent.code_name, "kanban", skillContent);
|
|
1766
|
+
}
|
|
1767
|
+
} catch (err) {
|
|
1768
|
+
log(`Kanban skill install failed for '${agent.code_name}': ${err.message}`);
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
let boardItems = [];
|
|
1773
|
+
const hasBoardTemplates = tasks.some((t) => BOARD_INJECT_TEMPLATES.has(t.template_id));
|
|
1774
|
+
if (hasBoardTemplates) {
|
|
1775
|
+
try {
|
|
1776
|
+
const boardData = await api.post("/host/my-kanban", { agent_id: agent.agent_id });
|
|
1777
|
+
boardItems = (boardData.items ?? []).map(sanitizeBoardItem);
|
|
1778
|
+
kanbanBoardCache.set(agent.code_name, boardItems);
|
|
1779
|
+
} catch {
|
|
1780
|
+
boardItems = kanbanBoardCache.get(agent.code_name) ?? [];
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
const agentFw = agentFrameworkCache.get(agent.code_name) ?? "openclaw";
|
|
1784
|
+
const sessionMode = refreshData.agent.session_mode ?? "oneshot";
|
|
1785
|
+
if (agentFw === "claude-code" && sessionMode === "persistent") {
|
|
1786
|
+
await ensurePersistentSession(agent, tasks, boardItems, refreshData);
|
|
1787
|
+
} else if (agentFw === "claude-code" && tasks.length > 0) {
|
|
1788
|
+
await syncAndCheckClaudeScheduler(agent, tasks, boardItems, refreshData);
|
|
1789
|
+
} else if (frameworkAdapter.syncScheduledTasks && gatewayRunning && gatewayPort) {
|
|
1790
|
+
const stableTasksHash = createHash("sha256").update(JSON.stringify(tasks)).digest("hex").slice(0, 16);
|
|
1791
|
+
const boardHash = boardItems.length > 0 ? createHash("sha256").update(JSON.stringify(boardItems.map((b) => ({ id: b.id, title: b.title, status: b.status, priority: b.priority, deliverable: b.deliverable })))).digest("hex").slice(0, 16) : "empty";
|
|
1792
|
+
const resolvedModels = resolveModelChain(refreshData);
|
|
1793
|
+
const modelsHash = createHash("sha256").update(JSON.stringify(resolvedModels)).digest("hex").slice(0, 16);
|
|
1794
|
+
const combinedHash = `${stableTasksHash}:${boardHash}:${modelsHash}`;
|
|
1795
|
+
const prevTasksHash = knownTasksHashes.get(agent.agent_id);
|
|
1796
|
+
if (combinedHash !== prevTasksHash) {
|
|
1797
|
+
const enrichedTasks = tasks.map((t) => {
|
|
1798
|
+
if (BOARD_INJECT_TEMPLATES.has(t.template_id) && boardItems.length > 0) {
|
|
1799
|
+
const template = PLAN_TEMPLATES.has(t.template_id) ? "morning-plan" : "follow-up";
|
|
1800
|
+
const boardPrefix = formatBoardForPrompt(boardItems, template);
|
|
1801
|
+
return { ...t, prompt: boardPrefix + t.prompt };
|
|
1802
|
+
}
|
|
1803
|
+
return t;
|
|
1804
|
+
});
|
|
1805
|
+
try {
|
|
1806
|
+
const token = readGatewayToken(agent.code_name);
|
|
1807
|
+
await frameworkAdapter.syncScheduledTasks(
|
|
1808
|
+
agent.code_name,
|
|
1809
|
+
enrichedTasks.map((t) => ({
|
|
1810
|
+
id: t.id,
|
|
1811
|
+
template_id: t.template_id,
|
|
1812
|
+
name: t.name,
|
|
1813
|
+
schedule_kind: t.schedule_kind,
|
|
1814
|
+
schedule_expr: t.schedule_expr,
|
|
1815
|
+
schedule_every: t.schedule_every,
|
|
1816
|
+
schedule_at: t.schedule_at,
|
|
1817
|
+
timezone: t.timezone,
|
|
1818
|
+
prompt: t.prompt,
|
|
1819
|
+
session_target: t.session_target,
|
|
1820
|
+
delivery_mode: t.delivery_mode,
|
|
1821
|
+
delivery_channel: t.delivery_channel,
|
|
1822
|
+
delivery_to: t.delivery_to,
|
|
1823
|
+
enabled: t.enabled,
|
|
1824
|
+
model_tier: t.model_tier ?? "primary"
|
|
1825
|
+
})),
|
|
1826
|
+
gatewayPort,
|
|
1827
|
+
token,
|
|
1828
|
+
{
|
|
1829
|
+
models: resolvedModels
|
|
1830
|
+
}
|
|
1831
|
+
);
|
|
1832
|
+
knownTasksHashes.set(agent.agent_id, combinedHash);
|
|
1833
|
+
log(`Scheduled tasks synced for '${agent.code_name}' (${enrichedTasks.length} task(s))`);
|
|
1834
|
+
} catch (err) {
|
|
1835
|
+
log(`Failed to sync scheduled tasks for '${agent.code_name}': ${err.message}`);
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
for (const t of tasks) {
|
|
1840
|
+
const jobName = `aug:${t.template_id}:${t.id ?? t.name.toLowerCase().replace(/\s+/g, "-")}`;
|
|
1841
|
+
taskDisplayInfo.set(`${agent.code_name}:${jobName}`, {
|
|
1842
|
+
taskName: t.name,
|
|
1843
|
+
schedule: t.schedule_expr ?? t.schedule_every ?? "",
|
|
1844
|
+
agentDisplayName: agent.display_name
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
const ccInFlight = agentFw === "claude-code" ? claudeTaskConcurrency.get(agent.code_name) ?? 0 : null;
|
|
1848
|
+
log(`[${agent.code_name}] Harvest gate: gatewayRunning=${gatewayRunning} gatewayPort=${gatewayPort} tasks=${tasks.length} fw=${agentFw}${ccInFlight !== null ? ` claude-p=${ccInFlight}/${MAX_CLAUDE_CONCURRENCY}` : ""}`);
|
|
1849
|
+
if (agentFw === "openclaw" && gatewayRunning && gatewayPort && tasks.length > 0) {
|
|
1850
|
+
const lastHarvest = lastHarvestAt.get(agent.code_name) ?? 0;
|
|
1851
|
+
if (Date.now() - lastHarvest >= HARVEST_INTERVAL_MS) {
|
|
1852
|
+
lastHarvestAt.set(agent.code_name, Date.now());
|
|
1853
|
+
harvestCronResults(agent.code_name, tasks, gatewayPort).catch((err) => {
|
|
1854
|
+
log(`Cron result harvest failed for '${agent.code_name}': ${err.message}`);
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
{
|
|
1859
|
+
const triggerAt = refreshData.agent.work_trigger_at;
|
|
1860
|
+
if (triggerAt) {
|
|
1861
|
+
const triggerTs = new Date(triggerAt).getTime();
|
|
1862
|
+
const lastTrigger = lastWorkTriggerAt.get(agent.code_name) ?? 0;
|
|
1863
|
+
if (triggerTs > lastTrigger) {
|
|
1864
|
+
lastWorkTriggerAt.set(agent.code_name, triggerTs);
|
|
1865
|
+
if (agentFw === "openclaw" && gatewayRunning && gatewayPort) {
|
|
1866
|
+
const homeDir = process.env["HOME"] ?? "/tmp";
|
|
1867
|
+
const jobsPath = join2(homeDir, `.openclaw-${agent.code_name}`, "cron", "jobs.json");
|
|
1868
|
+
if (existsSync2(jobsPath)) {
|
|
1869
|
+
try {
|
|
1870
|
+
const jobsData = JSON.parse(readFileSync2(jobsPath, "utf-8"));
|
|
1871
|
+
const kanbanJob = (jobsData.jobs ?? []).find(
|
|
1872
|
+
(j) => typeof j.name === "string" && j.name.includes("kanban-work")
|
|
1873
|
+
);
|
|
1874
|
+
if (kanbanJob?.id) {
|
|
1875
|
+
const cliBin = resolveAgentFramework(agent.code_name).cliBinary ?? "openclaw";
|
|
1876
|
+
log(`Work trigger: firing kanban-work for '${agent.code_name}'`);
|
|
1877
|
+
execFilePromise(cliBin, ["--profile", agent.code_name, "cron", "run", kanbanJob.id]).then(() => log(`Work trigger succeeded for '${agent.code_name}'`)).catch((err) => log(`Work trigger failed for '${agent.code_name}': ${err.message}`));
|
|
1878
|
+
}
|
|
1879
|
+
} catch {
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
} else if (agentFw === "claude-code") {
|
|
1883
|
+
fireClaudeWorkTrigger(agent.code_name, agent.agent_id, boardItems);
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
if (agentFw === "openclaw") {
|
|
1889
|
+
cleanupStaleSessions(agent.code_name);
|
|
1890
|
+
}
|
|
1891
|
+
{
|
|
1892
|
+
const agentId = codeNameToAgentId.get(agent.code_name);
|
|
1893
|
+
if (agentId) {
|
|
1894
|
+
try {
|
|
1895
|
+
const boardData = await api.post("/host/my-kanban", { agent_id: agentId });
|
|
1896
|
+
const freshBoard = (boardData.items ?? []).map(sanitizeBoardItem);
|
|
1897
|
+
const freshDoneIds = new Set(freshBoard.filter((b) => b.status === "done" || b.status === "failed").map((b) => b.id));
|
|
1898
|
+
const previousDoneIds = notifyBoardCache.get(agent.code_name);
|
|
1899
|
+
notifyBoardCache.set(agent.code_name, freshDoneIds);
|
|
1900
|
+
if (!previousDoneIds) {
|
|
1901
|
+
log(`[${agent.code_name}] Board diff: initial cache (${freshDoneIds.size} done items)`);
|
|
1902
|
+
} else {
|
|
1903
|
+
const newlyDone = freshBoard.filter(
|
|
1904
|
+
(b) => (b.status === "done" || b.status === "failed") && !previousDoneIds.has(b.id)
|
|
1905
|
+
);
|
|
1906
|
+
log(`[${agent.code_name}] Board diff: ${previousDoneIds.size} prev done \u2192 ${freshDoneIds.size} now, ${newlyDone.length} newly done`);
|
|
1907
|
+
if (newlyDone.length > 0) {
|
|
1908
|
+
const displayName = agentDisplayNames.get(agent.code_name) ?? agent.code_name;
|
|
1909
|
+
for (const item of newlyDone) {
|
|
1910
|
+
log(`Newly done: '${item.title}' notify_channel=${item.notify_channel ?? "none"} notify_to=${item.notify_to ?? "none"}`);
|
|
1911
|
+
if (item.notify_channel && item.notify_to) {
|
|
1912
|
+
const isFailed = item.status === "failed";
|
|
1913
|
+
const resultLine = item.result ? `
|
|
1914
|
+
${isFailed ? "Reason" : "Result"}: ${item.result}` : "";
|
|
1915
|
+
const emoji = isFailed ? "\u274C" : "\u2705";
|
|
1916
|
+
const message = `${emoji} ${isFailed ? "Task Failed" : "Task Complete"} \u2014 ${displayName}
|
|
1917
|
+
${item.title}${resultLine}`;
|
|
1918
|
+
sendTaskNotification(agent.code_name, item.notify_channel, item.notify_to, message).catch((err) => {
|
|
1919
|
+
log(`sendTaskNotification failed for '${agent.code_name}': ${err.message}`);
|
|
1920
|
+
});
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
}
|
|
1925
|
+
const staleItems = freshBoard.filter((b) => {
|
|
1926
|
+
if (b.status !== "in_progress" || !b.updated_at) return false;
|
|
1927
|
+
const age = Date.now() - new Date(b.updated_at).getTime();
|
|
1928
|
+
return age > STALE_TASK_THRESHOLD_MS && !alertedStaleItems.has(`${agent.code_name}:${b.id}`);
|
|
1929
|
+
});
|
|
1930
|
+
if (staleItems.length > 0) {
|
|
1931
|
+
const displayName = agentDisplayNames.get(agent.code_name) ?? agent.code_name;
|
|
1932
|
+
for (const item of staleItems) {
|
|
1933
|
+
const age = Math.round((Date.now() - new Date(item.updated_at).getTime()) / 6e4);
|
|
1934
|
+
log(`Stale task: '${item.title}' (id=${item.id}) in_progress for ${age}m \u2014 auto-failing for '${agent.code_name}'`);
|
|
1935
|
+
alertedStaleItems.add(`${agent.code_name}:${item.id}`);
|
|
1936
|
+
try {
|
|
1937
|
+
const failResult = await api.post("/host/kanban", {
|
|
1938
|
+
agent_id: agentId,
|
|
1939
|
+
update: [{ id: item.id, title: item.title, status: "failed", result: `Timed out \u2014 in progress for ${age} minutes with no update` }]
|
|
1940
|
+
});
|
|
1941
|
+
log(`Auto-fail result for '${item.title}': updated=${failResult.updated}`);
|
|
1942
|
+
if ((failResult.updated ?? 0) > 0 || failResult.ok === true) {
|
|
1943
|
+
freshDoneIds.add(item.id);
|
|
1944
|
+
}
|
|
1945
|
+
} catch (err) {
|
|
1946
|
+
log(`Auto-fail API error for '${item.title}': ${err.message}`);
|
|
1947
|
+
}
|
|
1948
|
+
const message = `\u23F3 Task Timed Out \u2014 ${displayName}
|
|
1949
|
+
${item.title}
|
|
1950
|
+
In progress for ${age} minutes with no update \u2014 auto-failed`;
|
|
1951
|
+
if (item.notify_channel && item.notify_to) {
|
|
1952
|
+
sendTaskNotification(agent.code_name, item.notify_channel, item.notify_to, message).catch(() => {
|
|
1953
|
+
});
|
|
1954
|
+
}
|
|
1955
|
+
sendSlackWebhookMessage(`\u23F3 *Task Timed Out* \u2014 *${displayName}*
|
|
1956
|
+
*${item.title}*
|
|
1957
|
+
In progress for ${age} minutes \u2014 auto-failed`).catch(() => {
|
|
1958
|
+
});
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
const prefix = `${agent.code_name}:`;
|
|
1962
|
+
for (const key of alertedStaleItems) {
|
|
1963
|
+
if (key.startsWith(prefix)) {
|
|
1964
|
+
const itemId = key.slice(prefix.length);
|
|
1965
|
+
if (!freshBoard.some((b) => b.id === itemId && b.status === "in_progress")) {
|
|
1966
|
+
alertedStaleItems.delete(key);
|
|
1967
|
+
}
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
} catch (err) {
|
|
1971
|
+
log(`Board diff failed for '${agent.code_name}': ${err.message}`);
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
agentStates.push({
|
|
1976
|
+
agentId: agent.agent_id,
|
|
1977
|
+
codeName: agent.code_name,
|
|
1978
|
+
status: agent.status,
|
|
1979
|
+
charterVersion,
|
|
1980
|
+
toolsVersion,
|
|
1981
|
+
secretsHash,
|
|
1982
|
+
lastRefreshAt: now,
|
|
1983
|
+
lastProvisionAt,
|
|
1984
|
+
lastDriftCheckAt,
|
|
1985
|
+
lastSecretsProvisionAt,
|
|
1986
|
+
gatewayPort,
|
|
1987
|
+
gatewayPid,
|
|
1988
|
+
gatewayRunning,
|
|
1989
|
+
acpSessions: []
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
1992
|
+
var CRON_SESSION_KEEP_COUNT = 10;
|
|
1993
|
+
var CRON_RUN_RETENTION_DAYS = 7;
|
|
1994
|
+
var lastCleanupAt = /* @__PURE__ */ new Map();
|
|
1995
|
+
var CLEANUP_INTERVAL_MS = 6e4;
|
|
1996
|
+
function cleanupStaleSessions(codeName) {
|
|
1997
|
+
const lastRun = lastCleanupAt.get(codeName) ?? 0;
|
|
1998
|
+
if (lastRun > 0 && Date.now() - lastRun < CLEANUP_INTERVAL_MS) return;
|
|
1999
|
+
lastCleanupAt.set(codeName, Date.now());
|
|
2000
|
+
const homeDir = process.env["HOME"] ?? "/tmp";
|
|
2001
|
+
for (const agentDir of ["main", codeName]) {
|
|
2002
|
+
const sessionsDir = join2(homeDir, `.openclaw-${codeName}`, "agents", agentDir, "sessions");
|
|
2003
|
+
cleanupCronSessions(sessionsDir, CRON_SESSION_KEEP_COUNT);
|
|
2004
|
+
}
|
|
2005
|
+
const cronRunsDir = join2(homeDir, `.openclaw-${codeName}`, "cron", "runs");
|
|
2006
|
+
cleanupOldFiles(cronRunsDir, CRON_RUN_RETENTION_DAYS, ".jsonl");
|
|
2007
|
+
const cronJobsPath = join2(homeDir, `.openclaw-${codeName}`, "cron", "jobs.json");
|
|
2008
|
+
clearStaleCronRunState(cronJobsPath);
|
|
2009
|
+
}
|
|
2010
|
+
function cleanupCronSessions(sessionsDir, keepCount) {
|
|
2011
|
+
const indexPath = join2(sessionsDir, "sessions.json");
|
|
2012
|
+
if (!existsSync2(indexPath)) return;
|
|
2013
|
+
try {
|
|
2014
|
+
const raw = readFileSync2(indexPath, "utf-8");
|
|
2015
|
+
const index = JSON.parse(raw);
|
|
2016
|
+
const cronRunKeys = Object.keys(index).filter((k) => k.includes(":cron:") && k.includes(":run:")).map((k) => ({
|
|
2017
|
+
key: k,
|
|
2018
|
+
sessionId: index[k]?.sessionId,
|
|
2019
|
+
updatedAt: index[k]?.updatedAt ?? 0
|
|
2020
|
+
})).sort((a, b) => b.updatedAt - a.updatedAt);
|
|
2021
|
+
if (cronRunKeys.length <= keepCount) return;
|
|
2022
|
+
const toDelete = cronRunKeys.slice(keepCount);
|
|
2023
|
+
let deletedFiles = 0;
|
|
2024
|
+
for (const entry of toDelete) {
|
|
2025
|
+
delete index[entry.key];
|
|
2026
|
+
if (entry.sessionId) {
|
|
2027
|
+
const sessionFile = join2(sessionsDir, `${entry.sessionId}.jsonl`);
|
|
2028
|
+
try {
|
|
2029
|
+
if (existsSync2(sessionFile)) {
|
|
2030
|
+
unlinkSync(sessionFile);
|
|
2031
|
+
deletedFiles++;
|
|
2032
|
+
}
|
|
2033
|
+
} catch {
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
const cronParentKeys = Object.keys(index).filter(
|
|
2038
|
+
(k) => k.includes(":cron:") && !k.includes(":run:") && k !== "agent:main:main"
|
|
2039
|
+
);
|
|
2040
|
+
for (const parentKey of cronParentKeys) {
|
|
2041
|
+
const hasRuns = Object.keys(index).some(
|
|
2042
|
+
(k) => k.startsWith(parentKey + ":run:")
|
|
2043
|
+
);
|
|
2044
|
+
if (!hasRuns) {
|
|
2045
|
+
const parentSessionId = index[parentKey]?.sessionId;
|
|
2046
|
+
delete index[parentKey];
|
|
2047
|
+
if (parentSessionId) {
|
|
2048
|
+
try {
|
|
2049
|
+
const f = join2(sessionsDir, `${parentSessionId}.jsonl`);
|
|
2050
|
+
if (existsSync2(f)) {
|
|
2051
|
+
unlinkSync(f);
|
|
2052
|
+
deletedFiles++;
|
|
2053
|
+
}
|
|
2054
|
+
} catch {
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
writeFileSync(indexPath, JSON.stringify(index));
|
|
2060
|
+
if (toDelete.length > 0) {
|
|
2061
|
+
log(`Cleaned ${toDelete.length} cron session(s) and ${deletedFiles} file(s) from ${sessionsDir}`);
|
|
2062
|
+
}
|
|
2063
|
+
} catch {
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
var STALE_RUN_TIMEOUT_MS = 5 * 6e4;
|
|
2067
|
+
function clearStaleCronRunState(jobsPath) {
|
|
2068
|
+
if (!existsSync2(jobsPath)) return;
|
|
2069
|
+
try {
|
|
2070
|
+
const raw = readFileSync2(jobsPath, "utf-8");
|
|
2071
|
+
const data = JSON.parse(raw);
|
|
2072
|
+
const jobs = data.jobs ?? data;
|
|
2073
|
+
if (!Array.isArray(jobs)) return;
|
|
2074
|
+
let changed = false;
|
|
2075
|
+
const now = Date.now();
|
|
2076
|
+
for (const job of jobs) {
|
|
2077
|
+
const state2 = job.state;
|
|
2078
|
+
if (!state2) continue;
|
|
2079
|
+
const runStartedAt = state2.runningAtMs ?? state2.runStartedAtMs;
|
|
2080
|
+
const isRunning = state2.running === true || state2.status === "running";
|
|
2081
|
+
if (isRunning && runStartedAt && now - runStartedAt > STALE_RUN_TIMEOUT_MS) {
|
|
2082
|
+
state2.running = false;
|
|
2083
|
+
delete state2.status;
|
|
2084
|
+
delete state2.runStartedAtMs;
|
|
2085
|
+
delete state2.currentRunId;
|
|
2086
|
+
changed = true;
|
|
2087
|
+
log(`Cleared stale running state for cron job '${job.name}' (stuck for ${Math.round((now - runStartedAt) / 6e4)}min)`);
|
|
2088
|
+
} else if (isRunning && !runStartedAt) {
|
|
2089
|
+
state2.running = false;
|
|
2090
|
+
changed = true;
|
|
2091
|
+
log(`Cleared stale running state for cron job '${job.name}' (no start timestamp)`);
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
if (changed) {
|
|
2095
|
+
writeFileSync(jobsPath, JSON.stringify(data, null, 2));
|
|
2096
|
+
}
|
|
2097
|
+
} catch {
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
function cleanupOldFiles(dir, maxAgeDays, ext) {
|
|
2101
|
+
if (!existsSync2(dir)) return;
|
|
2102
|
+
const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1e3;
|
|
2103
|
+
let removed = 0;
|
|
2104
|
+
try {
|
|
2105
|
+
for (const f of readdirSync(dir)) {
|
|
2106
|
+
if (!f.endsWith(ext)) continue;
|
|
2107
|
+
const fullPath = join2(dir, f);
|
|
2108
|
+
try {
|
|
2109
|
+
const st = statSync(fullPath);
|
|
2110
|
+
if (st.mtimeMs < cutoff) {
|
|
2111
|
+
unlinkSync(fullPath);
|
|
2112
|
+
removed++;
|
|
2113
|
+
}
|
|
2114
|
+
} catch {
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
if (removed > 0) {
|
|
2118
|
+
log(`Cleaned ${removed} old cron run log(s) from ${dir}`);
|
|
2119
|
+
}
|
|
2120
|
+
} catch {
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
var inFlightClaudeTasks = /* @__PURE__ */ new Set();
|
|
2124
|
+
var claudeTaskConcurrency = /* @__PURE__ */ new Map();
|
|
2125
|
+
var MAX_CLAUDE_CONCURRENCY = 2;
|
|
2126
|
+
var claudeSchedulerStates = /* @__PURE__ */ new Map();
|
|
2127
|
+
async function syncAndCheckClaudeScheduler(agent, tasks, boardItems, refreshData) {
|
|
2128
|
+
const codeName = agent.code_name;
|
|
2129
|
+
const stableTasksHash = createHash("sha256").update(JSON.stringify(tasks)).digest("hex").slice(0, 16);
|
|
2130
|
+
const boardHash = boardItems.length > 0 ? createHash("sha256").update(JSON.stringify(boardItems.map((b) => ({ id: b.id, title: b.title, status: b.status, priority: b.priority, deliverable: b.deliverable })))).digest("hex").slice(0, 16) : "empty";
|
|
2131
|
+
const resolvedModels = resolveModelChain(refreshData);
|
|
2132
|
+
const modelsHash = createHash("sha256").update(JSON.stringify(resolvedModels)).digest("hex").slice(0, 16);
|
|
2133
|
+
const combinedHash = `${stableTasksHash}:${boardHash}:${modelsHash}`;
|
|
2134
|
+
const prevHash = knownTasksHashes.get(agent.agent_id);
|
|
2135
|
+
if (combinedHash !== prevHash) {
|
|
2136
|
+
const taskInputs = tasks.map((t) => ({
|
|
2137
|
+
id: t.id,
|
|
2138
|
+
template_id: t.template_id,
|
|
2139
|
+
name: t.name,
|
|
2140
|
+
schedule_kind: t.schedule_kind,
|
|
2141
|
+
schedule_expr: t.schedule_expr ?? null,
|
|
2142
|
+
schedule_every: t.schedule_every ?? null,
|
|
2143
|
+
schedule_at: t.schedule_at ?? null,
|
|
2144
|
+
timezone: t.timezone ?? "UTC",
|
|
2145
|
+
prompt: t.prompt ?? "",
|
|
2146
|
+
session_target: t.session_target ?? "isolated",
|
|
2147
|
+
delivery_mode: t.delivery_mode ?? "none",
|
|
2148
|
+
delivery_channel: t.delivery_channel ?? null,
|
|
2149
|
+
delivery_to: t.delivery_to ?? null,
|
|
2150
|
+
enabled: t.enabled ?? true
|
|
2151
|
+
}));
|
|
2152
|
+
const state3 = syncTasksToScheduler(codeName, agent.agent_id, taskInputs);
|
|
2153
|
+
claudeSchedulerStates.set(codeName, state3);
|
|
2154
|
+
knownTasksHashes.set(agent.agent_id, combinedHash);
|
|
2155
|
+
log(`[claude-scheduler] Tasks synced for '${codeName}' (${taskInputs.length} task(s))`);
|
|
2156
|
+
}
|
|
2157
|
+
if (!claudeSchedulerStates.has(codeName)) {
|
|
2158
|
+
claudeSchedulerStates.set(codeName, loadSchedulerState(codeName));
|
|
2159
|
+
}
|
|
2160
|
+
const state2 = claudeSchedulerStates.get(codeName);
|
|
2161
|
+
const ready = getReadyTasks(state2);
|
|
2162
|
+
if (ready.length === 0) return;
|
|
2163
|
+
for (const task of ready) {
|
|
2164
|
+
if (inFlightClaudeTasks.has(task.taskId)) continue;
|
|
2165
|
+
if ((claudeTaskConcurrency.get(codeName) ?? 0) >= MAX_CLAUDE_CONCURRENCY) break;
|
|
2166
|
+
let prompt = task.prompt;
|
|
2167
|
+
if (BOARD_INJECT_TEMPLATES.has(task.templateId) && boardItems.length > 0) {
|
|
2168
|
+
const template = PLAN_TEMPLATES.has(task.templateId) ? "morning-plan" : "follow-up";
|
|
2169
|
+
const boardPrefix = formatBoardForPrompt(boardItems, template);
|
|
2170
|
+
prompt = boardPrefix + prompt;
|
|
2171
|
+
}
|
|
2172
|
+
if (KANBAN_WORK_TEMPLATES.has(task.templateId)) {
|
|
2173
|
+
const todayItem = boardItems.find((b) => b.status === "today");
|
|
2174
|
+
if (todayItem) {
|
|
2175
|
+
try {
|
|
2176
|
+
await api.post("/host/kanban", {
|
|
2177
|
+
agent_id: agent.agent_id,
|
|
2178
|
+
update: [{ id: todayItem.id, title: todayItem.title, status: "in_progress" }]
|
|
2179
|
+
});
|
|
2180
|
+
log(`[claude-scheduler] Moved '${todayItem.title}' to in_progress for '${codeName}'`);
|
|
2181
|
+
} catch (err) {
|
|
2182
|
+
log(`[claude-scheduler] Failed to move item to in_progress: ${err.message}`);
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2186
|
+
inFlightClaudeTasks.add(task.taskId);
|
|
2187
|
+
claudeTaskConcurrency.set(codeName, (claudeTaskConcurrency.get(codeName) ?? 0) + 1);
|
|
2188
|
+
log(`[claude-scheduler] Firing '${task.name}' for '${codeName}'`);
|
|
2189
|
+
executeAndProcessClaudeTask(codeName, agent.agent_id, task, prompt).finally(() => {
|
|
2190
|
+
inFlightClaudeTasks.delete(task.taskId);
|
|
2191
|
+
claudeTaskConcurrency.set(codeName, Math.max(0, (claudeTaskConcurrency.get(codeName) ?? 1) - 1));
|
|
2192
|
+
});
|
|
2193
|
+
}
|
|
2194
|
+
}
|
|
2195
|
+
async function executeAndProcessClaudeTask(codeName, agentId, task, prompt) {
|
|
2196
|
+
const projectDir = getProjectDir(codeName);
|
|
2197
|
+
const mcpConfigPath = join2(projectDir, ".mcp.json");
|
|
2198
|
+
try {
|
|
2199
|
+
const claudeMdPath = join2(projectDir, "CLAUDE.md");
|
|
2200
|
+
const claudeArgs = [
|
|
2201
|
+
"-p",
|
|
2202
|
+
prompt,
|
|
2203
|
+
"--output-format",
|
|
2204
|
+
"text",
|
|
2205
|
+
"--mcp-config",
|
|
2206
|
+
mcpConfigPath,
|
|
2207
|
+
"--allowedTools",
|
|
2208
|
+
"mcp__augmented__*,Bash,Read,Write,Edit,Grep,Glob"
|
|
2209
|
+
];
|
|
2210
|
+
if (existsSync2(claudeMdPath)) {
|
|
2211
|
+
claudeArgs.push("--system-prompt-file", claudeMdPath);
|
|
2212
|
+
}
|
|
2213
|
+
const { stdout, stderr } = await execFilePromiseLong("claude", claudeArgs, {
|
|
2214
|
+
cwd: projectDir,
|
|
2215
|
+
timeout: 3e5,
|
|
2216
|
+
stdin: "ignore"
|
|
2217
|
+
});
|
|
2218
|
+
if (stderr) {
|
|
2219
|
+
log(`[claude-scheduler] Task '${task.name}' stderr for '${codeName}': ${stderr.slice(0, 500)}`);
|
|
2220
|
+
}
|
|
2221
|
+
const output = stdout.trim();
|
|
2222
|
+
log(`[claude-scheduler] Task '${task.name}' completed for '${codeName}' (${output.length} chars): ${output.slice(0, 300)}`);
|
|
2223
|
+
await processClaudeTaskResult(codeName, agentId, task.templateId, output);
|
|
2224
|
+
const updated = markTaskFired(codeName, task.taskId, "ok");
|
|
2225
|
+
claudeSchedulerStates.set(codeName, updated);
|
|
2226
|
+
} catch (err) {
|
|
2227
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2228
|
+
log(`[claude-scheduler] Task '${task.name}' failed for '${codeName}': ${errMsg}`);
|
|
2229
|
+
const updated = markTaskFired(codeName, task.taskId, "error");
|
|
2230
|
+
claudeSchedulerStates.set(codeName, updated);
|
|
2231
|
+
}
|
|
2232
|
+
}
|
|
2233
|
+
async function processClaudeTaskResult(codeName, agentId, templateId, output) {
|
|
2234
|
+
try {
|
|
2235
|
+
if (STANDUP_TEMPLATES.has(templateId)) {
|
|
2236
|
+
const standup = parseStandupSummary(output);
|
|
2237
|
+
await api.post("/host/agent-status", {
|
|
2238
|
+
agent_id: agentId,
|
|
2239
|
+
standup,
|
|
2240
|
+
current_status: "idle"
|
|
2241
|
+
});
|
|
2242
|
+
log(`[claude-scheduler] Standup posted for '${codeName}'`);
|
|
2243
|
+
} else if (TASK_UPDATE_TEMPLATES.has(templateId)) {
|
|
2244
|
+
await api.post("/host/agent-status", {
|
|
2245
|
+
agent_id: agentId,
|
|
2246
|
+
current_tasks: output.slice(0, 2e3)
|
|
2247
|
+
});
|
|
2248
|
+
log(`[claude-scheduler] Task update posted for '${codeName}'`);
|
|
2249
|
+
} else if (PLAN_TEMPLATES.has(templateId)) {
|
|
2250
|
+
const planItems = parsePlanItems(output);
|
|
2251
|
+
if (planItems.length > 0) {
|
|
2252
|
+
await api.post("/host/kanban", {
|
|
2253
|
+
agent_id: agentId,
|
|
2254
|
+
add: planItems
|
|
2255
|
+
});
|
|
2256
|
+
log(`[claude-scheduler] Plan items posted for '${codeName}' (${planItems.length} items)`);
|
|
2257
|
+
}
|
|
2258
|
+
} else if (KANBAN_WORK_TEMPLATES.has(templateId)) {
|
|
2259
|
+
const kanbanUpdates = parseKanbanUpdates(output);
|
|
2260
|
+
if (kanbanUpdates.length > 0) {
|
|
2261
|
+
await api.post("/host/kanban", {
|
|
2262
|
+
agent_id: agentId,
|
|
2263
|
+
update: kanbanUpdates
|
|
2264
|
+
});
|
|
2265
|
+
log(`[claude-scheduler] Kanban updates posted for '${codeName}' (${kanbanUpdates.length} updates)`);
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
} catch (err) {
|
|
2269
|
+
log(`[claude-scheduler] Failed to post result for '${codeName}': ${err.message}`);
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
function fireClaudeWorkTrigger(codeName, agentId, boardItems) {
|
|
2273
|
+
const state2 = claudeSchedulerStates.get(codeName) ?? loadSchedulerState(codeName);
|
|
2274
|
+
const kanbanTask = findTaskByTemplate(state2, "kanban-work");
|
|
2275
|
+
if (!kanbanTask) {
|
|
2276
|
+
log(`[claude-scheduler] Work trigger: no kanban-work task found for '${codeName}'`);
|
|
2277
|
+
return;
|
|
2278
|
+
}
|
|
2279
|
+
if (inFlightClaudeTasks.has(kanbanTask.taskId)) {
|
|
2280
|
+
log(`[claude-scheduler] Work trigger: kanban-work already in-flight for '${codeName}'`);
|
|
2281
|
+
return;
|
|
2282
|
+
}
|
|
2283
|
+
let prompt = kanbanTask.prompt;
|
|
2284
|
+
if (boardItems.length > 0) {
|
|
2285
|
+
const boardPrefix = formatBoardForPrompt(boardItems, "follow-up");
|
|
2286
|
+
prompt = boardPrefix + prompt;
|
|
2287
|
+
}
|
|
2288
|
+
inFlightClaudeTasks.add(kanbanTask.taskId);
|
|
2289
|
+
claudeTaskConcurrency.set(codeName, (claudeTaskConcurrency.get(codeName) ?? 0) + 1);
|
|
2290
|
+
log(`[claude-scheduler] Work trigger: firing kanban-work for '${codeName}'`);
|
|
2291
|
+
executeAndProcessClaudeTask(codeName, agentId, kanbanTask, prompt).finally(() => {
|
|
2292
|
+
inFlightClaudeTasks.delete(kanbanTask.taskId);
|
|
2293
|
+
claudeTaskConcurrency.set(codeName, Math.max(0, (claudeTaskConcurrency.get(codeName) ?? 1) - 1));
|
|
2294
|
+
});
|
|
2295
|
+
}
|
|
2296
|
+
var persistentSessionAgents = /* @__PURE__ */ new Set();
|
|
2297
|
+
async function ensurePersistentSession(agent, tasks, boardItems, refreshData) {
|
|
2298
|
+
const codeName = agent.code_name;
|
|
2299
|
+
const projectDir = getProjectDir2(codeName);
|
|
2300
|
+
const mcpConfigPath = join2(projectDir, ".mcp.json");
|
|
2301
|
+
const claudeMdPath = join2(projectDir, "CLAUDE.md");
|
|
2302
|
+
const channelConfigs = refreshData.channel_configs;
|
|
2303
|
+
const channels = [];
|
|
2304
|
+
const devChannels = [];
|
|
2305
|
+
if (channelConfigs) {
|
|
2306
|
+
if ("telegram" in channelConfigs) {
|
|
2307
|
+
channels.push("plugin:telegram@claude-plugins-official");
|
|
2308
|
+
}
|
|
2309
|
+
if ("discord" in channelConfigs) {
|
|
2310
|
+
channels.push("plugin:discord@claude-plugins-official");
|
|
2311
|
+
}
|
|
2312
|
+
if ("slack" in channelConfigs) {
|
|
2313
|
+
devChannels.push("server:slack");
|
|
2314
|
+
}
|
|
2315
|
+
}
|
|
2316
|
+
if (!isSessionHealthy(codeName)) {
|
|
2317
|
+
if (persistentSessionAgents.has(codeName)) {
|
|
2318
|
+
log(`[persistent-session] Session for '${codeName}' is unhealthy, will restart`);
|
|
2319
|
+
}
|
|
2320
|
+
startPersistentSession({
|
|
2321
|
+
codeName,
|
|
2322
|
+
agentId: agent.agent_id,
|
|
2323
|
+
projectDir,
|
|
2324
|
+
mcpConfigPath,
|
|
2325
|
+
claudeMdPath,
|
|
2326
|
+
channels,
|
|
2327
|
+
devChannels,
|
|
2328
|
+
log
|
|
2329
|
+
});
|
|
2330
|
+
persistentSessionAgents.add(codeName);
|
|
2331
|
+
return;
|
|
2332
|
+
}
|
|
2333
|
+
resetRestartCount(codeName);
|
|
2334
|
+
const state2 = claudeSchedulerStates.get(codeName);
|
|
2335
|
+
if (state2) {
|
|
2336
|
+
const ready = getReadyTasks(state2);
|
|
2337
|
+
if (ready.length > 0) {
|
|
2338
|
+
log(`[persistent-session] ${ready.length} ready task(s) for '${codeName}': ${ready.map((t) => `${t.name}(next=${t.nextFireAt ? new Date(t.nextFireAt).toISOString() : "null"})`).join(", ")}`);
|
|
2339
|
+
}
|
|
2340
|
+
for (const task of ready) {
|
|
2341
|
+
let prompt = task.prompt;
|
|
2342
|
+
if (BOARD_INJECT_TEMPLATES.has(task.templateId) && boardItems.length > 0) {
|
|
2343
|
+
const template = PLAN_TEMPLATES.has(task.templateId) ? "morning-plan" : "follow-up";
|
|
2344
|
+
const boardPrefix = formatBoardForPrompt(boardItems, template);
|
|
2345
|
+
prompt = boardPrefix + prompt;
|
|
2346
|
+
}
|
|
2347
|
+
if (KANBAN_WORK_TEMPLATES.has(task.templateId)) {
|
|
2348
|
+
const todayItem = boardItems.find((b) => b.status === "today");
|
|
2349
|
+
if (todayItem) {
|
|
2350
|
+
try {
|
|
2351
|
+
await api.post("/host/kanban", {
|
|
2352
|
+
agent_id: agent.agent_id,
|
|
2353
|
+
update: [{ id: todayItem.id, title: todayItem.title, status: "in_progress" }]
|
|
2354
|
+
});
|
|
2355
|
+
log(`[persistent-session] Moved '${todayItem.title}' to in_progress for '${codeName}'`);
|
|
2356
|
+
} catch {
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
}
|
|
2360
|
+
log(`[persistent-session] Injecting task '${task.name}' into '${codeName}'`);
|
|
2361
|
+
const injected = await injectMessage(codeName, "task", prompt, {
|
|
2362
|
+
task_id: task.taskId,
|
|
2363
|
+
template_id: task.templateId,
|
|
2364
|
+
task_name: task.name
|
|
2365
|
+
});
|
|
2366
|
+
if (injected) {
|
|
2367
|
+
const updated = markTaskFired(codeName, task.taskId, "ok");
|
|
2368
|
+
claudeSchedulerStates.set(codeName, updated);
|
|
2369
|
+
log(`[persistent-session] Task '${task.name}' injected, next fire at ${new Date(updated.tasks[task.taskId]?.nextFireAt ?? 0).toISOString()}`);
|
|
2370
|
+
}
|
|
2371
|
+
break;
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
const stableTasksHash = createHash("sha256").update(JSON.stringify(tasks)).digest("hex").slice(0, 16);
|
|
2375
|
+
const prevHash = knownTasksHashes.get(agent.agent_id);
|
|
2376
|
+
if (stableTasksHash !== prevHash) {
|
|
2377
|
+
const taskInputs = tasks.map((t) => ({
|
|
2378
|
+
id: t.id,
|
|
2379
|
+
template_id: t.template_id,
|
|
2380
|
+
name: t.name,
|
|
2381
|
+
schedule_kind: t.schedule_kind,
|
|
2382
|
+
schedule_expr: t.schedule_expr ?? null,
|
|
2383
|
+
schedule_every: t.schedule_every ?? null,
|
|
2384
|
+
schedule_at: t.schedule_at ?? null,
|
|
2385
|
+
timezone: t.timezone ?? "UTC",
|
|
2386
|
+
prompt: t.prompt ?? "",
|
|
2387
|
+
session_target: t.session_target ?? "isolated",
|
|
2388
|
+
delivery_mode: t.delivery_mode ?? "none",
|
|
2389
|
+
delivery_channel: t.delivery_channel ?? null,
|
|
2390
|
+
delivery_to: t.delivery_to ?? null,
|
|
2391
|
+
enabled: t.enabled ?? true
|
|
2392
|
+
}));
|
|
2393
|
+
const schedulerState = syncTasksToScheduler(codeName, agent.agent_id, taskInputs);
|
|
2394
|
+
claudeSchedulerStates.set(codeName, schedulerState);
|
|
2395
|
+
knownTasksHashes.set(agent.agent_id, stableTasksHash);
|
|
2396
|
+
log(`[persistent-session] Tasks synced for '${codeName}' (${taskInputs.length} task(s))`);
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
var realtimeStarted = false;
|
|
2400
|
+
var realtimeDriftStarted = false;
|
|
2401
|
+
var realtimeKanbanStarted = false;
|
|
2402
|
+
var realtimeAssignStarted = false;
|
|
2403
|
+
var realtimeConfigStarted = false;
|
|
2404
|
+
function ensureRealtimeStarted(agentStates) {
|
|
2405
|
+
if (realtimeStarted) return;
|
|
2406
|
+
const activeAgentIds = agentStates.filter((a) => a.status === "active").map((a) => a.agentId);
|
|
2407
|
+
if (activeAgentIds.length === 0) return;
|
|
2408
|
+
const apiKey = process.env["AGT_API_KEY"];
|
|
2409
|
+
if (!apiKey) return;
|
|
2410
|
+
void exchangeApiKey(apiKey).then((exchange) => {
|
|
2411
|
+
if (!exchange.supabaseUrl || !exchange.supabaseAnonKey) {
|
|
2412
|
+
log("[realtime-chat] No Supabase URL/key from exchange \u2014 staying on polling");
|
|
2413
|
+
return;
|
|
2414
|
+
}
|
|
2415
|
+
startRealtimeChat({
|
|
2416
|
+
supabaseUrl: exchange.supabaseUrl,
|
|
2417
|
+
supabaseAnonKey: exchange.supabaseAnonKey,
|
|
2418
|
+
token: exchange.token,
|
|
2419
|
+
agentIds: activeAgentIds,
|
|
2420
|
+
onMessage: (msg) => {
|
|
2421
|
+
const agent = agentStates.find((a) => a.agentId === msg.agent_id);
|
|
2422
|
+
if (!agent) return;
|
|
2423
|
+
if (directChatInFlight.has(msg.id)) return;
|
|
2424
|
+
directChatInFlight.add(msg.id);
|
|
2425
|
+
processDirectChatMessage(agent, {
|
|
2426
|
+
id: msg.id,
|
|
2427
|
+
session_id: msg.session_id,
|
|
2428
|
+
content: msg.content
|
|
2429
|
+
}).finally(() => {
|
|
2430
|
+
directChatInFlight.delete(msg.id);
|
|
2431
|
+
});
|
|
2432
|
+
},
|
|
2433
|
+
onError: (err) => {
|
|
2434
|
+
log(`[realtime-chat] Error: ${err.message}`);
|
|
2435
|
+
},
|
|
2436
|
+
onStatusChange: (status) => {
|
|
2437
|
+
if (status === "disconnected" || status === "error") {
|
|
2438
|
+
log("[realtime] Disconnected \u2014 falling back to polling, will reconnect next cycle");
|
|
2439
|
+
realtimeStarted = false;
|
|
2440
|
+
realtimeDriftStarted = false;
|
|
2441
|
+
realtimeAssignStarted = false;
|
|
2442
|
+
realtimeConfigStarted = false;
|
|
2443
|
+
realtimeKanbanStarted = false;
|
|
2444
|
+
}
|
|
2445
|
+
},
|
|
2446
|
+
log
|
|
2447
|
+
});
|
|
2448
|
+
realtimeStarted = true;
|
|
2449
|
+
log(`[realtime-chat] Started for ${activeAgentIds.length} agent(s)`);
|
|
2450
|
+
}).catch((err) => {
|
|
2451
|
+
log(`[realtime-chat] Failed to start: ${err.message}`);
|
|
2452
|
+
});
|
|
2453
|
+
}
|
|
2454
|
+
function ensureRealtimeDriftStarted(agentStates) {
|
|
2455
|
+
if (realtimeDriftStarted) return;
|
|
2456
|
+
const activeAgentIds = agentStates.filter((a) => a.status === "active").map((a) => a.agentId);
|
|
2457
|
+
if (activeAgentIds.length === 0) return;
|
|
2458
|
+
const apiKey = process.env["AGT_API_KEY"];
|
|
2459
|
+
if (!apiKey) return;
|
|
2460
|
+
void exchangeApiKey(apiKey).then((exchange) => {
|
|
2461
|
+
if (!exchange.supabaseUrl || !exchange.supabaseAnonKey) return;
|
|
2462
|
+
startRealtimeDrift({
|
|
2463
|
+
supabaseUrl: exchange.supabaseUrl,
|
|
2464
|
+
supabaseAnonKey: exchange.supabaseAnonKey,
|
|
2465
|
+
token: exchange.token,
|
|
2466
|
+
agentIds: activeAgentIds,
|
|
2467
|
+
onDrift: (doc) => {
|
|
2468
|
+
const agentState = agentStates.find((a) => a.agentId === doc.agent_id);
|
|
2469
|
+
if (agentState) {
|
|
2470
|
+
knownVersions.delete(doc.agent_id);
|
|
2471
|
+
log(`[realtime] Drift invalidated for '${agentState.codeName}' \u2014 will re-provision next cycle`);
|
|
2472
|
+
}
|
|
2473
|
+
},
|
|
2474
|
+
log
|
|
2475
|
+
});
|
|
2476
|
+
realtimeDriftStarted = true;
|
|
2477
|
+
log(`[realtime] Drift subscription started for ${activeAgentIds.length} agent(s)`);
|
|
2478
|
+
}).catch((err) => {
|
|
2479
|
+
log(`[realtime] Drift subscription failed: ${err.message}`);
|
|
2480
|
+
});
|
|
2481
|
+
}
|
|
2482
|
+
function ensureRealtimeAssignStarted(agentStates) {
|
|
2483
|
+
if (realtimeAssignStarted) return;
|
|
2484
|
+
const apiKey = process.env["AGT_API_KEY"];
|
|
2485
|
+
if (!apiKey) return;
|
|
2486
|
+
void exchangeApiKey(apiKey).then((exchange) => {
|
|
2487
|
+
if (!exchange.supabaseUrl || !exchange.supabaseAnonKey || !exchange.hostId) return;
|
|
2488
|
+
startRealtimeAssignments({
|
|
2489
|
+
supabaseUrl: exchange.supabaseUrl,
|
|
2490
|
+
supabaseAnonKey: exchange.supabaseAnonKey,
|
|
2491
|
+
token: exchange.token,
|
|
2492
|
+
hostId: exchange.hostId,
|
|
2493
|
+
onAssign: (payload) => {
|
|
2494
|
+
log(`[realtime] Agent ${payload.agent_id} assigned \u2014 will pick up next cycle`);
|
|
2495
|
+
},
|
|
2496
|
+
onUnassign: (payload) => {
|
|
2497
|
+
log(`[realtime] Agent ${payload.agent_id} unassigned`);
|
|
2498
|
+
clearAgentCaches(payload.agent_id, "");
|
|
2499
|
+
},
|
|
2500
|
+
log
|
|
2501
|
+
});
|
|
2502
|
+
realtimeAssignStarted = true;
|
|
2503
|
+
log(`[realtime] Assignment subscription started for host ${exchange.hostId}`);
|
|
2504
|
+
}).catch((err) => {
|
|
2505
|
+
log(`[realtime] Assignment subscription failed: ${err.message}`);
|
|
2506
|
+
});
|
|
2507
|
+
}
|
|
2508
|
+
function ensureRealtimeConfigStarted(agentStates) {
|
|
2509
|
+
if (realtimeConfigStarted) return;
|
|
2510
|
+
const activeAgentIds = agentStates.filter((a) => a.status === "active").map((a) => a.agentId);
|
|
2511
|
+
if (activeAgentIds.length === 0) return;
|
|
2512
|
+
const apiKey = process.env["AGT_API_KEY"];
|
|
2513
|
+
if (!apiKey) return;
|
|
2514
|
+
void exchangeApiKey(apiKey).then((exchange) => {
|
|
2515
|
+
if (!exchange.supabaseUrl || !exchange.supabaseAnonKey) return;
|
|
2516
|
+
startRealtimeConfig({
|
|
2517
|
+
supabaseUrl: exchange.supabaseUrl,
|
|
2518
|
+
supabaseAnonKey: exchange.supabaseAnonKey,
|
|
2519
|
+
token: exchange.token,
|
|
2520
|
+
agentIds: activeAgentIds,
|
|
2521
|
+
onConfigChange: (agent) => {
|
|
2522
|
+
knownStatuses.delete(agent.agent_id);
|
|
2523
|
+
},
|
|
2524
|
+
log
|
|
2525
|
+
});
|
|
2526
|
+
realtimeConfigStarted = true;
|
|
2527
|
+
log(`[realtime] Config subscription started for ${activeAgentIds.length} agent(s)`);
|
|
2528
|
+
}).catch((err) => {
|
|
2529
|
+
log(`[realtime] Config subscription failed: ${err.message}`);
|
|
2530
|
+
});
|
|
2531
|
+
}
|
|
2532
|
+
function ensureRealtimeKanbanStarted(agentStates) {
|
|
2533
|
+
if (realtimeKanbanStarted) return;
|
|
2534
|
+
const activeAgentIds = agentStates.filter((a) => a.status === "active").map((a) => a.agentId);
|
|
2535
|
+
if (activeAgentIds.length === 0) return;
|
|
2536
|
+
const apiKey = process.env["AGT_API_KEY"];
|
|
2537
|
+
if (!apiKey) return;
|
|
2538
|
+
void exchangeApiKey(apiKey).then((exchange) => {
|
|
2539
|
+
if (!exchange.supabaseUrl || !exchange.supabaseAnonKey) return;
|
|
2540
|
+
startRealtimeKanban({
|
|
2541
|
+
supabaseUrl: exchange.supabaseUrl,
|
|
2542
|
+
supabaseAnonKey: exchange.supabaseAnonKey,
|
|
2543
|
+
token: exchange.token,
|
|
2544
|
+
agentIds: activeAgentIds,
|
|
2545
|
+
onTodayItem: (item) => {
|
|
2546
|
+
const agent = agentStates.find((a) => a.agentId === item.agent_id);
|
|
2547
|
+
if (!agent) return;
|
|
2548
|
+
const agentFw = agentFrameworkCache.get(agent.codeName) ?? "openclaw";
|
|
2549
|
+
if (agentFw === "claude-code") {
|
|
2550
|
+
const boardItems = kanbanBoardCache.get(agent.codeName) ?? [];
|
|
2551
|
+
if (isSessionHealthy(agent.codeName)) {
|
|
2552
|
+
injectMessage(agent.codeName, "task", `New task added to your board: "${item.title}" (priority ${item.priority}). Pick it up \u2014 move to in_progress and start working.`, {
|
|
2553
|
+
task_name: "kanban-work-trigger"
|
|
2554
|
+
});
|
|
2555
|
+
log(`[realtime] Injected kanban-work trigger for '${agent.codeName}': "${item.title}"`);
|
|
2556
|
+
} else {
|
|
2557
|
+
fireClaudeWorkTrigger(agent.codeName, agent.agentId, boardItems);
|
|
2558
|
+
log(`[realtime] Fired kanban-work for '${agent.codeName}': "${item.title}"`);
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
},
|
|
2562
|
+
log
|
|
2563
|
+
});
|
|
2564
|
+
realtimeKanbanStarted = true;
|
|
2565
|
+
log(`[realtime] Kanban subscription started for ${activeAgentIds.length} agent(s)`);
|
|
2566
|
+
}).catch((err) => {
|
|
2567
|
+
log(`[realtime] Kanban subscription failed: ${err.message}`);
|
|
2568
|
+
});
|
|
2569
|
+
}
|
|
2570
|
+
var directChatInFlight = /* @__PURE__ */ new Set();
|
|
2571
|
+
async function pollDirectChatMessages(agentStates) {
|
|
2572
|
+
for (const agent of agentStates) {
|
|
2573
|
+
if (agent.status !== "active") continue;
|
|
2574
|
+
const fw = agentFrameworkCache.get(agent.codeName) ?? "openclaw";
|
|
2575
|
+
if (fw === "openclaw" && (!agent.gatewayRunning || !agent.gatewayPort)) continue;
|
|
2576
|
+
try {
|
|
2577
|
+
const data = await api.post("/host/direct-chat/poll", { agent_id: agent.agentId });
|
|
2578
|
+
for (const msg of data.messages) {
|
|
2579
|
+
if (directChatInFlight.has(msg.id)) continue;
|
|
2580
|
+
directChatInFlight.add(msg.id);
|
|
2581
|
+
processDirectChatMessage(agent, msg).finally(() => {
|
|
2582
|
+
directChatInFlight.delete(msg.id);
|
|
2583
|
+
});
|
|
2584
|
+
}
|
|
2585
|
+
} catch (err) {
|
|
2586
|
+
log(`Direct chat poll failed for '${agent.codeName}': ${err.message}`);
|
|
2587
|
+
}
|
|
2588
|
+
}
|
|
2589
|
+
}
|
|
2590
|
+
async function processDirectChatMessage(agent, msg) {
|
|
2591
|
+
const fw = agentFrameworkCache.get(agent.codeName) ?? "openclaw";
|
|
2592
|
+
log(`[direct-chat] Processing message for '${agent.codeName}' (fw=${fw}): id=${msg.id} len=${msg.content.length}`);
|
|
2593
|
+
try {
|
|
2594
|
+
let reply;
|
|
2595
|
+
if (fw === "claude-code") {
|
|
2596
|
+
const { getProjectDir: ccProjectDir } = await import("../claude-scheduler-APXMZEK4.js");
|
|
2597
|
+
const projDir = ccProjectDir(agent.codeName);
|
|
2598
|
+
const chatArgs = [
|
|
2599
|
+
"-p",
|
|
2600
|
+
msg.content,
|
|
2601
|
+
"--output-format",
|
|
2602
|
+
"text",
|
|
2603
|
+
"--mcp-config",
|
|
2604
|
+
join2(projDir, ".mcp.json"),
|
|
2605
|
+
"--allowedTools",
|
|
2606
|
+
"mcp__augmented__*,Bash,Read,Write,Edit,Grep,Glob"
|
|
2607
|
+
];
|
|
2608
|
+
const chatClaudeMd = join2(projDir, "CLAUDE.md");
|
|
2609
|
+
if (existsSync2(chatClaudeMd)) {
|
|
2610
|
+
chatArgs.push("--system-prompt-file", chatClaudeMd);
|
|
2611
|
+
}
|
|
2612
|
+
const { stdout } = await execFilePromiseLong("claude", chatArgs, { cwd: projDir, stdin: "ignore" });
|
|
2613
|
+
reply = stdout.trim() || "[No response from agent]";
|
|
2614
|
+
} else {
|
|
2615
|
+
const { stdout } = await execFilePromiseLong("openclaw", [
|
|
2616
|
+
"--profile",
|
|
2617
|
+
agent.codeName,
|
|
2618
|
+
"agent",
|
|
2619
|
+
"--local",
|
|
2620
|
+
"--agent",
|
|
2621
|
+
agent.codeName,
|
|
2622
|
+
"--message",
|
|
2623
|
+
msg.content,
|
|
2624
|
+
"--session-id",
|
|
2625
|
+
msg.session_id,
|
|
2626
|
+
"--json"
|
|
2627
|
+
]);
|
|
2628
|
+
try {
|
|
2629
|
+
const parsed = JSON.parse(stdout);
|
|
2630
|
+
const payloads = parsed?.payloads ?? parsed?.result?.payloads;
|
|
2631
|
+
reply = payloads?.[0]?.text ?? parsed?.reply ?? parsed?.text ?? parsed?.message ?? stdout;
|
|
2632
|
+
} catch {
|
|
2633
|
+
reply = stdout.trim() || "[No response from agent]";
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2636
|
+
await api.post("/host/direct-chat/reply", {
|
|
2637
|
+
agent_id: agent.agentId,
|
|
2638
|
+
session_id: msg.session_id,
|
|
2639
|
+
content: reply
|
|
2640
|
+
});
|
|
2641
|
+
log(`[direct-chat] Reply sent for '${agent.codeName}'`);
|
|
2642
|
+
} catch (err) {
|
|
2643
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2644
|
+
const errorId = createHash("sha256").update(errMsg).digest("hex").slice(0, 12);
|
|
2645
|
+
log(`[direct-chat] Failed to process message for '${agent.codeName}': error_id=${errorId}`);
|
|
2646
|
+
try {
|
|
2647
|
+
await api.post("/host/direct-chat/reply", {
|
|
2648
|
+
agent_id: agent.agentId,
|
|
2649
|
+
session_id: msg.session_id,
|
|
2650
|
+
content: `[Error] Failed to process message (ref: ${errorId}). Please retry.`
|
|
2651
|
+
});
|
|
2652
|
+
} catch {
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
var STANDUP_TEMPLATES = /* @__PURE__ */ new Set(["daily-standup", "end-of-day-summary"]);
|
|
2657
|
+
var TASK_UPDATE_TEMPLATES = /* @__PURE__ */ new Set(["hourly-status", "task-update"]);
|
|
2658
|
+
var PLAN_TEMPLATES = /* @__PURE__ */ new Set(["morning-plan"]);
|
|
2659
|
+
var KANBAN_WORK_TEMPLATES = /* @__PURE__ */ new Set(["kanban-work"]);
|
|
2660
|
+
var BOARD_INJECT_TEMPLATES = /* @__PURE__ */ new Set(["morning-plan", "task-update", "hourly-status", "end-of-day-summary", "kanban-work"]);
|
|
2661
|
+
var lastHarvestAt = /* @__PURE__ */ new Map();
|
|
2662
|
+
var HARVEST_INTERVAL_MS = 3 * 60 * 1e3;
|
|
2663
|
+
var kanbanBoardCache = /* @__PURE__ */ new Map();
|
|
2664
|
+
var notifyBoardCache = /* @__PURE__ */ new Map();
|
|
2665
|
+
async function harvestCronResults(codeName, tasks, gatewayPort) {
|
|
2666
|
+
const token = readGatewayToken(codeName);
|
|
2667
|
+
const gwArgs = ["--url", `ws://127.0.0.1:${gatewayPort}`, ...token ? ["--token", token] : []];
|
|
2668
|
+
let gatewayJobs = [];
|
|
2669
|
+
try {
|
|
2670
|
+
const cliBin = resolveAgentFramework(codeName).cliBinary ?? "openclaw";
|
|
2671
|
+
const { stdout } = await execFilePromise(cliBin, ["--profile", codeName, "cron", "list", "--json", ...gwArgs]);
|
|
2672
|
+
const parsed = JSON.parse(stdout);
|
|
2673
|
+
gatewayJobs = parsed.jobs ?? [];
|
|
2674
|
+
} catch {
|
|
2675
|
+
return;
|
|
2676
|
+
}
|
|
2677
|
+
const jobTemplateMap = /* @__PURE__ */ new Map();
|
|
2678
|
+
for (const job of gatewayJobs) {
|
|
2679
|
+
if (!job.name.startsWith("aug:")) continue;
|
|
2680
|
+
const parts = job.name.split(":");
|
|
2681
|
+
if (parts.length >= 3) {
|
|
2682
|
+
jobTemplateMap.set(job.id, parts[1]);
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
for (const [jobId, templateId] of jobTemplateMap) {
|
|
2686
|
+
if (!STANDUP_TEMPLATES.has(templateId) && !TASK_UPDATE_TEMPLATES.has(templateId) && !PLAN_TEMPLATES.has(templateId) && !KANBAN_WORK_TEMPLATES.has(templateId)) continue;
|
|
2687
|
+
let runs = [];
|
|
2688
|
+
try {
|
|
2689
|
+
const cliBin2 = resolveAgentFramework(codeName).cliBinary ?? "openclaw";
|
|
2690
|
+
const { stdout } = await execFilePromise(cliBin2, ["--profile", codeName, "cron", "runs", "--id", jobId, ...gwArgs]);
|
|
2691
|
+
const parsed = JSON.parse(stdout);
|
|
2692
|
+
runs = parsed.entries ?? [];
|
|
2693
|
+
} catch {
|
|
2694
|
+
continue;
|
|
2695
|
+
}
|
|
2696
|
+
const latestRun = runs.filter((r) => r.action === "finished").sort((a, b) => b.ts - a.ts)[0];
|
|
2697
|
+
if (latestRun) {
|
|
2698
|
+
const agentId = codeNameToAgentId.get(codeName);
|
|
2699
|
+
const summary = latestRun.summary ?? "";
|
|
2700
|
+
const isKeyError = summary.includes("Key limit exceeded") || summary.includes("key limit") || summary.includes("rate limit") || summary.includes("insufficient_quota") || summary.includes("billing") || summary.includes("402");
|
|
2701
|
+
if (agentId) {
|
|
2702
|
+
if (latestRun.status === "error" && isKeyError) {
|
|
2703
|
+
const errorMsg = summary.slice(0, 200);
|
|
2704
|
+
if (!apiKeyStatusCache.get(codeName)) {
|
|
2705
|
+
apiKeyStatusCache.set(codeName, true);
|
|
2706
|
+
api.post("/host/agent-api-key-status", { agent_id: agentId, status: "rate_limited", error: errorMsg }).catch(() => {
|
|
2707
|
+
});
|
|
2708
|
+
log(`API key error detected for '${codeName}': ${errorMsg}`);
|
|
2709
|
+
}
|
|
2710
|
+
} else if (latestRun.status === "ok" && apiKeyStatusCache.get(codeName)) {
|
|
2711
|
+
apiKeyStatusCache.delete(codeName);
|
|
2712
|
+
api.post("/host/agent-api-key-status", { agent_id: agentId, status: null, error: null }).catch(() => {
|
|
2713
|
+
});
|
|
2714
|
+
log(`API key status cleared for '${codeName}' \u2014 run succeeded`);
|
|
2715
|
+
}
|
|
2716
|
+
}
|
|
2717
|
+
}
|
|
2718
|
+
const completed = runs.filter((r) => r.action === "finished" && r.status === "ok" && r.summary).sort((a, b) => b.ts - a.ts);
|
|
2719
|
+
if (completed.length === 0) continue;
|
|
2720
|
+
const latest = completed[0];
|
|
2721
|
+
const lastSeen = lastCronRunTs.get(jobId) ?? 0;
|
|
2722
|
+
if (latest.ts <= lastSeen) continue;
|
|
2723
|
+
lastCronRunTs.set(jobId, latest.ts);
|
|
2724
|
+
const statusUpdate = { agent_code_name: codeName };
|
|
2725
|
+
if (STANDUP_TEMPLATES.has(templateId)) {
|
|
2726
|
+
const summary = latest.summary ?? "";
|
|
2727
|
+
statusUpdate.standup = parseStandupSummary(summary);
|
|
2728
|
+
}
|
|
2729
|
+
if (TASK_UPDATE_TEMPLATES.has(templateId)) {
|
|
2730
|
+
statusUpdate.current_tasks = latest.summary ?? "";
|
|
2731
|
+
}
|
|
2732
|
+
if (statusUpdate.standup || statusUpdate.current_tasks !== void 0) {
|
|
2733
|
+
try {
|
|
2734
|
+
await api.post("/host/agent-status", statusUpdate);
|
|
2735
|
+
log(`Updated ${templateId} for '${codeName}' from cron result`);
|
|
2736
|
+
} catch (err) {
|
|
2737
|
+
log(`Failed to update ${templateId} for '${codeName}': ${err.message}`);
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
if (PLAN_TEMPLATES.has(templateId)) {
|
|
2741
|
+
const summary = latest.summary ?? "";
|
|
2742
|
+
const planItems = parsePlanItems(summary);
|
|
2743
|
+
if (planItems.length > 0) {
|
|
2744
|
+
try {
|
|
2745
|
+
const agentId = codeNameToAgentId.get(codeName);
|
|
2746
|
+
if (agentId) {
|
|
2747
|
+
await api.post("/host/kanban", {
|
|
2748
|
+
agent_id: agentId,
|
|
2749
|
+
add: planItems,
|
|
2750
|
+
archive_days: 7
|
|
2751
|
+
});
|
|
2752
|
+
log(`Added ${planItems.length} kanban items for '${codeName}' from morning-plan`);
|
|
2753
|
+
}
|
|
2754
|
+
} catch (err) {
|
|
2755
|
+
log(`Failed to update kanban for '${codeName}': ${err.message}`);
|
|
2756
|
+
}
|
|
2757
|
+
}
|
|
2758
|
+
}
|
|
2759
|
+
if (TASK_UPDATE_TEMPLATES.has(templateId) || KANBAN_WORK_TEMPLATES.has(templateId)) {
|
|
2760
|
+
const summary = latest.summary ?? "";
|
|
2761
|
+
const kanbanUpdates = parseKanbanUpdates(summary);
|
|
2762
|
+
if (kanbanUpdates.length > 0) {
|
|
2763
|
+
try {
|
|
2764
|
+
const agentId = codeNameToAgentId.get(codeName);
|
|
2765
|
+
if (agentId) {
|
|
2766
|
+
await api.post("/host/kanban", {
|
|
2767
|
+
agent_id: agentId,
|
|
2768
|
+
update: kanbanUpdates
|
|
2769
|
+
});
|
|
2770
|
+
log(`Updated ${kanbanUpdates.length} kanban items for '${codeName}'`);
|
|
2771
|
+
}
|
|
2772
|
+
} catch (err) {
|
|
2773
|
+
log(`Failed to update kanban for '${codeName}': ${err.message}`);
|
|
2774
|
+
}
|
|
2775
|
+
}
|
|
2776
|
+
}
|
|
2777
|
+
}
|
|
2778
|
+
}
|
|
2779
|
+
function parseStandupSummary(summary) {
|
|
2780
|
+
const lines = summary.split("\n");
|
|
2781
|
+
let yesterday = "";
|
|
2782
|
+
let today = "";
|
|
2783
|
+
let blockers = "";
|
|
2784
|
+
let currentSection = null;
|
|
2785
|
+
for (const line of lines) {
|
|
2786
|
+
const lower = line.toLowerCase();
|
|
2787
|
+
if (lower.includes("yesterday") || lower.includes("accomplished")) {
|
|
2788
|
+
currentSection = "yesterday";
|
|
2789
|
+
continue;
|
|
2790
|
+
} else if (lower.includes("today") || lower.includes("working on")) {
|
|
2791
|
+
currentSection = "today";
|
|
2792
|
+
continue;
|
|
2793
|
+
} else if (lower.includes("blocker")) {
|
|
2794
|
+
currentSection = "blockers";
|
|
2795
|
+
continue;
|
|
2796
|
+
}
|
|
2797
|
+
const trimmed = line.replace(/^[-*•]\s*/, "").trim();
|
|
2798
|
+
if (!trimmed) continue;
|
|
2799
|
+
switch (currentSection) {
|
|
2800
|
+
case "yesterday":
|
|
2801
|
+
yesterday += (yesterday ? "\n" : "") + trimmed;
|
|
2802
|
+
break;
|
|
2803
|
+
case "today":
|
|
2804
|
+
today += (today ? "\n" : "") + trimmed;
|
|
2805
|
+
break;
|
|
2806
|
+
case "blockers":
|
|
2807
|
+
blockers += (blockers ? "\n" : "") + trimmed;
|
|
2808
|
+
break;
|
|
2809
|
+
}
|
|
2810
|
+
}
|
|
2811
|
+
if (!yesterday && !today && !blockers) {
|
|
2812
|
+
today = summary;
|
|
2813
|
+
}
|
|
2814
|
+
return { yesterday, today, blockers };
|
|
2815
|
+
}
|
|
2816
|
+
function parsePlanItems(summary) {
|
|
2817
|
+
const items = [];
|
|
2818
|
+
const lines = summary.split("\n");
|
|
2819
|
+
let currentItem = null;
|
|
2820
|
+
for (const line of lines) {
|
|
2821
|
+
const trimmed = line.trim();
|
|
2822
|
+
const itemMatch = trimmed.match(/^(?:\d+[\.\)]\s*|[-*•]\s*)\[?(HIGH|MEDIUM|LOW|MED)\]?\s*(.+)/i);
|
|
2823
|
+
if (itemMatch) {
|
|
2824
|
+
if (currentItem) items.push(currentItem);
|
|
2825
|
+
const priorityStr = itemMatch[1].toUpperCase();
|
|
2826
|
+
const rest = itemMatch[2];
|
|
2827
|
+
let estimatedMinutes;
|
|
2828
|
+
const timeMatch = rest.match(/\(~?(\d+)\s*(min(?:utes?)?|hr?(?:ours?)?|h)\)/i);
|
|
2829
|
+
if (timeMatch) {
|
|
2830
|
+
const val = parseInt(timeMatch[1], 10);
|
|
2831
|
+
const unit = timeMatch[2].toLowerCase();
|
|
2832
|
+
estimatedMinutes = unit.startsWith("h") ? val * 60 : val;
|
|
2833
|
+
}
|
|
2834
|
+
const title = sanitizeKanbanString(
|
|
2835
|
+
rest.replace(/\(~?\d+\s*(?:min(?:utes?)?|hr?(?:ours?)?|h)\)/i, ""),
|
|
2836
|
+
MAX_KANBAN_TITLE_LENGTH
|
|
2837
|
+
);
|
|
2838
|
+
if (!title) continue;
|
|
2839
|
+
const priorityMap = { HIGH: 1, MEDIUM: 2, MED: 2, LOW: 3 };
|
|
2840
|
+
const priority = priorityMap[priorityStr] ?? 2;
|
|
2841
|
+
if (![1, 2, 3].includes(priority)) continue;
|
|
2842
|
+
currentItem = {
|
|
2843
|
+
title,
|
|
2844
|
+
priority,
|
|
2845
|
+
estimated_minutes: estimatedMinutes,
|
|
2846
|
+
status: "today"
|
|
2847
|
+
};
|
|
2848
|
+
} else if (currentItem && trimmed && !trimmed.match(/^(?:PLAN|---)/i)) {
|
|
2849
|
+
const descLine = sanitizeKanbanString(trimmed, MAX_KANBAN_NOTES_LENGTH);
|
|
2850
|
+
currentItem.description = currentItem.description ? sanitizeKanbanString(currentItem.description + "\n" + descLine, MAX_KANBAN_NOTES_LENGTH) : descLine;
|
|
2851
|
+
}
|
|
2852
|
+
}
|
|
2853
|
+
if (currentItem) items.push(currentItem);
|
|
2854
|
+
return items;
|
|
2855
|
+
}
|
|
2856
|
+
var VALID_KANBAN_STATUSES = /* @__PURE__ */ new Set(["backlog", "today", "in_progress", "done"]);
|
|
2857
|
+
var MAX_KANBAN_TITLE_LENGTH = 500;
|
|
2858
|
+
var MAX_KANBAN_NOTES_LENGTH = 2e3;
|
|
2859
|
+
function sanitizeKanbanString(value, maxLen) {
|
|
2860
|
+
if (!value) return "";
|
|
2861
|
+
return value.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/\{\{.*?\}\}/g, "").replace(/^(System|Assistant|Human):\s*/gmi, "").replace(/#{2,}/g, "").replace(/\s+/g, " ").trim().slice(0, maxLen);
|
|
2862
|
+
}
|
|
2863
|
+
function sanitizeBoardItem(item) {
|
|
2864
|
+
return {
|
|
2865
|
+
...item,
|
|
2866
|
+
title: sanitizeKanbanString(item.title, 200),
|
|
2867
|
+
status: sanitizeKanbanString(item.status, 50),
|
|
2868
|
+
...item.deliverable ? { deliverable: sanitizeKanbanString(item.deliverable, 500) } : {}
|
|
2869
|
+
};
|
|
2870
|
+
}
|
|
2871
|
+
var builtInSkillCache = /* @__PURE__ */ new Map();
|
|
2872
|
+
function getBuiltInSkillContent(skillId) {
|
|
2873
|
+
if (builtInSkillCache.has(skillId)) return builtInSkillCache.get(skillId);
|
|
2874
|
+
try {
|
|
2875
|
+
const candidates = [
|
|
2876
|
+
join2(process.cwd(), "skills", skillId, "SKILL.md"),
|
|
2877
|
+
join2(new URL(".", import.meta.url).pathname, "..", "..", "..", "..", "skills", skillId, "SKILL.md")
|
|
2878
|
+
];
|
|
2879
|
+
for (const candidate of candidates) {
|
|
2880
|
+
if (existsSync2(candidate)) {
|
|
2881
|
+
const content = readFileSync2(candidate, "utf-8");
|
|
2882
|
+
const files = [{ relativePath: "SKILL.md", content }];
|
|
2883
|
+
builtInSkillCache.set(skillId, files);
|
|
2884
|
+
return files;
|
|
2885
|
+
}
|
|
2886
|
+
}
|
|
2887
|
+
builtInSkillCache.set(skillId, null);
|
|
2888
|
+
return null;
|
|
2889
|
+
} catch {
|
|
2890
|
+
builtInSkillCache.set(skillId, null);
|
|
2891
|
+
return null;
|
|
2892
|
+
}
|
|
2893
|
+
}
|
|
2894
|
+
function parseKanbanUpdates(summary) {
|
|
2895
|
+
const updates = [];
|
|
2896
|
+
const kanbanIdx = summary.indexOf("KANBAN UPDATE:");
|
|
2897
|
+
if (kanbanIdx === -1) return updates;
|
|
2898
|
+
const kanbanSection = summary.slice(kanbanIdx + "KANBAN UPDATE:".length);
|
|
2899
|
+
const lines = kanbanSection.split("\n");
|
|
2900
|
+
for (const line of lines) {
|
|
2901
|
+
const trimmed = line.trim();
|
|
2902
|
+
const match = trimmed.match(/^[-*•]\s*"([^"]+)":\s*(backlog|today|in_progress|done)(?:\s*\((.+)\))?/i);
|
|
2903
|
+
if (match) {
|
|
2904
|
+
const status = match[2].toLowerCase();
|
|
2905
|
+
if (!VALID_KANBAN_STATUSES.has(status)) continue;
|
|
2906
|
+
const title = sanitizeKanbanString(match[1], MAX_KANBAN_TITLE_LENGTH);
|
|
2907
|
+
if (!title) continue;
|
|
2908
|
+
const parenthetical = match[3] ?? void 0;
|
|
2909
|
+
let notes;
|
|
2910
|
+
let result;
|
|
2911
|
+
if (parenthetical && status === "done") {
|
|
2912
|
+
const resultMatch = parenthetical.match(/^result:\s*(.+)/i);
|
|
2913
|
+
if (resultMatch) {
|
|
2914
|
+
result = sanitizeKanbanString(resultMatch[1], MAX_KANBAN_NOTES_LENGTH);
|
|
2915
|
+
} else {
|
|
2916
|
+
notes = sanitizeKanbanString(parenthetical, MAX_KANBAN_NOTES_LENGTH);
|
|
2917
|
+
}
|
|
2918
|
+
} else if (parenthetical) {
|
|
2919
|
+
notes = sanitizeKanbanString(parenthetical, MAX_KANBAN_NOTES_LENGTH);
|
|
2920
|
+
}
|
|
2921
|
+
updates.push({
|
|
2922
|
+
title,
|
|
2923
|
+
status,
|
|
2924
|
+
notes,
|
|
2925
|
+
result
|
|
2926
|
+
});
|
|
2927
|
+
}
|
|
2928
|
+
}
|
|
2929
|
+
return updates;
|
|
2930
|
+
}
|
|
2931
|
+
function formatBoardForPrompt(items, template) {
|
|
2932
|
+
if (items.length === 0) return "";
|
|
2933
|
+
const priorityLabel = (p) => p === 1 ? "HIGH" : p === 3 ? "LOW" : "MED";
|
|
2934
|
+
const timeLabel = (m) => m ? ` (~${m >= 60 ? `${Math.round(m / 60)}hr` : `${m}min`})` : "";
|
|
2935
|
+
const deliverableLine = (d) => d ? `
|
|
2936
|
+
Deliverable: ${d}` : "";
|
|
2937
|
+
const grouped = {};
|
|
2938
|
+
for (const item of items) {
|
|
2939
|
+
const key = item.status;
|
|
2940
|
+
if (!grouped[key]) grouped[key] = [];
|
|
2941
|
+
grouped[key].push(item);
|
|
2942
|
+
}
|
|
2943
|
+
const lines = [];
|
|
2944
|
+
if (template === "morning-plan") {
|
|
2945
|
+
lines.push("=== CURRENT BOARD ===");
|
|
2946
|
+
for (const [status, label] of [["backlog", "BACKLOG (carry-over)"], ["today", "TODAY"], ["in_progress", "IN PROGRESS"]]) {
|
|
2947
|
+
const statusItems = grouped[status];
|
|
2948
|
+
if (statusItems && statusItems.length > 0) {
|
|
2949
|
+
lines.push(`${label}:`);
|
|
2950
|
+
statusItems.forEach((item, i) => {
|
|
2951
|
+
lines.push(` ${i + 1}. [${priorityLabel(item.priority)}] ${item.title}${timeLabel(item.estimated_minutes)}${deliverableLine(item.deliverable)}`);
|
|
2952
|
+
});
|
|
2953
|
+
}
|
|
2954
|
+
}
|
|
2955
|
+
lines.push("=====================");
|
|
2956
|
+
lines.push("");
|
|
2957
|
+
lines.push("Create today's plan. You may:");
|
|
2958
|
+
lines.push('- Move backlog items to "today"');
|
|
2959
|
+
lines.push("- Add new items you've identified");
|
|
2960
|
+
lines.push("- Reprioritise existing items");
|
|
2961
|
+
lines.push("");
|
|
2962
|
+
} else {
|
|
2963
|
+
lines.push("=== YOUR KANBAN BOARD ===");
|
|
2964
|
+
for (const [status, label] of [["today", "TODAY"], ["in_progress", "IN PROGRESS"], ["backlog", "BACKLOG"]]) {
|
|
2965
|
+
const statusItems = grouped[status];
|
|
2966
|
+
if (statusItems && statusItems.length > 0) {
|
|
2967
|
+
lines.push(`${label}:`);
|
|
2968
|
+
statusItems.forEach((item, i) => {
|
|
2969
|
+
lines.push(` ${i + 1}. [${priorityLabel(item.priority)}] ${item.title}${timeLabel(item.estimated_minutes)}${deliverableLine(item.deliverable)}`);
|
|
2970
|
+
});
|
|
2971
|
+
}
|
|
2972
|
+
}
|
|
2973
|
+
const doneItems = grouped["done"];
|
|
2974
|
+
if (doneItems && doneItems.length > 0) {
|
|
2975
|
+
lines.push("DONE TODAY:");
|
|
2976
|
+
doneItems.forEach((item, i) => {
|
|
2977
|
+
lines.push(` ${i + 1}. ${item.title}`);
|
|
2978
|
+
});
|
|
2979
|
+
}
|
|
2980
|
+
lines.push("=========================");
|
|
2981
|
+
lines.push("");
|
|
2982
|
+
lines.push("IMPORTANT: Use kanban MCP tools to update the board IN REAL TIME:");
|
|
2983
|
+
lines.push("1. FIRST call kanban.move to move your chosen item to in_progress BEFORE starting work");
|
|
2984
|
+
lines.push("2. Do the work");
|
|
2985
|
+
lines.push("3. Call kanban.done with a result summary when finished");
|
|
2986
|
+
lines.push("4. If blocked, call kanban.update with notes, then pick the next item");
|
|
2987
|
+
lines.push("");
|
|
2988
|
+
lines.push("SELF-MANAGEMENT: When you receive a request from a channel (Slack, Telegram)");
|
|
2989
|
+
lines.push("that takes more than a quick response, create a task first with kanban.add,");
|
|
2990
|
+
lines.push("move to in_progress, do the work, then kanban.done with a result summary.");
|
|
2991
|
+
lines.push("");
|
|
2992
|
+
lines.push("If MCP tools are unavailable, include a KANBAN UPDATE section in your output:");
|
|
2993
|
+
lines.push("KANBAN UPDATE:");
|
|
2994
|
+
lines.push('- "item title": new_status (optional notes)');
|
|
2995
|
+
lines.push('- "item title": done (result: <what you produced>)');
|
|
2996
|
+
lines.push("Statuses: backlog, today, in_progress, done");
|
|
2997
|
+
lines.push("");
|
|
2998
|
+
}
|
|
2999
|
+
return lines.join("\n");
|
|
3000
|
+
}
|
|
3001
|
+
async function execFilePromise(cmd, args) {
|
|
3002
|
+
const { execFile: ef } = await import("child_process");
|
|
3003
|
+
return new Promise((resolve, reject) => {
|
|
3004
|
+
ef(cmd, args, { timeout: 15e3 }, (err, stdout, stderr) => {
|
|
3005
|
+
if (err) reject(err);
|
|
3006
|
+
else resolve({ stdout, stderr });
|
|
3007
|
+
});
|
|
3008
|
+
});
|
|
3009
|
+
}
|
|
3010
|
+
async function execFilePromiseLong(cmd, args, opts) {
|
|
3011
|
+
const { spawn: sp } = await import("child_process");
|
|
3012
|
+
return new Promise((resolve, reject) => {
|
|
3013
|
+
const child = sp(cmd, args, {
|
|
3014
|
+
cwd: opts?.cwd,
|
|
3015
|
+
stdio: [opts?.stdin === "ignore" ? "ignore" : "pipe", "pipe", "pipe"]
|
|
3016
|
+
});
|
|
3017
|
+
let stdout = "";
|
|
3018
|
+
let stderr = "";
|
|
3019
|
+
child.stdout?.on("data", (d) => {
|
|
3020
|
+
stdout += d.toString();
|
|
3021
|
+
});
|
|
3022
|
+
child.stderr?.on("data", (d) => {
|
|
3023
|
+
stderr += d.toString();
|
|
3024
|
+
});
|
|
3025
|
+
const timer = setTimeout(() => {
|
|
3026
|
+
child.kill();
|
|
3027
|
+
reject(new Error(`Timed out after ${opts?.timeout ?? 12e4}ms`));
|
|
3028
|
+
}, opts?.timeout ?? 12e4);
|
|
3029
|
+
child.on("close", (code) => {
|
|
3030
|
+
clearTimeout(timer);
|
|
3031
|
+
if (code !== 0) reject(new Error(`Exit code ${code}: ${stderr.slice(0, 500)}`));
|
|
3032
|
+
else resolve({ stdout, stderr });
|
|
3033
|
+
});
|
|
3034
|
+
child.on("error", (err) => {
|
|
3035
|
+
clearTimeout(timer);
|
|
3036
|
+
reject(err);
|
|
3037
|
+
});
|
|
3038
|
+
});
|
|
3039
|
+
}
|
|
3040
|
+
var LATE_THRESHOLD_MS = 5 * 60 * 1e3;
|
|
3041
|
+
async function monitorCronHealth(agentStates) {
|
|
3042
|
+
const alerts = [];
|
|
3043
|
+
const now = Date.now();
|
|
3044
|
+
for (const agent of agentStates) {
|
|
3045
|
+
if (!agent.gatewayRunning || !agent.gatewayPort) continue;
|
|
3046
|
+
const token = readGatewayToken(agent.codeName);
|
|
3047
|
+
const gwArgs = ["--url", `ws://127.0.0.1:${agent.gatewayPort}`, ...token ? ["--token", token] : []];
|
|
3048
|
+
let jobs = [];
|
|
3049
|
+
try {
|
|
3050
|
+
const cliBin = resolveAgentFramework(agent.codeName).cliBinary ?? "openclaw";
|
|
3051
|
+
const { stdout } = await execFilePromise(cliBin, ["--profile", agent.codeName, "cron", "list", "--json", ...gwArgs]);
|
|
3052
|
+
const parsed = JSON.parse(stdout);
|
|
3053
|
+
jobs = parsed.jobs ?? [];
|
|
3054
|
+
} catch {
|
|
3055
|
+
continue;
|
|
3056
|
+
}
|
|
3057
|
+
for (const job of jobs) {
|
|
3058
|
+
if (!job.enabled || !job.name.startsWith("aug:")) continue;
|
|
3059
|
+
const alertKey = `${agent.codeName}:${job.id}`;
|
|
3060
|
+
const displayInfo = taskDisplayInfo.get(`${agent.codeName}:${job.name}`);
|
|
3061
|
+
const taskName = displayInfo?.taskName ?? job.name;
|
|
3062
|
+
const schedule = displayInfo?.schedule ?? "";
|
|
3063
|
+
const agentDisplayName = displayInfo?.agentDisplayName ?? agent.codeName;
|
|
3064
|
+
if (job.state?.nextRunAtMs && job.state.nextRunAtMs + LATE_THRESHOLD_MS < now) {
|
|
3065
|
+
if (!alertedJobs.has(`late:${alertKey}`)) {
|
|
3066
|
+
const minsLate = Math.round((now - job.state.nextRunAtMs) / 6e4);
|
|
3067
|
+
alerts.push({
|
|
3068
|
+
type: "late_standup",
|
|
3069
|
+
agentCodeName: agent.codeName,
|
|
3070
|
+
agentDisplayName,
|
|
3071
|
+
jobName: job.name,
|
|
3072
|
+
taskName,
|
|
3073
|
+
schedule,
|
|
3074
|
+
jobId: job.id,
|
|
3075
|
+
detail: `Job is ${minsLate}m late (expected at ${new Date(job.state.nextRunAtMs).toISOString()})`
|
|
3076
|
+
});
|
|
3077
|
+
alertedJobs.add(`late:${alertKey}`);
|
|
3078
|
+
}
|
|
3079
|
+
} else {
|
|
3080
|
+
alertedJobs.delete(`late:${alertKey}`);
|
|
3081
|
+
}
|
|
3082
|
+
if (job.state?.lastRunStatus === "error" || job.state?.consecutiveErrors && job.state.consecutiveErrors > 0) {
|
|
3083
|
+
if (!alertedJobs.has(`fail:${alertKey}`)) {
|
|
3084
|
+
alerts.push({
|
|
3085
|
+
type: "cron_failure",
|
|
3086
|
+
agentCodeName: agent.codeName,
|
|
3087
|
+
agentDisplayName,
|
|
3088
|
+
jobName: job.name,
|
|
3089
|
+
taskName,
|
|
3090
|
+
schedule,
|
|
3091
|
+
jobId: job.id,
|
|
3092
|
+
detail: `Last run failed (${job.state.consecutiveErrors ?? 1} consecutive error(s))`
|
|
3093
|
+
});
|
|
3094
|
+
alertedJobs.add(`fail:${alertKey}`);
|
|
3095
|
+
}
|
|
3096
|
+
} else {
|
|
3097
|
+
alertedJobs.delete(`fail:${alertKey}`);
|
|
3098
|
+
}
|
|
3099
|
+
}
|
|
3100
|
+
}
|
|
3101
|
+
if (alerts.length === 0) return;
|
|
3102
|
+
for (const alert of alerts) {
|
|
3103
|
+
log(`ALERT [${alert.type}] ${alert.agentCodeName}/${alert.jobName}: ${alert.detail}`);
|
|
3104
|
+
}
|
|
3105
|
+
if (alertSlackWebhook) {
|
|
3106
|
+
await sendSlackAlert(alerts);
|
|
3107
|
+
}
|
|
3108
|
+
try {
|
|
3109
|
+
await api.post("/host/cron-alerts", { alerts });
|
|
3110
|
+
} catch {
|
|
3111
|
+
}
|
|
3112
|
+
}
|
|
3113
|
+
function telegramApiCall(botToken, method, body) {
|
|
3114
|
+
return new Promise((resolve, reject) => {
|
|
3115
|
+
const postData = JSON.stringify(body);
|
|
3116
|
+
const req = https.request({
|
|
3117
|
+
hostname: "api.telegram.org",
|
|
3118
|
+
port: 443,
|
|
3119
|
+
path: `/bot${botToken}/${method}`,
|
|
3120
|
+
method: "POST",
|
|
3121
|
+
family: 4,
|
|
3122
|
+
timeout: 1e4,
|
|
3123
|
+
headers: { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(postData) }
|
|
3124
|
+
}, (res) => {
|
|
3125
|
+
let data = "";
|
|
3126
|
+
res.on("data", (d) => {
|
|
3127
|
+
data += d;
|
|
3128
|
+
});
|
|
3129
|
+
res.on("end", () => {
|
|
3130
|
+
try {
|
|
3131
|
+
resolve(JSON.parse(data));
|
|
3132
|
+
} catch {
|
|
3133
|
+
reject(new Error("Invalid JSON from Telegram API"));
|
|
3134
|
+
}
|
|
3135
|
+
});
|
|
3136
|
+
});
|
|
3137
|
+
req.on("error", reject);
|
|
3138
|
+
req.on("timeout", () => {
|
|
3139
|
+
req.destroy();
|
|
3140
|
+
reject(new Error("Telegram API timeout"));
|
|
3141
|
+
});
|
|
3142
|
+
req.write(postData);
|
|
3143
|
+
req.end();
|
|
3144
|
+
});
|
|
3145
|
+
}
|
|
3146
|
+
async function sendSlackWebhookMessage(text) {
|
|
3147
|
+
if (!alertSlackWebhook) {
|
|
3148
|
+
log("sendSlackWebhookMessage: no alertSlackWebhook configured \u2014 message dropped");
|
|
3149
|
+
return;
|
|
3150
|
+
}
|
|
3151
|
+
try {
|
|
3152
|
+
const response = await fetch(alertSlackWebhook, {
|
|
3153
|
+
method: "POST",
|
|
3154
|
+
headers: { "Content-Type": "application/json" },
|
|
3155
|
+
body: JSON.stringify({ text })
|
|
3156
|
+
});
|
|
3157
|
+
if (!response.ok) {
|
|
3158
|
+
log(`Slack webhook failed: ${response.status} ${response.statusText}`);
|
|
3159
|
+
}
|
|
3160
|
+
} catch (err) {
|
|
3161
|
+
log(`Slack webhook error: ${err.message}`);
|
|
3162
|
+
}
|
|
3163
|
+
}
|
|
3164
|
+
async function sendSlackChannelMessage(agentCodeName, channelId, text) {
|
|
3165
|
+
const botToken = agentChannelTokens.get(agentCodeName)?.slack;
|
|
3166
|
+
if (!botToken) {
|
|
3167
|
+
log(`No Slack bot token cached for '${agentCodeName}' \u2014 cannot post to ${channelId}`);
|
|
3168
|
+
return false;
|
|
3169
|
+
}
|
|
3170
|
+
try {
|
|
3171
|
+
const controller = new AbortController();
|
|
3172
|
+
const timeout = setTimeout(() => controller.abort(), 5e3);
|
|
3173
|
+
try {
|
|
3174
|
+
const response = await fetch("https://slack.com/api/chat.postMessage", {
|
|
3175
|
+
method: "POST",
|
|
3176
|
+
headers: {
|
|
3177
|
+
"Authorization": `Bearer ${botToken}`,
|
|
3178
|
+
"Content-Type": "application/json"
|
|
3179
|
+
},
|
|
3180
|
+
body: JSON.stringify({ channel: channelId, text }),
|
|
3181
|
+
signal: controller.signal
|
|
3182
|
+
});
|
|
3183
|
+
clearTimeout(timeout);
|
|
3184
|
+
const data = await response.json();
|
|
3185
|
+
if (!data.ok) {
|
|
3186
|
+
log(`Slack chat.postMessage failed for '${agentCodeName}' to ${channelId}: ${data.error}`);
|
|
3187
|
+
return false;
|
|
3188
|
+
}
|
|
3189
|
+
return true;
|
|
3190
|
+
} finally {
|
|
3191
|
+
clearTimeout(timeout);
|
|
3192
|
+
}
|
|
3193
|
+
} catch (err) {
|
|
3194
|
+
log(`Slack channel message error for '${agentCodeName}': ${err.message}`);
|
|
3195
|
+
return false;
|
|
3196
|
+
}
|
|
3197
|
+
}
|
|
3198
|
+
async function sendSlackAlert(alerts) {
|
|
3199
|
+
const blocks = alerts.map((a) => {
|
|
3200
|
+
const emoji = a.type === "late_standup" ? ":warning:" : ":x:";
|
|
3201
|
+
const label = a.type === "late_standup" ? "Late" : "Failed";
|
|
3202
|
+
const scheduleInfo = a.schedule ? ` (${a.schedule})` : "";
|
|
3203
|
+
return `${emoji} *${label}* \u2014 *${a.agentDisplayName}* / ${a.taskName}${scheduleInfo}
|
|
3204
|
+
${a.detail}`;
|
|
3205
|
+
});
|
|
3206
|
+
await sendSlackWebhookMessage(`:rotating_light: *Cron Health Alert*
|
|
3207
|
+
|
|
3208
|
+
${blocks.join("\n\n")}`);
|
|
3209
|
+
}
|
|
3210
|
+
async function sendTaskNotification(agentCodeName, channel, to, text) {
|
|
3211
|
+
const tokens = agentChannelTokens.get(agentCodeName);
|
|
3212
|
+
if (channel === "slack") {
|
|
3213
|
+
const botToken = tokens?.slack;
|
|
3214
|
+
const channelId = to.replace(/^channel:/, "");
|
|
3215
|
+
if (botToken) {
|
|
3216
|
+
const sent = await sendSlackChannelMessage(agentCodeName, channelId, text);
|
|
3217
|
+
if (sent) return;
|
|
3218
|
+
}
|
|
3219
|
+
log(`No Slack bot token for '${agentCodeName}' \u2014 targeted notification dropped`);
|
|
3220
|
+
} else if (channel === "telegram") {
|
|
3221
|
+
const botToken = tokens?.telegram;
|
|
3222
|
+
const chatId = to.replace(/^chat:/, "");
|
|
3223
|
+
if (!botToken) {
|
|
3224
|
+
log(`No Telegram bot token for '${agentCodeName}' \u2014 notification dropped`);
|
|
3225
|
+
return;
|
|
3226
|
+
}
|
|
3227
|
+
const allowedChats = tokens?.telegramAllowedChats;
|
|
3228
|
+
if (allowedChats && allowedChats.length > 0 && !allowedChats.includes(chatId)) {
|
|
3229
|
+
log(`Telegram chat ${chatId} not in allowed_chat_ids for '${agentCodeName}'`);
|
|
3230
|
+
return;
|
|
3231
|
+
}
|
|
3232
|
+
try {
|
|
3233
|
+
const result = await telegramApiCall(botToken, "sendMessage", { chat_id: chatId, text });
|
|
3234
|
+
if (!result.ok) {
|
|
3235
|
+
log(`Telegram sendMessage failed for '${agentCodeName}': ${result.description}`);
|
|
3236
|
+
} else {
|
|
3237
|
+
log(`Telegram notification sent for '${agentCodeName}' to chat ${chatId}`);
|
|
3238
|
+
}
|
|
3239
|
+
} catch (err) {
|
|
3240
|
+
log(`Telegram API error for '${agentCodeName}': ${err.message}`);
|
|
3241
|
+
}
|
|
3242
|
+
} else {
|
|
3243
|
+
log(`Unknown notify_channel '${channel}' for '${agentCodeName}'`);
|
|
3244
|
+
}
|
|
3245
|
+
}
|
|
3246
|
+
function generateArtifacts(agent, refreshData, adapter) {
|
|
3247
|
+
if (!refreshData.charter || !refreshData.tools) {
|
|
3248
|
+
throw new Error("No charter/tools available");
|
|
3249
|
+
}
|
|
3250
|
+
const charterContent = refreshData.charter.raw_content;
|
|
3251
|
+
const toolsContent = refreshData.tools.raw_content;
|
|
3252
|
+
const charterParsed = extractFrontmatter(charterContent);
|
|
3253
|
+
if (!charterParsed.frontmatter) {
|
|
3254
|
+
throw new Error(`Failed to parse CHARTER.md frontmatter: ${charterParsed.error ?? "unknown"}`);
|
|
3255
|
+
}
|
|
3256
|
+
const toolsParsed = extractFrontmatter(toolsContent);
|
|
3257
|
+
if (!toolsParsed.frontmatter) {
|
|
3258
|
+
throw new Error(`Failed to parse TOOLS.md frontmatter: ${toolsParsed.error ?? "unknown"}`);
|
|
3259
|
+
}
|
|
3260
|
+
const charterFrontmatter = charterParsed.frontmatter;
|
|
3261
|
+
const toolsFrontmatter = toolsParsed.frontmatter;
|
|
3262
|
+
const configuredChannels = Object.keys(refreshData.channel_configs ?? {});
|
|
3263
|
+
const agentChannelPolicy = {
|
|
3264
|
+
policy: "allowlist",
|
|
3265
|
+
allowed: configuredChannels,
|
|
3266
|
+
denied: [],
|
|
3267
|
+
require_approval_to_change: true
|
|
3268
|
+
};
|
|
3269
|
+
let teamChannelPolicy;
|
|
3270
|
+
const policyData = refreshData.team_channel_policy;
|
|
3271
|
+
if (policyData) {
|
|
3272
|
+
teamChannelPolicy = {
|
|
3273
|
+
team_id: policyData.team_id,
|
|
3274
|
+
allowed_channels: policyData.allowed_channels ?? [],
|
|
3275
|
+
denied_channels: policyData.denied_channels ?? [],
|
|
3276
|
+
require_elevated_for_pii: policyData.require_elevated_for_pii ?? false
|
|
3277
|
+
};
|
|
3278
|
+
}
|
|
3279
|
+
const resolvedChannels = resolveChannels(agentChannelPolicy, teamChannelPolicy);
|
|
3280
|
+
const effectiveChannels = agent.status === "paused" ? [] : resolvedChannels;
|
|
3281
|
+
const provisionInput = {
|
|
3282
|
+
agent: refreshData.agent,
|
|
3283
|
+
charterFrontmatter,
|
|
3284
|
+
charterContent,
|
|
3285
|
+
toolsFrontmatter,
|
|
3286
|
+
toolsContent,
|
|
3287
|
+
resolvedChannels: effectiveChannels,
|
|
3288
|
+
deploymentTarget: "local_docker",
|
|
3289
|
+
gatewayPort: 9e3,
|
|
3290
|
+
team: refreshData.team ?? void 0
|
|
3291
|
+
};
|
|
3292
|
+
const provisionOutput = provision(provisionInput, adapter.id);
|
|
3293
|
+
return provisionOutput.artifacts;
|
|
3294
|
+
}
|
|
3295
|
+
async function cleanupAgentFiles(codeName, agentDir) {
|
|
3296
|
+
if (existsSync2(agentDir)) {
|
|
3297
|
+
try {
|
|
3298
|
+
rmSync(agentDir, { recursive: true, force: true });
|
|
3299
|
+
log(`Removed provision directory for '${codeName}'`);
|
|
3300
|
+
} catch (err) {
|
|
3301
|
+
log(`Failed to remove provision dir for '${codeName}': ${err.message}`);
|
|
3302
|
+
}
|
|
3303
|
+
}
|
|
3304
|
+
try {
|
|
3305
|
+
const adapter = resolveAgentFramework(codeName);
|
|
3306
|
+
const registered = await getOrCacheRegisteredAgents(adapter, codeName);
|
|
3307
|
+
if (registered.has(codeName)) {
|
|
3308
|
+
await adapter.deregisterAgent(codeName);
|
|
3309
|
+
registered.delete(codeName);
|
|
3310
|
+
log(`Deregistered '${codeName}' from ${adapter.label}`);
|
|
3311
|
+
}
|
|
3312
|
+
} catch (err) {
|
|
3313
|
+
log(`Failed to deregister '${codeName}': ${err.message}`);
|
|
3314
|
+
}
|
|
3315
|
+
}
|
|
3316
|
+
function startGatewayPool() {
|
|
3317
|
+
if (gatewayPool) return;
|
|
3318
|
+
gatewayPool = new GatewayClientPool();
|
|
3319
|
+
gatewayPool.on("connected", (codeName) => {
|
|
3320
|
+
log(`Gateway WebSocket connected for '${codeName}'`);
|
|
3321
|
+
});
|
|
3322
|
+
gatewayPool.on("disconnected", (codeName) => {
|
|
3323
|
+
log(`Gateway WebSocket disconnected for '${codeName}'`);
|
|
3324
|
+
});
|
|
3325
|
+
gatewayPool.on("error", (err, codeName) => {
|
|
3326
|
+
log(`Gateway WebSocket error for '${codeName}': ${err.message}`);
|
|
3327
|
+
});
|
|
3328
|
+
gatewayPool.on("event", (evt) => {
|
|
3329
|
+
send({ type: "gateway-event", event: evt.event, payload: evt.payload, agentCodeName: evt.agentCodeName });
|
|
3330
|
+
getHostId().then((hostId) => {
|
|
3331
|
+
if (!hostId) return;
|
|
3332
|
+
api.post("/host/events", {
|
|
3333
|
+
host_id: hostId,
|
|
3334
|
+
agent_code_name: evt.agentCodeName,
|
|
3335
|
+
event_type: evt.event,
|
|
3336
|
+
payload: evt.payload,
|
|
3337
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3338
|
+
}).catch((err) => {
|
|
3339
|
+
log(`Failed to forward gateway event: ${err.message}`);
|
|
3340
|
+
});
|
|
3341
|
+
}).catch(() => {
|
|
3342
|
+
});
|
|
3343
|
+
});
|
|
3344
|
+
}
|
|
3345
|
+
function stopGatewayPool() {
|
|
3346
|
+
if (gatewayPool) {
|
|
3347
|
+
gatewayPool.disconnectAll();
|
|
3348
|
+
gatewayPool = null;
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
function startPolling() {
|
|
3352
|
+
if (!config || running) return;
|
|
3353
|
+
running = true;
|
|
3354
|
+
log(`Starting poll loop (interval=${config.intervalMs}ms, configDir=${config.configDir})`);
|
|
3355
|
+
void migrateToProfiles().then(() => {
|
|
3356
|
+
startGatewayPool();
|
|
3357
|
+
return pollCycle();
|
|
3358
|
+
}).then(() => {
|
|
3359
|
+
scheduleNext();
|
|
3360
|
+
});
|
|
3361
|
+
}
|
|
3362
|
+
function scheduleNext() {
|
|
3363
|
+
if (!running || !config) return;
|
|
3364
|
+
pollTimer = setTimeout(() => {
|
|
3365
|
+
void pollCycle().then(scheduleNext);
|
|
3366
|
+
}, config.intervalMs);
|
|
3367
|
+
}
|
|
3368
|
+
async function stopPolling() {
|
|
3369
|
+
running = false;
|
|
3370
|
+
if (pollTimer) {
|
|
3371
|
+
clearTimeout(pollTimer);
|
|
3372
|
+
pollTimer = null;
|
|
3373
|
+
}
|
|
3374
|
+
stopRealtimeChat();
|
|
3375
|
+
stopGatewayPool();
|
|
3376
|
+
await stopAllGateways();
|
|
3377
|
+
}
|
|
3378
|
+
function startManager(opts) {
|
|
3379
|
+
config = opts;
|
|
3380
|
+
startPolling();
|
|
3381
|
+
}
|
|
3382
|
+
async function stopManager() {
|
|
3383
|
+
await stopPolling();
|
|
3384
|
+
}
|
|
3385
|
+
for (const sig of ["SIGTERM", "SIGINT"]) {
|
|
3386
|
+
process.on(sig, () => {
|
|
3387
|
+
log(`Received ${sig}, shutting down`);
|
|
3388
|
+
void stopPolling().then(() => {
|
|
3389
|
+
process.exit(0);
|
|
3390
|
+
});
|
|
3391
|
+
});
|
|
3392
|
+
}
|
|
3393
|
+
process.on("disconnect", () => {
|
|
3394
|
+
log("Parent disconnected, exiting");
|
|
3395
|
+
void stopPolling().then(() => {
|
|
3396
|
+
process.exit(0);
|
|
3397
|
+
});
|
|
3398
|
+
});
|
|
3399
|
+
export {
|
|
3400
|
+
startManager,
|
|
3401
|
+
stopManager
|
|
3402
|
+
};
|
|
3403
|
+
//# sourceMappingURL=manager-worker.js.map
|