assistme 0.6.9 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-4RHN77BN.js +3383 -0
- package/dist/{chunk-E2PAIRTS.js → chunk-QHMIXIWO.js} +6 -0
- package/dist/index.js +1446 -6270
- package/dist/{job-runner-K7R4WZD4.js → job-runner-YM2NBIL3.js} +1 -1
- package/dist/workers/entry.d.ts +1 -0
- package/dist/workers/entry.js +3595 -0
- package/package.json +8 -5
- package/src/agent/proactive-monitor.ts +819 -0
- package/src/agent/processor.ts +16 -0
- package/src/agent/session.ts +30 -0
- package/src/agent/system-prompt.ts +63 -12
- package/src/commands/monitor.ts +200 -0
- package/src/commands/start.ts +25 -13
- package/src/credentials/local-store.ts +18 -0
- package/src/credentials/program-store.ts +182 -0
- package/src/index.ts +2 -0
- package/src/mcp/agent-tools-server.ts +318 -1
- package/src/orchestrator.ts +391 -0
- package/src/utils/constants.ts +14 -0
- package/src/workers/base-handler.ts +83 -0
- package/src/workers/conversation.ts +71 -0
- package/src/workers/entry.ts +41 -0
- package/src/workers/index.ts +9 -0
- package/src/workers/job-analyzer.ts +415 -0
- package/src/workers/manager.ts +492 -0
- package/src/workers/types.ts +53 -0
- package/tests/agent/proactive-monitor.test.ts +930 -0
- package/tests/credentials/program-store.test.ts +166 -0
|
@@ -0,0 +1,3383 @@
|
|
|
1
|
+
import {
|
|
2
|
+
AppError,
|
|
3
|
+
BrowseSkillRowSchema,
|
|
4
|
+
CDP_COMMAND_TIMEOUT_MS,
|
|
5
|
+
FRAME_CONTEXTS_MAX_SIZE,
|
|
6
|
+
MEMORY_COMPRESSION_TARGET,
|
|
7
|
+
MEMORY_COMPRESSION_THRESHOLD,
|
|
8
|
+
MEMORY_DEDUP_SIMILARITY_THRESHOLD,
|
|
9
|
+
SCHEDULER_INTERVAL_MS,
|
|
10
|
+
SHELL_MAX_OUTPUT,
|
|
11
|
+
SHELL_TIMEOUT_MS,
|
|
12
|
+
SKILL_DESCRIPTION_BUDGET_CHARS,
|
|
13
|
+
SkillCreateResultSchema,
|
|
14
|
+
SkillRowSchema,
|
|
15
|
+
WS_CONNECT_TIMEOUT_MS,
|
|
16
|
+
callMcpHandler,
|
|
17
|
+
errorMessage,
|
|
18
|
+
log,
|
|
19
|
+
readAuthStore,
|
|
20
|
+
safeParse,
|
|
21
|
+
writeAuthStore
|
|
22
|
+
} from "./chunk-QHMIXIWO.js";
|
|
23
|
+
import {
|
|
24
|
+
getConfig
|
|
25
|
+
} from "./chunk-YYSJHZSO.js";
|
|
26
|
+
|
|
27
|
+
// src/db/auth.ts
|
|
28
|
+
async function loginWithToken(mcpToken) {
|
|
29
|
+
if (!mcpToken.startsWith("am_")) {
|
|
30
|
+
throw new Error("Invalid token format. Use an am_ token from the web page.");
|
|
31
|
+
}
|
|
32
|
+
const result = await callMcpHandler(
|
|
33
|
+
"auth.validate_token",
|
|
34
|
+
{},
|
|
35
|
+
mcpToken
|
|
36
|
+
);
|
|
37
|
+
const store = readAuthStore();
|
|
38
|
+
store["mcp_token"] = mcpToken;
|
|
39
|
+
writeAuthStore(store);
|
|
40
|
+
return result.user_id;
|
|
41
|
+
}
|
|
42
|
+
async function getCurrentUserId() {
|
|
43
|
+
const result = await callMcpHandler("auth.validate_token");
|
|
44
|
+
return result.user_id;
|
|
45
|
+
}
|
|
46
|
+
async function logout() {
|
|
47
|
+
try {
|
|
48
|
+
writeAuthStore({});
|
|
49
|
+
} catch {
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// src/db/session.ts
|
|
54
|
+
async function createSession(sessionName, workspacePath, version) {
|
|
55
|
+
const { getConfig: getConfig2 } = await import("./config-3RWSAUAZ.js");
|
|
56
|
+
const data = await callMcpHandler("session.create", {
|
|
57
|
+
session_name: sessionName,
|
|
58
|
+
workspace_path: workspacePath,
|
|
59
|
+
version,
|
|
60
|
+
model: getConfig2().model || null
|
|
61
|
+
});
|
|
62
|
+
return data;
|
|
63
|
+
}
|
|
64
|
+
async function updateHeartbeat(sessionId) {
|
|
65
|
+
try {
|
|
66
|
+
await callMcpHandler("session.heartbeat", { session_id: sessionId });
|
|
67
|
+
} catch (err) {
|
|
68
|
+
log.warn(`Heartbeat update failed: ${err instanceof Error ? err.message : err}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
async function endSession(sessionId) {
|
|
72
|
+
try {
|
|
73
|
+
await callMcpHandler("session.end", { session_id: sessionId });
|
|
74
|
+
} catch (err) {
|
|
75
|
+
log.error(`Failed to end session: ${err instanceof Error ? err.message : err}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async function setSessionBusy(sessionId, busy) {
|
|
79
|
+
await callMcpHandler("session.set_busy", {
|
|
80
|
+
session_id: sessionId,
|
|
81
|
+
busy
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
async function cleanupStaleSessions(currentSessionId, thresholdMs = 12e4) {
|
|
85
|
+
try {
|
|
86
|
+
const result = await callMcpHandler("session.cleanup_stale", {
|
|
87
|
+
current_session_id: currentSessionId,
|
|
88
|
+
threshold_ms: thresholdMs
|
|
89
|
+
});
|
|
90
|
+
return result.cleaned;
|
|
91
|
+
} catch {
|
|
92
|
+
return 0;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
async function getActiveSessions(limit = 5) {
|
|
96
|
+
return callMcpHandler("session.get_active", { limit });
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// src/db/task.ts
|
|
100
|
+
async function createTask(conversationId, sessionId, prompt) {
|
|
101
|
+
const data = await callMcpHandler("task.create", {
|
|
102
|
+
conversation_id: conversationId,
|
|
103
|
+
session_id: sessionId,
|
|
104
|
+
prompt
|
|
105
|
+
});
|
|
106
|
+
return { ...data, prompt };
|
|
107
|
+
}
|
|
108
|
+
async function pollAndClaimTask(sessionId) {
|
|
109
|
+
try {
|
|
110
|
+
const data = await callMcpHandler("task.poll_and_claim", {
|
|
111
|
+
session_id: sessionId
|
|
112
|
+
});
|
|
113
|
+
if (!data) return null;
|
|
114
|
+
return {
|
|
115
|
+
...data,
|
|
116
|
+
prompt: data.metadata?.prompt || data.content || ""
|
|
117
|
+
};
|
|
118
|
+
} catch (err) {
|
|
119
|
+
log.warn(`Poll-and-claim failed: ${err instanceof Error ? err.message : err}`);
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
async function claimTask(messageId) {
|
|
124
|
+
const data = await callMcpHandler("task.claim", {
|
|
125
|
+
message_id: messageId
|
|
126
|
+
});
|
|
127
|
+
return data;
|
|
128
|
+
}
|
|
129
|
+
async function completeTask(messageId, resultSummary, tokenUsage) {
|
|
130
|
+
await callMcpHandler("task.complete", {
|
|
131
|
+
message_id: messageId,
|
|
132
|
+
result: resultSummary,
|
|
133
|
+
token_usage: tokenUsage || null
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
async function failTask(messageId, errorMessage2) {
|
|
137
|
+
try {
|
|
138
|
+
await callMcpHandler("task.fail", {
|
|
139
|
+
message_id: messageId,
|
|
140
|
+
error: errorMessage2
|
|
141
|
+
});
|
|
142
|
+
} catch (err) {
|
|
143
|
+
log.error(`Failed to update task status: ${err instanceof Error ? err.message : err}`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// src/db/conversation.ts
|
|
148
|
+
async function getOrCreateCliConversation() {
|
|
149
|
+
const data = await callMcpHandler("conversation.get_or_create");
|
|
150
|
+
return data;
|
|
151
|
+
}
|
|
152
|
+
async function getConversationHistory(conversationId, excludeMessageId, limit = 20) {
|
|
153
|
+
try {
|
|
154
|
+
const rows = await callMcpHandler("conversation.get_history", {
|
|
155
|
+
conversation_id: conversationId,
|
|
156
|
+
exclude_message_id: excludeMessageId,
|
|
157
|
+
limit
|
|
158
|
+
});
|
|
159
|
+
return (rows || []).reverse().map((row) => {
|
|
160
|
+
const prompt = row.metadata?.prompt || "";
|
|
161
|
+
const content = row.content || "";
|
|
162
|
+
const response = row.status === "failed" ? `[Task failed] ${content}` : content;
|
|
163
|
+
return { prompt, response };
|
|
164
|
+
}).filter((entry) => entry.prompt && entry.response);
|
|
165
|
+
} catch (err) {
|
|
166
|
+
log.debug(`Failed to fetch conversation history: ${err instanceof Error ? err.message : err}`);
|
|
167
|
+
return [];
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// src/db/event.ts
|
|
172
|
+
var MAX_EMIT_RETRIES = 2;
|
|
173
|
+
var EMIT_RETRY_DELAY_MS = 500;
|
|
174
|
+
async function emitWithRetry(messageId, eventType, eventData, seq) {
|
|
175
|
+
for (let attempt = 0; attempt <= MAX_EMIT_RETRIES; attempt++) {
|
|
176
|
+
try {
|
|
177
|
+
await callMcpHandler("event.emit", {
|
|
178
|
+
message_id: messageId,
|
|
179
|
+
event_type: eventType,
|
|
180
|
+
event_data: eventData,
|
|
181
|
+
seq
|
|
182
|
+
});
|
|
183
|
+
return;
|
|
184
|
+
} catch (err) {
|
|
185
|
+
if (attempt < MAX_EMIT_RETRIES) {
|
|
186
|
+
await new Promise((r) => setTimeout(r, EMIT_RETRY_DELAY_MS * (attempt + 1)));
|
|
187
|
+
} else {
|
|
188
|
+
log.warn(
|
|
189
|
+
`Failed to emit event after ${MAX_EMIT_RETRIES + 1} attempts: ${err instanceof Error ? err.message : err}`
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
var eventSequence = 0;
|
|
196
|
+
function resetEventSequence() {
|
|
197
|
+
eventSequence = 0;
|
|
198
|
+
}
|
|
199
|
+
async function emitEvent(messageId, eventType, eventData) {
|
|
200
|
+
eventSequence++;
|
|
201
|
+
await emitWithRetry(messageId, eventType, eventData, eventSequence);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// src/db/action.ts
|
|
205
|
+
async function setActionRequest(messageId, actionData) {
|
|
206
|
+
await callMcpHandler("action.set_request", {
|
|
207
|
+
message_id: messageId,
|
|
208
|
+
action_data: actionData
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
async function pollActionResponse(messageId) {
|
|
212
|
+
return callMcpHandler("action.poll_response", {
|
|
213
|
+
message_id: messageId
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// src/db/job-poll.ts
|
|
218
|
+
async function pollAndClaimJobRun() {
|
|
219
|
+
try {
|
|
220
|
+
const data = await callMcpHandler("job.claim_pending_run");
|
|
221
|
+
return data;
|
|
222
|
+
} catch (err) {
|
|
223
|
+
log.debug(`Job run poll failed: ${err instanceof Error ? err.message : err}`);
|
|
224
|
+
return null;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// src/browser/chrome-launcher.ts
|
|
229
|
+
import { execSync, spawn } from "child_process";
|
|
230
|
+
import { platform as platform2, homedir } from "os";
|
|
231
|
+
import { existsSync, unlinkSync, mkdirSync, cpSync } from "fs";
|
|
232
|
+
import { join } from "path";
|
|
233
|
+
|
|
234
|
+
// src/browser/controller.ts
|
|
235
|
+
import { WebSocket } from "ws";
|
|
236
|
+
import { platform } from "os";
|
|
237
|
+
var BrowserController = class {
|
|
238
|
+
ws = null;
|
|
239
|
+
debugPort;
|
|
240
|
+
messageId = 0;
|
|
241
|
+
callbacks = /* @__PURE__ */ new Map();
|
|
242
|
+
connected = false;
|
|
243
|
+
currentTabId = null;
|
|
244
|
+
refCache = /* @__PURE__ */ new Map();
|
|
245
|
+
frameContexts = /* @__PURE__ */ new Map();
|
|
246
|
+
// refId → contextId
|
|
247
|
+
constructor(port = 9222) {
|
|
248
|
+
this.debugPort = port;
|
|
249
|
+
}
|
|
250
|
+
// ── Connection ──────────────────────────────────────────────────
|
|
251
|
+
async isAvailable() {
|
|
252
|
+
try {
|
|
253
|
+
const res = await fetch(`http://127.0.0.1:${this.debugPort}/json/version`, {
|
|
254
|
+
signal: AbortSignal.timeout(2e3)
|
|
255
|
+
});
|
|
256
|
+
return res.ok;
|
|
257
|
+
} catch {
|
|
258
|
+
return false;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
async connect(tabIndex) {
|
|
262
|
+
if (this.connected && this.ws?.readyState === WebSocket.OPEN) {
|
|
263
|
+
if (tabIndex === void 0) {
|
|
264
|
+
return "Already connected to browser.";
|
|
265
|
+
}
|
|
266
|
+
const tabs2 = await this.getTabs();
|
|
267
|
+
const pageTabs2 = tabs2.filter((t) => t.type === "page");
|
|
268
|
+
const targetTab2 = pageTabs2[tabIndex];
|
|
269
|
+
if (targetTab2 && targetTab2.id === this.currentTabId) {
|
|
270
|
+
return `Already connected to tab: "${targetTab2.title}"`;
|
|
271
|
+
}
|
|
272
|
+
await this.disconnect();
|
|
273
|
+
}
|
|
274
|
+
const available = await this.isAvailable();
|
|
275
|
+
if (!available) {
|
|
276
|
+
throw new Error(
|
|
277
|
+
`Cannot connect to browser on port ${this.debugPort}. Chrome remote debugging is not reachable. Please ensure Chrome is running with remote debugging enabled.`
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
const tabs = await this.getTabs();
|
|
281
|
+
const pageTabs = tabs.filter((t) => t.type === "page");
|
|
282
|
+
if (pageTabs.length === 0) {
|
|
283
|
+
throw new Error("No browser tabs found. Please open at least one tab.");
|
|
284
|
+
}
|
|
285
|
+
const targetTab = pageTabs[tabIndex ?? 0];
|
|
286
|
+
if (!targetTab.webSocketDebuggerUrl) {
|
|
287
|
+
throw new Error("Tab does not expose a WebSocket debugger URL.");
|
|
288
|
+
}
|
|
289
|
+
this.currentTabId = targetTab.id;
|
|
290
|
+
return new Promise((resolve, reject) => {
|
|
291
|
+
let settled = false;
|
|
292
|
+
this.ws = new WebSocket(targetTab.webSocketDebuggerUrl);
|
|
293
|
+
const connectTimeout = setTimeout(() => {
|
|
294
|
+
if (!settled) {
|
|
295
|
+
settled = true;
|
|
296
|
+
this.ws?.close();
|
|
297
|
+
reject(new Error(`Connection timeout (${WS_CONNECT_TIMEOUT_MS}ms)`));
|
|
298
|
+
}
|
|
299
|
+
}, WS_CONNECT_TIMEOUT_MS);
|
|
300
|
+
this.ws.on("open", () => {
|
|
301
|
+
if (settled) return;
|
|
302
|
+
settled = true;
|
|
303
|
+
clearTimeout(connectTimeout);
|
|
304
|
+
this.connected = true;
|
|
305
|
+
this.send("Page.enable").catch(() => {
|
|
306
|
+
});
|
|
307
|
+
this.send("Runtime.enable").catch(() => {
|
|
308
|
+
});
|
|
309
|
+
this.send("DOM.enable").catch(() => {
|
|
310
|
+
});
|
|
311
|
+
resolve(`Connected to tab: "${targetTab.title}" (${targetTab.url})`);
|
|
312
|
+
});
|
|
313
|
+
this.ws.on("message", (data) => {
|
|
314
|
+
try {
|
|
315
|
+
const msg = JSON.parse(data.toString());
|
|
316
|
+
if (msg.id !== void 0 && this.callbacks.has(msg.id)) {
|
|
317
|
+
this.callbacks.get(msg.id)(msg);
|
|
318
|
+
this.callbacks.delete(msg.id);
|
|
319
|
+
}
|
|
320
|
+
} catch {
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
this.ws.on("error", (err) => {
|
|
324
|
+
this.connected = false;
|
|
325
|
+
if (!settled) {
|
|
326
|
+
settled = true;
|
|
327
|
+
clearTimeout(connectTimeout);
|
|
328
|
+
reject(new Error(`WebSocket error: ${err.message}`));
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
this.ws.on("close", () => {
|
|
332
|
+
this.connected = false;
|
|
333
|
+
this.ws = null;
|
|
334
|
+
for (const [id, cb] of this.callbacks) {
|
|
335
|
+
cb({ id, error: { code: -1, message: "WebSocket closed" } });
|
|
336
|
+
}
|
|
337
|
+
this.callbacks.clear();
|
|
338
|
+
});
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
async disconnect() {
|
|
342
|
+
if (this.ws) {
|
|
343
|
+
this.ws.close();
|
|
344
|
+
this.ws = null;
|
|
345
|
+
this.connected = false;
|
|
346
|
+
}
|
|
347
|
+
this.refCache.clear();
|
|
348
|
+
this.frameContexts.clear();
|
|
349
|
+
return "Disconnected from browser.";
|
|
350
|
+
}
|
|
351
|
+
// ── CDP Protocol ────────────────────────────────────────────────
|
|
352
|
+
async getTabs() {
|
|
353
|
+
const res = await fetch(`http://127.0.0.1:${this.debugPort}/json`, {
|
|
354
|
+
signal: AbortSignal.timeout(3e3)
|
|
355
|
+
});
|
|
356
|
+
return await res.json();
|
|
357
|
+
}
|
|
358
|
+
send(method, params) {
|
|
359
|
+
return new Promise((resolve, reject) => {
|
|
360
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
361
|
+
reject(new Error("Not connected to browser. Call browser_connect first."));
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
const id = ++this.messageId;
|
|
365
|
+
const timeout = setTimeout(() => {
|
|
366
|
+
this.callbacks.delete(id);
|
|
367
|
+
reject(new Error(`CDP command timed out: ${method}`));
|
|
368
|
+
}, CDP_COMMAND_TIMEOUT_MS);
|
|
369
|
+
this.callbacks.set(id, (response) => {
|
|
370
|
+
clearTimeout(timeout);
|
|
371
|
+
if (response.error) {
|
|
372
|
+
reject(new Error(`CDP error: ${response.error.message}`));
|
|
373
|
+
} else {
|
|
374
|
+
resolve(response.result || {});
|
|
375
|
+
}
|
|
376
|
+
});
|
|
377
|
+
this.ws.send(JSON.stringify({ id, method, params }));
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
ensureConnected() {
|
|
381
|
+
if (!this.connected || !this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
382
|
+
throw new Error("Not connected to browser. Use browser_connect tool first.");
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
// ── Navigation ──────────────────────────────────────────────────
|
|
386
|
+
async navigate(url) {
|
|
387
|
+
this.ensureConnected();
|
|
388
|
+
await this.send("Page.navigate", { url });
|
|
389
|
+
await this.waitForLoad();
|
|
390
|
+
const info = await this.getPageInfo();
|
|
391
|
+
return `Navigated to: ${info.title}
|
|
392
|
+
URL: ${info.url}`;
|
|
393
|
+
}
|
|
394
|
+
async goBack() {
|
|
395
|
+
this.ensureConnected();
|
|
396
|
+
try {
|
|
397
|
+
const history = await this.send("Page.getNavigationHistory");
|
|
398
|
+
const idx = history.currentIndex ?? 0;
|
|
399
|
+
const entries = history.entries ?? [];
|
|
400
|
+
if (idx > 0 && entries[idx - 1]) {
|
|
401
|
+
await this.send("Page.navigateToHistoryEntry", {
|
|
402
|
+
entryId: entries[idx - 1].id
|
|
403
|
+
});
|
|
404
|
+
} else {
|
|
405
|
+
await this.evaluate("window.history.back()");
|
|
406
|
+
}
|
|
407
|
+
} catch {
|
|
408
|
+
await this.evaluate("window.history.back()");
|
|
409
|
+
}
|
|
410
|
+
await this.waitForLoad();
|
|
411
|
+
const info = await this.getPageInfo();
|
|
412
|
+
return `Went back to: ${info.title}`;
|
|
413
|
+
}
|
|
414
|
+
async reload() {
|
|
415
|
+
this.ensureConnected();
|
|
416
|
+
await this.send("Page.reload");
|
|
417
|
+
await this.waitForLoad();
|
|
418
|
+
return "Page reloaded.";
|
|
419
|
+
}
|
|
420
|
+
// ── Page Reading ────────────────────────────────────────────────
|
|
421
|
+
async readPage() {
|
|
422
|
+
this.ensureConnected();
|
|
423
|
+
const result = await this.send("Runtime.evaluate", {
|
|
424
|
+
expression: `
|
|
425
|
+
(function() {
|
|
426
|
+
// Get page title and URL
|
|
427
|
+
let output = "Title: " + document.title + "\\n";
|
|
428
|
+
output += "URL: " + window.location.href + "\\n\\n";
|
|
429
|
+
|
|
430
|
+
// Get main text content, cleaned up
|
|
431
|
+
const body = document.body.cloneNode(true);
|
|
432
|
+
// Remove scripts, styles, navs that add noise
|
|
433
|
+
body.querySelectorAll('script, style, noscript, svg, iframe').forEach(el => el.remove());
|
|
434
|
+
|
|
435
|
+
const text = body.innerText
|
|
436
|
+
.split('\\n')
|
|
437
|
+
.map(line => line.trim())
|
|
438
|
+
.filter(line => line.length > 0)
|
|
439
|
+
.join('\\n');
|
|
440
|
+
|
|
441
|
+
output += text;
|
|
442
|
+
return output.slice(0, 30000);
|
|
443
|
+
})()
|
|
444
|
+
`,
|
|
445
|
+
returnByValue: true
|
|
446
|
+
});
|
|
447
|
+
return result.result?.value || "Could not read page content.";
|
|
448
|
+
}
|
|
449
|
+
async readElement(selector) {
|
|
450
|
+
this.ensureConnected();
|
|
451
|
+
const selectorJS = JSON.stringify(selector);
|
|
452
|
+
const result = await this.send("Runtime.evaluate", {
|
|
453
|
+
expression: `
|
|
454
|
+
(function() {
|
|
455
|
+
const el = document.querySelector(${selectorJS});
|
|
456
|
+
if (!el) return 'Element not found: ' + ${selectorJS};
|
|
457
|
+
return el.innerText || el.textContent || el.value || '(empty)';
|
|
458
|
+
})()
|
|
459
|
+
`,
|
|
460
|
+
returnByValue: true
|
|
461
|
+
});
|
|
462
|
+
return result.result?.value || "Element not found.";
|
|
463
|
+
}
|
|
464
|
+
async getPageInfo() {
|
|
465
|
+
const result = await this.send("Runtime.evaluate", {
|
|
466
|
+
expression: `JSON.stringify({ title: document.title, url: window.location.href })`,
|
|
467
|
+
returnByValue: true
|
|
468
|
+
});
|
|
469
|
+
try {
|
|
470
|
+
return JSON.parse(result.result?.value || "{}");
|
|
471
|
+
} catch {
|
|
472
|
+
return { title: "Unknown", url: "unknown" };
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
// ── Screenshots (for Claude vision) ─────────────────────────────
|
|
476
|
+
async screenshot() {
|
|
477
|
+
this.ensureConnected();
|
|
478
|
+
const result = await this.send("Page.captureScreenshot", {
|
|
479
|
+
format: "png",
|
|
480
|
+
quality: 80,
|
|
481
|
+
captureBeyondViewport: false
|
|
482
|
+
});
|
|
483
|
+
return result.data || "";
|
|
484
|
+
}
|
|
485
|
+
// ── Interactions ────────────────────────────────────────────────
|
|
486
|
+
async click(selector) {
|
|
487
|
+
this.ensureConnected();
|
|
488
|
+
const selectorJS = JSON.stringify(selector);
|
|
489
|
+
const result = await this.send("Runtime.evaluate", {
|
|
490
|
+
expression: `
|
|
491
|
+
(function() {
|
|
492
|
+
var sel = ${selectorJS};
|
|
493
|
+
|
|
494
|
+
// Support :contains('text') pseudo-selector (not native CSS)
|
|
495
|
+
var containsMatch = sel.match(/^(.+?)?:contains\\(['"](.+?)['"]\\)$/);
|
|
496
|
+
if (containsMatch) {
|
|
497
|
+
var baseTag = (containsMatch[1] || '*').toLowerCase();
|
|
498
|
+
var searchText = containsMatch[2];
|
|
499
|
+
var candidates = document.querySelectorAll(baseTag === '*' ? '*' : baseTag);
|
|
500
|
+
var found = null;
|
|
501
|
+
for (var i = 0; i < candidates.length; i++) {
|
|
502
|
+
var c = candidates[i];
|
|
503
|
+
// Prefer exact text match on direct text content (not children)
|
|
504
|
+
var directText = Array.from(c.childNodes)
|
|
505
|
+
.filter(function(n) { return n.nodeType === 3; })
|
|
506
|
+
.map(function(n) { return n.textContent.trim(); })
|
|
507
|
+
.join(' ');
|
|
508
|
+
if (directText === searchText || c.textContent.trim() === searchText) {
|
|
509
|
+
// Prefer the deepest (most specific) matching element
|
|
510
|
+
if (!found || found.contains(c)) found = c;
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
if (!found) return 'Element not found: ' + sel;
|
|
514
|
+
found.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
515
|
+
found.click();
|
|
516
|
+
return 'Clicked: ' + (found.tagName || '') + ' ' + (found.textContent || '').slice(0, 50).trim();
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
var el = document.querySelector(sel);
|
|
520
|
+
if (!el) return 'Element not found: ' + sel;
|
|
521
|
+
|
|
522
|
+
// Scroll into view
|
|
523
|
+
el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
524
|
+
|
|
525
|
+
// Click
|
|
526
|
+
el.click();
|
|
527
|
+
return 'Clicked: ' + (el.tagName || '') + ' ' + (el.textContent || '').slice(0, 50).trim();
|
|
528
|
+
})()
|
|
529
|
+
`,
|
|
530
|
+
returnByValue: true
|
|
531
|
+
});
|
|
532
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
533
|
+
return result.result?.value || "Click executed.";
|
|
534
|
+
}
|
|
535
|
+
async typeText(selector, text) {
|
|
536
|
+
this.ensureConnected();
|
|
537
|
+
const selectorJS = JSON.stringify(selector);
|
|
538
|
+
const textJS = JSON.stringify(text);
|
|
539
|
+
const result = await this.send("Runtime.evaluate", {
|
|
540
|
+
expression: `
|
|
541
|
+
(function() {
|
|
542
|
+
var el = document.querySelector(${selectorJS});
|
|
543
|
+
|
|
544
|
+
// If not found in main document, search same-origin iframes
|
|
545
|
+
if (!el) {
|
|
546
|
+
var iframes = document.querySelectorAll('iframe');
|
|
547
|
+
for (var i = 0; i < iframes.length; i++) {
|
|
548
|
+
try {
|
|
549
|
+
var iframeDoc = iframes[i].contentDocument;
|
|
550
|
+
if (iframeDoc) {
|
|
551
|
+
el = iframeDoc.querySelector(${selectorJS});
|
|
552
|
+
if (el) break;
|
|
553
|
+
}
|
|
554
|
+
} catch(e) { /* cross-origin, skip */ }
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
if (!el) return 'Element not found: ' + ${selectorJS};
|
|
559
|
+
|
|
560
|
+
el.focus();
|
|
561
|
+
|
|
562
|
+
// Check if this is a contenteditable element (rich text editor)
|
|
563
|
+
var isContentEditable = el.isContentEditable ||
|
|
564
|
+
el.getAttribute('contenteditable') === 'true' ||
|
|
565
|
+
el.getAttribute('contenteditable') === '';
|
|
566
|
+
|
|
567
|
+
if (isContentEditable) {
|
|
568
|
+
// For contenteditable: select all content, then replace
|
|
569
|
+
var ownerDoc = el.ownerDocument;
|
|
570
|
+
var sel = ownerDoc.defaultView.getSelection();
|
|
571
|
+
var range = ownerDoc.createRange();
|
|
572
|
+
range.selectNodeContents(el);
|
|
573
|
+
sel.removeAllRanges();
|
|
574
|
+
sel.addRange(range);
|
|
575
|
+
// Use insertText command which respects undo stack and triggers input events
|
|
576
|
+
ownerDoc.execCommand('insertText', false, ${textJS});
|
|
577
|
+
return 'Typed into: ' + (el.tagName || '') + ' [contenteditable]';
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// For input/textarea: clear and set value
|
|
581
|
+
var nativeInputValueSetter = Object.getOwnPropertyDescriptor(
|
|
582
|
+
window.HTMLInputElement.prototype, 'value'
|
|
583
|
+
)?.set || Object.getOwnPropertyDescriptor(
|
|
584
|
+
window.HTMLTextAreaElement.prototype, 'value'
|
|
585
|
+
)?.set;
|
|
586
|
+
if (nativeInputValueSetter) {
|
|
587
|
+
nativeInputValueSetter.call(el, ${textJS});
|
|
588
|
+
} else {
|
|
589
|
+
el.value = ${textJS};
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
// Dispatch events that frameworks (React, Angular, Material) listen to
|
|
593
|
+
el.dispatchEvent(new Event('input', { bubbles: true, cancelable: true }));
|
|
594
|
+
el.dispatchEvent(new Event('change', { bubbles: true, cancelable: true }));
|
|
595
|
+
el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: ${textJS} }));
|
|
596
|
+
return 'Typed into: ' + (el.tagName || '') + ' [' + (el.name || el.id || '') + ']';
|
|
597
|
+
})()
|
|
598
|
+
`,
|
|
599
|
+
returnByValue: true
|
|
600
|
+
});
|
|
601
|
+
const textResult = result.result?.value || "";
|
|
602
|
+
if (textResult.startsWith("Element not found")) {
|
|
603
|
+
return this.typeAtFocus(text);
|
|
604
|
+
}
|
|
605
|
+
return textResult || "Text entered.";
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* Type text into the currently focused element using CDP Input.insertText.
|
|
609
|
+
* This bypasses DOM queries entirely and works with any focused element,
|
|
610
|
+
* including those inside cross-origin iframes or shadow DOM.
|
|
611
|
+
*/
|
|
612
|
+
async typeAtFocus(text) {
|
|
613
|
+
this.ensureConnected();
|
|
614
|
+
const modKey = platform() === "darwin" ? "Meta" : "Control";
|
|
615
|
+
await this.pressKey(`${modKey}+a`);
|
|
616
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
617
|
+
await this.pressKey("Backspace");
|
|
618
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
619
|
+
await this.send("Input.insertText", { text });
|
|
620
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
621
|
+
return "Text entered (into focused element).";
|
|
622
|
+
}
|
|
623
|
+
async pressKey(key) {
|
|
624
|
+
this.ensureConnected();
|
|
625
|
+
const keyMap = {
|
|
626
|
+
Enter: { keyCode: 13, code: "Enter" },
|
|
627
|
+
Tab: { keyCode: 9, code: "Tab" },
|
|
628
|
+
Escape: { keyCode: 27, code: "Escape" },
|
|
629
|
+
Backspace: { keyCode: 8, code: "Backspace" },
|
|
630
|
+
Delete: { keyCode: 46, code: "Delete" },
|
|
631
|
+
ArrowDown: { keyCode: 40, code: "ArrowDown" },
|
|
632
|
+
ArrowUp: { keyCode: 38, code: "ArrowUp" },
|
|
633
|
+
ArrowLeft: { keyCode: 37, code: "ArrowLeft" },
|
|
634
|
+
ArrowRight: { keyCode: 39, code: "ArrowRight" },
|
|
635
|
+
Home: { keyCode: 36, code: "Home" },
|
|
636
|
+
End: { keyCode: 35, code: "End" },
|
|
637
|
+
Space: { keyCode: 32, code: "Space" }
|
|
638
|
+
};
|
|
639
|
+
const modifierMap = {
|
|
640
|
+
Alt: 1,
|
|
641
|
+
Control: 2,
|
|
642
|
+
Meta: 4,
|
|
643
|
+
Shift: 8
|
|
644
|
+
};
|
|
645
|
+
const parts = key.split("+");
|
|
646
|
+
let modifiers = 0;
|
|
647
|
+
let actualKey = parts[parts.length - 1];
|
|
648
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
649
|
+
const mod = modifierMap[parts[i]];
|
|
650
|
+
if (mod) modifiers |= mod;
|
|
651
|
+
}
|
|
652
|
+
const mapped = keyMap[actualKey];
|
|
653
|
+
if (mapped) {
|
|
654
|
+
await this.send("Input.dispatchKeyEvent", {
|
|
655
|
+
type: "keyDown",
|
|
656
|
+
key: actualKey,
|
|
657
|
+
code: mapped.code,
|
|
658
|
+
windowsVirtualKeyCode: mapped.keyCode,
|
|
659
|
+
nativeVirtualKeyCode: mapped.keyCode,
|
|
660
|
+
modifiers
|
|
661
|
+
});
|
|
662
|
+
await this.send("Input.dispatchKeyEvent", {
|
|
663
|
+
type: "keyUp",
|
|
664
|
+
key: actualKey,
|
|
665
|
+
code: mapped.code,
|
|
666
|
+
windowsVirtualKeyCode: mapped.keyCode,
|
|
667
|
+
nativeVirtualKeyCode: mapped.keyCode,
|
|
668
|
+
modifiers
|
|
669
|
+
});
|
|
670
|
+
} else if (actualKey.length === 1) {
|
|
671
|
+
const code = `Key${actualKey.toUpperCase()}`;
|
|
672
|
+
const keyCode = actualKey.toUpperCase().charCodeAt(0);
|
|
673
|
+
await this.send("Input.dispatchKeyEvent", {
|
|
674
|
+
type: "keyDown",
|
|
675
|
+
key: actualKey,
|
|
676
|
+
code,
|
|
677
|
+
windowsVirtualKeyCode: keyCode,
|
|
678
|
+
nativeVirtualKeyCode: keyCode,
|
|
679
|
+
modifiers
|
|
680
|
+
});
|
|
681
|
+
if (!modifiers) {
|
|
682
|
+
await this.send("Input.dispatchKeyEvent", {
|
|
683
|
+
type: "char",
|
|
684
|
+
text: actualKey,
|
|
685
|
+
modifiers
|
|
686
|
+
});
|
|
687
|
+
}
|
|
688
|
+
await this.send("Input.dispatchKeyEvent", {
|
|
689
|
+
type: "keyUp",
|
|
690
|
+
key: actualKey,
|
|
691
|
+
code,
|
|
692
|
+
modifiers
|
|
693
|
+
});
|
|
694
|
+
} else {
|
|
695
|
+
await this.send("Input.dispatchKeyEvent", {
|
|
696
|
+
type: "keyDown",
|
|
697
|
+
key: actualKey,
|
|
698
|
+
modifiers
|
|
699
|
+
});
|
|
700
|
+
await this.send("Input.dispatchKeyEvent", {
|
|
701
|
+
type: "keyUp",
|
|
702
|
+
key: actualKey,
|
|
703
|
+
modifiers
|
|
704
|
+
});
|
|
705
|
+
}
|
|
706
|
+
return `Pressed key: ${key}`;
|
|
707
|
+
}
|
|
708
|
+
async scrollDown() {
|
|
709
|
+
this.ensureConnected();
|
|
710
|
+
await this.send("Runtime.evaluate", {
|
|
711
|
+
expression: "window.scrollBy(0, window.innerHeight * 0.8)"
|
|
712
|
+
});
|
|
713
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
714
|
+
return "Scrolled down.";
|
|
715
|
+
}
|
|
716
|
+
async scrollUp() {
|
|
717
|
+
this.ensureConnected();
|
|
718
|
+
await this.send("Runtime.evaluate", {
|
|
719
|
+
expression: "window.scrollBy(0, -window.innerHeight * 0.8)"
|
|
720
|
+
});
|
|
721
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
722
|
+
return "Scrolled up.";
|
|
723
|
+
}
|
|
724
|
+
// ── Annotated Snapshot (ref system) ─────────────────────────────
|
|
725
|
+
/**
|
|
726
|
+
* Take a snapshot of all interactive elements on the page.
|
|
727
|
+
*
|
|
728
|
+
* Strategy (informed by research — arxiv:2511.19477):
|
|
729
|
+
* - **Text ref table is ALWAYS returned** — compact, low-token, works for
|
|
730
|
+
* all page complexities including dense layouts (date pickers, tables).
|
|
731
|
+
* - **Annotated screenshot is OPTIONAL** (annotate parameter):
|
|
732
|
+
* - true: overlay ref badges on screenshot (best for simple pages with
|
|
733
|
+
* few interactive elements — gives visual context)
|
|
734
|
+
* - false: plain screenshot without overlays (default — avoids label
|
|
735
|
+
* clutter on dense pages; model still sees the page visually)
|
|
736
|
+
* - Research shows text-based grounding outperforms visual annotations
|
|
737
|
+
* on complex pages, and the hybrid approach (a11y text primary +
|
|
738
|
+
* selective vision) achieves ~85% vs ~50% for pure vision.
|
|
739
|
+
*/
|
|
740
|
+
async snapshot(annotate = false) {
|
|
741
|
+
this.ensureConnected();
|
|
742
|
+
await this.waitForLoad(5e3);
|
|
743
|
+
const findResult = await this.send("Runtime.evaluate", {
|
|
744
|
+
expression: `
|
|
745
|
+
(function() {
|
|
746
|
+
// Clean up previous refs
|
|
747
|
+
document.querySelectorAll('[data-assistme-ref]').forEach(function(el) {
|
|
748
|
+
el.removeAttribute('data-assistme-ref');
|
|
749
|
+
});
|
|
750
|
+
|
|
751
|
+
var selectors = [
|
|
752
|
+
'a[href]', 'button', 'input:not([type="hidden"])', 'select', 'textarea',
|
|
753
|
+
'[role="button"]', '[role="link"]', '[role="checkbox"]', '[role="radio"]',
|
|
754
|
+
'[role="combobox"]', '[role="listbox"]', '[role="menuitem"]', '[role="tab"]',
|
|
755
|
+
'[role="switch"]', '[role="slider"]', '[role="option"]', '[role="searchbox"]',
|
|
756
|
+
'[onclick]', '[tabindex]:not([tabindex="-1"])',
|
|
757
|
+
'[contenteditable="true"]'
|
|
758
|
+
].join(', ');
|
|
759
|
+
|
|
760
|
+
// Collect elements from main document AND same-origin iframes
|
|
761
|
+
var all = Array.from(document.querySelectorAll(selectors));
|
|
762
|
+
try {
|
|
763
|
+
var iframes = document.querySelectorAll('iframe');
|
|
764
|
+
for (var fi = 0; fi < iframes.length; fi++) {
|
|
765
|
+
try {
|
|
766
|
+
var iframeDoc = iframes[fi].contentDocument;
|
|
767
|
+
if (iframeDoc) {
|
|
768
|
+
var iframeRect = iframes[fi].getBoundingClientRect();
|
|
769
|
+
var iframeEls = iframeDoc.querySelectorAll(selectors);
|
|
770
|
+
for (var fe = 0; fe < iframeEls.length; fe++) {
|
|
771
|
+
// Tag iframe elements with offset for coordinate correction
|
|
772
|
+
iframeEls[fe].__iframeOffset = { x: iframeRect.x, y: iframeRect.y };
|
|
773
|
+
all.push(iframeEls[fe]);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
} catch(e) { /* cross-origin iframe, skip */ }
|
|
777
|
+
}
|
|
778
|
+
} catch(e) { /* iframe enumeration failed, continue */ }
|
|
779
|
+
|
|
780
|
+
var refs = [];
|
|
781
|
+
var vh = window.innerHeight;
|
|
782
|
+
var vw = window.innerWidth;
|
|
783
|
+
|
|
784
|
+
for (var i = 0; i < all.length && refs.length < 80; i++) {
|
|
785
|
+
var el = all[i];
|
|
786
|
+
var rect = el.getBoundingClientRect();
|
|
787
|
+
|
|
788
|
+
// Skip invisible / tiny elements
|
|
789
|
+
if (rect.width < 5 || rect.height < 5) continue;
|
|
790
|
+
var style = window.getComputedStyle(el);
|
|
791
|
+
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') continue;
|
|
792
|
+
|
|
793
|
+
// Skip elements far outside viewport
|
|
794
|
+
if (rect.bottom < -50 || rect.top > vh + 50) continue;
|
|
795
|
+
if (rect.right < -50 || rect.left > vw + 50) continue;
|
|
796
|
+
|
|
797
|
+
// Determine role
|
|
798
|
+
var role = el.getAttribute('role') || '';
|
|
799
|
+
if (!role) {
|
|
800
|
+
var tag = el.tagName.toLowerCase();
|
|
801
|
+
if (tag === 'a') role = 'link';
|
|
802
|
+
else if (tag === 'button') role = 'button';
|
|
803
|
+
else if (tag === 'input') {
|
|
804
|
+
var t = (el.type || 'text').toLowerCase();
|
|
805
|
+
if (t === 'checkbox') role = 'checkbox';
|
|
806
|
+
else if (t === 'radio') role = 'radio';
|
|
807
|
+
else if (t === 'submit' || t === 'button') role = 'button';
|
|
808
|
+
else role = 'textbox';
|
|
809
|
+
}
|
|
810
|
+
else if (tag === 'select') role = 'combobox';
|
|
811
|
+
else if (tag === 'textarea') role = 'textbox';
|
|
812
|
+
else role = tag;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
// Determine accessible name
|
|
816
|
+
var name = '';
|
|
817
|
+
var ariaLabel = el.getAttribute('aria-label');
|
|
818
|
+
var ariaLabelledBy = el.getAttribute('aria-labelledby');
|
|
819
|
+
if (ariaLabel) {
|
|
820
|
+
name = ariaLabel;
|
|
821
|
+
} else if (ariaLabelledBy) {
|
|
822
|
+
var labelEl = document.getElementById(ariaLabelledBy);
|
|
823
|
+
if (labelEl) name = labelEl.textContent.trim();
|
|
824
|
+
} else if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
|
|
825
|
+
if (el.id) {
|
|
826
|
+
var lbl = document.querySelector('label[for="' + CSS.escape(el.id) + '"]');
|
|
827
|
+
if (lbl) name = lbl.textContent.trim();
|
|
828
|
+
}
|
|
829
|
+
if (!name) name = el.getAttribute('placeholder') || el.getAttribute('name') || '';
|
|
830
|
+
} else {
|
|
831
|
+
name = (el.textContent || '').trim().slice(0, 60);
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
var refId = refs.length + 1;
|
|
835
|
+
el.setAttribute('data-assistme-ref', String(refId));
|
|
836
|
+
|
|
837
|
+
// Correct coordinates for elements inside iframes
|
|
838
|
+
var offsetX = el.__iframeOffset ? el.__iframeOffset.x : 0;
|
|
839
|
+
var offsetY = el.__iframeOffset ? el.__iframeOffset.y : 0;
|
|
840
|
+
|
|
841
|
+
refs.push({
|
|
842
|
+
id: refId,
|
|
843
|
+
role: role,
|
|
844
|
+
name: name,
|
|
845
|
+
tag: el.tagName.toLowerCase(),
|
|
846
|
+
type: el.getAttribute('type') || '',
|
|
847
|
+
box: {
|
|
848
|
+
x: Math.round(rect.x + offsetX),
|
|
849
|
+
y: Math.round(rect.y + offsetY),
|
|
850
|
+
width: Math.round(rect.width),
|
|
851
|
+
height: Math.round(rect.height)
|
|
852
|
+
}
|
|
853
|
+
});
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
return JSON.stringify(refs);
|
|
857
|
+
})()
|
|
858
|
+
`,
|
|
859
|
+
returnByValue: true
|
|
860
|
+
});
|
|
861
|
+
const refs = JSON.parse(
|
|
862
|
+
findResult.result?.value || "[]"
|
|
863
|
+
).map((r) => ({
|
|
864
|
+
id: r.id,
|
|
865
|
+
role: r.role,
|
|
866
|
+
name: r.name,
|
|
867
|
+
tag: r.tag,
|
|
868
|
+
inputType: r.type || "",
|
|
869
|
+
box: r.box
|
|
870
|
+
}));
|
|
871
|
+
await this.discoverCrossOriginFrameRefs(refs);
|
|
872
|
+
if (annotate && refs.length <= 40) {
|
|
873
|
+
const refsJson = JSON.stringify(refs);
|
|
874
|
+
await this.send("Runtime.evaluate", {
|
|
875
|
+
expression: `
|
|
876
|
+
(function() {
|
|
877
|
+
var old = document.getElementById('__assistme_refs__');
|
|
878
|
+
if (old) old.remove();
|
|
879
|
+
|
|
880
|
+
var overlay = document.createElement('div');
|
|
881
|
+
overlay.id = '__assistme_refs__';
|
|
882
|
+
overlay.style.cssText = 'position:fixed;top:0;left:0;width:100%;height:100%;pointer-events:none;z-index:2147483647;';
|
|
883
|
+
|
|
884
|
+
var refs = ${refsJson};
|
|
885
|
+
var vh = window.innerHeight;
|
|
886
|
+
var vw = window.innerWidth;
|
|
887
|
+
|
|
888
|
+
for (var i = 0; i < refs.length; i++) {
|
|
889
|
+
var b = refs[i].box;
|
|
890
|
+
if (b.y + b.height < 0 || b.y > vh || b.x + b.width < 0 || b.x > vw) continue;
|
|
891
|
+
|
|
892
|
+
// Red badge with ref number
|
|
893
|
+
var badge = document.createElement('div');
|
|
894
|
+
var badgeTop = Math.max(0, b.y - 14);
|
|
895
|
+
var badgeLeft = Math.max(0, b.x);
|
|
896
|
+
badge.style.cssText = 'position:fixed;background:#e8384f;color:#fff;font:bold 10px/1.2 monospace;padding:1px 3px;border-radius:2px;white-space:nowrap;'
|
|
897
|
+
+ 'left:' + badgeLeft + 'px;top:' + badgeTop + 'px;';
|
|
898
|
+
badge.textContent = String(refs[i].id);
|
|
899
|
+
overlay.appendChild(badge);
|
|
900
|
+
|
|
901
|
+
// Border around element
|
|
902
|
+
var border = document.createElement('div');
|
|
903
|
+
border.style.cssText = 'position:fixed;border:1.5px solid #e8384f;border-radius:2px;'
|
|
904
|
+
+ 'left:' + b.x + 'px;top:' + b.y + 'px;width:' + b.width + 'px;height:' + b.height + 'px;';
|
|
905
|
+
overlay.appendChild(border);
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
document.documentElement.appendChild(overlay);
|
|
909
|
+
})()
|
|
910
|
+
`
|
|
911
|
+
});
|
|
912
|
+
}
|
|
913
|
+
const image = await this.screenshot();
|
|
914
|
+
if (annotate) {
|
|
915
|
+
await this.send("Runtime.evaluate", {
|
|
916
|
+
expression: `(function() { var el = document.getElementById('__assistme_refs__'); if (el) el.remove(); })()`
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
this.refCache.clear();
|
|
920
|
+
for (const ref of refs) {
|
|
921
|
+
this.refCache.set(ref.id, ref);
|
|
922
|
+
}
|
|
923
|
+
const pageInfo = await this.getPageInfo();
|
|
924
|
+
return { image, refs, url: pageInfo.url, title: pageInfo.title };
|
|
925
|
+
}
|
|
926
|
+
/**
|
|
927
|
+
* Build a compact text table of refs for the model.
|
|
928
|
+
*/
|
|
929
|
+
static formatRefTable(result) {
|
|
930
|
+
let table = `Page: ${result.title}
|
|
931
|
+
URL: ${result.url}
|
|
932
|
+
|
|
933
|
+
Refs:
|
|
934
|
+
`;
|
|
935
|
+
for (const ref of result.refs) {
|
|
936
|
+
const extra = ref.inputType ? ` (${ref.inputType})` : "";
|
|
937
|
+
const nameStr = ref.name ? ` "${ref.name}"` : "";
|
|
938
|
+
table += `[${ref.id}] ${ref.role}${nameStr}${extra}
|
|
939
|
+
`;
|
|
940
|
+
}
|
|
941
|
+
if (result.refs.length === 0) {
|
|
942
|
+
table += "(no interactive elements found)\n";
|
|
943
|
+
}
|
|
944
|
+
return table;
|
|
945
|
+
}
|
|
946
|
+
// ── Cross-Origin Iframe Discovery ────────────────────────────────
|
|
947
|
+
/**
|
|
948
|
+
* Use CDP's Page.getFrameTree + Runtime.evaluate with contextId to discover
|
|
949
|
+
* interactive elements inside cross-origin iframes (e.g., ProtonMail editor,
|
|
950
|
+
* Google Docs, embedded rich text editors).
|
|
951
|
+
*
|
|
952
|
+
* Same-origin iframes are already handled inline by the main snapshot JS.
|
|
953
|
+
* This method handles the ones that threw cross-origin errors.
|
|
954
|
+
*/
|
|
955
|
+
async discoverCrossOriginFrameRefs(refs) {
|
|
956
|
+
this.frameContexts.clear();
|
|
957
|
+
try {
|
|
958
|
+
const frameTree = await this.send("Page.getFrameTree");
|
|
959
|
+
const mainFrameId = frameTree.frameTree?.frame?.id;
|
|
960
|
+
const childFrames = frameTree.frameTree?.childFrames || [];
|
|
961
|
+
if (childFrames.length === 0) return;
|
|
962
|
+
const contexts = await this.getFrameContexts(mainFrameId || "");
|
|
963
|
+
for (const child of childFrames) {
|
|
964
|
+
const frameId = child.frame.id;
|
|
965
|
+
const contextId = contexts.get(frameId);
|
|
966
|
+
if (!contextId) continue;
|
|
967
|
+
const iframeOffsetResult = await this.send("Runtime.evaluate", {
|
|
968
|
+
expression: `
|
|
969
|
+
(function() {
|
|
970
|
+
var iframes = document.querySelectorAll('iframe');
|
|
971
|
+
for (var i = 0; i < iframes.length; i++) {
|
|
972
|
+
try {
|
|
973
|
+
// Match by frame src or name
|
|
974
|
+
var f = iframes[i];
|
|
975
|
+
if (f.contentWindow) {
|
|
976
|
+
var r = f.getBoundingClientRect();
|
|
977
|
+
if (r.width > 10 && r.height > 10) {
|
|
978
|
+
return JSON.stringify({ x: r.x, y: r.y, width: r.width, height: r.height, index: i });
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
} catch(e) {}
|
|
982
|
+
}
|
|
983
|
+
return 'null';
|
|
984
|
+
})()
|
|
985
|
+
`,
|
|
986
|
+
returnByValue: true
|
|
987
|
+
});
|
|
988
|
+
let iframeOffset = { x: 0, y: 0 };
|
|
989
|
+
try {
|
|
990
|
+
const parsed = JSON.parse(
|
|
991
|
+
iframeOffsetResult.result?.value || "null"
|
|
992
|
+
);
|
|
993
|
+
if (parsed) iframeOffset = { x: parsed.x, y: parsed.y };
|
|
994
|
+
} catch {
|
|
995
|
+
}
|
|
996
|
+
const startRefId = refs.length + 1;
|
|
997
|
+
try {
|
|
998
|
+
const frameResult = await this.send("Runtime.evaluate", {
|
|
999
|
+
expression: `
|
|
1000
|
+
(function() {
|
|
1001
|
+
var selectors = [
|
|
1002
|
+
'a[href]', 'button', 'input:not([type="hidden"])', 'select', 'textarea',
|
|
1003
|
+
'[role="button"]', '[role="link"]', '[role="checkbox"]', '[role="radio"]',
|
|
1004
|
+
'[role="combobox"]', '[role="listbox"]', '[role="menuitem"]', '[role="tab"]',
|
|
1005
|
+
'[role="switch"]', '[role="slider"]', '[role="option"]', '[role="searchbox"]',
|
|
1006
|
+
'[onclick]', '[tabindex]:not([tabindex="-1"])',
|
|
1007
|
+
'[contenteditable="true"]', '[contenteditable=""]'
|
|
1008
|
+
].join(', ');
|
|
1009
|
+
|
|
1010
|
+
var all = document.querySelectorAll(selectors);
|
|
1011
|
+
// Also check if the body itself is contenteditable
|
|
1012
|
+
if (document.body && (document.body.isContentEditable || document.body.getAttribute('contenteditable') === 'true')) {
|
|
1013
|
+
all = [document.body].concat(Array.from(all));
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
var refs = [];
|
|
1017
|
+
var startId = ${startRefId};
|
|
1018
|
+
var vh = window.innerHeight;
|
|
1019
|
+
var vw = window.innerWidth;
|
|
1020
|
+
|
|
1021
|
+
for (var i = 0; i < all.length && refs.length < 20; i++) {
|
|
1022
|
+
var el = all[i];
|
|
1023
|
+
var rect = el.getBoundingClientRect();
|
|
1024
|
+
if (rect.width < 5 || rect.height < 5) continue;
|
|
1025
|
+
var style = window.getComputedStyle(el);
|
|
1026
|
+
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') continue;
|
|
1027
|
+
|
|
1028
|
+
var role = el.getAttribute('role') || '';
|
|
1029
|
+
if (!role) {
|
|
1030
|
+
var tag = el.tagName.toLowerCase();
|
|
1031
|
+
if (tag === 'a') role = 'link';
|
|
1032
|
+
else if (tag === 'button') role = 'button';
|
|
1033
|
+
else if (tag === 'input') {
|
|
1034
|
+
var t = (el.type || 'text').toLowerCase();
|
|
1035
|
+
if (t === 'checkbox') role = 'checkbox';
|
|
1036
|
+
else if (t === 'radio') role = 'radio';
|
|
1037
|
+
else if (t === 'submit' || t === 'button') role = 'button';
|
|
1038
|
+
else role = 'textbox';
|
|
1039
|
+
}
|
|
1040
|
+
else if (tag === 'select') role = 'combobox';
|
|
1041
|
+
else if (tag === 'textarea') role = 'textbox';
|
|
1042
|
+
else if (el.isContentEditable) role = 'textbox';
|
|
1043
|
+
else role = tag;
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
var name = '';
|
|
1047
|
+
var ariaLabel = el.getAttribute('aria-label');
|
|
1048
|
+
if (ariaLabel) {
|
|
1049
|
+
name = ariaLabel;
|
|
1050
|
+
} else if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
|
|
1051
|
+
name = el.getAttribute('placeholder') || el.getAttribute('name') || '';
|
|
1052
|
+
} else if (el.isContentEditable) {
|
|
1053
|
+
name = 'compose body';
|
|
1054
|
+
} else {
|
|
1055
|
+
name = (el.textContent || '').trim().slice(0, 60);
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
var refId = startId + refs.length;
|
|
1059
|
+
el.setAttribute('data-assistme-ref', String(refId));
|
|
1060
|
+
|
|
1061
|
+
refs.push({
|
|
1062
|
+
id: refId,
|
|
1063
|
+
role: role,
|
|
1064
|
+
name: name,
|
|
1065
|
+
tag: el.tagName.toLowerCase(),
|
|
1066
|
+
type: el.getAttribute('type') || '',
|
|
1067
|
+
box: {
|
|
1068
|
+
x: Math.round(rect.x),
|
|
1069
|
+
y: Math.round(rect.y),
|
|
1070
|
+
width: Math.round(rect.width),
|
|
1071
|
+
height: Math.round(rect.height)
|
|
1072
|
+
},
|
|
1073
|
+
inFrame: true
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
return JSON.stringify(refs);
|
|
1078
|
+
})()
|
|
1079
|
+
`,
|
|
1080
|
+
contextId,
|
|
1081
|
+
returnByValue: true
|
|
1082
|
+
});
|
|
1083
|
+
const frameRefs = JSON.parse(
|
|
1084
|
+
frameResult.result?.value || "[]"
|
|
1085
|
+
);
|
|
1086
|
+
for (const r of frameRefs) {
|
|
1087
|
+
refs.push({
|
|
1088
|
+
id: r.id,
|
|
1089
|
+
role: r.role,
|
|
1090
|
+
name: r.name,
|
|
1091
|
+
tag: r.tag,
|
|
1092
|
+
inputType: r.type || "",
|
|
1093
|
+
box: {
|
|
1094
|
+
x: Math.round(r.box.x + iframeOffset.x),
|
|
1095
|
+
y: Math.round(r.box.y + iframeOffset.y),
|
|
1096
|
+
width: r.box.width,
|
|
1097
|
+
height: r.box.height
|
|
1098
|
+
}
|
|
1099
|
+
});
|
|
1100
|
+
if (this.frameContexts.size < FRAME_CONTEXTS_MAX_SIZE) {
|
|
1101
|
+
this.frameContexts.set(r.id, contextId);
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
} catch {
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
} catch {
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
/**
|
|
1111
|
+
* Get execution context IDs for each frame in the page.
|
|
1112
|
+
* Uses Runtime.executionContextCreated events collected during the session,
|
|
1113
|
+
* or falls back to evaluating in known frames.
|
|
1114
|
+
*/
|
|
1115
|
+
async getFrameContexts(_mainFrameId) {
|
|
1116
|
+
const contexts = /* @__PURE__ */ new Map();
|
|
1117
|
+
try {
|
|
1118
|
+
await this.send("Runtime.enable").catch(() => {
|
|
1119
|
+
});
|
|
1120
|
+
const frameTree = await this.send("Page.getFrameTree");
|
|
1121
|
+
const childFrames = frameTree.frameTree?.childFrames || [];
|
|
1122
|
+
for (const child of childFrames) {
|
|
1123
|
+
try {
|
|
1124
|
+
const world = await this.send("Page.createIsolatedWorld", {
|
|
1125
|
+
frameId: child.frame.id,
|
|
1126
|
+
worldName: "assistme-snapshot",
|
|
1127
|
+
grantUniveralAccess: true
|
|
1128
|
+
});
|
|
1129
|
+
if (world.executionContextId) {
|
|
1130
|
+
contexts.set(child.frame.id, world.executionContextId);
|
|
1131
|
+
}
|
|
1132
|
+
} catch {
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
} catch {
|
|
1136
|
+
}
|
|
1137
|
+
return contexts;
|
|
1138
|
+
}
|
|
1139
|
+
// ── Ref Resolution ────────────────────────────────────────────────
|
|
1140
|
+
/**
|
|
1141
|
+
* Resolve a ref ID to its current center coordinates in the viewport.
|
|
1142
|
+
* Uses two strategies:
|
|
1143
|
+
* 1. Fast: find by data-assistme-ref attribute (set during snapshot)
|
|
1144
|
+
* 2. Stable: search by role + accessible name (survives DOM changes)
|
|
1145
|
+
*
|
|
1146
|
+
* Includes actionability checks (like Playwright):
|
|
1147
|
+
* - Element must be visible (not display:none, not zero-size)
|
|
1148
|
+
* - Element must be in viewport (scrolls into view if needed)
|
|
1149
|
+
* - Element must not be covered by another element (checks elementFromPoint)
|
|
1150
|
+
*
|
|
1151
|
+
* Returns null if the element cannot be found or is not actionable.
|
|
1152
|
+
* Returns { error: string } if found but not actionable (for diagnostics).
|
|
1153
|
+
*/
|
|
1154
|
+
async resolveRef(refId) {
|
|
1155
|
+
const cached = this.refCache.get(refId);
|
|
1156
|
+
const role = cached?.role || "";
|
|
1157
|
+
const name = cached?.name || "";
|
|
1158
|
+
const roleJS = JSON.stringify(role);
|
|
1159
|
+
const nameJS = JSON.stringify(name);
|
|
1160
|
+
const result = await this.send("Runtime.evaluate", {
|
|
1161
|
+
expression: `
|
|
1162
|
+
(function() {
|
|
1163
|
+
var refId = ${refId};
|
|
1164
|
+
var role = ${roleJS};
|
|
1165
|
+
var name = ${nameJS};
|
|
1166
|
+
|
|
1167
|
+
// Strategy 1: data attribute (fast, from last snapshot)
|
|
1168
|
+
var el = document.querySelector('[data-assistme-ref="' + refId + '"]');
|
|
1169
|
+
|
|
1170
|
+
// Strategy 2: role + name search (stable, survives DOM changes)
|
|
1171
|
+
if (!el && role && name) {
|
|
1172
|
+
var selectorMap = {
|
|
1173
|
+
textbox: 'input, textarea, [role="textbox"], [role="searchbox"]',
|
|
1174
|
+
button: 'button, [role="button"], input[type="submit"], input[type="button"]',
|
|
1175
|
+
link: 'a[href], [role="link"]',
|
|
1176
|
+
combobox: 'select, [role="combobox"]',
|
|
1177
|
+
checkbox: 'input[type="checkbox"], [role="checkbox"]',
|
|
1178
|
+
radio: 'input[type="radio"], [role="radio"]',
|
|
1179
|
+
tab: '[role="tab"]',
|
|
1180
|
+
menuitem: '[role="menuitem"]',
|
|
1181
|
+
option: '[role="option"], option',
|
|
1182
|
+
};
|
|
1183
|
+
var sel = selectorMap[role] || '*[role="' + role + '"]';
|
|
1184
|
+
var candidates = document.querySelectorAll(sel);
|
|
1185
|
+
for (var i = 0; i < candidates.length; i++) {
|
|
1186
|
+
var c = candidates[i];
|
|
1187
|
+
var cName = c.getAttribute('aria-label')
|
|
1188
|
+
|| c.getAttribute('placeholder')
|
|
1189
|
+
|| (c.textContent || '').trim().slice(0, 60);
|
|
1190
|
+
if (cName === name) { el = c; break; }
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
if (!el) return 'null';
|
|
1195
|
+
|
|
1196
|
+
// \u2500\u2500 Actionability checks (Playwright-style) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
1197
|
+
|
|
1198
|
+
// Check visibility
|
|
1199
|
+
var style = window.getComputedStyle(el);
|
|
1200
|
+
if (style.display === 'none')
|
|
1201
|
+
return JSON.stringify({ error: 'Element is hidden (display:none)' });
|
|
1202
|
+
if (style.visibility === 'hidden')
|
|
1203
|
+
return JSON.stringify({ error: 'Element is hidden (visibility:hidden)' });
|
|
1204
|
+
if (parseFloat(style.opacity) < 0.05)
|
|
1205
|
+
return JSON.stringify({ error: 'Element is hidden (opacity:0)' });
|
|
1206
|
+
|
|
1207
|
+
// Check disabled
|
|
1208
|
+
if (el.disabled || el.getAttribute('aria-disabled') === 'true')
|
|
1209
|
+
return JSON.stringify({ error: 'Element is disabled' });
|
|
1210
|
+
|
|
1211
|
+
// Scroll into view
|
|
1212
|
+
el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
1213
|
+
var r = el.getBoundingClientRect();
|
|
1214
|
+
|
|
1215
|
+
// Check non-zero size
|
|
1216
|
+
if (r.width < 1 || r.height < 1)
|
|
1217
|
+
return JSON.stringify({ error: 'Element has zero size (' + r.width + 'x' + r.height + ')' });
|
|
1218
|
+
|
|
1219
|
+
// Check element is in viewport
|
|
1220
|
+
if (r.bottom < 0 || r.top > window.innerHeight || r.right < 0 || r.left > window.innerWidth)
|
|
1221
|
+
return JSON.stringify({ error: 'Element is outside viewport after scroll' });
|
|
1222
|
+
|
|
1223
|
+
var cx = r.x + r.width / 2;
|
|
1224
|
+
var cy = r.y + r.height / 2;
|
|
1225
|
+
|
|
1226
|
+
// Check not covered by another element (hit test)
|
|
1227
|
+
var topEl = document.elementFromPoint(cx, cy);
|
|
1228
|
+
if (topEl && topEl !== el && !el.contains(topEl) && !topEl.closest('[data-assistme-ref="' + refId + '"]')) {
|
|
1229
|
+
// Check if the covering element is the overlay (ignore it)
|
|
1230
|
+
if (!topEl.closest('#__assistme_refs__')) {
|
|
1231
|
+
var coverTag = topEl.tagName.toLowerCase();
|
|
1232
|
+
var coverText = (topEl.textContent || '').trim().slice(0, 30);
|
|
1233
|
+
return JSON.stringify({
|
|
1234
|
+
error: 'Element is covered by <' + coverTag + '>' + (coverText ? ' "' + coverText + '"' : ''),
|
|
1235
|
+
x: cx, y: cy, width: r.width, height: r.height
|
|
1236
|
+
});
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
return JSON.stringify({
|
|
1241
|
+
x: cx,
|
|
1242
|
+
y: cy,
|
|
1243
|
+
width: r.width,
|
|
1244
|
+
height: r.height
|
|
1245
|
+
});
|
|
1246
|
+
})()
|
|
1247
|
+
`,
|
|
1248
|
+
returnByValue: true
|
|
1249
|
+
});
|
|
1250
|
+
const value = result.result?.value;
|
|
1251
|
+
if (value && value !== "null") {
|
|
1252
|
+
try {
|
|
1253
|
+
return JSON.parse(value);
|
|
1254
|
+
} catch {
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
const frameContextId = this.frameContexts.get(refId);
|
|
1258
|
+
if (frameContextId) {
|
|
1259
|
+
return this.resolveRefInFrame(refId, frameContextId, role, name);
|
|
1260
|
+
}
|
|
1261
|
+
return null;
|
|
1262
|
+
}
|
|
1263
|
+
/**
|
|
1264
|
+
* Resolve a ref inside a cross-origin iframe using its execution context.
|
|
1265
|
+
* Returns coordinates adjusted by the iframe's viewport offset.
|
|
1266
|
+
*/
|
|
1267
|
+
async resolveRefInFrame(refId, contextId, role, name) {
|
|
1268
|
+
const roleJS = JSON.stringify(role);
|
|
1269
|
+
const nameJS = JSON.stringify(name);
|
|
1270
|
+
try {
|
|
1271
|
+
const offsetResult = await this.send("Runtime.evaluate", {
|
|
1272
|
+
expression: `
|
|
1273
|
+
(function() {
|
|
1274
|
+
var iframes = document.querySelectorAll('iframe');
|
|
1275
|
+
for (var i = 0; i < iframes.length; i++) {
|
|
1276
|
+
var r = iframes[i].getBoundingClientRect();
|
|
1277
|
+
if (r.width > 10 && r.height > 10) {
|
|
1278
|
+
return JSON.stringify({ x: r.x, y: r.y });
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
return JSON.stringify({ x: 0, y: 0 });
|
|
1282
|
+
})()
|
|
1283
|
+
`,
|
|
1284
|
+
returnByValue: true
|
|
1285
|
+
});
|
|
1286
|
+
const offset = JSON.parse(
|
|
1287
|
+
offsetResult.result?.value || '{"x":0,"y":0}'
|
|
1288
|
+
);
|
|
1289
|
+
const frameResult = await this.send("Runtime.evaluate", {
|
|
1290
|
+
expression: `
|
|
1291
|
+
(function() {
|
|
1292
|
+
var el = document.querySelector('[data-assistme-ref="${refId}"]');
|
|
1293
|
+
if (!el && ${roleJS} && ${nameJS}) {
|
|
1294
|
+
// Fallback: search by role
|
|
1295
|
+
var candidates = document.querySelectorAll('*');
|
|
1296
|
+
for (var i = 0; i < candidates.length; i++) {
|
|
1297
|
+
var c = candidates[i];
|
|
1298
|
+
if (c.isContentEditable || c.getAttribute('contenteditable') === 'true') {
|
|
1299
|
+
el = c; break;
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
if (!el) return 'null';
|
|
1304
|
+
|
|
1305
|
+
el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
1306
|
+
var r = el.getBoundingClientRect();
|
|
1307
|
+
if (r.width < 1 || r.height < 1) return JSON.stringify({ error: 'Zero size' });
|
|
1308
|
+
|
|
1309
|
+
return JSON.stringify({
|
|
1310
|
+
x: r.x + r.width / 2,
|
|
1311
|
+
y: r.y + r.height / 2,
|
|
1312
|
+
width: r.width,
|
|
1313
|
+
height: r.height
|
|
1314
|
+
});
|
|
1315
|
+
})()
|
|
1316
|
+
`,
|
|
1317
|
+
contextId,
|
|
1318
|
+
returnByValue: true
|
|
1319
|
+
});
|
|
1320
|
+
const value = frameResult.result?.value;
|
|
1321
|
+
if (!value || value === "null") return null;
|
|
1322
|
+
const parsed = JSON.parse(value);
|
|
1323
|
+
if (parsed.error) return parsed;
|
|
1324
|
+
return {
|
|
1325
|
+
x: parsed.x + offset.x,
|
|
1326
|
+
y: parsed.y + offset.y,
|
|
1327
|
+
width: parsed.width,
|
|
1328
|
+
height: parsed.height
|
|
1329
|
+
};
|
|
1330
|
+
} catch {
|
|
1331
|
+
return null;
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
// ── Ref-based Interactions (CDP Input Events) ─────────────────────
|
|
1335
|
+
/**
|
|
1336
|
+
* Click an element by ref using CDP Input.dispatchMouseEvent.
|
|
1337
|
+
* This simulates a real mouse click through the browser's input pipeline,
|
|
1338
|
+
* triggering hover states, focus management, and all native browser events
|
|
1339
|
+
* — more reliable than el.click() for framework components.
|
|
1340
|
+
*
|
|
1341
|
+
* Includes auto-wait: retries up to 3 times (with 500ms intervals) if the
|
|
1342
|
+
* element is not yet actionable (e.g., covered by a loading overlay, still
|
|
1343
|
+
* animating into view). This matches Playwright's auto-waiting behavior.
|
|
1344
|
+
*/
|
|
1345
|
+
async clickRef(refId) {
|
|
1346
|
+
this.ensureConnected();
|
|
1347
|
+
const ref = this.refCache.get(refId);
|
|
1348
|
+
const refLabel = `[${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
|
|
1349
|
+
const maxRetries = 3;
|
|
1350
|
+
let lastError = "";
|
|
1351
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
1352
|
+
const resolved = await this.resolveRef(refId);
|
|
1353
|
+
if (!resolved) {
|
|
1354
|
+
return {
|
|
1355
|
+
success: false,
|
|
1356
|
+
message: `Ref ${refLabel} not found. Take a new snapshot with browser_snapshot.`
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1359
|
+
if (resolved.error) {
|
|
1360
|
+
lastError = resolved.error;
|
|
1361
|
+
if (attempt < maxRetries - 1) {
|
|
1362
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1363
|
+
continue;
|
|
1364
|
+
}
|
|
1365
|
+
return { success: false, message: `Cannot click ${refLabel}: ${lastError}` };
|
|
1366
|
+
}
|
|
1367
|
+
if (attempt === 0) {
|
|
1368
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
1369
|
+
const settled = await this.resolveRef(refId);
|
|
1370
|
+
if (settled && !settled.error) {
|
|
1371
|
+
resolved.x = settled.x;
|
|
1372
|
+
resolved.y = settled.y;
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
await this.send("Input.dispatchMouseEvent", {
|
|
1376
|
+
type: "mouseMoved",
|
|
1377
|
+
x: resolved.x,
|
|
1378
|
+
y: resolved.y
|
|
1379
|
+
});
|
|
1380
|
+
await this.send("Input.dispatchMouseEvent", {
|
|
1381
|
+
type: "mousePressed",
|
|
1382
|
+
x: resolved.x,
|
|
1383
|
+
y: resolved.y,
|
|
1384
|
+
button: "left",
|
|
1385
|
+
clickCount: 1
|
|
1386
|
+
});
|
|
1387
|
+
await this.send("Input.dispatchMouseEvent", {
|
|
1388
|
+
type: "mouseReleased",
|
|
1389
|
+
x: resolved.x,
|
|
1390
|
+
y: resolved.y,
|
|
1391
|
+
button: "left",
|
|
1392
|
+
clickCount: 1
|
|
1393
|
+
});
|
|
1394
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
1395
|
+
return { success: true, message: `Clicked ${refLabel}` };
|
|
1396
|
+
}
|
|
1397
|
+
return { success: false, message: `Cannot click ${refLabel}: ${lastError}` };
|
|
1398
|
+
}
|
|
1399
|
+
/**
|
|
1400
|
+
* Type text into an element by ref using CDP Input events.
|
|
1401
|
+
* Clicks to focus, selects all existing text (Ctrl/Cmd+A), then uses
|
|
1402
|
+
* Input.insertText for reliable text insertion across all frameworks.
|
|
1403
|
+
*/
|
|
1404
|
+
async typeRef(refId, text) {
|
|
1405
|
+
this.ensureConnected();
|
|
1406
|
+
const ref = this.refCache.get(refId);
|
|
1407
|
+
const refLabel = `[${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
|
|
1408
|
+
const clickResult = await this.clickRef(refId);
|
|
1409
|
+
if (!clickResult.success) return clickResult;
|
|
1410
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
1411
|
+
const selectAllKey = platform() === "darwin" ? "Meta+a" : "Control+a";
|
|
1412
|
+
await this.pressKey(selectAllKey);
|
|
1413
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
1414
|
+
await this.pressKey("Backspace");
|
|
1415
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
1416
|
+
const frameContextId = this.frameContexts.get(refId);
|
|
1417
|
+
const clearEvalOpts = {
|
|
1418
|
+
expression: `
|
|
1419
|
+
(function() {
|
|
1420
|
+
var el = document.querySelector('[data-assistme-ref="${refId}"]');
|
|
1421
|
+
if (!el) return 'no_element';
|
|
1422
|
+
|
|
1423
|
+
// For contenteditable elements, check textContent instead of value
|
|
1424
|
+
if (el.isContentEditable || el.getAttribute('contenteditable') === 'true') {
|
|
1425
|
+
if (el.textContent && el.textContent.trim() !== '') {
|
|
1426
|
+
el.textContent = '';
|
|
1427
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
1428
|
+
return 'js_cleared';
|
|
1429
|
+
}
|
|
1430
|
+
return 'ok';
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
if (el.value !== undefined && el.value !== '') {
|
|
1434
|
+
// Ctrl+A didn't work (some frameworks intercept it) \u2014 clear via JS
|
|
1435
|
+
var setter = Object.getOwnPropertyDescriptor(
|
|
1436
|
+
window.HTMLInputElement.prototype, 'value'
|
|
1437
|
+
)?.set || Object.getOwnPropertyDescriptor(
|
|
1438
|
+
window.HTMLTextAreaElement.prototype, 'value'
|
|
1439
|
+
)?.set;
|
|
1440
|
+
if (setter) setter.call(el, '');
|
|
1441
|
+
else el.value = '';
|
|
1442
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
1443
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
1444
|
+
return 'js_cleared';
|
|
1445
|
+
}
|
|
1446
|
+
return 'ok';
|
|
1447
|
+
})()
|
|
1448
|
+
`,
|
|
1449
|
+
returnByValue: true
|
|
1450
|
+
};
|
|
1451
|
+
if (frameContextId) {
|
|
1452
|
+
clearEvalOpts.contextId = frameContextId;
|
|
1453
|
+
}
|
|
1454
|
+
const cleared = await this.send("Runtime.evaluate", clearEvalOpts);
|
|
1455
|
+
const clearStatus = cleared.result?.value || "ok";
|
|
1456
|
+
if (clearStatus === "no_element" && !frameContextId) {
|
|
1457
|
+
return {
|
|
1458
|
+
success: false,
|
|
1459
|
+
message: `Ref ${refLabel} not found after click. Take a new snapshot.`
|
|
1460
|
+
};
|
|
1461
|
+
}
|
|
1462
|
+
await this.send("Input.insertText", { text });
|
|
1463
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
1464
|
+
return { success: true, message: `Typed "${text}" into ${refLabel}` };
|
|
1465
|
+
}
|
|
1466
|
+
/**
|
|
1467
|
+
* Select a dropdown option by ref. Delegates to selectOption with the
|
|
1468
|
+
* ref's data attribute as selector, handling both native <select> and
|
|
1469
|
+
* custom dropdown components.
|
|
1470
|
+
*/
|
|
1471
|
+
async selectRef(refId, option) {
|
|
1472
|
+
this.ensureConnected();
|
|
1473
|
+
const cached = this.refCache.get(refId);
|
|
1474
|
+
if (!cached) {
|
|
1475
|
+
return {
|
|
1476
|
+
success: false,
|
|
1477
|
+
message: `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`
|
|
1478
|
+
};
|
|
1479
|
+
}
|
|
1480
|
+
const refLabel = `[${refId}] ${cached.role} "${cached.name}"`;
|
|
1481
|
+
const result = await this.selectOption(`[data-assistme-ref="${refId}"]`, option);
|
|
1482
|
+
const message = result.replace(/\[data-assistme-ref="\d+"\]/, refLabel);
|
|
1483
|
+
const success = !result.includes("not found");
|
|
1484
|
+
return { success, message };
|
|
1485
|
+
}
|
|
1486
|
+
// ── Action Pipeline ───────────────────────────────────────────────
|
|
1487
|
+
/**
|
|
1488
|
+
* Execute a batch of actions sequentially using refs.
|
|
1489
|
+
* Reduces round-trips: instead of one tool call per action, the model
|
|
1490
|
+
* can specify a sequence of actions that execute atomically.
|
|
1491
|
+
*
|
|
1492
|
+
* Optionally takes a screenshot after all actions complete.
|
|
1493
|
+
*/
|
|
1494
|
+
async act(actions, takeScreenshot = false) {
|
|
1495
|
+
this.ensureConnected();
|
|
1496
|
+
const results = [];
|
|
1497
|
+
for (const spec of actions) {
|
|
1498
|
+
let result;
|
|
1499
|
+
let success = true;
|
|
1500
|
+
try {
|
|
1501
|
+
switch (spec.action) {
|
|
1502
|
+
case "click": {
|
|
1503
|
+
const r = await this.clickRef(spec.ref);
|
|
1504
|
+
result = r.message;
|
|
1505
|
+
success = r.success;
|
|
1506
|
+
break;
|
|
1507
|
+
}
|
|
1508
|
+
case "type": {
|
|
1509
|
+
const r = await this.typeRef(spec.ref, spec.text);
|
|
1510
|
+
result = r.message;
|
|
1511
|
+
success = r.success;
|
|
1512
|
+
break;
|
|
1513
|
+
}
|
|
1514
|
+
case "select": {
|
|
1515
|
+
const r = await this.selectRef(spec.ref, spec.option);
|
|
1516
|
+
result = r.message;
|
|
1517
|
+
success = r.success;
|
|
1518
|
+
break;
|
|
1519
|
+
}
|
|
1520
|
+
case "press":
|
|
1521
|
+
result = await this.pressKey(spec.key);
|
|
1522
|
+
break;
|
|
1523
|
+
case "scroll":
|
|
1524
|
+
result = spec.direction === "up" ? await this.scrollUp() : await this.scrollDown();
|
|
1525
|
+
break;
|
|
1526
|
+
case "wait":
|
|
1527
|
+
await new Promise((r) => setTimeout(r, Math.min(spec.ms, 5e3)));
|
|
1528
|
+
result = `Waited ${spec.ms}ms`;
|
|
1529
|
+
break;
|
|
1530
|
+
default:
|
|
1531
|
+
result = `Unknown action: ${spec.action}`;
|
|
1532
|
+
success = false;
|
|
1533
|
+
}
|
|
1534
|
+
} catch (err) {
|
|
1535
|
+
result = `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
1536
|
+
success = false;
|
|
1537
|
+
}
|
|
1538
|
+
results.push({
|
|
1539
|
+
action: spec.action,
|
|
1540
|
+
ref: "ref" in spec ? spec.ref : void 0,
|
|
1541
|
+
result,
|
|
1542
|
+
success
|
|
1543
|
+
});
|
|
1544
|
+
if (!success) break;
|
|
1545
|
+
if (spec.action !== "wait") {
|
|
1546
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
let screenshot;
|
|
1550
|
+
if (takeScreenshot) {
|
|
1551
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
1552
|
+
screenshot = await this.screenshot();
|
|
1553
|
+
}
|
|
1554
|
+
return { results, screenshot };
|
|
1555
|
+
}
|
|
1556
|
+
// ── Dropdown/Select ─────────────────────────────────────────────
|
|
1557
|
+
/**
|
|
1558
|
+
* Select an option from a dropdown — handles both native <select> elements
|
|
1559
|
+
* and custom Material Design / React / Angular dropdown components.
|
|
1560
|
+
*
|
|
1561
|
+
* Strategy:
|
|
1562
|
+
* 1. Try native <select> first (by selector or label text)
|
|
1563
|
+
* 2. Fall back to custom dropdown: click to open, then click the option by text
|
|
1564
|
+
*/
|
|
1565
|
+
async selectOption(selector, optionText) {
|
|
1566
|
+
this.ensureConnected();
|
|
1567
|
+
const selectorJS = JSON.stringify(selector);
|
|
1568
|
+
const optionJS = JSON.stringify(optionText);
|
|
1569
|
+
const result = await this.send("Runtime.evaluate", {
|
|
1570
|
+
expression: `
|
|
1571
|
+
(function() {
|
|
1572
|
+
var sel = ${selectorJS};
|
|
1573
|
+
var optText = ${optionJS};
|
|
1574
|
+
|
|
1575
|
+
// Strategy 1: Native <select> element
|
|
1576
|
+
var selectEl = document.querySelector(sel);
|
|
1577
|
+
if (selectEl && selectEl.tagName === 'SELECT') {
|
|
1578
|
+
var options = selectEl.querySelectorAll('option');
|
|
1579
|
+
for (var i = 0; i < options.length; i++) {
|
|
1580
|
+
if (options[i].textContent.trim() === optText) {
|
|
1581
|
+
selectEl.value = options[i].value;
|
|
1582
|
+
selectEl.dispatchEvent(new Event('change', { bubbles: true }));
|
|
1583
|
+
selectEl.dispatchEvent(new Event('input', { bubbles: true }));
|
|
1584
|
+
return 'Selected "' + optText + '" in native select';
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
return 'Option "' + optText + '" not found in select. Available: ' +
|
|
1588
|
+
Array.from(options).map(function(o) { return o.textContent.trim(); }).join(', ');
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
// Strategy 2: Custom dropdown \u2014 find the trigger element
|
|
1592
|
+
var trigger = selectEl;
|
|
1593
|
+
if (!trigger) {
|
|
1594
|
+
// Try finding by aria-label first (fast, indexed)
|
|
1595
|
+
trigger = document.querySelector('[aria-label="' + sel.replace(/"/g, '\\"') + '"]');
|
|
1596
|
+
}
|
|
1597
|
+
if (!trigger) {
|
|
1598
|
+
// Try finding by label/placeholder text in likely dropdown elements
|
|
1599
|
+
var dropdownCandidates = document.querySelectorAll(
|
|
1600
|
+
'button, [role="combobox"], [role="listbox"], [role="button"], ' +
|
|
1601
|
+
'select, input, .MuiSelect-root, .MuiInput-root, ' +
|
|
1602
|
+
'[class*="select"], [class*="dropdown"], [class*="picker"]'
|
|
1603
|
+
);
|
|
1604
|
+
for (var j = 0; j < dropdownCandidates.length; j++) {
|
|
1605
|
+
var el = dropdownCandidates[j];
|
|
1606
|
+
var ownText = Array.from(el.childNodes)
|
|
1607
|
+
.filter(function(n) { return n.nodeType === 3; })
|
|
1608
|
+
.map(function(n) { return n.textContent.trim(); })
|
|
1609
|
+
.join('');
|
|
1610
|
+
if (ownText === sel || el.getAttribute('aria-label') === sel ||
|
|
1611
|
+
el.getAttribute('placeholder') === sel) {
|
|
1612
|
+
trigger = el;
|
|
1613
|
+
break;
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
if (!trigger) return 'Dropdown not found: ' + sel;
|
|
1619
|
+
|
|
1620
|
+
// Click to open the dropdown
|
|
1621
|
+
trigger.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
1622
|
+
trigger.click();
|
|
1623
|
+
|
|
1624
|
+
// Wait a frame for the dropdown menu to render, then select the option
|
|
1625
|
+
return new Promise(function(resolve) {
|
|
1626
|
+
setTimeout(function() {
|
|
1627
|
+
// Look for the option in listbox/menu/dropdown overlays
|
|
1628
|
+
var optionContainers = document.querySelectorAll(
|
|
1629
|
+
'[role="listbox"], [role="menu"], [role="presentation"], .MuiMenu-list, .MuiList-root, ul.mdc-list, .VfPpkd-xl07Ob'
|
|
1630
|
+
);
|
|
1631
|
+
|
|
1632
|
+
// Also check all visible elements as fallback
|
|
1633
|
+
var searchIn = optionContainers.length > 0
|
|
1634
|
+
? Array.from(optionContainers).flatMap(function(c) { return Array.from(c.querySelectorAll('*')); })
|
|
1635
|
+
: Array.from(document.querySelectorAll('li, [role="option"], [role="menuitem"], div[data-value]'));
|
|
1636
|
+
|
|
1637
|
+
for (var k = 0; k < searchIn.length; k++) {
|
|
1638
|
+
var opt = searchIn[k];
|
|
1639
|
+
var txt = opt.textContent ? opt.textContent.trim() : '';
|
|
1640
|
+
if (txt === optText) {
|
|
1641
|
+
opt.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
1642
|
+
opt.click();
|
|
1643
|
+
resolve('Selected "' + optText + '" from custom dropdown');
|
|
1644
|
+
return;
|
|
1645
|
+
}
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
// Broader search: visible leaf elements in interactive containers
|
|
1649
|
+
var broadCandidates = document.querySelectorAll(
|
|
1650
|
+
'li, span, div, a, button, label, [role="option"], [role="menuitem"], ' +
|
|
1651
|
+
'[role="menuitemradio"], [role="menuitemcheckbox"], [data-value]'
|
|
1652
|
+
);
|
|
1653
|
+
for (var m = 0; m < broadCandidates.length; m++) {
|
|
1654
|
+
var candidate = broadCandidates[m];
|
|
1655
|
+
if (candidate.textContent && candidate.textContent.trim() === optText &&
|
|
1656
|
+
candidate.offsetParent !== null && candidate.children.length === 0) {
|
|
1657
|
+
candidate.click();
|
|
1658
|
+
resolve('Selected "' + optText + '" (broad match)');
|
|
1659
|
+
return;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
resolve('Option "' + optText + '" not found in dropdown');
|
|
1664
|
+
}, 300);
|
|
1665
|
+
});
|
|
1666
|
+
})()
|
|
1667
|
+
`,
|
|
1668
|
+
returnByValue: true,
|
|
1669
|
+
awaitPromise: true
|
|
1670
|
+
});
|
|
1671
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1672
|
+
return result.result?.value || "Selection attempted.";
|
|
1673
|
+
}
|
|
1674
|
+
// ── JavaScript Evaluation ───────────────────────────────────────
|
|
1675
|
+
async evaluate(expression) {
|
|
1676
|
+
this.ensureConnected();
|
|
1677
|
+
const result = await this.send("Runtime.evaluate", {
|
|
1678
|
+
expression,
|
|
1679
|
+
returnByValue: true,
|
|
1680
|
+
awaitPromise: true
|
|
1681
|
+
});
|
|
1682
|
+
const evalResult = result.result;
|
|
1683
|
+
const value = evalResult?.value;
|
|
1684
|
+
if (value === void 0) {
|
|
1685
|
+
const desc = evalResult?.description;
|
|
1686
|
+
return desc || "(undefined)";
|
|
1687
|
+
}
|
|
1688
|
+
return typeof value === "string" ? value : JSON.stringify(value, null, 2);
|
|
1689
|
+
}
|
|
1690
|
+
// ── Tab Management ──────────────────────────────────────────────
|
|
1691
|
+
async listTabs() {
|
|
1692
|
+
const tabs = await this.getTabs();
|
|
1693
|
+
const pageTabs = tabs.filter((t) => t.type === "page");
|
|
1694
|
+
if (pageTabs.length === 0) return "No tabs open.";
|
|
1695
|
+
return pageTabs.map(
|
|
1696
|
+
(t, i) => `[${i}] ${t.title.slice(0, 60)}${this.currentTabId === t.id ? " (active)" : ""}
|
|
1697
|
+
${t.url}`
|
|
1698
|
+
).join("\n\n");
|
|
1699
|
+
}
|
|
1700
|
+
async switchTab(index) {
|
|
1701
|
+
const tabs = await this.getTabs();
|
|
1702
|
+
const pageTabs = tabs.filter((t) => t.type === "page");
|
|
1703
|
+
if (index < 0 || index >= pageTabs.length) {
|
|
1704
|
+
return `Invalid tab index. Available: 0-${pageTabs.length - 1}`;
|
|
1705
|
+
}
|
|
1706
|
+
await this.disconnect();
|
|
1707
|
+
return this.connect(index);
|
|
1708
|
+
}
|
|
1709
|
+
async openNewTab(url) {
|
|
1710
|
+
const targetUrl = url || "about:blank";
|
|
1711
|
+
const res = await fetch(
|
|
1712
|
+
`http://127.0.0.1:${this.debugPort}/json/new?${encodeURIComponent(targetUrl)}`,
|
|
1713
|
+
{ signal: AbortSignal.timeout(5e3) }
|
|
1714
|
+
);
|
|
1715
|
+
const tab = await res.json();
|
|
1716
|
+
await this.disconnect();
|
|
1717
|
+
const tabs = await this.getTabs();
|
|
1718
|
+
const idx = tabs.filter((t) => t.type === "page").findIndex((t) => t.id === tab.id);
|
|
1719
|
+
if (idx >= 0) {
|
|
1720
|
+
await this.connect(idx);
|
|
1721
|
+
}
|
|
1722
|
+
return `Opened new tab: ${targetUrl}`;
|
|
1723
|
+
}
|
|
1724
|
+
// ── Helpers ─────────────────────────────────────────────────────
|
|
1725
|
+
async waitForLoad(timeoutMs = 8e3) {
|
|
1726
|
+
const start = Date.now();
|
|
1727
|
+
let sawInteractive = false;
|
|
1728
|
+
while (Date.now() - start < timeoutMs) {
|
|
1729
|
+
try {
|
|
1730
|
+
const result = await this.send("Runtime.evaluate", {
|
|
1731
|
+
expression: "document.readyState",
|
|
1732
|
+
returnByValue: true
|
|
1733
|
+
});
|
|
1734
|
+
const state = result.result?.value;
|
|
1735
|
+
if (state === "complete") {
|
|
1736
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
1737
|
+
return;
|
|
1738
|
+
}
|
|
1739
|
+
if (state === "interactive") {
|
|
1740
|
+
if (!sawInteractive) {
|
|
1741
|
+
sawInteractive = true;
|
|
1742
|
+
}
|
|
1743
|
+
}
|
|
1744
|
+
} catch {
|
|
1745
|
+
}
|
|
1746
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
1747
|
+
}
|
|
1748
|
+
if (sawInteractive) {
|
|
1749
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
isConnected() {
|
|
1753
|
+
return this.connected && this.ws?.readyState === WebSocket.OPEN;
|
|
1754
|
+
}
|
|
1755
|
+
// ── Login Detection ────────────────────────────────────────────
|
|
1756
|
+
/**
|
|
1757
|
+
* Detect if the current page appears to be a login/authentication page.
|
|
1758
|
+
* Checks URL patterns, password input fields, and login form actions.
|
|
1759
|
+
*/
|
|
1760
|
+
async detectLoginPage() {
|
|
1761
|
+
try {
|
|
1762
|
+
const result = await this.send("Runtime.evaluate", {
|
|
1763
|
+
expression: `
|
|
1764
|
+
(function() {
|
|
1765
|
+
var url = window.location.href.toLowerCase();
|
|
1766
|
+
|
|
1767
|
+
// Exclude signup/registration pages \u2014 these are NOT login pages
|
|
1768
|
+
var signupPatterns = [
|
|
1769
|
+
'/signup', '/sign-up', '/sign_up', '/register',
|
|
1770
|
+
'/registration', '/create-account', '/create_account',
|
|
1771
|
+
'/join', '/enroll',
|
|
1772
|
+
'accounts.google.com/lifecycle/steps/signup',
|
|
1773
|
+
'signup.live.com',
|
|
1774
|
+
];
|
|
1775
|
+
for (var s = 0; s < signupPatterns.length; s++) {
|
|
1776
|
+
if (url.indexOf(signupPatterns[s]) !== -1) {
|
|
1777
|
+
return JSON.stringify({ isLoginPage: false, reason: '' });
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
// URL-based detection
|
|
1782
|
+
var loginPatterns = [
|
|
1783
|
+
'/login', '/signin', '/sign-in', '/sign_in',
|
|
1784
|
+
'/auth/', '/sso/', '/oauth/', '/session/new',
|
|
1785
|
+
'/accounts/login', '/users/sign_in',
|
|
1786
|
+
'accounts.google.com/v3/signin',
|
|
1787
|
+
'accounts.google.com/servicelogin',
|
|
1788
|
+
'login.microsoftonline.com',
|
|
1789
|
+
'github.com/login', 'github.com/session',
|
|
1790
|
+
'login.live.com', 'appleid.apple.com'
|
|
1791
|
+
];
|
|
1792
|
+
for (var i = 0; i < loginPatterns.length; i++) {
|
|
1793
|
+
if (url.indexOf(loginPatterns[i]) !== -1) {
|
|
1794
|
+
return JSON.stringify({
|
|
1795
|
+
isLoginPage: true,
|
|
1796
|
+
reason: 'URL contains login pattern: ' + loginPatterns[i]
|
|
1797
|
+
});
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
// Password input detection (visible only)
|
|
1802
|
+
var passwordInputs = document.querySelectorAll('input[type="password"]');
|
|
1803
|
+
for (var j = 0; j < passwordInputs.length; j++) {
|
|
1804
|
+
var input = passwordInputs[j];
|
|
1805
|
+
var rect = input.getBoundingClientRect();
|
|
1806
|
+
var style = window.getComputedStyle(input);
|
|
1807
|
+
if (rect.width > 0 && rect.height > 0 &&
|
|
1808
|
+
style.display !== 'none' && style.visibility !== 'hidden') {
|
|
1809
|
+
return JSON.stringify({
|
|
1810
|
+
isLoginPage: true,
|
|
1811
|
+
reason: 'Page contains visible password input field'
|
|
1812
|
+
});
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
// Login form action detection
|
|
1817
|
+
var formSelectors = [
|
|
1818
|
+
'form[action*="login"]', 'form[action*="signin"]',
|
|
1819
|
+
'form[action*="session"]', 'form[action*="auth"]',
|
|
1820
|
+
'form[action*="authenticate"]'
|
|
1821
|
+
];
|
|
1822
|
+
var loginForms = document.querySelectorAll(formSelectors.join(','));
|
|
1823
|
+
if (loginForms.length > 0) {
|
|
1824
|
+
return JSON.stringify({
|
|
1825
|
+
isLoginPage: true,
|
|
1826
|
+
reason: 'Page contains login form'
|
|
1827
|
+
});
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
return JSON.stringify({ isLoginPage: false, reason: '' });
|
|
1831
|
+
})()
|
|
1832
|
+
`,
|
|
1833
|
+
returnByValue: true
|
|
1834
|
+
});
|
|
1835
|
+
const value = result.result?.value;
|
|
1836
|
+
return JSON.parse(value || '{"isLoginPage":false,"reason":""}');
|
|
1837
|
+
} catch {
|
|
1838
|
+
return { isLoginPage: false, reason: "" };
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
};
|
|
1842
|
+
|
|
1843
|
+
// src/browser/chrome-launcher.ts
|
|
1844
|
+
function findChromePath() {
|
|
1845
|
+
const os = platform2();
|
|
1846
|
+
if (os === "darwin") {
|
|
1847
|
+
const paths = [
|
|
1848
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
1849
|
+
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
|
1850
|
+
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
|
1851
|
+
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
|
1852
|
+
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"
|
|
1853
|
+
];
|
|
1854
|
+
return paths.find((p) => existsSync(p)) ?? null;
|
|
1855
|
+
}
|
|
1856
|
+
if (os === "linux") {
|
|
1857
|
+
const names = [
|
|
1858
|
+
"google-chrome",
|
|
1859
|
+
"google-chrome-stable",
|
|
1860
|
+
"chromium-browser",
|
|
1861
|
+
"chromium",
|
|
1862
|
+
"microsoft-edge",
|
|
1863
|
+
"microsoft-edge-stable",
|
|
1864
|
+
"brave-browser"
|
|
1865
|
+
];
|
|
1866
|
+
for (const name of names) {
|
|
1867
|
+
try {
|
|
1868
|
+
return execSync(`which ${name}`, {
|
|
1869
|
+
encoding: "utf-8",
|
|
1870
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
1871
|
+
}).trim();
|
|
1872
|
+
} catch {
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
return null;
|
|
1876
|
+
}
|
|
1877
|
+
if (os === "win32") {
|
|
1878
|
+
const prefixes = [
|
|
1879
|
+
process.env.PROGRAMFILES,
|
|
1880
|
+
process.env["PROGRAMFILES(X86)"],
|
|
1881
|
+
process.env.LOCALAPPDATA
|
|
1882
|
+
].filter(Boolean);
|
|
1883
|
+
const subPaths = [
|
|
1884
|
+
"Google\\Chrome\\Application\\chrome.exe",
|
|
1885
|
+
"Microsoft\\Edge\\Application\\msedge.exe",
|
|
1886
|
+
"BraveSoftware\\Brave-Browser\\Application\\brave.exe"
|
|
1887
|
+
];
|
|
1888
|
+
for (const prefix of prefixes) {
|
|
1889
|
+
for (const sub of subPaths) {
|
|
1890
|
+
const p = `${prefix}\\${sub}`;
|
|
1891
|
+
if (existsSync(p)) return p;
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
return null;
|
|
1895
|
+
}
|
|
1896
|
+
return null;
|
|
1897
|
+
}
|
|
1898
|
+
function getDefaultProfileDir(chromePath) {
|
|
1899
|
+
const home = homedir();
|
|
1900
|
+
const os = platform2();
|
|
1901
|
+
if (os === "darwin") {
|
|
1902
|
+
if (chromePath.includes("Brave Browser"))
|
|
1903
|
+
return join(home, "Library", "Application Support", "BraveSoftware", "Brave-Browser");
|
|
1904
|
+
if (chromePath.includes("Microsoft Edge"))
|
|
1905
|
+
return join(home, "Library", "Application Support", "Microsoft Edge");
|
|
1906
|
+
if (chromePath.includes("Chromium"))
|
|
1907
|
+
return join(home, "Library", "Application Support", "Chromium");
|
|
1908
|
+
if (chromePath.includes("Canary"))
|
|
1909
|
+
return join(home, "Library", "Application Support", "Google", "Chrome Canary");
|
|
1910
|
+
return join(home, "Library", "Application Support", "Google", "Chrome");
|
|
1911
|
+
}
|
|
1912
|
+
if (os === "win32") {
|
|
1913
|
+
const appData = process.env.LOCALAPPDATA || join(home, "AppData", "Local");
|
|
1914
|
+
if (chromePath.includes("brave"))
|
|
1915
|
+
return join(appData, "BraveSoftware", "Brave-Browser", "User Data");
|
|
1916
|
+
if (chromePath.includes("msedge")) return join(appData, "Microsoft", "Edge", "User Data");
|
|
1917
|
+
return join(appData, "Google", "Chrome", "User Data");
|
|
1918
|
+
}
|
|
1919
|
+
if (chromePath.includes("brave")) return join(home, ".config", "BraveSoftware", "Brave-Browser");
|
|
1920
|
+
if (chromePath.includes("microsoft-edge")) return join(home, ".config", "microsoft-edge");
|
|
1921
|
+
if (chromePath.includes("chromium")) return join(home, ".config", "chromium");
|
|
1922
|
+
return join(home, ".config", "google-chrome");
|
|
1923
|
+
}
|
|
1924
|
+
function getDebugProfileDir(chromePath) {
|
|
1925
|
+
const home = homedir();
|
|
1926
|
+
const debugDir = join(home, ".assistme", "browser-profile");
|
|
1927
|
+
if (!existsSync(debugDir)) {
|
|
1928
|
+
mkdirSync(debugDir, { recursive: true });
|
|
1929
|
+
log.debug(`Created debug profile directory: ${debugDir}`);
|
|
1930
|
+
const realDir = getDefaultProfileDir(chromePath);
|
|
1931
|
+
if (existsSync(realDir)) {
|
|
1932
|
+
seedDebugProfile(realDir, debugDir);
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
return debugDir;
|
|
1936
|
+
}
|
|
1937
|
+
function seedDebugProfile(realDir, debugDir) {
|
|
1938
|
+
const rootFiles = ["Local State"];
|
|
1939
|
+
const profileFiles = ["Bookmarks", "Preferences", "Favicons", "Top Sites", "Shortcuts"];
|
|
1940
|
+
for (const file of rootFiles) {
|
|
1941
|
+
const src = join(realDir, file);
|
|
1942
|
+
const dest = join(debugDir, file);
|
|
1943
|
+
try {
|
|
1944
|
+
if (existsSync(src)) {
|
|
1945
|
+
cpSync(src, dest, { force: true });
|
|
1946
|
+
log.debug(`Seeded: ${file}`);
|
|
1947
|
+
}
|
|
1948
|
+
} catch {
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1951
|
+
const srcProfile = join(realDir, "Default");
|
|
1952
|
+
const destProfile = join(debugDir, "Default");
|
|
1953
|
+
if (existsSync(srcProfile)) {
|
|
1954
|
+
mkdirSync(destProfile, { recursive: true });
|
|
1955
|
+
for (const file of profileFiles) {
|
|
1956
|
+
const src = join(srcProfile, file);
|
|
1957
|
+
const dest = join(destProfile, file);
|
|
1958
|
+
try {
|
|
1959
|
+
if (existsSync(src)) {
|
|
1960
|
+
cpSync(src, dest, { force: true });
|
|
1961
|
+
log.debug(`Seeded: Default/${file}`);
|
|
1962
|
+
}
|
|
1963
|
+
} catch {
|
|
1964
|
+
}
|
|
1965
|
+
}
|
|
1966
|
+
const srcExt = join(srcProfile, "Extensions");
|
|
1967
|
+
const destExt = join(destProfile, "Extensions");
|
|
1968
|
+
try {
|
|
1969
|
+
if (existsSync(srcExt)) {
|
|
1970
|
+
cpSync(srcExt, destExt, { recursive: true, force: true });
|
|
1971
|
+
log.debug("Seeded: Default/Extensions");
|
|
1972
|
+
}
|
|
1973
|
+
} catch {
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1977
|
+
function spawnChrome(chromePath, port) {
|
|
1978
|
+
const profileDir = getDebugProfileDir(chromePath);
|
|
1979
|
+
const flags = [
|
|
1980
|
+
`--remote-debugging-port=${port}`,
|
|
1981
|
+
`--user-data-dir=${profileDir}`,
|
|
1982
|
+
"--restore-last-session"
|
|
1983
|
+
];
|
|
1984
|
+
log.debug(`Spawning browser: ${chromePath} ${flags.join(" ")}`);
|
|
1985
|
+
const child = spawn(chromePath, flags, {
|
|
1986
|
+
detached: true,
|
|
1987
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1988
|
+
});
|
|
1989
|
+
let stderr = "";
|
|
1990
|
+
child.stderr?.on("data", (chunk) => {
|
|
1991
|
+
stderr += chunk.toString();
|
|
1992
|
+
});
|
|
1993
|
+
child.on("error", (err) => {
|
|
1994
|
+
log.error(`Chrome spawn error: ${err.message}`);
|
|
1995
|
+
});
|
|
1996
|
+
child.on("exit", (code, signal) => {
|
|
1997
|
+
if (code !== null && code !== 0) {
|
|
1998
|
+
log.debug(`Chrome exited with code ${code}${signal ? ` (signal: ${signal})` : ""}`);
|
|
1999
|
+
if (stderr) {
|
|
2000
|
+
const lines = stderr.split("\n").filter(Boolean).slice(0, 5);
|
|
2001
|
+
for (const line of lines) {
|
|
2002
|
+
log.debug(` chrome stderr: ${line}`);
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
});
|
|
2007
|
+
child.unref();
|
|
2008
|
+
return child;
|
|
2009
|
+
}
|
|
2010
|
+
async function waitForCDP(browser, timeoutMs = 3e4) {
|
|
2011
|
+
const start = Date.now();
|
|
2012
|
+
let attempts = 0;
|
|
2013
|
+
while (Date.now() - start < timeoutMs) {
|
|
2014
|
+
attempts++;
|
|
2015
|
+
if (await browser.isAvailable()) {
|
|
2016
|
+
log.debug(`CDP became reachable after ${attempts} attempts (${Date.now() - start}ms)`);
|
|
2017
|
+
return true;
|
|
2018
|
+
}
|
|
2019
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
2020
|
+
}
|
|
2021
|
+
log.debug(`CDP not reachable after ${attempts} attempts (${timeoutMs}ms timeout)`);
|
|
2022
|
+
return false;
|
|
2023
|
+
}
|
|
2024
|
+
async function isPortInUse(port) {
|
|
2025
|
+
try {
|
|
2026
|
+
const res = await fetch(`http://127.0.0.1:${port}/json/version`, {
|
|
2027
|
+
signal: AbortSignal.timeout(1e3)
|
|
2028
|
+
});
|
|
2029
|
+
const body = await res.text();
|
|
2030
|
+
return !body.includes("webSocketDebuggerUrl");
|
|
2031
|
+
} catch {
|
|
2032
|
+
return false;
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
async function ensureBrowserAvailable(port = 9222) {
|
|
2036
|
+
const browser = getBrowser(port);
|
|
2037
|
+
if (await browser.isAvailable()) {
|
|
2038
|
+
log.debug("CDP already reachable \u2014 no launch needed");
|
|
2039
|
+
return { success: true, action: "already_available" };
|
|
2040
|
+
}
|
|
2041
|
+
if (await isPortInUse(port)) {
|
|
2042
|
+
log.debug(`Port ${port} is in use by a non-Chrome process`);
|
|
2043
|
+
return {
|
|
2044
|
+
success: false,
|
|
2045
|
+
action: "port_conflict",
|
|
2046
|
+
detail: `Port ${port} is already in use by another process. Try a different port or stop the conflicting process.`
|
|
2047
|
+
};
|
|
2048
|
+
}
|
|
2049
|
+
const chromePath = findChromePath();
|
|
2050
|
+
if (!chromePath) {
|
|
2051
|
+
log.debug("Chrome binary not found on this system");
|
|
2052
|
+
return { success: false, action: "chrome_not_found" };
|
|
2053
|
+
}
|
|
2054
|
+
log.debug(`Found Chrome at: ${chromePath}`);
|
|
2055
|
+
spawnChrome(chromePath, port);
|
|
2056
|
+
if (await waitForCDP(browser)) {
|
|
2057
|
+
return { success: true, action: "launched", chromePath };
|
|
2058
|
+
}
|
|
2059
|
+
const debugDir = getDebugProfileDir(chromePath);
|
|
2060
|
+
const lockPath = join(debugDir, "SingletonLock");
|
|
2061
|
+
if (existsSync(lockPath)) {
|
|
2062
|
+
log.debug("Found stale SingletonLock in debug profile \u2014 removing and retrying");
|
|
2063
|
+
try {
|
|
2064
|
+
unlinkSync(lockPath);
|
|
2065
|
+
for (const f of ["SingletonSocket", "SingletonCookie"]) {
|
|
2066
|
+
try {
|
|
2067
|
+
unlinkSync(join(debugDir, f));
|
|
2068
|
+
} catch {
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
} catch {
|
|
2072
|
+
}
|
|
2073
|
+
spawnChrome(chromePath, port);
|
|
2074
|
+
if (await waitForCDP(browser, 15e3)) {
|
|
2075
|
+
return { success: true, action: "launched", chromePath };
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
return {
|
|
2079
|
+
success: false,
|
|
2080
|
+
action: "launch_failed",
|
|
2081
|
+
chromePath,
|
|
2082
|
+
detail: "Could not start browser with remote debugging. Possible causes:\n 1) Another assistme debug browser is already using port " + port + "\n 2) The browser crashed on startup\nTry: rm -rf ~/.assistme/browser-profile && assistme"
|
|
2083
|
+
};
|
|
2084
|
+
}
|
|
2085
|
+
var browserInstances = /* @__PURE__ */ new Map();
|
|
2086
|
+
function getBrowser(port = 9222) {
|
|
2087
|
+
let instance = browserInstances.get(port);
|
|
2088
|
+
if (!instance) {
|
|
2089
|
+
instance = new BrowserController(port);
|
|
2090
|
+
browserInstances.set(port, instance);
|
|
2091
|
+
}
|
|
2092
|
+
return instance;
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
// src/agent/scheduler.ts
|
|
2096
|
+
import { Cron } from "croner";
|
|
2097
|
+
function getNextRunTime(cronExpr, timezone, fromDate) {
|
|
2098
|
+
const now = fromDate || /* @__PURE__ */ new Date();
|
|
2099
|
+
try {
|
|
2100
|
+
const job = new Cron(cronExpr, { timezone: timezone || "UTC" });
|
|
2101
|
+
const next = job.nextRun(now);
|
|
2102
|
+
if (!next) {
|
|
2103
|
+
throw new Error(`No future run time found for cron expression: ${cronExpr}`);
|
|
2104
|
+
}
|
|
2105
|
+
return next;
|
|
2106
|
+
} catch (err) {
|
|
2107
|
+
if (err instanceof Error && err.message.includes("No future run time")) {
|
|
2108
|
+
throw err;
|
|
2109
|
+
}
|
|
2110
|
+
throw new Error(`Invalid cron expression "${cronExpr}": ${errorMessage(err)}`);
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
var Scheduler = class {
|
|
2114
|
+
timer = null;
|
|
2115
|
+
running = false;
|
|
2116
|
+
onScheduledTask = null;
|
|
2117
|
+
async start(onScheduledTask) {
|
|
2118
|
+
this.onScheduledTask = onScheduledTask;
|
|
2119
|
+
this.running = true;
|
|
2120
|
+
await this.initializeNextRuns();
|
|
2121
|
+
this.timer = setInterval(() => this.checkDueTasks(), SCHEDULER_INTERVAL_MS);
|
|
2122
|
+
log.info("Scheduler started (checking every 30s)");
|
|
2123
|
+
}
|
|
2124
|
+
stop() {
|
|
2125
|
+
this.running = false;
|
|
2126
|
+
if (this.timer) {
|
|
2127
|
+
clearInterval(this.timer);
|
|
2128
|
+
this.timer = null;
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
async initializeNextRuns() {
|
|
2132
|
+
try {
|
|
2133
|
+
const data = await callMcpHandler("schedule.get_uninitialized");
|
|
2134
|
+
if (data) {
|
|
2135
|
+
for (const task of data) {
|
|
2136
|
+
const nextRun = getNextRunTime(task.cron_expression, task.timezone);
|
|
2137
|
+
await callMcpHandler("schedule.update", {
|
|
2138
|
+
task_id: task.id,
|
|
2139
|
+
next_run_at: nextRun.toISOString()
|
|
2140
|
+
});
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
} catch (err) {
|
|
2144
|
+
log.debug(`Scheduler init: ${errorMessage(err)}`);
|
|
2145
|
+
}
|
|
2146
|
+
}
|
|
2147
|
+
async checkDueTasks() {
|
|
2148
|
+
if (!this.running || !this.onScheduledTask) return;
|
|
2149
|
+
try {
|
|
2150
|
+
const dueTasks = await callMcpHandler("schedule.check_due");
|
|
2151
|
+
if (!dueTasks || dueTasks.length === 0) return;
|
|
2152
|
+
const task = dueTasks[0];
|
|
2153
|
+
log.info(`Scheduled task due: "${task.name}"`);
|
|
2154
|
+
const nextRun = getNextRunTime(task.cron_expression, task.timezone);
|
|
2155
|
+
await callMcpHandler("schedule.update", {
|
|
2156
|
+
task_id: task.id,
|
|
2157
|
+
last_run_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2158
|
+
next_run_at: nextRun.toISOString(),
|
|
2159
|
+
run_count: task.run_count + 1
|
|
2160
|
+
});
|
|
2161
|
+
try {
|
|
2162
|
+
await this.onScheduledTask(task);
|
|
2163
|
+
await callMcpHandler("schedule.update", {
|
|
2164
|
+
task_id: task.id,
|
|
2165
|
+
last_error: null
|
|
2166
|
+
});
|
|
2167
|
+
} catch (err) {
|
|
2168
|
+
const errMsg = errorMessage(err);
|
|
2169
|
+
await callMcpHandler("schedule.update", {
|
|
2170
|
+
task_id: task.id,
|
|
2171
|
+
last_error: errMsg
|
|
2172
|
+
});
|
|
2173
|
+
log.error(`Scheduled task "${task.name}" failed: ${errMsg}`);
|
|
2174
|
+
}
|
|
2175
|
+
} catch (err) {
|
|
2176
|
+
log.debug(`Scheduler check error: ${errorMessage(err)}`);
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
};
|
|
2180
|
+
async function createScheduledTask(name, prompt, cronExpression, timezone = "UTC") {
|
|
2181
|
+
const nextRun = getNextRunTime(cronExpression, timezone);
|
|
2182
|
+
return callMcpHandler("schedule.create", {
|
|
2183
|
+
name,
|
|
2184
|
+
prompt,
|
|
2185
|
+
cron_expression: cronExpression,
|
|
2186
|
+
timezone,
|
|
2187
|
+
next_run_at: nextRun.toISOString()
|
|
2188
|
+
});
|
|
2189
|
+
}
|
|
2190
|
+
async function listScheduledTasks() {
|
|
2191
|
+
return callMcpHandler("schedule.list");
|
|
2192
|
+
}
|
|
2193
|
+
async function toggleScheduledTask(taskId, enabled) {
|
|
2194
|
+
const params = { task_id: taskId, enabled };
|
|
2195
|
+
if (enabled) {
|
|
2196
|
+
const taskData = await callMcpHandler(
|
|
2197
|
+
"schedule.get_task",
|
|
2198
|
+
{ task_id: taskId }
|
|
2199
|
+
);
|
|
2200
|
+
if (taskData) {
|
|
2201
|
+
const nextRun = getNextRunTime(taskData.cron_expression, taskData.timezone);
|
|
2202
|
+
params.next_run_at = nextRun.toISOString();
|
|
2203
|
+
}
|
|
2204
|
+
}
|
|
2205
|
+
await callMcpHandler("schedule.toggle", params);
|
|
2206
|
+
}
|
|
2207
|
+
async function deleteScheduledTask(taskId) {
|
|
2208
|
+
await callMcpHandler("schedule.delete", { task_id: taskId });
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
// src/agent/memory.ts
|
|
2212
|
+
var MemoryManager = class {
|
|
2213
|
+
/**
|
|
2214
|
+
* Store a new memory. Called by the agent after completing tasks
|
|
2215
|
+
* to remember important facts about the user.
|
|
2216
|
+
*/
|
|
2217
|
+
async remember(content, category = "general", options) {
|
|
2218
|
+
const expiresAt = options?.expiresInDays ? new Date(Date.now() + options.expiresInDays * 864e5).toISOString() : null;
|
|
2219
|
+
const data = await callMcpHandler("memory.store", {
|
|
2220
|
+
category,
|
|
2221
|
+
content,
|
|
2222
|
+
importance: options?.importance ?? 5,
|
|
2223
|
+
tags: options?.tags ?? [],
|
|
2224
|
+
source_message_id: options?.sourceMessageId ?? null,
|
|
2225
|
+
expires_at: expiresAt
|
|
2226
|
+
});
|
|
2227
|
+
log.debug(`Memory stored: [${category}] ${content.slice(0, 80)}...`);
|
|
2228
|
+
return data;
|
|
2229
|
+
}
|
|
2230
|
+
/**
|
|
2231
|
+
* Search memories by query text. Uses ILIKE + tag containment.
|
|
2232
|
+
*/
|
|
2233
|
+
async search(query, limit = 10) {
|
|
2234
|
+
try {
|
|
2235
|
+
return await callMcpHandler("memory.search", {
|
|
2236
|
+
query,
|
|
2237
|
+
limit
|
|
2238
|
+
});
|
|
2239
|
+
} catch (err) {
|
|
2240
|
+
log.warn(`Memory search failed: ${err instanceof Error ? err.message : err}`);
|
|
2241
|
+
return [];
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
/**
|
|
2245
|
+
* Get the most important/recent memories to include in context.
|
|
2246
|
+
* Called before each task to build the agent's "working memory".
|
|
2247
|
+
* Automatically filters out expired memories.
|
|
2248
|
+
*/
|
|
2249
|
+
async getContext(maxItems = 20) {
|
|
2250
|
+
const all = await callMcpHandler("memory.get_context", {
|
|
2251
|
+
max_items: maxItems
|
|
2252
|
+
});
|
|
2253
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2254
|
+
return (all || []).filter((m) => {
|
|
2255
|
+
if (seen.has(m.id)) return false;
|
|
2256
|
+
seen.add(m.id);
|
|
2257
|
+
return true;
|
|
2258
|
+
});
|
|
2259
|
+
}
|
|
2260
|
+
/**
|
|
2261
|
+
* Format memories into a string for the system prompt.
|
|
2262
|
+
*/
|
|
2263
|
+
async buildMemoryPrompt() {
|
|
2264
|
+
const memories = await this.getContext();
|
|
2265
|
+
if (memories.length === 0) return "";
|
|
2266
|
+
const sections = {};
|
|
2267
|
+
for (const m of memories) {
|
|
2268
|
+
const key = m.category;
|
|
2269
|
+
if (!sections[key]) sections[key] = [];
|
|
2270
|
+
sections[key].push(`- ${m.content}`);
|
|
2271
|
+
}
|
|
2272
|
+
const categoryLabels = {
|
|
2273
|
+
instruction: "Standing Instructions",
|
|
2274
|
+
preference: "User Preferences",
|
|
2275
|
+
general: "Known Facts",
|
|
2276
|
+
context: "Context",
|
|
2277
|
+
skill_learned: "Learned Skills",
|
|
2278
|
+
fact: "Facts"
|
|
2279
|
+
};
|
|
2280
|
+
let prompt = "\n\n## What You Know About The User\n";
|
|
2281
|
+
for (const [cat, items] of Object.entries(sections)) {
|
|
2282
|
+
prompt += `
|
|
2283
|
+
### ${categoryLabels[cat] || cat}
|
|
2284
|
+
`;
|
|
2285
|
+
prompt += items.join("\n") + "\n";
|
|
2286
|
+
}
|
|
2287
|
+
return prompt;
|
|
2288
|
+
}
|
|
2289
|
+
// ── CRUD for CLI ────────────────────────────────────────────
|
|
2290
|
+
async list(category, limit = 20) {
|
|
2291
|
+
const data = await callMcpHandler("memory.list", {
|
|
2292
|
+
category: category || null,
|
|
2293
|
+
limit
|
|
2294
|
+
});
|
|
2295
|
+
return data || [];
|
|
2296
|
+
}
|
|
2297
|
+
async add(content, category = "general", importance = 5, tags = []) {
|
|
2298
|
+
return this.remember(content, category, { importance, tags });
|
|
2299
|
+
}
|
|
2300
|
+
async remove(memoryId) {
|
|
2301
|
+
await callMcpHandler("memory.remove", { memory_id: memoryId });
|
|
2302
|
+
}
|
|
2303
|
+
async clear(category) {
|
|
2304
|
+
const result = await callMcpHandler("memory.clear", {
|
|
2305
|
+
category: category || null
|
|
2306
|
+
});
|
|
2307
|
+
return result.count;
|
|
2308
|
+
}
|
|
2309
|
+
// ── Compression & Deduplication ──────────────────────────────────
|
|
2310
|
+
/**
|
|
2311
|
+
* Check if memory count exceeds threshold and compress if needed.
|
|
2312
|
+
* Called automatically after task completion.
|
|
2313
|
+
*/
|
|
2314
|
+
async compressIfNeeded() {
|
|
2315
|
+
try {
|
|
2316
|
+
const all = await this.list(void 0, 200);
|
|
2317
|
+
if (all.length < MEMORY_COMPRESSION_THRESHOLD) {
|
|
2318
|
+
return 0;
|
|
2319
|
+
}
|
|
2320
|
+
log.info(`Memory compression triggered: ${all.length} memories (threshold: ${MEMORY_COMPRESSION_THRESHOLD})`);
|
|
2321
|
+
let removed = 0;
|
|
2322
|
+
const now = Date.now();
|
|
2323
|
+
for (const m of all) {
|
|
2324
|
+
if (m.expires_at && new Date(m.expires_at).getTime() < now) {
|
|
2325
|
+
await this.remove(m.id);
|
|
2326
|
+
removed++;
|
|
2327
|
+
}
|
|
2328
|
+
}
|
|
2329
|
+
const remaining = all.filter(
|
|
2330
|
+
(m) => !m.expires_at || new Date(m.expires_at).getTime() >= now
|
|
2331
|
+
);
|
|
2332
|
+
const duplicateIds = this.findDuplicates(remaining);
|
|
2333
|
+
for (const id of duplicateIds) {
|
|
2334
|
+
await this.remove(id);
|
|
2335
|
+
removed++;
|
|
2336
|
+
}
|
|
2337
|
+
const afterDedup = remaining.filter((m) => !duplicateIds.has(m.id));
|
|
2338
|
+
if (afterDedup.length > MEMORY_COMPRESSION_TARGET) {
|
|
2339
|
+
const toRemove = afterDedup.sort((a, b) => {
|
|
2340
|
+
if (a.importance !== b.importance) return a.importance - b.importance;
|
|
2341
|
+
if (a.access_count !== b.access_count) return a.access_count - b.access_count;
|
|
2342
|
+
return new Date(a.created_at).getTime() - new Date(b.created_at).getTime();
|
|
2343
|
+
}).slice(0, afterDedup.length - MEMORY_COMPRESSION_TARGET);
|
|
2344
|
+
for (const m of toRemove) {
|
|
2345
|
+
if (m.category === "instruction" && m.importance >= 8) continue;
|
|
2346
|
+
await this.remove(m.id);
|
|
2347
|
+
removed++;
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
if (removed > 0) {
|
|
2351
|
+
log.info(`Memory compression complete: removed ${removed} memories`);
|
|
2352
|
+
}
|
|
2353
|
+
return removed;
|
|
2354
|
+
} catch (err) {
|
|
2355
|
+
log.warn(`Memory compression error: ${err instanceof Error ? err.message : err}`);
|
|
2356
|
+
return 0;
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
/**
|
|
2360
|
+
* Find duplicate memories based on content similarity.
|
|
2361
|
+
* Returns the IDs of memories that should be removed (keeps the higher-importance duplicate).
|
|
2362
|
+
*/
|
|
2363
|
+
findDuplicates(memories) {
|
|
2364
|
+
const toRemove = /* @__PURE__ */ new Set();
|
|
2365
|
+
for (let i = 0; i < memories.length; i++) {
|
|
2366
|
+
if (toRemove.has(memories[i].id)) continue;
|
|
2367
|
+
for (let j = i + 1; j < memories.length; j++) {
|
|
2368
|
+
if (toRemove.has(memories[j].id)) continue;
|
|
2369
|
+
if (memories[i].category !== memories[j].category) continue;
|
|
2370
|
+
const similarity = computeWordOverlap(memories[i].content, memories[j].content);
|
|
2371
|
+
if (similarity >= MEMORY_DEDUP_SIMILARITY_THRESHOLD) {
|
|
2372
|
+
if (memories[i].importance > memories[j].importance || memories[i].importance === memories[j].importance && new Date(memories[i].created_at) > new Date(memories[j].created_at)) {
|
|
2373
|
+
toRemove.add(memories[j].id);
|
|
2374
|
+
} else {
|
|
2375
|
+
toRemove.add(memories[i].id);
|
|
2376
|
+
}
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
return toRemove;
|
|
2381
|
+
}
|
|
2382
|
+
};
|
|
2383
|
+
function computeWordOverlap(a, b) {
|
|
2384
|
+
const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(Boolean));
|
|
2385
|
+
const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(Boolean));
|
|
2386
|
+
if (wordsA.size === 0 && wordsB.size === 0) return 1;
|
|
2387
|
+
if (wordsA.size === 0 || wordsB.size === 0) return 0;
|
|
2388
|
+
let intersection = 0;
|
|
2389
|
+
for (const w of wordsA) {
|
|
2390
|
+
if (wordsB.has(w)) intersection++;
|
|
2391
|
+
}
|
|
2392
|
+
const union = wordsA.size + wordsB.size - intersection;
|
|
2393
|
+
return union === 0 ? 0 : intersection / union;
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
// src/agent/skills.ts
|
|
2397
|
+
import { execSync as execSync2 } from "child_process";
|
|
2398
|
+
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
2399
|
+
"the",
|
|
2400
|
+
"a",
|
|
2401
|
+
"an",
|
|
2402
|
+
"is",
|
|
2403
|
+
"are",
|
|
2404
|
+
"was",
|
|
2405
|
+
"were",
|
|
2406
|
+
"be",
|
|
2407
|
+
"been",
|
|
2408
|
+
"being",
|
|
2409
|
+
"have",
|
|
2410
|
+
"has",
|
|
2411
|
+
"had",
|
|
2412
|
+
"do",
|
|
2413
|
+
"does",
|
|
2414
|
+
"did",
|
|
2415
|
+
"will",
|
|
2416
|
+
"would",
|
|
2417
|
+
"could",
|
|
2418
|
+
"should",
|
|
2419
|
+
"may",
|
|
2420
|
+
"might",
|
|
2421
|
+
"shall",
|
|
2422
|
+
"can",
|
|
2423
|
+
"need",
|
|
2424
|
+
"dare",
|
|
2425
|
+
"ought",
|
|
2426
|
+
"used",
|
|
2427
|
+
"to",
|
|
2428
|
+
"of",
|
|
2429
|
+
"in",
|
|
2430
|
+
"for",
|
|
2431
|
+
"on",
|
|
2432
|
+
"with",
|
|
2433
|
+
"at",
|
|
2434
|
+
"by",
|
|
2435
|
+
"from",
|
|
2436
|
+
"as",
|
|
2437
|
+
"into",
|
|
2438
|
+
"through",
|
|
2439
|
+
"during",
|
|
2440
|
+
"before",
|
|
2441
|
+
"after",
|
|
2442
|
+
"above",
|
|
2443
|
+
"below",
|
|
2444
|
+
"between",
|
|
2445
|
+
"out",
|
|
2446
|
+
"off",
|
|
2447
|
+
"over",
|
|
2448
|
+
"under",
|
|
2449
|
+
"again",
|
|
2450
|
+
"further",
|
|
2451
|
+
"then",
|
|
2452
|
+
"once",
|
|
2453
|
+
"here",
|
|
2454
|
+
"there",
|
|
2455
|
+
"when",
|
|
2456
|
+
"where",
|
|
2457
|
+
"why",
|
|
2458
|
+
"how",
|
|
2459
|
+
"all",
|
|
2460
|
+
"each",
|
|
2461
|
+
"every",
|
|
2462
|
+
"both",
|
|
2463
|
+
"few",
|
|
2464
|
+
"more",
|
|
2465
|
+
"most",
|
|
2466
|
+
"other",
|
|
2467
|
+
"some",
|
|
2468
|
+
"such",
|
|
2469
|
+
"no",
|
|
2470
|
+
"nor",
|
|
2471
|
+
"not",
|
|
2472
|
+
"only",
|
|
2473
|
+
"own",
|
|
2474
|
+
"same",
|
|
2475
|
+
"so",
|
|
2476
|
+
"than",
|
|
2477
|
+
"too",
|
|
2478
|
+
"very",
|
|
2479
|
+
"and",
|
|
2480
|
+
"but",
|
|
2481
|
+
"or",
|
|
2482
|
+
"if",
|
|
2483
|
+
"this",
|
|
2484
|
+
"that",
|
|
2485
|
+
"these",
|
|
2486
|
+
"those",
|
|
2487
|
+
"it",
|
|
2488
|
+
"its",
|
|
2489
|
+
"i",
|
|
2490
|
+
"me",
|
|
2491
|
+
"my",
|
|
2492
|
+
"we",
|
|
2493
|
+
"our",
|
|
2494
|
+
"you",
|
|
2495
|
+
"your",
|
|
2496
|
+
"he",
|
|
2497
|
+
"him",
|
|
2498
|
+
"she",
|
|
2499
|
+
"her",
|
|
2500
|
+
"they",
|
|
2501
|
+
"them",
|
|
2502
|
+
"what",
|
|
2503
|
+
"which",
|
|
2504
|
+
"who",
|
|
2505
|
+
"whom"
|
|
2506
|
+
]);
|
|
2507
|
+
function tokenize(text) {
|
|
2508
|
+
const englishTokens = text.split(/[\s\-_/.,;:!?()[\]{}'"]+/).filter((w) => w.length > 2 && !STOP_WORDS.has(w));
|
|
2509
|
+
const chineseChars = text.replace(/[^\u4e00-\u9fff]/g, "");
|
|
2510
|
+
const chineseBigrams = [];
|
|
2511
|
+
for (let i = 0; i < chineseChars.length - 1; i++) {
|
|
2512
|
+
chineseBigrams.push(chineseChars.slice(i, i + 2));
|
|
2513
|
+
}
|
|
2514
|
+
return [...englishTokens, ...chineseBigrams];
|
|
2515
|
+
}
|
|
2516
|
+
function bigrams(tokens) {
|
|
2517
|
+
const result = /* @__PURE__ */ new Set();
|
|
2518
|
+
for (let i = 0; i < tokens.length - 1; i++) {
|
|
2519
|
+
result.add(`${tokens[i]} ${tokens[i + 1]}`);
|
|
2520
|
+
}
|
|
2521
|
+
return result;
|
|
2522
|
+
}
|
|
2523
|
+
function parseDbMetadata(raw) {
|
|
2524
|
+
if (!raw || typeof raw !== "object") return {};
|
|
2525
|
+
const obj = raw;
|
|
2526
|
+
const openclaw = obj.openclaw || obj;
|
|
2527
|
+
return {
|
|
2528
|
+
emoji: openclaw.emoji,
|
|
2529
|
+
requires: openclaw.requires,
|
|
2530
|
+
primaryEnv: openclaw.primaryEnv,
|
|
2531
|
+
os: openclaw.os,
|
|
2532
|
+
always: openclaw.always,
|
|
2533
|
+
skillKey: openclaw.skillKey,
|
|
2534
|
+
credentials: openclaw.credentials
|
|
2535
|
+
};
|
|
2536
|
+
}
|
|
2537
|
+
var SkillManager = class {
|
|
2538
|
+
skills = /* @__PURE__ */ new Map();
|
|
2539
|
+
idfCache = /* @__PURE__ */ new Map();
|
|
2540
|
+
userId = null;
|
|
2541
|
+
/** Cache for findRelevant() — keyed by prompt, invalidated on skill changes */
|
|
2542
|
+
relevanceCache = /* @__PURE__ */ new Map();
|
|
2543
|
+
DESCRIPTION_BUDGET_CHARS = SKILL_DESCRIPTION_BUDGET_CHARS;
|
|
2544
|
+
setUserId(userId) {
|
|
2545
|
+
this.userId = userId;
|
|
2546
|
+
}
|
|
2547
|
+
async loadFromDb() {
|
|
2548
|
+
if (!this.userId) return;
|
|
2549
|
+
try {
|
|
2550
|
+
const data = await callMcpHandler("skill.load");
|
|
2551
|
+
this.skills.clear();
|
|
2552
|
+
for (const raw of data || []) {
|
|
2553
|
+
const row = safeParse(SkillRowSchema, raw);
|
|
2554
|
+
if (!row) continue;
|
|
2555
|
+
const skill = this.rowToSkill(row);
|
|
2556
|
+
this.skills.set(skill.name, skill);
|
|
2557
|
+
}
|
|
2558
|
+
this.buildIdfCache();
|
|
2559
|
+
if (this.skills.size > 0) {
|
|
2560
|
+
log.info(`Loaded ${this.skills.size} skill(s) from DB`);
|
|
2561
|
+
}
|
|
2562
|
+
} catch (err) {
|
|
2563
|
+
log.debug(`DB skill load error: ${err}`);
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
rowToSkill(row) {
|
|
2567
|
+
return {
|
|
2568
|
+
name: String(row.name),
|
|
2569
|
+
description: String(row.description ?? ""),
|
|
2570
|
+
version: String(row.version ?? "1.0.0"),
|
|
2571
|
+
userInvocable: row.user_invocable !== false,
|
|
2572
|
+
disableModelInvocation: row.disable_model_invocation === true,
|
|
2573
|
+
keywords: Array.isArray(row.keywords) ? row.keywords : [],
|
|
2574
|
+
allowedTools: Array.isArray(row.allowed_tools) ? row.allowed_tools : [],
|
|
2575
|
+
argumentHint: String(row.argument_hint ?? ""),
|
|
2576
|
+
metadata: parseDbMetadata(row.metadata),
|
|
2577
|
+
homepage: String(row.homepage ?? ""),
|
|
2578
|
+
content: String(row.content ?? ""),
|
|
2579
|
+
filePath: "",
|
|
2580
|
+
source: row.source || "manual",
|
|
2581
|
+
dbId: row.id != null ? String(row.id) : void 0,
|
|
2582
|
+
sourceSkillId: row.source_skill_id != null ? String(row.source_skill_id) : void 0,
|
|
2583
|
+
invocationCount: typeof row.invocation_count === "number" ? row.invocation_count : 0
|
|
2584
|
+
};
|
|
2585
|
+
}
|
|
2586
|
+
/** Invalidate caches when skills change (create, add, update, remove). */
|
|
2587
|
+
invalidateCaches() {
|
|
2588
|
+
this.relevanceCache.clear();
|
|
2589
|
+
this.buildIdfCache();
|
|
2590
|
+
}
|
|
2591
|
+
buildIdfCache() {
|
|
2592
|
+
this.idfCache.clear();
|
|
2593
|
+
const docFreq = /* @__PURE__ */ new Map();
|
|
2594
|
+
const totalSkills = this.skills.size || 1;
|
|
2595
|
+
for (const skill of this.skills.values()) {
|
|
2596
|
+
const allText = `${skill.name} ${skill.description} ${skill.content} ${skill.keywords.join(" ")}`.toLowerCase();
|
|
2597
|
+
const words = new Set(tokenize(allText));
|
|
2598
|
+
for (const w of words) {
|
|
2599
|
+
docFreq.set(w, (docFreq.get(w) || 0) + 1);
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
for (const [word, df] of docFreq) {
|
|
2603
|
+
this.idfCache.set(word, Math.log(totalSkills / df) + 1);
|
|
2604
|
+
}
|
|
2605
|
+
}
|
|
2606
|
+
getAll() {
|
|
2607
|
+
return Array.from(this.skills.values());
|
|
2608
|
+
}
|
|
2609
|
+
get(name) {
|
|
2610
|
+
return this.skills.get(name);
|
|
2611
|
+
}
|
|
2612
|
+
findRelevant(prompt, maxResults = 3) {
|
|
2613
|
+
const cacheKey = prompt.toLowerCase();
|
|
2614
|
+
const cached = this.relevanceCache.get(cacheKey);
|
|
2615
|
+
if (cached && cached.maxResults >= maxResults) {
|
|
2616
|
+
return cached.results.slice(0, maxResults);
|
|
2617
|
+
}
|
|
2618
|
+
const lower = cacheKey;
|
|
2619
|
+
const promptTokens = tokenize(lower);
|
|
2620
|
+
const promptTokenSet = new Set(promptTokens);
|
|
2621
|
+
const idf = (word) => this.idfCache.get(word) || 1;
|
|
2622
|
+
const scored = [];
|
|
2623
|
+
for (const skill of this.skills.values()) {
|
|
2624
|
+
if (skill.disableModelInvocation) continue;
|
|
2625
|
+
let score = 0;
|
|
2626
|
+
if (lower.includes(skill.name.toLowerCase())) score += 10;
|
|
2627
|
+
for (const kw of skill.keywords) {
|
|
2628
|
+
if (lower.includes(kw.toLowerCase())) score += 8;
|
|
2629
|
+
}
|
|
2630
|
+
const descTokens = tokenize(skill.description.toLowerCase());
|
|
2631
|
+
for (const word of descTokens) {
|
|
2632
|
+
if (promptTokenSet.has(word)) score += 3 * idf(word);
|
|
2633
|
+
}
|
|
2634
|
+
const contentTokens = tokenize(skill.content.toLowerCase());
|
|
2635
|
+
for (const word of contentTokens) {
|
|
2636
|
+
if (promptTokenSet.has(word)) score += 0.5 * idf(word);
|
|
2637
|
+
}
|
|
2638
|
+
const promptBigrams = bigrams(promptTokens);
|
|
2639
|
+
const descBigrams = bigrams(descTokens);
|
|
2640
|
+
for (const bg of descBigrams) {
|
|
2641
|
+
if (promptBigrams.has(bg)) score += 5;
|
|
2642
|
+
}
|
|
2643
|
+
if (score > 0) scored.push({ skill, score });
|
|
2644
|
+
}
|
|
2645
|
+
const results = scored.sort((a, b) => b.score - a.score).slice(0, maxResults).map((s) => s.skill);
|
|
2646
|
+
this.relevanceCache.set(cacheKey, { results, maxResults });
|
|
2647
|
+
return results;
|
|
2648
|
+
}
|
|
2649
|
+
/**
|
|
2650
|
+
* Build lightweight skill descriptions for the system prompt.
|
|
2651
|
+
* When a taskPrompt is provided, relevant skills are prioritized to the top;
|
|
2652
|
+
* remaining skills are sorted by usage frequency (invocationCount).
|
|
2653
|
+
*/
|
|
2654
|
+
buildSkillDescriptions(taskPrompt) {
|
|
2655
|
+
const all = this.getAll().filter((s) => !s.disableModelInvocation);
|
|
2656
|
+
if (all.length === 0) return "";
|
|
2657
|
+
const alwaysSkills = all.filter((s) => s.metadata.always);
|
|
2658
|
+
const rest = all.filter((s) => !s.metadata.always);
|
|
2659
|
+
let relevantNames = null;
|
|
2660
|
+
if (taskPrompt) {
|
|
2661
|
+
const relevant = this.findRelevant(taskPrompt, 10);
|
|
2662
|
+
relevantNames = new Set(relevant.map((s) => s.name));
|
|
2663
|
+
}
|
|
2664
|
+
const sorted = rest.sort((a, b) => {
|
|
2665
|
+
if (relevantNames) {
|
|
2666
|
+
const aRelevant = relevantNames.has(a.name);
|
|
2667
|
+
const bRelevant = relevantNames.has(b.name);
|
|
2668
|
+
if (aRelevant && !bRelevant) return -1;
|
|
2669
|
+
if (!aRelevant && bRelevant) return 1;
|
|
2670
|
+
}
|
|
2671
|
+
return (b.invocationCount || 0) - (a.invocationCount || 0);
|
|
2672
|
+
});
|
|
2673
|
+
const skills = [...alwaysSkills, ...sorted];
|
|
2674
|
+
let budget = this.DESCRIPTION_BUDGET_CHARS;
|
|
2675
|
+
let prompt = "\n\n## Your Skills\n";
|
|
2676
|
+
prompt += "These are your approved skills. Use skill_invoke to load full instructions when a task matches.\n";
|
|
2677
|
+
prompt += "If no skill matches but the task is a reusable pattern, consider creating one with skill_create.\n\n";
|
|
2678
|
+
let included = 0;
|
|
2679
|
+
for (const skill of skills) {
|
|
2680
|
+
const emoji = skill.metadata.emoji || "";
|
|
2681
|
+
const hint = skill.argumentHint ? ` (${skill.argumentHint})` : "";
|
|
2682
|
+
const line = `- **${emoji ? emoji + " " : ""}${skill.name}**${hint}: ${skill.description}
|
|
2683
|
+
`;
|
|
2684
|
+
if (budget - line.length < 0) break;
|
|
2685
|
+
budget -= line.length;
|
|
2686
|
+
prompt += line;
|
|
2687
|
+
included++;
|
|
2688
|
+
}
|
|
2689
|
+
if (included < skills.length) {
|
|
2690
|
+
prompt += `
|
|
2691
|
+
_(${skills.length - included} additional skills available \u2014 use skill_search to find more)_
|
|
2692
|
+
`;
|
|
2693
|
+
}
|
|
2694
|
+
return prompt;
|
|
2695
|
+
}
|
|
2696
|
+
async create(name, description, content, options) {
|
|
2697
|
+
if (!this.userId) return null;
|
|
2698
|
+
try {
|
|
2699
|
+
const metadata = options?.emoji ? { openclaw: { emoji: options.emoji } } : {};
|
|
2700
|
+
const data = await callMcpHandler(
|
|
2701
|
+
"skill.create",
|
|
2702
|
+
{
|
|
2703
|
+
name,
|
|
2704
|
+
description,
|
|
2705
|
+
content,
|
|
2706
|
+
version: "1.0.0",
|
|
2707
|
+
source: options?.source || "manual",
|
|
2708
|
+
emoji: options?.emoji || null,
|
|
2709
|
+
keywords: options?.keywords || [],
|
|
2710
|
+
metadata
|
|
2711
|
+
}
|
|
2712
|
+
);
|
|
2713
|
+
const raw = Array.isArray(data) ? data[0] : data;
|
|
2714
|
+
const row = safeParse(SkillCreateResultSchema, raw);
|
|
2715
|
+
if (!row) {
|
|
2716
|
+
log.debug(`Skill create returned invalid data for "${name}"`);
|
|
2717
|
+
return null;
|
|
2718
|
+
}
|
|
2719
|
+
const id = row.out_id || row.id;
|
|
2720
|
+
const skillName = row.out_name || row.name || name;
|
|
2721
|
+
this.skills.set(skillName, {
|
|
2722
|
+
name: skillName,
|
|
2723
|
+
description,
|
|
2724
|
+
version: "1.0.0",
|
|
2725
|
+
userInvocable: true,
|
|
2726
|
+
disableModelInvocation: false,
|
|
2727
|
+
keywords: options?.keywords || [],
|
|
2728
|
+
allowedTools: [],
|
|
2729
|
+
argumentHint: "",
|
|
2730
|
+
metadata: options?.emoji ? { emoji: options.emoji } : {},
|
|
2731
|
+
homepage: "",
|
|
2732
|
+
content,
|
|
2733
|
+
filePath: "",
|
|
2734
|
+
source: options?.source || "manual",
|
|
2735
|
+
dbId: id,
|
|
2736
|
+
invocationCount: 0
|
|
2737
|
+
});
|
|
2738
|
+
this.invalidateCaches();
|
|
2739
|
+
log.info(`Skill "${skillName}" created in skills table (pending approval)`);
|
|
2740
|
+
return { id, name: skillName };
|
|
2741
|
+
} catch (err) {
|
|
2742
|
+
log.debug(`Skill create error: ${err}`);
|
|
2743
|
+
return null;
|
|
2744
|
+
}
|
|
2745
|
+
}
|
|
2746
|
+
async addSkill(skillId) {
|
|
2747
|
+
if (!this.userId) return null;
|
|
2748
|
+
try {
|
|
2749
|
+
const result = await callMcpHandler(
|
|
2750
|
+
"skill.fetch_and_add",
|
|
2751
|
+
{ skill_id: skillId }
|
|
2752
|
+
);
|
|
2753
|
+
const row = result.skill;
|
|
2754
|
+
const agentSkillRow = result.agent_skill && typeof result.agent_skill === "object" ? result.agent_skill : row;
|
|
2755
|
+
const skill = this.rowToSkill({
|
|
2756
|
+
...agentSkillRow,
|
|
2757
|
+
name: row.name,
|
|
2758
|
+
description: row.description,
|
|
2759
|
+
content: row.content,
|
|
2760
|
+
source_skill_id: skillId
|
|
2761
|
+
});
|
|
2762
|
+
this.skills.set(skill.name, skill);
|
|
2763
|
+
this.invalidateCaches();
|
|
2764
|
+
log.info(`Skill "${row.name}" added to user's collection`);
|
|
2765
|
+
return skill;
|
|
2766
|
+
} catch (err) {
|
|
2767
|
+
log.debug(`addSkill error: ${err}`);
|
|
2768
|
+
return null;
|
|
2769
|
+
}
|
|
2770
|
+
}
|
|
2771
|
+
remove(name) {
|
|
2772
|
+
const skill = this.skills.get(name);
|
|
2773
|
+
if (!skill) return false;
|
|
2774
|
+
this.skills.delete(name);
|
|
2775
|
+
this.invalidateCaches();
|
|
2776
|
+
this.removeFromDb(name).catch(() => {
|
|
2777
|
+
});
|
|
2778
|
+
return true;
|
|
2779
|
+
}
|
|
2780
|
+
listFormatted() {
|
|
2781
|
+
const skills = this.getAll();
|
|
2782
|
+
if (skills.length === 0) return "No skills in your collection.";
|
|
2783
|
+
return skills.map((s) => {
|
|
2784
|
+
const emoji = s.metadata.emoji || "";
|
|
2785
|
+
return ` ${emoji ? emoji + " " : ""}${s.name} (${s.version}) [${s.source}]
|
|
2786
|
+
${s.description}`;
|
|
2787
|
+
}).join("\n\n");
|
|
2788
|
+
}
|
|
2789
|
+
findSimilar(name) {
|
|
2790
|
+
if (this.skills.has(name)) return this.skills.get(name);
|
|
2791
|
+
const normalizedName = name.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
2792
|
+
for (const [existingName, skill] of this.skills) {
|
|
2793
|
+
const normalizedExisting = existingName.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
2794
|
+
if (normalizedName.includes(normalizedExisting) || normalizedExisting.includes(normalizedName)) {
|
|
2795
|
+
return skill;
|
|
2796
|
+
}
|
|
2797
|
+
const nameWords = new Set(name.split("-"));
|
|
2798
|
+
const existingWords = new Set(existingName.split("-"));
|
|
2799
|
+
let overlap = 0;
|
|
2800
|
+
for (const w of nameWords) {
|
|
2801
|
+
if (existingWords.has(w)) overlap++;
|
|
2802
|
+
}
|
|
2803
|
+
if (overlap >= 2 || overlap >= 1 && nameWords.size <= 2) {
|
|
2804
|
+
return skill;
|
|
2805
|
+
}
|
|
2806
|
+
}
|
|
2807
|
+
return null;
|
|
2808
|
+
}
|
|
2809
|
+
update(name, newContent, description) {
|
|
2810
|
+
const skill = this.skills.get(name);
|
|
2811
|
+
if (!skill) return false;
|
|
2812
|
+
const versionParts = skill.version.split(".").map(Number);
|
|
2813
|
+
versionParts[2] = (versionParts[2] || 0) + 1;
|
|
2814
|
+
const newVersion = versionParts.join(".");
|
|
2815
|
+
const newDescription = description || skill.description;
|
|
2816
|
+
skill.content = newContent;
|
|
2817
|
+
skill.description = newDescription;
|
|
2818
|
+
skill.version = newVersion;
|
|
2819
|
+
this.invalidateCaches();
|
|
2820
|
+
this.syncToAgentSkills(name, newDescription, newContent, newVersion, {
|
|
2821
|
+
source: "auto_improved"
|
|
2822
|
+
}).catch(() => {
|
|
2823
|
+
});
|
|
2824
|
+
if (skill.sourceSkillId && this.userId) {
|
|
2825
|
+
callMcpHandler("skill.update_source", {
|
|
2826
|
+
source_skill_id: skill.sourceSkillId,
|
|
2827
|
+
content: newContent,
|
|
2828
|
+
description: newDescription,
|
|
2829
|
+
version: newVersion
|
|
2830
|
+
}).catch(() => {
|
|
2831
|
+
});
|
|
2832
|
+
}
|
|
2833
|
+
log.info(`Skill "${name}" updated to v${newVersion}`);
|
|
2834
|
+
return true;
|
|
2835
|
+
}
|
|
2836
|
+
// ── DB Integration ─────────────────────────────────────────────────
|
|
2837
|
+
async syncToAgentSkills(name, description, content, version, options) {
|
|
2838
|
+
if (!this.userId) return;
|
|
2839
|
+
try {
|
|
2840
|
+
const data = await callMcpHandler("skill.upsert", {
|
|
2841
|
+
name,
|
|
2842
|
+
description,
|
|
2843
|
+
content,
|
|
2844
|
+
version,
|
|
2845
|
+
source: options?.source || "manual",
|
|
2846
|
+
emoji: options?.emoji || null,
|
|
2847
|
+
keywords: options?.keywords || [],
|
|
2848
|
+
change_summary: options?.changeSummary || null,
|
|
2849
|
+
source_skill_id: options?.sourceSkillId || null
|
|
2850
|
+
});
|
|
2851
|
+
const skill = this.skills.get(name);
|
|
2852
|
+
if (skill && data && typeof data === "object" && "id" in data) {
|
|
2853
|
+
skill.dbId = data.id;
|
|
2854
|
+
}
|
|
2855
|
+
log.debug(`Skill "${name}" synced to agent_skills`);
|
|
2856
|
+
} catch (err) {
|
|
2857
|
+
log.debug(`DB skill sync error for "${name}": ${err}`);
|
|
2858
|
+
}
|
|
2859
|
+
}
|
|
2860
|
+
async logInvocation(skillName, options) {
|
|
2861
|
+
if (!this.userId) return;
|
|
2862
|
+
const skill = this.skills.get(skillName);
|
|
2863
|
+
const skillDbId = skill?.dbId;
|
|
2864
|
+
if (!skillDbId) {
|
|
2865
|
+
log.debug(`Cannot log invocation: skill "${skillName}" has no DB ID`);
|
|
2866
|
+
return;
|
|
2867
|
+
}
|
|
2868
|
+
try {
|
|
2869
|
+
await callMcpHandler("skill.log_invocation", {
|
|
2870
|
+
skill_id: skillDbId,
|
|
2871
|
+
message_id: options?.messageId || null,
|
|
2872
|
+
session_id: options?.sessionId || null,
|
|
2873
|
+
task_prompt: options?.taskPrompt?.slice(0, 500) || null,
|
|
2874
|
+
arguments: options?.arguments || null,
|
|
2875
|
+
success: options?.success ?? null
|
|
2876
|
+
});
|
|
2877
|
+
} catch (err) {
|
|
2878
|
+
log.debug(`Invocation log error: ${err}`);
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
async searchDb(query, limit = 10) {
|
|
2882
|
+
if (this.userId) {
|
|
2883
|
+
try {
|
|
2884
|
+
const data = await callMcpHandler("skill.search", {
|
|
2885
|
+
query,
|
|
2886
|
+
limit
|
|
2887
|
+
});
|
|
2888
|
+
if (data) {
|
|
2889
|
+
return data.map((row) => ({
|
|
2890
|
+
name: String(row.name),
|
|
2891
|
+
description: String(row.description ?? ""),
|
|
2892
|
+
emoji: String(row.emoji ?? ""),
|
|
2893
|
+
source: String(row.source ?? "manual"),
|
|
2894
|
+
invocationCount: typeof row.invocation_count === "number" ? row.invocation_count : 0
|
|
2895
|
+
}));
|
|
2896
|
+
}
|
|
2897
|
+
} catch {
|
|
2898
|
+
}
|
|
2899
|
+
}
|
|
2900
|
+
const results = this.findRelevant(query, limit);
|
|
2901
|
+
return results.map((s) => ({
|
|
2902
|
+
name: s.name,
|
|
2903
|
+
description: s.description,
|
|
2904
|
+
emoji: s.metadata.emoji || "",
|
|
2905
|
+
source: s.source,
|
|
2906
|
+
invocationCount: 0
|
|
2907
|
+
}));
|
|
2908
|
+
}
|
|
2909
|
+
async removeFromDb(name) {
|
|
2910
|
+
if (!this.userId) return;
|
|
2911
|
+
try {
|
|
2912
|
+
await callMcpHandler("skill.remove", { name });
|
|
2913
|
+
} catch {
|
|
2914
|
+
}
|
|
2915
|
+
}
|
|
2916
|
+
// ── Marketplace ────────────────────────────────────────────────────
|
|
2917
|
+
async publish(name, options) {
|
|
2918
|
+
if (!this.userId) return null;
|
|
2919
|
+
const skill = this.skills.get(name);
|
|
2920
|
+
if (!skill) return null;
|
|
2921
|
+
if (skill.source === "external") {
|
|
2922
|
+
log.debug(`Cannot publish external skill "${name}"`);
|
|
2923
|
+
return null;
|
|
2924
|
+
}
|
|
2925
|
+
try {
|
|
2926
|
+
const data = await callMcpHandler("skill.publish", {
|
|
2927
|
+
name: skill.name,
|
|
2928
|
+
description: skill.description,
|
|
2929
|
+
version: skill.version,
|
|
2930
|
+
emoji: skill.metadata.emoji || null,
|
|
2931
|
+
content: skill.content,
|
|
2932
|
+
argument_hint: skill.argumentHint || null,
|
|
2933
|
+
keywords: skill.keywords,
|
|
2934
|
+
allowed_tools: skill.allowedTools,
|
|
2935
|
+
author_name: options?.authorName || null,
|
|
2936
|
+
metadata: skill.metadata,
|
|
2937
|
+
homepage: skill.homepage || null,
|
|
2938
|
+
category: options?.category || null,
|
|
2939
|
+
source: skill.source
|
|
2940
|
+
});
|
|
2941
|
+
log.info(`Skill "${name}" published to marketplace`);
|
|
2942
|
+
return data;
|
|
2943
|
+
} catch (err) {
|
|
2944
|
+
log.debug(`Publish error: ${err}`);
|
|
2945
|
+
return null;
|
|
2946
|
+
}
|
|
2947
|
+
}
|
|
2948
|
+
async browse(options) {
|
|
2949
|
+
try {
|
|
2950
|
+
const data = await callMcpHandler("skill.browse", {
|
|
2951
|
+
query: options?.query || null,
|
|
2952
|
+
category: options?.category || null,
|
|
2953
|
+
sort: options?.sort || "popular",
|
|
2954
|
+
limit: options?.limit || 20,
|
|
2955
|
+
offset: options?.offset || 0
|
|
2956
|
+
});
|
|
2957
|
+
return (data || []).map((r) => safeParse(BrowseSkillRowSchema, r)).filter(Boolean).map((r) => ({
|
|
2958
|
+
id: r.id,
|
|
2959
|
+
name: r.name,
|
|
2960
|
+
description: r.description,
|
|
2961
|
+
emoji: r.emoji,
|
|
2962
|
+
version: r.version,
|
|
2963
|
+
authorName: r.author_name,
|
|
2964
|
+
category: r.category,
|
|
2965
|
+
installCount: r.install_count,
|
|
2966
|
+
avgRating: r.avg_rating ?? null,
|
|
2967
|
+
ratingCount: r.rating_count
|
|
2968
|
+
}));
|
|
2969
|
+
} catch {
|
|
2970
|
+
return [];
|
|
2971
|
+
}
|
|
2972
|
+
}
|
|
2973
|
+
};
|
|
2974
|
+
function validateSkillName(name) {
|
|
2975
|
+
if (!name || name.length === 0) return "name is empty";
|
|
2976
|
+
if (name.length > 64) return `name too long (${name.length}/64 chars)`;
|
|
2977
|
+
if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name)) {
|
|
2978
|
+
return `name must be lowercase kebab-case (a-z, 0-9, hyphens), no leading/trailing/consecutive hyphens. Got: "${name}"`;
|
|
2979
|
+
}
|
|
2980
|
+
return null;
|
|
2981
|
+
}
|
|
2982
|
+
function normalizeSkillName(name) {
|
|
2983
|
+
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").replace(/-{2,}/g, "-").slice(0, 64);
|
|
2984
|
+
}
|
|
2985
|
+
function substituteArguments(content, args) {
|
|
2986
|
+
const parts = args.split(/\s+/);
|
|
2987
|
+
content = content.replace(/\$ARGUMENTS/g, args);
|
|
2988
|
+
content = content.replace(/\$ARGUMENTS\[(\d+)\]/g, (_, i) => parts[parseInt(i)] || "");
|
|
2989
|
+
content = content.replace(/\$(\d+)(?!\w)/g, (_, i) => parts[parseInt(i)] || "");
|
|
2990
|
+
return content;
|
|
2991
|
+
}
|
|
2992
|
+
var SAFE_DYNAMIC_COMMANDS = /^(date|whoami|hostname|uname|pwd|echo|node\s+--version|npm\s+--version|git\s+(branch|rev-parse|log\s+--oneline)|cat\s+)/;
|
|
2993
|
+
function preprocessDynamicContext(content, cwd) {
|
|
2994
|
+
return content.replace(/!`([^`]+)`/g, (_, cmd) => {
|
|
2995
|
+
if (!SAFE_DYNAMIC_COMMANDS.test(cmd.trim())) {
|
|
2996
|
+
return `[command blocked: ${cmd}]`;
|
|
2997
|
+
}
|
|
2998
|
+
try {
|
|
2999
|
+
return execSync2(cmd, { timeout: 1e4, encoding: "utf-8", cwd }).trim();
|
|
3000
|
+
} catch {
|
|
3001
|
+
return `[command failed: ${cmd}]`;
|
|
3002
|
+
}
|
|
3003
|
+
});
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
// src/credentials/credential-store.ts
|
|
3007
|
+
import { randomUUID } from "crypto";
|
|
3008
|
+
import { dirname } from "path";
|
|
3009
|
+
|
|
3010
|
+
// src/credentials/encryption.ts
|
|
3011
|
+
import { createCipheriv, createDecipheriv, randomBytes, scryptSync, createHash } from "crypto";
|
|
3012
|
+
import { existsSync as existsSync2, readFileSync, writeFileSync, mkdirSync as mkdirSync2 } from "fs";
|
|
3013
|
+
import { join as join2 } from "path";
|
|
3014
|
+
import { homedir as homedir2, hostname, userInfo } from "os";
|
|
3015
|
+
var ALGORITHM = "aes-256-gcm";
|
|
3016
|
+
var KEY_LENGTH = 32;
|
|
3017
|
+
var IV_LENGTH = 12;
|
|
3018
|
+
var AUTH_TAG_LENGTH = 16;
|
|
3019
|
+
var SALT_FILE = "encryption.salt";
|
|
3020
|
+
function deriveKey(basePath) {
|
|
3021
|
+
const saltPath = join2(basePath, SALT_FILE);
|
|
3022
|
+
let salt;
|
|
3023
|
+
if (existsSync2(saltPath)) {
|
|
3024
|
+
salt = readFileSync(saltPath);
|
|
3025
|
+
} else {
|
|
3026
|
+
salt = randomBytes(32);
|
|
3027
|
+
if (!existsSync2(basePath)) {
|
|
3028
|
+
mkdirSync2(basePath, { recursive: true, mode: 448 });
|
|
3029
|
+
}
|
|
3030
|
+
writeFileSync(saltPath, salt, { mode: 384 });
|
|
3031
|
+
}
|
|
3032
|
+
const machineId = createHash("sha256").update(hostname()).update(userInfo().username).update(homedir2()).digest();
|
|
3033
|
+
return scryptSync(machineId, salt, KEY_LENGTH);
|
|
3034
|
+
}
|
|
3035
|
+
function encrypt(plaintext, key) {
|
|
3036
|
+
const iv = randomBytes(IV_LENGTH);
|
|
3037
|
+
const cipher = createCipheriv(ALGORITHM, key, iv, { authTagLength: AUTH_TAG_LENGTH });
|
|
3038
|
+
const encrypted = Buffer.concat([
|
|
3039
|
+
cipher.update(plaintext, "utf-8"),
|
|
3040
|
+
cipher.final()
|
|
3041
|
+
]);
|
|
3042
|
+
return {
|
|
3043
|
+
iv: iv.toString("base64"),
|
|
3044
|
+
data: encrypted.toString("base64"),
|
|
3045
|
+
tag: cipher.getAuthTag().toString("base64")
|
|
3046
|
+
};
|
|
3047
|
+
}
|
|
3048
|
+
function decrypt(payload, key) {
|
|
3049
|
+
const iv = Buffer.from(payload.iv, "base64");
|
|
3050
|
+
const data = Buffer.from(payload.data, "base64");
|
|
3051
|
+
const tag = Buffer.from(payload.tag, "base64");
|
|
3052
|
+
const decipher = createDecipheriv(ALGORITHM, key, iv, { authTagLength: AUTH_TAG_LENGTH });
|
|
3053
|
+
decipher.setAuthTag(tag);
|
|
3054
|
+
return Buffer.concat([
|
|
3055
|
+
decipher.update(data),
|
|
3056
|
+
decipher.final()
|
|
3057
|
+
]).toString("utf-8");
|
|
3058
|
+
}
|
|
3059
|
+
|
|
3060
|
+
// src/credentials/local-store.ts
|
|
3061
|
+
import Database from "better-sqlite3";
|
|
3062
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3 } from "fs";
|
|
3063
|
+
import { join as join3 } from "path";
|
|
3064
|
+
import { homedir as homedir3 } from "os";
|
|
3065
|
+
var DEFAULT_DB_DIR = join3(homedir3(), ".config", "assistme");
|
|
3066
|
+
var DEFAULT_DB_NAME = "local.db";
|
|
3067
|
+
var LocalStore = class {
|
|
3068
|
+
db;
|
|
3069
|
+
dbPath;
|
|
3070
|
+
constructor(dbPath) {
|
|
3071
|
+
const dir = dbPath ? dbPath : DEFAULT_DB_DIR;
|
|
3072
|
+
if (!existsSync3(dir)) {
|
|
3073
|
+
mkdirSync3(dir, { recursive: true, mode: 448 });
|
|
3074
|
+
}
|
|
3075
|
+
this.dbPath = dbPath ? join3(dbPath, DEFAULT_DB_NAME) : join3(DEFAULT_DB_DIR, DEFAULT_DB_NAME);
|
|
3076
|
+
this.db = new Database(this.dbPath);
|
|
3077
|
+
this.db.pragma("journal_mode = WAL");
|
|
3078
|
+
this.db.pragma("foreign_keys = ON");
|
|
3079
|
+
this.migrate();
|
|
3080
|
+
}
|
|
3081
|
+
/** Run schema migrations. Idempotent — safe to call on every startup. */
|
|
3082
|
+
migrate() {
|
|
3083
|
+
this.db.exec(`
|
|
3084
|
+
CREATE TABLE IF NOT EXISTS credentials (
|
|
3085
|
+
id TEXT PRIMARY KEY,
|
|
3086
|
+
name TEXT NOT NULL UNIQUE,
|
|
3087
|
+
type TEXT NOT NULL DEFAULT 'secret',
|
|
3088
|
+
skill_name TEXT,
|
|
3089
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
3090
|
+
encrypted_data TEXT NOT NULL,
|
|
3091
|
+
created_at TEXT NOT NULL,
|
|
3092
|
+
updated_at TEXT NOT NULL
|
|
3093
|
+
);
|
|
3094
|
+
|
|
3095
|
+
CREATE INDEX IF NOT EXISTS idx_credentials_name ON credentials(name);
|
|
3096
|
+
CREATE INDEX IF NOT EXISTS idx_credentials_skill ON credentials(skill_name);
|
|
3097
|
+
CREATE INDEX IF NOT EXISTS idx_credentials_type ON credentials(type);
|
|
3098
|
+
|
|
3099
|
+
CREATE TABLE IF NOT EXISTS programs (
|
|
3100
|
+
id TEXT PRIMARY KEY,
|
|
3101
|
+
name TEXT NOT NULL UNIQUE,
|
|
3102
|
+
description TEXT NOT NULL DEFAULT '',
|
|
3103
|
+
language TEXT NOT NULL DEFAULT 'unknown',
|
|
3104
|
+
directory TEXT NOT NULL,
|
|
3105
|
+
skill_name TEXT,
|
|
3106
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
3107
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
3108
|
+
metadata TEXT NOT NULL DEFAULT '{}',
|
|
3109
|
+
created_at TEXT NOT NULL,
|
|
3110
|
+
updated_at TEXT NOT NULL
|
|
3111
|
+
);
|
|
3112
|
+
|
|
3113
|
+
CREATE INDEX IF NOT EXISTS idx_programs_name ON programs(name);
|
|
3114
|
+
CREATE INDEX IF NOT EXISTS idx_programs_skill ON programs(skill_name);
|
|
3115
|
+
CREATE INDEX IF NOT EXISTS idx_programs_status ON programs(status);
|
|
3116
|
+
`);
|
|
3117
|
+
}
|
|
3118
|
+
/** Get the raw database handle for direct queries. */
|
|
3119
|
+
getDb() {
|
|
3120
|
+
return this.db;
|
|
3121
|
+
}
|
|
3122
|
+
/** Close the database connection. */
|
|
3123
|
+
close() {
|
|
3124
|
+
this.db.close();
|
|
3125
|
+
}
|
|
3126
|
+
};
|
|
3127
|
+
var _instance = null;
|
|
3128
|
+
function getLocalStore(dbPath) {
|
|
3129
|
+
if (!_instance) {
|
|
3130
|
+
_instance = new LocalStore(dbPath);
|
|
3131
|
+
}
|
|
3132
|
+
return _instance;
|
|
3133
|
+
}
|
|
3134
|
+
|
|
3135
|
+
// src/credentials/credential-store.ts
|
|
3136
|
+
var CredentialStore = class {
|
|
3137
|
+
store;
|
|
3138
|
+
encryptionKey;
|
|
3139
|
+
constructor(dbPath) {
|
|
3140
|
+
this.store = getLocalStore(dbPath);
|
|
3141
|
+
this.encryptionKey = deriveKey(dirname(this.store.dbPath));
|
|
3142
|
+
}
|
|
3143
|
+
// ── CRUD ────────────────────────────────────────────────────────
|
|
3144
|
+
save(name, type, data, opts) {
|
|
3145
|
+
const db = this.store.getDb();
|
|
3146
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3147
|
+
const encryptedData = this.encryptData(data);
|
|
3148
|
+
const tags = JSON.stringify(opts?.tags || []);
|
|
3149
|
+
const existing = db.prepare("SELECT id FROM credentials WHERE name = ?").get(name);
|
|
3150
|
+
if (existing) {
|
|
3151
|
+
db.prepare(`
|
|
3152
|
+
UPDATE credentials
|
|
3153
|
+
SET type = ?, skill_name = ?, tags = ?, encrypted_data = ?, updated_at = ?
|
|
3154
|
+
WHERE id = ?
|
|
3155
|
+
`).run(type, opts?.skillName ?? null, tags, encryptedData, now, existing.id);
|
|
3156
|
+
log.debug(`Credential "${name}" updated (${existing.id})`);
|
|
3157
|
+
return this.toMeta({ id: existing.id, name, type, skill_name: opts?.skillName ?? null, tags, encrypted_data: encryptedData, created_at: now, updated_at: now });
|
|
3158
|
+
}
|
|
3159
|
+
const id = randomUUID();
|
|
3160
|
+
db.prepare(`
|
|
3161
|
+
INSERT INTO credentials (id, name, type, skill_name, tags, encrypted_data, created_at, updated_at)
|
|
3162
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
3163
|
+
`).run(id, name, type, opts?.skillName ?? null, tags, encryptedData, now, now);
|
|
3164
|
+
log.debug(`Credential "${name}" saved (${id})`);
|
|
3165
|
+
return { id, name, type, skillName: opts?.skillName, tags: opts?.tags || [], createdAt: now, updatedAt: now };
|
|
3166
|
+
}
|
|
3167
|
+
get(id) {
|
|
3168
|
+
const row = this.store.getDb().prepare("SELECT * FROM credentials WHERE id = ?").get(id);
|
|
3169
|
+
return row ? this.toCredential(row) : null;
|
|
3170
|
+
}
|
|
3171
|
+
getByName(name) {
|
|
3172
|
+
const row = this.store.getDb().prepare("SELECT * FROM credentials WHERE name = ?").get(name);
|
|
3173
|
+
return row ? this.toCredential(row) : null;
|
|
3174
|
+
}
|
|
3175
|
+
update(id, data) {
|
|
3176
|
+
const row = this.store.getDb().prepare("SELECT * FROM credentials WHERE id = ?").get(id);
|
|
3177
|
+
if (!row) return null;
|
|
3178
|
+
const existing = this.decryptData(row.encrypted_data);
|
|
3179
|
+
const merged = { ...existing };
|
|
3180
|
+
for (const [key, value] of Object.entries(data)) {
|
|
3181
|
+
if (value !== void 0) {
|
|
3182
|
+
merged[key] = value;
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3186
|
+
const encryptedData = this.encryptData(merged);
|
|
3187
|
+
this.store.getDb().prepare("UPDATE credentials SET encrypted_data = ?, updated_at = ? WHERE id = ?").run(encryptedData, now, id);
|
|
3188
|
+
log.debug(`Credential "${row.name}" updated`);
|
|
3189
|
+
return this.toMeta({ ...row, encrypted_data: encryptedData, updated_at: now });
|
|
3190
|
+
}
|
|
3191
|
+
remove(id) {
|
|
3192
|
+
const result = this.store.getDb().prepare("DELETE FROM credentials WHERE id = ?").run(id);
|
|
3193
|
+
if (result.changes > 0) {
|
|
3194
|
+
log.debug(`Credential ${id} removed`);
|
|
3195
|
+
return true;
|
|
3196
|
+
}
|
|
3197
|
+
return false;
|
|
3198
|
+
}
|
|
3199
|
+
removeByName(name) {
|
|
3200
|
+
const result = this.store.getDb().prepare("DELETE FROM credentials WHERE name = ?").run(name);
|
|
3201
|
+
if (result.changes > 0) {
|
|
3202
|
+
log.debug(`Credential "${name}" removed`);
|
|
3203
|
+
return true;
|
|
3204
|
+
}
|
|
3205
|
+
return false;
|
|
3206
|
+
}
|
|
3207
|
+
// ── Query ───────────────────────────────────────────────────────
|
|
3208
|
+
list() {
|
|
3209
|
+
const rows = this.store.getDb().prepare("SELECT * FROM credentials ORDER BY updated_at DESC").all();
|
|
3210
|
+
return rows.map((r) => this.toMeta(r));
|
|
3211
|
+
}
|
|
3212
|
+
findBySkill(skillName) {
|
|
3213
|
+
const rows = this.store.getDb().prepare("SELECT * FROM credentials WHERE skill_name = ? ORDER BY name").all(skillName);
|
|
3214
|
+
return rows.map((r) => this.toMeta(r));
|
|
3215
|
+
}
|
|
3216
|
+
findByTag(tag) {
|
|
3217
|
+
const rows = this.store.getDb().prepare("SELECT * FROM credentials WHERE tags LIKE ? ORDER BY name").all(`%${tag.toLowerCase()}%`);
|
|
3218
|
+
return rows.filter((r) => {
|
|
3219
|
+
const tags = JSON.parse(r.tags);
|
|
3220
|
+
return tags.some((t) => t.toLowerCase() === tag.toLowerCase());
|
|
3221
|
+
}).map((r) => this.toMeta(r));
|
|
3222
|
+
}
|
|
3223
|
+
findByType(type) {
|
|
3224
|
+
const rows = this.store.getDb().prepare("SELECT * FROM credentials WHERE type = ? ORDER BY name").all(type);
|
|
3225
|
+
return rows.map((r) => this.toMeta(r));
|
|
3226
|
+
}
|
|
3227
|
+
// ── Bulk ────────────────────────────────────────────────────────
|
|
3228
|
+
removeBySkill(skillName) {
|
|
3229
|
+
const result = this.store.getDb().prepare("DELETE FROM credentials WHERE skill_name = ?").run(skillName);
|
|
3230
|
+
return result.changes;
|
|
3231
|
+
}
|
|
3232
|
+
clear() {
|
|
3233
|
+
this.store.getDb().prepare("DELETE FROM credentials").run();
|
|
3234
|
+
}
|
|
3235
|
+
// ── Internal ────────────────────────────────────────────────────
|
|
3236
|
+
encryptData(data) {
|
|
3237
|
+
const payload = encrypt(JSON.stringify(data), this.encryptionKey);
|
|
3238
|
+
return JSON.stringify(payload);
|
|
3239
|
+
}
|
|
3240
|
+
decryptData(encrypted) {
|
|
3241
|
+
const payload = JSON.parse(encrypted);
|
|
3242
|
+
const decrypted = decrypt(payload, this.encryptionKey);
|
|
3243
|
+
return JSON.parse(decrypted);
|
|
3244
|
+
}
|
|
3245
|
+
toMeta(row) {
|
|
3246
|
+
return {
|
|
3247
|
+
id: row.id,
|
|
3248
|
+
name: row.name,
|
|
3249
|
+
type: row.type,
|
|
3250
|
+
skillName: row.skill_name || void 0,
|
|
3251
|
+
tags: JSON.parse(row.tags),
|
|
3252
|
+
createdAt: row.created_at,
|
|
3253
|
+
updatedAt: row.updated_at
|
|
3254
|
+
};
|
|
3255
|
+
}
|
|
3256
|
+
toCredential(row) {
|
|
3257
|
+
try {
|
|
3258
|
+
return {
|
|
3259
|
+
meta: this.toMeta(row),
|
|
3260
|
+
data: this.decryptData(row.encrypted_data)
|
|
3261
|
+
};
|
|
3262
|
+
} catch (err) {
|
|
3263
|
+
log.debug(`Failed to decrypt credential ${row.id}: ${err}`);
|
|
3264
|
+
return null;
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
3267
|
+
};
|
|
3268
|
+
var _instance2 = null;
|
|
3269
|
+
function getCredentialStore() {
|
|
3270
|
+
if (!_instance2) {
|
|
3271
|
+
_instance2 = new CredentialStore();
|
|
3272
|
+
}
|
|
3273
|
+
return _instance2;
|
|
3274
|
+
}
|
|
3275
|
+
|
|
3276
|
+
// src/tools/shell.ts
|
|
3277
|
+
import { exec } from "child_process";
|
|
3278
|
+
var BLOCKED_PATTERNS = [
|
|
3279
|
+
/rm\s+(-\w*\s+)*-\w*r\w*\s+\/($|\s)/i,
|
|
3280
|
+
// rm -rf /, rm -fr /, etc.
|
|
3281
|
+
/rm\s+(-\w*\s+)*-\w*r\w*\s+~($|\s|\/)/i,
|
|
3282
|
+
// rm -rf ~, rm -fr ~/, etc.
|
|
3283
|
+
/\bmkfs\b/i,
|
|
3284
|
+
// mkfs (any form)
|
|
3285
|
+
/\bdd\s+.*\bif=/i,
|
|
3286
|
+
// dd if=
|
|
3287
|
+
/:\s*\(\s*\)\s*\{/,
|
|
3288
|
+
// fork bomb :(){}
|
|
3289
|
+
/\bchmod\s+(-\w+\s+)*-R\s+777\s+\//i,
|
|
3290
|
+
// chmod -R 777 /
|
|
3291
|
+
/>\s*\/dev\/sd[a-z]/i,
|
|
3292
|
+
// write to raw disk
|
|
3293
|
+
/\bshutdown\b/i,
|
|
3294
|
+
// shutdown
|
|
3295
|
+
/\breboot\b/i,
|
|
3296
|
+
// reboot
|
|
3297
|
+
/\bsystemctl\s+(start|stop|disable|mask)\b/i
|
|
3298
|
+
// dangerous systemctl ops
|
|
3299
|
+
];
|
|
3300
|
+
function isBlocked(command) {
|
|
3301
|
+
return BLOCKED_PATTERNS.some((pattern) => pattern.test(command));
|
|
3302
|
+
}
|
|
3303
|
+
async function executeShell(command, cwd) {
|
|
3304
|
+
if (isBlocked(command)) {
|
|
3305
|
+
throw new AppError(`Command blocked for safety: "${command}"`, "COMMAND_BLOCKED");
|
|
3306
|
+
}
|
|
3307
|
+
const config = getConfig();
|
|
3308
|
+
const workDir = cwd || config.workspacePath;
|
|
3309
|
+
return new Promise((resolve) => {
|
|
3310
|
+
exec(
|
|
3311
|
+
command,
|
|
3312
|
+
{
|
|
3313
|
+
cwd: workDir,
|
|
3314
|
+
timeout: SHELL_TIMEOUT_MS,
|
|
3315
|
+
maxBuffer: 1024 * 1024,
|
|
3316
|
+
// 1MB buffer
|
|
3317
|
+
env: { ...process.env, TERM: "dumb" }
|
|
3318
|
+
},
|
|
3319
|
+
(error, stdout, stderr) => {
|
|
3320
|
+
let output = "";
|
|
3321
|
+
if (stdout) {
|
|
3322
|
+
output += stdout;
|
|
3323
|
+
}
|
|
3324
|
+
if (stderr) {
|
|
3325
|
+
output += stderr ? `
|
|
3326
|
+
[stderr]
|
|
3327
|
+
${stderr}` : "";
|
|
3328
|
+
}
|
|
3329
|
+
if (error && !stdout && !stderr) {
|
|
3330
|
+
output = `Error: ${error.message}`;
|
|
3331
|
+
}
|
|
3332
|
+
if (output.length > SHELL_MAX_OUTPUT) {
|
|
3333
|
+
output = output.slice(0, SHELL_MAX_OUTPUT) + `
|
|
3334
|
+
|
|
3335
|
+
[Output truncated at ${SHELL_MAX_OUTPUT} bytes]`;
|
|
3336
|
+
}
|
|
3337
|
+
resolve(output || "(no output)");
|
|
3338
|
+
}
|
|
3339
|
+
);
|
|
3340
|
+
});
|
|
3341
|
+
}
|
|
3342
|
+
|
|
3343
|
+
export {
|
|
3344
|
+
loginWithToken,
|
|
3345
|
+
getCurrentUserId,
|
|
3346
|
+
logout,
|
|
3347
|
+
createSession,
|
|
3348
|
+
updateHeartbeat,
|
|
3349
|
+
endSession,
|
|
3350
|
+
setSessionBusy,
|
|
3351
|
+
cleanupStaleSessions,
|
|
3352
|
+
getActiveSessions,
|
|
3353
|
+
createTask,
|
|
3354
|
+
pollAndClaimTask,
|
|
3355
|
+
claimTask,
|
|
3356
|
+
completeTask,
|
|
3357
|
+
failTask,
|
|
3358
|
+
getOrCreateCliConversation,
|
|
3359
|
+
getConversationHistory,
|
|
3360
|
+
resetEventSequence,
|
|
3361
|
+
emitEvent,
|
|
3362
|
+
setActionRequest,
|
|
3363
|
+
pollActionResponse,
|
|
3364
|
+
pollAndClaimJobRun,
|
|
3365
|
+
BrowserController,
|
|
3366
|
+
ensureBrowserAvailable,
|
|
3367
|
+
getBrowser,
|
|
3368
|
+
getNextRunTime,
|
|
3369
|
+
Scheduler,
|
|
3370
|
+
createScheduledTask,
|
|
3371
|
+
listScheduledTasks,
|
|
3372
|
+
toggleScheduledTask,
|
|
3373
|
+
deleteScheduledTask,
|
|
3374
|
+
executeShell,
|
|
3375
|
+
MemoryManager,
|
|
3376
|
+
SkillManager,
|
|
3377
|
+
validateSkillName,
|
|
3378
|
+
normalizeSkillName,
|
|
3379
|
+
substituteArguments,
|
|
3380
|
+
preprocessDynamicContext,
|
|
3381
|
+
getLocalStore,
|
|
3382
|
+
getCredentialStore
|
|
3383
|
+
};
|