amalgm 0.1.144 → 0.1.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/apps/supervisor.js +142 -25
- package/runtime/scripts/amalgm-mcp/browser/auth-tools.js +49 -10
- package/runtime/scripts/amalgm-mcp/browser/cua-tools.js +84 -111
- package/runtime/scripts/amalgm-mcp/browser/delta.js +254 -0
- package/runtime/scripts/amalgm-mcp/browser/tools.js +166 -12
- package/runtime/scripts/amalgm-mcp/lib/tool-result.js +13 -0
- package/runtime/scripts/amalgm-mcp/tests/apps-crash-loop.test.js +85 -0
- package/runtime/scripts/amalgm-mcp/tests/browser-tools-surface.test.js +165 -0
- package/runtime/scripts/amalgm-mcp/tests/core-tools.test.js +18 -15
- package/runtime/scripts/amalgm-mcp/toolbox/loadout-context.js +7 -1
- package/runtime/scripts/chat-core/tool-shape.js +10 -1
- package/runtime/scripts/platform-context.txt +5 -2
package/package.json
CHANGED
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
|
|
8
8
|
const fs = require('fs');
|
|
9
9
|
const path = require('path');
|
|
10
|
-
const { spawn
|
|
11
|
-
const { DEFAULT_CWD } = require('../config');
|
|
10
|
+
const { spawn } = require('child_process');
|
|
11
|
+
const { APPS_DIR, DEFAULT_CWD } = require('../config');
|
|
12
12
|
const { runtimePort } = require('../../../lib/runtime-manifest');
|
|
13
13
|
const { syncAppRoutesToGateway } = require('./advertise');
|
|
14
14
|
const {
|
|
@@ -25,6 +25,77 @@ const running = new Map();
|
|
|
25
25
|
const stopping = new Set();
|
|
26
26
|
const restartTimers = new Map();
|
|
27
27
|
|
|
28
|
+
// Crash containment: an app that keeps dying must not keep the supervisor busy
|
|
29
|
+
// forever. Restarts back off exponentially, and after maxCrashes unexpected
|
|
30
|
+
// exits inside windowMs the app is parked as 'degraded' until someone starts
|
|
31
|
+
// it explicitly (or the runtime reboots). Apps are allowed to fail; they are
|
|
32
|
+
// not allowed to consume the runtime.
|
|
33
|
+
const APP_RESTART_POLICY = {
|
|
34
|
+
initialDelayMs: 2000,
|
|
35
|
+
maxDelayMs: 60_000,
|
|
36
|
+
maxCrashes: 5,
|
|
37
|
+
windowMs: 5 * 60_000,
|
|
38
|
+
};
|
|
39
|
+
const crashHistory = new Map();
|
|
40
|
+
|
|
41
|
+
function recordCrash(appId) {
|
|
42
|
+
const now = Date.now();
|
|
43
|
+
const recent = (crashHistory.get(appId) || [])
|
|
44
|
+
.filter((time) => now - time <= APP_RESTART_POLICY.windowMs);
|
|
45
|
+
recent.push(now);
|
|
46
|
+
crashHistory.set(appId, recent);
|
|
47
|
+
return recent.length;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function restartDelayMs(crashCount) {
|
|
51
|
+
const exponent = Math.max(0, crashCount - 1);
|
|
52
|
+
return Math.min(APP_RESTART_POLICY.initialDelayMs * 2 ** exponent, APP_RESTART_POLICY.maxDelayMs);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Per-app log files: app output belongs to the app, not to the runtime's main
|
|
56
|
+
// log. The main log only carries lifecycle events (started/exited/degraded).
|
|
57
|
+
const appLogStreams = new Map();
|
|
58
|
+
|
|
59
|
+
function appLogPath(appId) {
|
|
60
|
+
return path.join(APPS_DIR, 'logs', `${appId}.log`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function appLogStream(appId) {
|
|
64
|
+
let stream = appLogStreams.get(appId);
|
|
65
|
+
if (stream && !stream.destroyed) return stream;
|
|
66
|
+
const file = appLogPath(appId);
|
|
67
|
+
try {
|
|
68
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
69
|
+
try {
|
|
70
|
+
const stats = fs.statSync(file);
|
|
71
|
+
if (stats.size > 5 * 1024 * 1024) fs.renameSync(file, `${file}.old`);
|
|
72
|
+
} catch {}
|
|
73
|
+
stream = fs.createWriteStream(file, { flags: 'a' });
|
|
74
|
+
stream.on('error', () => {});
|
|
75
|
+
appLogStreams.set(appId, stream);
|
|
76
|
+
return stream;
|
|
77
|
+
} catch {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function appendAppLog(appId, source, chunk) {
|
|
83
|
+
const stream = appLogStream(appId);
|
|
84
|
+
if (!stream) return;
|
|
85
|
+
const stamp = new Date().toISOString();
|
|
86
|
+
const text = chunk.toString();
|
|
87
|
+
const suffix = text.endsWith('\n') ? '' : '\n';
|
|
88
|
+
stream.write(`${stamp} [${source}] ${text}${suffix}`);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function closeAppLog(appId) {
|
|
92
|
+
const stream = appLogStreams.get(appId);
|
|
93
|
+
appLogStreams.delete(appId);
|
|
94
|
+
if (stream) {
|
|
95
|
+
try { stream.end(); } catch {}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
28
99
|
const APP_ENV_ALLOWLIST = [
|
|
29
100
|
'AMALGM_BIND_HOST',
|
|
30
101
|
'AMALGM_DIR',
|
|
@@ -92,29 +163,51 @@ function parsePids(output) {
|
|
|
92
163
|
return Array.from(pids);
|
|
93
164
|
}
|
|
94
165
|
|
|
95
|
-
|
|
166
|
+
// Async replacement for spawnSync: these probes (lsof/ps) can take seconds on
|
|
167
|
+
// a loaded machine, and a blocked event loop here stalls /healthz for the whole
|
|
168
|
+
// runtime — which is what turns one crash-looping app into a runtime restart.
|
|
169
|
+
function runProbe(command, args, timeoutMs = 5000) {
|
|
170
|
+
return new Promise((resolve) => {
|
|
171
|
+
let child;
|
|
172
|
+
try {
|
|
173
|
+
child = spawn(command, args, { stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true });
|
|
174
|
+
} catch {
|
|
175
|
+
resolve({ status: -1, stdout: '' });
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
let stdout = '';
|
|
179
|
+
const timer = setTimeout(() => {
|
|
180
|
+
try { child.kill('SIGKILL'); } catch {}
|
|
181
|
+
}, timeoutMs);
|
|
182
|
+
child.stdout?.on('data', (chunk) => { stdout += chunk.toString(); });
|
|
183
|
+
child.on('error', () => {
|
|
184
|
+
clearTimeout(timer);
|
|
185
|
+
resolve({ status: -1, stdout: '' });
|
|
186
|
+
});
|
|
187
|
+
child.on('close', (code) => {
|
|
188
|
+
clearTimeout(timer);
|
|
189
|
+
resolve({ status: code, stdout });
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
async function listeningPidsForPort(port) {
|
|
96
195
|
const parsed = Number(port);
|
|
97
196
|
if (!Number.isInteger(parsed) || parsed <= 0 || parsed > 65535) return [];
|
|
98
|
-
const result =
|
|
99
|
-
|
|
100
|
-
});
|
|
101
|
-
if (result.error || !result.stdout) return [];
|
|
197
|
+
const result = await runProbe('lsof', ['-nP', `-iTCP:${parsed}`, '-sTCP:LISTEN', '-t']);
|
|
198
|
+
if (!result.stdout) return [];
|
|
102
199
|
return parsePids(result.stdout);
|
|
103
200
|
}
|
|
104
201
|
|
|
105
|
-
function pidCommand(pid) {
|
|
106
|
-
const result =
|
|
107
|
-
|
|
108
|
-
});
|
|
109
|
-
if (result.error || result.status !== 0) return '';
|
|
202
|
+
async function pidCommand(pid) {
|
|
203
|
+
const result = await runProbe('ps', ['eww', '-p', String(pid), '-o', 'command=']);
|
|
204
|
+
if (result.status !== 0) return '';
|
|
110
205
|
return String(result.stdout || '');
|
|
111
206
|
}
|
|
112
207
|
|
|
113
|
-
function pidCwd(pid) {
|
|
114
|
-
const result =
|
|
115
|
-
|
|
116
|
-
});
|
|
117
|
-
if (result.error || result.status !== 0) return '';
|
|
208
|
+
async function pidCwd(pid) {
|
|
209
|
+
const result = await runProbe('lsof', ['-a', '-p', String(pid), '-d', 'cwd', '-Fn']);
|
|
210
|
+
if (result.status !== 0) return '';
|
|
118
211
|
const line = String(result.stdout || '').split('\n').find((item) => item.startsWith('n'));
|
|
119
212
|
return line ? line.slice(1) : '';
|
|
120
213
|
}
|
|
@@ -125,25 +218,25 @@ function isPathInside(parent, maybeChild) {
|
|
|
125
218
|
return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
|
|
126
219
|
}
|
|
127
220
|
|
|
128
|
-
function isAppOwnedPid(app, pid) {
|
|
221
|
+
async function isAppOwnedPid(app, pid) {
|
|
129
222
|
if (app.pid === pid) return true;
|
|
130
|
-
const command = pidCommand(pid);
|
|
223
|
+
const command = await pidCommand(pid);
|
|
131
224
|
if (command.includes(`AMALGM_APP_ID=${app.id}`) || command.includes(`AMALGM_APP_ID='${app.id}'`)) return true;
|
|
132
225
|
if (app.appRef && (command.includes(`AMALGM_APP_REF=${app.appRef}`) || command.includes(`AMALGM_APP_REF='${app.appRef}'`))) {
|
|
133
226
|
return true;
|
|
134
227
|
}
|
|
135
228
|
if (app.cwd && command.includes(path.resolve(app.cwd))) return true;
|
|
136
|
-
return isPathInside(app.cwd, pidCwd(pid));
|
|
229
|
+
return isPathInside(app.cwd, await pidCwd(pid));
|
|
137
230
|
}
|
|
138
231
|
|
|
139
232
|
async function killOwnedPortListeners(app, options = {}) {
|
|
140
|
-
const pids = listeningPidsForPort(app.port).filter((pid) => pid !== process.pid);
|
|
233
|
+
const pids = (await listeningPidsForPort(app.port)).filter((pid) => pid !== process.pid);
|
|
141
234
|
if (pids.length === 0) return [];
|
|
142
235
|
|
|
143
236
|
const owned = [];
|
|
144
237
|
const unknown = [];
|
|
145
238
|
for (const pid of pids) {
|
|
146
|
-
(isAppOwnedPid(app, pid) ? owned : unknown).push(pid);
|
|
239
|
+
((await isAppOwnedPid(app, pid)) ? owned : unknown).push(pid);
|
|
147
240
|
}
|
|
148
241
|
|
|
149
242
|
if (unknown.length > 0 && options.failOnUnknown !== false) {
|
|
@@ -245,16 +338,18 @@ function startAppProcess(app) {
|
|
|
245
338
|
|
|
246
339
|
running.set(app.id, child);
|
|
247
340
|
child.unref();
|
|
341
|
+
console.log(`[App:${app.id}] started pid=${child.pid} (output: ${appLogPath(app.id)})`);
|
|
248
342
|
child.stdout?.on('data', (chunk) => {
|
|
249
|
-
|
|
343
|
+
appendAppLog(app.id, 'stdout', chunk);
|
|
250
344
|
});
|
|
251
345
|
child.stderr?.on('data', (chunk) => {
|
|
252
|
-
|
|
346
|
+
appendAppLog(app.id, 'stderr', chunk);
|
|
253
347
|
});
|
|
254
348
|
child.on('exit', (code) => {
|
|
255
349
|
const activeChild = running.get(app.id);
|
|
256
350
|
if (activeChild && activeChild !== child) return;
|
|
257
351
|
running.delete(app.id);
|
|
352
|
+
closeAppLog(app.id);
|
|
258
353
|
const wasStopping = stopping.has(app.id);
|
|
259
354
|
stopping.delete(app.id);
|
|
260
355
|
if (wasStopping) return;
|
|
@@ -263,6 +358,20 @@ function startAppProcess(app) {
|
|
|
263
358
|
if (!current) return;
|
|
264
359
|
|
|
265
360
|
if (current.desiredState === 'running' && current.keepAlive !== false) {
|
|
361
|
+
const crashes = recordCrash(app.id);
|
|
362
|
+
if (crashes >= APP_RESTART_POLICY.maxCrashes) {
|
|
363
|
+
const windowMinutes = Math.round(APP_RESTART_POLICY.windowMs / 60_000);
|
|
364
|
+
console.error(`[App:${app.id}] degraded: crashed ${crashes} times in ${windowMinutes}m; automatic restarts paused (logs: ${appLogPath(app.id)})`);
|
|
365
|
+
updateAppMeta(app.id, {
|
|
366
|
+
status: 'degraded',
|
|
367
|
+
pid: null,
|
|
368
|
+
error: `Crashed ${crashes} times in ${windowMinutes} minutes; automatic restarts paused. Fix the app, then start it again. Logs: ${appLogPath(app.id)}`,
|
|
369
|
+
});
|
|
370
|
+
void syncRoutesBestEffort(`degraded:${app.id}`);
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const delayMs = restartDelayMs(crashes);
|
|
374
|
+
console.error(`[App:${app.id}] exited with code ${code}; restarting in ${Math.round(delayMs / 1000)}s (crash ${crashes}/${APP_RESTART_POLICY.maxCrashes})`);
|
|
266
375
|
updateAppMeta(app.id, {
|
|
267
376
|
status: 'restarting',
|
|
268
377
|
pid: null,
|
|
@@ -274,11 +383,12 @@ function startAppProcess(app) {
|
|
|
274
383
|
startApp(app.id).catch((err) => {
|
|
275
384
|
updateAppMeta(app.id, { status: 'error', error: err.message, pid: null });
|
|
276
385
|
});
|
|
277
|
-
},
|
|
386
|
+
}, delayMs);
|
|
278
387
|
restartTimers.set(app.id, timer);
|
|
279
388
|
return;
|
|
280
389
|
}
|
|
281
390
|
|
|
391
|
+
console.log(`[App:${app.id}] exited with code ${code}`);
|
|
282
392
|
updateAppMeta(app.id, {
|
|
283
393
|
status: code === 0 ? 'stopped' : 'error',
|
|
284
394
|
pid: null,
|
|
@@ -297,6 +407,10 @@ async function startApp(appId) {
|
|
|
297
407
|
if (existingTimer) clearTimeout(existingTimer);
|
|
298
408
|
restartTimers.delete(appId);
|
|
299
409
|
|
|
410
|
+
// A degraded app has no pending restart timer, so reaching here means an
|
|
411
|
+
// explicit start (user/API/boot) — give it a fresh crash budget.
|
|
412
|
+
if (app.status === 'degraded') crashHistory.delete(appId);
|
|
413
|
+
|
|
300
414
|
if (running.has(appId)) {
|
|
301
415
|
const current = updateAppMeta(app.id, {
|
|
302
416
|
desiredState: 'running',
|
|
@@ -362,6 +476,8 @@ async function stopApp(appId, options = {}) {
|
|
|
362
476
|
console.warn(`[Apps] Could not clean listeners for ${appId} on port ${app.port}: ${err.message}`);
|
|
363
477
|
});
|
|
364
478
|
running.delete(appId);
|
|
479
|
+
crashHistory.delete(appId);
|
|
480
|
+
closeAppLog(appId);
|
|
365
481
|
|
|
366
482
|
setTimeout(() => stopping.delete(appId), 1000);
|
|
367
483
|
const current = updateAppMeta(appId, {
|
|
@@ -524,6 +640,7 @@ async function startRegisteredApps() {
|
|
|
524
640
|
}
|
|
525
641
|
|
|
526
642
|
module.exports = {
|
|
643
|
+
APP_RESTART_POLICY,
|
|
527
644
|
deleteApp,
|
|
528
645
|
connectDns,
|
|
529
646
|
disconnectDns,
|
|
@@ -31,21 +31,60 @@ function jsonText(value) {
|
|
|
31
31
|
return textResult(JSON.stringify(value, null, 2));
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/** Every browser session leaves a durable profile behind, so the full lists
|
|
35
|
+
* grow without bound (a stress test found ~130 profiles / 86K chars from one
|
|
36
|
+
* unpaged dump). Summaries are the default; `full` is the opt-in. */
|
|
37
|
+
const SUMMARY_LIMIT = 15;
|
|
38
|
+
|
|
39
|
+
function summarizeEntry(entry) {
|
|
40
|
+
if (!entry || typeof entry !== 'object') return entry;
|
|
41
|
+
const out = {};
|
|
42
|
+
for (const key of ['id', 'name', 'profileId', 'domains', 'status', 'targetUrl', 'lastUsedAt', 'updatedAt', 'createdAt', 'expiresAt']) {
|
|
43
|
+
if (entry[key] !== undefined && entry[key] !== null && entry[key] !== '') out[key] = entry[key];
|
|
44
|
+
}
|
|
45
|
+
if (out.name === out.id) delete out.name;
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function summarizeList(entries) {
|
|
50
|
+
const list = Array.isArray(entries) ? entries : [];
|
|
51
|
+
return {
|
|
52
|
+
total: list.length,
|
|
53
|
+
// Store lists are already sorted most-recent-first.
|
|
54
|
+
recent: list.slice(0, SUMMARY_LIMIT).map(summarizeEntry),
|
|
55
|
+
...(list.length > SUMMARY_LIMIT ? { note: `${list.length - SUMMARY_LIMIT} older entries omitted — pass full: true for everything.` } : {}),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
34
59
|
module.exports = [
|
|
35
60
|
{
|
|
36
61
|
name: 'auth_list',
|
|
37
|
-
description: 'List saved browser auth state on this computer: durable profiles, encrypted auth bundles, and pending login links. Raw cookies and tokens are never returned.',
|
|
38
|
-
inputSchema: {
|
|
39
|
-
|
|
62
|
+
description: 'List saved browser auth state on this computer: durable profiles, encrypted auth bundles, and pending login links. Returns a recent-first summary by default; pass full: true for complete records. Raw cookies and tokens are never returned.',
|
|
63
|
+
inputSchema: {
|
|
64
|
+
type: 'object',
|
|
65
|
+
properties: {
|
|
66
|
+
full: { type: 'boolean', description: 'Return complete records instead of the recent-first summary' },
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
async handler(args = {}) {
|
|
40
70
|
try {
|
|
71
|
+
const cookieSource = {
|
|
72
|
+
bundleId: COOKIE_SOURCE_BUNDLE_ID,
|
|
73
|
+
profileId: DEFAULT_COOKIE_SOURCE_PROFILE_ID,
|
|
74
|
+
};
|
|
75
|
+
if (args.full) {
|
|
76
|
+
return jsonText({
|
|
77
|
+
profiles: listBrowserProfiles(),
|
|
78
|
+
bundles: listBrowserAuthBundles(),
|
|
79
|
+
loginSessions: listBrowserLoginSessions(),
|
|
80
|
+
cookieSource,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
41
83
|
return jsonText({
|
|
42
|
-
profiles: listBrowserProfiles(),
|
|
43
|
-
bundles: listBrowserAuthBundles(),
|
|
44
|
-
loginSessions: listBrowserLoginSessions(),
|
|
45
|
-
cookieSource
|
|
46
|
-
bundleId: COOKIE_SOURCE_BUNDLE_ID,
|
|
47
|
-
profileId: DEFAULT_COOKIE_SOURCE_PROFILE_ID,
|
|
48
|
-
},
|
|
84
|
+
profiles: summarizeList(listBrowserProfiles()),
|
|
85
|
+
bundles: summarizeList(listBrowserAuthBundles()),
|
|
86
|
+
loginSessions: summarizeList(listBrowserLoginSessions()),
|
|
87
|
+
cookieSource,
|
|
49
88
|
});
|
|
50
89
|
} catch (err) {
|
|
51
90
|
return errorResult(`auth_list failed: ${err.message}`);
|
|
@@ -1,147 +1,120 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* CUA (computer-use)
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* CUA (computer-use) — coordinate-based browser control as ONE tool with an
|
|
5
|
+
* action field, not eight. Agents need this maybe 5% of the time (canvas,
|
|
6
|
+
* webGL, hostile UIs, vision loops); it doesn't deserve eight schema slots.
|
|
7
|
+
*
|
|
8
|
+
* The contract that matters: cua screenshot coordinates map 1:1 to the x/y
|
|
9
|
+
* every other action takes. For normal pages the snapshot/@ref tools are
|
|
10
|
+
* faster and more reliable.
|
|
11
|
+
*
|
|
12
|
+
* Loadouts saved when this was eight actions (cua_click, cua_type, ...) keep
|
|
13
|
+
* working — toolbox/loadout-context.js maps the old grants onto `cua`.
|
|
8
14
|
*/
|
|
9
15
|
|
|
10
16
|
const { textResult, errorResult } = require('../lib/tool-result');
|
|
11
17
|
const engine = require('./engine');
|
|
18
|
+
const delta = require('./delta');
|
|
12
19
|
|
|
13
20
|
const sessionProperty = {
|
|
14
21
|
session: { type: 'string', description: 'Browser session/profile id. Defaults to your chat session.' },
|
|
15
22
|
};
|
|
16
23
|
|
|
24
|
+
const ACTIONS = ['screenshot', 'click', 'double_click', 'move', 'scroll', 'type', 'keypress', 'drag'];
|
|
25
|
+
|
|
17
26
|
function coordinate(value, name) {
|
|
18
27
|
const number = Number(value);
|
|
19
28
|
if (!Number.isFinite(number)) throw new Error(`${name} must be a finite number`);
|
|
20
29
|
return number;
|
|
21
30
|
}
|
|
22
31
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return errorResult(`${action} failed: ${err.message}`);
|
|
31
|
-
}
|
|
32
|
-
};
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
module.exports = [
|
|
36
|
-
{
|
|
37
|
-
name: 'cua_screenshot',
|
|
38
|
-
description: 'Capture the browser viewport for a vision loop. Coordinates in the image map 1:1 to cua_click/cua_move coordinates.',
|
|
39
|
-
inputSchema: { type: 'object', properties: sessionProperty },
|
|
40
|
-
async handler(args, ctx) {
|
|
41
|
-
try {
|
|
42
|
-
const capture = await engine.screenshot({ ...args, fullPage: false }, ctx);
|
|
43
|
-
return {
|
|
44
|
-
content: [
|
|
45
|
-
{ type: 'image', data: capture.base64, mimeType: 'image/png' },
|
|
46
|
-
{ type: 'text', text: `Viewport screenshot (${capture.bytes} bytes)` },
|
|
47
|
-
],
|
|
48
|
-
};
|
|
49
|
-
} catch (err) {
|
|
50
|
-
return errorResult(`cua_screenshot failed: ${err.message}`);
|
|
51
|
-
}
|
|
52
|
-
},
|
|
53
|
-
},
|
|
54
|
-
{
|
|
55
|
-
name: 'cua_click',
|
|
56
|
-
description: 'Click viewport coordinates.',
|
|
57
|
-
inputSchema: {
|
|
58
|
-
type: 'object',
|
|
59
|
-
properties: {
|
|
60
|
-
x: { type: 'number' },
|
|
61
|
-
y: { type: 'number' },
|
|
62
|
-
button: { type: 'number', description: '1 left (default), 2 middle, 3 right' },
|
|
63
|
-
...sessionProperty,
|
|
64
|
-
},
|
|
65
|
-
required: ['x', 'y'],
|
|
66
|
-
},
|
|
67
|
-
handler: handlerFor('cua_click', (args) => ({ ...args, x: coordinate(args.x, 'x'), y: coordinate(args.y, 'y') })),
|
|
68
|
-
},
|
|
69
|
-
{
|
|
70
|
-
name: 'cua_double_click',
|
|
71
|
-
description: 'Double-click viewport coordinates.',
|
|
72
|
-
inputSchema: {
|
|
73
|
-
type: 'object',
|
|
74
|
-
properties: { x: { type: 'number' }, y: { type: 'number' }, ...sessionProperty },
|
|
75
|
-
required: ['x', 'y'],
|
|
76
|
-
},
|
|
77
|
-
handler: handlerFor('cua_double_click', (args) => ({ ...args, x: coordinate(args.x, 'x'), y: coordinate(args.y, 'y') })),
|
|
78
|
-
},
|
|
79
|
-
{
|
|
80
|
-
name: 'cua_move',
|
|
81
|
-
description: 'Move the mouse to viewport coordinates.',
|
|
82
|
-
inputSchema: {
|
|
83
|
-
type: 'object',
|
|
84
|
-
properties: { x: { type: 'number' }, y: { type: 'number' }, ...sessionProperty },
|
|
85
|
-
required: ['x', 'y'],
|
|
86
|
-
},
|
|
87
|
-
handler: handlerFor('cua_move', (args) => ({ ...args, x: coordinate(args.x, 'x'), y: coordinate(args.y, 'y') })),
|
|
88
|
-
},
|
|
89
|
-
{
|
|
90
|
-
name: 'cua_scroll',
|
|
91
|
-
description: 'Scroll from viewport coordinates by scrollX/scrollY pixels.',
|
|
92
|
-
inputSchema: {
|
|
93
|
-
type: 'object',
|
|
94
|
-
properties: {
|
|
95
|
-
x: { type: 'number' },
|
|
96
|
-
y: { type: 'number' },
|
|
97
|
-
scrollX: { type: 'number' },
|
|
98
|
-
scrollY: { type: 'number' },
|
|
99
|
-
...sessionProperty,
|
|
100
|
-
},
|
|
101
|
-
required: ['x', 'y', 'scrollX', 'scrollY'],
|
|
102
|
-
},
|
|
103
|
-
handler: handlerFor('cua_scroll', (args) => ({
|
|
32
|
+
/** Per-action argument validation/coercion. Throws with a usable message. */
|
|
33
|
+
function prepare(action, args) {
|
|
34
|
+
if (action === 'click' || action === 'double_click' || action === 'move') {
|
|
35
|
+
return { ...args, x: coordinate(args.x, 'x'), y: coordinate(args.y, 'y') };
|
|
36
|
+
}
|
|
37
|
+
if (action === 'scroll') {
|
|
38
|
+
return {
|
|
104
39
|
...args,
|
|
105
40
|
x: coordinate(args.x, 'x'),
|
|
106
41
|
y: coordinate(args.y, 'y'),
|
|
107
42
|
scrollX: Number(args.scrollX || 0),
|
|
108
43
|
scrollY: Number(args.scrollY || 0),
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
{
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
if (action === 'type') {
|
|
47
|
+
if (typeof args.text !== 'string') throw new Error('text is required for type');
|
|
48
|
+
return args;
|
|
49
|
+
}
|
|
50
|
+
if (action === 'keypress') {
|
|
51
|
+
if (!Array.isArray(args.keys) || !args.keys.length) throw new Error('keys is required for keypress, e.g. ["Enter"]');
|
|
52
|
+
return args;
|
|
53
|
+
}
|
|
54
|
+
if (action === 'drag') {
|
|
55
|
+
if (!Array.isArray(args.path) || args.path.length < 2) throw new Error('path must contain at least two {x, y} points');
|
|
56
|
+
return args;
|
|
57
|
+
}
|
|
58
|
+
return args;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Actions that plausibly changed the page and deserve a result delta. */
|
|
62
|
+
const DELTA_ACTIONS = new Set(['click', 'double_click', 'keypress', 'drag']);
|
|
63
|
+
|
|
64
|
+
module.exports = [
|
|
131
65
|
{
|
|
132
|
-
name: '
|
|
133
|
-
description: '
|
|
66
|
+
name: 'cua',
|
|
67
|
+
description: 'Coordinate-based browser control for vision loops — one tool, pick an action: '
|
|
68
|
+
+ 'screenshot (capture the viewport; its pixel coordinates map 1:1 to x/y here), '
|
|
69
|
+
+ 'click / double_click / move (x, y), scroll (x, y + scrollX/scrollY), '
|
|
70
|
+
+ 'type (text, real keystrokes at the focused element), keypress (keys array like ["Enter"]), '
|
|
71
|
+
+ 'drag (path of {x,y} points). '
|
|
72
|
+
+ 'Use only when snapshot/@refs cannot see the UI (canvas, webGL, custom widgets); refs are faster and more reliable. '
|
|
73
|
+
+ 'In the desktop app, text/key actions are scoped inside the target page so they cannot type into Amalgm\'s own chat UI.',
|
|
134
74
|
inputSchema: {
|
|
135
75
|
type: 'object',
|
|
136
76
|
properties: {
|
|
77
|
+
action: { type: 'string', enum: ACTIONS, description: 'Which computer-use action to perform' },
|
|
78
|
+
x: { type: 'number', description: 'Viewport x for click/double_click/move/scroll' },
|
|
79
|
+
y: { type: 'number', description: 'Viewport y for click/double_click/move/scroll' },
|
|
80
|
+
button: { type: 'number', description: 'Mouse button for click: 1 left (default), 2 middle, 3 right' },
|
|
81
|
+
scrollX: { type: 'number', description: 'Horizontal scroll delta in px (scroll)' },
|
|
82
|
+
scrollY: { type: 'number', description: 'Vertical scroll delta in px (scroll)' },
|
|
83
|
+
text: { type: 'string', description: 'Text to type (type)' },
|
|
84
|
+
keys: { type: 'array', items: { type: 'string' }, description: 'Key combination, e.g. ["Control", "a"] or ["Enter"] (keypress)' },
|
|
137
85
|
path: {
|
|
138
86
|
type: 'array',
|
|
139
87
|
items: { type: 'object', properties: { x: { type: 'number' }, y: { type: 'number' } }, required: ['x', 'y'] },
|
|
88
|
+
description: 'Points to drag the mouse through (drag)',
|
|
140
89
|
},
|
|
141
90
|
...sessionProperty,
|
|
142
91
|
},
|
|
143
|
-
required: ['
|
|
92
|
+
required: ['action'],
|
|
93
|
+
},
|
|
94
|
+
async handler(args, ctx) {
|
|
95
|
+
const action = String(args.action || '');
|
|
96
|
+
if (!ACTIONS.includes(action)) {
|
|
97
|
+
return errorResult(`action must be one of: ${ACTIONS.join(', ')}`);
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
if (action === 'screenshot') {
|
|
101
|
+
const capture = await engine.screenshot({ ...args, fullPage: false }, ctx);
|
|
102
|
+
return {
|
|
103
|
+
content: [
|
|
104
|
+
{ type: 'image', data: capture.base64, mimeType: 'image/png' },
|
|
105
|
+
{ type: 'text', text: `Viewport screenshot (${capture.bytes} bytes). Coordinates map 1:1 to cua x/y.` },
|
|
106
|
+
],
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const result = await engine.cua(`cua_${action}`, prepare(action, args), ctx);
|
|
110
|
+
if (DELTA_ACTIONS.has(action)) {
|
|
111
|
+
const after = await delta.describeAfter(result.session, ctx);
|
|
112
|
+
return textResult([`cua ${action} ok`, after].filter(Boolean).join('\n'));
|
|
113
|
+
}
|
|
114
|
+
return textResult(`cua ${action} ok`);
|
|
115
|
+
} catch (err) {
|
|
116
|
+
return errorResult(`cua ${action} failed: ${err.message}`);
|
|
117
|
+
}
|
|
144
118
|
},
|
|
145
|
-
handler: handlerFor('cua_drag'),
|
|
146
119
|
},
|
|
147
120
|
];
|