amalgm 0.1.88 → 0.1.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.js +16 -6
- package/lib/service.js +19 -0
- package/lib/supervisor.js +51 -0
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/apps/supervisor.js +0 -25
- package/runtime/scripts/amalgm-mcp/browser/auth-tools.js +179 -0
- package/runtime/scripts/amalgm-mcp/browser/cli.js +217 -0
- package/runtime/scripts/amalgm-mcp/browser/cua-tools.js +147 -0
- package/runtime/scripts/amalgm-mcp/browser/engine.js +454 -0
- package/runtime/scripts/amalgm-mcp/browser/recorder-tools.js +82 -0
- package/runtime/scripts/amalgm-mcp/browser/recorder.js +199 -0
- package/runtime/scripts/amalgm-mcp/browser/rest.js +5 -1
- package/runtime/scripts/amalgm-mcp/browser/tools.js +104 -731
- package/runtime/scripts/amalgm-mcp/fs/rest.js +5 -6
- package/runtime/scripts/amalgm-mcp/index.js +2 -0
- package/runtime/scripts/amalgm-mcp/lib/tool-result.js +32 -66
- package/runtime/scripts/amalgm-mcp/project-context/paths.js +266 -0
- package/runtime/scripts/amalgm-mcp/project-context/prompt.js +232 -0
- package/runtime/scripts/amalgm-mcp/project-context/rest.js +100 -0
- package/runtime/scripts/amalgm-mcp/project-context/store.js +671 -0
- package/runtime/scripts/amalgm-mcp/project-context/tools.js +146 -0
- package/runtime/scripts/amalgm-mcp/server/core-tools.js +35 -2
- package/runtime/scripts/amalgm-mcp/server/local-service-router.js +2 -0
- package/runtime/scripts/amalgm-mcp/server/routes/project-context.js +33 -0
- package/runtime/scripts/amalgm-mcp/state/db.js +11 -0
- package/runtime/scripts/amalgm-mcp/state/snapshot.js +3 -3
- package/runtime/scripts/amalgm-mcp/tests/core-tools.test.js +37 -4
- package/runtime/scripts/amalgm-mcp/tests/project-context.test.js +328 -0
- package/runtime/scripts/amalgm-mcp/tests/system-catalog.test.js +9 -9
- package/runtime/scripts/amalgm-mcp/tests/toolbox-service.test.js +2 -2
- package/runtime/scripts/amalgm-mcp/toolbox/loadout-context.js +1 -0
- package/runtime/scripts/amalgm-mcp/toolbox/store.js +0 -15
- package/runtime/scripts/amalgm-mcp/toolbox/system-catalog.js +2 -0
- package/runtime/scripts/amalgm-mcp/workspace/rest.js +11 -7
- package/runtime/scripts/chat-core/adapters/opencode.js +1 -1
- package/runtime/scripts/chat-core/engine.js +16 -2
- package/runtime/scripts/chat-core/input.js +19 -1
- package/runtime/scripts/chat-core/tool-shape.js +6 -4
- package/runtime/scripts/chat-core/tooling/system-instructions.js +51 -0
- package/runtime/scripts/chat-core/tooling/system-prompt.js +34 -8
- package/runtime/scripts/local-gateway.js +1 -0
- package/runtime/scripts/amalgm-mcp/browser/agent-browser.js +0 -569
- package/runtime/scripts/amalgm-mcp/browser/page.js +0 -25
- package/runtime/scripts/chat-core/tooling/active-memory.js +0 -574
- package/runtime/scripts/chat-core/tooling/passive-memory.js +0 -576
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* CUA (computer-use) tools — coordinate-based browser control, split out from
|
|
5
|
+
* the main browser surface. Use these only for vision-model loops that work
|
|
6
|
+
* from screenshots; for everything else the snapshot/@ref tools in the
|
|
7
|
+
* browser group are faster and more reliable.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const { textResult, errorResult } = require('../lib/tool-result');
|
|
11
|
+
const engine = require('./engine');
|
|
12
|
+
|
|
13
|
+
const sessionProperty = {
|
|
14
|
+
session: { type: 'string', description: 'Browser session/profile id. Defaults to your chat session.' },
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
function coordinate(value, name) {
|
|
18
|
+
const number = Number(value);
|
|
19
|
+
if (!Number.isFinite(number)) throw new Error(`${name} must be a finite number`);
|
|
20
|
+
return number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function handlerFor(action, prepare) {
|
|
24
|
+
return async (args, ctx) => {
|
|
25
|
+
try {
|
|
26
|
+
const prepared = prepare ? prepare(args) : args;
|
|
27
|
+
const result = await engine.cua(action, prepared, ctx);
|
|
28
|
+
return textResult(`ok (session: ${result.session || result.browserSessionId || 'default'})`);
|
|
29
|
+
} catch (err) {
|
|
30
|
+
return errorResult(`${action} failed: ${err.message}`);
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
module.exports = [
|
|
36
|
+
{
|
|
37
|
+
name: 'cua_screenshot',
|
|
38
|
+
description: 'Capture the browser viewport for a vision loop. Coordinates in the image map 1:1 to cua_click/cua_move coordinates.',
|
|
39
|
+
inputSchema: { type: 'object', properties: sessionProperty },
|
|
40
|
+
async handler(args, ctx) {
|
|
41
|
+
try {
|
|
42
|
+
const capture = await engine.screenshot({ ...args, fullPage: false }, ctx);
|
|
43
|
+
return {
|
|
44
|
+
content: [
|
|
45
|
+
{ type: 'image', data: capture.base64, mimeType: 'image/png' },
|
|
46
|
+
{ type: 'text', text: `Viewport screenshot (${capture.bytes} bytes)` },
|
|
47
|
+
],
|
|
48
|
+
};
|
|
49
|
+
} catch (err) {
|
|
50
|
+
return errorResult(`cua_screenshot failed: ${err.message}`);
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
name: 'cua_click',
|
|
56
|
+
description: 'Click viewport coordinates.',
|
|
57
|
+
inputSchema: {
|
|
58
|
+
type: 'object',
|
|
59
|
+
properties: {
|
|
60
|
+
x: { type: 'number' },
|
|
61
|
+
y: { type: 'number' },
|
|
62
|
+
button: { type: 'number', description: '1 left (default), 2 middle, 3 right' },
|
|
63
|
+
...sessionProperty,
|
|
64
|
+
},
|
|
65
|
+
required: ['x', 'y'],
|
|
66
|
+
},
|
|
67
|
+
handler: handlerFor('cua_click', (args) => ({ ...args, x: coordinate(args.x, 'x'), y: coordinate(args.y, 'y') })),
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'cua_double_click',
|
|
71
|
+
description: 'Double-click viewport coordinates.',
|
|
72
|
+
inputSchema: {
|
|
73
|
+
type: 'object',
|
|
74
|
+
properties: { x: { type: 'number' }, y: { type: 'number' }, ...sessionProperty },
|
|
75
|
+
required: ['x', 'y'],
|
|
76
|
+
},
|
|
77
|
+
handler: handlerFor('cua_double_click', (args) => ({ ...args, x: coordinate(args.x, 'x'), y: coordinate(args.y, 'y') })),
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
name: 'cua_move',
|
|
81
|
+
description: 'Move the mouse to viewport coordinates.',
|
|
82
|
+
inputSchema: {
|
|
83
|
+
type: 'object',
|
|
84
|
+
properties: { x: { type: 'number' }, y: { type: 'number' }, ...sessionProperty },
|
|
85
|
+
required: ['x', 'y'],
|
|
86
|
+
},
|
|
87
|
+
handler: handlerFor('cua_move', (args) => ({ ...args, x: coordinate(args.x, 'x'), y: coordinate(args.y, 'y') })),
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
name: 'cua_scroll',
|
|
91
|
+
description: 'Scroll from viewport coordinates by scrollX/scrollY pixels.',
|
|
92
|
+
inputSchema: {
|
|
93
|
+
type: 'object',
|
|
94
|
+
properties: {
|
|
95
|
+
x: { type: 'number' },
|
|
96
|
+
y: { type: 'number' },
|
|
97
|
+
scrollX: { type: 'number' },
|
|
98
|
+
scrollY: { type: 'number' },
|
|
99
|
+
...sessionProperty,
|
|
100
|
+
},
|
|
101
|
+
required: ['x', 'y', 'scrollX', 'scrollY'],
|
|
102
|
+
},
|
|
103
|
+
handler: handlerFor('cua_scroll', (args) => ({
|
|
104
|
+
...args,
|
|
105
|
+
x: coordinate(args.x, 'x'),
|
|
106
|
+
y: coordinate(args.y, 'y'),
|
|
107
|
+
scrollX: Number(args.scrollX || 0),
|
|
108
|
+
scrollY: Number(args.scrollY || 0),
|
|
109
|
+
})),
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
name: 'cua_type',
|
|
113
|
+
description: 'Type text at the focused element with real keystrokes.',
|
|
114
|
+
inputSchema: {
|
|
115
|
+
type: 'object',
|
|
116
|
+
properties: { text: { type: 'string' }, ...sessionProperty },
|
|
117
|
+
required: ['text'],
|
|
118
|
+
},
|
|
119
|
+
handler: handlerFor('cua_type'),
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
name: 'cua_keypress',
|
|
123
|
+
description: 'Press a key combination, e.g. ["Control", "a"] or ["Enter"].',
|
|
124
|
+
inputSchema: {
|
|
125
|
+
type: 'object',
|
|
126
|
+
properties: { keys: { type: 'array', items: { type: 'string' } }, ...sessionProperty },
|
|
127
|
+
required: ['keys'],
|
|
128
|
+
},
|
|
129
|
+
handler: handlerFor('cua_keypress'),
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
name: 'cua_drag',
|
|
133
|
+
description: 'Drag the mouse through a list of viewport points.',
|
|
134
|
+
inputSchema: {
|
|
135
|
+
type: 'object',
|
|
136
|
+
properties: {
|
|
137
|
+
path: {
|
|
138
|
+
type: 'array',
|
|
139
|
+
items: { type: 'object', properties: { x: { type: 'number' }, y: { type: 'number' } }, required: ['x', 'y'] },
|
|
140
|
+
},
|
|
141
|
+
...sessionProperty,
|
|
142
|
+
},
|
|
143
|
+
required: ['path'],
|
|
144
|
+
},
|
|
145
|
+
handler: handlerFor('cua_drag'),
|
|
146
|
+
},
|
|
147
|
+
];
|
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Browser engine: the verb set behind the browser/cua/recorder tools, with
|
|
5
|
+
* backend routing.
|
|
6
|
+
*
|
|
7
|
+
* Backends, in order of preference:
|
|
8
|
+
* 1. "attached" — agent-browser connected over CDP to the Amalgm desktop
|
|
9
|
+
* app's visible in-tab webview. Requires the Electron bridge file to
|
|
10
|
+
* advertise a cdpUrl (or AMALGM_BROWSER_CDP_URL). Full agent-browser
|
|
11
|
+
* power against the splitscreen surface the user can see.
|
|
12
|
+
* 2. "electron" — legacy HTTP bridge into the desktop app's webview. Covers
|
|
13
|
+
* the core verbs; no CLI passthrough.
|
|
14
|
+
* 3. "cli" — standalone agent-browser session (headed window on
|
|
15
|
+
* desktop, headless on servers).
|
|
16
|
+
*
|
|
17
|
+
* Sessions map to durable profiles; the default session for a tool call is
|
|
18
|
+
* the caller's chat session, so one conversation keeps one browser.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const electron = require('./electron-bridge');
|
|
22
|
+
const cli = require('./cli');
|
|
23
|
+
|
|
24
|
+
const attachedSessions = new Set();
|
|
25
|
+
const knownSessions = new Map();
|
|
26
|
+
|
|
27
|
+
function isPackagedDesktopRuntime() {
|
|
28
|
+
const execPath = process.execPath || '';
|
|
29
|
+
return process.env.ELECTRON_RUN_AS_NODE === '1' && execPath.includes('.app/Contents/MacOS/');
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function cdpUrl() {
|
|
33
|
+
if (process.env.AMALGM_BROWSER_CDP_URL) return process.env.AMALGM_BROWSER_CDP_URL;
|
|
34
|
+
try {
|
|
35
|
+
const fs = require('fs');
|
|
36
|
+
const path = require('path');
|
|
37
|
+
const file = path.join(cli.amalgmDir(), 'electron-browser-bridge.json');
|
|
38
|
+
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
39
|
+
if (typeof data?.cdpUrl === 'string' && data.cdpUrl) return data.cdpUrl;
|
|
40
|
+
if (Number.isInteger(data?.cdpPort) && data.cdpPort > 0) return String(data.cdpPort);
|
|
41
|
+
} catch {}
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function mode() {
|
|
46
|
+
const requested = (process.env.AMALGM_BROWSER_BACKEND || '').toLowerCase();
|
|
47
|
+
if (requested === 'agent-browser' || requested === 'external' || requested === 'cli') return 'cli';
|
|
48
|
+
if (requested === 'electron') return 'electron';
|
|
49
|
+
|
|
50
|
+
if (electron.isConfigured() || isPackagedDesktopRuntime()) {
|
|
51
|
+
return cdpUrl() ? 'attached' : 'electron';
|
|
52
|
+
}
|
|
53
|
+
return 'cli';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function usesElectronBridge() {
|
|
57
|
+
return mode() === 'electron';
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function sessionFor(args = {}, context = {}) {
|
|
61
|
+
return cli.safeId(
|
|
62
|
+
args.session
|
|
63
|
+
|| args.profile
|
|
64
|
+
|| args.browserProfileId
|
|
65
|
+
|| args.browserSessionId
|
|
66
|
+
|| args.sessionId
|
|
67
|
+
|| args.tabId
|
|
68
|
+
|| context?.callerSessionId
|
|
69
|
+
|| cli.DEFAULT_SESSION,
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function rememberProfile(session) {
|
|
74
|
+
try {
|
|
75
|
+
require('./profiles').touchBrowserProfile({
|
|
76
|
+
id: session,
|
|
77
|
+
name: session,
|
|
78
|
+
source: 'agent-browser',
|
|
79
|
+
profilePath: cli.profileDir(session),
|
|
80
|
+
}, { source: 'agent-browser' });
|
|
81
|
+
} catch {}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function sessionInfo(session, extra = {}) {
|
|
85
|
+
rememberProfile(session);
|
|
86
|
+
const info = {
|
|
87
|
+
session,
|
|
88
|
+
browserSessionId: session,
|
|
89
|
+
browserProfileId: session,
|
|
90
|
+
backend: mode(),
|
|
91
|
+
...extra,
|
|
92
|
+
};
|
|
93
|
+
knownSessions.set(session, { ...knownSessions.get(session), ...info, lastActiveAt: Date.now() });
|
|
94
|
+
return info;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* In attached mode, connect the session to the desktop app's chromium on
|
|
99
|
+
* first use and select the in-tab webview target so commands drive the
|
|
100
|
+
* surface the user is looking at.
|
|
101
|
+
*/
|
|
102
|
+
async function ensureReady(session) {
|
|
103
|
+
if (mode() !== 'attached' || attachedSessions.has(session)) return;
|
|
104
|
+
const target = cdpUrl();
|
|
105
|
+
await cli.runCli(['connect', target], { session });
|
|
106
|
+
try {
|
|
107
|
+
const tabs = await cli.runCli(['tab'], { session, timeoutMs: 15_000 });
|
|
108
|
+
const list = tabs?.data?.tabs || tabs?.tabs || [];
|
|
109
|
+
const webview = list.find((tab) => tab.type === 'webview');
|
|
110
|
+
if (webview?.tabId) await cli.runCli(['tab', webview.tabId], { session, timeoutMs: 15_000 });
|
|
111
|
+
} catch {
|
|
112
|
+
// No webview target yet — the outer page target still works.
|
|
113
|
+
}
|
|
114
|
+
attachedSessions.add(session);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async function run(session, commandArgs, options = {}) {
|
|
118
|
+
await ensureReady(session);
|
|
119
|
+
return cli.runCli(commandArgs, { session, ...options });
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function readText(session, commandArgs, timeoutMs) {
|
|
123
|
+
await ensureReady(session);
|
|
124
|
+
return cli.readText(commandArgs, session, timeoutMs);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Target grammar shared by click/fill:
|
|
129
|
+
* "@e12" — ref from browser_snapshot (preferred)
|
|
130
|
+
* "text=Sign in" — find by visible text
|
|
131
|
+
* anything else — CSS selector
|
|
132
|
+
*/
|
|
133
|
+
function findArgs(target, action, value) {
|
|
134
|
+
if (target.startsWith('text=')) {
|
|
135
|
+
const command = ['find', 'text', target.slice('text='.length), action];
|
|
136
|
+
if (value !== undefined) command.push(value);
|
|
137
|
+
return command;
|
|
138
|
+
}
|
|
139
|
+
const command = [action === 'fill' || action === 'type' ? action : action, target];
|
|
140
|
+
if (value !== undefined) command.push(value);
|
|
141
|
+
return command;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function electronLocator(target) {
|
|
145
|
+
if (target.startsWith('@')) return { ref: target };
|
|
146
|
+
if (target.startsWith('text=')) return { text: target.slice('text='.length) };
|
|
147
|
+
return { selector: target };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// ---------------------------------------------------------------------------
|
|
151
|
+
// Core verbs
|
|
152
|
+
// ---------------------------------------------------------------------------
|
|
153
|
+
|
|
154
|
+
async function state(args = {}, context) {
|
|
155
|
+
const session = sessionFor(args, context);
|
|
156
|
+
if (usesElectronBridge()) return electron.stateBrowser({ browserSessionId: session }, context);
|
|
157
|
+
|
|
158
|
+
let url = '';
|
|
159
|
+
let title = '';
|
|
160
|
+
try { url = await readText(session, ['get', 'url']); } catch {}
|
|
161
|
+
try { title = await readText(session, ['get', 'title']); } catch {}
|
|
162
|
+
return sessionInfo(session, { url, title });
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function open(args = {}, context) {
|
|
166
|
+
const session = sessionFor(args, context);
|
|
167
|
+
if (usesElectronBridge()) {
|
|
168
|
+
const result = await electron.navigateBrowser({ url: args.url, browserSessionId: session }, context);
|
|
169
|
+
return sessionInfo(session, { url: result.url || args.url, title: result.title || '' });
|
|
170
|
+
}
|
|
171
|
+
await run(session, ['open', args.url]);
|
|
172
|
+
return state(args, context);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async function snapshot(args = {}, context) {
|
|
176
|
+
const session = sessionFor(args, context);
|
|
177
|
+
if (usesElectronBridge()) {
|
|
178
|
+
const result = await electron.snapshotBrowser({ browserSessionId: session }, context);
|
|
179
|
+
return sessionInfo(session, {
|
|
180
|
+
url: result.url,
|
|
181
|
+
title: result.title,
|
|
182
|
+
snapshotText: result.snapshotText || result.snapshot || '',
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
const result = await run(session, ['snapshot', '-i'], { timeoutMs: 30_000 });
|
|
186
|
+
const current = await state(args, context);
|
|
187
|
+
return { ...current, snapshotText: cli.asText(result) };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async function screenshot(args = {}, context) {
|
|
191
|
+
const session = sessionFor(args, context);
|
|
192
|
+
if (usesElectronBridge()) {
|
|
193
|
+
return electron.screenshotBrowser({ browserSessionId: session, fullPage: args.fullPage }, context);
|
|
194
|
+
}
|
|
195
|
+
const fs = require('fs');
|
|
196
|
+
const path = require('path');
|
|
197
|
+
const crypto = require('crypto');
|
|
198
|
+
const file = path.join(cli.screenshotDir(), `${session}-${Date.now()}-${crypto.randomUUID()}.png`);
|
|
199
|
+
await run(session, ['screenshot', ...(args.fullPage ? ['--full'] : []), file]);
|
|
200
|
+
const buffer = fs.readFileSync(file);
|
|
201
|
+
return {
|
|
202
|
+
...sessionInfo(session),
|
|
203
|
+
base64: buffer.toString('base64'),
|
|
204
|
+
bytes: buffer.length,
|
|
205
|
+
mode: args.fullPage ? 'full page' : 'viewport',
|
|
206
|
+
path: file,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async function click(args = {}, context) {
|
|
211
|
+
const session = sessionFor(args, context);
|
|
212
|
+
const target = String(args.target || '');
|
|
213
|
+
if (usesElectronBridge()) {
|
|
214
|
+
await electron.clickBrowser({ ...electronLocator(target), browserSessionId: session }, context);
|
|
215
|
+
return sessionInfo(session, { clicked: target });
|
|
216
|
+
}
|
|
217
|
+
await run(session, target.startsWith('text=')
|
|
218
|
+
? findArgs(target, 'click')
|
|
219
|
+
: ['click', target]);
|
|
220
|
+
return sessionInfo(session, { clicked: target });
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async function fill(args = {}, context) {
|
|
224
|
+
const session = sessionFor(args, context);
|
|
225
|
+
const target = String(args.target || '');
|
|
226
|
+
const text = String(args.text ?? '');
|
|
227
|
+
if (usesElectronBridge()) {
|
|
228
|
+
await electron.typeBrowser({ ...electronLocator(target), text, browserSessionId: session }, context);
|
|
229
|
+
if (args.submit) await electron.pressBrowser({ key: 'Enter', browserSessionId: session }, context);
|
|
230
|
+
return sessionInfo(session, { filled: target, submitted: Boolean(args.submit) });
|
|
231
|
+
}
|
|
232
|
+
await run(session, target.startsWith('text=')
|
|
233
|
+
? findArgs(target, 'fill', text)
|
|
234
|
+
: ['fill', target, text]);
|
|
235
|
+
if (args.submit) await run(session, ['press', 'Enter']);
|
|
236
|
+
return sessionInfo(session, { filled: target, submitted: Boolean(args.submit) });
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
async function press(args = {}, context) {
|
|
240
|
+
const session = sessionFor(args, context);
|
|
241
|
+
if (usesElectronBridge()) {
|
|
242
|
+
await electron.pressBrowser({ key: args.key, browserSessionId: session }, context);
|
|
243
|
+
return sessionInfo(session, { pressed: args.key });
|
|
244
|
+
}
|
|
245
|
+
await run(session, ['press', args.key]);
|
|
246
|
+
return sessionInfo(session, { pressed: args.key });
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
async function evaluate(args = {}, context) {
|
|
250
|
+
const session = sessionFor(args, context);
|
|
251
|
+
if (usesElectronBridge()) {
|
|
252
|
+
const { result } = await electron.evaluateBrowser({ script: args.script, browserSessionId: session }, context);
|
|
253
|
+
return { ...sessionInfo(session), result };
|
|
254
|
+
}
|
|
255
|
+
const result = await run(session, ['eval', args.script || '']);
|
|
256
|
+
const data = result?.data ?? result;
|
|
257
|
+
// agent-browser wraps eval results as { origin, result }.
|
|
258
|
+
const value = data && typeof data === 'object' && 'result' in data ? data.result : data;
|
|
259
|
+
return { ...sessionInfo(session), result: value };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async function wait(args = {}, context) {
|
|
263
|
+
const session = sessionFor(args, context);
|
|
264
|
+
const timeoutMs = args.timeoutMs || 30_000;
|
|
265
|
+
|
|
266
|
+
if (args.ms) {
|
|
267
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(Number(args.ms) || 0, 60_000)));
|
|
268
|
+
return sessionInfo(session, { waited: `${args.ms}ms` });
|
|
269
|
+
}
|
|
270
|
+
if (args.url) {
|
|
271
|
+
if (usesElectronBridge()) {
|
|
272
|
+
await electron.waitForURLBrowser({ url: args.url, timeoutMs, browserSessionId: session }, context);
|
|
273
|
+
return sessionInfo(session, { matched: args.url });
|
|
274
|
+
}
|
|
275
|
+
const deadline = Date.now() + timeoutMs;
|
|
276
|
+
while (Date.now() < deadline) {
|
|
277
|
+
const current = await readText(session, ['get', 'url']);
|
|
278
|
+
if (current.includes(args.url.replace(/\*/g, ''))) return sessionInfo(session, { matched: current });
|
|
279
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
280
|
+
}
|
|
281
|
+
throw new Error(`Timed out waiting for URL ${args.url}`);
|
|
282
|
+
}
|
|
283
|
+
if (args.selector) {
|
|
284
|
+
if (usesElectronBridge()) {
|
|
285
|
+
await electron.waitForSelectorBrowser({ selector: args.selector, timeoutMs, browserSessionId: session }, context);
|
|
286
|
+
return sessionInfo(session, { matched: args.selector });
|
|
287
|
+
}
|
|
288
|
+
await run(session, ['wait', args.selector], { timeoutMs });
|
|
289
|
+
return sessionInfo(session, { matched: args.selector });
|
|
290
|
+
}
|
|
291
|
+
throw new Error('wait requires one of: selector, url, ms');
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
async function tabs(args = {}, context) {
|
|
295
|
+
const session = sessionFor(args, context);
|
|
296
|
+
if (usesElectronBridge()) {
|
|
297
|
+
if (args.action === 'select' && args.tab) {
|
|
298
|
+
return electron.tabsSelectBrowser({ tabId: args.tab, browserSessionId: session }, context);
|
|
299
|
+
}
|
|
300
|
+
return electron.tabsListBrowser(context);
|
|
301
|
+
}
|
|
302
|
+
if (args.action === 'select' && args.tab) {
|
|
303
|
+
return run(session, ['tab', args.tab]);
|
|
304
|
+
}
|
|
305
|
+
if (args.action === 'new') {
|
|
306
|
+
return run(session, ['tab', 'new', ...(args.url ? [args.url] : [])]);
|
|
307
|
+
}
|
|
308
|
+
const result = await run(session, ['tab']);
|
|
309
|
+
return { ...sessionInfo(session), tabs: result?.data?.tabs || result?.tabs || [] };
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* Full agent-browser CLI passthrough — the escape hatch that keeps the tool
|
|
314
|
+
* surface small while staying turing complete for browsers. Network mocking,
|
|
315
|
+
* cookies, storage, traces, diffs, React inspection: all reachable here.
|
|
316
|
+
*/
|
|
317
|
+
async function passthrough(args = {}, context) {
|
|
318
|
+
const session = sessionFor(args, context);
|
|
319
|
+
if (usesElectronBridge()) {
|
|
320
|
+
throw new Error(
|
|
321
|
+
'browser_cli needs the agent-browser backend. The desktop bridge does not expose it; '
|
|
322
|
+
+ 'set AMALGM_BROWSER_BACKEND=agent-browser or enable the CDP bridge (cdpUrl).',
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
const commandArgs = (args.args || []).map(String);
|
|
326
|
+
if (!commandArgs.length) throw new Error('args is required, e.g. ["scroll", "down"] or ["get", "text", "@e1"]');
|
|
327
|
+
const result = await run(session, commandArgs, { timeoutMs: args.timeoutMs || 60_000 });
|
|
328
|
+
return { ...sessionInfo(session), result: result?.data ?? result };
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
async function cua(action, args = {}, context) {
|
|
332
|
+
const session = sessionFor(args, context);
|
|
333
|
+
if (usesElectronBridge()) return electron.cuaBrowser(action, { ...args, browserSessionId: session }, context);
|
|
334
|
+
|
|
335
|
+
const x = Math.round(Number(args.x || 0));
|
|
336
|
+
const y = Math.round(Number(args.y || 0));
|
|
337
|
+
if (action === 'cua_move') await run(session, ['mouse', 'move', x, y]);
|
|
338
|
+
if (action === 'cua_click') {
|
|
339
|
+
await run(session, ['mouse', 'move', x, y]);
|
|
340
|
+
await run(session, ['mouse', 'down', args.button || 1]);
|
|
341
|
+
await run(session, ['mouse', 'up', args.button || 1]);
|
|
342
|
+
}
|
|
343
|
+
if (action === 'cua_double_click') {
|
|
344
|
+
await run(session, ['mouse', 'move', x, y]);
|
|
345
|
+
for (let i = 0; i < 2; i += 1) {
|
|
346
|
+
await run(session, ['mouse', 'down', 1]);
|
|
347
|
+
await run(session, ['mouse', 'up', 1]);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
if (action === 'cua_scroll') await run(session, ['mouse', 'wheel', args.scrollY || 0, args.scrollX || 0]);
|
|
351
|
+
if (action === 'cua_type') await run(session, ['keyboard', 'type', args.text || '']);
|
|
352
|
+
if (action === 'cua_keypress') await run(session, ['press', (args.keys || []).join('+')]);
|
|
353
|
+
if (action === 'cua_drag') {
|
|
354
|
+
const points = Array.isArray(args.path) ? args.path : [];
|
|
355
|
+
if (points.length < 2) throw new Error('drag path must contain at least two points');
|
|
356
|
+
await run(session, ['mouse', 'move', points[0].x, points[0].y]);
|
|
357
|
+
await run(session, ['mouse', 'down', 1]);
|
|
358
|
+
for (const point of points.slice(1)) await run(session, ['mouse', 'move', point.x, point.y]);
|
|
359
|
+
await run(session, ['mouse', 'up', 1]);
|
|
360
|
+
}
|
|
361
|
+
return sessionInfo(session, { ok: true });
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// ---------------------------------------------------------------------------
|
|
365
|
+
// State save/load (used by browser auth bundles) and lifecycle
|
|
366
|
+
// ---------------------------------------------------------------------------
|
|
367
|
+
|
|
368
|
+
async function saveState(args = {}, context) {
|
|
369
|
+
const session = sessionFor(args, context);
|
|
370
|
+
if (usesElectronBridge()) return electron.saveStateBrowser({ ...args, browserSessionId: session }, context);
|
|
371
|
+
|
|
372
|
+
const fs = require('fs');
|
|
373
|
+
const os = require('os');
|
|
374
|
+
const path = require('path');
|
|
375
|
+
const crypto = require('crypto');
|
|
376
|
+
const file = args.path || path.join(os.tmpdir(), `amalgm-browser-state-${session}-${Date.now()}-${crypto.randomUUID()}.json`);
|
|
377
|
+
await run(session, ['state', 'save', file]);
|
|
378
|
+
const stateData = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
379
|
+
if (!args.path) {
|
|
380
|
+
try { fs.rmSync(file, { force: true }); } catch {}
|
|
381
|
+
}
|
|
382
|
+
const current = await state(args, context);
|
|
383
|
+
return {
|
|
384
|
+
...current,
|
|
385
|
+
state: stateData,
|
|
386
|
+
path: args.path ? file : undefined,
|
|
387
|
+
storageKinds: ['cookies', 'localStorage', 'sessionStorage'],
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
async function loadState(args = {}, context) {
|
|
392
|
+
const session = sessionFor(args, context);
|
|
393
|
+
if (usesElectronBridge()) return electron.loadStateBrowser({ ...args, browserSessionId: session }, context);
|
|
394
|
+
|
|
395
|
+
const fs = require('fs');
|
|
396
|
+
const os = require('os');
|
|
397
|
+
const path = require('path');
|
|
398
|
+
const crypto = require('crypto');
|
|
399
|
+
let file = args.path;
|
|
400
|
+
let temporary = false;
|
|
401
|
+
if (!file) {
|
|
402
|
+
if (!args.state || typeof args.state !== 'object') throw new Error('state or path is required');
|
|
403
|
+
file = path.join(os.tmpdir(), `amalgm-browser-state-load-${session}-${Date.now()}-${crypto.randomUUID()}.json`);
|
|
404
|
+
fs.writeFileSync(file, `${JSON.stringify(args.state, null, 2)}\n`, { mode: 0o600 });
|
|
405
|
+
temporary = true;
|
|
406
|
+
}
|
|
407
|
+
await run(session, ['state', 'load', file]);
|
|
408
|
+
if (temporary) {
|
|
409
|
+
try { fs.rmSync(file, { force: true }); } catch {}
|
|
410
|
+
}
|
|
411
|
+
return sessionInfo(session, { loaded: true, storageKinds: ['cookies', 'localStorage', 'sessionStorage'] });
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
async function close(args = {}, context) {
|
|
415
|
+
const session = sessionFor(args, context);
|
|
416
|
+
if (usesElectronBridge()) {
|
|
417
|
+
await electron.closeBrowser({ browserSessionId: session }, context);
|
|
418
|
+
} else {
|
|
419
|
+
await cli.runCli(['close'], { session });
|
|
420
|
+
}
|
|
421
|
+
attachedSessions.delete(session);
|
|
422
|
+
knownSessions.delete(session);
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** Resolve the CDP websocket URL for a session — recorder needs it. */
|
|
426
|
+
async function cdpUrlForSession(session) {
|
|
427
|
+
await ensureReady(session);
|
|
428
|
+
const result = await cli.runCli(['get', 'cdp-url'], { session, timeoutMs: 15_000 });
|
|
429
|
+
return result?.data?.cdpUrl || result?.cdpUrl || null;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
module.exports = {
|
|
433
|
+
cdpUrlForSession,
|
|
434
|
+
click,
|
|
435
|
+
close,
|
|
436
|
+
cua,
|
|
437
|
+
evaluate,
|
|
438
|
+
fill,
|
|
439
|
+
knownSessions,
|
|
440
|
+
loadState,
|
|
441
|
+
mode,
|
|
442
|
+
open,
|
|
443
|
+
passthrough,
|
|
444
|
+
press,
|
|
445
|
+
saveState,
|
|
446
|
+
screenshot,
|
|
447
|
+
sessionFor,
|
|
448
|
+
sessionInfo,
|
|
449
|
+
snapshot,
|
|
450
|
+
state,
|
|
451
|
+
tabs,
|
|
452
|
+
usesElectronBridge,
|
|
453
|
+
wait,
|
|
454
|
+
};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Recorder tools — session recording as a first-class primitive any agent can
|
|
5
|
+
* use (QA bots especially). Captures the browser session — including the
|
|
6
|
+
* desktop app's visible in-tab webview — to a WebM file stored with the
|
|
7
|
+
* Amalgm project the work belongs to.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const { textResult, errorResult } = require('../lib/tool-result');
|
|
11
|
+
const recorder = require('./recorder');
|
|
12
|
+
|
|
13
|
+
const commonProperties = {
|
|
14
|
+
session: { type: 'string', description: 'Browser session to record. Defaults to your chat session.' },
|
|
15
|
+
projectPath: {
|
|
16
|
+
type: 'string',
|
|
17
|
+
description: 'Amalgm project directory; the video is saved under <project>/.amalgm/recordings/. Defaults to the project this chat session belongs to, falling back to ~/.amalgm/browser-sessions/.',
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
module.exports = [
|
|
22
|
+
{
|
|
23
|
+
name: 'record_start',
|
|
24
|
+
description: 'Start recording the browser session to a WebM video. Stop with record_stop. One recording per session at a time.',
|
|
25
|
+
inputSchema: {
|
|
26
|
+
type: 'object',
|
|
27
|
+
properties: {
|
|
28
|
+
name: { type: 'string', description: 'Recording name used in the filename, e.g. "login-flow-qa"' },
|
|
29
|
+
fps: { type: 'number', description: 'Frames per second (1-30, default 10)' },
|
|
30
|
+
...commonProperties,
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
async handler(args, ctx) {
|
|
34
|
+
try {
|
|
35
|
+
const result = await recorder.start(args, ctx);
|
|
36
|
+
return textResult(`Recording started (${result.fps} fps)\nFile: ${result.path}\nSession: ${result.session}`);
|
|
37
|
+
} catch (err) {
|
|
38
|
+
return errorResult(`record_start failed: ${err.message}`);
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
name: 'record_stop',
|
|
44
|
+
description: 'Stop the active recording and save the WebM file. Returns the local file path.',
|
|
45
|
+
inputSchema: { type: 'object', properties: commonProperties },
|
|
46
|
+
async handler(args, ctx) {
|
|
47
|
+
try {
|
|
48
|
+
const result = await recorder.stop(args, ctx);
|
|
49
|
+
return {
|
|
50
|
+
content: [{
|
|
51
|
+
type: 'text',
|
|
52
|
+
text: `Recording saved: ${result.path}\n`
|
|
53
|
+
+ `${result.frames} frames, ${Math.round(result.durationMs / 1000)}s, ${result.bytes} bytes`,
|
|
54
|
+
}],
|
|
55
|
+
metadata: {
|
|
56
|
+
browserVideo: {
|
|
57
|
+
path: result.path,
|
|
58
|
+
relativePath: result.relativePath,
|
|
59
|
+
bytes: result.bytes,
|
|
60
|
+
mimeType: result.mimeType,
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
} catch (err) {
|
|
65
|
+
return errorResult(`record_stop failed: ${err.message}`);
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'record_list',
|
|
71
|
+
description: 'List saved recordings for a project (or the session fallback directory).',
|
|
72
|
+
inputSchema: { type: 'object', properties: commonProperties },
|
|
73
|
+
async handler(args, ctx) {
|
|
74
|
+
try {
|
|
75
|
+
const result = recorder.list(args, ctx);
|
|
76
|
+
return textResult(JSON.stringify(result, null, 2));
|
|
77
|
+
} catch (err) {
|
|
78
|
+
return errorResult(`record_list failed: ${err.message}`);
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
},
|
|
82
|
+
];
|