amalgm 0.0.0 → 0.0.1

package/runtime/scripts/amalgm-mcp/browser/tools.js

@@ -0,0 +1,688 @@
+/**
+ * Browser MCP tools. In desktop local mode they drive Amalgm's visible
+ * Electron browser surface; elsewhere they fall back to Playwright.
+ */
+
+const { textResult, errorResult } = require('../lib/tool-result');
+const {
+  clipboardBrowser,
+  closeBrowser,
+  clickBrowser,
+  cuaBrowser,
+  evaluateBrowser,
+  getTextBrowser,
+  locatorCountBrowser,
+  locatorTextBrowser,
+  navigateBrowser,
+  pressBrowser,
+  screenshotBrowser,
+  snapshotBrowser,
+  startVideoBrowser,
+  stateBrowser,
+  stopVideoBrowser,
+  tabNavigationBrowser,
+  tabsGetBrowser,
+  tabsListBrowser,
+  tabsNewBrowser,
+  tabsSelectBrowser,
+  tabsSelectedBrowser,
+  typeBrowser,
+  waitForLoadStateBrowser,
+  waitForSelectorBrowser,
+  waitForURLBrowser,
+} = require('./page');
+
+const tabTargetProperties = {
+  tabId: { type: 'string', description: 'Explicit browser tab id returned by browser_tabs_* or browser_navigate' },
+  browserSessionId: { type: 'string', description: 'Stable browser automation session id' },
+};
+
+const locatorProperties = {
+  ...tabTargetProperties,
+  selector: { type: 'string', description: 'CSS selector for the target element' },
+  text: { type: 'string', description: 'Visible text to match' },
+  role: { type: 'string', description: 'ARIA role, e.g. button, link, textbox' },
+  name: { type: 'string', description: 'Accessible name or visible name used with role/text matching' },
+  label: { type: 'string', description: 'Form label text' },
+  placeholder: { type: 'string', description: 'Input placeholder text' },
+  testId: { type: 'string', description: 'data-testid, data-test-id, or data-test value' },
+  exact: { type: 'boolean', description: 'Require exact text/name match' },
+  strict: { type: 'boolean', description: 'Require the locator to resolve to exactly one visible element' },
+  timeoutMs: { type: 'number', description: 'Maximum wait time in milliseconds' },
+};
+
+function jsonText(value) {
+  return textResult(JSON.stringify(value, null, 2));
+}
+
+function targetLine(result) {
+  const bits = [];
+  if (result?.tabId) bits.push(`Tab: ${result.tabId}`);
+  if (result?.browserSessionId) bits.push(`Session: ${result.browserSessionId}`);
+  return bits.length ? `\n${bits.join('\n')}` : '';
+}
+
+function requireFiniteCoordinate(value, name) {
+  const number = Number(value);
+  if (!Number.isFinite(number)) throw new Error(`${name} must be a finite number`);
+  return number;
+}
+
+module.exports = [
+  {
+    name: 'browser_tabs_list',
+    description: 'List visible Amalgm browser-use tabs/surfaces.',
+    inputSchema: { type: 'object', properties: {} },
+    async handler(_args, ctx) {
+      try {
+        return jsonText(await tabsListBrowser(ctx));
+      } catch (err) {
+        return errorResult(`List browser tabs failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_tabs_selected',
+    description: 'Return the most recently used browser-use tab.',
+    inputSchema: { type: 'object', properties: {} },
+    async handler(_args, ctx) {
+      try {
+        return jsonText(await tabsSelectedBrowser(ctx));
+      } catch (err) {
+        return errorResult(`Get selected browser tab failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_tabs_new',
+    description: 'Create a new visible browser-use tab/surface, optionally navigating it to a URL.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        url: { type: 'string', description: 'Optional URL to open' },
+        browserSessionId: { type: 'string', description: 'Optional session id; generated if omitted' },
+      },
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await tabsNewBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Create browser tab failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_tabs_get',
+    description: 'Get state for a browser tab by tabId or browserSessionId.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await tabsGetBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Get browser tab failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_tabs_select',
+    description: 'Select/focus a browser-use tab by tabId or browserSessionId.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await tabsSelectBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Select browser tab failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_tabs_close',
+    description: 'Close a browser-use tab by tabId or browserSessionId.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        await closeBrowser(args, ctx);
+        return textResult('Browser tab closed.');
+      } catch (err) {
+        return errorResult(`Close browser tab failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_state',
+    description: 'Return URL/title/loading state for the current or specified browser tab.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await stateBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Browser state failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_navigate',
+    description: 'Navigate a visible browser tab to a URL. Returns tabId and browserSessionId for follow-up calls.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        url: { type: 'string', description: 'The URL to navigate to' },
+      },
+      required: ['url'],
+    },
+    async handler(args, ctx) {
+      if (!args.url) return errorResult('url is required');
+      try {
+        const result = await navigateBrowser(args, ctx);
+        return textResult(`Navigated to ${result.url || args.url}\nStatus: ${result.status}\nTitle: ${result.title || ''}${targetLine(result)}`);
+      } catch (err) {
+        return errorResult(`Navigation failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_back',
+    description: 'Navigate the browser tab back in history.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await tabNavigationBrowser('back', args, ctx));
+      } catch (err) {
+        return errorResult(`Browser back failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_forward',
+    description: 'Navigate the browser tab forward in history.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await tabNavigationBrowser('forward', args, ctx));
+      } catch (err) {
+        return errorResult(`Browser forward failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_reload',
+    description: 'Reload the browser tab.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await tabNavigationBrowser('reload', args, ctx));
+      } catch (err) {
+        return errorResult(`Browser reload failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_screenshot',
+    description: 'Take a screenshot of the current or specified browser tab. Returns a base64-encoded PNG.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        full_page: { type: 'boolean', description: 'Capture the full scrollable page (Playwright fallback only; Electron captures viewport)' },
+      },
+    },
+    async handler(args, ctx) {
+      try {
+        const capture = await screenshotBrowser({ ...args, fullPage: args.full_page || false }, ctx);
+        return {
+          content: [
+            { type: 'image', data: capture.base64, mimeType: 'image/png' },
+            { type: 'text', text: `Screenshot captured (${capture.bytes} bytes, ${capture.mode})${targetLine(capture)}` },
+          ],
+        };
+      } catch (err) {
+        return errorResult(`Screenshot failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_snapshot',
+    description: 'Get a token-efficient DOM/text snapshot of the current or specified browser tab.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        const snapshot = await snapshotBrowser(args, ctx);
+        let result = `Page: ${snapshot.title}\nURL: ${snapshot.url}${targetLine(snapshot)}\n\n`;
+        result += (snapshot.snapshotText || '(empty)').slice(0, 10_000);
+        return textResult(result);
+      } catch (err) {
+        return errorResult(`Snapshot failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_click',
+    description: 'Click an element by selector, text, role/name, label, placeholder, or testId.',
+    inputSchema: { type: 'object', properties: locatorProperties },
+    async handler(args, ctx) {
+      if (!args.selector && !args.text && !args.role && !args.label && !args.placeholder && !args.testId && !args.name) {
+        return errorResult('A selector, text, role, label, placeholder, testId, or name is required');
+      }
+      try {
+        const result = await clickBrowser(args, ctx);
+        return textResult(`${result.description}${targetLine(result)}`);
+      } catch (err) {
+        return errorResult(`Click failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_type',
+    description: 'Type text into a focused element or a target matched by selector/role/label/placeholder/testId.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...locatorProperties,
+        text: { type: 'string', description: 'The text to type' },
+        clear: { type: 'boolean', description: 'Clear the field before typing (default: true)' },
+      },
+      required: ['text'],
+    },
+    async handler(args, ctx) {
+      if (typeof args.text !== 'string') return errorResult('text is required');
+      try {
+        const result = await typeBrowser(args, ctx);
+        return textResult(`${result.description}${targetLine(result)}`);
+      } catch (err) {
+        return errorResult(`Type failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_press',
+    description: 'Press a key on a target element or the focused element.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...locatorProperties,
+        key: { type: 'string', description: 'Key to press, e.g. Enter, Escape, A' },
+      },
+      required: ['key'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await pressBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Press failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_get_text',
+    description: 'Extract text content from the current page or a CSS selector.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        selector: { type: 'string', description: 'CSS selector to extract (omit for full page)' },
+      },
+    },
+    async handler(args, ctx) {
+      try {
+        let text = (await getTextBrowser(args, ctx)).text;
+        if (text && text.length > 15_000) {
+          text = `${text.slice(0, 15_000)}\n\n... (truncated, use a selector for specific content)`;
+        }
+        return textResult(text || '(empty)');
+      } catch (err) {
+        return errorResult(`Get text failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_locator_count',
+    description: 'Count visible elements matching a locator.',
+    inputSchema: { type: 'object', properties: locatorProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await locatorCountBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Locator count failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_locator_text',
+    description: 'Return visible text for the first element matching a locator, plus match count.',
+    inputSchema: { type: 'object', properties: locatorProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await locatorTextBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Locator text failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_evaluate',
+    description: 'Execute JavaScript in the browser page context.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        script: { type: 'string' },
+      },
+      required: ['script'],
+    },
+    async handler(args, ctx) {
+      if (!args.script) return errorResult('script is required');
+      try {
+        const { result } = await evaluateBrowser(args, ctx);
+        const output = typeof result === 'object' ? JSON.stringify(result, null, 2) : String(result);
+        return textResult(output);
+      } catch (err) {
+        return errorResult(`Evaluate failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_wait_for_load_state',
+    description: 'Wait for the browser tab to reach load, domcontentloaded, or networkidle.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        state: { type: 'string', enum: ['load', 'domcontentloaded', 'networkidle'] },
+        timeoutMs: { type: 'number' },
+      },
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await waitForLoadStateBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Wait for load state failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_wait_for_url',
+    description: 'Wait for the browser tab URL to match a string. Supports * wildcards in Electron mode.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        url: { type: 'string' },
+        waitUntil: { type: 'string', enum: ['commit', 'load', 'domcontentloaded', 'networkidle'] },
+        timeoutMs: { type: 'number' },
+      },
+      required: ['url'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await waitForURLBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Wait for URL failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_wait_for_selector',
+    description: 'Wait for a CSS selector to become attached, detached, visible, or hidden.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        selector: { type: 'string' },
+        state: { type: 'string', enum: ['attached', 'detached', 'visible', 'hidden'] },
+        timeoutMs: { type: 'number' },
+      },
+      required: ['selector'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await waitForSelectorBrowser(args, ctx));
+      } catch (err) {
+        return errorResult(`Wait for selector failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_cua_click',
+    description: 'Click viewport coordinates in the visible browser tab.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        x: { type: 'number' },
+        y: { type: 'number' },
+        button: { type: 'number', description: '1 left, 2 middle, 3 right' },
+        keypress: { type: 'array', items: { type: 'string' } },
+      },
+      required: ['x', 'y'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await cuaBrowser('cua_click', { ...args, x: requireFiniteCoordinate(args.x, 'x'), y: requireFiniteCoordinate(args.y, 'y') }, ctx));
+      } catch (err) {
+        return errorResult(`CUA click failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_cua_double_click',
+    description: 'Double-click viewport coordinates in the visible browser tab.',
+    inputSchema: {
+      type: 'object',
+      properties: { ...tabTargetProperties, x: { type: 'number' }, y: { type: 'number' }, keypress: { type: 'array', items: { type: 'string' } } },
+      required: ['x', 'y'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await cuaBrowser('cua_double_click', { ...args, x: requireFiniteCoordinate(args.x, 'x'), y: requireFiniteCoordinate(args.y, 'y') }, ctx));
+      } catch (err) {
+        return errorResult(`CUA double-click failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_cua_move',
+    description: 'Move the visible automation cursor to viewport coordinates.',
+    inputSchema: {
+      type: 'object',
+      properties: { ...tabTargetProperties, x: { type: 'number' }, y: { type: 'number' }, keys: { type: 'array', items: { type: 'string' } } },
+      required: ['x', 'y'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await cuaBrowser('cua_move', { ...args, x: requireFiniteCoordinate(args.x, 'x'), y: requireFiniteCoordinate(args.y, 'y') }, ctx));
+      } catch (err) {
+        return errorResult(`CUA move failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_cua_scroll',
+    description: 'Scroll the visible browser tab from viewport coordinates.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        x: { type: 'number' },
+        y: { type: 'number' },
+        scrollX: { type: 'number' },
+        scrollY: { type: 'number' },
+        keypress: { type: 'array', items: { type: 'string' } },
+      },
+      required: ['x', 'y', 'scrollX', 'scrollY'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await cuaBrowser('cua_scroll', {
+          ...args,
+          x: requireFiniteCoordinate(args.x, 'x'),
+          y: requireFiniteCoordinate(args.y, 'y'),
+          scrollX: Number(args.scrollX || 0),
+          scrollY: Number(args.scrollY || 0),
+        }, ctx));
+      } catch (err) {
+        return errorResult(`CUA scroll failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_cua_type',
+    description: 'Type text at the currently focused browser element.',
+    inputSchema: {
+      type: 'object',
+      properties: { ...tabTargetProperties, text: { type: 'string' } },
+      required: ['text'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await cuaBrowser('cua_type', args, ctx));
+      } catch (err) {
+        return errorResult(`CUA type failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_cua_keypress',
+    description: 'Press a key combination in the visible browser tab.',
+    inputSchema: {
+      type: 'object',
+      properties: { ...tabTargetProperties, keys: { type: 'array', items: { type: 'string' } } },
+      required: ['keys'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await cuaBrowser('cua_keypress', args, ctx));
+      } catch (err) {
+        return errorResult(`CUA keypress failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_cua_drag',
+    description: 'Drag through a list of viewport points.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        path: { type: 'array', items: { type: 'object', properties: { x: { type: 'number' }, y: { type: 'number' } }, required: ['x', 'y'] } },
+        keys: { type: 'array', items: { type: 'string' } },
+      },
+      required: ['path'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await cuaBrowser('cua_drag', args, ctx));
+      } catch (err) {
+        return errorResult(`CUA drag failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_cua_screenshot',
+    description: 'Capture the visible browser tab viewport.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        const capture = await screenshotBrowser({ ...args, fullPage: false }, ctx);
+        return {
+          content: [
+            { type: 'image', data: capture.base64, mimeType: 'image/png' },
+            { type: 'text', text: `Visible browser screenshot captured (${capture.bytes} bytes)${targetLine(capture)}` },
+          ],
+        };
+      } catch (err) {
+        return errorResult(`CUA screenshot failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_clipboard_read_text',
+    description: 'Read plain text from the browser page clipboard API.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await clipboardBrowser('clipboard_read_text', args, ctx));
+      } catch (err) {
+        return errorResult(`Clipboard read failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_clipboard_write_text',
+    description: 'Write plain text through the browser page clipboard API.',
+    inputSchema: {
+      type: 'object',
+      properties: { ...tabTargetProperties, text: { type: 'string' } },
+      required: ['text'],
+    },
+    async handler(args, ctx) {
+      try {
+        return jsonText(await clipboardBrowser('clipboard_write_text', args, ctx));
+      } catch (err) {
+        return errorResult(`Clipboard write failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_start_video',
+    description: 'Start recording the visible browser session to a local WebM video. Saved when browser_stop_video is called.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...tabTargetProperties,
+        fps: { type: 'number', description: 'Recording frames per second (default: 12, max: 30)' },
+        session_id: { type: 'string', description: 'Deprecated alias for recording_id. Optional recording folder name under ~/.amalgm/browser-sessions' },
+        recording_id: { type: 'string', description: 'Optional recording folder name under ~/.amalgm/browser-sessions' },
+      },
+    },
+    async handler(args, ctx) {
+      try {
+        const result = await startVideoBrowser({
+          ...args,
+          fps: args.fps,
+          sessionId: args.recording_id || args.session_id,
+        }, ctx);
+        return textResult(`Browser video recording started (${result.fps || 'default'} fps).${targetLine(result)}`);
+      } catch (err) {
+        return errorResult(`Start video failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_stop_video',
+    description: 'Stop browser video recording and save it under ~/.amalgm/browser-sessions/.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      try {
+        const result = await stopVideoBrowser(args, ctx);
+        if (!result.path) return textResult(`Browser video recording stopped.${targetLine(result)}`);
+        const urlLine = result.url ? `\nOpen: ${result.url}` : '';
+        return {
+          content: [
+            { type: 'text', text: `Browser video saved: ${result.path}\nBytes: ${result.bytes || 0}${urlLine}${targetLine(result)}` },
+          ],
+          metadata: {
+            browserVideo: result.browserVideo || {
+              path: result.path,
+              relativePath: result.relativePath,
+              url: result.url,
+              bytes: result.bytes,
+              mimeType: result.mimeType || 'video/webm',
+            },
+          },
+        };
+      } catch (err) {
+        return errorResult(`Stop video failed: ${err.message}`);
+      }
+    },
+  },
+  {
+    name: 'browser_close',
+    description: 'Close the current or specified browser tab/surface.',
+    inputSchema: { type: 'object', properties: tabTargetProperties },
+    async handler(args, ctx) {
+      await closeBrowser(args, ctx);
+      return textResult('Browser closed. A new instance will launch on the next browser_* call.');
+    },
+  },
+];