@monoes/monobrowse 1.0.0 → 1.0.1

package/src/browser/screenshot.ts

@@ -1,4 +1,6 @@
 import type { CdpClient } from './cdp.js';
+import type { ElementRef } from './types.js';
+import { getObjectIdForRef } from './snapshot.js';
 import { writeFile, mkdir } from 'fs/promises';
 import { join, dirname } from 'path';
 import { tmpdir } from 'os';
@@ -9,6 +11,68 @@ export interface ScreenshotOptions {
   quality?: number;
   format?: 'jpeg' | 'png' | 'webp';
   annotate?: boolean;
+  refs?: Map<string, ElementRef>;
+}
+
+async function getViewportTopLeft(
+  client: CdpClient,
+  sessionId: string,
+  ref: ElementRef
+): Promise<{ x: number; y: number } | null> {
+  const objectId = await getObjectIdForRef(client, sessionId, ref).catch(() => null);
+  if (!objectId) return null;
+  try {
+    const r = await client.send<{ result: { value?: string } }>('Runtime.callFunctionOn', {
+      functionDeclaration: 'function() { var r=this.getBoundingClientRect(); if(r.width===0&&r.height===0)return null; return JSON.stringify({x:Math.round(r.left),y:Math.round(r.top)}); }',
+      objectId,
+      returnByValue: true,
+    }, sessionId);
+    if (!r.result?.value) return null;
+    return JSON.parse(r.result.value) as { x: number; y: number };
+  } catch {
+    return null;
+  }
+}
+
+async function injectAnnotationOverlay(
+  client: CdpClient,
+  sessionId: string,
+  refs: Map<string, ElementRef>
+): Promise<void> {
+  const badges: Array<{ num: number; x: number; y: number }> = [];
+  for (const [key, ref] of refs) {
+    const m = key.match(/^e(\d+)$/);
+    if (!m) continue;
+    const pos = await getViewportTopLeft(client, sessionId, ref);
+    if (pos) badges.push({ num: parseInt(m[1], 10), x: pos.x, y: pos.y });
+  }
+  if (badges.length === 0) return;
+
+  const badgesJson = JSON.stringify(badges);
+  await client.send('Runtime.evaluate', {
+    expression: `(function(){
+      var p=document.getElementById('__mm_ann__');if(p)p.remove();
+      var c=document.createElement('div');
+      c.id='__mm_ann__';
+      c.style.cssText='position:fixed;top:0;left:0;width:0;height:0;z-index:2147483647;pointer-events:none;';
+      var bs=${badgesJson};
+      bs.forEach(function(b){
+        var d=document.createElement('div');
+        d.style.cssText='position:fixed;left:'+b.x+'px;top:'+b.y+'px;background:#e74c3c;color:#fff;border-radius:3px;padding:0 4px;font-size:11px;font-family:monospace;font-weight:bold;line-height:18px;box-shadow:0 1px 3px rgba(0,0,0,.5);';
+        d.textContent=b.num;
+        c.appendChild(d);
+      });
+      document.body.appendChild(c);
+    })()`,
+    returnByValue: false,
+  }, sessionId).catch(() => {});
+}
+
+async function removeAnnotationOverlay(client: CdpClient, sessionId: string): Promise<void> {
+  await client.send('Runtime.evaluate', {
+    expression: `(function(){var e=document.getElementById('__mm_ann__');if(e)e.remove();})()`,
+    returnByValue: false,
+  }, sessionId).catch(() => {});
 }
 
 export async function captureScreenshot(
@@ -33,10 +97,21 @@ export async function captureScreenshot(
     params.captureBeyondViewport = true;
   }
 
-  const result = await client.send<{ data: string }>('Page.captureScreenshot', params, sessionId);
-  const data = result.data;
-  const dataUrl = `data:image/${format};base64,${data}`;
+  if (options.annotate && options.refs) {
+    await injectAnnotationOverlay(client, sessionId, options.refs);
+  }
 
+  let data: string;
+  try {
+    const result = await client.send<{ data: string }>('Page.captureScreenshot', params, sessionId);
+    data = result.data;
+  } finally {
+    if (options.annotate) {
+      await removeAnnotationOverlay(client, sessionId);
+    }
+  }
+
+  const dataUrl = `data:image/${format};base64,${data}`;
   const outputPath = options.path ?? join(tmpdir(), `monomind-screenshot-${Date.now()}.${format}`);
   await mkdir(dirname(outputPath), { recursive: true });
   await writeFile(outputPath, Buffer.from(data, 'base64'));

package/src/browser/types.ts

@@ -125,11 +125,22 @@ export const INTERACTIVE_ROLES = new Set([
 ]);
 
 export const CHROME_EXECUTABLES = [
+  // macOS — Chrome
   '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
   '/Applications/Chromium.app/Contents/MacOS/Chromium',
+  // macOS — Edge
+  '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
+  // Linux — Chrome
   '/usr/bin/google-chrome',
   '/usr/bin/chromium-browser',
   '/usr/bin/chromium',
+  // Linux — Edge
+  '/usr/bin/microsoft-edge',
+  '/usr/bin/microsoft-edge-stable',
+  // Windows — Chrome
   'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
   'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
+  // Windows — Edge (pre-installed on all Windows 10/11 machines)
+  'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
+  'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
 ];

package/src/cli/action.ts

@@ -0,0 +1,145 @@
+// src/commands/browse-action.ts
+import { Command } from 'commander';
+import { readdir, writeFile, mkdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { analyzePageForAction, type AnalyzerPage, readAction } from '../index.js';
+import type { ActionDef } from '../index.js';
+
+// Built-in actions (shipped with the CLI) — actual step definitions live in adapters/
+const BUILTIN_ACTIONS: { id: string; platform: string; name: string }[] = [
+  { id: 'linkedin:comment_post', platform: 'linkedin', name: 'Comment on Post' },
+  { id: 'linkedin:like_post', platform: 'linkedin', name: 'Like Post' },
+  { id: 'linkedin:send_connection', platform: 'linkedin', name: 'Send Connection Request' },
+  { id: 'linkedin:publish_post', platform: 'linkedin', name: 'Publish Post' },
+  { id: 'linkedin:keyword_search', platform: 'linkedin', name: 'Keyword Search' },
+  { id: 'instagram:like_post', platform: 'instagram', name: 'Like Post' },
+  { id: 'instagram:comment_post', platform: 'instagram', name: 'Comment on Post' },
+  { id: 'instagram:follow_user', platform: 'instagram', name: 'Follow User' },
+  { id: 'instagram:send_dm', platform: 'instagram', name: 'Send DM' },
+  { id: 'instagram:hashtag_search', platform: 'instagram', name: 'Hashtag Search' },
+  { id: 'x:like_post', platform: 'x', name: 'Like Post' },
+  { id: 'x:reply_post', platform: 'x', name: 'Reply to Post' },
+  { id: 'x:follow_user', platform: 'x', name: 'Follow User' },
+  { id: 'x:publish_post', platform: 'x', name: 'Publish Post' },
+  { id: 'x:keyword_search', platform: 'x', name: 'Keyword Search' },
+  { id: 'gemini:submit_prompt', platform: 'gemini', name: 'Submit Prompt' },
+  { id: 'gemini:extract_response', platform: 'gemini', name: 'Extract Response' },
+];
+
+async function getCustomActions(): Promise<ActionDef[]> {
+  const dir = join(process.cwd(), '.monomind', 'actions');
+  if (!existsSync(dir)) return [];
+  const files = (await readdir(dir)).filter(f => f.endsWith('.json'));
+  const results: ActionDef[] = [];
+  for (const f of files) {
+    try {
+      results.push(await readAction(join(dir, f)));
+    } catch {
+      // skip malformed action files
+    }
+  }
+  return results;
+}
+
+export function createActionCommand(): Command {
+  const cmd = new Command('action').description('Manage browser actions');
+
+  cmd
+    .command('build')
+    .description('AI-powered: analyze a page and generate an action definition')
+    .requiredOption('--url <url>', 'URL to analyze')
+    .requiredOption('--task <description>', 'What the action should do')
+    .option('--output <file>', 'Output file path (default: .monomind/actions/<id>.json)')
+    .action(async (opts: { url: string; task: string; output?: string }) => {
+      console.log(`Analyzing ${opts.url} for task: "${opts.task}"`);
+      console.log('Opening browser...');
+
+      let actionDef: ActionDef;
+      try {
+        // Dynamic import to avoid crashing if CDP is unavailable
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const mod = await (import('@monoes/monobrowse' as string) as Promise<any>).catch(() => null);
+        const createBrowserPage = mod?.createBrowserPage as ((url: string) => Promise<AnalyzerPage>) | null;
+        if (!createBrowserPage) {
+          throw new Error('Browser CDP client not available. Ensure Chrome is running with --remote-debugging-port=9222');
+        }
+        const page = await createBrowserPage(opts.url);
+        actionDef = await analyzePageForAction(page, opts.task);
+      } catch (err) {
+        console.error(`Action build failed: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(1);
+      }
+
+      const dir = join(process.cwd(), '.monomind', 'actions');
+      await mkdir(dir, { recursive: true });
+      const outputPath = opts.output ?? join(dir, `${actionDef.id.replace(':', '-')}.json`);
+      await writeFile(outputPath, JSON.stringify(actionDef, null, 2));
+      console.log(`\nGenerated action: ${outputPath}`);
+      console.log(`Action ID: ${actionDef.id}`);
+      console.log(`Steps: ${actionDef.steps.length}`);
+    });
+
+  cmd
+    .command('run <action-id>')
+    .description('Run a single action')
+    .requiredOption('--account <user>', 'Account username to use')
+    .option('--params <pairs...>', 'Key=value params e.g. --params post_url=https://... text=hello')
+    .action(async (actionId: string, opts: { account: string; params?: string[] }) => {
+      const params: Record<string, string> = {};
+      for (const pair of opts.params ?? []) {
+        const eq = pair.indexOf('=');
+        if (eq > 0) params[pair.slice(0, eq)] = pair.slice(eq + 1);
+      }
+      console.log(`Running action: ${actionId} with account: ${opts.account}`);
+      console.log('Params:', params);
+      console.log('(Action runner not yet implemented — use browse playbook run for full execution)');
+    });
+
+  cmd
+    .command('list')
+    .description('List available actions (built-in + custom)')
+    .option('--platform <platform>', 'Filter by platform')
+    .action(async (opts: { platform?: string }) => {
+      const custom = await getCustomActions();
+      const all = [
+        ...BUILTIN_ACTIONS.map(a => ({ ...a, source: 'built-in' })),
+        ...custom.map(a => ({ id: a.id, platform: a.platform, name: a.name, source: 'custom' })),
+      ];
+      const filtered = opts.platform ? all.filter(a => a.platform === opts.platform) : all;
+      if (filtered.length === 0) {
+        console.log('No actions found.');
+        return;
+      }
+      console.log(`\n${'ID'.padEnd(40)} ${'PLATFORM'.padEnd(12)} ${'NAME'.padEnd(30)} SOURCE`);
+      console.log('─'.repeat(90));
+      for (const a of filtered) {
+        console.log(`${a.id.padEnd(40)} ${a.platform.padEnd(12)} ${a.name.padEnd(30)} ${a.source}`);
+      }
+    });
+
+  cmd
+    .command('show <action-id>')
+    .description('Print the action definition JSON')
+    .action(async (actionId: string) => {
+      // Check custom actions first
+      const dir = join(process.cwd(), '.monomind', 'actions');
+      const fileName = actionId.replace(':', '-') + '.json';
+      const customPath = join(dir, fileName);
+      if (existsSync(customPath)) {
+        const def = await readAction(customPath);
+        console.log(JSON.stringify(def, null, 2));
+        return;
+      }
+      // Check built-ins
+      const builtin = BUILTIN_ACTIONS.find(a => a.id === actionId);
+      if (builtin) {
+        console.log(JSON.stringify({ ...builtin, note: 'Built-in action — steps defined in browser/adapters/' }, null, 2));
+        return;
+      }
+      console.error(`Action not found: ${actionId}`);
+      process.exit(1);
+    });
+
+  return cmd;
+}