nothumanallowed 9.2.4 → 9.3.1

package/src/services/tool-executor.mjs

@@ -7,6 +7,8 @@
  * Zero external dependencies — pure Node.js 22.
  */
 
+import os from 'os';
+
 import {
   listMessages,
   getMessage,
@@ -276,6 +278,55 @@ TOOLS:
     Returns: title, excerpt, and body text (max 8000 chars). Only fetches text/html/json/xml.
     Use this when the user provides a specific URL to read, summarize, or analyze.
 
+--- BROWSER AUTOMATION ---
+
+49. browser_open(url: string, waitForLoad?: boolean)
+    Open a URL in a headless Chrome browser. Launches Chrome automatically on first use.
+    SSRF-protected (blocks private IPs, localhost). Renders JavaScript, SPAs, and dynamic pages.
+    Use this when you need to interact with a page (click, type, screenshot) or when fetch_url fails on JS-rendered content.
+
+50. browser_screenshot(fullPage?: boolean, format?: "png"|"jpeg"|"webp", quality?: number, saveTo?: string)
+    Capture a screenshot of the current browser page.
+    fullPage=true captures the entire scrollable page. saveTo saves to a file path (e.g. "~/screenshot.png").
+    Returns base64-encoded image data. Use after browser_open to see what's on screen.
+
+51. browser_click(selector?: string, x?: number, y?: number)
+    Click an element on the page by CSS selector or x/y coordinates.
+    Examples: selector="#submit-btn", selector="a.nav-link", selector="button:nth-of-type(2)"
+    Use coordinates for precise clicking when selector doesn't work.
+
+52. browser_type(text: string, selector?: string, clear?: boolean, delay?: number)
+    Type text into an input field. If selector is provided, clicks the element first to focus it.
+    clear=true clears existing content before typing. delay=50 types with 50ms delay between keys.
+
+53. browser_extract(selector?: string, mode?: "text"|"html"|"value"|"attribute", attribute?: string, all?: boolean)
+    Extract content from the page. Default: extract all text from body.
+    selector="h1" extracts the first h1 text. all=true extracts from ALL matching elements.
+    mode="value" gets input values. mode="attribute" with attribute="href" gets link URLs.
+    mode="html" gets raw HTML of the element.
+
+54. browser_js(code: string)
+    Execute arbitrary JavaScript in the browser page context and return the result.
+    The code runs inside the page — you have access to document, window, fetch, etc.
+    Example: "document.querySelectorAll('.item').length" returns the count of items.
+    Use for complex interactions, form filling, or data extraction that other tools can't handle.
+
+55. browser_wait(selector: string, timeout?: number, visible?: boolean)
+    Wait for an element to appear on the page. Default timeout: 10 seconds.
+    visible=true (default) waits for the element to be visible, not just in DOM.
+    Use after clicking or navigating to wait for dynamic content to load.
+
+56. browser_scroll(direction?: "up"|"down"|"top"|"bottom", amount?: number)
+    Scroll the page. direction="down" (default) scrolls down 500px. "top"/"bottom" go to extremes.
+    amount=1000 scrolls 1000px instead of default 500.
+
+57. browser_key(key: string, ctrl?: boolean, shift?: boolean, alt?: boolean)
+    Press a keyboard key. Keys: Enter, Tab, Escape, Backspace, Delete, ArrowUp, ArrowDown, ArrowLeft, ArrowRight, Space.
+    ctrl=true holds Ctrl (or Cmd on Mac). Example: key="Enter" to submit a form.
+
+58. browser_close()
+    Close the browser. Frees resources. Browser auto-closes when NHA exits.
+
 RULES:
 - For search/read operations, execute immediately and present results conversationally.
 - For write/send/delete operations (gmail_send, gmail_reply, gmail_delete, calendar_create, calendar_move, calendar_update, contact_delete, task_done, notify_remind), DESCRIBE what you're about to do and include the JSON block so the system can ask the user for confirmation.
@@ -1035,6 +1086,152 @@ export async function executeTool(action, params, config) {
       return `Birthday set for ${contact.name}: ${monthName} ${day}. It will appear in the Birthdays tab.`;
     }
 
+    // ── Browser Automation ────────────────────────────────────────────
+    case 'browser_open': {
+      const be = await import('./browser-engine.mjs');
+      const url = params.url;
+      if (!url) return 'A URL is required.';
+
+      const result = await be.browserOpen(url, {
+        waitForLoad: params.waitForLoad !== false,
+      });
+      if (result.error) return `Browser error: ${result.message}`;
+
+      return `Page loaded: "${result.title}"\nURL: ${result.url}`;
+    }
+
+    case 'browser_screenshot': {
+      const be = await import('./browser-engine.mjs');
+      if (!be.isBrowserRunning()) return 'No browser open. Use browser_open first.';
+
+      const saveTo = params.saveTo
+        ? params.saveTo.replace(/^~/, os.homedir())
+        : null;
+
+      const result = await be.browserScreenshot({
+        fullPage: params.fullPage || false,
+        format: params.format || 'png',
+        quality: params.quality,
+        saveTo,
+      });
+      if (result.error) return `Screenshot error: ${result.message}`;
+
+      if (result.savedTo) {
+        return `Screenshot saved to: ${result.savedTo} (${Math.round(result.size / 1024)}KB base64)`;
+      }
+
+      // Return base64 for LLM vision analysis
+      return `Screenshot captured (${Math.round(result.size / 1024)}KB base64 PNG).\n[Base64 data available — use browser_js or browser_extract to analyze page content instead]`;
+    }
+
+    case 'browser_click': {
+      const be = await import('./browser-engine.mjs');
+      if (!be.isBrowserRunning()) return 'No browser open. Use browser_open first.';
+
+      const result = await be.browserClick({
+        selector: params.selector,
+        x: params.x,
+        y: params.y,
+      });
+      if (result.error) return `Click error: ${result.message}`;
+
+      return `Clicked: ${result.selector} at (${result.x}, ${result.y})`;
+    }
+
+    case 'browser_type': {
+      const be = await import('./browser-engine.mjs');
+      if (!be.isBrowserRunning()) return 'No browser open. Use browser_open first.';
+
+      if (!params.text) return 'Text is required.';
+
+      const result = await be.browserType({
+        text: params.text,
+        selector: params.selector,
+        clear: params.clear || false,
+        delay: params.delay || 0,
+      });
+      if (result.error) return `Type error: ${result.message}`;
+
+      return `Typed ${result.length} chars into ${result.selector}`;
+    }
+
+    case 'browser_extract': {
+      const be = await import('./browser-engine.mjs');
+      if (!be.isBrowserRunning()) return 'No browser open. Use browser_open first.';
+
+      const result = await be.browserExtract({
+        selector: params.selector || 'body',
+        mode: params.mode || 'text',
+        attribute: params.attribute,
+        all: params.all || false,
+      });
+      if (result.error) return `Extract error: ${result.message}`;
+
+      return `[${result.selector}] (${result.length} chars):\n${result.content}`;
+    }
+
+    case 'browser_js': {
+      const be = await import('./browser-engine.mjs');
+      if (!be.isBrowserRunning()) return 'No browser open. Use browser_open first.';
+
+      if (!params.code) return 'JavaScript code is required.';
+
+      const result = await be.browserEval(params.code);
+      if (result.error) return `JS error: ${result.message}`;
+
+      return `[${result.type}] ${result.result}`;
+    }
+
+    case 'browser_wait': {
+      const be = await import('./browser-engine.mjs');
+      if (!be.isBrowserRunning()) return 'No browser open. Use browser_open first.';
+
+      if (!params.selector) return 'A CSS selector is required.';
+
+      const result = await be.browserWaitFor(params.selector, {
+        timeout: params.timeout || 10000,
+        visible: params.visible !== false,
+      });
+      if (result.error) return `Wait failed: ${result.message}`;
+
+      return `Element found: ${result.selector} (${result.elapsed}ms)`;
+    }
+
+    case 'browser_scroll': {
+      const be = await import('./browser-engine.mjs');
+      if (!be.isBrowserRunning()) return 'No browser open. Use browser_open first.';
+
+      const result = await be.browserScroll({
+        direction: params.direction || 'down',
+        amount: params.amount || 500,
+      });
+      if (result.error) return `Scroll error: ${result.message}`;
+
+      return `Scrolled ${result.direction}`;
+    }
+
+    case 'browser_key': {
+      const be = await import('./browser-engine.mjs');
+      if (!be.isBrowserRunning()) return 'No browser open. Use browser_open first.';
+
+      if (!params.key) return 'A key name is required (e.g. Enter, Tab, Escape).';
+
+      const result = await be.browserKeyPress(params.key, {
+        ctrl: params.ctrl,
+        shift: params.shift,
+        alt: params.alt,
+      });
+      if (result.error) return `Key error: ${result.message}`;
+
+      return `Pressed: ${result.key}`;
+    }
+
+    case 'browser_close': {
+      const be = await import('./browser-engine.mjs');
+      const result = await be.browserClose();
+      return result.message;
+    }
+
     // ── Web Search & Fetch ──────────────────────────────────────────────
     case 'web_search': {
       const wt = await import('./web-tools.mjs');

package/src/services/web-ui.mjs

@@ -95,6 +95,11 @@ input:focus,textarea:focus{border-color:var(--green3)}
 .msg--assistant .msg__bubble{background:var(--greendim);border:1px solid var(--green3);border-radius:8px 8px 8px 2px;padding:10px 14px;max-width:85%;color:var(--text);white-space:pre-wrap;word-wrap:break-word}
 .msg__label{font-size:10px;color:var(--dim);margin-bottom:2px}
 .msg--thinking{color:var(--dim);font-style:italic}
+.tool-indicator{display:inline-block;padding:2px 8px;margin:2px 0;border-radius:4px;font-size:11px;background:var(--bg3);border:1px solid var(--border)}
+.tool-indicator--browser{border-color:#9c27b0;color:#ce93d8}
+.tool-indicator--web{border-color:var(--cyan);color:var(--cyan)}
+.tool-indicator--email{border-color:var(--green3);color:var(--green)}
+.screenshot-preview{max-width:100%;border-radius:var(--r);margin:8px 0;border:1px solid var(--border)}
 .chat__bar{display:flex;gap:8px;padding:10px 0 0 0;border-top:1px solid var(--border);flex-shrink:0}
 .chat__input{flex:1;resize:none;min-height:40px;max-height:100px;padding:10px 14px}
 .chat__send{background:var(--green3);color:var(--bg);padding:10px 16px;border-radius:var(--r);font-weight:700;font-size:12px}
@@ -372,11 +377,16 @@ function exportConvMd(){if(!activeConvId)return;window.open(API+'/api/conversati
 function renderMessages(){
   var el=document.getElementById('chatMessages');if(!el)return;
   if(chatHistory.length===0){
-    el.innerHTML='<div class="chat__empty"><div class="chat__empty-title">NHA Chat</div><div>Personal Operations Assistant — Streaming + Web Search</div><div class="chat__empty-hint">Try: Show my unread emails / Search the web for React 19 / What is on my calendar?</div></div>';
+    el.innerHTML='<div class="chat__empty"><div class="chat__empty-title">NHA Chat</div><div>Personal Operations Assistant — Streaming + Web Search + Browser</div><div class="chat__empty-hint">Try: Show my unread emails / Search the web for React 19 / Open google.com and take a screenshot</div></div>';
     return;
   }
   var h='';chatHistory.forEach(function(m){
-    h+='<div class="msg msg--'+esc(m.role)+'"><div class="msg__label">'+esc(m.role==='user'?'You':'NHA')+'</div><div class="msg__bubble">'+esc(m.content)+'</div></div>';
+    var raw=m.content||'';
+    var imgs=[];var idx=0;
+    var safe=raw.replace(/!\\[([^\\]]*)\\]\\((data:image\\/[a-z]+;base64,[A-Za-z0-9+\\/=]+)\\)/g,function(_,alt,src){var ph='__IMG'+idx+'__';imgs.push({ph:ph,alt:alt,src:src});idx++;return ph;});
+    var content=esc(safe);
+    for(var i=0;i<imgs.length;i++){content=content.replace(imgs[i].ph,'<img class="screenshot-preview" alt="'+esc(imgs[i].alt)+'" src="'+imgs[i].src+'">');}
+    h+='<div class="msg msg--'+esc(m.role)+'"><div class="msg__label">'+esc(m.role==='user'?'You':'NHA')+'</div><div class="msg__bubble">'+content+'</div></div>';
   });
   el.innerHTML=h;el.scrollTop=el.scrollHeight;
 }
@@ -491,12 +501,21 @@ function sendChat(){
                 if(el){var msgs=el.querySelectorAll('.msg');var last=msgs[msgs.length-1];if(last){var bub=last.querySelector('.msg__bubble');if(bub)bub.textContent=chatHistory[streamIdx].content;}el.scrollTop=el.scrollHeight;}
               }
               if(currentEvent==='tool'){
-                var indicator=data.status==='executing'?'['+data.action+' executing...]':'['+data.action+': '+data.status+']';
+                var toolLabels={browser_open:'Opening page',browser_screenshot:'Taking screenshot',browser_click:'Clicking element',browser_type:'Typing text',browser_extract:'Extracting content',browser_js:'Running JavaScript',browser_wait:'Waiting for element',browser_scroll:'Scrolling page',browser_key:'Pressing key',browser_close:'Closing browser',web_search:'Searching the web',fetch_url:'Fetching URL',gmail_list:'Searching emails',gmail_read:'Reading email',gmail_send:'Sending email',calendar_today:'Loading calendar',calendar_create:'Creating event'};
+                var label=toolLabels[data.action]||data.action;
+                var indicator=data.status==='executing'?'\\u23f3 '+label+'...':'\\u2705 '+label;
+                if(data.status==='error')indicator='\\u274c '+label+' failed';
                 chatHistory[streamIdx].content+=indicator+'\\n';
                 renderMessages();
               }
+              if(currentEvent==='screenshot'&&data.base64){
+                if(!chatHistory[streamIdx]._screenshots)chatHistory[streamIdx]._screenshots=[];
+                chatHistory[streamIdx]._screenshots.push('data:image/'+(data.format||'png')+';base64,'+data.base64);
+                chatHistory[streamIdx].content+='\\n![Screenshot](data:image/'+(data.format||'png')+';base64,'+data.base64+')\\n';
+                renderMessages();
+              }
               if(currentEvent==='tool_synthesis'){chatHistory[streamIdx].content='';renderMessages();}
-              if(currentEvent==='done'){chatStreaming=false;if(data.content)chatHistory[streamIdx].content=data.content;renderMessages();loadConvList();}
+              if(currentEvent==='done'){chatStreaming=false;if(data.content)chatHistory[streamIdx].content=data.content;var ss=chatHistory[streamIdx]._screenshots;if(ss&&ss.length>0){for(var si=0;si<ss.length;si++){chatHistory[streamIdx].content+='\\n![Screenshot]('+ss[si]+')\\n';}}renderMessages();loadConvList();}
               if(currentEvent==='error'){chatStreaming=false;chatHistory[streamIdx].content='Error: '+(data.message||'Unknown');renderMessages();}
             }catch(e){}
           }