@swarmclawai/swarmclaw 0.7.7 → 0.8.0

package/src/lib/server/session-tools/web.ts

@@ -2,12 +2,13 @@ import { z } from 'zod'
 import { tool, type StructuredToolInterface } from '@langchain/core/tools'
 import fs from 'fs'
 import path from 'path'
+import { fileURLToPath, pathToFileURL } from 'url'
 import * as cheerio from 'cheerio'
 import { UPLOAD_DIR } from '../storage'
 import type { ToolBuildContext } from './context'
 import { spawnSync } from 'child_process'
 import { safePath, truncate, MAX_OUTPUT, findBinaryOnPath } from './context'
-import { getSearchProvider } from './search-providers'
+import { getSearchProvider, type SearchResult } from './search-providers'
 import { dedupeScreenshotMarkdownLines } from './web-output'
 import { withRetry } from '../tool-retry'
 import type { Plugin, PluginHooks } from '@/types'
@@ -16,44 +17,55 @@ import { normalizeToolInputArgs } from './normalize-tool-args'
 import {
   ensureSessionBrowserProfileId,
   getBrowserProfileDir,
+  loadBrowserSessionRecord,
   markBrowserSessionClosed,
   recordBrowserObservation,
   removeBrowserSessionRecord,
   upsertBrowserSessionRecord,
 } from '../browser-state'
 
-// --- Search result compression logic ---
-async function compressSearchResults(results: any[], query: string, bctx: any): Promise<string | null> {
-  const session = bctx.resolveCurrentSession?.()
-  if (!session?.provider || !session?.model) return null
-  const { getProvider } = await import('@/lib/providers')
-  const { loadCredentials, decryptKey } = await import('../storage')
-  const providerEntry = getProvider(session.provider)
-  if (!providerEntry?.handler?.streamChat) return null
-  let apiKey: string | undefined
-  if (session.credentialId) {
-    const creds = loadCredentials()
-    const cred = creds[session.credentialId]
-    if (cred) apiKey = decryptKey(cred.encryptedKey)
-  }
-  const systemPrompt = 'You are a search result summarizer. Condense search results into a concise reference. Keep key facts, URLs, and data points. Remove filler and redundancy. Output plain text, not JSON.'
-  const message = `Query: "${query}"\n\nResults:\n${JSON.stringify(results, null, 1)}\n\nSummarize these results concisely.`
-  let compressed = ''
-  await providerEntry.handler.streamChat({
-    session: { ...session, messages: [] }, message, apiKey, systemPrompt,
-    write: (raw: string) => {
-      const lines = raw.split('\n').filter(Boolean)
-      for (const line of lines) {
-        if (!line.startsWith('data: ')) continue
-        try {
-          const ev = JSON.parse(line.slice(6))
-          if (ev.t === 'd' && ev.text) compressed += ev.text
-        } catch { /* ignore */ }
+function cleanSearchField(value: string | undefined): string {
+  return (value || '').replace(/\s+/g, ' ').trim()
+}
+
+export function formatWebSearchResults(query: string, results: SearchResult[], maxChars = MAX_OUTPUT): string {
+  const cleanedQuery = cleanSearchField(query)
+  const header = cleanedQuery ? `Search results for: ${cleanedQuery}` : 'Search results'
+  const sections: string[] = [header]
+  const joinSections = (items: string[]) => items.filter(Boolean).join('\n\n')
+
+  for (let index = 0; index < results.length; index++) {
+    const result = results[index]
+    const title = cleanSearchField(result?.title) || cleanSearchField(result?.url) || `Result ${index + 1}`
+    const url = cleanSearchField(result?.url)
+    const snippet = cleanSearchField(result?.snippet)
+    const lines = [`${index + 1}. ${title}`]
+    if (url) lines.push(`URL: ${url}`)
+    if (snippet) lines.push(`Snippet: ${snippet}`)
+    const candidate = joinSections([...sections, lines.join('\n')])
+    if (candidate.length <= maxChars) {
+      sections.push(lines.join('\n'))
+      continue
+    }
+
+    if (url) {
+      const minimalLines = [`${index + 1}. ${title}`, `URL: ${url}`]
+      const minimalCandidate = joinSections([...sections, minimalLines.join('\n')])
+      if (minimalCandidate.length <= maxChars) {
+        sections.push(minimalLines.join('\n'))
       }
-    },
-    active: new Map(), loadHistory: () => [],
-  })
-  return compressed.trim() || null
+    }
+
+    const omitted = results.length - index
+    if (omitted > 0) {
+      const remainingNotice = `(${omitted} additional result${omitted === 1 ? '' : 's'} omitted for brevity)`
+      const withNotice = joinSections([...sections, remainingNotice])
+      if (withNotice.length <= maxChars) sections.push(remainingNotice)
+    }
+    return truncate(joinSections(sections), maxChars)
+  }
+
+  return truncate(joinSections(sections), maxChars)
 }
 
 type BrowserRuntimeEntry = {
@@ -159,13 +171,221 @@ export function cleanupSessionBrowser(sessionId: string): void {
 export function getActiveBrowserCount(): number { return activeBrowsers.size }
 export function hasActiveBrowser(sessionId: string): boolean { return activeBrowsers.has(sessionId) }
 
+export function inferWebActionFromArgs(params: {
+  action?: string
+  query?: string
+  url?: string
+}): 'search' | 'fetch' | undefined {
+  if (params.action === 'search' || params.action === 'fetch') return params.action
+  if (typeof params.url === 'string' && /^https?:\/\//i.test(params.url.trim())) return 'fetch'
+  if (typeof params.query === 'string' && params.query.trim()) return 'search'
+  if (typeof params.url === 'string' && params.url.trim()) return 'search'
+  return undefined
+}
+
+function parseStructuredJsonValue(value: unknown): unknown {
+  if (typeof value !== 'string') return value
+  const trimmed = value.trim()
+  if (!trimmed || (!trimmed.startsWith('{') && !trimmed.startsWith('['))) return value
+  try {
+    return JSON.parse(trimmed)
+  } catch {
+    return value
+  }
+}
+
+function parseJsonObjectValue(value: unknown): Record<string, unknown> | null {
+  const parsed = parseStructuredJsonValue(value)
+  return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+    ? parsed as Record<string, unknown>
+    : null
+}
+
+function parseJsonArrayValue(value: unknown): unknown[] | null {
+  const parsed = parseStructuredJsonValue(value)
+  return Array.isArray(parsed) ? parsed : null
+}
+
+function pickNonEmptyBrowserString(...values: unknown[]): string | undefined {
+  for (const value of values) {
+    if (typeof value !== 'string') continue
+    const trimmed = value.trim()
+    if (trimmed) return trimmed
+  }
+  return undefined
+}
+
+function wrapBrowserEvaluateFunction(code: string): string {
+  const trimmed = code.trim()
+  if (!trimmed) return trimmed
+  if (/^(?:async\s+)?function\b/.test(trimmed)) return trimmed
+  if (/^(?:async\s*)?\([^)]*\)\s*=>/.test(trimmed)) return trimmed
+  return /[;{}]/.test(trimmed)
+    ? `() => { ${trimmed} }`
+    : `() => (${trimmed})`
+}
+
+function wrapBrowserRunCodeFunction(code: string): string {
+  const trimmed = code.trim()
+  if (!trimmed) return trimmed
+  if (/^(?:async\s+)?function\b/.test(trimmed)) return trimmed
+  if (/^(?:async\s*)?\([^)]*\)\s*=>/.test(trimmed)) return trimmed
+  return /[;{}]/.test(trimmed)
+    ? `async (page) => { ${trimmed} }`
+    : `async (page) => (${trimmed})`
+}
+
+export function normalizeBrowserActionParams(rawParams: Record<string, unknown>): Record<string, unknown> {
+  const normalized = normalizeToolInputArgs(rawParams)
+  const action = String(normalized.action || '').trim().toLowerCase()
+  const params: Record<string, unknown> = { ...normalized }
+
+  const parsedFields = parseJsonArrayValue(params.fields)
+  if (parsedFields) params.fields = parsedFields
+
+  const parsedForm = parseJsonObjectValue(params.form)
+  if (parsedForm) params.form = parsedForm
+
+  if (typeof params.selector === 'string' && !pickNonEmptyBrowserString(params.element)) {
+    params.element = params.selector
+  }
+
+  if (action === 'submit_form' && typeof params.selector === 'string' && !pickNonEmptyBrowserString(params.submitElement)) {
+    params.submitElement = params.selector
+  }
+
+  if (action === 'select') {
+    const parsedValues = parseJsonArrayValue(params.values ?? params.option ?? params.value)
+    if (parsedValues) params.values = parsedValues
+    else if (params.values === undefined) {
+      const scalar = pickNonEmptyBrowserString(params.option, params.value, params.text)
+      if (scalar) params.values = [scalar]
+    }
+  }
+
+  if (action === 'evaluate' && !pickNonEmptyBrowserString(params.function)) {
+    const code = pickNonEmptyBrowserString(params.code, params.script, params.javascript, params.js)
+    if (code) params.function = wrapBrowserEvaluateFunction(code)
+  }
+
+  if (action === 'run_code') {
+    const code = pickNonEmptyBrowserString(params.code, params.function, params.script, params.javascript, params.js)
+    if (code) params.code = wrapBrowserRunCodeFunction(code)
+  }
+
+  return params
+}
+
+function pickBrowserTargetFromParams(params: Record<string, unknown>): string | null {
+  for (const value of [
+    params.url,
+    params.filePath,
+    params.path,
+    params.href,
+    params.link,
+    params.target,
+    params.page,
+  ]) {
+    if (typeof value !== 'string') continue
+    const trimmed = value.trim()
+    if (trimmed) return trimmed
+  }
+  return null
+}
+
+function resolveUploadFilePath(target: string): string | null {
+  const normalized = target.replace(/^sandbox:/, '')
+  const match = normalized.match(/^\/api\/uploads\/([^?#]+)/)
+  if (!match) return null
+  let decoded = match[1]
+  try {
+    decoded = decodeURIComponent(decoded)
+  } catch {
+    // keep raw segment
+  }
+  const safeName = decoded.replace(/[^a-zA-Z0-9._-]/g, '')
+  const resolved = path.join(UPLOAD_DIR, safeName)
+  return fs.existsSync(resolved) ? resolved : null
+}
+
+function resolveBrowserFileUrlPath(target: string): string | null {
+  if (!/^file:/i.test(target)) return null
+  try {
+    const resolved = fileURLToPath(target)
+    return fs.existsSync(resolved) ? resolved : null
+  } catch {
+    return null
+  }
+}
+
+function tryResolveBrowserLocalPath(cwd: string, target: string): string | null {
+  const uploadPath = resolveUploadFilePath(target)
+  if (uploadPath) return uploadPath
+
+  const fileUrlPath = resolveBrowserFileUrlPath(target)
+  if (fileUrlPath) return fileUrlPath
+
+  if (/^(?:https?:|about:|data:)/i.test(target)) return null
+
+  const normalized = target.replace(/^sandbox:/, '')
+  const looksLikePath = normalized.startsWith('/')
+    || normalized.startsWith('./')
+    || normalized.startsWith('../')
+    || normalized.includes('/')
+    || /\.(?:html?|xhtml|txt|md|json|ya?ml|csv|ts|tsx|js|jsx|mjs|cjs|css|png|jpe?g|gif|webp|svg|pdf)$/i.test(normalized)
+  if (!looksLikePath) return null
+
+  const candidates = new Set<string>()
+  if (path.isAbsolute(normalized)) candidates.add(normalized)
+  try { candidates.add(safePath(cwd, normalized)) } catch { /* ignore */ }
+  try { candidates.add(path.resolve(cwd, normalized)) } catch { /* ignore */ }
+
+  for (const candidate of candidates) {
+    if (!candidate || !fs.existsSync(candidate)) continue
+    const stat = fs.statSync(candidate)
+    if (stat.isDirectory()) {
+      const indexPath = path.join(candidate, 'index.html')
+      if (fs.existsSync(indexPath)) return indexPath
+      return null
+    }
+    return candidate
+  }
+  return null
+}
+
+function localHtmlFileToDataUrl(filePath: string): string | null {
+  const ext = path.extname(filePath).toLowerCase()
+  if (ext !== '.html' && ext !== '.htm') return null
+  try {
+    const html = fs.readFileSync(filePath, 'utf8')
+    const hasRelativeAssetReferences = /<(?:script|img|source|video|audio)\b[^>]+\b(?:src|poster)\s*=\s*["'](?![a-z]+:|\/\/|#|\/)([^"']+)["']|<link\b[^>]+\bhref\s*=\s*["'](?![a-z]+:|\/\/|#|\/)([^"']+)["']/i.test(html)
+    if (hasRelativeAssetReferences) return null
+    return `data:text/html;charset=utf-8,${encodeURIComponent(html)}`
+  } catch {
+    return null
+  }
+}
+
+export function resolveBrowserNavigationTarget(cwd: string, target: string): string {
+  const trimmed = target.trim()
+  if (!trimmed) return trimmed
+  const localPath = tryResolveBrowserLocalPath(cwd, trimmed)
+  if (localPath) return localHtmlFileToDataUrl(localPath) || pathToFileURL(localPath).toString()
+  return trimmed.replace(/^sandbox:/, '')
+}
+
 /**
  * Unified Web Execution Logic
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-async function executeWebAction(args: Record<string, unknown>, bctx: any) {
+async function executeWebAction(args: Record<string, unknown>) {
   const normalized = normalizeToolInputArgs(args)
-  const { action, query, url, maxResults } = normalized as { action: string; query?: string; url?: string; maxResults?: number }
+  const { query, url, maxResults } = normalized as { query?: string; url?: string; maxResults?: number }
+  const action = inferWebActionFromArgs({
+    action: (normalized as { action?: string }).action,
+    query,
+    url,
+  })
   try {
     if (action === 'search') {
       const searchQuery = query || url
@@ -176,12 +396,7 @@ async function executeWebAction(args: Record<string, unknown>, bctx: any) {
       const provider = await getSearchProvider(settings)
       const results = await provider.search(searchQuery, limit)
       if (results.length === 0) return 'No results found.'
-      const raw = JSON.stringify(results, null, 2)
-      if (raw.length > 2000) {
-        const compressed = await compressSearchResults(results, searchQuery, bctx)
-        if (compressed) return compressed
-      }
-      return raw
+      return formatWebSearchResults(searchQuery, results)
     } else if (action === 'fetch') {
       const fetchUrl = url || query
       if (!fetchUrl) return 'Error: "url" is required for fetch action.'
@@ -238,7 +453,7 @@ const WebPlugin: Plugin = {
         },
         required: ['action']
       },
-      execute: async (args, context) => executeWebAction(args, { ...context.session, resolveCurrentSession: () => context.session })
+      execute: async (args) => executeWebAction(args)
     }
   ]
 }
@@ -255,7 +470,7 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
   if (bctx.hasPlugin('web')) {
     tools.push(
       tool(
-        async (args) => executeWebAction(args, bctx),
+        async (args) => executeWebAction(args),
         {
           name: 'web',
           description: WebPlugin.tools![0].description,
@@ -329,13 +544,30 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
           pendingBrowserInitializations.set(sessionKey, connectPromise)
           const entry = await connectPromise
           acquireExistingEntry(entry)
+          const lastState = loadBrowserSessionRecord(sessionKey)
+          const restoreUrl = typeof lastState?.currentUrl === 'string' ? lastState.currentUrl.trim() : ''
+          if (restoreUrl && restoreUrl !== 'about:blank') {
+            try {
+              await entry.client.callTool({ name: 'browser_navigate', arguments: { url: restoreUrl } })
+            } catch (err: unknown) {
+              upsertBrowserSessionRecord({
+                sessionId: sessionKey,
+                profileId: profileInfo.profileId,
+                profileDir,
+                inheritedFromSessionId: profileInfo.inheritedFromSessionId,
+                status: 'error',
+                lastAction: 'browser_restore',
+                lastError: err instanceof Error ? err.message : String(err),
+              })
+            }
+          }
           upsertBrowserSessionRecord({
             sessionId: sessionKey,
             profileId: profileInfo.profileId,
             profileDir,
             inheritedFromSessionId: profileInfo.inheritedFromSessionId,
             status: 'active',
-            lastAction: 'browser_open',
+            lastAction: restoreUrl && restoreUrl !== 'about:blank' ? 'browser_restore' : 'browser_open',
           })
         } finally {
           if (pendingBrowserInitializations.get(sessionKey)) {
@@ -404,6 +636,105 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
       function: fn,
     })
 
+    const performSelectorDomAction = async (
+      action: 'click' | 'type' | 'select' | 'hover',
+      params: Record<string, unknown>,
+    ): Promise<{ ok: true; output: string } | { ok: false; error: string } | null> => {
+      const selector = pickNonEmptyBrowserString(params.element, params.selector)
+      if (!selector) return null
+      if (typeof params.ref === 'string' && params.ref.trim()) return null
+
+      const payload =
+        action === 'click'
+          ? `() => {
+              const selector = ${JSON.stringify(selector)};
+              try {
+                const element = document.querySelector(selector);
+                if (!element) return { ok: false, error: 'No element matched selector.' };
+                element.click?.();
+                return { ok: true, action: 'click', selector };
+              } catch (error) {
+                return { ok: false, error: String(error) };
+              }
+            }`
+          : action === 'hover'
+            ? `() => {
+                const selector = ${JSON.stringify(selector)};
+                try {
+                  const element = document.querySelector(selector);
+                  if (!element) return { ok: false, error: 'No element matched selector.' };
+                  element.dispatchEvent(new MouseEvent('mouseover', { bubbles: true }));
+                  element.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true }));
+                  return { ok: true, action: 'hover', selector };
+                } catch (error) {
+                  return { ok: false, error: String(error) };
+                }
+              }`
+            : action === 'type'
+              ? `() => {
+                  const selector = ${JSON.stringify(selector)};
+                  const value = ${JSON.stringify(String(params.text ?? params.value ?? ''))};
+                  const submit = ${params.submit === true ? 'true' : 'false'};
+                  try {
+                    const element = document.querySelector(selector);
+                    if (!element) return { ok: false, error: 'No element matched selector.' };
+                    element.focus?.();
+                    if ('value' in element) element.value = value;
+                    else if (element.isContentEditable) element.textContent = value;
+                    else return { ok: false, error: 'Matched element is not editable.' };
+                    element.dispatchEvent(new Event('input', { bubbles: true }));
+                    element.dispatchEvent(new Event('change', { bubbles: true }));
+                    if (submit) {
+                      if (element.form?.requestSubmit) element.form.requestSubmit();
+                      else {
+                        element.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }));
+                        element.dispatchEvent(new KeyboardEvent('keyup', { key: 'Enter', bubbles: true }));
+                      }
+                    }
+                    return { ok: true, action: 'type', selector, valueLength: value.length };
+                  } catch (error) {
+                    return { ok: false, error: String(error) };
+                  }
+                }`
+              : `() => {
+                  const selector = ${JSON.stringify(selector)};
+                  const desired = ${JSON.stringify(
+                    Array.isArray(params.values)
+                      ? params.values.map((value) => String(value ?? ''))
+                      : [String(params.value ?? params.option ?? '')],
+                  )};
+                  try {
+                    const element = document.querySelector(selector);
+                    if (!element) return { ok: false, error: 'No element matched selector.' };
+                    if (!(element instanceof HTMLSelectElement)) return { ok: false, error: 'Matched element is not a <select>.' };
+                    const selected = [];
+                    for (const option of Array.from(element.options)) {
+                      const match = desired.includes(option.value) || desired.includes(option.text);
+                      option.selected = match;
+                      if (match) selected.push(option.value || option.text || '');
+                    }
+                    element.dispatchEvent(new Event('input', { bubbles: true }));
+                    element.dispatchEvent(new Event('change', { bubbles: true }));
+                    return { ok: true, action: 'select', selector, selected };
+                  } catch (error) {
+                    return { ok: false, error: String(error) };
+                  }
+                }`
+
+      const raw = await callBrowserEvaluate(payload)
+      const parsed = extractJsonPayload(raw)
+      if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        return { ok: false, error: cleanPlaywrightOutput(raw) || `DOM ${action} fallback failed.` }
+      }
+      if ((parsed as Record<string, unknown>).ok !== true) {
+        return {
+          ok: false,
+          error: String((parsed as Record<string, unknown>).error || `DOM ${action} fallback failed.`),
+        }
+      }
+      return { ok: true, output: stringifyStructured(parsed) }
+    }
+
     const captureStructuredObservation = async () => {
       const expression = `() => {
         const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
@@ -650,12 +981,141 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
     }
 
     const dismissCookieBanners = async (mcpCall: (toolName: string, args: Record<string, unknown>) => Promise<string>) => {
-      await new Promise((r) => setTimeout(r, 1500))
+      await new Promise((r) => setTimeout(r, 1200))
       const js = `() => {
-        const sel = ['button[id*="reject" i]', 'button[class*="reject" i]', 'a[id*="reject" i]', 'a[class*="reject" i]', '#onetrust-reject-all-handler', '#CybotCookiebotDialogBodyButtonDecline', '#didomi-notice-disagree-button', '.qc-cmp2-summary-buttons button:first-child', 'button.sp_choice_type_12'];
-        for (const s of sel) { const el = document.querySelector(s); if (el && el.offsetParent !== null) { el.click(); return 'dismissed:' + s; } }
-        const btns = [...document.querySelectorAll('button, a[role="button"]')]; const rejectRe = /^(reject|reject all|decline|deny|refuse|no,? thanks|only necessary|necessary only)$/i;
-        for (const b of btns) { const txt = (b.textContent || '').trim(); if (rejectRe.test(txt) && b.offsetParent !== null) { b.click(); return 'dismissed:text=' + txt; } }
+        const docs = [document];
+        const roots = [document];
+        const seenRoots = new Set([document]);
+        const pushRoot = (root) => {
+          if (!root || seenRoots.has(root)) return;
+          seenRoots.add(root);
+          roots.push(root);
+        };
+        const collectFrames = (doc) => {
+          try {
+            const frames = doc.querySelectorAll('iframe');
+            for (const frame of frames) {
+              try {
+                const child = frame.contentDocument || frame.contentWindow?.document;
+                if (child && !docs.includes(child)) {
+                  docs.push(child);
+                  pushRoot(child);
+                }
+              } catch {}
+            }
+          } catch {}
+        };
+        const collectShadowRoots = () => {
+          for (const root of [...roots]) {
+            try {
+              const all = root.querySelectorAll('*');
+              for (const el of all) {
+                if (el.shadowRoot) pushRoot(el.shadowRoot);
+              }
+            } catch {}
+          }
+        };
+        collectFrames(document);
+        collectShadowRoots();
+        const allRoots = [...new Set([...docs, ...roots])];
+        const visible = (el) => {
+          if (!el) return false;
+          const style = window.getComputedStyle(el);
+          if (!style || style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;
+          const rect = el.getBoundingClientRect();
+          return rect.width > 0 && rect.height > 0;
+        };
+        const normalizedText = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
+        const candidateSelectors = [
+          '#onetrust-reject-all-handler',
+          '#CybotCookiebotDialogBodyButtonDecline',
+          '#didomi-notice-disagree-button',
+          '.qc-cmp2-summary-buttons button:first-child',
+          'button.sp_choice_type_12',
+          'button[id*="reject" i]',
+          'button[class*="reject" i]',
+          'button[id*="decline" i]',
+          'button[class*="decline" i]',
+          'button[id*="consent" i]',
+          'button[class*="consent" i]',
+          'a[id*="reject" i]',
+          'a[class*="reject" i]',
+          'a[id*="decline" i]',
+          'a[class*="decline" i]'
+        ];
+        for (const root of allRoots) {
+          for (const selector of candidateSelectors) {
+            try {
+              const el = root.querySelector(selector);
+              if (el && visible(el)) {
+                el.click();
+                return 'clicked:' + selector;
+              }
+            } catch {}
+          }
+        }
+        const buttonSelector = 'button, a[role="button"], [role="button"], input[type="button"], input[type="submit"]';
+        const rejectRe = /^(reject|reject all|decline|decline all|deny|deny all|refuse|no,? thanks|only necessary|necessary only|use necessary cookies only)$/i;
+        const acceptRe = /^(accept|accept all|allow all|agree|i agree|okay|ok|got it|continue|consent)$/i;
+        const closeRe = /^(close|dismiss|skip|not now|x|×)$/i;
+        const clickMatching = (matcher, label) => {
+          for (const root of allRoots) {
+            let buttons = [];
+            try { buttons = [...root.querySelectorAll(buttonSelector)]; } catch {}
+            for (const button of buttons) {
+              const txt = normalizedText(button.textContent || button.getAttribute?.('aria-label') || button.getAttribute?.('value'));
+              if (!txt || !matcher.test(txt) || !visible(button)) continue;
+              try {
+                button.click();
+                return label + ':' + txt.slice(0, 80);
+              } catch {}
+            }
+          }
+          return null;
+        };
+        const clicked = clickMatching(rejectRe, 'reject') || clickMatching(acceptRe, 'accept') || clickMatching(closeRe, 'close');
+        if (clicked) return clicked;
+        const overlaySelectors = [
+          '#onetrust-banner-sdk',
+          '#onetrust-consent-sdk',
+          '#CybotCookiebotDialog',
+          '.didomi-popup-container',
+          '.fc-consent-root',
+          '[id*="cookie" i]',
+          '[class*="cookie" i]',
+          '[id*="consent" i]',
+          '[class*="consent" i]',
+          '[id*="privacy" i]',
+          '[class*="privacy" i]',
+          '[id*="sp_message" i]',
+          '[class*="sp_message" i]'
+        ];
+        const hidden = [];
+        for (const root of allRoots) {
+          for (const selector of overlaySelectors) {
+            let nodes = [];
+            try { nodes = [...root.querySelectorAll(selector)]; } catch {}
+            for (const node of nodes) {
+              if (!visible(node)) continue;
+              const text = normalizedText(node.textContent).toLowerCase();
+              const attrs = normalizedText(node.id + ' ' + node.className).toLowerCase();
+              if (!text.includes('cookie') && !text.includes('consent') && !text.includes('privacy') && !attrs.includes('cookie') && !attrs.includes('consent') && !attrs.includes('privacy') && !attrs.includes('onetrust') && !attrs.includes('didomi') && !attrs.includes('sp_message')) continue;
+              try {
+                node.style.setProperty('display', 'none', 'important');
+                node.style.setProperty('visibility', 'hidden', 'important');
+                node.style.setProperty('pointer-events', 'none', 'important');
+                hidden.push(selector);
+              } catch {}
+            }
+          }
+        }
+        if (hidden.length) {
+          try {
+            document.documentElement.style.removeProperty('overflow');
+            document.body.style.removeProperty('overflow');
+          } catch {}
+          return 'hidden:' + hidden[0];
+        }
         return 'none';
       }`
       await mcpCall('browser_evaluate', { function: js })
@@ -665,9 +1125,9 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
       const fields = Array.isArray(params.fields)
         ? params.fields
         : (() => {
-            const form = params.form
-            if (!form || typeof form !== 'object' || Array.isArray(form)) return []
-            return Object.entries(form as Record<string, unknown>).map(([key, value]) => {
+          const form = params.form
+          if (!form || typeof form !== 'object' || Array.isArray(form)) return []
+          return Object.entries(form as Record<string, unknown>).map(([key, value]) => {
               const escapedId = String(key).replace(/[^a-zA-Z0-9_-]/g, '')
               const escapedAttr = String(key).replace(/["\\]/g, '\\$&')
               const inferredType = typeof value === 'boolean'
@@ -692,11 +1152,43 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
         const ref = typeof entry.ref === 'string' ? entry.ref : undefined
         const element = typeof entry.element === 'string' ? entry.element : undefined
         const fieldType = String(entry.type || 'text').toLowerCase()
-        const value = entry.value
+        const value = entry.value ?? entry.text
         if (!ref && !element) continue
+        const selectValues = Array.isArray(entry.values)
+          ? entry.values.map((item) => String(item ?? ''))
+          : Array.isArray(parseJsonArrayValue(entry.values ?? entry.option ?? value))
+            ? (parseJsonArrayValue(entry.values ?? entry.option ?? value) as unknown[]).map((item) => String(item ?? ''))
+            : [String(entry.option ?? value ?? '')]
+        if (!ref && element) {
+          if (fieldType === 'select') {
+            const result = await performSelectorDomAction('select', { element, values: selectValues })
+            if (!result) return { ok: false, error: 'Selector fallback failed for select field.' }
+            if (!result.ok) return result
+          } else if (fieldType === 'checkbox' || fieldType === 'radio') {
+            if (value === true || value === 'true' || value === 'on' || value === 'checked') {
+              const result = await performSelectorDomAction('click', { element })
+              if (!result) return { ok: false, error: 'Selector fallback failed for checkbox field.' }
+              if (!result.ok) return result
+            }
+          } else {
+            const result = await performSelectorDomAction('type', {
+              element,
+              text: String(value ?? ''),
+              submit: params.submit === true,
+            })
+            if (!result) return { ok: false, error: 'Selector fallback failed for input field.' }
+            if (!result.ok) return result
+          }
+          filled.push({
+            ref: null,
+            element,
+            type: fieldType,
+            value: value ?? null,
+          })
+          continue
+        }
         if (fieldType === 'select') {
-          const values = Array.isArray(value) ? value.map(String) : [String(value ?? '')]
-          await callMcpTool('browser_select_option', { ref, element, values })
+          await callMcpTool('browser_select_option', { ref, element, values: selectValues })
         } else if (fieldType === 'checkbox' || fieldType === 'radio') {
           if (value === true || value === 'true' || value === 'on' || value === 'checked') {
             await callMcpTool('browser_click', { ref, element })
@@ -720,11 +1212,19 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
     }
 
     const submitForm = async (params: Record<string, unknown>) => {
-      if (typeof params.submitRef === 'string' || typeof params.submitElement === 'string') {
-        await callMcpTool('browser_click', {
-          ref: typeof params.submitRef === 'string' ? params.submitRef : undefined,
-          element: typeof params.submitElement === 'string' ? params.submitElement : undefined,
-        })
+      const submitRef = pickNonEmptyBrowserString(params.submitRef)
+      const submitElement = pickNonEmptyBrowserString(params.submitElement, params.selector, params.element)
+      if (submitRef || submitElement) {
+        if (submitRef) {
+          await callMcpTool('browser_click', {
+            ref: submitRef,
+            element: submitElement,
+          })
+        } else if (submitElement) {
+          const result = await performSelectorDomAction('click', { element: submitElement })
+          if (!result) return { ok: false, error: 'submitElement is required for selector-based submit.' }
+          if (!result.ok) return result
+        }
       } else {
         await callBrowserEvaluate(`() => {
           const form = document.forms[0];
@@ -911,8 +1411,9 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
     const completeWebTask = async (params: Record<string, unknown>) => {
       const steps: string[] = []
       if (typeof params.url === 'string' && params.url.trim()) {
-        await callMcpTool('browser_navigate', { url: params.url.trim() })
-        steps.push(`navigate:${params.url.trim()}`)
+        const navigationTarget = resolveBrowserNavigationTarget(cwd, params.url.trim())
+        await callMcpTool('browser_navigate', { url: navigationTarget })
+        steps.push(`navigate:${navigationTarget}`)
         try { await dismissCookieBanners(callMcpTool) } catch { /* ignore */ }
       }
 
@@ -927,7 +1428,7 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
         if (scroll.ok) initialPage = scroll.page
       }
 
-      if (Array.isArray(params.fields) && params.fields.length > 0) {
+      if ((Array.isArray(params.fields) && params.fields.length > 0) || (params.form && typeof params.form === 'object' && !Array.isArray(params.form))) {
         const filled = await performFillForm(params)
         if (!filled.ok) return filled
         steps.push('fill_form')
@@ -991,7 +1492,7 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
     tools.push(
       tool(
         async (rawParams) => {
-          const params = normalizeToolInputArgs((rawParams ?? {}) as Record<string, unknown>)
+          const params = normalizeBrowserActionParams((rawParams ?? {}) as Record<string, unknown>)
           try {
             const action = String(params.action || '').trim()
 
@@ -1029,9 +1530,9 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
             }
 
             if (action === 'read_page') {
-              const url = typeof params.url === 'string' ? params.url : ''
-              if (url) {
-                await callMcpTool('browser_navigate', { url })
+              const target = pickBrowserTargetFromParams(params)
+              if (target) {
+                await callMcpTool('browser_navigate', { url: resolveBrowserNavigationTarget(cwd, target) })
                 try { await dismissCookieBanners(callMcpTool) } catch { /* ignore */ }
               }
               return stringifyStructured(await captureStructuredObservation())
@@ -1106,6 +1607,18 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
               if (v !== undefined && v !== null && v !== '') args[k] = v
             }
 
+            if (action === 'navigate') {
+              const target = pickBrowserTargetFromParams(params)
+              if (!target) return 'Error: url or filePath is required for navigate.'
+              args.url = resolveBrowserNavigationTarget(cwd, target)
+              delete args.filePath
+              delete args.path
+              delete args.href
+              delete args.link
+              delete args.target
+              delete args.page
+            }
+
             if (action === 'tabs') {
               args.action = typeof params.tabAction === 'string' ? params.tabAction : 'list'
               delete args.tabAction
@@ -1118,9 +1631,13 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
               args.values = Array.isArray(args.option) ? args.option : [String(args.option)]
               delete args.option
             }
+            if (action === 'select' && args.values === undefined && args.value !== undefined) {
+              args.values = Array.isArray(args.value) ? args.value : [String(args.value)]
+              delete args.value
+            }
 
             if ((action === 'screenshot' || action === 'snapshot') && args.url) {
-              const navUrl = args.url
+              const navUrl = resolveBrowserNavigationTarget(cwd, String(args.url))
               delete args.url
               await callMcpTool('browser_navigate', { url: navUrl })
               try { await dismissCookieBanners(callMcpTool) } catch { /* ignore */ }
@@ -1139,6 +1656,15 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
               } catch {
                 await new Promise((r) => setTimeout(r, 1200))
               }
+              try { await dismissCookieBanners(callMcpTool) } catch { /* ignore */ }
+            }
+
+            if (['click', 'hover', 'type', 'select'].includes(action) && !args.ref && typeof args.element === 'string') {
+              const selectorResult = await performSelectorDomAction(action as 'click' | 'hover' | 'type' | 'select', args)
+              if (!selectorResult) return `Error: ${action} requires a target element.`
+              if (!selectorResult.ok) return `Error: ${selectorResult.error}`
+              try { await captureStructuredObservation() } catch { /* ignore */ }
+              return selectorResult.output
             }
 
             let result = await callMcpTool(mcpTool, args, { saveTo: typeof params.saveTo === 'string' ? params.saveTo : undefined })
@@ -1219,9 +1745,34 @@ export function buildWebTools(bctx: ToolBuildContext): StructuredToolInterface[]
           const cmdArgs = (normalized.args ?? normalized.arguments) as string | undefined
           try {
             if (!command) return 'Error: command is required.'
-            const spawnArgs = ['browser', command, '--json']
-            if (cmdArgs) spawnArgs.push(...cmdArgs.split(/\s+/).filter(Boolean))
-            const result = spawnSync(openclawPath, spawnArgs, { encoding: 'utf-8', timeout: 60_000, maxBuffer: MAX_OUTPUT })
+            const parsedArgs = cmdArgs ? cmdArgs.split(/\s+/).filter(Boolean) : []
+            const runBrowserCommand = (browserCommand: string, browserArgs: string[]) => {
+              const spawnArgs = ['browser', '--json', browserCommand, ...browserArgs]
+              return spawnSync(openclawPath, spawnArgs, {
+                encoding: 'utf-8',
+                timeout: 60_000,
+                maxBuffer: MAX_OUTPUT,
+              })
+            }
+
+            if (command === 'capture') {
+              const outputs: string[] = []
+              if (parsedArgs.length > 0) {
+                const openResult = runBrowserCommand('open', parsedArgs)
+                if (openResult.status !== 0) {
+                  return `Error (exit ${openResult.status}): ${openResult.stderr || openResult.stdout || 'unknown'}`
+                }
+                if (openResult.stdout?.trim()) outputs.push(openResult.stdout.trim())
+              }
+              const screenshotResult = runBrowserCommand('screenshot', [])
+              if (screenshotResult.status !== 0) {
+                return `Error (exit ${screenshotResult.status}): ${screenshotResult.stderr || screenshotResult.stdout || 'unknown'}`
+              }
+              if (screenshotResult.stdout?.trim()) outputs.push(screenshotResult.stdout.trim())
+              return truncate(outputs.join('\n').trim() || '(no output)', MAX_OUTPUT)
+            }
+
+            const result = runBrowserCommand(command, parsedArgs)
             if (result.status !== 0) return `Error (exit ${result.status}): ${result.stderr || result.stdout || 'unknown'}`
             return truncate(result.stdout || '(no output)', MAX_OUTPUT)
           } catch (err: unknown) { return `Error: ${err instanceof Error ? err.message : String(err)}` }