osai-agent 4.0.0

package/src/tools/browser.js

@@ -0,0 +1,434 @@
+// =============================================================================
+// OS AI Agent — Browser Automation Module (Playwright + HTTP Fallback)
+// =============================================================================
+// Safe, stateless, per-request browser automation for the AI agent.
+// Each call is fully isolated — launch → close in finally block.
+// Falls back to HTTP fetch if Playwright/Chromium is unavailable.
+// =============================================================================
+
+import { logger } from '../utils/logger.js';
+import { searchDDG, searchDDGHttp } from './search-providers.js';
+
+let _chromium = null;
+async function getChromium() {
+  if (_chromium) return _chromium;
+  try {
+    const mod = await import('playwright-core');
+    _chromium = mod.chromium;
+    return _chromium;
+  } catch {
+    return null;
+  }
+}
+
+// ─── CONSTANTS ────────────────────────────────────────────────────────────────
+
+const NAVIGATION_TIMEOUT = 30000;
+const MAX_CONTENT_LENGTH = 20000;
+const MAX_LINKS = 50;
+const MAX_CACHE_ENTRIES = 100;
+const CACHE_TTL_MS = 5 * 60 * 1000;
+const MAX_RETRIES = 2;
+const UNSAFE_PROTOCOLS = ['file:', 'ftp:', 'javascript:', 'data:'];
+
+const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36';
+
+// ─── CACHE (bounded, TTL-based, FIFO eviction) ───────────────────────────────
+
+const _cache = new Map();
+const _cacheKeys = [];
+
+function _cacheGet(key) {
+  const entry = _cache.get(key);
+  if (!entry) return null;
+  if (Date.now() - entry.timestamp > CACHE_TTL_MS) {
+    _cache.delete(key);
+    const idx = _cacheKeys.indexOf(key);
+    if (idx >= 0) _cacheKeys.splice(idx, 1);
+    return null;
+  }
+  return entry.data;
+}
+
+function _cacheSet(key, data) {
+  if (_cacheKeys.length >= MAX_CACHE_ENTRIES) {
+    const oldest = _cacheKeys.shift();
+    _cache.delete(oldest);
+  }
+  _cache.set(key, { data, timestamp: Date.now() });
+  _cacheKeys.push(key);
+}
+
+// ─── URL VALIDATION ───────────────────────────────────────────────────────────
+
+function validateUrl(url) {
+  if (!url || typeof url !== 'string') {
+    return { valid: false, error: 'URL is required and must be a string' };
+  }
+  const trimmed = url.trim();
+  if (!trimmed.startsWith('http://') && !trimmed.startsWith('https://')) {
+    return { valid: false, error: `Unsupported protocol. Only http:// and https:// URLs are allowed. Got: "${trimmed.slice(0, 30)}"` };
+  }
+  const protocol = trimmed.split(':')[0] + ':';
+  if (UNSAFE_PROTOCOLS.includes(protocol)) {
+    return { valid: false, error: `Blocked unsafe protocol: ${protocol}` };
+  }
+  try {
+    new URL(trimmed);
+  } catch {
+    return { valid: false, error: `Invalid URL format: "${trimmed.slice(0, 60)}"` };
+  }
+  return { valid: true, url: trimmed };
+}
+
+// ─── TEXT SANITIZATION ────────────────────────────────────────────────────────
+
+function sanitizeText(text) {
+  if (!text) return '';
+  return text
+    .replace(/<script[\s\S]*?<\/script>/gi, '')
+    .replace(/<style[\s\S]*?<\/style>/gi, '')
+    .replace(/<noscript[\s\S]*?<\/noscript>/gi, '')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&#x27;/g, "'")
+    .replace(/&#x2F;/g, '/')
+    .replace(/\s{2,}/g, ' ')
+    .trim();
+}
+
+function truncateContent(text, maxLen = MAX_CONTENT_LENGTH) {
+  if (!text) return '';
+  if (text.length <= maxLen) return text;
+  return text.slice(0, maxLen) + `\n\n...[truncated: ${text.length - maxLen} more chars]`;
+}
+
+function dedupeUrls(urls) {
+  const seen = new Set();
+  const result = [];
+  for (const u of urls) {
+    if (seen.has(u)) continue;
+    seen.add(u);
+    if (result.length >= MAX_LINKS) break;
+    result.push(u);
+  }
+  return result;
+}
+
+// ─── BROWSER HELPER (stateless per-request) ───────────────────────────────────
+
+async function withBrowser(fn) {
+  const pw = await getChromium();
+  if (!pw) throw new Error('playwright-core not available');
+  let browser = null;
+  try {
+    browser = await pw.launch({
+      headless: true,
+      args: [
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-gpu',
+        '--disable-accelerated-2d-canvas',
+        '--no-first-run',
+        '--no-zygote',
+        '--single-process',
+        '--disable-background-networking',
+      ],
+    });
+    return await fn(browser);
+  } catch (err) {
+    throw err;
+  } finally {
+    if (browser) {
+      try { await browser.close(); } catch {}
+    }
+  }
+}
+
+// ─── BROWSER-BASED EXTRACTION ─────────────────────────────────────────────────
+
+async function extractWithBrowser(url, timeout = NAVIGATION_TIMEOUT) {
+  return withBrowser(async (browser) => {
+    const context = await browser.newContext({
+      userAgent: USER_AGENT,
+      locale: 'en-US',
+    });
+    const page = await context.newPage();
+    page.setDefaultTimeout(timeout);
+
+    try {
+      const response = await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+        timeout,
+      });
+
+      if (!response) {
+        return { success: false, error: 'Navigation returned no response' };
+      }
+
+      const status = response.status();
+      if (status >= 400) {
+        return { success: false, error: `HTTP ${status}: ${response.statusText()}` };
+      }
+
+      const title = await page.title().catch(() => '');
+      const content = await page.innerText('body').catch(() => '');
+      const rawLinks = await page.evaluate(() =>
+        Array.from(document.querySelectorAll('a[href]'))
+          .map(a => a.href)
+          .filter(h => h && (h.startsWith('http://') || h.startsWith('https://')))
+      ).catch(() => []);
+
+      return {
+        success: true,
+        source: 'browser',
+        url,
+        title: sanitizeText(title),
+        content: truncateContent(sanitizeText(content)),
+        links: dedupeUrls(rawLinks),
+      };
+    } finally {
+      try { await page.close(); } catch {}
+      try { await context.close(); } catch {}
+    }
+  });
+}
+
+// ─── HTTP FALLBACK EXTRACTION ─────────────────────────────────────────────────
+
+async function extractWithHttp(url, timeout = NAVIGATION_TIMEOUT) {
+  try {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: {
+        'User-Agent': USER_AGENT,
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9',
+      },
+      redirect: 'follow',
+    });
+
+    clearTimeout(timer);
+
+    if (!response.ok) {
+      return { success: false, error: `HTTP ${response.status}: ${response.statusText()}` };
+    }
+
+    const html = await response.text();
+    const contentType = response.headers.get('content-type') || '';
+
+    // Extract title from HTML
+    const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    const title = titleMatch ? sanitizeText(titleMatch[1]) : '';
+
+    // Extract text content
+    const content = sanitizeText(html);
+
+    // Extract links
+    const linkRegex = /<a[^>]+href="([^"]+)"[^>]*>/gi;
+    const links = [];
+    let match;
+    while ((match = linkRegex.exec(html)) !== null) {
+      const href = match[1];
+      if (href && (href.startsWith('http://') || href.startsWith('https://'))) {
+        links.push(href);
+      }
+    }
+
+    return {
+      success: true,
+      source: 'fallback',
+      url,
+      title,
+      content: truncateContent(content),
+      links: dedupeUrls(links),
+    };
+  } catch (err) {
+    return {
+      success: false,
+      error: `HTTP fallback failed: ${err.message}`,
+    };
+  }
+}
+
+// ─── SEARCH ENGINE PARSER (DuckDuckGo) ───────────────────────────────────────
+
+async function searchDuckDuckGo(query, timeout = NAVIGATION_TIMEOUT) {
+  try {
+    return await searchDDG(query, 5);
+  } catch {
+    return searchDuckDuckGoHttp(query, timeout);
+  }
+}
+
+async function searchDuckDuckGoHttp(searchUrl, timeout) {
+  try {
+    const url = typeof searchUrl === 'string' && searchUrl.startsWith('http')
+      ? new URL(searchUrl).searchParams.get('q') || searchUrl
+      : searchUrl;
+    const q = typeof url === 'string' && url.startsWith('http') ? url : searchUrl;
+    if (q.startsWith('http')) {
+      return await searchDDGHttp(q, 5);
+    }
+    return await searchDDGHttp(q, 5);
+  } catch {
+    return [];
+  }
+}
+
+// ─── PUBLIC API ───────────────────────────────────────────────────────────────
+
+/**
+ * Navigate to a URL and extract clean readable content.
+ * Uses Playwright Chromium by default, falls back to HTTP fetch on failure.
+ *
+ * @param {string} url - The URL to browse (http/https only)
+ * @returns {Promise<{success: boolean, source: string, url: string, title: string, content: string, links: string[]}>}
+ */
+export async function browse(url) {
+  // 1. Validate URL before any action
+  const validation = validateUrl(url);
+  if (!validation.valid) {
+    return { success: false, source: 'none', url: url || '', title: '', content: '', links: [], error: validation.error };
+  }
+
+  const safeUrl = validation.url;
+
+  // 2. Check cache
+  const cacheKey = safeUrl;
+  const cached = _cacheGet(cacheKey);
+  if (cached) {
+    logger.debug(`Browser cache hit: ${safeUrl}`);
+    return { ...cached, cached: true };
+  }
+
+  // 3. Try Playwright with retry
+  let lastError = null;
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    try {
+      const result = await extractWithBrowser(safeUrl);
+      if (result.success) {
+        _cacheSet(cacheKey, result);
+        return result;
+      }
+      lastError = result.error;
+    } catch (err) {
+      lastError = err.message;
+      logger.warn(`Browser attempt ${attempt + 1}/${MAX_RETRIES} failed for ${safeUrl}: ${err.message}`);
+    }
+  }
+
+  // 4. Fallback to HTTP fetch
+  logger.debug(`Browser failed (${lastError}), falling back to HTTP for: ${safeUrl}`);
+  const fallbackResult = await extractWithHttp(safeUrl);
+  if (fallbackResult.success) {
+    _cacheSet(cacheKey, fallbackResult);
+    return fallbackResult;
+  }
+
+  return {
+    success: false,
+    source: 'none',
+    url: safeUrl,
+    title: '',
+    content: '',
+    links: [],
+    error: `All methods failed. Last error: ${lastError}`,
+  };
+}
+
+/**
+ * Search the web using DuckDuckGo, then open the first valid result.
+ *
+ * @param {string} query - Search query
+ * @returns {Promise<{success: boolean, source: string, url: string, title: string, content: string, links: string[], searchResults?: Array}>}
+ */
+export async function searchAndBrowse(query) {
+  if (!query || typeof query !== 'string' || query.trim().length < 2) {
+    return { success: false, source: 'none', url: '', title: '', content: '', links: [], error: 'Search query too short (minimum 2 characters)' };
+  }
+
+  const searchResults = await searchDuckDuckGo(query);
+
+  if (!searchResults || searchResults.length === 0) {
+    return { success: false, source: 'none', url: '', title: '', content: '', links: [], error: `No search results found for "${query}"` };
+  }
+
+  // Pick the first valid result
+  const firstResult = searchResults[0];
+  const browseResult = await browse(firstResult.url);
+
+  return {
+    ...browseResult,
+    searchResults: searchResults.map(r => ({ title: r.title, url: r.url, snippet: r.snippet })),
+  };
+}
+
+/**
+ * Extract structured data from a page using CSS selectors.
+ *
+ * @param {Object} selectors - Map of key → CSS selector (e.g. { heading: 'h1', price: '.price' })
+ * @param {string} url - The page URL to extract from
+ * @returns {Promise<{success: boolean, source: string, url: string, data: Object}>}
+ */
+export async function extractStructuredData(selectors, url) {
+  if (!selectors || typeof selectors !== 'object' || Object.keys(selectors).length === 0) {
+    return { success: false, source: 'none', url: url || '', data: {}, error: 'Selectors map is required' };
+  }
+
+  const validation = validateUrl(url);
+  if (!validation.valid) {
+    return { success: false, source: 'none', url: url || '', data: {}, error: validation.error };
+  }
+
+  const safeUrl = validation.url;
+
+  try {
+    return withBrowser(async (browser) => {
+      const context = await browser.newContext({ userAgent: USER_AGENT });
+      const page = await context.newPage();
+      page.setDefaultTimeout(NAVIGATION_TIMEOUT);
+
+      try {
+        await page.goto(safeUrl, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT });
+
+        const data = {};
+        for (const [key, selector] of Object.entries(selectors)) {
+          try {
+            const element = page.locator(selector).first();
+            const text = await element.innerText();
+            data[key] = sanitizeText(text);
+          } catch {
+            data[key] = null;
+          }
+        }
+
+        return {
+          success: true,
+          source: 'browser',
+          url: safeUrl,
+          data,
+        };
+      } finally {
+        try { await page.close(); } catch {}
+        try { await context.close(); } catch {}
+      }
+    });
+  } catch (err) {
+    return {
+      success: false,
+      source: 'none',
+      url: safeUrl,
+      data: {},
+      error: `Structured extraction failed: ${err.message}`,
+    };
+  }
+}