@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/utils/url-utils.ts

@@ -0,0 +1,255 @@
+/**
+ * URL Utilities
+ *
+ * Unified functions for URL normalization and validation used across the
+ * knowledge store, writer queue, and shared link registry.
+ */
+
+import { logger } from '../logger.ts';
+
+/**
+ * Strips trailing punctuation an LLM commonly appends to a URL in prose
+ * (sentence periods, list commas, markdown emphasis/code markers) WITHOUT
+ * corrupting the URL itself:
+ * - A trailing closing bracket (')', ']', '}') is removed only when it is
+ *   unbalanced (e.g. from "(see http://x)" or "[1] http://x]"). A balanced
+ *   one is part of the URL — e.g. Wikipedia's
+ *   "…/Python_(programming_language)" or an IPv6 "http://[::1]/" — and is
+ *   preserved.
+ * - '_' and '~' are RFC 3986 unreserved characters and are legitimate at the
+ *   end of a URL, so they are never stripped (in the main parse path a leading
+ *   markdown marker would have made new URL() throw and diverted to the
+ *   aggressive fallback, so a trailing one here is far more likely genuine).
+ * The loop is bounded to defend against pathological input (ReDoS-free).
+ */
+export function stripTrailingLlmPunctuation(input: string): string {
+  let s = input;
+  for (let i = 0; i < 25 && s.length > 0; i++) {
+    const last = s[s.length - 1]!;
+    if (last === '*' || last === '`' || last === '.' || last === ',') {
+      s = s.slice(0, -1);
+      continue;
+    }
+    // Balance-aware removal of a trailing closing bracket: strip it only when it
+    // has no matching opener anywhere in the string (so a markdown/parenthetical
+    // noise bracket is removed, but a balanced one that's part of the URL —
+    // Wikipedia "..._(programming_language)", IPv6 "http://[::1]/" — is kept).
+    if (last === ')' || last === ']' || last === '}') {
+      const open = last === ')' ? '(' : last === ']' ? '[' : '{';
+      let opens = 0;
+      let closes = 0;
+      for (const ch of s) {
+        if (ch === open) opens++;
+        else if (ch === last) closes++;
+      }
+      if (closes > opens) {
+        s = s.slice(0, -1);
+        continue;
+      }
+    }
+    break;
+  }
+  return s;
+}
+
+/**
+ * Normalizes a URL for consistent deduplication and storage.
+ * 
+ * Features:
+ * - Forces HTTPS (deduplicates http/https as the same content)
+ * - Lowercases hostname
+ * - Removes default ports (80 for http, 443 for https)
+ * - Removes ALL hash fragments
+ * - Sorts query parameters for consistent ordering
+ * - Strips trailing slashes from pathnames
+ * - Strips trailing markdown markers (e.g. **) or trailing punctuation
+ *
+ * @param url - The URL to normalize
+ * @returns The normalized URL string, or the original string if parsing fails
+ */
+export function normalizeUrl(url: string): string {
+  if (!url || typeof url !== 'string') return url;
+
+  try {
+    // Guard against pathological inputs (ReDoS defense for subsequent regex)
+    if (url.length > 4096) return url.trim().split('#')[0]!.toLowerCase();
+    // 1. Strip trailing markdown markers and punctuation often found in LLM output
+    //    (balanced parens and unreserved trailing chars are preserved).
+    const cleanUrl = stripTrailingLlmPunctuation(url.trim());
+    
+    const parsed = new URL(cleanUrl);
+    
+    // 2. Force HTTPS (dedup check)
+    parsed.protocol = 'https:';
+    
+    // 3. Lowercase hostname
+    parsed.hostname = parsed.hostname.toLowerCase();
+    
+    // 4. Remove default ports
+    if (parsed.port === '443' || parsed.port === '80') {
+      parsed.port = '';
+    }
+    
+    // 5. Remove all hash fragments
+    parsed.hash = '';
+    
+    // 6. Sort query parameters
+    parsed.searchParams.sort();
+    
+    // 7. Strip trailing slashes from pathname (URL forces / for root, so we keep length > 1)
+    while (parsed.pathname.length > 1 && parsed.pathname.endsWith('/')) {
+      const trimmed = parsed.pathname.slice(0, -1);
+      parsed.pathname = trimmed;
+      // Non-special-scheme URLs (e.g. "foo.bar:baz/") carry an opaque path whose
+      // setter silently ignores assignment. Without this guard the loop would
+      // spin forever on such input — a synchronous infinite loop that even
+      // blocks vitest's own test timeout. Bail the instant a write doesn't take.
+      if (parsed.pathname !== trimmed) break;
+    }
+    
+    let result = parsed.toString();
+    
+    // 8. Final cleanup: strip trailing slash and question mark
+    if (result.endsWith('/')) {
+      result = result.slice(0, -1);
+    }
+    if (result.endsWith('?')) {
+      result = result.slice(0, -1);
+    }
+    
+    return result;
+  } catch (_err) {
+    // Fallback for invalid URLs: aggressive best-effort normalization.
+    // LLM hallucinations often produce broken URLs — log a warning so we can
+    // trace them, then do our best to clean them for dedup purposes.
+    logger.debug(`[url-utils] normalizing unparseable URL (${url.slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
+    
+    let cleaned = url.trim()
+      // Strip leading/trailing markdown and punctuation often added by LLMs.
+      // Quantifiers are bounded ({1,20}) rather than unbounded (+) to avoid a
+      // polynomial-backtracking (ReDoS) regex on unsanitized URL input; no
+      // legitimate URL carries 20+ leading/trailing punctuation characters.
+      .replace(/^[*_~`"']{1,20}/, '')
+      .replace(/[*_~`"',.)}\]]{1,20}$/, '');
+    
+    // Strip hash fragment (best-effort)
+    const hashIdx = cleaned.indexOf('#');
+    if (hashIdx >= 0) cleaned = cleaned.substring(0, hashIdx);
+    
+    // Strip trailing slash
+    if (cleaned.endsWith('/') && cleaned.length > 8) {
+      cleaned = cleaned.slice(0, -1);
+    }
+    
+    // Lowercase everything
+    cleaned = cleaned.toLowerCase();
+    
+    // Try to force https:// prefix if it looks like a domain
+    if (!cleaned.startsWith('http') && cleaned.includes('.')) {
+      cleaned = 'https://' + cleaned.replace(/^\/{2,}/, '');
+    }
+    
+    return cleaned;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Initial-links validation (shared by the CLI --initial-links flag and the
+// research tool's initialLinks parameter, so the two paths cannot drift).
+// These links are templated verbatim into the researcher prompt as trusted
+// seed evidence, so they must be bounded and shape-checked at every entry
+// point: http(s)-only, ≤ MAX_INITIAL_LINK_CHARS each, ≤ MAX_INITIAL_LINKS.
+// ---------------------------------------------------------------------------
+
+/** Maximum number of initial/seed links a caller may provide. */
+export const MAX_INITIAL_LINKS = 20;
+
+/** Maximum length (chars) of each initial/seed link. */
+export const MAX_INITIAL_LINK_CHARS = 2048;
+
+/**
+ * Validate a single initial/seed link.
+ * @returns null when valid, or a human-readable reason (no trailing period).
+ */
+export function validateInitialLink(link: string): string | null {
+  if (link.length > MAX_INITIAL_LINK_CHARS) {
+    return `each URL must be at most ${MAX_INITIAL_LINK_CHARS} characters`;
+  }
+  let parsed: URL;
+  try {
+    parsed = new URL(link);
+  } catch {
+    return `"${link}" is not a valid URL`;
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    return 'only http(s) URLs are allowed';
+  }
+  return null;
+}
+
+/**
+ * Validate a full initialLinks array (count cap + per-link checks).
+ * @returns null when valid, or a human-readable reason (no trailing period).
+ */
+export function validateInitialLinks(links: readonly string[]): string | null {
+  if (links.length > MAX_INITIAL_LINKS) {
+    return `at most ${MAX_INITIAL_LINKS} URLs may be provided`;
+  }
+  for (const link of links) {
+    const err = validateInitialLink(link);
+    if (err) return err;
+  }
+  return null;
+}
+
+/**
+ * Validates URL format and applies basic SSRF defense-in-depth.
+ *
+ * Rejects:
+ * - Non-HTTP/HTTPS protocols
+ * - Localhost, loopback, and private network ranges
+ * - Hostnames without a public TLD (e.g. .local, .internal)
+ * - Obvious hallucinations or malformed strings
+ *
+ * @param url - The URL to validate
+ * @returns True if the URL is considered safe and valid for research
+ */
+export function validateUrl(url: string): boolean {
+  if (!url || typeof url !== 'string') return false;
+  
+  try {
+    const parsed = new URL(url);
+    
+    // Only allow HTTP/HTTPS
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return false;
+    
+    const hostname = parsed.hostname.toLowerCase();
+    if (!hostname) return false;
+    
+    // SSRF Defense: Reject localhost and loopback
+    if (hostname === 'localhost' || hostname.endsWith('.localhost')) return false;
+    if (hostname === '::1' || hostname === '[::1]') return false;
+    
+    // SSRF Defense: Reject private/internal IP ranges
+    // IPv4: 10.x, 172.16-31.x, 192.168.x, 127.x, 169.254.x, 0.x
+    if (/^(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.|127\.|0\.|169\.254\.)/.test(hostname)) return false;
+    
+    // IPv6 private ranges
+    // Note: new URL().hostname wraps IPv6 literals in brackets, so IPv6 ranges
+    // must be matched on the bracketed form. Matching the bracketless prefix
+    // (e.g. "fc"/"fd") would instead reject ordinary domains like fc2.com,
+    // fcc.gov, fda.gov and fdic.gov — a false-negative, not SSRF protection
+    // (the connect-time IP pin in validateUrlForSSRF is the real gate).
+    if (hostname.startsWith('fe80:') || hostname.startsWith('[fe80:')) return false;
+    if (hostname.startsWith('[fc') || hostname.startsWith('[fd')) return false;
+    if (hostname.startsWith('::ffff:') || hostname.startsWith('[::ffff:')) return false;
+    
+    // Reject hostnames without a public TLD or with internal suffixes
+    if (hostname.endsWith('.local') || hostname.endsWith('.internal')) return false;
+    if (!hostname.includes('.')) return false; // Must have at least one dot (e.g. example.com)
+    
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/utils/user-agent.ts

@@ -0,0 +1,28 @@
+/**
+ * Realistic User-Agent Strings for Browser Rotation
+ *
+ * Provides current, realistic User-Agent strings that mimic real browsers.
+ * Updated regularly to match current browser versions.
+ */
+
+export const USER_AGENTS = [
+    // Chrome on Windows (updated 2025)
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
+
+    // Chrome on macOS
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
+
+    // Chrome on Linux
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
+
+    // Firefox on Windows (updated 2025)
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0',
+
+    // Firefox on macOS
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:138.0) Gecko/20100101 Firefox/138.0',
+];

package/src/web-research/browser-search.ts

@@ -0,0 +1,179 @@
+/**
+ * Web Research Extension - Browser-based Search (DuckDuckGo Lite)
+ * 
+ * Orchestrates exhaustive search bursts across multiple worker processes.
+ */
+
+import { runWorkerSearch } from '../infrastructure/browser/task-execution-service.ts';
+import { getMaxWorkers } from '../infrastructure/browser/config.ts';
+import { logger } from '../logger.ts';
+import { safeUnref } from '../utils/safe-unref.ts';
+import { normalizeUrl } from '../utils/url-utils.ts';
+import type { SearchResult } from './types.ts';
+import type { Config } from '../config.ts';
+import { metrics } from '../utils/metrics.ts';
+import { getServiceContainer } from '../core/service-registry.ts';
+import type { ServiceContainer } from '../core/service-registry.ts';
+
+/**
+ * Orchestrate high-fidelity search across multiple queries.
+ * Uses true multithreaded worker processes for maximum burst performance.
+ * Each worker maintains its own "warm" browser process.
+ */
+export async function performSearch(
+    queries: string[], 
+    config?: Config,
+    signal?: AbortSignal,
+    onProgress?: (links: number) => void,
+    container: ServiceContainer = getServiceContainer()
+): Promise<Map<string, SearchResult[]>> {
+    const startTime = Date.now();
+    const resultMap = new Map<string, SearchResult[]>();
+    const seenUrls = new Set<string>();
+
+    // FULL_MOCK_MODE: When both PI_RESEARCH_MOCK_SEARCH and PI_RESEARCH_MOCK_SCRAPE
+    // are true, return deterministic mock results without spawning any worker processes.
+    // This gate keeps CI tests deterministic and fast, and prevents real worker pool
+    // failures from cascading into test flakiness.
+    const fullMockMode = process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true' &&
+                         process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
+    if (fullMockMode) {
+      logger.log('[Search] FULL_MOCK_MODE enabled — returning mock results without worker pool');
+      for (const q of queries) {
+        const domain = q.replace(/[^a-zA-Z0-9]/g, '-').toLowerCase().slice(0, 30);
+        resultMap.set(q, [{
+          title: `Mock result for: ${q}`,
+          url: `https://mock.example.com/${domain}`,
+          content: `This is a mock search result for query "${q}".`,
+        }]);
+      }
+      return resultMap;
+    }
+
+    const maxWorkers = getMaxWorkers(config);
+
+    metrics.setGauge('browser_search_max_workers', maxWorkers);
+    metrics.increment('browser_search_orchestrations_total', 1);
+
+    logger.log(`[Search] Orchestrating ${queries.length} queries across ${maxWorkers} worker processes...`);
+
+    // Call onProgress(0) immediately to clear any 'searching' placeholder in the UI
+    if (onProgress) onProgress(0);
+
+    // Hard cap per query so a single Cloudflare block or hung browser worker
+    // cannot stall the entire Promise.all burst. Derived from the configured
+    // search timeout (default 45s) so raising SEARCH_TIMEOUT_MS actually takes
+    // effect here instead of being silently overridden by a lower literal.
+    const QUERY_TIMEOUT_MS = config?.SEARCH_TIMEOUT_MS ?? 45_000;
+
+    const filteredQueries = queries.filter(q => q.trim());
+    let timeoutCount = 0;
+    let errorCount = 0;
+    // Keep one representative worker error so total-failure surfaces the real
+    // cause (e.g. a worker that cannot load its module) instead of a generic
+    // timeout guess that misdirects to the network or system load.
+    let sampleWorkerError = '';
+
+    const searchTasks = filteredQueries.map(async (query) => {
+        const queryStartTime = Date.now();
+        const timeoutController = new AbortController();
+        const timeoutId = setTimeout(() => timeoutController.abort(), QUERY_TIMEOUT_MS);
+        safeUnref(timeoutId);
+
+        try {
+            if (signal?.aborted) {
+                resultMap.set(query, []);
+                return;
+            }
+
+            const querySignal = signal
+              ? AbortSignal.any([signal, timeoutController.signal])
+              : timeoutController.signal;
+
+            const results = await runWorkerSearch(query, config, querySignal, 1, undefined, container);
+            const queryDuration = Date.now() - queryStartTime;
+            metrics.observe('browser_search_query_duration_ms', queryDuration);
+
+            if (results?.length > 0) {
+                metrics.increment('browser_search_queries_total', 1, { status: 'success' });
+                metrics.increment('browser_search_results_total', results.length);
+                logger.debug(`[Search] Worker returned ${results.length} results for: ${query}`);
+
+                const uniqueResults: SearchResult[] = [];
+                for (const r of results) {
+                    // Dedup across ALL queries (seenUrls), not just within this query.
+                    // The same URL surfacing for several queries would otherwise repeat
+                    // its snippet in the combined output, wasting context tokens; this
+                    // also makes seenUrls.size a true cross-query unique count for the
+                    // progress callback and the unique-URLs metric.
+                    // Dedup on the NORMALIZED url (consistent with shared-links / knowledge store),
+                    // so http/https, trailing-slash, and tracking-param variants of the same page
+                    // collapse to one — otherwise snippets repeat across queries (wasting context)
+                    // and seenUrls.size over-counts uniques for the progress callback / metric.
+                    const key = r.url ? normalizeUrl(r.url) : '';
+                    if (key && !seenUrls.has(key)) {
+                        seenUrls.add(key);
+                        uniqueResults.push(r);
+                    }
+                }
+                resultMap.set(query, uniqueResults);
+            } else {
+                metrics.increment('browser_search_queries_total', 1, { status: 'no_results' });
+                resultMap.set(query, []);
+            }
+        } catch (error) {
+            const queryDuration = Date.now() - queryStartTime;
+            const isTimeout = timeoutController.signal.aborted && !signal?.aborted;
+            const status = isTimeout ? 'timeout' : 'error';
+            
+            if (isTimeout) timeoutCount++;
+            else errorCount++;
+
+            metrics.observe('browser_search_query_duration_ms', queryDuration, { status });
+            metrics.increment('browser_search_queries_total', 1, { status });
+            
+            if (isTimeout) {
+                logger.warn(`[Search] Query timed out after ${QUERY_TIMEOUT_MS}ms (likely blocked or slow startup): "${query}"`);
+            } else {
+                const msg = error instanceof Error ? error.message : String(error);
+                if (msg !== 'Aborted') {
+                    if (!sampleWorkerError) sampleWorkerError = msg;
+                    logger.error(`[Search] Worker failed for "${query}": ${msg}`);
+                }
+            }
+            resultMap.set(query, []);
+        } finally {
+            clearTimeout(timeoutId);
+            if (onProgress) onProgress(seenUrls.size);
+        }
+    });
+
+    await Promise.all(searchTasks);
+
+    // Detect total failure: if every valid query returned empty, the worker pool is likely dead.
+    const totalResults = Array.from(resultMap.values()).reduce((sum, r) => sum + r.length, 0);
+    if (totalResults === 0 && filteredQueries.length > 0) {
+        metrics.increment('browser_search_total_failures_total', 1);
+        
+        let reason = `all ${filteredQueries.length} queries returned no results`;
+        if (timeoutCount === filteredQueries.length) {
+            reason = `all ${filteredQueries.length} queries timed out after ${QUERY_TIMEOUT_MS}ms`;
+        } else if (errorCount === filteredQueries.length) {
+            reason = `all ${filteredQueries.length} queries encountered worker errors`;
+        } else if (timeoutCount + errorCount === filteredQueries.length) {
+            reason = `${timeoutCount} queries timed out and ${errorCount} queries failed`;
+        }
+
+        throw new Error(
+            `Search completely failed: ${reason}. ` +
+            `Browser workers may be unavailable, DuckDuckGo is unreachable, or the system is under extreme load.` +
+            (sampleWorkerError ? ` Last worker error: ${sampleWorkerError}` : '')
+        );
+    }
+
+    const totalDuration = Date.now() - startTime;
+    metrics.observe('browser_search_total_duration_ms', totalDuration);
+    metrics.increment('browser_search_unique_urls_total', seenUrls.size);
+
+    return resultMap;
+}