@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/web-research/search.ts

@@ -1,235 +1,95 @@
 /**
- * Web Research Extension - Search Only (search)
+ * Web Research Extension - Search Orchestrator
  *
- * Search via SearXNG and return URLs, titles, and snippets.
+ * Exclusively uses browser-based search (DuckDuckGo Lite) via a managed pool.
  */
 
-import { getSearxngUrl } from './utils.ts';
-import { createTimeoutSignal } from './retry-utils.ts';
-import type { SearXNGResult, QueryResultWithError, SearxngSearchOptions } from './types.ts';
 import { logger } from '../logger.ts';
-import { getConfig } from '../config.ts';
+import { performSearch } from './browser-search.ts';
+import type { QueryResultWithError } from './types.ts';
+import type { Config } from '../config.ts';
+import { metrics } from '../utils/metrics.ts';
+import { getServiceContainer } from '../core/service-registry.ts';
+import type { ServiceContainer } from '../core/service-registry.ts';
 
 // ============================================================================
-// Type Definitions
-// ============================================================================
-
-/**
- * Type for SearXNG API response
- */
-interface SearxngApiResponse {
-  results?: SearXNGResult[];
-  query: string;
-  total: number;
-}
-
-/**
- * Error with cause property
- */
-interface ErrorWithCause extends Error {
-  cause?: unknown;
-}
-
-// Re-export search options type for consumers that import from this module
-export type { SearxngSearchOptions } from './types.ts';
-
-// ============================================================================
-// SearXNG Search Functions
+// Multiple Query Search
 // ============================================================================
 
 /**
- * Map our sourceType to the SearXNG categories parameter value.
- * SearXNG category names: general, news, it (tech/GitHub), images, videos, etc.
- * 'docs' has no SearXNG equivalent so we leave it as general.
- */
-function sourceTypeToCategory(sourceType: SearxngSearchOptions['sourceType']): string {
-  switch (sourceType) {
-    case 'news':   return 'news';
-    case 'github': return 'it';
-    default:       return 'general';
-  }
-}
-
-/**
- * Map our freshness value to the SearXNG time_range parameter value.
- * Returns undefined when no filter should be applied.
- */
-function freshnessToTimeRange(freshness: SearxngSearchOptions['freshness']): string | undefined {
-  switch (freshness) {
-    case 'day':   return 'day';
-    case 'week':  return 'week';
-    case 'month': return 'month';
-    case 'year':  return 'year';
-    default:      return undefined;
-  }
-}
-
-/**
- * Search SearXNG API for a single query
+ * Search multiple queries via the Browser-based Search Queue.
  *
- * @param query - The search query string
- * @param options - Optional freshness and sourceType filters
- * @returns Promise<SearXNGResult[]> - Array of search results
- * @throws {Error} When network error, timeout, or API error occurs
+ * @param queries - Array of search query strings
+ * @param config - Optional configuration override
+ * @param signal - Optional AbortSignal
+ * @param onProgress - Optional callback with cumulative link count across completed queries
+ * @returns Promise<QueryResultWithError[]> - Array of search results with error information
  */
-async function searchSearxng(query: string, options?: SearxngSearchOptions, signal?: AbortSignal): Promise<SearXNGResult[]> {
-  const baseUrl = getSearxngUrl();
-
-  // Build URL with query parameters
-  const url = new URL(`${baseUrl}/search`);
-  url.searchParams.append('q', query);
-  url.searchParams.append('format', 'json');
-  // Restrict to the configured language. Without this, SearXNG uses its instance
-  // defaults which can return mixed-language results.
-  url.searchParams.append('language', getConfig().SEARCH_LANGUAGE);
-  url.searchParams.append('categories', sourceTypeToCategory(options?.sourceType));
-  // Apply time filter when requested (omit param entirely for 'any' / undefined)
-  const timeRange = freshnessToTimeRange(options?.freshness);
-  if (timeRange !== undefined) {
-    url.searchParams.append('time_range', timeRange);
-  }
-
-  // Make GET request
-  let response: Response;
-  const fetch = globalThis.fetch;
-
-  try {
-    // createTimeoutSignal combines the 45s timeout with any caller-provided signal,
-    // so Ctrl+C during a search aborts the fetch immediately rather than waiting for timeout.
-    response = await fetch(url.toString(), {
-      signal: createTimeoutSignal(45000, signal),
-    });
-  } catch (error) {
-    // Handle network errors, DNS failures, connection refused, etc.
-    if (error instanceof Error) {
-      if (error.name === 'AbortError' || error.name === 'TimeoutError' || error.name === 'DOMException') {
-        const timeoutError = new Error(
-          `SearXNG request timed out: ${error.message}`,
-        ) as ErrorWithCause;
-        timeoutError.cause = error;
-        throw timeoutError;
-      }
-      const networkError = new Error(
-        `SearXNG network error: ${error.message}`,
-      ) as ErrorWithCause;
-      networkError.cause = error;
-      throw networkError;
-    }
-    const unknownError = new Error(
-      `SearXNG network error: ${String(error)}`,
-    ) as ErrorWithCause;
-    unknownError.cause = error;
-    throw unknownError;
-  }
-
-  if (!response.ok) {
-    const apiError = new Error(
-      `SearXNG request failed: ${response.status} ${response.statusText}`,
-    ) as ErrorWithCause;
-    apiError.cause = { status: response.status, statusText: response.statusText };
-    throw apiError;
-  }
-
-  const data = await response.json() as SearxngApiResponse;
-
-  // Return results or empty array using nullish coalescing
-  const results = data.results ?? [];
+export async function search(
+  queries: string[],
+  config?: Config,
+  signal?: AbortSignal,
+  onProgress?: (links: number) => void,
+  container: ServiceContainer = getServiceContainer()
+): Promise<QueryResultWithError[]> {
+  if (queries.length === 0) return [];
   
-  // Diagnostics: Log if we got no results or suspiciously few results
-  if (results.length === 0) {
-    if (logger.debug) {
-      logger.debug(`[Web Research] No results for query: ${query}`);
-    }
-  } else if (results.length < 3 && query.length > 10) {
-    // Very few results for a long query might indicate search quality issues
-    if (logger.debug) {
-      logger.debug(`[Web Research] Low result count (${results.length}) for query: ${query}`);
-    }
-  }
+  logger.log(`[Search] Orchestrating ${queries.length} queries via Browser Queue...`);
+  metrics.observe('search_query_count_per_request', queries.length);
+  const searchStart = Date.now();
   
-  return results;
-}
-
-// ============================================================================
-// Error Classification
-// ============================================================================
-
-/**
- * Classify error type for better error reporting
- *
- * @param error - The unknown error to classify
- * @returns QueryResultWithError['error'] - Classified error object
- */
-function classifyError(error: unknown): QueryResultWithError['error'] {
-  const errorStr = String(error);
-
-  // Timeout errors — check BEFORE network errors because "SearXNG network error: ...timeout" messages
-  // contain "network" and would otherwise be misclassified as network_error.
-  if (errorStr.includes('timed out') || errorStr.includes('timeout') ||
-      errorStr.includes('AbortError') || errorStr.includes('TimeoutError')) {
-    return { type: 'timeout', message: 'Search request timed out' };
-  }
-
-  // Network-related errors: connection refused, DNS failures, network unreachable, etc.
-  if (errorStr.includes('fetch failed') ||
-      errorStr.includes('ECONNREFUSED') ||
-      errorStr.includes('ENOTFOUND') ||
-      errorStr.includes('ECONNRESET') ||
-      errorStr.includes('ETIMEDOUT') ||
-      errorStr.includes('EHOSTUNREACH') ||
-      errorStr.includes('ENETUNREACH') ||
-      errorStr.includes('network') ||
-      errorStr.includes('Network error')) {
-    return { type: 'network_error', message: 'Network error - unable to connect to SearXNG service' };
-  }
-
-  if (errorStr.includes('SearXNG request failed')) {
-    // Check status code for more specific info
-    const match = errorStr.match(/(\d{3})/);
-    if (match !== null) {
-      const status = parseInt(match[1] ?? '0', 10);
-      if (status >= 500) {
-        return { type: 'service_unavailable', message: `SearXNG server error (${status})` };
-      }
-      if (status >= 400) {
-        return { type: 'service_unavailable', message: `SearXNG client error (${status})` };
+  try {
+    const resultMap = await performSearch(queries, config, signal, onProgress, container);
+    const searchDuration = Date.now() - searchStart;
+    metrics.observe('search_latency_ms', searchDuration);
+    
+    let totalResults = 0;
+    let successfulQueries = 0;
+    let failedQueries = 0;
+    
+    const results = queries.map(q => {
+      const qResults = resultMap.get(q) || [];
+      const result: QueryResultWithError = { query: q, results: qResults };
+      
+      if (qResults.length > 0) {
+        totalResults += qResults.length;
+        successfulQueries++;
+        metrics.observe('search_results_per_query', qResults.length);
+      } else {
+        result.error = {
+          type: 'empty_results',
+          message: 'Browser-based search returned no relevant results for this query (the query may be too narrow, or the site returned nothing usable).'
+        };
+        failedQueries++;
       }
+      return result;
+    });
+    
+    metrics.observe('search_results_total', totalResults);
+    metrics.increment('search_queries_total', successfulQueries, { status: 'success' });
+    metrics.increment('search_queries_total', failedQueries, { status: 'failed' });
+    metrics.observe('search_success_ratio', successfulQueries / queries.length);
+    
+    return results;
+  } catch (error) {
+    const searchDuration = Date.now() - searchStart;
+    metrics.observe('search_latency_ms', searchDuration, { status: 'error' });
+    metrics.increment('search_queries_total', queries.length, { status: 'error' });
+    metrics.increment('search_errors_total', 1, { error_type: 'search_failed' });
+    
+    const message = error instanceof Error ? error.message : String(error);
+    // Re-throw total search failure so the orchestrator can surface a clear error
+    // rather than silently producing an empty synthesis with zero links.
+    if (message.includes('Search completely failed')) {
+      metrics.increment('search_errors_total', 1, { error_type: 'total_search_failure' });
+      throw error;
     }
+    logger.error(`[Search] Orchestration failed:`, error);
+    metrics.increment('search_errors_total', 1, { error_type: 'orchestration_error' });
+    return queries.map(q => ({
+      query: q,
+      results: [],
+      error: { type: 'unknown', message }
+    }));
   }
-
-  return { type: 'unknown', message: errorStr.length > 0 ? errorStr : 'Unknown error' };
-}
-
-// ============================================================================
-// Multiple Query Search
-// ============================================================================
-
-/**
- * Search multiple queries via SearXNG
- *
- * @param queries - Array of search query strings
- * @param options - Optional freshness and sourceType filters applied uniformly to all queries
- * @param signal - Optional AbortSignal; combined with the per-request timeout
- * @returns Promise<QueryResultWithError[]> - Array of search results with error information
- */
-export function search(queries: string[], options?: SearxngSearchOptions, signal?: AbortSignal): Promise<QueryResultWithError[]> {
-  const searchPromises = queries.map((query: string): Promise<QueryResultWithError> => {
-    return searchSearxng(query, options, signal)
-      .then((results: SearXNGResult[]): QueryResultWithError => {
-        const result: QueryResultWithError = { query, results };
-        // Explicitly mark as empty results if no results returned
-        if (results.length === 0) {
-          result.error = { type: 'empty_results', message: 'No results found for this query' };
-        }
-        return result;
-      })
-      .catch((error: unknown): QueryResultWithError => {
-        logger.warn(`[Web Research] Search failed for "${query}":`, error);
-        const result: QueryResultWithError = { query, results: [] };
-        result.error = classifyError(error);
-        return result;
-      });
-  });
-
-  return Promise.all(searchPromises);
-}
+}

package/src/web-research/types.ts

@@ -4,19 +4,14 @@
  * All TypeScript interfaces and types for the web research functionality
  */
 
-// Search filter options passed through to SearXNG
-export type SearchFreshness = 'any' | 'day' | 'week' | 'month' | 'year';
-export type SearchSourceType = 'general' | 'news' | 'github';
+import type { SearchResult, ScrapeResult } from '../core/interfaces/scheduler-interfaces.ts';
 
-export interface SearxngSearchOptions {
-  freshness?: SearchFreshness;
-  sourceType?: SearchSourceType;
-}
+export type { SearchResult, ScrapeResult };
 
 // Search result with per-query error context.
 export interface QueryResultWithError {
   query: string;
-  results: SearXNGResult[];
+  results: SearchResult[];
   error?: {
     type: 'empty_results' | 'service_unavailable' | 'timeout' | 'network_error' | 'unknown';
     message: string;
@@ -24,24 +19,14 @@ export interface QueryResultWithError {
 }
 
 // Scraper types
-export type Layer = 'fetch' | 'playwright+chromium';
+export type Layer = 'fetch' | 'playwright+camoufox';
 
 export interface ScrapeLayerResult {
+  source: 'fetch' | 'playwright';
+  layer: 'fetch' | 'playwright' | 'playwright+camoufox';
   markdown: string;
-  source: string;
-  layer: Layer;
-}
-
-// SearXNG types
-export interface SearXNGResult {
-  title: string;
-  url: string;
-  content: string;
-  engine?: string;
-  score?: number;
+  error?: string;
 }
 
-// Timeouts
-export const PRIMARY_SCRAPER_TIMEOUT = 15000;  // fetch layer
-export const FALLBACK_SCRAPER_TIMEOUT = 20000; // Chromium layer
-
+/** Timeout for the lightweight fetch layer only (not the full playwright scraper path). */
+export const FETCH_LAYER_TIMEOUT = 10000;

package/src/web-research/utils.ts

@@ -4,155 +4,14 @@
  * Helper functions, type guards, and utilities
  */
 
-import { createRequire } from 'module';
-
-const require = createRequire(import.meta.url);
-
-/**
- * SearXNG Manager interface
- * Provides method to retrieve the SearXNG URL
- */
-interface SearxngManager {
-  getSearxngUrl(): string;
-}
-
-/**
- * Browser context tracking interface
- * Contains browser, context, page objects and a unique identifier
- */
-interface TrackedContext {
-  browser: {
-    close(): Promise<void>;
-  };
-  context?: {
-    close(): Promise<void>;
-  };
-  page?: {
-    close(): Promise<void>;
-  };
-  id: string;
-}
-
-// Manager instance (set by extension)
-let searxngManager: SearxngManager | null = null;
-
 /**
- * Set the SearXNG manager instance (called by extension on session start)
- */
-export function setSearxngManager(manager: SearxngManager): void {
-  searxngManager = manager;
-}
-
-/**
- * Get SearXNG URL
- * Gets URL from manager instance
- */
-export function getSearxngUrl(): string {
-  if (!searxngManager) {
-    throw new Error('SearXNG manager not initialized. Session may not have started properly.');
-  }
-  return searxngManager.getSearxngUrl();
-}
-
-// Active browser contexts tracking - keyed by ID for O(1) lookup
-const activeContexts = new Map<string, TrackedContext>();
-
-/**
- * Reset connection counters (Legacy stub for compatibility)
- */
-export function resetConnectionCounters(): void {}
-
-/**
- * Clear all tracked contexts without closing them
- * Used when browser is closed externally (e.g., via stopChromium())
- */
-export function clearTrackedContexts(): void {
-  activeContexts.clear();
-}
-
-/**
- * Generate unique ID for context tracking
- */
-function generateContextId(): string {
-  return `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
-}
-
-/**
- * Check if a Node.js module is available
+ * Common module checking utility
  */
 export function checkModule(name: string): boolean {
   try {
-    require.resolve(name);
+    import.meta.resolve(name);
     return true;
   } catch {
     return false;
   }
 }
-
-/**
- * Clean up all active browser contexts
- * IMPORTANT: This does NOT close the browser - only pages and contexts
- * The singleton browser is managed exclusively by stopChromium() in scrapers.ts
- */
-export async function cleanupAllContexts(): Promise<void> {
-  const cleanupPromises: Promise<void>[] = [];
-
-  for (const ctx of activeContexts.values()) {
-    cleanupPromises.push(
-      (async (): Promise<void> => {
-        try {
-          // Close page first, then context
-          // Do NOT close browser here - it's managed by stopChromium()
-          if (ctx.page !== undefined) {
-            await ctx.page.close().catch(() => {});
-          }
-          if (ctx.context !== undefined) {
-            await ctx.context.close().catch(() => {});
-          }
-        } catch {
-          // Ignore cleanup errors
-        }
-      })(),
-    );
-  }
-
-  // Clear AFTER promises complete to prevent race condition
-  await Promise.allSettled(cleanupPromises);
-  activeContexts.clear();
-}
-
-/**
- * Track a browser context for cleanup
- */
-export function trackContext(
-  browser: { close(): Promise<void> },
-  context: { close(): Promise<void> },
-  page: { close(): Promise<void> },
-): string {
-  const id = generateContextId();
-  activeContexts.set(id, { browser, context, page, id });
-  return id;
-}
-
-/**
- * Untrack a browser context by ID
- */
-export function untrackContextById(id: string): void {
-  activeContexts.delete(id);
-}
-
-/**
- * Validate maxConcurrency
- */
-export function validateMaxConcurrency(value?: number): number {
-  if (value === undefined) {
-    return 10; // Default
-  }
-  return Math.min(Math.max(1, Math.floor(value)), 20); // Clamp between 1 and 20
-}
-
-// Legacy compatibility stubs
-export function incrementConnectionCount(): void {}
-export function decrementConnectionCount(): void {}
-export function onConnectionCountChange(_callback: any): () => void { return () => {}; }
-export function getActiveConnectionCount(): number { return 0; }