@lincoln504/pi-research 0.1.13 → 1.0.1

package/src/infrastructure/browser/config.ts

@@ -0,0 +1,288 @@
+/**
+ * Browser Configuration
+ *
+ * Consolidated configuration utilities for browser management.
+ * Combines functionality from browser-config.ts and browser-configuration.ts.
+ */
+
+import * as crypto from 'node:crypto';
+import { join } from 'node:path';
+import { existsSync, mkdirSync } from 'node:fs';
+import { platform, homedir } from 'node:os';
+import type { Config } from '../../config.ts';
+import { getConfig } from '../../config.ts';
+import { getLogger } from '../../logger.ts';
+
+// ============================================================================
+// Binary Cache Management
+// ============================================================================
+
+/**
+ * The directory where camoufox-js installs the browser on Windows.
+ *
+ * camoufox-js's userCacheDir("camoufox") resolves to
+ *   os.homedir()/AppData/Local/camoufox/camoufox/Cache
+ * — note the DOUBLED "camoufox" segment, and that it uses os.homedir() rather
+ * than %LOCALAPPDATA%. We must mirror it exactly, otherwise isBrowserAvailable()
+ * looks in the wrong place and every browser test silently skips on Windows
+ * even though the binary downloaded and launches fine.
+ */
+function getWindowsCamoufoxDir(): string {
+    return join(homedir(), 'AppData', 'Local', 'camoufox', 'camoufox', 'Cache');
+}
+
+/**
+ * Get the camoufox binary cache directory.
+ * Uses PLAYWRIGHT_BROWSERS_PATH if set, otherwise the standard user cache location
+ * that camoufox uses by default (~/.cache/camoufox on Linux).
+ * We do NOT override HOME — that trick was unreliable and caused install/runtime mismatches.
+ */
+export function getBrowserCacheDir(): string {
+    if (process.env['PLAYWRIGHT_BROWSERS_PATH']) {
+        return process.env['PLAYWRIGHT_BROWSERS_PATH'];
+    }
+    // Mirror getCamoufoxBinaryPath() so both functions agree on the cache location
+    const osPlatform = platform();
+    if (osPlatform === 'win32') {
+        return getWindowsCamoufoxDir();
+    } else if (osPlatform === 'darwin') {
+        return join(homedir(), 'Library', 'Caches', 'camoufox');
+    } else {
+        const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
+        return join(cacheHome, 'camoufox');
+    }
+}
+
+/**
+ * Directory for transient per-worker browser profiles (Playwright/Camoufox
+ * create one isolated profile per browser instance).
+ *
+ * Defaults to a DISK-backed location (~/.cache/pi-research/profiles) rather than
+ * os.tmpdir(): on systems where /tmp is tmpfs (RAM-backed), placing several
+ * browser profiles there consumes RAM and, with an OOM killer like earlyoom
+ * active, can contribute to the whole session being killed. Override with
+ * PI_RESEARCH_TMP_DIR (config.TMP_DIR) — e.g. point it back at the system temp
+ * dir to deliberately use tmpfs/RAM when there is memory headroom.
+ *
+ * The directory is created if missing, because Playwright requires the parent
+ * of its mkdtemp profile dir to already exist.
+ */
+export function getBrowserProfileDir(config?: Config): string {
+    const configured = (config || getConfig()).TMP_DIR;
+    let dir: string;
+    if (configured && configured.trim().length > 0) {
+        dir = configured;
+    } else {
+        const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
+        dir = join(cacheHome, 'pi-research', 'profiles');
+    }
+    if (!existsSync(dir)) {
+        try {
+            mkdirSync(dir, { recursive: true });
+        } catch (_e) {
+            // Ignore race / permission errors; Playwright will surface a clear
+            // error if the directory is genuinely unusable.
+        }
+    }
+    return dir;
+}
+
+/**
+ * Get environment for spawning browser worker processes.
+ * Does not override HOME so camoufox uses its natural install location.
+ *
+ * CRITICAL: Browser workers run as cluster child processes and inherit process.env.
+ * Since the config module loads .env file values into its internal config object
+ * but does NOT set them into process.env, we must explicitly inject config values
+ * here so workers can read them from their process.env copy.
+ */
+export function getBrowserEnv(config?: Config): NodeJS.ProcessEnv {
+    const env: NodeJS.ProcessEnv = { ...process.env };
+    const customPath = process.env['PLAYWRIGHT_BROWSERS_PATH'];
+    if (customPath) {
+        env['PLAYWRIGHT_BROWSERS_PATH'] = customPath;
+    } else {
+        delete env['PLAYWRIGHT_BROWSERS_PATH'];
+    }
+    // Tell thread-workers where to write their lifecycle/error log. Precedence:
+    //   1. A user-set PI_RESEARCH_LOG_FILE (already copied from process.env above) wins —
+    //      never clobber an explicit override.
+    //   2. Otherwise propagate the resolved main-process log path so workers log to the
+    //      same file. That path already honors the user's PI_RESEARCH_LOG_PATH.
+    if (!env['PI_RESEARCH_LOG_FILE']) {
+        const logFilePath = getLogger().getLogFilePath();
+        if (logFilePath) {
+            env['PI_RESEARCH_LOG_FILE'] = logFilePath;
+        }
+    }
+    // Redirect the worker's temp dir to a disk-backed profile directory so
+    // Playwright/Camoufox per-instance profiles do not land on a tmpfs (RAM)
+    // /tmp. The worker's own logging is unaffected: it writes to
+    // PI_RESEARCH_LOG_FILE (set above), not os.tmpdir(). TMP/TEMP are set too
+    // for Windows. os.tmpdir() in the worker honours these.
+    const profileDir = getBrowserProfileDir(config);
+    env['TMPDIR'] = profileDir;
+    env['TMP'] = profileDir;
+    env['TEMP'] = profileDir;
+    // Inject config values that browser workers read from process.env.
+    // Without this, workers only see shell-set values — .env file values would be lost.
+    const c = config || getConfig();
+    env['PI_RESEARCH_SCRAPE_TIMEOUT_MS'] = String(c.SCRAPE_TIMEOUT_MS);
+    env['PI_RESEARCH_SEARCH_TIMEOUT_MS'] = String(c.SEARCH_TIMEOUT_MS);
+    env['PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS'] = String(c.HEALTH_CHECK_TIMEOUT_MS);
+    return env;
+}
+
+/**
+ * Ensure the browser cache directory exists (only relevant for custom PLAYWRIGHT_BROWSERS_PATH).
+ */
+export function ensureBrowserCacheDir(): string {
+    const cacheDir = getBrowserCacheDir();
+    if (!existsSync(cacheDir)) {
+        try {
+            mkdirSync(cacheDir, { recursive: true });
+        } catch (_e) {
+            // Ignore race condition
+        }
+    }
+    return cacheDir;
+}
+
+/**
+ * Get the expected path where camoufox installs its binaries.
+ * Matches camoufox-js's own resolution logic.
+ */
+export function getCamoufoxBinaryPath(): string {
+    const customPath = process.env['PLAYWRIGHT_BROWSERS_PATH'];
+    if (customPath) {
+        return customPath;
+    }
+
+    const osPlatform = platform();
+
+    if (osPlatform === 'win32') {
+        return getWindowsCamoufoxDir();
+    } else if (osPlatform === 'darwin') {
+        return join(homedir(), 'Library', 'Caches', 'camoufox');
+    } else {
+        // Linux and others
+        const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
+        return join(cacheHome, 'camoufox');
+    }
+}
+
+// ============================================================================
+// Scheduler Configuration
+// ============================================================================
+
+/**
+ * Generate a version hash for the scheduler based on critical config values.
+ * This allows us to detect when configuration changes and invalidate the cache.
+ */
+export function generateSchedulerVersion(config?: Config): string {
+    const c = config || getConfig();
+    const versionString = `v2:${c.WORKER_THREADS}:${c.WORKER_CONCURRENCY}:${c.MAX_CONCURRENT_RESEARCHERS}`;
+    return crypto.createHash('sha256').update(versionString).digest('hex').substring(0, 16);
+}
+
+/**
+ * Get the current number of worker threads from config.
+ * This is a function instead of a constant to allow config changes to take effect
+ * without requiring a process restart.
+ */
+export function getMaxWorkers(config?: Config): number {
+    return (config || getConfig()).WORKER_THREADS;
+}
+
+/**
+ * Get the current scheduler version hash.
+ */
+export function getSchedulerVersion(config?: Config): string {
+    return generateSchedulerVersion(config);
+}
+
+// ============================================================================
+// Display Environment
+// ============================================================================
+
+/**
+ * Resolve the camoufox headless mode for the current platform.
+ *
+ * Returns true (true headless — no visible window) on Windows and macOS.
+ * Historically Windows used headless:false because headless:true crashed Firefox
+ * (exit 0x80000003, camoufox-js issue #614); that is fixed in camoufox-js >=0.10
+ * (verified on Windows 11 x64 with 0.10.2: headless:true launches and navigates
+ * reliably and, crucially, NO browser window pops up on the desktop). Using
+ * headless:false on a real Windows desktop flashed visible, sometimes fullscreen,
+ * browser windows during every scrape — true headless avoids that entirely.
+ *
+ * Returns true in all other cases (macOS, Linux+X11, Linux+Wayland, Linux TTY).
+ *
+ * On a bare Linux TTY (no DISPLAY, no WAYLAND_DISPLAY) true headless is the
+ * default: Firefox renders fully offscreen, needs no display server and no Xvfb,
+ * and navigates reliably (verified: cold launch ~0.8s, page nav ~0.3s on a bare
+ * Debian box). The earlier 'virtual' default forced a hard Xvfb dependency that
+ * is absent on minimal servers — when Xvfb is missing camoufox-js throws at
+ * launch, and even when present, GTK/Xvfb rendering on a stripped box can stall
+ * navigation, surfacing as a misleading "connection timed out" health error.
+ *
+ * Xvfb ('virtual') remains available as an explicit opt-in for users who have
+ * installed it and want the marginal anti-detection benefit of a real virtual
+ * framebuffer: set PI_RESEARCH_USE_XVFB=true. It only takes effect on a bare
+ * Linux TTY (when a real display is present that display is used instead).
+ *
+ * Note: the JS camoufox port does not strip WAYLAND_DISPLAY / GDK_BACKEND /
+ * MOZ_ENABLE_WAYLAND before spawning Xvfb (unlike the Python port), so
+ * headless:'virtual' can still attach to a live compositor instead of the
+ * spawned Xvfb — another reason true headless is the safe default.
+ */
+export function resolveHeadlessMode(): boolean | 'virtual' {
+  const osPlatform = platform();
+  // Windows + macOS: true headless (no visible window). camoufox-js >=0.10 no
+  // longer crashes on Windows with headless:true (see the doc comment above).
+  if (osPlatform !== 'linux') return true;
+  if (process.env['DISPLAY']) return true;
+  if (process.env['WAYLAND_DISPLAY']) return true;
+  // Bare Linux TTY: true headless by default (no Xvfb required); opt into Xvfb
+  // explicitly only when requested.
+  if (process.env['PI_RESEARCH_USE_XVFB'] === 'true') return 'virtual';
+  return true;
+}
+
+// ============================================================================
+// Browser Availability Check
+// ============================================================================
+
+/**
+ * Check if the browser is available for meaningful browser pool testing.
+ * 
+ * Returns false if:
+ * - camoufox-js package is not installed
+ * - camoufox binary is not present at expected path
+ * - FULL_MOCK_MODE is active (both search and scrape mocked)
+ * 
+ * In FULL_MOCK_MODE, browser pool tests are not meaningful because:
+ * - The FixedClusterPool deadlocks in Vitest fork context when both search and scrape are mocked
+ * - Mocked tasks short-circuit in runTask() and don't exercise real browser behavior
+ * - These tests verify real browser behavior: crash recovery, IPC routing, profile locking, etc.
+ * 
+ * Single point of control: all existing skipTests() checks use this function.
+ */
+export function isBrowserAvailable(): boolean {
+    if (isFullMockMode()) return false;
+    try {
+        import.meta.resolve('camoufox-js');
+        return existsSync(getCamoufoxBinaryPath());
+    } catch {
+        return false;
+    }
+}
+
+/**
+ * Check whether search and scrape are mocked (used by tests that need real
+ * browser behavior — mock mode short-circuits in runTask()).
+ */
+function isFullMockMode(): boolean {
+    return process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true' &&
+           process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
+}

package/src/infrastructure/browser/ensure-browser.ts

@@ -0,0 +1,205 @@
+/**
+ * Runtime browser (camoufox) provisioning.
+ *
+ * Normally the camoufox browser is fetched by the `postinstall` script
+ * (scripts/setup.cjs) when pi-research is installed. But some install flows skip
+ * lifecycle scripts (e.g. `npm install --ignore-scripts`, or a bare `git clone`
+ * whose `npm install` has not yet run) and therefore never run our postinstall.
+ * On those the browser binary is absent and the first scrape would fail with
+ * "Camoufox is not installed".
+ *
+ * `ensureBrowserInstalled()` closes that gap: it is called once before the worker
+ * pool spawns and lazily fetches the browser if (and only if) it is missing.
+ *
+ * Non-regression guarantee for the other install paths: when the browser is
+ * already present — which it always is after the pi-extension / plain-npm
+ * postinstall fetch — this is a cheap `existsSync` check that returns
+ * immediately and changes nothing. It only ever does work when the binary is
+ * genuinely absent.
+ */
+
+import { spawn } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, statSync, fstatSync, openSync, closeSync, rmSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { logger } from '../../logger.ts';
+import { getCamoufoxBinaryPath } from './config.ts';
+
+/** Bound the ~100MB download so a stalled network fails fast instead of hanging forever. */
+const FETCH_TIMEOUT_MS = 15 * 60 * 1000;
+/** A lock older than this is considered stale (crashed mid-fetch) and may be stolen. */
+const STALE_LOCK_MS = 20 * 60 * 1000;
+/** When another process holds the fetch lock, poll this long for the browser to appear. */
+const WAIT_FOR_PEER_MS = 16 * 60 * 1000;
+const POLL_INTERVAL_MS = 2000;
+
+/** In-process dedupe: concurrent pool inits / scrapers share one fetch. */
+let inFlight: Promise<void> | null = null;
+
+/**
+ * True when a camoufox browser binary is actually present (cache dir exists and
+ * contains at least one version subdirectory) — mirrors the check in setup.cjs.
+ */
+export function isBrowserBinaryPresent(): boolean {
+  try {
+    const cacheDir = getCamoufoxBinaryPath();
+    if (!existsSync(cacheDir)) return false;
+    return readdirSync(cacheDir).some((entry) => {
+      try {
+        return statSync(join(cacheDir, entry)).isDirectory();
+      } catch {
+        return false;
+      }
+    });
+  } catch {
+    return false;
+  }
+}
+
+/** Resolve the camoufox-js CLI bin, mirroring scripts/setup.cjs (handles hoisting). */
+function resolveCamoufoxBin(): string | null {
+  try {
+    const require = createRequire(import.meta.url);
+    const pkgJson = require.resolve('camoufox-js/package.json');
+    const bin = join(dirname(pkgJson), '..', '.bin', process.platform === 'win32' ? 'camoufox-js.cmd' : 'camoufox-js');
+    if (existsSync(bin)) return bin;
+  } catch {
+    /* fall through to npx */
+  }
+  return null;
+}
+
+const delay = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+
+/** Run `camoufox-js fetch`, resolving on exit 0 and rejecting otherwise / on timeout. */
+function runFetch(): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const bin = resolveCamoufoxBin();
+    const [cmd, args] = bin ? [bin, ['fetch']] : ['npx', ['camoufox-js', 'fetch']];
+    logger.info(`[ensure-browser] Camoufox not found; fetching the browser (this runs once, ~100MB): ${cmd} ${args.join(' ')}`);
+
+    // Windows needs shell:true to run the .cmd shim (Node CVE-2024-27980 fix makes a
+    // shell-less spawn of .cmd throw EINVAL), but shell:true performs NO quoting, so a
+    // spaced install path (C:\Users\John Smith\…) must be quoted explicitly. The args
+    // here are fixed literals ('fetch'), never user input, so quoting the command alone
+    // is sufficient and this stays injection-free.
+    const useShell = process.platform === 'win32';
+    const child = spawn(useShell ? `"${cmd}"` : cmd, args, { stdio: 'inherit', shell: useShell });
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      reject(new Error(`camoufox fetch timed out after ${FETCH_TIMEOUT_MS}ms`));
+    }, FETCH_TIMEOUT_MS);
+
+    child.on('error', (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+    child.on('exit', (code) => {
+      clearTimeout(timer);
+      if (code === 0) resolve();
+      else reject(new Error(`camoufox fetch exited with code ${code}`));
+    });
+  });
+}
+
+/** Wait for a peer process (holding the lock) to finish fetching the browser. */
+async function waitForPeer(): Promise<boolean> {
+  const deadline = Date.now() + WAIT_FOR_PEER_MS;
+  while (Date.now() < deadline) {
+    await delay(POLL_INTERVAL_MS);
+    if (isBrowserBinaryPresent()) return true;
+  }
+  return false;
+}
+
+async function provision(): Promise<void> {
+  // Fast path: already installed (pi-extension / plain-npm postinstall case).
+  if (isBrowserBinaryPresent()) return;
+
+  // Respect the same opt-out the postinstall fetch honours (used by CI/tests).
+  if (process.env['PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD'] === '1') {
+    logger.debug('[ensure-browser] Browser missing but PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1; skipping auto-fetch.');
+    return;
+  }
+
+  // Cross-process lock so two concurrent first-runs don't double-fetch. Scoped
+  // by a hash of the target cache dir so distinct cache locations (e.g. a
+  // PLAYWRIGHT_BROWSERS_PATH override, or different users) never share a lock.
+  const cacheKey = createHash('sha256').update(getCamoufoxBinaryPath()).digest('hex').slice(0, 16);
+  const lockPath = join(tmpdir(), `pi-research-camoufox-fetch-${cacheKey}.lock`);
+  let haveLock = false;
+  try {
+    // 'wx' = O_CREAT|O_EXCL: fails if another process holds it. Mode 0o600 keeps
+    // the (empty) lock owner-only, matching FileLockService and avoiding a
+    // world-readable temp file.
+    closeSync(openSync(lockPath, 'wx', 0o600));
+    haveLock = true;
+  } catch {
+    // Lock held — but steal it if it's stale (a crash left it behind). Read the
+    // mtime through a file descriptor rather than re-stat'ing by path (avoids a
+    // check-then-use race), then re-acquire with O_EXCL: a process that recreated
+    // the lock in the meantime makes our 'wx' create fail, so we never double-acquire.
+    try {
+      let mtimeMs: number;
+      const fd = openSync(lockPath, 'r');
+      try {
+        mtimeMs = fstatSync(fd).mtimeMs;
+      } finally {
+        closeSync(fd);
+      }
+      if (Date.now() - mtimeMs > STALE_LOCK_MS) {
+        rmSync(lockPath, { force: true });
+        closeSync(openSync(lockPath, 'wx', 0o600));
+        haveLock = true;
+      }
+    } catch {
+      /* someone won the race; fall through to wait */
+    }
+  }
+
+  if (!haveLock) {
+    // Another process is fetching — wait for it, then re-check.
+    logger.info('[ensure-browser] Another process is fetching the browser; waiting for it to finish…');
+    if (await waitForPeer()) return;
+    // Peer didn't deliver in time; fetch ourselves as a fallback.
+  }
+
+  try {
+    if (isBrowserBinaryPresent()) return; // re-check under lock
+    await runFetch();
+    if (isBrowserBinaryPresent()) {
+      logger.info('[ensure-browser] Camoufox browser is ready.');
+    } else {
+      logger.warn('[ensure-browser] camoufox fetch reported success but no binary was found.');
+    }
+  } finally {
+    if (haveLock) {
+      try {
+        rmSync(lockPath, { force: true });
+      } catch {
+        /* best-effort */
+      }
+    }
+  }
+}
+
+/**
+ * Ensure the camoufox browser is installed, fetching it once if missing.
+ *
+ * Idempotent and concurrency-safe: concurrent callers share a single fetch, and
+ * a cross-process lock prevents two processes from fetching at the same time.
+ * Best-effort — if the fetch fails, the subsequent browser launch surfaces the
+ * existing actionable "run npx camoufox-js fetch" error rather than this throwing.
+ */
+export function ensureBrowserInstalled(): Promise<void> {
+  if (!inFlight) {
+    inFlight = provision().catch((err) => {
+      // Reset so a later attempt (e.g. after the user fixes the network) can retry.
+      inFlight = null;
+      logger.warn(`[ensure-browser] Automatic browser provisioning failed: ${err instanceof Error ? err.message : String(err)}`);
+    });
+  }
+  return inFlight;
+}

package/src/infrastructure/browser/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Browser Infrastructure - Public API
+ *
+ * Barrel module for browser-related functionality. Source code imports
+ * directly from specific submodules; this file exists for test convenience.
+ */
+
+export {
+    getMaxWorkers,
+    getSchedulerVersion,
+    isBrowserAvailable,
+    getCamoufoxBinaryPath,
+    getBrowserCacheDir,
+} from './config.ts';
+
+export {
+    isTransientSocketError,
+    isPoolShutdownError,
+    resetBrowserCircuitBreaker,
+} from './browser-error-utils.ts';
+
+export type { IScheduler } from '../../core/interfaces/scheduler-interfaces.ts';
+
+export {
+    forceSchedulerRestart,
+} from './scheduler-factory.ts';
+
+export {
+    runBrowserTask,
+    runBrowserHealthCheck,
+    runWorkerSearch,
+} from './task-execution-service.ts';
+
+export {
+    stopBrowserManager,
+    waitForBrowserPoolIdle,
+} from './browser-lifecycle.ts';