@askjo/camofox-browser 1.8.12 → 1.9.1

package/scripts/postinstall.js

@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+// Postinstall: download Camoufox binaries and verify the cache is populated.
+//
+// Why a script instead of an inline `npx camoufox-js fetch`:
+//   1. Cross-platform: avoids POSIX-only `VAR= cmd` shell syntax (Windows
+//      cmd.exe does not honor it).
+//   2. Defends against PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 inherited from
+//      the user's shell or a CI/Docker base image. `camoufox-js` honors
+//      that flag by convention (same env name as `playwright`'s skip flag),
+//      which leaves the binary cache empty and makes the server crash at
+//      runtime with "Version information not found".
+//   3. Verifies the cache after fetch and exits non-zero with actionable
+//      remediation if the binary is still missing — failing the install
+//      is strictly better than a silent runtime crash.
+
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { homedir, platform } from 'node:os';
+import { join } from 'node:path';
+
+function camoufoxCacheDir() {
+  const home = homedir();
+  const plat = platform();
+  if (plat === 'darwin') return join(home, 'Library', 'Caches', 'camoufox');
+  if (plat === 'win32') {
+    // Matches camoufox-js/dist/pkgman.js:246 which nests the app name twice:
+    // %LOCALAPPDATA%\camoufox\camoufox\Cache
+    const base = process.env.LOCALAPPDATA || join(home, 'AppData', 'Local');
+    return join(base, 'camoufox', 'camoufox', 'Cache');
+  }
+  return join(process.env.XDG_CACHE_HOME || join(home, '.cache'), 'camoufox');
+}
+
+function fail(message) {
+  process.stderr.write(`[camofox-browser] postinstall: ${message}\n`);
+  process.exit(1);
+}
+
+const childEnv = { ...process.env };
+delete childEnv.PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD;
+
+const isWindows = platform() === 'win32';
+const result = spawnSync(isWindows ? 'npx.cmd' : 'npx', ['camoufox-js', 'fetch'], {
+  stdio: 'inherit',
+  env: childEnv,
+  shell: isWindows,
+});
+
+if (result.error) fail(`failed to spawn npx: ${result.error.message}`);
+if (result.status !== 0) fail(`\`npx camoufox-js fetch\` exited with code ${result.status}`);
+
+const versionFile = join(camoufoxCacheDir(), 'version.json');
+if (!existsSync(versionFile)) {
+  process.stderr.write('[camofox-browser] postinstall: Camoufox cache not populated.\n');
+  process.stderr.write(`  Expected file: ${versionFile}\n`);
+  process.stderr.write('  Possible causes:\n');
+  process.stderr.write('    - Network failure during binary download (check your connection)\n');
+  process.stderr.write('    - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD re-exported by a wrapping process\n');
+  process.stderr.write('  Manual fix:  PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD= npx camoufox-js fetch\n');
+  process.exit(1);
+}

package/server.js

@@ -384,6 +384,8 @@ const MAX_CONCURRENT_PER_USER = CONFIG.maxConcurrentPerUser;
 const PAGE_CLOSE_TIMEOUT_MS = 5000;
 const NAVIGATE_TIMEOUT_MS = CONFIG.navigateTimeoutMs;
 const BUILDREFS_TIMEOUT_MS = CONFIG.buildrefsTimeoutMs;
+const NATIVE_MEM_RESTART_THRESHOLD_MB = CONFIG.nativeMemRestartThresholdMb;
+let _nativeMemBaseline = null; // RSS - heapUsed at first idle measurement
 const FAILURE_THRESHOLD = 3;
 const MAX_CONSECUTIVE_TIMEOUTS = 3;
 const TAB_LOCK_TIMEOUT_MS = 35000; // Must be > HANDLER_TIMEOUT_MS so active op times out first
@@ -508,13 +510,16 @@ async function withUserLimit(userId, operation) {
 }
 
 async function safePageClose(page) {
+  if (!page || page.isClosed()) return;
   try {
     await Promise.race([
-      page.close(),
-      new Promise(resolve => setTimeout(resolve, PAGE_CLOSE_TIMEOUT_MS))
+      page.close({ runBeforeUnload: false }),
+      new Promise((_, reject) => setTimeout(() => reject(new Error('page close timed out')), PAGE_CLOSE_TIMEOUT_MS)),
     ]);
   } catch (e) {
-    log('warn', 'page close failed', { error: e.message });
+    log('warn', 'page close timed out or failed, force-closing', { error: e.message });
+    try { await page.close({ runBeforeUnload: false }); } catch (_) {}
+    page.removeAllListeners();
   }
 }
 
@@ -567,6 +572,20 @@ function clearBrowserIdleTimer() {
   }
 }
 
+// Detects errors that retrying cannot recover from (e.g., Camoufox binary
+// missing because postinstall was skipped). The user must run
+// `npx camoufox-js fetch` and restart; looping on this wastes resources
+// and buries the actionable error under noise.
+//
+// Sentinel: matches the human-readable message thrown by camoufox-js's
+// FileNotFoundError in dist/pkgman.js (Version.fromPath). FileNotFoundError
+// is not exported from the public API, so substring matching is the only
+// available hook. If the upstream message changes, this regex needs an
+// update; the dependency range in package.json controls exposure.
+function isFatalInstallError(err) {
+  return /Version information not found/i.test(err?.message || '');
+}
+
 function scheduleBrowserWarmRetry(delayMs = 5000) {
   if (browserWarmRetryTimer || browser || browserLaunchPromise) return;
   browserWarmRetryTimer = setTimeout(async () => {
@@ -576,6 +595,13 @@ function scheduleBrowserWarmRetry(delayMs = 5000) {
       await ensureBrowser();
       log('info', 'background browser warm retry succeeded', { ms: Date.now() - start });
     } catch (err) {
+      if (isFatalInstallError(err)) {
+        log('error', 'browser unavailable: Camoufox binaries are not installed; aborting retry loop', {
+          error: err.message,
+          remediation: 'run `npx camoufox-js fetch` then restart the server',
+        });
+        return;
+      }
       log('warn', 'background browser warm retry failed', { error: err.message, nextDelayMs: delayMs });
       scheduleBrowserWarmRetry(Math.min(delayMs * 2, 30000));
     }
@@ -625,8 +651,13 @@ async function restartBrowser(reason) {
 function getTotalTabCount() {
   let total = 0;
   for (const session of sessions.values()) {
-    for (const group of session.tabGroups.values()) {
-      total += group.size;
+    try {
+      // Use real Playwright page count so leaked pages exert backpressure
+      // on MAX_TABS_GLOBAL, surfacing leaks before Firefox starves.
+      total += session.context.pages().length;
+    } catch (_) {
+      // Context is dead — fall back to bookkeeping count for this session.
+      for (const group of session.tabGroups.values()) total += group.size;
     }
   }
   return total;
@@ -734,6 +765,7 @@ async function _closeBrowserFullyImpl(reason) {
 
   // Reset native memory baseline so next browser measures from fresh
   reporter.resetNativeMemBaseline();
+  _nativeMemBaseline = null;
 
   // Verify cleanup: check FD/handle counts dropped (after force-kill completes)
   const postCloseFds = _countOpenFds();
@@ -993,6 +1025,12 @@ async function closeSession(userId, session, {
 
   const key = normalizeUserId(userId);
 
+  // Drain locks BEFORE closing context — queued operations get clean "Tab destroyed"
+  // (410) instead of messy "Target page closed" (500) errors.
+  if (clearLocks) {
+    clearSessionLocks(session);
+  }
+
   if (clearDownloads) {
     await clearSessionDownloads(session).catch(() => {});
   }
@@ -1011,10 +1049,6 @@ async function closeSession(userId, session, {
   sessions.delete(key);
   await pluginEvents.emitAsync('session:destroyed', { userId: key, reason });
 
-  if (clearLocks) {
-    clearSessionLocks(session);
-  }
-
   refreshActiveTabsGauge();
 }
 
@@ -1166,8 +1200,21 @@ function handleRouteError(err, req, res, extraFields = {}) {
     browserRestartsTotal.labels('proxy_error').inc();
     destroySession(userId);
   }
+  // Navigation-related timeouts can poison the proxy session (e.g., Cloudflare holding
+  // the connection open for 30s). The browser context shares a single proxy session, so
+  // one poisoned page kills all subsequent navigations in that context. Destroy the
+  // entire session so the next request gets a fresh BrowserContext + proxy.
+  const NAVIGATION_TIMEOUT_ACTIONS = new Set(['click', 'navigate', 'open_url']);
+  if (isTimeoutError(err) && userId && NAVIGATION_TIMEOUT_ACTIONS.has(action)) {
+    log('warn', 'navigation timeout — destroying session for fresh proxy', {
+      action, userId, error: err.message,
+    });
+    browserRestartsTotal.labels('navigation_timeout').inc();
+    destroySession(userId);
+  }
   // Track consecutive timeouts per tab and auto-destroy stuck tabs
-  if (userId && isTimeoutError(err)) {
+  // (for non-navigation timeouts like type, scroll that don't poison the proxy)
+  if (userId && isTimeoutError(err) && !NAVIGATION_TIMEOUT_ACTIONS.has(action)) {
     const tabId = req.body?.tabId || req.query?.tabId || req.params?.tabId;
     const session = sessions.get(normalizeUserId(userId));
     if (session && tabId) {
@@ -1193,6 +1240,12 @@ function handleRouteError(err, req, res, extraFields = {}) {
   if (isTabDestroyedError(err)) {
     return res.status(410).json({ error: 'Tab was destroyed. Open a new tab.', ...extraFields });
   }
+  // Dead context = session torn down (by proxy error, timeout, or reaper) while this op
+  // was in flight. The ROOT CAUSE was already reported — this is a cascade error.
+  // Return 503 (retriable) so the client retries with a fresh session.
+  if (isDeadContextError(err)) {
+    return res.status(503).json({ error: 'Browser session expired. Retry to get a fresh session.', code: 'session_expired', ...extraFields });
+  }
   // --- Frustration detection: report when a tab hits a streak of failures ---
   // Individual failures are noise. 3+ consecutive = the site is persistently broken.
   const FRUSTRATION_TYPES = new Set(['timeout', 'dead_context', 'nav_aborted']);
@@ -2027,7 +2080,7 @@ app.post('/tabs', async (req, res) => {
           { statusCode: 409 },
         );
       }
-      const session = await getSession(userId, { trace: !!trace });
+      let session = await getSession(userId, { trace: !!trace });
       
       let totalTabs = 0;
       for (const group of session.tabGroups.values()) totalTabs += group.size;
@@ -2044,7 +2097,7 @@ app.post('/tabs', async (req, res) => {
       
       const page = await session.context.newPage();
       const tabId = fly.makeTabId();
-      const tabState = createTabState(page);
+      let tabState = createTabState(page);
       attachDownloadListener(tabState, tabId, log, pluginEvents, userId);
       group.set(tabId, tabState);
       refreshActiveTabsGauge();
@@ -2053,7 +2106,32 @@ app.post('/tabs', async (req, res) => {
         const urlErr = validateUrl(url);
         if (urlErr) throw Object.assign(new Error(urlErr), { statusCode: 400 });
         tabState.lastRequestedUrl = url;
-        await withPageLoadDuration('open_url', () => page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
+        try {
+          await withPageLoadDuration('open_url', () => page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
+        } catch (navErr) {
+          if ((isProxyError(navErr) || isTimeoutError(navErr)) && proxyPool?.canRotateSessions) {
+            log('warn', 'tab create navigate failed, retrying with fresh proxy', {
+              reqId: req.reqId, tabId, error: navErr.message,
+            });
+            browserRestartsTotal.labels('proxy_retry').inc();
+            const key = normalizeUserId(userId);
+            const oldSession = sessions.get(key);
+            if (oldSession) {
+              await closeSession(key, oldSession, { reason: 'proxy_retry_rotate', clearDownloads: true, clearLocks: true });
+            }
+            session = await getSession(userId, { trace: !!trace });
+            const retryGroup = getTabGroup(session, resolvedSessionKey);
+            const retryPage = await session.context.newPage();
+            tabState = createTabState(retryPage);
+            tabState.lastRequestedUrl = url;
+            attachDownloadListener(tabState, tabId, log, pluginEvents, userId);
+            retryGroup.set(tabId, tabState);
+            refreshActiveTabsGauge();
+            await withPageLoadDuration('open_url', () => retryPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
+          } else {
+            throw navErr;
+          }
+        }
         tabState.visitedUrls.add(url);
       }
       
@@ -2228,7 +2306,24 @@ app.post('/tabs/:tabId/navigate', async (req, res) => {
           await prewarmGoogleHome();
         }
 
-        await navigateCurrentPage();
+        // Navigate with transparent retry on proxy/timeout errors.
+        // If the proxy is blocked or the page times out, destroy the session,
+        // get a fresh proxy, and retry once before failing to the caller.
+        try {
+          await navigateCurrentPage();
+        } catch (navErr) {
+          if ((isProxyError(navErr) || isTimeoutError(navErr)) && proxyPool?.canRotateSessions) {
+            log('warn', 'navigate failed, retrying with fresh proxy session', {
+              reqId: req.reqId, tabId, error: navErr.message,
+            });
+            browserRestartsTotal.labels('proxy_retry').inc();
+            await recreateTabOnFreshContext();
+            if (isGoogleSearch) await prewarmGoogleHome();
+            await navigateCurrentPage();
+          } else {
+            throw navErr;
+          }
+        }
 
         if (isGoogleSearch && proxyPool?.canRotateSessions && await isGoogleSearchBlocked(tabState.page)) {
           log('warn', 'google search blocked, rotating browser proxy session', {
@@ -3040,6 +3135,88 @@ app.post('/tabs/:tabId/scroll', async (req, res) => {
   }
 });
 
+// Viewport
+/**
+ * @openapi
+ * /tabs/{tabId}/viewport:
+ *   post:
+ *     tags: [Interaction]
+ *     summary: Set the page viewport size
+ *     description: >
+ *       Physically resizes the page via Playwright's `page.setViewportSize`,
+ *       triggering a real layout reflow. Use for responsive testing —
+ *       `window.resizeTo()` is a no-op on non-popup windows.
+ *     parameters:
+ *       - name: tabId
+ *         in: path
+ *         required: true
+ *         schema:
+ *           type: string
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             type: object
+ *             required: [userId, width, height]
+ *             properties:
+ *               userId:
+ *                 type: string
+ *               width:
+ *                 type: integer
+ *                 minimum: 100
+ *                 maximum: 4000
+ *               height:
+ *                 type: integer
+ *                 minimum: 100
+ *                 maximum: 4000
+ *     responses:
+ *       200:
+ *         description: Viewport set.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 ok:
+ *                   type: boolean
+ *                 width:
+ *                   type: integer
+ *                 height:
+ *                   type: integer
+ *       400:
+ *         description: Width or height missing or out of range.
+ *       404:
+ *         description: Tab not found.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/Error'
+ */
+app.post('/tabs/:tabId/viewport', async (req, res) => {
+  try {
+    const { userId, width, height } = req.body;
+    if (!Number.isFinite(width) || !Number.isFinite(height) || width < 100 || height < 100 || width > 4000 || height > 4000) {
+      return res.status(400).json({ error: 'width and height required (100..4000 px)' });
+    }
+    const session = sessions.get(normalizeUserId(userId));
+    const found = session && findTab(session, req.params.tabId);
+    if (!found) return res.status(404).json({ error: 'Tab not found' });
+
+    const { tabState } = found;
+    tabState.toolCalls++; tabState.consecutiveTimeouts = 0; tabState.consecutiveFailures = 0;
+
+    await tabState.page.setViewportSize({ width: Math.round(width), height: Math.round(height) });
+    await tabState.page.waitForTimeout(150);
+
+    pluginEvents.emit('tab:viewport', { userId, tabId: req.params.tabId, width, height });
+    res.json({ ok: true, width: Math.round(width), height: Math.round(height) });
+  } catch (err) {
+    log('error', 'viewport failed', { reqId: req.reqId, error: err.message });
+    handleRouteError(err, req, res);
+  }
+});
+
 // Back
 /**
  * @openapi
@@ -3096,7 +3273,7 @@ app.post('/tabs/:tabId/back', async (req, res) => {
     
     const result = await withTabLock(tabId, async () => {
       try {
-        await tabState.page.goBack({ timeout: 10000 });
+        await tabState.page.goBack({ timeout: 20000 });
       } catch (navErr) {
         // NS_BINDING_CANCELLED_OLD_LOAD: Firefox cancels the old load when going back.
         // The navigation itself succeeded -- just the prior page's load was interrupted.
@@ -4289,6 +4466,64 @@ setInterval(() => {
   if (sessions.size === 0) scheduleBrowserIdleShutdown();
 }, 60_000);
 
+// Orphan page reaper -- force-closes Playwright pages that survived a safePageClose
+// timeout or were otherwise dropped from tabGroups tracking. Without this, leaked
+// pages starve Firefox of DOM threads and eventually block new tab creation.
+setInterval(() => {
+  let reaped = 0;
+  for (const session of sessions.values()) {
+    if (session._closing) continue;
+    let contextPages;
+    try {
+      contextPages = session.context.pages();
+    } catch (_) {
+      continue; // context already dead
+    }
+    const registered = new Set();
+    for (const group of session.tabGroups.values()) {
+      for (const tabState of group.values()) registered.add(tabState.page);
+    }
+    for (const page of contextPages) {
+      if (!registered.has(page)) {
+        reaped++;
+        page.removeAllListeners();
+        page.close({ runBeforeUnload: false }).catch(() => {});
+      }
+    }
+  }
+  if (reaped > 0) log('warn', 'orphan page reaper closed leaked pages', { reaped });
+}, 60_000);
+
+// Native memory pressure restart -- when all sessions are gone and the Node
+// process's native memory (RSS minus V8 heap) has grown beyond threshold, kill
+// the browser process immediately instead of waiting for the idle timer.
+// Note: This measures Node/Playwright internal state (CDP buffers, glibc arenas),
+// NOT Firefox's own memory (which is a separate child process). Firefox jemalloc
+// fragmentation is tracked separately via browser RSS in /proc/<pid>/status.
+// The restart reclaims Playwright state; Firefox's process dies with it.
+setInterval(() => {
+  if (sessions.size > 0 || !browser) return;
+  const mem = process.memoryUsage();
+  const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
+  if (_nativeMemBaseline === null) {
+    _nativeMemBaseline = nativeMemMb;
+    return;
+  }
+  const growth = nativeMemMb - _nativeMemBaseline;
+  if (growth >= NATIVE_MEM_RESTART_THRESHOLD_MB) {
+    log('warn', 'native memory pressure, restarting browser', {
+      baselineMb: _nativeMemBaseline,
+      currentMb: nativeMemMb,
+      growthMb: growth,
+      thresholdMb: NATIVE_MEM_RESTART_THRESHOLD_MB,
+    });
+    browserRestartsTotal.labels('memory_pressure').inc();
+    closeBrowserFully('memory_pressure').catch((err) => {
+      log('error', 'memory pressure browser close failed', { error: err.message });
+    });
+  }
+}, 30_000);
+
 // =============================================================================
 // OpenClaw-compatible endpoint aliases
 // These allow camoufox to be used as a profile backend for OpenClaw's browser tool
@@ -4453,7 +4688,7 @@ app.post('/tabs/open', async (req, res) => {
     const urlErr = validateUrl(url);
     if (urlErr) return res.status(400).json({ error: urlErr });
     
-    const session = await getSession(userId);
+    let session = await getSession(userId);
     
     // Recycle oldest tab when limits are reached instead of rejecting
     let totalTabs = 0;
@@ -4465,16 +4700,40 @@ app.post('/tabs/open', async (req, res) => {
       }
     }
     
-    const group = getTabGroup(session, listItemId);
+    let group = getTabGroup(session, listItemId);
     
-    const page = await session.context.newPage();
+    let page = await session.context.newPage();
     const tabId = fly.makeTabId();
-    const tabState = createTabState(page);
+    let tabState = createTabState(page);
     attachDownloadListener(tabState, tabId, log, pluginEvents, userId);
     group.set(tabId, tabState);
     refreshActiveTabsGauge();
     
-    await withPageLoadDuration('open_url', () => page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
+    try {
+      await withPageLoadDuration('open_url', () => page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
+    } catch (navErr) {
+      if ((isProxyError(navErr) || isTimeoutError(navErr)) && proxyPool?.canRotateSessions) {
+        log('warn', 'tab open failed, retrying with fresh proxy', {
+          reqId: req.reqId, tabId, error: navErr.message,
+        });
+        browserRestartsTotal.labels('proxy_retry').inc();
+        const key = normalizeUserId(userId);
+        const oldSession = sessions.get(key);
+        if (oldSession) {
+          await closeSession(key, oldSession, { reason: 'proxy_retry_rotate', clearDownloads: true, clearLocks: true });
+        }
+        session = await getSession(userId);
+        group = getTabGroup(session, listItemId);
+        page = await session.context.newPage();
+        tabState = createTabState(page);
+        attachDownloadListener(tabState, tabId, log, pluginEvents, userId);
+        group.set(tabId, tabState);
+        refreshActiveTabsGauge();
+        await withPageLoadDuration('open_url', () => page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
+      } else {
+        throw navErr;
+      }
+    }
     tabState.visitedUrls.add(url);
     
     log('info', 'openclaw tab opened', { reqId: req.reqId, tabId, url: page.url() });
@@ -5232,8 +5491,15 @@ const server = app.listen(PORT, async () => {
     log('info', 'browser pre-warmed', { ms: Date.now() - start });
     scheduleBrowserIdleShutdown();
   } catch (err) {
-    log('error', 'browser pre-warm failed (will retry in background)', { error: err.message });
-    scheduleBrowserWarmRetry();
+    if (isFatalInstallError(err)) {
+      log('error', 'browser pre-warm aborted: Camoufox binaries are not installed', {
+        error: err.message,
+        remediation: 'run `npx camoufox-js fetch` then restart the server',
+      });
+    } else {
+      log('error', 'browser pre-warm failed (will retry in background)', { error: err.message });
+      scheduleBrowserWarmRetry();
+    }
   }
   // Idle self-shutdown removed -- Fly manages machine lifecycle via fly.toml.
 });

package/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "noImplicitAny": false,
+    "types": ["node"]
+  },
+  "files": ["plugin.ts"]
+}