@askjo/camofox-browser 1.4.0 → 1.5.0

package/server.js

@@ -1,10 +1,13 @@
 import { Camoufox, launchOptions } from 'camoufox-js';
+import { VirtualDisplay } from 'camoufox-js/dist/virtdisplay.js';
 import { firefox } from 'playwright-core';
 import express from 'express';
 import crypto from 'crypto';
 import os from 'os';
 import { expandMacro } from './lib/macros.js';
 import { loadConfig } from './lib/config.js';
+import { normalizePlaywrightProxy, createProxyPool, buildProxyUrl } from './lib/proxy.js';
+import { createFlyHelpers } from './lib/fly.js';
 import { windowSnapshot } from './lib/snapshot.js';
 import {
   MAX_DOWNLOAD_INLINE_BYTES,
@@ -14,7 +17,15 @@ import {
   getDownloadsList,
   extractPageImages,
 } from './lib/downloads.js';
-import { detectYtDlp, hasYtDlp, ytDlpTranscript, parseJson3, parseVtt, parseXml } from './lib/youtube.js';
+import { detectYtDlp, hasYtDlp, ensureYtDlp, ytDlpTranscript, parseJson3, parseVtt, parseXml } from './lib/youtube.js';
+import {
+  register as metricsRegister,
+  requestsTotal, requestDuration, pageLoadDuration,
+  activeTabsGauge, tabLockQueueDepth,
+  tabLockTimeoutsTotal, startMemoryReporter, stopMemoryReporter, actionFromReq,
+  failuresTotal, browserRestartsTotal, tabsDestroyedTotal,
+  sessionsExpiredTotal, tabsReapedTotal, tabsRecycledTotal, classifyError,
+} from './lib/metrics.js';
 
 const CONFIG = loadConfig();
 
@@ -37,23 +48,44 @@ function log(level, msg, fields = {}) {
 const app = express();
 app.use(express.json({ limit: '100kb' }));
 
-// Request logging middleware
+// Request logging + metrics middleware
 app.use((req, res, next) => {
-  if (req.path === '/health') return next();
   const reqId = crypto.randomUUID().slice(0, 8);
   req.reqId = reqId;
   req.startTime = Date.now();
+
   const userId = req.body?.userId || req.query?.userId || '-';
-  log('info', 'req', { reqId, method: req.method, path: req.path, userId });
+  if (req.path !== '/health') {
+    log('info', 'req', { reqId, method: req.method, path: req.path, userId });
+  }
+
+  const action = actionFromReq(req);
+  const done = requestDuration.startTimer({ action });
+
   const origEnd = res.end.bind(res);
   res.end = function (...args) {
     const ms = Date.now() - req.startTime;
-    log('info', 'res', { reqId, status: res.statusCode, ms });
+    const isErrorStatus = res.statusCode >= 400;
+    requestsTotal.labels(action, isErrorStatus ? 'error' : 'success').inc();
+    done();
+
+    if (req.path !== '/health') {
+      log('info', 'res', { reqId, status: res.statusCode, ms });
+    }
+
     return origEnd(...args);
   };
+
   next();
 });
 
+// --- Horizontal scaling (Fly.io multi-machine) ---
+const fly = createFlyHelpers(CONFIG);
+const FLY_MACHINE_ID = fly.machineId;
+
+// Route tab requests to the owning machine via fly-replay header.
+app.use('/tabs/:tabId', fly.replayMiddleware(log));
+
 const ALLOWED_URL_SCHEMES = ['http:', 'https:'];
 
 // Interactive roles to include - exclude combobox to avoid opening complex widgets
@@ -203,6 +235,7 @@ app.post('/sessions/:userId/cookies', express.json({ limit: '512kb' }), async (r
     log('info', 'cookies imported', { reqId: req.reqId, userId: String(userId), count: sanitized.length });
     res.json(result);
   } catch (err) {
+    failuresTotal.labels(classifyError(err), 'set_cookies').inc();
     log('error', 'cookie import failed', { reqId: req.reqId, error: err.message });
     res.status(500).json({ error: safeError(err) });
   }
@@ -229,6 +262,8 @@ const FAILURE_THRESHOLD = 3;
 const MAX_CONSECUTIVE_TIMEOUTS = 3;
 const TAB_LOCK_TIMEOUT_MS = 35000; // Must be > HANDLER_TIMEOUT_MS so active op times out first
 
+
+
 // Proper mutex for tab serialization. The old Promise-chain lock on timeout proceeded
 // WITHOUT the lock, allowing concurrent Playwright operations that corrupt CDP state.
 class TabLock {
@@ -243,9 +278,12 @@ class TabLock {
       entry.timer = setTimeout(() => {
         const idx = this.queue.indexOf(entry);
         if (idx !== -1) this.queue.splice(idx, 1);
+        tabLockTimeoutsTotal.inc();
+        refreshTabLockQueueDepth();
         reject(new Error('Tab lock queue timeout'));
       }, timeoutMs);
       this.queue.push(entry);
+      refreshTabLockQueueDepth();
       this._tryNext();
     });
   }
@@ -253,6 +291,7 @@ class TabLock {
   release() {
     this.active = false;
     this._tryNext();
+    refreshTabLockQueueDepth();
   }
 
   _tryNext() {
@@ -260,6 +299,7 @@ class TabLock {
     this.active = true;
     const entry = this.queue.shift();
     clearTimeout(entry.timer);
+    refreshTabLockQueueDepth();
     entry.resolve();
   }
 
@@ -270,6 +310,7 @@ class TabLock {
       entry.reject(new Error('Tab destroyed'));
     }
     this.queue = [];
+    refreshTabLockQueueDepth();
   }
 }
 
@@ -302,6 +343,10 @@ function withTimeout(promise, ms, label) {
   ]);
 }
 
+function requestTimeoutMs(baseMs = HANDLER_TIMEOUT_MS) {
+  return proxyPool?.canRotateSessions ? Math.max(baseMs, 180000) : baseMs;
+}
+
 const userConcurrency = new Map();
 
 async function withUserLimit(userId, operation) {
@@ -355,25 +400,27 @@ function getHostOS() {
   return 'linux';
 }
 
-function buildProxyConfig() {
-  const { host, port, username, password } = CONFIG.proxy;
-  
-  if (!host || !port) {
-    log('info', 'no proxy configured');
-    return null;
-  }
-  
-  log('info', 'proxy configured', { host, port });
-  return {
-    server: `http://${host}:${port}`,
-    username,
-    password,
-  };
+// Proxy strategy for outbound browsing.
+const proxyPool = createProxyPool(CONFIG.proxy);
+
+if (proxyPool) {
+  log('info', 'proxy pool created', {
+    mode: proxyPool.mode,
+    host: proxyPool.canRotateSessions ? CONFIG.proxy.backconnectHost : CONFIG.proxy.host,
+    ports: proxyPool.canRotateSessions ? [CONFIG.proxy.backconnectPort] : CONFIG.proxy.ports,
+    poolSize: proxyPool.size,
+    country: CONFIG.proxy.country || null,
+    state: CONFIG.proxy.state || null,
+    city: CONFIG.proxy.city || null,
+  });
+} else {
+  log('info', 'no proxy configured');
 }
 
 const BROWSER_IDLE_TIMEOUT_MS = CONFIG.browserIdleTimeoutMs;
 let browserIdleTimer = null;
 let browserLaunchPromise = null;
+let browserWarmRetryTimer = null;
 
 function scheduleBrowserIdleShutdown() {
   clearBrowserIdleTimer();
@@ -396,6 +443,21 @@ function clearBrowserIdleTimer() {
   }
 }
 
+function scheduleBrowserWarmRetry(delayMs = 5000) {
+  if (browserWarmRetryTimer || browser || browserLaunchPromise) return;
+  browserWarmRetryTimer = setTimeout(async () => {
+    browserWarmRetryTimer = null;
+    try {
+      const start = Date.now();
+      await ensureBrowser();
+      log('info', 'background browser warm retry succeeded', { ms: Date.now() - start });
+    } catch (err) {
+      log('warn', 'background browser warm retry failed', { error: err.message, nextDelayMs: delayMs });
+      scheduleBrowserWarmRetry(Math.min(delayMs * 2, 30000));
+    }
+  }, delayMs);
+}
+
 // --- Browser health tracking ---
 const healthState = {
   consecutiveNavFailures: 0,
@@ -417,6 +479,7 @@ function recordNavFailure() {
 async function restartBrowser(reason) {
   if (healthState.isRecovering) return;
   healthState.isRecovering = true;
+  browserRestartsTotal.labels(reason).inc();
   log('error', 'restarting browser', { reason, failures: healthState.consecutiveNavFailures });
   try {
     for (const [, session] of sessions) {
@@ -449,29 +512,157 @@ function getTotalTabCount() {
   return total;
 }
 
+// Virtual display for WebGL support and anti-detection.
+// Xvfb gives Firefox a real X display with GLX, enabling software-rendered WebGL
+// via Mesa llvmpipe. Without this, WebGL returns "no context" — a massive bot signal.
+let virtualDisplay = null;
+let browserLaunchProxy = null;
+
+async function probeGoogleSearch(candidateBrowser) {
+  let context = null;
+  try {
+    context = await candidateBrowser.newContext({
+      viewport: { width: 1280, height: 720 },
+      permissions: ['geolocation'],
+    });
+    const page = await context.newPage();
+    await page.goto('https://www.google.com/', { waitUntil: 'domcontentloaded', timeout: 30000 });
+    await page.waitForTimeout(1200);
+    await page.goto('https://www.google.com/search?q=weather%20today', { waitUntil: 'domcontentloaded', timeout: 30000 });
+    await page.waitForTimeout(4000);
+
+    const blocked = await isGoogleSearchBlocked(page);
+    return {
+      ok: !blocked && isGoogleSerp(page.url()),
+      url: page.url(),
+      blocked,
+    };
+  } finally {
+    await context?.close().catch(() => {});
+  }
+}
+
+function attachBrowserCleanup(candidateBrowser, localVirtualDisplay) {
+  const origClose = candidateBrowser.close.bind(candidateBrowser);
+  candidateBrowser.close = async (...args) => {
+    await origClose(...args);
+    browserLaunchProxy = null;
+    if (localVirtualDisplay) {
+      localVirtualDisplay.kill();
+      if (virtualDisplay === localVirtualDisplay) virtualDisplay = null;
+    }
+  };
+}
+
 async function launchBrowserInstance() {
   const hostOS = getHostOS();
-  const proxy = buildProxyConfig();
-  
-  log('info', 'launching camoufox', { hostOS, geoip: !!proxy });
-  
-  const options = await launchOptions({
-    headless: true,
-    os: hostOS,
-    humanize: true,
-    enable_cache: true,
-    proxy: proxy,
-    geoip: !!proxy,
-  });
-  
-  browser = await firefox.launch(options);
-  log('info', 'camoufox launched');
-  return browser;
+  const maxAttempts = proxyPool?.launchRetries ?? 1;
+  let lastError = null;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    const launchProxy = proxyPool
+      ? proxyPool.getLaunchProxy(proxyPool.canRotateSessions ? `browser-${crypto.randomUUID().replace(/-/g, '').slice(0, 12)}` : undefined)
+      : null;
+
+    let localVirtualDisplay = null;
+    let vdDisplay = undefined;
+    let candidateBrowser = null;
+
+    try {
+      if (os.platform() === 'linux') {
+        localVirtualDisplay = new VirtualDisplay();
+        vdDisplay = localVirtualDisplay.get();
+        log('info', 'xvfb virtual display started', { display: vdDisplay, attempt });
+      }
+    } catch (err) {
+      log('warn', 'xvfb not available, falling back to headless', { error: err.message, attempt });
+      localVirtualDisplay = null;
+    }
+
+    const useVirtualDisplay = !!vdDisplay;
+    log('info', 'launching camoufox', {
+      hostOS,
+      attempt,
+      maxAttempts,
+      geoip: !!launchProxy,
+      proxyMode: proxyPool?.mode || null,
+      proxyServer: launchProxy?.server || null,
+      proxySession: launchProxy?.sessionId || null,
+      proxyPoolSize: proxyPool?.size || 0,
+      virtualDisplay: useVirtualDisplay,
+    });
+
+    try {
+      const options = await launchOptions({
+        headless: useVirtualDisplay ? false : true,
+        os: hostOS,
+        humanize: true,
+        enable_cache: true,
+        proxy: launchProxy,
+        geoip: !!launchProxy,
+        virtual_display: vdDisplay,
+      });
+      options.proxy = normalizePlaywrightProxy(options.proxy);
+
+      candidateBrowser = await firefox.launch(options);
+
+      if (proxyPool?.canRotateSessions) {
+        const probe = await probeGoogleSearch(candidateBrowser);
+        if (!probe.ok) {
+          log('warn', 'browser launch google probe failed', {
+            attempt,
+            maxAttempts,
+            proxySession: launchProxy?.sessionId || null,
+            url: probe.url,
+          });
+          if (attempt < maxAttempts) {
+            await candidateBrowser.close().catch(() => {});
+            if (localVirtualDisplay) localVirtualDisplay.kill();
+            continue;
+          }
+          // Last attempt: accept browser in degraded mode rather than death-spiraling.
+          // Non-Google sites will still work; Google requests will get blocked responses.
+          log('error', 'all proxy sessions Google-blocked, accepting browser in degraded mode', {
+            maxAttempts,
+            proxySession: launchProxy?.sessionId || null,
+          });
+        }
+      }
+
+      virtualDisplay = localVirtualDisplay;
+      browserLaunchProxy = launchProxy;
+      browser = candidateBrowser;
+      attachBrowserCleanup(browser, localVirtualDisplay);
+
+      log('info', 'camoufox launched', {
+        attempt,
+        maxAttempts,
+        virtualDisplay: useVirtualDisplay,
+        proxyMode: proxyPool?.mode || null,
+        proxyServer: launchProxy?.server || null,
+        proxySession: launchProxy?.sessionId || null,
+      });
+      return browser;
+    } catch (err) {
+      lastError = err;
+      log('warn', 'camoufox launch attempt failed', {
+        attempt,
+        maxAttempts,
+        error: err.message,
+        proxySession: launchProxy?.sessionId || null,
+      });
+      await candidateBrowser?.close().catch(() => {});
+      if (localVirtualDisplay) localVirtualDisplay.kill();
+    }
+  }
+
+  throw lastError || new Error('Failed to launch a usable browser');
 }
 
 async function ensureBrowser() {
   clearBrowserIdleTimer();
   if (browser && !browser.isConnected()) {
+    failuresTotal.labels('browser_disconnected', 'internal').inc();
     log('warn', 'browser disconnected, clearing dead sessions and relaunching', {
       deadSessions: sessions.size,
     });
@@ -479,13 +670,20 @@ async function ensureBrowser() {
       await session.context.close().catch(() => {});
     }
     sessions.clear();
+    // Clean up virtual display from dead browser before relaunching
+    if (virtualDisplay) {
+      virtualDisplay.kill();
+      virtualDisplay = null;
+    }
+    browserLaunchProxy = null;
     browser = null;
   }
   if (browser) return browser;
   if (browserLaunchPromise) return browserLaunchPromise;
+  const launchTimeoutMs = proxyPool?.launchTimeoutMs ?? 60000;
   browserLaunchPromise = Promise.race([
     launchBrowserInstance(),
-    new Promise((_, reject) => setTimeout(() => reject(new Error('Browser launch timeout (30s)')), 30000)),
+    new Promise((_, reject) => setTimeout(() => reject(new Error(`Browser launch timeout (${Math.round(launchTimeoutMs / 1000)}s)`)), launchTimeoutMs)),
   ]).finally(() => { browserLaunchPromise = null; });
   return browserLaunchPromise;
 }
@@ -528,11 +726,26 @@ async function getSession(userId) {
       contextOptions.timezoneId = 'America/Los_Angeles';
       contextOptions.geolocation = { latitude: 37.7749, longitude: -122.4194 };
     }
+    let sessionProxy = null;
+    if (proxyPool?.canRotateSessions) {
+      sessionProxy = proxyPool.getNext(`ctx-${key}-${crypto.randomUUID().replace(/-/g, '').slice(0, 8)}`);
+      contextOptions.proxy = normalizePlaywrightProxy(sessionProxy);
+      log('info', 'session proxy assigned', { userId: key, sessionId: sessionProxy.sessionId });
+    } else if (proxyPool) {
+      sessionProxy = proxyPool.getNext();
+      contextOptions.proxy = normalizePlaywrightProxy(sessionProxy);
+      log('info', 'session proxy assigned', { userId: key, proxy: sessionProxy.server });
+    }
     const context = await b.newContext(contextOptions);
     
-    session = { context, tabGroups: new Map(), lastAccess: Date.now() };
+    session = { context, tabGroups: new Map(), lastAccess: Date.now(), proxySessionId: sessionProxy?.sessionId || null };
     sessions.set(key, session);
-    log('info', 'session created', { userId: key });
+    log('info', 'session created', {
+      userId: key,
+      proxyMode: proxyPool?.mode || null,
+      proxyServer: sessionProxy?.server || browserLaunchProxy?.server || null,
+      proxySession: sessionProxy?.sessionId || browserLaunchProxy?.sessionId || null,
+    });
   }
   session.lastAccess = Date.now();
   return session;
@@ -571,11 +784,30 @@ function isTabDestroyedError(err) {
 
 // Centralized error handler for route catch blocks.
 // Auto-destroys dead browser sessions and returns appropriate status codes.
+function isProxyError(err) {
+  if (!err) return false;
+  const msg = err.message || '';
+  return msg.includes('NS_ERROR_PROXY') || msg.includes('proxy connection') || msg.includes('Proxy connection');
+}
+
 function handleRouteError(err, req, res, extraFields = {}) {
+  const failureType = classifyError(err);
+  const action = actionFromReq(req);
+  failuresTotal.labels(failureType, action).inc();
+
   const userId = req.body?.userId || req.query?.userId;
   if (userId && isDeadContextError(err)) {
     destroySession(userId);
   }
+  // Proxy errors mean the session is dead — rotate at context level.
+  // Destroy the user's session so the next request gets a fresh context with a new proxy.
+  if (isProxyError(err) && proxyPool?.canRotateSessions && userId) {
+    log('warn', 'proxy error detected, destroying user session for fresh proxy on next request', {
+      action, userId, error: err.message,
+    });
+    browserRestartsTotal.labels('proxy_error').inc();
+    destroySession(userId);
+  }
   // Track consecutive timeouts per tab and auto-destroy stuck tabs
   if (userId && isTimeoutError(err)) {
     const tabId = req.body?.tabId || req.query?.tabId || req.params?.tabId;
@@ -586,7 +818,7 @@ function handleRouteError(err, req, res, extraFields = {}) {
         found.tabState.consecutiveTimeouts++;
         if (found.tabState.consecutiveTimeouts >= MAX_CONSECUTIVE_TIMEOUTS) {
           log('warn', 'auto-destroying tab after consecutive timeouts', { tabId, count: found.tabState.consecutiveTimeouts });
-          destroyTab(session, tabId);
+          destroyTab(session, tabId, 'consecutive_timeouts');
         }
       }
     }
@@ -596,7 +828,7 @@ function handleRouteError(err, req, res, extraFields = {}) {
     const tabId = req.body?.tabId || req.query?.tabId || req.params?.tabId;
     const session = sessions.get(normalizeUserId(userId));
     if (session && tabId) {
-      destroyTab(session, tabId);
+      destroyTab(session, tabId, 'lock_queue');
     }
     return res.status(503).json({ error: 'Tab unresponsive and has been destroyed. Open a new tab.', ...extraFields });
   }
@@ -607,25 +839,61 @@ function handleRouteError(err, req, res, extraFields = {}) {
   sendError(res, err, extraFields);
 }
 
-function destroyTab(session, tabId) {
+function destroyTab(session, tabId, reason) {
   const lock = tabLocks.get(tabId);
   if (lock) {
     lock.drain();
     tabLocks.delete(tabId);
+    refreshTabLockQueueDepth();
   }
   for (const [listItemId, group] of session.tabGroups) {
     if (group.has(tabId)) {
       const tabState = group.get(tabId);
-      log('warn', 'destroying stuck tab', { tabId, listItemId, toolCalls: tabState.toolCalls });
+      log('warn', 'destroying stuck tab', { tabId, listItemId, toolCalls: tabState.toolCalls, reason: reason || 'unknown' });
       safePageClose(tabState.page);
       group.delete(tabId);
       if (group.size === 0) session.tabGroups.delete(listItemId);
+      refreshActiveTabsGauge();
+      if (reason) tabsDestroyedTotal.labels(reason).inc();
       return true;
     }
   }
   return false;
 }
 
+/**
+ * Recycle the oldest (least-used) tab in a session to free a slot.
+ * Closes the old tab's page and removes it from its group.
+ * Returns { recycledTabId, recycledFromGroup } or null if no tab to recycle.
+ */
+async function recycleOldestTab(session, reqId) {
+  let oldestTab = null;
+  let oldestGroup = null;
+  let oldestGroupKey = null;
+  let oldestTabId = null;
+  for (const [gKey, group] of session.tabGroups) {
+    for (const [tid, ts] of group) {
+      if (!oldestTab || ts.toolCalls < oldestTab.toolCalls) {
+        oldestTab = ts;
+        oldestGroup = group;
+        oldestGroupKey = gKey;
+        oldestTabId = tid;
+      }
+    }
+  }
+  if (!oldestTab) return null;
+
+  await safePageClose(oldestTab.page);
+  oldestGroup.delete(oldestTabId);
+  if (oldestGroup.size === 0) session.tabGroups.delete(oldestGroupKey);
+  const lock = tabLocks.get(oldestTabId);
+  if (lock) { lock.drain(); tabLocks.delete(oldestTabId); }
+  refreshTabLockQueueDepth();
+  tabsRecycledTotal.inc();
+  log('info', 'tab recycled (limit reached)', { reqId, recycledTabId: oldestTabId, recycledFromGroup: oldestGroupKey });
+  return { recycledTabId: oldestTabId, recycledFromGroup: oldestGroupKey };
+}
+
 function destroySession(userId) {
   const key = normalizeUserId(userId);
   const session = sessions.get(key);
@@ -654,13 +922,89 @@ function createTabState(page) {
     toolCalls: 0,
     consecutiveTimeouts: 0,
     lastSnapshot: null,
+    lastRequestedUrl: null,
+    googleRetryCount: 0,
   };
 }
 
+async function isGoogleUnavailable(page) {
+  if (!page || page.isClosed()) return false;
+  const bodyText = await page.evaluate(() => document.body?.innerText?.slice(0, 600) || '').catch(() => '');
+  return /Unable to connect|502 Bad Gateway or Proxy Error|Camoufox can’t establish a connection/.test(bodyText);
+}
+
+async function rotateGoogleTab(userId, sessionKey, tabId, previousTabState, reason, reqId) {
+  if (!previousTabState?.lastRequestedUrl || !isGoogleSearchUrl(previousTabState.lastRequestedUrl)) return null;
+  if ((previousTabState.googleRetryCount || 0) >= 3) return null;
+
+  browserRestartsTotal.labels(reason).inc(); // track rotation events (not a full restart)
+
+  // Rotate at context level — create a fresh context with a new proxy session
+  // instead of restarting the entire browser (which kills ALL sessions/tabs).
+  const key = normalizeUserId(userId);
+  const oldSession = sessions.get(key);
+  if (oldSession) {
+    await oldSession.context.close().catch(() => {});
+    sessions.delete(key);
+  }
+  const session = await getSession(userId);
+  const group = getTabGroup(session, sessionKey);
+  const page = await session.context.newPage();
+  const tabState = createTabState(page);
+  tabState.googleRetryCount = (previousTabState.googleRetryCount || 0) + 1;
+  tabState.lastRequestedUrl = previousTabState.lastRequestedUrl;
+  attachDownloadListener(tabState, tabId, log);
+  group.set(tabId, tabState);
+  refreshActiveTabsGauge();
+
+  log('warn', 'replaying google search on fresh context (per-context proxy rotation)', {
+    reqId,
+    tabId,
+    retryCount: tabState.googleRetryCount,
+    url: tabState.lastRequestedUrl,
+    proxySession: session.proxySessionId || null,
+  });
+
+  await withPageLoadDuration('navigate', () => page.goto('https://www.google.com/', { waitUntil: 'domcontentloaded', timeout: 30000 }));
+  tabState.visitedUrls.add('https://www.google.com/');
+  await page.waitForTimeout(1200);
+  await withPageLoadDuration('navigate', () => page.goto(tabState.lastRequestedUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }));
+  tabState.visitedUrls.add(tabState.lastRequestedUrl);
+  return { session, tabState };
+}
+
+function refreshActiveTabsGauge() {
+  activeTabsGauge.set(getTotalTabCount());
+}
+
+function refreshTabLockQueueDepth() {
+  let queued = 0;
+  for (const lock of tabLocks.values()) {
+    if (lock?.queue) queued += lock.queue.length;
+  }
+  tabLockQueueDepth.set(queued);
+}
+
+async function withPageLoadDuration(action, fn) {
+  const end = pageLoadDuration.startTimer();
+  try {
+    return await fn();
+  } finally {
+    end();
+  }
+}
+
 
 
 async function waitForPageReady(page, options = {}) {
-  const { timeout = 10000, waitForNetwork = true } = options;
+  const {
+    timeout = 10000,
+    waitForNetwork = true,
+    waitForHydration = true,
+    settleMs = 200,
+    hydrationPollMs = 250,
+    hydrationTimeoutMs = Math.min(timeout, 10000),
+  } = options;
   
   try {
     await page.waitForLoadState('domcontentloaded', { timeout });
@@ -671,27 +1015,28 @@ async function waitForPageReady(page, options = {}) {
       });
     }
     
-    // Framework hydration wait (React/Next.js/Vue) - mirrors Swift WebView.swift logic
-    // Wait for readyState === 'complete' + network quiet (40 iterations × 250ms max)
-    await page.evaluate(async () => {
-      for (let i = 0; i < 40; i++) {
-        // Check if network is quiet (no recent resource loads)
-        const entries = performance.getEntriesByType('resource');
-        const recentEntries = entries.slice(-5);
-        const netQuiet = recentEntries.every(e => (performance.now() - e.responseEnd) > 400);
-        
-        if (document.readyState === 'complete' && netQuiet) {
-          // Double RAF to ensure paint is complete
-          await new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)));
-          break;
+    if (waitForHydration) {
+      const maxIterations = Math.max(1, Math.floor(hydrationTimeoutMs / hydrationPollMs));
+      await page.evaluate(async ({ maxIterations, hydrationPollMs }) => {
+        for (let i = 0; i < maxIterations; i++) {
+          const entries = performance.getEntriesByType('resource');
+          const recentEntries = entries.slice(-5);
+          const netQuiet = recentEntries.every(e => (performance.now() - e.responseEnd) > 400);
+          
+          if (document.readyState === 'complete' && netQuiet) {
+            await new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)));
+            break;
+          }
+          await new Promise(r => setTimeout(r, hydrationPollMs));
         }
-        await new Promise(r => setTimeout(r, 250));
-      }
-    }).catch(() => {
-      log('warn', 'hydration wait failed, continuing');
-    });
+      }, { maxIterations, hydrationPollMs }).catch(() => {
+        log('warn', 'hydration wait failed, continuing');
+      });
+    }
     
-    await page.waitForTimeout(200);
+    if (settleMs > 0) {
+      await page.waitForTimeout(settleMs);
+    }
     
     // Auto-dismiss common consent/privacy dialogs
     await dismissConsentDialogs(page);
@@ -758,6 +1103,25 @@ function isGoogleSerp(url) {
   }
 }
 
+function isGoogleSearchUrl(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.hostname.includes('google.') && parsed.pathname === '/search';
+  } catch {
+    return false;
+  }
+}
+
+async function isGoogleSearchBlocked(page) {
+  if (!page || page.isClosed()) return false;
+
+  const url = page.url();
+  if (url.includes('google.com/sorry/')) return true;
+
+  const bodyText = await page.evaluate(() => document.body?.innerText?.slice(0, 600) || '').catch(() => '');
+  return /Our systems have detected unusual traffic|About this page|If you're having trouble accessing Google Search|SG_REL/.test(bodyText);
+}
+
 // --- Google SERP: combined extraction (refs + snapshot in one DOM pass) ---
 // Returns { refs: Map, snapshot: string }
 async function extractGoogleSerp(page) {
@@ -898,6 +1262,8 @@ async function extractGoogleSerp(page) {
   return { refs, snapshot: extracted.snapshot };
 }
 
+const REFRESH_READY_TIMEOUT_MS = 2500;
+
 async function buildRefs(page) {
   const refs = new Map();
   
@@ -916,16 +1282,20 @@ async function buildRefs(page) {
   const start = Date.now();
   
   // Hard total timeout on the entire buildRefs operation
-  const timeoutPromise = new Promise((_, reject) => 
-    setTimeout(() => reject(new Error('buildRefs_timeout')), BUILDREFS_TIMEOUT_MS)
-  );
+  let timerId;
+  const timeoutPromise = new Promise((_, reject) => {
+    timerId = setTimeout(() => reject(new Error('buildRefs_timeout')), BUILDREFS_TIMEOUT_MS);
+  });
   
   try {
-    return await Promise.race([
+    const result = await Promise.race([
       _buildRefsInner(page, refs, start),
       timeoutPromise
     ]);
+    clearTimeout(timerId);
+    return result;
   } catch (err) {
+    clearTimeout(timerId);
     if (err.message === 'buildRefs_timeout') {
       log('warn', 'buildRefs: total timeout exceeded', { elapsed: Date.now() - start });
       return refs;
@@ -935,7 +1305,12 @@ async function buildRefs(page) {
 }
 
 async function _buildRefsInner(page, refs, start) {
-  await waitForPageReady(page, { waitForNetwork: false });
+  await waitForPageReady(page, {
+    timeout: REFRESH_READY_TIMEOUT_MS,
+    waitForNetwork: false,
+    waitForHydration: false,
+    settleMs: 100,
+  });
   
   // Budget remaining time for ariaSnapshot
   const elapsed = Date.now() - start;
@@ -1004,7 +1379,12 @@ async function getAriaSnapshot(page) {
   if (!page || page.isClosed()) {
     return null;
   }
-  await waitForPageReady(page, { waitForNetwork: false });
+  await waitForPageReady(page, {
+    timeout: REFRESH_READY_TIMEOUT_MS,
+    waitForNetwork: false,
+    waitForHydration: false,
+    settleMs: 100,
+  });
   try {
     return await page.locator('body').ariaSnapshot({ timeout: 5000 });
   } catch (err) {
@@ -1027,11 +1407,46 @@ function refToLocator(page, ref, refs) {
   return locator;
 }
 
+async function refreshTabRefs(tabState, options = {}) {
+  const {
+    reason = 'refresh',
+    timeoutMs = null,
+    preserveExistingOnEmpty = true,
+  } = options;
+
+  const beforeUrl = tabState.page?.url?.() || '';
+  const existingRefs = tabState.refs instanceof Map ? tabState.refs : new Map();
+  const refreshPromise = buildRefs(tabState.page);
+
+  let refreshedRefs;
+  if (timeoutMs) {
+    const timeoutLabel = `${reason}_refs_timeout`;
+    refreshedRefs = await Promise.race([
+      refreshPromise,
+      new Promise((_, reject) => setTimeout(() => reject(new Error(timeoutLabel)), timeoutMs)),
+    ]);
+  } else {
+    refreshedRefs = await refreshPromise;
+  }
+
+  const afterUrl = tabState.page?.url?.() || beforeUrl;
+  if (preserveExistingOnEmpty && refreshedRefs.size === 0 && existingRefs.size > 0 && beforeUrl === afterUrl) {
+    log('warn', 'preserving previous refs after empty rebuild', {
+      reason,
+      url: afterUrl,
+      previousRefs: existingRefs.size,
+    });
+    return existingRefs;
+  }
+
+  return refreshedRefs;
+}
+
 // --- YouTube transcript ---
 // Implementation extracted to lib/youtube.js to avoid scanner false positives
 // (child_process + app.post in same file triggers OpenClaw skill-scanner)
 
-detectYtDlp(log);
+await detectYtDlp(log);
 
 app.post('/youtube/transcript', async (req, res) => {
   const reqId = req.reqId;
@@ -1051,14 +1466,23 @@ app.post('/youtube/transcript', async (req, res) => {
     const videoId = videoIdMatch[1];
     const lang = languages[0] || 'en';
 
-    log('info', 'youtube transcript: starting', { reqId, videoId, lang, method: hasYtDlp() ? 'yt-dlp' : 'browser' });
+    // Re-detect yt-dlp if startup detection failed (transient issue)
+    await ensureYtDlp(log);
+
+    const ytDlpProxyUrl = buildProxyUrl(proxyPool, CONFIG.proxy);
+    log('info', 'youtube transcript: starting', { reqId, videoId, lang, method: hasYtDlp() ? 'yt-dlp' : 'browser', hasProxy: !!ytDlpProxyUrl });
 
     let result;
     if (hasYtDlp()) {
       try {
-        result = await ytDlpTranscript(reqId, url, videoId, lang);
+        result = await ytDlpTranscript(reqId, url, videoId, lang, ytDlpProxyUrl);
       } catch (ytErr) {
-        log('warn', 'yt-dlp failed, falling back to browser', { reqId, error: ytErr.message });
+        log('warn', 'yt-dlp threw, falling back to browser', { reqId, error: ytErr.message });
+        result = null;
+      }
+      // If yt-dlp returned an error result (e.g. no captions) or threw, try browser
+      if (!result || result.status !== 'ok') {
+        if (result) log('warn', 'yt-dlp returned error, falling back to browser', { reqId, status: result.status, code: result.code });
         result = await browserTranscript(reqId, url, videoId, lang);
       }
     } else {
@@ -1068,6 +1492,7 @@ app.post('/youtube/transcript', async (req, res) => {
     log('info', 'youtube transcript: done', { reqId, videoId, status: result.status, words: result.total_words });
     res.json(result);
   } catch (err) {
+    failuresTotal.labels(classifyError(err), 'youtube_transcript').inc();
     log('error', 'youtube transcript failed', { reqId, error: err.message, stack: err.stack });
     res.status(500).json({ error: safeError(err) });
   }
@@ -1186,6 +1611,16 @@ async function browserTranscript(reqId, url, videoId, lang) {
       };
     } finally {
       await safePageClose(page);
+      // Clean up phantom transcript session if no tabs remain
+      const ytSession = sessions.get(normalizeUserId('__yt_transcript__'));
+      if (ytSession) {
+        let totalTabs = 0;
+        for (const g of ytSession.tabGroups.values()) totalTabs += g.size;
+        if (totalTabs === 0) {
+          ytSession.context.close().catch(() => {});
+          sessions.delete(normalizeUserId('__yt_transcript__'));
+        }
+      }
     }
   });
 }
@@ -1195,16 +1630,34 @@ app.get('/health', (req, res) => {
     return res.status(503).json({ ok: false, engine: 'camoufox', recovering: true });
   }
   const running = browser !== null && (browser.isConnected?.() ?? false);
+  if (proxyPool?.canRotateSessions && !running) {
+    scheduleBrowserWarmRetry();
+    return res.status(503).json({
+      ok: false,
+      engine: 'camoufox',
+      browserConnected: false,
+      browserRunning: false,
+      warming: true,
+      ...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
+    });
+  }
   res.json({ 
     ok: true, 
     engine: 'camoufox',
     browserConnected: running,
     browserRunning: running,
     activeTabs: getTotalTabCount(),
+    activeSessions: sessions.size,
     consecutiveFailures: healthState.consecutiveNavFailures,
+    ...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
   });
 });
 
+app.get('/metrics', async (_req, res) => {
+  res.set('Content-Type', metricsRegister.contentType);
+  res.send(await metricsRegister.metrics());
+});
+
 // Create new tab
 app.post('/tabs', async (req, res) => {
   try {
@@ -1220,32 +1673,35 @@ app.post('/tabs', async (req, res) => {
       
       let totalTabs = 0;
       for (const group of session.tabGroups.values()) totalTabs += group.size;
-      if (totalTabs >= MAX_TABS_PER_SESSION) {
-        throw Object.assign(new Error('Maximum tabs per session reached'), { statusCode: 429 });
-      }
       
-      if (getTotalTabCount() >= MAX_TABS_GLOBAL) {
-        throw Object.assign(new Error('Maximum global tabs reached'), { statusCode: 429 });
+      // Recycle oldest tab when limits are reached instead of rejecting
+      if (totalTabs >= MAX_TABS_PER_SESSION || getTotalTabCount() >= MAX_TABS_GLOBAL) {
+        const recycled = await recycleOldestTab(session, req.reqId);
+        if (!recycled) {
+          throw Object.assign(new Error('Maximum tabs per session reached'), { statusCode: 429 });
+        }
       }
       
       const group = getTabGroup(session, resolvedSessionKey);
       
       const page = await session.context.newPage();
-      const tabId = crypto.randomUUID();
+      const tabId = fly.makeTabId();
       const tabState = createTabState(page);
       attachDownloadListener(tabState, tabId);
       group.set(tabId, tabState);
+      refreshActiveTabsGauge();
       
       if (url) {
         const urlErr = validateUrl(url);
         if (urlErr) throw Object.assign(new Error(urlErr), { statusCode: 400 });
-        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
+        tabState.lastRequestedUrl = url;
+        await withPageLoadDuration('open_url', () => page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
         tabState.visitedUrls.add(url);
       }
       
       log('info', 'tab created', { reqId: req.reqId, tabId, userId, sessionKey: resolvedSessionKey, url: page.url() });
       return { tabId, url: page.url() };
-    })(), HANDLER_TIMEOUT_MS, 'tab create');
+    })(), requestTimeoutMs(), 'tab create');
 
     res.json(result);
   } catch (err) {
@@ -1264,45 +1720,29 @@ app.post('/tabs/:tabId/navigate', async (req, res) => {
 
     const result = await withUserLimit(userId, () => withTimeout((async () => {
       await ensureBrowser();
+      const resolvedSessionKey = sessionKey || listItemId || 'default';
       let session = sessions.get(normalizeUserId(userId));
       let found = session && findTab(session, tabId);
       
       let tabState;
       if (!found) {
-        const resolvedSessionKey = sessionKey || listItemId || 'default';
         session = await getSession(userId);
         let sessionTabs = 0;
         for (const g of session.tabGroups.values()) sessionTabs += g.size;
         if (getTotalTabCount() >= MAX_TABS_GLOBAL || sessionTabs >= MAX_TABS_PER_SESSION) {
-          // Reuse oldest tab in session instead of rejecting
-          let oldestTab = null;
-          let oldestGroup = null;
-          let oldestTabId = null;
-          for (const [gKey, group] of session.tabGroups) {
-            for (const [tid, ts] of group) {
-              if (!oldestTab || ts.toolCalls < oldestTab.toolCalls) {
-                oldestTab = ts;
-                oldestGroup = group;
-                oldestTabId = tid;
-              }
-            }
-          }
-          if (oldestTab) {
-            tabState = oldestTab;
-            const group = getTabGroup(session, resolvedSessionKey);
-            if (oldestGroup) oldestGroup.delete(oldestTabId);
-            group.set(tabId, tabState);
-            { const _l = tabLocks.get(oldestTabId); if (_l) _l.drain(); tabLocks.delete(oldestTabId); }
-            log('info', 'tab recycled (limit reached)', { reqId: req.reqId, tabId, recycledFrom: oldestTabId, userId });
-          } else {
+          // Recycle oldest tab to free a slot, then create new page
+          const recycled = await recycleOldestTab(session, req.reqId);
+          if (!recycled) {
             throw new Error('Maximum tabs per session reached');
           }
-        } else {
+        }
+        {
           const page = await session.context.newPage();
           tabState = createTabState(page);
           attachDownloadListener(tabState, tabId, log);
           const group = getTabGroup(session, resolvedSessionKey);
           group.set(tabId, tabState);
+          refreshActiveTabsGauge();
           log('info', 'tab auto-created on navigate', { reqId: req.reqId, tabId, userId });
         }
       } else {
@@ -1311,7 +1751,7 @@ app.post('/tabs/:tabId/navigate', async (req, res) => {
       tabState.toolCalls++; tabState.consecutiveTimeouts = 0;
       
       let targetUrl = url;
-      if (macro) {
+      if (macro && macro !== '__NO__' && macro !== 'none' && macro !== 'null') {
         targetUrl = expandMacro(macro, query) || url;
       }
       
@@ -1321,9 +1761,61 @@ app.post('/tabs/:tabId/navigate', async (req, res) => {
       if (urlErr) throw new Error(urlErr);
       
       return await withTabLock(tabId, async () => {
-        await tabState.page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
-        tabState.visitedUrls.add(targetUrl);
-        tabState.lastSnapshot = null;
+        const currentSessionKey = found?.listItemId || resolvedSessionKey;
+        const isGoogleSearch = isGoogleSearchUrl(targetUrl);
+
+        const navigateCurrentPage = async () => {
+          tabState.lastRequestedUrl = targetUrl;
+          await withPageLoadDuration('navigate', () => tabState.page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }));
+          tabState.visitedUrls.add(targetUrl);
+          tabState.lastSnapshot = null;
+        };
+
+        const prewarmGoogleHome = async () => {
+          if (!isGoogleSearch || tabState.visitedUrls.has('https://www.google.com/')) return;
+          await withPageLoadDuration('navigate', () => tabState.page.goto('https://www.google.com/', { waitUntil: 'domcontentloaded', timeout: 30000 }));
+          tabState.visitedUrls.add('https://www.google.com/');
+          await tabState.page.waitForTimeout(1200);
+        };
+
+        const recreateTabOnFreshContext = async () => {
+          const previousRetryCount = tabState.googleRetryCount || 0;
+          browserRestartsTotal.labels('google_search_block').inc();
+          // Rotate at context level — destroy this user's session and create
+          // a fresh one with a new proxy session. Does NOT restart the browser.
+          const key = normalizeUserId(userId);
+          const oldSession = sessions.get(key);
+          if (oldSession) {
+            await oldSession.context.close().catch(() => {});
+            sessions.delete(key);
+          }
+          session = await getSession(userId);
+          const group = getTabGroup(session, currentSessionKey);
+          const page = await session.context.newPage();
+          tabState = createTabState(page);
+          tabState.googleRetryCount = previousRetryCount + 1;
+          attachDownloadListener(tabState, tabId, log);
+          group.set(tabId, tabState);
+          refreshActiveTabsGauge();
+        };
+
+        if (isGoogleSearch && proxyPool?.canRotateSessions) {
+          await prewarmGoogleHome();
+        }
+
+        await navigateCurrentPage();
+
+        if (isGoogleSearch && proxyPool?.canRotateSessions && await isGoogleSearchBlocked(tabState.page)) {
+          log('warn', 'google search blocked, rotating browser proxy session', {
+            reqId: req.reqId,
+            tabId,
+            url: tabState.page.url(),
+            proxySession: browserLaunchProxy?.sessionId || null,
+          });
+          await recreateTabOnFreshContext();
+          await prewarmGoogleHome();
+          await navigateCurrentPage();
+        }
         
         // For Google SERP: skip eager ref building during navigate.
         // Results render asynchronously after DOMContentLoaded — the snapshot
@@ -1332,18 +1824,22 @@ app.post('/tabs/:tabId/navigate', async (req, res) => {
           tabState.refs = new Map();
           return { ok: true, tabId, url: tabState.page.url(), refsAvailable: false, googleSerp: true };
         }
+
+        if (isGoogleSearch && await isGoogleSearchBlocked(tabState.page)) {
+          return { ok: false, tabId, url: tabState.page.url(), refsAvailable: false, googleBlocked: true };
+        }
         
         tabState.refs = await buildRefs(tabState.page);
         return { ok: true, tabId, url: tabState.page.url(), refsAvailable: tabState.refs.size > 0 };
-      });
-    })(), HANDLER_TIMEOUT_MS, 'navigate'));
+      }, requestTimeoutMs());
+    })(), requestTimeoutMs(), 'navigate'));
     
     log('info', 'navigated', { reqId: req.reqId, tabId, url: result.url });
     res.json(result);
   } catch (err) {
     log('error', 'navigate failed', { reqId: req.reqId, tabId, error: err.message });
-    const status = err.message && err.message.startsWith('Blocked URL scheme') ? 400 : 500;
-    if (status === 400) {
+    const is400 = err.message && (err.message.startsWith('Blocked URL scheme') || err.message === 'url or macro required');
+    if (is400) {
       return res.status(400).json({ error: safeError(err) });
     }
     handleRouteError(err, req, res);
@@ -1377,6 +1873,25 @@ app.get('/tabs/:tabId/snapshot', async (req, res) => {
     }
 
     const result = await withUserLimit(userId, () => withTimeout((async () => {
+      if (proxyPool?.canRotateSessions && isGoogleSearchUrl(tabState.lastRequestedUrl || '')) {
+        const blocked = await isGoogleSearchBlocked(tabState.page);
+        const unavailable = !blocked && await isGoogleUnavailable(tabState.page);
+        if (blocked || unavailable) {
+          const rotated = await rotateGoogleTab(userId, found.listItemId, req.params.tabId, tabState, blocked ? 'google_search_block_snapshot' : 'google_search_unavailable_snapshot', req.reqId);
+          if (rotated) {
+            tabState.page = rotated.tabState.page;
+            tabState.refs = rotated.tabState.refs;
+            tabState.visitedUrls = rotated.tabState.visitedUrls;
+            tabState.downloads = rotated.tabState.downloads;
+            tabState.toolCalls = rotated.tabState.toolCalls;
+            tabState.consecutiveTimeouts = rotated.tabState.consecutiveTimeouts;
+            tabState.lastSnapshot = rotated.tabState.lastSnapshot;
+            tabState.lastRequestedUrl = rotated.tabState.lastRequestedUrl;
+            tabState.googleRetryCount = rotated.tabState.googleRetryCount;
+          }
+        }
+      }
+
       const pageUrl = tabState.page.url();
       
       // Google SERP fast path — DOM extraction instead of ariaSnapshot
@@ -1402,7 +1917,7 @@ app.get('/tabs/:tabId/snapshot', async (req, res) => {
         return response;
       }
       
-      tabState.refs = await buildRefs(tabState.page);
+      tabState.refs = await refreshTabRefs(tabState, { reason: 'snapshot' });
       const ariaYaml = await getAriaSnapshot(tabState.page);
       
       let annotatedYaml = ariaYaml || '';
@@ -1458,7 +1973,7 @@ app.get('/tabs/:tabId/snapshot', async (req, res) => {
       }
 
       return response;
-    })(), HANDLER_TIMEOUT_MS, 'snapshot'));
+    })(), requestTimeoutMs(), 'snapshot'));
 
     log('info', 'snapshot', { reqId: req.reqId, tabId: req.params.tabId, url: result.url, snapshotLen: result.snapshot?.length, refsCount: result.refsCount, hasScreenshot: !!result.screenshot, truncated: result.truncated });
     res.json(result);
@@ -1576,9 +2091,7 @@ app.post('/tabs/:tabId/click', async (req, res) => {
           log('info', 'auto-refreshing refs before click', { ref, hadRefs: tabState.refs.size });
           try {
             const preClickBudget = Math.min(4000, remainingBudget());
-            const refreshPromise = buildRefs(tabState.page);
-            const refreshBudget = new Promise((_, reject) => setTimeout(() => reject(new Error('pre_click_refs_timeout')), preClickBudget));
-            tabState.refs = await Promise.race([refreshPromise, refreshBudget]);
+            tabState.refs = await refreshTabRefs(tabState, { reason: 'pre_click', timeoutMs: preClickBudget });
           } catch (e) {
             if (e.message === 'pre_click_refs_timeout' || e.message === 'buildRefs_timeout') {
               log('warn', 'pre-click buildRefs timed out, proceeding without refresh');
@@ -1618,9 +2131,7 @@ app.post('/tabs/:tabId/click', async (req, res) => {
       // If it times out, return without refs (caller's next /snapshot will rebuild them).
       const postClickBudget = Math.max(2000, remainingBudget());
       try {
-        const refsPromise = buildRefs(tabState.page);
-        const refsBudget = new Promise((_, reject) => setTimeout(() => reject(new Error('post_click_refs_timeout')), postClickBudget));
-        tabState.refs = await Promise.race([refsPromise, refsBudget]);
+        tabState.refs = await refreshTabRefs(tabState, { reason: 'post_click', timeoutMs: postClickBudget });
       } catch (e) {
         if (e.message === 'post_click_refs_timeout' || e.message === 'buildRefs_timeout') {
           log('warn', 'post-click buildRefs timed out, returning without refs', { budget: postClickBudget, elapsed: Date.now() - clickStart });
@@ -1644,7 +2155,7 @@ app.post('/tabs/:tabId/click', async (req, res) => {
         const session = sessions.get(normalizeUserId(req.body.userId));
         const found = session && findTab(session, tabId);
         if (found?.tabState?.page && !found.tabState.page.isClosed()) {
-          found.tabState.refs = await buildRefs(found.tabState.page);
+          found.tabState.refs = await refreshTabRefs(found.tabState, { reason: 'click_timeout' });
           found.tabState.lastSnapshot = null;
           return res.status(500).json({
             error: safeError(err),
@@ -1683,7 +2194,7 @@ app.post('/tabs/:tabId/type', async (req, res) => {
         let locator = refToLocator(tabState.page, ref, tabState.refs);
         if (!locator) {
           log('info', 'auto-refreshing refs before fill', { ref, hadRefs: tabState.refs.size });
-          tabState.refs = await buildRefs(tabState.page);
+          tabState.refs = await refreshTabRefs(tabState, { reason: 'type' });
           locator = refToLocator(tabState.page, ref, tabState.refs);
         }
         if (!locator) { const maxRef = tabState.refs.size > 0 ? `e${tabState.refs.size}` : 'none'; throw new StaleRefsError(ref, maxRef, tabState.refs.size); }
@@ -1701,7 +2212,7 @@ app.post('/tabs/:tabId/type', async (req, res) => {
         const session = sessions.get(normalizeUserId(req.body.userId));
         const found = session && findTab(session, tabId);
         if (found?.tabState?.page && !found.tabState.page.isClosed()) {
-          found.tabState.refs = await buildRefs(found.tabState.page);
+          found.tabState.refs = await refreshTabRefs(found.tabState, { reason: 'type_timeout' });
           found.tabState.lastSnapshot = null;
           return res.status(500).json({
             error: safeError(err),
@@ -1753,8 +2264,9 @@ app.post('/tabs/:tabId/scroll', async (req, res) => {
     const { tabState } = found;
     tabState.toolCalls++; tabState.consecutiveTimeouts = 0;
     
-    const delta = direction === 'up' ? -amount : amount;
-    await tabState.page.mouse.wheel(0, delta);
+    const isVertical = direction === 'up' || direction === 'down';
+    const delta = (direction === 'up' || direction === 'left') ? -amount : amount;
+    await tabState.page.mouse.wheel(isVertical ? 0 : delta, isVertical ? delta : 0);
     await tabState.page.waitForTimeout(300);
     
     res.json({ ok: true });
@@ -1778,7 +2290,17 @@ app.post('/tabs/:tabId/back', async (req, res) => {
     tabState.toolCalls++; tabState.consecutiveTimeouts = 0;
     
     const result = await withTabLock(tabId, async () => {
-      await tabState.page.goBack({ timeout: 10000 });
+      try {
+        await tabState.page.goBack({ timeout: 10000 });
+      } catch (navErr) {
+        // NS_BINDING_CANCELLED_OLD_LOAD: Firefox cancels the old load when going back.
+        // The navigation itself succeeded — just the prior page's load was interrupted.
+        if (navErr.message && navErr.message.includes('NS_BINDING_CANCELLED')) {
+          log('info', 'goBack cancelled old load (expected)', { reqId: req.reqId, tabId });
+        } else {
+          throw navErr;
+        }
+      }
       tabState.refs = await buildRefs(tabState.page);
       return { ok: true, url: tabState.page.url() };
     });
@@ -1906,6 +2428,7 @@ app.get('/tabs/:tabId/downloads', async (req, res) => {
 
     res.json({ tabId: req.params.tabId, downloads });
   } catch (err) {
+    failuresTotal.labels(classifyError(err), 'downloads').inc();
     log('error', 'downloads failed', { reqId: req.reqId, error: err.message });
     res.status(500).json({ error: safeError(err) });
   }
@@ -1931,6 +2454,7 @@ app.get('/tabs/:tabId/images', async (req, res) => {
 
     res.json({ tabId: req.params.tabId, images });
   } catch (err) {
+    failuresTotal.labels(classifyError(err), 'images').inc();
     log('error', 'images failed', { reqId: req.reqId, error: err.message });
     res.status(500).json({ error: safeError(err) });
   }
@@ -1999,6 +2523,7 @@ app.post('/tabs/:tabId/evaluate', express.json({ limit: '1mb' }), async (req, re
     log('info', 'evaluate', { reqId: req.reqId, tabId: req.params.tabId, userId, resultType: typeof result });
     res.json({ ok: true, result });
   } catch (err) {
+    failuresTotal.labels(classifyError(err), 'evaluate').inc();
     log('error', 'evaluate failed', { reqId: req.reqId, error: err.message });
     res.status(500).json({ error: safeError(err) });
   }
@@ -2007,17 +2532,19 @@ app.post('/tabs/:tabId/evaluate', express.json({ limit: '1mb' }), async (req, re
 // Close tab
 app.delete('/tabs/:tabId', async (req, res) => {
   try {
-    const { userId } = req.body;
+    const userId = req.query.userId || req.body?.userId;
+    if (!userId) return res.status(400).json({ error: 'userId required (query or body)' });
     const session = sessions.get(normalizeUserId(userId));
     const found = session && findTab(session, req.params.tabId);
     if (found) {
       await clearTabDownloads(found.tabState);
       await safePageClose(found.tabState.page);
       found.group.delete(req.params.tabId);
-      { const _l = tabLocks.get(req.params.tabId); if (_l) _l.drain(); tabLocks.delete(req.params.tabId); }
+      { const _l = tabLocks.get(req.params.tabId); if (_l) _l.drain(); tabLocks.delete(req.params.tabId); refreshTabLockQueueDepth(); }
       if (found.group.size === 0) {
         session.tabGroups.delete(found.listItemId);
       }
+      refreshActiveTabsGauge();
       log('info', 'tab closed', { reqId: req.reqId, tabId: req.params.tabId, userId });
     }
     res.json({ ok: true });
@@ -2030,16 +2557,23 @@ app.delete('/tabs/:tabId', async (req, res) => {
 // Close tab group
 app.delete('/tabs/group/:listItemId', async (req, res) => {
   try {
-    const { userId } = req.body;
+    const userId = req.query.userId || req.body?.userId;
+    if (!userId) return res.status(400).json({ error: 'userId required (query or body)' });
     const session = sessions.get(normalizeUserId(userId));
     const group = session?.tabGroups.get(req.params.listItemId);
     if (group) {
       for (const [tabId, tabState] of group) {
         await clearTabDownloads(tabState);
         await safePageClose(tabState.page);
-        tabLocks.delete(tabId);
+        const lock = tabLocks.get(tabId);
+        if (lock) {
+          lock.drain();
+          tabLocks.delete(tabId);
+        }
       }
       session.tabGroups.delete(req.params.listItemId);
+      refreshTabLockQueueDepth();
+      refreshActiveTabsGauge();
       log('info', 'tab group closed', { reqId: req.reqId, listItemId: req.params.listItemId, userId });
     }
     res.json({ ok: true });
@@ -2058,6 +2592,18 @@ app.delete('/sessions/:userId', async (req, res) => {
       await clearSessionDownloads(session);
       await session.context.close();
       sessions.delete(userId);
+      // Remove any lingering tab locks for the session
+      for (const [listItemId, group] of session.tabGroups) {
+        for (const tabId of group.keys()) {
+          const lock = tabLocks.get(tabId);
+          if (lock) {
+            lock.drain();
+            tabLocks.delete(tabId);
+          }
+        }
+      }
+      refreshTabLockQueueDepth();
+      refreshActiveTabsGauge();
       log('info', 'session closed', { userId });
     }
     if (sessions.size === 0) scheduleBrowserIdleShutdown();
@@ -2073,9 +2619,11 @@ setInterval(() => {
   const now = Date.now();
   for (const [userId, session] of sessions) {
     if (now - session.lastAccess > SESSION_TIMEOUT_MS) {
+      sessionsExpiredTotal.inc();
       clearSessionDownloads(session).catch(() => {});
       session.context.close().catch(() => {});
       sessions.delete(userId);
+      refreshActiveTabsGauge();
       log('info', 'session expired', { userId });
     }
   }
@@ -2083,6 +2631,7 @@ setInterval(() => {
   if (sessions.size === 0) {
     scheduleBrowserIdleShutdown();
   }
+  refreshTabLockQueueDepth();
 }, 60_000);
 
 // Per-tab inactivity reaper — close tabs idle for TAB_INACTIVITY_MS
@@ -2099,10 +2648,13 @@ setInterval(() => {
         if (tabState.toolCalls === tabState._lastReaperToolCalls) {
           const idleMs = now - tabState._lastReaperCheck;
           if (idleMs >= TAB_INACTIVITY_MS) {
+            tabsReapedTotal.inc();
             log('info', 'tab reaped (inactive)', { userId, tabId, listItemId, idleMs, toolCalls: tabState.toolCalls });
             safePageClose(tabState.page);
             group.delete(tabId);
             { const _l = tabLocks.get(tabId); if (_l) _l.drain(); tabLocks.delete(tabId); }
+            refreshTabLockQueueDepth();
+            refreshActiveTabsGauge();
           }
         } else {
           tabState._lastReaperCheck = now;
@@ -2180,26 +2732,26 @@ app.post('/tabs/open', async (req, res) => {
     
     const session = await getSession(userId);
     
-    // Check global tab limit first
-    if (getTotalTabCount() >= MAX_TABS_GLOBAL) {
-      return res.status(429).json({ error: 'Maximum global tabs reached' });
-    }
-    
+    // Recycle oldest tab when limits are reached instead of rejecting
     let totalTabs = 0;
     for (const g of session.tabGroups.values()) totalTabs += g.size;
-    if (totalTabs >= MAX_TABS_PER_SESSION) {
-      return res.status(429).json({ error: 'Maximum tabs per session reached' });
+    if (totalTabs >= MAX_TABS_PER_SESSION || getTotalTabCount() >= MAX_TABS_GLOBAL) {
+      const recycled = await recycleOldestTab(session, req.reqId);
+      if (!recycled) {
+        return res.status(429).json({ error: 'Maximum tabs per session reached' });
+      }
     }
     
     const group = getTabGroup(session, listItemId);
     
     const page = await session.context.newPage();
-    const tabId = crypto.randomUUID();
+    const tabId = fly.makeTabId();
     const tabState = createTabState(page);
     attachDownloadListener(tabState, tabId, log);
     group.set(tabId, tabState);
+    refreshActiveTabsGauge();
     
-    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
+    await withPageLoadDuration('open_url', () => page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
     tabState.visitedUrls.add(url);
     
     log('info', 'openclaw tab opened', { reqId: req.reqId, tabId, url: page.url() });
@@ -2222,6 +2774,7 @@ app.post('/start', async (req, res) => {
     await ensureBrowser();
     res.json({ ok: true, profile: 'camoufox' });
   } catch (err) {
+    failuresTotal.labels('browser_launch', 'start').inc();
     res.status(500).json({ ok: false, error: safeError(err) });
   }
 });
@@ -2242,7 +2795,21 @@ app.post('/stop', async (req, res) => {
       cleanupTasks.push(clearSessionDownloads(session));
     }
     await Promise.all(cleanupTasks);
+    for (const session of sessions.values()) {
+      for (const [, group] of session.tabGroups) {
+        for (const tabId of group.keys()) {
+          const lock = tabLocks.get(tabId);
+          if (lock) {
+            lock.drain();
+            tabLocks.delete(tabId);
+          }
+        }
+      }
+    }
+    tabLocks.clear();
     sessions.clear();
+    refreshActiveTabsGauge();
+    refreshTabLockQueueDepth();
     res.json({ ok: true, stopped: true, profile: 'camoufox' });
   } catch (err) {
     res.status(500).json({ ok: false, error: safeError(err) });
@@ -2273,7 +2840,7 @@ app.post('/navigate', async (req, res) => {
     tabState.toolCalls++; tabState.consecutiveTimeouts = 0;
     
     const result = await withTabLock(targetId, async () => {
-      await tabState.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
+      await withPageLoadDuration('navigate', () => tabState.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }));
       tabState.visitedUrls.add(url);
       tabState.lastSnapshot = null;
       
@@ -2510,8 +3077,9 @@ app.post('/act', async (req, res) => {
             if (!locator) { const maxRef = tabState.refs.size > 0 ? `e${tabState.refs.size}` : 'none'; throw new StaleRefsError(ref, maxRef, tabState.refs.size); }
             await locator.scrollIntoViewIfNeeded({ timeout: 5000 });
           } else {
-            const delta = direction === 'up' ? -amount : amount;
-            await tabState.page.mouse.wheel(0, delta);
+            const isVertical = direction === 'up' || direction === 'down';
+            const delta = (direction === 'up' || direction === 'left') ? -amount : amount;
+            await tabState.page.mouse.wheel(isVertical ? 0 : delta, isVertical ? delta : 0);
           }
           await tabState.page.waitForTimeout(300);
           return { ok: true, targetId };
@@ -2611,6 +3179,7 @@ setInterval(async () => {
     await testContext.close();
     healthState.lastSuccessfulNav = Date.now();
   } catch (err) {
+    failuresTotal.labels('health_probe', 'internal').inc();
     log('warn', 'health probe failed', { error: err.message, timeSinceSuccessMs: timeSinceSuccess });
     if (testContext) await testContext.close().catch(() => {});
     restartBrowser('health probe failed').catch(() => {});
@@ -2641,6 +3210,7 @@ async function gracefulShutdown(signal) {
   forceTimeout.unref();
 
   server.close();
+  stopMemoryReporter();
 
   for (const [userId, session] of sessions) {
     await session.context.close().catch(() => {});
@@ -2652,17 +3222,32 @@ async function gracefulShutdown(signal) {
 process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
 process.on('SIGINT', () => gracefulShutdown('SIGINT'));
 
+// Idle self-shutdown REMOVED — it was racing with min_machines_running=2
+// and stopping machines that Fly couldn't auto-restart fast enough, leaving
+// only 1 machine to handle all browser traffic (causing timeouts for users).
+// Fly's auto_stop_machines=false + min_machines_running=2 handles scaling.
+
 const PORT = CONFIG.port;
 const server = app.listen(PORT, async () => {
-  log('info', 'server started', { port: PORT, pid: process.pid, nodeVersion: process.version });
+  startMemoryReporter();
+  refreshActiveTabsGauge();
+  refreshTabLockQueueDepth();
+  if (FLY_MACHINE_ID) {
+    log('info', 'server started (fly)', { port: PORT, pid: process.pid, machineId: FLY_MACHINE_ID, nodeVersion: process.version });
+  } else {
+    log('info', 'server started', { port: PORT, pid: process.pid, nodeVersion: process.version });
+  }
   // Pre-warm browser so first request doesn't eat a 6-7s cold start
   try {
     const start = Date.now();
     await ensureBrowser();
     log('info', 'browser pre-warmed', { ms: Date.now() - start });
+    scheduleBrowserIdleShutdown();
   } catch (err) {
-    log('error', 'browser pre-warm failed (will retry on first request)', { error: err.message });
+    log('error', 'browser pre-warm failed (will retry in background)', { error: err.message });
+    scheduleBrowserWarmRetry();
   }
+  // Idle self-shutdown removed — Fly manages machine lifecycle via fly.toml.
 });
 
 server.on('error', (err) => {