@ogulcancelik/pi-web-browse 1.0.3 → 1.0.5

package/README.md

@@ -10,7 +10,7 @@ Web search and content extraction skill for [pi](https://github.com/badlogic/pi-
 - 🌐 **Page Fetching** - Extract readable content from any URL
 - 🤖 **Bot Protection Bypass** - Handles JS challenges, Cloudflare, etc.
 - 🚀 **Persistent Daemon** - Warm browser session for fast subsequent requests
-- 🖥️ **Cross-Platform** - Auto-detects Brave, Chrome, Edge, Chromium
+- 🖥️ **Cross-Platform** - Auto-detects Chrome, Brave, Edge, Chromium
 
 ## Install
 
@@ -48,25 +48,29 @@ Environment variables (all optional):
 
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `WEB_BROWSE_BROWSER_BIN` | Browser binary path | Auto-detected |
-| `WEB_BROWSE_USER_AGENT` | User-Agent string | Chrome on Windows |
+| `WEB_BROWSE_BROWSER_BIN` | Browser binary path | Auto-detected, prefers Chrome |
+| `WEB_BROWSE_USER_AGENT` | Override User-Agent string | Auto-derived from detected browser + OS |
 | `WEB_BROWSE_DAEMON_PORT` | Daemon HTTP port | 9377 |
 | `WEB_BROWSE_CDP_PORT` | Chrome DevTools port | 9225 |
 | `WEB_BROWSE_DEBUG_DUMP` | Save debug files on failure | off |
 
+By default, the hidden profile is browser-specific (for example `~/.config/web-browse-cdp-profile-chrome` or `~/.config/web-browse-cdp-profile-brave`) so updates do not try to reuse the same hidden profile across different Chromium-family browsers.
+
 ## Browser Detection
 
-The skill auto-detects browsers in common locations:
+The skill auto-detects browsers in common locations and now prefers Chrome first, because Google Search is less likely to challenge headless Chrome than headless Brave on some setups.
+
+- **Linux:** google-chrome, google-chrome-stable, brave, brave-browser, chromium (from PATH)
+- **macOS:** Google Chrome, Google Chrome Canary, Brave Browser, Edge, Chromium (in /Applications)
+- **Windows:** Chrome, Brave, Edge, Chromium (Program Files, LocalAppData)
 
-- **Linux:** brave, brave-browser, google-chrome, chromium (from PATH)
-- **macOS:** Brave Browser, Google Chrome, Chromium, Edge (in /Applications)
-- **Windows:** Brave, Chrome, Edge, Chromium (Program Files, LocalAppData)
+If you want Brave anyway, set `WEB_BROWSE_BROWSER_BIN` or pass `--browser-bin`.
 
 ## How It Works
 
-1. **Search** - Uses Google via headless browser (falls back to DuckDuckGo if blocked)
-2. **Fetch** - Opens URL in headless Chromium, waits for JS, extracts readable content
-3. **Daemon** - Keeps a warm browser session for speed + bot-protection bypass
+1. **Search** - Uses Google via a persistent headless browser session. If Google blocks the request, it fails fast and falls back to DuckDuckGo.
+2. **Fetch** - Opens URLs in the same hidden browser session, waits for JS, and extracts readable content.
+3. **Daemon** - Keeps a warm, browser-specific hidden profile/session alive for speed and bot-protection resilience without touching your normal browser profile.
 
 ## License
 

package/lib/browser-bin.js

@@ -7,34 +7,37 @@ const IS_MACOS = PLATFORM === "darwin";
 const IS_WINDOWS = PLATFORM === "win32";
 
 // macOS .app bundle paths (checked as absolute paths)
+// Prefer Chrome first: on some setups Google Search is less likely to challenge headless Chrome than headless Brave.
 const MACOS_BROWSER_PATHS = [
-  "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
   "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
   "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
-  "/Applications/Chromium.app/Contents/MacOS/Chromium",
+  "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
   "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
+  "/Applications/Chromium.app/Contents/MacOS/Chromium",
 ];
 
 // Linux binary names (searched on PATH)
+// Prefer Chrome first: on some setups Google Search is less likely to challenge headless Chrome than headless Brave.
 const LINUX_BROWSER_NAMES = [
-  "brave",
-  "brave-browser",
   "google-chrome",
   "google-chrome-stable",
+  "brave",
+  "brave-browser",
   "chromium",
   "chromium-browser",
 ];
 
 // Windows browser paths (common install locations)
+// Prefer Chrome first: on some setups Google Search is less likely to challenge headless Chrome than headless Brave.
 const WINDOWS_BROWSER_PATHS = [
-  // Brave
-  join(process.env.LOCALAPPDATA || "", "BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
-  join(process.env.PROGRAMFILES || "", "BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
-  join(process.env["PROGRAMFILES(X86)"] || "", "BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
   // Chrome
   join(process.env.LOCALAPPDATA || "", "Google", "Chrome", "Application", "chrome.exe"),
   join(process.env.PROGRAMFILES || "", "Google", "Chrome", "Application", "chrome.exe"),
   join(process.env["PROGRAMFILES(X86)"] || "", "Google", "Chrome", "Application", "chrome.exe"),
+  // Brave
+  join(process.env.LOCALAPPDATA || "", "BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
+  join(process.env.PROGRAMFILES || "", "BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
+  join(process.env["PROGRAMFILES(X86)"] || "", "BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
   // Edge (comes with Windows 10/11)
   join(process.env.PROGRAMFILES || "", "Microsoft", "Edge", "Application", "msedge.exe"),
   join(process.env["PROGRAMFILES(X86)"] || "", "Microsoft", "Edge", "Application", "msedge.exe"),

package/lib/cdp.js

@@ -85,27 +85,23 @@ async function chooseAvailablePort(preferredPort) {
   return await getEphemeralPort();
 }
 
-export async function startBrowserForCdp(preferredPort, profileDir, browserBin = null, spawnedProcessGroupPids = null) {
+export async function startBrowserForCdp(
+  preferredPort,
+  profileDir,
+  browserBin = null,
+  spawnedProcessGroupPids = null,
+  extraArgs = [],
+) {
   const bin = resolveBrowserBin(browserBin);
   const port = await chooseAvailablePort(preferredPort);
 
   // OS-specific headless flags
   let headlessArgs;
   if (IS_MACOS || IS_WINDOWS) {
-    // macOS and Windows: use standard headless mode.
-    // --headless=new injects "HeadlessChrome" into the UA string which is
-    // trivially detected by Google (results in /sorry/ CAPTCHA).  Override
-    // the UA to look like a normal browser.
-    // On Linux this isn't needed: --ozone-platform=headless runs a full
-    // browser with a normal UA (no "Headless" marker).
-    const uaPlatform = IS_MACOS
-      ? "(Macintosh; Intel Mac OS X 10_15_7)"
-      : "(Windows NT 10.0; Win64; x64)";
-    const HEADLESS_UA = `Mozilla/5.0 ${uaPlatform} AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36`;
+    // macOS and Windows: use standard headless mode
     headlessArgs = [
       "--headless=new",
       "--window-size=1280,720",
-      `--user-agent=${HEADLESS_UA}`,
     ];
   } else {
     // Linux: use ozone headless platform (Wayland/X11 independent)
@@ -127,6 +123,7 @@ export async function startBrowserForCdp(preferredPort, profileDir, browserBin =
     "--disable-backgrounding-occluded-windows",
     "--disable-renderer-backgrounding",
 
+    ...extraArgs,
     `--remote-debugging-port=${port}`,
     "--remote-debugging-address=127.0.0.1",
     `--user-data-dir=${profileDir}`,

package/lib/daemon-client.js

@@ -11,9 +11,8 @@ export async function checkDaemonHealth({ daemonUrl, timeoutMs = 600 } = {}) {
     const response = await fetch(`${daemonUrl}/health`, { signal: controller.signal });
     clearTimeout(timeout);
 
-    if (!response.ok) return null;
     const payload = await response.json().catch(() => null);
-    return payload && payload.status === "ok" ? payload : null;
+    return payload && typeof payload === "object" && typeof payload.status === "string" ? payload : null;
   } catch {
     return null;
   }
@@ -67,8 +66,8 @@ export async function startDaemonInBackground({
 
   child.unref();
 
-  // Wait up to ~5s for daemon to become healthy.
-  for (let i = 0; i < 20; i += 1) {
+  // Wait up to ~12s for daemon to start responding. Browser startup can be slow.
+  for (let i = 0; i < 48; i += 1) {
     await new Promise((r) => setTimeout(r, 250));
     const health = await checkDaemonHealth({ daemonUrl, timeoutMs: 800 });
     if (health) return health;
@@ -77,15 +76,30 @@ export async function startDaemonInBackground({
   throw new Error(`daemon failed to start on ${daemonUrl}`);
 }
 
+function daemonMatchesExpectedConfig(health, { expectedProfileDir = null, expectedBrowserKind = null } = {}) {
+  if (!health) return false;
+  if (expectedProfileDir && health.profileDir && health.profileDir !== expectedProfileDir) return false;
+  if (expectedBrowserKind && health.browserKind && health.browserKind !== expectedBrowserKind) return false;
+  return true;
+}
+
 export async function ensureDaemonRunning({
   scriptPath,
   daemonUrl,
   daemonPidFile,
   forwardedArgs = [],
   env = process.env,
+  expectedProfileDir = null,
+  expectedBrowserKind = null,
 } = {}) {
   const health = await checkDaemonHealth({ daemonUrl });
-  if (health) return health;
+  if (health && daemonMatchesExpectedConfig(health, { expectedProfileDir, expectedBrowserKind })) {
+    return health;
+  }
+
+  if (health) {
+    await stopDaemon({ daemonUrl, daemonPidFile }).catch(() => {});
+  }
 
   // If a stale PID file exists, ignore it; health check is the source of truth.
   return await startDaemonInBackground({ scriptPath, daemonUrl, daemonPidFile, forwardedArgs, env });
@@ -115,7 +129,7 @@ export async function stopDaemon({ daemonUrl, daemonPidFile } = {}) {
   return { status: "stopping", pid };
 }
 
-export async function sendDaemonCommand({ daemonUrl, command, payload } = {}) {
+export async function sendDaemonCommand({ daemonUrl, command, payload, timeoutMs = 180000 } = {}) {
   if (!daemonUrl) throw new Error("sendDaemonCommand requires daemonUrl");
   if (!command) throw new Error("sendDaemonCommand requires command");
 
@@ -123,7 +137,7 @@ export async function sendDaemonCommand({ daemonUrl, command, payload } = {}) {
     method: "POST",
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify({ command, payload }),
-    signal: AbortSignal.timeout(120000),
+    signal: AbortSignal.timeout(timeoutMs),
   });
 
   const json = await response.json().catch(() => null);