browserclaw 0.4.0 → 0.4.1

package/README.md

@@ -2,6 +2,8 @@
 
 <p align="center">
   <a href="https://www.npmjs.com/package/browserclaw"><img src="https://img.shields.io/npm/v/browserclaw.svg" alt="npm version" /></a>
+  <a href="https://www.npmjs.com/package/browserclaw"><img src="https://img.shields.io/npm/dw/browserclaw" alt="npm downloads" /></a>
+  <a href="https://github.com/idan-rubin/browserclaw/stargazers"><img src="https://img.shields.io/github/stars/idan-rubin/browserclaw" alt="GitHub stars" /></a>
   <a href="./LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="License: MIT" /></a>
 </p>
 

package/dist/index.cjs

@@ -349,39 +349,109 @@ function resolveUserDataDir(profileName) {
   const configDir = process.env.XDG_CONFIG_HOME ?? path__default.default.join(os__default.default.homedir(), ".config");
   return path__default.default.join(configDir, "browserclaw", "profiles", profileName, "user-data");
 }
-async function isChromeReachable(cdpUrl, timeoutMs = 500, authToken) {
-  const ctrl = new AbortController();
-  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+function isWebSocketUrl(url) {
   try {
-    const headers = {};
-    if (authToken) headers["Authorization"] = `Bearer ${authToken}`;
-    const res = await fetch(`${cdpUrl.replace(/\/+$/, "")}/json/version`, { signal: ctrl.signal, headers });
-    if (!res.ok) return false;
-    const data = await res.json();
-    return data != null && typeof data === "object";
+    const parsed = new URL(url);
+    return parsed.protocol === "ws:" || parsed.protocol === "wss:";
   } catch {
     return false;
-  } finally {
-    clearTimeout(t);
   }
 }
-async function getChromeWebSocketUrl(cdpUrl, timeoutMs = 500, authToken) {
+function isLoopbackHost(hostname) {
+  return hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1" || hostname === "[::1]";
+}
+function normalizeCdpWsUrl(wsUrl, cdpUrl) {
+  const ws = new URL(wsUrl);
+  const cdp = new URL(cdpUrl);
+  const isWildcardBind = ws.hostname === "0.0.0.0" || ws.hostname === "[::]";
+  if ((isLoopbackHost(ws.hostname) || isWildcardBind) && !isLoopbackHost(cdp.hostname)) {
+    ws.hostname = cdp.hostname;
+    const cdpPort = cdp.port || (cdp.protocol === "https:" ? "443" : "80");
+    ws.port = cdpPort;
+    ws.protocol = cdp.protocol === "https:" ? "wss:" : "ws:";
+  }
+  if (cdp.protocol === "https:" && ws.protocol === "ws:") ws.protocol = "wss:";
+  if (!ws.username && !ws.password && (cdp.username || cdp.password)) {
+    ws.username = cdp.username;
+    ws.password = cdp.password;
+  }
+  for (const [key, value] of cdp.searchParams.entries()) {
+    if (!ws.searchParams.has(key)) ws.searchParams.append(key, value);
+  }
+  return ws.toString();
+}
+function normalizeCdpHttpBaseForJsonEndpoints(cdpUrl) {
+  try {
+    const url = new URL(cdpUrl);
+    if (url.protocol === "ws:") url.protocol = "http:";
+    else if (url.protocol === "wss:") url.protocol = "https:";
+    url.pathname = url.pathname.replace(/\/devtools\/browser\/.*$/, "");
+    url.pathname = url.pathname.replace(/\/cdp$/, "");
+    return url.toString().replace(/\/$/, "");
+  } catch {
+    return cdpUrl.replace(/^ws:/, "http:").replace(/^wss:/, "https:").replace(/\/devtools\/browser\/.*$/, "").replace(/\/cdp$/, "").replace(/\/$/, "");
+  }
+}
+function appendCdpPath(cdpUrl, cdpPath) {
+  const url = new URL(cdpUrl);
+  url.pathname = `${url.pathname.replace(/\/$/, "")}${cdpPath.startsWith("/") ? cdpPath : `/${cdpPath}`}`;
+  return url.toString();
+}
+async function canOpenWebSocket(url, timeoutMs) {
+  return new Promise((resolve2) => {
+    let settled = false;
+    const finish = (value) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      try {
+        ws.close();
+      } catch {
+      }
+      resolve2(value);
+    };
+    const timer = setTimeout(() => finish(false), Math.max(50, timeoutMs + 25));
+    let ws;
+    try {
+      ws = new WebSocket(url);
+    } catch {
+      finish(false);
+      return;
+    }
+    ws.onopen = () => finish(true);
+    ws.onerror = () => finish(false);
+  });
+}
+async function fetchChromeVersion(cdpUrl, timeoutMs = 500, authToken) {
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), timeoutMs);
   try {
+    const httpBase = isWebSocketUrl(cdpUrl) ? normalizeCdpHttpBaseForJsonEndpoints(cdpUrl) : cdpUrl;
     const headers = {};
     if (authToken) headers["Authorization"] = `Bearer ${authToken}`;
-    const res = await fetch(`${cdpUrl.replace(/\/+$/, "")}/json/version`, { signal: ctrl.signal, headers });
+    const res = await fetch(appendCdpPath(httpBase, "/json/version"), { signal: ctrl.signal, headers });
     if (!res.ok) return null;
     const data = await res.json();
     if (!data || typeof data !== "object") return null;
-    return String(data?.webSocketDebuggerUrl ?? "").trim() || null;
+    return data;
   } catch {
     return null;
   } finally {
     clearTimeout(t);
   }
 }
+async function isChromeReachable(cdpUrl, timeoutMs = 500, authToken) {
+  if (isWebSocketUrl(cdpUrl)) return await canOpenWebSocket(cdpUrl, timeoutMs);
+  const version = await fetchChromeVersion(cdpUrl, timeoutMs, authToken);
+  return Boolean(version);
+}
+async function getChromeWebSocketUrl(cdpUrl, timeoutMs = 500, authToken) {
+  if (isWebSocketUrl(cdpUrl)) return cdpUrl;
+  const version = await fetchChromeVersion(cdpUrl, timeoutMs, authToken);
+  const wsUrl = String(version?.webSocketDebuggerUrl ?? "").trim();
+  if (!wsUrl) return null;
+  return normalizeCdpWsUrl(wsUrl, cdpUrl);
+}
 async function isChromeCdpReady(cdpUrl, timeoutMs = 500, handshakeTimeoutMs = 800) {
   const wsUrl = await getChromeWebSocketUrl(cdpUrl, timeoutMs);
   if (!wsUrl) return false;
@@ -1267,6 +1337,20 @@ function withBrowserNavigationPolicy(ssrfPolicy) {
 }
 var NETWORK_NAVIGATION_PROTOCOLS = /* @__PURE__ */ new Set(["http:", "https:"]);
 var SAFE_NON_NETWORK_URLS = /* @__PURE__ */ new Set(["about:blank"]);
+var PROXY_ENV_KEYS = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"];
+function isAllowedNonNetworkNavigationUrl(parsed) {
+  return SAFE_NON_NETWORK_URLS.has(parsed.href);
+}
+function isPrivateNetworkAllowedByPolicy(policy) {
+  return policy?.dangerouslyAllowPrivateNetwork === true || policy?.allowPrivateNetwork === true;
+}
+function hasProxyEnvConfigured(env = process.env) {
+  for (const key of PROXY_ENV_KEYS) {
+    const value = env[key];
+    if (typeof value === "string" && value.trim().length > 0) return true;
+  }
+  return false;
+}
 async function assertBrowserNavigationAllowed(opts) {
   const rawUrl = String(opts.url ?? "").trim();
   let parsed;
@@ -1276,9 +1360,14 @@ async function assertBrowserNavigationAllowed(opts) {
     throw new InvalidBrowserNavigationUrlError(`Invalid URL: "${rawUrl}"`);
   }
   if (!NETWORK_NAVIGATION_PROTOCOLS.has(parsed.protocol)) {
-    if (SAFE_NON_NETWORK_URLS.has(parsed.href)) return;
+    if (isAllowedNonNetworkNavigationUrl(parsed)) return;
     throw new InvalidBrowserNavigationUrlError(`Navigation blocked: unsupported protocol "${parsed.protocol}"`);
   }
+  if (hasProxyEnvConfigured() && !isPrivateNetworkAllowedByPolicy(opts.ssrfPolicy)) {
+    throw new InvalidBrowserNavigationUrlError(
+      "Navigation blocked: strict browser SSRF policy cannot be enforced while env proxy variables are set"
+    );
+  }
   const policy = opts.ssrfPolicy;
   if (policy?.dangerouslyAllowPrivateNetwork ?? policy?.allowPrivateNetwork ?? true) return;
   const allowedHostnames = [
@@ -1483,10 +1572,24 @@ async function assertBrowserNavigationResultAllowed(opts) {
   } catch {
     return;
   }
-  if (NETWORK_NAVIGATION_PROTOCOLS.has(parsed.protocol) || SAFE_NON_NETWORK_URLS.has(parsed.href)) {
+  if (NETWORK_NAVIGATION_PROTOCOLS.has(parsed.protocol) || isAllowedNonNetworkNavigationUrl(parsed)) {
     await assertBrowserNavigationAllowed(opts);
   }
 }
+async function assertBrowserNavigationRedirectChainAllowed(opts) {
+  const chain = [];
+  let current = opts.request ?? null;
+  while (current) {
+    chain.push(current.url());
+    current = current.redirectedFrom();
+  }
+  for (const url of [...chain].reverse()) {
+    await assertBrowserNavigationAllowed({ url, lookupFn: opts.lookupFn, ssrfPolicy: opts.ssrfPolicy });
+  }
+}
+function requiresInspectableBrowserNavigationRedirects(ssrfPolicy) {
+  return !isPrivateNetworkAllowedByPolicy(ssrfPolicy);
+}
 
 // src/actions/interaction.ts
 async function clickViaPlaywright(opts) {
@@ -1691,7 +1794,8 @@ async function navigateViaPlaywright(opts) {
   await assertBrowserNavigationAllowed({ url, ssrfPolicy: policy });
   const page = await getPageForTargetId({ cdpUrl: opts.cdpUrl, targetId: opts.targetId });
   ensurePageState(page);
-  await page.goto(url, { timeout: normalizeTimeoutMs(opts.timeoutMs, 2e4) });
+  const response = await page.goto(url, { timeout: normalizeTimeoutMs(opts.timeoutMs, 2e4) });
+  await assertBrowserNavigationRedirectChainAllowed({ request: response?.request(), ...withBrowserNavigationPolicy(policy) });
   const finalUrl = page.url();
   await assertBrowserNavigationResultAllowed({ url: finalUrl, ssrfPolicy: policy });
   return { url: finalUrl };
@@ -1722,7 +1826,9 @@ async function createPageViaPlaywright(opts) {
   const page = await context.newPage();
   ensurePageState(page);
   if (targetUrl !== "about:blank") {
-    await page.goto(targetUrl, { timeout: normalizeTimeoutMs(void 0, 2e4) });
+    const navigationPolicy = withBrowserNavigationPolicy(policy);
+    const response = await page.goto(targetUrl, { timeout: normalizeTimeoutMs(void 0, 2e4) });
+    await assertBrowserNavigationRedirectChainAllowed({ request: response?.request(), ...navigationPolicy });
     await assertBrowserNavigationResultAllowed({ url: page.url(), ssrfPolicy: policy });
   }
   const tid = await pageTargetId(page).catch(() => null);
@@ -3252,7 +3358,13 @@ exports.BrowserClaw = BrowserClaw;
 exports.CrawlPage = CrawlPage;
 exports.InvalidBrowserNavigationUrlError = InvalidBrowserNavigationUrlError;
 exports.assertBrowserNavigationAllowed = assertBrowserNavigationAllowed;
+exports.assertBrowserNavigationRedirectChainAllowed = assertBrowserNavigationRedirectChainAllowed;
+exports.assertBrowserNavigationResultAllowed = assertBrowserNavigationResultAllowed;
+exports.getChromeWebSocketUrl = getChromeWebSocketUrl;
 exports.isChromeCdpReady = isChromeCdpReady;
+exports.isChromeReachable = isChromeReachable;
+exports.normalizeCdpHttpBaseForJsonEndpoints = normalizeCdpHttpBaseForJsonEndpoints;
+exports.requiresInspectableBrowserNavigationRedirects = requiresInspectableBrowserNavigationRedirects;
 exports.withBrowserNavigationPolicy = withBrowserNavigationPolicy;
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map