browserless 14.1.1 → 14.1.3

package/index.d.ts

@@ -40,7 +40,10 @@ export interface Context {
   text: (url: string, opts?: GotoOptions) => Promise<string>
   getDevice: (deviceName: string) => Viewport | undefined
   report: (opts?: { benchmark?: boolean }) => Promise<HardwareInfo>
-  destroyContext: (opts?: { force?: boolean }) => Promise<void>
+  destroyContext: (opts?: {
+    /** @deprecated A destroyed context never retries, so `force` has no effect. */
+    force?: boolean
+  }) => Promise<void>
   withPage: <T>(fn: (page: Page, goto: unknown) => Promise<T>, opts?: { timeout?: number, keepPage?: boolean }) => Promise<T>
 }
 

package/package.json

@@ -2,7 +2,7 @@
   "name": "browserless",
   "description": "The headless Chrome/Chromium driver on top of Puppeteer. Take screenshots, generate PDFs, extract text and HTML with a production-ready API.",
   "homepage": "https://browserless.js.org",
-  "version": "14.1.1",
+  "version": "14.1.3",
   "types": "index.d.ts",
   "main": "src/index.js",
   "author": {
@@ -36,14 +36,15 @@
   ],
   "dependencies": {
     "@browserless/errors": "14.0.22",
-    "@browserless/goto": "14.1.1",
-    "@browserless/pdf": "14.1.1",
-    "@browserless/screenshot": "14.1.1",
+    "@browserless/goto": "14.1.2",
+    "@browserless/pdf": "14.1.2",
+    "@browserless/screenshot": "14.1.2",
     "debug-logfmt": "~1.4.15",
     "kill-process-group": "~1.0.16",
     "p-reflect": "~2.1.0",
     "p-retry": "~4.6.1",
     "p-timeout": "~4.1.0",
+    "proxy-chain": "~3.0.1",
     "require-one-of": "~1.0.25",
     "superlock": "~1.4.2"
   },
@@ -69,5 +70,5 @@
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "0c8d09fa9aa1d210a9a3e1906048909b5bd102b6"
+  "gitHead": "c45e62a88191f36685a98af337762bc0c7a72953"
 }

package/src/index.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { ensureError, browserTimeout } = require('@browserless/errors')
+const { ensureError, browserTimeout, contextDisconnected } = require('@browserless/errors')
 const createScreenshot = require('@browserless/screenshot')
 const debug = require('debug-logfmt')('browserless')
 const createGoto = require('@browserless/goto')
@@ -12,6 +12,7 @@ const pRetry = require('p-retry')
 
 const { AbortError } = pRetry
 
+const { forwardProxy, closeForwarder } = require('./proxy-forwarder')
 const driver = require('./driver')
 
 // Chrome can stop answering CDP while `connected` stays true, so the
@@ -33,9 +34,11 @@ module.exports = ({ timeout: globalTimeout = 30000, ...launchOpts } = {}) => {
   const { defaultViewport } = goto
 
   let isClosed = false
+  const forwarders = new Set()
 
-  const close = opts => {
+  const close = async opts => {
     isClosed = true
+    await Promise.all([...forwarders].map(closeForwarder))
     return browserProcessPromise.then(browserProcess => driver.close(browserProcess, opts))
   }
 
@@ -108,11 +111,21 @@ module.exports = ({ timeout: globalTimeout = 30000, ...launchOpts } = {}) => {
   }
 
   const createContext = async ({ retry = 2, timeout: contextTimeout, ...contextOpts } = {}) => {
-    let _contextPromise = createBrowserContext(contextOpts)
-    let isDestroyedForced = false
+    let forwardedContextOpts
+    const forwardContextOpts = () => {
+      forwardedContextOpts = isClosed
+        ? Promise.resolve(contextOpts)
+        : forwardProxy(contextOpts, { mode: launchOpts.mode })
+      forwarders.add(forwardedContextOpts)
+      return forwardedContextOpts
+    }
+    const newBrowserContext = () =>
+      (forwardedContextOpts ?? forwardContextOpts()).then(createBrowserContext)
+    let _contextPromise = newBrowserContext()
+    let destroyCount = 0
     const pageMetadata = new WeakMap()
 
-    const getBrowserContext = () => _contextPromise
+    const getBrowserContext = () => (_contextPromise ??= newBrowserContext())
 
     const createPage = async name => {
       const duration = debug.duration('createPage')
@@ -144,6 +157,8 @@ module.exports = ({ timeout: globalTimeout = 30000, ...launchOpts } = {}) => {
 
       return async (...args) => {
         let isRejected = false
+        const destroysAtStart = destroyCount
+        const isAbandoned = () => isClosed || destroyCount !== destroysAtStart
 
         async function run () {
           let page
@@ -162,9 +177,11 @@ module.exports = ({ timeout: globalTimeout = 30000, ...launchOpts } = {}) => {
           }
 
           try {
+            if (isAbandoned()) throw contextDisconnected()
             page = await createPage(name)
             closePageTimeout = startCloseTimeout()
             const value = await fn(page, goto)(...args)
+            if (isAbandoned()) throw contextDisconnected()
             // A timeout already rejected the caller, who never received this
             // page. Retaining it would clear the in-call watchdog and arm a
             // new one with nobody holding the page.
@@ -190,17 +207,18 @@ module.exports = ({ timeout: globalTimeout = 30000, ...launchOpts } = {}) => {
             onFailedAttempt: async error => {
               debug('onFailedAttempt', { name: error.name, code: error.code, isRejected })
               if (error.name === 'AbortError') throw error
-              if (isRejected || isDestroyedForced) throw new AbortError()
+              if (isRejected) throw new AbortError()
               const isRetryable =
                 error.code === 'EBRWSRCONTEXTCONNRESET' || error.code === 'EPROTOCOL'
               if (!isRetryable) throw error
+              if (isAbandoned()) throw new AbortError()
               const { message, attemptNumber, retriesLeft } = error
               // Every attempt builds its own page, so both codes are retryable
               // inside the existing context: `EBRWSRCONTEXTCONNRESET` covers a
               // closed page, a detached frame and a destroyed execution context,
               // which a fresh page recovers from.
               const previousContextPromise = _contextPromise
-              _contextPromise = createBrowserContext(contextOpts)
+              _contextPromise = newBrowserContext()
               await pReflect(previousContextPromise.then(ctx => ctx.close()))
               debug('retry', { attemptNumber, retriesLeft, message })
             }
@@ -236,15 +254,25 @@ module.exports = ({ timeout: globalTimeout = 30000, ...launchOpts } = {}) => {
       )
     }
 
-    const destroyContext = async ({ force = false } = {}) => {
-      if (force) isDestroyedForced = true
+    const destroyContext = async () => {
+      destroyCount++
+      const closingContextPromise = _contextPromise
+      const closingContextOpts = forwardedContextOpts
+      _contextPromise = undefined
+      forwardedContextOpts = undefined
+      if (!closingContextPromise) return
 
-      const [browserProcess, browserContext] = await Promise.all([
-        getBrowser(),
-        getBrowserContext()
-      ])
-      await pReflect(browserContext.close())
-      debug('destroyContext', { pid: driver.pid(browserProcess), force, id: browserContext.id })
+      try {
+        const [browserProcess, browserContext] = await Promise.all([
+          getBrowser(),
+          closingContextPromise
+        ])
+        await pReflect(browserContext.close())
+        debug('destroyContext', { pid: driver.pid(browserProcess), id: browserContext.id })
+      } finally {
+        forwarders.delete(closingContextOpts)
+        await closeForwarder(closingContextOpts)
+      }
     }
 
     return {

package/src/proxy-forwarder.js

@@ -0,0 +1,50 @@
+'use strict'
+
+const { anonymizeProxy, closeAnonymizedProxy } = require('proxy-chain')
+
+const REDACTED = '***'
+const DEFAULT_SCHEME = 'http://'
+const CONNECT_MODE_ERROR =
+  "A proxyServer with credentials needs `mode: 'launch'`: a connected browser cannot reach the local forwarder"
+
+const forwardedProxyServers = new Set()
+
+const parseProxyServer = proxyServer =>
+  URL.parse(proxyServer.includes('://') ? proxyServer : `${DEFAULT_SCHEME}${proxyServer}`)
+
+const hasCredentials = proxyUrl => Boolean(proxyUrl?.username || proxyUrl?.password)
+
+const redactSecrets = (text, secrets) =>
+  secrets.reduce((redacted, secret) => redacted.replaceAll(secret, REDACTED), text)
+
+const withoutCredentials = (error, proxyUrl) => {
+  const redactedUrl = new URL(proxyUrl)
+  if (proxyUrl.username) redactedUrl.username = REDACTED
+  if (proxyUrl.password) redactedUrl.password = REDACTED
+  const secrets = [proxyUrl.username, proxyUrl.password].filter(Boolean)
+  error.message = error.message
+    .split(proxyUrl.href)
+    .map(part => redactSecrets(part, secrets))
+    .join(redactedUrl.href)
+  return error
+}
+
+const forwardProxy = async (contextOpts, { mode } = {}) => {
+  const proxyUrl = contextOpts.proxyServer && parseProxyServer(contextOpts.proxyServer)
+  if (!hasCredentials(proxyUrl)) return contextOpts
+  if (mode === 'connect') throw new TypeError(CONNECT_MODE_ERROR)
+  try {
+    const proxyServer = await anonymizeProxy(proxyUrl.href)
+    forwardedProxyServers.add(proxyServer)
+    return { ...contextOpts, proxyServer }
+  } catch (error) {
+    throw withoutCredentials(error, proxyUrl)
+  }
+}
+
+const closeForwarder = async contextOptsPromise => {
+  const { proxyServer } = await contextOptsPromise.catch(() => ({}))
+  if (forwardedProxyServers.delete(proxyServer)) await closeAnonymizedProxy(proxyServer, true)
+}
+
+module.exports = { forwardProxy, closeForwarder }