@browserless/goto 13.0.9 → 13.1.1

package/README.md

@@ -71,6 +71,7 @@ const { response, device, error } = await goto(page, {
 | `javascript` | `boolean` | `true` | Enable/disable JavaScript |
 | `animations` | `boolean` | `false` | Enable CSS animations |
 | `colorScheme` | `string` | — | `'light'` or `'dark'` preference |
+| `flattenShadowDOM` | `boolean` | `false` (`true` for `html()`) | Serialize open shadow DOM into HTML (mutates the page DOM; closed shadow roots are skipped) |
 | `mediaType` | `string` | — | CSS media type (`'screen'`, `'print'`) |
 | `timezone` | `string` | — | Timezone to emulate |
 | `authenticate` | `object` | — | HTTP authentication credentials |

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/goto",
   "description": "Navigate to web pages with built-in ad blocking, device emulation, and optimized loading for faster automation.",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "13.0.9",
+  "version": "13.1.1",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -31,7 +31,7 @@
   ],
   "dependencies": {
     "@browserless/devices": "13.0.0",
-    "@duckduckgo/autoconsent": "~14.90.0",
+    "@duckduckgo/autoconsent": "~14.92.0",
     "@ghostery/adblocker-puppeteer": "~2.18.0",
     "debug-logfmt": "~1.4.10",
     "got": "~11.8.6",
@@ -42,7 +42,7 @@
     "tough-cookie": "~6.0.1"
   },
   "devDependencies": {
-    "@browserless/test": "13.0.9",
+    "@browserless/test": "13.1.1",
     "ava": "5",
     "p-wait-for": "3"
   },
@@ -66,5 +66,5 @@
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "b4a515a809d6d7d33828e901d18d32e805e26d55"
+  "gitHead": "d491c5204deef804c923539063d92658f4e9649d"
 }

package/src/index.js

@@ -228,18 +228,11 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
 
   // related https://github.com/puppeteer/puppeteer/issues/1353
   const _waitUntilAuto = (page, { timeout }) => {
-    return Promise.all(
-      [
-        {
-          fn: () => page.waitForNavigation({ waitUntil: 'networkidle2' }),
-          debug: 'waitUntilAuto:waitForNavigation'
-        },
-        {
-          fn: () => page.evaluate(() => window.history.pushState(null, null, null)),
-          debug: 'waitUntilAuto:pushState'
-        }
-      ].map(({ fn, debug }) => run({ fn: fn(), debug, timeout }))
-    )
+    return run({
+      fn: page.waitForNetworkIdle({ idleTime: 500, concurrency: 2 }),
+      debug: 'waitUntilAuto:networkIdle',
+      timeout
+    })
   }
 
   const goto = async (
@@ -251,6 +244,7 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       authenticate,
       click,
       colorScheme,
+      flattenShadowDOM = false,
       headers: rawHeaders = {},
       html,
       javascript = true,
@@ -481,7 +475,7 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
             return Promise.race([page.goto(url, { waitUntil, ...args }), promise])
           })()
 
-      const { value: response, reason: error } = await run({
+      let { value: response, reason: error } = await run({
         fn: navigationPromise,
         timeout: gotoTimeout,
         debug: { fn: html ? 'html' : 'url', waitUntil }
@@ -547,6 +541,65 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
         await waitUntilAuto(page, { response, timeout: actionTimeout * 2 })
       }
 
+      if (flattenShadowDOM) {
+        const { isRejected, reason: flattenError } = await run({
+          fn: page.evaluate(() => {
+            ;(function flatten (root) {
+              const replaceSlot = (slot, nodes) => {
+                const parent = slot.parentNode
+                if (!parent) return
+                for (const node of nodes) parent.insertBefore(node, slot)
+                slot.remove()
+              }
+
+              const flattenNode = node => {
+                if (node.nodeType !== window.Node.ELEMENT_NODE) return
+                if (node.shadowRoot) flattenElement(node)
+                flattenChildren(node)
+              }
+
+              const flattenChildren = root => {
+                for (const el of root.querySelectorAll('*')) {
+                  if (el.shadowRoot) flattenElement(el)
+                  if (el.localName !== 'slot') continue
+
+                  const slot = el
+                  const nodes = slot.assignedNodes({ flatten: true })
+                  if (nodes.length > 0) {
+                    const clones = nodes.map(node => {
+                      flattenNode(node)
+                      return node.cloneNode(true)
+                    })
+
+                    replaceSlot(slot, clones)
+                  } else {
+                    const fallback = [...slot.childNodes]
+                    fallback.forEach(flattenNode)
+                    if (fallback.length > 0) {
+                      replaceSlot(
+                        slot,
+                        fallback.map(n => n.cloneNode(true))
+                      )
+                    } else slot.remove()
+                  }
+                }
+              }
+
+              const flattenElement = el => {
+                flattenChildren(el.shadowRoot)
+                el.innerHTML = el.shadowRoot.innerHTML
+              }
+
+              flattenChildren(root)
+            })(document.body)
+          }),
+          timeout: actionTimeout,
+          debug: 'flattenShadowDOM'
+        })
+
+        if (isRejected) error = error || flattenError || new Error('flattenShadowDOM failed')
+      }
+
       return { response, device, error }
     } finally {
       if (abortTypesHandler) page.off('request', abortTypesHandler)