html-get 2.22.3 → 2.24.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.22.3",
+  "version": "2.24.0",
   "types": "index.d.ts",
   "main": "src/index.js",
   "bin": {

package/src/index.js

@@ -155,6 +155,7 @@ const prerender = PCancelable.fn(
           }
         },
         {
+          flattenShadowDOM: true,
           timeout,
           headers,
           abortTypes
@@ -191,6 +192,11 @@ const prerender = PCancelable.fn(
 
 const modes = { fetch, prerender }
 
+const hasShadowDOM = $ =>
+  $('*')
+    .toArray()
+    .some(el => el.tagName?.includes('-'))
+
 const isFetchMode = url => {
   const parsedUrl = parseUrl(url)
   return autoDomains.some(conditions =>
@@ -308,7 +314,24 @@ module.exports = PCancelable.fn(
 
     onCancel(() => promise.cancel())
 
-    const { mode, html, $, ...payload } = await promise
+    let { mode, html, $, ...payload } = await promise
+
+    if (mode === 'fetch' && getBrowserless && hasShadowDOM($)) {
+      debug('shadow DOM detected, retrying with prerender', { url: targetUrl })
+      const prerenderPromise = getContent(targetUrl, 'prerender', {
+        getBrowserless,
+        getTemporalFile,
+        gotOpts,
+        headers,
+        mutool,
+        puppeteerOpts,
+        rewriteUrls,
+        rewriteHtml,
+        toEncode
+      })
+      onCancel(() => prerenderPromise.cancel())
+      ;({ mode, html, $, ...payload } = await prerenderPromise)
+    }
 
     return Object.assign(payload, {
       ...serializeHtml($),