html-get 2.23.0 → 2.24.1

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.23.0",
+  "version": "2.24.1",
   "types": "index.d.ts",
   "main": "src/index.js",
   "bin": {

package/src/index.js

@@ -192,6 +192,11 @@ const prerender = PCancelable.fn(
 
 const modes = { fetch, prerender }
 
+const hasShadowDOM = $ =>
+  $('*')
+    .toArray()
+    .some(el => el.tagName?.includes('-'))
+
 const isFetchMode = url => {
   const parsedUrl = parseUrl(url)
   return autoDomains.some(conditions =>
@@ -309,11 +314,31 @@ module.exports = PCancelable.fn(
 
     onCancel(() => promise.cancel())
 
-    const { mode, html, $, ...payload } = await promise
+    let { mode, html, $, ...payload } = await promise
+
+    let shadowDOM = hasShadowDOM($)
+
+    if (mode === 'fetch' && getBrowserless && shadowDOM) {
+      debug('shadow DOM detected, retrying with prerender', { url: targetUrl })
+      const prerenderPromise = getContent(targetUrl, 'prerender', {
+        getBrowserless,
+        getTemporalFile,
+        gotOpts,
+        headers,
+        mutool,
+        puppeteerOpts,
+        rewriteUrls,
+        rewriteHtml,
+        toEncode
+      })
+      onCancel(() => prerenderPromise.cancel())
+      ;({ mode, html, $, ...payload } = await prerenderPromise)
+      shadowDOM = hasShadowDOM($)
+    }
 
     return Object.assign(payload, {
       ...serializeHtml($),
-      stats: { mode, timing: duration() }
+      stats: { mode, timing: duration(), shadowDOM }
     })
   }
 )