html-get 2.10.7 → 2.11.1

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.10.7",
+  "version": "2.11.1",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -29,14 +29,14 @@
     "request"
   ],
   "dependencies": {
-    "@metascraper/helpers": "~5.34.1",
+    "@metascraper/helpers": "~5.34.7",
     "cheerio": "~1.0.0-rc.12",
     "css-url-regex": "~4.0.0",
     "debug-logfmt": "~1.0.4",
     "execall": "~2.0.0",
     "got": "~11.8.6",
     "html-encode": "~2.1.6",
-    "html-urls": "~2.4.39",
+    "html-urls": "~2.4.45",
     "is-html-content": "~1.0.0",
     "lodash": "~4.17.21",
     "mri": "~1.2.0",
@@ -44,7 +44,7 @@
     "p-retry": "~4.6.0",
     "replace-string": "~3.1.0",
     "time-span": "~4.0.0",
-    "top-sites": "~1.1.132",
+    "top-sites": "~1.1.169",
     "write-json-file": "~4.3.0"
   },
   "devDependencies": {

package/src/auto-domains.json

@@ -1,39 +1,39 @@
 [
 	"google",
 	"youtube",
-	"microsoft",
 	"apple",
-	"wikipedia",
+	"microsoft",
 	"wordpress",
+	"wikipedia",
 	"blogspot",
 	"vimeo",
 	"github",
 	"theguardian",
-	"imdb",
-	"bbc",
 	"nytimes",
 	"slideshare",
-	"soundcloud",
+	"bbc",
+	"imdb",
 	"pinterest",
 	"telegraph",
+	"spotify",
 	"huffingtonpost",
+	"soundcloud",
 	"engadget",
-	"techcrunch",
 	"zoom",
+	"techcrunch",
 	"yelp",
 	"eventbrite",
-	"spotify",
 	"theverge",
-	"dribbble",
-	"giphy",
-	"imgur",
+	"flickr",
+	"digg",
 	"csdn",
 	"deviantart",
-	"digg",
+	"dribbble",
 	"etsy",
-	"flickr",
 	"ghost",
+	"giphy",
 	"gitlab",
+	"imgur",
 	"meetup",
 	"producthunt",
 	"sourceforge",

package/src/index.js

@@ -59,78 +59,82 @@ const fetch = PCancelable.fn(
   }
 )
 
-const prerender = async (
-  url,
-  {
-    getBrowserless,
-    toEncode,
-    headers,
-    gotOpts,
-    timeout = REQ_TIMEOUT,
-    abortTypes = ['image', 'stylesheet', 'font'],
-    ...opts
-  }
-) => {
-  let fetchRes
-  let data = {}
-  let isFetchResRejected = false
-
-  try {
-    fetchRes = fetch(url, {
-      reflect: true,
+const prerender = PCancelable.fn(
+  async (
+    url,
+    {
+      getBrowserless,
       toEncode,
-      ...gotOpts,
       headers,
-      timeout
-    })
-    const browserless = await getBrowserless()
-
-    const getPayload = browserless.evaluate(
-      async (page, response) => {
-        if (!response) throw new AbortError('empty response')
-
-        return {
-          headers: response.headers(),
-          html: await page.content(),
-          mode: 'prerender',
-          url: response.url(),
-          statusCode: response.status()
-        }
-      },
-      {
-        timeout,
+      gotOpts,
+      timeout = REQ_TIMEOUT,
+      abortTypes = ['image', 'stylesheet', 'font'],
+      ...opts
+    },
+    onCancel
+  ) => {
+    let fetchRes
+    let data = {}
+    let isFetchResRejected = false
+
+    onCancel(() => fetchRes.cancel())
+
+    try {
+      fetchRes = fetch(url, {
+        reflect: true,
+        toEncode,
+        ...gotOpts,
         headers,
-        abortTypes
-      }
-    )
+        timeout
+      })
+      const browserless = await getBrowserless()
+
+      const getPayload = browserless.evaluate(
+        async (page, response) => {
+          if (!response) throw new AbortError('empty response')
+
+          return {
+            headers: response.headers(),
+            html: await page.content(),
+            mode: 'prerender',
+            url: response.url(),
+            statusCode: response.status()
+          }
+        },
+        {
+          timeout,
+          headers,
+          abortTypes
+        }
+      )
 
-    const payload = await getPayload(url, opts)
+      const payload = await getPayload(url, opts)
+      await fetchRes.cancel()
+      debug('prerender', { url, state: 'success' })
+      return payload
+    } catch (err) {
+      const { isRejected, ...dataProps } = await fetchRes
 
-    await fetchRes.cancel()
-    debug('prerender', { url, state: 'success' })
-    return payload
-  } catch (err) {
-    const { isRejected, ...dataProps } = await fetchRes
+      debug('prerender:error', {
+        url,
+        isRejected,
+        error: err.message
+      })
 
-    debug('prerender:error', {
-      url,
-      isRejected,
-      error: err.message
-    })
+      isFetchResRejected = isRejected
+      data = dataProps
+    }
 
-    isFetchResRejected = isRejected
-    data = dataProps
+    return isFetchResRejected
+      ? {
+          headers: data.headers || {},
+          html: '',
+          url,
+          mode: 'prerender'
+        }
+      : data
   }
-
-  return isFetchResRejected
-    ? {
-        headers: data.headers || {},
-        html: '',
-        url,
-        mode: 'prerender'
-      }
-    : data
-}
+)
 
 const modes = { fetch, prerender }
 
@@ -162,40 +166,47 @@ const getContent = async (
   return { ...content, html }
 }
 
-module.exports = async (
-  targetUrl,
-  {
-    encoding = 'utf-8',
-    getBrowserless,
-    getMode = determinateMode,
-    gotOpts,
-    headers,
-    prerender = 'auto',
-    puppeteerOpts,
-    rewriteUrls = false
-  } = {}
-) => {
-  if (!getBrowserless && prerender !== false) {
-    throw TypeError(
-      "Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
-    )
-  }
+module.exports = PCancelable.fn(
+  async (
+    targetUrl,
+    {
+      encoding = 'utf-8',
+      getBrowserless,
+      getMode = determinateMode,
+      gotOpts,
+      headers,
+      prerender = 'auto',
+      puppeteerOpts,
+      rewriteUrls = false
+    } = {},
+    onCancel
+  ) => {
+    if (!getBrowserless && prerender !== false) {
+      throw TypeError(
+        "Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
+      )
+    }
 
-  const toEncode = htmlEncode(encoding)
-  const reqMode = getMode(targetUrl, { prerender })
+    const toEncode = htmlEncode(encoding)
+    const reqMode = getMode(targetUrl, { prerender })
 
-  const time = timeSpan()
+    const time = timeSpan()
 
-  const { mode, ...payload } = await getContent(targetUrl, reqMode, {
-    getBrowserless,
-    gotOpts,
-    headers,
-    puppeteerOpts,
-    rewriteUrls,
-    toEncode
-  })
+    const promise = getContent(targetUrl, reqMode, {
+      getBrowserless,
+      gotOpts,
+      headers,
+      puppeteerOpts,
+      rewriteUrls,
+      toEncode
+    })
 
-  return { ...payload, stats: { mode, timing: time.rounded() } }
-}
+    onCancel(() => promise.onCancel())
+
+    const { mode, ...payload } = await promise
+
+    return { ...payload, stats: { mode, timing: time.rounded() } }
+  }
+)
 
 module.exports.REQ_TIMEOUT = REQ_TIMEOUT