html-get 2.10.6 → 2.11.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.10.6",
+  "version": "2.11.0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -29,14 +29,14 @@
     "request"
   ],
   "dependencies": {
-    "@metascraper/helpers": "~5.34.1",
+    "@metascraper/helpers": "~5.34.7",
     "cheerio": "~1.0.0-rc.12",
     "css-url-regex": "~4.0.0",
     "debug-logfmt": "~1.0.4",
     "execall": "~2.0.0",
     "got": "~11.8.6",
     "html-encode": "~2.1.6",
-    "html-urls": "~2.4.39",
+    "html-urls": "~2.4.45",
     "is-html-content": "~1.0.0",
     "lodash": "~4.17.21",
     "mri": "~1.2.0",
@@ -44,7 +44,7 @@
     "p-retry": "~4.6.0",
     "replace-string": "~3.1.0",
     "time-span": "~4.0.0",
-    "top-sites": "~1.1.132",
+    "top-sites": "~1.1.169",
     "write-json-file": "~4.3.0"
   },
   "devDependencies": {

package/src/auto-domains.json

@@ -1,6 +1,6 @@
 [
-	"youtube",
 	"google",
+	"youtube",
 	"microsoft",
 	"apple",
 	"wikipedia",
@@ -8,32 +8,32 @@
 	"blogspot",
 	"vimeo",
 	"github",
-	"imdb",
 	"theguardian",
+	"imdb",
 	"bbc",
-	"slideshare",
 	"nytimes",
-	"huffingtonpost",
-	"telegraph",
-	"pinterest",
+	"slideshare",
 	"soundcloud",
-	"eventbrite",
+	"pinterest",
+	"telegraph",
+	"huffingtonpost",
 	"engadget",
-	"yelp",
-	"zoom",
 	"techcrunch",
+	"zoom",
+	"yelp",
+	"eventbrite",
 	"spotify",
 	"theverge",
-	"etsy",
 	"dribbble",
+	"giphy",
+	"imgur",
 	"csdn",
 	"deviantart",
 	"digg",
+	"etsy",
 	"flickr",
 	"ghost",
-	"giphy",
 	"gitlab",
-	"imgur",
 	"meetup",
 	"producthunt",
 	"sourceforge",

package/src/index.js

@@ -13,11 +13,12 @@ const addHtml = require('./html')
 
 const REQ_TIMEOUT = 8000
 
-const fetch = (
-  url,
-  { reflect = false, toEncode, timeout = REQ_TIMEOUT, ...opts }
-) =>
-  new PCancelable(async (resolve, reject, onCancel) => {
+const fetch = PCancelable.fn(
+  async (
+    url,
+    { reflect = false, toEncode, timeout = REQ_TIMEOUT, ...opts },
+    onCancel
+  ) => {
     const reqTimeout = reflect ? timeout / 2 : timeout
 
     const req = got(url, {
@@ -36,99 +37,109 @@ const fetch = (
 
     try {
       const res = await req
-      return resolve({
+      return {
         headers: res.headers,
         html: await toEncode(res.body, res.headers['content-type']),
         mode: 'fetch',
         url: res.url,
         statusCode: res.statusCode
-      })
+      }
     } catch (error) {
       debug('fetch:error', { url, message: error.message || error, reflect })
       return reflect
-        ? resolve({ isRejected: true, error })
-        : resolve({
-          url,
-          html: '',
-          mode: 'fetch',
-          headers: error.response ? error.response.headers : {},
-          statusCode: error.response ? error.response.statusCode : undefined
-        })
+        ? { isRejected: true, error }
+        : {
+            url,
+            html: '',
+            mode: 'fetch',
+            headers: error.response ? error.response.headers : {},
+            statusCode: error.response ? error.response.statusCode : undefined
+          }
     }
-  })
-
-const prerender = async (
-  url,
-  {
-    getBrowserless,
-    toEncode,
-    headers,
-    gotOpts,
-    timeout = REQ_TIMEOUT,
-    abortTypes = ['image', 'stylesheet', 'font'],
-    ...opts
   }
-) => {
-  let fetchRes
-  let data = {}
-  let isFetchResRejected = false
+)
 
-  try {
-    fetchRes = fetch(url, {
-      reflect: true,
+const prerender = PCancelable.fn(
+  async (
+    url,
+    {
+      getBrowserless,
       toEncode,
-      ...gotOpts,
       headers,
-      timeout
-    })
-    const browserless = await getBrowserless()
-
-    const getPayload = browserless.evaluate(
-      async (page, response) => {
-        if (!response) throw new AbortError('empty response')
-
-        return {
-          headers: response.headers(),
-          html: await page.content(),
-          mode: 'prerender',
-          url: response.url(),
-          statusCode: response.status()
-        }
-      },
-      {
-        timeout,
+      gotOpts,
+      timeout = REQ_TIMEOUT,
+      abortTypes = ['image', 'stylesheet', 'font'],
+      ...opts
+    },
+    onCancel
+  ) => {
+    let fetchRes
+    let data = {}
+    let isFetchResRejected = false
+
+    onCancel(() => fetchRes.cancel())
+
+    try {
+      fetchRes = fetch(url, {
+        reflect: true,
+        toEncode,
+        ...gotOpts,
         headers,
-        abortTypes
-      }
-    )
+        timeout
+      })
+      const browserless = await getBrowserless()
+
+      const getPayload = browserless.evaluate(
+        async (page, response) => {
+          if (!response) throw new AbortError('empty response')
+
+          return {
+            headers: response.headers(),
+            html: await page.content(),
+            mode: 'prerender',
+            url: response.url(),
+            statusCode: response.status()
+          }
+        },
+        {
+          timeout,
+          headers,
+          abortTypes
+        }
+      )
 
-    const payload = await getPayload(url, opts)
+      onCancel(() => {
+        debug('prerender:cancel', { url })
+        getPayload.cancel()
+      })
 
-    await fetchRes.cancel()
-    debug('prerender', { url, state: 'success' })
-    return payload
-  } catch (err) {
-    const { isRejected, ...dataProps } = await fetchRes
+      const payload = await getPayload(url, opts)
+      await fetchRes.cancel()
+      debug('prerender', { url, state: 'success' })
+      return payload
+    } catch (err) {
+      const { isRejected, ...dataProps } = await fetchRes
 
-    debug('prerender:error', {
-      url,
-      isRejected,
-      error: err.message
-    })
+      debug('prerender:error', {
+        url,
+        isRejected,
+        error: err.message
+      })
 
-    isFetchResRejected = isRejected
-    data = dataProps
-  }
+      isFetchResRejected = isRejected
+      data = dataProps
+    }
 
-  return isFetchResRejected
-    ? {
-        headers: data.headers || {},
-        html: '',
-        url,
-        mode: 'prerender'
-      }
-    : data
-}
+    return isFetchResRejected
+      ? {
+          headers: data.headers || {},
+          html: '',
+          url,
+          mode: 'prerender'
+        }
+      : data
+  }
+)
 
 const modes = { fetch, prerender }
 
@@ -160,40 +171,47 @@ const getContent = async (
   return { ...content, html }
 }
 
-module.exports = async (
-  targetUrl,
-  {
-    encoding = 'utf-8',
-    getBrowserless,
-    getMode = determinateMode,
-    gotOpts,
-    headers,
-    prerender = 'auto',
-    puppeteerOpts,
-    rewriteUrls = false
-  } = {}
-) => {
-  if (!getBrowserless && prerender !== false) {
-    throw TypeError(
-      "Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
-    )
-  }
+module.exports = PCancelable.fn(
+  async (
+    targetUrl,
+    {
+      encoding = 'utf-8',
+      getBrowserless,
+      getMode = determinateMode,
+      gotOpts,
+      headers,
+      prerender = 'auto',
+      puppeteerOpts,
+      rewriteUrls = false
+    } = {},
+    onCancel
+  ) => {
+    if (!getBrowserless && prerender !== false) {
+      throw TypeError(
+        "Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
+      )
+    }
 
-  const toEncode = htmlEncode(encoding)
-  const reqMode = getMode(targetUrl, { prerender })
+    const toEncode = htmlEncode(encoding)
+    const reqMode = getMode(targetUrl, { prerender })
 
-  const time = timeSpan()
+    const time = timeSpan()
 
-  const { mode, ...payload } = await getContent(targetUrl, reqMode, {
-    getBrowserless,
-    gotOpts,
-    headers,
-    puppeteerOpts,
-    rewriteUrls,
-    toEncode
-  })
+    const promise = getContent(targetUrl, reqMode, {
+      getBrowserless,
+      gotOpts,
+      headers,
+      puppeteerOpts,
+      rewriteUrls,
+      toEncode
+    })
 
-  return { ...payload, stats: { mode, timing: time.rounded() } }
-}
+    onCancel(() => promise.onCancel())
+
+    const { mode, ...payload } = await promise
+
+    return { ...payload, stats: { mode, timing: time.rounded() } }
+  }
+)
 
 module.exports.REQ_TIMEOUT = REQ_TIMEOUT