html-get 2.9.2 → 2.9.6

package/CHANGELOG.md

@@ -2,6 +2,19 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### 2.9.6 (2021-10-25)
+
+### [2.9.5](https://github.com/microlinkhq/html-get/compare/v2.9.4...v2.9.5) (2021-09-08)
+
+
+### Bug Fixes
+
+* ensure reponse is a buffer ([842ed98](https://github.com/microlinkhq/html-get/commit/842ed98f713be107ee7a4b945d89e60376df6209))
+
+### [2.9.4](https://github.com/microlinkhq/html-get/compare/v2.9.3...v2.9.4) (2021-09-08)
+
+### 2.9.3 (2021-09-04)
+
 ### 2.9.2 (2021-08-16)
 
 ### 2.9.1 (2021-08-03)

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.9.2",
+  "version": "2.9.6",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -29,14 +29,14 @@
     "request"
   ],
   "dependencies": {
-    "@metascraper/helpers": "~5.24.0",
-    "cheerio": "~1.0.0-rc.9",
+    "@metascraper/helpers": "~5.25.0",
+    "cheerio": "~1.0.0-rc.10",
     "css-url-regex": "~4.0.0",
     "debug-logfmt": "~1.0.4",
     "execall": "~2.0.0",
     "got": "~11.8.2",
     "html-encode": "~2.1.6",
-    "html-urls": "~2.4.22",
+    "html-urls": "~2.4.26",
     "is-html-content": "~1.0.0",
     "lodash": "~4.17.21",
     "minimist": "~1.2.5",
@@ -44,8 +44,8 @@
     "p-retry": "~4.6.0",
     "replace-string": "~3.1.0",
     "time-span": "~4.0.0",
-    "tldts": "~5.7.38",
-    "top-sites": "~1.1.40",
+    "tldts": "~5.7.42",
+    "top-sites": "~1.1.63",
     "write-json-file": "~4.3.0"
   },
   "devDependencies": {

package/src/auto-domains.json

@@ -1,37 +1,37 @@
 [
-	"apple",
-	"youtube",
 	"google",
+	"youtube",
+	"apple",
 	"microsoft",
-	"wikipedia",
 	"wordpress",
-	"vimeo",
+	"wikipedia",
 	"github",
-	"bbc",
+	"vimeo",
 	"blogspot",
 	"imdb",
-	"slideshare",
 	"nytimes",
+	"bbc",
 	"theguardian",
-	"huffingtonpost",
-	"pinterest",
+	"slideshare",
 	"telegraph",
+	"pinterest",
+	"huffingtonpost",
 	"soundcloud",
-	"eventbrite",
-	"spotify",
-	"yelp",
 	"stackoverflow",
+	"eventbrite",
 	"zoom",
+	"yelp",
 	"techcrunch",
-	"engadget",
+	"spotify",
 	"theverge",
-	"imgur",
-	"csdn",
+	"engadget",
 	"digg",
+	"csdn",
 	"etsy",
 	"flickr",
 	"ghost",
 	"giphy",
+	"imgur",
 	"meetup",
 	"producthunt",
 	"reddit",

package/src/index.js

@@ -20,9 +20,9 @@ const fetch = (
 ) =>
   new PCancelable(async (resolve, reject, onCancel) => {
     const req = got(url, {
-      responseType: 'buffer',
       timeout: reflect ? timeout / 2 : timeout,
-      ...opts
+      ...opts,
+      responseType: 'buffer'
     })
 
     onCancel.shouldReject = false
@@ -67,8 +67,8 @@ const prerender = async (
     fetchRes = fetch(url, {
       reflect: true,
       toEncode,
-      headers,
       ...gotOpts,
+      headers,
       timeout
     })
     const browserless = await getBrowserless()
@@ -85,7 +85,11 @@ const prerender = async (
           statusCode: response.status()
         }
       },
-      { timeout, headers }
+      {
+        timeout,
+        headers,
+        abortTypes: ['stylesheet', 'fonts', 'image', 'media']
+      }
     )
 
     const payload = await getPayload(url, opts)
@@ -158,7 +162,9 @@ module.exports = async (
   } = {}
 ) => {
   if (!getBrowserless) {
-    throw TypeError('Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require(\'browserless\')`')
+    throw TypeError(
+      "Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
+    )
   }
 
   const toEncode = htmlEncode(encoding)