html-get 2.11.3 → 2.11.5

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.11.3",
+  "version": "2.11.5",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -50,6 +50,7 @@
   "devDependencies": {
     "@commitlint/cli": "latest",
     "@commitlint/config-conventional": "latest",
+    "@ksmithut/prettier-standard": "latest",
     "ava": "latest",
     "browserless": "latest",
     "c8": "latest",
@@ -59,7 +60,6 @@
     "git-authors-cli": "latest",
     "nano-staged": "latest",
     "npm-check-updates": "latest",
-    "prettier-standard": "latest",
     "pretty": "latest",
     "puppeteer": "latest",
     "regex-iso-date": "latest",
@@ -98,7 +98,8 @@
       "test/**/*.js",
       "!test/util.js"
     ],
-    "timeout": "2m"
+    "timeout": "2m",
+    "workerThreads": false
   },
   "commitlint": {
     "extends": [
@@ -107,7 +108,8 @@
   },
   "nano-staged": {
     "*.js": [
-      "prettier-standard"
+      "prettier-standard",
+      "standard --fix"
     ],
     "*.md": [
       "standard-markdown"

package/src/auto-domains.json

@@ -14,25 +14,25 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"wordpress"
+			"apple"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"apple"
+			"microsoft"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"wikipedia"
+			"wordpress"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"microsoft"
+			"wikipedia"
 		]
 	],
 	[
@@ -53,12 +53,6 @@
 			"github"
 		]
 	],
-	[
-		[
-			"domainWithoutSuffix",
-			"imdb"
-		]
-	],
 	[
 		[
 			"domainWithoutSuffix",
@@ -71,12 +65,6 @@
 			"slideshare"
 		]
 	],
-	[
-		[
-			"domainWithoutSuffix",
-			"theguardian"
-		]
-	],
 	[
 		[
 			"domainWithoutSuffix",
@@ -86,19 +74,19 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"spotify"
+			"theguardian"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"huffingtonpost"
+			"imdb"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"soundcloud"
+			"pinterest"
 		]
 	],
 	[
@@ -110,25 +98,25 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"pinterest"
+			"spotify"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"yelp"
+			"soundcloud"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"techcrunch"
+			"huffingtonpost"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"zoom"
+			"engadget"
 		]
 	],
 	[
@@ -140,7 +128,7 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"engadget"
+			"yelp"
 		]
 	],
 	[
@@ -152,25 +140,25 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"theverge"
+			"zoom"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"flickr"
+			"techcrunch"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"stackoverflow"
+			"theverge"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"digg"
+			"giphy"
 		]
 	],
 	[
@@ -191,6 +179,12 @@
 			"deviantart"
 		]
 	],
+	[
+		[
+			"domainWithoutSuffix",
+			"digg"
+		]
+	],
 	[
 		[
 			"domainWithoutSuffix",
@@ -200,13 +194,13 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"ghost"
+			"flickr"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"giphy"
+			"ghost"
 		]
 	],
 	[
@@ -239,6 +233,12 @@
 			"sourceforge"
 		]
 	],
+	[
+		[
+			"domainWithoutSuffix",
+			"stackoverflow"
+		]
+	],
 	[
 		[
 			"domainWithoutSuffix",

package/src/index.js

@@ -12,6 +12,7 @@ const autoDomains = require('./auto-domains')
 const addHtml = require('./html')
 
 const REQ_TIMEOUT = 8000
+const ABORT_TYPES = ['image', 'stylesheet', 'font']
 
 const fetch = PCancelable.fn(
   async (
@@ -68,7 +69,7 @@ const prerender = PCancelable.fn(
       headers,
       gotOpts,
       timeout = REQ_TIMEOUT,
-      abortTypes = ['image', 'stylesheet', 'font'],
+      abortTypes = ABORT_TYPES,
       ...opts
     },
     onCancel
@@ -151,24 +152,32 @@ const determinateMode = (url, { prerender }) => {
   return isFetchMode(url) ? 'fetch' : 'prerender'
 }
 
-const getContent = async (
-  url,
-  mode,
-  { getBrowserless, gotOpts, headers, puppeteerOpts, rewriteUrls, toEncode }
-) => {
-  const isFetchMode = mode === 'fetch'
-  const fetchOpts = isFetchMode
-    ? { headers, toEncode, ...gotOpts }
-    : { headers, toEncode, getBrowserless, gotOpts, ...puppeteerOpts }
-  const content = await modes[mode](url, fetchOpts)
-
-  const html = addHtml({
-    ...content,
-    ...(isFetchMode ? puppeteerOpts : undefined),
-    rewriteUrls
-  })
-  return { ...content, html }
-}
+const getContent = PCancelable.fn(
+  (
+    url,
+    mode,
+    { getBrowserless, gotOpts, headers, puppeteerOpts, rewriteUrls, toEncode },
+    onCancel
+  ) => {
+    const isFetchMode = mode === 'fetch'
+    const fetchOpts = isFetchMode
+      ? { headers, toEncode, ...gotOpts }
+      : { headers, toEncode, getBrowserless, gotOpts, ...puppeteerOpts }
+
+    const promise = modes[mode](url, fetchOpts)
+    onCancel(() => promise.cancel())
+
+    return promise.then(content => {
+      const html = addHtml({
+        ...content,
+        ...(isFetchMode ? puppeteerOpts : undefined),
+        rewriteUrls
+      })
+
+      return { ...content, html }
+    })
+  }
+)
 
 module.exports = PCancelable.fn(
   async (
@@ -205,13 +214,15 @@ module.exports = PCancelable.fn(
       toEncode
     })
 
-    onCancel(() => promise.onCancel())
+    onCancel(() => promise.cancel())
 
     const { mode, ...payload } = await promise
 
-    return { ...payload, stats: { mode, timing: time.rounded() } }
+    return Object.assign(payload, { stats: { mode, timing: time.rounded() } })
   }
 )
 
 module.exports.REQ_TIMEOUT = REQ_TIMEOUT
+module.exports.ABORT_TYPES = ABORT_TYPES
 module.exports.isFetchMode = isFetchMode
+module.exports.getContent = getContent