html-get 2.11.4 → 2.11.6

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.11.4",
+  "version": "2.11.6",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -98,7 +98,8 @@
       "test/**/*.js",
       "!test/util.js"
     ],
-    "timeout": "2m"
+    "timeout": "2m",
+    "workerThreads": false
   },
   "commitlint": {
     "extends": [
@@ -107,7 +108,8 @@
   },
   "nano-staged": {
     "*.js": [
-      "prettier-standard"
+      "prettier-standard",
+      "standard --fix"
     ],
     "*.md": [
       "standard-markdown"

package/src/auto-domains.json

@@ -2,37 +2,37 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"google"
+			"youtube"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"youtube"
+			"google"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"apple"
+			"wordpress"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"microsoft"
+			"apple"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"wordpress"
+			"wikipedia"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"wikipedia"
+			"microsoft"
 		]
 	],
 	[
@@ -44,85 +44,85 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"vimeo"
+			"github"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"github"
+			"vimeo"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"bbc"
+			"nytimes"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"slideshare"
+			"imdb"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"nytimes"
+			"theguardian"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"theguardian"
+			"bbc"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"imdb"
+			"slideshare"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"pinterest"
+			"soundcloud"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"telegraph"
+			"huffingtonpost"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"spotify"
+			"pinterest"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"soundcloud"
+			"telegraph"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"huffingtonpost"
+			"spotify"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"engadget"
+			"techcrunch"
 		]
 	],
 	[
 		[
-			"domain",
-			"abc.net.au"
+			"domainWithoutSuffix",
+			"zoom"
 		]
 	],
 	[
@@ -133,20 +133,20 @@
 	],
 	[
 		[
-			"domainWithoutSuffix",
-			"eventbrite"
+			"domain",
+			"abc.net.au"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"zoom"
+			"eventbrite"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"techcrunch"
+			"engadget"
 		]
 	],
 	[
@@ -164,7 +164,7 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"etsy"
+			"dribbble"
 		]
 	],
 	[
@@ -188,7 +188,7 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"dribbble"
+			"etsy"
 		]
 	],
 	[

package/src/html.js

@@ -28,16 +28,18 @@ const addHead = ({ $, url, headers }) => {
   const charset = nth(split(contentType, 'charset='), 1)
   const timestamp = get(headers, 'last-modified') || get(headers, 'date')
   const date = timestamp && toDate(timestamp)
-
+  const { domain } = parseUrl(url)
   const head = $('head')
 
   upsert(head.find('title'), tags, `<title>${path.basename(url)}</title>`)
 
-  upsert(
-    head.find('meta[property="og:site_name"]'),
-    tags,
-    `<meta property="og:site_name" content="${parseUrl(url).domain}">`
-  )
+  if (domain) {
+    upsert(
+      head.find('meta[property="og:site_name"]'),
+      tags,
+      `<meta property="og:site_name" content="${domain}">`
+    )
+  }
 
   if (date) {
     upsert(

package/src/index.js

@@ -152,24 +152,32 @@ const determinateMode = (url, { prerender }) => {
   return isFetchMode(url) ? 'fetch' : 'prerender'
 }
 
-const getContent = async (
-  url,
-  mode,
-  { getBrowserless, gotOpts, headers, puppeteerOpts, rewriteUrls, toEncode }
-) => {
-  const isFetchMode = mode === 'fetch'
-  const fetchOpts = isFetchMode
-    ? { headers, toEncode, ...gotOpts }
-    : { headers, toEncode, getBrowserless, gotOpts, ...puppeteerOpts }
-  const content = await modes[mode](url, fetchOpts)
-
-  const html = addHtml({
-    ...content,
-    ...(isFetchMode ? puppeteerOpts : undefined),
-    rewriteUrls
-  })
-  return { ...content, html }
-}
+const getContent = PCancelable.fn(
+  (
+    url,
+    mode,
+    { getBrowserless, gotOpts, headers, puppeteerOpts, rewriteUrls, toEncode },
+    onCancel
+  ) => {
+    const isFetchMode = mode === 'fetch'
+    const fetchOpts = isFetchMode
+      ? { headers, toEncode, ...gotOpts }
+      : { headers, toEncode, getBrowserless, gotOpts, ...puppeteerOpts }
+
+    const promise = modes[mode](url, fetchOpts)
+    onCancel(() => promise.cancel())
+
+    return promise.then(content => {
+      const html = addHtml({
+        ...content,
+        ...(isFetchMode ? puppeteerOpts : undefined),
+        rewriteUrls
+      })
+
+      return { ...content, html }
+    })
+  }
+)
 
 module.exports = PCancelable.fn(
   async (
@@ -206,14 +214,15 @@ module.exports = PCancelable.fn(
       toEncode
     })
 
-    onCancel(() => promise.onCancel())
+    onCancel(() => promise.cancel())
 
     const { mode, ...payload } = await promise
 
-    return { ...payload, stats: { mode, timing: time.rounded() } }
+    return Object.assign(payload, { stats: { mode, timing: time.rounded() } })
   }
 )
 
 module.exports.REQ_TIMEOUT = REQ_TIMEOUT
 module.exports.ABORT_TYPES = ABORT_TYPES
 module.exports.isFetchMode = isFetchMode
+module.exports.getContent = getContent