html-get 2.11.6 → 2.13.0

package/bin/index.js

@@ -3,8 +3,8 @@
 'use strict'
 
 const createBrowserless = require('browserless')
-const mri = require('mri')
 const { URL } = require('url')
+const mri = require('mri')
 
 const getHTML = require('..')
 
@@ -26,9 +26,9 @@ getHTML(url, { getBrowserless, ...args })
    headers: ${
      headers
        ? Object.keys(headers).reduce(
-           (acc, key) => `${acc}${key}=${headers[key]} `,
-           ''
-         )
+         (acc, key) => `${acc}${key}=${headers[key]} `,
+         ''
+       )
        : '-'
    }
 `)

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.11.6",
+  "version": "2.13.0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"

package/src/auto-domains.json

@@ -2,37 +2,37 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"youtube"
+			"google"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"google"
+			"youtube"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"wordpress"
+			"apple"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"apple"
+			"wordpress"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"wikipedia"
+			"microsoft"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"microsoft"
+			"wikipedia"
 		]
 	],
 	[
@@ -56,7 +56,7 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"nytimes"
+			"slideshare"
 		]
 	],
 	[
@@ -68,7 +68,7 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"theguardian"
+			"nytimes"
 		]
 	],
 	[
@@ -80,31 +80,31 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"slideshare"
+			"theguardian"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"soundcloud"
+			"telegraph"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"huffingtonpost"
+			"pinterest"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"pinterest"
+			"huffingtonpost"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"telegraph"
+			"soundcloud"
 		]
 	],
 	[
@@ -116,55 +116,61 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"techcrunch"
+			"eventbrite"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"zoom"
+			"yelp"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"yelp"
+			"engadget"
 		]
 	],
 	[
 		[
-			"domain",
-			"abc.net.au"
+			"domainWithoutSuffix",
+			"techcrunch"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"eventbrite"
+			"zoom"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"engadget"
+			"theverge"
+		]
+	],
+	[
+		[
+			"domain",
+			"abc.net.au"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"theverge"
+			"etsy"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"giphy"
+			"stackoverflow"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"dribbble"
+			"flickr"
 		]
 	],
 	[
@@ -188,19 +194,19 @@
 	[
 		[
 			"domainWithoutSuffix",
-			"etsy"
+			"dribbble"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"flickr"
+			"ghost"
 		]
 	],
 	[
 		[
 			"domainWithoutSuffix",
-			"ghost"
+			"giphy"
 		]
 	],
 	[
@@ -233,12 +239,6 @@
 			"sourceforge"
 		]
 	],
-	[
-		[
-			"domainWithoutSuffix",
-			"stackoverflow"
-		]
-	],
 	[
 		[
 			"domainWithoutSuffix",

package/src/html.js

@@ -22,12 +22,21 @@ const has = el => el.length !== 0
 
 const upsert = (el, collection, item) => !has(el) && collection.push(item)
 
+/**
+ * Infer timestamp from `last-modified`, `date`, or `age` response headers.
+ */
+const getDate = headers => {
+  const timestamp = get(headers, 'last-modified') || get(headers, 'date')
+  return timestamp
+    ? toDate(timestamp)
+    : toDate(Date.now() - Number(get(headers, 'age')) * 1000)
+}
+
 const addHead = ({ $, url, headers }) => {
   const tags = []
   const contentType = get(headers, 'content-type')
   const charset = nth(split(contentType, 'charset='), 1)
-  const timestamp = get(headers, 'last-modified') || get(headers, 'date')
-  const date = timestamp && toDate(timestamp)
+  const date = getDate(headers)
   const { domain } = parseUrl(url)
   const head = $('head')
 
@@ -70,9 +79,9 @@ const addBody = ({ url, headers, html }) => {
   if (isMime(contentType, 'image')) {
     element = `<img src="${url}"></img>`
   } else if (isMime(contentType, 'video')) {
-    element = `<video src="${url}"></video>`
+    element = `<video><source src="${url}" type="${contentType}"></source></video>`
   } else if (isMime(contentType, 'audio')) {
-    element = `<audio src="${url}"></audio>`
+    element = `<audio><source src="${url}" type="${contentType}"></source></audio>`
   } else if (mimeExtension(contentType) === 'json') {
     element = `<pre>${html}</pre>`
   }
@@ -179,3 +188,5 @@ module.exports = ({
 
   return rewriteUrls ? rewriteCssUrls({ html: $.html(), url }) : $.html()
 }
+
+module.exports.getDate = getDate