html-get 2.9.14 → 2.9.16

package/README.md

@@ -38,21 +38,37 @@ $ npm install puppeteer html-get --save
 ## Usage
 
 ```js
-'use strict'
-
+const createBrowserless = require('browserless')
 const getHTML = require('html-get')
 
-getHTML('https://example.com').then(
-  ({ url, html, stats, headers, statusCode }) =>
-    console.log(`
-       url: ${url}
-      html: ${Buffer.from(html).byteLength} bytes (HTTP ${statusCode})
-      time: ${stats.timing} (${stats.mode})
-   headers: ${Object.keys(headers).reduce(
-     (acc, key) => `${acc}${key}=${headers[key]} `,
-     ''
-   )}
-`))
+// Spawn Chromium process once
+const browserlessFactory = createBrowserless()
+
+// Kill the process when Node.js exit
+process.on('exit', () => {
+  console.log('closing resources!')
+  browserlessFactory.close()
+})
+
+const getContent = async url => {
+  // create a browser context inside Chromium process
+  const browserContext = browserlessFactory.createContext()
+  const getBrowserless = () => browserContext
+  const result = await getHTML(url, { getBrowserless })
+  // close the browser context after it's used
+  await getBrowserless((browser) => browser.destroyContext())
+  return result
+}
+
+getContent('https://example.com')
+  .then(content => {
+    console.log(content)
+    process.exit()
+  })
+  .catch(error => {
+    console.error(error)
+    process.exit(1)
+  })
 ```
 
 ### Command Line

package/bin/index.js

@@ -2,18 +2,24 @@
 
 'use strict'
 
+const createBrowserless = require('browserless')
 const minimist = require('minimist')
 const { URL } = require('url')
 
 const getHTML = require('..')
 
+const browserlessFactory = createBrowserless()
+
 const [input, ...argv] = process.argv.slice(2)
 const url = new URL(input).toString()
 
 const { debug: isDebug, ...args } = minimist(argv)
 
-getHTML(url, args)
-  .then(({ html, stats, headers, statusCode }) => {
+const browserContext = browserlessFactory.createContext()
+const getBrowserless = () => browserContext
+
+getHTML(url, { getBrowserless, ...args })
+  .then(async ({ html, stats, headers, statusCode }) => {
     if (isDebug) {
       console.log(`
        url: ${url}
@@ -31,10 +37,13 @@ getHTML(url, args)
     } else {
       console.log(html)
     }
-
     process.exit(0)
   })
-  .catch(err => {
+  .catch(async err => {
     console.error(err)
     process.exit(1)
   })
+  .finally(async () => {
+    await getBrowserless(browser => browser.destroyContext())
+    browserlessFactory.close()
+  })

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.9.14",
+  "version": "2.9.16",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -79,7 +79,7 @@
   ],
   "scripts": {
     "clean": "rm -rf node_modules",
-    "contributors": "(git-authors-cli && finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
+    "contributors": "(git-authors-cli && finepack --sort-ignore-object-at ava && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
     "lint": "standard-markdown README.md && standard",
     "postinstall": "node scripts/postinstall",
     "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
@@ -94,9 +94,10 @@
   },
   "license": "MIT",
   "ava": {
+    "workerThreads": false,
     "files": [
-      "!test/util.js",
-      "test/**/*.js"
+      "test/**/*.js",
+      "!test/util.js"
     ],
     "timeout": "2m"
   },

package/src/auto-domains.json

@@ -1,37 +1,37 @@
 [
 	"youtube",
 	"google",
-	"microsoft",
 	"apple",
-	"wikipedia",
+	"microsoft",
 	"wordpress",
+	"wikipedia",
 	"blogspot",
-	"vimeo",
 	"github",
-	"nytimes",
+	"vimeo",
 	"slideshare",
-	"bbc",
 	"imdb",
+	"bbc",
 	"theguardian",
+	"nytimes",
+	"huffingtonpost",
 	"telegraph",
 	"pinterest",
-	"huffingtonpost",
-	"spotify",
+	"yelp",
 	"eventbrite",
+	"engadget",
 	"zoom",
 	"techcrunch",
-	"yelp",
-	"soundcloud",
-	"engadget",
 	"theverge",
+	"spotify",
+	"soundcloud",
+	"etsy",
 	"flickr",
 	"stackoverflow",
-	"giphy",
-	"imgur",
 	"csdn",
 	"digg",
-	"etsy",
 	"ghost",
+	"giphy",
+	"imgur",
 	"meetup",
 	"producthunt",
 	"reddit",

package/src/html.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { startsWith, get, split, nth, castArray, forEach } = require('lodash')
 const { date: toDate, isUrl, isMime } = require('@metascraper/helpers')
+const { get, split, nth, castArray, forEach } = require('lodash')
 const { TAGS: URL_TAGS } = require('html-urls')
 const replaceString = require('replace-string')
 const isHTML = require('is-html-content')
@@ -75,7 +75,8 @@ const rewriteHtmlUrls = ({ $, url }) => {
     $(tagName.join(',')).each(function () {
       const el = $(this)
       const attr = el.attr(urlAttr)
-      if (startsWith(attr, '/')) {
+
+      if (typeof attr === 'string' && !attr.startsWith('http')) {
         try {
           const newAttr = new URL(attr, url).toString()
           el.attr(urlAttr, newAttr)
@@ -166,5 +167,3 @@ module.exports = ({
 
   return rewriteUrls ? rewriteCssUrls({ html: $.html(), url }) : $.html()
 }
-
-module.exports.isHTML = isHTML

package/src/index.js

@@ -161,7 +161,7 @@ module.exports = async (
     rewriteUrls = false
   } = {}
 ) => {
-  if (!getBrowserless) {
+  if (!getBrowserless && prerender !== false) {
     throw TypeError(
       "Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
     )