html-get 2.9.13 → 2.9.16-0

package/README.md

@@ -38,21 +38,37 @@ $ npm install puppeteer html-get --save
 ## Usage
 
 ```js
-'use strict'
-
+const createBrowserless = require('browserless')
 const getHTML = require('html-get')
 
-getHTML('https://example.com').then(
-  ({ url, html, stats, headers, statusCode }) =>
-    console.log(`
-       url: ${url}
-      html: ${Buffer.from(html).byteLength} bytes (HTTP ${statusCode})
-      time: ${stats.timing} (${stats.mode})
-   headers: ${Object.keys(headers).reduce(
-     (acc, key) => `${acc}${key}=${headers[key]} `,
-     ''
-   )}
-`))
+// Spawn Chromium process once
+const browserlessFactory = createBrowserless()
+
+// Kill the process when Node.js exit
+process.on('exit', () => {
+  console.log('closing resources!')
+  browserlessFactory.close()
+})
+
+const getContent = async url => {
+  // create a browser context inside Chromium process
+  const browserContext = browserlessFactory.createContext()
+  const getBrowserless = () => browserContext
+  const result = await getHTML(url, { getBrowserless })
+  // close the browser context after it's used
+  await getBrowserless((browser) => browser.destroyContext())
+  return result
+}
+
+getContent('https://example.com')
+  .then(content => {
+    console.log(content)
+    process.exit()
+  })
+  .catch(error => {
+    console.error(error)
+    process.exit(1)
+  })
 ```
 
 ### Command Line

package/bin/index.js

@@ -2,18 +2,24 @@
 
 'use strict'
 
+const createBrowserless = require('browserless')
 const minimist = require('minimist')
 const { URL } = require('url')
 
 const getHTML = require('..')
 
+const browserlessFactory = createBrowserless()
+
 const [input, ...argv] = process.argv.slice(2)
 const url = new URL(input).toString()
 
 const { debug: isDebug, ...args } = minimist(argv)
 
-getHTML(url, args)
-  .then(({ html, stats, headers, statusCode }) => {
+const browserContext = browserlessFactory.createContext()
+const getBrowserless = () => browserContext
+
+getHTML(url, { getBrowserless, ...args })
+  .then(async ({ html, stats, headers, statusCode }) => {
     if (isDebug) {
       console.log(`
        url: ${url}
@@ -31,10 +37,13 @@ getHTML(url, args)
     } else {
       console.log(html)
     }
-
     process.exit(0)
   })
-  .catch(err => {
+  .catch(async err => {
     console.error(err)
     process.exit(1)
   })
+  .finally(async () => {
+    await getBrowserless(browser => browser.destroyContext())
+    browserlessFactory.close()
+  })

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.9.13",
+  "version": "2.9.16-0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -29,14 +29,14 @@
     "request"
   ],
   "dependencies": {
-    "@metascraper/helpers": "~5.28.2",
+    "@metascraper/helpers": "~5.29.0",
     "cheerio": "~1.0.0-rc.10",
     "css-url-regex": "~4.0.0",
     "debug-logfmt": "~1.0.4",
     "execall": "~2.0.0",
     "got": "~11.8.2",
     "html-encode": "~2.1.6",
-    "html-urls": "~2.4.33",
+    "html-urls": "~2.4.34",
     "is-html-content": "~1.0.0",
     "lodash": "~4.17.21",
     "minimist": "~1.2.6",
@@ -44,8 +44,8 @@
     "p-retry": "~4.6.0",
     "replace-string": "~3.1.0",
     "time-span": "~4.0.0",
-    "tldts": "~5.7.73",
-    "top-sites": "~1.1.94",
+    "tldts": "~5.7.74",
+    "top-sites": "~1.1.96",
     "write-json-file": "~4.3.0"
   },
   "devDependencies": {
@@ -77,26 +77,12 @@
     "scripts",
     "src"
   ],
-  "scripts": {
-    "clean": "rm -rf node_modules",
-    "contributors": "(git-authors-cli && finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
-    "lint": "standard-markdown README.md && standard",
-    "postinstall": "node scripts/postinstall",
-    "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
-    "prerelease": "npm run update:check && npm run contributors",
-    "pretest": "npm run lint",
-    "release": "standard-version -a",
-    "release:github": "conventional-github-releaser -p angular",
-    "release:tags": "git push --follow-tags origin HEAD:master",
-    "test": "c8 ava",
-    "update": "ncu -u",
-    "update:check": "ncu -- --error-level 2"
-  },
   "license": "MIT",
   "ava": {
+    "workerThreads": false,
     "files": [
-      "!test/util.js",
-      "test/**/*.js"
+      "test/**/*.js",
+      "!test/util.js"
     ],
     "timeout": "2m"
   },
@@ -119,5 +105,21 @@
   "simple-git-hooks": {
     "commit-msg": "npx commitlint --edit",
     "pre-commit": "npx nano-staged"
-  }
-}
+  },
+  "scripts": {
+    "clean": "rm -rf node_modules",
+    "contributors": "(git-authors-cli && finepack --sort-ignore-object-at ava && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
+    "lint": "standard-markdown README.md && standard",
+    "postinstall": "node scripts/postinstall",
+    "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
+    "prerelease": "npm run update:check && npm run contributors",
+    "pretest": "npm run lint",
+    "release": "standard-version -a",
+    "release:github": "conventional-github-releaser -p angular",
+    "release:tags": "git push --follow-tags origin HEAD:master",
+    "test": "c8 ava",
+    "update": "ncu -u",
+    "update:check": "ncu -- --error-level 2"
+  },
+  "readme": "<div align=\"center\">\n  <img src=\"https://cdn.microlink.io/logo/banner.png\" alt=\"microlink cdn\">\n  <br>\n  <br>\n</div>\n\n![Last version](https://img.shields.io/github/tag/microlinkhq/html-get.svg?style=flat-square)\n[![Coverage Status](https://img.shields.io/coveralls/microlinkhq/html-get.svg?style=flat-square)](https://coveralls.io/github/microlinkhq/html-get)\n[![NPM Status](https://img.shields.io/npm/dm/html-get.svg?style=flat-square)](https://www.npmjs.org/package/html-get)\n\n> Get the HTML from any website, using prerendering when is necessary.\n\n## Features\n\n- Get HTML markup from any website (client side apps as well).\n- Prerendering detection based on domains list.\n- Speed up process blocking ads trackers.\n- Encoding body response properly.\n\n<br>\n\nHeadless technology like [puppeteer](https://github.com/GoogleChrome/puppeteer) brings us to get the HTML markup from any website, even when the target URL is client side app and we need to wait until dom events fire for getting the real markup.\n\nGenerally this approach better than a simple GET request from the target URL, but because you need to wait for dom events, prerendering could be slow and in some scenario unnecessary (sites that use server side rendering could be resolved with a simple GET).\n\n**html-get** bring the best of both worlds, doing the following algorithm:\n\n- Determinate if the target URL actually needs prerendering (internally it has a [list of popular site domains](https://github.com/microlinkhq/html-get/blob/master/src/auto-domains.js) that don't need it).\n- If it needs prerendering, perform the action using Headless technology, blocking ads trackers requests for speed up the process, trying to resolve the main request in the minimum amount of time.\n- If it does not need prerendering or prerendering fails for any reason (for example, timeout), the request will be resolved doing a GET request.\n\n## Install\n\n```bash\n$ npm install puppeteer html-get --save\n```\n\n## Usage\n\n```js\nconst createBrowserless = require('browserless')\nconst getHTML = require('html-get')\n\n// Spawn Chromium process once\nconst browserlessFactory = createBrowserless()\n\n// Kill the process when Node.js exit\nprocess.on('exit', () => {\n  console.log('closing resources!')\n  browserlessFactory.close()\n})\n\nconst getContent = async url => {\n  // create a browser context inside Chromium process\n  const browserContext = browserlessFactory.createContext()\n  const getBrowserless = () => browserContext\n  const result = await getHTML(url, { getBrowserless })\n  // close the browser context after it's used\n  await getBrowserless((browser) => browser.destroyContext())\n  return result\n}\n\ngetContent('https://example.com')\n  .then(content => {\n    console.log(content)\n    process.exit()\n  })\n  .catch(error => {\n    console.error(error)\n    process.exit(1)\n  })\n```\n\n### Command Line\n\n```\n$ npx html-get https://example.com\n```\n\n## API\n\n### getHTML(url, [options])\n\n#### url\n\n*Required*<br>\nType: `string`\n\nThe target URL for getting the HTML markup.\n\n##### getBrowserless\n\n*Required*<br>\nType: `function`<br>\n\nA function that should return a [browserless](https://browserless.js.org/) instance to be used for interact with puppeteer:\n\n#### options\n\n##### prerender\n\nType: `boolean`|`string`<br>\nDefault: `'auto'`\n\nEnable or disable prerendering as mechanism for getting the HTML markup explicitly.\n\nThe value `auto` means that that internally use a list of websites that don't need to use prerendering by default. This list is used for speedup the process, using `fetch` mode for these websites.\n\nSee [getMode parameter](#getMode) for know more.\n\n##### encoding\n\nType: `string`<br>\nDefault: `'utf-8'`\n\nEncoding the HTML markup properly from the body response.\n\nIt determines the encode to use A Node.js library for converting HTML documents of arbitrary encoding into a target encoding (utf8, utf16, etc).\n\n##### headers\n\nType: `object`<br>\n\nRequest headers that will be passed to fetch/prerender process.\n\n##### getMode\n\nType: `function`<br>\n\nA function evaluation that will be invoked to determinate the resolutive `mode` for getting the HTML markup from the target URL.\n\nThe default `getMode` is:\n\n```js\nconst getMode = (url, { prerender }) => {\n  if (prerender === false) return 'fetch'\n  if (prerender !== 'auto') return 'prerender'\n  return autoDomains.includes(getDomain(url)) ? 'fetch' : 'prerender'\n}\n```\n\n##### gotOptions\n\nType: `object`<br>\n\nUnder `mode=fetch`, pass configuration object to [got](https://www.npmjs.com/package/got).\n\n##### puppeteerOpts\n\nType: `object`\n\nUnder non `mode=fetch`, pass configuration object to [puppeteer](https://www.npmjs.com/package/puppeteer).\n\n##### rewriteUrls\n\nType: `boolean`<br>\nDefault: `false`\n\nWhen is `true`, it will be rewritten CSS/HTML relatives URLs present in the HTML markup into absolutes.\n\n## License\n\n**html-get** © [Microlink](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/html-get/blob/master/LICENSE.md) License.<br>\nAuthored and maintained by [Kiko Beats](https://kikobeats.com) with help from [contributors](https://github.com/microlinkhq/html-get/contributors).\n\n> [microlink.io](https://microlink.io) · GitHub [@MicrolinkHQ](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq)\n"
+}

package/src/auto-domains.json

@@ -1,41 +1,41 @@
 [
-	"google",
 	"youtube",
-	"microsoft",
 	"apple",
+	"google",
+	"microsoft",
 	"wikipedia",
 	"wordpress",
 	"blogspot",
-	"github",
 	"vimeo",
-	"imdb",
-	"theguardian",
-	"bbc",
+	"github",
 	"nytimes",
+	"imdb",
 	"slideshare",
-	"pinterest",
+	"bbc",
+	"theguardian",
 	"telegraph",
 	"huffingtonpost",
+	"pinterest",
+	"soundcloud",
+	"spotify",
+	"eventbrite",
+	"stackoverflow",
 	"zoom",
 	"techcrunch",
-	"spotify",
 	"yelp",
-	"eventbrite",
-	"soundcloud",
-	"theverge",
 	"engadget",
-	"digg",
-	"imgur",
+	"theverge",
+	"reddit",
 	"etsy",
-	"csdn",
 	"flickr",
+	"digg",
+	"csdn",
 	"ghost",
 	"giphy",
+	"imgur",
 	"meetup",
 	"producthunt",
-	"reddit",
 	"sourceforge",
-	"stackoverflow",
 	"tumblr",
 	"ycombinator"
 ]

package/src/html.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { startsWith, get, split, nth, castArray, forEach } = require('lodash')
 const { date: toDate, isUrl, isMime } = require('@metascraper/helpers')
+const { get, split, nth, castArray, forEach } = require('lodash')
 const { TAGS: URL_TAGS } = require('html-urls')
 const replaceString = require('replace-string')
 const isHTML = require('is-html-content')
@@ -75,7 +75,8 @@ const rewriteHtmlUrls = ({ $, url }) => {
     $(tagName.join(',')).each(function () {
       const el = $(this)
       const attr = el.attr(urlAttr)
-      if (startsWith(attr, '/')) {
+
+      if (typeof attr === 'string' && !attr.startsWith('http')) {
         try {
           const newAttr = new URL(attr, url).toString()
           el.attr(urlAttr, newAttr)
@@ -166,5 +167,3 @@ module.exports = ({
 
   return rewriteUrls ? rewriteCssUrls({ html: $.html(), url }) : $.html()
 }
-
-module.exports.isHTML = isHTML

package/src/index.js

@@ -161,7 +161,7 @@ module.exports = async (
     rewriteUrls = false
   } = {}
 ) => {
-  if (!getBrowserless) {
+  if (!getBrowserless && prerender !== false) {
     throw TypeError(
       "Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
     )