html-get 2.19.0 → 2.21.0

package/README.md

@@ -121,11 +121,14 @@ Type: `object`
 
 Request headers that will be passed to fetch/prerender process.
 
-##### mutoolPath
+##### mutool
 
-Type: `function`
+Type: `function`|`boolean`<br>
+Default: `source code`
+
+It returns a function that receives that executes [mutool](https://mupdf.com/) binary for turning PDF files into HTML markup.
 
-It returns the path for [mutool](https://mupdf.com/) binary, used for turning PDF files into HTML markup.
+It can explicitly disabled passing `false`.
 
 ##### prerender
 
@@ -158,6 +161,12 @@ Default: `false`
 
 When is `true`, it will rewrite some common mistake related with HTML meta tags.
 
+##### serializeHtml
+
+It determines how HTML should be serialied before returning.
+
+It's serialized `$ => ({ html: $.html() })` by default.
+
 ## License
 
 **html-get** © [Microlink](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/html-get/blob/master/LICENSE.md) License.<br>

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.19.0",
+  "version": "2.21.0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -38,6 +38,7 @@
     "@kikobeats/time-span": "~1.0.5",
     "@metascraper/helpers": "~5.46.1",
     "cheerio": "~1.0.0",
+    "content-type": "~1.0.5",
     "css-url-regex": "~4.0.0",
     "debug-logfmt": "~1.2.3",
     "execall": "~2.0.0",
@@ -71,7 +72,6 @@
     "regex-iso-date": "latest",
     "simple-git-hooks": "latest",
     "standard": "latest",
-    "standard-markdown": "latest",
     "standard-version": "latest"
   },
   "engines": {
@@ -85,7 +85,7 @@
   "scripts": {
     "clean": "rm -rf node_modules",
     "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
-    "lint": "standard-markdown README.md && standard",
+    "lint": "standard",
     "postinstall": "node scripts/postinstall",
     "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
     "pretest": "npm run lint",
@@ -98,7 +98,7 @@
   "ava": {
     "files": [
       "test/**/*.js",
-      "!test/util.js"
+      "!test/helpers.js"
     ],
     "timeout": "2m",
     "workerThreads": false
@@ -118,9 +118,6 @@
       "prettier-standard",
       "standard --fix"
     ],
-    "*.md": [
-      "standard-markdown"
-    ],
     "package.json": [
       "finepack"
     ]

package/src/auto-domains.json

@@ -1 +1 @@
-[[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","theguardian"]],[["domain","x.com"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","arxiv"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]
+[[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","bbc"]],[["domain","x.com"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","arxiv"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]

package/src/html.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { get, split, nth, castArray, forEach } = require('lodash')
 const debug = require('debug-logfmt')('html-get:rewrite')
+const { get, castArray, forEach } = require('lodash')
 const isLocalAddress = require('is-local-address')
 const { TAGS: URL_TAGS } = require('html-urls')
 const isHTML = require('is-html-content')
@@ -19,6 +19,8 @@ const {
   parseUrl
 } = require('@metascraper/helpers')
 
+const { getContentType, getCharset } = require('./util')
+
 const has = el => el.length !== 0
 
 const upsert = (el, collection, item) => !has(el) && collection.push(item)
@@ -35,8 +37,7 @@ const getDate = headers => {
 
 const addHead = ({ $, url, headers }) => {
   const tags = []
-  const contentType = get(headers, 'content-type')
-  const charset = nth(split(contentType, 'charset='), 1)
+  const charset = getCharset(headers)
   const date = getDate(headers)
   const { domain } = parseUrl(url)
   const head = $('head')
@@ -73,8 +74,7 @@ const addHead = ({ $, url, headers }) => {
 }
 
 const addBody = ({ url, headers, html }) => {
-  const contentType = get(headers, 'content-type')
-
+  const contentType = getContentType(headers)
   let element = ''
 
   if (isMime(contentType, 'image')) {
@@ -132,24 +132,44 @@ const rewriteHtmlUrls = ({ $, url }) => {
   })
 }
 
-const rewriteCssUrls = ({ html, url }) => {
-  const cssUrls = Array.from(
-    execall(cssUrl(), html).reduce((acc, match) => {
+const replaceCssUrls = (url, stylesheet) => {
+  const cssUrls = Array.from(execall(cssUrl(), stylesheet)).reduce(
+    (acc, match) => {
       match.subMatches.forEach(match => acc.add(match))
       return acc
-    }, new Set())
+    },
+    new Set()
   )
 
   cssUrls.forEach(cssUrl => {
     if (cssUrl.startsWith('/')) {
       try {
         const absoluteUrl = new URL(cssUrl, url).toString()
-        html = html.replaceAll(`url(${cssUrl})`, `url(${absoluteUrl})`)
+        stylesheet = stylesheet.replaceAll(
+          `url(${cssUrl})`,
+          `url(${absoluteUrl})`
+        )
       } catch (_) {}
     }
   })
 
-  return html
+  return stylesheet
+}
+
+const rewriteCssUrls = ({ $, url }) => {
+  // Process <style> tags
+  // e.g., <style>body { background-image: url('/image.jpg'); }</style>
+  $('style').each((_, element) =>
+    $(element).html(replaceCssUrls(url, $(element).html()))
+  )
+
+  // Process elements with style attributes
+  // e.g., <div style="background-image: url('/image.jpg');"></div>
+  $('[style]').each((_, element) =>
+    $(element).attr('style', replaceCssUrls(url, $(element).attr('style')))
+  )
+
+  return $
 }
 
 const injectStyle = ({ $, styles }) =>
@@ -216,7 +236,7 @@ module.exports = ({
   if (scripts) injectScripts({ $, scripts, type: 'text/javascript' })
   if (modules) injectScripts({ $, modules, type: 'module' })
 
-  return rewriteUrls ? rewriteCssUrls({ html: $.html(), url }) : $.html()
+  return rewriteUrls ? rewriteCssUrls({ $, url }) : $
 }
 
 module.exports.getDate = getDate

package/src/index.js

@@ -1,10 +1,10 @@
 'use strict'
 
 const { parseUrl, isMediaUrl, isPdfUrl } = require('@metascraper/helpers')
+const { readFile, writeFile } = require('fs/promises')
 const timeSpan = require('@kikobeats/time-span')()
 const debug = require('debug-logfmt')('html-get')
 const { execSync } = require('child_process')
-const { writeFile } = require('fs/promises')
 const PCancelable = require('p-cancelable')
 const { AbortError } = require('p-retry')
 const htmlEncode = require('html-encode')
@@ -14,6 +14,7 @@ const path = require('path')
 const got = require('got')
 const os = require('os')
 
+const { getContentLength, getContentType } = require('./util')
 const autoDomains = require('./auto-domains')
 const addHtml = require('./html')
 
@@ -21,12 +22,14 @@ const REQ_TIMEOUT = 8000
 
 const ABORT_TYPES = ['image', 'stylesheet', 'font']
 
+const PDF_SIZE_TRESHOLD = 150 * 1024 // 150kb
+
 const fetch = PCancelable.fn(
   async (
     url,
     {
       getTemporalFile,
-      mutoolPath,
+      mutool,
       reflect = false,
       timeout = REQ_TIMEOUT,
       toEncode,
@@ -58,14 +61,22 @@ const fetch = PCancelable.fn(
       const res = await req
 
       const html = await (async () => {
-        const contentType = res.headers['content-type'] ?? ''
-        if (mutoolPath && contentType === 'application/pdf') {
+        const contentType = getContentType(res.headers)
+
+        if (mutool && contentType === 'application/pdf') {
           const file = getTemporalFile(url, 'pdf')
           await writeFile(file.path, res.body)
-          return (await $(`${mutoolPath} draw -q -F html ${file.path}`)).stdout
+          if (getContentLength(res.headers) > PDF_SIZE_TRESHOLD) {
+            const ofile = getTemporalFile(`${url}-pdf`, 'pdf')
+            await mutool(`-o ${ofile.path} ${file.path}`)
+            return readFile(ofile.path, 'utf-8')
+          } else {
+            const { stdout } = await mutool(file.path)
+            return stdout
+          }
         }
 
-        return contentType.startsWith('text/html') || !isMediaUrl(url)
+        return contentType === 'text/html' || !isMediaUrl(url)
           ? await toEncode(res.body, res.headers['content-type'])
           : res.body.toString()
       })()
@@ -193,8 +204,8 @@ const defaultGetMode = (url, { prerender }) => {
   return isFetchMode(url) ? 'fetch' : 'prerender'
 }
 
-const defaultGetTemporalFile = (url, ext) => {
-  const hash = crypto.createHash('sha256').update(url).digest('hex')
+const defaultGetTemporalFile = (input, ext) => {
+  const hash = crypto.createHash('sha256').update(input).digest('hex')
   const filepath = path.join(
     os.tmpdir(),
     ext === undefined ? hash : `${hash}.${ext}`
@@ -202,10 +213,13 @@ const defaultGetTemporalFile = (url, ext) => {
   return { path: filepath }
 }
 
-const defaultMutoolPath = () =>
+const defaultMutool = () =>
   (() => {
     try {
-      return execSync('which mutool', { stdio: 'pipe' }).toString().trim()
+      const mutoolPath = execSync('which mutool', { stdio: 'pipe' })
+        .toString()
+        .trim()
+      return (...args) => $(`${mutoolPath} draw -q -F html ${args}`)
     } catch (_) {}
   })()
 
@@ -218,7 +232,7 @@ const getContent = PCancelable.fn(
       getTemporalFile,
       gotOpts,
       headers,
-      mutoolPath,
+      mutool,
       puppeteerOpts,
       rewriteUrls,
       rewriteHtml,
@@ -229,21 +243,21 @@ const getContent = PCancelable.fn(
     const isFetchMode = mode === 'fetch'
 
     const fetchOpts = isFetchMode
-      ? { headers, toEncode, mutoolPath, getTemporalFile, ...gotOpts }
+      ? { headers, toEncode, mutool, getTemporalFile, ...gotOpts }
       : { headers, toEncode, getBrowserless, gotOpts, ...puppeteerOpts }
 
     const promise = modes[mode](url, fetchOpts)
     onCancel(() => promise.cancel())
 
     return promise.then(content => {
-      const html = addHtml({
+      const $ = addHtml({
         ...content,
         ...(isFetchMode ? puppeteerOpts : undefined),
         rewriteUrls,
         rewriteHtml
       })
 
-      return { ...content, html }
+      return { ...content, $ }
     })
   }
 )
@@ -258,11 +272,12 @@ module.exports = PCancelable.fn(
       getTemporalFile = defaultGetTemporalFile,
       gotOpts,
       headers,
-      mutoolPath = defaultMutoolPath(),
+      mutool = defaultMutool(),
       prerender = 'auto',
       puppeteerOpts,
+      rewriteHtml = false,
       rewriteUrls = false,
-      rewriteHtml = false
+      serializeHtml = $ => ({ html: $.html() })
     } = {},
     onCancel
   ) => {
@@ -282,7 +297,7 @@ module.exports = PCancelable.fn(
       getTemporalFile,
       gotOpts,
       headers,
-      mutoolPath,
+      mutool,
       puppeteerOpts,
       rewriteUrls,
       rewriteHtml,
@@ -291,13 +306,18 @@ module.exports = PCancelable.fn(
 
     onCancel(() => promise.cancel())
 
-    const { mode, ...payload } = await promise
+    const { mode, $, ...payload } = await promise
 
-    return Object.assign(payload, { stats: { mode, timing: duration() } })
+    return Object.assign(payload, {
+      ...serializeHtml($),
+      stats: { mode, timing: duration() }
+    })
   }
 )
 
 module.exports.REQ_TIMEOUT = REQ_TIMEOUT
 module.exports.ABORT_TYPES = ABORT_TYPES
+module.exports.PDF_SIZE_TRESHOLD = PDF_SIZE_TRESHOLD
 module.exports.isFetchMode = isFetchMode
 module.exports.getContent = getContent
+module.exports.defaultMutool = defaultMutool

package/src/util.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { parse } = require('content-type')
+
+const CACHE = Object.create(null)
+
+const parseContentType = contentType =>
+  typeof contentType === 'string'
+    ? parse(contentType)
+    : { type: undefined, parameters: {} }
+
+const contentType = headers => {
+  const contentType = headers['content-type']
+  return (
+    CACHE[contentType] || (CACHE[contentType] = parseContentType(contentType))
+  )
+}
+
+const getContentType = headers => contentType(headers).type
+
+const getCharset = headers =>
+  contentType(headers).parameters.charset?.toLowerCase()
+
+const getContentLength = headers => Number(headers['content-length'])
+
+module.exports = {
+  getCharset,
+  getContentLength,
+  getContentType
+}