html-get 2.20.0 → 2.21.1

package/README.md

@@ -121,11 +121,14 @@ Type: `object`
 
 Request headers that will be passed to fetch/prerender process.
 
-##### mutoolPath
+##### mutool
 
-Type: `function`
+Type: `function`|`boolean`<br>
+Default: `source code`
+
+It returns a function that receives that executes [mutool](https://mupdf.com/) binary for turning PDF files into HTML markup.
 
-It returns the path for [mutool](https://mupdf.com/) binary, used for turning PDF files into HTML markup.
+It can explicitly disabled passing `false`.
 
 ##### prerender
 

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.20.0",
+  "version": "2.21.1",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -38,6 +38,7 @@
     "@kikobeats/time-span": "~1.0.5",
     "@metascraper/helpers": "~5.46.1",
     "cheerio": "~1.0.0",
+    "content-type": "~1.0.5",
     "css-url-regex": "~4.0.0",
     "debug-logfmt": "~1.2.3",
     "execall": "~2.0.0",
@@ -50,7 +51,7 @@
     "mri": "~1.2.0",
     "p-cancelable": "~2.1.0",
     "p-retry": "~4.6.0",
-    "tinyspawn": "~1.3.3",
+    "tinyspawn": "~1.4.0",
     "top-sites": "~1.1.220"
   },
   "devDependencies": {
@@ -97,7 +98,7 @@
   "ava": {
     "files": [
       "test/**/*.js",
-      "!test/util.js"
+      "!test/helpers.js"
     ],
     "timeout": "2m",
     "workerThreads": false

package/src/auto-domains.json

@@ -1 +1 @@
-[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","github"]],[["domain","x.com"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","arxiv"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]
+[[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","bbc"]],[["domain","x.com"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","arxiv"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]

package/src/html.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { get, split, nth, castArray, forEach } = require('lodash')
 const debug = require('debug-logfmt')('html-get:rewrite')
+const { get, castArray, forEach } = require('lodash')
 const isLocalAddress = require('is-local-address')
 const { TAGS: URL_TAGS } = require('html-urls')
 const isHTML = require('is-html-content')
@@ -19,6 +19,8 @@ const {
   parseUrl
 } = require('@metascraper/helpers')
 
+const { getContentType, getCharset } = require('./util')
+
 const has = el => el.length !== 0
 
 const upsert = (el, collection, item) => !has(el) && collection.push(item)
@@ -35,8 +37,7 @@ const getDate = headers => {
 
 const addHead = ({ $, url, headers }) => {
   const tags = []
-  const contentType = get(headers, 'content-type')
-  const charset = nth(split(contentType, 'charset='), 1)
+  const charset = getCharset(headers)
   const date = getDate(headers)
   const { domain } = parseUrl(url)
   const head = $('head')
@@ -73,8 +74,7 @@ const addHead = ({ $, url, headers }) => {
 }
 
 const addBody = ({ url, headers, html }) => {
-  const contentType = get(headers, 'content-type')
-
+  const contentType = getContentType(headers)
   let element = ''
 
   if (isMime(contentType, 'image')) {

package/src/index.js

@@ -1,10 +1,10 @@
 'use strict'
 
 const { parseUrl, isMediaUrl, isPdfUrl } = require('@metascraper/helpers')
+const { readFile, writeFile } = require('fs/promises')
 const timeSpan = require('@kikobeats/time-span')()
 const debug = require('debug-logfmt')('html-get')
 const { execSync } = require('child_process')
-const { writeFile } = require('fs/promises')
 const PCancelable = require('p-cancelable')
 const { AbortError } = require('p-retry')
 const htmlEncode = require('html-encode')
@@ -14,6 +14,7 @@ const path = require('path')
 const got = require('got')
 const os = require('os')
 
+const { getContentLength, getContentType } = require('./util')
 const autoDomains = require('./auto-domains')
 const addHtml = require('./html')
 
@@ -21,12 +22,14 @@ const REQ_TIMEOUT = 8000
 
 const ABORT_TYPES = ['image', 'stylesheet', 'font']
 
+const PDF_SIZE_TRESHOLD = 150 * 1024 // 150kb
+
 const fetch = PCancelable.fn(
   async (
     url,
     {
       getTemporalFile,
-      mutoolPath,
+      mutool,
       reflect = false,
       timeout = REQ_TIMEOUT,
       toEncode,
@@ -58,14 +61,22 @@ const fetch = PCancelable.fn(
       const res = await req
 
       const html = await (async () => {
-        const contentType = res.headers['content-type'] ?? ''
-        if (mutoolPath && contentType === 'application/pdf') {
+        const contentType = getContentType(res.headers)
+
+        if (mutool && contentType === 'application/pdf') {
           const file = getTemporalFile(url, 'pdf')
           await writeFile(file.path, res.body)
-          return (await $(`${mutoolPath} draw -q -F html ${file.path}`)).stdout
+          if (getContentLength(res.headers) > PDF_SIZE_TRESHOLD) {
+            const ofile = getTemporalFile(`${url}-pdf`, 'pdf')
+            await mutool(`-o ${ofile.path} ${file.path}`)
+            return readFile(ofile.path, 'utf-8')
+          } else {
+            const { stdout } = await mutool(file.path)
+            return stdout
+          }
         }
 
-        return contentType.startsWith('text/html') || !isMediaUrl(url)
+        return contentType === 'text/html' || !isMediaUrl(url)
           ? await toEncode(res.body, res.headers['content-type'])
           : res.body.toString()
       })()
@@ -193,8 +204,8 @@ const defaultGetMode = (url, { prerender }) => {
   return isFetchMode(url) ? 'fetch' : 'prerender'
 }
 
-const defaultGetTemporalFile = (url, ext) => {
-  const hash = crypto.createHash('sha256').update(url).digest('hex')
+const defaultGetTemporalFile = (input, ext) => {
+  const hash = crypto.createHash('sha256').update(input).digest('hex')
   const filepath = path.join(
     os.tmpdir(),
     ext === undefined ? hash : `${hash}.${ext}`
@@ -202,10 +213,13 @@ const defaultGetTemporalFile = (url, ext) => {
   return { path: filepath }
 }
 
-const defaultMutoolPath = () =>
+const defaultMutool = () =>
   (() => {
     try {
-      return execSync('which mutool', { stdio: 'pipe' }).toString().trim()
+      const mutoolPath = execSync('which mutool', { stdio: 'pipe' })
+        .toString()
+        .trim()
+      return (...args) => $(`${mutoolPath} draw -q -F html ${args}`)
     } catch (_) {}
   })()
 
@@ -218,7 +232,7 @@ const getContent = PCancelable.fn(
       getTemporalFile,
       gotOpts,
       headers,
-      mutoolPath,
+      mutool,
       puppeteerOpts,
       rewriteUrls,
       rewriteHtml,
@@ -229,7 +243,7 @@ const getContent = PCancelable.fn(
     const isFetchMode = mode === 'fetch'
 
     const fetchOpts = isFetchMode
-      ? { headers, toEncode, mutoolPath, getTemporalFile, ...gotOpts }
+      ? { headers, toEncode, mutool, getTemporalFile, ...gotOpts }
       : { headers, toEncode, getBrowserless, gotOpts, ...puppeteerOpts }
 
     const promise = modes[mode](url, fetchOpts)
@@ -258,7 +272,7 @@ module.exports = PCancelable.fn(
       getTemporalFile = defaultGetTemporalFile,
       gotOpts,
       headers,
-      mutoolPath = defaultMutoolPath(),
+      mutool = defaultMutool(),
       prerender = 'auto',
       puppeteerOpts,
       rewriteHtml = false,
@@ -283,7 +297,7 @@ module.exports = PCancelable.fn(
       getTemporalFile,
       gotOpts,
       headers,
-      mutoolPath,
+      mutool,
       puppeteerOpts,
       rewriteUrls,
       rewriteHtml,
@@ -303,5 +317,7 @@ module.exports = PCancelable.fn(
 
 module.exports.REQ_TIMEOUT = REQ_TIMEOUT
 module.exports.ABORT_TYPES = ABORT_TYPES
+module.exports.PDF_SIZE_TRESHOLD = PDF_SIZE_TRESHOLD
 module.exports.isFetchMode = isFetchMode
 module.exports.getContent = getContent
+module.exports.defaultMutool = defaultMutool

package/src/util.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { parse } = require('content-type')
+
+const CACHE = Object.create(null)
+
+const parseContentType = contentType =>
+  typeof contentType === 'string'
+    ? parse(contentType)
+    : { type: undefined, parameters: {} }
+
+const contentType = headers => {
+  const contentType = headers['content-type']
+  return (
+    CACHE[contentType] || (CACHE[contentType] = parseContentType(contentType))
+  )
+}
+
+const getContentType = headers => contentType(headers).type
+
+const getCharset = headers =>
+  contentType(headers).parameters.charset?.toLowerCase()
+
+const getContentLength = headers => Number(headers['content-length'])
+
+module.exports = {
+  getCharset,
+  getContentLength,
+  getContentType
+}