html-get 2.14.3 → 2.15.0-0

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, using prerendering when is necessary.",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.14.3",
+  "version": "2.15.0-0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -44,6 +44,7 @@
     "p-cancelable": "~2.1.0",
     "p-retry": "~4.6.0",
     "replace-string": "~3.1.0",
+    "tinyspawn": "~1.2.6",
     "top-sites": "~1.1.202"
   },
   "devDependencies": {
@@ -76,21 +77,6 @@
     "scripts",
     "src"
   ],
-  "scripts": {
-    "clean": "rm -rf node_modules",
-    "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
-    "lint": "standard-markdown README.md && standard",
-    "postinstall": "node scripts/postinstall",
-    "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
-    "prerelease": "npm run update:check && npm run contributors",
-    "pretest": "npm run lint",
-    "release": "standard-version -a",
-    "release:github": "github-generate-release",
-    "release:tags": "git push --follow-tags origin HEAD:master",
-    "test": "c8 ava",
-    "update": "ncu -u",
-    "update:check": "ncu -- --error-level 2"
-  },
   "license": "MIT",
   "ava": {
     "files": [
@@ -120,5 +106,20 @@
   "simple-git-hooks": {
     "commit-msg": "npx commitlint --edit",
     "pre-commit": "npx nano-staged"
+  },
+  "scripts": {
+    "clean": "rm -rf node_modules",
+    "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
+    "lint": "standard-markdown README.md && standard",
+    "postinstall": "node scripts/postinstall",
+    "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
+    "prerelease": "npm run update:check && npm run contributors",
+    "pretest": "npm run lint",
+    "release": "standard-version -a",
+    "release:github": "github-generate-release",
+    "release:tags": "git push --follow-tags origin HEAD:master",
+    "test": "c8 ava",
+    "update": "ncu -u",
+    "update:check": "ncu -- --error-level 2"
   }
-}
+}

package/src/auto-domains.json

@@ -1 +1 @@
-[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","soundcloud"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]
+[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","stackoverflow"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]

package/src/index.js

@@ -1,23 +1,30 @@
 'use strict'
 
-const { parseUrl, isMediaUrl } = require('@metascraper/helpers')
+const { parseUrl, isMediaUrl, isPdfUrl } = require('@metascraper/helpers')
 const timeSpan = require('@kikobeats/time-span')()
 const debug = require('debug-logfmt')('html-get')
+const { execSync } = require('child_process')
+const { writeFile } = require('fs/promises')
 const PCancelable = require('p-cancelable')
 const { AbortError } = require('p-retry')
 const htmlEncode = require('html-encode')
+const crypto = require('crypto')
+const $ = require('tinyspawn')
+const path = require('path')
 const got = require('got')
+const os = require('os')
 
 const autoDomains = require('./auto-domains')
 const addHtml = require('./html')
 
 const REQ_TIMEOUT = 8000
+
 const ABORT_TYPES = ['image', 'stylesheet', 'font']
 
 const fetch = PCancelable.fn(
   async (
     url,
-    { reflect = false, toEncode, timeout = REQ_TIMEOUT, ...opts },
+    { reflect = false, toEncode, timeout = REQ_TIMEOUT, mutoolPath, getTemporalFile, ...opts },
     onCancel
   ) => {
     const reqTimeout = reflect ? timeout / 2 : timeout
@@ -37,13 +44,24 @@ const fetch = PCancelable.fn(
 
     try {
       const res = await req
+
+      const html = await (async () => {
+        const contentType = res.headers['content-type'] ?? ''
+        if (mutoolPath && contentType === 'application/pdf') {
+          const file = getTemporalFile(url, 'pdf')
+          await writeFile(file.path, res.body)
+          return (await $(`mutool draw -q -F html ${file.path}`)).stdout
+        }
+
+        return contentType.startsWith('text/html') ||
+          !isMediaUrl(url)
+          ? await toEncode(res.body, res.headers['content-type'])
+          : res.body
+      })()
+
       return {
         headers: res.headers,
-        html:
-          res.headers['content-type'].startsWith('text/html') ||
-          !isMediaUrl(url)
-            ? await toEncode(res.body, res.headers['content-type'])
-            : res.body,
+        html,
         mode: 'fetch',
         url: res.url,
         statusCode: res.statusCode
@@ -149,22 +167,31 @@ const isFetchMode = url => {
   )
 }
 
-const determinateMode = (url, { prerender }) => {
-  if (prerender === false || isMediaUrl(url)) return 'fetch'
+const defaultGetMode = (url, { prerender }) => {
+  if (prerender === false || isMediaUrl(url) || isPdfUrl(url)) return 'fetch'
   if (prerender === true) return 'prerender'
   return isFetchMode(url) ? 'fetch' : 'prerender'
 }
 
+const defaultGetTemporalFile = (url, ext) => {
+  const hash = crypto.createHash('sha256').update(url).digest('hex')
+  const filepath = path.join(os.tmpdir(), ext === undefined ? hash : `${hash}.${ext}`)
+  return { path: filepath }
+}
+
+const defaultMutoolPath = () => (() => { try { return execSync('which mutool').toString().trim() } catch (_) {} })()
+
 const getContent = PCancelable.fn(
   (
     url,
     mode,
-    { getBrowserless, gotOpts, headers, puppeteerOpts, rewriteUrls, toEncode },
+    { getBrowserless, gotOpts, headers, puppeteerOpts, rewriteUrls, toEncode, mutoolPath, getTemporalFile },
     onCancel
   ) => {
     const isFetchMode = mode === 'fetch'
+
     const fetchOpts = isFetchMode
-      ? { headers, toEncode, ...gotOpts }
+      ? { headers, toEncode, mutoolPath, getTemporalFile, ...gotOpts }
       : { headers, toEncode, getBrowserless, gotOpts, ...puppeteerOpts }
 
     const promise = modes[mode](url, fetchOpts)
@@ -188,9 +215,11 @@ module.exports = PCancelable.fn(
     {
       encoding = 'utf-8',
       getBrowserless,
-      getMode = determinateMode,
+      getMode = defaultGetMode,
+      getTemporalFile = defaultGetTemporalFile,
       gotOpts,
       headers,
+      mutoolPath = defaultMutoolPath(),
       prerender = 'auto',
       puppeteerOpts,
       rewriteUrls = false
@@ -210,8 +239,10 @@ module.exports = PCancelable.fn(
 
     const promise = getContent(targetUrl, reqMode, {
       getBrowserless,
+      getTemporalFile,
       gotOpts,
       headers,
+      mutoolPath,
       puppeteerOpts,
       rewriteUrls,
       toEncode