html-get 2.24.7 → 2.24.9

package/README.md

@@ -13,7 +13,7 @@
 
 ## Features
 
-- Get HTML markup for any URL, including images, video, audio, or pdf.
+- Get HTML markup for any URL, including images, video, audio, pdf, or office documents (docx, xlsx, pptx, odt, rtf, epub).
 - Block ads tracker or any non-necessary network subrequest.
 - Handle unreachable or timeout URLs gracefully.
 - Ensure HTML markup is appropriately encoded.
@@ -130,6 +130,17 @@ It returns a function that receives that executes [mutool](https://mupdf.com/) b
 
 It can explicitly disabled passing `false`.
 
+##### pandoc
+
+Type: `function`|`boolean`<br>
+Default: `source code`
+
+It returns a function that executes the [pandoc](https://pandoc.org/) binary for turning office documents (docx, xlsx, pptx, odt, rtf, epub) into HTML markup.
+
+The format is resolved from the response content-type, falling back to the URL extension.
+
+It can be explicitly disabled passing `false`.
+
 ##### prerender
 
 Type: `boolean`|`string`<br>

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.24.7",
+  "version": "2.24.9",
   "types": "index.d.ts",
   "main": "src/index.js",
   "bin": {

package/src/index.js

@@ -15,6 +15,7 @@ const got = require('got')
 const os = require('os')
 
 const { getContentLength, getContentType } = require('./util')
+const { getOfficeFormat, isOfficeUrl } = require('./office')
 const autoDomains = require('./auto-domains')
 const addHtml = require('./html')
 
@@ -30,6 +31,7 @@ const fetch = PCancelable.fn(
     {
       getTemporalFile,
       mutool,
+      pandoc,
       reflect = false,
       timeout = REQ_TIMEOUT,
       toEncode,
@@ -63,7 +65,12 @@ const fetch = PCancelable.fn(
       const html = await (async () => {
         const contentType = getContentType(res.headers)
 
-        if (mutool && contentType === 'application/pdf') {
+        const officeFormat =
+          pandoc && getOfficeFormat({ contentType, url: [res.url, url] })
+
+        // a recognized office file never goes through mutool, even if the
+        // response is mislabeled as application/pdf
+        if (mutool && !officeFormat && contentType === 'application/pdf') {
           const file = getTemporalFile(url, 'pdf')
           await writeFile(file.path, res.body)
           if (getContentLength(res.headers) > PDF_SIZE_TRESHOLD) {
@@ -76,6 +83,12 @@ const fetch = PCancelable.fn(
           }
         }
 
+        if (officeFormat) {
+          const file = getTemporalFile(res.url, officeFormat)
+          await writeFile(file.path, res.body)
+          return pandoc(officeFormat, file.path)
+        }
+
         return contentType === 'text/html' || !isMediaUrl(url)
           ? await toEncode(res.body, res.headers['content-type'])
           : res.body.toString()
@@ -219,7 +232,14 @@ const isFetchMode = url => {
 }
 
 const defaultGetMode = (url, { prerender }) => {
-  if (prerender === false || isMediaUrl(url) || isPdfUrl(url)) return 'fetch'
+  if (
+    prerender === false ||
+    isMediaUrl(url) ||
+    isPdfUrl(url) ||
+    isOfficeUrl(url)
+  ) {
+    return 'fetch'
+  }
   if (prerender === true) return 'prerender'
   return isFetchMode(url) ? 'fetch' : 'prerender'
 }
@@ -245,6 +265,23 @@ const defaultMutool = () =>
     } catch (_) {}
   })()
 
+const defaultPandoc = () =>
+  (() => {
+    try {
+      const pandocPath = execSync('which pandoc', {
+        stdio: ['pipe', 'pipe', 'ignore']
+      })
+        .toString()
+        .trim()
+      return async (format, filepath) => {
+        const { stdout } = await $(
+          `${pandocPath} --from=${format} --to=html --standalone --embed-resources ${filepath}`
+        )
+        return stdout
+      }
+    } catch (_) {}
+  })()
+
 const getContent = PCancelable.fn(
   (
     url,
@@ -255,6 +292,7 @@ const getContent = PCancelable.fn(
       gotOpts,
       headers,
       mutool,
+      pandoc,
       puppeteerOpts,
       rewriteUrls,
       rewriteHtml,
@@ -265,7 +303,7 @@ const getContent = PCancelable.fn(
     const isFetchMode = mode === 'fetch'
 
     const fetchOpts = isFetchMode
-      ? { headers, toEncode, mutool, getTemporalFile, ...gotOpts }
+      ? { headers, toEncode, mutool, pandoc, getTemporalFile, ...gotOpts }
       : { headers, toEncode, getBrowserless, gotOpts, ...puppeteerOpts }
 
     const promise = modes[mode](url, fetchOpts)
@@ -295,6 +333,7 @@ module.exports = PCancelable.fn(
       gotOpts,
       headers,
       mutool = defaultMutool(),
+      pandoc = defaultPandoc(),
       prerender = 'auto',
       puppeteerOpts,
       rewriteHtml = false,
@@ -320,6 +359,7 @@ module.exports = PCancelable.fn(
       gotOpts,
       headers,
       mutool,
+      pandoc,
       puppeteerOpts,
       rewriteUrls,
       rewriteHtml,
@@ -332,7 +372,12 @@ module.exports = PCancelable.fn(
 
     let shadowDOM = hasShadowDOM($)
 
-    if (mode === 'fetch' && getBrowserless && shadowDOM) {
+    if (
+      mode === 'fetch' &&
+      getBrowserless &&
+      shadowDOM &&
+      prerender !== false
+    ) {
       debug('shadow DOM detected, retrying with prerender', { url: targetUrl })
       const prerenderPromise = getContent(targetUrl, 'prerender', {
         getBrowserless,

package/src/office.js

@@ -0,0 +1,92 @@
+'use strict'
+
+// Office document formats Pandoc reads natively (no LibreOffice needed). Each is
+// converted to HTML at fetch time, so the rest of the pipeline (metadata,
+// markdown, screenshot) treats the document like a regular web page.
+//
+// Legacy OLE binaries (.doc/.xls/.ppt) and OpenDocument spreadsheet/presentation
+// (.ods/.odp) are intentionally absent: Pandoc can't read them, they need a
+// headless LibreOffice in the container.
+const OFFICE_FORMATS = {
+  docx: {
+    extensions: ['docx'],
+    contentTypes: [
+      'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+    ]
+  },
+  xlsx: {
+    extensions: ['xlsx'],
+    contentTypes: [
+      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+    ]
+  },
+  pptx: {
+    extensions: ['pptx'],
+    contentTypes: [
+      'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+    ]
+  },
+  odt: {
+    extensions: ['odt'],
+    contentTypes: ['application/vnd.oasis.opendocument.text']
+  },
+  rtf: {
+    extensions: ['rtf'],
+    contentTypes: ['application/rtf', 'text/rtf']
+  },
+  epub: {
+    extensions: ['epub'],
+    contentTypes: ['application/epub+zip']
+  }
+}
+
+const byContentType = new Map()
+const byExtension = new Map()
+
+for (const [format, { contentTypes, extensions }] of Object.entries(
+  OFFICE_FORMATS
+)) {
+  for (const contentType of contentTypes) byContentType.set(contentType, format)
+  for (const extension of extensions) byExtension.set(extension, format)
+}
+
+const getUrlExtension = url => {
+  const clean = String(url).split('?')[0].split('#')[0]
+  const index = clean.lastIndexOf('.')
+  if (index === -1) return ''
+  const extension = clean.slice(index + 1).toLowerCase()
+  return /^[a-z0-9]+$/.test(extension) ? extension : ''
+}
+
+const getOfficeFormatFromUrl = url => byExtension.get(getUrlExtension(url))
+
+// Content-types that positively identify a web page or media. They are
+// authoritative: the URL extension must never override them (e.g. an HTML page
+// served at a `.docx` vanity URL is still a web page, not an office document).
+const isWebContentType = type =>
+  /^(text\/html|application\/xhtml\+xml|application\/json|image\/|audio\/|video\/)/.test(
+    type || ''
+  )
+
+// Resolve the Pandoc `--from` format for a document:
+//   1. an explicit office content-type wins
+//   2. an explicit web/media content-type means "not an office document"
+//   3. otherwise the content-type is non-committal (octet-stream, zip, missing,
+//      or a mislabeled binary type) so the URL extension decides; `url` may be a
+//      list of candidates, tried in order (e.g. final URL then original request
+//      URL after a redirect)
+const getOfficeFormat = ({ contentType, url } = {}) => {
+  const byType = byContentType.get(contentType)
+  if (byType) return byType
+  if (isWebContentType(contentType)) return undefined
+  const candidates = Array.isArray(url) ? url : [url]
+  for (const candidate of candidates) {
+    const format = candidate && getOfficeFormatFromUrl(candidate)
+    if (format) return format
+  }
+  return undefined
+}
+
+const isOfficeUrl = url => byExtension.has(getUrlExtension(url))
+
+module.exports = { OFFICE_FORMATS, getOfficeFormat, isOfficeUrl }