html-get 2.16.10 → 2.17.0-0

package/README.md

@@ -151,6 +151,13 @@ Default: `false`
 
 When is `true`, it will be rewritten CSS/HTML relatives URLs present in the HTML markup into absolutes.
 
+##### rewriteHtml
+
+Type: `boolean`<br>
+Default: `false`
+
+When is `true`, it will rewrite some common mistake related with HTML meta tags.
+
 ## License
 
 **html-get** © [Microlink](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/html-get/blob/master/LICENSE.md) License.<br>

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.16.10",
+  "version": "2.17.0-0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -81,18 +81,6 @@
     "scripts",
     "src"
   ],
-  "scripts": {
-    "clean": "rm -rf node_modules",
-    "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
-    "lint": "standard-markdown README.md && standard",
-    "postinstall": "node scripts/postinstall",
-    "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
-    "pretest": "npm run lint",
-    "release": "standard-version -a",
-    "release:github": "github-generate-release",
-    "release:tags": "git push --follow-tags origin HEAD:master",
-    "test": "c8 ava"
-  },
   "license": "MIT",
   "ava": {
     "files": [
@@ -127,5 +115,17 @@
   "simple-git-hooks": {
     "commit-msg": "npx commitlint --edit",
     "pre-commit": "npx nano-staged"
+  },
+  "scripts": {
+    "clean": "rm -rf node_modules",
+    "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
+    "lint": "standard-markdown README.md && standard",
+    "postinstall": "node scripts/postinstall",
+    "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
+    "pretest": "npm run lint",
+    "release": "standard-version -a",
+    "release:github": "github-generate-release",
+    "release:tags": "git push --follow-tags origin HEAD:master",
+    "test": "c8 ava"
   }
-}
+}

package/src/auto-domains.json

@@ -1 +1 @@
-[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domain","x.com"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","eventbrite"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","arxiv"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]
+[[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domain","x.com"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","arxiv"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]

package/src/html.js

@@ -89,6 +89,22 @@ const addBody = ({ url, headers, html }) => {
   return `<!DOCTYPE html><html><head></head><body>${element}</body></html>`
 }
 
+const rewriteOpenGraph = ({ $ }) =>
+  $('meta[name^="og:"]').each((_, element) => {
+    const el = $(element)
+    const name = el.attr('name')
+    el.removeAttr('name')
+    el.attr('property', name)
+  })
+
+const rewriteMetaProperty = ({ $ }) =>
+  $('meta[property]:not([property^="og"])').each((_, element) => {
+    const el = $(element)
+    const property = el.attr('property')
+    el.removeAttr('property')
+    el.attr('name', property)
+  })
+
 const rewriteHtmlUrls = ({ $, url }) => {
   forEach(URL_TAGS, (tagName, urlAttr) => {
     $(tagName.join(',')).each(function () {
@@ -156,6 +172,7 @@ module.exports = ({
   hide,
   remove,
   rewriteUrls,
+  rewriteHtml,
   scripts,
   modules
 }) => {
@@ -167,6 +184,11 @@ module.exports = ({
 
   if (rewriteUrls) rewriteHtmlUrls({ $, url })
 
+  if (rewriteHtml) {
+    rewriteOpenGraph({ $ })
+    rewriteMetaProperty({ $ })
+  }
+
   addHead({ $, url, headers })
 
   if (styles) injectStyle({ $, styles })

package/src/index.js

@@ -57,7 +57,7 @@ const fetch = PCancelable.fn(
         if (mutoolPath && contentType === 'application/pdf') {
           const file = getTemporalFile(url, 'pdf')
           await writeFile(file.path, res.body)
-          return (await $(`mutool draw -q -F html ${file.path}`)).stdout
+          return (await $(`${mutoolPath} draw -q -F html ${file.path}`)).stdout
         }
 
         return contentType.startsWith('text/html') || !isMediaUrl(url)
@@ -191,7 +191,7 @@ const defaultGetTemporalFile = (url, ext) => {
 const defaultMutoolPath = () =>
   (() => {
     try {
-      return execSync('which mutool').toString().trim()
+      return execSync('which mutool', { stdio: 'pipe' }).toString().trim()
     } catch (_) {}
   })()
 
@@ -207,6 +207,7 @@ const getContent = PCancelable.fn(
       mutoolPath,
       puppeteerOpts,
       rewriteUrls,
+      rewriteHtml,
       toEncode
     },
     onCancel
@@ -224,7 +225,8 @@ const getContent = PCancelable.fn(
       const html = addHtml({
         ...content,
         ...(isFetchMode ? puppeteerOpts : undefined),
-        rewriteUrls
+        rewriteUrls,
+        rewriteHtml
       })
 
       return { ...content, html }
@@ -245,7 +247,8 @@ module.exports = PCancelable.fn(
       mutoolPath = defaultMutoolPath(),
       prerender = 'auto',
       puppeteerOpts,
-      rewriteUrls = false
+      rewriteUrls = false,
+      rewriteHtml = false
     } = {},
     onCancel
   ) => {
@@ -268,6 +271,7 @@ module.exports = PCancelable.fn(
       mutoolPath,
       puppeteerOpts,
       rewriteUrls,
+      rewriteHtml,
       toEncode
     })