html-get 2.18.5 → 2.20.0

package/README.md

@@ -158,6 +158,12 @@ Default: `false`
 
 When is `true`, it will rewrite some common mistake related with HTML meta tags.
 
+##### serializeHtml
+
+It determines how HTML should be serialied before returning.
+
+It's serialized `$ => ({ html: $.html() })` by default.
+
 ## License
 
 **html-get** © [Microlink](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/html-get/blob/master/LICENSE.md) License.<br>

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.18.5",
+  "version": "2.20.0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -45,7 +45,7 @@
     "html-encode": "~2.1.7",
     "html-urls": "~2.4.62",
     "is-html-content": "~1.0.0",
-    "localhost-url-regex": "~1.0.13",
+    "is-local-address": "~2.2.0",
     "lodash": "~4.17.21",
     "mri": "~1.2.0",
     "p-cancelable": "~2.1.0",
@@ -71,7 +71,6 @@
     "regex-iso-date": "latest",
     "simple-git-hooks": "latest",
     "standard": "latest",
-    "standard-markdown": "latest",
     "standard-version": "latest"
   },
   "engines": {
@@ -85,7 +84,7 @@
   "scripts": {
     "clean": "rm -rf node_modules",
     "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
-    "lint": "standard-markdown README.md && standard",
+    "lint": "standard",
     "postinstall": "node scripts/postinstall",
     "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
     "pretest": "npm run lint",
@@ -118,13 +117,13 @@
       "prettier-standard",
       "standard --fix"
     ],
-    "*.md": [
-      "standard-markdown"
-    ],
     "package.json": [
       "finepack"
     ]
   },
+  "pnpm": {
+    "neverBuiltDependencies": []
+  },
   "simple-git-hooks": {
     "commit-msg": "npx commitlint --edit",
     "pre-commit": "npx nano-staged"

package/src/auto-domains.json

@@ -1 +1 @@
-[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","theguardian"]],[["domain","x.com"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","arxiv"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","theverge"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]
+[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","github"]],[["domain","x.com"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","arxiv"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]

package/src/html.js

@@ -2,7 +2,7 @@
 
 const { get, split, nth, castArray, forEach } = require('lodash')
 const debug = require('debug-logfmt')('html-get:rewrite')
-const localhostUrl = require('localhost-url-regex')
+const isLocalAddress = require('is-local-address')
 const { TAGS: URL_TAGS } = require('html-urls')
 const isHTML = require('is-html-content')
 const cssUrl = require('css-url-regex')
@@ -118,37 +118,58 @@ const rewriteHtmlUrls = ({ $, url }) => {
     $(tagName.join(',')).each(function () {
       const el = $(this)
       const attr = el.attr(urlAttr)
-
-      if (localhostUrl().test(attr)) {
-        el.remove()
-      } else if (typeof attr === 'string' && !attr.startsWith('http')) {
-        try {
-          const newAttr = new URL(attr, url).toString()
-          el.attr(urlAttr, newAttr)
-        } catch (_) {}
-      }
+      if (typeof attr !== 'string') return
+      try {
+        const urlObj = new URL(attr, url)
+        if (!urlObj.protocol.startsWith('http')) return
+        if (isLocalAddress(urlObj.hostname)) {
+          el.remove()
+        } else {
+          el.attr(urlAttr, urlObj.toString())
+        }
+      } catch (_) {}
     })
   })
 }
 
-const rewriteCssUrls = ({ html, url }) => {
-  const cssUrls = Array.from(
-    execall(cssUrl(), html).reduce((acc, match) => {
+const replaceCssUrls = (url, stylesheet) => {
+  const cssUrls = Array.from(execall(cssUrl(), stylesheet)).reduce(
+    (acc, match) => {
       match.subMatches.forEach(match => acc.add(match))
       return acc
-    }, new Set())
+    },
+    new Set()
   )
 
   cssUrls.forEach(cssUrl => {
     if (cssUrl.startsWith('/')) {
       try {
         const absoluteUrl = new URL(cssUrl, url).toString()
-        html = html.replaceAll(`url(${cssUrl})`, `url(${absoluteUrl})`)
+        stylesheet = stylesheet.replaceAll(
+          `url(${cssUrl})`,
+          `url(${absoluteUrl})`
+        )
       } catch (_) {}
     }
   })
 
-  return html
+  return stylesheet
+}
+
+const rewriteCssUrls = ({ $, url }) => {
+  // Process <style> tags
+  // e.g., <style>body { background-image: url('/image.jpg'); }</style>
+  $('style').each((_, element) =>
+    $(element).html(replaceCssUrls(url, $(element).html()))
+  )
+
+  // Process elements with style attributes
+  // e.g., <div style="background-image: url('/image.jpg');"></div>
+  $('[style]').each((_, element) =>
+    $(element).attr('style', replaceCssUrls(url, $(element).attr('style')))
+  )
+
+  return $
 }
 
 const injectStyle = ({ $, styles }) =>
@@ -215,7 +236,7 @@ module.exports = ({
   if (scripts) injectScripts({ $, scripts, type: 'text/javascript' })
   if (modules) injectScripts({ $, modules, type: 'module' })
 
-  return rewriteUrls ? rewriteCssUrls({ html: $.html(), url }) : $.html()
+  return rewriteUrls ? rewriteCssUrls({ $, url }) : $
 }
 
 module.exports.getDate = getDate

package/src/index.js

@@ -236,14 +236,14 @@ const getContent = PCancelable.fn(
     onCancel(() => promise.cancel())
 
     return promise.then(content => {
-      const html = addHtml({
+      const $ = addHtml({
         ...content,
         ...(isFetchMode ? puppeteerOpts : undefined),
         rewriteUrls,
         rewriteHtml
       })
 
-      return { ...content, html }
+      return { ...content, $ }
     })
   }
 )
@@ -261,8 +261,9 @@ module.exports = PCancelable.fn(
       mutoolPath = defaultMutoolPath(),
       prerender = 'auto',
       puppeteerOpts,
+      rewriteHtml = false,
       rewriteUrls = false,
-      rewriteHtml = false
+      serializeHtml = $ => ({ html: $.html() })
     } = {},
     onCancel
   ) => {
@@ -291,9 +292,12 @@ module.exports = PCancelable.fn(
 
     onCancel(() => promise.cancel())
 
-    const { mode, ...payload } = await promise
+    const { mode, $, ...payload } = await promise
 
-    return Object.assign(payload, { stats: { mode, timing: duration() } })
+    return Object.assign(payload, {
+      ...serializeHtml($),
+      stats: { mode, timing: duration() }
+    })
   }
 )