html-get 2.18.4 → 2.19.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.18.4",
+  "version": "2.19.0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -36,22 +36,22 @@
   ],
   "dependencies": {
     "@kikobeats/time-span": "~1.0.5",
-    "@metascraper/helpers": "~5.45.22",
+    "@metascraper/helpers": "~5.46.1",
     "cheerio": "~1.0.0",
     "css-url-regex": "~4.0.0",
-    "debug-logfmt": "~1.2.2",
+    "debug-logfmt": "~1.2.3",
     "execall": "~2.0.0",
     "got": "~11.8.6",
     "html-encode": "~2.1.7",
-    "html-urls": "~2.4.59",
+    "html-urls": "~2.4.62",
     "is-html-content": "~1.0.0",
-    "localhost-url-regex": "~1.0.11",
+    "is-local-address": "~2.2.0",
     "lodash": "~4.17.21",
     "mri": "~1.2.0",
     "p-cancelable": "~2.1.0",
     "p-retry": "~4.6.0",
-    "tinyspawn": "~1.3.2",
-    "top-sites": "~1.1.217"
+    "tinyspawn": "~1.3.3",
+    "top-sites": "~1.1.220"
   },
   "devDependencies": {
     "@commitlint/cli": "latest",
@@ -125,6 +125,9 @@
       "finepack"
     ]
   },
+  "pnpm": {
+    "neverBuiltDependencies": []
+  },
   "simple-git-hooks": {
     "commit-msg": "npx commitlint --edit",
     "pre-commit": "npx nano-staged"

package/src/auto-domains.json

@@ -1 +1 @@
-[[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","github"]],[["domain","x.com"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","arxiv"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]
+[[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","theguardian"]],[["domain","x.com"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","arxiv"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]

package/src/html.js

@@ -2,7 +2,7 @@
 
 const { get, split, nth, castArray, forEach } = require('lodash')
 const debug = require('debug-logfmt')('html-get:rewrite')
-const localhostUrl = require('localhost-url-regex')
+const isLocalAddress = require('is-local-address')
 const { TAGS: URL_TAGS } = require('html-urls')
 const isHTML = require('is-html-content')
 const cssUrl = require('css-url-regex')
@@ -118,15 +118,16 @@ const rewriteHtmlUrls = ({ $, url }) => {
     $(tagName.join(',')).each(function () {
       const el = $(this)
       const attr = el.attr(urlAttr)
-
-      if (localhostUrl().test(attr)) {
-        el.remove()
-      } else if (typeof attr === 'string' && !attr.startsWith('http')) {
-        try {
-          const newAttr = new URL(attr, url).toString()
-          el.attr(urlAttr, newAttr)
-        } catch (_) {}
-      }
+      if (typeof attr !== 'string') return
+      try {
+        const urlObj = new URL(attr, url)
+        if (!urlObj.protocol.startsWith('http')) return
+        if (isLocalAddress(urlObj.hostname)) {
+          el.remove()
+        } else {
+          el.attr(urlAttr, urlObj.toString())
+        }
+      } catch (_) {}
     })
   })
 }