html-urls 2.4.68 → 2.5.1

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "html-urls",
-  "description": "Get all links from a HTML markup",
+  "description": "Extract all href and src URLs from HTML markup for crawling, auditing, and link checks.",
   "homepage": "https://github.com/Kikobeats/html-urls",
-  "version": "2.4.68",
+  "version": "2.5.1",
   "main": "src/index.js",
   "author": {
     "email": "josefrancisco.verdu@gmail.com",
@@ -18,17 +18,19 @@
     "url": "https://github.com/Kikobeats/html-urls/issues"
   },
   "keywords": [
+    "anchors",
+    "crawl",
+    "extract",
     "href",
-    "hrefs",
     "html",
-    "link",
     "links",
+    "parser",
     "src",
     "url",
     "urls"
   ],
   "dependencies": {
-    "@metascraper/helpers": "~5.50.1",
+    "@metascraper/helpers": "~5.53.0",
     "cheerio": "~1.2.0",
     "is-uri": "~1.2.14",
     "is-url-http": "~2.3.13",

package/src/index.js

@@ -34,6 +34,17 @@ const reduceSelector = (collection, fn, acc = []) => {
 
 const includes = (collection, fn) => findIndex(collection, fn) !== -1
 
+const getDedupeKey = link => link.uri || link.value
+
+// normalizeUrl adds a trailing slash to bare hosts, and `*` is matcher's only
+// wildcard, so exact entries need the same treatment the uri already got.
+const normalizeWhitelist = whitelist => {
+  if (isEmpty(whitelist)) return whitelist
+  return whitelist.map(pattern =>
+    pattern.includes('*') ? pattern : normalizeUrl(pattern) || pattern
+  )
+}
+
 const getLink = ({ url, el, attribute }) => {
   const attr = get(el, `attribs.${attribute}`, '')
   if (isEmpty(attr)) return undefined
@@ -47,7 +58,7 @@ const getLink = ({ url, el, attribute }) => {
 
 const createGetLinksByAttribute = ({ removeDuplicates }) => {
   const has = removeDuplicates
-    ? (acc, uid) => includes(acc, item => get(item, UID) === uid)
+    ? (acc, key) => includes(acc, item => getDedupeKey(item) === key)
     : () => false
 
   return ({ selector, attribute, url, whitelist }) =>
@@ -55,10 +66,11 @@ const createGetLinksByAttribute = ({ removeDuplicates }) => {
       selector,
       (acc, el) => {
         const link = getLink({ url, el, attribute })
-        const uid = get(link, UID)
         if (isEmpty(link)) return acc
-        const isAlreadyAdded = has(acc, uid)
+        const key = getDedupeKey(link)
+        const isAlreadyAdded = has(acc, key)
         if (isAlreadyAdded) return acc
+        const uid = get(link, UID)
         const match = !isEmpty(whitelist) && matcher([uid], concat(whitelist))
         return isEmpty(match) ? concat(acc, link) : acc
       },
@@ -68,7 +80,7 @@ const createGetLinksByAttribute = ({ removeDuplicates }) => {
 
 const createAdd = ({ removeDuplicates }) =>
   removeDuplicates
-    ? (acc, links) => uniqBy(concat(acc, links), UID)
+    ? (acc, links) => uniqBy(concat(acc, links), getDedupeKey)
     : (acc, links) => concat(acc, links)
 
 module.exports = ({
@@ -79,6 +91,7 @@ module.exports = ({
   cheerioOpts = {}
 } = {}) => {
   const $ = cheerio.load(html, cheerioOpts)
+  const normalizedWhitelist = normalizeWhitelist(whitelist)
 
   const add = createAdd({ removeDuplicates })
   const getLinksByAttribute = createGetLinksByAttribute({ removeDuplicates })
@@ -90,7 +103,7 @@ module.exports = ({
         selector: $(htmlTags.join(',')),
         attribute,
         url,
-        whitelist
+        whitelist: normalizedWhitelist
       })
       return add(acc, links)
     },