html-urls 2.5.0 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.js +18 -5
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "html-urls",
|
|
3
3
|
"description": "Extract all href and src URLs from HTML markup for crawling, auditing, and link checks.",
|
|
4
4
|
"homepage": "https://github.com/Kikobeats/html-urls",
|
|
5
|
-
"version": "2.5.
|
|
5
|
+
"version": "2.5.1",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"author": {
|
|
8
8
|
"email": "josefrancisco.verdu@gmail.com",
|
package/src/index.js
CHANGED
|
@@ -34,6 +34,17 @@ const reduceSelector = (collection, fn, acc = []) => {
|
|
|
34
34
|
|
|
35
35
|
const includes = (collection, fn) => findIndex(collection, fn) !== -1
|
|
36
36
|
|
|
37
|
+
const getDedupeKey = link => link.uri || link.value
|
|
38
|
+
|
|
39
|
+
// normalizeUrl adds a trailing slash to bare hosts, and `*` is matcher's only
|
|
40
|
+
// wildcard, so exact entries need the same treatment the uri already got.
|
|
41
|
+
const normalizeWhitelist = whitelist => {
|
|
42
|
+
if (isEmpty(whitelist)) return whitelist
|
|
43
|
+
return whitelist.map(pattern =>
|
|
44
|
+
pattern.includes('*') ? pattern : normalizeUrl(pattern) || pattern
|
|
45
|
+
)
|
|
46
|
+
}
|
|
47
|
+
|
|
37
48
|
const getLink = ({ url, el, attribute }) => {
|
|
38
49
|
const attr = get(el, `attribs.${attribute}`, '')
|
|
39
50
|
if (isEmpty(attr)) return undefined
|
|
@@ -47,7 +58,7 @@ const getLink = ({ url, el, attribute }) => {
|
|
|
47
58
|
|
|
48
59
|
const createGetLinksByAttribute = ({ removeDuplicates }) => {
|
|
49
60
|
const has = removeDuplicates
|
|
50
|
-
? (acc,
|
|
61
|
+
? (acc, key) => includes(acc, item => getDedupeKey(item) === key)
|
|
51
62
|
: () => false
|
|
52
63
|
|
|
53
64
|
return ({ selector, attribute, url, whitelist }) =>
|
|
@@ -55,10 +66,11 @@ const createGetLinksByAttribute = ({ removeDuplicates }) => {
|
|
|
55
66
|
selector,
|
|
56
67
|
(acc, el) => {
|
|
57
68
|
const link = getLink({ url, el, attribute })
|
|
58
|
-
const uid = get(link, UID)
|
|
59
69
|
if (isEmpty(link)) return acc
|
|
60
|
-
const
|
|
70
|
+
const key = getDedupeKey(link)
|
|
71
|
+
const isAlreadyAdded = has(acc, key)
|
|
61
72
|
if (isAlreadyAdded) return acc
|
|
73
|
+
const uid = get(link, UID)
|
|
62
74
|
const match = !isEmpty(whitelist) && matcher([uid], concat(whitelist))
|
|
63
75
|
return isEmpty(match) ? concat(acc, link) : acc
|
|
64
76
|
},
|
|
@@ -68,7 +80,7 @@ const createGetLinksByAttribute = ({ removeDuplicates }) => {
|
|
|
68
80
|
|
|
69
81
|
const createAdd = ({ removeDuplicates }) =>
|
|
70
82
|
removeDuplicates
|
|
71
|
-
? (acc, links) => uniqBy(concat(acc, links),
|
|
83
|
+
? (acc, links) => uniqBy(concat(acc, links), getDedupeKey)
|
|
72
84
|
: (acc, links) => concat(acc, links)
|
|
73
85
|
|
|
74
86
|
module.exports = ({
|
|
@@ -79,6 +91,7 @@ module.exports = ({
|
|
|
79
91
|
cheerioOpts = {}
|
|
80
92
|
} = {}) => {
|
|
81
93
|
const $ = cheerio.load(html, cheerioOpts)
|
|
94
|
+
const normalizedWhitelist = normalizeWhitelist(whitelist)
|
|
82
95
|
|
|
83
96
|
const add = createAdd({ removeDuplicates })
|
|
84
97
|
const getLinksByAttribute = createGetLinksByAttribute({ removeDuplicates })
|
|
@@ -90,7 +103,7 @@ module.exports = ({
|
|
|
90
103
|
selector: $(htmlTags.join(',')),
|
|
91
104
|
attribute,
|
|
92
105
|
url,
|
|
93
|
-
whitelist
|
|
106
|
+
whitelist: normalizedWhitelist
|
|
94
107
|
})
|
|
95
108
|
return add(acc, links)
|
|
96
109
|
},
|