@zachleat/fetch-links 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/links.js +12 -9
- package/package.json +1 -1
package/links.js
CHANGED
|
@@ -3,9 +3,10 @@ import posthtml from "posthtml";
|
|
|
3
3
|
import posthtmlUrls from "@11ty/posthtml-urls";
|
|
4
4
|
|
|
5
5
|
export class Links {
|
|
6
|
-
static async find(originalUrl) {
|
|
6
|
+
static async find(originalUrl, options = {}) {
|
|
7
|
+
let { fetchOptions } = options;
|
|
7
8
|
let ln = new Links();
|
|
8
|
-
let html = await ln.fetch(originalUrl);
|
|
9
|
+
let html = await ln.fetch(originalUrl, fetchOptions);
|
|
9
10
|
let urls = await ln.findUrls(html, {
|
|
10
11
|
originalUrl
|
|
11
12
|
});
|
|
@@ -13,20 +14,22 @@ export class Links {
|
|
|
13
14
|
return Links.filterAll(urls, { originalUrl });
|
|
14
15
|
}
|
|
15
16
|
|
|
16
|
-
async fetch(url) {
|
|
17
|
-
|
|
17
|
+
async fetch(url, options = {}) {
|
|
18
|
+
let fetchOptions = Object.assign({
|
|
18
19
|
type: "text",
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
verbose: true,
|
|
20
|
+
// duration: "1d",
|
|
21
|
+
// verbose: true,
|
|
22
22
|
showErrors: true,
|
|
23
|
-
});
|
|
23
|
+
}, options);
|
|
24
|
+
|
|
25
|
+
return Fetch(url, fetchOptions);
|
|
24
26
|
}
|
|
25
27
|
|
|
26
28
|
static normalizeUrl(url, origin) {
|
|
27
29
|
return (new URL(url, origin)).toString();
|
|
28
30
|
}
|
|
29
31
|
|
|
32
|
+
// TODO de-duplicate via subdomain redirects (see speedlify 11ty leaderboards) e.g. 11ty.dev versus www.11ty.dev
|
|
30
33
|
async findUrls(html, options = {}) {
|
|
31
34
|
let { originalUrl } = options;
|
|
32
35
|
let urls = new Map();
|
|
@@ -68,7 +71,7 @@ export class Links {
|
|
|
68
71
|
.filter(entry => this.onlyKeepExternal(entry, originalUrl))
|
|
69
72
|
.filter(entry => this.onlyKeepRelevant(entry));
|
|
70
73
|
|
|
71
|
-
return
|
|
74
|
+
return filtered;
|
|
72
75
|
}
|
|
73
76
|
|
|
74
77
|
static sortByRelMe(results) {
|