@zachleat/fetch-links 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/links.js +111 -0
- package/package.json +31 -0
- package/test/test.js +16 -0
package/README.md
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Fetch Links
|
|
2
|
+
|
|
3
|
+
## Install
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
npm install @zachleat/fetch-links
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
|
|
11
|
+
```js
|
|
12
|
+
import { Links } from "@zachleat/fetch-links";
|
|
13
|
+
|
|
14
|
+
// returns <Array>
|
|
15
|
+
await Links.find("https://www.zachleat.com/");
|
|
16
|
+
|
|
17
|
+
/* [
|
|
18
|
+
{ url: 'mailto:zach@zachleat.com', via: [ 'link[href][rel="me"]' ] },
|
|
19
|
+
{ url: 'https://github.com/zachleat', via: [ 'link[href][rel="me"]' ] },
|
|
20
|
+
// …
|
|
21
|
+
] */
|
|
22
|
+
```
|
package/links.js
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import Fetch from "@11ty/eleventy-fetch";
|
|
2
|
+
import posthtml from "posthtml";
|
|
3
|
+
import posthtmlUrls from "@11ty/posthtml-urls";
|
|
4
|
+
|
|
5
|
+
export class Links {
|
|
6
|
+
static async find(originalUrl) {
|
|
7
|
+
let ln = new Links();
|
|
8
|
+
let html = await ln.fetch(originalUrl);
|
|
9
|
+
let urls = await ln.findUrls(html, {
|
|
10
|
+
originalUrl
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
return Links.filterAll(urls, { originalUrl });
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
async fetch(url) {
|
|
17
|
+
return Fetch(url, {
|
|
18
|
+
type: "text",
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
verbose: true,
|
|
22
|
+
showErrors: true,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
static normalizeUrl(url, origin) {
|
|
27
|
+
return (new URL(url, origin)).toString();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async findUrls(html, options = {}) {
|
|
31
|
+
let { originalUrl } = options;
|
|
32
|
+
let urls = new Map();
|
|
33
|
+
|
|
34
|
+
let posthtmlOptions = {
|
|
35
|
+
eachURL: function(url, attr, tagName, node) {
|
|
36
|
+
if(url.startsWith("#") || tagName === "img" || tagName === "video" || tagName === "source") {
|
|
37
|
+
return url;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
let normalized = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
|
|
41
|
+
let key = normalized;
|
|
42
|
+
let via = `${tagName}[${attr}]${node.attrs.rel ? `[rel="${node.attrs.rel}"]` : ""}`;
|
|
43
|
+
if(urls.has(key)) {
|
|
44
|
+
urls.get(key).via.add(via);
|
|
45
|
+
} else {
|
|
46
|
+
urls.set(key, {
|
|
47
|
+
url: normalized,
|
|
48
|
+
via: new Set([via]),
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return url;
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
await posthtml().use( posthtmlUrls(posthtmlOptions) ).process(html);
|
|
57
|
+
|
|
58
|
+
return Array.from(urls.values().map(entry => {
|
|
59
|
+
entry.via = Array.from(entry.via);
|
|
60
|
+
return entry;
|
|
61
|
+
}));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
static filterAll(results, options = {}) {
|
|
65
|
+
let { originalUrl } = options;
|
|
66
|
+
|
|
67
|
+
let filtered = results
|
|
68
|
+
.filter(entry => this.onlyKeepExternal(entry, originalUrl))
|
|
69
|
+
.filter(entry => this.onlyKeepRelevant(entry));
|
|
70
|
+
|
|
71
|
+
return this.sortByRelMe(filtered);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
static sortByRelMe(results) {
|
|
75
|
+
return results.sort((a, b) => {
|
|
76
|
+
let aHasRelMe = a.via.find(via => via.includes(`[rel="me"]`));
|
|
77
|
+
let bHasRelMe = b.via.find(via => via.includes(`[rel="me"]`));
|
|
78
|
+
if(aHasRelMe && bHasRelMe) {
|
|
79
|
+
return 0;
|
|
80
|
+
}
|
|
81
|
+
if(aHasRelMe) {
|
|
82
|
+
return -1;
|
|
83
|
+
}
|
|
84
|
+
if(bHasRelMe) {
|
|
85
|
+
return 1;
|
|
86
|
+
}
|
|
87
|
+
return a.url.length - b.url.length;
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
static onlyKeepExternal(entry, originalUrl) {
|
|
92
|
+
let comparison = originalUrl || "https://example.com";
|
|
93
|
+
let u = new URL(entry.url, comparison);
|
|
94
|
+
return !u.toString().startsWith(comparison);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
static onlyKeepRelevant(entry) {
|
|
98
|
+
if(entry.via.find(via => via.startsWith("script[") ||
|
|
99
|
+
via.includes(`[rel="dns-prefetch"]`) ||
|
|
100
|
+
via.includes(`[rel="token_endpoint"]`) ||
|
|
101
|
+
via.includes(`[rel="authorization_endpoint"]`) ||
|
|
102
|
+
via.includes(`[rel="publickey"]`) ||
|
|
103
|
+
via.includes(`[rel="preconnect"]`) ||
|
|
104
|
+
via.includes(`[rel="webmention"]`) ||
|
|
105
|
+
via.includes(`[rel="pingback"]`) ||
|
|
106
|
+
via.startsWith("link[") && via.includes(`[rel="stylesheet"]`))) {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
return true;
|
|
110
|
+
}
|
|
111
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@zachleat/fetch-links",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Utility to fetch hyperlinks from a URL.",
|
|
5
|
+
"main": "links.js",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"scripts": {
|
|
8
|
+
"test": "node --test"
|
|
9
|
+
},
|
|
10
|
+
"author": "Zach Leatherman <zachleatherman@gmail.com> (https://zachleat.com/)",
|
|
11
|
+
"license": "MIT",
|
|
12
|
+
"publishConfig": {
|
|
13
|
+
"access": "public"
|
|
14
|
+
},
|
|
15
|
+
"repository": {
|
|
16
|
+
"type": "git",
|
|
17
|
+
"url": "git+https://github.com/zachleat/fetch-links.git"
|
|
18
|
+
},
|
|
19
|
+
"bugs": {
|
|
20
|
+
"url": "https://github.com/zachleat/fetch-links/issues"
|
|
21
|
+
},
|
|
22
|
+
"homepage": "https://github.com/zachleat/fetch-links#readme",
|
|
23
|
+
"dependencies": {
|
|
24
|
+
"@11ty/eleventy-fetch": "^5.1.2",
|
|
25
|
+
"@11ty/posthtml-urls": "^1.0.3",
|
|
26
|
+
"posthtml": "^0.16.7"
|
|
27
|
+
},
|
|
28
|
+
"devDependencies": {
|
|
29
|
+
"http-server": "^14.1.1"
|
|
30
|
+
}
|
|
31
|
+
}
|
package/test/test.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
|
|
4
|
+
import { Links } from "../links.js";
|
|
5
|
+
|
|
6
|
+
test("zachleat.com", async (t) => {
|
|
7
|
+
let originalUrl = "https://www.zachleat.com/";
|
|
8
|
+
// let originalUrl = "https://matthiasott.com/";
|
|
9
|
+
// let originalUrl = "https://www.mikeaparicio.com/";
|
|
10
|
+
// let originalUrl = "https://www.miriamsuzanne.com/";
|
|
11
|
+
// let originalUrl = "https://daverupert.com/";
|
|
12
|
+
// let originalUrl = "https://aaadaaam.com/";
|
|
13
|
+
|
|
14
|
+
let filtered = await Links.find(originalUrl);
|
|
15
|
+
assert.ok(filtered.length > 0);
|
|
16
|
+
});
|