@zachleat/fetch-links 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +22 -0
  2. package/links.js +111 -0
  3. package/package.json +31 -0
  4. package/test/test.js +16 -0
package/README.md ADDED
@@ -0,0 +1,22 @@
1
+ # Fetch Links
2
+
3
+ ## Install
4
+
5
+ ```
6
+ npm install @zachleat/fetch-links
7
+ ```
8
+
9
+ ## Usage
10
+
11
+ ```js
12
+ import { Links } from "@zachleat/fetch-links";
13
+
14
+ // returns <Array>
15
+ await Links.find("https://www.zachleat.com/");
16
+
17
+ /* [
18
+ { url: 'mailto:zach@zachleat.com', via: [ 'link[href][rel="me"]' ] },
19
+ { url: 'https://github.com/zachleat', via: [ 'link[href][rel="me"]' ] },
20
+ // …
21
+ ] */
22
+ ```
package/links.js ADDED
@@ -0,0 +1,111 @@
1
+ import Fetch from "@11ty/eleventy-fetch";
2
+ import posthtml from "posthtml";
3
+ import posthtmlUrls from "@11ty/posthtml-urls";
4
+
5
+ export class Links {
6
+ static async find(originalUrl) {
7
+ let ln = new Links();
8
+ let html = await ln.fetch(originalUrl);
9
+ let urls = await ln.findUrls(html, {
10
+ originalUrl
11
+ });
12
+
13
+ return Links.filterAll(urls, { originalUrl });
14
+ }
15
+
16
+ async fetch(url) {
17
+ return Fetch(url, {
18
+ type: "text",
19
+ },
20
+ {
21
+ verbose: true,
22
+ showErrors: true,
23
+ });
24
+ }
25
+
26
+ static normalizeUrl(url, origin) {
27
+ return (new URL(url, origin)).toString();
28
+ }
29
+
30
+ async findUrls(html, options = {}) {
31
+ let { originalUrl } = options;
32
+ let urls = new Map();
33
+
34
+ let posthtmlOptions = {
35
+ eachURL: function(url, attr, tagName, node) {
36
+ if(url.startsWith("#") || tagName === "img" || tagName === "video" || tagName === "source") {
37
+ return url;
38
+ }
39
+
40
+ let normalized = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
41
+ let key = normalized;
42
+ let via = `${tagName}[${attr}]${node.attrs.rel ? `[rel="${node.attrs.rel}"]` : ""}`;
43
+ if(urls.has(key)) {
44
+ urls.get(key).via.add(via);
45
+ } else {
46
+ urls.set(key, {
47
+ url: normalized,
48
+ via: new Set([via]),
49
+ });
50
+ }
51
+
52
+ return url;
53
+ }
54
+ };
55
+
56
+ await posthtml().use( posthtmlUrls(posthtmlOptions) ).process(html);
57
+
58
+ return Array.from(urls.values().map(entry => {
59
+ entry.via = Array.from(entry.via);
60
+ return entry;
61
+ }));
62
+ }
63
+
64
+ static filterAll(results, options = {}) {
65
+ let { originalUrl } = options;
66
+
67
+ let filtered = results
68
+ .filter(entry => this.onlyKeepExternal(entry, originalUrl))
69
+ .filter(entry => this.onlyKeepRelevant(entry));
70
+
71
+ return this.sortByRelMe(filtered);
72
+ }
73
+
74
+ static sortByRelMe(results) {
75
+ return results.sort((a, b) => {
76
+ let aHasRelMe = a.via.find(via => via.includes(`[rel="me"]`));
77
+ let bHasRelMe = b.via.find(via => via.includes(`[rel="me"]`));
78
+ if(aHasRelMe && bHasRelMe) {
79
+ return 0;
80
+ }
81
+ if(aHasRelMe) {
82
+ return -1;
83
+ }
84
+ if(bHasRelMe) {
85
+ return 1;
86
+ }
87
+ return a.url.length - b.url.length;
88
+ });
89
+ }
90
+
91
+ static onlyKeepExternal(entry, originalUrl) {
92
+ let comparison = originalUrl || "https://example.com";
93
+ let u = new URL(entry.url, comparison);
94
+ return !u.toString().startsWith(comparison);
95
+ }
96
+
97
+ static onlyKeepRelevant(entry) {
98
+ if(entry.via.find(via => via.startsWith("script[") ||
99
+ via.includes(`[rel="dns-prefetch"]`) ||
100
+ via.includes(`[rel="token_endpoint"]`) ||
101
+ via.includes(`[rel="authorization_endpoint"]`) ||
102
+ via.includes(`[rel="publickey"]`) ||
103
+ via.includes(`[rel="preconnect"]`) ||
104
+ via.includes(`[rel="webmention"]`) ||
105
+ via.includes(`[rel="pingback"]`) ||
106
+ via.startsWith("link[") && via.includes(`[rel="stylesheet"]`))) {
107
+ return false;
108
+ }
109
+ return true;
110
+ }
111
+ }
package/package.json ADDED
@@ -0,0 +1,31 @@
1
+ {
2
+ "name": "@zachleat/fetch-links",
3
+ "version": "1.0.0",
4
+ "description": "Utility to fetch hyperlinks from a URL.",
5
+ "main": "links.js",
6
+ "type": "module",
7
+ "scripts": {
8
+ "test": "node --test"
9
+ },
10
+ "author": "Zach Leatherman <zachleatherman@gmail.com> (https://zachleat.com/)",
11
+ "license": "MIT",
12
+ "publishConfig": {
13
+ "access": "public"
14
+ },
15
+ "repository": {
16
+ "type": "git",
17
+ "url": "git+https://github.com/zachleat/fetch-links.git"
18
+ },
19
+ "bugs": {
20
+ "url": "https://github.com/zachleat/fetch-links/issues"
21
+ },
22
+ "homepage": "https://github.com/zachleat/fetch-links#readme",
23
+ "dependencies": {
24
+ "@11ty/eleventy-fetch": "^5.1.2",
25
+ "@11ty/posthtml-urls": "^1.0.3",
26
+ "posthtml": "^0.16.7"
27
+ },
28
+ "devDependencies": {
29
+ "http-server": "^14.1.1"
30
+ }
31
+ }
package/test/test.js ADDED
@@ -0,0 +1,16 @@
1
+ import test from 'node:test';
2
+ import assert from "node:assert/strict";
3
+
4
+ import { Links } from "../links.js";
5
+
6
+ test("zachleat.com", async (t) => {
7
+ let originalUrl = "https://www.zachleat.com/";
8
+ // let originalUrl = "https://matthiasott.com/";
9
+ // let originalUrl = "https://www.mikeaparicio.com/";
10
+ // let originalUrl = "https://www.miriamsuzanne.com/";
11
+ // let originalUrl = "https://daverupert.com/";
12
+ // let originalUrl = "https://aaadaaam.com/";
13
+
14
+ let filtered = await Links.find(originalUrl);
15
+ assert.ok(filtered.length > 0);
16
+ });