@zachleat/fetch-links 1.0.4 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/links.js +66 -4
- package/package.json +1 -1
- package/test/test.js +0 -20
package/links.js
CHANGED
|
@@ -26,7 +26,32 @@ export class Links {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
static normalizeUrl(url, origin) {
|
|
29
|
-
|
|
29
|
+
let u = new URL(url, origin);
|
|
30
|
+
|
|
31
|
+
// github.com/username/ becomes github.com/username (for no real good reason)
|
|
32
|
+
if(u.hostname === "github.com" && u.pathname.endsWith("/")) {
|
|
33
|
+
u.pathname = u.pathname.slice(0, -1);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
return u.toString();
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
static fetchContent(node) {
|
|
40
|
+
// node has children
|
|
41
|
+
if(Array.isArray(node?.content)) {
|
|
42
|
+
let children = node.content.filter(entry => {
|
|
43
|
+
return typeof entry !== "string" || !entry.startsWith("<!--") && !entry.endsWith("-->");
|
|
44
|
+
}).filter(Boolean);
|
|
45
|
+
|
|
46
|
+
if(children.length > 0) {
|
|
47
|
+
return " " + children.map(child => this.fetchContent(child)).filter(Boolean).join("");
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if(typeof node === "string") {
|
|
52
|
+
return node;
|
|
53
|
+
}
|
|
54
|
+
return "";
|
|
30
55
|
}
|
|
31
56
|
|
|
32
57
|
// TODO de-duplicate via subdomain redirects (see speedlify 11ty leaderboards) e.g. 11ty.dev versus www.11ty.dev
|
|
@@ -38,8 +63,25 @@ export class Links {
|
|
|
38
63
|
eachURL: function(url, attr, tagName, node) {
|
|
39
64
|
if(url.startsWith("#") ||
|
|
40
65
|
url.startsWith("javascript:") ||
|
|
41
|
-
|
|
42
|
-
|
|
66
|
+
url.startsWith("at://") ||
|
|
67
|
+
// forms
|
|
68
|
+
tagName === "form" && attr === "action" ||
|
|
69
|
+
(tagName === "link" && (
|
|
70
|
+
// app icons
|
|
71
|
+
node.attrs.rel === "icon" ||
|
|
72
|
+
node.attrs.rel === "mask-icon" ||
|
|
73
|
+
node.attrs.rel === "shortcut icon" ||
|
|
74
|
+
node.attrs.rel === "apple-touch-icon" ||
|
|
75
|
+
node.attrs.rel === "apple-touch-icon-precomposed") ||
|
|
76
|
+
// app manifest
|
|
77
|
+
node.attrs.rel === "manifest" ||
|
|
78
|
+
// script
|
|
79
|
+
node.attrs.rel === "modulepreload"
|
|
80
|
+
) ||
|
|
81
|
+
(tagName === "link" &&
|
|
82
|
+
node.attrs.as === "font" &&
|
|
83
|
+
node.attrs.type?.startsWith("font/")
|
|
84
|
+
) ||
|
|
43
85
|
// media
|
|
44
86
|
tagName === "img" ||
|
|
45
87
|
tagName === "audio" ||
|
|
@@ -49,11 +91,20 @@ export class Links {
|
|
|
49
91
|
}
|
|
50
92
|
|
|
51
93
|
let normalized = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
|
|
52
|
-
let key = normalized;
|
|
94
|
+
let key = Links.stripProtocol(normalized.toLowerCase());
|
|
53
95
|
let via = `${tagName}[${attr}]${node.attrs.rel ? `[rel="${node.attrs.rel}"]` : ""}`;
|
|
54
96
|
let type;
|
|
97
|
+
let priority = 0;
|
|
98
|
+
|
|
99
|
+
// TODO add a scope to the section (header/footer/main)
|
|
100
|
+
let content = Links.fetchContent(node).trim();
|
|
101
|
+
|
|
55
102
|
if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml")) {
|
|
56
103
|
type = "feed";
|
|
104
|
+
priority = 1;
|
|
105
|
+
}
|
|
106
|
+
if(node.attrs?.rel === "me") {
|
|
107
|
+
priority = 1;
|
|
57
108
|
}
|
|
58
109
|
|
|
59
110
|
if(urls.has(key)) {
|
|
@@ -62,11 +113,19 @@ export class Links {
|
|
|
62
113
|
if(!entry.type) {
|
|
63
114
|
entry.type = type;
|
|
64
115
|
}
|
|
116
|
+
if(!entry.priority) {
|
|
117
|
+
entry.priority = priority;
|
|
118
|
+
}
|
|
119
|
+
if(!entry.content) {
|
|
120
|
+
entry.content = content;
|
|
121
|
+
}
|
|
65
122
|
} else {
|
|
66
123
|
urls.set(key, {
|
|
67
124
|
url: normalized,
|
|
68
125
|
via: new Set([via]),
|
|
69
126
|
type,
|
|
127
|
+
priority,
|
|
128
|
+
content,
|
|
70
129
|
});
|
|
71
130
|
}
|
|
72
131
|
|
|
@@ -135,6 +194,9 @@ export class Links {
|
|
|
135
194
|
via.includes(`[rel="dns-prefetch"]`) ||
|
|
136
195
|
via.includes(`[rel="token_endpoint"]`) ||
|
|
137
196
|
via.includes(`[rel="authorization_endpoint"]`) ||
|
|
197
|
+
via.includes(`[rel="indieauth-metadata"]`) ||
|
|
198
|
+
via.includes(`[rel="micropub"]`) ||
|
|
199
|
+
via.includes(`[rel="microsub"]`) ||
|
|
138
200
|
via.includes(`[rel="publickey"]`) ||
|
|
139
201
|
via.includes(`[rel="preconnect"]`) ||
|
|
140
202
|
via.includes(`[rel="webmention"]`) ||
|
package/package.json
CHANGED
package/test/test.js
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import test from 'node:test';
|
|
2
|
-
import assert from "node:assert/strict";
|
|
3
|
-
|
|
4
|
-
import { Links } from "../links.js";
|
|
5
|
-
|
|
6
|
-
test("zachleat.com", async (t) => {
|
|
7
|
-
let originalUrl = "https://www.zachleat.com/";
|
|
8
|
-
// let originalUrl = "https://matthiasott.com/";
|
|
9
|
-
// let originalUrl = "https://www.mikeaparicio.com/";
|
|
10
|
-
// let originalUrl = "https://www.miriamsuzanne.com/";
|
|
11
|
-
// let originalUrl = "https://daverupert.com/";
|
|
12
|
-
// let originalUrl = "https://aaadaaam.com/";
|
|
13
|
-
// let originalUrl = "https://torvalds-family.blogspot.com/";
|
|
14
|
-
// let originalUrl = "https://wesbos.com/";
|
|
15
|
-
// let originalUrl = "https://chriscoyier.net/";
|
|
16
|
-
|
|
17
|
-
let filtered = await Links.find(originalUrl);
|
|
18
|
-
// console.log( filtered );
|
|
19
|
-
assert.ok(filtered.length > 0);
|
|
20
|
-
});
|