@zachleat/fetch-links 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/links.js +73 -3
- package/package.json +1 -1
- package/test/test.js +0 -19
package/links.js
CHANGED
|
@@ -26,7 +26,32 @@ export class Links {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
static normalizeUrl(url, origin) {
|
|
29
|
-
|
|
29
|
+
let u = new URL(url, origin);
|
|
30
|
+
|
|
31
|
+
// github.com/username/ becomes github.com/username (for no real good reason)
|
|
32
|
+
if(u.hostname === "github.com" && u.pathname.endsWith("/")) {
|
|
33
|
+
u.pathname = u.pathname.slice(0, -1);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
return u.toString();
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
static fetchContent(node) {
|
|
40
|
+
// node has children
|
|
41
|
+
if(Array.isArray(node?.content)) {
|
|
42
|
+
let children = node.content.filter(entry => {
|
|
43
|
+
return typeof entry !== "string" || !entry.startsWith("<!--") && !entry.endsWith("-->");
|
|
44
|
+
}).filter(Boolean);
|
|
45
|
+
|
|
46
|
+
if(children.length > 0) {
|
|
47
|
+
return " " + children.map(child => this.fetchContent(child)).filter(Boolean).join("");
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if(typeof node === "string") {
|
|
52
|
+
return node;
|
|
53
|
+
}
|
|
54
|
+
return "";
|
|
30
55
|
}
|
|
31
56
|
|
|
32
57
|
// TODO de-duplicate via subdomain redirects (see speedlify 11ty leaderboards) e.g. 11ty.dev versus www.11ty.dev
|
|
@@ -36,16 +61,50 @@ export class Links {
|
|
|
36
61
|
|
|
37
62
|
let posthtmlOptions = {
|
|
38
63
|
eachURL: function(url, attr, tagName, node) {
|
|
39
|
-
if(url.startsWith("#") ||
|
|
64
|
+
if(url.startsWith("#") ||
|
|
65
|
+
url.startsWith("javascript:") ||
|
|
66
|
+
url.startsWith("at://") ||
|
|
67
|
+
// forms
|
|
68
|
+
tagName === "form" && attr === "action" ||
|
|
69
|
+
(tagName === "link" && (
|
|
70
|
+
// app icons
|
|
71
|
+
node.attrs.rel === "icon" ||
|
|
72
|
+
node.attrs.rel === "mask-icon" ||
|
|
73
|
+
node.attrs.rel === "shortcut icon" ||
|
|
74
|
+
node.attrs.rel === "apple-touch-icon" ||
|
|
75
|
+
node.attrs.rel === "apple-touch-icon-precomposed") ||
|
|
76
|
+
// app manifest
|
|
77
|
+
node.attrs.rel === "manifest" ||
|
|
78
|
+
// script
|
|
79
|
+
node.attrs.rel === "modulepreload"
|
|
80
|
+
) ||
|
|
81
|
+
(tagName === "link" &&
|
|
82
|
+
node.attrs.as === "font" &&
|
|
83
|
+
node.attrs.type?.startsWith("font/")
|
|
84
|
+
) ||
|
|
85
|
+
// media
|
|
86
|
+
tagName === "img" ||
|
|
87
|
+
tagName === "audio" ||
|
|
88
|
+
tagName === "video" ||
|
|
89
|
+
tagName === "source") {
|
|
40
90
|
return url;
|
|
41
91
|
}
|
|
42
92
|
|
|
43
93
|
let normalized = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
|
|
44
|
-
let key = normalized;
|
|
94
|
+
let key = Links.stripProtocol(normalized.toLowerCase());
|
|
45
95
|
let via = `${tagName}[${attr}]${node.attrs.rel ? `[rel="${node.attrs.rel}"]` : ""}`;
|
|
46
96
|
let type;
|
|
97
|
+
let priority = 0;
|
|
98
|
+
|
|
99
|
+
// TODO add a scope to the section (header/footer/main)
|
|
100
|
+
let content = Links.fetchContent(node).trim();
|
|
101
|
+
|
|
47
102
|
if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml")) {
|
|
48
103
|
type = "feed";
|
|
104
|
+
priority = 1;
|
|
105
|
+
}
|
|
106
|
+
if(node.attrs?.rel === "me") {
|
|
107
|
+
priority = 1;
|
|
49
108
|
}
|
|
50
109
|
|
|
51
110
|
if(urls.has(key)) {
|
|
@@ -54,11 +113,19 @@ export class Links {
|
|
|
54
113
|
if(!entry.type) {
|
|
55
114
|
entry.type = type;
|
|
56
115
|
}
|
|
116
|
+
if(!entry.priority) {
|
|
117
|
+
entry.priority = priority;
|
|
118
|
+
}
|
|
119
|
+
if(!entry.content) {
|
|
120
|
+
entry.content = content;
|
|
121
|
+
}
|
|
57
122
|
} else {
|
|
58
123
|
urls.set(key, {
|
|
59
124
|
url: normalized,
|
|
60
125
|
via: new Set([via]),
|
|
61
126
|
type,
|
|
127
|
+
priority,
|
|
128
|
+
content,
|
|
62
129
|
});
|
|
63
130
|
}
|
|
64
131
|
|
|
@@ -127,6 +194,9 @@ export class Links {
|
|
|
127
194
|
via.includes(`[rel="dns-prefetch"]`) ||
|
|
128
195
|
via.includes(`[rel="token_endpoint"]`) ||
|
|
129
196
|
via.includes(`[rel="authorization_endpoint"]`) ||
|
|
197
|
+
via.includes(`[rel="indieauth-metadata"]`) ||
|
|
198
|
+
via.includes(`[rel="micropub"]`) ||
|
|
199
|
+
via.includes(`[rel="microsub"]`) ||
|
|
130
200
|
via.includes(`[rel="publickey"]`) ||
|
|
131
201
|
via.includes(`[rel="preconnect"]`) ||
|
|
132
202
|
via.includes(`[rel="webmention"]`) ||
|
package/package.json
CHANGED
package/test/test.js
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import test from 'node:test';
|
|
2
|
-
import assert from "node:assert/strict";
|
|
3
|
-
|
|
4
|
-
import { Links } from "../links.js";
|
|
5
|
-
|
|
6
|
-
test("zachleat.com", async (t) => {
|
|
7
|
-
let originalUrl = "https://www.zachleat.com/";
|
|
8
|
-
// let originalUrl = "https://matthiasott.com/";
|
|
9
|
-
// let originalUrl = "https://www.mikeaparicio.com/";
|
|
10
|
-
// let originalUrl = "https://www.miriamsuzanne.com/";
|
|
11
|
-
// let originalUrl = "https://daverupert.com/";
|
|
12
|
-
// let originalUrl = "https://aaadaaam.com/";
|
|
13
|
-
// let originalUrl = "https://torvalds-family.blogspot.com/";
|
|
14
|
-
// let originalUrl = "https://wesbos.com/";
|
|
15
|
-
|
|
16
|
-
let filtered = await Links.find(originalUrl);
|
|
17
|
-
// console.log( filtered );
|
|
18
|
-
assert.ok(filtered.length > 0);
|
|
19
|
-
});
|