@zachleat/fetch-links 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/links.js +80 -7
- package/package.json +1 -1
- package/test/test.js +0 -20
package/links.js
CHANGED
|
@@ -26,7 +26,40 @@ export class Links {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
static normalizeUrl(url, origin) {
|
|
29
|
-
|
|
29
|
+
let u = new URL(url, origin);
|
|
30
|
+
|
|
31
|
+
// github.com/username/ becomes github.com/username (for no real good reason)
|
|
32
|
+
if(u.hostname === "github.com" && u.pathname.endsWith("/")) {
|
|
33
|
+
u.pathname = u.pathname.slice(0, -1);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
return u.toString();
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
static isTopLevel(url) {
|
|
40
|
+
if(!url) {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
let u = new URL(url);
|
|
44
|
+
return u.pathname === "/" && !u.search && !u.hash;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
static fetchContent(node) {
|
|
48
|
+
// node has children
|
|
49
|
+
if(Array.isArray(node?.content)) {
|
|
50
|
+
let children = node.content.filter(entry => {
|
|
51
|
+
return typeof entry !== "string" || !entry.startsWith("<!--") && !entry.endsWith("-->");
|
|
52
|
+
}).filter(Boolean);
|
|
53
|
+
|
|
54
|
+
if(children.length > 0) {
|
|
55
|
+
return " " + children.map(child => this.fetchContent(child)).filter(Boolean).join("");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if(typeof node === "string") {
|
|
60
|
+
return node;
|
|
61
|
+
}
|
|
62
|
+
return "";
|
|
30
63
|
}
|
|
31
64
|
|
|
32
65
|
// TODO de-duplicate via subdomain redirects (see speedlify 11ty leaderboards) e.g. 11ty.dev versus www.11ty.dev
|
|
@@ -38,8 +71,25 @@ export class Links {
|
|
|
38
71
|
eachURL: function(url, attr, tagName, node) {
|
|
39
72
|
if(url.startsWith("#") ||
|
|
40
73
|
url.startsWith("javascript:") ||
|
|
41
|
-
|
|
42
|
-
|
|
74
|
+
url.startsWith("at://") ||
|
|
75
|
+
// forms
|
|
76
|
+
tagName === "form" && attr === "action" ||
|
|
77
|
+
(tagName === "link" && (
|
|
78
|
+
// app icons
|
|
79
|
+
node.attrs.rel === "icon" ||
|
|
80
|
+
node.attrs.rel === "mask-icon" ||
|
|
81
|
+
node.attrs.rel === "shortcut icon" ||
|
|
82
|
+
node.attrs.rel === "apple-touch-icon" ||
|
|
83
|
+
node.attrs.rel === "apple-touch-icon-precomposed") ||
|
|
84
|
+
// app manifest
|
|
85
|
+
node.attrs.rel === "manifest" ||
|
|
86
|
+
// script
|
|
87
|
+
node.attrs.rel === "modulepreload"
|
|
88
|
+
) ||
|
|
89
|
+
(tagName === "link" &&
|
|
90
|
+
node.attrs.as === "font" &&
|
|
91
|
+
node.attrs.type?.startsWith("font/")
|
|
92
|
+
) ||
|
|
43
93
|
// media
|
|
44
94
|
tagName === "img" ||
|
|
45
95
|
tagName === "audio" ||
|
|
@@ -48,13 +98,25 @@ export class Links {
|
|
|
48
98
|
return url;
|
|
49
99
|
}
|
|
50
100
|
|
|
51
|
-
let
|
|
52
|
-
let key =
|
|
101
|
+
let normalizedUrl = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
|
|
102
|
+
let key = Links.stripProtocol(normalizedUrl.toLowerCase());
|
|
53
103
|
let via = `${tagName}[${attr}]${node.attrs.rel ? `[rel="${node.attrs.rel}"]` : ""}`;
|
|
54
104
|
let type;
|
|
55
|
-
|
|
105
|
+
let priority = 0;
|
|
106
|
+
|
|
107
|
+
// TODO add a scope to the section (header/footer/main)
|
|
108
|
+
let content = Links.fetchContent(node).trim();
|
|
109
|
+
|
|
110
|
+
if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml" || node.attrs.type === "application/json")) {
|
|
56
111
|
type = "feed";
|
|
112
|
+
priority = 1;
|
|
113
|
+
}
|
|
114
|
+
if(node.attrs?.rel === "me") {
|
|
115
|
+
priority = 1;
|
|
57
116
|
}
|
|
117
|
+
// if(Links.isTopLevel(normalizedUrl)) {
|
|
118
|
+
// priority = 1;
|
|
119
|
+
// }
|
|
58
120
|
|
|
59
121
|
if(urls.has(key)) {
|
|
60
122
|
let entry = urls.get(key);
|
|
@@ -62,11 +124,19 @@ export class Links {
|
|
|
62
124
|
if(!entry.type) {
|
|
63
125
|
entry.type = type;
|
|
64
126
|
}
|
|
127
|
+
if(!entry.priority) {
|
|
128
|
+
entry.priority = priority;
|
|
129
|
+
}
|
|
130
|
+
if(!entry.content) {
|
|
131
|
+
entry.content = content;
|
|
132
|
+
}
|
|
65
133
|
} else {
|
|
66
134
|
urls.set(key, {
|
|
67
|
-
url:
|
|
135
|
+
url: normalizedUrl,
|
|
68
136
|
via: new Set([via]),
|
|
69
137
|
type,
|
|
138
|
+
priority,
|
|
139
|
+
content,
|
|
70
140
|
});
|
|
71
141
|
}
|
|
72
142
|
|
|
@@ -135,6 +205,9 @@ export class Links {
|
|
|
135
205
|
via.includes(`[rel="dns-prefetch"]`) ||
|
|
136
206
|
via.includes(`[rel="token_endpoint"]`) ||
|
|
137
207
|
via.includes(`[rel="authorization_endpoint"]`) ||
|
|
208
|
+
via.includes(`[rel="indieauth-metadata"]`) ||
|
|
209
|
+
via.includes(`[rel="micropub"]`) ||
|
|
210
|
+
via.includes(`[rel="microsub"]`) ||
|
|
138
211
|
via.includes(`[rel="publickey"]`) ||
|
|
139
212
|
via.includes(`[rel="preconnect"]`) ||
|
|
140
213
|
via.includes(`[rel="webmention"]`) ||
|
package/package.json
CHANGED
package/test/test.js
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import test from 'node:test';
|
|
2
|
-
import assert from "node:assert/strict";
|
|
3
|
-
|
|
4
|
-
import { Links } from "../links.js";
|
|
5
|
-
|
|
6
|
-
test("zachleat.com", async (t) => {
|
|
7
|
-
let originalUrl = "https://www.zachleat.com/";
|
|
8
|
-
// let originalUrl = "https://matthiasott.com/";
|
|
9
|
-
// let originalUrl = "https://www.mikeaparicio.com/";
|
|
10
|
-
// let originalUrl = "https://www.miriamsuzanne.com/";
|
|
11
|
-
// let originalUrl = "https://daverupert.com/";
|
|
12
|
-
// let originalUrl = "https://aaadaaam.com/";
|
|
13
|
-
// let originalUrl = "https://torvalds-family.blogspot.com/";
|
|
14
|
-
// let originalUrl = "https://wesbos.com/";
|
|
15
|
-
// let originalUrl = "https://chriscoyier.net/";
|
|
16
|
-
|
|
17
|
-
let filtered = await Links.find(originalUrl);
|
|
18
|
-
// console.log( filtered );
|
|
19
|
-
assert.ok(filtered.length > 0);
|
|
20
|
-
});
|