@zachleat/fetch-links 1.0.5 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/links.js +27 -8
- package/package.json +1 -1
package/links.js
CHANGED
|
@@ -36,6 +36,14 @@ export class Links {
|
|
|
36
36
|
return u.toString();
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
static isTopLevel(url) {
|
|
40
|
+
if(!url) {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
let u = new URL(url);
|
|
44
|
+
return u.pathname === "/" && !u.search && !u.hash;
|
|
45
|
+
}
|
|
46
|
+
|
|
39
47
|
static fetchContent(node) {
|
|
40
48
|
// node has children
|
|
41
49
|
if(Array.isArray(node?.content)) {
|
|
@@ -90,8 +98,8 @@ export class Links {
|
|
|
90
98
|
return url;
|
|
91
99
|
}
|
|
92
100
|
|
|
93
|
-
let
|
|
94
|
-
let key = Links.stripProtocol(
|
|
101
|
+
let normalizedUrl = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
|
|
102
|
+
let key = Links.stripProtocol(normalizedUrl.toLowerCase());
|
|
95
103
|
let via = `${tagName}[${attr}]${node.attrs.rel ? `[rel="${node.attrs.rel}"]` : ""}`;
|
|
96
104
|
let type;
|
|
97
105
|
let priority = 0;
|
|
@@ -99,13 +107,16 @@ export class Links {
|
|
|
99
107
|
// TODO add a scope to the section (header/footer/main)
|
|
100
108
|
let content = Links.fetchContent(node).trim();
|
|
101
109
|
|
|
102
|
-
if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml")) {
|
|
110
|
+
if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml" || node.attrs.type === "application/json")) {
|
|
103
111
|
type = "feed";
|
|
104
112
|
priority = 1;
|
|
105
113
|
}
|
|
106
114
|
if(node.attrs?.rel === "me") {
|
|
107
115
|
priority = 1;
|
|
108
116
|
}
|
|
117
|
+
// if(Links.isTopLevel(normalizedUrl)) {
|
|
118
|
+
// priority = 1;
|
|
119
|
+
// }
|
|
109
120
|
|
|
110
121
|
if(urls.has(key)) {
|
|
111
122
|
let entry = urls.get(key);
|
|
@@ -121,7 +132,7 @@ export class Links {
|
|
|
121
132
|
}
|
|
122
133
|
} else {
|
|
123
134
|
urls.set(key, {
|
|
124
|
-
url:
|
|
135
|
+
url: normalizedUrl,
|
|
125
136
|
via: new Set([via]),
|
|
126
137
|
type,
|
|
127
138
|
priority,
|
|
@@ -177,15 +188,23 @@ export class Links {
|
|
|
177
188
|
return url;
|
|
178
189
|
}
|
|
179
190
|
|
|
191
|
+
static stripSearch(url) {
|
|
192
|
+
let u = new URL(url);
|
|
193
|
+
u.search = "";
|
|
194
|
+
return u.toString()
|
|
195
|
+
}
|
|
196
|
+
|
|
180
197
|
static onlyKeepExternal(entry, originalUrl) {
|
|
181
198
|
if(entry.type) { // keep feeds and other important types
|
|
182
199
|
return true;
|
|
183
200
|
}
|
|
184
201
|
|
|
185
|
-
let
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
202
|
+
let contextUrl = originalUrl || "https://example.com";
|
|
203
|
+
|
|
204
|
+
let entryUrl = new URL(entry.url, contextUrl);
|
|
205
|
+
let u = this.stripProtocol(entryUrl.toString());
|
|
206
|
+
|
|
207
|
+
return !u.startsWith(this.stripProtocol(this.stripSearch(contextUrl)));
|
|
189
208
|
}
|
|
190
209
|
|
|
191
210
|
static onlyKeepRelevant(entry) {
|