@zachleat/fetch-links 1.0.6 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/links.js +77 -9
- package/package.json +8 -2
package/README.md
CHANGED
|
@@ -19,4 +19,8 @@ await Links.find("https://www.zachleat.com/");
|
|
|
19
19
|
{ url: 'https://github.com/zachleat', via: [ 'link[href][rel="me"]' ] },
|
|
20
20
|
// …
|
|
21
21
|
] */
|
|
22
|
-
```
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Changelog
|
|
25
|
+
|
|
26
|
+
- `v2.0.0` Strips hashes from URLs. Filters out social media post URLs (keeping user profile URLs).
|
package/links.js
CHANGED
|
@@ -27,6 +27,7 @@ export class Links {
|
|
|
27
27
|
|
|
28
28
|
static normalizeUrl(url, origin) {
|
|
29
29
|
let u = new URL(url, origin);
|
|
30
|
+
u.hash = "";
|
|
30
31
|
|
|
31
32
|
// github.com/username/ becomes github.com/username (for no real good reason)
|
|
32
33
|
if(u.hostname === "github.com" && u.pathname.endsWith("/")) {
|
|
@@ -69,9 +70,10 @@ export class Links {
|
|
|
69
70
|
|
|
70
71
|
let posthtmlOptions = {
|
|
71
72
|
eachURL: function(url, attr, tagName, node) {
|
|
72
|
-
if(url.startsWith("#") ||
|
|
73
|
-
url.startsWith("javascript:") ||
|
|
74
|
-
url.startsWith("
|
|
73
|
+
if(url.startsWith("#") || // hash links
|
|
74
|
+
url.startsWith("javascript:") || // JavaScript links
|
|
75
|
+
url.startsWith("about:") || // about:blank
|
|
76
|
+
url.startsWith("at://") || // atproto
|
|
75
77
|
// forms
|
|
76
78
|
tagName === "form" && attr === "action" ||
|
|
77
79
|
(tagName === "link" && (
|
|
@@ -95,6 +97,8 @@ export class Links {
|
|
|
95
97
|
tagName === "audio" ||
|
|
96
98
|
tagName === "video" ||
|
|
97
99
|
tagName === "source") {
|
|
100
|
+
|
|
101
|
+
// do nothing
|
|
98
102
|
return url;
|
|
99
103
|
}
|
|
100
104
|
|
|
@@ -105,7 +109,7 @@ export class Links {
|
|
|
105
109
|
let priority = 0;
|
|
106
110
|
|
|
107
111
|
// TODO add a scope to the section (header/footer/main)
|
|
108
|
-
let content = Links.fetchContent(node).trim();
|
|
112
|
+
let content = Links.fetchContent(node).trim() || node.attrs?.["aria-label"] || node.attrs?.title || "";
|
|
109
113
|
|
|
110
114
|
if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml" || node.attrs.type === "application/json")) {
|
|
111
115
|
type = "feed";
|
|
@@ -157,7 +161,9 @@ export class Links {
|
|
|
157
161
|
|
|
158
162
|
let filtered = results
|
|
159
163
|
.filter(entry => this.onlyKeepExternal(entry, originalUrl))
|
|
160
|
-
.filter(entry => this.onlyKeepRelevant(entry))
|
|
164
|
+
.filter(entry => this.onlyKeepRelevant(entry))
|
|
165
|
+
.filter(entry => this.onlyKeepLabeled(entry))
|
|
166
|
+
.filter(entry => this.onlyKeepNonSocialPosts(entry));
|
|
161
167
|
|
|
162
168
|
return filtered;
|
|
163
169
|
}
|
|
@@ -188,15 +194,77 @@ export class Links {
|
|
|
188
194
|
return url;
|
|
189
195
|
}
|
|
190
196
|
|
|
197
|
+
static stripSearch(url) {
|
|
198
|
+
let u = new URL(url);
|
|
199
|
+
u.search = "";
|
|
200
|
+
return u.toString()
|
|
201
|
+
}
|
|
202
|
+
|
|
191
203
|
static onlyKeepExternal(entry, originalUrl) {
|
|
192
204
|
if(entry.type) { // keep feeds and other important types
|
|
193
205
|
return true;
|
|
194
206
|
}
|
|
195
207
|
|
|
196
|
-
let
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
208
|
+
let contextUrl = originalUrl || "https://example.com";
|
|
209
|
+
|
|
210
|
+
let entryUrl = new URL(entry.url, contextUrl);
|
|
211
|
+
let u = this.stripProtocol(entryUrl.toString());
|
|
212
|
+
|
|
213
|
+
return !u.startsWith(this.stripProtocol(this.stripSearch(contextUrl)));
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
static isSocialPost(url) {
|
|
217
|
+
let u;
|
|
218
|
+
try {
|
|
219
|
+
u = new URL(url);
|
|
220
|
+
} catch(e) {
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
let host = u.hostname.replace(/^(www|m|mobile)\./, "");
|
|
225
|
+
let path = u.pathname;
|
|
226
|
+
|
|
227
|
+
// Mastodon/fediverse statuses (any host)
|
|
228
|
+
if(/^\/@[^/]+\/\d+\/?$/.test(path) || /^\/users\/[^/]+\/statuses\/\d+/.test(path)) {
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
switch(host) {
|
|
233
|
+
case "bsky.app":
|
|
234
|
+
return /^\/profile\/[^/]+\/post\//.test(path);
|
|
235
|
+
case "twitter.com":
|
|
236
|
+
case "x.com":
|
|
237
|
+
return /^\/[^/]+\/status\//.test(path);
|
|
238
|
+
case "youtube.com":
|
|
239
|
+
return path === "/watch" || /^\/(shorts|live)\//.test(path);
|
|
240
|
+
case "youtu.be":
|
|
241
|
+
return path.length > 1;
|
|
242
|
+
case "threads.net":
|
|
243
|
+
case "threads.com":
|
|
244
|
+
return /^\/@[^/]+\/post\//.test(path);
|
|
245
|
+
case "news.ycombinator.com":
|
|
246
|
+
return path === "/item";
|
|
247
|
+
case "lobste.rs":
|
|
248
|
+
return /^\/s\//.test(path);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
return false;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// Drop social posts, videos, and discussion threads
|
|
255
|
+
static onlyKeepNonSocialPosts(entry) {
|
|
256
|
+
if(entry.type) {
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
return !this.isSocialPost(entry.url);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// Drop unlabeled anchors (e.g. avatar-only webmention facepiles)
|
|
263
|
+
static onlyKeepLabeled(entry) {
|
|
264
|
+
if(entry.type || entry.priority || entry.content) {
|
|
265
|
+
return true;
|
|
266
|
+
}
|
|
267
|
+
return !entry.via.every(via => via.startsWith("a["));
|
|
200
268
|
}
|
|
201
269
|
|
|
202
270
|
static onlyKeepRelevant(entry) {
|
package/package.json
CHANGED
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zachleat/fetch-links",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "Utility to fetch hyperlinks from a URL.",
|
|
5
5
|
"main": "links.js",
|
|
6
|
+
"files": [
|
|
7
|
+
"links.js"
|
|
8
|
+
],
|
|
6
9
|
"type": "module",
|
|
7
10
|
"scripts": {
|
|
8
11
|
"test": "node --test"
|
|
9
12
|
},
|
|
10
13
|
"author": "Zach Leatherman <zachleatherman@gmail.com> (https://zachleat.com/)",
|
|
11
14
|
"license": "MIT",
|
|
15
|
+
"engines": {
|
|
16
|
+
"node": ">=22"
|
|
17
|
+
},
|
|
12
18
|
"publishConfig": {
|
|
13
19
|
"access": "public"
|
|
14
20
|
},
|
|
@@ -21,7 +27,7 @@
|
|
|
21
27
|
},
|
|
22
28
|
"homepage": "https://github.com/zachleat/fetch-links#readme",
|
|
23
29
|
"dependencies": {
|
|
24
|
-
"@11ty/eleventy-fetch": "^5.1.
|
|
30
|
+
"@11ty/eleventy-fetch": "^5.1.3",
|
|
25
31
|
"@11ty/posthtml-urls": "^1.0.3",
|
|
26
32
|
"posthtml": "^0.16.7"
|
|
27
33
|
},
|