@zachleat/fetch-links 1.0.7 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +5 -1
  2. package/links.js +65 -5
  3. package/package.json +8 -2
package/README.md CHANGED
@@ -19,4 +19,8 @@ await Links.find("https://www.zachleat.com/");
19
19
  { url: 'https://github.com/zachleat', via: [ 'link[href][rel="me"]' ] },
20
20
  // …
21
21
  ] */
22
- ```
22
+ ```
23
+
24
+ ## Changelog
25
+
26
+ - `v2.0.0` Strips hashes from URLs. Filters out social media post URLs (keeping user profile URLs).
package/links.js CHANGED
@@ -27,6 +27,7 @@ export class Links {
27
27
 
28
28
  static normalizeUrl(url, origin) {
29
29
  let u = new URL(url, origin);
30
+ u.hash = "";
30
31
 
31
32
  // github.com/username/ becomes github.com/username (for no real good reason)
32
33
  if(u.hostname === "github.com" && u.pathname.endsWith("/")) {
@@ -69,9 +70,10 @@ export class Links {
69
70
 
70
71
  let posthtmlOptions = {
71
72
  eachURL: function(url, attr, tagName, node) {
72
- if(url.startsWith("#") ||
73
- url.startsWith("javascript:") ||
74
- url.startsWith("at://") ||
73
+ if(url.startsWith("#") || // hash links
74
+ url.startsWith("javascript:") || // JavaScript links
75
+ url.startsWith("about:") || // about:blank
76
+ url.startsWith("at://") || // atproto
75
77
  // forms
76
78
  tagName === "form" && attr === "action" ||
77
79
  (tagName === "link" && (
@@ -95,6 +97,8 @@ export class Links {
95
97
  tagName === "audio" ||
96
98
  tagName === "video" ||
97
99
  tagName === "source") {
100
+
101
+ // do nothing
98
102
  return url;
99
103
  }
100
104
 
@@ -105,7 +109,7 @@ export class Links {
105
109
  let priority = 0;
106
110
 
107
111
  // TODO add a scope to the section (header/footer/main)
108
- let content = Links.fetchContent(node).trim();
112
+ let content = Links.fetchContent(node).trim() || node.attrs?.["aria-label"] || node.attrs?.title || "";
109
113
 
110
114
  if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml" || node.attrs.type === "application/json")) {
111
115
  type = "feed";
@@ -157,7 +161,9 @@ export class Links {
157
161
 
158
162
  let filtered = results
159
163
  .filter(entry => this.onlyKeepExternal(entry, originalUrl))
160
- .filter(entry => this.onlyKeepRelevant(entry));
164
+ .filter(entry => this.onlyKeepRelevant(entry))
165
+ .filter(entry => this.onlyKeepLabeled(entry))
166
+ .filter(entry => this.onlyKeepNonSocialPosts(entry));
161
167
 
162
168
  return filtered;
163
169
  }
@@ -207,6 +213,60 @@ export class Links {
207
213
  return !u.startsWith(this.stripProtocol(this.stripSearch(contextUrl)));
208
214
  }
209
215
 
216
+ static isSocialPost(url) {
217
+ let u;
218
+ try {
219
+ u = new URL(url);
220
+ } catch(e) {
221
+ return false;
222
+ }
223
+
224
+ let host = u.hostname.replace(/^(www|m|mobile)\./, "");
225
+ let path = u.pathname;
226
+
227
+ // Mastodon/fediverse statuses (any host)
228
+ if(/^\/@[^/]+\/\d+\/?$/.test(path) || /^\/users\/[^/]+\/statuses\/\d+/.test(path)) {
229
+ return true;
230
+ }
231
+
232
+ switch(host) {
233
+ case "bsky.app":
234
+ return /^\/profile\/[^/]+\/post\//.test(path);
235
+ case "twitter.com":
236
+ case "x.com":
237
+ return /^\/[^/]+\/status\//.test(path);
238
+ case "youtube.com":
239
+ return path === "/watch" || /^\/(shorts|live)\//.test(path);
240
+ case "youtu.be":
241
+ return path.length > 1;
242
+ case "threads.net":
243
+ case "threads.com":
244
+ return /^\/@[^/]+\/post\//.test(path);
245
+ case "news.ycombinator.com":
246
+ return path === "/item";
247
+ case "lobste.rs":
248
+ return /^\/s\//.test(path);
249
+ }
250
+
251
+ return false;
252
+ }
253
+
254
+ // Drop social posts, videos, and discussion threads
255
+ static onlyKeepNonSocialPosts(entry) {
256
+ if(entry.type) {
257
+ return true;
258
+ }
259
+ return !this.isSocialPost(entry.url);
260
+ }
261
+
262
+ // Drop unlabeled anchors (e.g. avatar-only webmention facepiles)
263
+ static onlyKeepLabeled(entry) {
264
+ if(entry.type || entry.priority || entry.content) {
265
+ return true;
266
+ }
267
+ return !entry.via.every(via => via.startsWith("a["));
268
+ }
269
+
210
270
  static onlyKeepRelevant(entry) {
211
271
  if(entry.via.find(via => via.startsWith("script[") ||
212
272
  via.includes(`[rel="preload"]`) ||
package/package.json CHANGED
@@ -1,14 +1,20 @@
1
1
  {
2
2
  "name": "@zachleat/fetch-links",
3
- "version": "1.0.7",
3
+ "version": "2.0.0",
4
4
  "description": "Utility to fetch hyperlinks from a URL.",
5
5
  "main": "links.js",
6
+ "files": [
7
+ "links.js"
8
+ ],
6
9
  "type": "module",
7
10
  "scripts": {
8
11
  "test": "node --test"
9
12
  },
10
13
  "author": "Zach Leatherman <zachleatherman@gmail.com> (https://zachleat.com/)",
11
14
  "license": "MIT",
15
+ "engines": {
16
+ "node": ">=22"
17
+ },
12
18
  "publishConfig": {
13
19
  "access": "public"
14
20
  },
@@ -21,7 +27,7 @@
21
27
  },
22
28
  "homepage": "https://github.com/zachleat/fetch-links#readme",
23
29
  "dependencies": {
24
- "@11ty/eleventy-fetch": "^5.1.2",
30
+ "@11ty/eleventy-fetch": "^5.1.3",
25
31
  "@11ty/posthtml-urls": "^1.0.3",
26
32
  "posthtml": "^0.16.7"
27
33
  },