@zachleat/fetch-links 1.0.6 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +5 -1
  2. package/links.js +77 -9
  3. package/package.json +8 -2
package/README.md CHANGED
@@ -19,4 +19,8 @@ await Links.find("https://www.zachleat.com/");
19
19
  { url: 'https://github.com/zachleat', via: [ 'link[href][rel="me"]' ] },
20
20
  // …
21
21
  ] */
22
- ```
22
+ ```
23
+
24
+ ## Changelog
25
+
26
+ - `v2.0.0` Strips hashes from URLs. Filters out social media post URLs (keeping user profile URLs).
package/links.js CHANGED
@@ -27,6 +27,7 @@ export class Links {
27
27
 
28
28
  static normalizeUrl(url, origin) {
29
29
  let u = new URL(url, origin);
30
+ u.hash = "";
30
31
 
31
32
  // github.com/username/ becomes github.com/username (for no real good reason)
32
33
  if(u.hostname === "github.com" && u.pathname.endsWith("/")) {
@@ -69,9 +70,10 @@ export class Links {
69
70
 
70
71
  let posthtmlOptions = {
71
72
  eachURL: function(url, attr, tagName, node) {
72
- if(url.startsWith("#") ||
73
- url.startsWith("javascript:") ||
74
- url.startsWith("at://") ||
73
+ if(url.startsWith("#") || // hash links
74
+ url.startsWith("javascript:") || // JavaScript links
75
+ url.startsWith("about:") || // about:blank
76
+ url.startsWith("at://") || // atproto
75
77
  // forms
76
78
  tagName === "form" && attr === "action" ||
77
79
  (tagName === "link" && (
@@ -95,6 +97,8 @@ export class Links {
95
97
  tagName === "audio" ||
96
98
  tagName === "video" ||
97
99
  tagName === "source") {
100
+
101
+ // do nothing
98
102
  return url;
99
103
  }
100
104
 
@@ -105,7 +109,7 @@ export class Links {
105
109
  let priority = 0;
106
110
 
107
111
  // TODO add a scope to the section (header/footer/main)
108
- let content = Links.fetchContent(node).trim();
112
+ let content = Links.fetchContent(node).trim() || node.attrs?.["aria-label"] || node.attrs?.title || "";
109
113
 
110
114
  if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml" || node.attrs.type === "application/json")) {
111
115
  type = "feed";
@@ -157,7 +161,9 @@ export class Links {
157
161
 
158
162
  let filtered = results
159
163
  .filter(entry => this.onlyKeepExternal(entry, originalUrl))
160
- .filter(entry => this.onlyKeepRelevant(entry));
164
+ .filter(entry => this.onlyKeepRelevant(entry))
165
+ .filter(entry => this.onlyKeepLabeled(entry))
166
+ .filter(entry => this.onlyKeepNonSocialPosts(entry));
161
167
 
162
168
  return filtered;
163
169
  }
@@ -188,15 +194,77 @@ export class Links {
188
194
  return url;
189
195
  }
190
196
 
197
+ static stripSearch(url) {
198
+ let u = new URL(url);
199
+ u.search = "";
200
+ return u.toString()
201
+ }
202
+
191
203
  static onlyKeepExternal(entry, originalUrl) {
192
204
  if(entry.type) { // keep feeds and other important types
193
205
  return true;
194
206
  }
195
207
 
196
- let comparison = originalUrl || "https://example.com";
197
- let u = this.stripProtocol((new URL(entry.url, comparison)).toString());
198
-
199
- return !u.startsWith(this.stripProtocol(comparison));
208
+ let contextUrl = originalUrl || "https://example.com";
209
+
210
+ let entryUrl = new URL(entry.url, contextUrl);
211
+ let u = this.stripProtocol(entryUrl.toString());
212
+
213
+ return !u.startsWith(this.stripProtocol(this.stripSearch(contextUrl)));
214
+ }
215
+
216
+ static isSocialPost(url) {
217
+ let u;
218
+ try {
219
+ u = new URL(url);
220
+ } catch(e) {
221
+ return false;
222
+ }
223
+
224
+ let host = u.hostname.replace(/^(www|m|mobile)\./, "");
225
+ let path = u.pathname;
226
+
227
+ // Mastodon/fediverse statuses (any host)
228
+ if(/^\/@[^/]+\/\d+\/?$/.test(path) || /^\/users\/[^/]+\/statuses\/\d+/.test(path)) {
229
+ return true;
230
+ }
231
+
232
+ switch(host) {
233
+ case "bsky.app":
234
+ return /^\/profile\/[^/]+\/post\//.test(path);
235
+ case "twitter.com":
236
+ case "x.com":
237
+ return /^\/[^/]+\/status\//.test(path);
238
+ case "youtube.com":
239
+ return path === "/watch" || /^\/(shorts|live)\//.test(path);
240
+ case "youtu.be":
241
+ return path.length > 1;
242
+ case "threads.net":
243
+ case "threads.com":
244
+ return /^\/@[^/]+\/post\//.test(path);
245
+ case "news.ycombinator.com":
246
+ return path === "/item";
247
+ case "lobste.rs":
248
+ return /^\/s\//.test(path);
249
+ }
250
+
251
+ return false;
252
+ }
253
+
254
+ // Drop social posts, videos, and discussion threads
255
+ static onlyKeepNonSocialPosts(entry) {
256
+ if(entry.type) {
257
+ return true;
258
+ }
259
+ return !this.isSocialPost(entry.url);
260
+ }
261
+
262
+ // Drop unlabeled anchors (e.g. avatar-only webmention facepiles)
263
+ static onlyKeepLabeled(entry) {
264
+ if(entry.type || entry.priority || entry.content) {
265
+ return true;
266
+ }
267
+ return !entry.via.every(via => via.startsWith("a["));
200
268
  }
201
269
 
202
270
  static onlyKeepRelevant(entry) {
package/package.json CHANGED
@@ -1,14 +1,20 @@
1
1
  {
2
2
  "name": "@zachleat/fetch-links",
3
- "version": "1.0.6",
3
+ "version": "2.0.0",
4
4
  "description": "Utility to fetch hyperlinks from a URL.",
5
5
  "main": "links.js",
6
+ "files": [
7
+ "links.js"
8
+ ],
6
9
  "type": "module",
7
10
  "scripts": {
8
11
  "test": "node --test"
9
12
  },
10
13
  "author": "Zach Leatherman <zachleatherman@gmail.com> (https://zachleat.com/)",
11
14
  "license": "MIT",
15
+ "engines": {
16
+ "node": ">=22"
17
+ },
12
18
  "publishConfig": {
13
19
  "access": "public"
14
20
  },
@@ -21,7 +27,7 @@
21
27
  },
22
28
  "homepage": "https://github.com/zachleat/fetch-links#readme",
23
29
  "dependencies": {
24
- "@11ty/eleventy-fetch": "^5.1.2",
30
+ "@11ty/eleventy-fetch": "^5.1.3",
25
31
  "@11ty/posthtml-urls": "^1.0.3",
26
32
  "posthtml": "^0.16.7"
27
33
  },