@zachleat/fetch-links 1.0.4 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/links.js +80 -7
  2. package/package.json +1 -1
  3. package/test/test.js +0 -20
package/links.js CHANGED
@@ -26,7 +26,40 @@ export class Links {
26
26
  }
27
27
 
28
28
  static normalizeUrl(url, origin) {
29
- return (new URL(url, origin)).toString();
29
+ let u = new URL(url, origin);
30
+
31
+ // github.com/username/ becomes github.com/username (for no real good reason)
32
+ if(u.hostname === "github.com" && u.pathname.endsWith("/")) {
33
+ u.pathname = u.pathname.slice(0, -1);
34
+ }
35
+
36
+ return u.toString();
37
+ }
38
+
39
+ static isTopLevel(url) {
40
+ if(!url) {
41
+ return false;
42
+ }
43
+ let u = new URL(url);
44
+ return u.pathname === "/" && !u.search && !u.hash;
45
+ }
46
+
47
+ static fetchContent(node) {
48
+ // node has children
49
+ if(Array.isArray(node?.content)) {
50
+ let children = node.content.filter(entry => {
51
+ return typeof entry !== "string" || !entry.startsWith("<!--") && !entry.endsWith("-->");
52
+ }).filter(Boolean);
53
+
54
+ if(children.length > 0) {
55
+ return " " + children.map(child => this.fetchContent(child)).filter(Boolean).join("");
56
+ }
57
+ }
58
+
59
+ if(typeof node === "string") {
60
+ return node;
61
+ }
62
+ return "";
30
63
  }
31
64
 
32
65
  // TODO de-duplicate via subdomain redirects (see speedlify 11ty leaderboards) e.g. 11ty.dev versus www.11ty.dev
@@ -38,8 +71,25 @@ export class Links {
38
71
  eachURL: function(url, attr, tagName, node) {
39
72
  if(url.startsWith("#") ||
40
73
  url.startsWith("javascript:") ||
41
- // app icons
42
- (tagName === "link" && (node.attrs.rel === "icon" || node.attrs.rel === "apple-touch-icon" || node.attrs.rel === "apple-touch-icon-precomposed")) ||
74
+ url.startsWith("at://") ||
75
+ // forms
76
+ tagName === "form" && attr === "action" ||
77
+ (tagName === "link" && (
78
+ // app icons
79
+ node.attrs.rel === "icon" ||
80
+ node.attrs.rel === "mask-icon" ||
81
+ node.attrs.rel === "shortcut icon" ||
82
+ node.attrs.rel === "apple-touch-icon" ||
83
+ node.attrs.rel === "apple-touch-icon-precomposed") ||
84
+ // app manifest
85
+ node.attrs.rel === "manifest" ||
86
+ // script
87
+ node.attrs.rel === "modulepreload"
88
+ ) ||
89
+ (tagName === "link" &&
90
+ node.attrs.as === "font" &&
91
+ node.attrs.type?.startsWith("font/")
92
+ ) ||
43
93
  // media
44
94
  tagName === "img" ||
45
95
  tagName === "audio" ||
@@ -48,13 +98,25 @@ export class Links {
48
98
  return url;
49
99
  }
50
100
 
51
- let normalized = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
52
- let key = normalized;
101
+ let normalizedUrl = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
102
+ let key = Links.stripProtocol(normalizedUrl.toLowerCase());
53
103
  let via = `${tagName}[${attr}]${node.attrs.rel ? `[rel="${node.attrs.rel}"]` : ""}`;
54
104
  let type;
55
- if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml")) {
105
+ let priority = 0;
106
+
107
+ // TODO add a scope to the section (header/footer/main)
108
+ let content = Links.fetchContent(node).trim();
109
+
110
+ if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml" || node.attrs.type === "application/json")) {
56
111
  type = "feed";
112
+ priority = 1;
113
+ }
114
+ if(node.attrs?.rel === "me") {
115
+ priority = 1;
57
116
  }
117
+ // if(Links.isTopLevel(normalizedUrl)) {
118
+ // priority = 1;
119
+ // }
58
120
 
59
121
  if(urls.has(key)) {
60
122
  let entry = urls.get(key);
@@ -62,11 +124,19 @@ export class Links {
62
124
  if(!entry.type) {
63
125
  entry.type = type;
64
126
  }
127
+ if(!entry.priority) {
128
+ entry.priority = priority;
129
+ }
130
+ if(!entry.content) {
131
+ entry.content = content;
132
+ }
65
133
  } else {
66
134
  urls.set(key, {
67
- url: normalized,
135
+ url: normalizedUrl,
68
136
  via: new Set([via]),
69
137
  type,
138
+ priority,
139
+ content,
70
140
  });
71
141
  }
72
142
 
@@ -135,6 +205,9 @@ export class Links {
135
205
  via.includes(`[rel="dns-prefetch"]`) ||
136
206
  via.includes(`[rel="token_endpoint"]`) ||
137
207
  via.includes(`[rel="authorization_endpoint"]`) ||
208
+ via.includes(`[rel="indieauth-metadata"]`) ||
209
+ via.includes(`[rel="micropub"]`) ||
210
+ via.includes(`[rel="microsub"]`) ||
138
211
  via.includes(`[rel="publickey"]`) ||
139
212
  via.includes(`[rel="preconnect"]`) ||
140
213
  via.includes(`[rel="webmention"]`) ||
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zachleat/fetch-links",
3
- "version": "1.0.4",
3
+ "version": "1.0.6",
4
4
  "description": "Utility to fetch hyperlinks from a URL.",
5
5
  "main": "links.js",
6
6
  "type": "module",
package/test/test.js DELETED
@@ -1,20 +0,0 @@
1
- import test from 'node:test';
2
- import assert from "node:assert/strict";
3
-
4
- import { Links } from "../links.js";
5
-
6
- test("zachleat.com", async (t) => {
7
- let originalUrl = "https://www.zachleat.com/";
8
- // let originalUrl = "https://matthiasott.com/";
9
- // let originalUrl = "https://www.mikeaparicio.com/";
10
- // let originalUrl = "https://www.miriamsuzanne.com/";
11
- // let originalUrl = "https://daverupert.com/";
12
- // let originalUrl = "https://aaadaaam.com/";
13
- // let originalUrl = "https://torvalds-family.blogspot.com/";
14
- // let originalUrl = "https://wesbos.com/";
15
- // let originalUrl = "https://chriscoyier.net/";
16
-
17
- let filtered = await Links.find(originalUrl);
18
- // console.log( filtered );
19
- assert.ok(filtered.length > 0);
20
- });