@zachleat/fetch-links 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/links.js +22 -3
  2. package/package.json +1 -1
  3. package/test/test.js +4 -0
package/links.js CHANGED
@@ -36,7 +36,15 @@ export class Links {
36
36
 
37
37
  let posthtmlOptions = {
38
38
  eachURL: function(url, attr, tagName, node) {
39
- if(url.startsWith("#") || tagName === "img" || tagName === "video" || tagName === "source") {
39
+ if(url.startsWith("#") ||
40
+ url.startsWith("javascript:") ||
41
+ // app icons
42
+ (tagName === "link" && (node.attrs.rel === "icon" || node.attrs.rel === "apple-touch-icon" || node.attrs.rel === "apple-touch-icon-precomposed")) ||
43
+ // media
44
+ tagName === "img" ||
45
+ tagName === "audio" ||
46
+ tagName === "video" ||
47
+ tagName === "source") {
40
48
  return url;
41
49
  }
42
50
 
@@ -101,18 +109,29 @@ export class Links {
101
109
  });
102
110
  }
103
111
 
112
+ static stripProtocol(url) {
113
+ if(url.startsWith("https://")) {
114
+ url = url.slice("https://".length);
115
+ } else if(url.startsWith("http://")) {
116
+ url = url.slice("http://".length);
117
+ }
118
+ return url;
119
+ }
120
+
104
121
  static onlyKeepExternal(entry, originalUrl) {
105
122
  if(entry.type) { // keep feeds and other important types
106
123
  return true;
107
124
  }
108
125
 
109
126
  let comparison = originalUrl || "https://example.com";
110
- let u = new URL(entry.url, comparison);
111
- return !u.toString().startsWith(comparison);
127
+ let u = this.stripProtocol((new URL(entry.url, comparison)).toString());
128
+
129
+ return !u.startsWith(this.stripProtocol(comparison));
112
130
  }
113
131
 
114
132
  static onlyKeepRelevant(entry) {
115
133
  if(entry.via.find(via => via.startsWith("script[") ||
134
+ via.includes(`[rel="preload"]`) ||
116
135
  via.includes(`[rel="dns-prefetch"]`) ||
117
136
  via.includes(`[rel="token_endpoint"]`) ||
118
137
  via.includes(`[rel="authorization_endpoint"]`) ||
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zachleat/fetch-links",
3
- "version": "1.0.2",
3
+ "version": "1.0.4",
4
4
  "description": "Utility to fetch hyperlinks from a URL.",
5
5
  "main": "links.js",
6
6
  "type": "module",
package/test/test.js CHANGED
@@ -10,7 +10,11 @@ test("zachleat.com", async (t) => {
10
10
  // let originalUrl = "https://www.miriamsuzanne.com/";
11
11
  // let originalUrl = "https://daverupert.com/";
12
12
  // let originalUrl = "https://aaadaaam.com/";
13
+ // let originalUrl = "https://torvalds-family.blogspot.com/";
14
+ // let originalUrl = "https://wesbos.com/";
15
+ // let originalUrl = "https://chriscoyier.net/";
13
16
 
14
17
  let filtered = await Links.find(originalUrl);
18
+ // console.log( filtered );
15
19
  assert.ok(filtered.length > 0);
16
20
  });