@zachleat/fetch-links 1.0.1 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/links.js +29 -4
  2. package/package.json +1 -1
  3. package/test/test.js +3 -0
package/links.js CHANGED
@@ -36,19 +36,29 @@ export class Links {
36
36
 
37
37
  let posthtmlOptions = {
38
38
  eachURL: function(url, attr, tagName, node) {
39
- if(url.startsWith("#") || tagName === "img" || tagName === "video" || tagName === "source") {
39
+ if(url.startsWith("#") || url.startsWith("javascript:") || tagName === "img" || tagName === "video" || tagName === "source") {
40
40
  return url;
41
41
  }
42
42
 
43
43
  let normalized = originalUrl ? Links.normalizeUrl(url, originalUrl) : url;
44
44
  let key = normalized;
45
45
  let via = `${tagName}[${attr}]${node.attrs.rel ? `[rel="${node.attrs.rel}"]` : ""}`;
46
+ let type;
47
+ if(node.attrs.rel && (node.attrs.type === "application/atom+xml" || node.attrs.type === "application/rss+xml")) {
48
+ type = "feed";
49
+ }
50
+
46
51
  if(urls.has(key)) {
47
- urls.get(key).via.add(via);
52
+ let entry = urls.get(key);
53
+ entry.via.add(via);
54
+ if(!entry.type) {
55
+ entry.type = type;
56
+ }
48
57
  } else {
49
58
  urls.set(key, {
50
59
  url: normalized,
51
60
  via: new Set([via]),
61
+ type,
52
62
  });
53
63
  }
54
64
 
@@ -91,14 +101,29 @@ export class Links {
91
101
  });
92
102
  }
93
103
 
104
+ static stripProtocol(url) {
105
+ if(url.startsWith("https://")) {
106
+ url = url.slice("https://".length);
107
+ } else if(url.startsWith("http://")) {
108
+ url = url.slice("http://".length);
109
+ }
110
+ return url;
111
+ }
112
+
94
113
  static onlyKeepExternal(entry, originalUrl) {
114
+ if(entry.type) { // keep feeds and other important types
115
+ return true;
116
+ }
117
+
95
118
  let comparison = originalUrl || "https://example.com";
96
- let u = new URL(entry.url, comparison);
97
- return !u.toString().startsWith(comparison);
119
+ let u = this.stripProtocol((new URL(entry.url, comparison)).toString());
120
+
121
+ return !u.startsWith(this.stripProtocol(comparison));
98
122
  }
99
123
 
100
124
  static onlyKeepRelevant(entry) {
101
125
  if(entry.via.find(via => via.startsWith("script[") ||
126
+ via.includes(`[rel="preload"]`) ||
102
127
  via.includes(`[rel="dns-prefetch"]`) ||
103
128
  via.includes(`[rel="token_endpoint"]`) ||
104
129
  via.includes(`[rel="authorization_endpoint"]`) ||
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zachleat/fetch-links",
3
- "version": "1.0.1",
3
+ "version": "1.0.3",
4
4
  "description": "Utility to fetch hyperlinks from a URL.",
5
5
  "main": "links.js",
6
6
  "type": "module",
package/test/test.js CHANGED
@@ -10,7 +10,10 @@ test("zachleat.com", async (t) => {
10
10
  // let originalUrl = "https://www.miriamsuzanne.com/";
11
11
  // let originalUrl = "https://daverupert.com/";
12
12
  // let originalUrl = "https://aaadaaam.com/";
13
+ // let originalUrl = "https://torvalds-family.blogspot.com/";
14
+ // let originalUrl = "https://wesbos.com/";
13
15
 
14
16
  let filtered = await Links.find(originalUrl);
17
+ // console.log( filtered );
15
18
  assert.ok(filtered.length > 0);
16
19
  });