feedcanon 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +88 -0
- package/dist/defaults.cjs +188 -0
- package/dist/defaults.d.cts +9 -0
- package/dist/defaults.d.ts +9 -0
- package/dist/defaults.js +185 -0
- package/dist/exports.cjs +15 -0
- package/dist/exports.d.cts +6 -0
- package/dist/exports.d.ts +6 -0
- package/dist/exports.js +6 -0
- package/dist/index.cjs +131 -0
- package/dist/index.d.cts +6 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +131 -0
- package/dist/platforms/feedburner.cjs +21 -0
- package/dist/platforms/feedburner.d.cts +6 -0
- package/dist/platforms/feedburner.d.ts +6 -0
- package/dist/platforms/feedburner.js +20 -0
- package/dist/types.d.cts +64 -0
- package/dist/types.d.ts +64 -0
- package/dist/utils.cjs +163 -0
- package/dist/utils.d.cts +11 -0
- package/dist/utils.d.ts +11 -0
- package/dist/utils.js +157 -0
- package/package.json +64 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Maciej Lamberski
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# Feedcanon
|
|
2
|
+
|
|
3
|
+
[](https://codecov.io/gh/macieklamberski/feedcanon)
|
|
4
|
+
[](https://www.npmjs.com/package/feedcanon)
|
|
5
|
+
[](https://github.com/macieklamberski/feedcanon/blob/main/LICENSE)
|
|
6
|
+
|
|
7
|
+
Find the canonical URL for any web feed by comparing actual content. Turn messy feed URLs into their cleanest, most reliable form.
|
|
8
|
+
|
|
9
|
+
Many URLs can point to the same feed—varying by protocol, www prefixes, trailing slashes, order of params, or domain aliases. Feedcanon compares actual feed content, considers the feed's declared self URL, and tests simpler URL alternatives to find the cleanest working one. Perfect for feed readers that need consistent, deduplicated subscriptions.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## How It Works
|
|
14
|
+
|
|
15
|
+
- Read the feed's declared self URL (`atom:link rel="self"`) and validate it serves identical content.
|
|
16
|
+
- Generate URL variants from cleanest to least clean, testing each until one works.
|
|
17
|
+
- Verify URLs serve the same feed using exact body match, then signature-based matching.
|
|
18
|
+
- Attempt to upgrade HTTP URLs to HTTPS when both serve identical content.
|
|
19
|
+
- Normalize platform-specific domains (e.g., FeedBurner aliases like `feedproxy.google.com` → `feeds.feedburner.com`).
|
|
20
|
+
|
|
21
|
+
## Quick Start
|
|
22
|
+
|
|
23
|
+
### Installation
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
npm install feedcanon
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### Basic Usage
|
|
30
|
+
|
|
31
|
+
```typescript
|
|
32
|
+
import { findCanonical } from 'feedcanon'
|
|
33
|
+
|
|
34
|
+
const url = await findCanonical('https://www.example.com/feed/?utm_source=twitter')
|
|
35
|
+
|
|
36
|
+
// 'https://example.com/feed'
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
### With Callbacks
|
|
40
|
+
|
|
41
|
+
```typescript
|
|
42
|
+
import { findCanonical } from 'feedcanon'
|
|
43
|
+
|
|
44
|
+
const url = await findCanonical('https://example.com/feed', {
|
|
45
|
+
onFetch: ({ url, response }) => {
|
|
46
|
+
console.log('Fetched:', url, response.status)
|
|
47
|
+
},
|
|
48
|
+
onMatch: ({ url, feed }) => {
|
|
49
|
+
console.log('Found matching URL:', url)
|
|
50
|
+
},
|
|
51
|
+
})
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### Custom Fetch
|
|
55
|
+
|
|
56
|
+
```typescript
|
|
57
|
+
import { findCanonical } from 'feedcanon'
|
|
58
|
+
import axios from 'axios'
|
|
59
|
+
|
|
60
|
+
const url = await findCanonical('https://example.com/feed', {
|
|
61
|
+
fetchFn: async (url) => {
|
|
62
|
+
const response = await axios.get(url)
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
status: response.status,
|
|
66
|
+
url: response.request.res.responseUrl,
|
|
67
|
+
body: response.data,
|
|
68
|
+
headers: new Headers(response.headers),
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
})
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Database Integration
|
|
75
|
+
|
|
76
|
+
```typescript
|
|
77
|
+
import { findCanonical } from 'feedcanon'
|
|
78
|
+
|
|
79
|
+
const url = await findCanonical('https://example.com/feed', {
|
|
80
|
+
existsFn: async (url) => {
|
|
81
|
+
// Return data if URL exists in your database, undefined otherwise.
|
|
82
|
+
return await db.feeds.findByUrl(url)
|
|
83
|
+
},
|
|
84
|
+
onExists: ({ url, data }) => {
|
|
85
|
+
console.log('URL already exists:', url)
|
|
86
|
+
},
|
|
87
|
+
})
|
|
88
|
+
```
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
const require_feedburner = require('./platforms/feedburner.cjs');
|
|
2
|
+
|
|
3
|
+
//#region src/defaults.ts
|
|
4
|
+
const defaultPlatforms = [require_feedburner.feedburnerHandler];
|
|
5
|
+
const defaultStrippedParams = [
|
|
6
|
+
"utm_source",
|
|
7
|
+
"utm_medium",
|
|
8
|
+
"utm_campaign",
|
|
9
|
+
"utm_term",
|
|
10
|
+
"utm_content",
|
|
11
|
+
"utm_reader",
|
|
12
|
+
"utm_name",
|
|
13
|
+
"utm_cid",
|
|
14
|
+
"utm_viz_id",
|
|
15
|
+
"gclid",
|
|
16
|
+
"dclid",
|
|
17
|
+
"gbraid",
|
|
18
|
+
"wbraid",
|
|
19
|
+
"gclsrc",
|
|
20
|
+
"gad_source",
|
|
21
|
+
"fbclid",
|
|
22
|
+
"fb_action_ids",
|
|
23
|
+
"fb_action_types",
|
|
24
|
+
"fb_source",
|
|
25
|
+
"fb_ref",
|
|
26
|
+
"_ga",
|
|
27
|
+
"_gl",
|
|
28
|
+
"_bk",
|
|
29
|
+
"_ke",
|
|
30
|
+
"mc_cid",
|
|
31
|
+
"mc_eid",
|
|
32
|
+
"mkt_tok",
|
|
33
|
+
"msclkid",
|
|
34
|
+
"twclid",
|
|
35
|
+
"ttclid",
|
|
36
|
+
"igshid",
|
|
37
|
+
"mtm_campaign",
|
|
38
|
+
"mtm_cid",
|
|
39
|
+
"mtm_content",
|
|
40
|
+
"mtm_group",
|
|
41
|
+
"mtm_keyword",
|
|
42
|
+
"mtm_medium",
|
|
43
|
+
"mtm_placement",
|
|
44
|
+
"mtm_source",
|
|
45
|
+
"pk_campaign",
|
|
46
|
+
"pk_cid",
|
|
47
|
+
"pk_content",
|
|
48
|
+
"pk_keyword",
|
|
49
|
+
"pk_medium",
|
|
50
|
+
"pk_source",
|
|
51
|
+
"ncid",
|
|
52
|
+
"sr_share",
|
|
53
|
+
"hsa_acc",
|
|
54
|
+
"hsa_ad",
|
|
55
|
+
"hsa_cam",
|
|
56
|
+
"hsa_grp",
|
|
57
|
+
"hsa_kw",
|
|
58
|
+
"hsa_mt",
|
|
59
|
+
"hsa_net",
|
|
60
|
+
"hsa_src",
|
|
61
|
+
"hsa_tgt",
|
|
62
|
+
"hsa_ver",
|
|
63
|
+
"cid",
|
|
64
|
+
"s_kwcid",
|
|
65
|
+
"ef_id",
|
|
66
|
+
"obOrigUrl",
|
|
67
|
+
"dicbo",
|
|
68
|
+
"yclid",
|
|
69
|
+
"_",
|
|
70
|
+
"timestamp",
|
|
71
|
+
"ts",
|
|
72
|
+
"cb",
|
|
73
|
+
"cachebuster",
|
|
74
|
+
"nocache",
|
|
75
|
+
"rand",
|
|
76
|
+
"random",
|
|
77
|
+
"action_object_map",
|
|
78
|
+
"action_ref_map",
|
|
79
|
+
"action_type_map",
|
|
80
|
+
"algo_expid",
|
|
81
|
+
"algo_pvid",
|
|
82
|
+
"at_campaign",
|
|
83
|
+
"at_custom1",
|
|
84
|
+
"at_custom2",
|
|
85
|
+
"at_custom3",
|
|
86
|
+
"at_custom4",
|
|
87
|
+
"at_medium",
|
|
88
|
+
"at_preview_index",
|
|
89
|
+
"campaign_id",
|
|
90
|
+
"click_sum",
|
|
91
|
+
"fref",
|
|
92
|
+
"gs_l",
|
|
93
|
+
"hmb_campaign",
|
|
94
|
+
"hmb_medium",
|
|
95
|
+
"hmb_source",
|
|
96
|
+
"itm_campaign",
|
|
97
|
+
"itm_medium",
|
|
98
|
+
"itm_source",
|
|
99
|
+
"ml_subscriber",
|
|
100
|
+
"ml_subscriber_hash",
|
|
101
|
+
"oly_anon_id",
|
|
102
|
+
"oly_enc_id",
|
|
103
|
+
"rb_clickid",
|
|
104
|
+
"referer",
|
|
105
|
+
"referrer",
|
|
106
|
+
"spm",
|
|
107
|
+
"trk",
|
|
108
|
+
"vero_conv",
|
|
109
|
+
"vero_id",
|
|
110
|
+
"wickedid",
|
|
111
|
+
"xtor"
|
|
112
|
+
];
|
|
113
|
+
const defaultNormalizeOptions = {
|
|
114
|
+
stripProtocol: true,
|
|
115
|
+
stripAuthentication: false,
|
|
116
|
+
stripWww: true,
|
|
117
|
+
stripTrailingSlash: true,
|
|
118
|
+
stripRootSlash: true,
|
|
119
|
+
collapseSlashes: true,
|
|
120
|
+
stripHash: true,
|
|
121
|
+
stripTextFragment: true,
|
|
122
|
+
sortQueryParams: true,
|
|
123
|
+
stripQueryParams: defaultStrippedParams,
|
|
124
|
+
stripEmptyQuery: true,
|
|
125
|
+
normalizeEncoding: true,
|
|
126
|
+
lowercaseHostname: true,
|
|
127
|
+
normalizeUnicode: true,
|
|
128
|
+
convertToPunycode: true
|
|
129
|
+
};
|
|
130
|
+
const defaultTiers = [
|
|
131
|
+
{
|
|
132
|
+
stripProtocol: false,
|
|
133
|
+
stripAuthentication: false,
|
|
134
|
+
stripWww: true,
|
|
135
|
+
stripTrailingSlash: true,
|
|
136
|
+
stripRootSlash: true,
|
|
137
|
+
collapseSlashes: true,
|
|
138
|
+
stripHash: true,
|
|
139
|
+
stripTextFragment: true,
|
|
140
|
+
sortQueryParams: true,
|
|
141
|
+
stripQueryParams: defaultStrippedParams,
|
|
142
|
+
stripEmptyQuery: true,
|
|
143
|
+
normalizeEncoding: true,
|
|
144
|
+
lowercaseHostname: true,
|
|
145
|
+
normalizeUnicode: true,
|
|
146
|
+
convertToPunycode: true
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
stripProtocol: false,
|
|
150
|
+
stripAuthentication: false,
|
|
151
|
+
stripWww: false,
|
|
152
|
+
stripTrailingSlash: true,
|
|
153
|
+
stripRootSlash: true,
|
|
154
|
+
collapseSlashes: true,
|
|
155
|
+
stripHash: true,
|
|
156
|
+
stripTextFragment: true,
|
|
157
|
+
sortQueryParams: true,
|
|
158
|
+
stripQueryParams: defaultStrippedParams,
|
|
159
|
+
stripEmptyQuery: true,
|
|
160
|
+
normalizeEncoding: true,
|
|
161
|
+
lowercaseHostname: true,
|
|
162
|
+
normalizeUnicode: true,
|
|
163
|
+
convertToPunycode: true
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
stripProtocol: false,
|
|
167
|
+
stripAuthentication: false,
|
|
168
|
+
stripWww: false,
|
|
169
|
+
stripTrailingSlash: false,
|
|
170
|
+
stripRootSlash: true,
|
|
171
|
+
collapseSlashes: true,
|
|
172
|
+
stripHash: true,
|
|
173
|
+
stripTextFragment: true,
|
|
174
|
+
sortQueryParams: true,
|
|
175
|
+
stripQueryParams: defaultStrippedParams,
|
|
176
|
+
stripEmptyQuery: true,
|
|
177
|
+
normalizeEncoding: true,
|
|
178
|
+
lowercaseHostname: true,
|
|
179
|
+
normalizeUnicode: true,
|
|
180
|
+
convertToPunycode: true
|
|
181
|
+
}
|
|
182
|
+
];
|
|
183
|
+
|
|
184
|
+
//#endregion
|
|
185
|
+
exports.defaultNormalizeOptions = defaultNormalizeOptions;
|
|
186
|
+
exports.defaultPlatforms = defaultPlatforms;
|
|
187
|
+
exports.defaultStrippedParams = defaultStrippedParams;
|
|
188
|
+
exports.defaultTiers = defaultTiers;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { NormalizeOptions, PlatformHandler } from "./types.cjs";
|
|
2
|
+
|
|
3
|
+
//#region src/defaults.d.ts
|
|
4
|
+
declare const defaultPlatforms: Array<PlatformHandler>;
|
|
5
|
+
declare const defaultStrippedParams: string[];
|
|
6
|
+
declare const defaultNormalizeOptions: NormalizeOptions;
|
|
7
|
+
declare const defaultTiers: Array<NormalizeOptions>;
|
|
8
|
+
//#endregion
|
|
9
|
+
export { defaultNormalizeOptions, defaultPlatforms, defaultStrippedParams, defaultTiers };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { NormalizeOptions, PlatformHandler } from "./types.js";
|
|
2
|
+
|
|
3
|
+
//#region src/defaults.d.ts
|
|
4
|
+
declare const defaultPlatforms: Array<PlatformHandler>;
|
|
5
|
+
declare const defaultStrippedParams: string[];
|
|
6
|
+
declare const defaultNormalizeOptions: NormalizeOptions;
|
|
7
|
+
declare const defaultTiers: Array<NormalizeOptions>;
|
|
8
|
+
//#endregion
|
|
9
|
+
export { defaultNormalizeOptions, defaultPlatforms, defaultStrippedParams, defaultTiers };
|
package/dist/defaults.js
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { feedburnerHandler } from "./platforms/feedburner.js";
|
|
2
|
+
|
|
3
|
+
//#region src/defaults.ts
|
|
4
|
+
const defaultPlatforms = [feedburnerHandler];
|
|
5
|
+
const defaultStrippedParams = [
|
|
6
|
+
"utm_source",
|
|
7
|
+
"utm_medium",
|
|
8
|
+
"utm_campaign",
|
|
9
|
+
"utm_term",
|
|
10
|
+
"utm_content",
|
|
11
|
+
"utm_reader",
|
|
12
|
+
"utm_name",
|
|
13
|
+
"utm_cid",
|
|
14
|
+
"utm_viz_id",
|
|
15
|
+
"gclid",
|
|
16
|
+
"dclid",
|
|
17
|
+
"gbraid",
|
|
18
|
+
"wbraid",
|
|
19
|
+
"gclsrc",
|
|
20
|
+
"gad_source",
|
|
21
|
+
"fbclid",
|
|
22
|
+
"fb_action_ids",
|
|
23
|
+
"fb_action_types",
|
|
24
|
+
"fb_source",
|
|
25
|
+
"fb_ref",
|
|
26
|
+
"_ga",
|
|
27
|
+
"_gl",
|
|
28
|
+
"_bk",
|
|
29
|
+
"_ke",
|
|
30
|
+
"mc_cid",
|
|
31
|
+
"mc_eid",
|
|
32
|
+
"mkt_tok",
|
|
33
|
+
"msclkid",
|
|
34
|
+
"twclid",
|
|
35
|
+
"ttclid",
|
|
36
|
+
"igshid",
|
|
37
|
+
"mtm_campaign",
|
|
38
|
+
"mtm_cid",
|
|
39
|
+
"mtm_content",
|
|
40
|
+
"mtm_group",
|
|
41
|
+
"mtm_keyword",
|
|
42
|
+
"mtm_medium",
|
|
43
|
+
"mtm_placement",
|
|
44
|
+
"mtm_source",
|
|
45
|
+
"pk_campaign",
|
|
46
|
+
"pk_cid",
|
|
47
|
+
"pk_content",
|
|
48
|
+
"pk_keyword",
|
|
49
|
+
"pk_medium",
|
|
50
|
+
"pk_source",
|
|
51
|
+
"ncid",
|
|
52
|
+
"sr_share",
|
|
53
|
+
"hsa_acc",
|
|
54
|
+
"hsa_ad",
|
|
55
|
+
"hsa_cam",
|
|
56
|
+
"hsa_grp",
|
|
57
|
+
"hsa_kw",
|
|
58
|
+
"hsa_mt",
|
|
59
|
+
"hsa_net",
|
|
60
|
+
"hsa_src",
|
|
61
|
+
"hsa_tgt",
|
|
62
|
+
"hsa_ver",
|
|
63
|
+
"cid",
|
|
64
|
+
"s_kwcid",
|
|
65
|
+
"ef_id",
|
|
66
|
+
"obOrigUrl",
|
|
67
|
+
"dicbo",
|
|
68
|
+
"yclid",
|
|
69
|
+
"_",
|
|
70
|
+
"timestamp",
|
|
71
|
+
"ts",
|
|
72
|
+
"cb",
|
|
73
|
+
"cachebuster",
|
|
74
|
+
"nocache",
|
|
75
|
+
"rand",
|
|
76
|
+
"random",
|
|
77
|
+
"action_object_map",
|
|
78
|
+
"action_ref_map",
|
|
79
|
+
"action_type_map",
|
|
80
|
+
"algo_expid",
|
|
81
|
+
"algo_pvid",
|
|
82
|
+
"at_campaign",
|
|
83
|
+
"at_custom1",
|
|
84
|
+
"at_custom2",
|
|
85
|
+
"at_custom3",
|
|
86
|
+
"at_custom4",
|
|
87
|
+
"at_medium",
|
|
88
|
+
"at_preview_index",
|
|
89
|
+
"campaign_id",
|
|
90
|
+
"click_sum",
|
|
91
|
+
"fref",
|
|
92
|
+
"gs_l",
|
|
93
|
+
"hmb_campaign",
|
|
94
|
+
"hmb_medium",
|
|
95
|
+
"hmb_source",
|
|
96
|
+
"itm_campaign",
|
|
97
|
+
"itm_medium",
|
|
98
|
+
"itm_source",
|
|
99
|
+
"ml_subscriber",
|
|
100
|
+
"ml_subscriber_hash",
|
|
101
|
+
"oly_anon_id",
|
|
102
|
+
"oly_enc_id",
|
|
103
|
+
"rb_clickid",
|
|
104
|
+
"referer",
|
|
105
|
+
"referrer",
|
|
106
|
+
"spm",
|
|
107
|
+
"trk",
|
|
108
|
+
"vero_conv",
|
|
109
|
+
"vero_id",
|
|
110
|
+
"wickedid",
|
|
111
|
+
"xtor"
|
|
112
|
+
];
|
|
113
|
+
const defaultNormalizeOptions = {
|
|
114
|
+
stripProtocol: true,
|
|
115
|
+
stripAuthentication: false,
|
|
116
|
+
stripWww: true,
|
|
117
|
+
stripTrailingSlash: true,
|
|
118
|
+
stripRootSlash: true,
|
|
119
|
+
collapseSlashes: true,
|
|
120
|
+
stripHash: true,
|
|
121
|
+
stripTextFragment: true,
|
|
122
|
+
sortQueryParams: true,
|
|
123
|
+
stripQueryParams: defaultStrippedParams,
|
|
124
|
+
stripEmptyQuery: true,
|
|
125
|
+
normalizeEncoding: true,
|
|
126
|
+
lowercaseHostname: true,
|
|
127
|
+
normalizeUnicode: true,
|
|
128
|
+
convertToPunycode: true
|
|
129
|
+
};
|
|
130
|
+
const defaultTiers = [
|
|
131
|
+
{
|
|
132
|
+
stripProtocol: false,
|
|
133
|
+
stripAuthentication: false,
|
|
134
|
+
stripWww: true,
|
|
135
|
+
stripTrailingSlash: true,
|
|
136
|
+
stripRootSlash: true,
|
|
137
|
+
collapseSlashes: true,
|
|
138
|
+
stripHash: true,
|
|
139
|
+
stripTextFragment: true,
|
|
140
|
+
sortQueryParams: true,
|
|
141
|
+
stripQueryParams: defaultStrippedParams,
|
|
142
|
+
stripEmptyQuery: true,
|
|
143
|
+
normalizeEncoding: true,
|
|
144
|
+
lowercaseHostname: true,
|
|
145
|
+
normalizeUnicode: true,
|
|
146
|
+
convertToPunycode: true
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
stripProtocol: false,
|
|
150
|
+
stripAuthentication: false,
|
|
151
|
+
stripWww: false,
|
|
152
|
+
stripTrailingSlash: true,
|
|
153
|
+
stripRootSlash: true,
|
|
154
|
+
collapseSlashes: true,
|
|
155
|
+
stripHash: true,
|
|
156
|
+
stripTextFragment: true,
|
|
157
|
+
sortQueryParams: true,
|
|
158
|
+
stripQueryParams: defaultStrippedParams,
|
|
159
|
+
stripEmptyQuery: true,
|
|
160
|
+
normalizeEncoding: true,
|
|
161
|
+
lowercaseHostname: true,
|
|
162
|
+
normalizeUnicode: true,
|
|
163
|
+
convertToPunycode: true
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
stripProtocol: false,
|
|
167
|
+
stripAuthentication: false,
|
|
168
|
+
stripWww: false,
|
|
169
|
+
stripTrailingSlash: false,
|
|
170
|
+
stripRootSlash: true,
|
|
171
|
+
collapseSlashes: true,
|
|
172
|
+
stripHash: true,
|
|
173
|
+
stripTextFragment: true,
|
|
174
|
+
sortQueryParams: true,
|
|
175
|
+
stripQueryParams: defaultStrippedParams,
|
|
176
|
+
stripEmptyQuery: true,
|
|
177
|
+
normalizeEncoding: true,
|
|
178
|
+
lowercaseHostname: true,
|
|
179
|
+
normalizeUnicode: true,
|
|
180
|
+
convertToPunycode: true
|
|
181
|
+
}
|
|
182
|
+
];
|
|
183
|
+
|
|
184
|
+
//#endregion
|
|
185
|
+
export { defaultNormalizeOptions, defaultPlatforms, defaultStrippedParams, defaultTiers };
|
package/dist/exports.cjs
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const require_feedburner = require('./platforms/feedburner.cjs');
|
|
2
|
+
const require_defaults = require('./defaults.cjs');
|
|
3
|
+
const require_utils = require('./utils.cjs');
|
|
4
|
+
const require_index = require('./index.cjs');
|
|
5
|
+
|
|
6
|
+
exports.addMissingProtocol = require_utils.addMissingProtocol;
|
|
7
|
+
exports.defaultPlatforms = require_defaults.defaultPlatforms;
|
|
8
|
+
exports.defaultStrippedParams = require_defaults.defaultStrippedParams;
|
|
9
|
+
exports.defaultTiers = require_defaults.defaultTiers;
|
|
10
|
+
exports.feedburnerHandler = require_feedburner.feedburnerHandler;
|
|
11
|
+
exports.feedsmithParser = require_utils.feedsmithParser;
|
|
12
|
+
exports.findCanonical = require_index.findCanonical;
|
|
13
|
+
exports.normalizeUrl = require_utils.normalizeUrl;
|
|
14
|
+
exports.resolveFeedProtocol = require_utils.resolveFeedProtocol;
|
|
15
|
+
exports.resolveUrl = require_utils.resolveUrl;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { ExistsFn, FetchFn, FetchFnOptions, FetchFnResponse, FindCanonicalOptions, NormalizeOptions, OnExistsFn, OnFetchFn, OnMatchFn, ParserAdapter, PlatformHandler } from "./types.cjs";
|
|
2
|
+
import { defaultPlatforms, defaultStrippedParams, defaultTiers } from "./defaults.cjs";
|
|
3
|
+
import { findCanonical } from "./index.cjs";
|
|
4
|
+
import { feedburnerHandler } from "./platforms/feedburner.cjs";
|
|
5
|
+
import { addMissingProtocol, feedsmithParser, normalizeUrl, resolveFeedProtocol, resolveUrl } from "./utils.cjs";
|
|
6
|
+
export { type ExistsFn, type FetchFn, type FetchFnOptions, type FetchFnResponse, type FindCanonicalOptions, type NormalizeOptions, type OnExistsFn, type OnFetchFn, type OnMatchFn, type ParserAdapter, type PlatformHandler, addMissingProtocol, defaultPlatforms, defaultStrippedParams, defaultTiers, feedburnerHandler, feedsmithParser, findCanonical, normalizeUrl, resolveFeedProtocol, resolveUrl };
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { ExistsFn, FetchFn, FetchFnOptions, FetchFnResponse, FindCanonicalOptions, NormalizeOptions, OnExistsFn, OnFetchFn, OnMatchFn, ParserAdapter, PlatformHandler } from "./types.js";
|
|
2
|
+
import { defaultPlatforms, defaultStrippedParams, defaultTiers } from "./defaults.js";
|
|
3
|
+
import { findCanonical } from "./index.js";
|
|
4
|
+
import { feedburnerHandler } from "./platforms/feedburner.js";
|
|
5
|
+
import { addMissingProtocol, feedsmithParser, normalizeUrl, resolveFeedProtocol, resolveUrl } from "./utils.js";
|
|
6
|
+
export { type ExistsFn, type FetchFn, type FetchFnOptions, type FetchFnResponse, type FindCanonicalOptions, type NormalizeOptions, type OnExistsFn, type OnFetchFn, type OnMatchFn, type ParserAdapter, type PlatformHandler, addMissingProtocol, defaultPlatforms, defaultStrippedParams, defaultTiers, feedburnerHandler, feedsmithParser, findCanonical, normalizeUrl, resolveFeedProtocol, resolveUrl };
|
package/dist/exports.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { feedburnerHandler } from "./platforms/feedburner.js";
|
|
2
|
+
import { defaultPlatforms, defaultStrippedParams, defaultTiers } from "./defaults.js";
|
|
3
|
+
import { addMissingProtocol, feedsmithParser, normalizeUrl, resolveFeedProtocol, resolveUrl } from "./utils.js";
|
|
4
|
+
import { findCanonical } from "./index.js";
|
|
5
|
+
|
|
6
|
+
export { addMissingProtocol, defaultPlatforms, defaultStrippedParams, defaultTiers, feedburnerHandler, feedsmithParser, findCanonical, normalizeUrl, resolveFeedProtocol, resolveUrl };
|
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
const require_defaults = require('./defaults.cjs');
|
|
2
|
+
const require_utils = require('./utils.cjs');
|
|
3
|
+
|
|
4
|
+
//#region src/index.ts
|
|
5
|
+
const findCanonical = async (inputUrl, options) => {
|
|
6
|
+
const { fetchFn = require_utils.nativeFetch, existsFn, parser = require_utils.feedsmithParser, tiers = require_defaults.defaultTiers, platforms = require_defaults.defaultPlatforms, onFetch, onMatch, onExists } = options ?? {};
|
|
7
|
+
const resolveAndApplyPlatformHandlers = (url, baseUrl) => {
|
|
8
|
+
const resolved = require_utils.resolveUrl(url, baseUrl);
|
|
9
|
+
return resolved ? require_utils.applyPlatformHandlers(resolved, platforms) : void 0;
|
|
10
|
+
};
|
|
11
|
+
const initialRequestUrl = resolveAndApplyPlatformHandlers(inputUrl);
|
|
12
|
+
if (!initialRequestUrl) return;
|
|
13
|
+
let initialResponse;
|
|
14
|
+
try {
|
|
15
|
+
initialResponse = await fetchFn(initialRequestUrl);
|
|
16
|
+
} catch {
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
onFetch?.({
|
|
20
|
+
url: initialRequestUrl,
|
|
21
|
+
response: initialResponse
|
|
22
|
+
});
|
|
23
|
+
if (initialResponse.status < 200 || initialResponse.status >= 300) return;
|
|
24
|
+
const initialResponseUrl = resolveAndApplyPlatformHandlers(initialResponse.url);
|
|
25
|
+
if (!initialResponseUrl) return;
|
|
26
|
+
const initialResponseBody = initialResponse.body;
|
|
27
|
+
if (!initialResponseBody) return;
|
|
28
|
+
let initialResponseSignature;
|
|
29
|
+
let selfRequestUrl;
|
|
30
|
+
const initialResponseFeed = parser.parse(initialResponseBody);
|
|
31
|
+
if (!initialResponseFeed) return;
|
|
32
|
+
onMatch?.({
|
|
33
|
+
url: initialRequestUrl,
|
|
34
|
+
response: initialResponse,
|
|
35
|
+
feed: initialResponseFeed
|
|
36
|
+
});
|
|
37
|
+
const selfRequestUrlRaw = parser.getSelfUrl(initialResponseFeed);
|
|
38
|
+
if (selfRequestUrlRaw) selfRequestUrl = resolveAndApplyPlatformHandlers(selfRequestUrlRaw, initialResponseUrl);
|
|
39
|
+
const compareWithInitialResponse = (comparedResponseBody) => {
|
|
40
|
+
if (!comparedResponseBody) return false;
|
|
41
|
+
if (initialResponseBody === comparedResponseBody) return true;
|
|
42
|
+
const comparedResponseFeed = parser.parse(comparedResponseBody);
|
|
43
|
+
if (comparedResponseFeed) {
|
|
44
|
+
initialResponseSignature ||= JSON.stringify(parser.getSignature(initialResponseFeed));
|
|
45
|
+
const comparedResponseSignature = JSON.stringify(parser.getSignature(comparedResponseFeed));
|
|
46
|
+
return initialResponseSignature === comparedResponseSignature;
|
|
47
|
+
}
|
|
48
|
+
return false;
|
|
49
|
+
};
|
|
50
|
+
const fetchAndCompare = async (url) => {
|
|
51
|
+
let response;
|
|
52
|
+
try {
|
|
53
|
+
response = await fetchFn(url);
|
|
54
|
+
} catch {
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
onFetch?.({
|
|
58
|
+
url,
|
|
59
|
+
response
|
|
60
|
+
});
|
|
61
|
+
if (response.status < 200 || response.status >= 300) return;
|
|
62
|
+
if (!compareWithInitialResponse(response.body)) return;
|
|
63
|
+
return response;
|
|
64
|
+
};
|
|
65
|
+
let variantSource = initialResponseUrl;
|
|
66
|
+
if (selfRequestUrl && selfRequestUrl !== initialResponseUrl) {
|
|
67
|
+
const urlsToTry = [selfRequestUrl];
|
|
68
|
+
if (selfRequestUrl.startsWith("https://")) urlsToTry.push(selfRequestUrl.replace("https://", "http://"));
|
|
69
|
+
else if (selfRequestUrl.startsWith("http://")) urlsToTry.push(selfRequestUrl.replace("http://", "https://"));
|
|
70
|
+
for (const urlToTry of urlsToTry) {
|
|
71
|
+
const response = await fetchAndCompare(urlToTry);
|
|
72
|
+
if (response) {
|
|
73
|
+
onMatch?.({
|
|
74
|
+
url: urlToTry,
|
|
75
|
+
response,
|
|
76
|
+
feed: initialResponseFeed
|
|
77
|
+
});
|
|
78
|
+
variantSource = resolveAndApplyPlatformHandlers(response.url) ?? initialResponseUrl;
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const variants = new Set(tiers.map((tier) => resolveAndApplyPlatformHandlers(require_utils.normalizeUrl(variantSource, tier))).filter((url) => url !== void 0));
|
|
84
|
+
variants.add(variantSource);
|
|
85
|
+
let winningUrl = variantSource;
|
|
86
|
+
for (const variant of variants) {
|
|
87
|
+
if (existsFn) {
|
|
88
|
+
const data = await existsFn(variant);
|
|
89
|
+
if (data !== void 0) {
|
|
90
|
+
onExists?.({
|
|
91
|
+
url: variant,
|
|
92
|
+
data
|
|
93
|
+
});
|
|
94
|
+
return variant;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (variant === variantSource) continue;
|
|
98
|
+
if (variant === initialResponseUrl) {
|
|
99
|
+
winningUrl = initialResponseUrl;
|
|
100
|
+
break;
|
|
101
|
+
}
|
|
102
|
+
const response = await fetchAndCompare(variant);
|
|
103
|
+
if (response) {
|
|
104
|
+
const preparedResponseUrl = resolveAndApplyPlatformHandlers(response.url);
|
|
105
|
+
if (preparedResponseUrl === variantSource || preparedResponseUrl === initialResponseUrl) continue;
|
|
106
|
+
onMatch?.({
|
|
107
|
+
url: variant,
|
|
108
|
+
response,
|
|
109
|
+
feed: initialResponseFeed
|
|
110
|
+
});
|
|
111
|
+
winningUrl = variant;
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (winningUrl.startsWith("http://")) {
|
|
116
|
+
const httpsUrl = winningUrl.replace("http://", "https://");
|
|
117
|
+
const response = await fetchAndCompare(httpsUrl);
|
|
118
|
+
if (response) {
|
|
119
|
+
onMatch?.({
|
|
120
|
+
url: httpsUrl,
|
|
121
|
+
response,
|
|
122
|
+
feed: initialResponseFeed
|
|
123
|
+
});
|
|
124
|
+
return httpsUrl;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return winningUrl;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
//#endregion
|
|
131
|
+
exports.findCanonical = findCanonical;
|
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { FindCanonicalOptions } from "./types.cjs";
|
|
2
|
+
|
|
3
|
+
//#region src/index.d.ts
|
|
4
|
+
declare const findCanonical: <TFeed, TExisting>(inputUrl: string, options?: FindCanonicalOptions<TFeed, TExisting>) => Promise<string | undefined>;
|
|
5
|
+
//#endregion
|
|
6
|
+
export { findCanonical };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { FindCanonicalOptions } from "./types.js";
|
|
2
|
+
|
|
3
|
+
//#region src/index.d.ts
|
|
4
|
+
declare const findCanonical: <TFeed, TExisting>(inputUrl: string, options?: FindCanonicalOptions<TFeed, TExisting>) => Promise<string | undefined>;
|
|
5
|
+
//#endregion
|
|
6
|
+
export { findCanonical };
|