feedscout 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -17
- package/dist/blogrolls/defaults.cjs +1 -3
- package/dist/blogrolls/defaults.js +1 -2
- package/dist/blogrolls/extractors.cjs +6 -9
- package/dist/blogrolls/extractors.js +6 -9
- package/dist/blogrolls/index.cjs +6 -9
- package/dist/blogrolls/index.d.cts +1 -1
- package/dist/blogrolls/index.d.ts +1 -1
- package/dist/blogrolls/index.js +1 -4
- package/dist/blogrolls.cjs +4 -5
- package/dist/blogrolls.js +1 -2
- package/dist/common/discover/index.cjs +24 -16
- package/dist/common/discover/index.js +20 -12
- package/dist/common/discover/utils.cjs +19 -9
- package/dist/common/discover/utils.js +18 -8
- package/dist/common/locales.cjs +59 -15
- package/dist/common/locales.js +49 -4
- package/dist/common/types.cjs +2 -3
- package/dist/common/types.d.cts +11 -6
- package/dist/common/types.d.ts +11 -6
- package/dist/common/types.js +2 -2
- package/dist/common/uris/feed/index.cjs +12 -0
- package/dist/common/uris/feed/index.d.cts +6 -0
- package/dist/common/uris/feed/index.d.ts +6 -0
- package/dist/common/uris/feed/index.js +12 -0
- package/dist/common/uris/feed/types.d.cts +9 -0
- package/dist/common/uris/feed/types.d.ts +9 -0
- package/dist/common/uris/guess/index.cjs +2 -4
- package/dist/common/uris/guess/index.js +1 -3
- package/dist/common/uris/guess/utils.cjs +1 -3
- package/dist/common/uris/guess/utils.js +1 -2
- package/dist/common/uris/headers/index.cjs +2 -4
- package/dist/common/uris/headers/index.js +1 -3
- package/dist/common/uris/html/handlers.cjs +2 -4
- package/dist/common/uris/html/handlers.js +1 -3
- package/dist/common/uris/html/index.cjs +2 -4
- package/dist/common/uris/html/index.js +1 -3
- package/dist/common/uris/index.cjs +15 -12
- package/dist/common/uris/index.js +8 -5
- package/dist/common/uris/platform/index.cjs +3 -5
- package/dist/common/uris/platform/index.d.cts +7 -0
- package/dist/common/uris/platform/index.d.ts +7 -0
- package/dist/common/uris/platform/index.js +3 -4
- package/dist/common/uris/platform/types.d.cts +3 -3
- package/dist/common/uris/platform/types.d.ts +3 -3
- package/dist/common/utils.cjs +20 -9
- package/dist/common/utils.d.cts +2 -2
- package/dist/common/utils.d.ts +2 -2
- package/dist/common/utils.js +18 -8
- package/dist/favicons/defaults.cjs +60 -0
- package/dist/favicons/defaults.d.cts +18 -0
- package/dist/favicons/defaults.d.ts +18 -0
- package/dist/favicons/defaults.js +53 -0
- package/dist/favicons/extractors.cjs +13 -0
- package/dist/favicons/extractors.d.cts +7 -0
- package/dist/favicons/extractors.d.ts +7 -0
- package/dist/favicons/extractors.js +13 -0
- package/dist/favicons/index.cjs +29 -0
- package/dist/favicons/index.d.cts +7 -0
- package/dist/favicons/index.d.ts +7 -0
- package/dist/favicons/index.js +29 -0
- package/dist/favicons/platform/handlers/bluesky.cjs +27 -0
- package/dist/favicons/platform/handlers/bluesky.js +27 -0
- package/dist/favicons/platform/handlers/codeberg.cjs +18 -0
- package/dist/favicons/platform/handlers/codeberg.js +18 -0
- package/dist/favicons/platform/handlers/deviantart.cjs +29 -0
- package/dist/favicons/platform/handlers/deviantart.js +29 -0
- package/dist/favicons/platform/handlers/github.cjs +18 -0
- package/dist/favicons/platform/handlers/github.js +18 -0
- package/dist/favicons/platform/handlers/githubGist.cjs +18 -0
- package/dist/favicons/platform/handlers/githubGist.js +18 -0
- package/dist/favicons/platform/handlers/lobsters.cjs +21 -0
- package/dist/favicons/platform/handlers/lobsters.js +21 -0
- package/dist/favicons/platform/handlers/mastodon.cjs +36 -0
- package/dist/favicons/platform/handlers/mastodon.js +34 -0
- package/dist/favicons/platform/handlers/reddit.cjs +41 -0
- package/dist/favicons/platform/handlers/reddit.js +41 -0
- package/dist/favicons/platform/handlers/sourceforge.cjs +21 -0
- package/dist/favicons/platform/handlers/sourceforge.js +21 -0
- package/dist/favicons/platform/handlers/tumblr.cjs +16 -0
- package/dist/favicons/platform/handlers/tumblr.js +16 -0
- package/dist/favicons/types.d.cts +4 -0
- package/dist/favicons/types.d.ts +4 -0
- package/dist/favicons.cjs +12 -0
- package/dist/favicons.d.cts +4 -0
- package/dist/favicons.d.ts +4 -0
- package/dist/favicons.js +3 -0
- package/dist/feeds/defaults.cjs +55 -25
- package/dist/feeds/defaults.js +40 -10
- package/dist/feeds/extractors.cjs +1 -3
- package/dist/feeds/extractors.js +1 -3
- package/dist/feeds/index.cjs +6 -8
- package/dist/feeds/index.d.cts +1 -1
- package/dist/feeds/index.d.ts +1 -1
- package/dist/feeds/index.js +1 -3
- package/dist/feeds/platform/handlers/behance.cjs +2 -4
- package/dist/feeds/platform/handlers/behance.js +1 -3
- package/dist/feeds/platform/handlers/blogspot.cjs +7 -6
- package/dist/feeds/platform/handlers/blogspot.js +6 -5
- package/dist/feeds/platform/handlers/bluesky.cjs +3 -5
- package/dist/feeds/platform/handlers/bluesky.js +2 -4
- package/dist/feeds/platform/handlers/codeberg.cjs +69 -0
- package/dist/feeds/platform/handlers/codeberg.js +67 -0
- package/dist/feeds/platform/handlers/csdn.cjs +19 -0
- package/dist/feeds/platform/handlers/csdn.js +19 -0
- package/dist/feeds/platform/handlers/dailymotion.cjs +2 -4
- package/dist/feeds/platform/handlers/dailymotion.js +1 -3
- package/dist/feeds/platform/handlers/deviantart.cjs +4 -4
- package/dist/feeds/platform/handlers/deviantart.js +1 -3
- package/dist/feeds/platform/handlers/devto.cjs +2 -4
- package/dist/feeds/platform/handlers/devto.js +1 -3
- package/dist/feeds/platform/handlers/douban.cjs +54 -0
- package/dist/feeds/platform/handlers/douban.js +54 -0
- package/dist/feeds/platform/handlers/github.cjs +5 -4
- package/dist/feeds/platform/handlers/github.js +2 -3
- package/dist/feeds/platform/handlers/githubGist.cjs +4 -4
- package/dist/feeds/platform/handlers/githubGist.js +1 -3
- package/dist/feeds/platform/handlers/gitlab.cjs +2 -4
- package/dist/feeds/platform/handlers/gitlab.js +1 -3
- package/dist/feeds/platform/handlers/goodreads.cjs +39 -0
- package/dist/feeds/platform/handlers/goodreads.js +39 -0
- package/dist/feeds/platform/handlers/hashnode.cjs +16 -0
- package/dist/feeds/platform/handlers/hashnode.js +16 -0
- package/dist/feeds/platform/handlers/hatenablog.cjs +53 -0
- package/dist/feeds/platform/handlers/hatenablog.js +53 -0
- package/dist/feeds/platform/handlers/itchio.cjs +88 -0
- package/dist/feeds/platform/handlers/itchio.js +88 -0
- package/dist/feeds/platform/handlers/kickstarter.cjs +2 -4
- package/dist/feeds/platform/handlers/kickstarter.js +1 -3
- package/dist/feeds/platform/handlers/letterboxd.cjs +42 -0
- package/dist/feeds/platform/handlers/letterboxd.js +42 -0
- package/dist/feeds/platform/handlers/lobsters.cjs +3 -4
- package/dist/feeds/platform/handlers/lobsters.js +1 -3
- package/dist/feeds/platform/handlers/mastodon.cjs +40 -0
- package/dist/feeds/platform/handlers/mastodon.js +40 -0
- package/dist/feeds/platform/handlers/medium.cjs +2 -4
- package/dist/feeds/platform/handlers/medium.js +1 -3
- package/dist/feeds/platform/handlers/paragraph.cjs +21 -0
- package/dist/feeds/platform/handlers/paragraph.js +21 -0
- package/dist/feeds/platform/handlers/pinterest.cjs +2 -15
- package/dist/feeds/platform/handlers/pinterest.js +1 -14
- package/dist/feeds/platform/handlers/producthunt.cjs +2 -4
- package/dist/feeds/platform/handlers/producthunt.js +1 -3
- package/dist/feeds/platform/handlers/reddit.cjs +3 -4
- package/dist/feeds/platform/handlers/reddit.js +1 -3
- package/dist/feeds/platform/handlers/soundcloud.cjs +2 -4
- package/dist/feeds/platform/handlers/soundcloud.js +1 -3
- package/dist/feeds/platform/handlers/sourceforge.cjs +20 -0
- package/dist/feeds/platform/handlers/sourceforge.js +19 -0
- package/dist/feeds/platform/handlers/stackExchange.cjs +37 -0
- package/dist/feeds/platform/handlers/stackExchange.js +37 -0
- package/dist/feeds/platform/handlers/steam.cjs +26 -0
- package/dist/feeds/platform/handlers/steam.js +26 -0
- package/dist/feeds/platform/handlers/substack.cjs +2 -4
- package/dist/feeds/platform/handlers/substack.js +1 -3
- package/dist/feeds/platform/handlers/tumblr.cjs +5 -5
- package/dist/feeds/platform/handlers/tumblr.js +3 -4
- package/dist/feeds/platform/handlers/v2ex.cjs +33 -0
- package/dist/feeds/platform/handlers/v2ex.js +33 -0
- package/dist/feeds/platform/handlers/vimeo.cjs +68 -0
- package/dist/feeds/platform/handlers/vimeo.js +68 -0
- package/dist/feeds/platform/handlers/wordpress.cjs +5 -7
- package/dist/feeds/platform/handlers/wordpress.js +4 -6
- package/dist/feeds/platform/handlers/wpengine.cjs +4 -7
- package/dist/feeds/platform/handlers/wpengine.js +2 -4
- package/dist/feeds/platform/handlers/ximalaya.cjs +19 -0
- package/dist/feeds/platform/handlers/ximalaya.js +19 -0
- package/dist/feeds/platform/handlers/youtube.cjs +19 -8
- package/dist/feeds/platform/handlers/youtube.js +18 -7
- package/dist/feeds.cjs +4 -5
- package/dist/feeds.js +1 -2
- package/dist/hubs/discover/index.cjs +7 -9
- package/dist/hubs/discover/index.js +1 -3
- package/dist/hubs/discover/utils.cjs +1 -3
- package/dist/hubs/discover/utils.js +1 -2
- package/dist/hubs/feed/index.cjs +1 -3
- package/dist/hubs/feed/index.js +1 -3
- package/dist/hubs/headers/index.cjs +3 -5
- package/dist/hubs/headers/index.js +1 -3
- package/dist/hubs/html/index.cjs +3 -5
- package/dist/hubs/html/index.js +1 -3
- package/dist/hubs.js +1 -1
- package/dist/index.cjs +8 -7
- package/dist/index.d.cts +3 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -2
- package/dist/methods.cjs +13 -10
- package/dist/methods.d.cts +3 -1
- package/dist/methods.d.ts +3 -1
- package/dist/methods.js +3 -2
- package/dist/utils.cjs +3 -4
- package/dist/utils.js +1 -2
- package/package.json +15 -5
package/dist/common/locales.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
//#region src/common/locales.json
|
|
2
2
|
var errors = {
|
|
3
3
|
"platformMethodRequiresUrl": "Platform method requires url to be provided in input",
|
|
4
|
+
"feedMethodRequiresContent": "Feed method requires content to be provided in input",
|
|
4
5
|
"htmlMethodRequiresContent": "HTML method requires content to be provided in input",
|
|
5
6
|
"headersMethodRequiresHeaders": "Headers method requires headers to be provided in input",
|
|
6
7
|
"guessMethodRequiresUrl": "Guess method requires url to be provided in input"
|
|
@@ -9,6 +10,7 @@ var hints = {
|
|
|
9
10
|
"youtube:all": "All uploads",
|
|
10
11
|
"youtube:videos": "Videos only",
|
|
11
12
|
"youtube:shorts": "Shorts only",
|
|
13
|
+
"youtube:live": "Live streams only",
|
|
12
14
|
"youtube:playlist": "Playlist",
|
|
13
15
|
"github:activity": "Activity",
|
|
14
16
|
"github:releases": "Releases",
|
|
@@ -25,6 +27,11 @@ var hints = {
|
|
|
25
27
|
"gitlab:activity": "Activity",
|
|
26
28
|
"gitlab:releases": "Releases",
|
|
27
29
|
"gitlab:tags": "Tags",
|
|
30
|
+
"codeberg:activity": "Activity",
|
|
31
|
+
"codeberg:releases": "Releases",
|
|
32
|
+
"codeberg:tags": "Tags",
|
|
33
|
+
"codeberg:branch-commits": "Branch commits",
|
|
34
|
+
"codeberg:file-history": "File history",
|
|
28
35
|
"wordpress:category": "Category",
|
|
29
36
|
"wordpress:tag": "Tag",
|
|
30
37
|
"wordpress:author": "Author",
|
|
@@ -54,26 +61,64 @@ var hints = {
|
|
|
54
61
|
"github-gist:starred": "Starred",
|
|
55
62
|
"kickstarter:updates": "Updates",
|
|
56
63
|
"kickstarter:projects": "Projects",
|
|
64
|
+
"letterboxd:diary": "Diary",
|
|
57
65
|
"lobsters:tag": "Tag",
|
|
58
66
|
"lobsters:domain": "Domain",
|
|
59
67
|
"lobsters:stories": "Stories",
|
|
60
68
|
"lobsters:top": "Top stories",
|
|
61
69
|
"lobsters:newest": "Newest",
|
|
62
70
|
"lobsters:comments": "Comments",
|
|
71
|
+
"mastodon:posts": "Posts",
|
|
72
|
+
"mastodon:tag": "Tag",
|
|
63
73
|
"medium:posts": "Posts",
|
|
64
74
|
"medium:tag": "Tag",
|
|
65
75
|
"medium:tagged": "Tagged",
|
|
66
76
|
"medium:publication": "Publication",
|
|
67
77
|
"pinterest:pins": "Pins",
|
|
68
|
-
"pinterest:board": "Board",
|
|
69
78
|
"producthunt:topic": "Topic",
|
|
70
79
|
"producthunt:category": "Category",
|
|
71
80
|
"producthunt:products": "Products",
|
|
72
81
|
"soundcloud:tracks": "Tracks",
|
|
82
|
+
"stackexchange:tag": "Tag",
|
|
83
|
+
"stackexchange:question": "Question",
|
|
84
|
+
"stackexchange:user": "User",
|
|
85
|
+
"steam:news": "News",
|
|
86
|
+
"steam:group": "Group",
|
|
73
87
|
"substack:newsletter": "Newsletter",
|
|
74
88
|
"tumblr:posts": "Posts",
|
|
75
|
-
"tumblr:tag": "Tag"
|
|
89
|
+
"tumblr:tag": "Tag",
|
|
90
|
+
"goodreads:updates": "Updates",
|
|
91
|
+
"goodreads:reviews": "Reviews",
|
|
92
|
+
"sourceforge:activity": "Activity",
|
|
93
|
+
"vimeo:videos": "Videos",
|
|
94
|
+
"vimeo:likes": "Likes",
|
|
95
|
+
"vimeo:channel": "Channel",
|
|
96
|
+
"vimeo:group": "Group",
|
|
97
|
+
"hashnode:blog": "Blog",
|
|
98
|
+
"paragraph:blog": "Blog",
|
|
99
|
+
"hatenablog:posts-rss": "Posts (RSS)",
|
|
100
|
+
"hatenablog:posts-atom": "Posts (Atom)",
|
|
101
|
+
"hatenablog:category-rss": "Category (RSS)",
|
|
102
|
+
"hatenablog:category-atom": "Category (Atom)",
|
|
103
|
+
"hatenablog:author-rss": "Author (RSS)",
|
|
104
|
+
"hatenablog:author-atom": "Author (Atom)",
|
|
105
|
+
"itchio:devlog": "Devlog",
|
|
106
|
+
"itchio:games": "Games",
|
|
107
|
+
"itchio:tag": "Tag",
|
|
108
|
+
"itchio:section": "Section",
|
|
109
|
+
"itchio:featured": "Featured",
|
|
110
|
+
"itchio:new": "New",
|
|
111
|
+
"itchio:sales": "Sales",
|
|
112
|
+
"csdn:blog": "Blog",
|
|
113
|
+
"douban:interests": "Interests",
|
|
114
|
+
"douban:reviews": "Reviews",
|
|
115
|
+
"douban:notes": "Notes",
|
|
116
|
+
"douban:subjectReviews": "Subject reviews",
|
|
117
|
+
"v2ex:index": "Index",
|
|
118
|
+
"v2ex:node": "Node",
|
|
119
|
+
"v2ex:member": "Member",
|
|
120
|
+
"v2ex:tab": "Tab",
|
|
121
|
+
"ximalaya:album": "Album"
|
|
76
122
|
};
|
|
77
|
-
|
|
78
123
|
//#endregion
|
|
79
|
-
export { errors, hints };
|
|
124
|
+
export { errors, hints };
|
package/dist/common/types.cjs
CHANGED
package/dist/common/types.d.cts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { FeedMethodOptions } from "./uris/feed/types.cjs";
|
|
1
2
|
import { GuessMethodOptions } from "./uris/guess/types.cjs";
|
|
2
3
|
import { HeadersMethodOptions } from "./uris/headers/types.cjs";
|
|
3
4
|
import { HtmlMethodOptions } from "./uris/html/types.cjs";
|
|
@@ -13,7 +14,7 @@ type DiscoverUriEntry = {
|
|
|
13
14
|
uri: UriEntry;
|
|
14
15
|
hint?: DiscoverUriHint;
|
|
15
16
|
};
|
|
16
|
-
declare const discoverMethodOrder: readonly ["platform", "html", "headers", "guess"];
|
|
17
|
+
declare const discoverMethodOrder: readonly ["platform", "feed", "html", "headers", "guess"];
|
|
17
18
|
type DiscoverMethod = (typeof discoverMethodOrder)[number];
|
|
18
19
|
type LinkSelector = {
|
|
19
20
|
rel: string;
|
|
@@ -42,10 +43,12 @@ type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
|
|
|
42
43
|
type DiscoverResult<TValid = object> = ({
|
|
43
44
|
url: string;
|
|
44
45
|
isValid: true;
|
|
46
|
+
method?: DiscoverMethod;
|
|
45
47
|
hint?: DiscoverUriHint;
|
|
46
48
|
} & TValid) | {
|
|
47
49
|
url: string;
|
|
48
50
|
isValid: false;
|
|
51
|
+
method?: DiscoverMethod;
|
|
49
52
|
hint?: DiscoverUriHint;
|
|
50
53
|
error?: unknown;
|
|
51
54
|
};
|
|
@@ -53,6 +56,7 @@ type DiscoverExtractFn<TValid> = (input: {
|
|
|
53
56
|
url: string;
|
|
54
57
|
content: string;
|
|
55
58
|
headers?: Headers;
|
|
59
|
+
status?: number;
|
|
56
60
|
}) => Promise<DiscoverResult<TValid>>;
|
|
57
61
|
type DiscoverInputObject = {
|
|
58
62
|
url: string;
|
|
@@ -60,14 +64,15 @@ type DiscoverInputObject = {
|
|
|
60
64
|
headers?: Headers;
|
|
61
65
|
};
|
|
62
66
|
type DiscoverInput = string | DiscoverInputObject;
|
|
63
|
-
type DiscoverMethodsConfig = Array<
|
|
67
|
+
type DiscoverMethodsConfig<TMethods extends DiscoverMethod = DiscoverMethod> = Array<TMethods> | Pick<{
|
|
64
68
|
platform?: true | Partial<PlatformMethodOptions>;
|
|
69
|
+
feed?: true | Partial<FeedMethodOptions>;
|
|
65
70
|
html?: true | Partial<Omit<HtmlMethodOptions, 'baseUrl'>>;
|
|
66
71
|
headers?: true | Partial<Omit<HeadersMethodOptions, 'baseUrl'>>;
|
|
67
72
|
guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
|
|
68
|
-
}
|
|
69
|
-
type DiscoverOptions<TValid> = {
|
|
70
|
-
methods?: DiscoverMethodsConfig
|
|
73
|
+
}, TMethods>;
|
|
74
|
+
type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> = {
|
|
75
|
+
methods?: DiscoverMethodsConfig<TMethods>;
|
|
71
76
|
fetchFn?: DiscoverFetchFn;
|
|
72
77
|
extractFn?: DiscoverExtractFn<TValid>;
|
|
73
78
|
normalizeUrlFn?: DiscoverNormalizeUrlFn;
|
|
@@ -78,4 +83,4 @@ type DiscoverOptions<TValid> = {
|
|
|
78
83
|
includeInvalid?: boolean;
|
|
79
84
|
};
|
|
80
85
|
//#endregion
|
|
81
|
-
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };
|
|
86
|
+
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };
|
package/dist/common/types.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { FeedMethodOptions } from "./uris/feed/types.js";
|
|
1
2
|
import { GuessMethodOptions } from "./uris/guess/types.js";
|
|
2
3
|
import { HeadersMethodOptions } from "./uris/headers/types.js";
|
|
3
4
|
import { HtmlMethodOptions } from "./uris/html/types.js";
|
|
@@ -13,7 +14,7 @@ type DiscoverUriEntry = {
|
|
|
13
14
|
uri: UriEntry;
|
|
14
15
|
hint?: DiscoverUriHint;
|
|
15
16
|
};
|
|
16
|
-
declare const discoverMethodOrder: readonly ["platform", "html", "headers", "guess"];
|
|
17
|
+
declare const discoverMethodOrder: readonly ["platform", "feed", "html", "headers", "guess"];
|
|
17
18
|
type DiscoverMethod = (typeof discoverMethodOrder)[number];
|
|
18
19
|
type LinkSelector = {
|
|
19
20
|
rel: string;
|
|
@@ -42,10 +43,12 @@ type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
|
|
|
42
43
|
type DiscoverResult<TValid = object> = ({
|
|
43
44
|
url: string;
|
|
44
45
|
isValid: true;
|
|
46
|
+
method?: DiscoverMethod;
|
|
45
47
|
hint?: DiscoverUriHint;
|
|
46
48
|
} & TValid) | {
|
|
47
49
|
url: string;
|
|
48
50
|
isValid: false;
|
|
51
|
+
method?: DiscoverMethod;
|
|
49
52
|
hint?: DiscoverUriHint;
|
|
50
53
|
error?: unknown;
|
|
51
54
|
};
|
|
@@ -53,6 +56,7 @@ type DiscoverExtractFn<TValid> = (input: {
|
|
|
53
56
|
url: string;
|
|
54
57
|
content: string;
|
|
55
58
|
headers?: Headers;
|
|
59
|
+
status?: number;
|
|
56
60
|
}) => Promise<DiscoverResult<TValid>>;
|
|
57
61
|
type DiscoverInputObject = {
|
|
58
62
|
url: string;
|
|
@@ -60,14 +64,15 @@ type DiscoverInputObject = {
|
|
|
60
64
|
headers?: Headers;
|
|
61
65
|
};
|
|
62
66
|
type DiscoverInput = string | DiscoverInputObject;
|
|
63
|
-
type DiscoverMethodsConfig = Array<
|
|
67
|
+
type DiscoverMethodsConfig<TMethods extends DiscoverMethod = DiscoverMethod> = Array<TMethods> | Pick<{
|
|
64
68
|
platform?: true | Partial<PlatformMethodOptions>;
|
|
69
|
+
feed?: true | Partial<FeedMethodOptions>;
|
|
65
70
|
html?: true | Partial<Omit<HtmlMethodOptions, 'baseUrl'>>;
|
|
66
71
|
headers?: true | Partial<Omit<HeadersMethodOptions, 'baseUrl'>>;
|
|
67
72
|
guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
|
|
68
|
-
}
|
|
69
|
-
type DiscoverOptions<TValid> = {
|
|
70
|
-
methods?: DiscoverMethodsConfig
|
|
73
|
+
}, TMethods>;
|
|
74
|
+
type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> = {
|
|
75
|
+
methods?: DiscoverMethodsConfig<TMethods>;
|
|
71
76
|
fetchFn?: DiscoverFetchFn;
|
|
72
77
|
extractFn?: DiscoverExtractFn<TValid>;
|
|
73
78
|
normalizeUrlFn?: DiscoverNormalizeUrlFn;
|
|
@@ -78,4 +83,4 @@ type DiscoverOptions<TValid> = {
|
|
|
78
83
|
includeInvalid?: boolean;
|
|
79
84
|
};
|
|
80
85
|
//#endregion
|
|
81
|
-
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };
|
|
86
|
+
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };
|
package/dist/common/types.js
CHANGED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
const require_utils = require("../../utils.cjs");
|
|
2
|
+
let feedsmith = require("feedsmith");
|
|
3
|
+
//#region src/common/uris/feed/index.ts
|
|
4
|
+
const discoverUrisFromFeed = (content, options) => {
|
|
5
|
+
try {
|
|
6
|
+
const result = (0, feedsmith.parseFeed)(content);
|
|
7
|
+
return require_utils.omitEmpty(options.extractUrls(result));
|
|
8
|
+
} catch {}
|
|
9
|
+
return [];
|
|
10
|
+
};
|
|
11
|
+
//#endregion
|
|
12
|
+
exports.discoverUrisFromFeed = discoverUrisFromFeed;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { omitEmpty } from "../../utils.js";
|
|
2
|
+
import { parseFeed } from "feedsmith";
|
|
3
|
+
//#region src/common/uris/feed/index.ts
|
|
4
|
+
const discoverUrisFromFeed = (content, options) => {
|
|
5
|
+
try {
|
|
6
|
+
const result = parseFeed(content);
|
|
7
|
+
return omitEmpty(options.extractUrls(result));
|
|
8
|
+
} catch {}
|
|
9
|
+
return [];
|
|
10
|
+
};
|
|
11
|
+
//#endregion
|
|
12
|
+
export { discoverUrisFromFeed };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { parseFeed } from "feedsmith";
|
|
2
|
+
|
|
3
|
+
//#region src/common/uris/feed/types.d.ts
|
|
4
|
+
type FeedMethodData = ReturnType<typeof parseFeed>;
|
|
5
|
+
type FeedMethodOptions = {
|
|
6
|
+
extractUrls: (params: FeedMethodData) => Array<string>;
|
|
7
|
+
};
|
|
8
|
+
//#endregion
|
|
9
|
+
export { FeedMethodOptions };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { parseFeed } from "feedsmith";
|
|
2
|
+
|
|
3
|
+
//#region src/common/uris/feed/types.d.ts
|
|
4
|
+
type FeedMethodData = ReturnType<typeof parseFeed>;
|
|
5
|
+
type FeedMethodOptions = {
|
|
6
|
+
extractUrls: (params: FeedMethodData) => Array<string>;
|
|
7
|
+
};
|
|
8
|
+
//#endregion
|
|
9
|
+
export { FeedMethodOptions };
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
const require_utils = require(
|
|
2
|
-
|
|
1
|
+
const require_utils = require("./utils.cjs");
|
|
3
2
|
//#region src/common/uris/guess/index.ts
|
|
4
3
|
const discoverUrisFromGuess = (options) => {
|
|
5
4
|
const { baseUrl, uris, additionalBaseUrls = [] } = options;
|
|
6
5
|
return require_utils.generateUrlCombinations([baseUrl, ...additionalBaseUrls], uris);
|
|
7
6
|
};
|
|
8
|
-
|
|
9
7
|
//#endregion
|
|
10
|
-
exports.discoverUrisFromGuess = discoverUrisFromGuess;
|
|
8
|
+
exports.discoverUrisFromGuess = discoverUrisFromGuess;
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import { generateUrlCombinations } from "./utils.js";
|
|
2
|
-
|
|
3
2
|
//#region src/common/uris/guess/index.ts
|
|
4
3
|
const discoverUrisFromGuess = (options) => {
|
|
5
4
|
const { baseUrl, uris, additionalBaseUrls = [] } = options;
|
|
6
5
|
return generateUrlCombinations([baseUrl, ...additionalBaseUrls], uris);
|
|
7
6
|
};
|
|
8
|
-
|
|
9
7
|
//#endregion
|
|
10
|
-
export { discoverUrisFromGuess };
|
|
8
|
+
export { discoverUrisFromGuess };
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
|
|
2
1
|
//#region src/common/uris/guess/utils.ts
|
|
3
2
|
const generateUrlCombinations = (baseUrls, uris) => {
|
|
4
3
|
return baseUrls.flatMap((base) => {
|
|
@@ -34,8 +33,7 @@ const getSubdomainVariants = (baseUrl, prefixes) => {
|
|
|
34
33
|
return `${protocol}//${prefix === "" ? rootDomain : `${prefix}.${rootDomain}`}${port}`;
|
|
35
34
|
});
|
|
36
35
|
};
|
|
37
|
-
|
|
38
36
|
//#endregion
|
|
39
37
|
exports.generateUrlCombinations = generateUrlCombinations;
|
|
40
38
|
exports.getSubdomainVariants = getSubdomainVariants;
|
|
41
|
-
exports.getWwwCounterpart = getWwwCounterpart;
|
|
39
|
+
exports.getWwwCounterpart = getWwwCounterpart;
|
|
@@ -33,6 +33,5 @@ const getSubdomainVariants = (baseUrl, prefixes) => {
|
|
|
33
33
|
return `${protocol}//${prefix === "" ? rootDomain : `${prefix}.${rootDomain}`}${port}`;
|
|
34
34
|
});
|
|
35
35
|
};
|
|
36
|
-
|
|
37
36
|
//#endregion
|
|
38
|
-
export { generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };
|
|
37
|
+
export { generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
const require_utils = require(
|
|
2
|
-
|
|
1
|
+
const require_utils = require("../../utils.cjs");
|
|
3
2
|
//#region src/common/uris/headers/index.ts
|
|
4
3
|
const urlRegex = /<([^<>]+)>/;
|
|
5
4
|
const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
|
|
@@ -22,6 +21,5 @@ const discoverUrisFromHeaders = (headers, options) => {
|
|
|
22
21
|
}
|
|
23
22
|
return Array.from(uris);
|
|
24
23
|
};
|
|
25
|
-
|
|
26
24
|
//#endregion
|
|
27
|
-
exports.discoverUrisFromHeaders = discoverUrisFromHeaders;
|
|
25
|
+
exports.discoverUrisFromHeaders = discoverUrisFromHeaders;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { matchesAnyOfLinkSelectors } from "../../utils.js";
|
|
2
|
-
|
|
3
2
|
//#region src/common/uris/headers/index.ts
|
|
4
3
|
const urlRegex = /<([^<>]+)>/;
|
|
5
4
|
const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
|
|
@@ -22,6 +21,5 @@ const discoverUrisFromHeaders = (headers, options) => {
|
|
|
22
21
|
}
|
|
23
22
|
return Array.from(uris);
|
|
24
23
|
};
|
|
25
|
-
|
|
26
24
|
//#endregion
|
|
27
|
-
export { discoverUrisFromHeaders };
|
|
25
|
+
export { discoverUrisFromHeaders };
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
const require_utils = require(
|
|
2
|
-
|
|
1
|
+
const require_utils = require("../../utils.cjs");
|
|
3
2
|
//#region src/common/uris/html/handlers.ts
|
|
4
3
|
const handleOpenTag = (context, name, attribs, _isImplied) => {
|
|
5
4
|
if (name === "link" && attribs.href) {
|
|
@@ -42,6 +41,5 @@ const createHtmlUrisHandlers = (context) => {
|
|
|
42
41
|
}
|
|
43
42
|
};
|
|
44
43
|
};
|
|
45
|
-
|
|
46
44
|
//#endregion
|
|
47
|
-
exports.createHtmlUrisHandlers = createHtmlUrisHandlers;
|
|
45
|
+
exports.createHtmlUrisHandlers = createHtmlUrisHandlers;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { endsWithAnyOf, includesAnyOf, matchesAnyOfLinkSelectors } from "../../utils.js";
|
|
2
|
-
|
|
3
2
|
//#region src/common/uris/html/handlers.ts
|
|
4
3
|
const handleOpenTag = (context, name, attribs, _isImplied) => {
|
|
5
4
|
if (name === "link" && attribs.href) {
|
|
@@ -42,6 +41,5 @@ const createHtmlUrisHandlers = (context) => {
|
|
|
42
41
|
}
|
|
43
42
|
};
|
|
44
43
|
};
|
|
45
|
-
|
|
46
44
|
//#endregion
|
|
47
|
-
export { createHtmlUrisHandlers };
|
|
45
|
+
export { createHtmlUrisHandlers };
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
const require_handlers = require(
|
|
1
|
+
const require_handlers = require("./handlers.cjs");
|
|
2
2
|
let htmlparser2 = require("htmlparser2");
|
|
3
|
-
|
|
4
3
|
//#region src/common/uris/html/index.ts
|
|
5
4
|
const discoverUrisFromHtml = (html, options) => {
|
|
6
5
|
const context = {
|
|
@@ -16,6 +15,5 @@ const discoverUrisFromHtml = (html, options) => {
|
|
|
16
15
|
parser.end();
|
|
17
16
|
return Array.from(context.discoveredUris);
|
|
18
17
|
};
|
|
19
|
-
|
|
20
18
|
//#endregion
|
|
21
|
-
exports.discoverUrisFromHtml = discoverUrisFromHtml;
|
|
19
|
+
exports.discoverUrisFromHtml = discoverUrisFromHtml;
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { createHtmlUrisHandlers } from "./handlers.js";
|
|
2
2
|
import { Parser } from "htmlparser2";
|
|
3
|
-
|
|
4
3
|
//#region src/common/uris/html/index.ts
|
|
5
4
|
const discoverUrisFromHtml = (html, options) => {
|
|
6
5
|
const context = {
|
|
@@ -16,6 +15,5 @@ const discoverUrisFromHtml = (html, options) => {
|
|
|
16
15
|
parser.end();
|
|
17
16
|
return Array.from(context.discoveredUris);
|
|
18
17
|
};
|
|
19
|
-
|
|
20
18
|
//#endregion
|
|
21
|
-
export { discoverUrisFromHtml };
|
|
19
|
+
export { discoverUrisFromHtml };
|
|
@@ -1,29 +1,32 @@
|
|
|
1
|
-
const require_index = require(
|
|
2
|
-
const require_index$1 = require(
|
|
3
|
-
const require_index$2 = require(
|
|
4
|
-
const require_index$3 = require(
|
|
5
|
-
|
|
1
|
+
const require_index = require("./feed/index.cjs");
|
|
2
|
+
const require_index$1 = require("./guess/index.cjs");
|
|
3
|
+
const require_index$2 = require("./headers/index.cjs");
|
|
4
|
+
const require_index$3 = require("./html/index.cjs");
|
|
5
|
+
const require_index$4 = require("./platform/index.cjs");
|
|
6
6
|
//#region src/common/uris/index.ts
|
|
7
|
-
const discoverUris = (config) => {
|
|
7
|
+
const discoverUris = async (config, fetchFn) => {
|
|
8
8
|
const result = {};
|
|
9
9
|
if (config.platform) {
|
|
10
|
-
const uris = require_index$
|
|
10
|
+
const uris = await require_index$4.discoverUrisFromPlatform(config.platform.content, config.platform.headers, config.platform.options, fetchFn);
|
|
11
11
|
if (uris.length > 0) result.platform = uris;
|
|
12
12
|
}
|
|
13
|
+
if (config.feed) {
|
|
14
|
+
const uris = require_index.discoverUrisFromFeed(config.feed.content, config.feed.options);
|
|
15
|
+
if (uris.length > 0) result.feed = uris.map((uri) => ({ uri }));
|
|
16
|
+
}
|
|
13
17
|
if (config.html) {
|
|
14
|
-
const uris = require_index$
|
|
18
|
+
const uris = require_index$3.discoverUrisFromHtml(config.html.html, config.html.options);
|
|
15
19
|
if (uris.length > 0) result.html = uris.map((uri) => ({ uri }));
|
|
16
20
|
}
|
|
17
21
|
if (config.headers) {
|
|
18
|
-
const uris = require_index$
|
|
22
|
+
const uris = require_index$2.discoverUrisFromHeaders(config.headers.headers, config.headers.options);
|
|
19
23
|
if (uris.length > 0) result.headers = uris.map((uri) => ({ uri }));
|
|
20
24
|
}
|
|
21
25
|
if (config.guess) {
|
|
22
|
-
const uris = require_index.discoverUrisFromGuess(config.guess.options);
|
|
26
|
+
const uris = require_index$1.discoverUrisFromGuess(config.guess.options);
|
|
23
27
|
if (uris.length > 0) result.guess = uris.map((uri) => ({ uri }));
|
|
24
28
|
}
|
|
25
29
|
return result;
|
|
26
30
|
};
|
|
27
|
-
|
|
28
31
|
//#endregion
|
|
29
|
-
exports.discoverUris = discoverUris;
|
|
32
|
+
exports.discoverUris = discoverUris;
|
|
@@ -1,15 +1,19 @@
|
|
|
1
|
+
import { discoverUrisFromFeed } from "./feed/index.js";
|
|
1
2
|
import { discoverUrisFromGuess } from "./guess/index.js";
|
|
2
3
|
import { discoverUrisFromHeaders } from "./headers/index.js";
|
|
3
4
|
import { discoverUrisFromHtml } from "./html/index.js";
|
|
4
5
|
import { discoverUrisFromPlatform } from "./platform/index.js";
|
|
5
|
-
|
|
6
6
|
//#region src/common/uris/index.ts
|
|
7
|
-
const discoverUris = (config) => {
|
|
7
|
+
const discoverUris = async (config, fetchFn) => {
|
|
8
8
|
const result = {};
|
|
9
9
|
if (config.platform) {
|
|
10
|
-
const uris = discoverUrisFromPlatform(config.platform.
|
|
10
|
+
const uris = await discoverUrisFromPlatform(config.platform.content, config.platform.headers, config.platform.options, fetchFn);
|
|
11
11
|
if (uris.length > 0) result.platform = uris;
|
|
12
12
|
}
|
|
13
|
+
if (config.feed) {
|
|
14
|
+
const uris = discoverUrisFromFeed(config.feed.content, config.feed.options);
|
|
15
|
+
if (uris.length > 0) result.feed = uris.map((uri) => ({ uri }));
|
|
16
|
+
}
|
|
13
17
|
if (config.html) {
|
|
14
18
|
const uris = discoverUrisFromHtml(config.html.html, config.html.options);
|
|
15
19
|
if (uris.length > 0) result.html = uris.map((uri) => ({ uri }));
|
|
@@ -24,6 +28,5 @@ const discoverUris = (config) => {
|
|
|
24
28
|
}
|
|
25
29
|
return result;
|
|
26
30
|
};
|
|
27
|
-
|
|
28
31
|
//#endregion
|
|
29
|
-
export { discoverUris };
|
|
32
|
+
export { discoverUris };
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
|
|
2
1
|
//#region src/common/uris/platform/index.ts
|
|
3
|
-
const discoverUrisFromPlatform = (
|
|
2
|
+
const discoverUrisFromPlatform = async (content, headers, options, fetchFn) => {
|
|
4
3
|
const { baseUrl, handlers } = options;
|
|
5
4
|
for (const handler of handlers) try {
|
|
6
|
-
if (handler.match(baseUrl)) return handler.resolve(baseUrl,
|
|
5
|
+
if (handler.match(baseUrl, content, headers)) return await handler.resolve(baseUrl, content, headers, fetchFn);
|
|
7
6
|
} catch {}
|
|
8
7
|
return [];
|
|
9
8
|
};
|
|
10
|
-
|
|
11
9
|
//#endregion
|
|
12
|
-
exports.discoverUrisFromPlatform = discoverUrisFromPlatform;
|
|
10
|
+
exports.discoverUrisFromPlatform = discoverUrisFromPlatform;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { PlatformMethodOptions } from "./types.cjs";
|
|
2
|
+
import { DiscoverFetchFn, DiscoverUriEntry } from "../../types.cjs";
|
|
3
|
+
|
|
4
|
+
//#region src/common/uris/platform/index.d.ts
|
|
5
|
+
declare const discoverUrisFromPlatform: (content: string | undefined, headers: Headers | undefined, options: PlatformMethodOptions, fetchFn?: DiscoverFetchFn) => Promise<Array<DiscoverUriEntry>>;
|
|
6
|
+
//#endregion
|
|
7
|
+
export { discoverUrisFromPlatform };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { PlatformMethodOptions } from "./types.js";
|
|
2
|
+
import { DiscoverFetchFn, DiscoverUriEntry } from "../../types.js";
|
|
3
|
+
|
|
4
|
+
//#region src/common/uris/platform/index.d.ts
|
|
5
|
+
declare const discoverUrisFromPlatform: (content: string | undefined, headers: Headers | undefined, options: PlatformMethodOptions, fetchFn?: DiscoverFetchFn) => Promise<Array<DiscoverUriEntry>>;
|
|
6
|
+
//#endregion
|
|
7
|
+
export { discoverUrisFromPlatform };
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
//#region src/common/uris/platform/index.ts
|
|
2
|
-
const discoverUrisFromPlatform = (
|
|
2
|
+
const discoverUrisFromPlatform = async (content, headers, options, fetchFn) => {
|
|
3
3
|
const { baseUrl, handlers } = options;
|
|
4
4
|
for (const handler of handlers) try {
|
|
5
|
-
if (handler.match(baseUrl)) return handler.resolve(baseUrl,
|
|
5
|
+
if (handler.match(baseUrl, content, headers)) return await handler.resolve(baseUrl, content, headers, fetchFn);
|
|
6
6
|
} catch {}
|
|
7
7
|
return [];
|
|
8
8
|
};
|
|
9
|
-
|
|
10
9
|
//#endregion
|
|
11
|
-
export { discoverUrisFromPlatform };
|
|
10
|
+
export { discoverUrisFromPlatform };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { DiscoverUriEntry } from "../../types.cjs";
|
|
1
|
+
import { DiscoverFetchFn, DiscoverUriEntry } from "../../types.cjs";
|
|
2
2
|
|
|
3
3
|
//#region src/common/uris/platform/types.d.ts
|
|
4
4
|
type PlatformHandler = {
|
|
5
|
-
match: (url: string) => boolean;
|
|
6
|
-
resolve: (url: string, content?: string) => Array<DiscoverUriEntry
|
|
5
|
+
match: (url: string, content?: string, headers?: Headers) => boolean;
|
|
6
|
+
resolve: (url: string, content?: string, headers?: Headers, fetchFn?: DiscoverFetchFn) => Array<DiscoverUriEntry> | Promise<Array<DiscoverUriEntry>>;
|
|
7
7
|
};
|
|
8
8
|
type PlatformMethodOptions = {
|
|
9
9
|
baseUrl: string;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { DiscoverUriEntry } from "../../types.js";
|
|
1
|
+
import { DiscoverFetchFn, DiscoverUriEntry } from "../../types.js";
|
|
2
2
|
|
|
3
3
|
//#region src/common/uris/platform/types.d.ts
|
|
4
4
|
type PlatformHandler = {
|
|
5
|
-
match: (url: string) => boolean;
|
|
6
|
-
resolve: (url: string, content?: string) => Array<DiscoverUriEntry
|
|
5
|
+
match: (url: string, content?: string, headers?: Headers) => boolean;
|
|
6
|
+
resolve: (url: string, content?: string, headers?: Headers, fetchFn?: DiscoverFetchFn) => Array<DiscoverUriEntry> | Promise<Array<DiscoverUriEntry>>;
|
|
7
7
|
};
|
|
8
8
|
type PlatformMethodOptions = {
|
|
9
9
|
baseUrl: string;
|