feedscout 1.8.0 → 2.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/blogrolls/extractors.d.ts +2 -2
- package/dist/blogrolls/extractors.js +2 -2
- package/dist/blogrolls/index.js +4 -5
- package/dist/blogrolls.d.ts +2 -2
- package/dist/blogrolls.js +2 -2
- package/dist/common/discover/defaults.d.ts +6 -0
- package/dist/common/discover/defaults.js +44 -0
- package/dist/common/discover/index.js +2 -2
- package/dist/common/discover/utils.d.ts +3 -6
- package/dist/common/discover/utils.js +1 -28
- package/dist/common/locales.js +214 -10
- package/dist/common/types.d.ts +3 -2
- package/dist/common/uris/feed/types.d.ts +2 -2
- package/dist/common/uris/html/types.d.ts +4 -4
- package/dist/common/utils.d.ts +6 -4
- package/dist/common/utils.js +13 -11
- package/dist/favicons/extractors.d.ts +2 -2
- package/dist/favicons/extractors.js +2 -2
- package/dist/favicons/index.js +4 -5
- package/dist/favicons.d.ts +2 -2
- package/dist/favicons.js +2 -2
- package/dist/feeds/defaults.d.ts +2 -1
- package/dist/feeds/defaults.js +116 -9
- package/dist/feeds/extractors.d.ts +2 -2
- package/dist/feeds/extractors.js +6 -6
- package/dist/feeds/index.js +4 -5
- package/dist/feeds/platform/handlers/acast.js +26 -0
- package/dist/feeds/platform/handlers/ameblo.js +36 -0
- package/dist/feeds/platform/handlers/applePodcasts.js +1 -1
- package/dist/feeds/platform/handlers/arena.js +42 -0
- package/dist/feeds/platform/handlers/artstation.js +52 -0
- package/dist/feeds/platform/handlers/audioboom.js +23 -0
- package/dist/feeds/platform/handlers/bearblog.js +45 -0
- package/dist/feeds/platform/handlers/behance.js +7 -0
- package/dist/feeds/platform/handlers/blogspot.js +38 -2
- package/dist/feeds/platform/handlers/bookwyrm.js +48 -0
- package/dist/feeds/platform/handlers/buttondown.js +43 -0
- package/dist/feeds/platform/handlers/buzzsprout.js +22 -0
- package/dist/feeds/platform/handlers/codeberg.js +5 -5
- package/dist/feeds/platform/handlers/dailymotion.js +16 -0
- package/dist/feeds/platform/handlers/deviantart.js +25 -6
- package/dist/feeds/platform/handlers/devto.js +8 -0
- package/dist/feeds/platform/handlers/discourse.js +70 -0
- package/dist/feeds/platform/handlers/dreamwidth.js +48 -0
- package/dist/feeds/platform/handlers/exblog.js +35 -0
- package/dist/feeds/platform/handlers/fireside.js +24 -0
- package/dist/feeds/platform/handlers/friendica.js +44 -0
- package/dist/feeds/platform/handlers/ghost.js +30 -0
- package/dist/feeds/platform/handlers/github.js +6 -0
- package/dist/feeds/platform/handlers/githubGist.js +16 -5
- package/dist/feeds/platform/handlers/gitlab.js +31 -13
- package/dist/feeds/platform/handlers/goodreads.js +18 -8
- package/dist/feeds/platform/handlers/hackernews.js +21 -0
- package/dist/feeds/platform/handlers/hashnode.js +1 -1
- package/dist/feeds/platform/handlers/hatenablog.js +4 -1
- package/dist/feeds/platform/handlers/hearthis.js +32 -0
- package/dist/feeds/platform/handlers/heyWorld.js +18 -0
- package/dist/feeds/platform/handlers/insanejournal.js +69 -0
- package/dist/feeds/platform/handlers/itchio.js +24 -1
- package/dist/feeds/platform/handlers/lemmy.js +46 -4
- package/dist/feeds/platform/handlers/letterboxd.js +4 -0
- package/dist/feeds/platform/handlers/libsyn.js +25 -0
- package/dist/feeds/platform/handlers/listed.js +20 -0
- package/dist/feeds/platform/handlers/livejournal.js +68 -0
- package/dist/feeds/platform/handlers/mastodon.js +32 -0
- package/dist/feeds/platform/handlers/mataroa.js +16 -0
- package/dist/feeds/platform/handlers/medium.js +2 -2
- package/dist/feeds/platform/handlers/microblog.js +55 -0
- package/dist/feeds/platform/handlers/misskey.js +40 -0
- package/dist/feeds/platform/handlers/myanimelist.js +43 -0
- package/dist/feeds/platform/handlers/naverBlog.js +21 -0
- package/dist/feeds/platform/handlers/nebula.js +68 -0
- package/dist/feeds/platform/handlers/note.js +53 -0
- package/dist/feeds/platform/handlers/observable.js +34 -0
- package/dist/feeds/platform/handlers/odysee.js +20 -0
- package/dist/feeds/platform/handlers/pagecord.js +16 -0
- package/dist/feeds/platform/handlers/paragraph.js +1 -2
- package/dist/feeds/platform/handlers/pika.js +35 -0
- package/dist/feeds/platform/handlers/pinterest.js +13 -0
- package/dist/feeds/platform/handlers/pixelfed.js +46 -0
- package/dist/feeds/platform/handlers/pleroma.js +34 -0
- package/dist/feeds/platform/handlers/podbean.js +29 -0
- package/dist/feeds/platform/handlers/podigee.js +29 -0
- package/dist/feeds/platform/handlers/posthaven.js +24 -0
- package/dist/feeds/platform/handlers/prose.js +26 -0
- package/dist/feeds/platform/handlers/qiita.js +58 -0
- package/dist/feeds/platform/handlers/reddit.js +83 -9
- package/dist/feeds/platform/handlers/rssCom.js +20 -0
- package/dist/feeds/platform/handlers/seesaa.js +22 -0
- package/dist/feeds/platform/handlers/sourceforge.js +37 -4
- package/dist/feeds/platform/handlers/spreaker.js +21 -0
- package/dist/feeds/platform/handlers/stackExchange.js +26 -2
- package/dist/feeds/platform/handlers/steam.js +7 -0
- package/dist/feeds/platform/handlers/tildes.js +41 -0
- package/dist/feeds/platform/handlers/tistory.js +16 -0
- package/dist/feeds/platform/handlers/transistor.js +29 -0
- package/dist/feeds/platform/handlers/velog.js +24 -0
- package/dist/feeds/platform/handlers/vimeo.js +6 -0
- package/dist/feeds/platform/handlers/weblogLol.js +26 -0
- package/dist/feeds/platform/handlers/weebly.js +25 -0
- package/dist/feeds/platform/handlers/wordpress.js +173 -28
- package/dist/feeds/platform/handlers/writeas.js +51 -0
- package/dist/feeds/platform/handlers/ximalaya.js +1 -1
- package/dist/feeds/platform/handlers/youtube.js +4 -1
- package/dist/feeds/platform/handlers/zenn.js +54 -0
- package/dist/feeds.d.ts +3 -3
- package/dist/feeds.js +3 -3
- package/dist/hubs/discover/index.js +3 -3
- package/dist/hubs/feed/index.js +2 -2
- package/dist/hubs/headers/index.js +2 -2
- package/dist/hubs/html/index.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/utils.d.ts +2 -1
- package/package.json +24 -64
- package/dist/blogrolls/defaults.cjs +0 -51
- package/dist/blogrolls/defaults.d.cts +0 -17
- package/dist/blogrolls/extractors.cjs +0 -21
- package/dist/blogrolls/extractors.d.cts +0 -7
- package/dist/blogrolls/index.cjs +0 -26
- package/dist/blogrolls/index.d.cts +0 -7
- package/dist/blogrolls/types.d.cts +0 -6
- package/dist/blogrolls.cjs +0 -13
- package/dist/blogrolls.d.cts +0 -4
- package/dist/common/discover/index.cjs +0 -105
- package/dist/common/discover/utils.cjs +0 -130
- package/dist/common/discover/utils.d.cts +0 -8
- package/dist/common/locales.cjs +0 -144
- package/dist/common/types.cjs +0 -10
- package/dist/common/types.d.cts +0 -89
- package/dist/common/uris/feed/index.cjs +0 -12
- package/dist/common/uris/feed/index.d.cts +0 -6
- package/dist/common/uris/feed/types.d.cts +0 -9
- package/dist/common/uris/guess/index.cjs +0 -8
- package/dist/common/uris/guess/index.d.cts +0 -7
- package/dist/common/uris/guess/types.d.cts +0 -10
- package/dist/common/uris/guess/utils.cjs +0 -42
- package/dist/common/uris/guess/utils.d.cts +0 -8
- package/dist/common/uris/headers/index.cjs +0 -26
- package/dist/common/uris/headers/index.d.cts +0 -6
- package/dist/common/uris/headers/types.d.cts +0 -9
- package/dist/common/uris/html/handlers.cjs +0 -45
- package/dist/common/uris/html/index.cjs +0 -19
- package/dist/common/uris/html/index.d.cts +0 -6
- package/dist/common/uris/html/types.d.cts +0 -12
- package/dist/common/uris/index.cjs +0 -32
- package/dist/common/uris/platform/index.cjs +0 -10
- package/dist/common/uris/platform/index.d.cts +0 -7
- package/dist/common/uris/platform/types.d.cts +0 -13
- package/dist/common/utils.cjs +0 -97
- package/dist/common/utils.d.cts +0 -10
- package/dist/favicons/defaults.cjs +0 -65
- package/dist/favicons/defaults.d.cts +0 -18
- package/dist/favicons/extractors.cjs +0 -25
- package/dist/favicons/extractors.d.cts +0 -7
- package/dist/favicons/index.cjs +0 -30
- package/dist/favicons/index.d.cts +0 -7
- package/dist/favicons/platform/handlers/bluesky.cjs +0 -27
- package/dist/favicons/platform/handlers/codeberg.cjs +0 -19
- package/dist/favicons/platform/handlers/deviantart.cjs +0 -29
- package/dist/favicons/platform/handlers/devto.cjs +0 -31
- package/dist/favicons/platform/handlers/github.cjs +0 -19
- package/dist/favicons/platform/handlers/githubGist.cjs +0 -19
- package/dist/favicons/platform/handlers/gitlab.cjs +0 -38
- package/dist/favicons/platform/handlers/lobsters.cjs +0 -21
- package/dist/favicons/platform/handlers/mastodon.cjs +0 -40
- package/dist/favicons/platform/handlers/reddit.cjs +0 -42
- package/dist/favicons/platform/handlers/sourceforge.cjs +0 -21
- package/dist/favicons/platform/handlers/tumblr.cjs +0 -16
- package/dist/favicons/types.d.cts +0 -4
- package/dist/favicons/utils.cjs +0 -10
- package/dist/favicons.cjs +0 -12
- package/dist/favicons.d.cts +0 -4
- package/dist/feeds/defaults.cjs +0 -178
- package/dist/feeds/defaults.d.cts +0 -20
- package/dist/feeds/extractors.cjs +0 -46
- package/dist/feeds/extractors.d.cts +0 -7
- package/dist/feeds/index.cjs +0 -27
- package/dist/feeds/index.d.cts +0 -7
- package/dist/feeds/platform/handlers/applePodcasts.cjs +0 -26
- package/dist/feeds/platform/handlers/behance.cjs +0 -49
- package/dist/feeds/platform/handlers/blogspot.cjs +0 -36
- package/dist/feeds/platform/handlers/bluesky.cjs +0 -20
- package/dist/feeds/platform/handlers/codeberg.cjs +0 -69
- package/dist/feeds/platform/handlers/csdn.cjs +0 -20
- package/dist/feeds/platform/handlers/dailymotion.cjs +0 -70
- package/dist/feeds/platform/handlers/deviantart.cjs +0 -66
- package/dist/feeds/platform/handlers/devto.cjs +0 -50
- package/dist/feeds/platform/handlers/douban.cjs +0 -56
- package/dist/feeds/platform/handlers/github.cjs +0 -116
- package/dist/feeds/platform/handlers/githubGist.cjs +0 -45
- package/dist/feeds/platform/handlers/gitlab.cjs +0 -80
- package/dist/feeds/platform/handlers/goodreads.cjs +0 -39
- package/dist/feeds/platform/handlers/hashnode.cjs +0 -16
- package/dist/feeds/platform/handlers/hatenablog.cjs +0 -53
- package/dist/feeds/platform/handlers/itchio.cjs +0 -88
- package/dist/feeds/platform/handlers/kickstarter.cjs +0 -22
- package/dist/feeds/platform/handlers/lemmy.cjs +0 -46
- package/dist/feeds/platform/handlers/letterboxd.cjs +0 -42
- package/dist/feeds/platform/handlers/lobsters.cjs +0 -57
- package/dist/feeds/platform/handlers/mastodon.cjs +0 -42
- package/dist/feeds/platform/handlers/medium.cjs +0 -68
- package/dist/feeds/platform/handlers/paragraph.cjs +0 -21
- package/dist/feeds/platform/handlers/pinterest.cjs +0 -44
- package/dist/feeds/platform/handlers/producthunt.cjs +0 -29
- package/dist/feeds/platform/handlers/reddit.cjs +0 -75
- package/dist/feeds/platform/handlers/soundcloud.cjs +0 -39
- package/dist/feeds/platform/handlers/sourceforge.cjs +0 -20
- package/dist/feeds/platform/handlers/stackExchange.cjs +0 -40
- package/dist/feeds/platform/handlers/steam.cjs +0 -28
- package/dist/feeds/platform/handlers/substack.cjs +0 -23
- package/dist/feeds/platform/handlers/tumblr.cjs +0 -24
- package/dist/feeds/platform/handlers/v2ex.cjs +0 -35
- package/dist/feeds/platform/handlers/vimeo.cjs +0 -69
- package/dist/feeds/platform/handlers/wordpress.cjs +0 -64
- package/dist/feeds/platform/handlers/wpengine.cjs +0 -10
- package/dist/feeds/platform/handlers/ximalaya.cjs +0 -20
- package/dist/feeds/platform/handlers/youtube.cjs +0 -94
- package/dist/feeds/types.d.cts +0 -9
- package/dist/feeds.cjs +0 -15
- package/dist/feeds.d.cts +0 -4
- package/dist/hubs/discover/index.cjs +0 -30
- package/dist/hubs/discover/index.d.cts +0 -7
- package/dist/hubs/discover/types.d.cts +0 -15
- package/dist/hubs/feed/index.cjs +0 -32
- package/dist/hubs/headers/index.cjs +0 -17
- package/dist/hubs/html/index.cjs +0 -28
- package/dist/hubs.cjs +0 -0
- package/dist/hubs.d.cts +0 -2
- package/dist/index.cjs +0 -12
- package/dist/index.d.cts +0 -7
- package/dist/methods.cjs +0 -15
- package/dist/methods.d.cts +0 -7
- package/dist/utils.cjs +0 -9
- package/dist/utils.d.cts +0 -2
|
@@ -1,130 +0,0 @@
|
|
|
1
|
-
const require_locales = require("../locales.cjs");
|
|
2
|
-
const require_utils = require("../utils.cjs");
|
|
3
|
-
let feedsmith = require("feedsmith");
|
|
4
|
-
//#region src/common/discover/utils.ts
|
|
5
|
-
const defaultFetchFn = async (url, options) => {
|
|
6
|
-
const response = await fetch(url, {
|
|
7
|
-
method: options?.method ?? "GET",
|
|
8
|
-
headers: options?.headers
|
|
9
|
-
});
|
|
10
|
-
return {
|
|
11
|
-
headers: response.headers,
|
|
12
|
-
body: await response.text(),
|
|
13
|
-
url: response.url,
|
|
14
|
-
status: response.status,
|
|
15
|
-
statusText: response.statusText
|
|
16
|
-
};
|
|
17
|
-
};
|
|
18
|
-
const normalizeInput = async (input, fetchFn) => {
|
|
19
|
-
if (typeof input === "object") return input;
|
|
20
|
-
try {
|
|
21
|
-
const response = await fetchFn(input);
|
|
22
|
-
return {
|
|
23
|
-
url: response.url,
|
|
24
|
-
content: typeof response.body === "string" ? response.body : void 0,
|
|
25
|
-
headers: response.headers
|
|
26
|
-
};
|
|
27
|
-
} catch {}
|
|
28
|
-
return { url: input };
|
|
29
|
-
};
|
|
30
|
-
const getLinkOfType = (links, rel) => {
|
|
31
|
-
return links?.find((link) => link.rel === rel);
|
|
32
|
-
};
|
|
33
|
-
const getFeedSiteUrl = (parsed) => {
|
|
34
|
-
const { format, feed } = parsed;
|
|
35
|
-
if (format === "rss" || format === "rdf") return getLinkOfType(feed.atom?.links, "alternate")?.href ?? feed.link;
|
|
36
|
-
if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
|
|
37
|
-
if (format === "json") return feed.home_page_url;
|
|
38
|
-
};
|
|
39
|
-
const defaultResolveSiteUrlFn = (input) => {
|
|
40
|
-
if (!input.content) return;
|
|
41
|
-
try {
|
|
42
|
-
let siteUrl = getFeedSiteUrl((0, feedsmith.parseFeed)(input.content));
|
|
43
|
-
if (!siteUrl) try {
|
|
44
|
-
siteUrl = new URL(input.url).origin;
|
|
45
|
-
} catch {}
|
|
46
|
-
else siteUrl = require_utils.resolveUrl(siteUrl, input.url);
|
|
47
|
-
if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
|
|
48
|
-
return siteUrl;
|
|
49
|
-
} catch {}
|
|
50
|
-
};
|
|
51
|
-
const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
|
|
52
|
-
if (typeof entry.uri === "string") return {
|
|
53
|
-
...entry,
|
|
54
|
-
uri: resolveUrlFn(entry.uri, baseUrl) ?? entry.uri
|
|
55
|
-
};
|
|
56
|
-
return {
|
|
57
|
-
...entry,
|
|
58
|
-
uri: entry.uri.map((uri) => resolveUrlFn(uri, baseUrl) ?? uri)
|
|
59
|
-
};
|
|
60
|
-
};
|
|
61
|
-
const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
|
|
62
|
-
const resolvedInput = siteInput ?? sourceInput;
|
|
63
|
-
const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
|
|
64
|
-
const methodsConfig = {};
|
|
65
|
-
if (methodsObj.platform && defaults.platform) {
|
|
66
|
-
if (!resolvedInput.url || resolvedInput.url === "") throw new Error(require_locales.errors.platformMethodRequiresUrl);
|
|
67
|
-
const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
|
|
68
|
-
methodsConfig.platform = {
|
|
69
|
-
content: resolvedInput.content,
|
|
70
|
-
headers: resolvedInput.headers,
|
|
71
|
-
options: {
|
|
72
|
-
...defaults.platform,
|
|
73
|
-
...platformOptions,
|
|
74
|
-
baseUrl: resolvedInput.url
|
|
75
|
-
}
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
if (methodsObj.feed && defaults.feed) {
|
|
79
|
-
if (sourceInput.content === void 0) throw new Error(require_locales.errors.feedMethodRequiresContent);
|
|
80
|
-
const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
|
|
81
|
-
methodsConfig.feed = {
|
|
82
|
-
content: sourceInput.content,
|
|
83
|
-
options: {
|
|
84
|
-
...defaults.feed,
|
|
85
|
-
...feedOptions
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
if (methodsObj.html && defaults.html) {
|
|
90
|
-
if (resolvedInput.content === void 0) throw new Error(require_locales.errors.htmlMethodRequiresContent);
|
|
91
|
-
const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
|
|
92
|
-
methodsConfig.html = {
|
|
93
|
-
html: resolvedInput.content,
|
|
94
|
-
options: {
|
|
95
|
-
...defaults.html,
|
|
96
|
-
...htmlOptions,
|
|
97
|
-
baseUrl: resolvedInput.url
|
|
98
|
-
}
|
|
99
|
-
};
|
|
100
|
-
}
|
|
101
|
-
if (methodsObj.headers && defaults.headers) {
|
|
102
|
-
if (resolvedInput.headers === void 0) throw new Error(require_locales.errors.headersMethodRequiresHeaders);
|
|
103
|
-
const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
|
|
104
|
-
methodsConfig.headers = {
|
|
105
|
-
headers: resolvedInput.headers,
|
|
106
|
-
options: {
|
|
107
|
-
...defaults.headers,
|
|
108
|
-
...headersOptions,
|
|
109
|
-
baseUrl: resolvedInput.url
|
|
110
|
-
}
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
if (methodsObj.guess && defaults.guess) {
|
|
114
|
-
if (!resolvedInput.url || resolvedInput.url === "") throw new Error(require_locales.errors.guessMethodRequiresUrl);
|
|
115
|
-
const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
|
|
116
|
-
methodsConfig.guess = { options: {
|
|
117
|
-
...defaults.guess,
|
|
118
|
-
...guessOptions,
|
|
119
|
-
baseUrl: resolvedInput.url
|
|
120
|
-
} };
|
|
121
|
-
}
|
|
122
|
-
return methodsConfig;
|
|
123
|
-
};
|
|
124
|
-
//#endregion
|
|
125
|
-
exports.defaultFetchFn = defaultFetchFn;
|
|
126
|
-
exports.defaultResolveSiteUrlFn = defaultResolveSiteUrlFn;
|
|
127
|
-
exports.getFeedSiteUrl = getFeedSiteUrl;
|
|
128
|
-
exports.normalizeInput = normalizeInput;
|
|
129
|
-
exports.normalizeMethodsConfig = normalizeMethodsConfig;
|
|
130
|
-
exports.normalizeUriEntry = normalizeUriEntry;
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { DiscoverResolveSiteUrlFn } from "../types.cjs";
|
|
2
|
-
import { parseFeed } from "feedsmith";
|
|
3
|
-
|
|
4
|
-
//#region src/common/discover/utils.d.ts
|
|
5
|
-
declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
|
|
6
|
-
declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
|
|
7
|
-
//#endregion
|
|
8
|
-
export { defaultResolveSiteUrlFn, getFeedSiteUrl };
|
package/dist/common/locales.cjs
DELETED
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
//#region src/common/locales.json
|
|
2
|
-
var errors = {
|
|
3
|
-
"platformMethodRequiresUrl": "Platform method requires url to be provided in input",
|
|
4
|
-
"feedMethodRequiresContent": "Feed method requires content to be provided in input",
|
|
5
|
-
"htmlMethodRequiresContent": "HTML method requires content to be provided in input",
|
|
6
|
-
"headersMethodRequiresHeaders": "Headers method requires headers to be provided in input",
|
|
7
|
-
"guessMethodRequiresUrl": "Guess method requires url to be provided in input"
|
|
8
|
-
};
|
|
9
|
-
var hints = {
|
|
10
|
-
"youtube:all": "All uploads",
|
|
11
|
-
"youtube:videos": "Videos",
|
|
12
|
-
"youtube:shorts": "Shorts",
|
|
13
|
-
"youtube:live": "Live streams",
|
|
14
|
-
"youtube:popular-videos": "Popular videos",
|
|
15
|
-
"youtube:popular-shorts": "Popular shorts",
|
|
16
|
-
"youtube:popular-live": "Popular live streams",
|
|
17
|
-
"youtube:member-videos": "Member videos",
|
|
18
|
-
"youtube:member-shorts": "Member shorts",
|
|
19
|
-
"youtube:member-live": "Member live streams",
|
|
20
|
-
"youtube:playlist": "Playlist",
|
|
21
|
-
"github:activity": "Activity",
|
|
22
|
-
"github:releases": "Releases",
|
|
23
|
-
"github:commits": "Commits",
|
|
24
|
-
"github:tags": "Tags",
|
|
25
|
-
"github:wiki": "Wiki",
|
|
26
|
-
"github:discussions": "Discussions",
|
|
27
|
-
"github:branch-commits": "Branch commits",
|
|
28
|
-
"github:file-history": "File history",
|
|
29
|
-
"reddit:posts": "Posts",
|
|
30
|
-
"reddit:post-comments": "Post comments",
|
|
31
|
-
"reddit:comments": "Comments",
|
|
32
|
-
"reddit:multireddit": "Multireddit",
|
|
33
|
-
"gitlab:activity": "Activity",
|
|
34
|
-
"gitlab:releases": "Releases",
|
|
35
|
-
"gitlab:tags": "Tags",
|
|
36
|
-
"codeberg:activity": "Activity",
|
|
37
|
-
"codeberg:releases": "Releases",
|
|
38
|
-
"codeberg:tags": "Tags",
|
|
39
|
-
"codeberg:branch-commits": "Branch commits",
|
|
40
|
-
"codeberg:file-history": "File history",
|
|
41
|
-
"wordpress:category": "Category",
|
|
42
|
-
"wordpress:tag": "Tag",
|
|
43
|
-
"wordpress:author": "Author",
|
|
44
|
-
"wordpress:posts-rss2": "Posts (RSS 2.0)",
|
|
45
|
-
"wordpress:posts-rss2-alt": "Posts (RSS 2.0)",
|
|
46
|
-
"wordpress:posts-rdf": "Posts (RDF)",
|
|
47
|
-
"wordpress:posts-atom": "Posts (Atom)",
|
|
48
|
-
"wordpress:comments": "Comments",
|
|
49
|
-
"wordpress:comments-rss2": "Comments (RSS 2.0)",
|
|
50
|
-
"wordpress:comments-rdf": "Comments (RDF)",
|
|
51
|
-
"wordpress:comments-atom": "Comments (Atom)",
|
|
52
|
-
"blogspot:label": "Label",
|
|
53
|
-
"blogspot:posts-atom": "Posts (Atom)",
|
|
54
|
-
"blogspot:posts-rss": "Posts (RSS)",
|
|
55
|
-
"behance:portfolio": "Portfolio",
|
|
56
|
-
"behance:appreciated": "Appreciated",
|
|
57
|
-
"bluesky:posts": "Posts",
|
|
58
|
-
"dailymotion:videos": "Videos",
|
|
59
|
-
"dailymotion:playlist": "Playlist",
|
|
60
|
-
"deviantart:tag": "Tag",
|
|
61
|
-
"deviantart:favorites": "Favorites",
|
|
62
|
-
"deviantart:gallery": "Gallery",
|
|
63
|
-
"deviantart:deviations": "Deviations",
|
|
64
|
-
"devto:posts": "Posts",
|
|
65
|
-
"devto:tag": "Tag",
|
|
66
|
-
"github-gist:gists": "Gists",
|
|
67
|
-
"github-gist:starred": "Starred",
|
|
68
|
-
"kickstarter:updates": "Updates",
|
|
69
|
-
"kickstarter:projects": "Projects",
|
|
70
|
-
"letterboxd:diary": "Diary",
|
|
71
|
-
"lobsters:tag": "Tag",
|
|
72
|
-
"lobsters:domain": "Domain",
|
|
73
|
-
"lobsters:stories": "Stories",
|
|
74
|
-
"lobsters:top": "Top stories",
|
|
75
|
-
"lobsters:newest": "Newest",
|
|
76
|
-
"lobsters:comments": "Comments",
|
|
77
|
-
"mastodon:posts": "Posts",
|
|
78
|
-
"mastodon:tag": "Tag",
|
|
79
|
-
"medium:posts": "Posts",
|
|
80
|
-
"medium:tag": "Tag",
|
|
81
|
-
"medium:tagged": "Tagged",
|
|
82
|
-
"medium:publication": "Publication",
|
|
83
|
-
"pinterest:pins": "Pins",
|
|
84
|
-
"producthunt:topic": "Topic",
|
|
85
|
-
"producthunt:category": "Category",
|
|
86
|
-
"producthunt:products": "Products",
|
|
87
|
-
"soundcloud:tracks": "Tracks",
|
|
88
|
-
"stackexchange:tag": "Tag",
|
|
89
|
-
"stackexchange:question": "Question",
|
|
90
|
-
"stackexchange:user": "User",
|
|
91
|
-
"steam:news": "News",
|
|
92
|
-
"steam:group": "Group",
|
|
93
|
-
"substack:newsletter": "Newsletter",
|
|
94
|
-
"tumblr:posts": "Posts",
|
|
95
|
-
"tumblr:tag": "Tag",
|
|
96
|
-
"goodreads:updates": "Updates",
|
|
97
|
-
"goodreads:reviews": "Reviews",
|
|
98
|
-
"sourceforge:activity": "Activity",
|
|
99
|
-
"vimeo:videos": "Videos",
|
|
100
|
-
"vimeo:likes": "Likes",
|
|
101
|
-
"vimeo:channel": "Channel",
|
|
102
|
-
"vimeo:group": "Group",
|
|
103
|
-
"hashnode:blog": "Blog",
|
|
104
|
-
"paragraph:blog": "Blog",
|
|
105
|
-
"hatenablog:posts-rss": "Posts (RSS)",
|
|
106
|
-
"hatenablog:posts-atom": "Posts (Atom)",
|
|
107
|
-
"hatenablog:category-rss": "Category (RSS)",
|
|
108
|
-
"hatenablog:category-atom": "Category (Atom)",
|
|
109
|
-
"hatenablog:author-rss": "Author (RSS)",
|
|
110
|
-
"hatenablog:author-atom": "Author (Atom)",
|
|
111
|
-
"itchio:devlog": "Devlog",
|
|
112
|
-
"itchio:games": "Games",
|
|
113
|
-
"itchio:tag": "Tag",
|
|
114
|
-
"itchio:section": "Section",
|
|
115
|
-
"itchio:featured": "Featured",
|
|
116
|
-
"itchio:new": "New",
|
|
117
|
-
"itchio:sales": "Sales",
|
|
118
|
-
"csdn:blog": "Blog",
|
|
119
|
-
"douban:interests": "Interests",
|
|
120
|
-
"douban:reviews": "Reviews",
|
|
121
|
-
"douban:notes": "Notes",
|
|
122
|
-
"douban:subjectReviews": "Subject reviews",
|
|
123
|
-
"v2ex:index": "Index",
|
|
124
|
-
"v2ex:node": "Node",
|
|
125
|
-
"v2ex:member": "Member",
|
|
126
|
-
"v2ex:tab": "Tab",
|
|
127
|
-
"ximalaya:album": "Album",
|
|
128
|
-
"lemmy:community": "Community",
|
|
129
|
-
"lemmy:user": "User",
|
|
130
|
-
"apple-podcasts:podcast": "Podcast"
|
|
131
|
-
};
|
|
132
|
-
//#endregion
|
|
133
|
-
Object.defineProperty(exports, "errors", {
|
|
134
|
-
enumerable: true,
|
|
135
|
-
get: function() {
|
|
136
|
-
return errors;
|
|
137
|
-
}
|
|
138
|
-
});
|
|
139
|
-
Object.defineProperty(exports, "hints", {
|
|
140
|
-
enumerable: true,
|
|
141
|
-
get: function() {
|
|
142
|
-
return hints;
|
|
143
|
-
}
|
|
144
|
-
});
|
package/dist/common/types.cjs
DELETED
package/dist/common/types.d.cts
DELETED
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
import { FeedMethodOptions } from "./uris/feed/types.cjs";
|
|
2
|
-
import { GuessMethodOptions } from "./uris/guess/types.cjs";
|
|
3
|
-
import { HeadersMethodOptions } from "./uris/headers/types.cjs";
|
|
4
|
-
import { HtmlMethodOptions } from "./uris/html/types.cjs";
|
|
5
|
-
import { PlatformMethodOptions } from "./uris/platform/types.cjs";
|
|
6
|
-
|
|
7
|
-
//#region src/common/types.d.ts
|
|
8
|
-
type MaybePromise<T> = T | Promise<T>;
|
|
9
|
-
type UriEntry = string | Array<string>;
|
|
10
|
-
type DiscoverUriHint = {
|
|
11
|
-
key: string;
|
|
12
|
-
label: string;
|
|
13
|
-
};
|
|
14
|
-
type DiscoverUriEntry = {
|
|
15
|
-
uri: UriEntry;
|
|
16
|
-
hint?: DiscoverUriHint;
|
|
17
|
-
};
|
|
18
|
-
declare const discoverMethodOrder: readonly ["platform", "feed", "html", "headers", "guess"];
|
|
19
|
-
type DiscoverMethod = (typeof discoverMethodOrder)[number];
|
|
20
|
-
type LinkSelector = {
|
|
21
|
-
rel: string;
|
|
22
|
-
types?: Array<string>;
|
|
23
|
-
};
|
|
24
|
-
type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string | undefined;
|
|
25
|
-
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
|
|
26
|
-
type DiscoverFetchFnOptions = {
|
|
27
|
-
method?: 'GET' | 'HEAD';
|
|
28
|
-
headers?: Record<string, string>;
|
|
29
|
-
};
|
|
30
|
-
type DiscoverFetchFnResponse = {
|
|
31
|
-
headers: Headers;
|
|
32
|
-
body: string | ReadableStream<Uint8Array>;
|
|
33
|
-
url: string;
|
|
34
|
-
status: number;
|
|
35
|
-
statusText: string;
|
|
36
|
-
};
|
|
37
|
-
type DiscoverFetchFn = (url: string, options?: DiscoverFetchFnOptions) => MaybePromise<DiscoverFetchFnResponse>;
|
|
38
|
-
type DiscoverProgress = {
|
|
39
|
-
tested: number;
|
|
40
|
-
total: number;
|
|
41
|
-
found: number;
|
|
42
|
-
current: string;
|
|
43
|
-
};
|
|
44
|
-
type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
|
|
45
|
-
type DiscoverResult<TValid = object> = ({
|
|
46
|
-
url: string;
|
|
47
|
-
isValid: true;
|
|
48
|
-
method?: DiscoverMethod;
|
|
49
|
-
hint?: DiscoverUriHint;
|
|
50
|
-
} & TValid) | {
|
|
51
|
-
url: string;
|
|
52
|
-
isValid: false;
|
|
53
|
-
method?: DiscoverMethod;
|
|
54
|
-
hint?: DiscoverUriHint;
|
|
55
|
-
error?: unknown;
|
|
56
|
-
};
|
|
57
|
-
type DiscoverExtractFn<TValid> = (input: {
|
|
58
|
-
url: string;
|
|
59
|
-
content: string;
|
|
60
|
-
headers?: Headers;
|
|
61
|
-
status?: number;
|
|
62
|
-
}) => MaybePromise<DiscoverResult<TValid>>;
|
|
63
|
-
type DiscoverInputObject = {
|
|
64
|
-
url: string;
|
|
65
|
-
content?: string;
|
|
66
|
-
headers?: Headers;
|
|
67
|
-
};
|
|
68
|
-
type DiscoverInput = string | DiscoverInputObject;
|
|
69
|
-
type DiscoverMethodsConfig<TMethods extends DiscoverMethod = DiscoverMethod> = Array<TMethods> | Pick<{
|
|
70
|
-
platform?: true | Partial<PlatformMethodOptions>;
|
|
71
|
-
feed?: true | Partial<FeedMethodOptions>;
|
|
72
|
-
html?: true | Partial<Omit<HtmlMethodOptions, 'baseUrl'>>;
|
|
73
|
-
headers?: true | Partial<Omit<HeadersMethodOptions, 'baseUrl'>>;
|
|
74
|
-
guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
|
|
75
|
-
}, TMethods>;
|
|
76
|
-
type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> = {
|
|
77
|
-
methods?: DiscoverMethodsConfig<TMethods>;
|
|
78
|
-
fetchFn?: DiscoverFetchFn;
|
|
79
|
-
extractFn?: DiscoverExtractFn<TValid>;
|
|
80
|
-
resolveUrlFn?: DiscoverResolveUrlFn;
|
|
81
|
-
resolveSiteUrlFn?: DiscoverResolveSiteUrlFn;
|
|
82
|
-
stopOnFirstMethod?: boolean;
|
|
83
|
-
stopOnFirstResult?: boolean;
|
|
84
|
-
concurrency?: number;
|
|
85
|
-
onProgress?: DiscoverOnProgressFn;
|
|
86
|
-
includeInvalid?: boolean;
|
|
87
|
-
};
|
|
88
|
-
//#endregion
|
|
89
|
-
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, MaybePromise, UriEntry };
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
const require_utils = require("../../utils.cjs");
|
|
2
|
-
let feedsmith = require("feedsmith");
|
|
3
|
-
//#region src/common/uris/feed/index.ts
|
|
4
|
-
const discoverUrisFromFeed = (content, options) => {
|
|
5
|
-
try {
|
|
6
|
-
const result = (0, feedsmith.parseFeed)(content);
|
|
7
|
-
return require_utils.omitEmpty(options.extractUrls(result));
|
|
8
|
-
} catch {}
|
|
9
|
-
return [];
|
|
10
|
-
};
|
|
11
|
-
//#endregion
|
|
12
|
-
exports.discoverUrisFromFeed = discoverUrisFromFeed;
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { parseFeed } from "feedsmith";
|
|
2
|
-
|
|
3
|
-
//#region src/common/uris/feed/types.d.ts
|
|
4
|
-
type FeedMethodData = ReturnType<typeof parseFeed>;
|
|
5
|
-
type FeedMethodOptions = {
|
|
6
|
-
extractUrls: (params: FeedMethodData) => Array<string>;
|
|
7
|
-
};
|
|
8
|
-
//#endregion
|
|
9
|
-
export { FeedMethodOptions };
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
const require_utils = require("./utils.cjs");
|
|
2
|
-
//#region src/common/uris/guess/index.ts
|
|
3
|
-
const discoverUrisFromGuess = (options) => {
|
|
4
|
-
const { baseUrl, uris, additionalBaseUrls = [] } = options;
|
|
5
|
-
return require_utils.generateUrlCombinations([baseUrl, ...additionalBaseUrls], uris);
|
|
6
|
-
};
|
|
7
|
-
//#endregion
|
|
8
|
-
exports.discoverUrisFromGuess = discoverUrisFromGuess;
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import { GuessMethodOptions } from "./types.cjs";
|
|
2
|
-
import { UriEntry } from "../../types.cjs";
|
|
3
|
-
|
|
4
|
-
//#region src/common/uris/guess/index.d.ts
|
|
5
|
-
declare const discoverUrisFromGuess: (options: GuessMethodOptions) => Array<UriEntry>;
|
|
6
|
-
//#endregion
|
|
7
|
-
export { discoverUrisFromGuess };
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
//#region src/common/uris/guess/utils.ts
|
|
2
|
-
const ipAddressRegex = /^\d+\.\d+\.\d+\.\d+$/;
|
|
3
|
-
const resolveUri = (uri, base, origin, pathname) => {
|
|
4
|
-
if (uri.startsWith("/")) return `${origin}${uri}`;
|
|
5
|
-
if (uri.startsWith("?")) return `${origin}${pathname}${uri}`;
|
|
6
|
-
return new URL(uri, base).href;
|
|
7
|
-
};
|
|
8
|
-
const generateUrlCombinations = (baseUrls, uris) => {
|
|
9
|
-
return baseUrls.flatMap((base) => {
|
|
10
|
-
const parsed = new URL(base);
|
|
11
|
-
const origin = parsed.origin;
|
|
12
|
-
const pathname = parsed.pathname;
|
|
13
|
-
return uris.map((uri) => {
|
|
14
|
-
if (typeof uri === "string") return resolveUri(uri, base, origin, pathname);
|
|
15
|
-
return uri.map((alternative) => resolveUri(alternative, base, origin, pathname));
|
|
16
|
-
});
|
|
17
|
-
});
|
|
18
|
-
};
|
|
19
|
-
const getWwwCounterpart = (baseUrl) => {
|
|
20
|
-
const url = new URL(baseUrl);
|
|
21
|
-
const port = url.port ? `:${url.port}` : "";
|
|
22
|
-
if (url.hostname.startsWith("www.")) return `${url.protocol}//${url.hostname.slice(4)}${port}`;
|
|
23
|
-
return `${url.protocol}//www.${url.hostname}${port}`;
|
|
24
|
-
};
|
|
25
|
-
const getSubdomainVariants = (baseUrl, prefixes) => {
|
|
26
|
-
const url = new URL(baseUrl);
|
|
27
|
-
const hostname = url.hostname;
|
|
28
|
-
const isIpAddress = ipAddressRegex.test(hostname);
|
|
29
|
-
if (hostname === "localhost" || isIpAddress) return [];
|
|
30
|
-
const hostnameParts = hostname.split(".");
|
|
31
|
-
if (hostnameParts.length < 2) return [];
|
|
32
|
-
const rootDomain = hostnameParts.slice(-2).join(".");
|
|
33
|
-
const protocol = url.protocol;
|
|
34
|
-
const port = url.port ? `:${url.port}` : "";
|
|
35
|
-
return prefixes.map((prefix) => {
|
|
36
|
-
return `${protocol}//${prefix === "" ? rootDomain : `${prefix}.${rootDomain}`}${port}`;
|
|
37
|
-
});
|
|
38
|
-
};
|
|
39
|
-
//#endregion
|
|
40
|
-
exports.generateUrlCombinations = generateUrlCombinations;
|
|
41
|
-
exports.getSubdomainVariants = getSubdomainVariants;
|
|
42
|
-
exports.getWwwCounterpart = getWwwCounterpart;
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { UriEntry } from "../../types.cjs";
|
|
2
|
-
|
|
3
|
-
//#region src/common/uris/guess/utils.d.ts
|
|
4
|
-
declare const generateUrlCombinations: (baseUrls: Array<string>, uris: Array<UriEntry>) => Array<UriEntry>;
|
|
5
|
-
declare const getWwwCounterpart: (baseUrl: string) => string;
|
|
6
|
-
declare const getSubdomainVariants: (baseUrl: string, prefixes: Array<string>) => Array<string>;
|
|
7
|
-
//#endregion
|
|
8
|
-
export { generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
const require_utils = require("../../utils.cjs");
|
|
2
|
-
//#region src/common/uris/headers/index.ts
|
|
3
|
-
const linkSplitRegex = /,(?=\s*<)/;
|
|
4
|
-
const urlRegex = /<([^<>]+)>/;
|
|
5
|
-
const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
|
|
6
|
-
const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
|
|
7
|
-
const discoverUrisFromHeaders = (headers, options) => {
|
|
8
|
-
const uris = /* @__PURE__ */ new Set();
|
|
9
|
-
const linkHeader = headers.get("link");
|
|
10
|
-
if (!linkHeader) return [];
|
|
11
|
-
const links = linkHeader.split(linkSplitRegex);
|
|
12
|
-
for (const link of links) {
|
|
13
|
-
const urlMatch = link.match(urlRegex);
|
|
14
|
-
const relMatch = link.match(relRegex);
|
|
15
|
-
const typeMatch = link.match(typeRegex);
|
|
16
|
-
if (!urlMatch) continue;
|
|
17
|
-
const url = urlMatch[1];
|
|
18
|
-
const rel = relMatch?.[1]?.toLowerCase();
|
|
19
|
-
const type = typeMatch?.[1];
|
|
20
|
-
if (!rel) continue;
|
|
21
|
-
if (require_utils.matchesAnyOfLinkSelectors(rel, type, options.linkSelectors)) uris.add(url);
|
|
22
|
-
}
|
|
23
|
-
return [...uris];
|
|
24
|
-
};
|
|
25
|
-
//#endregion
|
|
26
|
-
exports.discoverUrisFromHeaders = discoverUrisFromHeaders;
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
const require_utils = require("../../utils.cjs");
|
|
2
|
-
//#region src/common/uris/html/handlers.ts
|
|
3
|
-
const handleOpenTag = (context, name, attribs, _isImplied) => {
|
|
4
|
-
if (name === "link" && attribs.href) {
|
|
5
|
-
const rel = attribs.rel?.toLowerCase();
|
|
6
|
-
if (!rel) return;
|
|
7
|
-
if (require_utils.matchesAnyOfLinkSelectors(rel, attribs.type, context.options.linkSelectors)) context.discoveredUris.add(attribs.href);
|
|
8
|
-
}
|
|
9
|
-
if (name === "a" && attribs.href) {
|
|
10
|
-
const lowerHref = attribs.href.toLowerCase();
|
|
11
|
-
if (require_utils.includesAnyOf(lowerHref, context.options.anchorIgnoredUris)) {
|
|
12
|
-
context.currentAnchor.href = "";
|
|
13
|
-
context.currentAnchor.text = "";
|
|
14
|
-
return;
|
|
15
|
-
}
|
|
16
|
-
context.currentAnchor.href = attribs.href;
|
|
17
|
-
context.currentAnchor.text = "";
|
|
18
|
-
if (require_utils.endsWithAnyOf(lowerHref, context.options.anchorUris)) context.discoveredUris.add(attribs.href);
|
|
19
|
-
}
|
|
20
|
-
};
|
|
21
|
-
const handleText = (context, text) => {
|
|
22
|
-
if (context.currentAnchor.href) context.currentAnchor.text += text;
|
|
23
|
-
};
|
|
24
|
-
const handleCloseTag = (context, name, _isImplied) => {
|
|
25
|
-
if (name === "a" && context.currentAnchor.href && context.currentAnchor.text) {
|
|
26
|
-
if (require_utils.includesAnyOf(context.currentAnchor.text.toLowerCase().trim(), context.options.anchorLabels)) context.discoveredUris.add(context.currentAnchor.href);
|
|
27
|
-
context.currentAnchor.href = "";
|
|
28
|
-
context.currentAnchor.text = "";
|
|
29
|
-
}
|
|
30
|
-
};
|
|
31
|
-
const createHtmlUrisHandlers = (context) => {
|
|
32
|
-
return {
|
|
33
|
-
onopentag: (name, attribs, isImplied) => {
|
|
34
|
-
return handleOpenTag(context, name, attribs, isImplied);
|
|
35
|
-
},
|
|
36
|
-
ontext: (text) => {
|
|
37
|
-
return handleText(context, text);
|
|
38
|
-
},
|
|
39
|
-
onclosetag: (name, isImplied) => {
|
|
40
|
-
return handleCloseTag(context, name, isImplied);
|
|
41
|
-
}
|
|
42
|
-
};
|
|
43
|
-
};
|
|
44
|
-
//#endregion
|
|
45
|
-
exports.createHtmlUrisHandlers = createHtmlUrisHandlers;
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
const require_handlers = require("./handlers.cjs");
|
|
2
|
-
let htmlparser2 = require("htmlparser2");
|
|
3
|
-
//#region src/common/uris/html/index.ts
|
|
4
|
-
const discoverUrisFromHtml = (html, options) => {
|
|
5
|
-
const context = {
|
|
6
|
-
discoveredUris: /* @__PURE__ */ new Set(),
|
|
7
|
-
currentAnchor: {
|
|
8
|
-
href: "",
|
|
9
|
-
text: ""
|
|
10
|
-
},
|
|
11
|
-
options
|
|
12
|
-
};
|
|
13
|
-
const parser = new htmlparser2.Parser(require_handlers.createHtmlUrisHandlers(context), { decodeEntities: true });
|
|
14
|
-
parser.write(html);
|
|
15
|
-
parser.end();
|
|
16
|
-
return [...context.discoveredUris];
|
|
17
|
-
};
|
|
18
|
-
//#endregion
|
|
19
|
-
exports.discoverUrisFromHtml = discoverUrisFromHtml;
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { LinkSelector } from "../../types.cjs";
|
|
2
|
-
|
|
3
|
-
//#region src/common/uris/html/types.d.ts
|
|
4
|
-
type HtmlMethodOptions = {
|
|
5
|
-
baseUrl?: string;
|
|
6
|
-
linkSelectors: Array<LinkSelector>;
|
|
7
|
-
anchorUris: Array<string>;
|
|
8
|
-
anchorIgnoredUris: Array<string>;
|
|
9
|
-
anchorLabels: Array<string>;
|
|
10
|
-
};
|
|
11
|
-
//#endregion
|
|
12
|
-
export { HtmlMethodOptions };
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
const require_index = require("./feed/index.cjs");
|
|
2
|
-
const require_index$1 = require("./guess/index.cjs");
|
|
3
|
-
const require_index$2 = require("./headers/index.cjs");
|
|
4
|
-
const require_index$3 = require("./html/index.cjs");
|
|
5
|
-
const require_index$4 = require("./platform/index.cjs");
|
|
6
|
-
//#region src/common/uris/index.ts
|
|
7
|
-
const discoverUris = async (config, fetchFn) => {
|
|
8
|
-
const result = {};
|
|
9
|
-
if (config.platform) {
|
|
10
|
-
const uris = await require_index$4.discoverUrisFromPlatform(config.platform.content, config.platform.headers, config.platform.options, fetchFn);
|
|
11
|
-
if (uris.length > 0) result.platform = uris;
|
|
12
|
-
}
|
|
13
|
-
if (config.feed) {
|
|
14
|
-
const uris = require_index.discoverUrisFromFeed(config.feed.content, config.feed.options);
|
|
15
|
-
if (uris.length > 0) result.feed = uris.map((uri) => ({ uri }));
|
|
16
|
-
}
|
|
17
|
-
if (config.html) {
|
|
18
|
-
const uris = require_index$3.discoverUrisFromHtml(config.html.html, config.html.options);
|
|
19
|
-
if (uris.length > 0) result.html = uris.map((uri) => ({ uri }));
|
|
20
|
-
}
|
|
21
|
-
if (config.headers) {
|
|
22
|
-
const uris = require_index$2.discoverUrisFromHeaders(config.headers.headers, config.headers.options);
|
|
23
|
-
if (uris.length > 0) result.headers = uris.map((uri) => ({ uri }));
|
|
24
|
-
}
|
|
25
|
-
if (config.guess) {
|
|
26
|
-
const uris = require_index$1.discoverUrisFromGuess(config.guess.options);
|
|
27
|
-
if (uris.length > 0) result.guess = uris.map((uri) => ({ uri }));
|
|
28
|
-
}
|
|
29
|
-
return result;
|
|
30
|
-
};
|
|
31
|
-
//#endregion
|
|
32
|
-
exports.discoverUris = discoverUris;
|