feedscout 1.8.0 → 2.0.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blogrolls/extractors.d.ts +2 -2
- package/dist/blogrolls/extractors.js +2 -2
- package/dist/blogrolls/index.js +4 -5
- package/dist/blogrolls.d.ts +2 -2
- package/dist/blogrolls.js +2 -2
- package/dist/common/discover/defaults.d.ts +6 -0
- package/dist/common/discover/defaults.js +44 -0
- package/dist/common/discover/index.js +2 -2
- package/dist/common/discover/utils.d.ts +3 -6
- package/dist/common/discover/utils.js +1 -28
- package/dist/common/types.d.ts +1 -1
- package/dist/common/uris/feed/types.d.ts +2 -2
- package/dist/common/utils.js +1 -8
- package/dist/favicons/extractors.d.ts +2 -2
- package/dist/favicons/extractors.js +2 -2
- package/dist/favicons/index.js +4 -5
- package/dist/favicons.d.ts +2 -2
- package/dist/favicons.js +2 -2
- package/dist/feeds/extractors.d.ts +2 -2
- package/dist/feeds/extractors.js +6 -6
- package/dist/feeds/index.js +4 -5
- package/dist/feeds.d.ts +2 -2
- package/dist/feeds.js +2 -2
- package/dist/hubs/discover/index.js +3 -3
- package/dist/hubs/feed/index.js +2 -2
- package/dist/hubs/headers/index.js +2 -2
- package/dist/hubs/html/index.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/package.json +18 -61
- package/dist/blogrolls/defaults.cjs +0 -51
- package/dist/blogrolls/defaults.d.cts +0 -17
- package/dist/blogrolls/extractors.cjs +0 -21
- package/dist/blogrolls/extractors.d.cts +0 -7
- package/dist/blogrolls/index.cjs +0 -26
- package/dist/blogrolls/index.d.cts +0 -7
- package/dist/blogrolls/types.d.cts +0 -6
- package/dist/blogrolls.cjs +0 -13
- package/dist/blogrolls.d.cts +0 -4
- package/dist/common/discover/index.cjs +0 -105
- package/dist/common/discover/utils.cjs +0 -130
- package/dist/common/discover/utils.d.cts +0 -8
- package/dist/common/locales.cjs +0 -144
- package/dist/common/types.cjs +0 -10
- package/dist/common/types.d.cts +0 -89
- package/dist/common/uris/feed/index.cjs +0 -12
- package/dist/common/uris/feed/index.d.cts +0 -6
- package/dist/common/uris/feed/types.d.cts +0 -9
- package/dist/common/uris/guess/index.cjs +0 -8
- package/dist/common/uris/guess/index.d.cts +0 -7
- package/dist/common/uris/guess/types.d.cts +0 -10
- package/dist/common/uris/guess/utils.cjs +0 -42
- package/dist/common/uris/guess/utils.d.cts +0 -8
- package/dist/common/uris/headers/index.cjs +0 -26
- package/dist/common/uris/headers/index.d.cts +0 -6
- package/dist/common/uris/headers/types.d.cts +0 -9
- package/dist/common/uris/html/handlers.cjs +0 -45
- package/dist/common/uris/html/index.cjs +0 -19
- package/dist/common/uris/html/index.d.cts +0 -6
- package/dist/common/uris/html/types.d.cts +0 -12
- package/dist/common/uris/index.cjs +0 -32
- package/dist/common/uris/platform/index.cjs +0 -10
- package/dist/common/uris/platform/index.d.cts +0 -7
- package/dist/common/uris/platform/types.d.cts +0 -13
- package/dist/common/utils.cjs +0 -97
- package/dist/common/utils.d.cts +0 -10
- package/dist/favicons/defaults.cjs +0 -65
- package/dist/favicons/defaults.d.cts +0 -18
- package/dist/favicons/extractors.cjs +0 -25
- package/dist/favicons/extractors.d.cts +0 -7
- package/dist/favicons/index.cjs +0 -30
- package/dist/favicons/index.d.cts +0 -7
- package/dist/favicons/platform/handlers/bluesky.cjs +0 -27
- package/dist/favicons/platform/handlers/codeberg.cjs +0 -19
- package/dist/favicons/platform/handlers/deviantart.cjs +0 -29
- package/dist/favicons/platform/handlers/devto.cjs +0 -31
- package/dist/favicons/platform/handlers/github.cjs +0 -19
- package/dist/favicons/platform/handlers/githubGist.cjs +0 -19
- package/dist/favicons/platform/handlers/gitlab.cjs +0 -38
- package/dist/favicons/platform/handlers/lobsters.cjs +0 -21
- package/dist/favicons/platform/handlers/mastodon.cjs +0 -40
- package/dist/favicons/platform/handlers/reddit.cjs +0 -42
- package/dist/favicons/platform/handlers/sourceforge.cjs +0 -21
- package/dist/favicons/platform/handlers/tumblr.cjs +0 -16
- package/dist/favicons/types.d.cts +0 -4
- package/dist/favicons/utils.cjs +0 -10
- package/dist/favicons.cjs +0 -12
- package/dist/favicons.d.cts +0 -4
- package/dist/feeds/defaults.cjs +0 -178
- package/dist/feeds/defaults.d.cts +0 -20
- package/dist/feeds/extractors.cjs +0 -46
- package/dist/feeds/extractors.d.cts +0 -7
- package/dist/feeds/index.cjs +0 -27
- package/dist/feeds/index.d.cts +0 -7
- package/dist/feeds/platform/handlers/applePodcasts.cjs +0 -26
- package/dist/feeds/platform/handlers/behance.cjs +0 -49
- package/dist/feeds/platform/handlers/blogspot.cjs +0 -36
- package/dist/feeds/platform/handlers/bluesky.cjs +0 -20
- package/dist/feeds/platform/handlers/codeberg.cjs +0 -69
- package/dist/feeds/platform/handlers/csdn.cjs +0 -20
- package/dist/feeds/platform/handlers/dailymotion.cjs +0 -70
- package/dist/feeds/platform/handlers/deviantart.cjs +0 -66
- package/dist/feeds/platform/handlers/devto.cjs +0 -50
- package/dist/feeds/platform/handlers/douban.cjs +0 -56
- package/dist/feeds/platform/handlers/github.cjs +0 -116
- package/dist/feeds/platform/handlers/githubGist.cjs +0 -45
- package/dist/feeds/platform/handlers/gitlab.cjs +0 -80
- package/dist/feeds/platform/handlers/goodreads.cjs +0 -39
- package/dist/feeds/platform/handlers/hashnode.cjs +0 -16
- package/dist/feeds/platform/handlers/hatenablog.cjs +0 -53
- package/dist/feeds/platform/handlers/itchio.cjs +0 -88
- package/dist/feeds/platform/handlers/kickstarter.cjs +0 -22
- package/dist/feeds/platform/handlers/lemmy.cjs +0 -46
- package/dist/feeds/platform/handlers/letterboxd.cjs +0 -42
- package/dist/feeds/platform/handlers/lobsters.cjs +0 -57
- package/dist/feeds/platform/handlers/mastodon.cjs +0 -42
- package/dist/feeds/platform/handlers/medium.cjs +0 -68
- package/dist/feeds/platform/handlers/paragraph.cjs +0 -21
- package/dist/feeds/platform/handlers/pinterest.cjs +0 -44
- package/dist/feeds/platform/handlers/producthunt.cjs +0 -29
- package/dist/feeds/platform/handlers/reddit.cjs +0 -75
- package/dist/feeds/platform/handlers/soundcloud.cjs +0 -39
- package/dist/feeds/platform/handlers/sourceforge.cjs +0 -20
- package/dist/feeds/platform/handlers/stackExchange.cjs +0 -40
- package/dist/feeds/platform/handlers/steam.cjs +0 -28
- package/dist/feeds/platform/handlers/substack.cjs +0 -23
- package/dist/feeds/platform/handlers/tumblr.cjs +0 -24
- package/dist/feeds/platform/handlers/v2ex.cjs +0 -35
- package/dist/feeds/platform/handlers/vimeo.cjs +0 -69
- package/dist/feeds/platform/handlers/wordpress.cjs +0 -64
- package/dist/feeds/platform/handlers/wpengine.cjs +0 -10
- package/dist/feeds/platform/handlers/ximalaya.cjs +0 -20
- package/dist/feeds/platform/handlers/youtube.cjs +0 -94
- package/dist/feeds/types.d.cts +0 -9
- package/dist/feeds.cjs +0 -15
- package/dist/feeds.d.cts +0 -4
- package/dist/hubs/discover/index.cjs +0 -30
- package/dist/hubs/discover/index.d.cts +0 -7
- package/dist/hubs/discover/types.d.cts +0 -15
- package/dist/hubs/feed/index.cjs +0 -32
- package/dist/hubs/headers/index.cjs +0 -17
- package/dist/hubs/html/index.cjs +0 -28
- package/dist/hubs.cjs +0 -0
- package/dist/hubs.d.cts +0 -2
- package/dist/index.cjs +0 -12
- package/dist/index.d.cts +0 -7
- package/dist/methods.cjs +0 -15
- package/dist/methods.d.cts +0 -7
- package/dist/utils.cjs +0 -9
- package/dist/utils.d.cts +0 -2
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.js";
|
|
|
2
2
|
import { BlogrollResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/blogrolls/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<BlogrollResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parseOpml } from "feedsmith";
|
|
2
2
|
//#region src/blogrolls/extractors.ts
|
|
3
|
-
const
|
|
3
|
+
const defaultExtractFn = ({ content, url }) => {
|
|
4
4
|
if (!content) return {
|
|
5
5
|
url,
|
|
6
6
|
isValid: false
|
|
@@ -18,4 +18,4 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
18
18
|
};
|
|
19
19
|
};
|
|
20
20
|
//#endregion
|
|
21
|
-
export {
|
|
21
|
+
export { defaultExtractFn };
|
package/dist/blogrolls/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn, defaultResolveSiteUrlFn } from "../common/discover/utils.js";
|
|
1
|
+
import { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
2
|
import { discover } from "../common/discover/index.js";
|
|
4
3
|
import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
|
|
5
|
-
import {
|
|
4
|
+
import { defaultExtractFn } from "./extractors.js";
|
|
6
5
|
//#region src/blogrolls/index.ts
|
|
7
6
|
const discoverBlogrolls = (input, options = {}) => {
|
|
8
7
|
return discover(input, {
|
|
@@ -13,8 +12,8 @@ const discoverBlogrolls = (input, options = {}) => {
|
|
|
13
12
|
"guess"
|
|
14
13
|
],
|
|
15
14
|
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
16
|
-
extractFn: options.extractFn ??
|
|
17
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
15
|
+
extractFn: options.extractFn ?? defaultExtractFn,
|
|
16
|
+
resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn,
|
|
18
17
|
resolveSiteUrlFn: options.resolveSiteUrlFn ?? defaultResolveSiteUrlFn
|
|
19
18
|
}, {
|
|
20
19
|
html: defaultHtmlOptions,
|
package/dist/blogrolls.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.js";
|
|
2
2
|
import { BlogrollResult } from "./blogrolls/types.js";
|
|
3
|
-
import {
|
|
4
|
-
export { BlogrollResult, anchorLabels,
|
|
3
|
+
import { defaultExtractFn } from "./blogrolls/extractors.js";
|
|
4
|
+
export { BlogrollResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
package/dist/blogrolls.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.js";
|
|
2
|
-
import {
|
|
3
|
-
export { anchorLabels,
|
|
2
|
+
import { defaultExtractFn } from "./blogrolls/extractors.js";
|
|
3
|
+
export { anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { getFeedSiteUrl } from "./utils.js";
|
|
2
|
+
import { parseFeed } from "feedsmith";
|
|
3
|
+
//#region src/common/discover/defaults.ts
|
|
4
|
+
const defaultFetchFn = async (url, options) => {
|
|
5
|
+
const response = await fetch(url, {
|
|
6
|
+
method: options?.method ?? "GET",
|
|
7
|
+
headers: options?.headers
|
|
8
|
+
});
|
|
9
|
+
return {
|
|
10
|
+
headers: response.headers,
|
|
11
|
+
body: await response.text(),
|
|
12
|
+
url: response.url,
|
|
13
|
+
status: response.status,
|
|
14
|
+
statusText: response.statusText
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
const defaultResolveUrlFn = (url, baseUrl) => {
|
|
18
|
+
try {
|
|
19
|
+
return new URL(url, baseUrl).href;
|
|
20
|
+
} catch {
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
const defaultResolveSiteUrlFn = (input, resolveUrlFn) => {
|
|
25
|
+
if (!input.content) return;
|
|
26
|
+
try {
|
|
27
|
+
let siteUrl = getFeedSiteUrl(parseFeed(input.content));
|
|
28
|
+
if (siteUrl) {
|
|
29
|
+
siteUrl = resolveUrlFn(siteUrl, input.url);
|
|
30
|
+
if (siteUrl) {
|
|
31
|
+
const parsed = new URL(siteUrl);
|
|
32
|
+
parsed.hash = "";
|
|
33
|
+
siteUrl = parsed.href;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (!siteUrl || siteUrl === new URL(input.url).href) try {
|
|
37
|
+
siteUrl = new URL(input.url).origin;
|
|
38
|
+
} catch {}
|
|
39
|
+
if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
|
|
40
|
+
return siteUrl;
|
|
41
|
+
} catch {}
|
|
42
|
+
};
|
|
43
|
+
//#endregion
|
|
44
|
+
export { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
|
|
1
2
|
import { discoverMethodOrder } from "../types.js";
|
|
2
3
|
import { processConcurrently } from "../utils.js";
|
|
3
4
|
import { discoverUris } from "../uris/index.js";
|
|
4
|
-
import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
|
|
5
5
|
//#region src/common/discover/index.ts
|
|
6
6
|
const discover = async (input, options, defaults) => {
|
|
7
7
|
const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
|
|
@@ -16,7 +16,7 @@ const discover = async (input, options, defaults) => {
|
|
|
16
16
|
}
|
|
17
17
|
let siteInput;
|
|
18
18
|
if (resolveSiteUrlFn) {
|
|
19
|
-
const siteUrl = resolveSiteUrlFn(sourceInput);
|
|
19
|
+
const siteUrl = resolveSiteUrlFn(sourceInput, resolveUrlFn);
|
|
20
20
|
if (siteUrl) try {
|
|
21
21
|
const response = await fetchFn(siteUrl);
|
|
22
22
|
siteInput = {
|
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { parseFeed } from "feedsmith";
|
|
3
|
-
|
|
1
|
+
import { FeedMethodData } from "../uris/feed/types.js";
|
|
4
2
|
//#region src/common/discover/utils.d.ts
|
|
5
|
-
declare const getFeedSiteUrl: (parsed:
|
|
6
|
-
declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
|
|
3
|
+
declare const getFeedSiteUrl: (parsed: FeedMethodData) => string | undefined;
|
|
7
4
|
//#endregion
|
|
8
|
-
export {
|
|
5
|
+
export { getFeedSiteUrl };
|
|
@@ -1,20 +1,5 @@
|
|
|
1
1
|
import { errors } from "../locales.js";
|
|
2
|
-
import { resolveUrl } from "../utils.js";
|
|
3
|
-
import { parseFeed } from "feedsmith";
|
|
4
2
|
//#region src/common/discover/utils.ts
|
|
5
|
-
const defaultFetchFn = async (url, options) => {
|
|
6
|
-
const response = await fetch(url, {
|
|
7
|
-
method: options?.method ?? "GET",
|
|
8
|
-
headers: options?.headers
|
|
9
|
-
});
|
|
10
|
-
return {
|
|
11
|
-
headers: response.headers,
|
|
12
|
-
body: await response.text(),
|
|
13
|
-
url: response.url,
|
|
14
|
-
status: response.status,
|
|
15
|
-
statusText: response.statusText
|
|
16
|
-
};
|
|
17
|
-
};
|
|
18
3
|
const normalizeInput = async (input, fetchFn) => {
|
|
19
4
|
if (typeof input === "object") return input;
|
|
20
5
|
try {
|
|
@@ -36,18 +21,6 @@ const getFeedSiteUrl = (parsed) => {
|
|
|
36
21
|
if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
|
|
37
22
|
if (format === "json") return feed.home_page_url;
|
|
38
23
|
};
|
|
39
|
-
const defaultResolveSiteUrlFn = (input) => {
|
|
40
|
-
if (!input.content) return;
|
|
41
|
-
try {
|
|
42
|
-
let siteUrl = getFeedSiteUrl(parseFeed(input.content));
|
|
43
|
-
if (!siteUrl) try {
|
|
44
|
-
siteUrl = new URL(input.url).origin;
|
|
45
|
-
} catch {}
|
|
46
|
-
else siteUrl = resolveUrl(siteUrl, input.url);
|
|
47
|
-
if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
|
|
48
|
-
return siteUrl;
|
|
49
|
-
} catch {}
|
|
50
|
-
};
|
|
51
24
|
const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
|
|
52
25
|
if (typeof entry.uri === "string") return {
|
|
53
26
|
...entry,
|
|
@@ -122,4 +95,4 @@ const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
|
|
|
122
95
|
return methodsConfig;
|
|
123
96
|
};
|
|
124
97
|
//#endregion
|
|
125
|
-
export {
|
|
98
|
+
export { getFeedSiteUrl, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };
|
package/dist/common/types.d.ts
CHANGED
|
@@ -22,7 +22,7 @@ type LinkSelector = {
|
|
|
22
22
|
types?: Array<string>;
|
|
23
23
|
};
|
|
24
24
|
type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string | undefined;
|
|
25
|
-
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
|
|
25
|
+
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject, resolveUrlFn: DiscoverResolveUrlFn) => string | undefined;
|
|
26
26
|
type DiscoverFetchFnOptions = {
|
|
27
27
|
method?: 'GET' | 'HEAD';
|
|
28
28
|
headers?: Record<string, string>;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { parseFeed } from "feedsmith";
|
|
2
2
|
|
|
3
3
|
//#region src/common/uris/feed/types.d.ts
|
|
4
|
-
type FeedMethodData = ReturnType<typeof parseFeed
|
|
4
|
+
type FeedMethodData = ReturnType<typeof parseFeed<string>>;
|
|
5
5
|
type FeedMethodOptions = {
|
|
6
6
|
extractUrls: (params: FeedMethodData) => Array<string>;
|
|
7
7
|
};
|
|
8
8
|
//#endregion
|
|
9
|
-
export { FeedMethodOptions };
|
|
9
|
+
export { FeedMethodData, FeedMethodOptions };
|
package/dist/common/utils.js
CHANGED
|
@@ -52,13 +52,6 @@ const omitEmpty = (array) => {
|
|
|
52
52
|
for (const item of array) if (item != null && item !== "") result.push(item);
|
|
53
53
|
return result;
|
|
54
54
|
};
|
|
55
|
-
const resolveUrl = (url, baseUrl) => {
|
|
56
|
-
try {
|
|
57
|
-
return new URL(url, baseUrl).href;
|
|
58
|
-
} catch {
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
55
|
const matchesAnyOfLinkSelectors = (rel, type, selectors) => {
|
|
63
56
|
return selectors.some((selector) => {
|
|
64
57
|
if (!anyWordMatchesAnyOf(rel, [selector.rel])) return false;
|
|
@@ -83,4 +76,4 @@ const processConcurrently = async (items, processFn, options) => {
|
|
|
83
76
|
}
|
|
84
77
|
};
|
|
85
78
|
//#endregion
|
|
86
|
-
export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently
|
|
79
|
+
export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently };
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.js";
|
|
|
2
2
|
import { FaviconResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/favicons/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<FaviconResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -11,7 +11,7 @@ const isImageContent = (content) => {
|
|
|
11
11
|
const isSuccessStatus = (status) => {
|
|
12
12
|
return status !== void 0 && status >= 200 && status < 400;
|
|
13
13
|
};
|
|
14
|
-
const
|
|
14
|
+
const defaultExtractFn = (input) => {
|
|
15
15
|
if (isImageContentType(input.headers) || isImageContent(input.content) || isSuccessStatus(input.status)) return {
|
|
16
16
|
url: input.url,
|
|
17
17
|
isValid: true
|
|
@@ -22,4 +22,4 @@ const defaultExtractor = (input) => {
|
|
|
22
22
|
};
|
|
23
23
|
};
|
|
24
24
|
//#endregion
|
|
25
|
-
export {
|
|
25
|
+
export { defaultExtractFn };
|
package/dist/favicons/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn, defaultResolveSiteUrlFn } from "../common/discover/utils.js";
|
|
1
|
+
import { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
2
|
import { discover } from "../common/discover/index.js";
|
|
4
3
|
import { defaultFeedOptions, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions } from "./defaults.js";
|
|
5
|
-
import {
|
|
4
|
+
import { defaultExtractFn } from "./extractors.js";
|
|
6
5
|
//#region src/favicons/index.ts
|
|
7
6
|
const discoverFavicons = (input, options = {}) => {
|
|
8
7
|
return discover(input, {
|
|
@@ -15,8 +14,8 @@ const discoverFavicons = (input, options = {}) => {
|
|
|
15
14
|
"guess"
|
|
16
15
|
],
|
|
17
16
|
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
18
|
-
extractFn: options.extractFn ??
|
|
19
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
17
|
+
extractFn: options.extractFn ?? defaultExtractFn,
|
|
18
|
+
resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn,
|
|
20
19
|
resolveSiteUrlFn: options.resolveSiteUrlFn ?? defaultResolveSiteUrlFn
|
|
21
20
|
}, {
|
|
22
21
|
platform: defaultPlatformOptions,
|
package/dist/favicons.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors } from "./favicons/defaults.js";
|
|
2
2
|
import { FaviconResult } from "./favicons/types.js";
|
|
3
|
-
import {
|
|
4
|
-
export { FaviconResult,
|
|
3
|
+
import { defaultExtractFn } from "./favicons/extractors.js";
|
|
4
|
+
export { FaviconResult, defaultExtractFn, defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors };
|
package/dist/favicons.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors } from "./favicons/defaults.js";
|
|
2
|
-
import {
|
|
3
|
-
export {
|
|
2
|
+
import { defaultExtractFn } from "./favicons/extractors.js";
|
|
3
|
+
export { defaultExtractFn, defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors };
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.js";
|
|
|
2
2
|
import { FeedResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/feeds/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<FeedResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
package/dist/feeds/extractors.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { resolveUrl } from "../common/utils.js";
|
|
2
1
|
import { getFeedSiteUrl } from "../common/discover/utils.js";
|
|
2
|
+
import { defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
3
|
import { parseFeed } from "feedsmith";
|
|
4
4
|
//#region src/feeds/extractors.ts
|
|
5
|
-
const
|
|
5
|
+
const defaultExtractFn = ({ content, url }) => {
|
|
6
6
|
if (!content) return {
|
|
7
7
|
url,
|
|
8
8
|
isValid: false
|
|
@@ -11,7 +11,7 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
11
11
|
const parsed = parseFeed(content);
|
|
12
12
|
const { format, feed } = parsed;
|
|
13
13
|
const rawSiteUrl = getFeedSiteUrl(parsed);
|
|
14
|
-
const siteUrl = rawSiteUrl ?
|
|
14
|
+
const siteUrl = rawSiteUrl ? defaultResolveUrlFn(rawSiteUrl, url) : void 0;
|
|
15
15
|
if (format === "rss" || format === "rdf") return {
|
|
16
16
|
url,
|
|
17
17
|
isValid: true,
|
|
@@ -24,8 +24,8 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
24
24
|
url,
|
|
25
25
|
isValid: true,
|
|
26
26
|
format,
|
|
27
|
-
title: feed.title,
|
|
28
|
-
description: feed.subtitle,
|
|
27
|
+
title: feed.title?.value,
|
|
28
|
+
description: feed.subtitle?.value,
|
|
29
29
|
siteUrl
|
|
30
30
|
};
|
|
31
31
|
if (format === "json") return {
|
|
@@ -43,4 +43,4 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
43
43
|
};
|
|
44
44
|
};
|
|
45
45
|
//#endregion
|
|
46
|
-
export {
|
|
46
|
+
export { defaultExtractFn };
|
package/dist/feeds/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn } from "../common/discover/utils.js";
|
|
1
|
+
import { defaultFetchFn, defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
2
|
import { discover } from "../common/discover/index.js";
|
|
4
3
|
import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions } from "./defaults.js";
|
|
5
|
-
import {
|
|
4
|
+
import { defaultExtractFn } from "./extractors.js";
|
|
6
5
|
//#region src/feeds/index.ts
|
|
7
6
|
const discoverFeeds = (input, options = {}) => {
|
|
8
7
|
return discover(input, {
|
|
@@ -14,8 +13,8 @@ const discoverFeeds = (input, options = {}) => {
|
|
|
14
13
|
"guess"
|
|
15
14
|
],
|
|
16
15
|
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
17
|
-
extractFn: options.extractFn ??
|
|
18
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
16
|
+
extractFn: options.extractFn ?? defaultExtractFn,
|
|
17
|
+
resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn
|
|
19
18
|
}, {
|
|
20
19
|
platform: defaultPlatformOptions,
|
|
21
20
|
html: defaultHtmlOptions,
|
package/dist/feeds.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.js";
|
|
2
2
|
import { FeedResult } from "./feeds/types.js";
|
|
3
|
-
import {
|
|
4
|
-
export { FeedResult, anchorLabels,
|
|
3
|
+
import { defaultExtractFn } from "./feeds/extractors.js";
|
|
4
|
+
export { FeedResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
package/dist/feeds.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.js";
|
|
2
|
-
import {
|
|
3
|
-
export { anchorLabels,
|
|
2
|
+
import { defaultExtractFn } from "./feeds/extractors.js";
|
|
3
|
+
export { anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn,
|
|
1
|
+
import { normalizeInput } from "../../common/discover/utils.js";
|
|
2
|
+
import { defaultFetchFn, defaultResolveUrlFn } from "../../common/discover/defaults.js";
|
|
3
3
|
import { discoverHubsFromFeed } from "../feed/index.js";
|
|
4
4
|
import { discoverHubsFromHeaders } from "../headers/index.js";
|
|
5
5
|
import { discoverHubsFromHtml } from "../html/index.js";
|
|
@@ -9,7 +9,7 @@ const discoverHubs = async (input, options = {}) => {
|
|
|
9
9
|
"headers",
|
|
10
10
|
"feed",
|
|
11
11
|
"html"
|
|
12
|
-
], fetchFn = defaultFetchFn, resolveUrlFn =
|
|
12
|
+
], fetchFn = defaultFetchFn, resolveUrlFn = defaultResolveUrlFn } = options;
|
|
13
13
|
const normalizedInput = await normalizeInput(input, fetchFn);
|
|
14
14
|
const results = [];
|
|
15
15
|
if (methods.includes("headers") && normalizedInput.headers) {
|
package/dist/hubs/feed/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
|
|
2
2
|
import { parseFeed } from "feedsmith";
|
|
3
3
|
//#region src/hubs/feed/index.ts
|
|
4
4
|
const getLinksWithRel = (links, rel) => {
|
|
5
5
|
return links?.filter((link) => link.rel === rel && link.href).map((link) => link.href) ?? [];
|
|
6
6
|
};
|
|
7
|
-
const discoverHubsFromFeed = (content, baseUrl, resolveUrlFn =
|
|
7
|
+
const discoverHubsFromFeed = (content, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
|
|
8
8
|
try {
|
|
9
9
|
const { format, feed } = parseFeed(content);
|
|
10
10
|
if (format === "json") {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
|
|
2
2
|
import { discoverUrisFromHeaders } from "../../common/uris/headers/index.js";
|
|
3
3
|
//#region src/hubs/headers/index.ts
|
|
4
4
|
const hubSelector = [{ rel: "hub" }];
|
|
5
5
|
const selfSelector = [{ rel: "self" }];
|
|
6
|
-
const discoverHubsFromHeaders = (headers, baseUrl, resolveUrlFn =
|
|
6
|
+
const discoverHubsFromHeaders = (headers, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
|
|
7
7
|
const hubUris = discoverUrisFromHeaders(headers, { linkSelectors: hubSelector });
|
|
8
8
|
if (hubUris.length === 0) return [];
|
|
9
9
|
const selfUris = discoverUrisFromHeaders(headers, { linkSelectors: selfSelector });
|
package/dist/hubs/html/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
|
|
2
2
|
import { discoverUrisFromHtml } from "../../common/uris/html/index.js";
|
|
3
3
|
//#region src/hubs/html/index.ts
|
|
4
4
|
const hubSelector = [{ rel: "hub" }];
|
|
@@ -8,7 +8,7 @@ const htmlOptions = {
|
|
|
8
8
|
anchorIgnoredUris: [],
|
|
9
9
|
anchorLabels: []
|
|
10
10
|
};
|
|
11
|
-
const discoverHubsFromHtml = (content, baseUrl, resolveUrlFn =
|
|
11
|
+
const discoverHubsFromHtml = (content, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
|
|
12
12
|
const hubUris = discoverUrisFromHtml(content, {
|
|
13
13
|
...htmlOptions,
|
|
14
14
|
linkSelectors: hubSelector
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, UriEntry } from "./common/types.js";
|
|
2
2
|
import { discoverBlogrolls } from "./blogrolls/index.js";
|
|
3
|
-
import { defaultResolveSiteUrlFn
|
|
3
|
+
import { defaultResolveSiteUrlFn } from "./common/discover/defaults.js";
|
|
4
|
+
import { getFeedSiteUrl } from "./common/discover/utils.js";
|
|
4
5
|
import { discoverFavicons } from "./favicons/index.js";
|
|
5
6
|
import { discoverFeeds } from "./feeds/index.js";
|
|
6
7
|
import { discoverHubs } from "./hubs/discover/index.js";
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getFeedSiteUrl } from "./common/discover/utils.js";
|
|
2
|
+
import { defaultResolveSiteUrlFn } from "./common/discover/defaults.js";
|
|
2
3
|
import { discoverBlogrolls } from "./blogrolls/index.js";
|
|
3
4
|
import { discoverFavicons } from "./favicons/index.js";
|
|
4
5
|
import { discoverFeeds } from "./feeds/index.js";
|
package/package.json
CHANGED
|
@@ -26,74 +26,32 @@
|
|
|
26
26
|
"type": "module",
|
|
27
27
|
"exports": {
|
|
28
28
|
".": {
|
|
29
|
-
"
|
|
30
|
-
|
|
31
|
-
"default": "./dist/index.js"
|
|
32
|
-
},
|
|
33
|
-
"require": {
|
|
34
|
-
"types": "./dist/index.d.cts",
|
|
35
|
-
"default": "./dist/index.cjs"
|
|
36
|
-
}
|
|
29
|
+
"types": "./dist/index.d.ts",
|
|
30
|
+
"default": "./dist/index.js"
|
|
37
31
|
},
|
|
38
32
|
"./feeds": {
|
|
39
|
-
"
|
|
40
|
-
|
|
41
|
-
"default": "./dist/feeds.js"
|
|
42
|
-
},
|
|
43
|
-
"require": {
|
|
44
|
-
"types": "./dist/feeds.d.cts",
|
|
45
|
-
"default": "./dist/feeds.cjs"
|
|
46
|
-
}
|
|
33
|
+
"types": "./dist/feeds.d.ts",
|
|
34
|
+
"default": "./dist/feeds.js"
|
|
47
35
|
},
|
|
48
36
|
"./blogrolls": {
|
|
49
|
-
"
|
|
50
|
-
|
|
51
|
-
"default": "./dist/blogrolls.js"
|
|
52
|
-
},
|
|
53
|
-
"require": {
|
|
54
|
-
"types": "./dist/blogrolls.d.cts",
|
|
55
|
-
"default": "./dist/blogrolls.cjs"
|
|
56
|
-
}
|
|
37
|
+
"types": "./dist/blogrolls.d.ts",
|
|
38
|
+
"default": "./dist/blogrolls.js"
|
|
57
39
|
},
|
|
58
40
|
"./favicons": {
|
|
59
|
-
"
|
|
60
|
-
|
|
61
|
-
"default": "./dist/favicons.js"
|
|
62
|
-
},
|
|
63
|
-
"require": {
|
|
64
|
-
"types": "./dist/favicons.d.cts",
|
|
65
|
-
"default": "./dist/favicons.cjs"
|
|
66
|
-
}
|
|
41
|
+
"types": "./dist/favicons.d.ts",
|
|
42
|
+
"default": "./dist/favicons.js"
|
|
67
43
|
},
|
|
68
44
|
"./hubs": {
|
|
69
|
-
"
|
|
70
|
-
|
|
71
|
-
"default": "./dist/hubs.js"
|
|
72
|
-
},
|
|
73
|
-
"require": {
|
|
74
|
-
"types": "./dist/hubs.d.cts",
|
|
75
|
-
"default": "./dist/hubs.cjs"
|
|
76
|
-
}
|
|
45
|
+
"types": "./dist/hubs.d.ts",
|
|
46
|
+
"default": "./dist/hubs.js"
|
|
77
47
|
},
|
|
78
48
|
"./methods": {
|
|
79
|
-
"
|
|
80
|
-
|
|
81
|
-
"default": "./dist/methods.js"
|
|
82
|
-
},
|
|
83
|
-
"require": {
|
|
84
|
-
"types": "./dist/methods.d.cts",
|
|
85
|
-
"default": "./dist/methods.cjs"
|
|
86
|
-
}
|
|
49
|
+
"types": "./dist/methods.d.ts",
|
|
50
|
+
"default": "./dist/methods.js"
|
|
87
51
|
},
|
|
88
52
|
"./utils": {
|
|
89
|
-
"
|
|
90
|
-
|
|
91
|
-
"default": "./dist/utils.js"
|
|
92
|
-
},
|
|
93
|
-
"require": {
|
|
94
|
-
"types": "./dist/utils.d.cts",
|
|
95
|
-
"default": "./dist/utils.cjs"
|
|
96
|
-
}
|
|
53
|
+
"types": "./dist/utils.d.ts",
|
|
54
|
+
"default": "./dist/utils.js"
|
|
97
55
|
},
|
|
98
56
|
"./package.json": "./package.json"
|
|
99
57
|
},
|
|
@@ -102,14 +60,13 @@
|
|
|
102
60
|
],
|
|
103
61
|
"scripts": {
|
|
104
62
|
"prepare": "lefthook install",
|
|
105
|
-
"build": "tsdown src/exports/index.ts src/exports/feeds.ts src/exports/blogrolls.ts src/exports/favicons.ts src/exports/hubs.ts src/exports/methods.ts src/exports/utils.ts --format
|
|
63
|
+
"build": "tsdown src/exports/index.ts src/exports/feeds.ts src/exports/blogrolls.ts src/exports/favicons.ts src/exports/hubs.ts src/exports/methods.ts src/exports/utils.ts --format esm --dts --clean --unbundle --no-fixed-extension",
|
|
106
64
|
"docs:dev": "vitepress dev docs",
|
|
107
65
|
"docs:build": "vitepress build docs"
|
|
108
66
|
},
|
|
109
67
|
"dependencies": {
|
|
110
|
-
"feedsmith": "^
|
|
111
|
-
"htmlparser2": "^
|
|
112
|
-
"typescript": "^6.0.2"
|
|
68
|
+
"feedsmith": "^3.0.0-beta.1",
|
|
69
|
+
"htmlparser2": "^12.0.0"
|
|
113
70
|
},
|
|
114
71
|
"devDependencies": {
|
|
115
72
|
"@types/bun": "^1.3.11",
|
|
@@ -117,5 +74,5 @@
|
|
|
117
74
|
"tsdown": "^0.21.7",
|
|
118
75
|
"vitepress": "^2.0.0-alpha.17"
|
|
119
76
|
},
|
|
120
|
-
"version": "
|
|
77
|
+
"version": "2.0.0-next.1"
|
|
121
78
|
}
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
//#region src/blogrolls/defaults.ts
|
|
2
|
-
const mimeTypes = [
|
|
3
|
-
"text/x-opml",
|
|
4
|
-
"application/xml",
|
|
5
|
-
"text/xml"
|
|
6
|
-
];
|
|
7
|
-
const urisMinimal = [
|
|
8
|
-
"/.well-known/recommendations.opml",
|
|
9
|
-
"/blogroll.opml",
|
|
10
|
-
"/opml.xml"
|
|
11
|
-
];
|
|
12
|
-
const urisBalanced = [
|
|
13
|
-
...urisMinimal,
|
|
14
|
-
"/blogroll.xml",
|
|
15
|
-
"/subscriptions.opml",
|
|
16
|
-
"/recommendations.opml"
|
|
17
|
-
];
|
|
18
|
-
const urisComprehensive = [
|
|
19
|
-
...urisBalanced,
|
|
20
|
-
"/links.opml",
|
|
21
|
-
"/feeds.opml",
|
|
22
|
-
"/subscriptions.xml"
|
|
23
|
-
];
|
|
24
|
-
const anchorLabels = [
|
|
25
|
-
"blogroll",
|
|
26
|
-
"opml",
|
|
27
|
-
"subscriptions",
|
|
28
|
-
"reading list"
|
|
29
|
-
];
|
|
30
|
-
const linkSelectors = [{ rel: "blogroll" }, {
|
|
31
|
-
rel: "outline",
|
|
32
|
-
types: mimeTypes
|
|
33
|
-
}];
|
|
34
|
-
const defaultHtmlOptions = {
|
|
35
|
-
linkSelectors,
|
|
36
|
-
anchorUris: urisComprehensive,
|
|
37
|
-
anchorIgnoredUris: [],
|
|
38
|
-
anchorLabels
|
|
39
|
-
};
|
|
40
|
-
const defaultHeadersOptions = { linkSelectors };
|
|
41
|
-
const defaultGuessOptions = { uris: urisBalanced };
|
|
42
|
-
//#endregion
|
|
43
|
-
exports.anchorLabels = anchorLabels;
|
|
44
|
-
exports.defaultGuessOptions = defaultGuessOptions;
|
|
45
|
-
exports.defaultHeadersOptions = defaultHeadersOptions;
|
|
46
|
-
exports.defaultHtmlOptions = defaultHtmlOptions;
|
|
47
|
-
exports.linkSelectors = linkSelectors;
|
|
48
|
-
exports.mimeTypes = mimeTypes;
|
|
49
|
-
exports.urisBalanced = urisBalanced;
|
|
50
|
-
exports.urisComprehensive = urisComprehensive;
|
|
51
|
-
exports.urisMinimal = urisMinimal;
|