feedscout 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/blogrolls/defaults.cjs +16 -10
- package/dist/blogrolls/defaults.d.cts +1 -2
- package/dist/blogrolls/defaults.d.ts +1 -2
- package/dist/blogrolls/defaults.js +17 -10
- package/dist/blogrolls/extractors.cjs +2 -2
- package/dist/blogrolls/extractors.d.cts +2 -2
- package/dist/blogrolls/extractors.d.ts +2 -2
- package/dist/blogrolls/extractors.js +2 -2
- package/dist/blogrolls/index.cjs +9 -10
- package/dist/blogrolls/index.js +4 -5
- package/dist/blogrolls.cjs +1 -2
- package/dist/blogrolls.d.cts +3 -3
- package/dist/blogrolls.d.ts +3 -3
- package/dist/blogrolls.js +3 -3
- package/dist/common/discover/defaults.cjs +46 -0
- package/dist/common/discover/defaults.d.cts +6 -0
- package/dist/common/discover/defaults.d.ts +6 -0
- package/dist/common/discover/defaults.js +44 -0
- package/dist/common/discover/index.cjs +7 -7
- package/dist/common/discover/index.js +2 -2
- package/dist/common/discover/utils.cjs +0 -29
- package/dist/common/discover/utils.d.cts +1 -3
- package/dist/common/discover/utils.d.ts +1 -3
- package/dist/common/discover/utils.js +1 -28
- package/dist/common/locales.cjs +214 -10
- package/dist/common/locales.js +214 -10
- package/dist/common/types.d.cts +3 -2
- package/dist/common/types.d.ts +3 -2
- package/dist/common/uris/html/handlers.cjs +14 -0
- package/dist/common/uris/html/handlers.js +14 -0
- package/dist/common/uris/html/index.cjs +17 -1
- package/dist/common/uris/html/index.js +17 -1
- package/dist/common/uris/html/types.d.cts +5 -4
- package/dist/common/uris/html/types.d.ts +5 -4
- package/dist/common/utils.cjs +12 -11
- package/dist/common/utils.d.cts +6 -4
- package/dist/common/utils.d.ts +6 -4
- package/dist/common/utils.js +13 -11
- package/dist/favicons/extractors.cjs +2 -2
- package/dist/favicons/extractors.d.cts +2 -2
- package/dist/favicons/extractors.d.ts +2 -2
- package/dist/favicons/extractors.js +2 -2
- package/dist/favicons/index.cjs +11 -12
- package/dist/favicons/index.js +4 -5
- package/dist/favicons.cjs +1 -1
- package/dist/favicons.d.cts +2 -2
- package/dist/favicons.d.ts +2 -2
- package/dist/favicons.js +2 -2
- package/dist/feeds/defaults.cjs +121 -9
- package/dist/feeds/defaults.d.cts +4 -3
- package/dist/feeds/defaults.d.ts +4 -3
- package/dist/feeds/defaults.js +121 -10
- package/dist/feeds/extractors.cjs +6 -6
- package/dist/feeds/extractors.d.cts +2 -2
- package/dist/feeds/extractors.d.ts +2 -2
- package/dist/feeds/extractors.js +4 -4
- package/dist/feeds/index.cjs +9 -10
- package/dist/feeds/index.js +4 -5
- package/dist/feeds/platform/handlers/acast.cjs +26 -0
- package/dist/feeds/platform/handlers/acast.js +26 -0
- package/dist/feeds/platform/handlers/ameblo.cjs +36 -0
- package/dist/feeds/platform/handlers/ameblo.js +36 -0
- package/dist/feeds/platform/handlers/applePodcasts.cjs +1 -1
- package/dist/feeds/platform/handlers/applePodcasts.js +1 -1
- package/dist/feeds/platform/handlers/arena.cjs +42 -0
- package/dist/feeds/platform/handlers/arena.js +42 -0
- package/dist/feeds/platform/handlers/artstation.cjs +52 -0
- package/dist/feeds/platform/handlers/artstation.js +52 -0
- package/dist/feeds/platform/handlers/audioboom.cjs +23 -0
- package/dist/feeds/platform/handlers/audioboom.js +23 -0
- package/dist/feeds/platform/handlers/bearblog.cjs +45 -0
- package/dist/feeds/platform/handlers/bearblog.js +45 -0
- package/dist/feeds/platform/handlers/behance.cjs +7 -0
- package/dist/feeds/platform/handlers/behance.js +7 -0
- package/dist/feeds/platform/handlers/blogspot.cjs +38 -2
- package/dist/feeds/platform/handlers/blogspot.js +38 -2
- package/dist/feeds/platform/handlers/bookwyrm.cjs +48 -0
- package/dist/feeds/platform/handlers/bookwyrm.js +48 -0
- package/dist/feeds/platform/handlers/buttondown.cjs +43 -0
- package/dist/feeds/platform/handlers/buttondown.js +43 -0
- package/dist/feeds/platform/handlers/buzzsprout.cjs +22 -0
- package/dist/feeds/platform/handlers/buzzsprout.js +22 -0
- package/dist/feeds/platform/handlers/codeberg.cjs +5 -5
- package/dist/feeds/platform/handlers/codeberg.js +5 -5
- package/dist/feeds/platform/handlers/dailymotion.cjs +16 -0
- package/dist/feeds/platform/handlers/dailymotion.js +16 -0
- package/dist/feeds/platform/handlers/deviantart.cjs +25 -6
- package/dist/feeds/platform/handlers/deviantart.js +25 -6
- package/dist/feeds/platform/handlers/devto.cjs +8 -0
- package/dist/feeds/platform/handlers/devto.js +8 -0
- package/dist/feeds/platform/handlers/discourse.cjs +70 -0
- package/dist/feeds/platform/handlers/discourse.js +70 -0
- package/dist/feeds/platform/handlers/dreamwidth.cjs +48 -0
- package/dist/feeds/platform/handlers/dreamwidth.js +48 -0
- package/dist/feeds/platform/handlers/exblog.cjs +35 -0
- package/dist/feeds/platform/handlers/exblog.js +35 -0
- package/dist/feeds/platform/handlers/fireside.cjs +24 -0
- package/dist/feeds/platform/handlers/fireside.js +24 -0
- package/dist/feeds/platform/handlers/friendica.cjs +44 -0
- package/dist/feeds/platform/handlers/friendica.js +44 -0
- package/dist/feeds/platform/handlers/ghost.cjs +30 -0
- package/dist/feeds/platform/handlers/ghost.js +30 -0
- package/dist/feeds/platform/handlers/github.cjs +6 -0
- package/dist/feeds/platform/handlers/github.js +6 -0
- package/dist/feeds/platform/handlers/githubGist.cjs +16 -5
- package/dist/feeds/platform/handlers/githubGist.js +16 -5
- package/dist/feeds/platform/handlers/gitlab.cjs +31 -13
- package/dist/feeds/platform/handlers/gitlab.js +31 -13
- package/dist/feeds/platform/handlers/goodreads.cjs +18 -8
- package/dist/feeds/platform/handlers/goodreads.js +18 -8
- package/dist/feeds/platform/handlers/hackernews.cjs +21 -0
- package/dist/feeds/platform/handlers/hackernews.js +21 -0
- package/dist/feeds/platform/handlers/hashnode.cjs +1 -1
- package/dist/feeds/platform/handlers/hashnode.js +1 -1
- package/dist/feeds/platform/handlers/hatenablog.cjs +4 -1
- package/dist/feeds/platform/handlers/hatenablog.js +4 -1
- package/dist/feeds/platform/handlers/hearthis.cjs +32 -0
- package/dist/feeds/platform/handlers/hearthis.js +32 -0
- package/dist/feeds/platform/handlers/heyWorld.cjs +18 -0
- package/dist/feeds/platform/handlers/heyWorld.js +18 -0
- package/dist/feeds/platform/handlers/insanejournal.cjs +69 -0
- package/dist/feeds/platform/handlers/insanejournal.js +69 -0
- package/dist/feeds/platform/handlers/itchio.cjs +24 -1
- package/dist/feeds/platform/handlers/itchio.js +24 -1
- package/dist/feeds/platform/handlers/lemmy.cjs +46 -4
- package/dist/feeds/platform/handlers/lemmy.js +46 -4
- package/dist/feeds/platform/handlers/letterboxd.cjs +4 -0
- package/dist/feeds/platform/handlers/letterboxd.js +4 -0
- package/dist/feeds/platform/handlers/libsyn.cjs +25 -0
- package/dist/feeds/platform/handlers/libsyn.js +25 -0
- package/dist/feeds/platform/handlers/listed.cjs +20 -0
- package/dist/feeds/platform/handlers/listed.js +20 -0
- package/dist/feeds/platform/handlers/livejournal.cjs +68 -0
- package/dist/feeds/platform/handlers/livejournal.js +68 -0
- package/dist/feeds/platform/handlers/mastodon.cjs +32 -0
- package/dist/feeds/platform/handlers/mastodon.js +32 -0
- package/dist/feeds/platform/handlers/mataroa.cjs +16 -0
- package/dist/feeds/platform/handlers/mataroa.js +16 -0
- package/dist/feeds/platform/handlers/medium.cjs +2 -2
- package/dist/feeds/platform/handlers/medium.js +2 -2
- package/dist/feeds/platform/handlers/microblog.cjs +55 -0
- package/dist/feeds/platform/handlers/microblog.js +55 -0
- package/dist/feeds/platform/handlers/misskey.cjs +40 -0
- package/dist/feeds/platform/handlers/misskey.js +40 -0
- package/dist/feeds/platform/handlers/myanimelist.cjs +43 -0
- package/dist/feeds/platform/handlers/myanimelist.js +43 -0
- package/dist/feeds/platform/handlers/naverBlog.cjs +21 -0
- package/dist/feeds/platform/handlers/naverBlog.js +21 -0
- package/dist/feeds/platform/handlers/nebula.cjs +68 -0
- package/dist/feeds/platform/handlers/nebula.js +68 -0
- package/dist/feeds/platform/handlers/note.cjs +53 -0
- package/dist/feeds/platform/handlers/note.js +53 -0
- package/dist/feeds/platform/handlers/observable.cjs +34 -0
- package/dist/feeds/platform/handlers/observable.js +34 -0
- package/dist/feeds/platform/handlers/odysee.cjs +20 -0
- package/dist/feeds/platform/handlers/odysee.js +20 -0
- package/dist/feeds/platform/handlers/pagecord.cjs +16 -0
- package/dist/feeds/platform/handlers/pagecord.js +16 -0
- package/dist/feeds/platform/handlers/paragraph.cjs +1 -2
- package/dist/feeds/platform/handlers/paragraph.js +1 -2
- package/dist/feeds/platform/handlers/pika.cjs +35 -0
- package/dist/feeds/platform/handlers/pika.js +35 -0
- package/dist/feeds/platform/handlers/pinterest.cjs +13 -0
- package/dist/feeds/platform/handlers/pinterest.js +13 -0
- package/dist/feeds/platform/handlers/pixelfed.cjs +46 -0
- package/dist/feeds/platform/handlers/pixelfed.js +46 -0
- package/dist/feeds/platform/handlers/pleroma.cjs +34 -0
- package/dist/feeds/platform/handlers/pleroma.js +34 -0
- package/dist/feeds/platform/handlers/podbean.cjs +29 -0
- package/dist/feeds/platform/handlers/podbean.js +29 -0
- package/dist/feeds/platform/handlers/podigee.cjs +29 -0
- package/dist/feeds/platform/handlers/podigee.js +29 -0
- package/dist/feeds/platform/handlers/posthaven.cjs +24 -0
- package/dist/feeds/platform/handlers/posthaven.js +24 -0
- package/dist/feeds/platform/handlers/prose.cjs +26 -0
- package/dist/feeds/platform/handlers/prose.js +26 -0
- package/dist/feeds/platform/handlers/qiita.cjs +58 -0
- package/dist/feeds/platform/handlers/qiita.js +58 -0
- package/dist/feeds/platform/handlers/reddit.cjs +83 -9
- package/dist/feeds/platform/handlers/reddit.js +83 -9
- package/dist/feeds/platform/handlers/rssCom.cjs +20 -0
- package/dist/feeds/platform/handlers/rssCom.js +20 -0
- package/dist/feeds/platform/handlers/seesaa.cjs +22 -0
- package/dist/feeds/platform/handlers/seesaa.js +22 -0
- package/dist/feeds/platform/handlers/sourceforge.cjs +37 -4
- package/dist/feeds/platform/handlers/sourceforge.js +37 -4
- package/dist/feeds/platform/handlers/spreaker.cjs +21 -0
- package/dist/feeds/platform/handlers/spreaker.js +21 -0
- package/dist/feeds/platform/handlers/stackExchange.cjs +26 -2
- package/dist/feeds/platform/handlers/stackExchange.js +26 -2
- package/dist/feeds/platform/handlers/steam.cjs +7 -0
- package/dist/feeds/platform/handlers/steam.js +7 -0
- package/dist/feeds/platform/handlers/tildes.cjs +41 -0
- package/dist/feeds/platform/handlers/tildes.js +41 -0
- package/dist/feeds/platform/handlers/tistory.cjs +16 -0
- package/dist/feeds/platform/handlers/tistory.js +16 -0
- package/dist/feeds/platform/handlers/transistor.cjs +29 -0
- package/dist/feeds/platform/handlers/transistor.js +29 -0
- package/dist/feeds/platform/handlers/velog.cjs +24 -0
- package/dist/feeds/platform/handlers/velog.js +24 -0
- package/dist/feeds/platform/handlers/vimeo.cjs +6 -0
- package/dist/feeds/platform/handlers/vimeo.js +6 -0
- package/dist/feeds/platform/handlers/weblogLol.cjs +26 -0
- package/dist/feeds/platform/handlers/weblogLol.js +26 -0
- package/dist/feeds/platform/handlers/weebly.cjs +25 -0
- package/dist/feeds/platform/handlers/weebly.js +25 -0
- package/dist/feeds/platform/handlers/wordpress.cjs +173 -28
- package/dist/feeds/platform/handlers/wordpress.js +173 -28
- package/dist/feeds/platform/handlers/writeas.cjs +51 -0
- package/dist/feeds/platform/handlers/writeas.js +51 -0
- package/dist/feeds/platform/handlers/ximalaya.cjs +1 -1
- package/dist/feeds/platform/handlers/ximalaya.js +1 -1
- package/dist/feeds/platform/handlers/youtube.cjs +4 -1
- package/dist/feeds/platform/handlers/youtube.js +4 -1
- package/dist/feeds/platform/handlers/zenn.cjs +54 -0
- package/dist/feeds/platform/handlers/zenn.js +54 -0
- package/dist/feeds.cjs +2 -1
- package/dist/feeds.d.cts +3 -3
- package/dist/feeds.d.ts +3 -3
- package/dist/feeds.js +3 -3
- package/dist/hubs/discover/index.cjs +4 -4
- package/dist/hubs/discover/index.js +3 -3
- package/dist/hubs/feed/index.cjs +2 -2
- package/dist/hubs/feed/index.js +2 -2
- package/dist/hubs/headers/index.cjs +2 -2
- package/dist/hubs/headers/index.js +2 -2
- package/dist/hubs/html/index.cjs +2 -2
- package/dist/hubs/html/index.js +2 -2
- package/dist/index.cjs +2 -1
- package/dist/index.d.cts +2 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/utils.d.cts +2 -1
- package/dist/utils.d.ts +2 -1
- package/package.json +6 -7
package/dist/common/types.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { PlatformMethodOptions } from "./uris/platform/types.js";
|
|
|
6
6
|
|
|
7
7
|
//#region src/common/types.d.ts
|
|
8
8
|
type MaybePromise<T> = T | Promise<T>;
|
|
9
|
+
type Pattern = string | RegExp;
|
|
9
10
|
type UriEntry = string | Array<string>;
|
|
10
11
|
type DiscoverUriHint = {
|
|
11
12
|
key: string;
|
|
@@ -22,7 +23,7 @@ type LinkSelector = {
|
|
|
22
23
|
types?: Array<string>;
|
|
23
24
|
};
|
|
24
25
|
type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string | undefined;
|
|
25
|
-
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
|
|
26
|
+
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject, resolveUrlFn: DiscoverResolveUrlFn) => string | undefined;
|
|
26
27
|
type DiscoverFetchFnOptions = {
|
|
27
28
|
method?: 'GET' | 'HEAD';
|
|
28
29
|
headers?: Record<string, string>;
|
|
@@ -86,4 +87,4 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
|
|
|
86
87
|
includeInvalid?: boolean;
|
|
87
88
|
};
|
|
88
89
|
//#endregion
|
|
89
|
-
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, MaybePromise, UriEntry };
|
|
90
|
+
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, MaybePromise, Pattern, UriEntry };
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
const require_utils = require("../../utils.cjs");
|
|
2
2
|
//#region src/common/uris/html/handlers.ts
|
|
3
3
|
const handleOpenTag = (context, name, attribs, _isImplied) => {
|
|
4
|
+
if (name === "base" && context.baseHref === void 0) {
|
|
5
|
+
const href = attribs.href?.trim();
|
|
6
|
+
if (href) context.baseHref = href;
|
|
7
|
+
}
|
|
4
8
|
if (name === "link" && attribs.href) {
|
|
5
9
|
const rel = attribs.rel?.toLowerCase();
|
|
6
10
|
if (!rel) return;
|
|
@@ -16,6 +20,16 @@ const handleOpenTag = (context, name, attribs, _isImplied) => {
|
|
|
16
20
|
context.currentAnchor.href = attribs.href;
|
|
17
21
|
context.currentAnchor.text = "";
|
|
18
22
|
if (require_utils.endsWithAnyOf(lowerHref, context.options.anchorUris)) context.discoveredUris.add(attribs.href);
|
|
23
|
+
if (context.options.anchorPathSegments?.length) {
|
|
24
|
+
let pathname;
|
|
25
|
+
try {
|
|
26
|
+
pathname = new URL(attribs.href, "https://feedscout.invalid").pathname;
|
|
27
|
+
} catch {}
|
|
28
|
+
if (pathname && require_utils.includesAnyOf(pathname, context.options.anchorPathSegments)) context.discoveredUris.add(attribs.href);
|
|
29
|
+
}
|
|
30
|
+
const ariaLabel = attribs["aria-label"];
|
|
31
|
+
const title = attribs.title;
|
|
32
|
+
if (ariaLabel && require_utils.includesAnyOf(ariaLabel, context.options.anchorLabels) || title && require_utils.includesAnyOf(title, context.options.anchorLabels)) context.discoveredUris.add(attribs.href);
|
|
19
33
|
}
|
|
20
34
|
};
|
|
21
35
|
const handleText = (context, text) => {
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { endsWithAnyOf, includesAnyOf, matchesAnyOfLinkSelectors } from "../../utils.js";
|
|
2
2
|
//#region src/common/uris/html/handlers.ts
|
|
3
3
|
const handleOpenTag = (context, name, attribs, _isImplied) => {
|
|
4
|
+
if (name === "base" && context.baseHref === void 0) {
|
|
5
|
+
const href = attribs.href?.trim();
|
|
6
|
+
if (href) context.baseHref = href;
|
|
7
|
+
}
|
|
4
8
|
if (name === "link" && attribs.href) {
|
|
5
9
|
const rel = attribs.rel?.toLowerCase();
|
|
6
10
|
if (!rel) return;
|
|
@@ -16,6 +20,16 @@ const handleOpenTag = (context, name, attribs, _isImplied) => {
|
|
|
16
20
|
context.currentAnchor.href = attribs.href;
|
|
17
21
|
context.currentAnchor.text = "";
|
|
18
22
|
if (endsWithAnyOf(lowerHref, context.options.anchorUris)) context.discoveredUris.add(attribs.href);
|
|
23
|
+
if (context.options.anchorPathSegments?.length) {
|
|
24
|
+
let pathname;
|
|
25
|
+
try {
|
|
26
|
+
pathname = new URL(attribs.href, "https://feedscout.invalid").pathname;
|
|
27
|
+
} catch {}
|
|
28
|
+
if (pathname && includesAnyOf(pathname, context.options.anchorPathSegments)) context.discoveredUris.add(attribs.href);
|
|
29
|
+
}
|
|
30
|
+
const ariaLabel = attribs["aria-label"];
|
|
31
|
+
const title = attribs.title;
|
|
32
|
+
if (ariaLabel && includesAnyOf(ariaLabel, context.options.anchorLabels) || title && includesAnyOf(title, context.options.anchorLabels)) context.discoveredUris.add(attribs.href);
|
|
19
33
|
}
|
|
20
34
|
};
|
|
21
35
|
const handleText = (context, text) => {
|
|
@@ -13,7 +13,23 @@ const discoverUrisFromHtml = (html, options) => {
|
|
|
13
13
|
const parser = new htmlparser2.Parser(require_handlers.createHtmlUrisHandlers(context), { decodeEntities: true });
|
|
14
14
|
parser.write(html);
|
|
15
15
|
parser.end();
|
|
16
|
-
|
|
16
|
+
const uris = [...context.discoveredUris];
|
|
17
|
+
if (context.baseHref) {
|
|
18
|
+
let base;
|
|
19
|
+
try {
|
|
20
|
+
base = options.baseUrl ? new URL(context.baseHref, options.baseUrl).href : context.baseHref;
|
|
21
|
+
} catch {
|
|
22
|
+
base = options.baseUrl;
|
|
23
|
+
}
|
|
24
|
+
return uris.map((uri) => {
|
|
25
|
+
try {
|
|
26
|
+
return new URL(uri, base).href;
|
|
27
|
+
} catch {
|
|
28
|
+
return uri;
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return uris;
|
|
17
33
|
};
|
|
18
34
|
//#endregion
|
|
19
35
|
exports.discoverUrisFromHtml = discoverUrisFromHtml;
|
|
@@ -13,7 +13,23 @@ const discoverUrisFromHtml = (html, options) => {
|
|
|
13
13
|
const parser = new Parser(createHtmlUrisHandlers(context), { decodeEntities: true });
|
|
14
14
|
parser.write(html);
|
|
15
15
|
parser.end();
|
|
16
|
-
|
|
16
|
+
const uris = [...context.discoveredUris];
|
|
17
|
+
if (context.baseHref) {
|
|
18
|
+
let base;
|
|
19
|
+
try {
|
|
20
|
+
base = options.baseUrl ? new URL(context.baseHref, options.baseUrl).href : context.baseHref;
|
|
21
|
+
} catch {
|
|
22
|
+
base = options.baseUrl;
|
|
23
|
+
}
|
|
24
|
+
return uris.map((uri) => {
|
|
25
|
+
try {
|
|
26
|
+
return new URL(uri, base).href;
|
|
27
|
+
} catch {
|
|
28
|
+
return uri;
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return uris;
|
|
17
33
|
};
|
|
18
34
|
//#endregion
|
|
19
35
|
export { discoverUrisFromHtml };
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import { LinkSelector } from "../../types.cjs";
|
|
1
|
+
import { LinkSelector, Pattern } from "../../types.cjs";
|
|
2
2
|
|
|
3
3
|
//#region src/common/uris/html/types.d.ts
|
|
4
4
|
type HtmlMethodOptions = {
|
|
5
5
|
baseUrl?: string;
|
|
6
6
|
linkSelectors: Array<LinkSelector>;
|
|
7
|
-
anchorUris: Array<
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
anchorUris: Array<Pattern>;
|
|
8
|
+
anchorPathSegments?: Array<Pattern>;
|
|
9
|
+
anchorIgnoredUris: Array<Pattern>;
|
|
10
|
+
anchorLabels: Array<Pattern>;
|
|
10
11
|
};
|
|
11
12
|
//#endregion
|
|
12
13
|
export { HtmlMethodOptions };
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import { LinkSelector } from "../../types.js";
|
|
1
|
+
import { LinkSelector, Pattern } from "../../types.js";
|
|
2
2
|
|
|
3
3
|
//#region src/common/uris/html/types.d.ts
|
|
4
4
|
type HtmlMethodOptions = {
|
|
5
5
|
baseUrl?: string;
|
|
6
6
|
linkSelectors: Array<LinkSelector>;
|
|
7
|
-
anchorUris: Array<
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
anchorUris: Array<Pattern>;
|
|
8
|
+
anchorPathSegments?: Array<Pattern>;
|
|
9
|
+
anchorIgnoredUris: Array<Pattern>;
|
|
10
|
+
anchorLabels: Array<Pattern>;
|
|
10
11
|
};
|
|
11
12
|
//#endregion
|
|
12
13
|
export { HtmlMethodOptions };
|
package/dist/common/utils.cjs
CHANGED
|
@@ -24,18 +24,27 @@ const isHostOf = (url, hosts) => {
|
|
|
24
24
|
};
|
|
25
25
|
const includesAnyOf = (value, patterns, parser) => {
|
|
26
26
|
const parsedValue = parser ? parser(value) : value?.toLowerCase();
|
|
27
|
-
return patterns.some((pattern) =>
|
|
27
|
+
return patterns.some((pattern) => {
|
|
28
|
+
if (pattern instanceof RegExp) return pattern.test(parsedValue);
|
|
29
|
+
return pattern && parsedValue?.includes(pattern.toLowerCase());
|
|
30
|
+
});
|
|
28
31
|
};
|
|
29
32
|
const isAnyOf = (value, patterns, parser) => {
|
|
30
33
|
const parsedValue = parser ? parser(value) : value?.toLowerCase()?.trim();
|
|
31
|
-
return patterns.some((pattern) =>
|
|
34
|
+
return patterns.some((pattern) => {
|
|
35
|
+
if (pattern instanceof RegExp) return pattern.test(parsedValue);
|
|
36
|
+
return parsedValue === pattern.toLowerCase().trim();
|
|
37
|
+
});
|
|
32
38
|
};
|
|
33
39
|
const anyWordMatchesAnyOf = (value, patterns) => {
|
|
34
40
|
return value.toLowerCase().split(whitespaceRegex).some((word) => isAnyOf(word, patterns));
|
|
35
41
|
};
|
|
36
42
|
const endsWithAnyOf = (value, patterns) => {
|
|
37
43
|
const lowerValue = value.toLowerCase();
|
|
38
|
-
return patterns.some((pattern) =>
|
|
44
|
+
return patterns.some((pattern) => {
|
|
45
|
+
if (pattern instanceof RegExp) return pattern.test(lowerValue);
|
|
46
|
+
return pattern && lowerValue.endsWith(pattern.toLowerCase());
|
|
47
|
+
});
|
|
39
48
|
};
|
|
40
49
|
const isOfAllowedMimeType = (type, allowedTypes) => {
|
|
41
50
|
if (allowedTypes.length === 0) return true;
|
|
@@ -52,13 +61,6 @@ const omitEmpty = (array) => {
|
|
|
52
61
|
for (const item of array) if (item != null && item !== "") result.push(item);
|
|
53
62
|
return result;
|
|
54
63
|
};
|
|
55
|
-
const resolveUrl = (url, baseUrl) => {
|
|
56
|
-
try {
|
|
57
|
-
return new URL(url, baseUrl).href;
|
|
58
|
-
} catch {
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
64
|
const matchesAnyOfLinkSelectors = (rel, type, selectors) => {
|
|
63
65
|
return selectors.some((selector) => {
|
|
64
66
|
if (!anyWordMatchesAnyOf(rel, [selector.rel])) return false;
|
|
@@ -94,4 +96,3 @@ exports.isSubdomainOf = isSubdomainOf;
|
|
|
94
96
|
exports.matchesAnyOfLinkSelectors = matchesAnyOfLinkSelectors;
|
|
95
97
|
exports.omitEmpty = omitEmpty;
|
|
96
98
|
exports.processConcurrently = processConcurrently;
|
|
97
|
-
exports.resolveUrl = resolveUrl;
|
package/dist/common/utils.d.cts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import { Pattern } from "./types.cjs";
|
|
2
|
+
|
|
1
3
|
//#region src/common/utils.d.ts
|
|
2
4
|
declare const isSubdomainOf: (url: string, domains: string | Array<string>) => boolean;
|
|
3
5
|
declare const isHostOf: (url: string, hosts: string | Array<string>) => boolean;
|
|
4
|
-
declare const includesAnyOf: (value: string, patterns: Array<
|
|
5
|
-
declare const isAnyOf: (value: string, patterns: Array<
|
|
6
|
-
declare const anyWordMatchesAnyOf: (value: string, patterns: Array<
|
|
7
|
-
declare const endsWithAnyOf: (value: string, patterns: Array<
|
|
6
|
+
declare const includesAnyOf: (value: string, patterns: Array<Pattern>, parser?: (value: string) => string) => boolean;
|
|
7
|
+
declare const isAnyOf: (value: string, patterns: Array<Pattern>, parser?: (value: string) => string) => boolean;
|
|
8
|
+
declare const anyWordMatchesAnyOf: (value: string, patterns: Array<Pattern>) => boolean;
|
|
9
|
+
declare const endsWithAnyOf: (value: string, patterns: Array<Pattern>) => boolean;
|
|
8
10
|
declare const omitEmpty: <T>(array: Array<T | null | undefined>) => Array<T>;
|
|
9
11
|
//#endregion
|
|
10
12
|
export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, omitEmpty };
|
package/dist/common/utils.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import { Pattern } from "./types.js";
|
|
2
|
+
|
|
1
3
|
//#region src/common/utils.d.ts
|
|
2
4
|
declare const isSubdomainOf: (url: string, domains: string | Array<string>) => boolean;
|
|
3
5
|
declare const isHostOf: (url: string, hosts: string | Array<string>) => boolean;
|
|
4
|
-
declare const includesAnyOf: (value: string, patterns: Array<
|
|
5
|
-
declare const isAnyOf: (value: string, patterns: Array<
|
|
6
|
-
declare const anyWordMatchesAnyOf: (value: string, patterns: Array<
|
|
7
|
-
declare const endsWithAnyOf: (value: string, patterns: Array<
|
|
6
|
+
declare const includesAnyOf: (value: string, patterns: Array<Pattern>, parser?: (value: string) => string) => boolean;
|
|
7
|
+
declare const isAnyOf: (value: string, patterns: Array<Pattern>, parser?: (value: string) => string) => boolean;
|
|
8
|
+
declare const anyWordMatchesAnyOf: (value: string, patterns: Array<Pattern>) => boolean;
|
|
9
|
+
declare const endsWithAnyOf: (value: string, patterns: Array<Pattern>) => boolean;
|
|
8
10
|
declare const omitEmpty: <T>(array: Array<T | null | undefined>) => Array<T>;
|
|
9
11
|
//#endregion
|
|
10
12
|
export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, omitEmpty };
|
package/dist/common/utils.js
CHANGED
|
@@ -24,18 +24,27 @@ const isHostOf = (url, hosts) => {
|
|
|
24
24
|
};
|
|
25
25
|
const includesAnyOf = (value, patterns, parser) => {
|
|
26
26
|
const parsedValue = parser ? parser(value) : value?.toLowerCase();
|
|
27
|
-
return patterns.some((pattern) =>
|
|
27
|
+
return patterns.some((pattern) => {
|
|
28
|
+
if (pattern instanceof RegExp) return pattern.test(parsedValue);
|
|
29
|
+
return pattern && parsedValue?.includes(pattern.toLowerCase());
|
|
30
|
+
});
|
|
28
31
|
};
|
|
29
32
|
const isAnyOf = (value, patterns, parser) => {
|
|
30
33
|
const parsedValue = parser ? parser(value) : value?.toLowerCase()?.trim();
|
|
31
|
-
return patterns.some((pattern) =>
|
|
34
|
+
return patterns.some((pattern) => {
|
|
35
|
+
if (pattern instanceof RegExp) return pattern.test(parsedValue);
|
|
36
|
+
return parsedValue === pattern.toLowerCase().trim();
|
|
37
|
+
});
|
|
32
38
|
};
|
|
33
39
|
const anyWordMatchesAnyOf = (value, patterns) => {
|
|
34
40
|
return value.toLowerCase().split(whitespaceRegex).some((word) => isAnyOf(word, patterns));
|
|
35
41
|
};
|
|
36
42
|
const endsWithAnyOf = (value, patterns) => {
|
|
37
43
|
const lowerValue = value.toLowerCase();
|
|
38
|
-
return patterns.some((pattern) =>
|
|
44
|
+
return patterns.some((pattern) => {
|
|
45
|
+
if (pattern instanceof RegExp) return pattern.test(lowerValue);
|
|
46
|
+
return pattern && lowerValue.endsWith(pattern.toLowerCase());
|
|
47
|
+
});
|
|
39
48
|
};
|
|
40
49
|
const isOfAllowedMimeType = (type, allowedTypes) => {
|
|
41
50
|
if (allowedTypes.length === 0) return true;
|
|
@@ -52,13 +61,6 @@ const omitEmpty = (array) => {
|
|
|
52
61
|
for (const item of array) if (item != null && item !== "") result.push(item);
|
|
53
62
|
return result;
|
|
54
63
|
};
|
|
55
|
-
const resolveUrl = (url, baseUrl) => {
|
|
56
|
-
try {
|
|
57
|
-
return new URL(url, baseUrl).href;
|
|
58
|
-
} catch {
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
64
|
const matchesAnyOfLinkSelectors = (rel, type, selectors) => {
|
|
63
65
|
return selectors.some((selector) => {
|
|
64
66
|
if (!anyWordMatchesAnyOf(rel, [selector.rel])) return false;
|
|
@@ -83,4 +85,4 @@ const processConcurrently = async (items, processFn, options) => {
|
|
|
83
85
|
}
|
|
84
86
|
};
|
|
85
87
|
//#endregion
|
|
86
|
-
export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently
|
|
88
|
+
export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently };
|
|
@@ -11,7 +11,7 @@ const isImageContent = (content) => {
|
|
|
11
11
|
const isSuccessStatus = (status) => {
|
|
12
12
|
return status !== void 0 && status >= 200 && status < 400;
|
|
13
13
|
};
|
|
14
|
-
const
|
|
14
|
+
const defaultExtractFn = (input) => {
|
|
15
15
|
if (isImageContentType(input.headers) || isImageContent(input.content) || isSuccessStatus(input.status)) return {
|
|
16
16
|
url: input.url,
|
|
17
17
|
isValid: true
|
|
@@ -22,4 +22,4 @@ const defaultExtractor = (input) => {
|
|
|
22
22
|
};
|
|
23
23
|
};
|
|
24
24
|
//#endregion
|
|
25
|
-
exports.
|
|
25
|
+
exports.defaultExtractFn = defaultExtractFn;
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.cjs";
|
|
|
2
2
|
import { FaviconResult } from "./types.cjs";
|
|
3
3
|
|
|
4
4
|
//#region src/favicons/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<FaviconResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.js";
|
|
|
2
2
|
import { FaviconResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/favicons/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<FaviconResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -11,7 +11,7 @@ const isImageContent = (content) => {
|
|
|
11
11
|
const isSuccessStatus = (status) => {
|
|
12
12
|
return status !== void 0 && status >= 200 && status < 400;
|
|
13
13
|
};
|
|
14
|
-
const
|
|
14
|
+
const defaultExtractFn = (input) => {
|
|
15
15
|
if (isImageContentType(input.headers) || isImageContent(input.content) || isSuccessStatus(input.status)) return {
|
|
16
16
|
url: input.url,
|
|
17
17
|
isValid: true
|
|
@@ -22,4 +22,4 @@ const defaultExtractor = (input) => {
|
|
|
22
22
|
};
|
|
23
23
|
};
|
|
24
24
|
//#endregion
|
|
25
|
-
export {
|
|
25
|
+
export { defaultExtractFn };
|
package/dist/favicons/index.cjs
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
const require_utils$1 = require("../common/discover/utils.cjs");
|
|
1
|
+
const require_defaults = require("../common/discover/defaults.cjs");
|
|
3
2
|
const require_index = require("../common/discover/index.cjs");
|
|
4
|
-
const require_defaults = require("./defaults.cjs");
|
|
3
|
+
const require_defaults$1 = require("./defaults.cjs");
|
|
5
4
|
const require_extractors = require("./extractors.cjs");
|
|
6
5
|
//#region src/favicons/index.ts
|
|
7
6
|
const discoverFavicons = (input, options = {}) => {
|
|
@@ -14,16 +13,16 @@ const discoverFavicons = (input, options = {}) => {
|
|
|
14
13
|
"headers",
|
|
15
14
|
"guess"
|
|
16
15
|
],
|
|
17
|
-
fetchFn: options.fetchFn ??
|
|
18
|
-
extractFn: options.extractFn ?? require_extractors.
|
|
19
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
20
|
-
resolveSiteUrlFn: options.resolveSiteUrlFn ??
|
|
16
|
+
fetchFn: options.fetchFn ?? require_defaults.defaultFetchFn,
|
|
17
|
+
extractFn: options.extractFn ?? require_extractors.defaultExtractFn,
|
|
18
|
+
resolveUrlFn: options.resolveUrlFn ?? require_defaults.defaultResolveUrlFn,
|
|
19
|
+
resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_defaults.defaultResolveSiteUrlFn
|
|
21
20
|
}, {
|
|
22
|
-
platform: require_defaults.defaultPlatformOptions,
|
|
23
|
-
feed: require_defaults.defaultFeedOptions,
|
|
24
|
-
html: require_defaults.defaultHtmlOptions,
|
|
25
|
-
headers: require_defaults.defaultHeadersOptions,
|
|
26
|
-
guess: require_defaults.defaultGuessOptions
|
|
21
|
+
platform: require_defaults$1.defaultPlatformOptions,
|
|
22
|
+
feed: require_defaults$1.defaultFeedOptions,
|
|
23
|
+
html: require_defaults$1.defaultHtmlOptions,
|
|
24
|
+
headers: require_defaults$1.defaultHeadersOptions,
|
|
25
|
+
guess: require_defaults$1.defaultGuessOptions
|
|
27
26
|
});
|
|
28
27
|
};
|
|
29
28
|
//#endregion
|
package/dist/favicons/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn, defaultResolveSiteUrlFn } from "../common/discover/utils.js";
|
|
1
|
+
import { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
2
|
import { discover } from "../common/discover/index.js";
|
|
4
3
|
import { defaultFeedOptions, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions } from "./defaults.js";
|
|
5
|
-
import {
|
|
4
|
+
import { defaultExtractFn } from "./extractors.js";
|
|
6
5
|
//#region src/favicons/index.ts
|
|
7
6
|
const discoverFavicons = (input, options = {}) => {
|
|
8
7
|
return discover(input, {
|
|
@@ -15,8 +14,8 @@ const discoverFavicons = (input, options = {}) => {
|
|
|
15
14
|
"guess"
|
|
16
15
|
],
|
|
17
16
|
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
18
|
-
extractFn: options.extractFn ??
|
|
19
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
17
|
+
extractFn: options.extractFn ?? defaultExtractFn,
|
|
18
|
+
resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn,
|
|
20
19
|
resolveSiteUrlFn: options.resolveSiteUrlFn ?? defaultResolveSiteUrlFn
|
|
21
20
|
}, {
|
|
22
21
|
platform: defaultPlatformOptions,
|
package/dist/favicons.cjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_defaults = require("./favicons/defaults.cjs");
|
|
3
3
|
const require_extractors = require("./favicons/extractors.cjs");
|
|
4
|
-
exports.
|
|
4
|
+
exports.defaultExtractFn = require_extractors.defaultExtractFn;
|
|
5
5
|
exports.defaultFeedOptions = require_defaults.defaultFeedOptions;
|
|
6
6
|
exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
|
|
7
7
|
exports.defaultGuessPaths = require_defaults.defaultGuessPaths;
|
package/dist/favicons.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors } from "./favicons/defaults.cjs";
|
|
2
2
|
import { FaviconResult } from "./favicons/types.cjs";
|
|
3
|
-
import {
|
|
4
|
-
export { FaviconResult,
|
|
3
|
+
import { defaultExtractFn } from "./favicons/extractors.cjs";
|
|
4
|
+
export { FaviconResult, defaultExtractFn, defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors };
|
package/dist/favicons.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors } from "./favicons/defaults.js";
|
|
2
2
|
import { FaviconResult } from "./favicons/types.js";
|
|
3
|
-
import {
|
|
4
|
-
export { FaviconResult,
|
|
3
|
+
import { defaultExtractFn } from "./favicons/extractors.js";
|
|
4
|
+
export { FaviconResult, defaultExtractFn, defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors };
|
package/dist/favicons.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors } from "./favicons/defaults.js";
|
|
2
|
-
import {
|
|
3
|
-
export {
|
|
2
|
+
import { defaultExtractFn } from "./favicons/extractors.js";
|
|
3
|
+
export { defaultExtractFn, defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors };
|