feedscout 1.8.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blogrolls/extractors.cjs +2 -2
- package/dist/blogrolls/extractors.d.cts +2 -2
- package/dist/blogrolls/extractors.d.ts +2 -2
- package/dist/blogrolls/extractors.js +2 -2
- package/dist/blogrolls/index.cjs +9 -10
- package/dist/blogrolls/index.js +4 -5
- package/dist/blogrolls.cjs +1 -1
- package/dist/blogrolls.d.cts +2 -2
- package/dist/blogrolls.d.ts +2 -2
- package/dist/blogrolls.js +2 -2
- package/dist/common/discover/defaults.cjs +46 -0
- package/dist/common/discover/defaults.d.cts +6 -0
- package/dist/common/discover/defaults.d.ts +6 -0
- package/dist/common/discover/defaults.js +44 -0
- package/dist/common/discover/index.cjs +7 -7
- package/dist/common/discover/index.js +2 -2
- package/dist/common/discover/utils.cjs +0 -29
- package/dist/common/discover/utils.d.cts +1 -3
- package/dist/common/discover/utils.d.ts +1 -3
- package/dist/common/discover/utils.js +1 -28
- package/dist/common/types.d.cts +1 -1
- package/dist/common/types.d.ts +1 -1
- package/dist/common/utils.cjs +0 -8
- package/dist/common/utils.js +1 -8
- package/dist/favicons/extractors.cjs +2 -2
- package/dist/favicons/extractors.d.cts +2 -2
- package/dist/favicons/extractors.d.ts +2 -2
- package/dist/favicons/extractors.js +2 -2
- package/dist/favicons/index.cjs +11 -12
- package/dist/favicons/index.js +4 -5
- package/dist/favicons.cjs +1 -1
- package/dist/favicons.d.cts +2 -2
- package/dist/favicons.d.ts +2 -2
- package/dist/favicons.js +2 -2
- package/dist/feeds/extractors.cjs +6 -6
- package/dist/feeds/extractors.d.cts +2 -2
- package/dist/feeds/extractors.d.ts +2 -2
- package/dist/feeds/extractors.js +4 -4
- package/dist/feeds/index.cjs +9 -10
- package/dist/feeds/index.js +4 -5
- package/dist/feeds.cjs +1 -1
- package/dist/feeds.d.cts +2 -2
- package/dist/feeds.d.ts +2 -2
- package/dist/feeds.js +2 -2
- package/dist/hubs/discover/index.cjs +4 -4
- package/dist/hubs/discover/index.js +3 -3
- package/dist/hubs/feed/index.cjs +2 -2
- package/dist/hubs/feed/index.js +2 -2
- package/dist/hubs/headers/index.cjs +2 -2
- package/dist/hubs/headers/index.js +2 -2
- package/dist/hubs/html/index.cjs +2 -2
- package/dist/hubs/html/index.js +2 -2
- package/dist/index.cjs +2 -1
- package/dist/index.d.cts +2 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
let feedsmith = require("feedsmith");
|
|
2
2
|
//#region src/blogrolls/extractors.ts
|
|
3
|
-
const
|
|
3
|
+
const defaultExtractFn = ({ content, url }) => {
|
|
4
4
|
if (!content) return {
|
|
5
5
|
url,
|
|
6
6
|
isValid: false
|
|
@@ -18,4 +18,4 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
18
18
|
};
|
|
19
19
|
};
|
|
20
20
|
//#endregion
|
|
21
|
-
exports.
|
|
21
|
+
exports.defaultExtractFn = defaultExtractFn;
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.cjs";
|
|
|
2
2
|
import { BlogrollResult } from "./types.cjs";
|
|
3
3
|
|
|
4
4
|
//#region src/blogrolls/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<BlogrollResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.js";
|
|
|
2
2
|
import { BlogrollResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/blogrolls/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<BlogrollResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parseOpml } from "feedsmith";
|
|
2
2
|
//#region src/blogrolls/extractors.ts
|
|
3
|
-
const
|
|
3
|
+
const defaultExtractFn = ({ content, url }) => {
|
|
4
4
|
if (!content) return {
|
|
5
5
|
url,
|
|
6
6
|
isValid: false
|
|
@@ -18,4 +18,4 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
18
18
|
};
|
|
19
19
|
};
|
|
20
20
|
//#endregion
|
|
21
|
-
export {
|
|
21
|
+
export { defaultExtractFn };
|
package/dist/blogrolls/index.cjs
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
const require_utils$1 = require("../common/discover/utils.cjs");
|
|
1
|
+
const require_defaults = require("../common/discover/defaults.cjs");
|
|
3
2
|
const require_index = require("../common/discover/index.cjs");
|
|
4
|
-
const require_defaults = require("./defaults.cjs");
|
|
3
|
+
const require_defaults$1 = require("./defaults.cjs");
|
|
5
4
|
const require_extractors = require("./extractors.cjs");
|
|
6
5
|
//#region src/blogrolls/index.ts
|
|
7
6
|
const discoverBlogrolls = (input, options = {}) => {
|
|
@@ -12,14 +11,14 @@ const discoverBlogrolls = (input, options = {}) => {
|
|
|
12
11
|
"headers",
|
|
13
12
|
"guess"
|
|
14
13
|
],
|
|
15
|
-
fetchFn: options.fetchFn ??
|
|
16
|
-
extractFn: options.extractFn ?? require_extractors.
|
|
17
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
18
|
-
resolveSiteUrlFn: options.resolveSiteUrlFn ??
|
|
14
|
+
fetchFn: options.fetchFn ?? require_defaults.defaultFetchFn,
|
|
15
|
+
extractFn: options.extractFn ?? require_extractors.defaultExtractFn,
|
|
16
|
+
resolveUrlFn: options.resolveUrlFn ?? require_defaults.defaultResolveUrlFn,
|
|
17
|
+
resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_defaults.defaultResolveSiteUrlFn
|
|
19
18
|
}, {
|
|
20
|
-
html: require_defaults.defaultHtmlOptions,
|
|
21
|
-
headers: require_defaults.defaultHeadersOptions,
|
|
22
|
-
guess: require_defaults.defaultGuessOptions
|
|
19
|
+
html: require_defaults$1.defaultHtmlOptions,
|
|
20
|
+
headers: require_defaults$1.defaultHeadersOptions,
|
|
21
|
+
guess: require_defaults$1.defaultGuessOptions
|
|
23
22
|
});
|
|
24
23
|
};
|
|
25
24
|
//#endregion
|
package/dist/blogrolls/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn, defaultResolveSiteUrlFn } from "../common/discover/utils.js";
|
|
1
|
+
import { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
2
|
import { discover } from "../common/discover/index.js";
|
|
4
3
|
import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
|
|
5
|
-
import {
|
|
4
|
+
import { defaultExtractFn } from "./extractors.js";
|
|
6
5
|
//#region src/blogrolls/index.ts
|
|
7
6
|
const discoverBlogrolls = (input, options = {}) => {
|
|
8
7
|
return discover(input, {
|
|
@@ -13,8 +12,8 @@ const discoverBlogrolls = (input, options = {}) => {
|
|
|
13
12
|
"guess"
|
|
14
13
|
],
|
|
15
14
|
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
16
|
-
extractFn: options.extractFn ??
|
|
17
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
15
|
+
extractFn: options.extractFn ?? defaultExtractFn,
|
|
16
|
+
resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn,
|
|
18
17
|
resolveSiteUrlFn: options.resolveSiteUrlFn ?? defaultResolveSiteUrlFn
|
|
19
18
|
}, {
|
|
20
19
|
html: defaultHtmlOptions,
|
package/dist/blogrolls.cjs
CHANGED
|
@@ -2,7 +2,7 @@ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
|
2
2
|
const require_defaults = require("./blogrolls/defaults.cjs");
|
|
3
3
|
const require_extractors = require("./blogrolls/extractors.cjs");
|
|
4
4
|
exports.anchorLabels = require_defaults.anchorLabels;
|
|
5
|
-
exports.
|
|
5
|
+
exports.defaultExtractFn = require_extractors.defaultExtractFn;
|
|
6
6
|
exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
|
|
7
7
|
exports.defaultHeadersOptions = require_defaults.defaultHeadersOptions;
|
|
8
8
|
exports.defaultHtmlOptions = require_defaults.defaultHtmlOptions;
|
package/dist/blogrolls.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.cjs";
|
|
2
2
|
import { BlogrollResult } from "./blogrolls/types.cjs";
|
|
3
|
-
import {
|
|
4
|
-
export { BlogrollResult, anchorLabels,
|
|
3
|
+
import { defaultExtractFn } from "./blogrolls/extractors.cjs";
|
|
4
|
+
export { BlogrollResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
package/dist/blogrolls.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.js";
|
|
2
2
|
import { BlogrollResult } from "./blogrolls/types.js";
|
|
3
|
-
import {
|
|
4
|
-
export { BlogrollResult, anchorLabels,
|
|
3
|
+
import { defaultExtractFn } from "./blogrolls/extractors.js";
|
|
4
|
+
export { BlogrollResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
package/dist/blogrolls.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.js";
|
|
2
|
-
import {
|
|
3
|
-
export { anchorLabels,
|
|
2
|
+
import { defaultExtractFn } from "./blogrolls/extractors.js";
|
|
3
|
+
export { anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
const require_utils = require("./utils.cjs");
|
|
2
|
+
let feedsmith = require("feedsmith");
|
|
3
|
+
//#region src/common/discover/defaults.ts
|
|
4
|
+
const defaultFetchFn = async (url, options) => {
|
|
5
|
+
const response = await fetch(url, {
|
|
6
|
+
method: options?.method ?? "GET",
|
|
7
|
+
headers: options?.headers
|
|
8
|
+
});
|
|
9
|
+
return {
|
|
10
|
+
headers: response.headers,
|
|
11
|
+
body: await response.text(),
|
|
12
|
+
url: response.url,
|
|
13
|
+
status: response.status,
|
|
14
|
+
statusText: response.statusText
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
const defaultResolveUrlFn = (url, baseUrl) => {
|
|
18
|
+
try {
|
|
19
|
+
return new URL(url, baseUrl).href;
|
|
20
|
+
} catch {
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
const defaultResolveSiteUrlFn = (input, resolveUrlFn) => {
|
|
25
|
+
if (!input.content) return;
|
|
26
|
+
try {
|
|
27
|
+
let siteUrl = require_utils.getFeedSiteUrl((0, feedsmith.parseFeed)(input.content));
|
|
28
|
+
if (siteUrl) {
|
|
29
|
+
siteUrl = resolveUrlFn(siteUrl, input.url);
|
|
30
|
+
if (siteUrl) {
|
|
31
|
+
const parsed = new URL(siteUrl);
|
|
32
|
+
parsed.hash = "";
|
|
33
|
+
siteUrl = parsed.href;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (!siteUrl || siteUrl === new URL(input.url).href) try {
|
|
37
|
+
siteUrl = new URL(input.url).origin;
|
|
38
|
+
} catch {}
|
|
39
|
+
if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
|
|
40
|
+
return siteUrl;
|
|
41
|
+
} catch {}
|
|
42
|
+
};
|
|
43
|
+
//#endregion
|
|
44
|
+
exports.defaultFetchFn = defaultFetchFn;
|
|
45
|
+
exports.defaultResolveSiteUrlFn = defaultResolveSiteUrlFn;
|
|
46
|
+
exports.defaultResolveUrlFn = defaultResolveUrlFn;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { getFeedSiteUrl } from "./utils.js";
|
|
2
|
+
import { parseFeed } from "feedsmith";
|
|
3
|
+
//#region src/common/discover/defaults.ts
|
|
4
|
+
const defaultFetchFn = async (url, options) => {
|
|
5
|
+
const response = await fetch(url, {
|
|
6
|
+
method: options?.method ?? "GET",
|
|
7
|
+
headers: options?.headers
|
|
8
|
+
});
|
|
9
|
+
return {
|
|
10
|
+
headers: response.headers,
|
|
11
|
+
body: await response.text(),
|
|
12
|
+
url: response.url,
|
|
13
|
+
status: response.status,
|
|
14
|
+
statusText: response.statusText
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
const defaultResolveUrlFn = (url, baseUrl) => {
|
|
18
|
+
try {
|
|
19
|
+
return new URL(url, baseUrl).href;
|
|
20
|
+
} catch {
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
const defaultResolveSiteUrlFn = (input, resolveUrlFn) => {
|
|
25
|
+
if (!input.content) return;
|
|
26
|
+
try {
|
|
27
|
+
let siteUrl = getFeedSiteUrl(parseFeed(input.content));
|
|
28
|
+
if (siteUrl) {
|
|
29
|
+
siteUrl = resolveUrlFn(siteUrl, input.url);
|
|
30
|
+
if (siteUrl) {
|
|
31
|
+
const parsed = new URL(siteUrl);
|
|
32
|
+
parsed.hash = "";
|
|
33
|
+
siteUrl = parsed.href;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (!siteUrl || siteUrl === new URL(input.url).href) try {
|
|
37
|
+
siteUrl = new URL(input.url).origin;
|
|
38
|
+
} catch {}
|
|
39
|
+
if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
|
|
40
|
+
return siteUrl;
|
|
41
|
+
} catch {}
|
|
42
|
+
};
|
|
43
|
+
//#endregion
|
|
44
|
+
export { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn };
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
+
const require_utils = require("./utils.cjs");
|
|
1
2
|
const require_types = require("../types.cjs");
|
|
2
|
-
const require_utils = require("../utils.cjs");
|
|
3
|
+
const require_utils$1 = require("../utils.cjs");
|
|
3
4
|
const require_index = require("../uris/index.cjs");
|
|
4
|
-
const require_utils$1 = require("./utils.cjs");
|
|
5
5
|
//#region src/common/discover/index.ts
|
|
6
6
|
const discover = async (input, options, defaults) => {
|
|
7
7
|
const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
|
|
8
|
-
const sourceInput = await require_utils
|
|
8
|
+
const sourceInput = await require_utils.normalizeInput(input, fetchFn);
|
|
9
9
|
if (sourceInput.content) {
|
|
10
10
|
const result = await extractFn({
|
|
11
11
|
url: sourceInput.url,
|
|
@@ -16,7 +16,7 @@ const discover = async (input, options, defaults) => {
|
|
|
16
16
|
}
|
|
17
17
|
let siteInput;
|
|
18
18
|
if (resolveSiteUrlFn) {
|
|
19
|
-
const siteUrl = resolveSiteUrlFn(sourceInput);
|
|
19
|
+
const siteUrl = resolveSiteUrlFn(sourceInput, resolveUrlFn);
|
|
20
20
|
if (siteUrl) try {
|
|
21
21
|
const response = await fetchFn(siteUrl);
|
|
22
22
|
siteInput = {
|
|
@@ -26,14 +26,14 @@ const discover = async (input, options, defaults) => {
|
|
|
26
26
|
};
|
|
27
27
|
} catch {}
|
|
28
28
|
}
|
|
29
|
-
const urisByMethod = await require_index.discoverUris(require_utils
|
|
29
|
+
const urisByMethod = await require_index.discoverUris(require_utils.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
|
|
30
30
|
const seen = /* @__PURE__ */ new Set();
|
|
31
31
|
const methodGroups = [];
|
|
32
32
|
for (const method of require_types.discoverMethodOrder) {
|
|
33
33
|
const rawUris = urisByMethod[method];
|
|
34
34
|
if (!rawUris || rawUris.length === 0) continue;
|
|
35
35
|
const unique = rawUris.map((entry) => {
|
|
36
|
-
return require_utils
|
|
36
|
+
return require_utils.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
|
|
37
37
|
}).filter((entry) => {
|
|
38
38
|
const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
|
|
39
39
|
if (seen.has(key)) return false;
|
|
@@ -91,7 +91,7 @@ const discover = async (input, options, defaults) => {
|
|
|
91
91
|
};
|
|
92
92
|
for (const { method, entries } of methodGroups) {
|
|
93
93
|
const foundBefore = found;
|
|
94
|
-
await require_utils.processConcurrently(entries, (entry) => processUri(entry, method), {
|
|
94
|
+
await require_utils$1.processConcurrently(entries, (entry) => processUri(entry, method), {
|
|
95
95
|
concurrency,
|
|
96
96
|
shouldStop: () => {
|
|
97
97
|
return stopOnFirstResult && found > 0;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
|
|
1
2
|
import { discoverMethodOrder } from "../types.js";
|
|
2
3
|
import { processConcurrently } from "../utils.js";
|
|
3
4
|
import { discoverUris } from "../uris/index.js";
|
|
4
|
-
import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
|
|
5
5
|
//#region src/common/discover/index.ts
|
|
6
6
|
const discover = async (input, options, defaults) => {
|
|
7
7
|
const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
|
|
@@ -16,7 +16,7 @@ const discover = async (input, options, defaults) => {
|
|
|
16
16
|
}
|
|
17
17
|
let siteInput;
|
|
18
18
|
if (resolveSiteUrlFn) {
|
|
19
|
-
const siteUrl = resolveSiteUrlFn(sourceInput);
|
|
19
|
+
const siteUrl = resolveSiteUrlFn(sourceInput, resolveUrlFn);
|
|
20
20
|
if (siteUrl) try {
|
|
21
21
|
const response = await fetchFn(siteUrl);
|
|
22
22
|
siteInput = {
|
|
@@ -1,20 +1,5 @@
|
|
|
1
1
|
const require_locales = require("../locales.cjs");
|
|
2
|
-
const require_utils = require("../utils.cjs");
|
|
3
|
-
let feedsmith = require("feedsmith");
|
|
4
2
|
//#region src/common/discover/utils.ts
|
|
5
|
-
const defaultFetchFn = async (url, options) => {
|
|
6
|
-
const response = await fetch(url, {
|
|
7
|
-
method: options?.method ?? "GET",
|
|
8
|
-
headers: options?.headers
|
|
9
|
-
});
|
|
10
|
-
return {
|
|
11
|
-
headers: response.headers,
|
|
12
|
-
body: await response.text(),
|
|
13
|
-
url: response.url,
|
|
14
|
-
status: response.status,
|
|
15
|
-
statusText: response.statusText
|
|
16
|
-
};
|
|
17
|
-
};
|
|
18
3
|
const normalizeInput = async (input, fetchFn) => {
|
|
19
4
|
if (typeof input === "object") return input;
|
|
20
5
|
try {
|
|
@@ -36,18 +21,6 @@ const getFeedSiteUrl = (parsed) => {
|
|
|
36
21
|
if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
|
|
37
22
|
if (format === "json") return feed.home_page_url;
|
|
38
23
|
};
|
|
39
|
-
const defaultResolveSiteUrlFn = (input) => {
|
|
40
|
-
if (!input.content) return;
|
|
41
|
-
try {
|
|
42
|
-
let siteUrl = getFeedSiteUrl((0, feedsmith.parseFeed)(input.content));
|
|
43
|
-
if (!siteUrl) try {
|
|
44
|
-
siteUrl = new URL(input.url).origin;
|
|
45
|
-
} catch {}
|
|
46
|
-
else siteUrl = require_utils.resolveUrl(siteUrl, input.url);
|
|
47
|
-
if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
|
|
48
|
-
return siteUrl;
|
|
49
|
-
} catch {}
|
|
50
|
-
};
|
|
51
24
|
const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
|
|
52
25
|
if (typeof entry.uri === "string") return {
|
|
53
26
|
...entry,
|
|
@@ -122,8 +95,6 @@ const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
|
|
|
122
95
|
return methodsConfig;
|
|
123
96
|
};
|
|
124
97
|
//#endregion
|
|
125
|
-
exports.defaultFetchFn = defaultFetchFn;
|
|
126
|
-
exports.defaultResolveSiteUrlFn = defaultResolveSiteUrlFn;
|
|
127
98
|
exports.getFeedSiteUrl = getFeedSiteUrl;
|
|
128
99
|
exports.normalizeInput = normalizeInput;
|
|
129
100
|
exports.normalizeMethodsConfig = normalizeMethodsConfig;
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { DiscoverResolveSiteUrlFn } from "../types.cjs";
|
|
2
1
|
import { parseFeed } from "feedsmith";
|
|
3
2
|
|
|
4
3
|
//#region src/common/discover/utils.d.ts
|
|
5
4
|
declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
|
|
6
|
-
declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
|
|
7
5
|
//#endregion
|
|
8
|
-
export {
|
|
6
|
+
export { getFeedSiteUrl };
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { DiscoverResolveSiteUrlFn } from "../types.js";
|
|
2
1
|
import { parseFeed } from "feedsmith";
|
|
3
2
|
|
|
4
3
|
//#region src/common/discover/utils.d.ts
|
|
5
4
|
declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
|
|
6
|
-
declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
|
|
7
5
|
//#endregion
|
|
8
|
-
export {
|
|
6
|
+
export { getFeedSiteUrl };
|
|
@@ -1,20 +1,5 @@
|
|
|
1
1
|
import { errors } from "../locales.js";
|
|
2
|
-
import { resolveUrl } from "../utils.js";
|
|
3
|
-
import { parseFeed } from "feedsmith";
|
|
4
2
|
//#region src/common/discover/utils.ts
|
|
5
|
-
const defaultFetchFn = async (url, options) => {
|
|
6
|
-
const response = await fetch(url, {
|
|
7
|
-
method: options?.method ?? "GET",
|
|
8
|
-
headers: options?.headers
|
|
9
|
-
});
|
|
10
|
-
return {
|
|
11
|
-
headers: response.headers,
|
|
12
|
-
body: await response.text(),
|
|
13
|
-
url: response.url,
|
|
14
|
-
status: response.status,
|
|
15
|
-
statusText: response.statusText
|
|
16
|
-
};
|
|
17
|
-
};
|
|
18
3
|
const normalizeInput = async (input, fetchFn) => {
|
|
19
4
|
if (typeof input === "object") return input;
|
|
20
5
|
try {
|
|
@@ -36,18 +21,6 @@ const getFeedSiteUrl = (parsed) => {
|
|
|
36
21
|
if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
|
|
37
22
|
if (format === "json") return feed.home_page_url;
|
|
38
23
|
};
|
|
39
|
-
const defaultResolveSiteUrlFn = (input) => {
|
|
40
|
-
if (!input.content) return;
|
|
41
|
-
try {
|
|
42
|
-
let siteUrl = getFeedSiteUrl(parseFeed(input.content));
|
|
43
|
-
if (!siteUrl) try {
|
|
44
|
-
siteUrl = new URL(input.url).origin;
|
|
45
|
-
} catch {}
|
|
46
|
-
else siteUrl = resolveUrl(siteUrl, input.url);
|
|
47
|
-
if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
|
|
48
|
-
return siteUrl;
|
|
49
|
-
} catch {}
|
|
50
|
-
};
|
|
51
24
|
const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
|
|
52
25
|
if (typeof entry.uri === "string") return {
|
|
53
26
|
...entry,
|
|
@@ -122,4 +95,4 @@ const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
|
|
|
122
95
|
return methodsConfig;
|
|
123
96
|
};
|
|
124
97
|
//#endregion
|
|
125
|
-
export {
|
|
98
|
+
export { getFeedSiteUrl, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };
|
package/dist/common/types.d.cts
CHANGED
|
@@ -22,7 +22,7 @@ type LinkSelector = {
|
|
|
22
22
|
types?: Array<string>;
|
|
23
23
|
};
|
|
24
24
|
type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string | undefined;
|
|
25
|
-
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
|
|
25
|
+
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject, resolveUrlFn: DiscoverResolveUrlFn) => string | undefined;
|
|
26
26
|
type DiscoverFetchFnOptions = {
|
|
27
27
|
method?: 'GET' | 'HEAD';
|
|
28
28
|
headers?: Record<string, string>;
|
package/dist/common/types.d.ts
CHANGED
|
@@ -22,7 +22,7 @@ type LinkSelector = {
|
|
|
22
22
|
types?: Array<string>;
|
|
23
23
|
};
|
|
24
24
|
type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string | undefined;
|
|
25
|
-
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
|
|
25
|
+
type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject, resolveUrlFn: DiscoverResolveUrlFn) => string | undefined;
|
|
26
26
|
type DiscoverFetchFnOptions = {
|
|
27
27
|
method?: 'GET' | 'HEAD';
|
|
28
28
|
headers?: Record<string, string>;
|
package/dist/common/utils.cjs
CHANGED
|
@@ -52,13 +52,6 @@ const omitEmpty = (array) => {
|
|
|
52
52
|
for (const item of array) if (item != null && item !== "") result.push(item);
|
|
53
53
|
return result;
|
|
54
54
|
};
|
|
55
|
-
const resolveUrl = (url, baseUrl) => {
|
|
56
|
-
try {
|
|
57
|
-
return new URL(url, baseUrl).href;
|
|
58
|
-
} catch {
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
55
|
const matchesAnyOfLinkSelectors = (rel, type, selectors) => {
|
|
63
56
|
return selectors.some((selector) => {
|
|
64
57
|
if (!anyWordMatchesAnyOf(rel, [selector.rel])) return false;
|
|
@@ -94,4 +87,3 @@ exports.isSubdomainOf = isSubdomainOf;
|
|
|
94
87
|
exports.matchesAnyOfLinkSelectors = matchesAnyOfLinkSelectors;
|
|
95
88
|
exports.omitEmpty = omitEmpty;
|
|
96
89
|
exports.processConcurrently = processConcurrently;
|
|
97
|
-
exports.resolveUrl = resolveUrl;
|
package/dist/common/utils.js
CHANGED
|
@@ -52,13 +52,6 @@ const omitEmpty = (array) => {
|
|
|
52
52
|
for (const item of array) if (item != null && item !== "") result.push(item);
|
|
53
53
|
return result;
|
|
54
54
|
};
|
|
55
|
-
const resolveUrl = (url, baseUrl) => {
|
|
56
|
-
try {
|
|
57
|
-
return new URL(url, baseUrl).href;
|
|
58
|
-
} catch {
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
55
|
const matchesAnyOfLinkSelectors = (rel, type, selectors) => {
|
|
63
56
|
return selectors.some((selector) => {
|
|
64
57
|
if (!anyWordMatchesAnyOf(rel, [selector.rel])) return false;
|
|
@@ -83,4 +76,4 @@ const processConcurrently = async (items, processFn, options) => {
|
|
|
83
76
|
}
|
|
84
77
|
};
|
|
85
78
|
//#endregion
|
|
86
|
-
export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently
|
|
79
|
+
export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently };
|
|
@@ -11,7 +11,7 @@ const isImageContent = (content) => {
|
|
|
11
11
|
const isSuccessStatus = (status) => {
|
|
12
12
|
return status !== void 0 && status >= 200 && status < 400;
|
|
13
13
|
};
|
|
14
|
-
const
|
|
14
|
+
const defaultExtractFn = (input) => {
|
|
15
15
|
if (isImageContentType(input.headers) || isImageContent(input.content) || isSuccessStatus(input.status)) return {
|
|
16
16
|
url: input.url,
|
|
17
17
|
isValid: true
|
|
@@ -22,4 +22,4 @@ const defaultExtractor = (input) => {
|
|
|
22
22
|
};
|
|
23
23
|
};
|
|
24
24
|
//#endregion
|
|
25
|
-
exports.
|
|
25
|
+
exports.defaultExtractFn = defaultExtractFn;
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.cjs";
|
|
|
2
2
|
import { FaviconResult } from "./types.cjs";
|
|
3
3
|
|
|
4
4
|
//#region src/favicons/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<FaviconResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.js";
|
|
|
2
2
|
import { FaviconResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/favicons/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<FaviconResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -11,7 +11,7 @@ const isImageContent = (content) => {
|
|
|
11
11
|
const isSuccessStatus = (status) => {
|
|
12
12
|
return status !== void 0 && status >= 200 && status < 400;
|
|
13
13
|
};
|
|
14
|
-
const
|
|
14
|
+
const defaultExtractFn = (input) => {
|
|
15
15
|
if (isImageContentType(input.headers) || isImageContent(input.content) || isSuccessStatus(input.status)) return {
|
|
16
16
|
url: input.url,
|
|
17
17
|
isValid: true
|
|
@@ -22,4 +22,4 @@ const defaultExtractor = (input) => {
|
|
|
22
22
|
};
|
|
23
23
|
};
|
|
24
24
|
//#endregion
|
|
25
|
-
export {
|
|
25
|
+
export { defaultExtractFn };
|
package/dist/favicons/index.cjs
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
const require_utils$1 = require("../common/discover/utils.cjs");
|
|
1
|
+
const require_defaults = require("../common/discover/defaults.cjs");
|
|
3
2
|
const require_index = require("../common/discover/index.cjs");
|
|
4
|
-
const require_defaults = require("./defaults.cjs");
|
|
3
|
+
const require_defaults$1 = require("./defaults.cjs");
|
|
5
4
|
const require_extractors = require("./extractors.cjs");
|
|
6
5
|
//#region src/favicons/index.ts
|
|
7
6
|
const discoverFavicons = (input, options = {}) => {
|
|
@@ -14,16 +13,16 @@ const discoverFavicons = (input, options = {}) => {
|
|
|
14
13
|
"headers",
|
|
15
14
|
"guess"
|
|
16
15
|
],
|
|
17
|
-
fetchFn: options.fetchFn ??
|
|
18
|
-
extractFn: options.extractFn ?? require_extractors.
|
|
19
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
20
|
-
resolveSiteUrlFn: options.resolveSiteUrlFn ??
|
|
16
|
+
fetchFn: options.fetchFn ?? require_defaults.defaultFetchFn,
|
|
17
|
+
extractFn: options.extractFn ?? require_extractors.defaultExtractFn,
|
|
18
|
+
resolveUrlFn: options.resolveUrlFn ?? require_defaults.defaultResolveUrlFn,
|
|
19
|
+
resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_defaults.defaultResolveSiteUrlFn
|
|
21
20
|
}, {
|
|
22
|
-
platform: require_defaults.defaultPlatformOptions,
|
|
23
|
-
feed: require_defaults.defaultFeedOptions,
|
|
24
|
-
html: require_defaults.defaultHtmlOptions,
|
|
25
|
-
headers: require_defaults.defaultHeadersOptions,
|
|
26
|
-
guess: require_defaults.defaultGuessOptions
|
|
21
|
+
platform: require_defaults$1.defaultPlatformOptions,
|
|
22
|
+
feed: require_defaults$1.defaultFeedOptions,
|
|
23
|
+
html: require_defaults$1.defaultHtmlOptions,
|
|
24
|
+
headers: require_defaults$1.defaultHeadersOptions,
|
|
25
|
+
guess: require_defaults$1.defaultGuessOptions
|
|
27
26
|
});
|
|
28
27
|
};
|
|
29
28
|
//#endregion
|
package/dist/favicons/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn, defaultResolveSiteUrlFn } from "../common/discover/utils.js";
|
|
1
|
+
import { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
2
|
import { discover } from "../common/discover/index.js";
|
|
4
3
|
import { defaultFeedOptions, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions } from "./defaults.js";
|
|
5
|
-
import {
|
|
4
|
+
import { defaultExtractFn } from "./extractors.js";
|
|
6
5
|
//#region src/favicons/index.ts
|
|
7
6
|
const discoverFavicons = (input, options = {}) => {
|
|
8
7
|
return discover(input, {
|
|
@@ -15,8 +14,8 @@ const discoverFavicons = (input, options = {}) => {
|
|
|
15
14
|
"guess"
|
|
16
15
|
],
|
|
17
16
|
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
18
|
-
extractFn: options.extractFn ??
|
|
19
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
17
|
+
extractFn: options.extractFn ?? defaultExtractFn,
|
|
18
|
+
resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn,
|
|
20
19
|
resolveSiteUrlFn: options.resolveSiteUrlFn ?? defaultResolveSiteUrlFn
|
|
21
20
|
}, {
|
|
22
21
|
platform: defaultPlatformOptions,
|
package/dist/favicons.cjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_defaults = require("./favicons/defaults.cjs");
|
|
3
3
|
const require_extractors = require("./favicons/extractors.cjs");
|
|
4
|
-
exports.
|
|
4
|
+
exports.defaultExtractFn = require_extractors.defaultExtractFn;
|
|
5
5
|
exports.defaultFeedOptions = require_defaults.defaultFeedOptions;
|
|
6
6
|
exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
|
|
7
7
|
exports.defaultGuessPaths = require_defaults.defaultGuessPaths;
|
package/dist/favicons.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors } from "./favicons/defaults.cjs";
|
|
2
2
|
import { FaviconResult } from "./favicons/types.cjs";
|
|
3
|
-
import {
|
|
4
|
-
export { FaviconResult,
|
|
3
|
+
import { defaultExtractFn } from "./favicons/extractors.cjs";
|
|
4
|
+
export { FaviconResult, defaultExtractFn, defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors };
|
package/dist/favicons.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors } from "./favicons/defaults.js";
|
|
2
2
|
import { FaviconResult } from "./favicons/types.js";
|
|
3
|
-
import {
|
|
4
|
-
export { FaviconResult,
|
|
3
|
+
import { defaultExtractFn } from "./favicons/extractors.js";
|
|
4
|
+
export { FaviconResult, defaultExtractFn, defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors };
|
package/dist/favicons.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors } from "./favicons/defaults.js";
|
|
2
|
-
import {
|
|
3
|
-
export {
|
|
2
|
+
import { defaultExtractFn } from "./favicons/extractors.js";
|
|
3
|
+
export { defaultExtractFn, defaultFeedOptions, defaultGuessOptions, defaultGuessPaths, defaultHeadersOptions, defaultHtmlOptions, defaultIconRels, defaultPlatformOptions, linkSelectors };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
const require_utils = require("../common/utils.cjs");
|
|
2
|
-
const
|
|
1
|
+
const require_utils = require("../common/discover/utils.cjs");
|
|
2
|
+
const require_defaults = require("../common/discover/defaults.cjs");
|
|
3
3
|
let feedsmith = require("feedsmith");
|
|
4
4
|
//#region src/feeds/extractors.ts
|
|
5
|
-
const
|
|
5
|
+
const defaultExtractFn = ({ content, url }) => {
|
|
6
6
|
if (!content) return {
|
|
7
7
|
url,
|
|
8
8
|
isValid: false
|
|
@@ -10,8 +10,8 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
10
10
|
try {
|
|
11
11
|
const parsed = (0, feedsmith.parseFeed)(content);
|
|
12
12
|
const { format, feed } = parsed;
|
|
13
|
-
const rawSiteUrl = require_utils
|
|
14
|
-
const siteUrl = rawSiteUrl ?
|
|
13
|
+
const rawSiteUrl = require_utils.getFeedSiteUrl(parsed);
|
|
14
|
+
const siteUrl = rawSiteUrl ? require_defaults.defaultResolveUrlFn(rawSiteUrl, url) : void 0;
|
|
15
15
|
if (format === "rss" || format === "rdf") return {
|
|
16
16
|
url,
|
|
17
17
|
isValid: true,
|
|
@@ -43,4 +43,4 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
43
43
|
};
|
|
44
44
|
};
|
|
45
45
|
//#endregion
|
|
46
|
-
exports.
|
|
46
|
+
exports.defaultExtractFn = defaultExtractFn;
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.cjs";
|
|
|
2
2
|
import { FeedResult } from "./types.cjs";
|
|
3
3
|
|
|
4
4
|
//#region src/feeds/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<FeedResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
|
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.js";
|
|
|
2
2
|
import { FeedResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/feeds/extractors.d.ts
|
|
5
|
-
declare const
|
|
5
|
+
declare const defaultExtractFn: DiscoverExtractFn<FeedResult>;
|
|
6
6
|
//#endregion
|
|
7
|
-
export {
|
|
7
|
+
export { defaultExtractFn };
|
package/dist/feeds/extractors.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { resolveUrl } from "../common/utils.js";
|
|
2
1
|
import { getFeedSiteUrl } from "../common/discover/utils.js";
|
|
2
|
+
import { defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
3
|
import { parseFeed } from "feedsmith";
|
|
4
4
|
//#region src/feeds/extractors.ts
|
|
5
|
-
const
|
|
5
|
+
const defaultExtractFn = ({ content, url }) => {
|
|
6
6
|
if (!content) return {
|
|
7
7
|
url,
|
|
8
8
|
isValid: false
|
|
@@ -11,7 +11,7 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
11
11
|
const parsed = parseFeed(content);
|
|
12
12
|
const { format, feed } = parsed;
|
|
13
13
|
const rawSiteUrl = getFeedSiteUrl(parsed);
|
|
14
|
-
const siteUrl = rawSiteUrl ?
|
|
14
|
+
const siteUrl = rawSiteUrl ? defaultResolveUrlFn(rawSiteUrl, url) : void 0;
|
|
15
15
|
if (format === "rss" || format === "rdf") return {
|
|
16
16
|
url,
|
|
17
17
|
isValid: true,
|
|
@@ -43,4 +43,4 @@ const defaultExtractor = ({ content, url }) => {
|
|
|
43
43
|
};
|
|
44
44
|
};
|
|
45
45
|
//#endregion
|
|
46
|
-
export {
|
|
46
|
+
export { defaultExtractFn };
|
package/dist/feeds/index.cjs
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
const require_utils$1 = require("../common/discover/utils.cjs");
|
|
1
|
+
const require_defaults = require("../common/discover/defaults.cjs");
|
|
3
2
|
const require_index = require("../common/discover/index.cjs");
|
|
4
|
-
const require_defaults = require("./defaults.cjs");
|
|
3
|
+
const require_defaults$1 = require("./defaults.cjs");
|
|
5
4
|
const require_extractors = require("./extractors.cjs");
|
|
6
5
|
//#region src/feeds/index.ts
|
|
7
6
|
const discoverFeeds = (input, options = {}) => {
|
|
@@ -13,14 +12,14 @@ const discoverFeeds = (input, options = {}) => {
|
|
|
13
12
|
"headers",
|
|
14
13
|
"guess"
|
|
15
14
|
],
|
|
16
|
-
fetchFn: options.fetchFn ??
|
|
17
|
-
extractFn: options.extractFn ?? require_extractors.
|
|
18
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
15
|
+
fetchFn: options.fetchFn ?? require_defaults.defaultFetchFn,
|
|
16
|
+
extractFn: options.extractFn ?? require_extractors.defaultExtractFn,
|
|
17
|
+
resolveUrlFn: options.resolveUrlFn ?? require_defaults.defaultResolveUrlFn
|
|
19
18
|
}, {
|
|
20
|
-
platform: require_defaults.defaultPlatformOptions,
|
|
21
|
-
html: require_defaults.defaultHtmlOptions,
|
|
22
|
-
headers: require_defaults.defaultHeadersOptions,
|
|
23
|
-
guess: require_defaults.defaultGuessOptions
|
|
19
|
+
platform: require_defaults$1.defaultPlatformOptions,
|
|
20
|
+
html: require_defaults$1.defaultHtmlOptions,
|
|
21
|
+
headers: require_defaults$1.defaultHeadersOptions,
|
|
22
|
+
guess: require_defaults$1.defaultGuessOptions
|
|
24
23
|
});
|
|
25
24
|
};
|
|
26
25
|
//#endregion
|
package/dist/feeds/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn } from "../common/discover/utils.js";
|
|
1
|
+
import { defaultFetchFn, defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
2
|
import { discover } from "../common/discover/index.js";
|
|
4
3
|
import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions } from "./defaults.js";
|
|
5
|
-
import {
|
|
4
|
+
import { defaultExtractFn } from "./extractors.js";
|
|
6
5
|
//#region src/feeds/index.ts
|
|
7
6
|
const discoverFeeds = (input, options = {}) => {
|
|
8
7
|
return discover(input, {
|
|
@@ -14,8 +13,8 @@ const discoverFeeds = (input, options = {}) => {
|
|
|
14
13
|
"guess"
|
|
15
14
|
],
|
|
16
15
|
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
17
|
-
extractFn: options.extractFn ??
|
|
18
|
-
resolveUrlFn: options.resolveUrlFn ??
|
|
16
|
+
extractFn: options.extractFn ?? defaultExtractFn,
|
|
17
|
+
resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn
|
|
19
18
|
}, {
|
|
20
19
|
platform: defaultPlatformOptions,
|
|
21
20
|
html: defaultHtmlOptions,
|
package/dist/feeds.cjs
CHANGED
|
@@ -2,7 +2,7 @@ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
|
2
2
|
const require_defaults = require("./feeds/defaults.cjs");
|
|
3
3
|
const require_extractors = require("./feeds/extractors.cjs");
|
|
4
4
|
exports.anchorLabels = require_defaults.anchorLabels;
|
|
5
|
-
exports.
|
|
5
|
+
exports.defaultExtractFn = require_extractors.defaultExtractFn;
|
|
6
6
|
exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
|
|
7
7
|
exports.defaultHeadersOptions = require_defaults.defaultHeadersOptions;
|
|
8
8
|
exports.defaultHtmlOptions = require_defaults.defaultHtmlOptions;
|
package/dist/feeds.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.cjs";
|
|
2
2
|
import { FeedResult } from "./feeds/types.cjs";
|
|
3
|
-
import {
|
|
4
|
-
export { FeedResult, anchorLabels,
|
|
3
|
+
import { defaultExtractFn } from "./feeds/extractors.cjs";
|
|
4
|
+
export { FeedResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
package/dist/feeds.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.js";
|
|
2
2
|
import { FeedResult } from "./feeds/types.js";
|
|
3
|
-
import {
|
|
4
|
-
export { FeedResult, anchorLabels,
|
|
3
|
+
import { defaultExtractFn } from "./feeds/extractors.js";
|
|
4
|
+
export { FeedResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
package/dist/feeds.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.js";
|
|
2
|
-
import {
|
|
3
|
-
export { anchorLabels,
|
|
2
|
+
import { defaultExtractFn } from "./feeds/extractors.js";
|
|
3
|
+
export { anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const require_utils = require("../../common/utils.cjs");
|
|
2
|
-
const
|
|
1
|
+
const require_utils = require("../../common/discover/utils.cjs");
|
|
2
|
+
const require_defaults = require("../../common/discover/defaults.cjs");
|
|
3
3
|
const require_index = require("../feed/index.cjs");
|
|
4
4
|
const require_index$1 = require("../headers/index.cjs");
|
|
5
5
|
const require_index$2 = require("../html/index.cjs");
|
|
@@ -9,8 +9,8 @@ const discoverHubs = async (input, options = {}) => {
|
|
|
9
9
|
"headers",
|
|
10
10
|
"feed",
|
|
11
11
|
"html"
|
|
12
|
-
], fetchFn =
|
|
13
|
-
const normalizedInput = await require_utils
|
|
12
|
+
], fetchFn = require_defaults.defaultFetchFn, resolveUrlFn = require_defaults.defaultResolveUrlFn } = options;
|
|
13
|
+
const normalizedInput = await require_utils.normalizeInput(input, fetchFn);
|
|
14
14
|
const results = [];
|
|
15
15
|
if (methods.includes("headers") && normalizedInput.headers) {
|
|
16
16
|
const headerHubs = require_index$1.discoverHubsFromHeaders(normalizedInput.headers, normalizedInput.url, resolveUrlFn);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { defaultFetchFn,
|
|
1
|
+
import { normalizeInput } from "../../common/discover/utils.js";
|
|
2
|
+
import { defaultFetchFn, defaultResolveUrlFn } from "../../common/discover/defaults.js";
|
|
3
3
|
import { discoverHubsFromFeed } from "../feed/index.js";
|
|
4
4
|
import { discoverHubsFromHeaders } from "../headers/index.js";
|
|
5
5
|
import { discoverHubsFromHtml } from "../html/index.js";
|
|
@@ -9,7 +9,7 @@ const discoverHubs = async (input, options = {}) => {
|
|
|
9
9
|
"headers",
|
|
10
10
|
"feed",
|
|
11
11
|
"html"
|
|
12
|
-
], fetchFn = defaultFetchFn, resolveUrlFn =
|
|
12
|
+
], fetchFn = defaultFetchFn, resolveUrlFn = defaultResolveUrlFn } = options;
|
|
13
13
|
const normalizedInput = await normalizeInput(input, fetchFn);
|
|
14
14
|
const results = [];
|
|
15
15
|
if (methods.includes("headers") && normalizedInput.headers) {
|
package/dist/hubs/feed/index.cjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
const
|
|
1
|
+
const require_defaults = require("../../common/discover/defaults.cjs");
|
|
2
2
|
let feedsmith = require("feedsmith");
|
|
3
3
|
//#region src/hubs/feed/index.ts
|
|
4
4
|
const getLinksWithRel = (links, rel) => {
|
|
5
5
|
return links?.filter((link) => link.rel === rel && link.href).map((link) => link.href) ?? [];
|
|
6
6
|
};
|
|
7
|
-
const discoverHubsFromFeed = (content, baseUrl, resolveUrlFn =
|
|
7
|
+
const discoverHubsFromFeed = (content, baseUrl, resolveUrlFn = require_defaults.defaultResolveUrlFn) => {
|
|
8
8
|
try {
|
|
9
9
|
const { format, feed } = (0, feedsmith.parseFeed)(content);
|
|
10
10
|
if (format === "json") {
|
package/dist/hubs/feed/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
|
|
2
2
|
import { parseFeed } from "feedsmith";
|
|
3
3
|
//#region src/hubs/feed/index.ts
|
|
4
4
|
const getLinksWithRel = (links, rel) => {
|
|
5
5
|
return links?.filter((link) => link.rel === rel && link.href).map((link) => link.href) ?? [];
|
|
6
6
|
};
|
|
7
|
-
const discoverHubsFromFeed = (content, baseUrl, resolveUrlFn =
|
|
7
|
+
const discoverHubsFromFeed = (content, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
|
|
8
8
|
try {
|
|
9
9
|
const { format, feed } = parseFeed(content);
|
|
10
10
|
if (format === "json") {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
const
|
|
1
|
+
const require_defaults = require("../../common/discover/defaults.cjs");
|
|
2
2
|
const require_index = require("../../common/uris/headers/index.cjs");
|
|
3
3
|
//#region src/hubs/headers/index.ts
|
|
4
4
|
const hubSelector = [{ rel: "hub" }];
|
|
5
5
|
const selfSelector = [{ rel: "self" }];
|
|
6
|
-
const discoverHubsFromHeaders = (headers, baseUrl, resolveUrlFn =
|
|
6
|
+
const discoverHubsFromHeaders = (headers, baseUrl, resolveUrlFn = require_defaults.defaultResolveUrlFn) => {
|
|
7
7
|
const hubUris = require_index.discoverUrisFromHeaders(headers, { linkSelectors: hubSelector });
|
|
8
8
|
if (hubUris.length === 0) return [];
|
|
9
9
|
const selfUris = require_index.discoverUrisFromHeaders(headers, { linkSelectors: selfSelector });
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
|
|
2
2
|
import { discoverUrisFromHeaders } from "../../common/uris/headers/index.js";
|
|
3
3
|
//#region src/hubs/headers/index.ts
|
|
4
4
|
const hubSelector = [{ rel: "hub" }];
|
|
5
5
|
const selfSelector = [{ rel: "self" }];
|
|
6
|
-
const discoverHubsFromHeaders = (headers, baseUrl, resolveUrlFn =
|
|
6
|
+
const discoverHubsFromHeaders = (headers, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
|
|
7
7
|
const hubUris = discoverUrisFromHeaders(headers, { linkSelectors: hubSelector });
|
|
8
8
|
if (hubUris.length === 0) return [];
|
|
9
9
|
const selfUris = discoverUrisFromHeaders(headers, { linkSelectors: selfSelector });
|
package/dist/hubs/html/index.cjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const
|
|
1
|
+
const require_defaults = require("../../common/discover/defaults.cjs");
|
|
2
2
|
const require_index = require("../../common/uris/html/index.cjs");
|
|
3
3
|
//#region src/hubs/html/index.ts
|
|
4
4
|
const hubSelector = [{ rel: "hub" }];
|
|
@@ -8,7 +8,7 @@ const htmlOptions = {
|
|
|
8
8
|
anchorIgnoredUris: [],
|
|
9
9
|
anchorLabels: []
|
|
10
10
|
};
|
|
11
|
-
const discoverHubsFromHtml = (content, baseUrl, resolveUrlFn =
|
|
11
|
+
const discoverHubsFromHtml = (content, baseUrl, resolveUrlFn = require_defaults.defaultResolveUrlFn) => {
|
|
12
12
|
const hubUris = require_index.discoverUrisFromHtml(content, {
|
|
13
13
|
...htmlOptions,
|
|
14
14
|
linkSelectors: hubSelector
|
package/dist/hubs/html/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
|
|
2
2
|
import { discoverUrisFromHtml } from "../../common/uris/html/index.js";
|
|
3
3
|
//#region src/hubs/html/index.ts
|
|
4
4
|
const hubSelector = [{ rel: "hub" }];
|
|
@@ -8,7 +8,7 @@ const htmlOptions = {
|
|
|
8
8
|
anchorIgnoredUris: [],
|
|
9
9
|
anchorLabels: []
|
|
10
10
|
};
|
|
11
|
-
const discoverHubsFromHtml = (content, baseUrl, resolveUrlFn =
|
|
11
|
+
const discoverHubsFromHtml = (content, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
|
|
12
12
|
const hubUris = discoverUrisFromHtml(content, {
|
|
13
13
|
...htmlOptions,
|
|
14
14
|
linkSelectors: hubSelector
|
package/dist/index.cjs
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_utils = require("./common/discover/utils.cjs");
|
|
3
|
+
const require_defaults = require("./common/discover/defaults.cjs");
|
|
3
4
|
const require_index = require("./blogrolls/index.cjs");
|
|
4
5
|
const require_index$1 = require("./favicons/index.cjs");
|
|
5
6
|
const require_index$2 = require("./feeds/index.cjs");
|
|
6
7
|
const require_index$3 = require("./hubs/discover/index.cjs");
|
|
7
|
-
exports.defaultResolveSiteUrlFn =
|
|
8
|
+
exports.defaultResolveSiteUrlFn = require_defaults.defaultResolveSiteUrlFn;
|
|
8
9
|
exports.discoverBlogrolls = require_index.discoverBlogrolls;
|
|
9
10
|
exports.discoverFavicons = require_index$1.discoverFavicons;
|
|
10
11
|
exports.discoverFeeds = require_index$2.discoverFeeds;
|
package/dist/index.d.cts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, UriEntry } from "./common/types.cjs";
|
|
2
2
|
import { discoverBlogrolls } from "./blogrolls/index.cjs";
|
|
3
|
-
import { defaultResolveSiteUrlFn
|
|
3
|
+
import { defaultResolveSiteUrlFn } from "./common/discover/defaults.cjs";
|
|
4
|
+
import { getFeedSiteUrl } from "./common/discover/utils.cjs";
|
|
4
5
|
import { discoverFavicons } from "./favicons/index.cjs";
|
|
5
6
|
import { discoverFeeds } from "./feeds/index.cjs";
|
|
6
7
|
import { discoverHubs } from "./hubs/discover/index.cjs";
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, UriEntry } from "./common/types.js";
|
|
2
2
|
import { discoverBlogrolls } from "./blogrolls/index.js";
|
|
3
|
-
import { defaultResolveSiteUrlFn
|
|
3
|
+
import { defaultResolveSiteUrlFn } from "./common/discover/defaults.js";
|
|
4
|
+
import { getFeedSiteUrl } from "./common/discover/utils.js";
|
|
4
5
|
import { discoverFavicons } from "./favicons/index.js";
|
|
5
6
|
import { discoverFeeds } from "./feeds/index.js";
|
|
6
7
|
import { discoverHubs } from "./hubs/discover/index.js";
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getFeedSiteUrl } from "./common/discover/utils.js";
|
|
2
|
+
import { defaultResolveSiteUrlFn } from "./common/discover/defaults.js";
|
|
2
3
|
import { discoverBlogrolls } from "./blogrolls/index.js";
|
|
3
4
|
import { discoverFavicons } from "./favicons/index.js";
|
|
4
5
|
import { discoverFeeds } from "./feeds/index.js";
|
package/package.json
CHANGED