feedscout 1.9.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_virtual/_rolldown/runtime.cjs +23 -0
- package/dist/blogrolls/extractors.cjs +3 -2
- package/dist/blogrolls/extractors.js +3 -2
- package/dist/common/discover/index.cjs +27 -14
- package/dist/common/discover/index.js +23 -10
- package/dist/common/discover/utils.cjs +9 -3
- package/dist/common/discover/utils.js +9 -3
- package/dist/common/types.d.cts +7 -0
- package/dist/common/types.d.ts +7 -0
- package/dist/common/uris/headers/index.cjs +7 -17
- package/dist/common/uris/headers/index.js +5 -17
- package/dist/common/utils.cjs +13 -1
- package/dist/common/utils.js +11 -2
- package/dist/favicons/extractors.cjs +3 -8
- package/dist/favicons/extractors.js +3 -8
- package/dist/feeds/extractors.cjs +5 -4
- package/dist/feeds/extractors.js +3 -2
- package/dist/feeds/platform/handlers/artstation.cjs +2 -2
- package/dist/feeds/platform/handlers/artstation.js +2 -2
- package/dist/feeds/platform/handlers/fireside.cjs +2 -2
- package/dist/feeds/platform/handlers/fireside.js +2 -2
- package/dist/feeds/platform/handlers/podbean.cjs +3 -3
- package/dist/feeds/platform/handlers/podbean.js +3 -3
- package/dist/feeds/platform/handlers/podigee.cjs +2 -2
- package/dist/feeds/platform/handlers/podigee.js +2 -2
- package/dist/feeds/platform/handlers/transistor.cjs +3 -3
- package/dist/feeds/platform/handlers/transistor.js +3 -3
- package/package.json +4 -2
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
//#region \0rolldown/runtime.js
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __copyProps = (to, from, except, desc) => {
|
|
9
|
+
if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
|
|
10
|
+
key = keys[i];
|
|
11
|
+
if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
|
|
12
|
+
get: ((k) => from[k]).bind(null, key),
|
|
13
|
+
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
|
|
19
|
+
value: mod,
|
|
20
|
+
enumerable: true
|
|
21
|
+
}) : target, mod));
|
|
22
|
+
//#endregion
|
|
23
|
+
exports.__toESM = __toESM;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
const require_utils = require("../common/utils.cjs");
|
|
1
2
|
let feedsmith = require("feedsmith");
|
|
2
3
|
//#region src/blogrolls/extractors.ts
|
|
3
|
-
const defaultExtractFn = ({ content, url }) => {
|
|
4
|
-
if (!content) return {
|
|
4
|
+
const defaultExtractFn = ({ content, url, status }) => {
|
|
5
|
+
if (!content || !require_utils.isSuccessfulStatus(status)) return {
|
|
5
6
|
url,
|
|
6
7
|
isValid: false
|
|
7
8
|
};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { isSuccessfulStatus } from "../common/utils.js";
|
|
1
2
|
import { parseOpml } from "feedsmith";
|
|
2
3
|
//#region src/blogrolls/extractors.ts
|
|
3
|
-
const defaultExtractFn = ({ content, url }) => {
|
|
4
|
-
if (!content) return {
|
|
4
|
+
const defaultExtractFn = ({ content, url, status }) => {
|
|
5
|
+
if (!content || !isSuccessfulStatus(status)) return {
|
|
5
6
|
url,
|
|
6
7
|
isValid: false
|
|
7
8
|
};
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
const require_utils = require("
|
|
1
|
+
const require_utils = require("../utils.cjs");
|
|
2
|
+
const require_utils$1 = require("./utils.cjs");
|
|
2
3
|
const require_types = require("../types.cjs");
|
|
3
|
-
const require_utils$1 = require("../utils.cjs");
|
|
4
4
|
const require_index = require("../uris/index.cjs");
|
|
5
5
|
//#region src/common/discover/index.ts
|
|
6
6
|
const discover = async (input, options, defaults) => {
|
|
7
|
-
const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
|
|
8
|
-
const
|
|
7
|
+
const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, maxUris = 50, includeInvalid = false, onProgress, onError } = options;
|
|
8
|
+
const safeConcurrency = require_utils.toPositiveInteger(concurrency, 3);
|
|
9
|
+
const safeMaxUris = require_utils.toPositiveInteger(maxUris, 50);
|
|
10
|
+
const sourceInput = await require_utils$1.normalizeInput(input, fetchFn, onError);
|
|
9
11
|
if (sourceInput.content) {
|
|
10
12
|
const result = await extractFn({
|
|
11
13
|
url: sourceInput.url,
|
|
@@ -24,26 +26,37 @@ const discover = async (input, options, defaults) => {
|
|
|
24
26
|
content: typeof response.body === "string" ? response.body : "",
|
|
25
27
|
headers: response.headers
|
|
26
28
|
};
|
|
27
|
-
} catch {
|
|
29
|
+
} catch (error) {
|
|
30
|
+
onError?.(error, {
|
|
31
|
+
phase: "resolveSiteUrl",
|
|
32
|
+
url: siteUrl
|
|
33
|
+
});
|
|
34
|
+
}
|
|
28
35
|
}
|
|
29
|
-
const urisByMethod = await require_index.discoverUris(require_utils.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
|
|
36
|
+
const urisByMethod = await require_index.discoverUris(require_utils$1.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
|
|
30
37
|
const seen = /* @__PURE__ */ new Set();
|
|
31
38
|
const methodGroups = [];
|
|
39
|
+
let remaining = safeMaxUris;
|
|
32
40
|
for (const method of require_types.discoverMethodOrder) {
|
|
41
|
+
if (remaining <= 0) break;
|
|
33
42
|
const rawUris = urisByMethod[method];
|
|
34
43
|
if (!rawUris || rawUris.length === 0) continue;
|
|
35
44
|
const unique = rawUris.map((entry) => {
|
|
36
|
-
return require_utils.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
|
|
45
|
+
return require_utils$1.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
|
|
37
46
|
}).filter((entry) => {
|
|
38
|
-
const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
|
|
47
|
+
const key = typeof entry.uri === "string" ? entry.uri : [...entry.uri].sort().join("\0");
|
|
39
48
|
if (seen.has(key)) return false;
|
|
40
49
|
seen.add(key);
|
|
41
50
|
return true;
|
|
42
51
|
});
|
|
43
|
-
if (unique.length > 0)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
52
|
+
if (unique.length > 0) {
|
|
53
|
+
const capped = unique.slice(0, remaining);
|
|
54
|
+
remaining -= capped.length;
|
|
55
|
+
methodGroups.push({
|
|
56
|
+
method,
|
|
57
|
+
entries: capped
|
|
58
|
+
});
|
|
59
|
+
}
|
|
47
60
|
}
|
|
48
61
|
const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
|
|
49
62
|
const results = [];
|
|
@@ -91,8 +104,8 @@ const discover = async (input, options, defaults) => {
|
|
|
91
104
|
};
|
|
92
105
|
for (const { method, entries } of methodGroups) {
|
|
93
106
|
const foundBefore = found;
|
|
94
|
-
await require_utils
|
|
95
|
-
concurrency,
|
|
107
|
+
await require_utils.processConcurrently(entries, (entry) => processUri(entry, method), {
|
|
108
|
+
concurrency: safeConcurrency,
|
|
96
109
|
shouldStop: () => {
|
|
97
110
|
return stopOnFirstResult && found > 0;
|
|
98
111
|
}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
import { processConcurrently, toPositiveInteger } from "../utils.js";
|
|
1
2
|
import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
|
|
2
3
|
import { discoverMethodOrder } from "../types.js";
|
|
3
|
-
import { processConcurrently } from "../utils.js";
|
|
4
4
|
import { discoverUris } from "../uris/index.js";
|
|
5
5
|
//#region src/common/discover/index.ts
|
|
6
6
|
const discover = async (input, options, defaults) => {
|
|
7
|
-
const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
|
|
8
|
-
const
|
|
7
|
+
const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, maxUris = 50, includeInvalid = false, onProgress, onError } = options;
|
|
8
|
+
const safeConcurrency = toPositiveInteger(concurrency, 3);
|
|
9
|
+
const safeMaxUris = toPositiveInteger(maxUris, 50);
|
|
10
|
+
const sourceInput = await normalizeInput(input, fetchFn, onError);
|
|
9
11
|
if (sourceInput.content) {
|
|
10
12
|
const result = await extractFn({
|
|
11
13
|
url: sourceInput.url,
|
|
@@ -24,26 +26,37 @@ const discover = async (input, options, defaults) => {
|
|
|
24
26
|
content: typeof response.body === "string" ? response.body : "",
|
|
25
27
|
headers: response.headers
|
|
26
28
|
};
|
|
27
|
-
} catch {
|
|
29
|
+
} catch (error) {
|
|
30
|
+
onError?.(error, {
|
|
31
|
+
phase: "resolveSiteUrl",
|
|
32
|
+
url: siteUrl
|
|
33
|
+
});
|
|
34
|
+
}
|
|
28
35
|
}
|
|
29
36
|
const urisByMethod = await discoverUris(normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
|
|
30
37
|
const seen = /* @__PURE__ */ new Set();
|
|
31
38
|
const methodGroups = [];
|
|
39
|
+
let remaining = safeMaxUris;
|
|
32
40
|
for (const method of discoverMethodOrder) {
|
|
41
|
+
if (remaining <= 0) break;
|
|
33
42
|
const rawUris = urisByMethod[method];
|
|
34
43
|
if (!rawUris || rawUris.length === 0) continue;
|
|
35
44
|
const unique = rawUris.map((entry) => {
|
|
36
45
|
return normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
|
|
37
46
|
}).filter((entry) => {
|
|
38
|
-
const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
|
|
47
|
+
const key = typeof entry.uri === "string" ? entry.uri : [...entry.uri].sort().join("\0");
|
|
39
48
|
if (seen.has(key)) return false;
|
|
40
49
|
seen.add(key);
|
|
41
50
|
return true;
|
|
42
51
|
});
|
|
43
|
-
if (unique.length > 0)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
52
|
+
if (unique.length > 0) {
|
|
53
|
+
const capped = unique.slice(0, remaining);
|
|
54
|
+
remaining -= capped.length;
|
|
55
|
+
methodGroups.push({
|
|
56
|
+
method,
|
|
57
|
+
entries: capped
|
|
58
|
+
});
|
|
59
|
+
}
|
|
47
60
|
}
|
|
48
61
|
const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
|
|
49
62
|
const results = [];
|
|
@@ -92,7 +105,7 @@ const discover = async (input, options, defaults) => {
|
|
|
92
105
|
for (const { method, entries } of methodGroups) {
|
|
93
106
|
const foundBefore = found;
|
|
94
107
|
await processConcurrently(entries, (entry) => processUri(entry, method), {
|
|
95
|
-
concurrency,
|
|
108
|
+
concurrency: safeConcurrency,
|
|
96
109
|
shouldStop: () => {
|
|
97
110
|
return stopOnFirstResult && found > 0;
|
|
98
111
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
const require_locales = require("../locales.cjs");
|
|
2
|
+
const require_utils = require("../utils.cjs");
|
|
2
3
|
//#region src/common/discover/utils.ts
|
|
3
|
-
const normalizeInput = async (input, fetchFn) => {
|
|
4
|
-
if (
|
|
4
|
+
const normalizeInput = async (input, fetchFn, onError) => {
|
|
5
|
+
if (require_utils.isObject(input)) return input;
|
|
5
6
|
try {
|
|
6
7
|
const response = await fetchFn(input);
|
|
7
8
|
return {
|
|
@@ -9,7 +10,12 @@ const normalizeInput = async (input, fetchFn) => {
|
|
|
9
10
|
content: typeof response.body === "string" ? response.body : void 0,
|
|
10
11
|
headers: response.headers
|
|
11
12
|
};
|
|
12
|
-
} catch {
|
|
13
|
+
} catch (error) {
|
|
14
|
+
onError?.(error, {
|
|
15
|
+
phase: "fetchInput",
|
|
16
|
+
url: input
|
|
17
|
+
});
|
|
18
|
+
}
|
|
13
19
|
return { url: input };
|
|
14
20
|
};
|
|
15
21
|
const getLinkOfType = (links, rel) => {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { errors } from "../locales.js";
|
|
2
|
+
import { isObject } from "../utils.js";
|
|
2
3
|
//#region src/common/discover/utils.ts
|
|
3
|
-
const normalizeInput = async (input, fetchFn) => {
|
|
4
|
-
if (
|
|
4
|
+
const normalizeInput = async (input, fetchFn, onError) => {
|
|
5
|
+
if (isObject(input)) return input;
|
|
5
6
|
try {
|
|
6
7
|
const response = await fetchFn(input);
|
|
7
8
|
return {
|
|
@@ -9,7 +10,12 @@ const normalizeInput = async (input, fetchFn) => {
|
|
|
9
10
|
content: typeof response.body === "string" ? response.body : void 0,
|
|
10
11
|
headers: response.headers
|
|
11
12
|
};
|
|
12
|
-
} catch {
|
|
13
|
+
} catch (error) {
|
|
14
|
+
onError?.(error, {
|
|
15
|
+
phase: "fetchInput",
|
|
16
|
+
url: input
|
|
17
|
+
});
|
|
18
|
+
}
|
|
13
19
|
return { url: input };
|
|
14
20
|
};
|
|
15
21
|
const getLinkOfType = (links, rel) => {
|
package/dist/common/types.d.cts
CHANGED
|
@@ -43,6 +43,11 @@ type DiscoverProgress = {
|
|
|
43
43
|
current: string;
|
|
44
44
|
};
|
|
45
45
|
type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
|
|
46
|
+
type DiscoverErrorContext = {
|
|
47
|
+
phase: 'fetchInput' | 'resolveSiteUrl';
|
|
48
|
+
url?: string;
|
|
49
|
+
};
|
|
50
|
+
type DiscoverOnErrorFn = (error: unknown, context: DiscoverErrorContext) => void;
|
|
46
51
|
type DiscoverResult<TValid = object> = ({
|
|
47
52
|
url: string;
|
|
48
53
|
isValid: true;
|
|
@@ -83,7 +88,9 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
|
|
|
83
88
|
stopOnFirstMethod?: boolean;
|
|
84
89
|
stopOnFirstResult?: boolean;
|
|
85
90
|
concurrency?: number;
|
|
91
|
+
maxUris?: number;
|
|
86
92
|
onProgress?: DiscoverOnProgressFn;
|
|
93
|
+
onError?: DiscoverOnErrorFn;
|
|
87
94
|
includeInvalid?: boolean;
|
|
88
95
|
};
|
|
89
96
|
//#endregion
|
package/dist/common/types.d.ts
CHANGED
|
@@ -43,6 +43,11 @@ type DiscoverProgress = {
|
|
|
43
43
|
current: string;
|
|
44
44
|
};
|
|
45
45
|
type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
|
|
46
|
+
type DiscoverErrorContext = {
|
|
47
|
+
phase: 'fetchInput' | 'resolveSiteUrl';
|
|
48
|
+
url?: string;
|
|
49
|
+
};
|
|
50
|
+
type DiscoverOnErrorFn = (error: unknown, context: DiscoverErrorContext) => void;
|
|
46
51
|
type DiscoverResult<TValid = object> = ({
|
|
47
52
|
url: string;
|
|
48
53
|
isValid: true;
|
|
@@ -83,7 +88,9 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
|
|
|
83
88
|
stopOnFirstMethod?: boolean;
|
|
84
89
|
stopOnFirstResult?: boolean;
|
|
85
90
|
concurrency?: number;
|
|
91
|
+
maxUris?: number;
|
|
86
92
|
onProgress?: DiscoverOnProgressFn;
|
|
93
|
+
onError?: DiscoverOnErrorFn;
|
|
87
94
|
includeInvalid?: boolean;
|
|
88
95
|
};
|
|
89
96
|
//#endregion
|
|
@@ -1,25 +1,15 @@
|
|
|
1
|
+
const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
|
|
1
2
|
const require_utils = require("../../utils.cjs");
|
|
3
|
+
let http_link_header = require("http-link-header");
|
|
4
|
+
http_link_header = require_runtime.__toESM(http_link_header, 1);
|
|
2
5
|
//#region src/common/uris/headers/index.ts
|
|
3
|
-
const linkSplitRegex = /,(?=\s*<)/;
|
|
4
|
-
const urlRegex = /<([^<>]+)>/;
|
|
5
|
-
const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
|
|
6
|
-
const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
|
|
7
6
|
const discoverUrisFromHeaders = (headers, options) => {
|
|
8
|
-
const uris = /* @__PURE__ */ new Set();
|
|
9
7
|
const linkHeader = headers.get("link");
|
|
10
8
|
if (!linkHeader) return [];
|
|
11
|
-
const
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
const typeMatch = link.match(typeRegex);
|
|
16
|
-
if (!urlMatch) continue;
|
|
17
|
-
const url = urlMatch[1];
|
|
18
|
-
const rel = relMatch?.[1]?.toLowerCase();
|
|
19
|
-
const type = typeMatch?.[1];
|
|
20
|
-
if (!rel) continue;
|
|
21
|
-
if (require_utils.matchesAnyOfLinkSelectors(rel, type, options.linkSelectors)) uris.add(url);
|
|
22
|
-
}
|
|
9
|
+
const uris = /* @__PURE__ */ new Set();
|
|
10
|
+
try {
|
|
11
|
+
for (const ref of http_link_header.default.parse(linkHeader).refs) if (ref.rel && require_utils.matchesAnyOfLinkSelectors(ref.rel, ref.type, options.linkSelectors)) uris.add(ref.uri);
|
|
12
|
+
} catch {}
|
|
23
13
|
return [...uris];
|
|
24
14
|
};
|
|
25
15
|
//#endregion
|
|
@@ -1,25 +1,13 @@
|
|
|
1
1
|
import { matchesAnyOfLinkSelectors } from "../../utils.js";
|
|
2
|
+
import LinkHeader from "http-link-header";
|
|
2
3
|
//#region src/common/uris/headers/index.ts
|
|
3
|
-
const linkSplitRegex = /,(?=\s*<)/;
|
|
4
|
-
const urlRegex = /<([^<>]+)>/;
|
|
5
|
-
const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
|
|
6
|
-
const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
|
|
7
4
|
const discoverUrisFromHeaders = (headers, options) => {
|
|
8
|
-
const uris = /* @__PURE__ */ new Set();
|
|
9
5
|
const linkHeader = headers.get("link");
|
|
10
6
|
if (!linkHeader) return [];
|
|
11
|
-
const
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
const typeMatch = link.match(typeRegex);
|
|
16
|
-
if (!urlMatch) continue;
|
|
17
|
-
const url = urlMatch[1];
|
|
18
|
-
const rel = relMatch?.[1]?.toLowerCase();
|
|
19
|
-
const type = typeMatch?.[1];
|
|
20
|
-
if (!rel) continue;
|
|
21
|
-
if (matchesAnyOfLinkSelectors(rel, type, options.linkSelectors)) uris.add(url);
|
|
22
|
-
}
|
|
7
|
+
const uris = /* @__PURE__ */ new Set();
|
|
8
|
+
try {
|
|
9
|
+
for (const ref of LinkHeader.parse(linkHeader).refs) if (ref.rel && matchesAnyOfLinkSelectors(ref.rel, ref.type, options.linkSelectors)) uris.add(ref.uri);
|
|
10
|
+
} catch {}
|
|
23
11
|
return [...uris];
|
|
24
12
|
};
|
|
25
13
|
//#endregion
|
package/dist/common/utils.cjs
CHANGED
|
@@ -5,6 +5,15 @@ const composeHint = (key) => ({
|
|
|
5
5
|
key,
|
|
6
6
|
label: require_locales.hints[key]
|
|
7
7
|
});
|
|
8
|
+
const isSuccessfulStatus = (status) => {
|
|
9
|
+
return status === void 0 || status >= 200 && status < 300;
|
|
10
|
+
};
|
|
11
|
+
const isObject = (value) => {
|
|
12
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
13
|
+
};
|
|
14
|
+
const toPositiveInteger = (value, fallback) => {
|
|
15
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 1 ? value : fallback;
|
|
16
|
+
};
|
|
8
17
|
const normalizeMimeType = (type) => {
|
|
9
18
|
return type.split(";")[0].trim().toLowerCase();
|
|
10
19
|
};
|
|
@@ -69,7 +78,7 @@ const matchesAnyOfLinkSelectors = (rel, type, selectors) => {
|
|
|
69
78
|
});
|
|
70
79
|
};
|
|
71
80
|
const processConcurrently = async (items, processFn, options) => {
|
|
72
|
-
if (options.concurrency
|
|
81
|
+
if (!(options.concurrency >= 1)) return;
|
|
73
82
|
const active = /* @__PURE__ */ new Set();
|
|
74
83
|
let index = 0;
|
|
75
84
|
while (index < items.length || active.size > 0) {
|
|
@@ -92,7 +101,10 @@ exports.hasMetaContent = hasMetaContent;
|
|
|
92
101
|
exports.includesAnyOf = includesAnyOf;
|
|
93
102
|
exports.isAnyOf = isAnyOf;
|
|
94
103
|
exports.isHostOf = isHostOf;
|
|
104
|
+
exports.isObject = isObject;
|
|
95
105
|
exports.isSubdomainOf = isSubdomainOf;
|
|
106
|
+
exports.isSuccessfulStatus = isSuccessfulStatus;
|
|
96
107
|
exports.matchesAnyOfLinkSelectors = matchesAnyOfLinkSelectors;
|
|
97
108
|
exports.omitEmpty = omitEmpty;
|
|
98
109
|
exports.processConcurrently = processConcurrently;
|
|
110
|
+
exports.toPositiveInteger = toPositiveInteger;
|
package/dist/common/utils.js
CHANGED
|
@@ -5,6 +5,15 @@ const composeHint = (key) => ({
|
|
|
5
5
|
key,
|
|
6
6
|
label: hints[key]
|
|
7
7
|
});
|
|
8
|
+
const isSuccessfulStatus = (status) => {
|
|
9
|
+
return status === void 0 || status >= 200 && status < 300;
|
|
10
|
+
};
|
|
11
|
+
const isObject = (value) => {
|
|
12
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
13
|
+
};
|
|
14
|
+
const toPositiveInteger = (value, fallback) => {
|
|
15
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 1 ? value : fallback;
|
|
16
|
+
};
|
|
8
17
|
const normalizeMimeType = (type) => {
|
|
9
18
|
return type.split(";")[0].trim().toLowerCase();
|
|
10
19
|
};
|
|
@@ -69,7 +78,7 @@ const matchesAnyOfLinkSelectors = (rel, type, selectors) => {
|
|
|
69
78
|
});
|
|
70
79
|
};
|
|
71
80
|
const processConcurrently = async (items, processFn, options) => {
|
|
72
|
-
if (options.concurrency
|
|
81
|
+
if (!(options.concurrency >= 1)) return;
|
|
73
82
|
const active = /* @__PURE__ */ new Set();
|
|
74
83
|
let index = 0;
|
|
75
84
|
while (index < items.length || active.size > 0) {
|
|
@@ -85,4 +94,4 @@ const processConcurrently = async (items, processFn, options) => {
|
|
|
85
94
|
}
|
|
86
95
|
};
|
|
87
96
|
//#endregion
|
|
88
|
-
export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently };
|
|
97
|
+
export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isObject, isSubdomainOf, isSuccessfulStatus, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently, toPositiveInteger };
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
const require_utils = require("../common/utils.cjs");
|
|
1
2
|
//#region src/favicons/extractors.ts
|
|
2
3
|
const isImageContentType = (headers) => {
|
|
3
4
|
return headers?.get("content-type")?.startsWith("image/") ?? false;
|
|
@@ -8,17 +9,11 @@ const isImageContent = (content) => {
|
|
|
8
9
|
const head = trimmed.slice(0, 200);
|
|
9
10
|
return trimmed.startsWith("<svg") || trimmed.startsWith("<?xml") && head.includes("<svg") || content.slice(1, 4) === "PNG" || content.startsWith("GIF8") || content.startsWith("RIFF") && content.includes("WEBP");
|
|
10
11
|
};
|
|
11
|
-
const isSuccessStatus = (status) => {
|
|
12
|
-
return status !== void 0 && status >= 200 && status < 400;
|
|
13
|
-
};
|
|
14
12
|
const defaultExtractFn = (input) => {
|
|
15
|
-
|
|
16
|
-
url: input.url,
|
|
17
|
-
isValid: true
|
|
18
|
-
};
|
|
13
|
+
const isImage = isImageContentType(input.headers) || isImageContent(input.content);
|
|
19
14
|
return {
|
|
20
15
|
url: input.url,
|
|
21
|
-
isValid:
|
|
16
|
+
isValid: isImage && require_utils.isSuccessfulStatus(input.status)
|
|
22
17
|
};
|
|
23
18
|
};
|
|
24
19
|
//#endregion
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isSuccessfulStatus } from "../common/utils.js";
|
|
1
2
|
//#region src/favicons/extractors.ts
|
|
2
3
|
const isImageContentType = (headers) => {
|
|
3
4
|
return headers?.get("content-type")?.startsWith("image/") ?? false;
|
|
@@ -8,17 +9,11 @@ const isImageContent = (content) => {
|
|
|
8
9
|
const head = trimmed.slice(0, 200);
|
|
9
10
|
return trimmed.startsWith("<svg") || trimmed.startsWith("<?xml") && head.includes("<svg") || content.slice(1, 4) === "PNG" || content.startsWith("GIF8") || content.startsWith("RIFF") && content.includes("WEBP");
|
|
10
11
|
};
|
|
11
|
-
const isSuccessStatus = (status) => {
|
|
12
|
-
return status !== void 0 && status >= 200 && status < 400;
|
|
13
|
-
};
|
|
14
12
|
const defaultExtractFn = (input) => {
|
|
15
|
-
|
|
16
|
-
url: input.url,
|
|
17
|
-
isValid: true
|
|
18
|
-
};
|
|
13
|
+
const isImage = isImageContentType(input.headers) || isImageContent(input.content);
|
|
19
14
|
return {
|
|
20
15
|
url: input.url,
|
|
21
|
-
isValid:
|
|
16
|
+
isValid: isImage && isSuccessfulStatus(input.status)
|
|
22
17
|
};
|
|
23
18
|
};
|
|
24
19
|
//#endregion
|
|
@@ -1,16 +1,17 @@
|
|
|
1
|
-
const require_utils = require("../common/
|
|
1
|
+
const require_utils = require("../common/utils.cjs");
|
|
2
|
+
const require_utils$1 = require("../common/discover/utils.cjs");
|
|
2
3
|
const require_defaults = require("../common/discover/defaults.cjs");
|
|
3
4
|
let feedsmith = require("feedsmith");
|
|
4
5
|
//#region src/feeds/extractors.ts
|
|
5
|
-
const defaultExtractFn = ({ content, url }) => {
|
|
6
|
-
if (!content) return {
|
|
6
|
+
const defaultExtractFn = ({ content, url, status }) => {
|
|
7
|
+
if (!content || !require_utils.isSuccessfulStatus(status)) return {
|
|
7
8
|
url,
|
|
8
9
|
isValid: false
|
|
9
10
|
};
|
|
10
11
|
try {
|
|
11
12
|
const parsed = (0, feedsmith.parseFeed)(content);
|
|
12
13
|
const { format, feed } = parsed;
|
|
13
|
-
const rawSiteUrl = require_utils.getFeedSiteUrl(parsed);
|
|
14
|
+
const rawSiteUrl = require_utils$1.getFeedSiteUrl(parsed);
|
|
14
15
|
const siteUrl = rawSiteUrl ? require_defaults.defaultResolveUrlFn(rawSiteUrl, url) : void 0;
|
|
15
16
|
if (format === "rss" || format === "rdf") return {
|
|
16
17
|
url,
|
package/dist/feeds/extractors.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { isSuccessfulStatus } from "../common/utils.js";
|
|
1
2
|
import { getFeedSiteUrl } from "../common/discover/utils.js";
|
|
2
3
|
import { defaultResolveUrlFn } from "../common/discover/defaults.js";
|
|
3
4
|
import { parseFeed } from "feedsmith";
|
|
4
5
|
//#region src/feeds/extractors.ts
|
|
5
|
-
const defaultExtractFn = ({ content, url }) => {
|
|
6
|
-
if (!content) return {
|
|
6
|
+
const defaultExtractFn = ({ content, url, status }) => {
|
|
7
|
+
if (!content || !isSuccessfulStatus(status)) return {
|
|
7
8
|
url,
|
|
8
9
|
isValid: false
|
|
9
10
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const require_utils = require("../../../common/utils.cjs");
|
|
2
2
|
//#region src/feeds/platform/handlers/artstation.ts
|
|
3
3
|
const hosts = ["artstation.com", "www.artstation.com"];
|
|
4
|
-
const
|
|
4
|
+
const domainSuffixRegex = /\.artstation\.com$/i;
|
|
5
5
|
const excludedPaths = [
|
|
6
6
|
"blogs",
|
|
7
7
|
"channels",
|
|
@@ -24,7 +24,7 @@ const artstationHandler = {
|
|
|
24
24
|
resolve: (url) => {
|
|
25
25
|
const parsed = new URL(url);
|
|
26
26
|
if (!require_utils.isHostOf(url, hosts) && require_utils.isSubdomainOf(url, "artstation.com")) return [{
|
|
27
|
-
uri: `https://www.artstation.com/${parsed.hostname.replace(
|
|
27
|
+
uri: `https://www.artstation.com/${parsed.hostname.replace(domainSuffixRegex, "")}.rss`,
|
|
28
28
|
hint: require_utils.composeHint("artstation:portfolio")
|
|
29
29
|
}];
|
|
30
30
|
const pathSegments = parsed.pathname.split("/").filter(Boolean);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { composeHint, isAnyOf, isHostOf, isSubdomainOf } from "../../../common/utils.js";
|
|
2
2
|
//#region src/feeds/platform/handlers/artstation.ts
|
|
3
3
|
const hosts = ["artstation.com", "www.artstation.com"];
|
|
4
|
-
const
|
|
4
|
+
const domainSuffixRegex = /\.artstation\.com$/i;
|
|
5
5
|
const excludedPaths = [
|
|
6
6
|
"blogs",
|
|
7
7
|
"channels",
|
|
@@ -24,7 +24,7 @@ const artstationHandler = {
|
|
|
24
24
|
resolve: (url) => {
|
|
25
25
|
const parsed = new URL(url);
|
|
26
26
|
if (!isHostOf(url, hosts) && isSubdomainOf(url, "artstation.com")) return [{
|
|
27
|
-
uri: `https://www.artstation.com/${parsed.hostname.replace(
|
|
27
|
+
uri: `https://www.artstation.com/${parsed.hostname.replace(domainSuffixRegex, "")}.rss`,
|
|
28
28
|
hint: composeHint("artstation:portfolio")
|
|
29
29
|
}];
|
|
30
30
|
const pathSegments = parsed.pathname.split("/").filter(Boolean);
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
const require_utils = require("../../../common/utils.cjs");
|
|
2
2
|
//#region src/feeds/platform/handlers/fireside.ts
|
|
3
|
-
const
|
|
3
|
+
const domainSuffixRegex = /\.fireside\.fm$/i;
|
|
4
4
|
const firesideHandler = {
|
|
5
5
|
match: (url) => {
|
|
6
6
|
return require_utils.isSubdomainOf(url, "fireside.fm");
|
|
7
7
|
},
|
|
8
8
|
resolve: (url) => {
|
|
9
9
|
const { hostname } = new URL(url);
|
|
10
|
-
const slug = hostname.replace(
|
|
10
|
+
const slug = hostname.replace(domainSuffixRegex, "");
|
|
11
11
|
const uris = [];
|
|
12
12
|
uris.push({
|
|
13
13
|
uri: `https://feeds.fireside.fm/${slug}/rss`,
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { composeHint, isSubdomainOf } from "../../../common/utils.js";
|
|
2
2
|
//#region src/feeds/platform/handlers/fireside.ts
|
|
3
|
-
const
|
|
3
|
+
const domainSuffixRegex = /\.fireside\.fm$/i;
|
|
4
4
|
const firesideHandler = {
|
|
5
5
|
match: (url) => {
|
|
6
6
|
return isSubdomainOf(url, "fireside.fm");
|
|
7
7
|
},
|
|
8
8
|
resolve: (url) => {
|
|
9
9
|
const { hostname } = new URL(url);
|
|
10
|
-
const slug = hostname.replace(
|
|
10
|
+
const slug = hostname.replace(domainSuffixRegex, "");
|
|
11
11
|
const uris = [];
|
|
12
12
|
uris.push({
|
|
13
13
|
uri: `https://feeds.fireside.fm/${slug}/rss`,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const require_utils = require("../../../common/utils.cjs");
|
|
2
2
|
//#region src/feeds/platform/handlers/podbean.ts
|
|
3
|
-
const
|
|
3
|
+
const domainSuffixRegex = /\.podbean\.com$/i;
|
|
4
4
|
const reservedSlugs = new Set([
|
|
5
5
|
"www",
|
|
6
6
|
"feed",
|
|
@@ -14,13 +14,13 @@ const reservedSlugs = new Set([
|
|
|
14
14
|
const podbeanHandler = {
|
|
15
15
|
match: (url) => {
|
|
16
16
|
if (!require_utils.isSubdomainOf(url, "podbean.com")) return false;
|
|
17
|
-
const slug = new URL(url).hostname.toLowerCase().replace(
|
|
17
|
+
const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
|
|
18
18
|
return !reservedSlugs.has(slug);
|
|
19
19
|
},
|
|
20
20
|
resolve: (url) => {
|
|
21
21
|
const { hostname } = new URL(url);
|
|
22
22
|
return [{
|
|
23
|
-
uri: `https://feed.podbean.com/${hostname.replace(
|
|
23
|
+
uri: `https://feed.podbean.com/${hostname.replace(domainSuffixRegex, "")}/feed.xml`,
|
|
24
24
|
hint: require_utils.composeHint("podbean:podcast")
|
|
25
25
|
}];
|
|
26
26
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { composeHint, isSubdomainOf } from "../../../common/utils.js";
|
|
2
2
|
//#region src/feeds/platform/handlers/podbean.ts
|
|
3
|
-
const
|
|
3
|
+
const domainSuffixRegex = /\.podbean\.com$/i;
|
|
4
4
|
const reservedSlugs = new Set([
|
|
5
5
|
"www",
|
|
6
6
|
"feed",
|
|
@@ -14,13 +14,13 @@ const reservedSlugs = new Set([
|
|
|
14
14
|
const podbeanHandler = {
|
|
15
15
|
match: (url) => {
|
|
16
16
|
if (!isSubdomainOf(url, "podbean.com")) return false;
|
|
17
|
-
const slug = new URL(url).hostname.toLowerCase().replace(
|
|
17
|
+
const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
|
|
18
18
|
return !reservedSlugs.has(slug);
|
|
19
19
|
},
|
|
20
20
|
resolve: (url) => {
|
|
21
21
|
const { hostname } = new URL(url);
|
|
22
22
|
return [{
|
|
23
|
-
uri: `https://feed.podbean.com/${hostname.replace(
|
|
23
|
+
uri: `https://feed.podbean.com/${hostname.replace(domainSuffixRegex, "")}/feed.xml`,
|
|
24
24
|
hint: composeHint("podbean:podcast")
|
|
25
25
|
}];
|
|
26
26
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const require_utils = require("../../../common/utils.cjs");
|
|
2
2
|
//#region src/feeds/platform/handlers/podigee.ts
|
|
3
|
-
const
|
|
3
|
+
const domainSuffixRegex = /\.podigee\.io$/i;
|
|
4
4
|
const reservedSlugs = new Set([
|
|
5
5
|
"www",
|
|
6
6
|
"app",
|
|
@@ -14,7 +14,7 @@ const reservedSlugs = new Set([
|
|
|
14
14
|
const podigeeHandler = {
|
|
15
15
|
match: (url) => {
|
|
16
16
|
if (!require_utils.isSubdomainOf(url, "podigee.io")) return false;
|
|
17
|
-
const slug = new URL(url).hostname.toLowerCase().replace(
|
|
17
|
+
const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
|
|
18
18
|
return !reservedSlugs.has(slug);
|
|
19
19
|
},
|
|
20
20
|
resolve: (url) => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { composeHint, isSubdomainOf } from "../../../common/utils.js";
|
|
2
2
|
//#region src/feeds/platform/handlers/podigee.ts
|
|
3
|
-
const
|
|
3
|
+
const domainSuffixRegex = /\.podigee\.io$/i;
|
|
4
4
|
const reservedSlugs = new Set([
|
|
5
5
|
"www",
|
|
6
6
|
"app",
|
|
@@ -14,7 +14,7 @@ const reservedSlugs = new Set([
|
|
|
14
14
|
const podigeeHandler = {
|
|
15
15
|
match: (url) => {
|
|
16
16
|
if (!isSubdomainOf(url, "podigee.io")) return false;
|
|
17
|
-
const slug = new URL(url).hostname.toLowerCase().replace(
|
|
17
|
+
const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
|
|
18
18
|
return !reservedSlugs.has(slug);
|
|
19
19
|
},
|
|
20
20
|
resolve: (url) => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const require_utils = require("../../../common/utils.cjs");
|
|
2
2
|
//#region src/feeds/platform/handlers/transistor.ts
|
|
3
|
-
const
|
|
3
|
+
const domainSuffixRegex = /\.transistor\.fm$/i;
|
|
4
4
|
const reservedSlugs = new Set([
|
|
5
5
|
"www",
|
|
6
6
|
"feeds",
|
|
@@ -14,13 +14,13 @@ const reservedSlugs = new Set([
|
|
|
14
14
|
const transistorHandler = {
|
|
15
15
|
match: (url) => {
|
|
16
16
|
if (!require_utils.isSubdomainOf(url, "transistor.fm")) return false;
|
|
17
|
-
const slug = new URL(url).hostname.toLowerCase().replace(
|
|
17
|
+
const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
|
|
18
18
|
return !reservedSlugs.has(slug);
|
|
19
19
|
},
|
|
20
20
|
resolve: (url) => {
|
|
21
21
|
const { hostname } = new URL(url);
|
|
22
22
|
return [{
|
|
23
|
-
uri: `https://feeds.transistor.fm/${hostname.replace(
|
|
23
|
+
uri: `https://feeds.transistor.fm/${hostname.replace(domainSuffixRegex, "")}`,
|
|
24
24
|
hint: require_utils.composeHint("transistor:podcast")
|
|
25
25
|
}];
|
|
26
26
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { composeHint, isSubdomainOf } from "../../../common/utils.js";
|
|
2
2
|
//#region src/feeds/platform/handlers/transistor.ts
|
|
3
|
-
const
|
|
3
|
+
const domainSuffixRegex = /\.transistor\.fm$/i;
|
|
4
4
|
const reservedSlugs = new Set([
|
|
5
5
|
"www",
|
|
6
6
|
"feeds",
|
|
@@ -14,13 +14,13 @@ const reservedSlugs = new Set([
|
|
|
14
14
|
const transistorHandler = {
|
|
15
15
|
match: (url) => {
|
|
16
16
|
if (!isSubdomainOf(url, "transistor.fm")) return false;
|
|
17
|
-
const slug = new URL(url).hostname.toLowerCase().replace(
|
|
17
|
+
const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
|
|
18
18
|
return !reservedSlugs.has(slug);
|
|
19
19
|
},
|
|
20
20
|
resolve: (url) => {
|
|
21
21
|
const { hostname } = new URL(url);
|
|
22
22
|
return [{
|
|
23
|
-
uri: `https://feeds.transistor.fm/${hostname.replace(
|
|
23
|
+
uri: `https://feeds.transistor.fm/${hostname.replace(domainSuffixRegex, "")}`,
|
|
24
24
|
hint: composeHint("transistor:podcast")
|
|
25
25
|
}];
|
|
26
26
|
}
|
package/package.json
CHANGED
|
@@ -108,13 +108,15 @@
|
|
|
108
108
|
},
|
|
109
109
|
"dependencies": {
|
|
110
110
|
"feedsmith": "^2.9.4",
|
|
111
|
-
"htmlparser2": "^10.1.0"
|
|
111
|
+
"htmlparser2": "^10.1.0",
|
|
112
|
+
"http-link-header": "^1.1.3"
|
|
112
113
|
},
|
|
113
114
|
"devDependencies": {
|
|
114
115
|
"@types/bun": "^1.3.14",
|
|
116
|
+
"@types/http-link-header": "^1.0.7",
|
|
115
117
|
"kvalita": "1.15.1",
|
|
116
118
|
"tsdown": "^0.22.2",
|
|
117
119
|
"vitepress": "^2.0.0-alpha.17"
|
|
118
120
|
},
|
|
119
|
-
"version": "1.
|
|
121
|
+
"version": "1.10.0"
|
|
120
122
|
}
|