feedscout 2.0.0-beta.3 → 2.0.0-beta.4

package/dist/blogrolls/extractors.js

@@ -1,7 +1,8 @@
+import { isSuccessfulStatus } from "../common/utils.js";
 import { parseOpml } from "feedsmith";
 //#region src/blogrolls/extractors.ts
-const defaultExtractFn = ({ content, url }) => {
-	if (!content) return {
+const defaultExtractFn = ({ content, url, status }) => {
+	if (!content || !isSuccessfulStatus(status)) return {
 		url,
 		isValid: false
 	};

package/dist/common/discover/index.js

@@ -1,11 +1,13 @@
+import { processConcurrently, toPositiveInteger } from "../utils.js";
 import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
 import { discoverMethodOrder } from "../types.js";
-import { processConcurrently } from "../utils.js";
 import { discoverUris } from "../uris/index.js";
 //#region src/common/discover/index.ts
 const discover = async (input, options, defaults) => {
-	const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
-	const sourceInput = await normalizeInput(input, fetchFn);
+	const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, maxUris = 50, includeInvalid = false, onProgress, onError } = options;
+	const safeConcurrency = toPositiveInteger(concurrency, 3);
+	const safeMaxUris = toPositiveInteger(maxUris, 50);
+	const sourceInput = await normalizeInput(input, fetchFn, onError);
 	if (sourceInput.content) {
 		const result = await extractFn({
 			url: sourceInput.url,
@@ -24,26 +26,37 @@ const discover = async (input, options, defaults) => {
 				content: typeof response.body === "string" ? response.body : "",
 				headers: response.headers
 			};
-		} catch {}
+		} catch (error) {
+			onError?.(error, {
+				phase: "resolveSiteUrl",
+				url: siteUrl
+			});
+		}
 	}
 	const urisByMethod = await discoverUris(normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
 	const seen = /* @__PURE__ */ new Set();
 	const methodGroups = [];
+	let remaining = safeMaxUris;
 	for (const method of discoverMethodOrder) {
+		if (remaining <= 0) break;
 		const rawUris = urisByMethod[method];
 		if (!rawUris || rawUris.length === 0) continue;
 		const unique = rawUris.map((entry) => {
 			return normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
 		}).filter((entry) => {
-			const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
+			const key = typeof entry.uri === "string" ? entry.uri : [...entry.uri].sort().join("\0");
 			if (seen.has(key)) return false;
 			seen.add(key);
 			return true;
 		});
-		if (unique.length > 0) methodGroups.push({
-			method,
-			entries: unique
-		});
+		if (unique.length > 0) {
+			const capped = unique.slice(0, remaining);
+			remaining -= capped.length;
+			methodGroups.push({
+				method,
+				entries: capped
+			});
+		}
 	}
 	const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
 	const results = [];
@@ -92,7 +105,7 @@ const discover = async (input, options, defaults) => {
 	for (const { method, entries } of methodGroups) {
 		const foundBefore = found;
 		await processConcurrently(entries, (entry) => processUri(entry, method), {
-			concurrency,
+			concurrency: safeConcurrency,
 			shouldStop: () => {
 				return stopOnFirstResult && found > 0;
 			}

package/dist/common/discover/utils.js

@@ -1,7 +1,8 @@
 import { errors } from "../locales.js";
+import { isObject } from "../utils.js";
 //#region src/common/discover/utils.ts
-const normalizeInput = async (input, fetchFn) => {
-	if (typeof input === "object") return input;
+const normalizeInput = async (input, fetchFn, onError) => {
+	if (isObject(input)) return input;
 	try {
 		const response = await fetchFn(input);
 		return {
@@ -9,7 +10,12 @@ const normalizeInput = async (input, fetchFn) => {
 			content: typeof response.body === "string" ? response.body : void 0,
 			headers: response.headers
 		};
-	} catch {}
+	} catch (error) {
+		onError?.(error, {
+			phase: "fetchInput",
+			url: input
+		});
+	}
 	return { url: input };
 };
 const getLinkOfType = (links, rel) => {

package/dist/common/types.d.ts

@@ -43,6 +43,11 @@ type DiscoverProgress = {
   current: string;
 };
 type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
+type DiscoverErrorContext = {
+  phase: 'fetchInput' | 'resolveSiteUrl';
+  url?: string;
+};
+type DiscoverOnErrorFn = (error: unknown, context: DiscoverErrorContext) => void;
 type DiscoverResult<TValid = object> = ({
   url: string;
   isValid: true;
@@ -83,7 +88,9 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
   stopOnFirstMethod?: boolean;
   stopOnFirstResult?: boolean;
   concurrency?: number;
+  maxUris?: number;
   onProgress?: DiscoverOnProgressFn;
+  onError?: DiscoverOnErrorFn;
   includeInvalid?: boolean;
 };
 //#endregion

package/dist/common/uris/headers/index.js

@@ -1,25 +1,13 @@
 import { matchesAnyOfLinkSelectors } from "../../utils.js";
+import LinkHeader from "http-link-header";
 //#region src/common/uris/headers/index.ts
-const linkSplitRegex = /,(?=\s*<)/;
-const urlRegex = /<([^<>]+)>/;
-const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
-const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
 const discoverUrisFromHeaders = (headers, options) => {
-	const uris = /* @__PURE__ */ new Set();
 	const linkHeader = headers.get("link");
 	if (!linkHeader) return [];
-	const links = linkHeader.split(linkSplitRegex);
-	for (const link of links) {
-		const urlMatch = link.match(urlRegex);
-		const relMatch = link.match(relRegex);
-		const typeMatch = link.match(typeRegex);
-		if (!urlMatch) continue;
-		const url = urlMatch[1];
-		const rel = relMatch?.[1]?.toLowerCase();
-		const type = typeMatch?.[1];
-		if (!rel) continue;
-		if (matchesAnyOfLinkSelectors(rel, type, options.linkSelectors)) uris.add(url);
-	}
+	const uris = /* @__PURE__ */ new Set();
+	try {
+		for (const ref of LinkHeader.parse(linkHeader).refs) if (ref.rel && matchesAnyOfLinkSelectors(ref.rel, ref.type, options.linkSelectors)) uris.add(ref.uri);
+	} catch {}
 	return [...uris];
 };
 //#endregion

package/dist/common/utils.js

@@ -5,6 +5,15 @@ const composeHint = (key) => ({
 	key,
 	label: hints[key]
 });
+const isSuccessfulStatus = (status) => {
+	return status === void 0 || status >= 200 && status < 300;
+};
+const isObject = (value) => {
+	return value !== null && typeof value === "object" && !Array.isArray(value);
+};
+const toPositiveInteger = (value, fallback) => {
+	return typeof value === "number" && Number.isInteger(value) && value >= 1 ? value : fallback;
+};
 const normalizeMimeType = (type) => {
 	return type.split(";")[0].trim().toLowerCase();
 };
@@ -69,7 +78,7 @@ const matchesAnyOfLinkSelectors = (rel, type, selectors) => {
 	});
 };
 const processConcurrently = async (items, processFn, options) => {
-	if (options.concurrency < 1) return;
+	if (!(options.concurrency >= 1)) return;
 	const active = /* @__PURE__ */ new Set();
 	let index = 0;
 	while (index < items.length || active.size > 0) {
@@ -85,4 +94,4 @@ const processConcurrently = async (items, processFn, options) => {
 	}
 };
 //#endregion
-export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently };
+export { anyWordMatchesAnyOf, composeHint, endsWithAnyOf, hasMetaContent, includesAnyOf, isAnyOf, isHostOf, isObject, isSubdomainOf, isSuccessfulStatus, matchesAnyOfLinkSelectors, omitEmpty, processConcurrently, toPositiveInteger };

package/dist/favicons/extractors.js

@@ -1,3 +1,4 @@
+import { isSuccessfulStatus } from "../common/utils.js";
 //#region src/favicons/extractors.ts
 const isImageContentType = (headers) => {
 	return headers?.get("content-type")?.startsWith("image/") ?? false;
@@ -8,17 +9,11 @@ const isImageContent = (content) => {
 	const head = trimmed.slice(0, 200);
 	return trimmed.startsWith("<svg") || trimmed.startsWith("<?xml") && head.includes("<svg") || content.slice(1, 4) === "PNG" || content.startsWith("GIF8") || content.startsWith("RIFF") && content.includes("WEBP");
 };
-const isSuccessStatus = (status) => {
-	return status !== void 0 && status >= 200 && status < 400;
-};
 const defaultExtractFn = (input) => {
-	if (isImageContentType(input.headers) || isImageContent(input.content) || isSuccessStatus(input.status)) return {
-		url: input.url,
-		isValid: true
-	};
+	const isImage = isImageContentType(input.headers) || isImageContent(input.content);
 	return {
 		url: input.url,
-		isValid: false
+		isValid: isImage && isSuccessfulStatus(input.status)
 	};
 };
 //#endregion

package/dist/feeds/extractors.js

@@ -1,9 +1,10 @@
+import { isSuccessfulStatus } from "../common/utils.js";
 import { getFeedSiteUrl } from "../common/discover/utils.js";
 import { defaultResolveUrlFn } from "../common/discover/defaults.js";
 import { parseFeed } from "feedsmith";
 //#region src/feeds/extractors.ts
-const defaultExtractFn = ({ content, url }) => {
-	if (!content) return {
+const defaultExtractFn = ({ content, url, status }) => {
+	if (!content || !isSuccessfulStatus(status)) return {
 		url,
 		isValid: false
 	};

package/dist/feeds/platform/handlers/artstation.js

@@ -1,7 +1,7 @@
 import { composeHint, isAnyOf, isHostOf, isSubdomainOf } from "../../../common/utils.js";
 //#region src/feeds/platform/handlers/artstation.ts
 const hosts = ["artstation.com", "www.artstation.com"];
-const domainSuffix = /\.artstation\.com$/i;
+const domainSuffixRegex = /\.artstation\.com$/i;
 const excludedPaths = [
 	"blogs",
 	"channels",
@@ -24,7 +24,7 @@ const artstationHandler = {
 	resolve: (url) => {
 		const parsed = new URL(url);
 		if (!isHostOf(url, hosts) && isSubdomainOf(url, "artstation.com")) return [{
-			uri: `https://www.artstation.com/${parsed.hostname.replace(domainSuffix, "")}.rss`,
+			uri: `https://www.artstation.com/${parsed.hostname.replace(domainSuffixRegex, "")}.rss`,
 			hint: composeHint("artstation:portfolio")
 		}];
 		const pathSegments = parsed.pathname.split("/").filter(Boolean);

package/dist/feeds/platform/handlers/discourse.js

@@ -4,7 +4,7 @@ const userRegex = /^\/u\/([^/]+)/;
 const categoryRegex = /^\/c\/(.+?)\/?$/;
 const topicRegex = /^\/t\/([^/]+)\/(\d+)/;
 const topRegex = /^\/top(?:\/([^/]+))?\/?$/;
-const validTopPeriods = new Set([
+const validTopPeriods = /* @__PURE__ */ new Set([
 	"daily",
 	"weekly",
 	"monthly",

package/dist/feeds/platform/handlers/fireside.js

@@ -1,13 +1,13 @@
 import { composeHint, isSubdomainOf } from "../../../common/utils.js";
 //#region src/feeds/platform/handlers/fireside.ts
-const domainSuffix = /\.fireside\.fm$/i;
+const domainSuffixRegex = /\.fireside\.fm$/i;
 const firesideHandler = {
 	match: (url) => {
 		return isSubdomainOf(url, "fireside.fm");
 	},
 	resolve: (url) => {
 		const { hostname } = new URL(url);
-		const slug = hostname.replace(domainSuffix, "");
+		const slug = hostname.replace(domainSuffixRegex, "");
 		const uris = [];
 		uris.push({
 			uri: `https://feeds.fireside.fm/${slug}/rss`,

package/dist/feeds/platform/handlers/lemmy.js

@@ -1,7 +1,7 @@
 import { composeHint, hasMetaContent } from "../../../common/utils.js";
 //#region src/feeds/platform/handlers/lemmy.ts
 const lemmyPoweredByRegex = /lemmy/i;
-const validSorts = new Set([
+const validSorts = /* @__PURE__ */ new Set([
 	"Active",
 	"Hot",
 	"New",

package/dist/feeds/platform/handlers/livejournal.js

@@ -3,7 +3,7 @@ import { composeHint, isSubdomainOf } from "../../../common/utils.js";
 const wwwUsersPathRegex = /^\/(?:users\/|~)([^/]+)/;
 const legacyUserPathRegex = /^\/([^/]+)/;
 const tagRegex = /^\/tag\/([^/]+)/;
-const reservedHosts = new Set([
+const reservedHosts = /* @__PURE__ */ new Set([
 	"livejournal.com",
 	"www.livejournal.com",
 	"users.livejournal.com",

package/dist/feeds/platform/handlers/nebula.js

@@ -14,7 +14,7 @@ const excludedPaths = [
 	"signup",
 	"terms"
 ];
-const globalPaths = new Set(["videos", "explore"]);
+const globalPaths = /* @__PURE__ */ new Set(["videos", "explore"]);
 const nebulaHandler = {
 	match: (url) => {
 		return isHostOf(url, hosts);

package/dist/feeds/platform/handlers/pinterest.js

@@ -34,7 +34,7 @@ const pinterestHandler = {
 		if (pathSegments.length === 0) return [];
 		const username = pathSegments[0];
 		if (isAnyOf(username, excludedPaths)) return [];
-		const reservedBoardSlugs = new Set([
+		const reservedBoardSlugs = /* @__PURE__ */ new Set([
 			"pins",
 			"boards",
 			"_saved",

package/dist/feeds/platform/handlers/podbean.js

@@ -1,7 +1,7 @@
 import { composeHint, isSubdomainOf } from "../../../common/utils.js";
 //#region src/feeds/platform/handlers/podbean.ts
-const domainSuffix = /\.podbean\.com$/i;
-const reservedSlugs = new Set([
+const domainSuffixRegex = /\.podbean\.com$/i;
+const reservedSlugs = /* @__PURE__ */ new Set([
 	"www",
 	"feed",
 	"pbcdn1",
@@ -14,13 +14,13 @@ const reservedSlugs = new Set([
 const podbeanHandler = {
 	match: (url) => {
 		if (!isSubdomainOf(url, "podbean.com")) return false;
-		const slug = new URL(url).hostname.toLowerCase().replace(domainSuffix, "");
+		const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
 		return !reservedSlugs.has(slug);
 	},
 	resolve: (url) => {
 		const { hostname } = new URL(url);
 		return [{
-			uri: `https://feed.podbean.com/${hostname.replace(domainSuffix, "")}/feed.xml`,
+			uri: `https://feed.podbean.com/${hostname.replace(domainSuffixRegex, "")}/feed.xml`,
 			hint: composeHint("podbean:podcast")
 		}];
 	}

package/dist/feeds/platform/handlers/podigee.js

@@ -1,7 +1,7 @@
 import { composeHint, isSubdomainOf } from "../../../common/utils.js";
 //#region src/feeds/platform/handlers/podigee.ts
-const domainSuffix = /\.podigee\.io$/i;
-const reservedSlugs = new Set([
+const domainSuffixRegex = /\.podigee\.io$/i;
+const reservedSlugs = /* @__PURE__ */ new Set([
 	"www",
 	"app",
 	"help",
@@ -14,7 +14,7 @@ const reservedSlugs = new Set([
 const podigeeHandler = {
 	match: (url) => {
 		if (!isSubdomainOf(url, "podigee.io")) return false;
-		const slug = new URL(url).hostname.toLowerCase().replace(domainSuffix, "");
+		const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
 		return !reservedSlugs.has(slug);
 	},
 	resolve: (url) => {

package/dist/feeds/platform/handlers/reddit.js

@@ -22,7 +22,7 @@ const sortOptions = [
 	"top",
 	"best"
 ];
-const timeOptions = new Set([
+const timeOptions = /* @__PURE__ */ new Set([
 	"hour",
 	"day",
 	"week",
@@ -30,7 +30,7 @@ const timeOptions = new Set([
 	"year",
 	"all"
 ]);
-const timeFilteredSorts = new Set(["top", "controversial"]);
+const timeFilteredSorts = /* @__PURE__ */ new Set(["top", "controversial"]);
 const getTimeframeSuffix = (sort, searchParams) => {
 	if (!timeFilteredSorts.has(sort)) return "";
 	const timeframe = searchParams.get("t");

package/dist/feeds/platform/handlers/stackExchange.js

@@ -13,7 +13,7 @@ const domains = [
 	"mathoverflow.net",
 	"stackexchange.com"
 ];
-const validSorts = new Set([
+const validSorts = /* @__PURE__ */ new Set([
 	"newest",
 	"active",
 	"votes",

package/dist/feeds/platform/handlers/transistor.js

@@ -1,7 +1,7 @@
 import { composeHint, isSubdomainOf } from "../../../common/utils.js";
 //#region src/feeds/platform/handlers/transistor.ts
-const domainSuffix = /\.transistor\.fm$/i;
-const reservedSlugs = new Set([
+const domainSuffixRegex = /\.transistor\.fm$/i;
+const reservedSlugs = /* @__PURE__ */ new Set([
 	"www",
 	"feeds",
 	"share",
@@ -14,13 +14,13 @@ const reservedSlugs = new Set([
 const transistorHandler = {
 	match: (url) => {
 		if (!isSubdomainOf(url, "transistor.fm")) return false;
-		const slug = new URL(url).hostname.toLowerCase().replace(domainSuffix, "");
+		const slug = new URL(url).hostname.toLowerCase().replace(domainSuffixRegex, "");
 		return !reservedSlugs.has(slug);
 	},
 	resolve: (url) => {
 		const { hostname } = new URL(url);
 		return [{
-			uri: `https://feeds.transistor.fm/${hostname.replace(domainSuffix, "")}`,
+			uri: `https://feeds.transistor.fm/${hostname.replace(domainSuffixRegex, "")}`,
 			hint: composeHint("transistor:podcast")
 		}];
 	}

package/package.json

@@ -65,17 +65,19 @@
     "docs:build": "vitepress build docs"
   },
   "dependencies": {
-    "htmlparser2": "^12.0.0"
+    "htmlparser2": "^12.0.0",
+    "http-link-header": "^1.1.3"
   },
   "peerDependencies": {
     "feedsmith": "3.0.0-beta.5"
   },
   "devDependencies": {
     "@types/bun": "^1.3.13",
+    "@types/http-link-header": "^1.0.7",
     "feedsmith": "3.0.0-beta.5",
     "kvalita": "^1.15.1",
     "tsdown": "^0.22.2",
     "vitepress": "^2.0.0-alpha.17"
   },
-  "version": "2.0.0-beta.3"
+  "version": "2.0.0-beta.4"
 }