feedscout 1.8.1 → 2.0.0-next.1

package/dist/common/discover/utils.d.ts

@@ -1,6 +1,5 @@
-import { parseFeed } from "feedsmith";
-
+import { FeedMethodData } from "../uris/feed/types.js";
 //#region src/common/discover/utils.d.ts
-declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
+declare const getFeedSiteUrl: (parsed: FeedMethodData) => string | undefined;
 //#endregion
 export { getFeedSiteUrl };

package/dist/common/uris/feed/types.d.ts

@@ -1,9 +1,9 @@
 import { parseFeed } from "feedsmith";
 
 //#region src/common/uris/feed/types.d.ts
-type FeedMethodData = ReturnType<typeof parseFeed>;
+type FeedMethodData = ReturnType<typeof parseFeed<string>>;
 type FeedMethodOptions = {
   extractUrls: (params: FeedMethodData) => Array<string>;
 };
 //#endregion
-export { FeedMethodOptions };
+export { FeedMethodData, FeedMethodOptions };

package/dist/feeds/extractors.js

@@ -24,8 +24,8 @@ const defaultExtractFn = ({ content, url }) => {
 			url,
 			isValid: true,
 			format,
-			title: feed.title,
-			description: feed.subtitle,
+			title: feed.title?.value,
+			description: feed.subtitle?.value,
 			siteUrl
 		};
 		if (format === "json") return {

package/package.json

@@ -26,74 +26,32 @@
   "type": "module",
   "exports": {
     ".": {
-      "import": {
-        "types": "./dist/index.d.ts",
-        "default": "./dist/index.js"
-      },
-      "require": {
-        "types": "./dist/index.d.cts",
-        "default": "./dist/index.cjs"
-      }
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
     },
     "./feeds": {
-      "import": {
-        "types": "./dist/feeds.d.ts",
-        "default": "./dist/feeds.js"
-      },
-      "require": {
-        "types": "./dist/feeds.d.cts",
-        "default": "./dist/feeds.cjs"
-      }
+      "types": "./dist/feeds.d.ts",
+      "default": "./dist/feeds.js"
     },
     "./blogrolls": {
-      "import": {
-        "types": "./dist/blogrolls.d.ts",
-        "default": "./dist/blogrolls.js"
-      },
-      "require": {
-        "types": "./dist/blogrolls.d.cts",
-        "default": "./dist/blogrolls.cjs"
-      }
+      "types": "./dist/blogrolls.d.ts",
+      "default": "./dist/blogrolls.js"
     },
     "./favicons": {
-      "import": {
-        "types": "./dist/favicons.d.ts",
-        "default": "./dist/favicons.js"
-      },
-      "require": {
-        "types": "./dist/favicons.d.cts",
-        "default": "./dist/favicons.cjs"
-      }
+      "types": "./dist/favicons.d.ts",
+      "default": "./dist/favicons.js"
     },
     "./hubs": {
-      "import": {
-        "types": "./dist/hubs.d.ts",
-        "default": "./dist/hubs.js"
-      },
-      "require": {
-        "types": "./dist/hubs.d.cts",
-        "default": "./dist/hubs.cjs"
-      }
+      "types": "./dist/hubs.d.ts",
+      "default": "./dist/hubs.js"
     },
     "./methods": {
-      "import": {
-        "types": "./dist/methods.d.ts",
-        "default": "./dist/methods.js"
-      },
-      "require": {
-        "types": "./dist/methods.d.cts",
-        "default": "./dist/methods.cjs"
-      }
+      "types": "./dist/methods.d.ts",
+      "default": "./dist/methods.js"
     },
     "./utils": {
-      "import": {
-        "types": "./dist/utils.d.ts",
-        "default": "./dist/utils.js"
-      },
-      "require": {
-        "types": "./dist/utils.d.cts",
-        "default": "./dist/utils.cjs"
-      }
+      "types": "./dist/utils.d.ts",
+      "default": "./dist/utils.js"
     },
     "./package.json": "./package.json"
   },
@@ -102,14 +60,13 @@
   ],
   "scripts": {
     "prepare": "lefthook install",
-    "build": "tsdown src/exports/index.ts src/exports/feeds.ts src/exports/blogrolls.ts src/exports/favicons.ts src/exports/hubs.ts src/exports/methods.ts src/exports/utils.ts --format cjs,esm --dts --clean --unbundle --no-fixed-extension",
+    "build": "tsdown src/exports/index.ts src/exports/feeds.ts src/exports/blogrolls.ts src/exports/favicons.ts src/exports/hubs.ts src/exports/methods.ts src/exports/utils.ts --format esm --dts --clean --unbundle --no-fixed-extension",
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs"
   },
   "dependencies": {
-    "feedsmith": "^2.9.1",
-    "htmlparser2": "^10.1.0",
-    "typescript": "^6.0.2"
+    "feedsmith": "^3.0.0-beta.1",
+    "htmlparser2": "^12.0.0"
   },
   "devDependencies": {
     "@types/bun": "^1.3.11",
@@ -117,5 +74,5 @@
     "tsdown": "^0.21.7",
     "vitepress": "^2.0.0-alpha.17"
   },
-  "version": "1.8.1"
+  "version": "2.0.0-next.1"
 }

package/dist/blogrolls/defaults.cjs

@@ -1,51 +0,0 @@
-//#region src/blogrolls/defaults.ts
-const mimeTypes = [
-	"text/x-opml",
-	"application/xml",
-	"text/xml"
-];
-const urisMinimal = [
-	"/.well-known/recommendations.opml",
-	"/blogroll.opml",
-	"/opml.xml"
-];
-const urisBalanced = [
-	...urisMinimal,
-	"/blogroll.xml",
-	"/subscriptions.opml",
-	"/recommendations.opml"
-];
-const urisComprehensive = [
-	...urisBalanced,
-	"/links.opml",
-	"/feeds.opml",
-	"/subscriptions.xml"
-];
-const anchorLabels = [
-	"blogroll",
-	"opml",
-	"subscriptions",
-	"reading list"
-];
-const linkSelectors = [{ rel: "blogroll" }, {
-	rel: "outline",
-	types: mimeTypes
-}];
-const defaultHtmlOptions = {
-	linkSelectors,
-	anchorUris: urisComprehensive,
-	anchorIgnoredUris: [],
-	anchorLabels
-};
-const defaultHeadersOptions = { linkSelectors };
-const defaultGuessOptions = { uris: urisBalanced };
-//#endregion
-exports.anchorLabels = anchorLabels;
-exports.defaultGuessOptions = defaultGuessOptions;
-exports.defaultHeadersOptions = defaultHeadersOptions;
-exports.defaultHtmlOptions = defaultHtmlOptions;
-exports.linkSelectors = linkSelectors;
-exports.mimeTypes = mimeTypes;
-exports.urisBalanced = urisBalanced;
-exports.urisComprehensive = urisComprehensive;
-exports.urisMinimal = urisMinimal;

package/dist/blogrolls/defaults.d.cts

@@ -1,17 +0,0 @@
-import { GuessMethodOptions } from "../common/uris/guess/types.cjs";
-import { HeadersMethodOptions } from "../common/uris/headers/types.cjs";
-import { HtmlMethodOptions } from "../common/uris/html/types.cjs";
-import { LinkSelector } from "../common/types.cjs";
-
-//#region src/blogrolls/defaults.d.ts
-declare const mimeTypes: string[];
-declare const urisMinimal: string[];
-declare const urisBalanced: string[];
-declare const urisComprehensive: string[];
-declare const anchorLabels: string[];
-declare const linkSelectors: Array<LinkSelector>;
-declare const defaultHtmlOptions: Omit<HtmlMethodOptions, 'baseUrl'>;
-declare const defaultHeadersOptions: Omit<HeadersMethodOptions, 'baseUrl'>;
-declare const defaultGuessOptions: Omit<GuessMethodOptions, 'baseUrl'>;
-//#endregion
-export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };

package/dist/blogrolls/extractors.cjs

@@ -1,21 +0,0 @@
-let feedsmith = require("feedsmith");
-//#region src/blogrolls/extractors.ts
-const defaultExtractFn = ({ content, url }) => {
-	if (!content) return {
-		url,
-		isValid: false
-	};
-	try {
-		return {
-			url,
-			isValid: true,
-			title: (0, feedsmith.parseOpml)(content).head?.title
-		};
-	} catch {}
-	return {
-		url,
-		isValid: false
-	};
-};
-//#endregion
-exports.defaultExtractFn = defaultExtractFn;

package/dist/blogrolls/extractors.d.cts

@@ -1,7 +0,0 @@
-import { DiscoverExtractFn } from "../common/types.cjs";
-import { BlogrollResult } from "./types.cjs";
-
-//#region src/blogrolls/extractors.d.ts
-declare const defaultExtractFn: DiscoverExtractFn<BlogrollResult>;
-//#endregion
-export { defaultExtractFn };

package/dist/blogrolls/index.cjs

@@ -1,25 +0,0 @@
-const require_defaults = require("../common/discover/defaults.cjs");
-const require_index = require("../common/discover/index.cjs");
-const require_defaults$1 = require("./defaults.cjs");
-const require_extractors = require("./extractors.cjs");
-//#region src/blogrolls/index.ts
-const discoverBlogrolls = (input, options = {}) => {
-	return require_index.discover(input, {
-		...options,
-		methods: options.methods ?? [
-			"html",
-			"headers",
-			"guess"
-		],
-		fetchFn: options.fetchFn ?? require_defaults.defaultFetchFn,
-		extractFn: options.extractFn ?? require_extractors.defaultExtractFn,
-		resolveUrlFn: options.resolveUrlFn ?? require_defaults.defaultResolveUrlFn,
-		resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_defaults.defaultResolveSiteUrlFn
-	}, {
-		html: require_defaults$1.defaultHtmlOptions,
-		headers: require_defaults$1.defaultHeadersOptions,
-		guess: require_defaults$1.defaultGuessOptions
-	});
-};
-//#endregion
-exports.discoverBlogrolls = discoverBlogrolls;

package/dist/blogrolls/index.d.cts

@@ -1,7 +0,0 @@
-import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.cjs";
-import { BlogrollResult } from "./types.cjs";
-
-//#region src/blogrolls/index.d.ts
-declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid, "html" | "headers" | "guess">) => Promise<Array<DiscoverResult<TValid>>>;
-//#endregion
-export { discoverBlogrolls };

package/dist/blogrolls/types.d.cts

@@ -1,6 +0,0 @@
-//#region src/blogrolls/types.d.ts
-type BlogrollResult = {
-  title?: string;
-};
-//#endregion
-export { BlogrollResult };

package/dist/blogrolls.cjs

@@ -1,13 +0,0 @@
-Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_defaults = require("./blogrolls/defaults.cjs");
-const require_extractors = require("./blogrolls/extractors.cjs");
-exports.anchorLabels = require_defaults.anchorLabels;
-exports.defaultExtractFn = require_extractors.defaultExtractFn;
-exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
-exports.defaultHeadersOptions = require_defaults.defaultHeadersOptions;
-exports.defaultHtmlOptions = require_defaults.defaultHtmlOptions;
-exports.linkSelectors = require_defaults.linkSelectors;
-exports.mimeTypes = require_defaults.mimeTypes;
-exports.urisBalanced = require_defaults.urisBalanced;
-exports.urisComprehensive = require_defaults.urisComprehensive;
-exports.urisMinimal = require_defaults.urisMinimal;

package/dist/blogrolls.d.cts

@@ -1,4 +0,0 @@
-import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.cjs";
-import { BlogrollResult } from "./blogrolls/types.cjs";
-import { defaultExtractFn } from "./blogrolls/extractors.cjs";
-export { BlogrollResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };

package/dist/common/discover/defaults.cjs

@@ -1,46 +0,0 @@
-const require_utils = require("./utils.cjs");
-let feedsmith = require("feedsmith");
-//#region src/common/discover/defaults.ts
-const defaultFetchFn = async (url, options) => {
-	const response = await fetch(url, {
-		method: options?.method ?? "GET",
-		headers: options?.headers
-	});
-	return {
-		headers: response.headers,
-		body: await response.text(),
-		url: response.url,
-		status: response.status,
-		statusText: response.statusText
-	};
-};
-const defaultResolveUrlFn = (url, baseUrl) => {
-	try {
-		return new URL(url, baseUrl).href;
-	} catch {
-		return;
-	}
-};
-const defaultResolveSiteUrlFn = (input, resolveUrlFn) => {
-	if (!input.content) return;
-	try {
-		let siteUrl = require_utils.getFeedSiteUrl((0, feedsmith.parseFeed)(input.content));
-		if (siteUrl) {
-			siteUrl = resolveUrlFn(siteUrl, input.url);
-			if (siteUrl) {
-				const parsed = new URL(siteUrl);
-				parsed.hash = "";
-				siteUrl = parsed.href;
-			}
-		}
-		if (!siteUrl || siteUrl === new URL(input.url).href) try {
-			siteUrl = new URL(input.url).origin;
-		} catch {}
-		if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
-		return siteUrl;
-	} catch {}
-};
-//#endregion
-exports.defaultFetchFn = defaultFetchFn;
-exports.defaultResolveSiteUrlFn = defaultResolveSiteUrlFn;
-exports.defaultResolveUrlFn = defaultResolveUrlFn;

package/dist/common/discover/defaults.d.cts

@@ -1,6 +0,0 @@
-import { DiscoverResolveSiteUrlFn } from "../types.cjs";
-
-//#region src/common/discover/defaults.d.ts
-declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
-//#endregion
-export { defaultResolveSiteUrlFn };

package/dist/common/discover/index.cjs

@@ -1,105 +0,0 @@
-const require_utils = require("./utils.cjs");
-const require_types = require("../types.cjs");
-const require_utils$1 = require("../utils.cjs");
-const require_index = require("../uris/index.cjs");
-//#region src/common/discover/index.ts
-const discover = async (input, options, defaults) => {
-	const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
-	const sourceInput = await require_utils.normalizeInput(input, fetchFn);
-	if (sourceInput.content) {
-		const result = await extractFn({
-			url: sourceInput.url,
-			content: sourceInput.content,
-			headers: sourceInput.headers
-		});
-		if (result.isValid) return [result];
-	}
-	let siteInput;
-	if (resolveSiteUrlFn) {
-		const siteUrl = resolveSiteUrlFn(sourceInput, resolveUrlFn);
-		if (siteUrl) try {
-			const response = await fetchFn(siteUrl);
-			siteInput = {
-				url: response.url,
-				content: typeof response.body === "string" ? response.body : "",
-				headers: response.headers
-			};
-		} catch {}
-	}
-	const urisByMethod = await require_index.discoverUris(require_utils.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
-	const seen = /* @__PURE__ */ new Set();
-	const methodGroups = [];
-	for (const method of require_types.discoverMethodOrder) {
-		const rawUris = urisByMethod[method];
-		if (!rawUris || rawUris.length === 0) continue;
-		const unique = rawUris.map((entry) => {
-			return require_utils.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
-		}).filter((entry) => {
-			const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
-			if (seen.has(key)) return false;
-			seen.add(key);
-			return true;
-		});
-		if (unique.length > 0) methodGroups.push({
-			method,
-			entries: unique
-		});
-	}
-	const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
-	const results = [];
-	let tested = 0;
-	let found = 0;
-	const fetchAndExtract = async (url) => {
-		try {
-			const fetchResult = await fetchFn(url);
-			return await extractFn({
-				url: fetchResult.url,
-				content: typeof fetchResult.body === "string" ? fetchResult.body : "",
-				headers: fetchResult.headers,
-				status: fetchResult.status
-			});
-		} catch (error) {
-			return {
-				url,
-				isValid: false,
-				error
-			};
-		}
-	};
-	const processUri = async (entry, method) => {
-		const alternatives = typeof entry.uri === "string" ? [entry.uri] : entry.uri;
-		for (const url of alternatives) {
-			const result = await fetchAndExtract(url);
-			results.push(entry.hint ? {
-				...result,
-				method,
-				hint: entry.hint
-			} : {
-				...result,
-				method
-			});
-			tested += 1;
-			if (result.isValid) found += 1;
-			onProgress?.({
-				tested,
-				total,
-				found,
-				current: url
-			});
-			if (result.isValid) break;
-		}
-	};
-	for (const { method, entries } of methodGroups) {
-		const foundBefore = found;
-		await require_utils$1.processConcurrently(entries, (entry) => processUri(entry, method), {
-			concurrency,
-			shouldStop: () => {
-				return stopOnFirstResult && found > 0;
-			}
-		});
-		if (stopOnFirstMethod && found > foundBefore) break;
-	}
-	return includeInvalid ? results : results.filter((result) => result.isValid);
-};
-//#endregion
-exports.discover = discover;

package/dist/common/discover/utils.cjs

@@ -1,101 +0,0 @@
-const require_locales = require("../locales.cjs");
-//#region src/common/discover/utils.ts
-const normalizeInput = async (input, fetchFn) => {
-	if (typeof input === "object") return input;
-	try {
-		const response = await fetchFn(input);
-		return {
-			url: response.url,
-			content: typeof response.body === "string" ? response.body : void 0,
-			headers: response.headers
-		};
-	} catch {}
-	return { url: input };
-};
-const getLinkOfType = (links, rel) => {
-	return links?.find((link) => link.rel === rel);
-};
-const getFeedSiteUrl = (parsed) => {
-	const { format, feed } = parsed;
-	if (format === "rss" || format === "rdf") return getLinkOfType(feed.atom?.links, "alternate")?.href ?? feed.link;
-	if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
-	if (format === "json") return feed.home_page_url;
-};
-const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
-	if (typeof entry.uri === "string") return {
-		...entry,
-		uri: resolveUrlFn(entry.uri, baseUrl) ?? entry.uri
-	};
-	return {
-		...entry,
-		uri: entry.uri.map((uri) => resolveUrlFn(uri, baseUrl) ?? uri)
-	};
-};
-const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
-	const resolvedInput = siteInput ?? sourceInput;
-	const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
-	const methodsConfig = {};
-	if (methodsObj.platform && defaults.platform) {
-		if (!resolvedInput.url || resolvedInput.url === "") throw new Error(require_locales.errors.platformMethodRequiresUrl);
-		const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
-		methodsConfig.platform = {
-			content: resolvedInput.content,
-			headers: resolvedInput.headers,
-			options: {
-				...defaults.platform,
-				...platformOptions,
-				baseUrl: resolvedInput.url
-			}
-		};
-	}
-	if (methodsObj.feed && defaults.feed) {
-		if (sourceInput.content === void 0) throw new Error(require_locales.errors.feedMethodRequiresContent);
-		const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
-		methodsConfig.feed = {
-			content: sourceInput.content,
-			options: {
-				...defaults.feed,
-				...feedOptions
-			}
-		};
-	}
-	if (methodsObj.html && defaults.html) {
-		if (resolvedInput.content === void 0) throw new Error(require_locales.errors.htmlMethodRequiresContent);
-		const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
-		methodsConfig.html = {
-			html: resolvedInput.content,
-			options: {
-				...defaults.html,
-				...htmlOptions,
-				baseUrl: resolvedInput.url
-			}
-		};
-	}
-	if (methodsObj.headers && defaults.headers) {
-		if (resolvedInput.headers === void 0) throw new Error(require_locales.errors.headersMethodRequiresHeaders);
-		const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
-		methodsConfig.headers = {
-			headers: resolvedInput.headers,
-			options: {
-				...defaults.headers,
-				...headersOptions,
-				baseUrl: resolvedInput.url
-			}
-		};
-	}
-	if (methodsObj.guess && defaults.guess) {
-		if (!resolvedInput.url || resolvedInput.url === "") throw new Error(require_locales.errors.guessMethodRequiresUrl);
-		const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
-		methodsConfig.guess = { options: {
-			...defaults.guess,
-			...guessOptions,
-			baseUrl: resolvedInput.url
-		} };
-	}
-	return methodsConfig;
-};
-//#endregion
-exports.getFeedSiteUrl = getFeedSiteUrl;
-exports.normalizeInput = normalizeInput;
-exports.normalizeMethodsConfig = normalizeMethodsConfig;
-exports.normalizeUriEntry = normalizeUriEntry;

package/dist/common/discover/utils.d.cts

@@ -1,6 +0,0 @@
-import { parseFeed } from "feedsmith";
-
-//#region src/common/discover/utils.d.ts
-declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
-//#endregion
-export { getFeedSiteUrl };