feedscout 1.6.1 → 1.7.0

package/README.md

@@ -22,6 +22,7 @@ Finds feeds by scanning links and anchors in HTML content, parsing HTTP headers,
 | --- | --- |
 | Feeds | RSS, Atom, JSON Feed, and RDF. Each feed is validated and returns metadata like format, title, description, and site URL. |
 | Blogrolls | OPML files containing feed subscriptions. Validated and returns title. |
+| Favicons | Site icons from HTML, feeds, platform APIs, and common paths. Validated to ensure URLs point to actual images. |
 | WebSub hubs | Find hubs for real-time feed update notifications. |
 
 ### Discovery Methods
@@ -144,6 +145,19 @@ const blogrolls = await discoverBlogrolls('https://example.com')
 // }]
 ```
 
+### Discover Favicons
+
+```typescript
+import { discoverFavicons } from 'feedscout'
+
+const favicons = await discoverFavicons('https://example.com')
+
+// [{
+//   url: 'https://example.com/apple-touch-icon.png',
+//   isValid: true,
+// }]
+```
+
 ### Discover WebSub Hubs
 
 ```typescript

package/dist/blogrolls/extractors.cjs

@@ -1,6 +1,6 @@
 let feedsmith = require("feedsmith");
 //#region src/blogrolls/extractors.ts
-const defaultExtractor = async ({ content, url }) => {
+const defaultExtractor = ({ content, url }) => {
 	if (!content) return {
 		url,
 		isValid: false

package/dist/blogrolls/extractors.js

@@ -1,6 +1,6 @@
 import { parseOpml } from "feedsmith";
 //#region src/blogrolls/extractors.ts
-const defaultExtractor = async ({ content, url }) => {
+const defaultExtractor = ({ content, url }) => {
 	if (!content) return {
 		url,
 		isValid: false

package/dist/blogrolls/index.cjs

@@ -4,7 +4,7 @@ const require_index = require("../common/discover/index.cjs");
 const require_defaults = require("./defaults.cjs");
 const require_extractors = require("./extractors.cjs");
 //#region src/blogrolls/index.ts
-const discoverBlogrolls = async (input, options = {}) => {
+const discoverBlogrolls = (input, options = {}) => {
 	return require_index.discover(input, {
 		...options,
 		methods: options.methods ?? [
@@ -14,7 +14,8 @@ const discoverBlogrolls = async (input, options = {}) => {
 		],
 		fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
 		extractFn: options.extractFn ?? require_extractors.defaultExtractor,
-		normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
+		resolveUrlFn: options.resolveUrlFn ?? require_utils.resolveUrl,
+		resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_utils$1.defaultResolveSiteUrlFn
 	}, {
 		html: require_defaults.defaultHtmlOptions,
 		headers: require_defaults.defaultHeadersOptions,

package/dist/blogrolls/index.js

@@ -1,10 +1,10 @@
-import { normalizeUrl } from "../common/utils.js";
-import { defaultFetchFn } from "../common/discover/utils.js";
+import { resolveUrl } from "../common/utils.js";
+import { defaultFetchFn, defaultResolveSiteUrlFn } from "../common/discover/utils.js";
 import { discover } from "../common/discover/index.js";
 import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
 import { defaultExtractor } from "./extractors.js";
 //#region src/blogrolls/index.ts
-const discoverBlogrolls = async (input, options = {}) => {
+const discoverBlogrolls = (input, options = {}) => {
 	return discover(input, {
 		...options,
 		methods: options.methods ?? [
@@ -14,7 +14,8 @@ const discoverBlogrolls = async (input, options = {}) => {
 		],
 		fetchFn: options.fetchFn ?? defaultFetchFn,
 		extractFn: options.extractFn ?? defaultExtractor,
-		normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
+		resolveUrlFn: options.resolveUrlFn ?? resolveUrl,
+		resolveSiteUrlFn: options.resolveSiteUrlFn ?? defaultResolveSiteUrlFn
 	}, {
 		html: defaultHtmlOptions,
 		headers: defaultHeadersOptions,

package/dist/common/discover/index.cjs

@@ -4,24 +4,36 @@ const require_index = require("../uris/index.cjs");
 const require_utils$1 = require("./utils.cjs");
 //#region src/common/discover/index.ts
 const discover = async (input, options, defaults) => {
-	const { methods, fetchFn, extractFn, normalizeUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
-	const normalizedInput = await require_utils$1.normalizeInput(input, fetchFn);
-	if (normalizedInput.content) {
+	const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
+	const sourceInput = await require_utils$1.normalizeInput(input, fetchFn);
+	if (sourceInput.content) {
 		const result = await extractFn({
-			url: normalizedInput.url,
-			content: normalizedInput.content,
-			headers: normalizedInput.headers
+			url: sourceInput.url,
+			content: sourceInput.content,
+			headers: sourceInput.headers
 		});
 		if (result.isValid) return [result];
 	}
-	const urisByMethod = await require_index.discoverUris(require_utils$1.normalizeMethodsConfig(normalizedInput, methods, defaults), fetchFn);
+	let siteInput;
+	if (resolveSiteUrlFn) {
+		const siteUrl = resolveSiteUrlFn(sourceInput);
+		if (siteUrl) try {
+			const response = await fetchFn(siteUrl);
+			siteInput = {
+				url: response.url,
+				content: typeof response.body === "string" ? response.body : "",
+				headers: response.headers
+			};
+		} catch {}
+	}
+	const urisByMethod = await require_index.discoverUris(require_utils$1.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
 	const seen = /* @__PURE__ */ new Set();
 	const methodGroups = [];
 	for (const method of require_types.discoverMethodOrder) {
 		const rawUris = urisByMethod[method];
-		if (!rawUris?.length) continue;
+		if (!rawUris || rawUris.length === 0) continue;
 		const unique = rawUris.map((entry) => {
-			return require_utils$1.normalizeUriEntry(entry, normalizeUrlFn, normalizedInput.url);
+			return require_utils$1.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
 		}).filter((entry) => {
 			const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
 			if (seen.has(key)) return false;

package/dist/common/discover/index.js

@@ -4,24 +4,36 @@ import { discoverUris } from "../uris/index.js";
 import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
 //#region src/common/discover/index.ts
 const discover = async (input, options, defaults) => {
-	const { methods, fetchFn, extractFn, normalizeUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
-	const normalizedInput = await normalizeInput(input, fetchFn);
-	if (normalizedInput.content) {
+	const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
+	const sourceInput = await normalizeInput(input, fetchFn);
+	if (sourceInput.content) {
 		const result = await extractFn({
-			url: normalizedInput.url,
-			content: normalizedInput.content,
-			headers: normalizedInput.headers
+			url: sourceInput.url,
+			content: sourceInput.content,
+			headers: sourceInput.headers
 		});
 		if (result.isValid) return [result];
 	}
-	const urisByMethod = await discoverUris(normalizeMethodsConfig(normalizedInput, methods, defaults), fetchFn);
+	let siteInput;
+	if (resolveSiteUrlFn) {
+		const siteUrl = resolveSiteUrlFn(sourceInput);
+		if (siteUrl) try {
+			const response = await fetchFn(siteUrl);
+			siteInput = {
+				url: response.url,
+				content: typeof response.body === "string" ? response.body : "",
+				headers: response.headers
+			};
+		} catch {}
+	}
+	const urisByMethod = await discoverUris(normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
 	const seen = /* @__PURE__ */ new Set();
 	const methodGroups = [];
 	for (const method of discoverMethodOrder) {
 		const rawUris = urisByMethod[method];
-		if (!rawUris?.length) continue;
+		if (!rawUris || rawUris.length === 0) continue;
 		const unique = rawUris.map((entry) => {
-			return normalizeUriEntry(entry, normalizeUrlFn, normalizedInput.url);
+			return normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
 		}).filter((entry) => {
 			const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
 			if (seen.has(key)) return false;

package/dist/common/discover/utils.cjs

@@ -1,4 +1,6 @@
 const require_locales = require("../locales.cjs");
+const require_utils = require("../utils.cjs");
+let feedsmith = require("feedsmith");
 //#region src/common/discover/utils.ts
 const defaultFetchFn = async (url, options) => {
 	const response = await fetch(url, {
@@ -22,37 +24,59 @@ const normalizeInput = async (input, fetchFn) => {
 		headers: response.headers
 	};
 };
-const normalizeUriEntry = (entry, normalizeUrlFn, baseUrl) => {
+const getLinkOfType = (links, rel) => {
+	return links?.find((link) => link.rel === rel);
+};
+const getFeedSiteUrl = (parsed) => {
+	const { format, feed } = parsed;
+	if (format === "rss" || format === "rdf") return getLinkOfType(feed.atom?.links, "alternate")?.href ?? feed.link;
+	if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
+	if (format === "json") return feed.home_page_url;
+};
+const defaultResolveSiteUrlFn = (input) => {
+	if (!input.content) return;
+	try {
+		let siteUrl = getFeedSiteUrl((0, feedsmith.parseFeed)(input.content));
+		if (!siteUrl) try {
+			siteUrl = new URL(input.url).origin;
+		} catch {}
+		else siteUrl = require_utils.resolveUrl(siteUrl, input.url);
+		if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
+		return siteUrl;
+	} catch {}
+};
+const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
 	if (typeof entry.uri === "string") return {
 		...entry,
-		uri: normalizeUrlFn(entry.uri, baseUrl)
+		uri: resolveUrlFn(entry.uri, baseUrl)
 	};
 	return {
 		...entry,
-		uri: entry.uri.map((uri) => normalizeUrlFn(uri, baseUrl))
+		uri: entry.uri.map((uri) => resolveUrlFn(uri, baseUrl))
 	};
 };
-const normalizeMethodsConfig = (input, methods, defaults) => {
+const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
+	const resolvedInput = siteInput ?? sourceInput;
 	const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
 	const methodsConfig = {};
 	if (methodsObj.platform && defaults.platform) {
-		if (!input.url || input.url === "") throw new Error(require_locales.errors.platformMethodRequiresUrl);
+		if (!resolvedInput.url || resolvedInput.url === "") throw new Error(require_locales.errors.platformMethodRequiresUrl);
 		const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
 		methodsConfig.platform = {
-			content: input.content,
-			headers: input.headers,
+			content: resolvedInput.content,
+			headers: resolvedInput.headers,
 			options: {
 				...defaults.platform,
 				...platformOptions,
-				baseUrl: input.url
+				baseUrl: resolvedInput.url
 			}
 		};
 	}
 	if (methodsObj.feed && defaults.feed) {
-		if (input.content === void 0) throw new Error(require_locales.errors.feedMethodRequiresContent);
+		if (sourceInput.content === void 0) throw new Error(require_locales.errors.feedMethodRequiresContent);
 		const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
 		methodsConfig.feed = {
-			content: input.content,
+			content: sourceInput.content,
 			options: {
 				...defaults.feed,
 				...feedOptions
@@ -60,42 +84,44 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
 		};
 	}
 	if (methodsObj.html && defaults.html) {
-		if (input.content === void 0) throw new Error(require_locales.errors.htmlMethodRequiresContent);
+		if (resolvedInput.content === void 0) throw new Error(require_locales.errors.htmlMethodRequiresContent);
 		const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
 		methodsConfig.html = {
-			html: input.content,
+			html: resolvedInput.content,
 			options: {
 				...defaults.html,
 				...htmlOptions,
-				baseUrl: input.url
+				baseUrl: resolvedInput.url
 			}
 		};
 	}
 	if (methodsObj.headers && defaults.headers) {
-		if (input.headers === void 0) throw new Error(require_locales.errors.headersMethodRequiresHeaders);
+		if (resolvedInput.headers === void 0) throw new Error(require_locales.errors.headersMethodRequiresHeaders);
 		const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
 		methodsConfig.headers = {
-			headers: input.headers,
+			headers: resolvedInput.headers,
 			options: {
 				...defaults.headers,
 				...headersOptions,
-				baseUrl: input.url
+				baseUrl: resolvedInput.url
 			}
 		};
 	}
 	if (methodsObj.guess && defaults.guess) {
-		if (!input.url || input.url === "") throw new Error(require_locales.errors.guessMethodRequiresUrl);
+		if (!resolvedInput.url || resolvedInput.url === "") throw new Error(require_locales.errors.guessMethodRequiresUrl);
 		const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
 		methodsConfig.guess = { options: {
 			...defaults.guess,
 			...guessOptions,
-			baseUrl: input.url
+			baseUrl: resolvedInput.url
 		} };
 	}
 	return methodsConfig;
 };
 //#endregion
 exports.defaultFetchFn = defaultFetchFn;
+exports.defaultResolveSiteUrlFn = defaultResolveSiteUrlFn;
+exports.getFeedSiteUrl = getFeedSiteUrl;
 exports.normalizeInput = normalizeInput;
 exports.normalizeMethodsConfig = normalizeMethodsConfig;
 exports.normalizeUriEntry = normalizeUriEntry;

package/dist/common/discover/utils.d.cts

@@ -0,0 +1,8 @@
+import { DiscoverResolveSiteUrlFn } from "../types.cjs";
+import { parseFeed } from "feedsmith";
+
+//#region src/common/discover/utils.d.ts
+declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
+declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
+//#endregion
+export { defaultResolveSiteUrlFn, getFeedSiteUrl };

package/dist/common/discover/utils.d.ts

@@ -0,0 +1,8 @@
+import { DiscoverResolveSiteUrlFn } from "../types.js";
+import { parseFeed } from "feedsmith";
+
+//#region src/common/discover/utils.d.ts
+declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
+declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
+//#endregion
+export { defaultResolveSiteUrlFn, getFeedSiteUrl };

package/dist/common/discover/utils.js

@@ -1,4 +1,6 @@
 import { errors } from "../locales.js";
+import { resolveUrl } from "../utils.js";
+import { parseFeed } from "feedsmith";
 //#region src/common/discover/utils.ts
 const defaultFetchFn = async (url, options) => {
 	const response = await fetch(url, {
@@ -22,37 +24,59 @@ const normalizeInput = async (input, fetchFn) => {
 		headers: response.headers
 	};
 };
-const normalizeUriEntry = (entry, normalizeUrlFn, baseUrl) => {
+const getLinkOfType = (links, rel) => {
+	return links?.find((link) => link.rel === rel);
+};
+const getFeedSiteUrl = (parsed) => {
+	const { format, feed } = parsed;
+	if (format === "rss" || format === "rdf") return getLinkOfType(feed.atom?.links, "alternate")?.href ?? feed.link;
+	if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
+	if (format === "json") return feed.home_page_url;
+};
+const defaultResolveSiteUrlFn = (input) => {
+	if (!input.content) return;
+	try {
+		let siteUrl = getFeedSiteUrl(parseFeed(input.content));
+		if (!siteUrl) try {
+			siteUrl = new URL(input.url).origin;
+		} catch {}
+		else siteUrl = resolveUrl(siteUrl, input.url);
+		if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
+		return siteUrl;
+	} catch {}
+};
+const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
 	if (typeof entry.uri === "string") return {
 		...entry,
-		uri: normalizeUrlFn(entry.uri, baseUrl)
+		uri: resolveUrlFn(entry.uri, baseUrl)
 	};
 	return {
 		...entry,
-		uri: entry.uri.map((uri) => normalizeUrlFn(uri, baseUrl))
+		uri: entry.uri.map((uri) => resolveUrlFn(uri, baseUrl))
 	};
 };
-const normalizeMethodsConfig = (input, methods, defaults) => {
+const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
+	const resolvedInput = siteInput ?? sourceInput;
 	const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
 	const methodsConfig = {};
 	if (methodsObj.platform && defaults.platform) {
-		if (!input.url || input.url === "") throw new Error(errors.platformMethodRequiresUrl);
+		if (!resolvedInput.url || resolvedInput.url === "") throw new Error(errors.platformMethodRequiresUrl);
 		const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
 		methodsConfig.platform = {
-			content: input.content,
-			headers: input.headers,
+			content: resolvedInput.content,
+			headers: resolvedInput.headers,
 			options: {
 				...defaults.platform,
 				...platformOptions,
-				baseUrl: input.url
+				baseUrl: resolvedInput.url
 			}
 		};
 	}
 	if (methodsObj.feed && defaults.feed) {
-		if (input.content === void 0) throw new Error(errors.feedMethodRequiresContent);
+		if (sourceInput.content === void 0) throw new Error(errors.feedMethodRequiresContent);
 		const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
 		methodsConfig.feed = {
-			content: input.content,
+			content: sourceInput.content,
 			options: {
 				...defaults.feed,
 				...feedOptions
@@ -60,39 +84,39 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
 		};
 	}
 	if (methodsObj.html && defaults.html) {
-		if (input.content === void 0) throw new Error(errors.htmlMethodRequiresContent);
+		if (resolvedInput.content === void 0) throw new Error(errors.htmlMethodRequiresContent);
 		const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
 		methodsConfig.html = {
-			html: input.content,
+			html: resolvedInput.content,
 			options: {
 				...defaults.html,
 				...htmlOptions,
-				baseUrl: input.url
+				baseUrl: resolvedInput.url
 			}
 		};
 	}
 	if (methodsObj.headers && defaults.headers) {
-		if (input.headers === void 0) throw new Error(errors.headersMethodRequiresHeaders);
+		if (resolvedInput.headers === void 0) throw new Error(errors.headersMethodRequiresHeaders);
 		const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
 		methodsConfig.headers = {
-			headers: input.headers,
+			headers: resolvedInput.headers,
 			options: {
 				...defaults.headers,
 				...headersOptions,
-				baseUrl: input.url
+				baseUrl: resolvedInput.url
 			}
 		};
 	}
 	if (methodsObj.guess && defaults.guess) {
-		if (!input.url || input.url === "") throw new Error(errors.guessMethodRequiresUrl);
+		if (!resolvedInput.url || resolvedInput.url === "") throw new Error(errors.guessMethodRequiresUrl);
 		const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
 		methodsConfig.guess = { options: {
 			...defaults.guess,
 			...guessOptions,
-			baseUrl: input.url
+			baseUrl: resolvedInput.url
 		} };
 	}
 	return methodsConfig;
 };
 //#endregion
-export { defaultFetchFn, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };
+export { defaultFetchFn, defaultResolveSiteUrlFn, getFeedSiteUrl, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };

package/dist/common/locales.cjs

@@ -118,7 +118,9 @@ var hints = {
 	"v2ex:node": "Node",
 	"v2ex:member": "Member",
 	"v2ex:tab": "Tab",
-	"ximalaya:album": "Album"
+	"ximalaya:album": "Album",
+	"lemmy:community": "Community",
+	"lemmy:user": "User"
 };
 //#endregion
 Object.defineProperty(exports, "errors", {

package/dist/common/locales.js

@@ -118,7 +118,9 @@ var hints = {
 	"v2ex:node": "Node",
 	"v2ex:member": "Member",
 	"v2ex:tab": "Tab",
-	"ximalaya:album": "Album"
+	"ximalaya:album": "Album",
+	"lemmy:community": "Community",
+	"lemmy:user": "User"
 };
 //#endregion
 export { errors, hints };

package/dist/common/types.d.cts

@@ -20,7 +20,8 @@ type LinkSelector = {
   rel: string;
   types?: Array<string>;
 };
-type DiscoverNormalizeUrlFn = (url: string, baseUrl: string | undefined) => string;
+type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string;
+type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
 type DiscoverFetchFnOptions = {
   method?: 'GET' | 'HEAD';
   headers?: Record<string, string>;
@@ -57,7 +58,7 @@ type DiscoverExtractFn<TValid> = (input: {
   content: string;
   headers?: Headers;
   status?: number;
-}) => Promise<DiscoverResult<TValid>>;
+}) => Promise<DiscoverResult<TValid>> | DiscoverResult<TValid>;
 type DiscoverInputObject = {
   url: string;
   content?: string;
@@ -75,7 +76,8 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
   methods?: DiscoverMethodsConfig<TMethods>;
   fetchFn?: DiscoverFetchFn;
   extractFn?: DiscoverExtractFn<TValid>;
-  normalizeUrlFn?: DiscoverNormalizeUrlFn;
+  resolveUrlFn?: DiscoverResolveUrlFn;
+  resolveSiteUrlFn?: DiscoverResolveSiteUrlFn;
   stopOnFirstMethod?: boolean;
   stopOnFirstResult?: boolean;
   concurrency?: number;
@@ -83,4 +85,4 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
   includeInvalid?: boolean;
 };
 //#endregion
-export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };
+export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };

package/dist/common/types.d.ts

@@ -20,7 +20,8 @@ type LinkSelector = {
   rel: string;
   types?: Array<string>;
 };
-type DiscoverNormalizeUrlFn = (url: string, baseUrl: string | undefined) => string;
+type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string;
+type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
 type DiscoverFetchFnOptions = {
   method?: 'GET' | 'HEAD';
   headers?: Record<string, string>;
@@ -57,7 +58,7 @@ type DiscoverExtractFn<TValid> = (input: {
   content: string;
   headers?: Headers;
   status?: number;
-}) => Promise<DiscoverResult<TValid>>;
+}) => Promise<DiscoverResult<TValid>> | DiscoverResult<TValid>;
 type DiscoverInputObject = {
   url: string;
   content?: string;
@@ -75,7 +76,8 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
   methods?: DiscoverMethodsConfig<TMethods>;
   fetchFn?: DiscoverFetchFn;
   extractFn?: DiscoverExtractFn<TValid>;
-  normalizeUrlFn?: DiscoverNormalizeUrlFn;
+  resolveUrlFn?: DiscoverResolveUrlFn;
+  resolveSiteUrlFn?: DiscoverResolveSiteUrlFn;
   stopOnFirstMethod?: boolean;
   stopOnFirstResult?: boolean;
   concurrency?: number;
@@ -83,4 +85,4 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
   includeInvalid?: boolean;
 };
 //#endregion
-export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };
+export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };

package/dist/common/uris/guess/utils.cjs

@@ -1,4 +1,5 @@
 //#region src/common/uris/guess/utils.ts
+const ipAddressRegex = /^\d+\.\d+\.\d+\.\d+$/;
 const resolveUri = (uri, base, origin, pathname) => {
 	if (uri.startsWith("/")) return `${origin}${uri}`;
 	if (uri.startsWith("?")) return `${origin}${pathname}${uri}`;
@@ -24,7 +25,7 @@ const getWwwCounterpart = (baseUrl) => {
 const getSubdomainVariants = (baseUrl, prefixes) => {
 	const url = new URL(baseUrl);
 	const hostname = url.hostname;
-	const isIpAddress = /^\d+\.\d+\.\d+\.\d+$/.test(hostname);
+	const isIpAddress = ipAddressRegex.test(hostname);
 	if (hostname === "localhost" || isIpAddress) return [];
 	const hostnameParts = hostname.split(".");
 	if (hostnameParts.length < 2) return [];

package/dist/common/uris/guess/utils.js

@@ -1,4 +1,5 @@
 //#region src/common/uris/guess/utils.ts
+const ipAddressRegex = /^\d+\.\d+\.\d+\.\d+$/;
 const resolveUri = (uri, base, origin, pathname) => {
 	if (uri.startsWith("/")) return `${origin}${uri}`;
 	if (uri.startsWith("?")) return `${origin}${pathname}${uri}`;
@@ -24,7 +25,7 @@ const getWwwCounterpart = (baseUrl) => {
 const getSubdomainVariants = (baseUrl, prefixes) => {
 	const url = new URL(baseUrl);
 	const hostname = url.hostname;
-	const isIpAddress = /^\d+\.\d+\.\d+\.\d+$/.test(hostname);
+	const isIpAddress = ipAddressRegex.test(hostname);
 	if (hostname === "localhost" || isIpAddress) return [];
 	const hostnameParts = hostname.split(".");
 	if (hostnameParts.length < 2) return [];

package/dist/common/uris/headers/index.cjs

@@ -3,11 +3,12 @@ const require_utils = require("../../utils.cjs");
 const urlRegex = /<([^<>]+)>/;
 const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
 const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
+const linkSplitRegex = /,(?=\s*<)/;
 const discoverUrisFromHeaders = (headers, options) => {
 	const uris = /* @__PURE__ */ new Set();
 	const linkHeader = headers.get("link");
 	if (!linkHeader) return [];
-	const links = linkHeader.split(/,(?=\s*<)/);
+	const links = linkHeader.split(linkSplitRegex);
 	for (const link of links) {
 		const urlMatch = link.match(urlRegex);
 		const relMatch = link.match(relRegex);

package/dist/common/uris/headers/index.js

@@ -3,11 +3,12 @@ import { matchesAnyOfLinkSelectors } from "../../utils.js";
 const urlRegex = /<([^<>]+)>/;
 const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
 const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
+const linkSplitRegex = /,(?=\s*<)/;
 const discoverUrisFromHeaders = (headers, options) => {
 	const uris = /* @__PURE__ */ new Set();
 	const linkHeader = headers.get("link");
 	if (!linkHeader) return [];
-	const links = linkHeader.split(/,(?=\s*<)/);
+	const links = linkHeader.split(linkSplitRegex);
 	for (const link of links) {
 		const urlMatch = link.match(urlRegex);
 		const relMatch = link.match(relRegex);