feedcanon 0.9.2 → 1.0.0-beta.2

package/dist/defaults.cjs

@@ -18,6 +18,8 @@ const defaultStrippedParams = [
 	"wbraid",
 	"gclsrc",
 	"gad_source",
+	"gad_campaignid",
+	"srsltid",
 	"fbclid",
 	"fb_action_ids",
 	"fb_action_types",
@@ -60,12 +62,43 @@ const defaultStrippedParams = [
 	"hsa_src",
 	"hsa_tgt",
 	"hsa_ver",
+	"hsCtaTracking",
+	"__hstc",
+	"__hsfp",
+	"__hssc",
 	"cid",
 	"s_kwcid",
 	"ef_id",
 	"obOrigUrl",
 	"dicbo",
 	"yclid",
+	"awinaffid",
+	"awinmid",
+	"clickref",
+	"afftrack",
+	"itm_source",
+	"itm_medium",
+	"itm_campaign",
+	"itm_content",
+	"itm_channel",
+	"itm_audience",
+	"int_source",
+	"int_medium",
+	"int_campaign",
+	"int_content",
+	"int_placement",
+	"int_campaign_type",
+	"int_keycode",
+	"g2i_source",
+	"g2i_medium",
+	"g2i_campaign",
+	"g2i_or_o",
+	"g2i_or_p",
+	"doing_wp_cron",
+	"preview",
+	"preview_id",
+	"preview_nonce",
+	"replytocom",
 	"_",
 	"timestamp",
 	"ts",
@@ -74,6 +107,14 @@ const defaultStrippedParams = [
 	"nocache",
 	"rand",
 	"random",
+	"forceByPassCache",
+	"sucurianticache",
+	"cleancache",
+	"rebuildcache",
+	"kontrol_health_check_timestamp",
+	"_x_tr_sl",
+	"_x_tr_tl",
+	"_x_tr_hl",
 	"action_object_map",
 	"action_ref_map",
 	"action_type_map",
@@ -93,9 +134,6 @@ const defaultStrippedParams = [
 	"hmb_campaign",
 	"hmb_medium",
 	"hmb_source",
-	"itm_campaign",
-	"itm_medium",
-	"itm_source",
 	"ml_subscriber",
 	"ml_subscriber_hash",
 	"oly_anon_id",
@@ -118,7 +156,6 @@ const defaultNormalizeOptions = {
 	stripRootSlash: true,
 	collapseSlashes: true,
 	stripHash: true,
-	stripTextFragment: true,
 	sortQueryParams: true,
 	stripQueryParams: defaultStrippedParams,
 	stripEmptyQuery: true,
@@ -136,9 +173,7 @@ const defaultTiers = [
 		stripRootSlash: true,
 		collapseSlashes: true,
 		stripHash: true,
-		stripTextFragment: true,
 		sortQueryParams: true,
-		stripQueryParams: defaultStrippedParams,
 		stripEmptyQuery: true,
 		normalizeEncoding: true,
 		lowercaseHostname: true,
@@ -153,9 +188,7 @@ const defaultTiers = [
 		stripRootSlash: true,
 		collapseSlashes: true,
 		stripHash: true,
-		stripTextFragment: true,
 		sortQueryParams: true,
-		stripQueryParams: defaultStrippedParams,
 		stripEmptyQuery: true,
 		normalizeEncoding: true,
 		lowercaseHostname: true,
@@ -170,9 +203,7 @@ const defaultTiers = [
 		stripRootSlash: true,
 		collapseSlashes: true,
 		stripHash: true,
-		stripTextFragment: true,
 		sortQueryParams: true,
-		stripQueryParams: defaultStrippedParams,
 		stripEmptyQuery: true,
 		normalizeEncoding: true,
 		lowercaseHostname: true,

package/dist/defaults.d.cts

@@ -1,9 +1,9 @@
-import { NormalizeOptions, PlatformHandler } from "./types.cjs";
+import { NormalizeOptions, PlatformHandler, Tier } from "./types.cjs";
 
 //#region src/defaults.d.ts
 declare const defaultPlatforms: Array<PlatformHandler>;
 declare const defaultStrippedParams: string[];
 declare const defaultNormalizeOptions: NormalizeOptions;
-declare const defaultTiers: Array<NormalizeOptions>;
+declare const defaultTiers: Array<Tier>;
 //#endregion
 export { defaultNormalizeOptions, defaultPlatforms, defaultStrippedParams, defaultTiers };

package/dist/defaults.d.ts

@@ -1,9 +1,9 @@
-import { NormalizeOptions, PlatformHandler } from "./types.js";
+import { NormalizeOptions, PlatformHandler, Tier } from "./types.js";
 
 //#region src/defaults.d.ts
 declare const defaultPlatforms: Array<PlatformHandler>;
 declare const defaultStrippedParams: string[];
 declare const defaultNormalizeOptions: NormalizeOptions;
-declare const defaultTiers: Array<NormalizeOptions>;
+declare const defaultTiers: Array<Tier>;
 //#endregion
 export { defaultNormalizeOptions, defaultPlatforms, defaultStrippedParams, defaultTiers };

package/dist/defaults.js

@@ -18,6 +18,8 @@ const defaultStrippedParams = [
 	"wbraid",
 	"gclsrc",
 	"gad_source",
+	"gad_campaignid",
+	"srsltid",
 	"fbclid",
 	"fb_action_ids",
 	"fb_action_types",
@@ -60,12 +62,43 @@ const defaultStrippedParams = [
 	"hsa_src",
 	"hsa_tgt",
 	"hsa_ver",
+	"hsCtaTracking",
+	"__hstc",
+	"__hsfp",
+	"__hssc",
 	"cid",
 	"s_kwcid",
 	"ef_id",
 	"obOrigUrl",
 	"dicbo",
 	"yclid",
+	"awinaffid",
+	"awinmid",
+	"clickref",
+	"afftrack",
+	"itm_source",
+	"itm_medium",
+	"itm_campaign",
+	"itm_content",
+	"itm_channel",
+	"itm_audience",
+	"int_source",
+	"int_medium",
+	"int_campaign",
+	"int_content",
+	"int_placement",
+	"int_campaign_type",
+	"int_keycode",
+	"g2i_source",
+	"g2i_medium",
+	"g2i_campaign",
+	"g2i_or_o",
+	"g2i_or_p",
+	"doing_wp_cron",
+	"preview",
+	"preview_id",
+	"preview_nonce",
+	"replytocom",
 	"_",
 	"timestamp",
 	"ts",
@@ -74,6 +107,14 @@ const defaultStrippedParams = [
 	"nocache",
 	"rand",
 	"random",
+	"forceByPassCache",
+	"sucurianticache",
+	"cleancache",
+	"rebuildcache",
+	"kontrol_health_check_timestamp",
+	"_x_tr_sl",
+	"_x_tr_tl",
+	"_x_tr_hl",
 	"action_object_map",
 	"action_ref_map",
 	"action_type_map",
@@ -93,9 +134,6 @@ const defaultStrippedParams = [
 	"hmb_campaign",
 	"hmb_medium",
 	"hmb_source",
-	"itm_campaign",
-	"itm_medium",
-	"itm_source",
 	"ml_subscriber",
 	"ml_subscriber_hash",
 	"oly_anon_id",
@@ -118,7 +156,6 @@ const defaultNormalizeOptions = {
 	stripRootSlash: true,
 	collapseSlashes: true,
 	stripHash: true,
-	stripTextFragment: true,
 	sortQueryParams: true,
 	stripQueryParams: defaultStrippedParams,
 	stripEmptyQuery: true,
@@ -136,9 +173,7 @@ const defaultTiers = [
 		stripRootSlash: true,
 		collapseSlashes: true,
 		stripHash: true,
-		stripTextFragment: true,
 		sortQueryParams: true,
-		stripQueryParams: defaultStrippedParams,
 		stripEmptyQuery: true,
 		normalizeEncoding: true,
 		lowercaseHostname: true,
@@ -153,9 +188,7 @@ const defaultTiers = [
 		stripRootSlash: true,
 		collapseSlashes: true,
 		stripHash: true,
-		stripTextFragment: true,
 		sortQueryParams: true,
-		stripQueryParams: defaultStrippedParams,
 		stripEmptyQuery: true,
 		normalizeEncoding: true,
 		lowercaseHostname: true,
@@ -170,9 +203,7 @@ const defaultTiers = [
 		stripRootSlash: true,
 		collapseSlashes: true,
 		stripHash: true,
-		stripTextFragment: true,
 		sortQueryParams: true,
-		stripQueryParams: defaultStrippedParams,
 		stripEmptyQuery: true,
 		normalizeEncoding: true,
 		lowercaseHostname: true,

package/dist/index.cjs

@@ -2,8 +2,15 @@ const require_defaults = require('./defaults.cjs');
 const require_utils = require('./utils.cjs');
 
 //#region src/index.ts
-const findCanonical = async (inputUrl, options) => {
-	const { fetchFn = require_utils.nativeFetch, existsFn, parser = require_utils.feedsmithParser, tiers = require_defaults.defaultTiers, platforms = require_defaults.defaultPlatforms, onFetch, onMatch, onExists } = options ?? {};
+async function findCanonical(inputUrl, options) {
+	const { parser = require_utils.feedsmithParser, fetchFn = require_utils.nativeFetch, existsFn, tiers = require_defaults.defaultTiers, platforms = require_defaults.defaultPlatforms, stripQueryParams = require_defaults.defaultStrippedParams, onFetch, onMatch, onExists } = options ?? {};
+	const stripParams = (url) => {
+		return stripQueryParams?.length ? require_utils.normalizeUrl(url, {
+			stripQueryParams,
+			sortQueryParams: true,
+			stripEmptyQuery: true
+		}) : url;
+	};
 	const resolveAndApplyPlatformHandlers = (url, baseUrl) => {
 		const resolved = require_utils.resolveUrl(url, baseUrl);
 		return resolved ? require_utils.applyPlatformHandlers(resolved, platforms) : void 0;
@@ -21,8 +28,9 @@ const findCanonical = async (inputUrl, options) => {
 		response: initialResponse
 	});
 	if (initialResponse.status < 200 || initialResponse.status >= 300) return;
-	const initialResponseUrl = resolveAndApplyPlatformHandlers(initialResponse.url);
-	if (!initialResponseUrl) return;
+	const initialResponseUrlRaw = resolveAndApplyPlatformHandlers(initialResponse.url);
+	if (!initialResponseUrlRaw) return;
+	const initialResponseUrl = stripParams(initialResponseUrlRaw);
 	const initialResponseBody = initialResponse.body;
 	if (!initialResponseBody) return;
 	let initialResponseSignature;
@@ -35,14 +43,17 @@ const findCanonical = async (inputUrl, options) => {
 		feed: initialResponseFeed
 	});
 	const selfRequestUrlRaw = parser.getSelfUrl(initialResponseFeed);
-	if (selfRequestUrlRaw) selfRequestUrl = resolveAndApplyPlatformHandlers(selfRequestUrlRaw, initialResponseUrl);
+	if (selfRequestUrlRaw) {
+		selfRequestUrl = resolveAndApplyPlatformHandlers(selfRequestUrlRaw, initialResponseUrl);
+		selfRequestUrl = selfRequestUrl ? stripParams(selfRequestUrl) : void 0;
+	}
 	const compareWithInitialResponse = (comparedResponseBody) => {
 		if (!comparedResponseBody) return false;
 		if (initialResponseBody === comparedResponseBody) return true;
 		const comparedResponseFeed = parser.parse(comparedResponseBody);
 		if (comparedResponseFeed) {
-			initialResponseSignature ||= JSON.stringify(parser.getSignature(initialResponseFeed));
-			const comparedResponseSignature = JSON.stringify(parser.getSignature(comparedResponseFeed));
+			initialResponseSignature ||= parser.getSignature(initialResponseFeed, parser.getSelfUrl(initialResponseFeed));
+			const comparedResponseSignature = parser.getSignature(comparedResponseFeed, parser.getSelfUrl(comparedResponseFeed));
 			return initialResponseSignature === comparedResponseSignature;
 		}
 		return false;
@@ -62,7 +73,7 @@ const findCanonical = async (inputUrl, options) => {
 		if (!compareWithInitialResponse(response.body)) return;
 		return response;
 	};
-	let variantSource = initialResponseUrl;
+	let variantSourceUrl = initialResponseUrl;
 	if (selfRequestUrl && selfRequestUrl !== initialResponseUrl) {
 		const urlsToTry = [selfRequestUrl];
 		if (selfRequestUrl.startsWith("https://")) urlsToTry.push(selfRequestUrl.replace("https://", "http://"));
@@ -75,40 +86,42 @@ const findCanonical = async (inputUrl, options) => {
 					response,
 					feed: initialResponseFeed
 				});
-				variantSource = resolveAndApplyPlatformHandlers(response.url) ?? initialResponseUrl;
+				variantSourceUrl = resolveAndApplyPlatformHandlers(response.url) ?? initialResponseUrl;
+				variantSourceUrl = stripParams(variantSourceUrl);
 				break;
 			}
 		}
 	}
-	const variants = new Set(tiers.map((tier) => resolveAndApplyPlatformHandlers(require_utils.normalizeUrl(variantSource, tier))).filter((url) => url !== void 0));
-	variants.add(variantSource);
-	let winningUrl = variantSource;
-	for (const variant of variants) {
+	const variantUrls = new Set(tiers.map((tier) => resolveAndApplyPlatformHandlers(require_utils.normalizeUrl(variantSourceUrl, tier))).filter((variantUrl) => !!variantUrl));
+	variantUrls.add(variantSourceUrl);
+	let winningUrl = variantSourceUrl;
+	for (const variantUrl of variantUrls) {
 		if (existsFn) {
-			const data = await existsFn(variant);
+			const data = await existsFn(variantUrl);
 			if (data !== void 0) {
 				onExists?.({
-					url: variant,
+					url: variantUrl,
 					data
 				});
-				return variant;
+				return variantUrl;
 			}
 		}
-		if (variant === variantSource) continue;
-		if (variant === initialResponseUrl) {
+		if (variantUrl === variantSourceUrl) continue;
+		if (variantUrl === initialResponseUrl) {
 			winningUrl = initialResponseUrl;
 			break;
 		}
-		const response = await fetchAndCompare(variant);
-		if (response) {
-			const preparedResponseUrl = resolveAndApplyPlatformHandlers(response.url);
-			if (preparedResponseUrl === variantSource || preparedResponseUrl === initialResponseUrl) continue;
+		const variantResponse = await fetchAndCompare(variantUrl);
+		if (variantResponse) {
+			let variantResponseUrl = resolveAndApplyPlatformHandlers(variantResponse.url);
+			if (variantResponseUrl) variantResponseUrl = stripParams(variantResponseUrl);
+			if (variantResponseUrl === variantSourceUrl || variantResponseUrl === initialResponseUrl) continue;
 			onMatch?.({
-				url: variant,
-				response,
+				url: variantUrl,
+				response: variantResponse,
 				feed: initialResponseFeed
 			});
-			winningUrl = variant;
+			winningUrl = variantUrl;
 			break;
 		}
 	}
@@ -125,7 +138,7 @@ const findCanonical = async (inputUrl, options) => {
 		}
 	}
 	return winningUrl;
-};
+}
 
 //#endregion
 exports.findCanonical = findCanonical;

package/dist/index.d.cts

@@ -1,6 +1,9 @@
-import { FeedsmithFeed, FindCanonicalOptions } from "./types.cjs";
+import { FeedsmithFeed, FetchFnResponse, FindCanonicalOptions, ParserAdapter } from "./types.cjs";
 
 //#region src/index.d.ts
-declare const findCanonical: <TFeed = FeedsmithFeed, TExisting = unknown>(inputUrl: string, options?: FindCanonicalOptions<TFeed, TExisting>) => Promise<string | undefined>;
+declare function findCanonical<TResponse extends FetchFnResponse = FetchFnResponse, TExisting = unknown>(inputUrl: string, options?: Omit<FindCanonicalOptions<FeedsmithFeed, TResponse, TExisting>, 'parser'>): Promise<string | undefined>;
+declare function findCanonical<TFeed, TResponse extends FetchFnResponse = FetchFnResponse, TExisting = unknown>(inputUrl: string, options: FindCanonicalOptions<TFeed, TResponse, TExisting> & {
+  parser: ParserAdapter<TFeed>;
+}): Promise<string | undefined>;
 //#endregion
 export { findCanonical };

package/dist/index.d.ts

@@ -1,6 +1,9 @@
-import { FeedsmithFeed, FindCanonicalOptions } from "./types.js";
+import { FeedsmithFeed, FetchFnResponse, FindCanonicalOptions, ParserAdapter } from "./types.js";
 
 //#region src/index.d.ts
-declare const findCanonical: <TFeed = FeedsmithFeed, TExisting = unknown>(inputUrl: string, options?: FindCanonicalOptions<TFeed, TExisting>) => Promise<string | undefined>;
+declare function findCanonical<TResponse extends FetchFnResponse = FetchFnResponse, TExisting = unknown>(inputUrl: string, options?: Omit<FindCanonicalOptions<FeedsmithFeed, TResponse, TExisting>, 'parser'>): Promise<string | undefined>;
+declare function findCanonical<TFeed, TResponse extends FetchFnResponse = FetchFnResponse, TExisting = unknown>(inputUrl: string, options: FindCanonicalOptions<TFeed, TResponse, TExisting> & {
+  parser: ParserAdapter<TFeed>;
+}): Promise<string | undefined>;
 //#endregion
 export { findCanonical };

package/dist/index.js

@@ -1,9 +1,16 @@
-import { defaultPlatforms, defaultTiers } from "./defaults.js";
+import { defaultPlatforms, defaultStrippedParams, defaultTiers } from "./defaults.js";
 import { applyPlatformHandlers, feedsmithParser, nativeFetch, normalizeUrl, resolveUrl } from "./utils.js";
 
 //#region src/index.ts
-const findCanonical = async (inputUrl, options) => {
-	const { fetchFn = nativeFetch, existsFn, parser = feedsmithParser, tiers = defaultTiers, platforms = defaultPlatforms, onFetch, onMatch, onExists } = options ?? {};
+async function findCanonical(inputUrl, options) {
+	const { parser = feedsmithParser, fetchFn = nativeFetch, existsFn, tiers = defaultTiers, platforms = defaultPlatforms, stripQueryParams = defaultStrippedParams, onFetch, onMatch, onExists } = options ?? {};
+	const stripParams = (url) => {
+		return stripQueryParams?.length ? normalizeUrl(url, {
+			stripQueryParams,
+			sortQueryParams: true,
+			stripEmptyQuery: true
+		}) : url;
+	};
 	const resolveAndApplyPlatformHandlers = (url, baseUrl) => {
 		const resolved = resolveUrl(url, baseUrl);
 		return resolved ? applyPlatformHandlers(resolved, platforms) : void 0;
@@ -21,8 +28,9 @@ const findCanonical = async (inputUrl, options) => {
 		response: initialResponse
 	});
 	if (initialResponse.status < 200 || initialResponse.status >= 300) return;
-	const initialResponseUrl = resolveAndApplyPlatformHandlers(initialResponse.url);
-	if (!initialResponseUrl) return;
+	const initialResponseUrlRaw = resolveAndApplyPlatformHandlers(initialResponse.url);
+	if (!initialResponseUrlRaw) return;
+	const initialResponseUrl = stripParams(initialResponseUrlRaw);
 	const initialResponseBody = initialResponse.body;
 	if (!initialResponseBody) return;
 	let initialResponseSignature;
@@ -35,14 +43,17 @@ const findCanonical = async (inputUrl, options) => {
 		feed: initialResponseFeed
 	});
 	const selfRequestUrlRaw = parser.getSelfUrl(initialResponseFeed);
-	if (selfRequestUrlRaw) selfRequestUrl = resolveAndApplyPlatformHandlers(selfRequestUrlRaw, initialResponseUrl);
+	if (selfRequestUrlRaw) {
+		selfRequestUrl = resolveAndApplyPlatformHandlers(selfRequestUrlRaw, initialResponseUrl);
+		selfRequestUrl = selfRequestUrl ? stripParams(selfRequestUrl) : void 0;
+	}
 	const compareWithInitialResponse = (comparedResponseBody) => {
 		if (!comparedResponseBody) return false;
 		if (initialResponseBody === comparedResponseBody) return true;
 		const comparedResponseFeed = parser.parse(comparedResponseBody);
 		if (comparedResponseFeed) {
-			initialResponseSignature ||= JSON.stringify(parser.getSignature(initialResponseFeed));
-			const comparedResponseSignature = JSON.stringify(parser.getSignature(comparedResponseFeed));
+			initialResponseSignature ||= parser.getSignature(initialResponseFeed, parser.getSelfUrl(initialResponseFeed));
+			const comparedResponseSignature = parser.getSignature(comparedResponseFeed, parser.getSelfUrl(comparedResponseFeed));
 			return initialResponseSignature === comparedResponseSignature;
 		}
 		return false;
@@ -62,7 +73,7 @@ const findCanonical = async (inputUrl, options) => {
 		if (!compareWithInitialResponse(response.body)) return;
 		return response;
 	};
-	let variantSource = initialResponseUrl;
+	let variantSourceUrl = initialResponseUrl;
 	if (selfRequestUrl && selfRequestUrl !== initialResponseUrl) {
 		const urlsToTry = [selfRequestUrl];
 		if (selfRequestUrl.startsWith("https://")) urlsToTry.push(selfRequestUrl.replace("https://", "http://"));
@@ -75,40 +86,42 @@ const findCanonical = async (inputUrl, options) => {
 					response,
 					feed: initialResponseFeed
 				});
-				variantSource = resolveAndApplyPlatformHandlers(response.url) ?? initialResponseUrl;
+				variantSourceUrl = resolveAndApplyPlatformHandlers(response.url) ?? initialResponseUrl;
+				variantSourceUrl = stripParams(variantSourceUrl);
 				break;
 			}
 		}
 	}
-	const variants = new Set(tiers.map((tier) => resolveAndApplyPlatformHandlers(normalizeUrl(variantSource, tier))).filter((url) => url !== void 0));
-	variants.add(variantSource);
-	let winningUrl = variantSource;
-	for (const variant of variants) {
+	const variantUrls = new Set(tiers.map((tier) => resolveAndApplyPlatformHandlers(normalizeUrl(variantSourceUrl, tier))).filter((variantUrl) => !!variantUrl));
+	variantUrls.add(variantSourceUrl);
+	let winningUrl = variantSourceUrl;
+	for (const variantUrl of variantUrls) {
 		if (existsFn) {
-			const data = await existsFn(variant);
+			const data = await existsFn(variantUrl);
 			if (data !== void 0) {
 				onExists?.({
-					url: variant,
+					url: variantUrl,
 					data
 				});
-				return variant;
+				return variantUrl;
 			}
 		}
-		if (variant === variantSource) continue;
-		if (variant === initialResponseUrl) {
+		if (variantUrl === variantSourceUrl) continue;
+		if (variantUrl === initialResponseUrl) {
 			winningUrl = initialResponseUrl;
 			break;
 		}
-		const response = await fetchAndCompare(variant);
-		if (response) {
-			const preparedResponseUrl = resolveAndApplyPlatformHandlers(response.url);
-			if (preparedResponseUrl === variantSource || preparedResponseUrl === initialResponseUrl) continue;
+		const variantResponse = await fetchAndCompare(variantUrl);
+		if (variantResponse) {
+			let variantResponseUrl = resolveAndApplyPlatformHandlers(variantResponse.url);
+			if (variantResponseUrl) variantResponseUrl = stripParams(variantResponseUrl);
+			if (variantResponseUrl === variantSourceUrl || variantResponseUrl === initialResponseUrl) continue;
 			onMatch?.({
-				url: variant,
-				response,
+				url: variantUrl,
+				response: variantResponse,
 				feed: initialResponseFeed
 			});
-			winningUrl = variant;
+			winningUrl = variantUrl;
 			break;
 		}
 	}
@@ -125,7 +138,7 @@ const findCanonical = async (inputUrl, options) => {
 		}
 	}
 	return winningUrl;
-};
+}
 
 //#endregion
 export { findCanonical };

package/dist/types.d.cts

@@ -5,7 +5,7 @@ type FeedsmithFeed = ReturnType<typeof feedsmith0.parseFeed>;
 type ParserAdapter<T> = {
   parse: (body: string) => T | undefined;
   getSelfUrl: (parsed: T) => string | undefined;
-  getSignature: (parsed: T) => object;
+  getSignature: (parsed: T, selfUrl?: string) => string;
 };
 type PlatformHandler = {
   match: (url: URL) => boolean;
@@ -19,7 +19,6 @@ type NormalizeOptions = {
   stripRootSlash?: boolean;
   collapseSlashes?: boolean;
   stripHash?: boolean;
-  stripTextFragment?: boolean;
   sortQueryParams?: boolean;
   stripQueryParams?: Array<string>;
   stripEmptyQuery?: boolean;
@@ -28,40 +27,42 @@ type NormalizeOptions = {
   normalizeUnicode?: boolean;
   convertToPunycode?: boolean;
 };
-type OnFetchFn = (data: {
+type Tier = Omit<NormalizeOptions, 'stripQueryParams'>;
+type OnFetchFn<TResponse extends FetchFnResponse = FetchFnResponse> = (data: {
   url: string;
-  response: FetchFnResponse;
+  response: TResponse;
 }) => void;
-type OnMatchFn<TFeed = unknown> = (data: {
+type OnMatchFn<TFeed = unknown, TResponse extends FetchFnResponse = FetchFnResponse> = (data: {
   url: string;
-  response: FetchFnResponse;
+  response: TResponse;
   feed: TFeed;
 }) => void;
 type OnExistsFn<T> = (data: {
   url: string;
   data: T;
 }) => void;
-type ExistsFn<T = unknown> = (url: string) => Promise<T | undefined>;
-type FindCanonicalOptions<TFeed = FeedsmithFeed, TExisting = unknown> = {
+type FindCanonicalOptions<TFeed = FeedsmithFeed, TResponse extends FetchFnResponse = FetchFnResponse, TExisting = unknown> = {
   parser?: ParserAdapter<TFeed>;
-  fetchFn?: FetchFn;
+  fetchFn?: FetchFn<TResponse>;
   existsFn?: ExistsFn<TExisting>;
-  tiers?: Array<NormalizeOptions>;
+  tiers?: Array<Tier>;
   platforms?: Array<PlatformHandler>;
-  onFetch?: OnFetchFn;
-  onMatch?: OnMatchFn<TFeed>;
+  stripQueryParams?: Array<string>;
+  onFetch?: OnFetchFn<TResponse>;
+  onMatch?: OnMatchFn<TFeed, TResponse>;
   onExists?: OnExistsFn<TExisting>;
 };
 type FetchFnOptions = {
   method?: 'GET' | 'HEAD';
   headers?: Record<string, string>;
 };
+type ExistsFn<T = unknown> = (url: string) => Promise<T | undefined>;
 type FetchFnResponse = {
   headers: Headers;
   body: string;
   url: string;
   status: number;
 };
-type FetchFn = (url: string, options?: FetchFnOptions) => Promise<FetchFnResponse>;
+type FetchFn<TResponse extends FetchFnResponse = FetchFnResponse> = (url: string, options?: FetchFnOptions) => Promise<TResponse>;
 //#endregion
-export { ExistsFn, FeedsmithFeed, FetchFn, FetchFnOptions, FetchFnResponse, FindCanonicalOptions, NormalizeOptions, OnExistsFn, OnFetchFn, OnMatchFn, ParserAdapter, PlatformHandler };
+export { ExistsFn, FeedsmithFeed, FetchFn, FetchFnOptions, FetchFnResponse, FindCanonicalOptions, NormalizeOptions, OnExistsFn, OnFetchFn, OnMatchFn, ParserAdapter, PlatformHandler, Tier };

package/dist/types.d.ts

@@ -5,7 +5,7 @@ type FeedsmithFeed = ReturnType<typeof feedsmith0.parseFeed>;
 type ParserAdapter<T> = {
   parse: (body: string) => T | undefined;
   getSelfUrl: (parsed: T) => string | undefined;
-  getSignature: (parsed: T) => object;
+  getSignature: (parsed: T, selfUrl?: string) => string;
 };
 type PlatformHandler = {
   match: (url: URL) => boolean;
@@ -19,7 +19,6 @@ type NormalizeOptions = {
   stripRootSlash?: boolean;
   collapseSlashes?: boolean;
   stripHash?: boolean;
-  stripTextFragment?: boolean;
   sortQueryParams?: boolean;
   stripQueryParams?: Array<string>;
   stripEmptyQuery?: boolean;
@@ -28,40 +27,42 @@ type NormalizeOptions = {
   normalizeUnicode?: boolean;
   convertToPunycode?: boolean;
 };
-type OnFetchFn = (data: {
+type Tier = Omit<NormalizeOptions, 'stripQueryParams'>;
+type OnFetchFn<TResponse extends FetchFnResponse = FetchFnResponse> = (data: {
   url: string;
-  response: FetchFnResponse;
+  response: TResponse;
 }) => void;
-type OnMatchFn<TFeed = unknown> = (data: {
+type OnMatchFn<TFeed = unknown, TResponse extends FetchFnResponse = FetchFnResponse> = (data: {
   url: string;
-  response: FetchFnResponse;
+  response: TResponse;
   feed: TFeed;
 }) => void;
 type OnExistsFn<T> = (data: {
   url: string;
   data: T;
 }) => void;
-type ExistsFn<T = unknown> = (url: string) => Promise<T | undefined>;
-type FindCanonicalOptions<TFeed = FeedsmithFeed, TExisting = unknown> = {
+type FindCanonicalOptions<TFeed = FeedsmithFeed, TResponse extends FetchFnResponse = FetchFnResponse, TExisting = unknown> = {
   parser?: ParserAdapter<TFeed>;
-  fetchFn?: FetchFn;
+  fetchFn?: FetchFn<TResponse>;
   existsFn?: ExistsFn<TExisting>;
-  tiers?: Array<NormalizeOptions>;
+  tiers?: Array<Tier>;
   platforms?: Array<PlatformHandler>;
-  onFetch?: OnFetchFn;
-  onMatch?: OnMatchFn<TFeed>;
+  stripQueryParams?: Array<string>;
+  onFetch?: OnFetchFn<TResponse>;
+  onMatch?: OnMatchFn<TFeed, TResponse>;
   onExists?: OnExistsFn<TExisting>;
 };
 type FetchFnOptions = {
   method?: 'GET' | 'HEAD';
   headers?: Record<string, string>;
 };
+type ExistsFn<T = unknown> = (url: string) => Promise<T | undefined>;
 type FetchFnResponse = {
   headers: Headers;
   body: string;
   url: string;
   status: number;
 };
-type FetchFn = (url: string, options?: FetchFnOptions) => Promise<FetchFnResponse>;
+type FetchFn<TResponse extends FetchFnResponse = FetchFnResponse> = (url: string, options?: FetchFnOptions) => Promise<TResponse>;
 //#endregion
-export { ExistsFn, FeedsmithFeed, FetchFn, FetchFnOptions, FetchFnResponse, FindCanonicalOptions, NormalizeOptions, OnExistsFn, OnFetchFn, OnMatchFn, ParserAdapter, PlatformHandler };
+export { ExistsFn, FeedsmithFeed, FetchFn, FetchFnOptions, FetchFnResponse, FindCanonicalOptions, NormalizeOptions, OnExistsFn, OnFetchFn, OnMatchFn, ParserAdapter, PlatformHandler, Tier };

package/dist/utils.cjs

@@ -91,7 +91,6 @@ const normalizeUrl = (url, options = require_defaults.defaultNormalizeOptions) =
 		}
 		if (options.stripWww && parsed.hostname.startsWith("www.")) parsed.hostname = parsed.hostname.slice(4);
 		if (options.stripHash) parsed.hash = "";
-		if (options.stripTextFragment && parsed.hash.startsWith("#:~:")) parsed.hash = "";
 		let pathname = parsed.pathname;
 		if (options.normalizeEncoding) pathname = decodeAndNormalizeEncoding(pathname);
 		if (options.collapseSlashes) pathname = pathname.replace(/\/+/g, "/");
@@ -146,8 +145,9 @@ const feedsmithParser = {
 			case "json": return parsed.feed.feed_url;
 		}
 	},
-	getSignature: (parsed) => {
-		return parsed.feed;
+	getSignature: (parsed, selfUrl) => {
+		const signature = JSON.stringify(parsed.feed);
+		return selfUrl ? signature.replaceAll(`"${selfUrl}"`, "\"__SELF_URL__\"") : signature;
 	}
 };
 

package/dist/utils.js

@@ -91,7 +91,6 @@ const normalizeUrl = (url, options = defaultNormalizeOptions) => {
 		}
 		if (options.stripWww && parsed.hostname.startsWith("www.")) parsed.hostname = parsed.hostname.slice(4);
 		if (options.stripHash) parsed.hash = "";
-		if (options.stripTextFragment && parsed.hash.startsWith("#:~:")) parsed.hash = "";
 		let pathname = parsed.pathname;
 		if (options.normalizeEncoding) pathname = decodeAndNormalizeEncoding(pathname);
 		if (options.collapseSlashes) pathname = pathname.replace(/\/+/g, "/");
@@ -146,8 +145,9 @@ const feedsmithParser = {
 			case "json": return parsed.feed.feed_url;
 		}
 	},
-	getSignature: (parsed) => {
-		return parsed.feed;
+	getSignature: (parsed, selfUrl) => {
+		const signature = JSON.stringify(parsed.feed);
+		return selfUrl ? signature.replaceAll(`"${selfUrl}"`, "\"__SELF_URL__\"") : signature;
 	}
 };
 

package/package.json

@@ -60,5 +60,5 @@
     "kvalita": "1.9.0",
     "tsdown": "^0.18.2"
   },
-  "version": "0.9.2"
+  "version": "1.0.0-beta.2"
 }