feedsweep 1.0.0 → 1.1.0

package/README.md

@@ -19,7 +19,7 @@ npm install feedsweep
 ```typescript
 import { transformContent } from 'feedsweep'
 
-const result = transformContent('<p>Check <img data-src="photo.jpg"> and visit /about</p>', {
+const result = await transformContent('<p>Check <img data-src="photo.jpg"> and visit /about</p>', {
   baseUrl: 'https://example.com/post/1',
 })
 ```
@@ -30,27 +30,33 @@ Inventory of every transform exported from the package. Most are enabled by defa
 
 | Transform | Description |
 | --- | --- |
-| `stripOrphanedClosingTags` | Remove unmatched `</p>` / `</div>` close tags |
 | `decodeDoubleEncodedTags` | Decode `&lt;tag&gt;` back to `<tag>` in mixed content |
-| `unwrapWrappers` | Remove outer `<div>`, `<article>`, `<section>` wrappers |
-| `paragraphizePlainText` | Wrap plain text in `<p>` tags |
-| `stripEmptyTags` | Remove empty `<p>`, `<div>`, `<span>` and other tags |
-| `stripComments` | Remove HTML `<!-- comments -->` |
 | `fixLazyImages` | Move `data-src` / `data-original` to real `src` |
-| `resolveRelativeUrls` | Convert relative URLs to absolute using base URL |
-| `unwrapRedirectUrls` | Remove Google/Bing/Facebook/etc. redirect wrappers |
-| `stripTrackingParams` | Remove UTM and other tracking parameters |
-| `removeTrackingPixels` | Strip 1×1 tracking pixel images |
-| `stripInterBlockBreaks` | Remove `<br>` tags between block elements |
-| `stripParagraphBoundaryBreaks` | Remove `<br>` tags adjacent to paragraph boundaries |
-| `highlightCode` | Syntax-highlight `<code>` blocks with highlight.js |
 | `mergeConsecutiveOneLinerPres` | Merge consecutive single-line `<pre>` tags |
 | `replacePreLineBreaks` | Replace `<br>` with `\n` inside `<pre>` |
-| `trimPreWhitespace` | Remove common leading indentation from `<pre>` |
-| `linkifyUrls` | Wrap bare URLs in `<a>` tags |
+| `stripInterBlockBreaks` | Remove `<br>` tags between block elements |
+| `stripParagraphBoundaryBreaks` | Remove `<br>` tags adjacent to paragraph boundaries |
+| `stripDuplicateTitleHeading` | Remove first `<h1>`–`<h6>` matching article title |
+| `unwrapRedirectUrls` | Remove Google/Bing/Facebook/etc. redirect wrappers |
+| `stripDeadAnchors` | Unwrap `<a>` with empty, `#`, or `javascript:` href |
+| `removeTrackingPixels` | Strip 1×1 tracking pixel images |
+| `stripTrackingParams` | Remove UTM and other tracking parameters |
+| `convertBreaksToParagraphs` | Convert `<br><br>` runs into semantic `<p>` blocks |
+| `injectEnclosures` | Inject feed enclosures into content as native `<audio>`/`<video>` or iframe placeholders |
 | `replaceEmbedsWithPlaceholders` | Convert `<iframe>` to embed placeholders |
-| `injectEnclosureEmbedPlaceholders` | Add audio/video enclosures to content |
-| `simplifyFigures` | Unwrap `<figure>` when the figcaption is empty or redundant |
+| `enrichEmbedPlaceholders` | Populate placeholder metadata (`title`, `description`, `duration`, etc.) via a caller-supplied async fn. Opt-in; not in defaults |
+| `proxyAssetUrls` | Rewrite image, video, and audio URLs through a caller-supplied proxy |
+| `resolveRelativeUrls` | Convert relative URLs to absolute using base URL |
+| `unwrapWrappers` | Remove outer `<div>`, `<article>`, `<section>` wrappers |
+| `unwrapDoublyNestedLists` | Unwrap `<ul>`/`<ol>` that wrap a single `<li>` containing a same-type list |
+| `mergeFragmentedLists` | Merge consecutive sibling `<ul>` / `<ol>` lists with matching attributes |
+| `paragraphizePlainText` | Wrap plain text in `<p>` tags |
+| `linkifyUrls` | Wrap bare URLs in `<a>` tags |
+| `trimPreWhitespace` | Remove common leading indentation from `<pre>` |
+| `highlightCode` | Syntax-highlight `<code>` blocks with highlight.js |
+| `stripEmptyTags` | Remove empty `<p>`, `<div>`, `<span>` and other tags |
+| `stripComments` | Remove HTML `<!-- comments -->` |
+| `unwrapCdataComments` | Strip malformed `<!--[CDATA[ … ]]-->` wrappers before parsing so the wrapped article reaches the DOM as real HTML |
 
 ## Options
 
@@ -62,6 +68,12 @@ const result = transformContent(html, {
   baseUrl: 'https://example.com/post/1',
   // Feed item enclosures (audio/video).
   enclosures: [{ url: 'https://example.com/audio.mp3', type: 'audio/mpeg' }],
+  // Route image/video/audio URLs through a proxy. Return `undefined` to leave a URL untouched.
+  assetProxyFn: (url, type) => `https://proxy.example.com/?type=${type}&url=${encodeURIComponent(url)}`,
+  // Populate embed placeholder metadata from a remote source (e.g. YouTube oEmbed).
+  enrichEmbedFn: async (embeds) => {
+    return new Map(embeds.map(({ provider, id }) => [`${provider}:${id}`, { title: '…' }]))
+  },
   // Run a custom DOM transform pipeline (omit to use defaults).
   domTransforms: [fixLazyImages, resolveRelativeUrls],
 })

package/dist/common.d.ts

@@ -1,11 +1,15 @@
-import { EmbedResolverResult } from "./types.js";
+import { EmbedResolverResult, MaybePromise } from "./types.js";
 
 //#region src/common.d.ts
 declare const stripOversizedBase64Sources: (html: string, maxSize: number) => string;
+declare const expandSvgSelfClose: (html: string) => string;
 declare const parseFragment: (html: string) => Document;
-declare const transformHtml: (html: string, transform: (document: Document) => void) => string;
-declare const applyDomTransforms: (html: string, transforms: Array<(document: Document) => void>) => string;
-declare const applyStringTransforms: (html: string, transforms: Array<(html: string) => string>) => string;
-declare const createEmbedPlaceholder: (document: Document, src: string, type: "video" | "audio" | "iframe", metadata?: Partial<EmbedResolverResult>) => HTMLElement;
+declare const transformHtml: (html: string, transform: (document: Document) => MaybePromise<void>) => Promise<string>;
+declare const applyDomTransforms: (html: string, transforms: Array<(document: Document) => MaybePromise<void>>) => Promise<string>;
+declare const applyStringTransforms: (html: string, transforms: Array<(html: string) => MaybePromise<string>>) => Promise<string>;
+declare const applyEmbedMetadata: (element: HTMLElement, metadata: Partial<EmbedResolverResult>, options?: {
+  setIfMissing?: boolean;
+}) => void;
+declare const createEmbedPlaceholder: (document: Document, src: string, metadata?: Partial<EmbedResolverResult>) => HTMLElement;
 //#endregion
-export { applyDomTransforms, applyStringTransforms, createEmbedPlaceholder, parseFragment, stripOversizedBase64Sources, transformHtml };
+export { applyDomTransforms, applyEmbedMetadata, applyStringTransforms, createEmbedPlaceholder, expandSvgSelfClose, parseFragment, stripOversizedBase64Sources, transformHtml };

package/dist/common.js

@@ -1,3 +1,4 @@
+import { coerceNumber } from "./utils.js";
 import { resolveUrl } from "feedcanon";
 import { parseHTML } from "linkedom";
 //#region src/common.ts
@@ -6,6 +7,11 @@ const Node = {
 	TEXT_NODE: 3,
 	COMMENT_NODE: 8
 };
+const NodeFilter = {
+	SHOW_ELEMENT: 1,
+	SHOW_TEXT: 4,
+	SHOW_COMMENT: 128
+};
 const base64SrcRegex = /((?:src|srcset|poster)=["'])data:[^"']*;base64,[^"']*(["'])/g;
 const safeThumbnailDataUrlRegex = /^data:image\/(png|jpe?g|gif|webp|avif);/i;
 const isSafeThumbnailUrl = (url) => {
@@ -17,23 +23,53 @@ const stripOversizedBase64Sources = (html, maxSize) => {
 		return `${prefix}${suffix}`;
 	});
 };
+const normalizeAttributeCase = (document) => {
+	for (const element of document.querySelectorAll("*")) {
+		const original = Array.from(element.attributes).map((attribute) => ({
+			name: attribute.name,
+			value: attribute.value
+		}));
+		const final = /* @__PURE__ */ new Map();
+		let needsRewrite = false;
+		for (const { name, value } of original) {
+			const lower = name.toLowerCase();
+			if (lower !== name) needsRewrite = true;
+			if (final.has(lower)) {
+				needsRewrite = true;
+				continue;
+			}
+			final.set(lower, value);
+		}
+		if (!needsRewrite) continue;
+		for (const { name } of original) element.removeAttribute(name);
+		for (const [name, value] of final) element.setAttribute(name, value);
+	}
+};
+const svgRegionRegex = /<svg\b[^>]*>[\s\S]*?<\/svg>/gi;
+const svgSelfCloseRegex = /<([a-z][a-z0-9-]*)((?:\s[^>]*)?)\s*\/>/gi;
+const expandSvgSelfClose = (html) => {
+	return html.replace(svgRegionRegex, (svgBlock) => {
+		return svgBlock.replace(svgSelfCloseRegex, "<$1$2></$1>");
+	});
+};
 const parseFragment = (html) => {
-	const { document } = parseHTML(`<!doctype html><html><head></head><body>${html}</body></html>`);
+	const { document } = parseHTML(`<!doctype html><html><head></head><body>${expandSvgSelfClose(html)}</body></html>`);
+	normalizeAttributeCase(document);
 	return document;
 };
-const transformHtml = (html, transform) => {
+const transformHtml = async (html, transform) => {
 	const document = parseFragment(html);
-	transform(document);
+	await transform(document);
 	return document.body.innerHTML;
 };
-const applyDomTransforms = (html, transforms) => {
+const applyDomTransforms = async (html, transforms) => {
 	const document = parseFragment(stripOversizedBase64Sources(html, 50 * 1024));
-	for (const transform of transforms) transform(document);
+	for (const transform of transforms) await transform(document);
 	return document.body.innerHTML;
 };
-const applyStringTransforms = (html, transforms) => {
+const applyStringTransforms = async (html, transforms) => {
 	let output = html;
-	for (const transform of transforms) output = transform(output);
+	for (const transform of transforms) output = await transform(output);
 	return output;
 };
 const blockElements = new Set([
@@ -71,10 +107,10 @@ const blockElements = new Set([
 	"ul"
 ]);
 const isWhitespaceText = (node) => {
-	return node.nodeType === Node.TEXT_NODE && !(node.textContent ?? "").trim();
+	return node.nodeType === Node.TEXT_NODE && !node.textContent?.trim();
 };
 const isBr = (node) => {
-	return node.nodeType === Node.ELEMENT_NODE && node.tagName.toLowerCase() === "br";
+	return node.nodeType === Node.ELEMENT_NODE && node.localName === "br";
 };
 const isComment = (node) => {
 	return node.nodeType === Node.COMMENT_NODE;
@@ -83,28 +119,63 @@ const isSkippable = (node) => {
 	return isWhitespaceText(node) || isBr(node) || isComment(node);
 };
 const isBlockElement = (node) => {
-	return node.nodeType === Node.ELEMENT_NODE && blockElements.has(node.tagName.toLowerCase());
-};
-const unwrapOuterTag = (html, pattern) => {
-	let result = html.trim();
-	let match = pattern.exec(result);
-	while (match) {
-		result = match[3].trim();
-		match = pattern.exec(result);
+	return node.nodeType === Node.ELEMENT_NODE && blockElements.has(node.localName);
+};
+const hasAncestorWithTagName = (node, tagSet, stopAt) => {
+	let ancestor = node.parentNode;
+	while (ancestor !== null && ancestor !== stopAt) {
+		if (ancestor.nodeType === Node.ELEMENT_NODE && tagSet.has(ancestor.localName)) return true;
+		ancestor = ancestor.parentNode;
 	}
-	return result;
+	return false;
+};
+const styleWidthRegex = /(?:^|;)\s*width\s*:\s*([0-9]*\.?[0-9]+)\s*(?:px)?\s*(?:;|$)/i;
+const styleHeightRegex = /(?:^|;)\s*height\s*:\s*([0-9]*\.?[0-9]+)\s*(?:px)?\s*(?:;|$)/i;
+const getDimensions = (element) => {
+	const width = coerceNumber(element.getAttribute("width"));
+	const height = coerceNumber(element.getAttribute("height"));
+	if (width !== void 0 && height !== void 0) return {
+		width,
+		height
+	};
+	const style = element.getAttribute("style");
+	if (!style) return {
+		width,
+		height
+	};
+	const fromStyle = (regex) => {
+		const match = regex.exec(style);
+		return match ? coerceNumber(match[1]) : void 0;
+	};
+	return {
+		width: width ?? fromStyle(styleWidthRegex),
+		height: height ?? fromStyle(styleHeightRegex)
+	};
+};
+const applyEmbedMetadata = (element, metadata, options) => {
+	const setIfMissing = options?.setIfMissing ?? false;
+	const set = (name, value) => {
+		if (setIfMissing && element.hasAttribute(name)) return;
+		element.setAttribute(name, value);
+	};
+	if (metadata.provider) set("data-embed-provider", metadata.provider);
+	if (metadata.id) set("data-embed-id", metadata.id);
+	if (metadata.src) set("data-embed-src", metadata.src);
+	if (metadata.url) set("data-embed-url", metadata.url);
+	if (metadata.thumbnail && isSafeThumbnailUrl(metadata.thumbnail)) set("data-embed-thumbnail", metadata.thumbnail);
+	if (metadata.width) set("data-embed-width", String(metadata.width));
+	if (metadata.height) set("data-embed-height", String(metadata.height));
+	if (metadata.title) set("data-embed-title", metadata.title);
+	if (metadata.description) set("data-embed-description", metadata.description);
+	if (metadata.author) set("data-embed-author", metadata.author);
+	if (metadata.avatar && isSafeThumbnailUrl(metadata.avatar)) set("data-embed-avatar", metadata.avatar);
+	if (metadata.duration) set("data-embed-duration", String(metadata.duration));
 };
-const createEmbedPlaceholder = (document, src, type, metadata) => {
+const createEmbedPlaceholder = (document, src, metadata) => {
 	const element = document.createElement("div");
-	element.setAttribute("data-embed", metadata?.type ?? type);
+	element.setAttribute("data-embed", "iframe");
 	element.setAttribute("data-embed-src", metadata?.src ?? src);
-	if (metadata?.provider) element.setAttribute("data-embed-provider", metadata.provider);
-	if (metadata?.url) element.setAttribute("data-embed-url", metadata.url);
-	if (metadata?.thumbnail && isSafeThumbnailUrl(metadata.thumbnail)) element.setAttribute("data-embed-thumbnail", metadata.thumbnail);
-	if (metadata?.width) element.setAttribute("data-embed-width", String(metadata.width));
-	if (metadata?.height) element.setAttribute("data-embed-height", String(metadata.height));
-	if (metadata?.author) element.setAttribute("data-embed-author", metadata.author);
-	if (metadata?.text) element.setAttribute("data-embed-text", metadata.text);
+	if (metadata) applyEmbedMetadata(element, metadata);
 	const fallbackUrl = metadata?.url ?? metadata?.src ?? src;
 	const link = document.createElement("a");
 	link.setAttribute("href", fallbackUrl);
@@ -113,4 +184,4 @@ const createEmbedPlaceholder = (document, src, type, metadata) => {
 	return element;
 };
 //#endregion
-export { Node, applyDomTransforms, applyStringTransforms, createEmbedPlaceholder, isBlockElement, isBr, isSkippable, parseFragment, stripOversizedBase64Sources, transformHtml, unwrapOuterTag };
+export { Node, NodeFilter, applyDomTransforms, applyEmbedMetadata, applyStringTransforms, createEmbedPlaceholder, expandSvgSelfClose, getDimensions, hasAncestorWithTagName, isBlockElement, isBr, isSkippable, isWhitespaceText, normalizeAttributeCase, parseFragment, stripOversizedBase64Sources, transformHtml };

package/dist/defaults.d.ts

@@ -7,8 +7,9 @@ declare const defaultFinalStringTransforms: Array<StringTransform>;
 declare const defaultEmbedResolvers: Array<EmbedResolver>;
 declare const defaultResolveUrlFn: ResolveUrlFn;
 declare const defaultLazySrcAttributes: string[];
+declare const defaultLazySrcsetAttributes: string[];
 declare const defaultTrackingHosts: string[];
 declare const defaultTrackingPathSegments: string[];
 declare const defaultUrlUnwrappers: Array<UrlUnwrapper>;
 //#endregion
-export { defaultDomTransforms, defaultEmbedResolvers, defaultFinalStringTransforms, defaultLazySrcAttributes, defaultResolveUrlFn, defaultStringTransforms, defaultTrackingHosts, defaultTrackingPathSegments, defaultUrlUnwrappers };
+export { defaultDomTransforms, defaultEmbedResolvers, defaultFinalStringTransforms, defaultLazySrcAttributes, defaultLazySrcsetAttributes, defaultResolveUrlFn, defaultStringTransforms, defaultTrackingHosts, defaultTrackingPathSegments, defaultUrlUnwrappers };

package/dist/defaults.js

@@ -1,24 +1,30 @@
 import { youtubeEmbedResolver } from "./embeds/youtube.js";
+import { convertBreaksToParagraphs } from "./transforms/dom/convertBreaksToParagraphs.js";
+import { decodeDoubleEncodedTags } from "./transforms/dom/decodeDoubleEncodedTags.js";
 import { fixLazyImages } from "./transforms/dom/fixLazyImages.js";
 import { highlightCode } from "./transforms/dom/highlightCode.js";
-import { injectEnclosureEmbedPlaceholders } from "./transforms/dom/injectEnclosureEmbedPlaceholders.js";
+import { injectEnclosures } from "./transforms/dom/injectEnclosures.js";
 import { linkifyUrls } from "./transforms/dom/linkifyUrls.js";
 import { mergeConsecutiveOneLinerPres } from "./transforms/dom/mergeConsecutiveOneLinerPres.js";
+import { mergeFragmentedLists } from "./transforms/dom/mergeFragmentedLists.js";
+import { proxyAssetUrls } from "./transforms/dom/proxyAssetUrls.js";
 import { removeTrackingPixels } from "./transforms/dom/removeTrackingPixels.js";
 import { replaceEmbedsWithPlaceholders } from "./transforms/dom/replaceEmbedsWithPlaceholders.js";
 import { replacePreLineBreaks } from "./transforms/dom/replacePreLineBreaks.js";
 import { resolveRelativeUrls } from "./transforms/dom/resolveRelativeUrls.js";
 import { stripComments } from "./transforms/dom/stripComments.js";
+import { stripDeadAnchors } from "./transforms/dom/stripDeadAnchors.js";
+import { stripDuplicateTitleHeading } from "./transforms/dom/stripDuplicateTitleHeading.js";
+import { stripEmptyTags } from "./transforms/dom/stripEmptyTags.js";
 import { stripInterBlockBreaks } from "./transforms/dom/stripInterBlockBreaks.js";
 import { stripParagraphBoundaryBreaks } from "./transforms/dom/stripParagraphBoundaryBreaks.js";
 import { stripTrackingParams } from "./transforms/dom/stripTrackingParams.js";
 import { trimPreWhitespace } from "./transforms/dom/trimPreWhitespace.js";
+import { unwrapDoublyNestedLists } from "./transforms/dom/unwrapDoublyNestedLists.js";
 import { unwrapRedirectUrls } from "./transforms/dom/unwrapRedirectUrls.js";
-import { decodeDoubleEncodedTags } from "./transforms/string/decodeDoubleEncodedTags.js";
+import { unwrapWrappers } from "./transforms/dom/unwrapWrappers.js";
 import { paragraphizePlainText } from "./transforms/string/paragraphizePlainText.js";
-import { stripEmptyTags } from "./transforms/string/stripEmptyTags.js";
-import { stripOrphanedClosingTags } from "./transforms/string/stripOrphanedClosingTags.js";
-import { unwrapWrappers } from "./transforms/string/unwrapWrappers.js";
+import { unwrapCdataComments } from "./transforms/string/unwrapCdataComments.js";
 import { unwrapBing } from "./unwraps/bing.js";
 import { unwrapFacebookShim } from "./unwraps/facebook.js";
 import { unwrapGoogle } from "./unwraps/google.js";
@@ -33,31 +39,34 @@ import { unwrapYahooSearch } from "./unwraps/yahooSearch.js";
 import { unwrapYouTube } from "./unwraps/youtube.js";
 import { resolveUrl } from "feedcanon";
 //#region src/defaults.ts
-const defaultStringTransforms = [
-	stripOrphanedClosingTags,
-	decodeDoubleEncodedTags,
-	unwrapWrappers,
-	paragraphizePlainText,
-	stripEmptyTags
-];
+const defaultStringTransforms = [unwrapCdataComments, paragraphizePlainText];
 const defaultDomTransforms = [
+	decodeDoubleEncodedTags,
 	stripComments,
+	unwrapDoublyNestedLists,
+	stripDuplicateTitleHeading,
 	fixLazyImages,
 	resolveRelativeUrls,
 	unwrapRedirectUrls,
+	stripDeadAnchors,
 	stripTrackingParams,
 	removeTrackingPixels,
+	convertBreaksToParagraphs,
 	stripInterBlockBreaks,
 	stripParagraphBoundaryBreaks,
+	mergeFragmentedLists,
 	highlightCode,
 	mergeConsecutiveOneLinerPres,
 	replacePreLineBreaks,
 	trimPreWhitespace,
 	linkifyUrls,
 	replaceEmbedsWithPlaceholders,
-	injectEnclosureEmbedPlaceholders
+	injectEnclosures,
+	proxyAssetUrls,
+	unwrapWrappers,
+	stripEmptyTags
 ];
-const defaultFinalStringTransforms = [stripEmptyTags];
+const defaultFinalStringTransforms = [];
 const defaultEmbedResolvers = [youtubeEmbedResolver];
 const defaultResolveUrlFn = (url, baseUrl) => resolveUrl(url, baseUrl);
 const defaultLazySrcAttributes = [
@@ -75,9 +84,27 @@ const defaultLazySrcAttributes = [
 	"data-image-src",
 	"data-canonical-src",
 	"data-img-url",
+	"nitro-lazy-src",
 	"data-orig",
 	"data-runner-src"
 ];
+const defaultLazySrcsetAttributes = [
+	"data-srcset",
+	"data-tf-srcset",
+	"data-lazy-srcset",
+	"data-image-srcset",
+	"data-modal-srcset",
+	"data-splide-lazy-srcset",
+	"data-alt-srcset",
+	"fifu-data-srcset",
+	"data-thumb-srcset",
+	"data-vp-popup-img-srcset",
+	"data-original-srcset",
+	"data-pswp-srcset",
+	"data-nectar-img-srcset",
+	"nitro-lazy-srcset",
+	"data-flickity-lazyload-srcset"
+];
 const defaultTrackingHosts = [
 	"feedsportal.com",
 	"stats.wordpress.com",
@@ -102,12 +129,14 @@ const defaultTrackingHosts = [
 	"quantserve.com",
 	"chartbeat.com",
 	"moatads.com",
-	"sentry.io"
+	"sentry.io",
+	"hubspot.com"
 ];
 const defaultTrackingPathSegments = [
 	"pixel",
 	"beacon",
-	"count"
+	"count",
+	"impression"
 ];
 const defaultUrlUnwrappers = [
 	unwrapBing,
@@ -124,4 +153,4 @@ const defaultUrlUnwrappers = [
 	unwrapRedditOut
 ];
 //#endregion
-export { defaultDomTransforms, defaultEmbedResolvers, defaultFinalStringTransforms, defaultLazySrcAttributes, defaultResolveUrlFn, defaultStringTransforms, defaultTrackingHosts, defaultTrackingPathSegments, defaultUrlUnwrappers };
+export { defaultDomTransforms, defaultEmbedResolvers, defaultFinalStringTransforms, defaultLazySrcAttributes, defaultLazySrcsetAttributes, defaultResolveUrlFn, defaultStringTransforms, defaultTrackingHosts, defaultTrackingPathSegments, defaultUrlUnwrappers };

package/dist/embeds/youtube.js

@@ -32,10 +32,10 @@ const youtubeResolveEmbed = (url) => {
 	if (!videoId) return;
 	return {
 		provider: "youtube",
+		id: videoId,
 		src: `https://www.youtube-nocookie.com/embed/${videoId}`,
 		url: `https://www.youtube.com/watch?v=${videoId}`,
-		thumbnail: composeThumbnailUrl(videoId),
-		type: "iframe"
+		thumbnail: composeThumbnailUrl(videoId)
 	};
 };
 const youtubeEmbedResolver = {

package/dist/index.d.ts

@@ -1,28 +1,34 @@
-import { DomTransform, EmbedResolver, EmbedResolverResult, Enclosure, ResolveUrlFn, StringTransform, TransformContentOptions, TransformContext } from "./types.js";
+import { AssetProxyFn, AssetType, DomTransform, EmbedResolver, EmbedResolverResult, Enclosure, EnrichEmbedFn, MaybePromise, ResolveUrlFn, StringTransform, TransformContentOptions, TransformContext } from "./types.js";
 import { defaultResolveUrlFn } from "./defaults.js";
-import { applyDomTransforms, applyStringTransforms, createEmbedPlaceholder, parseFragment, stripOversizedBase64Sources, transformHtml } from "./common.js";
+import { applyDomTransforms, applyEmbedMetadata, applyStringTransforms, createEmbedPlaceholder, expandSvgSelfClose, parseFragment, stripOversizedBase64Sources, transformHtml } from "./common.js";
 import { composeThumbnailUrl, extractVideoId, youtubeEmbedResolver, youtubeResolveEmbed } from "./embeds/youtube.js";
+import { convertBreaksToParagraphs } from "./transforms/dom/convertBreaksToParagraphs.js";
+import { decodeDoubleEncodedTags } from "./transforms/dom/decodeDoubleEncodedTags.js";
+import { enrichEmbedPlaceholders } from "./transforms/dom/enrichEmbedPlaceholders.js";
 import { fixLazyImages } from "./transforms/dom/fixLazyImages.js";
 import { detectLanguage, highlightCode } from "./transforms/dom/highlightCode.js";
-import { injectEnclosureEmbedPlaceholders } from "./transforms/dom/injectEnclosureEmbedPlaceholders.js";
+import { injectEnclosures } from "./transforms/dom/injectEnclosures.js";
 import { linkifyUrls } from "./transforms/dom/linkifyUrls.js";
 import { mergeConsecutiveOneLinerPres } from "./transforms/dom/mergeConsecutiveOneLinerPres.js";
+import { mergeFragmentedLists } from "./transforms/dom/mergeFragmentedLists.js";
+import { proxyAssetUrls } from "./transforms/dom/proxyAssetUrls.js";
 import { removeTrackingPixels } from "./transforms/dom/removeTrackingPixels.js";
 import { replaceEmbedsWithPlaceholders } from "./transforms/dom/replaceEmbedsWithPlaceholders.js";
 import { replacePreLineBreaks } from "./transforms/dom/replacePreLineBreaks.js";
 import { resolveRelativeUrls } from "./transforms/dom/resolveRelativeUrls.js";
-import { simplifyFigures } from "./transforms/dom/simplifyFigures.js";
 import { stripComments } from "./transforms/dom/stripComments.js";
+import { stripDeadAnchors } from "./transforms/dom/stripDeadAnchors.js";
+import { stripDuplicateTitleHeading } from "./transforms/dom/stripDuplicateTitleHeading.js";
+import { stripEmptyTags } from "./transforms/dom/stripEmptyTags.js";
 import { stripInterBlockBreaks } from "./transforms/dom/stripInterBlockBreaks.js";
 import { stripParagraphBoundaryBreaks } from "./transforms/dom/stripParagraphBoundaryBreaks.js";
 import { stripTrackingParams } from "./transforms/dom/stripTrackingParams.js";
 import { trimPreWhitespace } from "./transforms/dom/trimPreWhitespace.js";
+import { unwrapDoublyNestedLists } from "./transforms/dom/unwrapDoublyNestedLists.js";
 import { extractRedirectTarget, unwrapRedirectUrls } from "./transforms/dom/unwrapRedirectUrls.js";
-import { decodeDoubleEncodedTags } from "./transforms/string/decodeDoubleEncodedTags.js";
+import { unwrapWrappers } from "./transforms/dom/unwrapWrappers.js";
 import { paragraphizePlainText } from "./transforms/string/paragraphizePlainText.js";
-import { stripEmptyTags } from "./transforms/string/stripEmptyTags.js";
-import { stripOrphanedClosingTags } from "./transforms/string/stripOrphanedClosingTags.js";
-import { unwrapWrappers } from "./transforms/string/unwrapWrappers.js";
+import { unwrapCdataComments } from "./transforms/string/unwrapCdataComments.js";
 import { unwrapAceml } from "./unwraps/aceml.js";
 import { unwrapAdjust } from "./unwraps/adjust.js";
 import { unwrapAmazonAffiliate } from "./unwraps/amazonAffiliate.js";
@@ -100,6 +106,6 @@ import { unwrapZhihu } from "./unwraps/zhihu.js";
 import { ParamExtractorConfig, chooseBaseUrl, coerceNumber, createParamExtractor } from "./utils.js";
 
 //#region src/index.d.ts
-declare const transformContent: (html: string, options?: TransformContentOptions) => string;
+declare const transformContent: (html: string, options?: TransformContentOptions) => Promise<string>;
 //#endregion
-export { type DomTransform, type EmbedResolver, type EmbedResolverResult, type Enclosure, type ParamExtractorConfig, type ResolveUrlFn, type StringTransform, type TransformContentOptions, type TransformContext, applyDomTransforms, applyStringTransforms, chooseBaseUrl, coerceNumber, composeThumbnailUrl, createEmbedPlaceholder, createParamExtractor, decodeDoubleEncodedTags, defaultResolveUrlFn, detectLanguage, extractRedirectTarget, extractVideoId, fixLazyImages, highlightCode, injectEnclosureEmbedPlaceholders, linkifyUrls, mergeConsecutiveOneLinerPres, paragraphizePlainText, parseFragment, removeTrackingPixels, replaceEmbedsWithPlaceholders, replacePreLineBreaks, resolveRelativeUrls, simplifyFigures, stripComments, stripEmptyTags, stripInterBlockBreaks, stripOrphanedClosingTags, stripOversizedBase64Sources, stripParagraphBoundaryBreaks, stripTrackingParams, transformContent, transformHtml, trimPreWhitespace, unwrapAceml, unwrapAdjust, unwrapAmazonAffiliate, unwrapAmpCache, unwrapAwin, unwrapBing, unwrapCjNetwork, unwrapDigidip, unwrapDisqus, unwrapDouban, unwrapDuckduckgo, unwrapEbayRover, unwrapEffiliation, unwrapEmbedly, unwrapFacebookShim, unwrapFeedsportal, unwrapFirebaseDynamicLinks, unwrapFlipboard, unwrapGateSc, unwrapGeoriot, unwrapGitee, unwrapGoogle, unwrapGoogleAmpViewer, unwrapGoogleNews, unwrapGoogleNewsModern, unwrapGoogleScholar, unwrapGoogleTranslate, unwrapHashnode, unwrapIcptrack, unwrapIdealoPartner, unwrapInstagramShim, unwrapJianshuGo, unwrapJuejin, unwrapLeverAnalytics, unwrapLinksynergy, unwrapMailchimp, unwrapMailpanion, unwrapMailpgn, unwrapMailtrack, unwrapMedium, unwrapMimecast, unwrapMozillaOutgoing, unwrapNarrativ, unwrapNicoMs, unwrapOutlookSafelinks, unwrapPartnerAds, unwrapPocket, unwrapPostmark, unwrapProofpointV1, unwrapProofpointV2, unwrapProofpointV3, unwrapPxf, unwrapRecruitics, unwrapRedditOut, unwrapRedirectUrls, unwrapRedirectingat, unwrapSegmentfault, unwrapShareasale, unwrapSjv, unwrapSkimlinks, unwrapSlack, unwrapSmartredirect, unwrapSspai, unwrapSteamLinkfilter, unwrapTelegramIv, unwrapTradedoubler, unwrapTumblr, unwrapValuecommerce, unwrapViglink, unwrapVkAway, unwrapWebArchive, unwrapWrappers, unwrapYahooSearch, unwrapYandexTurbo, unwrapYouTube, unwrapZhihu, youtubeEmbedResolver, youtubeResolveEmbed };
+export { type AssetProxyFn, type AssetType, type DomTransform, type EmbedResolver, type EmbedResolverResult, type Enclosure, type EnrichEmbedFn, type MaybePromise, type ParamExtractorConfig, type ResolveUrlFn, type StringTransform, type TransformContentOptions, type TransformContext, applyDomTransforms, applyEmbedMetadata, applyStringTransforms, chooseBaseUrl, coerceNumber, composeThumbnailUrl, convertBreaksToParagraphs, createEmbedPlaceholder, createParamExtractor, decodeDoubleEncodedTags, defaultResolveUrlFn, detectLanguage, enrichEmbedPlaceholders, expandSvgSelfClose, extractRedirectTarget, extractVideoId, fixLazyImages, highlightCode, injectEnclosures, linkifyUrls, mergeConsecutiveOneLinerPres, mergeFragmentedLists, paragraphizePlainText, parseFragment, proxyAssetUrls, removeTrackingPixels, replaceEmbedsWithPlaceholders, replacePreLineBreaks, resolveRelativeUrls, stripComments, stripDeadAnchors, stripDuplicateTitleHeading, stripEmptyTags, stripInterBlockBreaks, stripOversizedBase64Sources, stripParagraphBoundaryBreaks, stripTrackingParams, transformContent, transformHtml, trimPreWhitespace, unwrapAceml, unwrapAdjust, unwrapAmazonAffiliate, unwrapAmpCache, unwrapAwin, unwrapBing, unwrapCdataComments, unwrapCjNetwork, unwrapDigidip, unwrapDisqus, unwrapDouban, unwrapDoublyNestedLists, unwrapDuckduckgo, unwrapEbayRover, unwrapEffiliation, unwrapEmbedly, unwrapFacebookShim, unwrapFeedsportal, unwrapFirebaseDynamicLinks, unwrapFlipboard, unwrapGateSc, unwrapGeoriot, unwrapGitee, unwrapGoogle, unwrapGoogleAmpViewer, unwrapGoogleNews, unwrapGoogleNewsModern, unwrapGoogleScholar, unwrapGoogleTranslate, unwrapHashnode, unwrapIcptrack, unwrapIdealoPartner, unwrapInstagramShim, unwrapJianshuGo, unwrapJuejin, unwrapLeverAnalytics, unwrapLinksynergy, unwrapMailchimp, unwrapMailpanion, unwrapMailpgn, unwrapMailtrack, unwrapMedium, unwrapMimecast, unwrapMozillaOutgoing, unwrapNarrativ, unwrapNicoMs, unwrapOutlookSafelinks, unwrapPartnerAds, unwrapPocket, unwrapPostmark, unwrapProofpointV1, unwrapProofpointV2, unwrapProofpointV3, unwrapPxf, unwrapRecruitics, unwrapRedditOut, unwrapRedirectUrls, unwrapRedirectingat, unwrapSegmentfault, unwrapShareasale, unwrapSjv, unwrapSkimlinks, unwrapSlack, unwrapSmartredirect, unwrapSspai, unwrapSteamLinkfilter, unwrapTelegramIv, unwrapTradedoubler, unwrapTumblr, unwrapValuecommerce, unwrapViglink, unwrapVkAway, unwrapWebArchive, unwrapWrappers, unwrapYahooSearch, unwrapYandexTurbo, unwrapYouTube, unwrapZhihu, youtubeEmbedResolver, youtubeResolveEmbed };

package/dist/index.js

@@ -1,26 +1,32 @@
-import { applyDomTransforms, applyStringTransforms, createEmbedPlaceholder, parseFragment, stripOversizedBase64Sources, transformHtml } from "./common.js";
+import { chooseBaseUrl, coerceNumber, createParamExtractor } from "./utils.js";
+import { applyDomTransforms, applyEmbedMetadata, applyStringTransforms, createEmbedPlaceholder, expandSvgSelfClose, parseFragment, stripOversizedBase64Sources, transformHtml } from "./common.js";
 import { composeThumbnailUrl, extractVideoId, youtubeEmbedResolver, youtubeResolveEmbed } from "./embeds/youtube.js";
+import { convertBreaksToParagraphs } from "./transforms/dom/convertBreaksToParagraphs.js";
+import { decodeDoubleEncodedTags } from "./transforms/dom/decodeDoubleEncodedTags.js";
 import { fixLazyImages } from "./transforms/dom/fixLazyImages.js";
 import { detectLanguage, highlightCode } from "./transforms/dom/highlightCode.js";
-import { injectEnclosureEmbedPlaceholders } from "./transforms/dom/injectEnclosureEmbedPlaceholders.js";
+import { injectEnclosures } from "./transforms/dom/injectEnclosures.js";
 import { linkifyUrls } from "./transforms/dom/linkifyUrls.js";
 import { mergeConsecutiveOneLinerPres } from "./transforms/dom/mergeConsecutiveOneLinerPres.js";
+import { mergeFragmentedLists } from "./transforms/dom/mergeFragmentedLists.js";
+import { proxyAssetUrls } from "./transforms/dom/proxyAssetUrls.js";
 import { removeTrackingPixels } from "./transforms/dom/removeTrackingPixels.js";
-import { chooseBaseUrl, coerceNumber, createParamExtractor } from "./utils.js";
 import { replaceEmbedsWithPlaceholders } from "./transforms/dom/replaceEmbedsWithPlaceholders.js";
 import { replacePreLineBreaks } from "./transforms/dom/replacePreLineBreaks.js";
 import { resolveRelativeUrls } from "./transforms/dom/resolveRelativeUrls.js";
 import { stripComments } from "./transforms/dom/stripComments.js";
+import { stripDeadAnchors } from "./transforms/dom/stripDeadAnchors.js";
+import { stripDuplicateTitleHeading } from "./transforms/dom/stripDuplicateTitleHeading.js";
+import { stripEmptyTags } from "./transforms/dom/stripEmptyTags.js";
 import { stripInterBlockBreaks } from "./transforms/dom/stripInterBlockBreaks.js";
 import { stripParagraphBoundaryBreaks } from "./transforms/dom/stripParagraphBoundaryBreaks.js";
 import { stripTrackingParams } from "./transforms/dom/stripTrackingParams.js";
 import { trimPreWhitespace } from "./transforms/dom/trimPreWhitespace.js";
+import { unwrapDoublyNestedLists } from "./transforms/dom/unwrapDoublyNestedLists.js";
 import { extractRedirectTarget, unwrapRedirectUrls } from "./transforms/dom/unwrapRedirectUrls.js";
-import { decodeDoubleEncodedTags } from "./transforms/string/decodeDoubleEncodedTags.js";
+import { unwrapWrappers } from "./transforms/dom/unwrapWrappers.js";
 import { paragraphizePlainText } from "./transforms/string/paragraphizePlainText.js";
-import { stripEmptyTags } from "./transforms/string/stripEmptyTags.js";
-import { stripOrphanedClosingTags } from "./transforms/string/stripOrphanedClosingTags.js";
-import { unwrapWrappers } from "./transforms/string/unwrapWrappers.js";
+import { unwrapCdataComments } from "./transforms/string/unwrapCdataComments.js";
 import { unwrapBing } from "./unwraps/bing.js";
 import { unwrapFacebookShim } from "./unwraps/facebook.js";
 import { unwrapGoogle } from "./unwraps/google.js";
@@ -33,8 +39,8 @@ import { unwrapRedditOut } from "./unwraps/redditOut.js";
 import { unwrapVkAway } from "./unwraps/vkAway.js";
 import { unwrapYahooSearch } from "./unwraps/yahooSearch.js";
 import { unwrapYouTube } from "./unwraps/youtube.js";
-import { defaultDomTransforms, defaultEmbedResolvers, defaultFinalStringTransforms, defaultLazySrcAttributes, defaultResolveUrlFn, defaultStringTransforms, defaultTrackingHosts, defaultTrackingPathSegments, defaultUrlUnwrappers } from "./defaults.js";
-import { simplifyFigures } from "./transforms/dom/simplifyFigures.js";
+import { defaultDomTransforms, defaultEmbedResolvers, defaultFinalStringTransforms, defaultLazySrcAttributes, defaultLazySrcsetAttributes, defaultResolveUrlFn, defaultStringTransforms, defaultTrackingHosts, defaultTrackingPathSegments, defaultUrlUnwrappers } from "./defaults.js";
+import { enrichEmbedPlaceholders } from "./transforms/dom/enrichEmbedPlaceholders.js";
 import { unwrapAceml } from "./unwraps/aceml.js";
 import { unwrapAdjust } from "./unwraps/adjust.js";
 import { unwrapAmazonAffiliate } from "./unwraps/amazonAffiliate.js";
@@ -98,21 +104,25 @@ import { unwrapWebArchive } from "./unwraps/webArchive.js";
 import { unwrapYandexTurbo } from "./unwraps/yandexTurbo.js";
 import { unwrapZhihu } from "./unwraps/zhihu.js";
 //#region src/index.ts
-const transformContent = (html, options = {}) => {
+const transformContent = async (html, options = {}) => {
 	const context = {
 		baseUrl: options.baseUrl,
 		enclosures: options.enclosures,
 		embedResolvers: options.embedResolvers ?? defaultEmbedResolvers,
 		lazySrcAttributes: options.lazySrcAttributes ?? defaultLazySrcAttributes,
+		lazySrcsetAttributes: options.lazySrcsetAttributes ?? defaultLazySrcsetAttributes,
 		trackingHosts: options.trackingHosts ?? defaultTrackingHosts,
 		trackingPathSegments: options.trackingPathSegments ?? defaultTrackingPathSegments,
 		urlUnwrappers: options.urlUnwrappers ?? defaultUrlUnwrappers,
-		resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn
+		resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn,
+		assetProxyFn: options.assetProxyFn,
+		enrichEmbedFn: options.enrichEmbedFn,
+		articleTitle: options.articleTitle
 	};
 	const stringFns = options.stringTransforms ?? defaultStringTransforms;
 	const domFns = options.domTransforms ?? defaultDomTransforms;
 	const finalFns = options.finalStringTransforms ?? defaultFinalStringTransforms;
-	return applyStringTransforms(applyDomTransforms(applyStringTransforms(html, stringFns.map((transform) => transform(context))), domFns.map((transform) => transform(context))), finalFns.map((transform) => transform(context)));
+	return await applyStringTransforms(await applyDomTransforms(await applyStringTransforms(html, stringFns.map((transform) => transform(context))), domFns.map((transform) => transform(context))), finalFns.map((transform) => transform(context)));
 };
 //#endregion
-export { applyDomTransforms, applyStringTransforms, chooseBaseUrl, coerceNumber, composeThumbnailUrl, createEmbedPlaceholder, createParamExtractor, decodeDoubleEncodedTags, defaultResolveUrlFn, detectLanguage, extractRedirectTarget, extractVideoId, fixLazyImages, highlightCode, injectEnclosureEmbedPlaceholders, linkifyUrls, mergeConsecutiveOneLinerPres, paragraphizePlainText, parseFragment, removeTrackingPixels, replaceEmbedsWithPlaceholders, replacePreLineBreaks, resolveRelativeUrls, simplifyFigures, stripComments, stripEmptyTags, stripInterBlockBreaks, stripOrphanedClosingTags, stripOversizedBase64Sources, stripParagraphBoundaryBreaks, stripTrackingParams, transformContent, transformHtml, trimPreWhitespace, unwrapAceml, unwrapAdjust, unwrapAmazonAffiliate, unwrapAmpCache, unwrapAwin, unwrapBing, unwrapCjNetwork, unwrapDigidip, unwrapDisqus, unwrapDouban, unwrapDuckduckgo, unwrapEbayRover, unwrapEffiliation, unwrapEmbedly, unwrapFacebookShim, unwrapFeedsportal, unwrapFirebaseDynamicLinks, unwrapFlipboard, unwrapGateSc, unwrapGeoriot, unwrapGitee, unwrapGoogle, unwrapGoogleAmpViewer, unwrapGoogleNews, unwrapGoogleNewsModern, unwrapGoogleScholar, unwrapGoogleTranslate, unwrapHashnode, unwrapIcptrack, unwrapIdealoPartner, unwrapInstagramShim, unwrapJianshuGo, unwrapJuejin, unwrapLeverAnalytics, unwrapLinksynergy, unwrapMailchimp, unwrapMailpanion, unwrapMailpgn, unwrapMailtrack, unwrapMedium, unwrapMimecast, unwrapMozillaOutgoing, unwrapNarrativ, unwrapNicoMs, unwrapOutlookSafelinks, unwrapPartnerAds, unwrapPocket, unwrapPostmark, unwrapProofpointV1, unwrapProofpointV2, unwrapProofpointV3, unwrapPxf, unwrapRecruitics, unwrapRedditOut, unwrapRedirectUrls, unwrapRedirectingat, unwrapSegmentfault, unwrapShareasale, unwrapSjv, unwrapSkimlinks, unwrapSlack, unwrapSmartredirect, unwrapSspai, unwrapSteamLinkfilter, unwrapTelegramIv, unwrapTradedoubler, unwrapTumblr, unwrapValuecommerce, unwrapViglink, unwrapVkAway, unwrapWebArchive, unwrapWrappers, unwrapYahooSearch, unwrapYandexTurbo, unwrapYouTube, unwrapZhihu, youtubeEmbedResolver, youtubeResolveEmbed };
+export { applyDomTransforms, applyEmbedMetadata, applyStringTransforms, chooseBaseUrl, coerceNumber, composeThumbnailUrl, convertBreaksToParagraphs, createEmbedPlaceholder, createParamExtractor, decodeDoubleEncodedTags, defaultResolveUrlFn, detectLanguage, enrichEmbedPlaceholders, expandSvgSelfClose, extractRedirectTarget, extractVideoId, fixLazyImages, highlightCode, injectEnclosures, linkifyUrls, mergeConsecutiveOneLinerPres, mergeFragmentedLists, paragraphizePlainText, parseFragment, proxyAssetUrls, removeTrackingPixels, replaceEmbedsWithPlaceholders, replacePreLineBreaks, resolveRelativeUrls, stripComments, stripDeadAnchors, stripDuplicateTitleHeading, stripEmptyTags, stripInterBlockBreaks, stripOversizedBase64Sources, stripParagraphBoundaryBreaks, stripTrackingParams, transformContent, transformHtml, trimPreWhitespace, unwrapAceml, unwrapAdjust, unwrapAmazonAffiliate, unwrapAmpCache, unwrapAwin, unwrapBing, unwrapCdataComments, unwrapCjNetwork, unwrapDigidip, unwrapDisqus, unwrapDouban, unwrapDoublyNestedLists, unwrapDuckduckgo, unwrapEbayRover, unwrapEffiliation, unwrapEmbedly, unwrapFacebookShim, unwrapFeedsportal, unwrapFirebaseDynamicLinks, unwrapFlipboard, unwrapGateSc, unwrapGeoriot, unwrapGitee, unwrapGoogle, unwrapGoogleAmpViewer, unwrapGoogleNews, unwrapGoogleNewsModern, unwrapGoogleScholar, unwrapGoogleTranslate, unwrapHashnode, unwrapIcptrack, unwrapIdealoPartner, unwrapInstagramShim, unwrapJianshuGo, unwrapJuejin, unwrapLeverAnalytics, unwrapLinksynergy, unwrapMailchimp, unwrapMailpanion, unwrapMailpgn, unwrapMailtrack, unwrapMedium, unwrapMimecast, unwrapMozillaOutgoing, unwrapNarrativ, unwrapNicoMs, unwrapOutlookSafelinks, unwrapPartnerAds, unwrapPocket, unwrapPostmark, unwrapProofpointV1, unwrapProofpointV2, unwrapProofpointV3, unwrapPxf, unwrapRecruitics, unwrapRedditOut, unwrapRedirectUrls, unwrapRedirectingat, unwrapSegmentfault, unwrapShareasale, unwrapSjv, unwrapSkimlinks, unwrapSlack, unwrapSmartredirect, unwrapSspai, unwrapSteamLinkfilter, unwrapTelegramIv, unwrapTradedoubler, unwrapTumblr, unwrapValuecommerce, unwrapViglink, unwrapVkAway, unwrapWebArchive, unwrapWrappers, unwrapYahooSearch, unwrapYandexTurbo, unwrapYouTube, unwrapZhihu, youtubeEmbedResolver, youtubeResolveEmbed };

package/dist/transforms/dom/convertBreaksToParagraphs.d.ts

@@ -0,0 +1,6 @@
+import { DomTransform } from "../../types.js";
+
+//#region src/transforms/dom/convertBreaksToParagraphs.d.ts
+declare const convertBreaksToParagraphs: DomTransform;
+//#endregion
+export { convertBreaksToParagraphs };