defuddle 0.14.0 → 0.15.0

package/dist/defuddle.js

@@ -2,34 +2,21 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Defuddle = void 0;
 const metadata_1 = require("./metadata");
-const headings_1 = require("./elements/headings");
 const extractor_registry_1 = require("./extractor-registry");
 const constants_1 = require("./constants");
 const standardize_1 = require("./standardize");
 const footnotes_1 = require("./elements/footnotes");
 const callouts_1 = require("./elements/callouts");
-const scoring_1 = require("./scoring");
+const scoring_1 = require("./removals/scoring");
+const small_images_1 = require("./removals/small-images");
+const hidden_1 = require("./removals/hidden");
+const selectors_1 = require("./removals/selectors");
+const content_patterns_1 = require("./removals/content-patterns");
+const metadata_block_1 = require("./removals/metadata-block");
 const utils_1 = require("./utils");
 const dom_1 = require("./utils/dom");
 /** Keys from extractor variables that map to top-level DefuddleResponse fields */
 const STANDARD_VARIABLE_KEYS = new Set(['title', 'author', 'published', 'site', 'description', 'image', 'language']);
-// Content pattern detection constants
-const STYLE_WIDTH_PATTERN = /width\s*:\s*(\d+)/;
-const STYLE_HEIGHT_PATTERN = /height\s*:\s*(\d+)/;
-const CONTENT_DATE_PATTERN = /(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{1,2}/i;
-const CONTENT_READ_TIME_PATTERN = /\d+\s*min(?:ute)?s?\s+read\b/i;
-const BOILERPLATE_PATTERNS = [
-    /^This (?:article|story|piece) (?:appeared|was published|originally appeared) in\b/i,
-    /^A version of this (?:article|story) (?:appeared|was published) in\b/i,
-    /^Originally (?:published|appeared) (?:in|on|at)\b/i,
-];
-const METADATA_STRIP_PATTERNS = [
-    /\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b/gi,
-    /\b\d+(?:st|nd|rd|th)?\b/g,
-    /\bmin(?:ute)?s?\b/gi,
-    /\bread\b/gi,
-    /[|·•—–\-,.\s]/g,
-];
 class Defuddle {
     /**
      * Create a new Defuddle instance
@@ -39,6 +26,7 @@ class Defuddle {
     constructor(doc, options = {}) {
         this._schemaOrgData = undefined;
         this._schemaOrgExtracted = false;
+        this._inExtractorPipelineRun = false;
         this.doc = doc;
         this.options = options;
         this.debug = options.debug || false;
@@ -121,20 +109,23 @@ class Defuddle {
             }
         }
         // Strip dangerous elements from this.doc before any fallback paths
-        // that read from it (e.g. _findContentBySchemaText).
-        // This must happen after parseInternal, which needs script tags
-        // for schema.org extraction, site-specific extractors, and math.
+        // that read from it. This must happen after parseInternal, which needs
+        // script tags for schema.org extraction, site-specific extractors, and math.
         this._stripUnsafeElements();
-        // If schema.org has a SocialMediaPosting with text content that is
-        // longer than what we extracted, the scorer likely picked the wrong
-        // element from a feed. Find the correct element in the DOM.
+        // If schema.org has text content that is significantly longer than what we
+        // extracted, the scorer likely picked the wrong element from a feed page.
+        // Use a 1.5x threshold to avoid triggering when the difference is small
+        // (e.g. just related-content link text removed).
         const schemaText = this._getSchemaText(result.schemaOrgData);
-        if (schemaText && this.countHtmlWords(schemaText) > result.wordCount) {
-            const contentHtml = this._findContentBySchemaText(schemaText);
-            if (contentHtml) {
-                this._log('Found DOM content matching schema.org text');
-                result.content = contentHtml;
-                result.wordCount = this.countHtmlWords(contentHtml);
+        if (schemaText && this.countHtmlWords(schemaText) > result.wordCount * 1.5) {
+            const bestMatch = this._findElementBySchemaText(this.doc.body, schemaText);
+            if (bestMatch) {
+                // Re-run the full pipeline with the schema-identified element as the
+                // content root so it benefits from the same cleanup as normal extraction.
+                const selector = this.getElementSelector(bestMatch);
+                this._log('Schema.org suggests a better content element, retrying with selector:', selector);
+                const schemaRetry = this.parseInternal({ contentSelector: selector });
+                result = schemaRetry;
             }
             else {
                 this._log('Using schema.org text as content (DOM element not found)');
@@ -238,62 +229,6 @@ class Defuddle {
         }
         return bestMatch;
     }
-    /**
-     * Find a DOM element whose text matches the schema.org text content.
-     * Used when the content scorer picked the wrong element from a feed page.
-     * Returns the element's inner HTML including sibling media (images, etc.)
-     */
-    _findContentBySchemaText(schemaText) {
-        const body = this.doc.body;
-        if (!body)
-            return '';
-        const bestMatch = this._findElementBySchemaText(body, schemaText);
-        if (!bestMatch)
-            return '';
-        // Read the largest sibling image src BEFORE resolveRelativeUrls
-        // can mangle comma-containing CDN URLs in srcset attributes
-        let imageSrc = '';
-        let imageAlt = '';
-        const parent = bestMatch.parentElement;
-        if (parent && parent !== body) {
-            const images = parent.querySelectorAll('img');
-            let largestImg = null;
-            let largestArea = 0;
-            for (const img of images) {
-                if (bestMatch.contains(img))
-                    continue;
-                const w = parseInt(img.getAttribute('width') || '0', 10);
-                const h = parseInt(img.getAttribute('height') || '0', 10);
-                const area = w * h;
-                if (area > largestArea) {
-                    largestArea = area;
-                    largestImg = img;
-                }
-            }
-            if (largestImg) {
-                imageSrc = this._getLargestImageSrc(largestImg);
-                imageAlt = largestImg.getAttribute('alt') || '';
-                try {
-                    const baseUrl = this.options.url || this.doc.URL;
-                    if (baseUrl)
-                        imageSrc = new URL(imageSrc, baseUrl).href;
-                }
-                catch { }
-            }
-        }
-        // Remove heading anchor links before serialization (e.g. <h2>Title<a href="#foo">#</a></h2>)
-        (0, headings_1.removeHeadingAnchors)(bestMatch);
-        // Now resolve URLs in the text content
-        this.resolveRelativeUrls(bestMatch);
-        let html = (0, dom_1.serializeHTML)(bestMatch);
-        if (imageSrc) {
-            const img = this.doc.createElement('img');
-            img.setAttribute('src', imageSrc);
-            img.setAttribute('alt', imageAlt);
-            html += img.outerHTML;
-        }
-        return html;
-    }
     findLargestHiddenContentSelector() {
         const body = this.doc.body;
         if (!body)
@@ -412,6 +347,16 @@ class Defuddle {
      */
     parseInternal(overrideOptions = {}) {
         const startTime = Date.now();
+        const profile = {};
+        const doProfile = this.options.profile ?? false;
+        const profileStep = (name, fn) => {
+            if (!doProfile)
+                return fn();
+            const t = performance.now();
+            const result = fn();
+            profile[name] = Math.round(performance.now() - t);
+            return result;
+        };
         // Guard against empty/broken documents (e.g. empty HTML, bot-blocked pages)
         if (!this.doc.documentElement) {
             const url = this.options.url || '';
@@ -465,10 +410,37 @@ class Defuddle {
                 includeReplies: options.includeReplies,
                 language: options.language,
             };
-            const extractor = extractor_registry_1.ExtractorRegistry.findExtractor(this.doc, url, schemaOrgData, extractorOpts);
-            if (extractor && extractor.canExtract()) {
-                const extracted = extractor.extract();
-                return this.buildExtractorResponse(extracted, metadata, startTime, extractor, pageMetaTags);
+            if (!this._inExtractorPipelineRun) {
+                const extractor = extractor_registry_1.ExtractorRegistry.findExtractor(this.doc, url, schemaOrgData, extractorOpts);
+                if (extractor && extractor.canExtract()) {
+                    const extracted = extractor.extract();
+                    if (extracted.contentSelector) {
+                        this._inExtractorPipelineRun = true;
+                        try {
+                            const pipelineResult = this.parseInternal({
+                                contentSelector: extracted.contentSelector,
+                                removeLowScoring: false,
+                                removeHiddenElements: false,
+                            });
+                            const variables = this.getExtractorVariables(extracted.variables);
+                            return {
+                                ...pipelineResult,
+                                title: extracted.variables?.title || pipelineResult.title,
+                                description: extracted.variables?.description || pipelineResult.description,
+                                author: extracted.variables?.author || pipelineResult.author,
+                                published: extracted.variables?.published || pipelineResult.published,
+                                site: extracted.variables?.site || pipelineResult.site,
+                                language: extracted.variables?.language || pipelineResult.language,
+                                extractorType: extractor.constructor.name.replace('Extractor', '').toLowerCase(),
+                                ...(variables ? { variables } : {}),
+                            };
+                        }
+                        finally {
+                            this._inExtractorPipelineRun = false;
+                        }
+                    }
+                    return this.buildExtractorResponse(extracted, metadata, startTime, extractor, pageMetaTags);
+                }
             }
             // Continue if there is no extractor...
             // Evaluate mobile styles and sizes on original document (cached across retries)
@@ -478,41 +450,47 @@ class Defuddle {
             const mobileStyles = this._mobileStyles;
             // Find small images in original document (cached across retries)
             if (!this._smallImages) {
-                this._smallImages = this.findSmallImages(this.doc);
+                this._smallImages = (0, small_images_1.findSmallImages)(this.doc, this.debug);
             }
             const smallImages = this._smallImages;
             // Clone document
-            const clone = this.doc.cloneNode(true);
-            // Merge adjacent text nodes that some DOM implementations (e.g. linkedom)
-            // create when parsing HTML entities like &#39;
-            clone.body?.normalize();
+            let clone;
+            profileStep('cloneDocument', () => {
+                clone = this.doc.cloneNode(true);
+                // Merge adjacent text nodes that some DOM implementations (e.g. linkedom)
+                // create when parsing HTML entities like &#39;
+                clone.body?.normalize();
+            });
             // Flatten shadow DOM content into the clone
-            this.flattenShadowRoots(this.doc, clone);
+            profileStep('flattenShadowRoots', () => this.flattenShadowRoots(this.doc, clone));
             // Resolve React streaming SSR suspense boundaries
-            this.resolveStreamedContent(clone);
+            profileStep('resolveStreamedContent', () => this.resolveStreamedContent(clone));
             // Apply mobile styles to clone
-            this.applyMobileStyles(clone, mobileStyles);
+            profileStep('applyMobileStyles', () => this.applyMobileStyles(clone, mobileStyles));
             // Find main content
-            let mainContent = null;
-            if (options.contentSelector) {
-                mainContent = clone.querySelector(options.contentSelector);
-                this._log('Using contentSelector:', options.contentSelector, mainContent ? 'found' : 'not found');
-            }
-            if (!mainContent) {
-                mainContent = this.findMainContent(clone);
-            }
-            // If we fell back to <body>, try using schema.org articleBody/text
-            // to find a more specific content element within the DOM.
-            if (mainContent && mainContent.tagName.toLowerCase() === 'body') {
-                const schemaText = this._getSchemaText(schemaOrgData);
-                if (schemaText) {
-                    const schemaContent = this._findElementBySchemaText(clone.body, schemaText);
-                    if (schemaContent) {
-                        this._log('Found content element via schema.org text');
-                        mainContent = schemaContent;
+            const mainContent = profileStep('findMainContent', () => {
+                let found = null;
+                if (options.contentSelector) {
+                    found = clone.querySelector(options.contentSelector);
+                    this._log('Using contentSelector:', options.contentSelector, found ? 'found' : 'not found');
+                }
+                if (!found) {
+                    found = this.findMainContent(clone);
+                }
+                // If we fell back to <body>, try using schema.org articleBody/text
+                // to find a more specific content element within the DOM.
+                if (found && found.tagName.toLowerCase() === 'body') {
+                    const schemaText = this._getSchemaText(schemaOrgData);
+                    if (schemaText) {
+                        const schemaContent = this._findElementBySchemaText(clone.body, schemaText);
+                        if (schemaContent) {
+                            this._log('Found content element via schema.org text');
+                            found = schemaContent;
+                        }
                     }
                 }
-            }
+                return found;
+            });
             if (!mainContent) {
                 const fallbackContent = this.doc.body ? this.resolveContentUrls((0, dom_1.serializeHTML)(this.doc.body)) : '';
                 const endTime = Date.now();
@@ -524,43 +502,66 @@ class Defuddle {
                     metaTags: pageMetaTags
                 };
             }
-            // Remove <wbr> elements — word break opportunity hints that carry no
-            // content but cause unwanted whitespace during standardization.
-            mainContent.querySelectorAll('wbr').forEach(el => el.remove());
+            // Remove h1-adjacent date/author metadata blocks from the content.
+            // These are extracted as frontmatter but also appear in the body when a
+            // wide container (e.g. <main>) is selected as the content element.
+            profileStep('removeMetadataBlock', () => {
+                if (metadata.published || metadata.author) {
+                    (0, metadata_block_1.removeMetadataBlock)(mainContent);
+                }
+                // Remove <wbr> elements — word break opportunity hints that carry no
+                // content but cause unwanted whitespace during standardization.
+                mainContent.querySelectorAll('wbr').forEach(el => el.remove());
+            });
             // Standardize footnotes before cleanup (CSS sidenotes use display:none)
-            if (options.standardize) {
-                (0, footnotes_1.standardizeFootnotes)(mainContent);
-                (0, callouts_1.standardizeCallouts)(mainContent);
-            }
+            profileStep('standardizeFootnotesCallouts', () => {
+                if (options.standardize) {
+                    (0, footnotes_1.standardizeFootnotes)(mainContent);
+                    (0, callouts_1.standardizeCallouts)(mainContent);
+                }
+            });
             // Remove small images
-            if (options.removeSmallImages) {
-                this.removeSmallImages(clone, smallImages);
-            }
+            profileStep('removeSmallImages', () => {
+                if (options.removeSmallImages) {
+                    (0, small_images_1.removeSmallImages)(clone, smallImages, this.debug);
+                }
+            });
             // Remove hidden elements using computed styles
-            if (options.removeHiddenElements) {
-                this.removeHiddenElements(clone, debugRemovals);
-            }
+            profileStep('removeHiddenElements', () => {
+                if (options.removeHiddenElements) {
+                    (0, hidden_1.removeHiddenElements)(clone, this.debug, debugRemovals);
+                }
+            });
             // Remove clutter using selectors — deterministic removal of known
             // non-content elements (nav, footer, .sidebar, etc.) by class/id.
             // Runs before scoring so the heuristic scorer sees a cleaner DOM.
-            if (options.removeExactSelectors || options.removePartialSelectors) {
-                this.removeBySelector(clone, options.removeExactSelectors, options.removePartialSelectors, mainContent, debugRemovals, options.removeHiddenElements === false);
-            }
+            profileStep('removeBySelector', () => {
+                if (options.removeExactSelectors || options.removePartialSelectors) {
+                    (0, selectors_1.removeBySelector)(clone, this.debug, options.removeExactSelectors, options.removePartialSelectors, mainContent, debugRemovals, options.removeHiddenElements === false);
+                }
+            });
             // Remove non-content blocks by scoring — heuristic removal based
             // on link density, text ratios, and navigation indicators.
-            if (options.removeLowScoring) {
-                scoring_1.ContentScorer.scoreAndRemove(clone, this.debug, debugRemovals, mainContent);
-            }
+            profileStep('removeLowScoring', () => {
+                if (options.removeLowScoring) {
+                    scoring_1.ContentScorer.scoreAndRemove(clone, this.debug, debugRemovals, mainContent);
+                }
+            });
             // Remove elements by content patterns (read time, boilerplate, article cards)
-            if (options.removeContentPatterns && mainContent) {
-                this.removeByContentPattern(mainContent, this.debug ? debugRemovals : undefined);
-            }
+            profileStep('removeByContentPattern', () => {
+                if (options.removeContentPatterns && mainContent) {
+                    const url = this.options.url || this.doc.URL || '';
+                    (0, content_patterns_1.removeByContentPattern)(mainContent, this.debug, url, debugRemovals);
+                }
+            });
             // Normalize the main content
-            if (options.standardize) {
-                (0, standardize_1.standardizeContent)(mainContent, metadata, this.doc, this.debug);
-            }
+            profileStep('standardizeContent', () => {
+                if (options.standardize) {
+                    (0, standardize_1.standardizeContent)(mainContent, metadata, this.doc, this.debug, doProfile ? profile : undefined);
+                }
+            });
             // Resolve relative URLs to absolute
-            this.resolveRelativeUrls(mainContent);
+            profileStep('resolveRelativeUrls', () => this.resolveRelativeUrls(mainContent));
             const content = mainContent.outerHTML;
             const endTime = Date.now();
             const result = {
@@ -576,6 +577,9 @@ class Defuddle {
                     removals: debugRemovals
                 };
             }
+            if (this.options.profile) {
+                result.profile = profile;
+            }
             return result;
         }
         catch (error) {
@@ -700,282 +704,6 @@ class Defuddle {
             image.remove();
         });
     }
-    removeHiddenElements(doc, debugRemovals) {
-        let count = 0;
-        const elementsToRemove = new Map();
-        // Check inline styles and CSS class-based hidden patterns.
-        const hiddenStylePattern = /(?:^|;\s*)(?:display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0)(?:\s*;|\s*$)/i;
-        // Only use getComputedStyle in browser environments where it's meaningful.
-        // In JSDOM/linkedom without stylesheets, it's extremely slow and unreliable.
-        const defaultView = doc.defaultView;
-        const isBrowser = typeof window !== 'undefined' && defaultView === window;
-        const allElements = doc.querySelectorAll('*');
-        for (const element of allElements) {
-            // Skip elements that contain math — sites like Wikipedia wrap MathML
-            // in display:none spans for accessibility (the visible version is an
-            // image/SVG fallback). We need to preserve these for math extraction.
-            if (element.querySelector('math, [data-mathml], .katex-mathml') ||
-                element.tagName.toLowerCase() === 'math') {
-                continue;
-            }
-            // Check inline style for hidden patterns
-            const style = element.getAttribute('style');
-            if (style && hiddenStylePattern.test(style)) {
-                const reason = style.includes('display') ? 'display:none' :
-                    style.includes('visibility') ? 'visibility:hidden' : 'opacity:0';
-                elementsToRemove.set(element, reason);
-                count++;
-                continue;
-            }
-            // Use getComputedStyle only in real browser environments
-            if (isBrowser) {
-                try {
-                    const computedStyle = defaultView.getComputedStyle(element);
-                    let reason = '';
-                    if (computedStyle.display === 'none')
-                        reason = 'display:none';
-                    else if (computedStyle.visibility === 'hidden')
-                        reason = 'visibility:hidden';
-                    else if (computedStyle.opacity === '0')
-                        reason = 'opacity:0';
-                    if (reason) {
-                        elementsToRemove.set(element, reason);
-                        count++;
-                        continue;
-                    }
-                }
-                catch (e) { }
-            }
-            // Detect CSS framework hidden utilities (e.g. Tailwind's "hidden",
-            // "sm:hidden", "not-machine:hidden")
-            const className = element.getAttribute('class') || '';
-            if (className) {
-                const tokens = className.split(/\s+/);
-                for (const token of tokens) {
-                    if (token === 'hidden' || token.endsWith(':hidden') || token === 'invisible' || token.endsWith(':invisible')) {
-                        elementsToRemove.set(element, `class:${token}`);
-                        count++;
-                        break;
-                    }
-                }
-            }
-        }
-        // Batch remove all hidden elements
-        elementsToRemove.forEach((reason, el) => {
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeHiddenElements',
-                    reason,
-                    text: (0, utils_1.textPreview)(el)
-                });
-            }
-            el.remove();
-        });
-        this._log('Removed hidden elements:', count);
-    }
-    removeBySelector(doc, removeExact = true, removePartial = true, mainContent, debugRemovals, skipHiddenExactSelectors = false) {
-        const startTime = Date.now();
-        let exactSelectorCount = 0;
-        let partialSelectorCount = 0;
-        // Track all elements to be removed, with their match type
-        const elementsToRemove = new Map();
-        // First collect elements matching exact selectors
-        if (removeExact) {
-            const exactElements = doc.querySelectorAll(constants_1.EXACT_SELECTORS_JOINED);
-            exactElements.forEach(el => {
-                if (el?.parentNode) {
-                    if (skipHiddenExactSelectors) {
-                        const hiddenAncestor = el.closest(constants_1.HIDDEN_EXACT_SKIP_SELECTOR);
-                        const role = (el.getAttribute('role') || '').toLowerCase();
-                        if (el.matches(constants_1.HIDDEN_EXACT_SELECTOR) ||
-                            (hiddenAncestor && role === 'dialog')) {
-                            return;
-                        }
-                    }
-                    // Skip elements inside code blocks (e.g. syntax highlighting spans)
-                    if (el.closest('pre, code')) {
-                        return;
-                    }
-                    elementsToRemove.set(el, { type: 'exact' });
-                    exactSelectorCount++;
-                }
-            });
-        }
-        if (removePartial) {
-            // Pre-compile individual regexes for debug pattern identification only
-            const individualRegexes = this.debug
-                ? constants_1.PARTIAL_SELECTORS.map(p => ({ pattern: p, regex: new RegExp(p, 'i') }))
-                : null;
-            // Use pre-built attribute selector for elements we care about
-            const allElements = doc.querySelectorAll(constants_1.TEST_ATTRIBUTES_SELECTOR);
-            // Process elements for partial matches
-            allElements.forEach(el => {
-                // Skip if already marked for removal
-                if (elementsToRemove.has(el)) {
-                    return;
-                }
-                // Skip code elements and elements containing code blocks
-                // where class names indicate language/syntax, not page structure
-                const tag = el.tagName;
-                if (tag === 'CODE' || tag === 'PRE' || el.querySelector('pre') || el.closest('code, pre')) {
-                    return;
-                }
-                // Get all relevant attributes and combine into a single string
-                const attrs = constants_1.TEST_ATTRIBUTES.map(attr => {
-                    if (attr === 'class') {
-                        return (0, dom_1.getClassName)(el);
-                    }
-                    if (attr === 'id') {
-                        return el.id || '';
-                    }
-                    return el.getAttribute(attr) || '';
-                }).join(' ').toLowerCase();
-                // Skip if no attributes to check
-                if (!attrs.trim()) {
-                    return;
-                }
-                // Check for partial match using single regex test
-                if (constants_1.PARTIAL_SELECTORS_REGEX.test(attrs)) {
-                    const matchedPattern = individualRegexes
-                        ? individualRegexes.find(r => r.regex.test(attrs))?.pattern
-                        : undefined;
-                    elementsToRemove.set(el, { type: 'partial', selector: matchedPattern });
-                    partialSelectorCount++;
-                }
-            });
-        }
-        // Remove all collected elements in a single pass
-        // Skip elements that are ancestors of mainContent to avoid disconnecting it
-        // Skip footnote list containers, their parents, and immediate children
-        // Skip anchor links inside headings - the heading transform handles these
-        elementsToRemove.forEach(({ type, selector }, el) => {
-            if (mainContent && el.contains(mainContent)) {
-                return;
-            }
-            if (el.tagName === 'A' && el.closest('h1, h2, h3, h4, h5, h6')) {
-                return;
-            }
-            try {
-                if (el.matches(constants_1.FOOTNOTE_LIST_SELECTORS) || el.querySelector(constants_1.FOOTNOTE_LIST_SELECTORS)) {
-                    return;
-                }
-                // Protect immediate children of footnote containers (e.g. wikidot div.footnote-footer)
-                const parent = el.parentElement;
-                if (parent && parent.matches(constants_1.FOOTNOTE_LIST_SELECTORS)) {
-                    return;
-                }
-            }
-            catch (e) { }
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeBySelector',
-                    selector: type === 'exact' ? 'exact' : selector,
-                    reason: type === 'exact' ? 'exact selector match' : `partial match: ${selector}`,
-                    text: (0, utils_1.textPreview)(el)
-                });
-            }
-            el.remove();
-        });
-        const endTime = Date.now();
-        this._log('Removed clutter elements:', {
-            exactSelectors: exactSelectorCount,
-            partialSelectors: partialSelectorCount,
-            total: elementsToRemove.size,
-            processingTime: `${(endTime - startTime).toFixed(2)}ms`
-        });
-    }
-    // Find small IMG and SVG elements
-    findSmallImages(doc) {
-        const MIN_DIMENSION = 33;
-        const smallImages = new Set();
-        let processedCount = 0;
-        const elements = doc.querySelectorAll('img, svg');
-        const defaultView = doc.defaultView;
-        const isBrowser = typeof window !== 'undefined' && defaultView === window;
-        for (const element of elements) {
-            const attrWidth = parseInt(element.getAttribute('width') || '0');
-            const attrHeight = parseInt(element.getAttribute('height') || '0');
-            // Check inline style dimensions
-            const style = element.getAttribute('style') || '';
-            const styleWidth = parseInt(style.match(STYLE_WIDTH_PATTERN)?.[1] || '0');
-            const styleHeight = parseInt(style.match(STYLE_HEIGHT_PATTERN)?.[1] || '0');
-            // Use getComputedStyle and getBoundingClientRect only in browser
-            let computedWidth = 0, computedHeight = 0;
-            if (isBrowser) {
-                try {
-                    const cs = defaultView.getComputedStyle(element);
-                    computedWidth = parseInt(cs.width) || 0;
-                    computedHeight = parseInt(cs.height) || 0;
-                }
-                catch (e) { }
-                try {
-                    const rect = element.getBoundingClientRect();
-                    if (rect.width > 0)
-                        computedWidth = computedWidth || rect.width;
-                    if (rect.height > 0)
-                        computedHeight = computedHeight || rect.height;
-                }
-                catch (e) { }
-            }
-            const widths = [attrWidth, styleWidth, computedWidth].filter(d => d > 0);
-            const heights = [attrHeight, styleHeight, computedHeight].filter(d => d > 0);
-            if (widths.length > 0 && heights.length > 0) {
-                const effectiveWidth = Math.min(...widths);
-                const effectiveHeight = Math.min(...heights);
-                if (effectiveWidth < MIN_DIMENSION || effectiveHeight < MIN_DIMENSION) {
-                    const identifier = this.getElementIdentifier(element);
-                    if (identifier) {
-                        smallImages.add(identifier);
-                        processedCount++;
-                    }
-                }
-            }
-        }
-        this._log('Found small elements:', processedCount);
-        return smallImages;
-    }
-    removeSmallImages(doc, smallImages) {
-        let removedCount = 0;
-        ['img', 'svg'].forEach(tag => {
-            const elements = doc.getElementsByTagName(tag);
-            Array.from(elements).forEach(element => {
-                const identifier = this.getElementIdentifier(element);
-                if (identifier && smallImages.has(identifier)) {
-                    element.remove();
-                    removedCount++;
-                }
-            });
-        });
-        this._log('Removed small elements:', removedCount);
-    }
-    getElementIdentifier(element) {
-        // Try to create a unique identifier using various attributes
-        if (element.tagName.toLowerCase() === 'img') {
-            // For lazy-loaded images, use data-src as identifier if available
-            const dataSrc = element.getAttribute('data-src');
-            if (dataSrc)
-                return `src:${dataSrc}`;
-            const src = element.getAttribute('src') || '';
-            const srcset = element.getAttribute('srcset') || '';
-            const dataSrcset = element.getAttribute('data-srcset');
-            if (src)
-                return `src:${src}`;
-            if (srcset)
-                return `srcset:${srcset}`;
-            if (dataSrcset)
-                return `srcset:${dataSrcset}`;
-        }
-        const id = element.id || '';
-        const className = (0, dom_1.getClassName)(element);
-        const viewBox = element.tagName.toLowerCase() === 'svg' ? element.getAttribute('viewBox') || '' : '';
-        if (id)
-            return `id:${id}`;
-        if (viewBox)
-            return `viewBox:${viewBox}`;
-        if (className)
-            return `class:${className}`;
-        return null;
-    }
     findMainContent(doc) {
         // Find all potential content containers
         const candidates = [];
@@ -1121,6 +849,9 @@ class Defuddle {
                 .trim()
                 .replace(/^\\?["']+/, '')
                 .replace(/\\?["']+$/, '');
+            // Fragment-only hrefs reference anchors within the same document — keep them relative.
+            if (normalized.startsWith('#'))
+                return normalized;
             try {
                 return new URL(normalized, baseUrl).href;
             }
@@ -1391,7 +1122,7 @@ class Defuddle {
         return {
             content: contentHtml,
             title: extracted.variables?.title || metadata.title,
-            description: metadata.description,
+            description: extracted.variables?.description || metadata.description,
             domain: metadata.domain,
             favicon: metadata.favicon,
             image: metadata.image,
@@ -1424,393 +1155,6 @@ class Defuddle {
         }
         return hasCustom ? custom : undefined;
     }
-    /**
-     * Content-based pattern removal for elements that can't be detected by
-     * CSS selectors (e.g. Tailwind/CSS-in-JS sites with non-semantic class names).
-     */
-    removeByContentPattern(mainContent, debugRemovals) {
-        // Remove read time metadata (e.g. "Mar 4th 2026 | 3 min read")
-        // Only removes leaf elements whose text is PURELY date + read time,
-        // not mixed with other meaningful content like tag names.
-        const candidates = Array.from(mainContent.querySelectorAll('p, span, div, time'));
-        for (const el of candidates) {
-            if (!el.parentNode)
-                continue;
-            if (el.closest('pre') || el.closest('code'))
-                continue;
-            const text = el.textContent?.trim() || '';
-            const words = (0, utils_1.countWords)(text);
-            // Match date + read time in short elements
-            if (words <= 15 && CONTENT_DATE_PATTERN.test(text) && CONTENT_READ_TIME_PATTERN.test(text)) {
-                // Ensure this is a leaf-ish element, not a large container
-                if (el.querySelectorAll('p, div, section, article').length === 0) {
-                    // Verify the text is ONLY date + read time metadata
-                    // by stripping all date/time words and checking nothing remains
-                    let cleaned = text;
-                    for (const pattern of METADATA_STRIP_PATTERNS) {
-                        cleaned = cleaned.replace(pattern, '');
-                    }
-                    if (cleaned.trim().length > 0)
-                        continue;
-                    if (this.debug && debugRemovals) {
-                        debugRemovals.push({
-                            step: 'removeByContentPattern',
-                            reason: 'read time metadata',
-                            text: (0, utils_1.textPreview)(el)
-                        });
-                    }
-                    el.remove();
-                }
-            }
-        }
-        // Remove standalone time/date elements near the start or end of content.
-        // A <time> in its own paragraph at the boundary is metadata (publish date),
-        // but <time> inline within prose should be preserved (see issue #136).
-        const timeElements = Array.from(mainContent.querySelectorAll('time'));
-        const contentText = mainContent.textContent || '';
-        for (const time of timeElements) {
-            if (!time.parentNode)
-                continue;
-            // Walk up through inline/formatting wrappers only (i, em, span, b, strong)
-            // Stop at block elements to avoid removing containers with other content.
-            let target = time;
-            let targetText = target.textContent?.trim() || '';
-            while (target.parentElement && target.parentElement !== mainContent) {
-                const parentTag = target.parentElement.tagName.toLowerCase();
-                const parentText = target.parentElement.textContent?.trim() || '';
-                // If parent is a <p> that only wraps this time, include it
-                if (parentTag === 'p' && parentText === targetText) {
-                    target = target.parentElement;
-                    break;
-                }
-                // Only walk through inline formatting wrappers
-                if (['i', 'em', 'span', 'b', 'strong', 'small'].includes(parentTag) &&
-                    parentText === targetText) {
-                    target = target.parentElement;
-                    targetText = parentText;
-                    continue;
-                }
-                break;
-            }
-            const text = target.textContent?.trim() || '';
-            const words = (0, utils_1.countWords)(text);
-            if (words > 10)
-                continue;
-            // Check if this element is near the start or end of mainContent
-            const pos = contentText.indexOf(text);
-            const distFromEnd = contentText.length - (pos + text.length);
-            if (pos > 200 && distFromEnd > 200)
-                continue;
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeByContentPattern',
-                    reason: 'boundary date element',
-                    text: (0, utils_1.textPreview)(target)
-                });
-            }
-            target.remove();
-        }
-        // Remove blog post metadata lists near content boundaries.
-        // These are short <ul>/<ol> elements where every item is a brief
-        // label + value pair (date, reading time, share, etc.) with no
-        // prose sentences. Detected structurally: all items are very short,
-        // none contain sentence-ending punctuation, and the total text is minimal.
-        const metadataLists = mainContent.querySelectorAll('ul, ol');
-        for (const list of metadataLists) {
-            if (!list.parentNode)
-                continue;
-            const items = Array.from(list.children).filter(el => el.tagName === 'LI');
-            if (items.length < 2 || items.length > 8)
-                continue;
-            // Must be near the start or end of content
-            const listText = list.textContent?.trim() || '';
-            const listPos = contentText.indexOf(listText);
-            const distFromEnd = contentText.length - (listPos + listText.length);
-            if (listPos > 500 && distFromEnd > 500)
-                continue;
-            // Skip lists introduced by a preceding paragraph (e.g. "Features include:")
-            // — those are content lists, not standalone metadata
-            const prevSibling = list.previousElementSibling;
-            if (prevSibling) {
-                const prevText = prevSibling.textContent?.trim() || '';
-                if (prevText.endsWith(':'))
-                    continue;
-            }
-            // Every item must be very short (label + value) with no prose
-            let isMetadata = true;
-            for (const item of items) {
-                const text = item.textContent?.trim() || '';
-                const words = (0, utils_1.countWords)(text);
-                if (words > 8) {
-                    isMetadata = false;
-                    break;
-                }
-                // Prose has sentence-ending punctuation; metadata doesn't
-                if (/[.!?]$/.test(text)) {
-                    isMetadata = false;
-                    break;
-                }
-            }
-            if (!isMetadata)
-                continue;
-            // Total text should be very short — this is metadata, not content
-            if ((0, utils_1.countWords)(listText) > 30)
-                continue;
-            // Walk up to find the container to remove (e.g. a wrapper div)
-            let target = list;
-            while (target.parentElement && target.parentElement !== mainContent) {
-                const parentText = target.parentElement.textContent?.trim() || '';
-                if (parentText !== listText)
-                    break;
-                target = target.parentElement;
-            }
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeByContentPattern',
-                    reason: 'blog metadata list',
-                    text: (0, utils_1.textPreview)(target)
-                });
-            }
-            target.remove();
-        }
-        // Remove section breadcrumbs
-        // Short elements containing a link to a parent section of the current URL.
-        const url = this.options.url || this.doc.URL || '';
-        let urlPath = '';
-        let pageHost = '';
-        try {
-            const parsedUrl = new URL(url);
-            urlPath = parsedUrl.pathname;
-            pageHost = parsedUrl.hostname.replace(/^www\./, '');
-        }
-        catch { }
-        if (urlPath) {
-            const shortElements = mainContent.querySelectorAll('div, span, p');
-            for (const el of shortElements) {
-                if (!el.parentNode)
-                    continue;
-                const text = el.textContent?.trim() || '';
-                const words = (0, utils_1.countWords)(text);
-                if (words > 10)
-                    continue;
-                // Must be a leaf-ish element (no block children)
-                if (el.querySelectorAll('p, div, section, article').length > 0)
-                    continue;
-                const link = el.querySelector('a[href]');
-                if (!link)
-                    continue;
-                try {
-                    const linkPath = new URL(link.getAttribute('href') || '', url).pathname;
-                    if (linkPath !== '/' && linkPath !== urlPath && urlPath.startsWith(linkPath)) {
-                        if (this.debug && debugRemovals) {
-                            debugRemovals.push({
-                                step: 'removeByContentPattern',
-                                reason: 'section breadcrumb',
-                                text: (0, utils_1.textPreview)(el)
-                            });
-                        }
-                        el.remove();
-                    }
-                }
-                catch { }
-            }
-        }
-        // Remove trailing external link lists — a heading + list of purely
-        // off-site links as the last content block (affiliate picks, product
-        // roundups, etc.). Only removed when nothing meaningful follows.
-        if (pageHost) {
-            const headings = mainContent.querySelectorAll('h2, h3, h4, h5, h6');
-            for (const heading of headings) {
-                if (!heading.parentNode)
-                    continue;
-                const list = heading.nextElementSibling;
-                if (!list || (list.tagName !== 'UL' && list.tagName !== 'OL'))
-                    continue;
-                const items = Array.from(list.children).filter(el => el.tagName === 'LI');
-                if (items.length < 2)
-                    continue;
-                // The list must be the last meaningful block — nothing after it
-                // except whitespace or empty elements. Walk up through ancestors
-                // to check siblings at each level up to mainContent.
-                let trailingContent = false;
-                let checkEl = list;
-                while (checkEl && checkEl !== mainContent) {
-                    let sibling = checkEl.nextElementSibling;
-                    while (sibling) {
-                        if ((sibling.textContent?.trim() || '').length > 0) {
-                            trailingContent = true;
-                            break;
-                        }
-                        sibling = sibling.nextElementSibling;
-                    }
-                    if (trailingContent)
-                        break;
-                    checkEl = checkEl.parentElement;
-                }
-                if (trailingContent)
-                    continue;
-                // Every list item must be primarily a link pointing off-site
-                let allExternalLinks = true;
-                for (const item of items) {
-                    const links = item.querySelectorAll('a[href]');
-                    if (links.length === 0) {
-                        allExternalLinks = false;
-                        break;
-                    }
-                    const itemText = item.textContent?.trim() || '';
-                    let linkTextLen = 0;
-                    for (const link of links) {
-                        linkTextLen += (link.textContent?.trim() || '').length;
-                        try {
-                            const linkHost = new URL(link.getAttribute('href') || '', url).hostname.replace(/^www\./, '');
-                            if (linkHost === pageHost) {
-                                allExternalLinks = false;
-                                break;
-                            }
-                        }
-                        catch { }
-                    }
-                    if (!allExternalLinks)
-                        break;
-                    if (linkTextLen < itemText.length * 0.6) {
-                        allExternalLinks = false;
-                        break;
-                    }
-                }
-                if (!allExternalLinks)
-                    continue;
-                if (this.debug && debugRemovals) {
-                    debugRemovals.push({
-                        step: 'removeByContentPattern',
-                        reason: 'trailing external link list',
-                        text: (0, utils_1.textPreview)(heading)
-                    });
-                    debugRemovals.push({
-                        step: 'removeByContentPattern',
-                        reason: 'trailing external link list',
-                        text: (0, utils_1.textPreview)(list)
-                    });
-                }
-                list.remove();
-                heading.remove();
-            }
-        }
-        // Remove trailing thin sections — the last few direct children of
-        // mainContent that contain a heading but very little prose. These are
-        // typically CTAs, newsletter prompts, or promotional sections that
-        // have been partially stripped by prior removal steps.
-        const totalWords = (0, utils_1.countWords)(mainContent.textContent || '');
-        if (totalWords > 300) {
-            // Walk backwards from the last direct child of mainContent,
-            // collecting trailing elements that are thin (empty or very short prose).
-            // Exclude SVG text (path data) from word counts — it's not prose.
-            const trailingEls = [];
-            let trailingWords = 0;
-            let child = mainContent.lastElementChild;
-            while (child) {
-                // Count prose words, excluding SVG path data which inflates word counts
-                let svgWords = 0;
-                for (const svg of child.querySelectorAll('svg')) {
-                    svgWords += (0, utils_1.countWords)(svg.textContent || '');
-                }
-                const words = (0, utils_1.countWords)(child.textContent?.trim() || '') - svgWords;
-                if (words > 25)
-                    break;
-                trailingWords += words;
-                trailingEls.push(child);
-                child = child.previousElementSibling;
-            }
-            // Must have a heading in the trailing elements and total < 15% of content.
-            // Skip if trailing elements contain content indicators (math, code, tables, images).
-            if (trailingEls.length >= 1 && trailingWords < totalWords * 0.15) {
-                const hasHeading = trailingEls.some(el => /^H[1-6]$/.test(el.tagName) || el.querySelector('h1, h2, h3, h4, h5, h6'));
-                const hasContent = trailingEls.some(el => el.querySelector(constants_1.CONTENT_ELEMENT_SELECTOR));
-                if (hasHeading && !hasContent) {
-                    for (const el of trailingEls) {
-                        if (this.debug && debugRemovals) {
-                            debugRemovals.push({ step: 'removeByContentPattern', reason: 'trailing thin section', text: (0, utils_1.textPreview)(el) });
-                        }
-                        el.remove();
-                    }
-                }
-            }
-        }
-        // Remove boilerplate sentences and trailing non-content.
-        // Search elements for end-of-article boilerplate, then truncate
-        // from the best ancestor that has siblings to remove.
-        const fullText = mainContent.textContent || '';
-        const boilerplateElements = mainContent.querySelectorAll('p, div, span, section');
-        for (const el of boilerplateElements) {
-            if (!el.parentNode)
-                continue;
-            const text = el.textContent?.trim() || '';
-            const words = (0, utils_1.countWords)(text);
-            if (words > 50 || words < 3)
-                continue;
-            for (const pattern of BOILERPLATE_PATTERNS) {
-                if (pattern.test(text)) {
-                    // Walk up to find an ancestor that has next siblings to truncate.
-                    // Don't walk all the way to mainContent's direct child — if there's
-                    // a single wrapper div, that would remove everything.
-                    let target = el;
-                    while (target.parentElement && target.parentElement !== mainContent) {
-                        if (target.nextElementSibling)
-                            break;
-                        target = target.parentElement;
-                    }
-                    // Only truncate if there's substantial content before the boilerplate
-                    const targetText = target.textContent || '';
-                    const targetPos = fullText.indexOf(targetText);
-                    if (targetPos < 200)
-                        continue;
-                    // Collect ancestors before modifying the DOM
-                    const ancestors = [];
-                    let anc = target.parentElement;
-                    while (anc && anc !== mainContent) {
-                        ancestors.push(anc);
-                        anc = anc.parentElement;
-                    }
-                    // Remove target element and its following siblings
-                    this.removeTrailingSiblings(target, true, debugRemovals);
-                    // Cascade upward: remove following siblings at each
-                    // ancestor level too. Everything after the boilerplate
-                    // in document order is non-content.
-                    for (const ancestor of ancestors) {
-                        this.removeTrailingSiblings(ancestor, false, debugRemovals);
-                    }
-                    return;
-                }
-            }
-        }
-    }
-    /**
-     * Remove an element's following siblings, and optionally the element itself.
-     */
-    removeTrailingSiblings(element, removeSelf, debugRemovals) {
-        let sibling = element.nextElementSibling;
-        while (sibling) {
-            const next = sibling.nextElementSibling;
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeByContentPattern',
-                    reason: 'trailing non-content',
-                    text: (0, utils_1.textPreview)(sibling)
-                });
-            }
-            sibling.remove();
-            sibling = next;
-        }
-        if (removeSelf) {
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeByContentPattern',
-                    reason: 'boilerplate text',
-                    text: (0, utils_1.textPreview)(element)
-                });
-            }
-            element.remove();
-        }
-    }
 }
 exports.Defuddle = Defuddle;
 //# sourceMappingURL=defuddle.js.map