defuddle 0.13.0 → 0.15.0

package/dist/defuddle.js

@@ -6,26 +6,17 @@ const extractor_registry_1 = require("./extractor-registry");
 const constants_1 = require("./constants");
 const standardize_1 = require("./standardize");
 const footnotes_1 = require("./elements/footnotes");
-const scoring_1 = require("./scoring");
+const callouts_1 = require("./elements/callouts");
+const scoring_1 = require("./removals/scoring");
+const small_images_1 = require("./removals/small-images");
+const hidden_1 = require("./removals/hidden");
+const selectors_1 = require("./removals/selectors");
+const content_patterns_1 = require("./removals/content-patterns");
+const metadata_block_1 = require("./removals/metadata-block");
 const utils_1 = require("./utils");
 const dom_1 = require("./utils/dom");
 /** Keys from extractor variables that map to top-level DefuddleResponse fields */
 const STANDARD_VARIABLE_KEYS = new Set(['title', 'author', 'published', 'site', 'description', 'image', 'language']);
-// Content pattern detection constants
-const CONTENT_DATE_PATTERN = /(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{1,2}/i;
-const CONTENT_READ_TIME_PATTERN = /\d+\s*min(?:ute)?s?\s+read\b/i;
-const BOILERPLATE_PATTERNS = [
-    /^This (?:article|story|piece) (?:appeared|was published|originally appeared) in\b/i,
-    /^A version of this (?:article|story) (?:appeared|was published) in\b/i,
-    /^Originally (?:published|appeared) (?:in|on|at)\b/i,
-];
-const METADATA_STRIP_PATTERNS = [
-    /\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b/gi,
-    /\b\d+(?:st|nd|rd|th)?\b/g,
-    /\bmin(?:ute)?s?\b/gi,
-    /\bread\b/gi,
-    /[|·•—–\-,.\s]/g,
-];
 class Defuddle {
     /**
      * Create a new Defuddle instance
@@ -35,6 +26,7 @@ class Defuddle {
     constructor(doc, options = {}) {
         this._schemaOrgData = undefined;
         this._schemaOrgExtracted = false;
+        this._inExtractorPipelineRun = false;
         this.doc = doc;
         this.options = options;
         this.debug = options.debug || false;
@@ -117,20 +109,23 @@ class Defuddle {
             }
         }
         // Strip dangerous elements from this.doc before any fallback paths
-        // that read from it (e.g. _findContentBySchemaText).
-        // This must happen after parseInternal, which needs script tags
-        // for schema.org extraction, site-specific extractors, and math.
+        // that read from it. This must happen after parseInternal, which needs
+        // script tags for schema.org extraction, site-specific extractors, and math.
         this._stripUnsafeElements();
-        // If schema.org has a SocialMediaPosting with text content that is
-        // longer than what we extracted, the scorer likely picked the wrong
-        // element from a feed. Find the correct element in the DOM.
+        // If schema.org has text content that is significantly longer than what we
+        // extracted, the scorer likely picked the wrong element from a feed page.
+        // Use a 1.5x threshold to avoid triggering when the difference is small
+        // (e.g. just related-content link text removed).
         const schemaText = this._getSchemaText(result.schemaOrgData);
-        if (schemaText && this.countHtmlWords(schemaText) > result.wordCount) {
-            const contentHtml = this._findContentBySchemaText(schemaText);
-            if (contentHtml) {
-                this._log('Found DOM content matching schema.org text');
-                result.content = contentHtml;
-                result.wordCount = this.countHtmlWords(contentHtml);
+        if (schemaText && this.countHtmlWords(schemaText) > result.wordCount * 1.5) {
+            const bestMatch = this._findElementBySchemaText(this.doc.body, schemaText);
+            if (bestMatch) {
+                // Re-run the full pipeline with the schema-identified element as the
+                // content root so it benefits from the same cleanup as normal extraction.
+                const selector = this.getElementSelector(bestMatch);
+                this._log('Schema.org suggests a better content element, retrying with selector:', selector);
+                const schemaRetry = this.parseInternal({ contentSelector: selector });
+                result = schemaRetry;
             }
             else {
                 this._log('Using schema.org text as content (DOM element not found)');
@@ -143,17 +138,30 @@ class Defuddle {
     /**
      * Extract text content from schema.org data (e.g. SocialMediaPosting, Article)
      */
-    _getSchemaText(schemaOrgData) {
-        if (!schemaOrgData)
+    _getSchemaText(schemaOrgData, depth = 0) {
+        if (!schemaOrgData || depth > 10)
             return '';
         const items = Array.isArray(schemaOrgData) ? schemaOrgData : [schemaOrgData];
         for (const item of items) {
+            // Recurse into nested arrays
+            if (Array.isArray(item)) {
+                const found = this._getSchemaText(item, depth + 1);
+                if (found)
+                    return found;
+                continue;
+            }
             if (item?.text && typeof item.text === 'string') {
                 return item.text;
             }
             if (item?.articleBody && typeof item.articleBody === 'string') {
                 return item.articleBody;
             }
+            // Traverse @graph arrays (common in JSON-LD with multiple entities)
+            if (item?.['@graph'] && Array.isArray(item['@graph'])) {
+                const found = this._getSchemaText(item['@graph'], depth + 1);
+                if (found)
+                    return found;
+            }
         }
         return '';
     }
@@ -194,82 +202,32 @@ class Defuddle {
         }
     }
     /**
-     * Find a DOM element whose text matches the schema.org text content.
-     * Used when the content scorer picked the wrong element from a feed page.
-     * Returns the element's inner HTML including sibling media (images, etc.)
+     * Find the smallest DOM element whose text contains the search phrase
+     * and whose word count is at least 80% of the expected count.
+     * Shared by _findSchemaContentElement and _findContentBySchemaText.
      */
-    _findContentBySchemaText(schemaText) {
-        const body = this.doc.body;
-        if (!body)
-            return '';
-        // Use the first paragraph as the search phrase.
-        // DOM textContent concatenates <p> elements without separators,
-        // so we can't cross paragraph boundaries when matching.
+    _findElementBySchemaText(root, schemaText) {
         const firstPara = schemaText.split(/\n\s*\n/)[0]?.trim() || '';
         const searchPhrase = firstPara.substring(0, 100).trim();
         if (!searchPhrase)
-            return '';
-        const schemaWordCount = this.countHtmlWords(schemaText);
-        // Find the smallest element whose text contains the search phrase
-        // and whose word count is close to the schema text's word count
+            return null;
+        const schemaWordCount = (0, utils_1.countWords)(schemaText);
         let bestMatch = null;
         let bestSize = Infinity;
-        const allElements = body.querySelectorAll('*');
+        const allElements = root.querySelectorAll('*');
         for (const el of allElements) {
-            const elText = (el.textContent || '');
+            if (el === root)
+                continue;
+            const elText = el.textContent || '';
             if (!elText.includes(searchPhrase))
                 continue;
             const elWords = (0, utils_1.countWords)(elText);
-            // Element should contain roughly the same amount of text
-            // (allow some slack for surrounding whitespace / minor extras)
             if (elWords >= schemaWordCount * 0.8 && elWords < bestSize) {
                 bestSize = elWords;
                 bestMatch = el;
             }
         }
-        if (!bestMatch)
-            return '';
-        // Read the largest sibling image src BEFORE resolveRelativeUrls
-        // can mangle comma-containing CDN URLs in srcset attributes
-        let imageSrc = '';
-        let imageAlt = '';
-        const parent = bestMatch.parentElement;
-        if (parent && parent !== body) {
-            const images = parent.querySelectorAll('img');
-            let largestImg = null;
-            let largestArea = 0;
-            for (const img of images) {
-                if (bestMatch.contains(img))
-                    continue;
-                const w = parseInt(img.getAttribute('width') || '0', 10);
-                const h = parseInt(img.getAttribute('height') || '0', 10);
-                const area = w * h;
-                if (area > largestArea) {
-                    largestArea = area;
-                    largestImg = img;
-                }
-            }
-            if (largestImg) {
-                imageSrc = this._getLargestImageSrc(largestImg);
-                imageAlt = largestImg.getAttribute('alt') || '';
-                try {
-                    const baseUrl = this.options.url || this.doc.URL;
-                    if (baseUrl)
-                        imageSrc = new URL(imageSrc, baseUrl).href;
-                }
-                catch { }
-            }
-        }
-        // Now resolve URLs in the text content
-        this.resolveRelativeUrls(bestMatch);
-        let html = (0, dom_1.serializeHTML)(bestMatch);
-        if (imageSrc) {
-            const img = this.doc.createElement('img');
-            img.setAttribute('src', imageSrc);
-            img.setAttribute('alt', imageAlt);
-            html += img.outerHTML;
-        }
-        return html;
+        return bestMatch;
     }
     findLargestHiddenContentSelector() {
         const body = this.doc.body;
@@ -353,7 +311,8 @@ class Defuddle {
         try {
             const url = this.options.url || this.doc.URL;
             const schemaOrgData = this.getSchemaOrgData();
-            const extractor = extractor_registry_1.ExtractorRegistry.findPreferredAsyncExtractor(this.doc, url, schemaOrgData);
+            const extractorOpts = { includeReplies: this.options.includeReplies ?? 'extractors', language: this.options.language };
+            const extractor = extractor_registry_1.ExtractorRegistry.findPreferredAsyncExtractor(this.doc, url, schemaOrgData, extractorOpts);
             if (extractor) {
                 const extracted = await extractor.extractAsync();
                 return this.getExtractorVariables(extracted.variables) || null;
@@ -368,7 +327,8 @@ class Defuddle {
         try {
             const url = this.options.url || this.doc.URL;
             const schemaOrgData = this.getSchemaOrgData();
-            const extractor = finder(this.doc, url, schemaOrgData);
+            const extractorOpts = { includeReplies: this.options.includeReplies ?? 'extractors', language: this.options.language };
+            const extractor = finder(this.doc, url, schemaOrgData, extractorOpts);
             if (extractor) {
                 const startTime = Date.now();
                 const extracted = await extractor.extractAsync();
@@ -387,6 +347,16 @@ class Defuddle {
      */
     parseInternal(overrideOptions = {}) {
         const startTime = Date.now();
+        const profile = {};
+        const doProfile = this.options.profile ?? false;
+        const profileStep = (name, fn) => {
+            if (!doProfile)
+                return fn();
+            const t = performance.now();
+            const result = fn();
+            profile[name] = Math.round(performance.now() - t);
+            return result;
+        };
         // Guard against empty/broken documents (e.g. empty HTML, bot-blocked pages)
         if (!this.doc.documentElement) {
             const url = this.options.url || '';
@@ -414,6 +384,7 @@ class Defuddle {
             removeSmallImages: true,
             removeContentPatterns: true,
             standardize: true,
+            includeReplies: 'extractors',
             ...this.options,
             ...overrideOptions
         };
@@ -435,10 +406,41 @@ class Defuddle {
         try {
             // Use site-specific extractor first, if there is one
             const url = options.url || this.doc.URL;
-            const extractor = extractor_registry_1.ExtractorRegistry.findExtractor(this.doc, url, schemaOrgData);
-            if (extractor && extractor.canExtract()) {
-                const extracted = extractor.extract();
-                return this.buildExtractorResponse(extracted, metadata, startTime, extractor, pageMetaTags);
+            const extractorOpts = {
+                includeReplies: options.includeReplies,
+                language: options.language,
+            };
+            if (!this._inExtractorPipelineRun) {
+                const extractor = extractor_registry_1.ExtractorRegistry.findExtractor(this.doc, url, schemaOrgData, extractorOpts);
+                if (extractor && extractor.canExtract()) {
+                    const extracted = extractor.extract();
+                    if (extracted.contentSelector) {
+                        this._inExtractorPipelineRun = true;
+                        try {
+                            const pipelineResult = this.parseInternal({
+                                contentSelector: extracted.contentSelector,
+                                removeLowScoring: false,
+                                removeHiddenElements: false,
+                            });
+                            const variables = this.getExtractorVariables(extracted.variables);
+                            return {
+                                ...pipelineResult,
+                                title: extracted.variables?.title || pipelineResult.title,
+                                description: extracted.variables?.description || pipelineResult.description,
+                                author: extracted.variables?.author || pipelineResult.author,
+                                published: extracted.variables?.published || pipelineResult.published,
+                                site: extracted.variables?.site || pipelineResult.site,
+                                language: extracted.variables?.language || pipelineResult.language,
+                                extractorType: extractor.constructor.name.replace('Extractor', '').toLowerCase(),
+                                ...(variables ? { variables } : {}),
+                            };
+                        }
+                        finally {
+                            this._inExtractorPipelineRun = false;
+                        }
+                    }
+                    return this.buildExtractorResponse(extracted, metadata, startTime, extractor, pageMetaTags);
+                }
             }
             // Continue if there is no extractor...
             // Evaluate mobile styles and sizes on original document (cached across retries)
@@ -448,29 +450,47 @@ class Defuddle {
             const mobileStyles = this._mobileStyles;
             // Find small images in original document (cached across retries)
             if (!this._smallImages) {
-                this._smallImages = this.findSmallImages(this.doc);
+                this._smallImages = (0, small_images_1.findSmallImages)(this.doc, this.debug);
             }
             const smallImages = this._smallImages;
             // Clone document
-            const clone = this.doc.cloneNode(true);
-            // Merge adjacent text nodes that some DOM implementations (e.g. linkedom)
-            // create when parsing HTML entities like &#39;
-            clone.body?.normalize();
+            let clone;
+            profileStep('cloneDocument', () => {
+                clone = this.doc.cloneNode(true);
+                // Merge adjacent text nodes that some DOM implementations (e.g. linkedom)
+                // create when parsing HTML entities like &#39;
+                clone.body?.normalize();
+            });
             // Flatten shadow DOM content into the clone
-            this.flattenShadowRoots(this.doc, clone);
+            profileStep('flattenShadowRoots', () => this.flattenShadowRoots(this.doc, clone));
             // Resolve React streaming SSR suspense boundaries
-            this.resolveStreamedContent(clone);
+            profileStep('resolveStreamedContent', () => this.resolveStreamedContent(clone));
             // Apply mobile styles to clone
-            this.applyMobileStyles(clone, mobileStyles);
+            profileStep('applyMobileStyles', () => this.applyMobileStyles(clone, mobileStyles));
             // Find main content
-            let mainContent = null;
-            if (options.contentSelector) {
-                mainContent = clone.querySelector(options.contentSelector);
-                this._log('Using contentSelector:', options.contentSelector, mainContent ? 'found' : 'not found');
-            }
-            if (!mainContent) {
-                mainContent = this.findMainContent(clone);
-            }
+            const mainContent = profileStep('findMainContent', () => {
+                let found = null;
+                if (options.contentSelector) {
+                    found = clone.querySelector(options.contentSelector);
+                    this._log('Using contentSelector:', options.contentSelector, found ? 'found' : 'not found');
+                }
+                if (!found) {
+                    found = this.findMainContent(clone);
+                }
+                // If we fell back to <body>, try using schema.org articleBody/text
+                // to find a more specific content element within the DOM.
+                if (found && found.tagName.toLowerCase() === 'body') {
+                    const schemaText = this._getSchemaText(schemaOrgData);
+                    if (schemaText) {
+                        const schemaContent = this._findElementBySchemaText(clone.body, schemaText);
+                        if (schemaContent) {
+                            this._log('Found content element via schema.org text');
+                            found = schemaContent;
+                        }
+                    }
+                }
+                return found;
+            });
             if (!mainContent) {
                 const fallbackContent = this.doc.body ? this.resolveContentUrls((0, dom_1.serializeHTML)(this.doc.body)) : '';
                 const endTime = Date.now();
@@ -482,37 +502,66 @@ class Defuddle {
                     metaTags: pageMetaTags
                 };
             }
+            // Remove h1-adjacent date/author metadata blocks from the content.
+            // These are extracted as frontmatter but also appear in the body when a
+            // wide container (e.g. <main>) is selected as the content element.
+            profileStep('removeMetadataBlock', () => {
+                if (metadata.published || metadata.author) {
+                    (0, metadata_block_1.removeMetadataBlock)(mainContent);
+                }
+                // Remove <wbr> elements — word break opportunity hints that carry no
+                // content but cause unwanted whitespace during standardization.
+                mainContent.querySelectorAll('wbr').forEach(el => el.remove());
+            });
             // Standardize footnotes before cleanup (CSS sidenotes use display:none)
-            if (options.standardize) {
-                (0, footnotes_1.standardizeFootnotes)(mainContent);
-            }
+            profileStep('standardizeFootnotesCallouts', () => {
+                if (options.standardize) {
+                    (0, footnotes_1.standardizeFootnotes)(mainContent);
+                    (0, callouts_1.standardizeCallouts)(mainContent);
+                }
+            });
             // Remove small images
-            if (options.removeSmallImages) {
-                this.removeSmallImages(clone, smallImages);
-            }
+            profileStep('removeSmallImages', () => {
+                if (options.removeSmallImages) {
+                    (0, small_images_1.removeSmallImages)(clone, smallImages, this.debug);
+                }
+            });
             // Remove hidden elements using computed styles
-            if (options.removeHiddenElements) {
-                this.removeHiddenElements(clone, debugRemovals);
-            }
-            // Remove non-content blocks by scoring
-            // Tries to find lists, navigation based on text content and link density
-            if (options.removeLowScoring) {
-                scoring_1.ContentScorer.scoreAndRemove(clone, this.debug, debugRemovals, mainContent);
-            }
-            // Remove clutter using selectors
-            if (options.removeExactSelectors || options.removePartialSelectors) {
-                this.removeBySelector(clone, options.removeExactSelectors, options.removePartialSelectors, mainContent, debugRemovals, options.removeHiddenElements === false);
-            }
+            profileStep('removeHiddenElements', () => {
+                if (options.removeHiddenElements) {
+                    (0, hidden_1.removeHiddenElements)(clone, this.debug, debugRemovals);
+                }
+            });
+            // Remove clutter using selectors — deterministic removal of known
+            // non-content elements (nav, footer, .sidebar, etc.) by class/id.
+            // Runs before scoring so the heuristic scorer sees a cleaner DOM.
+            profileStep('removeBySelector', () => {
+                if (options.removeExactSelectors || options.removePartialSelectors) {
+                    (0, selectors_1.removeBySelector)(clone, this.debug, options.removeExactSelectors, options.removePartialSelectors, mainContent, debugRemovals, options.removeHiddenElements === false);
+                }
+            });
+            // Remove non-content blocks by scoring — heuristic removal based
+            // on link density, text ratios, and navigation indicators.
+            profileStep('removeLowScoring', () => {
+                if (options.removeLowScoring) {
+                    scoring_1.ContentScorer.scoreAndRemove(clone, this.debug, debugRemovals, mainContent);
+                }
+            });
             // Remove elements by content patterns (read time, boilerplate, article cards)
-            if (options.removeContentPatterns && mainContent) {
-                this.removeByContentPattern(mainContent, this.debug ? debugRemovals : undefined);
-            }
+            profileStep('removeByContentPattern', () => {
+                if (options.removeContentPatterns && mainContent) {
+                    const url = this.options.url || this.doc.URL || '';
+                    (0, content_patterns_1.removeByContentPattern)(mainContent, this.debug, url, debugRemovals);
+                }
+            });
             // Normalize the main content
-            if (options.standardize) {
-                (0, standardize_1.standardizeContent)(mainContent, metadata, this.doc, this.debug);
-            }
+            profileStep('standardizeContent', () => {
+                if (options.standardize) {
+                    (0, standardize_1.standardizeContent)(mainContent, metadata, this.doc, this.debug, doProfile ? profile : undefined);
+                }
+            });
             // Resolve relative URLs to absolute
-            this.resolveRelativeUrls(mainContent);
+            profileStep('resolveRelativeUrls', () => this.resolveRelativeUrls(mainContent));
             const content = mainContent.outerHTML;
             const endTime = Date.now();
             const result = {
@@ -528,6 +577,9 @@ class Defuddle {
                     removals: debugRemovals
                 };
             }
+            if (this.options.profile) {
+                result.profile = profile;
+            }
             return result;
         }
         catch (error) {
@@ -556,7 +608,6 @@ class Defuddle {
             .replace(/&\w+;/g, ' ');
         return (0, utils_1.countWords)(text);
     }
-    // Make all other methods private by removing the static keyword and using private
     _log(...args) {
         if (this.debug) {
             console.log('Defuddle:', ...args);
@@ -653,282 +704,6 @@ class Defuddle {
             image.remove();
         });
     }
-    removeHiddenElements(doc, debugRemovals) {
-        let count = 0;
-        const elementsToRemove = new Map();
-        // Check inline styles and CSS class-based hidden patterns.
-        const hiddenStylePattern = /(?:^|;\s*)(?:display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0)(?:\s*;|\s*$)/i;
-        // Only use getComputedStyle in browser environments where it's meaningful.
-        // In JSDOM/linkedom without stylesheets, it's extremely slow and unreliable.
-        const defaultView = doc.defaultView;
-        const isBrowser = typeof window !== 'undefined' && defaultView === window;
-        const allElements = doc.querySelectorAll('*');
-        for (const element of allElements) {
-            // Skip elements that contain math — sites like Wikipedia wrap MathML
-            // in display:none spans for accessibility (the visible version is an
-            // image/SVG fallback). We need to preserve these for math extraction.
-            if (element.querySelector('math, [data-mathml], .katex-mathml') ||
-                element.tagName.toLowerCase() === 'math') {
-                continue;
-            }
-            // Check inline style for hidden patterns
-            const style = element.getAttribute('style');
-            if (style && hiddenStylePattern.test(style)) {
-                const reason = style.includes('display') ? 'display:none' :
-                    style.includes('visibility') ? 'visibility:hidden' : 'opacity:0';
-                elementsToRemove.set(element, reason);
-                count++;
-                continue;
-            }
-            // Use getComputedStyle only in real browser environments
-            if (isBrowser) {
-                try {
-                    const computedStyle = defaultView.getComputedStyle(element);
-                    let reason = '';
-                    if (computedStyle.display === 'none')
-                        reason = 'display:none';
-                    else if (computedStyle.visibility === 'hidden')
-                        reason = 'visibility:hidden';
-                    else if (computedStyle.opacity === '0')
-                        reason = 'opacity:0';
-                    if (reason) {
-                        elementsToRemove.set(element, reason);
-                        count++;
-                        continue;
-                    }
-                }
-                catch (e) { }
-            }
-            // Detect CSS framework hidden utilities (e.g. Tailwind's "hidden",
-            // "sm:hidden", "not-machine:hidden")
-            const className = element.getAttribute('class') || '';
-            if (className) {
-                const tokens = className.split(/\s+/);
-                for (const token of tokens) {
-                    if (token === 'hidden' || token.endsWith(':hidden') || token === 'invisible' || token.endsWith(':invisible')) {
-                        elementsToRemove.set(element, `class:${token}`);
-                        count++;
-                        break;
-                    }
-                }
-            }
-        }
-        // Batch remove all hidden elements
-        elementsToRemove.forEach((reason, el) => {
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeHiddenElements',
-                    reason,
-                    text: (0, utils_1.textPreview)(el)
-                });
-            }
-            el.remove();
-        });
-        this._log('Removed hidden elements:', count);
-    }
-    removeBySelector(doc, removeExact = true, removePartial = true, mainContent, debugRemovals, skipHiddenExactSelectors = false) {
-        const startTime = Date.now();
-        let exactSelectorCount = 0;
-        let partialSelectorCount = 0;
-        // Track all elements to be removed, with their match type
-        const elementsToRemove = new Map();
-        // First collect elements matching exact selectors
-        if (removeExact) {
-            const exactElements = doc.querySelectorAll(constants_1.EXACT_SELECTORS_JOINED);
-            exactElements.forEach(el => {
-                if (el?.parentNode) {
-                    if (skipHiddenExactSelectors) {
-                        const hiddenAncestor = el.closest(constants_1.HIDDEN_EXACT_SKIP_SELECTOR);
-                        const role = (el.getAttribute('role') || '').toLowerCase();
-                        if (el.matches(constants_1.HIDDEN_EXACT_SELECTOR) ||
-                            (hiddenAncestor && role === 'dialog')) {
-                            return;
-                        }
-                    }
-                    // Skip elements inside code blocks (e.g. syntax highlighting spans)
-                    if (el.closest('pre, code')) {
-                        return;
-                    }
-                    elementsToRemove.set(el, { type: 'exact' });
-                    exactSelectorCount++;
-                }
-            });
-        }
-        if (removePartial) {
-            // Pre-compile individual regexes for debug pattern identification only
-            const individualRegexes = this.debug
-                ? constants_1.PARTIAL_SELECTORS.map(p => ({ pattern: p, regex: new RegExp(p, 'i') }))
-                : null;
-            // Use pre-built attribute selector for elements we care about
-            const allElements = doc.querySelectorAll(constants_1.TEST_ATTRIBUTES_SELECTOR);
-            // Process elements for partial matches
-            allElements.forEach(el => {
-                // Skip if already marked for removal
-                if (elementsToRemove.has(el)) {
-                    return;
-                }
-                // Skip code elements and elements containing code blocks
-                // where class names indicate language/syntax, not page structure
-                const tag = el.tagName;
-                if (tag === 'CODE' || tag === 'PRE' || el.querySelector('pre')) {
-                    return;
-                }
-                // Get all relevant attributes and combine into a single string
-                const attrs = constants_1.TEST_ATTRIBUTES.map(attr => {
-                    if (attr === 'class') {
-                        return el.className && typeof el.className === 'string' ? el.className : '';
-                    }
-                    if (attr === 'id') {
-                        return el.id || '';
-                    }
-                    return el.getAttribute(attr) || '';
-                }).join(' ').toLowerCase();
-                // Skip if no attributes to check
-                if (!attrs.trim()) {
-                    return;
-                }
-                // Check for partial match using single regex test
-                if (constants_1.PARTIAL_SELECTORS_REGEX.test(attrs)) {
-                    const matchedPattern = individualRegexes
-                        ? individualRegexes.find(r => r.regex.test(attrs))?.pattern
-                        : undefined;
-                    elementsToRemove.set(el, { type: 'partial', selector: matchedPattern });
-                    partialSelectorCount++;
-                }
-            });
-        }
-        // Remove all collected elements in a single pass
-        // Skip elements that are ancestors of mainContent to avoid disconnecting it
-        // Skip footnote list containers, their parents, and immediate children
-        // Skip anchor links inside headings - the heading transform handles these
-        elementsToRemove.forEach(({ type, selector }, el) => {
-            if (mainContent && el.contains(mainContent)) {
-                return;
-            }
-            if (el.tagName === 'A' && el.closest('h1, h2, h3, h4, h5, h6')) {
-                return;
-            }
-            try {
-                if (el.matches(constants_1.FOOTNOTE_LIST_SELECTORS) || el.querySelector(constants_1.FOOTNOTE_LIST_SELECTORS)) {
-                    return;
-                }
-                // Protect immediate children of footnote containers (e.g. wikidot div.footnote-footer)
-                const parent = el.parentElement;
-                if (parent && parent.matches(constants_1.FOOTNOTE_LIST_SELECTORS)) {
-                    return;
-                }
-            }
-            catch (e) { }
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeBySelector',
-                    selector: type === 'exact' ? 'exact' : selector,
-                    reason: type === 'exact' ? 'exact selector match' : `partial match: ${selector}`,
-                    text: (0, utils_1.textPreview)(el)
-                });
-            }
-            el.remove();
-        });
-        const endTime = Date.now();
-        this._log('Removed clutter elements:', {
-            exactSelectors: exactSelectorCount,
-            partialSelectors: partialSelectorCount,
-            total: elementsToRemove.size,
-            processingTime: `${(endTime - startTime).toFixed(2)}ms`
-        });
-    }
-    // Find small IMG and SVG elements
-    findSmallImages(doc) {
-        const MIN_DIMENSION = 33;
-        const smallImages = new Set();
-        let processedCount = 0;
-        const elements = doc.querySelectorAll('img, svg');
-        const defaultView = doc.defaultView;
-        const isBrowser = typeof window !== 'undefined' && defaultView === window;
-        for (const element of elements) {
-            const attrWidth = parseInt(element.getAttribute('width') || '0');
-            const attrHeight = parseInt(element.getAttribute('height') || '0');
-            // Check inline style dimensions
-            const style = element.getAttribute('style') || '';
-            const styleWidth = parseInt(style.match(/width\s*:\s*(\d+)/)?.[1] || '0');
-            const styleHeight = parseInt(style.match(/height\s*:\s*(\d+)/)?.[1] || '0');
-            // Use getComputedStyle and getBoundingClientRect only in browser
-            let computedWidth = 0, computedHeight = 0;
-            if (isBrowser) {
-                try {
-                    const cs = defaultView.getComputedStyle(element);
-                    computedWidth = parseInt(cs.width) || 0;
-                    computedHeight = parseInt(cs.height) || 0;
-                }
-                catch (e) { }
-                try {
-                    const rect = element.getBoundingClientRect();
-                    if (rect.width > 0)
-                        computedWidth = computedWidth || rect.width;
-                    if (rect.height > 0)
-                        computedHeight = computedHeight || rect.height;
-                }
-                catch (e) { }
-            }
-            const widths = [attrWidth, styleWidth, computedWidth].filter(d => d > 0);
-            const heights = [attrHeight, styleHeight, computedHeight].filter(d => d > 0);
-            if (widths.length > 0 && heights.length > 0) {
-                const effectiveWidth = Math.min(...widths);
-                const effectiveHeight = Math.min(...heights);
-                if (effectiveWidth < MIN_DIMENSION || effectiveHeight < MIN_DIMENSION) {
-                    const identifier = this.getElementIdentifier(element);
-                    if (identifier) {
-                        smallImages.add(identifier);
-                        processedCount++;
-                    }
-                }
-            }
-        }
-        this._log('Found small elements:', processedCount);
-        return smallImages;
-    }
-    removeSmallImages(doc, smallImages) {
-        let removedCount = 0;
-        ['img', 'svg'].forEach(tag => {
-            const elements = doc.getElementsByTagName(tag);
-            Array.from(elements).forEach(element => {
-                const identifier = this.getElementIdentifier(element);
-                if (identifier && smallImages.has(identifier)) {
-                    element.remove();
-                    removedCount++;
-                }
-            });
-        });
-        this._log('Removed small elements:', removedCount);
-    }
-    getElementIdentifier(element) {
-        // Try to create a unique identifier using various attributes
-        if (element.tagName.toLowerCase() === 'img') {
-            // For lazy-loaded images, use data-src as identifier if available
-            const dataSrc = element.getAttribute('data-src');
-            if (dataSrc)
-                return `src:${dataSrc}`;
-            const src = element.getAttribute('src') || '';
-            const srcset = element.getAttribute('srcset') || '';
-            const dataSrcset = element.getAttribute('data-srcset');
-            if (src)
-                return `src:${src}`;
-            if (srcset)
-                return `srcset:${srcset}`;
-            if (dataSrcset)
-                return `srcset:${dataSrcset}`;
-        }
-        const id = element.id || '';
-        const className = element.className || '';
-        const viewBox = element.tagName.toLowerCase() === 'svg' ? element.getAttribute('viewBox') || '' : '';
-        if (id)
-            return `id:${id}`;
-        if (viewBox)
-            return `viewBox:${viewBox}`;
-        if (className)
-            return `class:${className}`;
-        return null;
-    }
     findMainContent(doc) {
         // Find all potential content containers
         const candidates = [];
@@ -1035,8 +810,8 @@ class Defuddle {
             if (current.id) {
                 selector += '#' + current.id;
             }
-            else if (current.className && typeof current.className === 'string') {
-                selector += '.' + current.className.trim().split(/\s+/).join('.');
+            else if ((0, dom_1.getClassName)(current)) {
+                selector += '.' + (0, dom_1.getClassName)(current).trim().split(/\s+/).join('.');
             }
             parts.unshift(selector);
             current = current.parentElement;
@@ -1050,9 +825,23 @@ class Defuddle {
      * Resolve relative URLs to absolute within a DOM element
      */
     resolveRelativeUrls(element) {
-        const baseUrl = this.options.url || this.doc.URL;
-        if (!baseUrl)
+        const docUrl = this.options.url || this.doc.URL;
+        if (!docUrl)
             return;
+        // Respect <base href> for relative URL resolution, matching browser behavior
+        let baseUrl = docUrl;
+        const baseEl = this.doc.querySelector('base[href]');
+        if (baseEl) {
+            const baseHref = baseEl.getAttribute('href');
+            if (baseHref) {
+                try {
+                    baseUrl = new URL(baseHref, docUrl).href;
+                }
+                catch {
+                    // Invalid base href, fall back to document URL
+                }
+            }
+        }
         const resolve = (url) => {
             // Some pages ship escaped quoted hrefs like \"mailto:...\" in server templates.
             // Normalize these before URL resolution.
@@ -1060,6 +849,9 @@ class Defuddle {
                 .trim()
                 .replace(/^\\?["']+/, '')
                 .replace(/\\?["']+$/, '');
+            // Fragment-only hrefs reference anchors within the same document — keep them relative.
+            if (normalized.startsWith('#'))
+                return normalized;
             try {
                 return new URL(normalized, baseUrl).href;
             }
@@ -1330,7 +1122,7 @@ class Defuddle {
         return {
             content: contentHtml,
             title: extracted.variables?.title || metadata.title,
-            description: metadata.description,
+            description: extracted.variables?.description || metadata.description,
             domain: metadata.domain,
             favicon: metadata.favicon,
             image: metadata.image,
@@ -1363,207 +1155,6 @@ class Defuddle {
         }
         return hasCustom ? custom : undefined;
     }
-    /**
-     * Content-based pattern removal for elements that can't be detected by
-     * CSS selectors (e.g. Tailwind/CSS-in-JS sites with non-semantic class names).
-     */
-    removeByContentPattern(mainContent, debugRemovals) {
-        // Remove read time metadata (e.g. "Mar 4th 2026 | 3 min read")
-        // Only removes leaf elements whose text is PURELY date + read time,
-        // not mixed with other meaningful content like tag names.
-        const candidates = Array.from(mainContent.querySelectorAll('p, span, div, time'));
-        for (const el of candidates) {
-            if (!el.parentNode)
-                continue;
-            if (el.closest('pre') || el.closest('code'))
-                continue;
-            const text = el.textContent?.trim() || '';
-            const words = (0, utils_1.countWords)(text);
-            // Match date + read time in short elements
-            if (words <= 15 && CONTENT_DATE_PATTERN.test(text) && CONTENT_READ_TIME_PATTERN.test(text)) {
-                // Ensure this is a leaf-ish element, not a large container
-                if (el.querySelectorAll('p, div, section, article').length === 0) {
-                    // Verify the text is ONLY date + read time metadata
-                    // by stripping all date/time words and checking nothing remains
-                    let cleaned = text;
-                    for (const pattern of METADATA_STRIP_PATTERNS) {
-                        cleaned = cleaned.replace(pattern, '');
-                    }
-                    if (cleaned.trim().length > 0)
-                        continue;
-                    if (this.debug && debugRemovals) {
-                        debugRemovals.push({
-                            step: 'removeByContentPattern',
-                            reason: 'read time metadata',
-                            text: (0, utils_1.textPreview)(el)
-                        });
-                    }
-                    el.remove();
-                }
-            }
-        }
-        // Remove standalone time/date elements near the start or end of content.
-        // A <time> in its own paragraph at the boundary is metadata (publish date),
-        // but <time> inline within prose should be preserved (see issue #136).
-        const timeElements = Array.from(mainContent.querySelectorAll('time'));
-        const contentText = mainContent.textContent || '';
-        for (const time of timeElements) {
-            if (!time.parentNode)
-                continue;
-            // Walk up through inline/formatting wrappers only (i, em, span, b, strong)
-            // Stop at block elements to avoid removing containers with other content.
-            let target = time;
-            let targetText = target.textContent?.trim() || '';
-            while (target.parentElement && target.parentElement !== mainContent) {
-                const parentTag = target.parentElement.tagName.toLowerCase();
-                const parentText = target.parentElement.textContent?.trim() || '';
-                // If parent is a <p> that only wraps this time, include it
-                if (parentTag === 'p' && parentText === targetText) {
-                    target = target.parentElement;
-                    break;
-                }
-                // Only walk through inline formatting wrappers
-                if (['i', 'em', 'span', 'b', 'strong', 'small'].includes(parentTag) &&
-                    parentText === targetText) {
-                    target = target.parentElement;
-                    targetText = parentText;
-                    continue;
-                }
-                break;
-            }
-            const text = target.textContent?.trim() || '';
-            const words = (0, utils_1.countWords)(text);
-            if (words > 10)
-                continue;
-            // Check if this element is near the start or end of mainContent
-            const pos = contentText.indexOf(text);
-            const distFromEnd = contentText.length - (pos + text.length);
-            if (pos > 200 && distFromEnd > 200)
-                continue;
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeByContentPattern',
-                    reason: 'boundary date element',
-                    text: (0, utils_1.textPreview)(target)
-                });
-            }
-            target.remove();
-        }
-        // Remove section breadcrumbs
-        // Short elements containing a link to a parent section of the current URL.
-        const url = this.options.url || this.doc.URL || '';
-        let urlPath = '';
-        try {
-            urlPath = new URL(url).pathname;
-        }
-        catch { }
-        if (urlPath) {
-            const shortElements = mainContent.querySelectorAll('div, span, p');
-            for (const el of shortElements) {
-                if (!el.parentNode)
-                    continue;
-                const text = el.textContent?.trim() || '';
-                const words = (0, utils_1.countWords)(text);
-                if (words > 10)
-                    continue;
-                // Must be a leaf-ish element (no block children)
-                if (el.querySelectorAll('p, div, section, article').length > 0)
-                    continue;
-                const link = el.querySelector('a[href]');
-                if (!link)
-                    continue;
-                try {
-                    const linkPath = new URL(link.getAttribute('href') || '', url).pathname;
-                    if (linkPath !== '/' && linkPath !== urlPath && urlPath.startsWith(linkPath)) {
-                        if (this.debug && debugRemovals) {
-                            debugRemovals.push({
-                                step: 'removeByContentPattern',
-                                reason: 'section breadcrumb',
-                                text: (0, utils_1.textPreview)(el)
-                            });
-                        }
-                        el.remove();
-                    }
-                }
-                catch { }
-            }
-        }
-        // Remove boilerplate sentences and trailing non-content.
-        // Search elements for end-of-article boilerplate, then truncate
-        // from the best ancestor that has siblings to remove.
-        const fullText = mainContent.textContent || '';
-        const boilerplateElements = mainContent.querySelectorAll('p, div, span, section');
-        for (const el of boilerplateElements) {
-            if (!el.parentNode)
-                continue;
-            const text = el.textContent?.trim() || '';
-            const words = (0, utils_1.countWords)(text);
-            if (words > 50 || words < 3)
-                continue;
-            for (const pattern of BOILERPLATE_PATTERNS) {
-                if (pattern.test(text)) {
-                    // Walk up to find an ancestor that has next siblings to truncate.
-                    // Don't walk all the way to mainContent's direct child — if there's
-                    // a single wrapper div, that would remove everything.
-                    let target = el;
-                    while (target.parentElement && target.parentElement !== mainContent) {
-                        if (target.nextElementSibling)
-                            break;
-                        target = target.parentElement;
-                    }
-                    // Only truncate if there's substantial content before the boilerplate
-                    const targetText = target.textContent || '';
-                    const targetPos = fullText.indexOf(targetText);
-                    if (targetPos < 200)
-                        continue;
-                    // Collect ancestors before modifying the DOM
-                    const ancestors = [];
-                    let anc = target.parentElement;
-                    while (anc && anc !== mainContent) {
-                        ancestors.push(anc);
-                        anc = anc.parentElement;
-                    }
-                    // Remove target element and its following siblings
-                    this.removeTrailingSiblings(target, true, debugRemovals);
-                    // Cascade upward: remove following siblings at each
-                    // ancestor level too. Everything after the boilerplate
-                    // in document order is non-content.
-                    for (const ancestor of ancestors) {
-                        this.removeTrailingSiblings(ancestor, false, debugRemovals);
-                    }
-                    return;
-                }
-            }
-        }
-    }
-    /**
-     * Remove an element's following siblings, and optionally the element itself.
-     */
-    removeTrailingSiblings(element, removeSelf, debugRemovals) {
-        let sibling = element.nextElementSibling;
-        while (sibling) {
-            const next = sibling.nextElementSibling;
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeByContentPattern',
-                    reason: 'trailing non-content',
-                    text: (0, utils_1.textPreview)(sibling)
-                });
-            }
-            sibling.remove();
-            sibling = next;
-        }
-        if (removeSelf) {
-            if (this.debug && debugRemovals) {
-                debugRemovals.push({
-                    step: 'removeByContentPattern',
-                    reason: 'boilerplate text',
-                    text: (0, utils_1.textPreview)(element)
-                });
-            }
-            element.remove();
-        }
-    }
 }
 exports.Defuddle = Defuddle;
 //# sourceMappingURL=defuddle.js.map