@j0hanz/superfetch 1.0.2 → 1.0.5

package/dist/services/card-extractor.js

@@ -0,0 +1,194 @@
+/**
+ * Card link extraction utilities for preserving card-style navigation
+ * from documentation sites before Readability strips them.
+ */
+const NOISE_SELECTORS = 'style, svg, [class*="icon"], [aria-hidden="true"]';
+/**
+ * Clean element by removing noise (styles, SVGs, icons)
+ */
+function cleanElement(element) {
+    const clone = element.cloneNode(true);
+    clone.querySelectorAll(NOISE_SELECTORS).forEach((el) => {
+        el.remove();
+    });
+    return clone;
+}
+/**
+ * Extract clean title from a card-like link element
+ */
+function extractCardTitle(link) {
+    const clone = cleanElement(link);
+    // Look for the first div child which typically contains the title in card layouts
+    for (const div of clone.querySelectorAll('div')) {
+        if (div.querySelector('div'))
+            continue; // Skip container divs
+        const text = div.textContent.trim();
+        if (text.length > 1 &&
+            text.length < 50 &&
+            !text.includes(' with ') &&
+            !text.includes('Use ')) {
+            return text;
+        }
+    }
+    // Look for structured title elements
+    const titleEl = clone.querySelector('[class*="title"], h2, h3, h4, h5, strong');
+    if (titleEl) {
+        const title = titleEl.textContent.trim();
+        if (title.length > 1 && title.length < 100)
+            return title;
+    }
+    // Fall back to first meaningful text content
+    const text = clone.textContent.trim().replace(/\s+/g, ' ');
+    if (!text || text.length <= 1 || text.length >= 100)
+        return null;
+    // Extract title part (first word/phrase before description)
+    const words = text.split(/(?=Use |Try |Learn |Get )/);
+    if (words.length > 1 && words[0])
+        return words[0].trim();
+    const firstLine = text
+        .split(/[.\n]/)
+        .find((s) => s.trim().length > 1)
+        ?.trim();
+    return firstLine ?? text;
+}
+/**
+ * Extract description from a card-like link element
+ */
+function extractCardDescription(link) {
+    const clone = cleanElement(link);
+    const descEl = clone.querySelector('p, [class*="description"], [class*="muted"]');
+    if (descEl) {
+        const desc = descEl.textContent.trim();
+        if (desc.length > 5 && desc.length < 200)
+            return desc;
+    }
+    const text = clone.textContent.trim().replace(/\s+/g, ' ');
+    if (!text)
+        return null;
+    const descMatch = /(Use |Try |Learn |Get ).*$/.exec(text);
+    if (descMatch && descMatch[0].length > 10)
+        return descMatch[0];
+    return null;
+}
+/**
+ * Create a list item with link and optional description
+ * Formats as markdown-style link to preserve href for AI parsing
+ */
+function createLinkListItem(document, href, title, description) {
+    const li = document.createElement('li');
+    const link = document.createElement('a');
+    link.setAttribute('href', href);
+    link.textContent = title;
+    li.appendChild(link);
+    if (description && description !== title && !title.includes(description)) {
+        li.appendChild(document.createTextNode(` - ${description}`));
+    }
+    return li;
+}
+/**
+ * Process custom <card> elements (used by MDX-based docs)
+ */
+function processCustomCards(document) {
+    const customCards = document.querySelectorAll('card[href], card[title]');
+    if (customCards.length === 0)
+        return;
+    const list = document.createElement('ul');
+    list.setAttribute('data-preserved-cards', 'true');
+    for (const card of customCards) {
+        const href = card.getAttribute('href');
+        const title = card.getAttribute('title') ?? card.textContent.trim();
+        if (href && title) {
+            const desc = card.querySelector('p')?.textContent.trim();
+            list.appendChild(createLinkListItem(document, href, title, desc));
+        }
+    }
+    if (list.children.length > 0) {
+        const firstCard = customCards[0];
+        firstCard?.parentNode?.insertBefore(list, firstCard);
+        customCards.forEach((card) => {
+            card.remove();
+        });
+    }
+}
+/**
+ * Process CSS grid card containers
+ * Optimized to use more specific selectors to reduce iteration overhead
+ */
+function processCardGrids(document) {
+    // Use querySelectorAll on all divs but filter early with direct child selector
+    for (const div of document.querySelectorAll('div')) {
+        // Use :scope > a[href] for direct child links only (more efficient than Array.from + filter)
+        const childLinks = div.querySelectorAll(':scope > a[href]');
+        if (childLinks.length < 2)
+            continue;
+        const looksLikeCards = Array.from(childLinks).every((link) => {
+            const hasStructuredContent = link.querySelector('svg, div, p, span');
+            const hasReasonableText = link.textContent.trim().length > 3;
+            return hasStructuredContent && hasReasonableText;
+        });
+        if (!looksLikeCards)
+            continue;
+        const section = document.createElement('div');
+        section.setAttribute('data-preserved-cards', 'true');
+        const list = document.createElement('ul');
+        for (const link of childLinks) {
+            const href = link.getAttribute('href');
+            const title = extractCardTitle(link);
+            const desc = extractCardDescription(link);
+            if (href && title) {
+                list.appendChild(createLinkListItem(document, href, title, desc));
+            }
+        }
+        if (list.children.length > 0) {
+            section.appendChild(list);
+            div.parentNode?.replaceChild(section, div);
+        }
+    }
+}
+/**
+ * Process semantic card containers
+ */
+function processSemanticCards(document) {
+    const cardSelectors = [
+        '[class*="card-group"]',
+        '[class*="card-grid"]',
+        '[class*="cards"]',
+        '[data-cards]',
+        '[class*="link-card"]',
+        '[class*="feature-card"]',
+    ];
+    for (const selector of cardSelectors) {
+        try {
+            for (const container of document.querySelectorAll(selector)) {
+                const links = container.querySelectorAll('a[href]');
+                if (links.length === 0)
+                    continue;
+                const list = document.createElement('ul');
+                list.setAttribute('data-preserved-cards', 'true');
+                for (const link of links) {
+                    const href = link.getAttribute('href');
+                    const title = extractCardTitle(link);
+                    if (href && title) {
+                        list.appendChild(createLinkListItem(document, href, title));
+                    }
+                }
+                if (list.children.length > 0) {
+                    container.parentNode?.replaceChild(list, container);
+                }
+            }
+        }
+        catch {
+            // Selector might be invalid, skip it
+        }
+    }
+}
+/**
+ * Pre-process HTML to preserve card links that Readability might strip.
+ * Converts card-like elements into simple link lists.
+ */
+export function preserveCardLinks(document) {
+    processCustomCards(document);
+    processCardGrids(document);
+    processSemanticCards(document);
+}
+//# sourceMappingURL=card-extractor.js.map

package/dist/services/card-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"card-extractor.js","sourceRoot":"","sources":["../../src/services/card-extractor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,eAAe,GAAG,mDAAmD,CAAC;AAE5E;;GAEG;AACH,SAAS,YAAY,CAAC,OAAgB;IACpC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;IACjD,KAAK,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;QACrD,EAAE,CAAC,MAAM,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAa;IACrC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAEjC,kFAAkF;IAClF,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;QAChD,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC;YAAE,SAAS,CAAC,sBAAsB;QAE9D,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QACpC,IACE,IAAI,CAAC,MAAM,GAAG,CAAC;YACf,IAAI,CAAC,MAAM,GAAG,EAAE;YAChB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACxB,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EACtB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CACjC,0CAA0C,CAC3C,CAAC;IACF,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QACzC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,KAAK,CAAC;IAC3D,CAAC;IAED,6CAA6C;IAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IAEjE,4DAA4D;IAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAEzD,MAAM,SAAS,GAAG,IAAI;SACnB,KAAK,CAAC,OAAO,CAAC;SACd,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACjC,EAAE,IAAI,EAAE,CAAC;IACX,OAAO,SAAS,IAAI,IAAI,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,IAAa;IAC3C,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAChC,6CAA6C,CAC9C,CAAC;IACF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,IAAI,CAAC;IACxD,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,SAAS,GAAG,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1D,IAAI,SAAS,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC;IAE/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CACzB,QAAkB,EAClB,IAAY,EACZ,KAAa,EACb,WAA2B;IAE3B,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IACzB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAErB,IAAI,WAAW,IAAI,WAAW,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACzE,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,WAAW,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,QAAkB;IAC5C,MAAM,WAAW,GAAG,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC;IACzE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAErC,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,YAAY,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QAEpE,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC;YACzD,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QACjC,SAAS,EAAE,UAAU,EAAE,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACrD,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,QAAkB;IAC1C,+EAA+E;IAC/E,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,6FAA6F;QAC7F,MAAM,UAAU,GAAG,GAAG,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;QAE5D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAEpC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3D,MAAM,oBAAoB,GAAG,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;YACrE,MAAM,iBAAiB,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7D,OAAO,oBAAoB,IAAI,iBAAiB,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc;YAAE,SAAS;QAE9B,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC9C,OAAO,CAAC,YAAY,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAE1C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAE1C,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;gBAClB,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC1B,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,QAAkB;IAC9C,MAAM,aAAa,GAAG;QACpB,uBAAuB;QACvB,sBAAsB;QACtB,kBAAkB;QAClB,cAAc;QACd,sBAAsB;QACtB,yBAAyB;KAC1B,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;gBACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEjC,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;gBAElD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;oBACvC,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;oBAErC,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;wBAClB,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;oBAC9D,CAAC;gBACH,CAAC;gBAED,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7B,SAAS,CAAC,UAAU,EAAE,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB;IAClD,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC7B,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC3B,oBAAoB,CAAC,QAAQ,CAAC,CAAC;AACjC,CAAC"}

package/dist/services/extractor.d.ts

@@ -1,22 +1,15 @@
-import type { ExtractedArticle } from '../types/index.js';
-/** Metadata extracted from HTML document (internal) */
-interface ExtractedMetadata {
-    title?: string;
-    description?: string;
-    author?: string;
-}
-/** Combined extraction result (internal) */
-interface ExtractionResult {
-    article: ExtractedArticle | null;
-    metadata: ExtractedMetadata;
-}
+import type { CheerioAPI } from 'cheerio';
+import type { ExtractedMetadata, ExtractionResult } from '../config/types.js';
 /**
- * Extracts both article content and metadata from HTML in a single JSDOM parse.
- * This is more efficient than calling extractArticle and extractMetadata separately.
- * @param html - HTML string to extract content from
- * @param url - URL of the page (used for resolving relative links)
- * @returns Extraction result with article and metadata
+ * Extract metadata using Cheerio (fast, no full DOM)
+ * This avoids JSDOM overhead for simple meta tag extraction
  */
-export declare function extractContent(html: string, url: string): ExtractionResult;
-export {};
+export declare function extractMetadataWithCheerio($: CheerioAPI): ExtractedMetadata;
+/**
+ * Main extraction function - uses Cheerio for metadata (fast)
+ * and lazy-loads JSDOM only when article extraction is needed
+ */
+export declare function extractContent(html: string, url: string, options?: {
+    extractArticle?: boolean;
+}): ExtractionResult;
 //# sourceMappingURL=extractor.d.ts.map

package/dist/services/extractor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAM1D,uDAAuD;AACvD,UAAU,iBAAiB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,4CAA4C;AAC5C,UAAU,gBAAgB;IACxB,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AA8DD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,gBAAgB,CAwC1E"}
+{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C,OAAO,KAAK,EAEV,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAgB5B;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,CAAC,EAAE,UAAU,GAAG,iBAAiB,CA4B3E;AAsCD;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAA6B,GAC/D,gBAAgB,CAsClB"}

package/dist/services/extractor.js

@@ -1,65 +1,79 @@
-import { JSDOM } from 'jsdom';
+import * as cheerio from 'cheerio';
+import { JSDOM, VirtualConsole } from 'jsdom';
 import { Readability } from '@mozilla/readability';
+import { preserveCardLinks } from './card-extractor.js';
 import { logError, logWarn } from './logger.js';
-// Maximum HTML size to process (10MB)
 const MAX_HTML_SIZE = 10 * 1024 * 1024;
-function getMetaContent(document, selectors) {
-    for (const selector of selectors) {
-        const content = document.querySelector(selector)?.getAttribute('content');
-        if (content)
-            return content;
-    }
-    return undefined;
-}
+// Shared VirtualConsole to suppress JSDOM warnings/errors
+const sharedVirtualConsole = new VirtualConsole();
+sharedVirtualConsole.on('error', () => {
+    /* suppress JSDOM errors */
+});
+sharedVirtualConsole.on('warn', () => {
+    /* suppress JSDOM warnings */
+});
 /**
- * Extracts metadata from a pre-parsed Document
+ * Extract metadata using Cheerio (fast, no full DOM)
+ * This avoids JSDOM overhead for simple meta tag extraction
  */
-function extractMetadataFromDocument(document) {
-    const title = getMetaContent(document, [
+export function extractMetadataWithCheerio($) {
+    const getMetaContent = (selectors) => {
+        for (const selector of selectors) {
+            const content = $(selector).attr('content');
+            if (content)
+                return content;
+        }
+        return undefined;
+    };
+    const title = getMetaContent([
         'meta[property="og:title"]',
         'meta[name="twitter:title"]',
     ]) ??
-        document.querySelector('title')?.textContent ??
-        undefined;
-    const description = getMetaContent(document, [
+        ($('title').text() || undefined);
+    const description = getMetaContent([
         'meta[property="og:description"]',
         'meta[name="twitter:description"]',
         'meta[name="description"]',
     ]);
-    const author = getMetaContent(document, [
+    const author = getMetaContent([
         'meta[name="author"]',
         'meta[property="article:author"]',
     ]);
     return { title, description, author };
 }
 /**
- * Extracts article content from a pre-parsed Document using Readability
+ * Extract article content using JSDOM + Readability
+ * Only called when extractMainContent is true (lazy loading)
  */
-function extractArticleFromDocument(document) {
-    // Clone the document since Readability mutates it
-    const clonedDoc = document.cloneNode(true);
-    const reader = new Readability(clonedDoc);
-    const article = reader.parse();
-    if (!article)
+function extractArticleWithJsdom(html, url) {
+    try {
+        // Use shared VirtualConsole to reduce per-parse overhead
+        const dom = new JSDOM(html, { url, virtualConsole: sharedVirtualConsole });
+        const { document } = dom.window;
+        preserveCardLinks(document);
+        const reader = new Readability(document);
+        const article = reader.parse();
+        if (!article)
+            return null;
+        return {
+            title: article.title ?? undefined,
+            byline: article.byline ?? undefined,
+            content: article.content ?? '',
+            textContent: article.textContent ?? '',
+            excerpt: article.excerpt ?? undefined,
+            siteName: article.siteName ?? undefined,
+        };
+    }
+    catch (error) {
+        logError('Failed to extract article with JSDOM', error instanceof Error ? error : undefined);
         return null;
-    return {
-        title: article.title ?? undefined,
-        byline: article.byline ?? undefined,
-        content: article.content ?? '',
-        textContent: article.textContent ?? '',
-        excerpt: article.excerpt ?? undefined,
-        siteName: article.siteName ?? undefined,
-    };
+    }
 }
 /**
- * Extracts both article content and metadata from HTML in a single JSDOM parse.
- * This is more efficient than calling extractArticle and extractMetadata separately.
- * @param html - HTML string to extract content from
- * @param url - URL of the page (used for resolving relative links)
- * @returns Extraction result with article and metadata
+ * Main extraction function - uses Cheerio for metadata (fast)
+ * and lazy-loads JSDOM only when article extraction is needed
  */
-export function extractContent(html, url) {
-    // Input validation
+export function extractContent(html, url, options = { extractArticle: true }) {
     if (!html || typeof html !== 'string') {
         logWarn('extractContent called with invalid HTML input');
         return { article: null, metadata: {} };
@@ -68,7 +82,6 @@ export function extractContent(html, url) {
         logWarn('extractContent called with invalid URL');
         return { article: null, metadata: {} };
     }
-    // Size validation to prevent memory issues
     let processedHtml = html;
     if (html.length > MAX_HTML_SIZE) {
         logWarn('HTML content exceeds maximum size for extraction, truncating', {
@@ -78,12 +91,13 @@ export function extractContent(html, url) {
         processedHtml = html.substring(0, MAX_HTML_SIZE);
     }
     try {
-        const dom = new JSDOM(processedHtml, { url });
-        const document = dom.window.document;
-        // Extract metadata first (non-destructive)
-        const metadata = extractMetadataFromDocument(document);
-        // Extract article (uses cloned document since Readability mutates)
-        const article = extractArticleFromDocument(document);
+        // Fast path: Extract metadata with Cheerio (no full DOM parsing)
+        const $ = cheerio.load(processedHtml);
+        const metadata = extractMetadataWithCheerio($);
+        // Lazy path: Only use JSDOM when article extraction is requested
+        const article = options.extractArticle
+            ? extractArticleWithJsdom(processedHtml, url)
+            : null;
         return { article, metadata };
     }
     catch (error) {

package/dist/services/extractor.js.map

@@ -1 +1 @@
-{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEhD,sCAAsC;AACtC,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAevC,SAAS,cAAc,CACrB,QAAkB,EAClB,SAAmB;IAEnB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1E,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,QAAkB;IACrD,MAAM,KAAK,GACT,cAAc,CAAC,QAAQ,EAAE;QACvB,2BAA2B;QAC3B,4BAA4B;KAC7B,CAAC;QACF,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW;QAC5C,SAAS,CAAC;IAEZ,MAAM,WAAW,GAAG,cAAc,CAAC,QAAQ,EAAE;QAC3C,iCAAiC;QACjC,kCAAkC;QAClC,0BAA0B;KAC3B,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE;QACtC,qBAAqB;QACrB,iCAAiC;KAClC,CAAC,CAAC;IAEH,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AACxC,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CACjC,QAAkB;IAElB,kDAAkD;IAClD,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAa,CAAC;IACvD,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;IAE/B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,SAAS;QACjC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;QACnC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;QACtC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,SAAS;QACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,SAAS;KACxC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IACtD,mBAAmB;IACnB,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,CAAC,+CAA+C,CAAC,CAAC;QACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,2CAA2C;IAC3C,IAAI,aAAa,GAAG,IAAI,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QAChC,OAAO,CAAC,8DAA8D,EAAE;YACtE,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;QACH,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QAErC,2CAA2C;QAC3C,MAAM,QAAQ,GAAG,2BAA2B,CAAC,QAAQ,CAAC,CAAC;QAEvD,mEAAmE;QACnE,MAAM,OAAO,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;QAErD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;AACH,CAAC"}
+{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAE9C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAQnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,0DAA0D;AAC1D,MAAM,oBAAoB,GAAG,IAAI,cAAc,EAAE,CAAC;AAClD,oBAAoB,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;IACpC,2BAA2B;AAC7B,CAAC,CAAC,CAAC;AACH,oBAAoB,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;IACnC,6BAA6B;AAC/B,CAAC,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,0BAA0B,CAAC,CAAa;IACtD,MAAM,cAAc,GAAG,CAAC,SAAmB,EAAsB,EAAE;QACjE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5C,IAAI,OAAO;gBAAE,OAAO,OAAO,CAAC;QAC9B,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC;IAEF,MAAM,KAAK,GACT,cAAc,CAAC;QACb,2BAA2B;QAC3B,4BAA4B;KAC7B,CAAC;QACF,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,CAAC;IAEnC,MAAM,WAAW,GAAG,cAAc,CAAC;QACjC,iCAAiC;QACjC,kCAAkC;QAClC,0BAA0B;KAC3B,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,cAAc,CAAC;QAC5B,qBAAqB;QACrB,iCAAiC;KAClC,CAAC,CAAC;IAEH,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAC9B,IAAY,EACZ,GAAW;IAEX,IAAI,CAAC;QACH,yDAAyD;QACzD,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAC3E,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QAEhC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1B,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,SAAS;YACjC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;YACnC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;YACtC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,SAAS;YACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,SAAS;SACxC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,sCAAsC,EACtC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,GAAW,EACX,UAAwC,EAAE,cAAc,EAAE,IAAI,EAAE;IAEhE,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,CAAC,+CAA+C,CAAC,CAAC;QACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,aAAa,GAAG,IAAI,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QAChC,OAAO,CAAC,8DAA8D,EAAE;YACtE,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;QACH,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,CAAC;QACH,iEAAiE;QACjE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,0BAA0B,CAAC,CAAC,CAAC,CAAC;QAE/C,iEAAiE;QACjE,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc;YACpC,CAAC,CAAC,uBAAuB,CAAC,aAAa,EAAE,GAAG,CAAC;YAC7C,CAAC,CAAC,IAAI,CAAC;QAET,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;AACH,CAAC"}

package/dist/services/fetcher.d.ts

@@ -1,13 +1,15 @@
-/**
- * Destroys HTTP agents and closes all sockets
- * Should be called during graceful shutdown
- */
+/** Options for fetch operations */
+export interface FetchOptions {
+    /** Custom HTTP headers to include in the request */
+    customHeaders?: Record<string, string>;
+    /** AbortSignal for request cancellation */
+    signal?: AbortSignal;
+    /** Per-request timeout override in milliseconds */
+    timeout?: number;
+}
 export declare function destroyAgents(): void;
-/**
- * Fetches URL with exponential backoff retry logic
- * @param url - URL to fetch
- * @param customHeaders - Optional custom headers
- * @param maxRetries - Maximum retry attempts (1-10, defaults to 3)
- */
-export declare function fetchUrlWithRetry(url: string, customHeaders?: Record<string, string>, maxRetries?: number): Promise<string>;
+export declare function fetchUrlWithRetry(url: string, options?: FetchOptions, maxRetries?: number, skipCache?: boolean): Promise<{
+    html: string;
+    fromHtmlCache: boolean;
+}>;
 //# sourceMappingURL=fetcher.d.ts.map

package/dist/services/fetcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAyCA;;;GAGG;AACH,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AAqID;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACtC,UAAU,SAAI,GACb,OAAO,CAAC,MAAM,CAAC,CA8BjB"}
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAqBA,mCAAmC;AACnC,MAAM,WAAW,YAAY;IAC3B,oDAAoD;IACpD,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,2CAA2C;IAC3C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,mDAAmD;IACnD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AA6DD,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AA4LD,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,YAAY,EACtB,UAAU,SAAI,EACd,SAAS,UAAQ,GAChB,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,OAAO,CAAA;CAAE,CAAC,CAoEnD"}