@cosmocoder/mcp-web-docs 2.0.13 → 2.0.14

package/build/processor/content.js

@@ -1,287 +0,0 @@
-import { Readability } from '@mozilla/readability';
-import { JSDOM } from 'jsdom';
-import { logger } from '../util/logger.js';
-function cleanText(text) {
-    return text
-        .replace(/\s+/g, ' ')
-        .replace(/\n\s*\n/g, '\n\n')
-        .trim();
-}
-/**
- * Extract text content from HTML while preserving code blocks with markdown fences.
- * This ensures code examples aren't lost when using Readability fallback.
- */
-function extractTextWithCodeBlocks(html) {
-    const dom = new JSDOM(html);
-    const doc = dom.window.document;
-    // First, wrap all code blocks with markdown fences
-    const codeElements = doc.querySelectorAll('pre, code');
-    codeElements.forEach((el) => {
-        // Skip inline <code> inside <pre> (already handled by parent)
-        if (el.tagName === 'CODE' && el.parentElement?.tagName === 'PRE') {
-            return;
-        }
-        const code = el.textContent?.trim();
-        if (code && code.length > 0) {
-            // Replace the element's content with fenced code
-            const isBlock = el.tagName === 'PRE' || code.includes('\n');
-            if (isBlock) {
-                el.textContent = `\n\`\`\`\n${code}\n\`\`\`\n`;
-            }
-            else {
-                el.textContent = `\`${code}\``;
-            }
-        }
-    });
-    // Now get the text content which includes our fenced markers
-    return doc.body?.textContent || '';
-}
-function findMainContent(doc) {
-    const selectors = [
-        // Storybook specific selectors
-        '[class*="story-content"]',
-        '[class*="storybook-"]',
-        '[class*="docs-content"]',
-        '[class*="sbdocs-"]',
-        '[class*="docblock-"]',
-        // Jimdo UI specific selectors
-        '[class*="docs-"]',
-        '[class*="documentation"]',
-        '[class*="content"]',
-        '[class*="main"]',
-        // Common documentation selectors
-        'main',
-        '[role="main"]',
-        '#root',
-        '#app',
-        '#__next',
-        '#storybook-root',
-        '.documentation',
-        '.docs-content',
-        '.markdown-body',
-        'article',
-        '.article',
-        '.content',
-        '.page-content',
-        '.docusaurus-content',
-        '.vuepress-content',
-        '.gatsby-content',
-        '.mdx-content',
-        '.nextra-content',
-        '.nuxt-content',
-    ];
-    // Try each selector
-    for (const selector of selectors) {
-        const elements = Array.from(doc.querySelectorAll(selector));
-        if (elements.length > 0) {
-            // If multiple elements found, return the one with most content
-            return elements.reduce((best, current) => {
-                const bestLength = best.textContent?.length || 0;
-                const currentLength = current.textContent?.length || 0;
-                return currentLength > bestLength ? current : best;
-            });
-        }
-    }
-    // Fallback: try to find the element with the most content
-    const candidates = Array.from(doc.body.children);
-    if (candidates.length === 0) {
-        return null;
-    }
-    return candidates.reduce((best, current) => {
-        const bestLength = best.textContent?.length || 0;
-        const currentLength = current.textContent?.length || 0;
-        return currentLength > bestLength ? current : best;
-    });
-}
-function getAllTextContent(element) {
-    let content = '';
-    // Handle code blocks specially
-    if (element.tagName === 'PRE' || element.classList.contains('code')) {
-        const code = element.textContent?.trim();
-        if (code) {
-            return '\n```\n' + code + '\n```\n';
-        }
-        return '';
-    }
-    // Handle lists specially
-    if (element.tagName === 'UL' || element.tagName === 'OL') {
-        const items = Array.from(element.querySelectorAll('li'))
-            .map((li) => '- ' + li.textContent?.trim())
-            .filter(Boolean)
-            .join('\n');
-        if (items) {
-            return '\n' + items + '\n';
-        }
-        return '';
-    }
-    // Handle tables specially
-    if (element.tagName === 'TABLE') {
-        const rows = Array.from(element.querySelectorAll('tr'))
-            .map((tr) => Array.from(tr.querySelectorAll('td, th'))
-            .map((cell) => cell.textContent?.trim() || '')
-            .join(' | '))
-            .filter(Boolean)
-            .join('\n');
-        if (rows) {
-            return '\n' + rows + '\n';
-        }
-        return '';
-    }
-    // Skip unwanted elements
-    if (['SCRIPT', 'STYLE', 'NAV', 'HEADER', 'FOOTER'].includes(element.tagName)) {
-        return '';
-    }
-    // Get text content of this element
-    const text = element.textContent?.trim();
-    if (text) {
-        content += text + '\n';
-    }
-    return content;
-}
-function extractContentBetweenElements(start, end) {
-    let content = '';
-    let current = start;
-    // Process all elements between start and end
-    while (current && current !== end) {
-        content += getAllTextContent(current);
-        // Check children first (depth-first)
-        if (current.firstElementChild && current !== start) {
-            current = current.firstElementChild;
-        }
-        // Then try next sibling
-        else if (current.nextElementSibling) {
-            current = current.nextElementSibling;
-        }
-        // Finally try parent's next sibling
-        else {
-            let parent = current.parentElement;
-            while (parent && parent !== end && !parent.nextElementSibling) {
-                parent = parent.parentElement;
-            }
-            if (parent && parent !== end) {
-                current = parent.nextElementSibling;
-            }
-            else {
-                break;
-            }
-        }
-    }
-    return cleanText(content);
-}
-function extractSections(mainContent) {
-    const headerSelectors = [
-        'h1',
-        'h2',
-        'h3',
-        'h4',
-        '[class*="heading"]',
-        '[class*="title"]',
-        '[class*="sbdocs-h"]',
-        '[class*="story-title"]',
-        '[class*="docblock-title"]',
-        '[class*="docs-title"]',
-    ];
-    const headers = Array.from(mainContent.querySelectorAll(headerSelectors.join(','))).filter((header) => {
-        const text = header.textContent?.trim();
-        return text && text.length > 0;
-    });
-    if (headers.length === 0) {
-        // No headers found, treat entire content as one section
-        const title = mainContent.querySelector('h1, [class*="title"]')?.textContent?.trim() || 'Content';
-        const body = getAllTextContent(mainContent);
-        if (body.length > 0) {
-            return [{ title, body: cleanText(body) }];
-        }
-        return [];
-    }
-    const components = [];
-    // Process content before first header
-    if (headers[0].previousElementSibling) {
-        const introContent = extractContentBetweenElements(mainContent.firstElementChild, headers[0]);
-        if (introContent.length > 0) {
-            components.push({
-                title: 'Introduction',
-                body: introContent,
-            });
-        }
-    }
-    // Process sections between headers
-    headers.forEach((header, index) => {
-        const nextHeader = headers[index + 1];
-        const title = header.textContent?.trim() || '';
-        const body = extractContentBetweenElements(header, nextHeader);
-        if (body.length > 0) {
-            components.push({ title, body });
-        }
-    });
-    // Filter out empty components and normalize
-    return components
-        .filter((comp) => comp.body.length > 0)
-        .map((comp) => ({
-        title: comp.title,
-        body: cleanText(comp.body),
-    }));
-}
-export async function processHtmlContent(page) {
-    try {
-        logger.debug(`[ContentProcessor] Processing content for ${page.url}`);
-        const dom = new JSDOM(page.content);
-        const doc = dom.window.document;
-        // Try to find main content first
-        const mainContent = findMainContent(doc);
-        // If no main content found, use Readability
-        if (!mainContent) {
-            logger.debug('[ContentProcessor] No main content found, trying Readability');
-            const reader = new Readability(doc);
-            const readability = reader.parse();
-            if (!readability) {
-                logger.debug(`[ContentProcessor] No content could be extracted from ${page.url}`);
-                return undefined;
-            }
-            // Use HTML content to preserve code blocks with markdown fences
-            // This prevents code examples from triggering false positive security detections
-            const contentWithCodeBlocks = readability.content ? extractTextWithCodeBlocks(readability.content) : readability.textContent || '';
-            return {
-                article: {
-                    url: page.url,
-                    path: page.path,
-                    title: readability.title || page.path,
-                    components: [
-                        {
-                            title: readability.title || 'Content',
-                            body: cleanText(contentWithCodeBlocks),
-                        },
-                    ],
-                },
-                content: cleanText(contentWithCodeBlocks),
-            };
-        }
-        logger.debug('[ContentProcessor] Found main content, extracting sections');
-        // Extract sections from main content
-        const components = extractSections(mainContent);
-        if (components.length === 0) {
-            logger.debug(`[ContentProcessor] No valid content sections found in ${page.url}`);
-            return undefined;
-        }
-        logger.debug(`[ContentProcessor] Extracted ${components.length} sections`);
-        const article = {
-            url: page.url,
-            path: page.path,
-            title: page.title || components[0].title,
-            components,
-        };
-        return {
-            article,
-            content: components
-                .map((comp) => `${comp.title}\n\n${comp.body}`)
-                .join('\n\n')
-                .trim(),
-        };
-    }
-    catch (error) {
-        logger.debug('[ContentProcessor] Error processing HTML content:', error);
-        logger.debug('[ContentProcessor] Error details:', error instanceof Error ? error.stack : error);
-        return undefined;
-    }
-}
-//# sourceMappingURL=content.js.map

package/build/processor/content.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"content.js","sourceRoot":"","sources":["../../src/processor/content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAmB3C,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI;SACR,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC;SAC3B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,yBAAyB,CAAC,IAAY;IAC7C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;IAEhC,mDAAmD;IACnD,MAAM,YAAY,GAAG,GAAG,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IACvD,YAAY,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;QAC1B,8DAA8D;QAC9D,IAAI,EAAE,CAAC,OAAO,KAAK,MAAM,IAAI,EAAE,CAAC,aAAa,EAAE,OAAO,KAAK,KAAK,EAAE,CAAC;YACjE,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,iDAAiD;YACjD,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,OAAO,EAAE,CAAC;gBACZ,EAAE,CAAC,WAAW,GAAG,aAAa,IAAI,YAAY,CAAC;YACjD,CAAC;iBACI,CAAC;gBACJ,EAAE,CAAC,WAAW,GAAG,KAAK,IAAI,IAAI,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,6DAA6D;IAC7D,OAAO,GAAG,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,eAAe,CAAC,GAAa;IACpC,MAAM,SAAS,GAAG;QAChB,+BAA+B;QAC/B,0BAA0B;QAC1B,uBAAuB;QACvB,yBAAyB;QACzB,oBAAoB;QACpB,sBAAsB;QACtB,8BAA8B;QAC9B,kBAAkB;QAClB,0BAA0B;QAC1B,oBAAoB;QACpB,iBAAiB;QACjB,iCAAiC;QACjC,MAAM;QACN,eAAe;QACf,OAAO;QACP,MAAM;QACN,SAAS;QACT,iBAAiB;QACjB,gBAAgB;QAChB,eAAe;QACf,gBAAgB;QAChB,SAAS;QACT,UAAU;QACV,UAAU;QACV,eAAe;QACf,qBAAqB;QACrB,mBAAmB;QACnB,iBAAiB;QACjB,cAAc;QACd,iBAAiB;QACjB,eAAe;KAChB,CAAC;IAEF,oBAAoB;IACpB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,+DAA+D;YAC/D,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC;gBACjD,MAAM,aAAa,GAAG,OAAO,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC;gBACvD,OAAO,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YACrD,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,0DAA0D;IAC1D,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,OAAO,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC;QACvD,OAAO,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAgB;IACzC,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,+BAA+B;IAC/B,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,IAAI,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;QACzC,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,SAAS,GAAG,IAAI,GAAG,SAAS,CAAC;QACtC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;aACrD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;aAC1C,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,0BAA0B;IAC1B,IAAI,OAAO,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;aACpD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CACV,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aAC7C,IAAI,CAAC,KAAK,CAAC,CACf;aACA,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7E,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,mCAAmC;IACnC,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IACzC,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,6BAA6B,CAAC,KAAc,EAAE,GAAmB;IACxE,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,OAAO,GAAmB,KAAK,CAAC;IAEpC,6CAA6C;IAC7C,OAAO,OAAO,IAAI,OAAO,KAAK,GAAG,EAAE,CAAC;QAClC,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAEtC,qCAAqC;QACrC,IAAI,OAAO,CAAC,iBAAiB,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;YACnD,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACtC,CAAC;QACD,wBAAwB;aACnB,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;YACpC,OAAO,GAAG,OAAO,CAAC,kBAAkB,CAAC;QACvC,CAAC;QACD,oCAAoC;aAC/B,CAAC;YACJ,IAAI,MAAM,GAAmB,OAAO,CAAC,aAAa,CAAC;YACnD,OAAO,MAAM,IAAI,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBAC9D,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC;YAChC,CAAC;YACD,IAAI,MAAM,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC7B,OAAO,GAAG,MAAM,CAAC,kBAAkB,CAAC;YACtC,CAAC;iBACI,CAAC;gBACJ,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,eAAe,CAAC,WAAoB;IAC3C,MAAM,eAAe,GAAG;QACtB,IAAI;QACJ,IAAI;QACJ,IAAI;QACJ,IAAI;QACJ,oBAAoB;QACpB,kBAAkB;QAClB,qBAAqB;QACrB,wBAAwB;QACxB,2BAA2B;QAC3B,uBAAuB;KACxB,CAAC;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;QACpG,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;QACxC,OAAO,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,wDAAwD;QACxD,MAAM,KAAK,GAAG,WAAW,CAAC,aAAa,CAAC,sBAAsB,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;QAClG,MAAM,IAAI,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC5C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAuB,EAAE,CAAC;IAE1C,sCAAsC;IACtC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,sBAAsB,EAAE,CAAC;QACtC,MAAM,YAAY,GAAG,6BAA6B,CAAC,WAAW,CAAC,iBAA4B,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzG,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,cAAc;gBACrB,IAAI,EAAE,YAAY;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,6BAA6B,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAE/D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,4CAA4C;IAC5C,OAAO,UAAU;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACd,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;KAC3B,CAAC,CAAC,CAAC;AACR,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAiB;IACxD,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,6CAA6C,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAEtE,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QAEhC,iCAAiC;QACjC,MAAM,WAAW,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QAEzC,4CAA4C;QAC5C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;YAC7E,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;YAEnC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,CAAC,KAAK,CAAC,yDAAyD,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBAClF,OAAO,SAAS,CAAC;YACnB,CAAC;YAED,gEAAgE;YAChE,iFAAiF;YACjF,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,IAAI,EAAE,CAAC;YAEnI,OAAO;gBACL,OAAO,EAAE;oBACP,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,KAAK,EAAE,WAAW,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI;oBACrC,UAAU,EAAE;wBACV;4BACE,KAAK,EAAE,WAAW,CAAC,KAAK,IAAI,SAAS;4BACrC,IAAI,EAAE,SAAS,CAAC,qBAAqB,CAAC;yBACvC;qBACF;iBACF;gBACD,OAAO,EAAE,SAAS,CAAC,qBAAqB,CAAC;aAC1C,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAE3E,qCAAqC;QACrC,MAAM,UAAU,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QAEhD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,KAAK,CAAC,yDAAyD,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAClF,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,gCAAgC,UAAU,CAAC,MAAM,WAAW,CAAC,CAAC;QAE3E,MAAM,OAAO,GAAY;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK;YACxC,UAAU;SACX,CAAC;QAEF,OAAO;YACL,OAAO;YACP,OAAO,EAAE,UAAU;iBAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC9C,IAAI,CAAC,MAAM,CAAC;iBACZ,IAAI,EAAE;SACV,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,mDAAmD,EAAE,KAAK,CAAC,CAAC;QACzE,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAChG,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}

package/build/processor/content.test.d.ts

@@ -1 +0,0 @@
-export {};

package/build/processor/content.test.js

@@ -1,369 +0,0 @@
-import { processHtmlContent } from './content.js';
-describe('HTML Content Processor', () => {
-    describe('processHtmlContent', () => {
-        it('should process simple HTML content', async () => {
-            const page = {
-                url: 'https://example.com/docs/page',
-                path: '/docs/page',
-                title: 'Documentation Page',
-                content: `
-          <!DOCTYPE html>
-          <html>
-            <head><title>Page Title</title></head>
-            <body>
-              <main>
-                <h1>Welcome to the Documentation</h1>
-                <p>This is the introduction paragraph.</p>
-                <h2>Getting Started</h2>
-                <p>Here's how to get started.</p>
-              </main>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.article.url).toBe(page.url);
-            expect(result?.article.components.length).toBeGreaterThan(0);
-        });
-        it('should extract content from article tags', async () => {
-            const page = {
-                url: 'https://example.com/blog/post',
-                path: '/blog/post',
-                title: 'Blog Post',
-                content: `
-          <html>
-            <body>
-              <nav>Navigation items</nav>
-              <article>
-                <h1>Article Title</h1>
-                <p>Article content goes here.</p>
-                <h2>Section One</h2>
-                <p>More content.</p>
-              </article>
-              <footer>Footer content</footer>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).toContain('Article content');
-            // Should not include nav/footer content
-            expect(result?.content).not.toContain('Navigation items');
-        });
-        it('should handle documentation-specific selectors', async () => {
-            const page = {
-                url: 'https://example.com/docs',
-                path: '/docs',
-                title: 'Docs',
-                content: `
-          <html>
-            <body>
-              <div class="sidebar">Sidebar</div>
-              <div class="markdown-body">
-                <h1>Documentation</h1>
-                <p>Main documentation content.</p>
-              </div>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).toContain('Main documentation content');
-        });
-        it('should preserve code blocks', async () => {
-            const page = {
-                url: 'https://example.com/code',
-                path: '/code',
-                title: 'Code',
-                content: `
-          <html>
-            <body>
-              <main>
-                <h1>Code Examples</h1>
-                <pre><code>function example() {
-  return "Hello";
-}</code></pre>
-              </main>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).toContain('function example');
-        });
-        it('should handle lists properly', async () => {
-            const page = {
-                url: 'https://example.com/list',
-                path: '/list',
-                title: 'List',
-                content: `
-          <html>
-            <body>
-              <main>
-                <h1>Features</h1>
-                <ul>
-                  <li>Feature one</li>
-                  <li>Feature two</li>
-                  <li>Feature three</li>
-                </ul>
-              </main>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).toContain('Feature one');
-            expect(result?.content).toContain('Feature two');
-        });
-        it('should handle tables', async () => {
-            const page = {
-                url: 'https://example.com/table',
-                path: '/table',
-                title: 'Table',
-                content: `
-          <html>
-            <body>
-              <main>
-                <h1>API Reference</h1>
-                <table>
-                  <tr><th>Method</th><th>Description</th></tr>
-                  <tr><td>GET</td><td>Retrieve data</td></tr>
-                  <tr><td>POST</td><td>Create data</td></tr>
-                </table>
-              </main>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).toContain('GET');
-            expect(result?.content).toContain('POST');
-        });
-        it('should skip script and style tags', async () => {
-            const page = {
-                url: 'https://example.com/scripts',
-                path: '/scripts',
-                title: 'Scripts',
-                content: `
-          <html>
-            <head>
-              <style>.class { color: red; }</style>
-            </head>
-            <body>
-              <script>alert('hello');</script>
-              <main>
-                <h1>Content</h1>
-                <p>Actual content here.</p>
-              </main>
-              <script>console.log('end');</script>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).not.toContain('alert');
-            expect(result?.content).not.toContain('color: red');
-            expect(result?.content).toContain('Actual content');
-        });
-        it('should use Readability as fallback', async () => {
-            const page = {
-                url: 'https://example.com/blog',
-                path: '/blog',
-                title: 'Blog',
-                content: `
-          <html>
-            <head><title>Blog Post</title></head>
-            <body>
-              <div>
-                <p>This is a paragraph of content.</p>
-                <p>Another paragraph with more information.</p>
-                <p>Yet another paragraph to provide enough content for Readability.</p>
-                <p>Additional paragraph for proper content extraction.</p>
-                <p>Final paragraph of meaningful content.</p>
-              </div>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            // Should still extract something even without clear main content
-            expect(result).toBeDefined();
-        });
-        it('should return undefined for pages without extractable content', async () => {
-            const page = {
-                url: 'https://example.com/empty',
-                path: '/empty',
-                title: 'Empty',
-                content: `
-          <html>
-            <body>
-              <nav>Just navigation</nav>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            // May return undefined or minimal content depending on parser
-            if (result) {
-                expect(result.content.length).toBeLessThan(100);
-            }
-        });
-        it('should handle Storybook-specific classes', async () => {
-            const page = {
-                url: 'https://storybook.example.com/docs',
-                path: '/docs',
-                title: 'Storybook',
-                content: `
-          <html>
-            <body>
-              <div class="sbdocs-wrapper">
-                <div class="sbdocs-content">
-                  <h1 class="sbdocs-h1">Component Documentation</h1>
-                  <div class="docblock-description">
-                    <p>Description of the component.</p>
-                  </div>
-                </div>
-              </div>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).toContain('Component Documentation');
-        });
-        it('should handle React app root containers', async () => {
-            const page = {
-                url: 'https://example.com/react-app',
-                path: '/react-app',
-                title: 'React App',
-                content: `
-          <html>
-            <body>
-              <div id="root">
-                <div>
-                  <h1>React App Content</h1>
-                  <p>This is rendered by React.</p>
-                </div>
-              </div>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).toContain('React App Content');
-        });
-        it('should handle Next.js containers', async () => {
-            const page = {
-                url: 'https://example.com/nextjs',
-                path: '/nextjs',
-                title: 'Next.js',
-                content: `
-          <html>
-            <body>
-              <div id="__next">
-                <main>
-                  <h1>Next.js Page</h1>
-                  <p>Content from Next.js.</p>
-                </main>
-              </div>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.content).toContain('Next.js Page');
-        });
-        it('should handle multiple heading levels for section extraction', async () => {
-            const page = {
-                url: 'https://example.com/headings',
-                path: '/headings',
-                title: 'Headings',
-                content: `
-          <html>
-            <body>
-              <main>
-                <h1>Main Title</h1>
-                <p>Intro text.</p>
-                <h2>Section 1</h2>
-                <p>Section 1 content.</p>
-                <h3>Subsection 1.1</h3>
-                <p>Subsection content.</p>
-                <h2>Section 2</h2>
-                <p>Section 2 content.</p>
-                <h4>Deep Section</h4>
-                <p>Deep content.</p>
-              </main>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            expect(result?.article.components.length).toBeGreaterThan(3);
-        });
-        it('should handle malformed HTML gracefully', async () => {
-            const page = {
-                url: 'https://example.com/malformed',
-                path: '/malformed',
-                title: 'Malformed',
-                content: `
-          <html>
-            <body>
-              <div>
-                <p>Unclosed paragraph
-                <p>Another unclosed
-                <span>Nested <b>incorrectly</span></b>
-                <main>
-                  <h1>Still Works</h1>
-                  <p>Content here.</p>
-                </main>
-              </div>
-            </body>
-          </html>
-        `,
-            };
-            // Should not throw
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-        });
-        it('should clean text and remove extra whitespace', async () => {
-            const page = {
-                url: 'https://example.com/whitespace',
-                path: '/whitespace',
-                title: 'Whitespace',
-                content: `
-          <html>
-            <body>
-              <main>
-                <h1>Title</h1>
-                <p>Text    with    extra     spaces.</p>
-                <p>
-
-
-                Multiple newlines here.
-
-
-                </p>
-              </main>
-            </body>
-          </html>
-        `,
-            };
-            const result = await processHtmlContent(page);
-            expect(result).toBeDefined();
-            // Should normalize whitespace
-            expect(result?.content).not.toMatch(/\s{3,}/);
-        });
-    });
-});
-//# sourceMappingURL=content.test.js.map