@nanocollective/get-md 1.0.0

package/dist/optimizers/structure-enhancer.js

@@ -0,0 +1,92 @@
+// src/optimizers/structure-enhancer.ts
+import * as cheerio from "cheerio";
+/**
+ * Enhance HTML structure for better markdown conversion
+ * - Improve heading hierarchy
+ * - Clean up nested elements
+ * - Normalize structure
+ */
+export function enhanceStructure(html) {
+    const $ = cheerio.load(html);
+    // 1. Normalize heading hierarchy
+    normalizeHeadings($);
+    // 2. Unwrap unnecessary nested elements
+    unwrapRedundantElements($);
+    // 3. Convert divs with heading-like content to actual headings
+    convertPseudoHeadings($);
+    return $.html();
+}
+function normalizeHeadings($) {
+    // Ensure headings have proper hierarchy
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const headings = [];
+    $("h1, h2, h3, h4, h5, h6").each((_, el) => {
+        const $el = $(el);
+        const tagName = el.tagName?.toLowerCase() || "";
+        const level = parseInt(tagName.substring(1));
+        headings.push({ level, $el });
+    });
+    // Adjust heading levels if they skip
+    let lastLevel = 0;
+    headings.forEach(({ level, $el }) => {
+        if (level > lastLevel + 1) {
+            const newLevel = lastLevel + 1;
+            const newTag = `h${newLevel}`;
+            const html = $el.html();
+            if (html) {
+                $el.replaceWith($(`<${newTag}>${html}</${newTag}>`));
+            }
+            lastLevel = newLevel;
+        }
+        else {
+            lastLevel = level;
+        }
+    });
+}
+function unwrapRedundantElements($) {
+    // Remove redundant nested divs and spans
+    $("div > div:only-child, span > span:only-child").each((_, el) => {
+        const $el = $(el);
+        const html = $el.html();
+        if (html)
+            $el.replaceWith(html);
+    });
+    // Unwrap paragraphs that only contain another block element
+    $("p").each((_, el) => {
+        const $el = $(el);
+        const children = $el.children();
+        if (children.length === 1) {
+            const tagName = children.first().prop("tagName")?.toLowerCase();
+            if (tagName &&
+                ["div", "blockquote", "pre", "ul", "ol", "table"].includes(tagName)) {
+                const html = $el.html();
+                if (html)
+                    $el.replaceWith(html);
+            }
+        }
+    });
+}
+function convertPseudoHeadings($) {
+    // Convert divs/spans that look like headings into actual headings
+    $("div, span").each((_, el) => {
+        const $el = $(el);
+        const text = $el.text().trim();
+        // Skip if it has children that aren't just text
+        if ($el.children().length > 0)
+            return;
+        // Check if it looks like a heading (short, possibly styled)
+        const className = $el.attr("class") || "";
+        const style = $el.attr("style") || "";
+        if (text.length > 0 &&
+            text.length < 100 &&
+            (className.toLowerCase().includes("title") ||
+                className.toLowerCase().includes("heading") ||
+                style.includes("font-weight: bold") ||
+                style.includes("font-weight:bold"))) {
+            // Convert to h3 by default
+            const newHeading = cheerio.load(`<h3>${text}</h3>`)("h3");
+            $el.replaceWith(newHeading);
+        }
+    });
+}
+//# sourceMappingURL=structure-enhancer.js.map

package/dist/optimizers/structure-enhancer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structure-enhancer.js","sourceRoot":"","sources":["../../src/optimizers/structure-enhancer.ts"],"names":[],"mappings":"AAAA,uCAAuC;AAEvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,iCAAiC;IACjC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAErB,wCAAwC;IACxC,uBAAuB,CAAC,CAAC,CAAC,CAAC;IAE3B,+DAA+D;IAC/D,qBAAqB,CAAC,CAAC,CAAC,CAAC;IAEzB,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAqB;IAC9C,wCAAwC;IACxC,8DAA8D;IAC9D,MAAM,QAAQ,GAAmD,EAAE,CAAC;IAEpE,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACzC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,qCAAqC;IACrC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE;QAClC,IAAI,KAAK,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,EAAE,CAAC;gBACT,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,MAAM,IAAI,IAAI,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YACD,SAAS,GAAG,QAAQ,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,uBAAuB,CAAC,CAAqB;IACpD,yCAAyC;IACzC,CAAC,CAAC,8CAA8C,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC/D,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI;YAAE,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,4DAA4D;IAC5D,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACpB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAEhC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;YAChE,IACE,OAAO;gBACP,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,EACnE,CAAC;gBACD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,IAAI,IAAI;oBAAE,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAAC,CAAqB;IAClD,kEAAkE;IAClE,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC5B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAE/B,gDAAgD;QAChD,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO;QAEtC,4DAA4D;QAC5D,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAEtC,IACE,IAAI,CAAC,MAAM,GAAG,CAAC;YACf,IAAI,CAAC,MAAM,GAAG,GAAG;YACjB,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACxC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3C,KAAK,CAAC,QAAQ,CAAC,mBAAmB,CAAC;gBACnC,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,EACrC,CAAC;YACD,2BAA2B;YAC3B,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1D,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/parsers/markdown-parser.d.ts

@@ -0,0 +1,16 @@
+import type { MarkdownOptions, MarkdownResult } from "../types.js";
+export declare class MarkdownParser {
+    private turndown;
+    constructor();
+    convert(html: string, options?: MarkdownOptions): MarkdownResult;
+    private extractMainContent;
+    private filterContent;
+    private setupLLMRules;
+    private convertTableToMarkdown;
+    private applyCustomRules;
+    private addFrontmatter;
+    private calculateMarkdownStats;
+    private postProcess;
+    private normalizeOptions;
+}
+//# sourceMappingURL=markdown-parser.d.ts.map

package/dist/parsers/markdown-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/markdown-parser.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EAIf,MAAM,aAAa,CAAC;AAErB,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAkB;;IAuBlC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,cAAc;IA0FpE,OAAO,CAAC,kBAAkB;IA4B1B,OAAO,CAAC,aAAa;IA2BrB,OAAO,CAAC,aAAa;IAsErB,OAAO,CAAC,sBAAsB;IAiE9B,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,cAAc;IAoBtB,OAAO,CAAC,sBAAsB;IAsC9B,OAAO,CAAC,WAAW;IAwDnB,OAAO,CAAC,gBAAgB;CAgBzB"}

package/dist/parsers/markdown-parser.js

@@ -0,0 +1,369 @@
+// src/parsers/markdown-parser.ts
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+import { Readability } from "@mozilla/readability";
+import { JSDOM } from "jsdom";
+import * as cheerio from "cheerio";
+import { cleanHTML } from "../optimizers/html-cleaner.js";
+import { formatForLLM } from "../optimizers/llm-formatter.js";
+import { enhanceStructure } from "../optimizers/structure-enhancer.js";
+import { extractMetadata } from "../extractors/metadata-extractor.js";
+export class MarkdownParser {
+    turndown;
+    constructor() {
+        // Initialize Turndown with LLM-friendly settings
+        this.turndown = new TurndownService({
+            headingStyle: "atx", // Use # style headings
+            hr: "---", // Horizontal rule style
+            bulletListMarker: "-", // Use - for lists
+            codeBlockStyle: "fenced", // Use ``` for code blocks
+            fence: "```", // Fence marker
+            emDelimiter: "*", // Emphasis delimiter
+            strongDelimiter: "**", // Strong delimiter
+            linkStyle: "inlined", // Inline links
+            linkReferenceStyle: "full", // Full reference links
+        });
+        // Add GitHub Flavored Markdown support (tables, strikethrough, etc.)
+        this.turndown.use(gfm);
+        // Set up custom rules optimized for LLMs
+        this.setupLLMRules();
+    }
+    convert(html, options = {}) {
+        const startTime = Date.now();
+        const opts = this.normalizeOptions(options);
+        // Step 1: Extract main content using Readability
+        let contentHtml = html;
+        let metadata = {};
+        let readabilitySuccess = false;
+        if (opts.extractContent) {
+            try {
+                const extracted = this.extractMainContent(html, opts.baseUrl);
+                if (extracted) {
+                    contentHtml = extracted.content;
+                    metadata = extracted.metadata;
+                    readabilitySuccess = true;
+                }
+            }
+            catch {
+                // Fallback to raw HTML if Readability fails
+                console.warn("Readability extraction failed, using raw HTML");
+            }
+        }
+        // Step 2: Additional metadata extraction
+        const additionalMeta = extractMetadata(contentHtml, opts.baseUrl);
+        metadata = { ...additionalMeta, ...metadata };
+        // Step 3: Clean HTML (remove scripts, styles, ads, etc.)
+        contentHtml = cleanHTML(contentHtml, {
+            aggressive: opts.aggressiveCleanup,
+            baseUrl: opts.baseUrl,
+        });
+        // Step 4: Enhance structure (improve heading hierarchy, etc.)
+        contentHtml = enhanceStructure(contentHtml);
+        // Step 5: Filter content based on options
+        contentHtml = this.filterContent(contentHtml, opts);
+        // Step 6: Apply custom rules if provided
+        if (opts.customRules && opts.customRules.length > 0) {
+            this.applyCustomRules(opts.customRules);
+        }
+        // Step 7: Convert to markdown
+        let markdown = this.turndown.turndown(contentHtml);
+        // Step 8: Apply LLM-specific formatting
+        markdown = formatForLLM(markdown);
+        // Step 9: Calculate word count and reading time from markdown (before adding frontmatter)
+        const { wordCount, readingTime } = this.calculateMarkdownStats(markdown);
+        metadata.wordCount = wordCount;
+        metadata.readingTime = readingTime;
+        // Step 10: Add frontmatter if requested
+        if (opts.includeMeta && Object.keys(metadata).length > 0) {
+            markdown = this.addFrontmatter(markdown, metadata);
+        }
+        // Step 11: Final cleanup
+        markdown = this.postProcess(markdown);
+        // Step 12: Validate length
+        if (opts.maxLength && markdown.length > opts.maxLength) {
+            markdown =
+                markdown.substring(0, opts.maxLength) + "\n\n[Content truncated]";
+        }
+        const processingTime = Date.now() - startTime;
+        // Calculate statistics
+        const $ = cheerio.load(contentHtml);
+        const imageCount = opts.includeImages ? $("img").length : 0;
+        const linkCount = opts.includeLinks ? $("a").length : 0;
+        return {
+            markdown,
+            metadata,
+            stats: {
+                inputLength: html.length,
+                outputLength: markdown.length,
+                processingTime,
+                readabilitySuccess,
+                imageCount,
+                linkCount,
+            },
+        };
+    }
+    extractMainContent(html, baseUrl) {
+        const doc = new JSDOM(html, { url: baseUrl });
+        const reader = new Readability(doc.window.document, {
+            // Increase content threshold to avoid extracting navigation/sidebars
+            charThreshold: 500,
+        });
+        const article = reader.parse();
+        if (!article) {
+            return null;
+        }
+        return {
+            content: article.content || "",
+            metadata: {
+                title: article.title || undefined,
+                author: article.byline || undefined,
+                excerpt: article.excerpt || undefined,
+                siteName: article.siteName || undefined,
+                // wordCount and readingTime will be calculated from final markdown
+            },
+        };
+    }
+    filterContent(html, options) {
+        const $ = cheerio.load(html);
+        // Remove images if disabled
+        if (!options.includeImages) {
+            $("img, picture, figure").remove();
+        }
+        // Remove links if disabled (keep text content)
+        if (!options.includeLinks) {
+            $("a").each((_, el) => {
+                const $el = $(el);
+                $el.replaceWith($el.text());
+            });
+        }
+        // Remove tables if disabled
+        if (!options.includeTables) {
+            $("table").remove();
+        }
+        return $.html();
+    }
+    setupLLMRules() {
+        // Custom rule for better table formatting
+        this.turndown.addRule("tables", {
+            filter: "table",
+            replacement: (_content, node) => {
+                return this.convertTableToMarkdown(node);
+            },
+        });
+        // Custom rule for code blocks with language detection
+        this.turndown.addRule("codeBlocks", {
+            filter: (node) => {
+                return node.nodeName === "PRE" && node.querySelector?.("code") !== null;
+            },
+            replacement: (_content, node) => {
+                const code = node.querySelector?.("code");
+                if (!code)
+                    return "";
+                // Detect language from class name
+                const className = code.className || "";
+                const langMatch = className.match(/language-(\w+)|lang-(\w+)/);
+                const language = langMatch?.[1] || langMatch?.[2] || "";
+                const codeContent = code.textContent || "";
+                return `\n\`\`\`${language}\n${codeContent}\n\`\`\`\n`;
+            },
+        });
+        // Custom rule for better image handling with alt text
+        this.turndown.addRule("images", {
+            filter: "img",
+            replacement: (_content, node) => {
+                const alt = node.alt || "Image";
+                const src = node.src || node.getAttribute?.("data-src") || "";
+                const title = node.title || "";
+                if (!src)
+                    return "";
+                if (title) {
+                    return `![${alt}](${src} "${title}")`;
+                }
+                return `![${alt}](${src})`;
+            },
+        });
+        // Custom rule for blockquotes with better formatting
+        this.turndown.addRule("blockquotes", {
+            filter: "blockquote",
+            replacement: (_content, node) => {
+                const text = node.textContent || "";
+                const lines = text.trim().split("\n");
+                return ("\n" + lines.map((line) => `> ${line}`).join("\n") + "\n");
+            },
+        });
+        // Remove empty paragraphs and whitespace-only elements
+        this.turndown.addRule("removeEmpty", {
+            filter: (node) => {
+                return (["P", "DIV", "SPAN"].includes(node.nodeName.toUpperCase()) &&
+                    (!node.textContent || node.textContent.trim() === ""));
+            },
+            replacement: () => "",
+        });
+    }
+    convertTableToMarkdown(table) {
+        const $ = cheerio.load(table.outerHTML || "");
+        const headers = [];
+        const rows = [];
+        const alignments = [];
+        // Extract headers and alignments
+        $("thead tr, tr:first-child")
+            .first()
+            .find("th, td")
+            .each((_, el) => {
+            const $el = $(el);
+            headers.push($el.text().trim().replace(/\n/g, " "));
+            // Detect alignment from style or align attribute
+            const align = $el.attr("align") ||
+                ($el.css("text-align") === "center"
+                    ? "center"
+                    : $el.css("text-align") === "right"
+                        ? "right"
+                        : "left");
+            alignments.push(align);
+        });
+        if (headers.length === 0)
+            return "";
+        // Extract rows
+        const rowSelector = $("thead").length > 0 ? "tbody tr" : "tr:not(:first-child)";
+        $(rowSelector).each((_, tr) => {
+            const row = [];
+            $(tr)
+                .find("td")
+                .each((_, td) => {
+                row.push($(td).text().trim().replace(/\n/g, " "));
+            });
+            if (row.length > 0) {
+                // Ensure row has same number of columns as headers
+                while (row.length < headers.length) {
+                    row.push("");
+                }
+                rows.push(row.slice(0, headers.length));
+            }
+        });
+        // Build markdown table
+        let markdown = "\n| " + headers.join(" | ") + " |\n";
+        // Add alignment row
+        const alignRow = alignments.map((align) => {
+            if (align === "center")
+                return ":---:";
+            if (align === "right")
+                return "---:";
+            return "---";
+        });
+        markdown += "| " + alignRow.join(" | ") + " |\n";
+        // Add data rows
+        rows.forEach((row) => {
+            markdown += "| " + row.join(" | ") + " |\n";
+        });
+        return markdown + "\n";
+    }
+    applyCustomRules(rules) {
+        rules.forEach((rule) => {
+            this.turndown.addRule(rule.name, {
+                filter: rule.filter,
+                replacement: rule.replacement,
+            });
+        });
+    }
+    addFrontmatter(markdown, metadata) {
+        // Build YAML frontmatter
+        const yaml = ["---"];
+        Object.entries(metadata).forEach(([key, value]) => {
+            if (value !== undefined && value !== null) {
+                // Escape string values with quotes if they contain special chars
+                const yamlValue = typeof value === "string" && /[:\n\r]/.test(value)
+                    ? `"${value.replace(/"/g, '\\"')}"`
+                    : value;
+                yaml.push(`${key}: ${String(yamlValue)}`);
+            }
+        });
+        yaml.push("---");
+        return yaml.join("\n") + "\n\n" + markdown;
+    }
+    calculateMarkdownStats(markdown) {
+        // Remove frontmatter if present
+        let contentOnly = markdown;
+        if (markdown.startsWith("---")) {
+            const frontmatterEnd = markdown.indexOf("---", 3);
+            if (frontmatterEnd !== -1) {
+                contentOnly = markdown.substring(frontmatterEnd + 3).trim();
+            }
+        }
+        // Count words in the actual content (excluding code blocks and URLs)
+        // Remove code blocks first
+        contentOnly = contentOnly.replace(/```[\s\S]*?```/g, "");
+        // Remove inline code
+        contentOnly = contentOnly.replace(/`[^`]+`/g, "");
+        // Remove URLs
+        contentOnly = contentOnly.replace(/https?:\/\/[^\s)]+/g, "");
+        // Remove markdown link syntax but keep the text
+        contentOnly = contentOnly.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
+        // Remove image syntax
+        contentOnly = contentOnly.replace(/!\[([^\]]*)\]\([^)]+\)/g, "");
+        // Count words
+        const words = contentOnly
+            .trim()
+            .split(/\s+/)
+            .filter((w) => w.length > 0);
+        const wordCount = words.length;
+        // Calculate reading time (250 words per minute is more realistic)
+        const readingTime = Math.ceil(wordCount / 250);
+        return { wordCount, readingTime };
+    }
+    postProcess(markdown) {
+        // Remove excessive blank lines (more than 2 consecutive)
+        markdown = markdown.replace(/\n{3,}/g, "\n\n");
+        // Clean up list formatting (remove blank lines within lists)
+        markdown = markdown.replace(/^(-|\d+\.)\s+(.+?)(\n\n)(-|\d+\.)/gm, "$1 $2\n$4");
+        // Ensure code blocks have spacing
+        markdown = markdown.replace(/([^\n])\n```/g, "$1\n\n```");
+        markdown = markdown.replace(/```\n([^`\n])/g, "```\n\n$1");
+        // Fix heading spacing (ensure blank line before headings, but not right after frontmatter)
+        markdown = markdown.replace(/([^\n-])\n(#{1,6}\s)/g, "$1\n\n$2");
+        // Ensure consistent spacing around horizontal rules (but not frontmatter delimiters)
+        // Only add spacing to --- that are not at the start and not part of frontmatter
+        const lines = markdown.split("\n");
+        let frontmatterCount = 0;
+        for (let i = 0; i < lines.length; i++) {
+            if (lines[i].trim() === "---") {
+                frontmatterCount++;
+                if (frontmatterCount <= 2) {
+                    continue;
+                }
+            }
+            // After frontmatter is closed, add spacing around horizontal rules
+            if (frontmatterCount >= 2 &&
+                lines[i].trim() === "---" &&
+                i > 0 &&
+                i < lines.length - 1) {
+                if (lines[i - 1].trim() !== "" && !lines[i - 1].startsWith("#")) {
+                    lines.splice(i, 0, "");
+                    i++;
+                }
+                if (lines[i + 1].trim() !== "" && !lines[i + 1].startsWith("#")) {
+                    lines.splice(i + 1, 0, "");
+                    i++;
+                }
+            }
+        }
+        markdown = lines.join("\n");
+        // Remove trailing whitespace from lines
+        markdown = markdown.replace(/[^\S\n]+$/gm, "");
+        return markdown.trim() + "\n";
+    }
+    normalizeOptions(options) {
+        return {
+            extractContent: options.extractContent ?? true,
+            includeMeta: options.includeMeta ?? true,
+            customRules: options.customRules ?? [],
+            preserveElements: options.preserveElements ?? [],
+            maxLength: options.maxLength ?? 1000000,
+            baseUrl: options.baseUrl,
+            includeImages: options.includeImages ?? true,
+            includeLinks: options.includeLinks ?? true,
+            includeTables: options.includeTables ?? true,
+            aggressiveCleanup: options.aggressiveCleanup ?? true, // Back to true with smarter selectors
+        };
+    }
+}
+//# sourceMappingURL=markdown-parser.js.map

package/dist/parsers/markdown-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown-parser.js","sourceRoot":"","sources":["../../src/parsers/markdown-parser.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qCAAqC,CAAC;AAStE,MAAM,OAAO,cAAc;IACjB,QAAQ,CAAkB;IAElC;QACE,iDAAiD;QACjD,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC;YAClC,YAAY,EAAE,KAAK,EAAE,uBAAuB;YAC5C,EAAE,EAAE,KAAK,EAAE,wBAAwB;YACnC,gBAAgB,EAAE,GAAG,EAAE,kBAAkB;YACzC,cAAc,EAAE,QAAQ,EAAE,0BAA0B;YACpD,KAAK,EAAE,KAAK,EAAE,eAAe;YAC7B,WAAW,EAAE,GAAG,EAAE,qBAAqB;YACvC,eAAe,EAAE,IAAI,EAAE,mBAAmB;YAC1C,SAAS,EAAE,SAAS,EAAE,eAAe;YACrC,kBAAkB,EAAE,MAAM,EAAE,uBAAuB;SACpD,CAAC,CAAC;QAEH,qEAAqE;QACrE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEvB,yCAAyC;QACzC,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,UAA2B,EAAE;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAE5C,iDAAiD;QACjD,IAAI,WAAW,GAAG,IAAI,CAAC;QACvB,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAE/B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC9D,IAAI,SAAS,EAAE,CAAC;oBACd,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC;oBAChC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;oBAC9B,kBAAkB,GAAG,IAAI,CAAC;gBAC5B,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,4CAA4C;gBAC5C,OAAO,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,yCAAyC;QACzC,MAAM,cAAc,GAAG,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClE,QAAQ,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,QAAQ,EAAE,CAAC;QAE9C,yDAAyD;QACzD,WAAW,GAAG,SAAS,CAAC,WAAW,EAAE;YACnC,UAAU,EAAE,IAAI,CAAC,iBAAiB;YAClC,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,8DAA8D;QAC9D,WAAW,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAE5C,0CAA0C;QAC1C,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAEpD,yCAAyC;QACzC,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC1C,CAAC;QAED,8BAA8B;QAC9B,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAEnD,wCAAwC;QACxC,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QAElC,0FAA0F;QAC1F,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QACzE,QAAQ,CAAC,SAAS,GAAG,SAAS,CAAC;QAC/B,QAAQ,CAAC,WAAW,GAAG,WAAW,CAAC;QAEnC,wCAAwC;QACxC,IAAI,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzD,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACrD,CAAC;QAED,yBAAyB;QACzB,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QAEtC,2BAA2B;QAC3B,IAAI,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YACvD,QAAQ;gBACN,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,yBAAyB,CAAC;QACtE,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAE9C,uBAAuB;QACvB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAExD,OAAO;YACL,QAAQ;YACR,QAAQ;YACR,KAAK,EAAE;gBACL,WAAW,EAAE,IAAI,CAAC,MAAM;gBACxB,YAAY,EAAE,QAAQ,CAAC,MAAM;gBAC7B,cAAc;gBACd,kBAAkB;gBAClB,UAAU;gBACV,SAAS;aACV;SACF,CAAC;IACJ,CAAC;IAEO,kBAAkB,CACxB,IAAY,EACZ,OAAgB;QAEhB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE;YAClD,qEAAqE;YACrE,aAAa,EAAE,GAAG;SACnB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;YAC9B,QAAQ,EAAE;gBACR,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,SAAS;gBACjC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;gBACnC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,SAAS;gBACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,SAAS;gBACvC,mEAAmE;aACpE;SACF,CAAC;IACJ,CAAC;IAEO,aAAa,CACnB,IAAY,EACZ,OAAkC;QAElC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,4BAA4B;QAC5B,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;YAC3B,CAAC,CAAC,sBAAsB,CAAC,CAAC,MAAM,EAAE,CAAC;QACrC,CAAC;QAED,+CAA+C;QAC/C,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;YAC1B,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBACpB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;gBAClB,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;YAC3B,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACtB,CAAC;QAED,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAClB,CAAC;IAEO,aAAa;QACnB,0CAA0C;QAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE;YAC9B,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;gBAC9B,OAAO,IAAI,CAAC,sBAAsB,CAAC,IAAoB,CAAC,CAAC;YAC3D,CAAC;SACF,CAAC,CAAC;QAEH,sDAAsD;QACtD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE;YAClC,MAAM,EAAE,CAAC,IAAkB,EAAE,EAAE;gBAC7B,OAAO,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC;YAC1E,CAAC;YACD,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAkB,EAAE,EAAE;gBAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,CAAC,IAAI;oBAAE,OAAO,EAAE,CAAC;gBAErB,kCAAkC;gBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;gBACvC,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;gBAC/D,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAExD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;gBAE3C,OAAO,WAAW,QAAQ,KAAK,WAAW,YAAY,CAAC;YACzD,CAAC;SACF,CAAC,CAAC;QAEH,sDAAsD;QACtD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE;YAC9B,MAAM,EAAE,KAAK;YACb,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAkB,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC;gBAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBAE/B,IAAI,CAAC,GAAG;oBAAE,OAAO,EAAE,CAAC;gBAEpB,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,KAAK,GAAG,KAAK,GAAG,KAAK,KAAK,IAAI,CAAC;gBACxC,CAAC;gBACD,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;YAC7B,CAAC;SACF,CAAC,CAAC;QAEH,qDAAqD;QACrD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAE;YACnC,MAAM,EAAE,YAAY;YACpB,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAkB,EAAE,EAAE;gBAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACtC,OAAO,CACL,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAClE,CAAC;YACJ,CAAC;SACF,CAAC,CAAC;QAEH,uDAAuD;QACvD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAE;YACnC,MAAM,EAAE,CAAC,IAAkB,EAAE,EAAE;gBAC7B,OAAO,CACL,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC1D,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CACtD,CAAC;YACJ,CAAC;YACD,WAAW,EAAE,GAAG,EAAE,CAAC,EAAE;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,sBAAsB,CAAC,KAAmB;QAChD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,iCAAiC;QACjC,CAAC,CAAC,0BAA0B,CAAC;aAC1B,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACd,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAEpD,iDAAiD;YACjD,MAAM,KAAK,GACT,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC;gBACjB,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,QAAQ;oBACjC,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,OAAO;wBACjC,CAAC,CAAC,OAAO;wBACT,CAAC,CAAC,MAAM,CAAC,CAAC;YAChB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEL,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,eAAe;QACf,MAAM,WAAW,GACf,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,sBAAsB,CAAC;QAC9D,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC5B,MAAM,GAAG,GAAa,EAAE,CAAC;YACzB,CAAC,CAAC,EAAE,CAAC;iBACF,IAAI,CAAC,IAAI,CAAC;iBACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YACL,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnB,mDAAmD;gBACnD,OAAO,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACf,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,IAAI,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;QAErD,oBAAoB;QACpB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACxC,IAAI,KAAK,KAAK,QAAQ;gBAAE,OAAO,OAAO,CAAC;YACvC,IAAI,KAAK,KAAK,OAAO;gBAAE,OAAO,MAAM,CAAC;YACrC,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QACH,QAAQ,IAAI,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;QAEjD,gBAAgB;QAChB,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACnB,QAAQ,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAEO,gBAAgB,CAAC,KAAqB;QAC5C,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE;gBAC/B,MAAM,EAAE,IAAI,CAAC,MAA2C;gBACxD,WAAW,EAAE,IAAI,CAAC,WAAkD;aACrE,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,cAAc,CAAC,QAAgB,EAAE,QAAyB;QAChE,yBAAyB;QACzB,MAAM,IAAI,GAAa,CAAC,KAAK,CAAC,CAAC;QAE/B,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAoB,EAAE,EAAE;YACnE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAC1C,iEAAiE;gBACjE,MAAM,SAAS,GACb,OAAO,KAAK,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC;oBAChD,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG;oBACnC,CAAC,CAAE,KAAyB,CAAC;gBACjC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEjB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC7C,CAAC;IAEO,sBAAsB,CAAC,QAAgB;QAI7C,gCAAgC;QAChC,IAAI,WAAW,GAAG,QAAQ,CAAC;QAC3B,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAClD,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC1B,WAAW,GAAG,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,qEAAqE;QACrE,2BAA2B;QAC3B,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACzD,qBAAqB;QACrB,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAClD,cAAc;QACd,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAC7D,gDAAgD;QAChD,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC;QAClE,sBAAsB;QACtB,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,yBAAyB,EAAE,EAAE,CAAC,CAAC;QAEjE,cAAc;QACd,MAAM,KAAK,GAAG,WAAW;aACtB,IAAI,EAAE;aACN,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;QAE/B,kEAAkE;QAClE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;QAE/C,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;IACpC,CAAC;IAEO,WAAW,CAAC,QAAgB;QAClC,yDAAyD;QACzD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE/C,6DAA6D;QAC7D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CACzB,qCAAqC,EACrC,WAAW,CACZ,CAAC;QAEF,kCAAkC;QAClC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;QAC1D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;QAE3D,2FAA2F;QAC3F,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC;QAEjE,qFAAqF;QACrF,gFAAgF;QAChF,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;gBAC9B,gBAAgB,EAAE,CAAC;gBACnB,IAAI,gBAAgB,IAAI,CAAC,EAAE,CAAC;oBAC1B,SAAS;gBACX,CAAC;YACH,CAAC;YAED,mEAAmE;YACnE,IACE,gBAAgB,IAAI,CAAC;gBACrB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,KAAK;gBACzB,CAAC,GAAG,CAAC;gBACL,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EACpB,CAAC;gBACD,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;oBACvB,CAAC,EAAE,CAAC;gBACN,CAAC;gBACD,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3B,CAAC,EAAE,CAAC;gBACN,CAAC;YACH,CAAC;QACH,CAAC;QAED,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE5B,wCAAwC;QACxC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAE/C,OAAO,QAAQ,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;IAChC,CAAC;IAEO,gBAAgB,CACtB,OAAwB;QAExB,OAAO;YACL,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI;YAC9C,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI;YACxC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;YACtC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,EAAE;YAChD,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,OAAO;YACvC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;YAC5C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI;YAC1C,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;YAC5C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,EAAE,sCAAsC;SAChE,CAAC;IACjC,CAAC;CACF"}

package/dist/types.d.ts

@@ -0,0 +1,115 @@
+/** Options for HTML to Markdown conversion */
+export interface MarkdownOptions {
+    /** Extract only main content using Readability (default: true) */
+    extractContent?: boolean;
+    /** Include metadata as YAML frontmatter (default: true) */
+    includeMeta?: boolean;
+    /** Custom Turndown rules */
+    customRules?: TurndownRule[];
+    /** Preserve specific HTML elements as HTML */
+    preserveElements?: string[];
+    /** Maximum content length in characters (default: 1000000) */
+    maxLength?: number;
+    /** Base URL for resolving relative links */
+    baseUrl?: string;
+    /** Include images (default: true) */
+    includeImages?: boolean;
+    /** Include links (default: true) */
+    includeLinks?: boolean;
+    /** Include tables (default: true) */
+    includeTables?: boolean;
+    /** Aggressive noise removal (default: true) */
+    aggressiveCleanup?: boolean;
+    /** Force treat input as URL (bypasses auto-detection) */
+    isUrl?: boolean;
+    /** Request timeout in milliseconds (default: 15000) */
+    timeout?: number;
+    /** Follow redirects (default: true) */
+    followRedirects?: boolean;
+    /** Maximum redirects to follow (default: 5) */
+    maxRedirects?: number;
+    /** Custom headers for URL fetching */
+    headers?: Record<string, string>;
+    /** User agent string for URL fetching */
+    userAgent?: string;
+}
+/** Result from markdown conversion */
+export interface MarkdownResult {
+    /** Generated markdown content */
+    markdown: string;
+    /** Extracted metadata */
+    metadata: ContentMetadata;
+    /** Conversion statistics */
+    stats: ConversionStats;
+}
+/** Extracted content metadata */
+export interface ContentMetadata {
+    /** Page title */
+    title?: string;
+    /** Author/byline */
+    author?: string;
+    /** Excerpt/description */
+    excerpt?: string;
+    /** Site name */
+    siteName?: string;
+    /** Publication date */
+    publishedTime?: string;
+    /** Language code */
+    language?: string;
+    /** Canonical URL */
+    canonicalUrl?: string;
+    /** Reading time estimate (minutes) */
+    readingTime?: number;
+    /** Word count */
+    wordCount?: number;
+}
+/** Conversion statistics */
+export interface ConversionStats {
+    /** Input HTML length */
+    inputLength: number;
+    /** Output markdown length */
+    outputLength: number;
+    /** Processing time in milliseconds */
+    processingTime: number;
+    /** Whether Readability extraction succeeded */
+    readabilitySuccess: boolean;
+    /** Number of images found */
+    imageCount: number;
+    /** Number of links found */
+    linkCount: number;
+}
+/** Options for URL fetching */
+export interface FetchOptions {
+    /** Request timeout in milliseconds (default: 15000) */
+    timeout?: number;
+    /** Follow redirects (default: true) */
+    followRedirects?: boolean;
+    /** Maximum redirects to follow (default: 5) */
+    maxRedirects?: number;
+    /** Custom headers */
+    headers?: Record<string, string>;
+    /** User agent string */
+    userAgent?: string;
+}
+/** Custom Turndown rule */
+export interface TurndownRule {
+    /** Rule name */
+    name: string;
+    /** Filter for elements to apply rule to */
+    filter: string | string[] | ((node: TurndownNode) => boolean);
+    /** Replacement function */
+    replacement: (content: string, node: TurndownNode) => string;
+}
+/** Turndown DOM node type */
+export interface TurndownNode {
+    nodeName: string;
+    textContent?: string | null;
+    outerHTML?: string;
+    className?: string;
+    alt?: string;
+    src?: string;
+    title?: string;
+    querySelector?: (selector: string) => TurndownNode | null;
+    getAttribute?: (name: string) => string | null;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,8CAA8C;AAC9C,MAAM,WAAW,eAAe;IAC9B,kEAAkE;IAClE,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB,2DAA2D;IAC3D,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,4BAA4B;IAC5B,WAAW,CAAC,EAAE,YAAY,EAAE,CAAC;IAE7B,8CAA8C;IAC9C,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE5B,8DAA8D;IAC9D,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,qCAAqC;IACrC,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,oCAAoC;IACpC,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB,qCAAqC;IACrC,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,+CAA+C;IAC/C,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,yDAAyD;IACzD,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uCAAuC;IACvC,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,+CAA+C;IAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,sCAAsC;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,sCAAsC;AACtC,MAAM,WAAW,cAAc;IAC7B,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAC;IAEjB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAE1B,4BAA4B;IAC5B,KAAK,EAAE,eAAe,CAAC;CACxB;AAED,iCAAiC;AACjC,MAAM,WAAW,eAAe;IAC9B,iBAAiB;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,oBAAoB;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,gBAAgB;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,uBAAuB;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,oBAAoB;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oBAAoB;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,sCAAsC;IACtC,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,iBAAiB;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,4BAA4B;AAC5B,MAAM,WAAW,eAAe;IAC9B,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IAEpB,6BAA6B;IAC7B,YAAY,EAAE,MAAM,CAAC;IAErB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IAEvB,+CAA+C;IAC/C,kBAAkB,EAAE,OAAO,CAAC;IAE5B,6BAA6B;IAC7B,UAAU,EAAE,MAAM,CAAC;IAEnB,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,+BAA+B;AAC/B,MAAM,WAAW,YAAY;IAC3B,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uCAAuC;IACvC,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,+CAA+C;IAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,qBAAqB;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,2BAA2B;AAC3B,MAAM,WAAW,YAAY;IAC3B,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IAEb,2CAA2C;IAC3C,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,YAAY,KAAK,OAAO,CAAC,CAAC;IAE9D,2BAA2B;IAC3B,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,KAAK,MAAM,CAAC;CAC9D;AAED,6BAA6B;AAC7B,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,YAAY,GAAG,IAAI,CAAC;IAC1D,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;CAChD"}

package/dist/types.js

@@ -0,0 +1,3 @@
+// src/types.ts
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,eAAe"}

package/dist/utils/url-fetcher.d.ts

@@ -0,0 +1,10 @@
+import type { FetchOptions } from "../types.js";
+/**
+ * Fetch HTML from a URL with timeout and redirect handling
+ */
+export declare function fetchUrl(url: string, options?: FetchOptions): Promise<string>;
+/**
+ * Validate if a string is a valid URL
+ */
+export declare function isValidUrl(urlString: string): boolean;
+//# sourceMappingURL=url-fetcher.d.ts.map

package/dist/utils/url-fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/url-fetcher.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAGhD;;GAEG;AACH,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,MAAM,CAAC,CAwCjB;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAOrD"}