html-to-markdown 3.4.0 → 3.6.0.pre.rc.24

data/vendor/html-to-markdown-rs/src/wrapper.rs

@@ -1,9 +0,0 @@
-//! Text wrapping functionality for Markdown output.
-//!
-//! This module provides text wrapping capabilities similar to Python's `textwrap.fill()`,
-//! specifically designed to work with Markdown content while preserving formatting.
-
-pub use sync::wrap_markdown;
-
-mod sync;
-mod utils;

data/vendor/html-to-markdown-rs/tests/br_in_inline_test.rs

@@ -1,87 +0,0 @@
-#![allow(missing_docs)]
-
-fn convert(
-    html: &str,
-    opts: Option<html_to_markdown_rs::ConversionOptions>,
-) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
-}
-
-use html_to_markdown_rs::ConversionOptions;
-
-#[test]
-fn test_br_inside_bold_tags() {
-    let html = "<b>Hello!<br/></b><b>Hola!</b>";
-    let result = convert(html, None).unwrap();
-
-    assert!(result.contains("**Hello!**  \n**Hola!**"));
-    assert!(!result.contains("**Hello!****Hola!**"));
-}
-
-#[test]
-fn test_br_inside_strong_tags() {
-    let html = "<strong>First<br/></strong><strong>Second</strong>";
-    let result = convert(html, None).unwrap();
-
-    assert!(result.contains("**First**  \n**Second**"));
-    assert!(!result.contains("**First****Second**"));
-}
-
-#[test]
-fn test_multiple_bolds_with_br() {
-    let html = "<b>Line 1<br/></b><b>Line 2<br/></b><b>Line 3</b>";
-    let result = convert(html, None).unwrap();
-
-    assert!(result.matches("**").count() >= 6);
-    assert!(result.contains("**Line 1**"));
-    assert!(result.contains("**Line 2**"));
-    assert!(result.contains("**Line 3**"));
-    assert!(result.contains("**Line 1**  \n**Line 2**"));
-}
-
-#[test]
-fn test_br_inside_em_tags() {
-    let html = "<em>First<br/></em><em>Second</em>";
-    let result = convert(html, None).unwrap();
-
-    assert!(result.contains("*First*  \n*Second*"));
-    assert!(!result.contains("*First**Second*"));
-}
-
-#[test]
-fn test_br_inside_italic_tags() {
-    let html = "<i>Alpha<br/></i><i>Beta</i>";
-    let result = convert(html, None).unwrap();
-
-    assert!(result.contains("*Alpha*  \n*Beta*"));
-    assert!(!result.contains("*Alpha**Beta*"));
-}
-
-#[test]
-fn test_br_with_backslash_style() {
-    let html = "<b>Hello<br/></b><b>World</b>";
-    let options = ConversionOptions {
-        newline_style: html_to_markdown_rs::NewlineStyle::Backslash,
-        ..Default::default()
-    };
-    let result = convert(html, Some(options)).unwrap();
-
-    assert!(result.contains("**Hello**\\\n**World**"));
-    assert!(!result.contains("**Hello****World**"));
-}
-
-#[test]
-fn test_br_inside_nested_formatting() {
-    let html = "<b><i>Bold italic<br/></i></b><b>Just bold</b>";
-    let result = convert(html, None).unwrap();
-
-    assert!(result.contains("***Bold italic***  \n**Just bold**"));
-}
-
-#[test]
-fn test_br_at_end_of_paragraph_with_bold() {
-    let html = "<p><b>Line 1<br/></b><b>Line 2</b></p>";
-    let result = convert(html, None).unwrap();
-
-    assert!(result.contains("**Line 1**  \n**Line 2**"));
-}

data/vendor/html-to-markdown-rs/tests/commonmark_compliance_test.rs

@@ -1,297 +0,0 @@
-#![allow(missing_docs)]
-
-use html_to_markdown_rs::ConversionOptions;
-use serde::Deserialize;
-
-#[derive(Debug, Deserialize)]
-struct CommonMarkTest {
-    markdown: String,
-    html: String,
-    example: u32,
-    start_line: u32,
-    #[allow(dead_code)]
-    end_line: u32,
-    section: String,
-}
-
-fn load_spec_tests() -> Vec<CommonMarkTest> {
-    let spec_json = include_str!("../../../packages/python/tests/commonmark_spec.json");
-    serde_json::from_str(spec_json).expect("Failed to parse commonmark_spec.json")
-}
-
-#[test]
-#[allow(clippy::too_many_lines)]
-fn test_commonmark_compliance() {
-    let tests = load_spec_tests();
-    let mut passed = 0;
-    let mut failed = 0;
-    let mut skipped = 0;
-    let mut failures = Vec::new();
-
-    for test in &tests {
-        let options = ConversionOptions {
-            bullets: "-".to_string(),
-            ..ConversionOptions::default()
-        };
-
-        if !options.escape_misc
-            && !options.escape_asterisks
-            && !options.escape_underscores
-            && !options.escape_ascii
-            && test.section == "Backslash escapes"
-        {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "HTML blocks" || test.section == "Raw HTML" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Fenced code blocks" {
-            skipped += 1;
-            continue;
-        }
-
-        // Example 231: expects indented code blocks but we default to backtick-fenced
-        if test.example == 231 {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Tabs" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Entity and numeric character references" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Thematic breaks" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Hard line breaks" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Soft line breaks" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Blank lines" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Textual content" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Lists" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "List items" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Paragraphs" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Setext headings" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Link reference definitions" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "ATX headings" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Indented code blocks" {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Code spans"
-            && [
-                329, 334, 335, 336, 337, 338, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
-            ]
-            .contains(&test.example)
-        {
-            skipped += 1;
-            continue;
-        }
-
-        if test.section == "Links" {
-            if !test.html.contains("<a") {
-                skipped += 1;
-                continue;
-            }
-            let passing_examples = [
-                482, 483, 484, 486, 496, 498, 501, 512, 514, 516, 517, 518, 519, 521, 522,
-            ];
-            if !passing_examples.contains(&test.example) {
-                skipped += 1;
-                continue;
-            }
-        }
-
-        if test.section == "Images" {
-            if !test.html.contains("<img") {
-                skipped += 1;
-                continue;
-            }
-            let passing_examples = [572, 578, 581];
-            if !passing_examples.contains(&test.example) {
-                skipped += 1;
-                continue;
-            }
-        }
-
-        if test.section == "Emphasis and strong emphasis" {
-            let passing_examples = [
-                350, 351, 352, 354, 355, 356, 358, 359, 360, 361, 362, 363, 365, 366, 367, 368, 369, 370, 371, 372,
-                374, 375, 378, 379, 380, 381, 383, 384, 385, 386, 387, 388, 391, 392, 393, 395, 396, 397, 398, 400,
-                401, 404, 405, 409, 410, 411, 412, 413, 414, 415, 416, 418, 419, 420, 421, 422, 423, 428, 429, 430,
-                431, 433, 434, 435, 436, 438, 439, 441, 442, 443, 444, 445, 446, 447, 448, 451, 460, 467, 469, 471,
-                472, 473, 474, 478, 480, 481,
-            ];
-            if !passing_examples.contains(&test.example) {
-                skipped += 1;
-                continue;
-            }
-        }
-
-        if test.section == "Block quotes" {
-            let passing_examples = [228, 231, 234, 243, 245, 248];
-            if !passing_examples.contains(&test.example) {
-                skipped += 1;
-                continue;
-            }
-        }
-
-        if test.section == "Autolinks" {
-            let passing_examples = [594, 595, 596, 597, 600, 602, 604, 605, 607, 608, 609, 610, 611, 612];
-            if !passing_examples.contains(&test.example) {
-                skipped += 1;
-                continue;
-            }
-        }
-
-        let result = convert(&test.html, Some(options));
-
-        match result {
-            Ok(output) => {
-                let output_normalized = normalize_markdown(&output);
-                let expected_normalized = normalize_markdown(&test.markdown);
-
-                if output_normalized == expected_normalized {
-                    passed += 1;
-                } else {
-                    failed += 1;
-                    failures.push((test, output));
-                }
-            }
-            Err(e) => {
-                failed += 1;
-                failures.push((test, format!("Error: {e:?}")));
-            }
-        }
-    }
-
-    let total = tests.len();
-    let tested = passed + failed;
-    let pass_rate = if tested > 0 {
-        (f64::from(passed) / f64::from(tested)) * 100.0
-    } else {
-        0.0
-    };
-
-    println!("\n=== CommonMark Compliance Test Results ===");
-    println!("Total tests: {total}");
-    if skipped > 0 {
-        println!("Skipped: {skipped} (escaping tests with escaping disabled)");
-        println!("Tested: {tested}");
-    }
-    println!("Passed: {passed} ({pass_rate:.1}%)");
-    println!(
-        "Failed: {} ({:.1}%)",
-        failed,
-        (f64::from(failed) / f64::from(tested)) * 100.0
-    );
-
-    if !failures.is_empty() {
-        println!("\n=== Failed Tests (first 20) ===");
-        for (test, output) in failures.iter().take(20) {
-            println!("\nExample {} ({}:{})", test.example, test.section, test.start_line);
-            println!("HTML input:\n{}", test.html);
-            println!("Expected markdown:\n{}", test.markdown);
-            println!("Got:\n{output}");
-            println!("---");
-        }
-
-        println!("\n=== Autolinks Failures ===");
-        for (test, output) in &failures {
-            if test.section == "Autolinks" {
-                println!("\nExample {} ({}:{})", test.example, test.section, test.start_line);
-                println!("HTML input:\n{}", test.html);
-                println!("Expected markdown:\n{}", test.markdown);
-                println!("Got:\n{output}");
-                println!("---");
-            }
-        }
-
-        let mut section_failures: std::collections::HashMap<&str, usize> = std::collections::HashMap::new();
-        for (test, _) in &failures {
-            *section_failures.entry(&test.section).or_insert(0) += 1;
-        }
-
-        println!("\n=== Failures by Section ===");
-        let mut sections: Vec<_> = section_failures.iter().collect();
-        sections.sort_by_key(|(_, count)| std::cmp::Reverse(*count));
-        for (section, count) in sections {
-            println!("{section}: {count} failures");
-        }
-
-        panic!(
-            "\nCommonMark compliance test FAILED: {passed}/{tested} tests passing ({pass_rate:.1}%)\n\
-                {skipped} tests skipped (escaping tests with escaping disabled)\n\
-                Default library settings must be CommonMark compliant!\n\
-                This is a mandatory test for v2.0 release."
-        );
-    }
-
-    println!(
-        "\n✓ All {tested} CommonMark tests passed! ({skipped} skipped) Library defaults are CommonMark compliant."
-    );
-}
-
-/// Normalize markdown for comparison
-fn normalize_markdown(md: &str) -> String {
-    md.trim_end().to_string()
-}
-
-fn convert(
-    html: &str,
-    opts: Option<html_to_markdown_rs::ConversionOptions>,
-) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
-}

data/vendor/html-to-markdown-rs/tests/djot_output_test.rs

@@ -1,153 +0,0 @@
-#![allow(missing_docs)]
-fn convert(
-    html: &str,
-    opts: Option<html_to_markdown_rs::ConversionOptions>,
-) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
-}
-
-use html_to_markdown_rs::{ConversionOptions, OutputFormat};
-
-fn djot_options() -> ConversionOptions {
-    ConversionOptions {
-        output_format: OutputFormat::Djot,
-        ..Default::default()
-    }
-}
-
-#[test]
-fn test_djot_emphasis() {
-    let html = "<p>Text with <em>emphasis</em> word</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(result.contains("_emphasis_"), "Expected _emphasis_, got: {result}");
-}
-
-#[test]
-fn test_djot_italic() {
-    let html = "<p>Text with <i>italic</i> word</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(result.contains("_italic_"), "Expected _italic_, got: {result}");
-}
-
-#[test]
-fn test_djot_strong() {
-    let html = "<p>Text with <strong>strong</strong> word</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(result.contains("*strong*"), "Expected *strong*, got: {result}");
-    // Should NOT have double asterisks
-    assert!(
-        !result.contains("**strong**"),
-        "Should not have **strong**, got: {result}"
-    );
-}
-
-#[test]
-fn test_djot_bold() {
-    let html = "<p>Text with <b>bold</b> word</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(result.contains("*bold*"), "Expected *bold*, got: {result}");
-    // Should NOT have double asterisks
-    assert!(
-        !result.contains("**bold**"),
-        "Should not have double asterisks, got: {result}"
-    );
-}
-
-#[test]
-fn test_djot_options_debug() {
-    // Debug test to verify options are set correctly
-    let options = djot_options();
-    println!("Output format: {:?}", options.output_format);
-    assert_eq!(options.output_format, OutputFormat::Djot);
-}
-
-#[test]
-fn test_djot_strikethrough() {
-    let html = "<p>Text with <del>deleted</del> word</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(result.contains("{-deleted-}"), "Expected {{-deleted-}}, got: {result}");
-}
-
-#[test]
-fn test_djot_strikethrough_s_tag() {
-    let html = "<p>Text with <s>strikethrough</s> word</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(
-        result.contains("{-strikethrough-}"),
-        "Expected {{-strikethrough-}}, got: {result}"
-    );
-}
-
-#[test]
-fn test_djot_inserted() {
-    let html = "<p>Text with <ins>inserted</ins> word</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(
-        result.contains("{+inserted+}"),
-        "Expected {{+inserted+}}, got: {result}"
-    );
-}
-
-#[test]
-fn test_djot_highlight() {
-    let html = "<p>Text with <mark>highlighted</mark> word</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(
-        result.contains("{=highlighted=}"),
-        "Expected {{=highlighted=}}, got: {result}"
-    );
-}
-
-#[test]
-fn test_djot_subscript() {
-    let html = "<p>H<sub>2</sub>O</p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(result.contains("~2~"), "Expected ~2~, got: {result}");
-}
-
-#[test]
-fn test_djot_superscript() {
-    let html = "<p>x<sup>2</sup></p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(result.contains("^2^"), "Expected ^2^, got: {result}");
-}
-
-#[test]
-fn test_djot_combined_formatting() {
-    let html = "<p><strong>Bold</strong> and <em>italic</em> and <del>deleted</del></p>";
-    let result = convert(html, Some(djot_options())).unwrap();
-    assert!(result.contains("*Bold*"), "Expected *Bold*, got: {result}");
-    assert!(result.contains("_italic_"), "Expected _italic_, got: {result}");
-    assert!(result.contains("{-deleted-}"), "Expected {{-deleted-}}, got: {result}");
-}
-
-#[test]
-fn test_markdown_output_unchanged() {
-    // Verify Markdown output still works as expected
-    let html = "<p><strong>Bold</strong> and <em>italic</em></p>";
-    let result = convert(html, None).unwrap();
-    assert!(
-        result.contains("**Bold**"),
-        "Expected **Bold** for Markdown, got: {result}"
-    );
-    assert!(
-        result.contains("*italic*"),
-        "Expected *italic* for Markdown, got: {result}"
-    );
-}
-
-#[test]
-fn test_markdown_strikethrough_unchanged() {
-    let html = "<p><del>deleted</del></p>";
-    let result = convert(html, None).unwrap();
-    assert!(
-        result.contains("~~deleted~~"),
-        "Expected ~~deleted~~ for Markdown, got: {result}"
-    );
-}
-
-#[test]
-fn test_output_format_default_is_markdown() {
-    let options = ConversionOptions::default();
-    assert_eq!(options.output_format, OutputFormat::Markdown);
-}

data/vendor/html-to-markdown-rs/tests/exclude_selectors_test.rs

@@ -1,132 +0,0 @@
-#![allow(missing_docs)]
-
-use html_to_markdown_rs::ConversionOptions;
-
-fn convert(html: &str, opts: Option<ConversionOptions>) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
-}
-
-#[test]
-fn test_exclude_selectors_drops_matching_elements() {
-    let html = r#"<body>
-        <div class="cookie-banner">Accept cookies</div>
-        <article><p>Main content here.</p></article>
-        <div id="ad-container">Buy stuff</div>
-    </body>"#;
-
-    let options = ConversionOptions {
-        exclude_selectors: vec![".cookie-banner".to_string(), "#ad-container".to_string()],
-        ..Default::default()
-    };
-
-    let result = convert(html, Some(options)).unwrap();
-
-    assert!(result.contains("Main content"), "Should keep main content");
-    assert!(!result.contains("cookie"), "Should drop .cookie-banner element");
-    assert!(!result.contains("Buy stuff"), "Should drop #ad-container element");
-}
-
-#[test]
-fn test_exclude_selectors_drops_nested_content() {
-    let html = r#"<body>
-        <aside class="sidebar">
-            <h2>Related articles</h2>
-            <p>Some sidebar content</p>
-        </aside>
-        <main><p>Primary content.</p></main>
-    </body>"#;
-
-    let options = ConversionOptions {
-        exclude_selectors: vec![".sidebar".to_string()],
-        ..Default::default()
-    };
-
-    let result = convert(html, Some(options)).unwrap();
-
-    assert!(result.contains("Primary content"), "Should keep main content");
-    assert!(
-        !result.contains("Related articles"),
-        "Should drop heading inside excluded element"
-    );
-    assert!(
-        !result.contains("sidebar content"),
-        "Should drop paragraph inside excluded element"
-    );
-}
-
-#[test]
-fn test_exclude_selectors_empty_list_is_noop() {
-    let html = r"<body><p>Hello world</p></body>";
-
-    let options = ConversionOptions {
-        exclude_selectors: vec![],
-        ..Default::default()
-    };
-
-    let result = convert(html, Some(options)).unwrap();
-    assert!(
-        result.contains("Hello world"),
-        "Empty exclude_selectors should not affect output"
-    );
-}
-
-#[test]
-fn test_exclude_selectors_invalid_selector_is_skipped() {
-    let html = r"<body><p>Visible text</p></body>";
-
-    // An empty string or garbled selector should not panic or error — just be ignored.
-    let options = ConversionOptions {
-        exclude_selectors: vec![String::new(), "p".to_string()],
-        ..Default::default()
-    };
-
-    // Should not return an error; whether the paragraph is excluded depends on the
-    // selector, but it must not panic.
-    let _ = convert(html, Some(options));
-}
-
-#[test]
-fn test_exclude_selectors_attribute_selector() {
-    let html = r#"<body>
-        <div role="complementary">Sidebar</div>
-        <p>Main text</p>
-    </body>"#;
-
-    let options = ConversionOptions {
-        exclude_selectors: vec!["[role='complementary']".to_string()],
-        ..Default::default()
-    };
-
-    let result = convert(html, Some(options)).unwrap();
-
-    assert!(result.contains("Main text"), "Should keep non-excluded content");
-    assert!(
-        !result.contains("Sidebar"),
-        "Should drop element matching attribute selector"
-    );
-}
-
-#[test]
-fn test_exclude_selectors_plain_text_output() {
-    let html = r#"<body>
-        <div class="nav">Navigation links</div>
-        <p>Article body text.</p>
-    </body>"#;
-
-    let options = ConversionOptions {
-        exclude_selectors: vec![".nav".to_string()],
-        output_format: html_to_markdown_rs::OutputFormat::Plain,
-        ..Default::default()
-    };
-
-    let result = convert(html, Some(options)).unwrap();
-
-    assert!(
-        result.contains("Article body text"),
-        "Should keep body text in plain output"
-    );
-    assert!(
-        !result.contains("Navigation links"),
-        "Should drop excluded element in plain output"
-    );
-}