html-to-markdown 3.4.0 → 3.6.0.pre.rc.23

data/vendor/html-to-markdown-rs/tests/test_custom_elements.rs

@@ -1,41 +0,0 @@
-#![allow(missing_docs)]
-
-fn convert(
-    html: &str,
-    opts: Option<html_to_markdown_rs::ConversionOptions>,
-) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
-}
-
-use std::fs;
-use std::path::PathBuf;
-
-use html_to_markdown_rs::ConversionOptions;
-
-fn fixture_path(name: &str) -> PathBuf {
-    [env!("CARGO_MANIFEST_DIR"), "../../test_documents/html/issues", name]
-        .iter()
-        .collect()
-}
-
-fn default_options() -> ConversionOptions {
-    ConversionOptions {
-        extract_metadata: false,
-        autolinks: false,
-        ..Default::default()
-    }
-}
-
-#[test]
-fn test_custom_elements() {
-    let html = fs::read_to_string(fixture_path("test-with-custom-elements.html")).expect("read html");
-
-    eprintln!("HTML: {html}");
-
-    let result = convert(&html, Some(default_options())).expect("convert html");
-    eprintln!("Result: {result}");
-
-    assert!(result.contains("Team Appraisal"));
-    assert!(result.contains("Team"));
-    assert!(result.contains("Pending"));
-}

data/vendor/html-to-markdown-rs/tests/test_issue_187.rs

@@ -1,204 +0,0 @@
-//! Test to reproduce issue #187: `visit_div` is not executed
-//!
-//! This test demonstrates that:
-//! 1. There is NO `visit_div`, `visit_script`, `visit_style` method
-//! 2. Instead, users should use `visit_element_start` with `tag_name` filtering
-//! 3. The documentation incorrectly shows these non-existent methods
-
-#![cfg(feature = "visitor")]
-
-use html_to_markdown_rs::visitor::{HtmlVisitor, NodeContext, VisitResult, VisitorHandle};
-use html_to_markdown_rs::{ConversionError, ConversionOptions, ConversionResult};
-use std::cell::RefCell;
-use std::rc::Rc;
-
-fn convert(
-    html: &str,
-    options: Option<ConversionOptions>,
-    visitor: Option<VisitorHandle>,
-) -> Result<ConversionResult, ConversionError> {
-    let mut opts = options.unwrap_or_default();
-    if visitor.is_some() {
-        opts.visitor = visitor;
-    }
-    html_to_markdown_rs::convert(html, Some(opts))
-}
-
-/// This is what the documentation SHOWS (but doesn't work)
-/// The methods `visit_div`, `visit_script`, `visit_style` DO NOT EXIST
-#[allow(dead_code)]
-#[derive(Debug, Default)]
-struct DocumentedButBrokenVisitor {
-    skipped_elements: Vec<(String, String)>,
-}
-
-// NOTE: These methods will NEVER be called because they don't exist in HtmlVisitor trait!
-// impl DocumentedButBrokenVisitor {
-//     fn visit_div(&mut self, ctx: &NodeContext, _content: &str) -> VisitResult {
-//         // This will NEVER be called!
-//         let classes = ctx.attributes.get("class").map(|s| s.as_str()).unwrap_or("");
-//         if classes.contains("ad") || classes.contains("advertisement") {
-//             self.skipped_elements.push(("div".to_string(), classes.to_string()));
-//             return VisitResult::Skip;
-//         }
-//         VisitResult::Continue
-//     }
-//
-//     fn visit_script(&mut self, ctx: &NodeContext) -> VisitResult {
-//         // This will NEVER be called!
-//         self.skipped_elements.push(("script".to_string(), "".to_string()));
-//         VisitResult::Skip
-//     }
-// }
-
-/// This is the CORRECT way to filter divs, scripts, and styles
-#[derive(Debug, Default)]
-struct ContentFilter {
-    skipped_elements: Vec<(String, String)>,
-}
-
-impl HtmlVisitor for ContentFilter {
-    /// Use `visit_element_start` to filter ANY element by tag name
-    fn visit_element_start(&mut self, ctx: &NodeContext) -> VisitResult {
-        let tag_name = ctx.tag_name.as_str();
-
-        match tag_name {
-            "div" => {
-                // Filter divs with unwanted classes
-                let classes = ctx.attributes.get("class").map_or("", std::string::String::as_str);
-                if classes.contains("ad")
-                    || classes.contains("advertisement")
-                    || classes.contains("tracking")
-                    || classes.contains("analytics")
-                {
-                    self.skipped_elements.push(("div".to_string(), classes.to_string()));
-                    return VisitResult::Skip;
-                }
-            }
-            "script" => {
-                // Always remove script tags
-                self.skipped_elements.push(("script".to_string(), String::new()));
-                return VisitResult::Skip;
-            }
-            "style" => {
-                // Always remove style tags
-                self.skipped_elements.push(("style".to_string(), String::new()));
-                return VisitResult::Skip;
-            }
-            _ => {}
-        }
-
-        VisitResult::Continue
-    }
-
-    /// Still use specific methods for links and images
-    fn visit_image(&mut self, ctx: &NodeContext, src: &str, _alt: &str, _title: Option<&str>) -> VisitResult {
-        // Remove tracking pixels (1x1 images)
-        let width = ctx.attributes.get("width").map_or("", std::string::String::as_str);
-        let height = ctx.attributes.get("height").map_or("", std::string::String::as_str);
-
-        if width == "1" && height == "1" {
-            self.skipped_elements
-                .push(("img".to_string(), format!("tracking pixel: {src}")));
-            return VisitResult::Skip;
-        }
-
-        // Skip images with "tracking" or "analytics" in the URL
-        if src.to_lowercase().contains("tracking") || src.to_lowercase().contains("analytics") {
-            self.skipped_elements
-                .push(("img".to_string(), format!("tracking URL: {src}")));
-            return VisitResult::Skip;
-        }
-
-        VisitResult::Continue
-    }
-
-    fn visit_link(&mut self, _ctx: &NodeContext, href: &str, text: &str, _title: Option<&str>) -> VisitResult {
-        // Remove links with utm_* tracking parameters
-        if href.to_lowercase().contains("utm_") {
-            // Strip tracking params but keep the link
-            if let Some(base_url) = href.split('?').next() {
-                return VisitResult::Custom(format!("[{text}]({base_url})"));
-            }
-        }
-
-        VisitResult::Continue
-    }
-}
-
-#[test]
-fn test_issue_187_content_filter() {
-    let html = r#"
-    <article>
-        <h1>Blog Post Title</h1>
-        <p>This is the main content of the article.</p>
-
-        <div class="ad advertisement">
-            <p>This is an advertisement block that should be removed.</p>
-        </div>
-
-        <p>More content here.</p>
-
-        <img src="https://tracking.example.com/pixel.gif" width="1" height="1" alt="">
-
-        <div class="content">
-            <p>Legitimate content in a div.</p>
-            <img src="https://cdn.example.com/image.jpg" alt="Article image" width="800">
-        </div>
-
-        <script>
-            console.log("This script should be removed");
-        </script>
-
-        <p>Read more on <a href="https://example.com/article?utm_source=newsletter&utm_medium=email">our website</a>.</p>
-
-        <div class="tracking analytics">
-            <img src="https://analytics.example.com/track.png" alt="">
-        </div>
-    </article>
-    "#;
-
-    let visitor = Rc::new(RefCell::new(ContentFilter::default()));
-    let result = convert(html, None, Some(visitor.clone()))
-        .unwrap()
-        .content
-        .unwrap_or_default();
-
-    println!("Converted Markdown:\n{result}");
-    println!("\nSkipped Elements:");
-    for (tag, info) in &visitor.borrow().skipped_elements {
-        println!("- {tag}: {info}");
-    }
-
-    // Verify that unwanted content was filtered out
-    assert!(
-        !result.contains("advertisement block that should be removed"),
-        "Ad div should be removed"
-    );
-    assert!(!result.contains("console.log"), "Script tag should be removed");
-
-    // Verify that legitimate content remains
-    assert!(result.contains("Blog Post Title"), "Heading should be preserved");
-    assert!(result.contains("main content"), "Main content should be preserved");
-    assert!(result.contains("Legitimate content"), "Content div should be preserved");
-    assert!(result.contains("Article image"), "Legitimate image should be preserved");
-
-    // Verify tracking parameters were stripped from link
-    assert!(
-        result.contains("[our website](https://example.com/article)"),
-        "Link tracking params should be stripped"
-    );
-
-    // Verify skipped elements were tracked
-    let borrowed = visitor.borrow();
-    assert!(
-        borrowed.skipped_elements.iter().any(|(tag, _)| tag == "div"),
-        "Should have skipped ad divs"
-    );
-    // Note: script and style tags are stripped during preprocessing before the visitor sees them,
-    // so they won't appear in skipped_elements. Only the visitor can control div, img, etc.
-    assert!(
-        borrowed.skipped_elements.iter().any(|(tag, _)| tag == "img"),
-        "Should have skipped tracking pixel"
-    );
-}

data/vendor/html-to-markdown-rs/tests/test_issue_218.rs

@@ -1,68 +0,0 @@
-//! Test to reproduce issue #218: Rust panic with Cyrillic HTML
-//!
-//! When converting HTML containing multi-byte UTF-8 characters (e.g., Cyrillic)
-//! with tabs between block elements and any visitor, a panic occurs:
-//! "byte index N is not a char boundary"
-
-#![cfg(feature = "visitor")]
-
-use std::cell::RefCell;
-use std::rc::Rc;
-
-use html_to_markdown_rs::visitor::{HtmlVisitor, VisitorHandle};
-use html_to_markdown_rs::{ConversionError, ConversionOptions, ConversionResult};
-
-fn convert(
-    html: &str,
-    options: Option<ConversionOptions>,
-    visitor: Option<VisitorHandle>,
-) -> Result<ConversionResult, ConversionError> {
-    let mut opts = options.unwrap_or_default();
-    if visitor.is_some() {
-        opts.visitor = visitor;
-    }
-    html_to_markdown_rs::convert(html, Some(opts))
-}
-
-/// Empty visitor — does nothing, just uses default implementations.
-#[derive(Debug, Default)]
-struct EmptyVisitor;
-
-impl HtmlVisitor for EmptyVisitor {}
-
-fn make_visitor() -> Rc<RefCell<dyn HtmlVisitor>> {
-    Rc::new(RefCell::new(EmptyVisitor))
-}
-
-#[test]
-fn test_cyrillic_with_tabs_between_divs_and_visitor() {
-    // Exact reproduction from the issue
-    let html = "<div><span>А</span></div>\t\t\t<div><span>По";
-    let result = convert(html, None, Some(make_visitor()));
-    assert!(result.is_ok(), "Should not panic: {result:?}");
-}
-
-#[test]
-fn test_multibyte_utf8_with_tabs_and_visitor() {
-    let cases = [
-        "<div><span>日本語</span></div>\t\t\t<div><span>テスト",
-        "<div><span>한국어</span></div>\t\t\t<div><span>테스트",
-        "<div><span>Привет</span></div>\t\t\t<div><span>Мир",
-        "<div><span>🎉</span></div>\t\t\t<div><span>🚀",
-    ];
-
-    for html in &cases {
-        let result = convert(html, None, Some(make_visitor()));
-        assert!(result.is_ok(), "Should not panic for: {html}\nError: {result:?}");
-    }
-}
-
-#[test]
-fn test_cyrillic_with_varying_tab_counts_and_visitor() {
-    for n in 1..=5 {
-        let tabs = "\t".repeat(n);
-        let html = format!("<div><span>А</span></div>{tabs}<div><span>По");
-        let result = convert(&html, None, Some(make_visitor()));
-        assert!(result.is_ok(), "Should not panic with {n} tabs: {result:?}");
-    }
-}

data/vendor/html-to-markdown-rs/tests/test_issue_277.rs

@@ -1,77 +0,0 @@
-#![allow(missing_docs)]
-
-//! Regression test for issue #277: silent truncation on large HTML inputs.
-//!
-//! The bug was caused by `repair_with_html5ever` re-introducing `<script>` elements
-//! that had already been stripped, and `preprocess_html` failing to find the closing
-//! tag when script content contained unbalanced literal `<script>` strings.
-
-fn convert(html: &str) -> String {
-    html_to_markdown_rs::convert(html, None)
-        .expect("conversion should not fail")
-        .content
-        .unwrap_or_default()
-}
-
-/// When custom elements trigger html5ever repair, scripts must be re-stripped.
-/// Without the fix, content after a script with unbalanced `<script>` literals
-/// would be silently truncated.
-#[test]
-fn test_no_truncation_after_repair_with_scripts() {
-    // Custom element triggers repair_with_html5ever
-    // Script content has an unbalanced literal `<script>` that confuses depth tracking
-    let html = r"<html>
-<head>
-<script>
-  var example = '<script>';
-  console.log(example);
-</script>
-</head>
-<body>
-<custom-widget>widget</custom-widget>
-<p>Content before</p>
-<p>Content after scripts that must not be truncated</p>
-<p>Final paragraph</p>
-</body>
-</html>";
-
-    let result = convert(html);
-    assert!(
-        result.contains("Content before"),
-        "Should contain content before script region"
-    );
-    assert!(
-        result.contains("Content after scripts that must not be truncated"),
-        "Content after scripts should NOT be silently truncated. Got:\n{result}"
-    );
-    assert!(
-        result.contains("Final paragraph"),
-        "Final content should be present. Got:\n{result}"
-    );
-}
-
-/// Ensure `preprocess_html` doesn't truncate the rest of the document when
-/// `find_closing_tag` returns None (unmatched script opening).
-#[test]
-fn test_preprocess_unmatched_script_preserves_remaining_content() {
-    // Even without custom elements, preprocess_html's unwrap_or fallback
-    // should not consume the entire rest of the document.
-    let html = r"<html><body>
-<p>Before</p>
-<script>var x = '<script>'; var y = '<script>';</script>
-<p>After first script</p>
-<script>var z = 1;</script>
-<p>After second script</p>
-</body></html>";
-
-    let result = convert(html);
-    assert!(result.contains("Before"), "Content before scripts should be present");
-    assert!(
-        result.contains("After first script"),
-        "Content after first script should be present. Got:\n{result}"
-    );
-    assert!(
-        result.contains("After second script"),
-        "Content after second script should be present. Got:\n{result}"
-    );
-}

data/vendor/html-to-markdown-rs/tests/test_max_depth.rs

@@ -1,82 +0,0 @@
-#![allow(missing_docs)]
-
-//! Tests for the `max_depth` recursion-safety option.
-
-use html_to_markdown_rs::ConversionOptions;
-
-fn convert_with_options(html: &str, options: ConversionOptions) -> String {
-    html_to_markdown_rs::convert(html, Some(options))
-        .expect("conversion should not fail")
-        .content
-        .unwrap_or_default()
-}
-
-/// With the default `max_depth: None`, deeply nested content should be fully converted.
-#[test]
-fn test_max_depth_none_converts_deeply_nested() {
-    // Build 100 levels of nesting around a leaf text node.
-    let mut html = String::from("<p>deep</p>");
-    for _ in 0..100 {
-        html = format!("<div>{html}</div>");
-    }
-
-    let options = ConversionOptions {
-        extract_metadata: false,
-        max_depth: None,
-        ..Default::default()
-    };
-
-    let result = convert_with_options(&html, options);
-    assert!(
-        result.contains("deep"),
-        "Deeply nested text should be present when max_depth is None. Got:\n{result}"
-    );
-}
-
-/// With `max_depth: Some(2)`, block elements at depth 2 are not visited, so
-/// their text content is excluded from the output.
-#[test]
-fn test_max_depth_truncates_at_limit() {
-    // Depth counting (each handler passes depth+1 to its children):
-    // depth 0: outer <div>  — visited
-    //   depth 1: <p>        — visited, paragraph handler passes depth+1 to children
-    //     depth 2: "shallow" — visited (2 < 3), appears in output
-    //   depth 1: inner <div> — visited, div handler passes depth+1 to children
-    //     depth 2: <p>       — visited, paragraph handler passes depth+1 to children
-    //       depth 3: "deep"  — skipped (3 >= 3), absent from output
-    let html = "<div><p>shallow</p><div><p>deep</p></div></div>";
-
-    let options = ConversionOptions {
-        extract_metadata: false,
-        max_depth: Some(3),
-        ..Default::default()
-    };
-
-    let result = convert_with_options(html, options);
-    assert!(
-        result.contains("shallow"),
-        "Content at depth < max_depth should be present. Got:\n{result}"
-    );
-    assert!(
-        !result.contains("deep"),
-        "Content at depth >= max_depth should be absent. Got:\n{result}"
-    );
-}
-
-/// With `max_depth: Some(0)`, no nodes are processed and the output is empty or whitespace only.
-#[test]
-fn test_max_depth_zero_produces_empty() {
-    let html = "<p>hello</p>";
-
-    let options = ConversionOptions {
-        extract_metadata: false,
-        max_depth: Some(0),
-        ..Default::default()
-    };
-
-    let result = convert_with_options(html, options);
-    assert!(
-        result.trim().is_empty(),
-        "max_depth: Some(0) should produce no output. Got:\n{result}"
-    );
-}

data/vendor/html-to-markdown-rs/tests/test_nested_simple.rs

@@ -1,45 +0,0 @@
-#![allow(missing_docs)]
-
-fn convert(
-    html: &str,
-    opts: Option<html_to_markdown_rs::ConversionOptions>,
-) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
-}
-
-use std::fs;
-use std::path::PathBuf;
-
-use html_to_markdown_rs::ConversionOptions;
-
-fn fixture_path(name: &str) -> PathBuf {
-    [env!("CARGO_MANIFEST_DIR"), "../../test_documents/html/issues", name]
-        .iter()
-        .collect()
-}
-
-fn default_options() -> ConversionOptions {
-    ConversionOptions {
-        extract_metadata: false,
-        autolinks: false,
-        ..Default::default()
-    }
-}
-
-fn normalize_newlines(input: &str) -> String {
-    input.replace("\r\n", "\n").replace('\r', "\n")
-}
-
-#[test]
-fn test_nested_simple() {
-    let html = fs::read_to_string(fixture_path("test-nested-simple.html")).expect("read html");
-    let expected = fs::read_to_string(fixture_path("test-nested-simple.md")).expect("read markdown");
-
-    eprintln!("HTML: {html}");
-    eprintln!("Expected: {expected}");
-
-    let result = convert(&html, Some(default_options())).expect("convert html");
-    eprintln!("Result: {result}");
-
-    assert_eq!(normalize_newlines(&result), normalize_newlines(&expected));
-}