html-to-markdown 3.2.4 → 3.4.0.pre.rc.13

data/vendor/html-to-markdown-rs/tests/issue_200_regressions.rs

@@ -4,7 +4,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 #[test]

data/vendor/html-to-markdown-rs/tests/issue_212_regressions.rs

@@ -4,7 +4,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 /// Regression test for <https://github.com/kreuzberg-dev/html-to-markdown/issues/212>

data/vendor/html-to-markdown-rs/tests/issue_216_217_regressions.rs

@@ -11,7 +11,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 /// Minimal reproducer: a <details> containing a <p> with <strong> inside.

data/vendor/html-to-markdown-rs/tests/json_ld_script_extraction.rs

@@ -14,7 +14,7 @@ fn extracts_json_ld_from_head_script() {
         </html>
     "#;
 
-    let result = html_to_markdown_rs::convert(html, None).expect("convert failed");
+    let result = html_to_markdown_rs::convert(html, None, None).expect("convert failed");
     let metadata = result.metadata;
 
     assert_eq!(metadata.structured_data.len(), 1);
@@ -35,7 +35,7 @@ fn extracts_json_ld_from_body_script_and_keeps_content() {
         </html>
     "#;
 
-    let result = html_to_markdown_rs::convert(html, None).expect("convert failed");
+    let result = html_to_markdown_rs::convert(html, None, None).expect("convert failed");
     let metadata = result.metadata;
 
     assert_eq!(metadata.structured_data.len(), 1);

data/vendor/html-to-markdown-rs/tests/lists_test.rs

@@ -4,7 +4,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 use html_to_markdown_rs::ConversionOptions;

data/vendor/html-to-markdown-rs/tests/plain_output_test.rs

@@ -3,7 +3,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 use html_to_markdown_rs::{ConversionOptions, OutputFormat};

data/vendor/html-to-markdown-rs/tests/preprocessing_tests.rs

@@ -4,7 +4,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 use html_to_markdown_rs::ConversionOptions;

data/vendor/html-to-markdown-rs/tests/reference_links_test.rs

@@ -3,7 +3,7 @@
 use html_to_markdown_rs::{ConversionOptions, LinkStyle};
 
 fn convert(html: &str, options: Option<ConversionOptions>) -> String {
-    html_to_markdown_rs::convert(html, options)
+    html_to_markdown_rs::convert(html, options, None)
         .unwrap()
         .content
         .unwrap_or_default()

data/vendor/html-to-markdown-rs/tests/sectioning_elements_test.rs

@@ -0,0 +1,137 @@
+#![allow(missing_docs)]
+
+fn convert(html: &str) -> String {
+    html_to_markdown_rs::convert(html, None, None)
+        .map(|r| r.content.unwrap_or_default())
+        .expect("conversion should succeed")
+}
+
+// --- header ---
+
+#[test]
+fn test_h1_inside_header() {
+    let html = "<header><h1>Title in header not exported???</h1></header>";
+    let result = convert(html);
+    assert_eq!(result, "# Title in header not exported???\n");
+}
+
+#[test]
+fn test_paragraph_inside_header() {
+    let html = "<header><p>Intro text</p></header>";
+    let result = convert(html);
+    assert_eq!(result, "Intro text\n");
+}
+
+#[test]
+fn test_header_with_nested_elements() {
+    let html = "<header><h1>Title</h1><p>Subtitle</p></header>";
+    let result = convert(html);
+    assert!(result.contains("# Title"), "Should contain h1: {result}");
+    assert!(result.contains("Subtitle"), "Should contain paragraph: {result}");
+}
+
+// --- footer ---
+
+#[test]
+fn test_paragraph_inside_footer() {
+    let html = "<footer><p>Footer content</p></footer>";
+    let result = convert(html);
+    assert_eq!(result, "Footer content\n");
+}
+
+// --- main ---
+
+#[test]
+fn test_h2_inside_main() {
+    let html = "<main><h2>Main heading</h2></main>";
+    let result = convert(html);
+    assert_eq!(result, "## Main heading\n");
+}
+
+// --- article ---
+
+#[test]
+fn test_article_with_header_and_section() {
+    let html = "<article><header><h1>Title</h1></header><section><p>Content here</p></section></article>";
+    let result = convert(html);
+    assert!(result.contains("# Title"), "Should contain heading: {result}");
+    assert!(result.contains("Content here"), "Should contain content: {result}");
+}
+
+// --- section ---
+
+#[test]
+fn test_heading_inside_section() {
+    let html = "<section><h2>Section Heading</h2><p>Section body</p></section>";
+    let result = convert(html);
+    assert!(result.contains("## Section Heading"), "Should contain h2: {result}");
+    assert!(result.contains("Section body"), "Should contain body: {result}");
+}
+
+// --- nav ---
+
+#[test]
+fn test_nav_dropped_by_default() {
+    // nav is dropped by default when remove_navigation is true (the default)
+    let html = r#"<nav><a href="/home">Home</a><a href="/about">About</a></nav>"#;
+    let result = convert(html);
+    assert!(result.is_empty(), "nav should be dropped by default: '{result}'");
+}
+
+#[test]
+fn test_nav_preserved_when_remove_navigation_disabled() {
+    use html_to_markdown_rs::{ConversionOptions, PreprocessingOptions};
+    let opts = ConversionOptions {
+        preprocessing: PreprocessingOptions {
+            remove_navigation: false,
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    let html = r#"<nav><a href="/home">Home</a></nav>"#;
+    let result = html_to_markdown_rs::convert(html, Some(opts), None)
+        .map(|r| r.content.unwrap_or_default())
+        .expect("conversion should succeed");
+    assert!(
+        result.contains("Home"),
+        "nav should pass through when remove_navigation=false: '{result}'"
+    );
+}
+
+// --- aside ---
+
+#[test]
+fn test_paragraph_inside_aside() {
+    let html = "<aside><p>Side note</p></aside>";
+    let result = convert(html);
+    assert_eq!(result, "Side note\n");
+}
+
+// --- navigation-hinted header should still be dropped ---
+
+#[test]
+fn test_site_chrome_header_dropped() {
+    // A <header> with class="site-header" is site chrome and should be removed
+    let html = r#"<header class="site-header"><a href="/">Logo</a></header><p>Content</p>"#;
+    let result = convert(html);
+    assert!(
+        !result.contains("Logo"),
+        "site-chrome header should be dropped: '{result}'"
+    );
+    assert!(
+        result.contains("Content"),
+        "body content should be preserved: '{result}'"
+    );
+}
+
+#[test]
+fn test_header_with_role_navigation_dropped() {
+    // A <header role="navigation"> is nav chrome and should be removed
+    let html = r#"<header role="navigation"><a href="/">Home</a></header><p>Body</p>"#;
+    let result = convert(html);
+    assert!(
+        !result.contains("Home"),
+        "navigation header should be dropped: '{result}'"
+    );
+    assert!(result.contains("Body"), "body content should be preserved: '{result}'");
+}

data/vendor/html-to-markdown-rs/tests/skip_images_test.rs

@@ -518,5 +518,5 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }

data/vendor/html-to-markdown-rs/tests/tables_test.rs

@@ -4,7 +4,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 use html_to_markdown_rs::ConversionOptions;
@@ -718,7 +718,7 @@ fn test_table_colspan_no_header_issue_233() {
         <td>Cell 2</td>
       </tr>
     </table>"#;
-    let result = html_to_markdown_rs::convert(html, None)
+    let result = html_to_markdown_rs::convert(html, None, None)
         .unwrap()
         .content
         .unwrap_or_default();

data/vendor/html-to-markdown-rs/tests/test_custom_elements.rs

@@ -4,7 +4,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 use std::fs;

data/vendor/html-to-markdown-rs/tests/test_issue_187.rs

@@ -7,7 +7,7 @@
 
 #![cfg(feature = "visitor")]
 
-use html_to_markdown_rs::convert_with_visitor;
+use html_to_markdown_rs::convert;
 use html_to_markdown_rs::visitor::{HtmlVisitor, NodeContext, VisitResult};
 use std::cell::RefCell;
 use std::rc::Rc;
@@ -147,7 +147,10 @@ fn test_issue_187_content_filter() {
     "#;
 
     let visitor = Rc::new(RefCell::new(ContentFilter::default()));
-    let result = convert_with_visitor(html, None, Some(visitor.clone())).unwrap();
+    let result = convert(html, None, Some(visitor.clone()))
+        .unwrap()
+        .content
+        .unwrap_or_default();
 
     println!("Converted Markdown:\n{result}");
     println!("\nSkipped Elements:");

data/vendor/html-to-markdown-rs/tests/test_issue_218.rs

@@ -9,7 +9,7 @@
 use std::cell::RefCell;
 use std::rc::Rc;
 
-use html_to_markdown_rs::convert_with_visitor;
+use html_to_markdown_rs::convert;
 use html_to_markdown_rs::visitor::HtmlVisitor;
 
 /// Empty visitor — does nothing, just uses default implementations.
@@ -26,7 +26,7 @@ fn make_visitor() -> Rc<RefCell<dyn HtmlVisitor>> {
 fn test_cyrillic_with_tabs_between_divs_and_visitor() {
     // Exact reproduction from the issue
     let html = "<div><span>А</span></div>\t\t\t<div><span>По";
-    let result = convert_with_visitor(html, None, Some(make_visitor()));
+    let result = convert(html, None, Some(make_visitor()));
     assert!(result.is_ok(), "Should not panic: {result:?}");
 }
 
@@ -40,7 +40,7 @@ fn test_multibyte_utf8_with_tabs_and_visitor() {
     ];
 
     for html in &cases {
-        let result = convert_with_visitor(html, None, Some(make_visitor()));
+        let result = convert(html, None, Some(make_visitor()));
         assert!(result.is_ok(), "Should not panic for: {html}\nError: {result:?}");
     }
 }
@@ -50,7 +50,7 @@ fn test_cyrillic_with_varying_tab_counts_and_visitor() {
     for n in 1..=5 {
         let tabs = "\t".repeat(n);
         let html = format!("<div><span>А</span></div>{tabs}<div><span>По");
-        let result = convert_with_visitor(&html, None, Some(make_visitor()));
+        let result = convert(&html, None, Some(make_visitor()));
         assert!(result.is_ok(), "Should not panic with {n} tabs: {result:?}");
     }
 }

data/vendor/html-to-markdown-rs/tests/test_issue_277.rs

@@ -0,0 +1,77 @@
+#![allow(missing_docs)]
+
+//! Regression test for issue #277: silent truncation on large HTML inputs.
+//!
+//! The bug was caused by `repair_with_html5ever` re-introducing `<script>` elements
+//! that had already been stripped, and `preprocess_html` failing to find the closing
+//! tag when script content contained unbalanced literal `<script>` strings.
+
+fn convert(html: &str) -> String {
+    html_to_markdown_rs::convert(html, None, None)
+        .expect("conversion should not fail")
+        .content
+        .unwrap_or_default()
+}
+
+/// When custom elements trigger html5ever repair, scripts must be re-stripped.
+/// Without the fix, content after a script with unbalanced `<script>` literals
+/// would be silently truncated.
+#[test]
+fn test_no_truncation_after_repair_with_scripts() {
+    // Custom element triggers repair_with_html5ever
+    // Script content has an unbalanced literal `<script>` that confuses depth tracking
+    let html = r"<html>
+<head>
+<script>
+  var example = '<script>';
+  console.log(example);
+</script>
+</head>
+<body>
+<custom-widget>widget</custom-widget>
+<p>Content before</p>
+<p>Content after scripts that must not be truncated</p>
+<p>Final paragraph</p>
+</body>
+</html>";
+
+    let result = convert(html);
+    assert!(
+        result.contains("Content before"),
+        "Should contain content before script region"
+    );
+    assert!(
+        result.contains("Content after scripts that must not be truncated"),
+        "Content after scripts should NOT be silently truncated. Got:\n{result}"
+    );
+    assert!(
+        result.contains("Final paragraph"),
+        "Final content should be present. Got:\n{result}"
+    );
+}
+
+/// Ensure `preprocess_html` doesn't truncate the rest of the document when
+/// `find_closing_tag` returns None (unmatched script opening).
+#[test]
+fn test_preprocess_unmatched_script_preserves_remaining_content() {
+    // Even without custom elements, preprocess_html's unwrap_or fallback
+    // should not consume the entire rest of the document.
+    let html = r"<html><body>
+<p>Before</p>
+<script>var x = '<script>'; var y = '<script>';</script>
+<p>After first script</p>
+<script>var z = 1;</script>
+<p>After second script</p>
+</body></html>";
+
+    let result = convert(html);
+    assert!(result.contains("Before"), "Content before scripts should be present");
+    assert!(
+        result.contains("After first script"),
+        "Content after first script should be present. Got:\n{result}"
+    );
+    assert!(
+        result.contains("After second script"),
+        "Content after second script should be present. Got:\n{result}"
+    );
+}

data/vendor/html-to-markdown-rs/tests/test_max_depth.rs

@@ -0,0 +1,82 @@
+#![allow(missing_docs)]
+
+//! Tests for the `max_depth` recursion-safety option.
+
+use html_to_markdown_rs::ConversionOptions;
+
+fn convert_with_options(html: &str, options: ConversionOptions) -> String {
+    html_to_markdown_rs::convert(html, Some(options), None)
+        .expect("conversion should not fail")
+        .content
+        .unwrap_or_default()
+}
+
+/// With the default `max_depth: None`, deeply nested content should be fully converted.
+#[test]
+fn test_max_depth_none_converts_deeply_nested() {
+    // Build 100 levels of nesting around a leaf text node.
+    let mut html = String::from("<p>deep</p>");
+    for _ in 0..100 {
+        html = format!("<div>{html}</div>");
+    }
+
+    let options = ConversionOptions {
+        extract_metadata: false,
+        max_depth: None,
+        ..Default::default()
+    };
+
+    let result = convert_with_options(&html, options);
+    assert!(
+        result.contains("deep"),
+        "Deeply nested text should be present when max_depth is None. Got:\n{result}"
+    );
+}
+
+/// With `max_depth: Some(2)`, block elements at depth 2 are not visited, so
+/// their text content is excluded from the output.
+#[test]
+fn test_max_depth_truncates_at_limit() {
+    // Depth counting (each handler passes depth+1 to its children):
+    // depth 0: outer <div>  — visited
+    //   depth 1: <p>        — visited, paragraph handler passes depth+1 to children
+    //     depth 2: "shallow" — visited (2 < 3), appears in output
+    //   depth 1: inner <div> — visited, div handler passes depth+1 to children
+    //     depth 2: <p>       — visited, paragraph handler passes depth+1 to children
+    //       depth 3: "deep"  — skipped (3 >= 3), absent from output
+    let html = "<div><p>shallow</p><div><p>deep</p></div></div>";
+
+    let options = ConversionOptions {
+        extract_metadata: false,
+        max_depth: Some(3),
+        ..Default::default()
+    };
+
+    let result = convert_with_options(html, options);
+    assert!(
+        result.contains("shallow"),
+        "Content at depth < max_depth should be present. Got:\n{result}"
+    );
+    assert!(
+        !result.contains("deep"),
+        "Content at depth >= max_depth should be absent. Got:\n{result}"
+    );
+}
+
+/// With `max_depth: Some(0)`, no nodes are processed and the output is empty or whitespace only.
+#[test]
+fn test_max_depth_zero_produces_empty() {
+    let html = "<p>hello</p>";
+
+    let options = ConversionOptions {
+        extract_metadata: false,
+        max_depth: Some(0),
+        ..Default::default()
+    };
+
+    let result = convert_with_options(html, options);
+    assert!(
+        result.trim().is_empty(),
+        "max_depth: Some(0) should produce no output. Got:\n{result}"
+    );
+}

data/vendor/html-to-markdown-rs/tests/test_nested_simple.rs

@@ -4,7 +4,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 use std::fs;

data/vendor/html-to-markdown-rs/tests/test_script_style_stripping.rs

@@ -111,7 +111,7 @@ fn test_preserve_json_ld_script() {
 </body>
 </html>"#;
 
-    let result = html_to_markdown_rs::convert(html, None).expect("Failed to convert");
+    let result = html_to_markdown_rs::convert(html, None, None).expect("Failed to convert");
     let metadata = result.metadata;
     let markdown = result.content.unwrap_or_default();
 
@@ -164,7 +164,7 @@ fn test_multiple_script_tags() {
 </body>
 </html>"#;
 
-    let result = html_to_markdown_rs::convert(html, None).expect("Failed to convert");
+    let result = html_to_markdown_rs::convert(html, None, None).expect("Failed to convert");
     let metadata = result.metadata;
     let markdown = result.content.unwrap_or_default();
 
@@ -221,7 +221,7 @@ fn test_reuters_like_structure() {
 </body>
 </html>"#;
 
-    let result = html_to_markdown_rs::convert(html, None).expect("Failed to convert");
+    let result = html_to_markdown_rs::convert(html, None, None).expect("Failed to convert");
     let metadata = result.metadata;
     let markdown = result.content.unwrap_or_default();
 
@@ -392,5 +392,5 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }

data/vendor/html-to-markdown-rs/tests/test_spa_bisect.rs

@@ -4,7 +4,7 @@ fn convert(
     html: &str,
     opts: Option<html_to_markdown_rs::ConversionOptions>,
 ) -> html_to_markdown_rs::error::Result<String> {
-    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+    html_to_markdown_rs::convert(html, opts, None).map(|r| r.content.unwrap_or_default())
 }
 
 use std::fs;

data/vendor/html-to-markdown-rs/tests/visitor_code_integration_test.rs

@@ -1,7 +1,7 @@
 #![allow(missing_docs)]
 #![cfg(feature = "visitor")]
 
-use html_to_markdown_rs::convert_with_visitor;
+use html_to_markdown_rs::convert;
 use html_to_markdown_rs::visitor::{HtmlVisitor, NodeContext, VisitResult};
 use std::cell::RefCell;
 use std::rc::Rc;
@@ -33,7 +33,7 @@ fn test_code_block_visitor() {
         inline_codes: vec![],
     }));
 
-    let result = convert_with_visitor(html, None, Some(visitor.clone()));
+    let result = convert(html, None, Some(visitor.clone()));
     assert!(result.is_ok());
 
     let visitor_ref = visitor.borrow();
@@ -49,7 +49,7 @@ fn test_inline_code_visitor() {
         inline_codes: vec![],
     }));
 
-    let result = convert_with_visitor(html, None, Some(visitor.clone()));
+    let result = convert(html, None, Some(visitor.clone()));
     assert!(result.is_ok());
 
     let visitor_ref = visitor.borrow();
@@ -71,9 +71,9 @@ fn test_code_block_skip() {
     let html = "<pre><code>skipped code</code></pre>";
     let visitor = Rc::new(RefCell::new(SkipCodeVisitor));
 
-    let result = convert_with_visitor(html, None, Some(visitor));
+    let result = convert(html, None, Some(visitor));
     assert!(result.is_ok());
-    let markdown = result.unwrap();
+    let markdown = result.unwrap().content.unwrap_or_default();
     assert!(!markdown.contains("skipped code"));
 }
 
@@ -97,7 +97,7 @@ fn test_code_block_language_detection() {
             inline_codes: vec![],
         }));
 
-        let result = convert_with_visitor(html, None, Some(visitor.clone()));
+        let result = convert(html, None, Some(visitor.clone()));
         assert!(result.is_ok(), "Failed to convert: {html}");
 
         let visitor_ref = visitor.borrow();