html-to-markdown 2.25.1 → 2.26.2

data/rust-vendor/futures-util/src/stream/unfold.rs

@@ -0,0 +1,119 @@
+use super::assert_stream;
+use crate::unfold_state::UnfoldState;
+use core::fmt;
+use core::pin::Pin;
+use futures_core::future::Future;
+use futures_core::ready;
+use futures_core::stream::{FusedStream, Stream};
+use futures_core::task::{Context, Poll};
+use pin_project_lite::pin_project;
+
+/// Creates a `Stream` from a seed and a closure returning a `Future`.
+///
+/// This function is the dual for the `Stream::fold()` adapter: while
+/// `Stream::fold()` reduces a `Stream` to one single value, `unfold()` creates a
+/// `Stream` from a seed value.
+///
+/// `unfold()` will call the provided closure with the provided seed, then wait
+/// for the returned `Future` to complete with `(a, b)`. It will then yield the
+/// value `a`, and use `b` as the next internal state.
+///
+/// If the closure returns `None` instead of `Some(Future)`, then the `unfold()`
+/// will stop producing items and return `Poll::Ready(None)` in future
+/// calls to `poll()`.
+///
+/// This function can typically be used when wanting to go from the "world of
+/// futures" to the "world of streams": the provided closure can build a
+/// `Future` using other library functions working on futures, and `unfold()`
+/// will turn it into a `Stream` by repeating the operation.
+///
+/// # Example
+///
+/// ```
+/// # futures::executor::block_on(async {
+/// use futures::stream::{self, StreamExt};
+///
+/// let stream = stream::unfold(0, |state| async move {
+///     if state <= 2 {
+///         let next_state = state + 1;
+///         let yielded = state * 2;
+///         Some((yielded, next_state))
+///     } else {
+///         None
+///     }
+/// });
+///
+/// let result = stream.collect::<Vec<i32>>().await;
+/// assert_eq!(result, vec![0, 2, 4]);
+/// # });
+/// ```
+pub fn unfold<T, F, Fut, Item>(init: T, f: F) -> Unfold<T, F, Fut>
+where
+    F: FnMut(T) -> Fut,
+    Fut: Future<Output = Option<(Item, T)>>,
+{
+    assert_stream::<Item, _>(Unfold { f, state: UnfoldState::Value { value: init } })
+}
+
+pin_project! {
+    /// Stream for the [`unfold`] function.
+    #[must_use = "streams do nothing unless polled"]
+    pub struct Unfold<T, F, Fut> {
+        f: F,
+        #[pin]
+        state: UnfoldState<T, Fut>,
+    }
+}
+
+impl<T, F, Fut> fmt::Debug for Unfold<T, F, Fut>
+where
+    T: fmt::Debug,
+    Fut: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Unfold").field("state", &self.state).finish()
+    }
+}
+
+impl<T, F, Fut, Item> FusedStream for Unfold<T, F, Fut>
+where
+    F: FnMut(T) -> Fut,
+    Fut: Future<Output = Option<(Item, T)>>,
+{
+    fn is_terminated(&self) -> bool {
+        if let UnfoldState::Empty = self.state {
+            true
+        } else {
+            false
+        }
+    }
+}
+
+impl<T, F, Fut, Item> Stream for Unfold<T, F, Fut>
+where
+    F: FnMut(T) -> Fut,
+    Fut: Future<Output = Option<(Item, T)>>,
+{
+    type Item = Item;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let mut this = self.project();
+
+        if let Some(state) = this.state.as_mut().take_value() {
+            this.state.set(UnfoldState::Future { future: (this.f)(state) });
+        }
+
+        let step = match this.state.as_mut().project_future() {
+            Some(fut) => ready!(fut.poll(cx)),
+            None => panic!("Unfold must not be polled after it returned `Poll::Ready(None)`"),
+        };
+
+        if let Some((item, next_state)) = step {
+            this.state.set(UnfoldState::Value { value: next_state });
+            Poll::Ready(Some(item))
+        } else {
+            this.state.set(UnfoldState::Empty);
+            Poll::Ready(None)
+        }
+    }
+}

data/rust-vendor/futures-util/src/task/mod.rs

@@ -0,0 +1,40 @@
+//! Tools for working with tasks.
+//!
+//! This module contains:
+//!
+//! - [`Spawn`], a trait for spawning new tasks.
+//! - [`Context`], a context of an asynchronous task,
+//!   including a handle for waking up the task.
+//! - [`Waker`], a handle for waking up a task.
+//!
+//! The remaining types and traits in the module are used for implementing
+//! executors or dealing with synchronization issues around task wakeup.
+
+#[doc(no_inline)]
+pub use core::task::{Context, Poll, RawWaker, RawWakerVTable, Waker};
+
+pub use futures_task::{FutureObj, LocalFutureObj, LocalSpawn, Spawn, SpawnError, UnsafeFutureObj};
+
+pub use futures_task::noop_waker;
+pub use futures_task::noop_waker_ref;
+
+#[cfg_attr(target_os = "none", cfg(target_has_atomic = "ptr"))]
+#[cfg(feature = "alloc")]
+pub use futures_task::ArcWake;
+
+#[cfg_attr(target_os = "none", cfg(target_has_atomic = "ptr"))]
+#[cfg(feature = "alloc")]
+pub use futures_task::waker;
+
+#[cfg_attr(target_os = "none", cfg(target_has_atomic = "ptr"))]
+#[cfg(feature = "alloc")]
+pub use futures_task::{waker_ref, WakerRef};
+
+#[cfg_attr(
+    target_os = "none",
+    cfg(any(target_has_atomic = "ptr", feature = "portable-atomic"))
+)]
+pub use futures_core::task::__internal::AtomicWaker;
+
+mod spawn;
+pub use self::spawn::{LocalSpawnExt, SpawnExt};

data/rust-vendor/futures-util/src/task/spawn.rs

@@ -0,0 +1,169 @@
+use futures_task::{LocalSpawn, Spawn};
+
+#[cfg(feature = "compat")]
+use crate::compat::Compat;
+
+#[cfg(feature = "channel")]
+#[cfg(feature = "std")]
+use crate::future::{FutureExt, RemoteHandle};
+#[cfg(feature = "alloc")]
+use alloc::boxed::Box;
+#[cfg(feature = "alloc")]
+use futures_core::future::Future;
+#[cfg(feature = "alloc")]
+use futures_task::{FutureObj, LocalFutureObj, SpawnError};
+
+impl<Sp: ?Sized> SpawnExt for Sp where Sp: Spawn {}
+impl<Sp: ?Sized> LocalSpawnExt for Sp where Sp: LocalSpawn {}
+
+/// Extension trait for `Spawn`.
+pub trait SpawnExt: Spawn {
+    /// Spawns a task that polls the given future with output `()` to
+    /// completion.
+    ///
+    /// This method returns a [`Result`] that contains a [`SpawnError`] if
+    /// spawning fails.
+    ///
+    /// You can use [`spawn_with_handle`](SpawnExt::spawn_with_handle) if
+    /// you want to spawn a future with output other than `()` or if you want
+    /// to be able to await its completion.
+    ///
+    /// Note this method will eventually be replaced with the upcoming
+    /// `Spawn::spawn` method which will take a `dyn Future` as input.
+    /// Technical limitations prevent `Spawn::spawn` from being implemented
+    /// today. Feel free to use this method in the meantime.
+    ///
+    /// ```
+    /// # {
+    /// use futures::executor::ThreadPool;
+    /// use futures::task::SpawnExt;
+    ///
+    /// let executor = ThreadPool::new().unwrap();
+    ///
+    /// let future = async { /* ... */ };
+    /// executor.spawn(future).unwrap();
+    /// # }
+    /// # std::thread::sleep(std::time::Duration::from_millis(500)); // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371
+    /// ```
+    #[cfg(feature = "alloc")]
+    fn spawn<Fut>(&self, future: Fut) -> Result<(), SpawnError>
+    where
+        Fut: Future<Output = ()> + Send + 'static,
+    {
+        self.spawn_obj(FutureObj::new(Box::new(future)))
+    }
+
+    /// Spawns a task that polls the given future to completion and returns a
+    /// future that resolves to the spawned future's output.
+    ///
+    /// This method returns a [`Result`] that contains a [`RemoteHandle`](crate::future::RemoteHandle), or, if
+    /// spawning fails, a [`SpawnError`]. [`RemoteHandle`](crate::future::RemoteHandle) is a future that
+    /// resolves to the output of the spawned future.
+    ///
+    /// ```
+    /// # {
+    /// use futures::executor::{block_on, ThreadPool};
+    /// use futures::future;
+    /// use futures::task::SpawnExt;
+    ///
+    /// let executor = ThreadPool::new().unwrap();
+    ///
+    /// let future = future::ready(1);
+    /// let join_handle_fut = executor.spawn_with_handle(future).unwrap();
+    /// assert_eq!(block_on(join_handle_fut), 1);
+    /// # }
+    /// # std::thread::sleep(std::time::Duration::from_millis(500)); // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371
+    /// ```
+    #[cfg(feature = "channel")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "channel")))]
+    #[cfg(feature = "std")]
+    fn spawn_with_handle<Fut>(&self, future: Fut) -> Result<RemoteHandle<Fut::Output>, SpawnError>
+    where
+        Fut: Future + Send + 'static,
+        Fut::Output: Send,
+    {
+        let (future, handle) = future.remote_handle();
+        self.spawn(future)?;
+        Ok(handle)
+    }
+
+    /// Wraps a [`Spawn`] and makes it usable as a futures 0.1 `Executor`.
+    /// Requires the `compat` feature to enable.
+    #[cfg(feature = "compat")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "compat")))]
+    fn compat(self) -> Compat<Self>
+    where
+        Self: Sized,
+    {
+        Compat::new(self)
+    }
+}
+
+/// Extension trait for `LocalSpawn`.
+pub trait LocalSpawnExt: LocalSpawn {
+    /// Spawns a task that polls the given future with output `()` to
+    /// completion.
+    ///
+    /// This method returns a [`Result`] that contains a [`SpawnError`] if
+    /// spawning fails.
+    ///
+    /// You can use [`spawn_with_handle`](SpawnExt::spawn_with_handle) if
+    /// you want to spawn a future with output other than `()` or if you want
+    /// to be able to await its completion.
+    ///
+    /// Note this method will eventually be replaced with the upcoming
+    /// `Spawn::spawn` method which will take a `dyn Future` as input.
+    /// Technical limitations prevent `Spawn::spawn` from being implemented
+    /// today. Feel free to use this method in the meantime.
+    ///
+    /// ```
+    /// use futures::executor::LocalPool;
+    /// use futures::task::LocalSpawnExt;
+    ///
+    /// let executor = LocalPool::new();
+    /// let spawner = executor.spawner();
+    ///
+    /// let future = async { /* ... */ };
+    /// spawner.spawn_local(future).unwrap();
+    /// ```
+    #[cfg(feature = "alloc")]
+    fn spawn_local<Fut>(&self, future: Fut) -> Result<(), SpawnError>
+    where
+        Fut: Future<Output = ()> + 'static,
+    {
+        self.spawn_local_obj(LocalFutureObj::new(Box::new(future)))
+    }
+
+    /// Spawns a task that polls the given future to completion and returns a
+    /// future that resolves to the spawned future's output.
+    ///
+    /// This method returns a [`Result`] that contains a [`RemoteHandle`](crate::future::RemoteHandle), or, if
+    /// spawning fails, a [`SpawnError`]. [`RemoteHandle`](crate::future::RemoteHandle) is a future that
+    /// resolves to the output of the spawned future.
+    ///
+    /// ```
+    /// use futures::executor::LocalPool;
+    /// use futures::task::LocalSpawnExt;
+    ///
+    /// let mut executor = LocalPool::new();
+    /// let spawner = executor.spawner();
+    ///
+    /// let future = async { 1 };
+    /// let join_handle_fut = spawner.spawn_local_with_handle(future).unwrap();
+    /// assert_eq!(executor.run_until(join_handle_fut), 1);
+    /// ```
+    #[cfg(feature = "channel")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "channel")))]
+    #[cfg(feature = "std")]
+    fn spawn_local_with_handle<Fut>(
+        &self,
+        future: Fut,
+    ) -> Result<RemoteHandle<Fut::Output>, SpawnError>
+    where
+        Fut: Future + 'static,
+    {
+        let (future, handle) = future.remote_handle();
+        self.spawn_local(future)?;
+        Ok(handle)
+    }
+}

data/rust-vendor/futures-util/src/unfold_state.rs

@@ -0,0 +1,39 @@
+use core::pin::Pin;
+
+use pin_project_lite::pin_project;
+
+pin_project! {
+    /// UnfoldState used for stream and sink unfolds
+    #[project = UnfoldStateProj]
+    #[project_replace = UnfoldStateProjReplace]
+    #[derive(Debug)]
+    pub(crate) enum UnfoldState<T, R> {
+        Value {
+            value: T,
+        },
+        Future {
+            #[pin]
+            future: R,
+        },
+        Empty,
+    }
+}
+
+impl<T, R> UnfoldState<T, R> {
+    pub(crate) fn project_future(self: Pin<&mut Self>) -> Option<Pin<&mut R>> {
+        match self.project() {
+            UnfoldStateProj::Future { future } => Some(future),
+            _ => None,
+        }
+    }
+
+    pub(crate) fn take_value(self: Pin<&mut Self>) -> Option<T> {
+        match &*self {
+            Self::Value { .. } => match self.project_replace(Self::Empty) {
+                UnfoldStateProjReplace::Value { value } => Some(value),
+                _ => unreachable!(),
+            },
+            _ => None,
+        }
+    }
+}

data/rust-vendor/html-to-markdown-rs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "html-to-markdown-rs"
-version = "2.23.0"
+version = "2.26.2"
 edition = "2024"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
 license = "MIT"
@@ -34,7 +34,7 @@ base64 = "0.22"
 ahash = "0.8"
 html-escape = "0.2.13"
 image = { version = "0.25", default-features = false, features = ["gif", "jpeg", "png", "bmp", "webp"], optional = true }
-html5ever = "0.36"
+html5ever = "0.38.0"
 lru = "0.16"
 serde = { version = "1.0", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }

data/rust-vendor/html-to-markdown-rs/src/convert_api.rs

@@ -255,7 +255,11 @@ pub fn convert_with_metadata(
     use std::cell::RefCell;
     use std::rc::Rc;
 
-    let options = options.unwrap_or_default();
+    // Disable YAML frontmatter prepending: metadata is returned as a struct,
+    // so embedding it in the content string is redundant and pollutes the output.
+    let mut options = options.unwrap_or_default();
+    options.extract_metadata = false;
+
     let normalized_html = normalize_input(html)?;
     if !metadata_cfg.any_enabled() {
         #[cfg(feature = "visitor")]

data/rust-vendor/html-to-markdown-rs/src/converter/block/paragraph.rs

@@ -61,6 +61,7 @@ pub(crate) fn handle(
 
     let p_ctx = Context {
         in_paragraph: true,
+        block_content_start: output.len(),
         ..ctx.clone()
     };
 

data/rust-vendor/html-to-markdown-rs/src/converter/context.rs

@@ -63,6 +63,10 @@ pub struct Context {
     pub(crate) heading_allow_inline_images: bool,
     /// Are we inside a paragraph element?
     pub(crate) in_paragraph: bool,
+    /// Output buffer position where the current block's content starts.
+    /// Used to distinguish paragraph-break newlines from a previous block
+    /// vs. newlines generated within the current block.
+    pub(crate) block_content_start: usize,
     /// Are we inside a ruby element?
     pub(crate) in_ruby: bool,
     /// Are we inside a `<strong>` / `<b>` element?
@@ -152,6 +156,7 @@ impl Context {
             in_heading: false,
             heading_allow_inline_images: false,
             in_paragraph: false,
+            block_content_start: 0,
             in_ruby: false,
             in_strong: false,
             in_link: false,

data/rust-vendor/html-to-markdown-rs/src/converter/text_node.rs

@@ -178,7 +178,12 @@ pub fn process_text_node(
         if !suffix.is_empty() {
             final_text.push_str(suffix);
         } else if has_trailing_single_newline {
-            let at_paragraph_break = output.ends_with("\n\n");
+            // Check if the "\n\n" at the end of the output buffer came from within
+            // the current block's content, not from a previous block's closing.
+            // Without this distinction, the second paragraph after a "\n\n" boundary
+            // would incorrectly suppress the trailing space before inline elements.
+            let current_block_output = &output[ctx.block_content_start..];
+            let at_paragraph_break = current_block_output.ends_with("\n\n");
             if !at_paragraph_break {
                 if has_double_newline {
                     final_text.push('\n');

data/rust-vendor/html-to-markdown-rs/src/converter/visitor_hooks.rs

@@ -151,17 +151,23 @@ pub fn handle_visitor_element_end(
         is_inline: !is_block_level_element(tag_name),
     };
 
-    let element_content = &output[element_output_start..];
+    // When a child element's visitor returns Custom or Skip, the handler
+    // truncates `output` back to that child's `element_output_start`.  This
+    // can make the *parent* element's saved `element_output_start` stale —
+    // pointing past the end of the now-shorter `output`.  Clamp to prevent
+    // a panic on the string slice.
+    let safe_start = element_output_start.min(output.len());
+    let element_content = &output[safe_start..];
 
     let mut visitor = visitor_handle.borrow_mut();
     match visitor.visit_element_end(&node_ctx, element_content) {
         VisitResult::Continue => {}
         VisitResult::Custom(custom) => {
-            output.truncate(element_output_start);
+            output.truncate(safe_start);
             output.push_str(&custom);
         }
         VisitResult::Skip => {
-            output.truncate(element_output_start);
+            output.truncate(safe_start);
         }
         VisitResult::Error(err) => {
             if ctx.visitor_error.borrow().is_none() {

data/rust-vendor/html-to-markdown-rs/tests/convert_with_metadata_no_frontmatter.rs

@@ -0,0 +1,100 @@
+#![allow(missing_docs)]
+
+//! Regression tests: `convert_with_metadata` must never prepend YAML frontmatter
+//! to the markdown string. Metadata is returned as a struct, so embedding it
+//! in the content is redundant and pollutes the output.
+
+use html_to_markdown_rs::ConversionOptions;
+use html_to_markdown_rs::metadata::MetadataConfig;
+
+#[test]
+fn convert_with_metadata_omits_yaml_frontmatter_default_options() {
+    let html = r#"
+        <html>
+          <head>
+            <title>My Page Title</title>
+            <meta name="description" content="A page description">
+            <meta name="author" content="Jane Doe">
+          </head>
+          <body><p>Hello world</p></body>
+        </html>
+    "#;
+
+    let (markdown, metadata) = html_to_markdown_rs::convert_with_metadata(html, None, MetadataConfig::default(), None)
+        .expect("convert_with_metadata failed");
+
+    // Metadata struct should contain the extracted data
+    assert!(
+        metadata.document.title.is_some(),
+        "metadata.document.title should be populated"
+    );
+
+    // Markdown output must NOT contain YAML frontmatter delimiters
+    assert!(
+        !markdown.contains("---"),
+        "markdown should not contain YAML frontmatter delimiters, got:\n{markdown}"
+    );
+    assert!(
+        !markdown.starts_with("---\n"),
+        "markdown should not start with YAML frontmatter, got:\n{markdown}"
+    );
+}
+
+#[test]
+fn convert_with_metadata_omits_frontmatter_even_when_extract_metadata_is_true() {
+    let html = r#"
+        <html>
+          <head>
+            <title>Test Title</title>
+            <meta name="description" content="Test description">
+          </head>
+          <body><p>Content here</p></body>
+        </html>
+    "#;
+
+    // Explicitly pass extract_metadata: true — convert_with_metadata should override it
+    let options = ConversionOptions {
+        extract_metadata: true,
+        ..Default::default()
+    };
+
+    let (markdown, metadata) =
+        html_to_markdown_rs::convert_with_metadata(html, Some(options), MetadataConfig::default(), None)
+            .expect("convert_with_metadata failed");
+
+    assert!(
+        metadata.document.title.is_some(),
+        "metadata struct should still contain title"
+    );
+    assert!(
+        !markdown.contains("---"),
+        "YAML frontmatter must not appear even when extract_metadata was explicitly true, got:\n{markdown}"
+    );
+}
+
+#[test]
+fn convert_with_metadata_body_content_is_clean() {
+    let html = r#"
+        <html>
+          <head>
+            <title>Page</title>
+            <meta name="keywords" content="rust, html, markdown">
+          </head>
+          <body><h1>Heading</h1><p>Paragraph text.</p></body>
+        </html>
+    "#;
+
+    let (markdown, _metadata) = html_to_markdown_rs::convert_with_metadata(html, None, MetadataConfig::default(), None)
+        .expect("convert_with_metadata failed");
+
+    // The output should start with the body content, not frontmatter
+    let trimmed = markdown.trim();
+    assert!(
+        trimmed.starts_with('#') || trimmed.starts_with("Heading"),
+        "markdown should start with body content, got:\n{markdown}"
+    );
+    assert!(
+        trimmed.contains("Paragraph text."),
+        "markdown should contain paragraph text, got:\n{markdown}"
+    );
+}

data/rust-vendor/html-to-markdown-rs/tests/issue_212_regressions.rs

@@ -0,0 +1,63 @@
+#![allow(missing_docs)]
+
+use html_to_markdown_rs::convert;
+
+/// Regression test for https://github.com/kreuzberg-dev/html-to-markdown/issues/212
+///
+/// When `\n` precedes an `<a>` tag inside a `<p>`, the whitespace was
+/// inconsistently handled between the first and subsequent paragraphs.
+/// The bug was stateful: identical HTML structures produced different
+/// results depending on their position in the document.
+#[test]
+fn consistent_whitespace_before_link_across_paragraphs_issue_212() {
+    let html = r#"<p>text before
+<a href="https://example.com">the link</a>
+after</p>
+<p>text before
+<a href="https://example.com">the link</a>
+after</p>"#;
+
+    let result = convert(html, None).unwrap();
+    assert_eq!(
+        result,
+        "text before [the link](https://example.com) after\n\ntext before [the link](https://example.com) after\n"
+    );
+}
+
+/// Same bug but with three paragraphs to ensure no accumulation effects.
+#[test]
+fn consistent_whitespace_three_paragraphs_issue_212() {
+    let html = r#"<p>click
+<a href="/a">here</a></p>
+<p>click
+<a href="/b">here</a></p>
+<p>click
+<a href="/c">here</a></p>"#;
+
+    let result = convert(html, None).unwrap();
+    assert_eq!(result, "click [here](/a)\n\nclick [here](/b)\n\nclick [here](/c)\n");
+}
+
+/// Verify the fix doesn't break whitespace between text nodes without links.
+#[test]
+fn newline_before_inline_elements_consistent_issue_212() {
+    let html = r#"<p>before
+<strong>bold</strong> after</p>
+<p>before
+<strong>bold</strong> after</p>"#;
+
+    let result = convert(html, None).unwrap();
+    assert_eq!(result, "before **bold** after\n\nbefore **bold** after\n");
+}
+
+/// Verify with `<em>` tags across multiple paragraphs.
+#[test]
+fn newline_before_em_across_paragraphs_issue_212() {
+    let html = r#"<p>some text
+<em>emphasized</em> end</p>
+<p>some text
+<em>emphasized</em> end</p>"#;
+
+    let result = convert(html, None).unwrap();
+    assert_eq!(result, "some text *emphasized* end\n\nsome text *emphasized* end\n");
+}