html-to-markdown 2.27.2 → 2.27.3

data/rust-vendor/tokio/tests/io_async_fd_memory_leak.rs

@@ -0,0 +1,209 @@
+//! Regression test for issue #7563 - Memory leak when fd closed before AsyncFd drop
+//!
+//! This test uses a custom global allocator to track actual memory usage,
+//! avoiding false positives from RSS measurements which include freed-but-retained memory.
+
+#![cfg(all(unix, target_os = "linux", feature = "full"))]
+
+use std::alloc::{GlobalAlloc, Layout, System};
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+/// A tracking allocator that counts bytes currently allocated
+struct TrackingAllocator {
+    allocated: AtomicUsize,
+}
+
+impl TrackingAllocator {
+    const fn new() -> Self {
+        Self {
+            allocated: AtomicUsize::new(0),
+        }
+    }
+}
+
+unsafe impl GlobalAlloc for TrackingAllocator {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        let ptr = unsafe { System.alloc(layout) };
+        if !ptr.is_null() {
+            self.allocated.fetch_add(layout.size(), Ordering::Relaxed);
+        }
+        ptr
+    }
+
+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+        unsafe { System.dealloc(ptr, layout) };
+        self.allocated.fetch_sub(layout.size(), Ordering::Relaxed);
+    }
+
+    unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
+        let new_ptr = unsafe { System.realloc(ptr, layout, new_size) };
+        if !new_ptr.is_null() {
+            // Subtract old size, add new size
+            if new_size > layout.size() {
+                self.allocated
+                    .fetch_add(new_size - layout.size(), Ordering::Relaxed);
+            } else {
+                self.allocated
+                    .fetch_sub(layout.size() - new_size, Ordering::Relaxed);
+            }
+        }
+        new_ptr
+    }
+}
+
+#[global_allocator]
+static GLOBAL: TrackingAllocator = TrackingAllocator::new();
+
+fn allocated_bytes() -> usize {
+    GLOBAL.allocated.load(Ordering::Relaxed)
+}
+
+#[tokio::test]
+async fn memory_leak_when_fd_closed_before_drop() {
+    use nix::sys::socket::{self, AddressFamily, SockFlag, SockType};
+    use std::os::unix::io::{AsRawFd, RawFd};
+    use std::sync::Arc;
+    use tokio::io::unix::AsyncFd;
+
+    struct RawFdWrapper {
+        fd: RawFd,
+    }
+
+    impl AsRawFd for RawFdWrapper {
+        fn as_raw_fd(&self) -> RawFd {
+            self.fd
+        }
+    }
+
+    struct ArcFd(Arc<RawFdWrapper>);
+
+    impl AsRawFd for ArcFd {
+        fn as_raw_fd(&self) -> RawFd {
+            self.0.as_raw_fd()
+        }
+    }
+
+    fn set_nonblocking(fd: RawFd) {
+        use nix::fcntl::{OFlag, F_GETFL, F_SETFL};
+
+        let flags = nix::fcntl::fcntl(fd, F_GETFL).expect("fcntl(F_GETFL)");
+
+        if flags < 0 {
+            panic!(
+                "bad return value from fcntl(F_GETFL): {} ({:?})",
+                flags,
+                nix::Error::last()
+            );
+        }
+
+        let flags = OFlag::from_bits_truncate(flags) | OFlag::O_NONBLOCK;
+
+        nix::fcntl::fcntl(fd, F_SETFL(flags)).expect("fcntl(F_SETFL)");
+    }
+
+    // Warm up - let runtime and allocator stabilize
+    for _ in 0..100 {
+        tokio::task::yield_now().await;
+    }
+
+    const ITERATIONS: usize = 1000;
+
+    // Phase 1: Warm up allocations
+    for _ in 0..ITERATIONS {
+        let (fd_a, _fd_b) = socket::socketpair(
+            AddressFamily::Unix,
+            SockType::Stream,
+            None,
+            SockFlag::empty(),
+        )
+        .unwrap();
+
+        let raw_fd = fd_a.as_raw_fd();
+        set_nonblocking(raw_fd);
+        std::mem::forget(fd_a);
+
+        let wrapper = Arc::new(RawFdWrapper { fd: raw_fd });
+        let async_fd = AsyncFd::new(ArcFd(wrapper)).unwrap();
+
+        // Close fd before dropping AsyncFd - this triggers the bug
+        unsafe {
+            libc::close(raw_fd);
+        }
+
+        drop(async_fd);
+    }
+
+    // Let things settle
+    tokio::task::yield_now().await;
+    let baseline = allocated_bytes();
+
+    // Phase 2: Run more iterations and check for growth
+    for _ in 0..ITERATIONS {
+        let (fd_a, _fd_b) = socket::socketpair(
+            AddressFamily::Unix,
+            SockType::Stream,
+            None,
+            SockFlag::empty(),
+        )
+        .unwrap();
+
+        let raw_fd = fd_a.as_raw_fd();
+        set_nonblocking(raw_fd);
+        std::mem::forget(fd_a);
+
+        let wrapper = Arc::new(RawFdWrapper { fd: raw_fd });
+        let async_fd = AsyncFd::new(ArcFd(wrapper)).unwrap();
+
+        unsafe {
+            libc::close(raw_fd);
+        }
+
+        drop(async_fd);
+    }
+
+    tokio::task::yield_now().await;
+    let after_phase2 = allocated_bytes();
+
+    // Phase 3: Run even more iterations
+    for _ in 0..ITERATIONS {
+        let (fd_a, _fd_b) = socket::socketpair(
+            AddressFamily::Unix,
+            SockType::Stream,
+            None,
+            SockFlag::empty(),
+        )
+        .unwrap();
+
+        let raw_fd = fd_a.as_raw_fd();
+        set_nonblocking(raw_fd);
+        std::mem::forget(fd_a);
+
+        let wrapper = Arc::new(RawFdWrapper { fd: raw_fd });
+        let async_fd = AsyncFd::new(ArcFd(wrapper)).unwrap();
+
+        unsafe {
+            libc::close(raw_fd);
+        }
+
+        drop(async_fd);
+    }
+
+    tokio::task::yield_now().await;
+    let after_phase3 = allocated_bytes();
+
+    let growth_phase2 = after_phase2.saturating_sub(baseline);
+    let growth_phase3 = after_phase3.saturating_sub(after_phase2);
+
+    // If there's a leak, each phase adds ~250KB (1000 * ~256 bytes per ScheduledIo)
+    // If fixed, memory should stabilize (minimal growth between phases)
+    // Allow 64KB tolerance for normal allocation variance
+    let threshold = 64 * 1024; // 64KB
+
+    assert!(
+        growth_phase2 < threshold || growth_phase3 < threshold,
+        "Memory leak detected: allocations keep growing without stabilizing. \
+         Phase 1->2: +{growth_phase2} bytes, Phase 2->3: +{growth_phase3} bytes. \
+         (baseline: {baseline} bytes, phase2: {after_phase2} bytes, phase3: {after_phase3} bytes). \
+         Expected at least one phase with <{threshold} bytes growth.",
+    );
+}

data/rust-vendor/tokio/tests/io_write_buf.rs

@@ -54,3 +54,59 @@ async fn write_all() {
     assert_eq!(wr.cnt, 1);
     assert_eq!(buf.position(), 4);
 }
+
+#[tokio::test]
+async fn write_buf_vectored() {
+    struct Wr {
+        buf: BytesMut,
+        cnt: usize,
+    }
+
+    impl AsyncWrite for Wr {
+        fn poll_write(
+            self: Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+            _buf: &[u8],
+        ) -> Poll<io::Result<usize>> {
+            panic!("shouldn't be called")
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Ok(()).into()
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Ok(()).into()
+        }
+
+        fn poll_write_vectored(
+            mut self: Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+            bufs: &[io::IoSlice<'_>],
+        ) -> Poll<Result<usize, io::Error>> {
+            let mut n = 0;
+            for buf in bufs {
+                self.buf.extend_from_slice(buf);
+                n += buf.len();
+            }
+            self.cnt += 1;
+            Ok(n).into()
+        }
+
+        fn is_write_vectored(&self) -> bool {
+            true
+        }
+    }
+
+    let mut wr = Wr {
+        buf: BytesMut::with_capacity(64),
+        cnt: 0,
+    };
+
+    let mut buf = Cursor::new(&b"hello world"[..]);
+
+    assert_ok!(wr.write_buf(&mut buf).await);
+    assert_eq!(wr.buf, b"hello world"[..]);
+    assert_eq!(wr.cnt, 1);
+    assert_eq!(buf.position(), 11);
+}

data/rust-vendor/tokio/tests/process_issue_7144.rs

@@ -17,7 +17,15 @@ async fn issue_7144() {
     }
 }
 
+async fn has_strace() -> bool {
+    Command::new("strace").arg("-V").output().await.is_ok()
+}
+
 async fn test_one() {
+    if !has_strace().await {
+        return;
+    }
+
     let mut t = Command::new("strace")
         .args("-o /dev/null -D sleep 5".split(' '))
         .spawn()

data/rust-vendor/tokio/tests/rt_basic.rs

@@ -456,3 +456,44 @@ fn rt() -> Runtime {
         .build()
         .unwrap()
 }
+
+#[test]
+fn before_park_yields() {
+    use futures::task::ArcWake;
+    use std::sync::Arc;
+    use tokio::runtime::Builder;
+    use tokio::sync::Notify;
+
+    struct MyWaker(Notify);
+
+    impl ArcWake for MyWaker {
+        fn wake_by_ref(arc_self: &Arc<Self>) {
+            arc_self.0.notify_one();
+        }
+    }
+
+    let notify = Arc::new(MyWaker(Notify::new()));
+    let notify2 = notify.clone();
+    let waker = futures::task::waker(notify2);
+    let woken = Arc::new(AtomicBool::new(false));
+    let woken2 = woken.clone();
+
+    let rt = Builder::new_current_thread()
+        .enable_all()
+        .on_thread_park(move || {
+            if !woken2.swap(true, Ordering::SeqCst) {
+                let mut cx = Context::from_waker(&waker);
+                // `yield_now` pushes the waker to the defer slot.
+                let fut = std::pin::pin!(tokio::task::yield_now());
+                let _ = fut.poll(&mut cx);
+            }
+        })
+        .build()
+        .unwrap();
+
+    rt.block_on(async {
+        notify.0.notified().await;
+    });
+
+    assert!(woken.load(Ordering::SeqCst));
+}

data/rust-vendor/tokio/tests/rt_common_before_park.rs

@@ -0,0 +1,92 @@
+#![warn(rust_2018_idioms)]
+#![cfg(feature = "full")]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support threads
+
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use tokio::runtime::Builder;
+use tokio::sync::Notify;
+
+#[test]
+fn before_park_wakes_block_on_task() {
+    let notify = Arc::new(Notify::new());
+    let notify2 = notify.clone();
+    let woken = Arc::new(AtomicBool::new(false));
+    let woken2 = woken.clone();
+
+    let rt = Builder::new_current_thread()
+        .enable_all()
+        .on_thread_park(move || {
+            // Only wake once to avoid busy loop if something goes wrong,
+            // though in this test we expect it to unpark immediately.
+            if !woken2.swap(true, Ordering::SeqCst) {
+                notify2.notify_one();
+            }
+        })
+        .build()
+        .unwrap();
+
+    rt.block_on(async {
+        // This will block until `notify` is notified.
+        // `before_park` should run when the runtime is about to park.
+        // It will notify `notify`, which should wake this task.
+        // The runtime should then see the task is woken and NOT park.
+        notify.notified().await;
+    });
+
+    assert!(woken.load(Ordering::SeqCst));
+}
+
+#[test]
+fn before_park_spawns_task() {
+    let notify = Arc::new(Notify::new());
+    let notify2 = notify.clone();
+    let woken = Arc::new(AtomicBool::new(false));
+    let woken2 = woken.clone();
+
+    let rt = Builder::new_current_thread()
+        .enable_all()
+        .on_thread_park(move || {
+            if !woken2.swap(true, Ordering::SeqCst) {
+                let notify = notify2.clone();
+                tokio::spawn(async move {
+                    notify.notify_one();
+                });
+            }
+        })
+        .build()
+        .unwrap();
+
+    rt.block_on(async {
+        // This will block until `notify` is notified.
+        // `before_park` should run when the runtime is about to park.
+        // It will spawn a task that notifies `notify`.
+        // The runtime should see the new task and NOT park.
+        // If it parks, it will deadlock.
+        notify.notified().await;
+    });
+
+    assert!(woken.load(Ordering::SeqCst));
+}
+
+#[test]
+fn wake_from_other_thread_block_on() {
+    let rt = Builder::new_current_thread().enable_all().build().unwrap();
+    let handle = rt.handle().clone();
+    let notify = Arc::new(Notify::new());
+    let notify2 = notify.clone();
+
+    let th = std::thread::spawn(move || {
+        // Give the main thread time to park
+        std::thread::sleep(std::time::Duration::from_millis(5));
+        handle.block_on(async move {
+            notify2.notify_one();
+        });
+    });
+
+    rt.block_on(async {
+        notify.notified().await;
+    });
+
+    th.join().unwrap();
+}

data/rust-vendor/tokio/tests/rt_metrics.rs

@@ -152,7 +152,7 @@ fn worker_park_unpark_count() {
     let metrics = rt.metrics();
     rt.block_on(rt.spawn(async {})).unwrap();
     drop(rt);
-    assert!(2 <= metrics.worker_park_unpark_count(0));
+    assert_eq!(0, metrics.worker_park_unpark_count(0) % 2);
 
     let rt = threaded();
     let metrics = rt.metrics();

data/rust-vendor/tokio/tests/rt_panic.rs

@@ -82,6 +82,18 @@ fn builder_global_queue_interval_panic_caller() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+#[test]
+fn builder_event_interval_interval_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let _ = Builder::new_multi_thread().event_interval(0).build();
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
 fn current_thread() -> Runtime {
     tokio::runtime::Builder::new_current_thread()
         .enable_all()

data/rust-vendor/tokio/tests/rt_shutdown_err.rs

@@ -0,0 +1,82 @@
+#![warn(rust_2018_idioms)]
+#![cfg(feature = "full")]
+#![cfg(not(miri))] // No socket in miri.
+
+use std::io;
+use tokio::net::TcpListener;
+use tokio::runtime::Builder;
+
+fn rt() -> tokio::runtime::Runtime {
+    Builder::new_current_thread().enable_all().build().unwrap()
+}
+
+#[test]
+fn test_is_rt_shutdown_err() {
+    let rt1 = rt();
+    let rt2 = rt();
+
+    let listener = rt1.block_on(async { TcpListener::bind("127.0.0.1:0").await.unwrap() });
+
+    drop(rt1);
+
+    rt2.block_on(async {
+        let res = listener.accept().await;
+        assert!(res.is_err());
+        let err = res.as_ref().unwrap_err();
+        assert!(tokio::runtime::is_rt_shutdown_err(err));
+    });
+}
+
+#[test]
+fn test_is_not_rt_shutdown_err() {
+    let err = io::Error::new(io::ErrorKind::Other, "some other error");
+    assert!(!tokio::runtime::is_rt_shutdown_err(&err));
+
+    let err = io::Error::new(io::ErrorKind::NotFound, "not found");
+    assert!(!tokio::runtime::is_rt_shutdown_err(&err));
+}
+
+#[test]
+#[cfg_attr(panic = "abort", ignore)]
+fn test_join_error_panic() {
+    let rt = rt();
+    let handle = rt.spawn(async {
+        panic!("oops");
+    });
+
+    let join_err = rt.block_on(handle).unwrap_err();
+    let io_err: io::Error = join_err.into();
+    assert!(!tokio::runtime::is_rt_shutdown_err(&io_err));
+}
+
+#[test]
+fn test_join_error_cancelled() {
+    let rt = rt();
+    let handle = rt.spawn(async {
+        std::future::pending::<()>().await;
+    });
+    handle.abort();
+    let join_err = rt.block_on(handle).unwrap_err();
+    let io_err: io::Error = join_err.into();
+    assert!(!tokio::runtime::is_rt_shutdown_err(&io_err));
+}
+
+#[test]
+fn test_other_error_kinds_and_strings() {
+    // TimedOut
+    let err = io::Error::new(io::ErrorKind::TimedOut, "timed out");
+    assert!(!tokio::runtime::is_rt_shutdown_err(&err));
+
+    // Interrupted
+    let err = io::Error::from(io::ErrorKind::Interrupted);
+    assert!(!tokio::runtime::is_rt_shutdown_err(&err));
+
+    // String that contains the shutdown message but has a prefix/suffix
+    let msg = "A Tokio 1.x context was found, but it is being shutdown. (extra info)";
+    let err = io::Error::new(io::ErrorKind::Other, msg);
+    assert!(!tokio::runtime::is_rt_shutdown_err(&err));
+
+    let msg = "Error: A Tokio 1.x context was found, but it is being shutdown.";
+    let err = io::Error::new(io::ErrorKind::Other, msg);
+    assert!(!tokio::runtime::is_rt_shutdown_err(&err));
+}

data/rust-vendor/tokio/tests/rt_threaded.rs

@@ -6,7 +6,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream};
 use tokio::runtime;
 use tokio::sync::oneshot;
-use tokio_test::{assert_err, assert_ok};
+use tokio_test::{assert_err, assert_ok, assert_pending};
 
 use std::future::{poll_fn, Future};
 use std::pin::Pin;
@@ -692,6 +692,54 @@ fn mutex_in_block_in_place() {
     })
 }
 
+#[test]
+/// Deferred tasks should be woken before starting the [`tokio::task::block_in_place`]
+// https://github.com/tokio-rs/tokio/issues/7877
+fn wake_deferred_tasks_before_block_in_place() {
+    let (tx1, rx1) = oneshot::channel::<()>();
+    let (tx2, rx2) = oneshot::channel::<()>();
+
+    let deferred_task = tokio_test::task::spawn(tokio::task::yield_now());
+    let deffered_task = Arc::new(Mutex::new(deferred_task));
+
+    let rt = runtime::Builder::new_multi_thread()
+        .worker_threads(1)
+        .build()
+        .unwrap();
+
+    let jh = {
+        let deferred_task = Arc::clone(&deffered_task);
+        rt.spawn(async move {
+            {
+                let mut lock = deferred_task.lock().unwrap();
+                assert_pending!(lock.poll());
+            }
+            tokio::task::block_in_place(|| {
+                // signal that the `block_in_place` has started
+                tx2.send(()).unwrap();
+                // wait for the shutdown signal
+                rx1.blocking_recv().unwrap();
+            });
+        })
+    };
+
+    // wait for the `block_in_place` to start
+    rx2.blocking_recv().unwrap();
+
+    // check that the deferred task was woken before the `block_in_place` ends
+    let is_woken = {
+        let lock = deffered_task.lock().unwrap();
+        lock.is_woken()
+    };
+
+    // signal the `block_in_place` to shutdown
+    tx1.send(()).unwrap();
+
+    rt.block_on(jh).unwrap();
+
+    assert!(is_woken);
+}
+
 // Testing the tuning logic is tricky as it is inherently timing based, and more
 // of a heuristic than an exact behavior. This test checks that the interval
 // changes over time based on load factors. There are no assertions, completion

data/rust-vendor/tokio/tests/rt_unstable_metrics.rs

@@ -62,6 +62,38 @@ fn num_idle_blocking_threads() {
     assert_eq!(1, rt.metrics().num_idle_blocking_threads());
 }
 
+#[test]
+fn num_idle_blocking_threads_is_zero_after_shutdown() {
+    let rt = current_thread();
+    let handle = rt.handle().clone();
+
+    // Spawn a blocking task to create a worker thread.
+    let _ = rt.block_on(rt.spawn_blocking(move || {}));
+
+    // Wait for the thread to become idle.
+    rt.block_on(async {
+        time::sleep(Duration::from_millis(5)).await;
+    });
+    if handle.metrics().num_idle_blocking_threads() == 0 {
+        rt.block_on(async {
+            time::sleep(Duration::from_secs(1)).await;
+        });
+    }
+    assert_eq!(1, handle.metrics().num_idle_blocking_threads());
+
+    // Drop the runtime, which triggers shutdown and joins all blocking
+    // threads. Before the fix for #6439, the shutdown path incremented
+    // num_idle_threads a second time for each idle worker, so this
+    // counter stayed at 1 instead of going back to 0.
+    drop(rt);
+
+    assert_eq!(
+        0,
+        handle.metrics().num_idle_blocking_threads(),
+        "num_idle_blocking_threads should be 0 after shutdown (see #6439)"
+    );
+}
+
 #[test]
 fn blocking_queue_depth() {
     let rt = tokio::runtime::Builder::new_current_thread()

data/rust-vendor/tokio/tests/tcp_connect.rs

@@ -181,7 +181,6 @@ mod linux {
     use std::{net, thread};
 
     #[tokio::test]
-    #[expect(deprecated)] // set_linger is deprecated
     fn poll_hup() {
         let addr = assert_ok!("127.0.0.1:0".parse());
         let mut srv = assert_ok!(TcpListener::bind(&addr));
@@ -189,7 +188,7 @@ mod linux {
 
         tokio::spawn(async move {
             let (mut client, _) = assert_ok!(srv.accept().await);
-            assert_ok!(client.set_linger(Some(Duration::from_millis(0))));
+            assert_ok!(client.set_zero_linger());
             assert_ok!(client.write_all(b"hello world").await);
 
             // TODO: Drop?
@@ -198,7 +197,7 @@ mod linux {
         /*
         let t = thread::spawn(move || {
             let mut client = assert_ok!(srv.accept()).0;
-            client.set_linger(Some(Duration::from_millis(0))).unwrap();
+            client.set_zero_linger().unwrap();
             client.write(b"hello world").unwrap();
             thread::sleep(Duration::from_millis(200));
         });

data/rust-vendor/tokio/tests/tcp_shutdown.rs

@@ -2,7 +2,6 @@
 #![cfg(all(feature = "full", not(target_os = "wasi"), not(miri)))] // Wasi doesn't support bind
                                                                    // No `socket` on miri.
 
-use std::time::Duration;
 use tokio::io::{self, AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream};
 use tokio::sync::oneshot::channel;
@@ -33,7 +32,6 @@ async fn shutdown() {
 }
 
 #[tokio::test]
-#[expect(deprecated)] // set_linger is deprecated
 async fn shutdown_after_tcp_reset() {
     let srv = assert_ok!(TcpListener::bind("127.0.0.1:0").await);
     let addr = assert_ok!(srv.local_addr());
@@ -51,7 +49,7 @@ async fn shutdown_after_tcp_reset() {
 
     let (stream, _) = assert_ok!(srv.accept().await);
     // By setting linger to 0 we will trigger a TCP reset
-    stream.set_linger(Some(Duration::new(0, 0))).unwrap();
+    stream.set_zero_linger().unwrap();
     connected_rx.await.unwrap();
 
     drop(stream);