wreq-rb 0.3.0

data/vendor/wreq/tests/retry.rs

@@ -0,0 +1,135 @@
+mod support;
+
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+
+use support::server;
+use wreq::Client;
+
+#[tokio::test]
+async fn retries_apply_in_scope() {
+    let _ = pretty_env_logger::try_init();
+
+    let cnt = Arc::new(AtomicUsize::new(0));
+    let server = server::http(move |_req| {
+        let cnt = cnt.clone();
+        async move {
+            if cnt.fetch_add(1, Ordering::Relaxed) == 0 {
+                // first req is bad
+                http::Response::builder()
+                    .status(http::StatusCode::SERVICE_UNAVAILABLE)
+                    .body(Default::default())
+                    .unwrap()
+            } else {
+                http::Response::default()
+            }
+        }
+    });
+
+    let scope = server.addr().ip().to_string();
+    let policy = wreq::retry::Policy::for_host(scope).classify_fn(|req_rep| {
+        if req_rep.status() == Some(http::StatusCode::SERVICE_UNAVAILABLE) {
+            req_rep.retryable()
+        } else {
+            req_rep.success()
+        }
+    });
+
+    let url = format!("http://{}", server.addr());
+    let resp = Client::builder()
+        .retry(policy)
+        .build()
+        .unwrap()
+        .get(url)
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(resp.status(), 200);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn default_retries_have_a_limit() {
+    let _ = pretty_env_logger::try_init();
+
+    let server = server::http_with_config(
+        move |req| async move {
+            assert_eq!(req.version(), http::Version::HTTP_2);
+            // refused forever
+            Err(http2::Error::from(http2::Reason::REFUSED_STREAM))
+        },
+        |_| {},
+    );
+
+    let client = Client::builder().http2_only().build().unwrap();
+
+    let url = format!("http://{}", server.addr());
+
+    let _err = client.get(url).send().await.unwrap_err();
+}
+
+// NOTE: using the default "current_thread" runtime here would cause the test to
+// fail, because the only thread would block until `panic_rx` receives a
+// notification while the client needs to be driven to get the graceful shutdown
+// done.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn highly_concurrent_requests_to_http2_server_with_low_max_concurrent_streams() {
+    let client = Client::builder().http2_only().no_proxy().build().unwrap();
+
+    let server = server::http_with_config(
+        move |req| async move {
+            assert_eq!(req.version(), http::Version::HTTP_2);
+            Ok::<_, std::convert::Infallible>(http::Response::default())
+        },
+        |builder| {
+            builder.http2().max_concurrent_streams(1);
+        },
+    );
+
+    let url = format!("http://{}", server.addr());
+
+    let futs = (0..100).map(|_| {
+        let client = client.clone();
+        let url = url.clone();
+        async move {
+            let res = client.get(&url).send().await.unwrap();
+            assert_eq!(res.status(), wreq::StatusCode::OK);
+        }
+    });
+    futures_util::future::join_all(futs).await;
+}
+
+#[tokio::test]
+async fn highly_concurrent_requests_to_slow_http2_server_with_low_max_concurrent_streams() {
+    use support::delay_server;
+
+    let client = Client::builder().http2_only().no_proxy().build().unwrap();
+
+    let server = delay_server::Server::new(
+        move |req| async move {
+            assert_eq!(req.version(), http::Version::HTTP_2);
+            http::Response::default()
+        },
+        |http| {
+            http.http2().max_concurrent_streams(1);
+        },
+        std::time::Duration::from_secs(2),
+    )
+    .await;
+
+    let url = format!("http://{}", server.addr());
+
+    let futs = (0..100).map(|_| {
+        let client = client.clone();
+        let url = url.clone();
+        async move {
+            let res = client.get(&url).send().await.unwrap();
+            assert_eq!(res.status(), wreq::StatusCode::OK);
+        }
+    });
+    futures_util::future::join_all(futs).await;
+
+    server.shutdown().await;
+}

data/vendor/wreq/tests/support/delay_server.rs

@@ -0,0 +1,117 @@
+#![allow(unused)]
+use std::{convert::Infallible, future::Future, net, time::Duration};
+
+use futures_util::FutureExt;
+use http::{Request, Response};
+use hyper::service::service_fn;
+use tokio::{net::TcpListener, select, sync::oneshot};
+
+/// This server, unlike [`super::server::Server`], allows for delaying the
+/// specified amount of time after each TCP connection is established. This is
+/// useful for testing the behavior of the client when the server is slow.
+///
+/// For example, in case of HTTP/2, once the TCP/TLS connection is established,
+/// both endpoints are supposed to send a preface and an initial `SETTINGS`
+/// frame (See [RFC9113 3.4] for details). What if these frames are delayed for
+/// whatever reason? This server allows for testing such scenarios.
+///
+/// [RFC9113 3.4]: https://www.rfc-editor.org/rfc/rfc9113.html#name-http-2-connection-preface
+pub struct Server {
+    addr: net::SocketAddr,
+    shutdown_tx: Option<oneshot::Sender<()>>,
+    server_terminated_rx: oneshot::Receiver<()>,
+}
+
+type Builder = hyper_util::server::conn::auto::Builder<hyper_util::rt::TokioExecutor>;
+
+impl Server {
+    pub async fn new<F1, Fut, F2, Bu>(func: F1, apply_config: F2, delay: Duration) -> Self
+    where
+        F1: Fn(Request<hyper::body::Incoming>) -> Fut + Clone + Send + 'static,
+        Fut: Future<Output = Response<wreq::Body>> + Send + 'static,
+        F2: FnOnce(&mut Builder) -> Bu + Send + 'static,
+    {
+        let (shutdown_tx, shutdown_rx) = oneshot::channel();
+        let (server_terminated_tx, server_terminated_rx) = oneshot::channel();
+
+        let tcp_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = tcp_listener.local_addr().unwrap();
+
+        tokio::spawn(async move {
+            let mut builder =
+                hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new());
+            apply_config(&mut builder);
+
+            tokio::spawn(async move {
+                let builder = builder;
+                let (connection_shutdown_tx, connection_shutdown_rx) = oneshot::channel();
+                let connection_shutdown_rx = connection_shutdown_rx.shared();
+                let mut shutdown_rx = std::pin::pin!(shutdown_rx);
+
+                let mut handles = Vec::new();
+                loop {
+                    select! {
+                        _ = shutdown_rx.as_mut() => {
+                            connection_shutdown_tx.send(()).unwrap();
+                            break;
+                        }
+                        res = tcp_listener.accept() => {
+                            let (stream, _) = res.unwrap();
+                            let io = hyper_util::rt::TokioIo::new(stream);
+
+
+                            let handle = tokio::spawn({
+                                let connection_shutdown_rx = connection_shutdown_rx.clone();
+                                let func = func.clone();
+                                let svc = service_fn(move |req| {
+                                    let fut = func(req);
+                                    async move {
+                                    Ok::<_, Infallible>(fut.await)
+                                }});
+                                let builder = builder.clone();
+
+                                async move {
+                                    let fut = builder.serve_connection_with_upgrades(io, svc);
+                                    tokio::time::sleep(delay).await;
+
+                                    let mut conn = std::pin::pin!(fut);
+
+                                    select! {
+                                        _ = conn.as_mut() => {}
+                                        _ = connection_shutdown_rx => {
+                                            conn.as_mut().graceful_shutdown();
+                                            conn.await.unwrap();
+                                        }
+                                    }
+                                }
+                            });
+
+                            handles.push(handle);
+                        }
+                    }
+                }
+
+                futures_util::future::join_all(handles).await;
+                server_terminated_tx.send(()).unwrap();
+            });
+        });
+
+        Self {
+            addr,
+            shutdown_tx: Some(shutdown_tx),
+            server_terminated_rx,
+        }
+    }
+
+    pub async fn shutdown(mut self) {
+        if let Some(tx) = self.shutdown_tx.take() {
+            let _ = tx.send(());
+        }
+
+        self.server_terminated_rx.await.unwrap();
+    }
+
+    pub fn addr(&self) -> net::SocketAddr {
+        self.addr
+    }
+}

data/vendor/wreq/tests/support/error.rs

@@ -0,0 +1,16 @@
+use std::error::Error as StdError;
+
+#[allow(unused)]
+pub fn inspect<E>(err: E) -> Vec<String>
+where
+    E: Into<Box<dyn StdError + Send + Sync>>,
+{
+    let berr = err.into();
+    let mut err = Some(&*berr as &(dyn StdError + 'static));
+    let mut errs = Vec::new();
+    while let Some(e) = err {
+        errs.push(e.to_string());
+        err = e.source();
+    }
+    errs
+}

data/vendor/wreq/tests/support/layer.rs

@@ -0,0 +1,183 @@
+use std::{
+    future::Future,
+    pin::Pin,
+    task::{Context, Poll},
+    time::Duration,
+};
+
+use futures::future::BoxFuture;
+use pin_project_lite::pin_project;
+use tokio::time::Sleep;
+use tower::{BoxError, Layer, Service};
+
+/// This tower layer injects an arbitrary delay before calling downstream layers.
+#[derive(Clone)]
+pub struct DelayLayer {
+    delay: Duration,
+}
+
+impl DelayLayer {
+    #[allow(unused)]
+    pub const fn new(delay: Duration) -> Self {
+        DelayLayer { delay }
+    }
+}
+
+impl<S> Layer<S> for DelayLayer {
+    type Service = Delay<S>;
+    fn layer(&self, service: S) -> Self::Service {
+        Delay::new(service, self.delay)
+    }
+}
+
+impl std::fmt::Debug for DelayLayer {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        f.debug_struct("DelayLayer")
+            .field("delay", &self.delay)
+            .finish()
+    }
+}
+
+/// This tower service injects an arbitrary delay before calling downstream layers.
+#[derive(Debug, Clone)]
+pub struct Delay<S> {
+    inner: S,
+    delay: Duration,
+}
+impl<S> Delay<S> {
+    pub fn new(inner: S, delay: Duration) -> Self {
+        Delay { inner, delay }
+    }
+}
+
+impl<S, Request> Service<Request> for Delay<S>
+where
+    S: Service<Request>,
+    S::Error: Into<BoxError>,
+{
+    type Response = S::Response;
+
+    type Error = BoxError;
+
+    type Future = ResponseFuture<S::Future>;
+
+    fn poll_ready(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        println!("Delay::poll_ready called");
+        match self.inner.poll_ready(cx) {
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(r) => Poll::Ready(r.map_err(Into::into)),
+        }
+    }
+
+    fn call(&mut self, req: Request) -> Self::Future {
+        println!("Delay::call executed");
+        let response = self.inner.call(req);
+        let sleep = tokio::time::sleep(self.delay);
+
+        ResponseFuture::new(response, sleep)
+    }
+}
+
+// `Delay` response future
+pin_project! {
+    #[derive(Debug)]
+    pub struct ResponseFuture<S> {
+        #[pin]
+        response: S,
+        #[pin]
+        sleep: Sleep,
+    }
+}
+
+impl<S> ResponseFuture<S> {
+    pub(crate) fn new(response: S, sleep: Sleep) -> Self {
+        ResponseFuture { response, sleep }
+    }
+}
+
+impl<F, S, E> Future for ResponseFuture<F>
+where
+    F: Future<Output = Result<S, E>>,
+    E: Into<BoxError>,
+{
+    type Output = Result<S, BoxError>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let this = self.project();
+
+        // First poll the sleep until complete
+        match this.sleep.poll(cx) {
+            Poll::Pending => return Poll::Pending,
+            Poll::Ready(_) => {}
+        }
+
+        // Then poll the inner future
+        match this.response.poll(cx) {
+            Poll::Ready(v) => Poll::Ready(v.map_err(Into::into)),
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct SharedConcurrencyLimitLayer {
+    semaphore: std::sync::Arc<tokio::sync::Semaphore>,
+}
+
+impl SharedConcurrencyLimitLayer {
+    #[allow(unused)]
+    pub fn new(limit: usize) -> Self {
+        Self {
+            semaphore: std::sync::Arc::new(tokio::sync::Semaphore::new(limit)),
+        }
+    }
+}
+
+impl<S> tower::Layer<S> for SharedConcurrencyLimitLayer {
+    type Service = SharedConcurrencyLimit<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        SharedConcurrencyLimit {
+            inner,
+            semaphore: self.semaphore.clone(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct SharedConcurrencyLimit<S> {
+    inner: S,
+    semaphore: std::sync::Arc<tokio::sync::Semaphore>,
+}
+
+impl<S, Req> tower::Service<Req> for SharedConcurrencyLimit<S>
+where
+    S: tower::Service<Req> + Clone + Send + 'static,
+    S::Future: Send + 'static,
+    Req: Send + 'static,
+{
+    type Response = S::Response;
+    type Error = S::Error;
+    type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
+
+    fn poll_ready(
+        &mut self,
+        _cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        // always ready, we handle limits in call
+        std::task::Poll::Ready(Ok(()))
+    }
+
+    fn call(&mut self, req: Req) -> Self::Future {
+        let semaphore = self.semaphore.clone();
+        let mut inner = self.inner.clone();
+
+        Box::pin(async move {
+            let _permit = semaphore.acquire_owned().await.unwrap();
+            inner.call(req).await
+        })
+    }
+}

data/vendor/wreq/tests/support/mod.rs

@@ -0,0 +1,9 @@
+pub mod delay_server;
+pub mod error;
+pub mod layer;
+pub mod server;
+
+// TODO: remove once done converting to new support server?
+#[allow(unused)]
+pub static DEFAULT_USER_AGENT: &str =
+    concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"));

data/vendor/wreq/tests/support/server.rs

@@ -0,0 +1,232 @@
+use std::{
+    convert::Infallible, future::Future, io, net, sync::mpsc as std_mpsc, thread, time::Duration,
+};
+
+use tokio::{io::AsyncReadExt, net::TcpStream, runtime, sync::oneshot};
+use wreq::Body;
+
+pub struct Server {
+    addr: net::SocketAddr,
+    panic_rx: std_mpsc::Receiver<()>,
+    events_rx: std_mpsc::Receiver<Event>,
+    shutdown_tx: Option<oneshot::Sender<()>>,
+}
+
+#[non_exhaustive]
+pub enum Event {
+    ConnectionClosed,
+}
+
+impl Server {
+    pub fn addr(&self) -> net::SocketAddr {
+        self.addr
+    }
+
+    #[allow(unused)]
+    pub fn events(&mut self) -> Vec<Event> {
+        let mut events = Vec::new();
+        while let Ok(event) = self.events_rx.try_recv() {
+            events.push(event);
+        }
+        events
+    }
+}
+
+impl Drop for Server {
+    fn drop(&mut self) {
+        if let Some(tx) = self.shutdown_tx.take() {
+            let _ = tx.send(());
+        }
+
+        if !::std::thread::panicking() {
+            self.panic_rx
+                .recv_timeout(Duration::from_secs(3))
+                .expect("test server should not panic");
+        }
+    }
+}
+
+#[allow(unused)]
+pub fn http<F, Fut>(func: F) -> Server
+where
+    F: Fn(http::Request<hyper::body::Incoming>) -> Fut + Clone + Send + 'static,
+    Fut: Future<Output = http::Response<Body>> + Send + 'static,
+{
+    let infall = move |req| {
+        let fut = func(req);
+        async move { Ok::<_, Infallible>(fut.await) }
+    };
+    http_with_config(infall, |_builder| {})
+}
+
+type Builder = hyper_util::server::conn::auto::Builder<hyper_util::rt::TokioExecutor>;
+
+pub fn http_with_config<F1, Fut, E, F2, Bu>(func: F1, apply_config: F2) -> Server
+where
+    F1: Fn(http::Request<hyper::body::Incoming>) -> Fut + Clone + Send + 'static,
+    Fut: Future<Output = Result<http::Response<Body>, E>> + Send + 'static,
+    E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    F2: FnOnce(&mut Builder) -> Bu + Send + 'static,
+{
+    // Spawn new runtime in thread to prevent reactor execution context conflict
+    let test_name = thread::current().name().unwrap_or("<unknown>").to_string();
+    thread::spawn(move || {
+        let rt = runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("new rt");
+        let listener = rt.block_on(async move {
+            tokio::net::TcpListener::bind(&std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
+                .await
+                .unwrap()
+        });
+        let addr = listener.local_addr().unwrap();
+
+        let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
+        let (panic_tx, panic_rx) = std_mpsc::channel();
+        let (events_tx, events_rx) = std_mpsc::channel();
+        let tname = format!(
+            "test({test_name})-support-server",
+        );
+        thread::Builder::new()
+            .name(tname)
+            .spawn(move || {
+                rt.block_on(async move {
+                    let mut builder =
+                        hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new());
+                    apply_config(&mut builder);
+
+                    loop {
+                        tokio::select! {
+                            _ = &mut shutdown_rx => {
+                                break;
+                            }
+                            accepted = listener.accept() => {
+                                let (io, _) = accepted.expect("accepted");
+                                let func = func.clone();
+                                let svc = hyper::service::service_fn(func);
+                                let builder = builder.clone();
+                                let events_tx = events_tx.clone();
+                                tokio::spawn(async move {
+                                    let _ = builder.serve_connection_with_upgrades(hyper_util::rt::TokioIo::new(io), svc).await;
+                                    let _ = events_tx.send(Event::ConnectionClosed);
+                                });
+                            }
+                        }
+                    }
+                    let _ = panic_tx.send(());
+                });
+            })
+            .expect("thread spawn");
+        Server {
+            addr,
+            panic_rx,
+            events_rx,
+            shutdown_tx: Some(shutdown_tx),
+        }
+    })
+    .join()
+    .unwrap()
+}
+
+#[allow(unused)]
+pub fn low_level_with_response<F>(do_response: F) -> Server
+where
+    for<'c> F: Fn(&'c [u8], &'c mut TcpStream) -> Box<dyn Future<Output = ()> + Send + 'c>
+        + Clone
+        + Send
+        + 'static,
+{
+    // Spawn new runtime in thread to prevent reactor execution context conflict
+    let test_name = thread::current().name().unwrap_or("<unknown>").to_string();
+    thread::spawn(move || {
+        let rt = runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("new rt");
+        let listener = rt.block_on(async move {
+            tokio::net::TcpListener::bind(&std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
+                .await
+                .unwrap()
+        });
+        let addr = listener.local_addr().unwrap();
+
+        let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
+        let (panic_tx, panic_rx) = std_mpsc::channel();
+        let (events_tx, events_rx) = std_mpsc::channel();
+        let tname = format!("test({test_name})-support-server",);
+        thread::Builder::new()
+            .name(tname)
+            .spawn(move || {
+                rt.block_on(async move {
+                    loop {
+                        tokio::select! {
+                            _ = &mut shutdown_rx => {
+                                break;
+                            }
+                            accepted = listener.accept() => {
+                                let (io, _) = accepted.expect("accepted");
+                                let do_response = do_response.clone();
+                                let events_tx = events_tx.clone();
+                                tokio::spawn(async move {
+                                    low_level_server_client(io, do_response).await;
+                                    let _ = events_tx.send(Event::ConnectionClosed);
+                                });
+                            }
+                        }
+                    }
+                    let _ = panic_tx.send(());
+                });
+            })
+            .expect("thread spawn");
+        Server {
+            addr,
+            panic_rx,
+            events_rx,
+            shutdown_tx: Some(shutdown_tx),
+        }
+    })
+    .join()
+    .unwrap()
+}
+
+#[allow(unused)]
+async fn low_level_server_client<F>(mut client_socket: TcpStream, do_response: F)
+where
+    for<'c> F: Fn(&'c [u8], &'c mut TcpStream) -> Box<dyn Future<Output = ()> + Send + 'c>,
+{
+    loop {
+        let request = low_level_read_http_request(&mut client_socket)
+            .await
+            .expect("read_http_request failed");
+        if request.is_empty() {
+            // connection closed by client
+            break;
+        }
+
+        Box::into_pin(do_response(&request, &mut client_socket)).await;
+    }
+}
+
+#[allow(unused)]
+async fn low_level_read_http_request(client_socket: &mut TcpStream) -> io::Result<Vec<u8>> {
+    let mut buf = Vec::new();
+
+    // Read until the delimiter "\r\n\r\n" is found
+    loop {
+        let mut temp_buffer = [0; 1024];
+        let n = client_socket.read(&mut temp_buffer).await?;
+
+        if n == 0 {
+            break;
+        }
+
+        buf.extend_from_slice(&temp_buffer[..n]);
+
+        if let Some(pos) = buf.windows(4).position(|window| window == b"\r\n\r\n") {
+            return Ok(buf.drain(..pos + 4).collect());
+        }
+    }
+
+    Ok(buf)
+}