wreq-rb 0.3.0

data/vendor/wreq/examples/cert_store.rs

@@ -0,0 +1,77 @@
+use std::time::Duration;
+
+use wreq::{
+    Client,
+    tls::{CertStore, TlsInfo},
+};
+
+/// Certificate Store Example
+///
+/// In most cases, you don't need to manually configure certificate stores. wreq automatically
+/// uses appropriate default certificates:
+/// - With `webpki-roots` feature enabled: Uses Mozilla's maintained root certificate collection
+/// - Without this feature: Uses system default certificate store paths
+///
+/// Manual certificate store configuration is only needed in the following special cases:
+///
+/// ## Scenarios requiring custom certificate store:
+///
+/// ### 1. Self-signed Certificates
+/// - Connect to internal services using self-signed certificates
+/// - Test servers in development environments
+///
+/// ### 2. Enterprise Internal CA
+/// - Add root certificates from enterprise internal certificate authorities
+/// - Access HTTPS services on corporate intranets
+///
+/// ### 3. Certificate Updates and Management
+/// - Dynamically update certificates in the certificate store
+/// - Remove revoked or expired certificates
+///
+/// ### 4. Compliance Requirements
+/// - Special compliance requirements for certain industries or regions
+/// - Need to use specific certificate collections
+///
+/// ### 5. Performance Optimization
+/// - Reduce certificate store size to improve TLS handshake performance
+/// - Include only necessary root certificates
+#[tokio::main]
+async fn main() -> wreq::Result<()> {
+    // Create a client with a custom certificate store using webpki-roots
+    let client = Client::builder()
+        .cert_store(CertStore::from_der_certs(
+            webpki_root_certs::TLS_SERVER_ROOT_CERTS,
+        )?)
+        .build()?;
+
+    // Use the API you're already familiar with
+    client.get("https://www.google.com").send().await?;
+
+    // Self-signed certificate Client
+    // Skip certificate verification for self-signed certificates
+    let client = Client::builder()
+        .tls_info(true)
+        .cert_verification(false)
+        .build()?;
+
+    // Use the API you're already familiar with
+    let resp = client.get("https://self-signed.badssl.com/").send().await?;
+    if let Some(tls_info) = resp.extensions().get::<TlsInfo>() {
+        if let Some(peer_cert_der) = tls_info.peer_certificate() {
+            // Create self-signed certificate Store
+            let self_signed_store = CertStore::from_der_certs(&[peer_cert_der])?;
+
+            // Create a client with self-signed certificate store
+            let client = Client::builder()
+                .cert_store(self_signed_store)
+                .connect_timeout(Duration::from_secs(10))
+                .build()?;
+
+            // Use the API you're already familiar with
+            let resp = client.get("https://self-signed.badssl.com/").send().await?;
+            println!("{}", resp.text().await?);
+        }
+    }
+
+    Ok(())
+}

data/vendor/wreq/examples/connect_via_lower_priority_tokio_runtime.rs

@@ -0,0 +1,258 @@
+// This example demonstrates how to delegate the connect calls, which contain TLS handshakes,
+// to a secondary tokio runtime of lower OS thread priority using a custom tower layer.
+// This helps to ensure that long-running futures during handshake crypto operations don't block
+// other I/O futures.
+//
+// This does introduce overhead of additional threads, channels, extra vtables, etc,
+// so it is best suited to services with large numbers of incoming connections or that
+// are otherwise very sensitive to any blocking futures.  Or, you might want fewer threads
+// and/or to use the current_thread runtime.
+//
+// This is using the `tokio` runtime and certain other dependencies:
+//
+// `tokio = { version = "1", features = ["full"] }`
+// `libc = "0"`
+// `pin-project-lite = "0.2"`
+// `tower = { version = "0.5", default-features = false}`
+
+#[tokio::main]
+async fn main() -> wreq::Result<()> {
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::TRACE)
+        .init();
+    background_threadpool::init_background_runtime();
+    tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+
+    let client = wreq::Client::builder()
+        .connector_layer(background_threadpool::BackgroundProcessorLayer::new())
+        .build()
+        .expect("should be able to build wreq client");
+
+    let url = if let Some(url) = std::env::args().nth(1) {
+        url
+    } else {
+        println!("No CLI URL provided, using default.");
+        "https://hyper.rs".into()
+    };
+
+    eprintln!("Fetching {url:?}...");
+
+    let res = client.get(url).send().await?;
+
+    eprintln!("Response: {:?} {}", res.version(), res.status());
+    eprintln!("Headers: {:#?}\n", res.headers());
+
+    let body = res.text().await?;
+
+    println!("{body}");
+
+    Ok(())
+}
+
+// separating out for convenience to avoid a million
+mod background_threadpool {
+    use std::{
+        future::Future,
+        pin::Pin,
+        sync::OnceLock,
+        task::{Context, Poll},
+    };
+
+    use futures_util::TryFutureExt;
+    use pin_project_lite::pin_project;
+    use tokio::{runtime::Handle, select, sync::mpsc::error::TrySendError};
+    use tower::{BoxError, Layer, Service};
+
+    static CPU_HEAVY_THREAD_POOL: OnceLock<
+        tokio::sync::mpsc::Sender<Pin<Box<dyn Future<Output = ()> + Send + 'static>>>,
+    > = OnceLock::new();
+
+    pub(crate) fn init_background_runtime() {
+        std::thread::Builder::new()
+            .name("cpu-heavy-background-threadpool".to_string())
+            .spawn(move || {
+                let rt = tokio::runtime::Builder::new_multi_thread()
+                    .thread_name("cpu-heavy-background-pool-thread")
+                    .worker_threads(std::thread::available_parallelism().unwrap().get())
+                    // ref: https://github.com/tokio-rs/tokio/issues/4941
+                    // consider uncommenting if seeing heavy task contention
+                    // .disable_lifo_slot()
+                    .on_thread_start(move || {
+                        #[cfg(target_os = "linux")]
+                        unsafe {
+                            // Increase thread pool thread niceness, so they are lower priority
+                            // than the foreground executor and don't interfere with I/O tasks
+                            {
+                                *libc::__errno_location() = 0;
+                                if libc::nice(10) == -1 && *libc::__errno_location() != 0 {
+                                    let error = std::io::Error::last_os_error();
+                                    tracing::error!("failed to set threadpool niceness: {error}");
+                                }
+                            }
+                        }
+                    })
+                    .enable_all()
+                    .build()
+                    .unwrap_or_else(|e| panic!("cpu heavy runtime failed_to_initialize: {e}"));
+                rt.block_on(async {
+                    tracing::debug!("starting background cpu-heavy work");
+                    process_cpu_work().await;
+                });
+            })
+            .unwrap_or_else(|e| panic!("cpu heavy thread failed_to_initialize: {e}"));
+    }
+
+    async fn process_cpu_work() {
+        // we only use this channel for routing work, it should move pretty quick, it can be small
+        let (tx, mut rx) = tokio::sync::mpsc::channel(10);
+        // share the handle to the background channel globally
+        CPU_HEAVY_THREAD_POOL.set(tx).unwrap();
+
+        while let Some(work) = rx.recv().await {
+            tokio::task::spawn(work);
+        }
+    }
+
+    // retrieve the sender to the background channel, and send the future over to it for execution
+    fn send_to_background_runtime(future: impl Future<Output = ()> + Send + 'static) {
+        let tx = CPU_HEAVY_THREAD_POOL.get().expect(
+            "start up the secondary tokio runtime before sending to `CPU_HEAVY_THREAD_POOL`",
+        );
+
+        match tx.try_send(Box::pin(future)) {
+            Ok(_) => (),
+            Err(TrySendError::Closed(_)) => {
+                panic!("background cpu heavy runtime channel is closed")
+            }
+            Err(TrySendError::Full(msg)) => {
+                tracing::warn!(
+                    "background cpu heavy runtime channel is full, task spawning loop delayed"
+                );
+                let tx = tx.clone();
+                Handle::current().spawn(async move {
+                    tx.send(msg)
+                        .await
+                        .expect("background cpu heavy runtime channel is closed")
+                });
+            }
+        }
+    }
+
+    // This tower layer injects futures with a oneshot channel, and then sends them to the
+    // background runtime for processing. We don't use the Buffer service because that is
+    // intended to process sequentially on a single task, whereas we want to spawn a new task
+    // per call.
+    #[derive(Copy, Clone)]
+    pub struct BackgroundProcessorLayer {}
+    impl BackgroundProcessorLayer {
+        pub fn new() -> Self {
+            Self {}
+        }
+    }
+    impl<S> Layer<S> for BackgroundProcessorLayer {
+        type Service = BackgroundProcessor<S>;
+        fn layer(&self, service: S) -> Self::Service {
+            BackgroundProcessor::new(service)
+        }
+    }
+
+    impl std::fmt::Debug for BackgroundProcessorLayer {
+        fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+            f.debug_struct("BackgroundProcessorLayer").finish()
+        }
+    }
+
+    // This tower service injects futures with a oneshot channel, and then sends them to the
+    // background runtime for processing.
+    #[derive(Debug, Clone)]
+    pub struct BackgroundProcessor<S> {
+        inner: S,
+    }
+
+    impl<S> BackgroundProcessor<S> {
+        pub fn new(inner: S) -> Self {
+            BackgroundProcessor { inner }
+        }
+    }
+
+    impl<S, Request> Service<Request> for BackgroundProcessor<S>
+    where
+        S: Service<Request>,
+        S::Response: Send + 'static,
+        S::Error: Into<BoxError> + Send,
+        S::Future: Send + 'static,
+    {
+        type Response = S::Response;
+
+        type Error = BoxError;
+
+        type Future = BackgroundResponseFuture<S::Response>;
+
+        fn poll_ready(
+            &mut self,
+            cx: &mut std::task::Context<'_>,
+        ) -> std::task::Poll<Result<(), Self::Error>> {
+            match self.inner.poll_ready(cx) {
+                Poll::Pending => Poll::Pending,
+                Poll::Ready(r) => Poll::Ready(r.map_err(Into::into)),
+            }
+        }
+
+        fn call(&mut self, req: Request) -> Self::Future {
+            let response = self.inner.call(req);
+
+            // wrap our inner service's future with a future that writes to this oneshot channel
+            let (mut tx, rx) = tokio::sync::oneshot::channel();
+            let future = async move {
+                select!(
+                    _ = tx.closed() => {
+                        // receiver already dropped, don't need to do anything
+                    }
+                    result = response.map_err(Into::<BoxError>::into) => {
+                        // if this fails, the receiver already dropped, so we don't need to do anything
+                        let _ = tx.send(result);
+                    }
+                )
+            };
+            // send the wrapped future to the background
+            send_to_background_runtime(future);
+
+            BackgroundResponseFuture::new(rx)
+        }
+    }
+
+    // `BackgroundProcessor` response future
+    pin_project! {
+        #[derive(Debug)]
+        pub struct BackgroundResponseFuture<S> {
+            #[pin]
+            rx: tokio::sync::oneshot::Receiver<Result<S, BoxError>>,
+        }
+    }
+
+    impl<S> BackgroundResponseFuture<S> {
+        pub(crate) fn new(rx: tokio::sync::oneshot::Receiver<Result<S, BoxError>>) -> Self {
+            BackgroundResponseFuture { rx }
+        }
+    }
+
+    impl<S> Future for BackgroundResponseFuture<S>
+    where
+        S: Send + 'static,
+    {
+        type Output = Result<S, BoxError>;
+
+        fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+            let this = self.project();
+
+            // now poll on the receiver end of the oneshot to get the result
+            match this.rx.poll(cx) {
+                Poll::Ready(v) => match v {
+                    Ok(v) => Poll::Ready(v),
+                    Err(err) => Poll::Ready(Err(Box::new(err) as BoxError)),
+                },
+                Poll::Pending => Poll::Pending,
+            }
+        }
+    }
+}

data/vendor/wreq/examples/emulation.rs

@@ -0,0 +1,118 @@
+use wreq::{
+    Client, Emulation,
+    header::{self, HeaderMap, HeaderValue, OrigHeaderMap},
+    http2::{Http2Options, PseudoId, PseudoOrder},
+    tls::{AlpnProtocol, TlsOptions, TlsVersion},
+};
+
+macro_rules! join {
+    ($sep:expr, $first:expr $(, $rest:expr)*) => {
+        concat!($first $(, $sep, $rest)*)
+    };
+}
+
+#[tokio::main]
+async fn main() -> wreq::Result<()> {
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::TRACE)
+        .init();
+
+    //  TLS options config
+    let tls = TlsOptions::builder()
+        .enable_ocsp_stapling(true)
+        .curves_list(join!(":", "X25519", "P-256", "P-384"))
+        .cipher_list(join!(
+            ":",
+            "TLS_AES_128_GCM_SHA256",
+            "TLS_AES_256_GCM_SHA384",
+            "TLS_CHACHA20_POLY1305_SHA256",
+            "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+            "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+            "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
+            "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+            "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"
+        ))
+        .sigalgs_list(join!(
+            ":",
+            "ecdsa_secp256r1_sha256",
+            "rsa_pss_rsae_sha256",
+            "rsa_pkcs1_sha256",
+            "ecdsa_secp384r1_sha384",
+            "rsa_pss_rsae_sha384",
+            "rsa_pkcs1_sha384",
+            "rsa_pss_rsae_sha512",
+            "rsa_pkcs1_sha512",
+            "rsa_pkcs1_sha1"
+        ))
+        .alpn_protocols([AlpnProtocol::HTTP2, AlpnProtocol::HTTP1])
+        .min_tls_version(TlsVersion::TLS_1_2)
+        .max_tls_version(TlsVersion::TLS_1_3)
+        .build();
+
+    // HTTP/2 options config
+    let http2 = Http2Options::builder()
+        .initial_stream_id(3)
+        .initial_window_size(16777216)
+        .initial_connection_window_size(16711681 + 65535)
+        .headers_pseudo_order(
+            PseudoOrder::builder()
+                .extend([
+                    PseudoId::Method,
+                    PseudoId::Path,
+                    PseudoId::Authority,
+                    PseudoId::Scheme,
+                ])
+                .build(),
+        )
+        .build();
+
+    // Default headers
+    let headers = {
+        let mut headers = HeaderMap::new();
+        headers.insert(header::USER_AGENT, HeaderValue::from_static("TwitterAndroid/10.89.0-release.0 (310890000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)"));
+        headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US"));
+        headers.insert(
+            header::ACCEPT_ENCODING,
+            HeaderValue::from_static("br, gzip, deflate"),
+        );
+        headers.insert(header::ACCEPT, HeaderValue::from_static("application/json"));
+        headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
+        headers.insert(
+            header::COOKIE,
+            HeaderValue::from_static("ct0=YOUR_CT0_VALUE;"),
+        );
+        headers
+    };
+
+    // The headers keep the original case and order
+    let orig_headers = {
+        let mut orig_headers = OrigHeaderMap::new();
+        orig_headers.insert("cookie");
+        orig_headers.insert("content-length");
+        orig_headers.insert("User-Agent");
+        orig_headers.insert("Accept-Language");
+        orig_headers.insert("Accept-Encoding");
+        orig_headers
+    };
+
+    // This provider encapsulates TLS, HTTP/1, HTTP/2, default headers, and original headers
+    let emulation = Emulation::builder()
+        .tls_options(tls)
+        .http2_options(http2)
+        .headers(headers)
+        .orig_headers(orig_headers)
+        .build();
+
+    // Build a client with emulation config
+    let client = Client::builder()
+        .emulation(emulation)
+        .cert_verification(false)
+        .build()?;
+
+    // Use the API you're already familiar with
+    let resp = client.get("https://tls.browserleaks.com/").send().await?;
+    println!("{}", resp.text().await?);
+
+    Ok(())
+}

data/vendor/wreq/examples/form.rs

@@ -0,0 +1,14 @@
+// Short example of a POST request with form data.
+//
+// This is using the `tokio` runtime. You'll need the following dependency:
+//
+// `tokio = { version = "1", features = ["full"] }`
+#[tokio::main]
+async fn main() {
+    let response = wreq::post("http://www.baidu.com")
+        .form(&[("one", "1")])
+        .send()
+        .await
+        .expect("send");
+    println!("Response status {}", response.status());
+}

data/vendor/wreq/examples/http1_websocket.rs

@@ -0,0 +1,37 @@
+use futures_util::{SinkExt, StreamExt, TryStreamExt};
+use wreq::{header, ws::message::Message};
+
+#[tokio::main]
+async fn main() -> wreq::Result<()> {
+    // Use the API you're already familiar with
+    let resp = wreq::websocket("wss://echo.websocket.org")
+        .header(header::USER_AGENT, env!("CARGO_PKG_NAME"))
+        .read_buffer_size(1024 * 1024)
+        .send()
+        .await?;
+
+    assert_eq!(resp.version(), http::Version::HTTP_11);
+
+    let websocket = resp.into_websocket().await?;
+    if let Some(protocol) = websocket.protocol() {
+        println!("WebSocket subprotocol: {:?}", protocol);
+    }
+
+    let (mut tx, mut rx) = websocket.split();
+
+    tokio::spawn(async move {
+        for i in 1..11 {
+            if let Err(err) = tx.send(Message::text(format!("Hello, World! {i}"))).await {
+                eprintln!("failed to send message: {err}");
+            }
+        }
+    });
+
+    while let Some(message) = rx.try_next().await? {
+        if let Message::Text(text) = message {
+            println!("received: {text}");
+        }
+    }
+
+    Ok(())
+}

data/vendor/wreq/examples/http2_websocket.rs

@@ -0,0 +1,45 @@
+//! Run websocket server
+//!
+//! ```not_rust
+//! git clone https://github.com/tokio-rs/axum && cd axum
+//! cargo run -p example-websockets-http2
+//! ```
+
+use futures_util::{SinkExt, StreamExt, TryStreamExt};
+use wreq::{header, ws::message::Message};
+
+#[tokio::main]
+async fn main() -> wreq::Result<()> {
+    // Use the API you're already familiar with
+    let resp = wreq::websocket("wss://127.0.0.1:3000/ws")
+        .force_http2()
+        .header(header::USER_AGENT, env!("CARGO_PKG_NAME"))
+        .read_buffer_size(1024 * 1024)
+        .send()
+        .await?;
+
+    assert_eq!(resp.version(), http::Version::HTTP_2);
+
+    let websocket = resp.into_websocket().await?;
+    if let Some(protocol) = websocket.protocol() {
+        println!("WebSocket subprotocol: {:?}", protocol);
+    }
+
+    let (mut tx, mut rx) = websocket.split();
+
+    tokio::spawn(async move {
+        for i in 1..11 {
+            if let Err(err) = tx.send(Message::text(format!("Hello, World! #{i}"))).await {
+                eprintln!("failed to send message: {err}");
+            }
+        }
+    });
+
+    while let Some(message) = rx.try_next().await? {
+        if let Message::Text(text) = message {
+            println!("received: {text}");
+        }
+    }
+
+    Ok(())
+}

data/vendor/wreq/examples/json_dynamic.rs

@@ -0,0 +1,41 @@
+//! This example illustrates the way to send and receive arbitrary JSON.
+//!
+//! This is useful for some ad-hoc experiments and situations when you don't
+//! really care about the structure of the JSON and just need to display it or
+//! process it at runtime.
+
+// This is using the `tokio` runtime. You'll need the following dependency:
+//
+// `tokio = { version = "1", features = ["full"] }`
+#[tokio::main]
+async fn main() -> wreq::Result<()> {
+    let echo_json: serde_json::Value = wreq::post("https://jsonplaceholder.typicode.com/posts")
+        .json(&serde_json::json!({
+            "title": "wreq.rs",
+            "body": "https://docs.rs/wreq",
+            "userId": 1
+        }))
+        .send()
+        .await?
+        .json()
+        .await?;
+
+    println!("{echo_json:#?}");
+    // Object(
+    //     {
+    //         "body": String(
+    //             "https://docs.rs/wreq"
+    //         ),
+    //         "id": Number(
+    //             101
+    //         ),
+    //         "title": String(
+    //             "wreq.rs"
+    //         ),
+    //         "userId": Number(
+    //             1
+    //         )
+    //     }
+    // )
+    Ok(())
+}

data/vendor/wreq/examples/json_typed.rs

@@ -0,0 +1,47 @@
+//! This example illustrates the way to send and receive statically typed JSON.
+//!
+//! In contrast to the arbitrary JSON example, this brings up the full power of
+//! Rust compile-time type system guaranties though it requires a little bit
+//! more code.
+
+// These require the `serde` dependency.
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Post {
+    id: Option<i32>,
+    title: String,
+    body: String,
+    #[serde(rename = "userId")]
+    user_id: i32,
+}
+
+// This is using the `tokio` runtime. You'll need the following dependency:
+//
+// `tokio = { version = "1", features = ["full"] }`
+#[tokio::main]
+async fn main() -> wreq::Result<()> {
+    let new_post = Post {
+        id: None,
+        title: "wreq.rs".into(),
+        body: "https://docs.rs/wreq".into(),
+        user_id: 1,
+    };
+    let new_post: Post = wreq::post("https://jsonplaceholder.typicode.com/posts")
+        .json(&new_post)
+        .send()
+        .await?
+        .json()
+        .await?;
+
+    println!("{new_post:#?}");
+    // Post {
+    //     id: Some(
+    //         101
+    //     ),
+    //     title: "wreq.rs",
+    //     body: "https://docs.rs/wreq",
+    //     user_id: 1
+    // }
+    Ok(())
+}

data/vendor/wreq/examples/keylog.rs

@@ -0,0 +1,16 @@
+use wreq::tls::KeyLog;
+
+#[tokio::main]
+async fn main() -> wreq::Result<()> {
+    // Build a client
+    let client = wreq::Client::builder()
+        .keylog(KeyLog::from_file("keylog.txt"))
+        .cert_verification(false)
+        .build()?;
+
+    // Use the API you're already familiar with
+    let resp = client.get("https://yande.re/post.json").send().await?;
+    println!("{}", resp.text().await?);
+
+    Ok(())
+}