wreq-rb 0.3.0

data/vendor/wreq/tests/deflate.rs

@@ -0,0 +1,347 @@
+mod support;
+use std::io::Write;
+
+use flate2::{Compression, write::ZlibEncoder};
+use support::server;
+use tokio::io::AsyncWriteExt;
+
+#[tokio::test]
+async fn deflate_response() {
+    deflate_case(10_000, 4096).await;
+}
+
+#[tokio::test]
+async fn deflate_single_byte_chunks() {
+    deflate_case(10, 1).await;
+}
+
+#[tokio::test]
+async fn test_deflate_empty_body() {
+    let server = server::http(move |req| async move {
+        assert_eq!(req.method(), "HEAD");
+
+        http::Response::builder()
+            .header("content-encoding", "deflate")
+            .body(Default::default())
+            .unwrap()
+    });
+
+    let res = wreq::head(format!("http://{}/deflate", server.addr()))
+        .send()
+        .await
+        .unwrap();
+
+    let body = res.text().await.unwrap();
+
+    assert_eq!(body, "");
+}
+
+#[tokio::test]
+async fn test_accept_header_is_not_changed_if_set() {
+    let server = server::http(move |req| async move {
+        assert_eq!(req.headers()["accept"], "application/json");
+        assert!(
+            req.headers()["accept-encoding"]
+                .to_str()
+                .unwrap()
+                .contains("deflate")
+        );
+        http::Response::default()
+    });
+
+    let res = wreq::get(format!("http://{}/accept", server.addr()))
+        .header(
+            wreq::header::ACCEPT,
+            wreq::header::HeaderValue::from_static("application/json"),
+        )
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(res.status(), wreq::StatusCode::OK);
+}
+
+#[tokio::test]
+async fn test_accept_encoding_header_is_not_changed_if_set() {
+    let server = server::http(move |req| async move {
+        assert_eq!(req.headers()["accept"], "*/*");
+        assert_eq!(req.headers()["accept-encoding"], "identity");
+        http::Response::default()
+    });
+
+    let res = wreq::get(format!("http://{}/accept-encoding", server.addr()))
+        .header(wreq::header::ACCEPT, "*/*")
+        .header(
+            wreq::header::ACCEPT_ENCODING,
+            wreq::header::HeaderValue::from_static("identity"),
+        )
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(res.status(), wreq::StatusCode::OK);
+}
+
+async fn deflate_case(response_size: usize, chunk_size: usize) {
+    use futures_util::stream::StreamExt;
+
+    let content: String = (0..response_size).fold(String::new(), |mut acc, i| {
+        acc.push_str(&format!("test {i}"));
+        acc
+    });
+    let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
+    encoder.write_all(content.as_bytes()).unwrap();
+    let deflated_content = encoder.finish().unwrap();
+
+    let mut response = format!(
+        "\
+         HTTP/1.1 200 OK\r\n\
+         Server: test-accept\r\n\
+         Content-Encoding: deflate\r\n\
+         Content-Length: {}\r\n\
+         \r\n",
+        &deflated_content.len()
+    )
+    .into_bytes();
+    response.extend(&deflated_content);
+
+    let server = server::http(move |req| {
+        assert!(
+            req.headers()["accept-encoding"]
+                .to_str()
+                .unwrap()
+                .contains("deflate")
+        );
+
+        let deflated = deflated_content.clone();
+        async move {
+            let len = deflated.len();
+            let stream =
+                futures_util::stream::unfold((deflated, 0), move |(deflated, pos)| async move {
+                    let chunk = deflated.chunks(chunk_size).nth(pos)?.to_vec();
+
+                    Some((chunk, (deflated, pos + 1)))
+                });
+
+            let body = wreq::Body::wrap_stream(stream.map(Ok::<_, std::convert::Infallible>));
+
+            http::Response::builder()
+                .header("content-encoding", "deflate")
+                .header("content-length", len)
+                .body(body)
+                .unwrap()
+        }
+    });
+
+    let res = wreq::get(format!("http://{}/deflate", server.addr()))
+        .send()
+        .await
+        .expect("response");
+
+    let body = res.text().await.expect("text");
+    assert_eq!(body, content);
+}
+
+const COMPRESSED_RESPONSE_HEADERS: &[u8] = b"HTTP/1.1 200 OK\x0d\x0a\
+            Content-Type: text/plain\x0d\x0a\
+            Connection: keep-alive\x0d\x0a\
+            Content-Encoding: deflate\x0d\x0a";
+
+const RESPONSE_CONTENT: &str = "some message here";
+
+fn deflate_compress(input: &[u8]) -> Vec<u8> {
+    let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
+    encoder.write_all(input).unwrap();
+    encoder.finish().unwrap()
+}
+
+#[tokio::test]
+async fn test_non_chunked_non_fragmented_response() {
+    let server = server::low_level_with_response(|_raw_request, client_socket| {
+        Box::new(async move {
+            let deflated_content = deflate_compress(RESPONSE_CONTENT.as_bytes());
+            let content_length_header =
+                format!("Content-Length: {}\r\n\r\n", deflated_content.len()).into_bytes();
+            let response = [
+                COMPRESSED_RESPONSE_HEADERS,
+                &content_length_header,
+                &deflated_content,
+            ]
+            .concat();
+
+            client_socket
+                .write_all(response.as_slice())
+                .await
+                .expect("response write_all failed");
+            client_socket.flush().await.expect("response flush failed");
+        })
+    });
+
+    let res = wreq::get(format!("http://{}/", server.addr()))
+        .send()
+        .await
+        .expect("response");
+
+    assert_eq!(res.text().await.expect("text"), RESPONSE_CONTENT);
+}
+
+#[tokio::test]
+async fn test_chunked_fragmented_response_1() {
+    const DELAY_BETWEEN_RESPONSE_PARTS: tokio::time::Duration =
+        tokio::time::Duration::from_millis(1000);
+    const DELAY_MARGIN: tokio::time::Duration = tokio::time::Duration::from_millis(50);
+
+    let server = server::low_level_with_response(|_raw_request, client_socket| {
+        Box::new(async move {
+            let deflated_content = deflate_compress(RESPONSE_CONTENT.as_bytes());
+            let response_first_part = [
+                COMPRESSED_RESPONSE_HEADERS,
+                format!(
+                    "Transfer-Encoding: chunked\r\n\r\n{:x}\r\n",
+                    deflated_content.len()
+                )
+                .as_bytes(),
+                &deflated_content,
+            ]
+            .concat();
+            let response_second_part = b"\r\n0\r\n\r\n";
+
+            client_socket
+                .write_all(response_first_part.as_slice())
+                .await
+                .expect("response_first_part write_all failed");
+            client_socket
+                .flush()
+                .await
+                .expect("response_first_part flush failed");
+
+            tokio::time::sleep(DELAY_BETWEEN_RESPONSE_PARTS).await;
+
+            client_socket
+                .write_all(response_second_part)
+                .await
+                .expect("response_second_part write_all failed");
+            client_socket
+                .flush()
+                .await
+                .expect("response_second_part flush failed");
+        })
+    });
+
+    let start = tokio::time::Instant::now();
+    let res = wreq::get(format!("http://{}/", server.addr()))
+        .send()
+        .await
+        .expect("response");
+
+    assert_eq!(res.text().await.expect("text"), RESPONSE_CONTENT);
+    assert!(start.elapsed() >= DELAY_BETWEEN_RESPONSE_PARTS - DELAY_MARGIN);
+}
+
+#[tokio::test]
+async fn test_chunked_fragmented_response_2() {
+    const DELAY_BETWEEN_RESPONSE_PARTS: tokio::time::Duration =
+        tokio::time::Duration::from_millis(1000);
+    const DELAY_MARGIN: tokio::time::Duration = tokio::time::Duration::from_millis(50);
+
+    let server = server::low_level_with_response(|_raw_request, client_socket| {
+        Box::new(async move {
+            let deflated_content = deflate_compress(RESPONSE_CONTENT.as_bytes());
+            let response_first_part = [
+                COMPRESSED_RESPONSE_HEADERS,
+                format!(
+                    "Transfer-Encoding: chunked\r\n\r\n{:x}\r\n",
+                    deflated_content.len()
+                )
+                .as_bytes(),
+                &deflated_content,
+                b"\r\n",
+            ]
+            .concat();
+            let response_second_part = b"0\r\n\r\n";
+
+            client_socket
+                .write_all(response_first_part.as_slice())
+                .await
+                .expect("response_first_part write_all failed");
+            client_socket
+                .flush()
+                .await
+                .expect("response_first_part flush failed");
+
+            tokio::time::sleep(DELAY_BETWEEN_RESPONSE_PARTS).await;
+
+            client_socket
+                .write_all(response_second_part)
+                .await
+                .expect("response_second_part write_all failed");
+            client_socket
+                .flush()
+                .await
+                .expect("response_second_part flush failed");
+        })
+    });
+
+    let start = tokio::time::Instant::now();
+    let res = wreq::get(format!("http://{}/", server.addr()))
+        .send()
+        .await
+        .expect("response");
+
+    assert_eq!(res.text().await.expect("text"), RESPONSE_CONTENT);
+    assert!(start.elapsed() >= DELAY_BETWEEN_RESPONSE_PARTS - DELAY_MARGIN);
+}
+
+#[tokio::test]
+async fn test_chunked_fragmented_response_with_extra_bytes() {
+    const DELAY_BETWEEN_RESPONSE_PARTS: tokio::time::Duration =
+        tokio::time::Duration::from_millis(1000);
+    const DELAY_MARGIN: tokio::time::Duration = tokio::time::Duration::from_millis(50);
+
+    let server = server::low_level_with_response(|_raw_request, client_socket| {
+        Box::new(async move {
+            let deflated_content = deflate_compress(RESPONSE_CONTENT.as_bytes());
+            let response_first_part = [
+                COMPRESSED_RESPONSE_HEADERS,
+                format!(
+                    "Transfer-Encoding: chunked\r\n\r\n{:x}\r\n",
+                    deflated_content.len()
+                )
+                .as_bytes(),
+                &deflated_content,
+            ]
+            .concat();
+            let response_second_part = b"\r\n2ab\r\n0\r\n\r\n";
+
+            client_socket
+                .write_all(response_first_part.as_slice())
+                .await
+                .expect("response_first_part write_all failed");
+            client_socket
+                .flush()
+                .await
+                .expect("response_first_part flush failed");
+
+            tokio::time::sleep(DELAY_BETWEEN_RESPONSE_PARTS).await;
+
+            client_socket
+                .write_all(response_second_part)
+                .await
+                .expect("response_second_part write_all failed");
+            client_socket
+                .flush()
+                .await
+                .expect("response_second_part flush failed");
+        })
+    });
+
+    let start = tokio::time::Instant::now();
+    let res = wreq::get(format!("http://{}/", server.addr()))
+        .send()
+        .await
+        .expect("response");
+
+    let err = res.text().await.expect_err("there must be an error");
+    assert!(err.is_decode());
+    assert!(start.elapsed() >= DELAY_BETWEEN_RESPONSE_PARTS - DELAY_MARGIN);
+}

data/vendor/wreq/tests/emulation.rs

@@ -0,0 +1,260 @@
+use std::time::Duration;
+
+use wreq::{
+    Client, Emulation,
+    http1::Http1Options,
+    http2::{
+        Http2Options, PseudoId, PseudoOrder, SettingId, SettingsOrder, StreamDependency, StreamId,
+    },
+    tls::{AlpnProtocol, CertificateCompressionAlgorithm, ExtensionType, TlsOptions, TlsVersion},
+};
+
+macro_rules! join {
+    ($sep:expr, $first:expr $(, $rest:expr)*) => {
+        concat!($first $(, $sep, $rest)*)
+    };
+}
+
+fn tls_options_template() -> TlsOptions {
+    //  TLS options config
+    TlsOptions::builder()
+        .curves_list(join!(
+            ":",
+            "X25519MLKEM768",
+            "X25519",
+            "P-256",
+            "P-384",
+            "P-521",
+            "ffdhe2048",
+            "ffdhe3072"
+        ))
+        .cipher_list(join!(
+            ":",
+            "TLS_AES_128_GCM_SHA256",
+            "TLS_CHACHA20_POLY1305_SHA256",
+            "TLS_AES_256_GCM_SHA384",
+            "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+            "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
+            "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
+            "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+            "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+            "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
+            "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA",
+            "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
+            "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
+            "TLS_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_RSA_WITH_AES_256_GCM_SHA384",
+            "TLS_RSA_WITH_AES_128_CBC_SHA",
+            "TLS_RSA_WITH_AES_256_CBC_SHA"
+        ))
+        .sigalgs_list(join!(
+            ":",
+            "ecdsa_secp256r1_sha256",
+            "ecdsa_secp384r1_sha384",
+            "ecdsa_secp521r1_sha512",
+            "rsa_pss_rsae_sha256",
+            "rsa_pss_rsae_sha384",
+            "rsa_pss_rsae_sha512",
+            "rsa_pkcs1_sha256",
+            "rsa_pkcs1_sha384",
+            "rsa_pkcs1_sha512",
+            "ecdsa_sha1",
+            "rsa_pkcs1_sha1"
+        ))
+        .delegated_credentials(join!(
+            ":",
+            "ecdsa_secp256r1_sha256",
+            "ecdsa_secp384r1_sha384",
+            "ecdsa_secp521r1_sha512",
+            "ecdsa_sha1"
+        ))
+        .certificate_compression_algorithms(&[
+            CertificateCompressionAlgorithm::ZLIB,
+            CertificateCompressionAlgorithm::BROTLI,
+            CertificateCompressionAlgorithm::ZSTD,
+        ])
+        .alpn_protocols([AlpnProtocol::HTTP2, AlpnProtocol::HTTP1])
+        .record_size_limit(0x4001)
+        .pre_shared_key(true)
+        .enable_ech_grease(true)
+        .enable_ocsp_stapling(true)
+        .enable_signed_cert_timestamps(true)
+        .min_tls_version(TlsVersion::TLS_1_2)
+        .max_tls_version(TlsVersion::TLS_1_3)
+        .key_shares_limit(3)
+        .preserve_tls13_cipher_list(true)
+        .aes_hw_override(true)
+        .random_aes_hw_override(true)
+        .extension_permutation(&[
+            ExtensionType::SERVER_NAME,
+            ExtensionType::EXTENDED_MASTER_SECRET,
+            ExtensionType::RENEGOTIATE,
+            ExtensionType::SUPPORTED_GROUPS,
+            ExtensionType::EC_POINT_FORMATS,
+            ExtensionType::SESSION_TICKET,
+            ExtensionType::APPLICATION_LAYER_PROTOCOL_NEGOTIATION,
+            ExtensionType::STATUS_REQUEST,
+            ExtensionType::DELEGATED_CREDENTIAL,
+            ExtensionType::KEY_SHARE,
+            ExtensionType::SUPPORTED_VERSIONS,
+            ExtensionType::SIGNATURE_ALGORITHMS,
+            ExtensionType::PSK_KEY_EXCHANGE_MODES,
+            ExtensionType::RECORD_SIZE_LIMIT,
+            ExtensionType::CERT_COMPRESSION,
+            ExtensionType::ENCRYPTED_CLIENT_HELLO,
+        ])
+        .build()
+}
+
+fn http2_options_template() -> Http2Options {
+    // HTTP/2 headers frame pseudo-header order
+    let headers_pseudo_order = PseudoOrder::builder()
+        .extend([
+            PseudoId::Method,
+            PseudoId::Path,
+            PseudoId::Authority,
+            PseudoId::Scheme,
+        ])
+        .build();
+
+    // HTTP/2 settings frame order
+    let settings_order = SettingsOrder::builder()
+        .extend([
+            SettingId::HeaderTableSize,
+            SettingId::EnablePush,
+            SettingId::MaxConcurrentStreams,
+            SettingId::InitialWindowSize,
+            SettingId::MaxFrameSize,
+            SettingId::MaxHeaderListSize,
+            SettingId::EnableConnectProtocol,
+            SettingId::NoRfc7540Priorities,
+        ])
+        .build();
+
+    Http2Options::builder()
+        .header_table_size(65536)
+        .enable_push(false)
+        .initial_window_size(131072)
+        .max_frame_size(16384)
+        .initial_connection_window_size(12517377 + 65535)
+        .headers_stream_dependency(StreamDependency::new(StreamId::ZERO, 41, false))
+        .headers_pseudo_order(headers_pseudo_order)
+        .settings_order(settings_order)
+        .build()
+}
+
+fn emulation_template() -> Emulation {
+    //  HTTP/1 options config
+    let http1 = Http1Options::builder()
+        .allow_obsolete_multiline_headers_in_responses(true)
+        .max_headers(100)
+        .build();
+
+    // This provider encapsulates TLS, HTTP/1, HTTP/2, default headers, and original headers
+    Emulation::builder()
+        .tls_options(tls_options_template())
+        .http1_options(http1)
+        .http2_options(http2_options_template())
+        .build()
+}
+
+#[tokio::test]
+async fn test_emulation() -> wreq::Result<()> {
+    let client = Client::builder()
+        .emulation(emulation_template())
+        .connect_timeout(Duration::from_secs(10))
+        .cert_verification(false)
+        .build()?;
+
+    let text = client
+        .get("https://tls.browserleaks.com/")
+        .send()
+        .await?
+        .text()
+        .await?;
+
+    assert!(
+        text.contains("t13d1717h2_5b57614c22b0_3cbfd9057e0d"),
+        "Response ja4_hash fingerprint not found: {text}"
+    );
+    assert!(
+        text.contains("6ea73faa8fc5aac76bded7bd238f6433"),
+        "Response akamai_hash fingerprint not found: {text}"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_request_with_emulation() -> wreq::Result<()> {
+    let client = Client::builder()
+        .connect_timeout(Duration::from_secs(10))
+        .cert_verification(false)
+        .build()?;
+
+    let text = client
+        .get("https://tls.browserleaks.com/")
+        .emulation(emulation_template())
+        .send()
+        .await?
+        .text()
+        .await?;
+
+    assert!(
+        text.contains("t13d1717h2_5b57614c22b0_3cbfd9057e0d"),
+        "Response ja4_hash fingerprint not found: {text}"
+    );
+    assert!(
+        text.contains("6ea73faa8fc5aac76bded7bd238f6433"),
+        "Response akamai_hash fingerprint not found: {text}"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_request_with_emulation_tls() -> wreq::Result<()> {
+    let client = Client::builder()
+        .connect_timeout(Duration::from_secs(10))
+        .cert_verification(false)
+        .build()?;
+
+    let text = client
+        .get("https://tls.browserleaks.com/")
+        .emulation(tls_options_template())
+        .send()
+        .await?
+        .text()
+        .await?;
+
+    assert!(
+        text.contains("t13d1717h2_5b57614c22b0_3cbfd9057e0d"),
+        "Response ja4_hash fingerprint not found: {text}"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_request_with_emulation_http2() -> wreq::Result<()> {
+    let client = Client::builder()
+        .connect_timeout(Duration::from_secs(10))
+        .cert_verification(false)
+        .build()?;
+
+    let text = client
+        .get("https://tls.browserleaks.com/")
+        .emulation(http2_options_template())
+        .send()
+        .await?
+        .text()
+        .await?;
+
+    assert!(
+        text.contains("6ea73faa8fc5aac76bded7bd238f6433"),
+        "Response akamai_hash fingerprint not found: {text}"
+    );
+
+    Ok(())
+}