wreq 1.2.5 → 1.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32fccd5de0a2dc902b5ff76a4354b5271f30151718388715d9563c72d702fce3
-  data.tar.gz: 6623c40db0fbe229026c4214b0c183e700b595329583d96b19aa3858d6a07943
+  metadata.gz: 87d989f8582c97e03c95ef1fd90d1f089eb35c787da5d9527942548483c1dd55
+  data.tar.gz: 413a4656384561df504e9b6a175c2f82c3373693df41bb88cc2ba78c2d319cb4
 SHA512:
-  metadata.gz: b17c6b0586311dc5144cba1ac348a7ac6754028ad51d9727b451f213fa77ab341a7ece9969ba62bd4eb6762de2a8940c12e4c0210e64016b39be72cf2bb63ada
-  data.tar.gz: bff84ec0a74f02523966d3f28d4772db404e142a7f9b4b16f3e74e62f322120d18750431c5dbc81db5489da75e7145219c39d13d7b6534955fc250b0c9b3b78e
+  metadata.gz: 2c0b23be854a1bdf4421ec7e32118a41e3c209bded0a32cde470614bf487fdaf48082ad432913a597ef176d14b12eeabcf976ccbddf49d99446ed3f08b5e3710
+  data.tar.gz: 53190348795c05829f6119753c6402a6c64a662a72c37ab88616275bbfce47292577348c8f53665d7abf8004d626ded649ff7b7d31d31f99fbc87e7caddef8e7

data/Cargo.lock

@@ -999,12 +999,23 @@ dependencies = [
  "zmij",
 ]
 
+[[package]]
+name = "serde_ignored"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "115dffd5f3853e06e746965a20dcbae6ee747ae30b543d91b0e089668bb07798"
+dependencies = [
+ "serde",
+ "serde_core",
+]
+
 [[package]]
 name = "serde_json"
 version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
+ "indexmap",
  "itoa",
  "memchr",
  "serde",
@@ -1013,14 +1024,14 @@ dependencies = [
 ]
 
 [[package]]
-name = "serde_magnus"
-version = "0.11.0"
+name = "serde_path_to_error"
+version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ff64c88ddd26acdcad5a501f18bcc339927b77b69f4a03bfaf2a6fc5ba2ac4b"
+checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
 dependencies = [
- "magnus",
+ "itoa",
  "serde",
- "tap",
+ "serde_core",
 ]
 
 [[package]]
@@ -1137,12 +1148,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "tap"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
-
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -1535,7 +1540,7 @@ dependencies = [
 
 [[package]]
 name = "wreq-ruby"
-version = "1.2.5"
+version = "1.2.6"
 dependencies = [
  "arc-swap",
  "bytes",
@@ -1548,7 +1553,9 @@ dependencies = [
  "rb-sys",
  "rb-sys-env",
  "serde",
- "serde_magnus",
+ "serde_ignored",
+ "serde_json",
+ "serde_path_to_error",
  "tokio",
  "wreq",
  "wreq-util",

data/Cargo.toml

@@ -6,7 +6,7 @@ homepage = "https://github.com/SearchApi/wreq-ruby"
 repository = "https://github.com/SearchApi/wreq-ruby"
 edition = "2024"
 rust-version = "1.85"
-version = "1.2.5"
+version = "1.2.6"
 
 [lib]
 crate-type = ["cdylib"]
@@ -33,7 +33,12 @@ wreq = { version = "6.0.0-rc", features = [
 ] }
 wreq-util = { version = "3.0.0-rc", features = ["emulation-compression"] }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_magnus = "0.11.0"
+serde_ignored = "0.1.14"
+serde_json = { version = "1.0.150", features = [
+    "arbitrary_precision",
+    "preserve_order",
+] }
+serde_path_to_error = "0.1.20"
 indexmap = { version = "2.14.0", features = ["serde"] }
 cookie = "0.18.1"
 bytes = "1.12.1"
@@ -42,6 +47,9 @@ http = "1.4.1"
 http-body-util = "0.1.3"
 futures-util = { version = "0.3.32", default-features = false }
 
+[dev-dependencies]
+magnus = { version = "0.8.2", features = ["embed"] }
+
 [build-dependencies]
 rb-sys-env = "0.2.2"
 

data/crates/wreq-util/src/emulate/macros.rs

@@ -95,9 +95,10 @@ macro_rules! header_chrome_sec_ch_ua {
 
 macro_rules! header_chrome_sec_fetch {
     ($headers:expr) => {
-        $headers.insert("sec-fetch-dest", HeaderValue::from_static("document"));
-        $headers.insert("sec-fetch-mode", HeaderValue::from_static("navigate"));
         $headers.insert("sec-fetch-site", HeaderValue::from_static("none"));
+        $headers.insert("sec-fetch-mode", HeaderValue::from_static("navigate"));
+        $headers.insert("sec-fetch-user", HeaderValue::from_static("?1"));
+        $headers.insert("sec-fetch-dest", HeaderValue::from_static("document"));
     };
 }
 
@@ -110,6 +111,11 @@ macro_rules! header_chrome_ua {
 macro_rules! header_chrome_accept {
     ($headers:expr) => {
         $headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
+    };
+}
+
+macro_rules! header_chrome_accept_encoding {
+    ($headers:expr) => {
         #[cfg(feature = "emulation-compression")]
         $headers.insert(
             ACCEPT_ENCODING,
@@ -118,14 +124,13 @@ macro_rules! header_chrome_accept {
         $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
     };
     (zstd, $headers:expr) => {
-        $headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
         #[cfg(feature = "emulation-compression")]
         $headers.insert(
             ACCEPT_ENCODING,
             HeaderValue::from_static("gzip, deflate, br, zstd"),
         );
         $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
-    }
+    };
 }
 
 macro_rules! header_firefox_sec_fetch {
@@ -133,6 +138,7 @@ macro_rules! header_firefox_sec_fetch {
         $headers.insert("sec-fetch-dest", HeaderValue::from_static("document"));
         $headers.insert("sec-fetch-mode", HeaderValue::from_static("navigate"));
         $headers.insert("sec-fetch-site", HeaderValue::from_static("none"));
+        $headers.insert("sec-fetch-user", HeaderValue::from_static("?1"));
     };
 }
 
@@ -144,12 +150,12 @@ macro_rules! header_firefox_accept {
                 "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             ),
         );
+        $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
         #[cfg(feature = "emulation-compression")]
         $headers.insert(
             ACCEPT_ENCODING,
             HeaderValue::from_static("gzip, deflate, br"),
         );
-        $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
     };
     (zstd, $headers:expr) => {
         $headers.insert(
@@ -158,21 +164,17 @@ macro_rules! header_firefox_accept {
                 "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             ),
         );
+        $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
         #[cfg(feature = "emulation-compression")]
         $headers.insert(
             ACCEPT_ENCODING,
             HeaderValue::from_static("gzip, deflate, br, zstd"),
         );
-        $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
     };
 }
 
 macro_rules! header_firefox_ua {
     ($headers:expr, $ua:expr) => {
-        $headers.insert(
-            HeaderName::from_static("te"),
-            HeaderValue::from_static("trailers"),
-        );
         $headers.insert(USER_AGENT, HeaderValue::from_static($ua));
     };
 }

data/crates/wreq-util/src/emulate/profile/chrome/header.rs

@@ -12,9 +12,14 @@ pub fn header_initializer(
         emulation_os.platform(),
         emulation_os.is_mobile()
     );
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_chrome_ua!(headers, ua);
-    header_chrome_sec_fetch!(headers);
     header_chrome_accept!(headers);
+    header_chrome_sec_fetch!(headers);
+    header_chrome_accept_encoding!(headers);
     headers
 }
 
@@ -30,9 +35,14 @@ pub fn header_initializer_with_zstd(
         emulation_os.platform(),
         emulation_os.is_mobile()
     );
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_chrome_ua!(headers, ua);
+    header_chrome_accept!(headers);
     header_chrome_sec_fetch!(headers);
-    header_chrome_accept!(zstd, headers);
+    header_chrome_accept_encoding!(zstd, headers);
     headers
 }
 
@@ -48,9 +58,14 @@ pub fn header_initializer_with_zstd_priority(
         emulation_os.platform(),
         emulation_os.is_mobile()
     );
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_chrome_ua!(headers, ua);
+    header_chrome_accept!(headers);
     header_chrome_sec_fetch!(headers);
-    header_chrome_accept!(zstd, headers);
+    header_chrome_accept_encoding!(zstd, headers);
     headers.insert(
         HeaderName::from_static("priority"),
         HeaderValue::from_static("u=0, i"),

data/crates/wreq-util/src/emulate/profile/firefox/header.rs

@@ -4,7 +4,15 @@ pub fn header_initializer(ua: &'static str) -> HeaderMap {
     let mut headers = HeaderMap::new();
     header_firefox_ua!(headers, ua);
     header_firefox_accept!(headers);
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_firefox_sec_fetch!(headers);
+    headers.insert(
+        HeaderName::from_static("te"),
+        HeaderValue::from_static("trailers"),
+    );
     headers
 }
 
@@ -12,10 +20,18 @@ pub fn header_initializer_with_zstd(ua: &'static str) -> HeaderMap {
     let mut headers = HeaderMap::new();
     header_firefox_ua!(headers, ua);
     header_firefox_accept!(zstd, headers);
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_firefox_sec_fetch!(headers);
     headers.insert(
         HeaderName::from_static("priority"),
         HeaderValue::from_static("u=0, i"),
     );
+    headers.insert(
+        HeaderName::from_static("te"),
+        HeaderValue::from_static("trailers"),
+    );
     headers
 }

data/crates/wreq-util/src/emulate/profile/opera/header.rs

@@ -13,15 +13,14 @@ pub fn header_initializer_with_zstd_priority(
         emulation_os.platform(),
         emulation_os.is_mobile()
     );
-    header_chrome_ua!(headers, ua);
-
-    headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
-    #[cfg(feature = "emulation-compression")]
     headers.insert(
-        ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br, zstd"),
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
     );
-    headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
+    header_chrome_ua!(headers, ua);
+    header_chrome_accept!(headers);
+    header_chrome_sec_fetch!(headers);
+    header_chrome_accept_encoding!(zstd, headers);
     headers.insert(
         HeaderName::from_static("priority"),
         HeaderValue::from_static("u=0, i"),

data/crates/wreq-util/tests/client.rs

@@ -1,9 +1,96 @@
 #![cfg(not(target_arch = "wasm32"))]
 mod support;
 
+use std::future::Future;
+
 use support::server;
+use tokio::{io::AsyncWriteExt, net::TcpStream};
 use wreq::Client;
-use wreq_util::{Emulation, Platform};
+use wreq_util::{Emulation, Platform, Profile};
+
+const CHROMIUM_HEADER_ORDER: &[&str] = &[
+    "sec-ch-ua",
+    "sec-ch-ua-mobile",
+    "sec-ch-ua-platform",
+    "upgrade-insecure-requests",
+    "user-agent",
+    "accept",
+    "sec-fetch-site",
+    "sec-fetch-mode",
+    "sec-fetch-user",
+    "sec-fetch-dest",
+    #[cfg(feature = "emulation-compression")]
+    "accept-encoding",
+    "accept-language",
+    "priority",
+];
+
+const FIREFOX_HEADER_ORDER: &[&str] = &[
+    "user-agent",
+    "accept",
+    "accept-language",
+    #[cfg(feature = "emulation-compression")]
+    "accept-encoding",
+    "upgrade-insecure-requests",
+    "sec-fetch-dest",
+    "sec-fetch-mode",
+    "sec-fetch-site",
+    "sec-fetch-user",
+    "te",
+];
+
+fn check_header_order<'a>(
+    request: &'a [u8],
+    stream: &'a mut TcpStream,
+    expected: &'static [&'static str],
+) -> Box<dyn Future<Output = ()> + Send + 'a> {
+    Box::new(async move {
+        let request = std::str::from_utf8(request)
+            .expect("request should be valid UTF-8")
+            .to_ascii_lowercase();
+        let mut previous = None;
+
+        for name in expected {
+            let marker = format!("\r\n{name}:");
+            let position = request
+                .find(&marker)
+                .unwrap_or_else(|| panic!("missing `{name}` header:\n{request}"));
+
+            if let Some(previous) = previous {
+                assert!(
+                    position > previous,
+                    "`{name}` header is out of order:\n{request}"
+                );
+            }
+
+            previous = Some(position);
+        }
+
+        assert!(request.contains("\r\nupgrade-insecure-requests: 1\r\n"));
+        assert!(request.contains("\r\nsec-fetch-user: ?1\r\n"));
+
+        stream
+            .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
+            .await
+            .expect("response");
+    })
+}
+
+async fn assert_emulation_headers(profile: Profile, expected: &'static [&'static str]) {
+    let server = server::low_level_with_response(move |request, stream| {
+        check_header_order(request, stream, expected)
+    });
+    let response = Client::builder()
+        .emulation(profile)
+        .build()
+        .expect("client")
+        .get(format!("http://{}/headers", server.addr()))
+        .send()
+        .await
+        .expect("request");
+
+    assert_eq!(response.status(), wreq::StatusCode::OK);
+}
 
 #[tokio::test]
 async fn test_client_emulation_device() {
@@ -49,3 +136,18 @@ async fn test_client_emulation_device() {
 
     assert_eq!(res.status(), wreq::StatusCode::OK);
 }
+
+#[tokio::test]
+async fn test_chrome_default_header_order() {
+    assert_emulation_headers(Emulation::Chrome133, CHROMIUM_HEADER_ORDER).await;
+}
+
+#[tokio::test]
+async fn test_firefox_default_header_order() {
+    assert_emulation_headers(Emulation::Firefox109, FIREFOX_HEADER_ORDER).await;
+}
+
+#[tokio::test]
+async fn test_opera_default_header_order() {
+    assert_emulation_headers(Emulation::Opera116, CHROMIUM_HEADER_ORDER).await;
+}