wreq 1.2.5 → 1.2.7

data/Cargo.toml

@@ -5,8 +5,8 @@ authors = ["SearchApi <support@searchapi.io>"]
 homepage = "https://github.com/SearchApi/wreq-ruby"
 repository = "https://github.com/SearchApi/wreq-ruby"
 edition = "2024"
-rust-version = "1.85"
-version = "1.2.5"
+version = "1.2.7"
+rust-version = "1.95"
 
 [lib]
 crate-type = ["cdylib"]
@@ -33,7 +33,12 @@ wreq = { version = "6.0.0-rc", features = [
 ] }
 wreq-util = { version = "3.0.0-rc", features = ["emulation-compression"] }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_magnus = "0.11.0"
+serde_ignored = "0.1.14"
+serde_json = { version = "1.0.150", features = [
+    "arbitrary_precision",
+    "preserve_order",
+] }
+serde_path_to_error = "0.1.20"
 indexmap = { version = "2.14.0", features = ["serde"] }
 cookie = "0.18.1"
 bytes = "1.12.1"
@@ -42,6 +47,9 @@ http = "1.4.1"
 http-body-util = "0.1.3"
 futures-util = { version = "0.3.32", default-features = false }
 
+[dev-dependencies]
+magnus = { version = "0.8.2", features = ["embed"] }
+
 [build-dependencies]
 rb-sys-env = "0.2.2"
 

data/Rakefile

@@ -1,5 +1,11 @@
 # frozen_string_literal: true
 
+require_relative "script/rust_env"
+
+# Cargo needs RubyInstaller's UCRT tools when Rake builds the native extension
+# on Windows. Other platforms keep their existing native Cargo environment.
+Wreq::RustEnv.activate
+
 require "bundler/gem_tasks"
 require "rake/testtask"
 require "rb_sys/extensiontask"

data/crates/wreq-util/README.md

@@ -43,7 +43,7 @@ async fn main() -> wreq::Result<()> {
         .build()?;
 
     // Use the API you're already familiar with
-    let resp = client.get("https://www.google.com").send().await?;
+    let resp = client.get("https://pingly.us.kg/api/all").send().await?;
     println!("{}", resp.text().await?);
     Ok(())
 }

data/crates/wreq-util/examples/emulate.rs

@@ -3,7 +3,7 @@ use wreq_util::{Emulation, Platform, Profile};
 #[tokio::main]
 async fn main() -> Result<(), wreq::Error> {
     // Example 1: Basic emulation - Safari26
-    let text = wreq::get("https://tls.browserleaks.com/")
+    let text = wreq::get("https://pingly.us.kg/api/all")
         .emulation(Emulation::Safari26)
         .send()
         .await?
@@ -12,7 +12,7 @@ async fn main() -> Result<(), wreq::Error> {
     println!("{}\n", text);
 
     // Example 2: Advanced emulation with options - OkHttp5
-    let text = wreq::get("https://tls.peet.ws/api/all")
+    let text = wreq::get("https://pingly.us.kg/api/all")
         .emulation(
             Emulation::builder()
                 .profile(Profile::OkHttp5)
@@ -27,7 +27,7 @@ async fn main() -> Result<(), wreq::Error> {
     println!("{}\n", text);
 
     // Example 3: Random device emulation
-    let text = wreq::get("https://tls.peet.ws/api/all")
+    let text = wreq::get("https://pingly.us.kg/api/all")
         .emulation(Emulation::random())
         .send()
         .await?

data/crates/wreq-util/src/emulate/macros.rs

@@ -95,9 +95,10 @@ macro_rules! header_chrome_sec_ch_ua {
 
 macro_rules! header_chrome_sec_fetch {
     ($headers:expr) => {
-        $headers.insert("sec-fetch-dest", HeaderValue::from_static("document"));
-        $headers.insert("sec-fetch-mode", HeaderValue::from_static("navigate"));
         $headers.insert("sec-fetch-site", HeaderValue::from_static("none"));
+        $headers.insert("sec-fetch-mode", HeaderValue::from_static("navigate"));
+        $headers.insert("sec-fetch-user", HeaderValue::from_static("?1"));
+        $headers.insert("sec-fetch-dest", HeaderValue::from_static("document"));
     };
 }
 
@@ -109,7 +110,12 @@ macro_rules! header_chrome_ua {
 
 macro_rules! header_chrome_accept {
     ($headers:expr) => {
-        $headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
+        $headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"));
+    };
+}
+
+macro_rules! header_chrome_accept_encoding {
+    ($headers:expr) => {
         #[cfg(feature = "emulation-compression")]
         $headers.insert(
             ACCEPT_ENCODING,
@@ -118,14 +124,13 @@ macro_rules! header_chrome_accept {
         $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
     };
     (zstd, $headers:expr) => {
-        $headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
         #[cfg(feature = "emulation-compression")]
         $headers.insert(
             ACCEPT_ENCODING,
             HeaderValue::from_static("gzip, deflate, br, zstd"),
         );
         $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
-    }
+    };
 }
 
 macro_rules! header_firefox_sec_fetch {
@@ -133,6 +138,7 @@ macro_rules! header_firefox_sec_fetch {
         $headers.insert("sec-fetch-dest", HeaderValue::from_static("document"));
         $headers.insert("sec-fetch-mode", HeaderValue::from_static("navigate"));
         $headers.insert("sec-fetch-site", HeaderValue::from_static("none"));
+        $headers.insert("sec-fetch-user", HeaderValue::from_static("?1"));
     };
 }
 
@@ -144,12 +150,12 @@ macro_rules! header_firefox_accept {
                 "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             ),
         );
+        $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
         #[cfg(feature = "emulation-compression")]
         $headers.insert(
             ACCEPT_ENCODING,
             HeaderValue::from_static("gzip, deflate, br"),
         );
-        $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
     };
     (zstd, $headers:expr) => {
         $headers.insert(
@@ -158,21 +164,17 @@ macro_rules! header_firefox_accept {
                 "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             ),
         );
+        $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
         #[cfg(feature = "emulation-compression")]
         $headers.insert(
             ACCEPT_ENCODING,
             HeaderValue::from_static("gzip, deflate, br, zstd"),
         );
-        $headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
     };
 }
 
 macro_rules! header_firefox_ua {
     ($headers:expr, $ua:expr) => {
-        $headers.insert(
-            HeaderName::from_static("te"),
-            HeaderValue::from_static("trailers"),
-        );
         $headers.insert(USER_AGENT, HeaderValue::from_static($ua));
     };
 }

data/crates/wreq-util/src/emulate/profile/chrome/header.rs

@@ -12,9 +12,14 @@ pub fn header_initializer(
         emulation_os.platform(),
         emulation_os.is_mobile()
     );
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_chrome_ua!(headers, ua);
-    header_chrome_sec_fetch!(headers);
     header_chrome_accept!(headers);
+    header_chrome_sec_fetch!(headers);
+    header_chrome_accept_encoding!(headers);
     headers
 }
 
@@ -30,9 +35,14 @@ pub fn header_initializer_with_zstd(
         emulation_os.platform(),
         emulation_os.is_mobile()
     );
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_chrome_ua!(headers, ua);
+    header_chrome_accept!(headers);
     header_chrome_sec_fetch!(headers);
-    header_chrome_accept!(zstd, headers);
+    header_chrome_accept_encoding!(zstd, headers);
     headers
 }
 
@@ -48,9 +58,14 @@ pub fn header_initializer_with_zstd_priority(
         emulation_os.platform(),
         emulation_os.is_mobile()
     );
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_chrome_ua!(headers, ua);
+    header_chrome_accept!(headers);
     header_chrome_sec_fetch!(headers);
-    header_chrome_accept!(zstd, headers);
+    header_chrome_accept_encoding!(zstd, headers);
     headers.insert(
         HeaderName::from_static("priority"),
         HeaderValue::from_static("u=0, i"),

data/crates/wreq-util/src/emulate/profile/firefox/header.rs

@@ -4,7 +4,15 @@ pub fn header_initializer(ua: &'static str) -> HeaderMap {
     let mut headers = HeaderMap::new();
     header_firefox_ua!(headers, ua);
     header_firefox_accept!(headers);
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_firefox_sec_fetch!(headers);
+    headers.insert(
+        HeaderName::from_static("te"),
+        HeaderValue::from_static("trailers"),
+    );
     headers
 }
 
@@ -12,10 +20,18 @@ pub fn header_initializer_with_zstd(ua: &'static str) -> HeaderMap {
     let mut headers = HeaderMap::new();
     header_firefox_ua!(headers, ua);
     header_firefox_accept!(zstd, headers);
+    headers.insert(
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
+    );
     header_firefox_sec_fetch!(headers);
     headers.insert(
         HeaderName::from_static("priority"),
         HeaderValue::from_static("u=0, i"),
     );
+    headers.insert(
+        HeaderName::from_static("te"),
+        HeaderValue::from_static("trailers"),
+    );
     headers
 }

data/crates/wreq-util/src/emulate/profile/opera/header.rs

@@ -13,15 +13,14 @@ pub fn header_initializer_with_zstd_priority(
         emulation_os.platform(),
         emulation_os.is_mobile()
     );
-    header_chrome_ua!(headers, ua);
-
-    headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
-    #[cfg(feature = "emulation-compression")]
     headers.insert(
-        ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br, zstd"),
+        HeaderName::from_static("upgrade-insecure-requests"),
+        HeaderValue::from_static("1"),
     );
-    headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
+    header_chrome_ua!(headers, ua);
+    header_chrome_accept!(headers);
+    header_chrome_sec_fetch!(headers);
+    header_chrome_accept_encoding!(zstd, headers);
     headers.insert(
         HeaderName::from_static("priority"),
         HeaderValue::from_static("u=0, i"),

data/crates/wreq-util/tests/client.rs

@@ -1,9 +1,96 @@
 #![cfg(not(target_arch = "wasm32"))]
 mod support;
 
+use std::future::Future;
+
 use support::server;
+use tokio::{io::AsyncWriteExt, net::TcpStream};
 use wreq::Client;
-use wreq_util::{Emulation, Platform};
+use wreq_util::{Emulation, Platform, Profile};
+
+const CHROMIUM_HEADER_ORDER: &[&str] = &[
+    "sec-ch-ua",
+    "sec-ch-ua-mobile",
+    "sec-ch-ua-platform",
+    "upgrade-insecure-requests",
+    "user-agent",
+    "accept",
+    "sec-fetch-site",
+    "sec-fetch-mode",
+    "sec-fetch-user",
+    "sec-fetch-dest",
+    #[cfg(feature = "emulation-compression")]
+    "accept-encoding",
+    "accept-language",
+    "priority",
+];
+
+const FIREFOX_HEADER_ORDER: &[&str] = &[
+    "user-agent",
+    "accept",
+    "accept-language",
+    #[cfg(feature = "emulation-compression")]
+    "accept-encoding",
+    "upgrade-insecure-requests",
+    "sec-fetch-dest",
+    "sec-fetch-mode",
+    "sec-fetch-site",
+    "sec-fetch-user",
+    "te",
+];
+
+fn check_header_order<'a>(
+    request: &'a [u8],
+    stream: &'a mut TcpStream,
+    expected: &'static [&'static str],
+) -> Box<dyn Future<Output = ()> + Send + 'a> {
+    Box::new(async move {
+        let request = std::str::from_utf8(request)
+            .expect("request should be valid UTF-8")
+            .to_ascii_lowercase();
+        let mut previous = None;
+
+        for name in expected {
+            let marker = format!("\r\n{name}:");
+            let position = request
+                .find(&marker)
+                .unwrap_or_else(|| panic!("missing `{name}` header:\n{request}"));
+
+            if let Some(previous) = previous {
+                assert!(
+                    position > previous,
+                    "`{name}` header is out of order:\n{request}"
+                );
+            }
+
+            previous = Some(position);
+        }
+
+        assert!(request.contains("\r\nupgrade-insecure-requests: 1\r\n"));
+        assert!(request.contains("\r\nsec-fetch-user: ?1\r\n"));
+
+        stream
+            .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
+            .await
+            .expect("response");
+    })
+}
+
+async fn assert_emulation_headers(profile: Profile, expected: &'static [&'static str]) {
+    let server = server::low_level_with_response(move |request, stream| {
+        check_header_order(request, stream, expected)
+    });
+    let response = Client::builder()
+        .emulation(profile)
+        .build()
+        .expect("client")
+        .get(format!("http://{}/headers", server.addr()))
+        .send()
+        .await
+        .expect("request");
+
+    assert_eq!(response.status(), wreq::StatusCode::OK);
+}
 
 #[tokio::test]
 async fn test_client_emulation_device() {
@@ -49,3 +136,18 @@ async fn test_client_emulation_device() {
 
     assert_eq!(res.status(), wreq::StatusCode::OK);
 }
+
+#[tokio::test]
+async fn test_chrome_default_header_order() {
+    assert_emulation_headers(Emulation::Chrome133, CHROMIUM_HEADER_ORDER).await;
+}
+
+#[tokio::test]
+async fn test_firefox_default_header_order() {
+    assert_emulation_headers(Emulation::Firefox109, FIREFOX_HEADER_ORDER).await;
+}
+
+#[tokio::test]
+async fn test_opera_default_header_order() {
+    assert_emulation_headers(Emulation::Opera116, CHROMIUM_HEADER_ORDER).await;
+}

data/docs/windows-gnu-tokio-crash.md

@@ -57,14 +57,60 @@ DLL Name: KERNEL32.dll
 problematic `Sleep` reference can come from Rust std, MinGW, or pthread-related
 linking paths, not only from `windows-sys`.
 
-Verified with:
+Windows builds use the `x86_64-pc-windows-gnu` target and the RubyInstaller
+UCRT toolchain. The Rakefile configures that environment automatically, so the
+extension and tests can be built with:
 
 ```powershell
-.\script\build_windows_gnu.ps1 -SkipBundleInstall -SkipToolInstall -RunTests
+rustup target add x86_64-pc-windows-gnu
+ridk exec pacman -S --needed --noconfirm `
+    mingw-w64-ucrt-x86_64-gcc `
+    mingw-w64-ucrt-x86_64-clang `
+    mingw-w64-ucrt-x86_64-cmake `
+    mingw-w64-ucrt-x86_64-pkgconf
+bundle exec rake test
 ```
 
+The first Rake invocation also writes a machine-local `.cargo/windows.toml`.
+The checked-in `.cargo/config.toml` loads it automatically, so Cargo and Rust
+language servers use the same GNU target and RubyInstaller tools without
+editor-specific settings.
+
+## RubyInstaller development tools
+
+RubyInstaller adds Ruby itself to `PATH`, but it does not enable the bundled
+MSYS2 development environment globally. Running `ridk enable` adds tools such
+as GCC, CMake, and Make to the current terminal only. This is normally useful
+because it avoids conflicts with other Windows toolchains.
+
+Rust language servers start Cargo directly and do not run `ridk enable` or the
+Rakefile first. Add RubyInstaller's `ucrt64\bin` and `usr\bin` directories to
+the user `PATH` so Cargo can load GCC, libclang, and their dependent DLLs from
+any editor:
+
+```powershell
+$rubyRoot = ruby -rrbconfig -e "print RbConfig::CONFIG.fetch('prefix')"
+$required = @(
+    (Join-Path $rubyRoot "msys64\ucrt64\bin"),
+    (Join-Path $rubyRoot "msys64\usr\bin")
+)
+$userPath = [Environment]::GetEnvironmentVariable("Path", "User")
+$entries = @($userPath -split ";" | Where-Object {
+    $_ -and $_ -notin $required
+})
+$entries = $required + $entries
+[Environment]::SetEnvironmentVariable("Path", ($entries -join ";"), "User")
+```
+
+Restart the editor after changing `PATH` so its language server inherits the
+updated environment. Run `bundle exec rake compile` once after installing or
+switching Ruby so `.cargo/windows.toml` points to the active RubyInstaller.
+Run a Rake command after switching the same checkout between Windows and WSL;
+Windows generates this local configuration, while other platforms remove it.
+
 Result:
 
 ```text
-160 runs, 775 assertions, 0 failures, 0 errors
+254 runs, 1198 assertions, 0 failures, 0 errors
+2 Rust tests passed
 ```

data/extconf.rb

@@ -1,4 +1,8 @@
 require "mkmf"
+require_relative "script/rust_env"
+
+Wreq::RustEnv.activate
+
 # We use rb_sys for makefile generation only.
 # We can use `RB_SYS_CARGO_PROFILE` to choose Cargo profile
 # Read more https://github.com/oxidize-rb/rb-sys/blob/main/gem/README.md