wreq-rb 0.5.0 → 0.6.0

data/ext/wreq_rb/src/client.rs

@@ -2,6 +2,7 @@ use std::ffi::c_void;
 use std::panic::{self, AssertUnwindSafe};
 use std::ptr;
 use std::any::Any;
+use std::sync::Arc;
 use std::time::Duration;
 
 use magnus::{
@@ -9,13 +10,15 @@ use magnus::{
     try_convert::TryConvert, Value,
 };
 use tokio::runtime::Runtime;
+use tokio::sync::Semaphore;
+use tokio::task::JoinSet;
 use tokio_util::sync::CancellationToken;
 use std::net::IpAddr;
 use wreq::header::{HeaderMap, HeaderName, HeaderValue, OrigHeaderMap};
 use wreq::tls::TlsVersion;
-use wreq_util::{Emulation as BrowserEmulation, EmulationOS, EmulationOption};
+use wreq_util::{Emulation as BrowserEmulation, Platform as EmulationPlatform, Profile as BrowserProfile};
 
-use crate::error::{generic_error, to_magnus_error};
+use crate::error::{generic_error, to_magnus_error, wreq_error};
 use crate::response::Response;
 
 // --------------------------------------------------------------------------
@@ -137,39 +140,95 @@ async fn execute_request(req: wreq::RequestBuilder) -> Result<ResponseData, wreq
     Ok(ResponseData { status, headers, body, url, version, content_length, transfer_size })
 }
 
+// --------------------------------------------------------------------------
+// Batch execution
+// --------------------------------------------------------------------------
+
+const DEFAULT_BATCH_CONCURRENCY: usize = 16;
+
+/// Per-item batch result as pure Rust types (no Ruby objects). Errors are kept
+/// as messages so one failure never discards the rest of the batch.
+enum BatchItem {
+    Ok(ResponseData),
+    Err(String),
+}
+
+/// Outcome of a whole batch performed outside the GVL.
+enum BatchOutcome {
+    Done(Vec<BatchItem>),
+    Interrupted,
+}
+
+/// Run every request concurrently with at most `concurrency` in flight,
+/// returning results in input order.
+async fn execute_batch(reqs: Vec<wreq::RequestBuilder>, concurrency: usize) -> Vec<BatchItem> {
+    let permits = Arc::new(Semaphore::new(concurrency));
+    let mut set: JoinSet<(usize, BatchItem)> = JoinSet::new();
+
+    for (idx, req) in reqs.into_iter().enumerate() {
+        let permits = Arc::clone(&permits);
+        set.spawn(async move {
+            let _permit = match permits.acquire_owned().await {
+                Ok(p) => p,
+                Err(_) => return (idx, BatchItem::Err("batch semaphore closed".to_owned())),
+            };
+            let item = match execute_request(req).await {
+                Ok(data) => BatchItem::Ok(data),
+                Err(e) => BatchItem::Err(e.to_string()),
+            };
+            (idx, item)
+        });
+    }
+
+    let mut slots: Vec<Option<BatchItem>> = Vec::new();
+    slots.resize_with(set.len(), || None);
+    while let Some(joined) = set.join_next().await {
+        // A JoinError means the task panicked or was aborted; its slot is left
+        // empty and filled with a generic error below.
+        if let Ok((idx, item)) = joined {
+            slots[idx] = Some(item);
+        }
+    }
+
+    slots
+        .into_iter()
+        .map(|slot| slot.unwrap_or_else(|| BatchItem::Err("request task failed".to_owned())))
+        .collect()
+}
+
 // --------------------------------------------------------------------------
 // Emulation helpers
 // --------------------------------------------------------------------------
 
-/// The default emulation to apply when none is specified.
-const DEFAULT_EMULATION: BrowserEmulation = BrowserEmulation::Chrome145;
+/// The default browser profile to apply when none is specified.
+const DEFAULT_EMULATION: BrowserProfile = BrowserProfile::Chrome148;
 
-/// Parse a Ruby string like "chrome_143" into a BrowserEmulation variant.
-fn parse_emulation(name: &str) -> Result<BrowserEmulation, magnus::Error> {
+/// Parse a Ruby string like "chrome_143" into a BrowserProfile variant.
+fn parse_emulation(name: &str) -> Result<BrowserProfile, magnus::Error> {
     let json_val = serde_json::Value::String(name.to_string());
-    serde_json::from_value::<BrowserEmulation>(json_val)
-        .map_err(|_| generic_error(format!("unknown emulation: '{}'. Use names like 'chrome_145', 'firefox_147', 'safari_18.5', etc.", name)))
+    serde_json::from_value::<BrowserProfile>(json_val)
+        .map_err(|_| generic_error(format!("unknown emulation: '{}'. Use names like 'chrome_148', 'firefox_151', 'safari_18.5', etc.", name)))
 }
 
-/// Parse a Ruby string like "windows" into an EmulationOS variant.
-fn parse_emulation_os(name: &str) -> Result<EmulationOS, magnus::Error> {
+/// Parse a Ruby string like "windows" into an EmulationPlatform variant.
+fn parse_emulation_os(name: &str) -> Result<EmulationPlatform, magnus::Error> {
     let json_val = serde_json::Value::String(name.to_string());
-    serde_json::from_value::<EmulationOS>(json_val)
+    serde_json::from_value::<EmulationPlatform>(json_val)
         .map_err(|_| generic_error("unknown emulation_os. Use: 'windows', 'macos', 'linux', 'android', 'ios'"))
 }
 
-/// Build an EmulationOption from an Emulation and an optional OS from the opts hash.
+/// Build a BrowserEmulation from a profile and an optional platform from the opts hash.
 fn build_emulation_option(
-    emu: BrowserEmulation,
+    profile: BrowserProfile,
     opts: &RHash,
-) -> Result<EmulationOption, magnus::Error> {
-    let os = match hash_get_string(opts, "emulation_os")? {
-        Some(os_name) => parse_emulation_os(&os_name)?,
-        None => EmulationOS::default(),
+) -> Result<BrowserEmulation, magnus::Error> {
+    let platform = match hash_get_string(opts, "emulation_os")? {
+        Some(platform_name) => parse_emulation_os(&platform_name)?,
+        None => EmulationPlatform::default(),
     };
-    Ok(EmulationOption::builder()
-        .emulation(emu)
-        .emulation_os(os)
+    Ok(BrowserEmulation::builder()
+        .profile(profile)
+        .platform(platform)
         .build())
 }
 
@@ -192,7 +251,8 @@ impl Client {
             Some(RHash::try_convert(args[0])?)
         };
 
-        let mut builder = wreq::Client::builder();
+        let mut builder = wreq::Client::builder()
+            .retry(wreq::retry::Policy::never());
 
         if let Some(opts) = opts {
             // Apply header_order BEFORE emulation so the user's ordering takes precedence
@@ -276,11 +336,11 @@ impl Client {
             }
 
             if let Some(v) = hash_get_bool(&opts, "verify_host")? {
-                builder = builder.verify_hostname(v);
+                builder = builder.tls_verify_hostname(v);
             }
 
             if let Some(v) = hash_get_bool(&opts, "verify_cert")? {
-                builder = builder.cert_verification(v);
+                builder = builder.tls_cert_verification(v);
             }
 
             if let Some(true) = hash_get_bool(&opts, "http1_only")? {
@@ -311,7 +371,7 @@ impl Client {
                 builder = builder.pool_max_idle_per_host(n);
             }
 
-            if let Some(n) = hash_get_u32(&opts, "pool_max_size")? {
+            if let Some(n) = hash_get_usize(&opts, "pool_max_size")? {
                 builder = builder.pool_max_size(n);
             }
 
@@ -334,11 +394,11 @@ impl Client {
             }
 
             if let Some(s) = hash_get_string(&opts, "min_tls_version")? {
-                builder = builder.min_tls_version(parse_tls_version(&s)?);
+                builder = builder.tls_min_version(parse_tls_version(&s)?);
             }
 
             if let Some(s) = hash_get_string(&opts, "max_tls_version")? {
-                builder = builder.max_tls_version(parse_tls_version(&s)?);
+                builder = builder.tls_max_version(parse_tls_version(&s)?);
             }
         } else {
             builder = builder.emulation(DEFAULT_EMULATION);
@@ -387,7 +447,6 @@ impl Client {
             old
         };
         old_token.cancel();
-        self.inner.cancel_connections();
     }
 
     fn execute_method(&self, method_str: &str, args: &[Value]) -> Result<Response, magnus::Error> {
@@ -439,12 +498,141 @@ impl Client {
         };
         Ok(Response::new(data.status, data.headers, data.body, data.url, data.version, data.content_length, data.transfer_size))
     }
+
+    /// Wreq::Client#request_batch(specs) or #request_batch(specs, options)
+    fn request_batch(&self, args: &[Value]) -> Result<RArray, magnus::Error> {
+        if args.is_empty() {
+            return Err(generic_error("an array of requests is required"));
+        }
+        let specs = RArray::try_convert(args[0])?;
+
+        let opts: Option<RHash> = if args.len() > 1 {
+            Some(RHash::try_convert(args[1])?)
+        } else {
+            None
+        };
+
+        let concurrency = match opts.as_ref() {
+            Some(o) => hash_get_usize(o, "concurrency")?.unwrap_or(DEFAULT_BATCH_CONCURRENCY),
+            None => DEFAULT_BATCH_CONCURRENCY,
+        };
+        if concurrency == 0 {
+            return Err(generic_error("concurrency must be >= 1"));
+        }
+
+        // All Ruby -> Rust conversion happens here, while we still hold the GVL.
+        let mut reqs: Vec<wreq::RequestBuilder> = Vec::with_capacity(specs.len());
+        for spec in specs.into_iter() {
+            reqs.push(self.build_request(spec, opts.as_ref())?);
+        }
+
+        let ruby = unsafe { Ruby::get_unchecked() };
+        if reqs.is_empty() {
+            return Ok(ruby.ary_new());
+        }
+
+        let client_token = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner()).clone();
+
+        // One GVL release covering the whole batch.
+        let outcome: BatchOutcome = unsafe {
+            without_gvl(|thread_token| {
+                runtime().block_on(async {
+                    tokio::select! {
+                        biased;
+                        _ = thread_token.cancelled() => BatchOutcome::Interrupted,
+                        _ = client_token.cancelled() => BatchOutcome::Interrupted,
+                        items = execute_batch(reqs, concurrency) => BatchOutcome::Done(items),
+                    }
+                })
+            })
+        };
+
+        let items = match outcome {
+            BatchOutcome::Done(items) => items,
+            BatchOutcome::Interrupted => return Err(generic_error("request interrupted")),
+        };
+
+        // Back under the GVL: Ruby objects may be created again.
+        let results = ruby.ary_new_capa(items.len());
+        for item in items {
+            match item {
+                BatchItem::Ok(d) => {
+                    let resp = Response::new(
+                        d.status, d.headers, d.body, d.url, d.version, d.content_length,
+                        d.transfer_size,
+                    );
+                    results.push(ruby.obj_wrap(resp))?;
+                }
+                BatchItem::Err(msg) => {
+                    let err: Value = wreq_error().funcall("new", (msg,))?;
+                    results.push(err)?;
+                }
+            }
+        }
+        Ok(results)
+    }
+
+    /// Convert a single batch spec into a RequestBuilder. Accepted forms:
+    ///   "https://example.com"      -> GET
+    ///   { method:, url:, **opts }
+    fn build_request(
+        &self,
+        spec: Value,
+        shared: Option<&RHash>,
+    ) -> Result<wreq::RequestBuilder, magnus::Error> {
+        let mut method = wreq::Method::GET;
+        let url: String;
+        let mut item_opts: Option<RHash> = None;
+
+        if let Some(hash) = RHash::from_value(spec) {
+            url = hash_get_string(&hash, "url")?
+                .ok_or_else(|| generic_error("each request hash requires a :url"))?;
+            if let Some(val) = hash_get_value(&hash, "method")? {
+                method = value_to_method(val)?;
+            }
+            item_opts = Some(hash);
+        } else {
+            url = TryConvert::try_convert(spec)?;
+        }
+
+        let mut req = self.inner.request(method, &url);
+        if let Some(shared) = shared {
+            req = apply_request_options(req, shared)?;
+        }
+        if let Some(item_opts) = item_opts {
+            req = apply_request_options(req, &item_opts)?;
+        }
+        Ok(req)
+    }
+}
+
+/// Parse a String or Symbol like "post" / :post into an HTTP method.
+fn value_to_method(val: Value) -> Result<wreq::Method, magnus::Error> {
+    let name: String = val.funcall("to_s", ())?;
+    name.to_uppercase()
+        .parse()
+        .map_err(|_| generic_error(format!("invalid HTTP method: {}", name)))
 }
 
 fn apply_request_options(
     mut req: wreq::RequestBuilder,
     opts: &RHash,
 ) -> Result<wreq::RequestBuilder, magnus::Error> {
+    if let Some(val) = hash_get_value(opts, "emulation")? {
+        let ruby = unsafe { Ruby::get_unchecked() };
+        if val.is_kind_of(ruby.class_false_class()) {
+            // emulation: false — no per-request emulation override
+        } else if val.is_kind_of(ruby.class_true_class()) {
+            let opt = build_emulation_option(DEFAULT_EMULATION, opts)?;
+            req = req.emulation(opt);
+        } else {
+            let name: String = TryConvert::try_convert(val)?;
+            let emu = parse_emulation(&name)?;
+            let opt = build_emulation_option(emu, opts)?;
+            req = req.emulation(opt);
+        }
+    }
+
     if let Some(hdr_hash) = hash_get_hash(opts, "headers")? {
         let hmap = hash_to_header_map(&hdr_hash)?;
         req = req.headers(hmap);
@@ -499,21 +687,6 @@ fn apply_request_options(
         req = req.proxy(proxy);
     }
 
-    if let Some(val) = hash_get_value(opts, "emulation")? {
-        let ruby = unsafe { Ruby::get_unchecked() };
-        if val.is_kind_of(ruby.class_false_class()) {
-            // emulation: false — no per-request emulation override
-        } else if val.is_kind_of(ruby.class_true_class()) {
-            let opt = build_emulation_option(DEFAULT_EMULATION, opts)?;
-            req = req.emulation(opt);
-        } else {
-            let name: String = TryConvert::try_convert(val)?;
-            let emu = parse_emulation(&name)?;
-            let opt = build_emulation_option(emu, opts)?;
-            req = req.emulation(opt);
-        }
-    }
-
     Ok(req)
 }
 
@@ -599,13 +772,6 @@ fn hash_get_usize(hash: &RHash, key: &str) -> Result<Option<usize>, magnus::Erro
     }
 }
 
-fn hash_get_u32(hash: &RHash, key: &str) -> Result<Option<u32>, magnus::Error> {
-    match hash_get_value(hash, key)? {
-        Some(v) => Ok(Some(TryConvert::try_convert(v)?)),
-        None => Ok(None),
-    }
-}
-
 fn parse_tls_version(s: &str) -> Result<TlsVersion, magnus::Error> {
     match s {
         "tls1.0" | "tls_1_0" | "1.0" => Ok(TlsVersion::TLS_1_0),
@@ -699,6 +865,7 @@ pub fn init(_ruby: &magnus::Ruby, module: &magnus::RModule) -> Result<(), magnus
     client_class.define_method("delete", method!(Client::delete, -1))?;
     client_class.define_method("head", method!(Client::head, -1))?;
     client_class.define_method("options", method!(Client::options, -1))?;
+    client_class.define_method("request_batch", method!(Client::request_batch, -1))?;
     client_class.define_method("cancel", method!(Client::cancel, 0))?;
 
     module.define_module_function("get", function!(wreq_get, -1))?;

data/lib/wreq-rb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wreq
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/patches/0001-add-transfer-size-tracking.patch

@@ -1,60 +1,51 @@
-diff --git a/src/client/http.rs b/src/client/http.rs
-index 10be0c83..462c77d0 100644
---- a/src/client/http.rs
-+++ b/src/client/http.rs
-@@ -33,6 +33,7 @@ use self::future::Pending;
-     feature = "deflate",
- ))]
- use super::layer::decoder::{AcceptEncoding, DecompressionLayer};
-+use super::layer::transfer_size::{CountingBody, TransferSizeLayer, TransferSizeService};
- #[cfg(feature = "ws")]
- use super::ws::WebSocketRequestBuilder;
- use super::{
-@@ -104,7 +105,7 @@ type Decompression<T> = super::layer::decoder::Decompression<T>;
-     feature = "brotli",
-     feature = "deflate"
- ))]
--type ResponseBody = TimeoutBody<tower_http::decompression::DecompressionBody<Incoming>>;
-+type ResponseBody = TimeoutBody<tower_http::decompression::DecompressionBody<CountingBody<Incoming>>>;
+diff --git a/src/client.rs b/src/client.rs
+index 4abe25d8..76e709a8 100644
+--- a/src/client.rs
++++ b/src/client.rs
+@@ -49,6 +49,7 @@ use self::{
+         redirect::{FollowRedirect, FollowRedirectLayer},
+         retry::RetryPolicy,
+         timeout::{Timeout, TimeoutLayer, TimeoutOptions, body::TimeoutBody},
++        transfer_size::{CountingBody, TransferSizeLayer, TransferSizeService},
+     },
+     request::{Request, RequestBuilder},
+     response::Response,
+@@ -115,25 +116,30 @@ type MaybeDecompressionBody<T> = tower_http::decompression::DecompressionBody<T>
+ type ClientService = Timeout<
+     ConfigService<
+         MaybeDecompression<
+-            Retry<RetryPolicy, FollowRedirect<HttpClient<Connector, Body>, FollowRedirectPolicy>>,
++            TransferSizeService<
++                Retry<
++                    RetryPolicy,
++                    FollowRedirect<HttpClient<Connector, Body>, FollowRedirectPolicy>,
++                >,
++            >,
+         >,
+     >,
+ >;
  
- /// Response body type with timeout only (no compression features).
- #[cfg(not(any(
-@@ -113,23 +114,25 @@ type ResponseBody = TimeoutBody<tower_http::decompression::DecompressionBody<Inc
-     feature = "brotli",
-     feature = "deflate"
- )))]
--type ResponseBody = TimeoutBody<Incoming>;
-+type ResponseBody = TimeoutBody<CountingBody<Incoming>>;
+ type BoxedClientService = BoxCloneSyncService<
+     http::Request<Body>,
+-    http::Response<TimeoutBody<MaybeDecompressionBody<Incoming>>>,
++    http::Response<TimeoutBody<MaybeDecompressionBody<CountingBody<Incoming>>>>,
+     BoxError,
+ >;
  
- /// The complete HTTP client service stack with all middleware layers.
- type ClientService = Timeout<
-     ResponseBodyTimeout<
-         ConfigService<
-             Decompression<
--                Retry<
--                    RetryPolicy,
--                    FollowRedirect<
--                        CookieService<
--                            MapErr<
--                                HttpClient<Connector, Body>,
--                                fn(client::error::Error) -> BoxError,
-+                TransferSizeService<
-+                    Retry<
-+                        RetryPolicy,
-+                        FollowRedirect<
-+                            CookieService<
-+                                MapErr<
-+                                    HttpClient<Connector, Body>,
-+                                    fn(client::error::Error) -> BoxError,
-+                                >,
-                             >,
-+                            FollowRedirectPolicy,
-                         >,
--                        FollowRedirectPolicy,
-                     >,
-                 >,
-             >,
-@@ -582,6 +585,10 @@ impl ClientBuilder {
+ type BoxedClientServiceLayer = BoxCloneSyncServiceLayer<
+     BoxCloneSyncService<
+         http::Request<Body>,
+-        http::Response<MaybeDecompressionBody<Incoming>>,
++        http::Response<MaybeDecompressionBody<CountingBody<Incoming>>>,
+         BoxError,
+     >,
+     http::Request<Body>,
+-    http::Response<MaybeDecompressionBody<Incoming>>,
++    http::Response<MaybeDecompressionBody<CountingBody<Incoming>>>,
+     BoxError,
+ >;
+ 
+@@ -593,6 +599,10 @@ impl ClientBuilder {
                  })
                  .service(service);
  
@@ -65,18 +56,36 @@ index 10be0c83..462c77d0 100644
              #[cfg(any(
                  feature = "gzip",
                  feature = "zstd",
+@@ -1585,7 +1595,7 @@ impl ClientBuilder {
+         L: Layer<
+                 BoxCloneSyncService<
+                     http::Request<Body>,
+-                    http::Response<MaybeDecompressionBody<Incoming>>,
++                    http::Response<MaybeDecompressionBody<CountingBody<Incoming>>>,
+                     BoxError,
+                 >,
+             > + Clone
+@@ -1594,7 +1604,7 @@ impl ClientBuilder {
+             + 'static,
+         L::Service: Service<
+                 http::Request<Body>,
+-                Response = http::Response<MaybeDecompressionBody<Incoming>>,
++                Response = http::Response<MaybeDecompressionBody<CountingBody<Incoming>>>,
+                 Error = BoxError,
+             > + Clone
+             + Send
 diff --git a/src/client/layer.rs b/src/client/layer.rs
-index 05bb533f..cb0b0866 100644
+index 5c03ef4d..fe37cff0 100644
 --- a/src/client/layer.rs
 +++ b/src/client/layer.rs
-@@ -13,3 +13,4 @@ pub mod decoder;
+@@ -12,3 +12,4 @@ pub mod decoder;
  pub mod redirect;
  pub mod retry;
  pub mod timeout;
 +pub mod transfer_size;
 diff --git a/src/client/layer/transfer_size.rs b/src/client/layer/transfer_size.rs
 new file mode 100644
-index 00000000..7e8a4390
+index 00000000..f155ec8d
 --- /dev/null
 +++ b/src/client/layer/transfer_size.rs
 @@ -0,0 +1,176 @@
@@ -91,8 +100,8 @@ index 00000000..7e8a4390
 +use std::{
 +    pin::Pin,
 +    sync::{
-+        atomic::{AtomicU64, Ordering},
 +        Arc,
++        atomic::{AtomicU64, Ordering},
 +    },
 +    task::{Context, Poll},
 +};
@@ -257,18 +266,18 @@ index 00000000..7e8a4390
 +    }
 +}
 diff --git a/src/client/response.rs b/src/client/response.rs
-index d98859b9..16ed35c3 100644
+index 3d3532af..ed478804 100644
 --- a/src/client/response.rs
 +++ b/src/client/response.rs
-@@ -20,6 +20,7 @@ use serde::de::DeserializeOwned;
- use super::{
-     conn::HttpInfo,
-     core::{ext::ReasonPhrase, upgrade},
-+    layer::transfer_size::TransferSizeHandle,
- };
+@@ -18,6 +18,7 @@ use mime::Mime;
+ use serde::de::DeserializeOwned;
+ use wreq_proto::ext::ReasonPhrase;
+ 
++use super::layer::transfer_size::TransferSizeHandle;
  #[cfg(feature = "cookies")]
  use crate::cookie;
-@@ -91,6 +92,21 @@ impl Response {
+ use crate::{
+@@ -97,6 +98,21 @@ impl Response {
          HttpBody::size_hint(self.res.body()).exact()
      }