wreq-rb 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +257 -102
  3. data/README.md +28 -0
  4. data/ext/wreq_rb/Cargo.toml +4 -4
  5. data/ext/wreq_rb/src/client.rs +176 -2
  6. data/lib/wreq-rb/version.rb +1 -1
  7. data/patches/0001-add-transfer-size-tracking.patch +11 -15
  8. data/vendor/wreq/Cargo.toml +9 -8
  9. data/vendor/wreq/README.md +5 -5
  10. data/vendor/wreq/bench/support/bench.rs +6 -2
  11. data/vendor/wreq/bench/support/client.rs +88 -1
  12. data/vendor/wreq/bench/support/exec.rs +0 -0
  13. data/vendor/wreq/bench/support/rt.rs +34 -0
  14. data/vendor/wreq/bench/support/server.rs +1 -1
  15. data/vendor/wreq/bench/support.rs +1 -15
  16. data/vendor/wreq/examples/cert_store.rs +13 -13
  17. data/vendor/wreq/examples/request_with_emulate.rs +1 -1
  18. data/vendor/wreq/examples/tcp_linger.rs +22 -0
  19. data/vendor/wreq/src/client/layer/client/pool.rs +17 -17
  20. data/vendor/wreq/src/client/layer/client.rs +2 -0
  21. data/vendor/wreq/src/client/layer/decoder.rs +71 -17
  22. data/vendor/wreq/src/client/layer/redirect/future.rs +49 -63
  23. data/vendor/wreq/src/client/layer/redirect/policy.rs +2 -26
  24. data/vendor/wreq/src/client/layer/redirect.rs +48 -60
  25. data/vendor/wreq/src/client/layer/retry.rs +12 -15
  26. data/vendor/wreq/src/client/layer/timeout/body.rs +27 -21
  27. data/vendor/wreq/src/client/layer/timeout/future.rs +33 -58
  28. data/vendor/wreq/src/client/layer/timeout.rs +8 -14
  29. data/vendor/wreq/src/client/request.rs +4 -0
  30. data/vendor/wreq/src/client.rs +53 -31
  31. data/vendor/wreq/src/conn/connector.rs +99 -129
  32. data/vendor/wreq/src/conn/http.rs +25 -18
  33. data/vendor/wreq/src/conn/net/tcp.rs +601 -107
  34. data/vendor/wreq/src/conn/proxy/socks.rs +6 -6
  35. data/vendor/wreq/src/conn/timeout.rs +166 -0
  36. data/vendor/wreq/src/conn.rs +5 -4
  37. data/vendor/wreq/src/cookie/jar.rs +1225 -0
  38. data/vendor/wreq/src/cookie/store.rs +321 -0
  39. data/vendor/wreq/src/cookie.rs +108 -612
  40. data/vendor/wreq/src/dns/resolve.rs +8 -2
  41. data/vendor/wreq/src/dns.rs +4 -4
  42. data/vendor/wreq/src/error.rs +53 -20
  43. data/vendor/wreq/src/lib.rs +1 -0
  44. data/vendor/wreq/src/proxy/matcher.rs +26 -12
  45. data/vendor/wreq/src/proxy/win.rs +39 -9
  46. data/vendor/wreq/src/redirect.rs +515 -100
  47. data/vendor/wreq/src/tls/conn.rs +3 -11
  48. data/vendor/wreq/src/tls/session.rs +7 -8
  49. data/vendor/wreq/src/tls/trust/store.rs +4 -4
  50. data/vendor/wreq/src/util.rs +23 -0
  51. data/vendor/wreq/tests/badssl.rs +72 -7
  52. data/vendor/wreq/tests/brotli.rs +1 -1
  53. data/vendor/wreq/tests/client.rs +24 -0
  54. data/vendor/wreq/tests/connector_layers.rs +8 -4
  55. data/vendor/wreq/tests/cookie.rs +59 -0
  56. data/vendor/wreq/tests/deflate.rs +1 -1
  57. data/vendor/wreq/tests/gzip.rs +53 -1
  58. data/vendor/wreq/tests/layers.rs +8 -4
  59. data/vendor/wreq/tests/redirect.rs +180 -97
  60. data/vendor/wreq/tests/timeouts.rs +47 -12
  61. data/vendor/wreq/tests/zstd.rs +1 -1
  62. metadata +8 -2
data/README.md CHANGED
@@ -114,6 +114,34 @@ All methods are available on both `Wreq` (module-level) and `Wreq::Client` (inst
114
114
  | `head(url, **opts)` | HEAD request |
115
115
  | `options(url, **opts)` | OPTIONS request |
116
116
 
117
+ ### Batch Requests
118
+
119
+ `Wreq::Client#request_batch` runs many requests concurrently inside a **single**
120
+ GVL release, multiplexed over the client's connection pool. This avoids one Ruby
121
+ thread per request.
122
+
123
+ ```ruby
124
+ client = Wreq::Client.new(redirect: false, pool_max_idle_per_host: 32)
125
+
126
+ responses = client.request_batch(urls, concurrency: 128)
127
+ ```
128
+
129
+ Each element of the array is either a URL string or a hash:
130
+
131
+ ```ruby
132
+ client.request_batch([
133
+ "https://example.com/a", # GET
134
+ { method: :put, url: "https://example.com/b", body: "hi" }
135
+ ], concurrency: 32)
136
+ ```
137
+
138
+ - Results are returned **in input order**.
139
+ - Each element is a `Wreq::Response` **or** a `Wreq::Error` — a single failed
140
+ request never discards the rest of the batch. Errors are returned, not raised.
141
+ - `concurrency` caps the number of in-flight requests and defaults to `16`.
142
+ - `cancel` and Ruby thread interrupts abort the whole batch, raising
143
+ `Wreq::Error` with `"request interrupted"`.
144
+
117
145
  ### Cancelling Requests
118
146
 
119
147
  Call `cancel` on a client to interrupt all in-flight requests immediately:
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "wreq_rb"
3
- version = "0.5.1"
3
+ version = "0.6.1"
4
4
  edition = "2021"
5
5
  publish = false
6
6
 
@@ -10,7 +10,7 @@ crate-type = ["cdylib"]
10
10
  [dependencies]
11
11
  magnus = { version = "0.8", features = ["rb-sys"] }
12
12
  rb-sys = "0.9"
13
- wreq = { path = "../../vendor/wreq", version = "=6.0.0-rc.29", features = [
13
+ wreq = { path = "../../vendor/wreq", version = "=0.16.1", features = [
14
14
  "cookies",
15
15
  "json",
16
16
  "gzip",
@@ -24,7 +24,7 @@ wreq = { path = "../../vendor/wreq", version = "=6.0.0-rc.29", features = [
24
24
  "query",
25
25
  "form",
26
26
  ] }
27
- wreq-util = { version = "=3.0.0-rc.12", features = ["emulation", "emulation-serde", "emulation-compression"] }
27
+ wreq-util = { version = "=0.2.0", features = ["emulation", "emulation-serde", "emulation-compression"] }
28
28
  tokio = { version = "1", features = ["full"] }
29
29
  tokio-util = "0.7"
30
30
  serde_json = "1.0"
@@ -32,6 +32,6 @@ bytes = "1"
32
32
  http = "1"
33
33
 
34
34
  [target.'cfg(target_os = "linux")'.dependencies]
35
- wreq = { path = "../../vendor/wreq", version = "=6.0.0-rc.29", features = [
35
+ wreq = { path = "../../vendor/wreq", version = "=0.16.1", features = [
36
36
  "prefix-symbols",
37
37
  ] }
@@ -2,6 +2,7 @@ use std::ffi::c_void;
2
2
  use std::panic::{self, AssertUnwindSafe};
3
3
  use std::ptr;
4
4
  use std::any::Any;
5
+ use std::sync::Arc;
5
6
  use std::time::Duration;
6
7
 
7
8
  use magnus::{
@@ -9,13 +10,15 @@ use magnus::{
9
10
  try_convert::TryConvert, Value,
10
11
  };
11
12
  use tokio::runtime::Runtime;
13
+ use tokio::sync::Semaphore;
14
+ use tokio::task::JoinSet;
12
15
  use tokio_util::sync::CancellationToken;
13
16
  use std::net::IpAddr;
14
17
  use wreq::header::{HeaderMap, HeaderName, HeaderValue, OrigHeaderMap};
15
18
  use wreq::tls::TlsVersion;
16
19
  use wreq_util::{Emulation as BrowserEmulation, Platform as EmulationPlatform, Profile as BrowserProfile};
17
20
 
18
- use crate::error::{generic_error, to_magnus_error};
21
+ use crate::error::{generic_error, to_magnus_error, wreq_error};
19
22
  use crate::response::Response;
20
23
 
21
24
  // --------------------------------------------------------------------------
@@ -137,12 +140,68 @@ async fn execute_request(req: wreq::RequestBuilder) -> Result<ResponseData, wreq
137
140
  Ok(ResponseData { status, headers, body, url, version, content_length, transfer_size })
138
141
  }
139
142
 
143
+ // --------------------------------------------------------------------------
144
+ // Batch execution
145
+ // --------------------------------------------------------------------------
146
+
147
+ const DEFAULT_BATCH_CONCURRENCY: usize = 16;
148
+
149
+ /// Per-item batch result as pure Rust types (no Ruby objects). Errors are kept
150
+ /// as messages so one failure never discards the rest of the batch.
151
+ enum BatchItem {
152
+ Ok(ResponseData),
153
+ Err(String),
154
+ }
155
+
156
+ /// Outcome of a whole batch performed outside the GVL.
157
+ enum BatchOutcome {
158
+ Done(Vec<BatchItem>),
159
+ Interrupted,
160
+ }
161
+
162
+ /// Run every request concurrently with at most `concurrency` in flight,
163
+ /// returning results in input order.
164
+ async fn execute_batch(reqs: Vec<wreq::RequestBuilder>, concurrency: usize) -> Vec<BatchItem> {
165
+ let permits = Arc::new(Semaphore::new(concurrency));
166
+ let mut set: JoinSet<(usize, BatchItem)> = JoinSet::new();
167
+
168
+ for (idx, req) in reqs.into_iter().enumerate() {
169
+ let permits = Arc::clone(&permits);
170
+ set.spawn(async move {
171
+ let _permit = match permits.acquire_owned().await {
172
+ Ok(p) => p,
173
+ Err(_) => return (idx, BatchItem::Err("batch semaphore closed".to_owned())),
174
+ };
175
+ let item = match execute_request(req).await {
176
+ Ok(data) => BatchItem::Ok(data),
177
+ Err(e) => BatchItem::Err(e.to_string()),
178
+ };
179
+ (idx, item)
180
+ });
181
+ }
182
+
183
+ let mut slots: Vec<Option<BatchItem>> = Vec::new();
184
+ slots.resize_with(set.len(), || None);
185
+ while let Some(joined) = set.join_next().await {
186
+ // A JoinError means the task panicked or was aborted; its slot is left
187
+ // empty and filled with a generic error below.
188
+ if let Ok((idx, item)) = joined {
189
+ slots[idx] = Some(item);
190
+ }
191
+ }
192
+
193
+ slots
194
+ .into_iter()
195
+ .map(|slot| slot.unwrap_or_else(|| BatchItem::Err("request task failed".to_owned())))
196
+ .collect()
197
+ }
198
+
140
199
  // --------------------------------------------------------------------------
141
200
  // Emulation helpers
142
201
  // --------------------------------------------------------------------------
143
202
 
144
203
  /// The default browser profile to apply when none is specified.
145
- const DEFAULT_EMULATION: BrowserProfile = BrowserProfile::Chrome148;
204
+ const DEFAULT_EMULATION: BrowserProfile = BrowserProfile::Chrome149;
146
205
 
147
206
  /// Parse a Ruby string like "chrome_143" into a BrowserProfile variant.
148
207
  fn parse_emulation(name: &str) -> Result<BrowserProfile, magnus::Error> {
@@ -439,6 +498,120 @@ impl Client {
439
498
  };
440
499
  Ok(Response::new(data.status, data.headers, data.body, data.url, data.version, data.content_length, data.transfer_size))
441
500
  }
501
+
502
+ /// Wreq::Client#request_batch(specs) or #request_batch(specs, options)
503
+ fn request_batch(&self, args: &[Value]) -> Result<RArray, magnus::Error> {
504
+ if args.is_empty() {
505
+ return Err(generic_error("an array of requests is required"));
506
+ }
507
+ let specs = RArray::try_convert(args[0])?;
508
+
509
+ let opts: Option<RHash> = if args.len() > 1 {
510
+ Some(RHash::try_convert(args[1])?)
511
+ } else {
512
+ None
513
+ };
514
+
515
+ let concurrency = match opts.as_ref() {
516
+ Some(o) => hash_get_usize(o, "concurrency")?.unwrap_or(DEFAULT_BATCH_CONCURRENCY),
517
+ None => DEFAULT_BATCH_CONCURRENCY,
518
+ };
519
+ if concurrency == 0 {
520
+ return Err(generic_error("concurrency must be >= 1"));
521
+ }
522
+
523
+ // All Ruby -> Rust conversion happens here, while we still hold the GVL.
524
+ let mut reqs: Vec<wreq::RequestBuilder> = Vec::with_capacity(specs.len());
525
+ for spec in specs.into_iter() {
526
+ reqs.push(self.build_request(spec, opts.as_ref())?);
527
+ }
528
+
529
+ let ruby = unsafe { Ruby::get_unchecked() };
530
+ if reqs.is_empty() {
531
+ return Ok(ruby.ary_new());
532
+ }
533
+
534
+ let client_token = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner()).clone();
535
+
536
+ // One GVL release covering the whole batch.
537
+ let outcome: BatchOutcome = unsafe {
538
+ without_gvl(|thread_token| {
539
+ runtime().block_on(async {
540
+ tokio::select! {
541
+ biased;
542
+ _ = thread_token.cancelled() => BatchOutcome::Interrupted,
543
+ _ = client_token.cancelled() => BatchOutcome::Interrupted,
544
+ items = execute_batch(reqs, concurrency) => BatchOutcome::Done(items),
545
+ }
546
+ })
547
+ })
548
+ };
549
+
550
+ let items = match outcome {
551
+ BatchOutcome::Done(items) => items,
552
+ BatchOutcome::Interrupted => return Err(generic_error("request interrupted")),
553
+ };
554
+
555
+ // Back under the GVL: Ruby objects may be created again.
556
+ let results = ruby.ary_new_capa(items.len());
557
+ for item in items {
558
+ match item {
559
+ BatchItem::Ok(d) => {
560
+ let resp = Response::new(
561
+ d.status, d.headers, d.body, d.url, d.version, d.content_length,
562
+ d.transfer_size,
563
+ );
564
+ results.push(ruby.obj_wrap(resp))?;
565
+ }
566
+ BatchItem::Err(msg) => {
567
+ let err: Value = wreq_error().funcall("new", (msg,))?;
568
+ results.push(err)?;
569
+ }
570
+ }
571
+ }
572
+ Ok(results)
573
+ }
574
+
575
+ /// Convert a single batch spec into a RequestBuilder. Accepted forms:
576
+ /// "https://example.com" -> GET
577
+ /// { method:, url:, **opts }
578
+ fn build_request(
579
+ &self,
580
+ spec: Value,
581
+ shared: Option<&RHash>,
582
+ ) -> Result<wreq::RequestBuilder, magnus::Error> {
583
+ let mut method = wreq::Method::GET;
584
+ let url: String;
585
+ let mut item_opts: Option<RHash> = None;
586
+
587
+ if let Some(hash) = RHash::from_value(spec) {
588
+ url = hash_get_string(&hash, "url")?
589
+ .ok_or_else(|| generic_error("each request hash requires a :url"))?;
590
+ if let Some(val) = hash_get_value(&hash, "method")? {
591
+ method = value_to_method(val)?;
592
+ }
593
+ item_opts = Some(hash);
594
+ } else {
595
+ url = TryConvert::try_convert(spec)?;
596
+ }
597
+
598
+ let mut req = self.inner.request(method, &url);
599
+ if let Some(shared) = shared {
600
+ req = apply_request_options(req, shared)?;
601
+ }
602
+ if let Some(item_opts) = item_opts {
603
+ req = apply_request_options(req, &item_opts)?;
604
+ }
605
+ Ok(req)
606
+ }
607
+ }
608
+
609
+ /// Parse a String or Symbol like "post" / :post into an HTTP method.
610
+ fn value_to_method(val: Value) -> Result<wreq::Method, magnus::Error> {
611
+ let name: String = val.funcall("to_s", ())?;
612
+ name.to_uppercase()
613
+ .parse()
614
+ .map_err(|_| generic_error(format!("invalid HTTP method: {}", name)))
442
615
  }
443
616
 
444
617
  fn apply_request_options(
@@ -692,6 +865,7 @@ pub fn init(_ruby: &magnus::Ruby, module: &magnus::RModule) -> Result<(), magnus
692
865
  client_class.define_method("delete", method!(Client::delete, -1))?;
693
866
  client_class.define_method("head", method!(Client::head, -1))?;
694
867
  client_class.define_method("options", method!(Client::options, -1))?;
868
+ client_class.define_method("request_batch", method!(Client::request_batch, -1))?;
695
869
  client_class.define_method("cancel", method!(Client::cancel, 0))?;
696
870
 
697
871
  module.define_module_function("get", function!(wreq_get, -1))?;
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Wreq
4
- VERSION = "0.5.1"
4
+ VERSION = "0.6.1"
5
5
  end
@@ -1,5 +1,5 @@
1
1
  diff --git a/src/client.rs b/src/client.rs
2
- index 4abe25d8..76e709a8 100644
2
+ index 214fe423..ccbdb0e9 100644
3
3
  --- a/src/client.rs
4
4
  +++ b/src/client.rs
5
5
  @@ -49,6 +49,7 @@ use self::{
@@ -10,18 +10,14 @@ index 4abe25d8..76e709a8 100644
10
10
  },
11
11
  request::{Request, RequestBuilder},
12
12
  response::Response,
13
- @@ -115,25 +116,30 @@ type MaybeDecompressionBody<T> = tower_http::decompression::DecompressionBody<T>
13
+ @@ -116,24 +117,26 @@ type MaybeDecompressionBody<T> = tower_http::decompression::DecompressionBody<T>
14
+
14
15
  type ClientService = Timeout<
15
16
  ConfigService<
16
- MaybeDecompression<
17
- - Retry<RetryPolicy, FollowRedirect<HttpClient<Connector, Body>, FollowRedirectPolicy>>,
18
- + TransferSizeService<
19
- + Retry<
20
- + RetryPolicy,
21
- + FollowRedirect<HttpClient<Connector, Body>, FollowRedirectPolicy>,
22
- + >,
23
- + >,
24
- >,
17
+ - MaybeDecompression<Retry<RetryPolicy, FollowRedirect<HttpClient<Connector, Body>>>>,
18
+ + MaybeDecompression<
19
+ + TransferSizeService<Retry<RetryPolicy, FollowRedirect<HttpClient<Connector, Body>>>>,
20
+ + >,
25
21
  >,
26
22
  >;
27
23
 
@@ -45,7 +41,7 @@ index 4abe25d8..76e709a8 100644
45
41
  BoxError,
46
42
  >;
47
43
 
48
- @@ -593,6 +599,10 @@ impl ClientBuilder {
44
+ @@ -597,6 +600,10 @@ impl ClientBuilder {
49
45
  })
50
46
  .service(service);
51
47
 
@@ -56,7 +52,7 @@ index 4abe25d8..76e709a8 100644
56
52
  #[cfg(any(
57
53
  feature = "gzip",
58
54
  feature = "zstd",
59
- @@ -1585,7 +1595,7 @@ impl ClientBuilder {
55
+ @@ -1607,7 +1614,7 @@ impl ClientBuilder {
60
56
  L: Layer<
61
57
  BoxCloneSyncService<
62
58
  http::Request<Body>,
@@ -65,7 +61,7 @@ index 4abe25d8..76e709a8 100644
65
61
  BoxError,
66
62
  >,
67
63
  > + Clone
68
- @@ -1594,7 +1604,7 @@ impl ClientBuilder {
64
+ @@ -1616,7 +1623,7 @@ impl ClientBuilder {
69
65
  + 'static,
70
66
  L::Service: Service<
71
67
  http::Request<Body>,
@@ -266,7 +262,7 @@ index 00000000..f155ec8d
266
262
  + }
267
263
  +}
268
264
  diff --git a/src/client/response.rs b/src/client/response.rs
269
- index 3d3532af..ed478804 100644
265
+ index 3d3532af..7d29ec45 100644
270
266
  --- a/src/client/response.rs
271
267
  +++ b/src/client/response.rs
272
268
  @@ -18,6 +18,7 @@ use mime::Mime;
@@ -1,7 +1,7 @@
1
1
  [package]
2
2
  name = "wreq"
3
- version = "6.0.0-rc.29"
4
- description = "An ergonomic Rust HTTP Client with TLS fingerprint"
3
+ version = "0.16.1"
4
+ description = "An ergonomic, privacy-aware Rust HTTP Client"
5
5
  keywords = ["http", "client", "websocket", "ja3", "ja4"]
6
6
  categories = ["web-programming::http-client"]
7
7
  repository = "https://github.com/0x676e67/wreq"
@@ -10,8 +10,8 @@ authors = ["0x676e67 <gngppz@gmail.com>"]
10
10
  readme = "README.md"
11
11
  license = "Apache-2.0"
12
12
  edition = "2024"
13
- rust-version = "1.85"
14
- include = ["README.md", "LICENSE", "src/**/*.rs"]
13
+ rust-version = "1.98"
14
+ include = ["/README.md", "/LICENSE", "/src/**/*.rs"]
15
15
 
16
16
  [package.metadata.docs.rs]
17
17
  all-features = true
@@ -92,10 +92,10 @@ http = "1.4.0"
92
92
  http2 = "0.5.17"
93
93
  httparse = "1.10.1"
94
94
  http-body = "1.0.1"
95
- http-body-util = "0.1.3"
95
+ http-body-util = "0.1.4"
96
96
  percent-encoding = "2.3.2"
97
97
  pin-project-lite = "0.2.17"
98
- futures-util = { version = "0.3.32", default-features = false }
98
+ futures-util = { version = "0.3.33", default-features = false }
99
99
  socket2 = { version = "0.6.3", features = ["all"] }
100
100
  ipnet = "2.12.0"
101
101
  lru = "0.18.0"
@@ -103,7 +103,7 @@ btls = "0.5.6"
103
103
  btls-sys = "0.5.6"
104
104
  tokio-btls = "0.5.6"
105
105
  tokio = { version = "1.52.3", default-features = false }
106
- compio = { version = "0.19.0-rc.1", features = ["io", "io-compat"], optional = true }
106
+ compio = { version = "0.19.2", features = ["io", "io-compat"], optional = true }
107
107
  wreq-rt = { version = "0.2.2-rc.2", default-features = false }
108
108
  wreq-proto = { version = "0.2.3", default-features = false }
109
109
  tower = { version = "0.5.3", default-features = false, features = [
@@ -174,7 +174,7 @@ tokio = { version = "1.0", default-features = false, features = [
174
174
  "macros",
175
175
  "rt-multi-thread",
176
176
  ] }
177
- compio = { version = "0.19.0-rc.1", features = ["macros", "net", "runtime", "time", "io", "io-compat"] }
177
+ compio = { version = "0.19.2", features = ["macros", "net", "runtime", "time", "io", "io-compat"] }
178
178
  hyper = { version = "1.7.0", default-features = false, features = [
179
179
  "http1",
180
180
  "http2",
@@ -202,6 +202,7 @@ zstd = "0.13.3"
202
202
  sysinfo = { version = "0.39.1", default-features = false, features = ["system"] }
203
203
  criterion = { version = "0.8.2", features = ["async_tokio"] }
204
204
  reqwest = { version = "0.13.3", default-features = false, features = ["rustls", "stream", "http2"] }
205
+ cyper = { version = "0.9.0", default-features = false, features = ["rustls", "stream"] }
205
206
 
206
207
  [profile.release]
207
208
  codegen-units = 1
@@ -2,8 +2,8 @@
2
2
 
3
3
  [![CI](https://github.com/0x676e67/wreq/actions/workflows/ci.yml/badge.svg)](https://github.com/0x676e67/wreq/actions/workflows/ci.yml)
4
4
  [![Crates.io License](https://img.shields.io/crates/l/wreq)](https://github.com/0x676e67/wreq/blob/main/LICENSE)
5
- [![Crates.io MSRV](https://img.shields.io/crates/msrv/wreq?logo=rust)](https://crates.io/crates/wreq)
6
- [![crates.io](https://img.shields.io/crates/v/wreq.svg?logo=rust)](https://crates.io/crates/wreq)
5
+ [![Crates.io MSRV](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fcrates.io%2Fapi%2Fv1%2Fcrates%2Fwreq%2Fversions%3Fsort%3Ddate%26per_page%3D1&query=%24.versions%5B0%5D.rust_version&suffix=.0&label=MSRV&color=blue&logo=rust)](https://crates.io/crates/wreq)
6
+ [![crates.io](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fcrates.io%2Fapi%2Fv1%2Fcrates%2Fwreq&query=%24.crate.max_stable_version&prefix=v&label=crates.io&color=orange&logo=rust)](https://crates.io/crates/wreq)
7
7
  [![Discord chat][discord-badge]][discord-url]
8
8
 
9
9
  [discord-badge]: https://img.shields.io/discord/1486741856397164788.svg?logo=discord
@@ -35,8 +35,8 @@ The following example uses the [Tokio](https://tokio.rs) runtime with optional f
35
35
  ```toml
36
36
  [dependencies]
37
37
  tokio = { version = "1", features = ["full"] }
38
- wreq = "6.0.0-rc"
39
- wreq-util = "3.0.0-rc"
38
+ wreq = "0.16"
39
+ wreq-util = "0.2"
40
40
  ```
41
41
 
42
42
  And then the code:
@@ -53,7 +53,7 @@ async fn main() -> wreq::Result<()> {
53
53
  .build()?;
54
54
 
55
55
  // Use the API you're already familiar with
56
- let resp = client.get("https://tls.peet.ws/api/all").send().await?;
56
+ let resp = client.get("https://pingly.us.kg/api/all").send().await?;
57
57
  println!("{}", resp.text().await?);
58
58
  Ok(())
59
59
  }
@@ -1,8 +1,10 @@
1
1
  use criterion::Criterion;
2
2
 
3
3
  use crate::support::{
4
- BoxError, HttpVersion, Tls, client::bench_clients, current_thread_runtime,
5
- multi_thread_runtime, server::with_server,
4
+ BoxError, HttpVersion, Tls,
5
+ client::bench_clients,
6
+ rt::{current_thread_runtime, multi_thread_runtime},
7
+ server::with_server,
6
8
  };
7
9
 
8
10
  pub const CURRENT_THREAD_LABEL: &str = "current_thread";
@@ -54,6 +56,7 @@ pub fn bench(
54
56
  bench_clients(
55
57
  &mut group,
56
58
  current_thread_runtime,
59
+ true,
57
60
  addr,
58
61
  tls,
59
62
  http_version,
@@ -74,6 +77,7 @@ pub fn bench(
74
77
  bench_clients(
75
78
  &mut group,
76
79
  multi_thread_runtime,
80
+ false,
77
81
  addr,
78
82
  tls,
79
83
  http_version,
@@ -2,10 +2,11 @@ use std::{convert::Infallible, net::SocketAddr, sync::Arc};
2
2
 
3
3
  use bytes::Bytes;
4
4
  use criterion::{BenchmarkGroup, measurement::WallTime};
5
+ use futures::{StreamExt, TryStreamExt};
5
6
  use http_body_util::BodyExt;
6
7
  use tokio::{runtime::Runtime, sync::Semaphore};
7
8
 
8
- use super::{BoxError, HttpVersion, Tls};
9
+ use super::{BoxError, HttpVersion, Tls, rt::CompioBenchExecutor};
9
10
 
10
11
  fn create_wreq_client(tls: Tls, http_version: HttpVersion) -> Result<wreq::Client, BoxError> {
11
12
  let builder = wreq::Client::builder()
@@ -35,6 +36,20 @@ fn create_reqwest_client(tls: Tls, http_version: HttpVersion) -> Result<reqwest:
35
36
  Ok(builder.build()?)
36
37
  }
37
38
 
39
+ fn create_cyper_client(tls: Tls, http_version: HttpVersion) -> Result<cyper::Client, BoxError> {
40
+ let builder = cyper::Client::builder()
41
+ .no_proxy()
42
+ .redirect(cyper::redirect::Policy::none())
43
+ .danger_accept_invalid_certs(matches!(tls, Tls::Enabled));
44
+
45
+ let builder = match http_version {
46
+ HttpVersion::Http1 => builder,
47
+ HttpVersion::Http2 => builder.http2_prior_knowledge(),
48
+ };
49
+
50
+ Ok(builder.build()?)
51
+ }
52
+
38
53
  async fn wreq_body_assert(mut response: wreq::Response, expected_body_size: usize) {
39
54
  let mut body_size = 0;
40
55
  while let Some(Ok(chunk)) = response.frame().await {
@@ -59,6 +74,17 @@ async fn reqwest_body_assert(mut response: reqwest::Response, expected_body_size
59
74
  );
60
75
  }
61
76
 
77
+ async fn cyper_body_assert(mut response: cyper::Response, expected_body_size: usize) {
78
+ let mut body_size = 0;
79
+ while let Some(Ok(chunk)) = response.next().await {
80
+ body_size += chunk.len();
81
+ }
82
+ assert!(
83
+ body_size == expected_body_size,
84
+ "Unexpected response body: got {body_size} bytes, expected {expected_body_size} bytes"
85
+ );
86
+ }
87
+
62
88
  fn stream_from_bytes(
63
89
  body: &'static [u8],
64
90
  chunk_size: usize,
@@ -94,6 +120,16 @@ fn reqwest_body(stream: bool, (body, chunk_size): (&'static [u8], usize)) -> req
94
120
  }
95
121
  }
96
122
 
123
+ #[inline]
124
+ fn cyper_body(stream: bool, (body, chunk_size): (&'static [u8], usize)) -> cyper::Body {
125
+ if stream {
126
+ let stream = stream_from_bytes(body, chunk_size).map_err(|never| match never {});
127
+ cyper::Body::stream(stream)
128
+ } else {
129
+ cyper::Body::from(body)
130
+ }
131
+ }
132
+
97
133
  async fn wreq_requests_concurrent(
98
134
  client: &wreq::Client,
99
135
  url: &str,
@@ -158,10 +194,44 @@ async fn reqwest_requests_concurrent(
158
194
  futures_util::future::join_all(handles).await;
159
195
  }
160
196
 
197
+ async fn cyper_requests_concurrent(
198
+ client: &cyper::Client,
199
+ url: &str,
200
+ num_requests: usize,
201
+ concurrent_limit: usize,
202
+ body: (&'static [u8], usize),
203
+ stream: bool,
204
+ ) {
205
+ let semaphore = Arc::new(Semaphore::new(concurrent_limit));
206
+ let mut handles = Vec::with_capacity(num_requests);
207
+ for _ in 0..num_requests {
208
+ let client = client.clone();
209
+ let url = url.to_string();
210
+ let semaphore = semaphore.clone();
211
+ let fut = async move {
212
+ let _permit = semaphore
213
+ .acquire()
214
+ .await
215
+ .expect("Semaphore should be acquirable");
216
+ let response = client
217
+ .post(url)
218
+ .expect("Unexpected request failure")
219
+ .body(cyper_body(stream, body))
220
+ .send()
221
+ .await
222
+ .expect("Unexpected request failure");
223
+ cyper_body_assert(response, body.0.len()).await;
224
+ };
225
+ handles.push(compio::runtime::spawn(fut));
226
+ }
227
+ futures_util::future::join_all(handles).await;
228
+ }
229
+
161
230
  #[allow(clippy::too_many_arguments)]
162
231
  pub fn bench_clients(
163
232
  group: &mut BenchmarkGroup<'_, WallTime>,
164
233
  rt: fn() -> Runtime,
234
+ include_cyper: bool,
165
235
  addr: SocketAddr,
166
236
  tls: Tls,
167
237
  http_version: HttpVersion,
@@ -211,6 +281,23 @@ pub fn bench_clients(
211
281
  })
212
282
  });
213
283
  ::std::mem::drop(client);
284
+
285
+ if include_cyper {
286
+ let compio_executor = CompioBenchExecutor::new()?;
287
+ let client = create_cyper_client(tls, http_version)?;
288
+ group.bench_function(make_benchmark_label::<cyper::Client>(stream), |b| {
289
+ b.to_async(&compio_executor).iter(|| {
290
+ cyper_requests_concurrent(
291
+ &client,
292
+ &url,
293
+ num_requests,
294
+ concurrent_limit,
295
+ body,
296
+ stream,
297
+ )
298
+ })
299
+ });
300
+ }
214
301
  }
215
302
 
216
303
  Ok(())
File without changes
@@ -0,0 +1,34 @@
1
+ use criterion::async_executor::AsyncExecutor;
2
+
3
+ pub struct CompioBenchExecutor {
4
+ runtime: compio::runtime::Runtime,
5
+ }
6
+
7
+ impl CompioBenchExecutor {
8
+ pub fn new() -> std::io::Result<Self> {
9
+ Ok(Self {
10
+ runtime: compio::runtime::Runtime::new()?,
11
+ })
12
+ }
13
+ }
14
+
15
+ impl AsyncExecutor for &CompioBenchExecutor {
16
+ fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
17
+ self.runtime.block_on(future)
18
+ }
19
+ }
20
+
21
+ pub fn current_thread_runtime() -> tokio::runtime::Runtime {
22
+ tokio::runtime::Builder::new_current_thread()
23
+ .enable_all()
24
+ .build()
25
+ .expect("Failed to build current-thread runtime")
26
+ }
27
+
28
+ pub fn multi_thread_runtime() -> tokio::runtime::Runtime {
29
+ tokio::runtime::Builder::new_multi_thread()
30
+ .worker_threads(4)
31
+ .enable_all()
32
+ .build()
33
+ .expect("Failed to build multi-thread runtime")
34
+ }
@@ -20,7 +20,7 @@ use tokio::{
20
20
  };
21
21
  use tokio_btls::SslStream;
22
22
 
23
- use super::{BoxError, Tls, multi_thread_runtime};
23
+ use super::{BoxError, Tls, rt::multi_thread_runtime};
24
24
 
25
25
  pub struct Server {
26
26
  listener: std::net::TcpListener,