wreq-rb 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +257 -102
- data/README.md +28 -0
- data/ext/wreq_rb/Cargo.toml +4 -4
- data/ext/wreq_rb/src/client.rs +176 -2
- data/lib/wreq-rb/version.rb +1 -1
- data/patches/0001-add-transfer-size-tracking.patch +11 -15
- data/vendor/wreq/Cargo.toml +9 -8
- data/vendor/wreq/README.md +5 -5
- data/vendor/wreq/bench/support/bench.rs +6 -2
- data/vendor/wreq/bench/support/client.rs +88 -1
- data/vendor/wreq/bench/support/exec.rs +0 -0
- data/vendor/wreq/bench/support/rt.rs +34 -0
- data/vendor/wreq/bench/support/server.rs +1 -1
- data/vendor/wreq/bench/support.rs +1 -15
- data/vendor/wreq/examples/cert_store.rs +13 -13
- data/vendor/wreq/examples/request_with_emulate.rs +1 -1
- data/vendor/wreq/examples/tcp_linger.rs +22 -0
- data/vendor/wreq/src/client/layer/client/pool.rs +17 -17
- data/vendor/wreq/src/client/layer/client.rs +2 -0
- data/vendor/wreq/src/client/layer/decoder.rs +71 -17
- data/vendor/wreq/src/client/layer/redirect/future.rs +49 -63
- data/vendor/wreq/src/client/layer/redirect/policy.rs +2 -26
- data/vendor/wreq/src/client/layer/redirect.rs +48 -60
- data/vendor/wreq/src/client/layer/retry.rs +12 -15
- data/vendor/wreq/src/client/layer/timeout/body.rs +27 -21
- data/vendor/wreq/src/client/layer/timeout/future.rs +33 -58
- data/vendor/wreq/src/client/layer/timeout.rs +8 -14
- data/vendor/wreq/src/client/request.rs +4 -0
- data/vendor/wreq/src/client.rs +53 -31
- data/vendor/wreq/src/conn/connector.rs +99 -129
- data/vendor/wreq/src/conn/http.rs +25 -18
- data/vendor/wreq/src/conn/net/tcp.rs +601 -107
- data/vendor/wreq/src/conn/proxy/socks.rs +6 -6
- data/vendor/wreq/src/conn/timeout.rs +166 -0
- data/vendor/wreq/src/conn.rs +5 -4
- data/vendor/wreq/src/cookie/jar.rs +1225 -0
- data/vendor/wreq/src/cookie/store.rs +321 -0
- data/vendor/wreq/src/cookie.rs +108 -612
- data/vendor/wreq/src/dns/resolve.rs +8 -2
- data/vendor/wreq/src/dns.rs +4 -4
- data/vendor/wreq/src/error.rs +53 -20
- data/vendor/wreq/src/lib.rs +1 -0
- data/vendor/wreq/src/proxy/matcher.rs +26 -12
- data/vendor/wreq/src/proxy/win.rs +39 -9
- data/vendor/wreq/src/redirect.rs +515 -100
- data/vendor/wreq/src/tls/conn.rs +3 -11
- data/vendor/wreq/src/tls/session.rs +7 -8
- data/vendor/wreq/src/tls/trust/store.rs +4 -4
- data/vendor/wreq/src/util.rs +23 -0
- data/vendor/wreq/tests/badssl.rs +72 -7
- data/vendor/wreq/tests/brotli.rs +1 -1
- data/vendor/wreq/tests/client.rs +24 -0
- data/vendor/wreq/tests/connector_layers.rs +8 -4
- data/vendor/wreq/tests/cookie.rs +59 -0
- data/vendor/wreq/tests/deflate.rs +1 -1
- data/vendor/wreq/tests/gzip.rs +53 -1
- data/vendor/wreq/tests/layers.rs +8 -4
- data/vendor/wreq/tests/redirect.rs +180 -97
- data/vendor/wreq/tests/timeouts.rs +47 -12
- data/vendor/wreq/tests/zstd.rs +1 -1
- metadata +8 -2
data/README.md
CHANGED
|
@@ -114,6 +114,34 @@ All methods are available on both `Wreq` (module-level) and `Wreq::Client` (inst
|
|
|
114
114
|
| `head(url, **opts)` | HEAD request |
|
|
115
115
|
| `options(url, **opts)` | OPTIONS request |
|
|
116
116
|
|
|
117
|
+
### Batch Requests
|
|
118
|
+
|
|
119
|
+
`Wreq::Client#request_batch` runs many requests concurrently inside a **single**
|
|
120
|
+
GVL release, multiplexed over the client's connection pool. This avoids one Ruby
|
|
121
|
+
thread per request.
|
|
122
|
+
|
|
123
|
+
```ruby
|
|
124
|
+
client = Wreq::Client.new(redirect: false, pool_max_idle_per_host: 32)
|
|
125
|
+
|
|
126
|
+
responses = client.request_batch(urls, concurrency: 128)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Each element of the array is either a URL string or a hash:
|
|
130
|
+
|
|
131
|
+
```ruby
|
|
132
|
+
client.request_batch([
|
|
133
|
+
"https://example.com/a", # GET
|
|
134
|
+
{ method: :put, url: "https://example.com/b", body: "hi" }
|
|
135
|
+
], concurrency: 32)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
- Results are returned **in input order**.
|
|
139
|
+
- Each element is a `Wreq::Response` **or** a `Wreq::Error` — a single failed
|
|
140
|
+
request never discards the rest of the batch. Errors are returned, not raised.
|
|
141
|
+
- `concurrency` caps the number of in-flight requests and defaults to `16`.
|
|
142
|
+
- `cancel` and Ruby thread interrupts abort the whole batch, raising
|
|
143
|
+
`Wreq::Error` with `"request interrupted"`.
|
|
144
|
+
|
|
117
145
|
### Cancelling Requests
|
|
118
146
|
|
|
119
147
|
Call `cancel` on a client to interrupt all in-flight requests immediately:
|
data/ext/wreq_rb/Cargo.toml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "wreq_rb"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.6.1"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
publish = false
|
|
6
6
|
|
|
@@ -10,7 +10,7 @@ crate-type = ["cdylib"]
|
|
|
10
10
|
[dependencies]
|
|
11
11
|
magnus = { version = "0.8", features = ["rb-sys"] }
|
|
12
12
|
rb-sys = "0.9"
|
|
13
|
-
wreq = { path = "../../vendor/wreq", version = "=
|
|
13
|
+
wreq = { path = "../../vendor/wreq", version = "=0.16.1", features = [
|
|
14
14
|
"cookies",
|
|
15
15
|
"json",
|
|
16
16
|
"gzip",
|
|
@@ -24,7 +24,7 @@ wreq = { path = "../../vendor/wreq", version = "=6.0.0-rc.29", features = [
|
|
|
24
24
|
"query",
|
|
25
25
|
"form",
|
|
26
26
|
] }
|
|
27
|
-
wreq-util = { version = "=
|
|
27
|
+
wreq-util = { version = "=0.2.0", features = ["emulation", "emulation-serde", "emulation-compression"] }
|
|
28
28
|
tokio = { version = "1", features = ["full"] }
|
|
29
29
|
tokio-util = "0.7"
|
|
30
30
|
serde_json = "1.0"
|
|
@@ -32,6 +32,6 @@ bytes = "1"
|
|
|
32
32
|
http = "1"
|
|
33
33
|
|
|
34
34
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
35
|
-
wreq = { path = "../../vendor/wreq", version = "=
|
|
35
|
+
wreq = { path = "../../vendor/wreq", version = "=0.16.1", features = [
|
|
36
36
|
"prefix-symbols",
|
|
37
37
|
] }
|
data/ext/wreq_rb/src/client.rs
CHANGED
|
@@ -2,6 +2,7 @@ use std::ffi::c_void;
|
|
|
2
2
|
use std::panic::{self, AssertUnwindSafe};
|
|
3
3
|
use std::ptr;
|
|
4
4
|
use std::any::Any;
|
|
5
|
+
use std::sync::Arc;
|
|
5
6
|
use std::time::Duration;
|
|
6
7
|
|
|
7
8
|
use magnus::{
|
|
@@ -9,13 +10,15 @@ use magnus::{
|
|
|
9
10
|
try_convert::TryConvert, Value,
|
|
10
11
|
};
|
|
11
12
|
use tokio::runtime::Runtime;
|
|
13
|
+
use tokio::sync::Semaphore;
|
|
14
|
+
use tokio::task::JoinSet;
|
|
12
15
|
use tokio_util::sync::CancellationToken;
|
|
13
16
|
use std::net::IpAddr;
|
|
14
17
|
use wreq::header::{HeaderMap, HeaderName, HeaderValue, OrigHeaderMap};
|
|
15
18
|
use wreq::tls::TlsVersion;
|
|
16
19
|
use wreq_util::{Emulation as BrowserEmulation, Platform as EmulationPlatform, Profile as BrowserProfile};
|
|
17
20
|
|
|
18
|
-
use crate::error::{generic_error, to_magnus_error};
|
|
21
|
+
use crate::error::{generic_error, to_magnus_error, wreq_error};
|
|
19
22
|
use crate::response::Response;
|
|
20
23
|
|
|
21
24
|
// --------------------------------------------------------------------------
|
|
@@ -137,12 +140,68 @@ async fn execute_request(req: wreq::RequestBuilder) -> Result<ResponseData, wreq
|
|
|
137
140
|
Ok(ResponseData { status, headers, body, url, version, content_length, transfer_size })
|
|
138
141
|
}
|
|
139
142
|
|
|
143
|
+
// --------------------------------------------------------------------------
|
|
144
|
+
// Batch execution
|
|
145
|
+
// --------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
const DEFAULT_BATCH_CONCURRENCY: usize = 16;
|
|
148
|
+
|
|
149
|
+
/// Per-item batch result as pure Rust types (no Ruby objects). Errors are kept
|
|
150
|
+
/// as messages so one failure never discards the rest of the batch.
|
|
151
|
+
enum BatchItem {
|
|
152
|
+
Ok(ResponseData),
|
|
153
|
+
Err(String),
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/// Outcome of a whole batch performed outside the GVL.
|
|
157
|
+
enum BatchOutcome {
|
|
158
|
+
Done(Vec<BatchItem>),
|
|
159
|
+
Interrupted,
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/// Run every request concurrently with at most `concurrency` in flight,
|
|
163
|
+
/// returning results in input order.
|
|
164
|
+
async fn execute_batch(reqs: Vec<wreq::RequestBuilder>, concurrency: usize) -> Vec<BatchItem> {
|
|
165
|
+
let permits = Arc::new(Semaphore::new(concurrency));
|
|
166
|
+
let mut set: JoinSet<(usize, BatchItem)> = JoinSet::new();
|
|
167
|
+
|
|
168
|
+
for (idx, req) in reqs.into_iter().enumerate() {
|
|
169
|
+
let permits = Arc::clone(&permits);
|
|
170
|
+
set.spawn(async move {
|
|
171
|
+
let _permit = match permits.acquire_owned().await {
|
|
172
|
+
Ok(p) => p,
|
|
173
|
+
Err(_) => return (idx, BatchItem::Err("batch semaphore closed".to_owned())),
|
|
174
|
+
};
|
|
175
|
+
let item = match execute_request(req).await {
|
|
176
|
+
Ok(data) => BatchItem::Ok(data),
|
|
177
|
+
Err(e) => BatchItem::Err(e.to_string()),
|
|
178
|
+
};
|
|
179
|
+
(idx, item)
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
let mut slots: Vec<Option<BatchItem>> = Vec::new();
|
|
184
|
+
slots.resize_with(set.len(), || None);
|
|
185
|
+
while let Some(joined) = set.join_next().await {
|
|
186
|
+
// A JoinError means the task panicked or was aborted; its slot is left
|
|
187
|
+
// empty and filled with a generic error below.
|
|
188
|
+
if let Ok((idx, item)) = joined {
|
|
189
|
+
slots[idx] = Some(item);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
slots
|
|
194
|
+
.into_iter()
|
|
195
|
+
.map(|slot| slot.unwrap_or_else(|| BatchItem::Err("request task failed".to_owned())))
|
|
196
|
+
.collect()
|
|
197
|
+
}
|
|
198
|
+
|
|
140
199
|
// --------------------------------------------------------------------------
|
|
141
200
|
// Emulation helpers
|
|
142
201
|
// --------------------------------------------------------------------------
|
|
143
202
|
|
|
144
203
|
/// The default browser profile to apply when none is specified.
|
|
145
|
-
const DEFAULT_EMULATION: BrowserProfile = BrowserProfile::
|
|
204
|
+
const DEFAULT_EMULATION: BrowserProfile = BrowserProfile::Chrome149;
|
|
146
205
|
|
|
147
206
|
/// Parse a Ruby string like "chrome_143" into a BrowserProfile variant.
|
|
148
207
|
fn parse_emulation(name: &str) -> Result<BrowserProfile, magnus::Error> {
|
|
@@ -439,6 +498,120 @@ impl Client {
|
|
|
439
498
|
};
|
|
440
499
|
Ok(Response::new(data.status, data.headers, data.body, data.url, data.version, data.content_length, data.transfer_size))
|
|
441
500
|
}
|
|
501
|
+
|
|
502
|
+
/// Wreq::Client#request_batch(specs) or #request_batch(specs, options)
|
|
503
|
+
fn request_batch(&self, args: &[Value]) -> Result<RArray, magnus::Error> {
|
|
504
|
+
if args.is_empty() {
|
|
505
|
+
return Err(generic_error("an array of requests is required"));
|
|
506
|
+
}
|
|
507
|
+
let specs = RArray::try_convert(args[0])?;
|
|
508
|
+
|
|
509
|
+
let opts: Option<RHash> = if args.len() > 1 {
|
|
510
|
+
Some(RHash::try_convert(args[1])?)
|
|
511
|
+
} else {
|
|
512
|
+
None
|
|
513
|
+
};
|
|
514
|
+
|
|
515
|
+
let concurrency = match opts.as_ref() {
|
|
516
|
+
Some(o) => hash_get_usize(o, "concurrency")?.unwrap_or(DEFAULT_BATCH_CONCURRENCY),
|
|
517
|
+
None => DEFAULT_BATCH_CONCURRENCY,
|
|
518
|
+
};
|
|
519
|
+
if concurrency == 0 {
|
|
520
|
+
return Err(generic_error("concurrency must be >= 1"));
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// All Ruby -> Rust conversion happens here, while we still hold the GVL.
|
|
524
|
+
let mut reqs: Vec<wreq::RequestBuilder> = Vec::with_capacity(specs.len());
|
|
525
|
+
for spec in specs.into_iter() {
|
|
526
|
+
reqs.push(self.build_request(spec, opts.as_ref())?);
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
let ruby = unsafe { Ruby::get_unchecked() };
|
|
530
|
+
if reqs.is_empty() {
|
|
531
|
+
return Ok(ruby.ary_new());
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
let client_token = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
|
535
|
+
|
|
536
|
+
// One GVL release covering the whole batch.
|
|
537
|
+
let outcome: BatchOutcome = unsafe {
|
|
538
|
+
without_gvl(|thread_token| {
|
|
539
|
+
runtime().block_on(async {
|
|
540
|
+
tokio::select! {
|
|
541
|
+
biased;
|
|
542
|
+
_ = thread_token.cancelled() => BatchOutcome::Interrupted,
|
|
543
|
+
_ = client_token.cancelled() => BatchOutcome::Interrupted,
|
|
544
|
+
items = execute_batch(reqs, concurrency) => BatchOutcome::Done(items),
|
|
545
|
+
}
|
|
546
|
+
})
|
|
547
|
+
})
|
|
548
|
+
};
|
|
549
|
+
|
|
550
|
+
let items = match outcome {
|
|
551
|
+
BatchOutcome::Done(items) => items,
|
|
552
|
+
BatchOutcome::Interrupted => return Err(generic_error("request interrupted")),
|
|
553
|
+
};
|
|
554
|
+
|
|
555
|
+
// Back under the GVL: Ruby objects may be created again.
|
|
556
|
+
let results = ruby.ary_new_capa(items.len());
|
|
557
|
+
for item in items {
|
|
558
|
+
match item {
|
|
559
|
+
BatchItem::Ok(d) => {
|
|
560
|
+
let resp = Response::new(
|
|
561
|
+
d.status, d.headers, d.body, d.url, d.version, d.content_length,
|
|
562
|
+
d.transfer_size,
|
|
563
|
+
);
|
|
564
|
+
results.push(ruby.obj_wrap(resp))?;
|
|
565
|
+
}
|
|
566
|
+
BatchItem::Err(msg) => {
|
|
567
|
+
let err: Value = wreq_error().funcall("new", (msg,))?;
|
|
568
|
+
results.push(err)?;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
Ok(results)
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/// Convert a single batch spec into a RequestBuilder. Accepted forms:
|
|
576
|
+
/// "https://example.com" -> GET
|
|
577
|
+
/// { method:, url:, **opts }
|
|
578
|
+
fn build_request(
|
|
579
|
+
&self,
|
|
580
|
+
spec: Value,
|
|
581
|
+
shared: Option<&RHash>,
|
|
582
|
+
) -> Result<wreq::RequestBuilder, magnus::Error> {
|
|
583
|
+
let mut method = wreq::Method::GET;
|
|
584
|
+
let url: String;
|
|
585
|
+
let mut item_opts: Option<RHash> = None;
|
|
586
|
+
|
|
587
|
+
if let Some(hash) = RHash::from_value(spec) {
|
|
588
|
+
url = hash_get_string(&hash, "url")?
|
|
589
|
+
.ok_or_else(|| generic_error("each request hash requires a :url"))?;
|
|
590
|
+
if let Some(val) = hash_get_value(&hash, "method")? {
|
|
591
|
+
method = value_to_method(val)?;
|
|
592
|
+
}
|
|
593
|
+
item_opts = Some(hash);
|
|
594
|
+
} else {
|
|
595
|
+
url = TryConvert::try_convert(spec)?;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
let mut req = self.inner.request(method, &url);
|
|
599
|
+
if let Some(shared) = shared {
|
|
600
|
+
req = apply_request_options(req, shared)?;
|
|
601
|
+
}
|
|
602
|
+
if let Some(item_opts) = item_opts {
|
|
603
|
+
req = apply_request_options(req, &item_opts)?;
|
|
604
|
+
}
|
|
605
|
+
Ok(req)
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/// Parse a String or Symbol like "post" / :post into an HTTP method.
|
|
610
|
+
fn value_to_method(val: Value) -> Result<wreq::Method, magnus::Error> {
|
|
611
|
+
let name: String = val.funcall("to_s", ())?;
|
|
612
|
+
name.to_uppercase()
|
|
613
|
+
.parse()
|
|
614
|
+
.map_err(|_| generic_error(format!("invalid HTTP method: {}", name)))
|
|
442
615
|
}
|
|
443
616
|
|
|
444
617
|
fn apply_request_options(
|
|
@@ -692,6 +865,7 @@ pub fn init(_ruby: &magnus::Ruby, module: &magnus::RModule) -> Result<(), magnus
|
|
|
692
865
|
client_class.define_method("delete", method!(Client::delete, -1))?;
|
|
693
866
|
client_class.define_method("head", method!(Client::head, -1))?;
|
|
694
867
|
client_class.define_method("options", method!(Client::options, -1))?;
|
|
868
|
+
client_class.define_method("request_batch", method!(Client::request_batch, -1))?;
|
|
695
869
|
client_class.define_method("cancel", method!(Client::cancel, 0))?;
|
|
696
870
|
|
|
697
871
|
module.define_module_function("get", function!(wreq_get, -1))?;
|
data/lib/wreq-rb/version.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
diff --git a/src/client.rs b/src/client.rs
|
|
2
|
-
index
|
|
2
|
+
index 214fe423..ccbdb0e9 100644
|
|
3
3
|
--- a/src/client.rs
|
|
4
4
|
+++ b/src/client.rs
|
|
5
5
|
@@ -49,6 +49,7 @@ use self::{
|
|
@@ -10,18 +10,14 @@ index 4abe25d8..76e709a8 100644
|
|
|
10
10
|
},
|
|
11
11
|
request::{Request, RequestBuilder},
|
|
12
12
|
response::Response,
|
|
13
|
-
@@ -
|
|
13
|
+
@@ -116,24 +117,26 @@ type MaybeDecompressionBody<T> = tower_http::decompression::DecompressionBody<T>
|
|
14
|
+
|
|
14
15
|
type ClientService = Timeout<
|
|
15
16
|
ConfigService<
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
+ TransferSizeService<
|
|
19
|
-
+
|
|
20
|
-
+ RetryPolicy,
|
|
21
|
-
+ FollowRedirect<HttpClient<Connector, Body>, FollowRedirectPolicy>,
|
|
22
|
-
+ >,
|
|
23
|
-
+ >,
|
|
24
|
-
>,
|
|
17
|
+
- MaybeDecompression<Retry<RetryPolicy, FollowRedirect<HttpClient<Connector, Body>>>>,
|
|
18
|
+
+ MaybeDecompression<
|
|
19
|
+
+ TransferSizeService<Retry<RetryPolicy, FollowRedirect<HttpClient<Connector, Body>>>>,
|
|
20
|
+
+ >,
|
|
25
21
|
>,
|
|
26
22
|
>;
|
|
27
23
|
|
|
@@ -45,7 +41,7 @@ index 4abe25d8..76e709a8 100644
|
|
|
45
41
|
BoxError,
|
|
46
42
|
>;
|
|
47
43
|
|
|
48
|
-
@@ -
|
|
44
|
+
@@ -597,6 +600,10 @@ impl ClientBuilder {
|
|
49
45
|
})
|
|
50
46
|
.service(service);
|
|
51
47
|
|
|
@@ -56,7 +52,7 @@ index 4abe25d8..76e709a8 100644
|
|
|
56
52
|
#[cfg(any(
|
|
57
53
|
feature = "gzip",
|
|
58
54
|
feature = "zstd",
|
|
59
|
-
@@ -
|
|
55
|
+
@@ -1607,7 +1614,7 @@ impl ClientBuilder {
|
|
60
56
|
L: Layer<
|
|
61
57
|
BoxCloneSyncService<
|
|
62
58
|
http::Request<Body>,
|
|
@@ -65,7 +61,7 @@ index 4abe25d8..76e709a8 100644
|
|
|
65
61
|
BoxError,
|
|
66
62
|
>,
|
|
67
63
|
> + Clone
|
|
68
|
-
@@ -
|
|
64
|
+
@@ -1616,7 +1623,7 @@ impl ClientBuilder {
|
|
69
65
|
+ 'static,
|
|
70
66
|
L::Service: Service<
|
|
71
67
|
http::Request<Body>,
|
|
@@ -266,7 +262,7 @@ index 00000000..f155ec8d
|
|
|
266
262
|
+ }
|
|
267
263
|
+}
|
|
268
264
|
diff --git a/src/client/response.rs b/src/client/response.rs
|
|
269
|
-
index 3d3532af..
|
|
265
|
+
index 3d3532af..7d29ec45 100644
|
|
270
266
|
--- a/src/client/response.rs
|
|
271
267
|
+++ b/src/client/response.rs
|
|
272
268
|
@@ -18,6 +18,7 @@ use mime::Mime;
|
data/vendor/wreq/Cargo.toml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "wreq"
|
|
3
|
-
version = "
|
|
4
|
-
description = "An ergonomic Rust HTTP Client
|
|
3
|
+
version = "0.16.1"
|
|
4
|
+
description = "An ergonomic, privacy-aware Rust HTTP Client"
|
|
5
5
|
keywords = ["http", "client", "websocket", "ja3", "ja4"]
|
|
6
6
|
categories = ["web-programming::http-client"]
|
|
7
7
|
repository = "https://github.com/0x676e67/wreq"
|
|
@@ -10,8 +10,8 @@ authors = ["0x676e67 <gngppz@gmail.com>"]
|
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
edition = "2024"
|
|
13
|
-
rust-version = "1.
|
|
14
|
-
include = ["README.md", "LICENSE", "src/**/*.rs"]
|
|
13
|
+
rust-version = "1.98"
|
|
14
|
+
include = ["/README.md", "/LICENSE", "/src/**/*.rs"]
|
|
15
15
|
|
|
16
16
|
[package.metadata.docs.rs]
|
|
17
17
|
all-features = true
|
|
@@ -92,10 +92,10 @@ http = "1.4.0"
|
|
|
92
92
|
http2 = "0.5.17"
|
|
93
93
|
httparse = "1.10.1"
|
|
94
94
|
http-body = "1.0.1"
|
|
95
|
-
http-body-util = "0.1.
|
|
95
|
+
http-body-util = "0.1.4"
|
|
96
96
|
percent-encoding = "2.3.2"
|
|
97
97
|
pin-project-lite = "0.2.17"
|
|
98
|
-
futures-util = { version = "0.3.
|
|
98
|
+
futures-util = { version = "0.3.33", default-features = false }
|
|
99
99
|
socket2 = { version = "0.6.3", features = ["all"] }
|
|
100
100
|
ipnet = "2.12.0"
|
|
101
101
|
lru = "0.18.0"
|
|
@@ -103,7 +103,7 @@ btls = "0.5.6"
|
|
|
103
103
|
btls-sys = "0.5.6"
|
|
104
104
|
tokio-btls = "0.5.6"
|
|
105
105
|
tokio = { version = "1.52.3", default-features = false }
|
|
106
|
-
compio = { version = "0.19.
|
|
106
|
+
compio = { version = "0.19.2", features = ["io", "io-compat"], optional = true }
|
|
107
107
|
wreq-rt = { version = "0.2.2-rc.2", default-features = false }
|
|
108
108
|
wreq-proto = { version = "0.2.3", default-features = false }
|
|
109
109
|
tower = { version = "0.5.3", default-features = false, features = [
|
|
@@ -174,7 +174,7 @@ tokio = { version = "1.0", default-features = false, features = [
|
|
|
174
174
|
"macros",
|
|
175
175
|
"rt-multi-thread",
|
|
176
176
|
] }
|
|
177
|
-
compio = { version = "0.19.
|
|
177
|
+
compio = { version = "0.19.2", features = ["macros", "net", "runtime", "time", "io", "io-compat"] }
|
|
178
178
|
hyper = { version = "1.7.0", default-features = false, features = [
|
|
179
179
|
"http1",
|
|
180
180
|
"http2",
|
|
@@ -202,6 +202,7 @@ zstd = "0.13.3"
|
|
|
202
202
|
sysinfo = { version = "0.39.1", default-features = false, features = ["system"] }
|
|
203
203
|
criterion = { version = "0.8.2", features = ["async_tokio"] }
|
|
204
204
|
reqwest = { version = "0.13.3", default-features = false, features = ["rustls", "stream", "http2"] }
|
|
205
|
+
cyper = { version = "0.9.0", default-features = false, features = ["rustls", "stream"] }
|
|
205
206
|
|
|
206
207
|
[profile.release]
|
|
207
208
|
codegen-units = 1
|
data/vendor/wreq/README.md
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://github.com/0x676e67/wreq/actions/workflows/ci.yml)
|
|
4
4
|
[](https://github.com/0x676e67/wreq/blob/main/LICENSE)
|
|
5
|
-
[](https://crates.io/crates/wreq)
|
|
6
|
+
[](https://crates.io/crates/wreq)
|
|
7
7
|
[![Discord chat][discord-badge]][discord-url]
|
|
8
8
|
|
|
9
9
|
[discord-badge]: https://img.shields.io/discord/1486741856397164788.svg?logo=discord
|
|
@@ -35,8 +35,8 @@ The following example uses the [Tokio](https://tokio.rs) runtime with optional f
|
|
|
35
35
|
```toml
|
|
36
36
|
[dependencies]
|
|
37
37
|
tokio = { version = "1", features = ["full"] }
|
|
38
|
-
wreq = "
|
|
39
|
-
wreq-util = "
|
|
38
|
+
wreq = "0.16"
|
|
39
|
+
wreq-util = "0.2"
|
|
40
40
|
```
|
|
41
41
|
|
|
42
42
|
And then the code:
|
|
@@ -53,7 +53,7 @@ async fn main() -> wreq::Result<()> {
|
|
|
53
53
|
.build()?;
|
|
54
54
|
|
|
55
55
|
// Use the API you're already familiar with
|
|
56
|
-
let resp = client.get("https://
|
|
56
|
+
let resp = client.get("https://pingly.us.kg/api/all").send().await?;
|
|
57
57
|
println!("{}", resp.text().await?);
|
|
58
58
|
Ok(())
|
|
59
59
|
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
use criterion::Criterion;
|
|
2
2
|
|
|
3
3
|
use crate::support::{
|
|
4
|
-
BoxError, HttpVersion, Tls,
|
|
5
|
-
|
|
4
|
+
BoxError, HttpVersion, Tls,
|
|
5
|
+
client::bench_clients,
|
|
6
|
+
rt::{current_thread_runtime, multi_thread_runtime},
|
|
7
|
+
server::with_server,
|
|
6
8
|
};
|
|
7
9
|
|
|
8
10
|
pub const CURRENT_THREAD_LABEL: &str = "current_thread";
|
|
@@ -54,6 +56,7 @@ pub fn bench(
|
|
|
54
56
|
bench_clients(
|
|
55
57
|
&mut group,
|
|
56
58
|
current_thread_runtime,
|
|
59
|
+
true,
|
|
57
60
|
addr,
|
|
58
61
|
tls,
|
|
59
62
|
http_version,
|
|
@@ -74,6 +77,7 @@ pub fn bench(
|
|
|
74
77
|
bench_clients(
|
|
75
78
|
&mut group,
|
|
76
79
|
multi_thread_runtime,
|
|
80
|
+
false,
|
|
77
81
|
addr,
|
|
78
82
|
tls,
|
|
79
83
|
http_version,
|
|
@@ -2,10 +2,11 @@ use std::{convert::Infallible, net::SocketAddr, sync::Arc};
|
|
|
2
2
|
|
|
3
3
|
use bytes::Bytes;
|
|
4
4
|
use criterion::{BenchmarkGroup, measurement::WallTime};
|
|
5
|
+
use futures::{StreamExt, TryStreamExt};
|
|
5
6
|
use http_body_util::BodyExt;
|
|
6
7
|
use tokio::{runtime::Runtime, sync::Semaphore};
|
|
7
8
|
|
|
8
|
-
use super::{BoxError, HttpVersion, Tls};
|
|
9
|
+
use super::{BoxError, HttpVersion, Tls, rt::CompioBenchExecutor};
|
|
9
10
|
|
|
10
11
|
fn create_wreq_client(tls: Tls, http_version: HttpVersion) -> Result<wreq::Client, BoxError> {
|
|
11
12
|
let builder = wreq::Client::builder()
|
|
@@ -35,6 +36,20 @@ fn create_reqwest_client(tls: Tls, http_version: HttpVersion) -> Result<reqwest:
|
|
|
35
36
|
Ok(builder.build()?)
|
|
36
37
|
}
|
|
37
38
|
|
|
39
|
+
fn create_cyper_client(tls: Tls, http_version: HttpVersion) -> Result<cyper::Client, BoxError> {
|
|
40
|
+
let builder = cyper::Client::builder()
|
|
41
|
+
.no_proxy()
|
|
42
|
+
.redirect(cyper::redirect::Policy::none())
|
|
43
|
+
.danger_accept_invalid_certs(matches!(tls, Tls::Enabled));
|
|
44
|
+
|
|
45
|
+
let builder = match http_version {
|
|
46
|
+
HttpVersion::Http1 => builder,
|
|
47
|
+
HttpVersion::Http2 => builder.http2_prior_knowledge(),
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
Ok(builder.build()?)
|
|
51
|
+
}
|
|
52
|
+
|
|
38
53
|
async fn wreq_body_assert(mut response: wreq::Response, expected_body_size: usize) {
|
|
39
54
|
let mut body_size = 0;
|
|
40
55
|
while let Some(Ok(chunk)) = response.frame().await {
|
|
@@ -59,6 +74,17 @@ async fn reqwest_body_assert(mut response: reqwest::Response, expected_body_size
|
|
|
59
74
|
);
|
|
60
75
|
}
|
|
61
76
|
|
|
77
|
+
async fn cyper_body_assert(mut response: cyper::Response, expected_body_size: usize) {
|
|
78
|
+
let mut body_size = 0;
|
|
79
|
+
while let Some(Ok(chunk)) = response.next().await {
|
|
80
|
+
body_size += chunk.len();
|
|
81
|
+
}
|
|
82
|
+
assert!(
|
|
83
|
+
body_size == expected_body_size,
|
|
84
|
+
"Unexpected response body: got {body_size} bytes, expected {expected_body_size} bytes"
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
|
|
62
88
|
fn stream_from_bytes(
|
|
63
89
|
body: &'static [u8],
|
|
64
90
|
chunk_size: usize,
|
|
@@ -94,6 +120,16 @@ fn reqwest_body(stream: bool, (body, chunk_size): (&'static [u8], usize)) -> req
|
|
|
94
120
|
}
|
|
95
121
|
}
|
|
96
122
|
|
|
123
|
+
#[inline]
|
|
124
|
+
fn cyper_body(stream: bool, (body, chunk_size): (&'static [u8], usize)) -> cyper::Body {
|
|
125
|
+
if stream {
|
|
126
|
+
let stream = stream_from_bytes(body, chunk_size).map_err(|never| match never {});
|
|
127
|
+
cyper::Body::stream(stream)
|
|
128
|
+
} else {
|
|
129
|
+
cyper::Body::from(body)
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
97
133
|
async fn wreq_requests_concurrent(
|
|
98
134
|
client: &wreq::Client,
|
|
99
135
|
url: &str,
|
|
@@ -158,10 +194,44 @@ async fn reqwest_requests_concurrent(
|
|
|
158
194
|
futures_util::future::join_all(handles).await;
|
|
159
195
|
}
|
|
160
196
|
|
|
197
|
+
async fn cyper_requests_concurrent(
|
|
198
|
+
client: &cyper::Client,
|
|
199
|
+
url: &str,
|
|
200
|
+
num_requests: usize,
|
|
201
|
+
concurrent_limit: usize,
|
|
202
|
+
body: (&'static [u8], usize),
|
|
203
|
+
stream: bool,
|
|
204
|
+
) {
|
|
205
|
+
let semaphore = Arc::new(Semaphore::new(concurrent_limit));
|
|
206
|
+
let mut handles = Vec::with_capacity(num_requests);
|
|
207
|
+
for _ in 0..num_requests {
|
|
208
|
+
let client = client.clone();
|
|
209
|
+
let url = url.to_string();
|
|
210
|
+
let semaphore = semaphore.clone();
|
|
211
|
+
let fut = async move {
|
|
212
|
+
let _permit = semaphore
|
|
213
|
+
.acquire()
|
|
214
|
+
.await
|
|
215
|
+
.expect("Semaphore should be acquirable");
|
|
216
|
+
let response = client
|
|
217
|
+
.post(url)
|
|
218
|
+
.expect("Unexpected request failure")
|
|
219
|
+
.body(cyper_body(stream, body))
|
|
220
|
+
.send()
|
|
221
|
+
.await
|
|
222
|
+
.expect("Unexpected request failure");
|
|
223
|
+
cyper_body_assert(response, body.0.len()).await;
|
|
224
|
+
};
|
|
225
|
+
handles.push(compio::runtime::spawn(fut));
|
|
226
|
+
}
|
|
227
|
+
futures_util::future::join_all(handles).await;
|
|
228
|
+
}
|
|
229
|
+
|
|
161
230
|
#[allow(clippy::too_many_arguments)]
|
|
162
231
|
pub fn bench_clients(
|
|
163
232
|
group: &mut BenchmarkGroup<'_, WallTime>,
|
|
164
233
|
rt: fn() -> Runtime,
|
|
234
|
+
include_cyper: bool,
|
|
165
235
|
addr: SocketAddr,
|
|
166
236
|
tls: Tls,
|
|
167
237
|
http_version: HttpVersion,
|
|
@@ -211,6 +281,23 @@ pub fn bench_clients(
|
|
|
211
281
|
})
|
|
212
282
|
});
|
|
213
283
|
::std::mem::drop(client);
|
|
284
|
+
|
|
285
|
+
if include_cyper {
|
|
286
|
+
let compio_executor = CompioBenchExecutor::new()?;
|
|
287
|
+
let client = create_cyper_client(tls, http_version)?;
|
|
288
|
+
group.bench_function(make_benchmark_label::<cyper::Client>(stream), |b| {
|
|
289
|
+
b.to_async(&compio_executor).iter(|| {
|
|
290
|
+
cyper_requests_concurrent(
|
|
291
|
+
&client,
|
|
292
|
+
&url,
|
|
293
|
+
num_requests,
|
|
294
|
+
concurrent_limit,
|
|
295
|
+
body,
|
|
296
|
+
stream,
|
|
297
|
+
)
|
|
298
|
+
})
|
|
299
|
+
});
|
|
300
|
+
}
|
|
214
301
|
}
|
|
215
302
|
|
|
216
303
|
Ok(())
|
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
use criterion::async_executor::AsyncExecutor;
|
|
2
|
+
|
|
3
|
+
pub struct CompioBenchExecutor {
|
|
4
|
+
runtime: compio::runtime::Runtime,
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
impl CompioBenchExecutor {
|
|
8
|
+
pub fn new() -> std::io::Result<Self> {
|
|
9
|
+
Ok(Self {
|
|
10
|
+
runtime: compio::runtime::Runtime::new()?,
|
|
11
|
+
})
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
impl AsyncExecutor for &CompioBenchExecutor {
|
|
16
|
+
fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
|
|
17
|
+
self.runtime.block_on(future)
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
pub fn current_thread_runtime() -> tokio::runtime::Runtime {
|
|
22
|
+
tokio::runtime::Builder::new_current_thread()
|
|
23
|
+
.enable_all()
|
|
24
|
+
.build()
|
|
25
|
+
.expect("Failed to build current-thread runtime")
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
pub fn multi_thread_runtime() -> tokio::runtime::Runtime {
|
|
29
|
+
tokio::runtime::Builder::new_multi_thread()
|
|
30
|
+
.worker_threads(4)
|
|
31
|
+
.enable_all()
|
|
32
|
+
.build()
|
|
33
|
+
.expect("Failed to build multi-thread runtime")
|
|
34
|
+
}
|